From 785795bad6c839dd2dc273ca960f51889a3039ee Mon Sep 17 00:00:00 2001 From: Mateusz Stadnik Date: Wed, 30 Sep 2026 22:57:00 +0200 Subject: [PATCH 1/4] zig-abi: Zig compiler built by tcc, size tier, SRA, big perf and fixes - abi: structs of any size by value (AAPCS32), even-register alignment - sra: frame objects to scalars (narrow/wide/partial fields, pointer VARs, whole copies, struct returns, by-value args as words) - inline: called-once inlining, small leaves, nested bodies, struct params - frame: lifetime-based colouring, slot sharing, relayout, dead objects - regalloc: spill-slot reuse, eviction by cost, spill splitting, r8-r11 for call-crossing values, coalescing fixes - thumb/arm: LDM/STM copies, UXT/SXT, UMAAL, IT-block MOV, inline atomics, module-local calls keep R9, word-copy stubs, inline const-size copies - size: -Os/-Oz size tier, machine outliner, cross-jump tail/case merge, self_store, sret_nrvo, branch narrowing with pool dumps and pads - loops: IVSR sliding windows, bottom-test, branch-target alignment, unroll - lib: ARMv8-M mem* in assembly, fast 64-bit division - frontend: __has_attribute, __has_builtin, overflow builtins, dropped unused statics, fputs/printf lowering, __builtin_unreachable - compile time: quadratic passes made linear (zig.c -O2 3m21 -> 47s) - build: unity TUs, big files split, dead passes removed, ASan opt-in, device self-compile heap 37.4 -> 5.5 MB - YAFF v3/v4 headers; many self-host and Zig-compiler miscompile fixes Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 17 +- AGENTS.md | 33 +- Makefile | 67 +- README | 4 +- configure | 19 +- docs/bugs/README.md | 22 + .../bugs/ctags-entry-c-kconfig-choice-name.md | 63 + .../dce-dead-var-store-volatile-lval-src.md | 81 - ...oad-cse-lval-store-src-tracked-as-value.md | 90 - ...st-const-compare-fold-breaks-native-tcc.md | 91 + .../bugs/volatile-local-folded-to-constant.md | 90 - docs/coding_guidelines.md | 2 +- docs/plan_switch_case_constant_merge.md | 205 + docs/zig_frame_promotion_plan.md | 236 + include/stdatomic.h | 8 +- include/tccdefs.h | 2 - lib/Makefile | 8 +- lib/arm_copystub.S | 218 + lib/arm_mem.S | 336 + lib/armeabi.c | 130 +- lib/armeabi_divmod.S | 49 + lib/builtin.c | 21 + lib/fp/Makefile | 3 + scripts/bisect_pass.sh | 2 +- scripts/gen_unity.py | 504 + source/backend/arch/arm/Makefile | 4 + source/backend/arch/arm/arm-link.c | 19 + source/backend/arch/arm/arm_aapcs.c | 49 +- source/backend/arch/arm/ssa_opt_arm.c | 2 +- source/backend/arch/arm/thumb/Makefile | 6 +- source/backend/arch/arm/thumb/arm-thumb-alu.c | 3484 ++++ source/backend/arch/arm/thumb/arm-thumb-asm.c | 16 +- .../backend/arch/arm/thumb/arm-thumb-call.c | 4833 ++++++ .../arch/arm/thumb/arm-thumb-callsite.c | 2 +- .../backend/arch/arm/thumb/arm-thumb-emit.c | 2251 +++ source/backend/arch/arm/thumb/arm-thumb-fp.c | 1310 ++ .../backend/arch/arm/thumb/arm-thumb-frame.c | 1283 ++ source/backend/arch/arm/thumb/arm-thumb-gen.c | 13564 +--------------- source/backend/arch/arm/thumb/arm-thumb-gen.h | 621 + source/backend/arch/arm/thumb/arm-thumb-mem.c | 1223 ++ source/backend/arch/arm/thumb/thop_mul.c | 19 + source/backend/arch/arm/thumb/thop_mul.h | 1 + source/backend/arch/arm/thumb/thumb-outline.c | 1060 ++ source/backend/arch/arm/thumb/thumb-tok.h | 1 + source/backend/arch/unity.mk | 30 + source/backend/generators/Makefile | 12 +- source/backend/generators/function.c | 258 +- source/backend/generators/icf.c | 426 + source/backend/generators/regalloc.c | 342 +- source/backend/generators/regalloc.h | 1 + source/driver/Makefile | 2 +- source/driver/libtcc.c | 51 +- source/driver/tcc.c | 2 + source/frontend/Makefile | 2 +- source/frontend/gen/builtin/call.c | 234 +- source/frontend/gen/builtin/misc.c | 22 + source/frontend/gen/builtin/overflow.c | 70 +- source/frontend/gen/builtin/simd.c | 7 +- source/frontend/gen/builtin/string.c | 4 +- source/frontend/gen/core/state.c | 76 +- source/frontend/gen/decl/attribute.c | 73 +- source/frontend/gen/decl/decl.c | 1376 +- source/frontend/gen/decl/declarator.c | 5 + source/frontend/gen/expr/atomic.c | 95 +- source/frontend/gen/expr/cond.c | 37 +- source/frontend/gen/expr/indir.c | 135 +- source/frontend/gen/expr/primary.c | 33 +- source/frontend/gen/gen_priv.h | 36 +- source/frontend/gen/init/alloc.c | 132 +- source/frontend/gen/init/initializer.c | 33 +- source/frontend/gen/inline/analysis.c | 20 +- source/frontend/gen/inline/const_eval.c | 5 +- source/frontend/gen/inline/emit.c | 983 +- source/frontend/gen/nested/nested.c | 7 +- source/frontend/gen/op/fold_math.c | 44 +- source/frontend/gen/op/vector.c | 21 +- source/frontend/gen/stmt/block.c | 21 +- source/frontend/gen/stmt/cleanup.c | 5 +- source/frontend/gen/stmt/ret.c | 18 +- source/frontend/gen/store/struct_copy.c | 162 + source/frontend/gen/store/vstore.c | 127 +- source/frontend/gen/sym/attr_merge.c | 25 +- source/frontend/gen/sym/elfsym.c | 4 +- source/frontend/gen/sym/symtab.c | 35 +- source/frontend/gen/type/cast.c | 6 + source/frontend/gen/type/size.c | 7 + source/frontend/gen/value/load.c | 10 +- source/frontend/gen/value/longlong.c | 6 +- source/frontend/gen/value/strlit_pool.c | 2 +- source/frontend/gen/value/vstack.c | 3 +- source/frontend/tccasm.c | 7 + source/frontend/tccpp.c | 173 +- source/frontend/tcctok.h | 5 + source/include/tcc.h | 267 +- source/ir/Makefile | 10 +- source/ir/codegen.c | 1016 +- source/ir/core.h | 2 + source/ir/cross_jump.c | 1571 ++ source/ir/dead_def.c | 239 + source/ir/dump.c | 19 +- source/ir/frame.c | 2253 +++ source/ir/gen/config.c | 3 + source/ir/gen/params.c | 138 +- source/ir/gen/put.c | 54 +- source/ir/gen/state.c | 28 +- source/ir/machine_op.c | 9 +- source/ir/machine_op.h | 32 +- source/ir/regalloc.c | 6893 +------- source/ir/regalloc.h | 1 + source/ir/regalloc_entry.c | 1834 +++ source/ir/regalloc_phi.c | 2149 +++ source/ir/regalloc_post.c | 2018 +++ source/ir/regalloc_priv.h | 355 + source/ir/regalloc_scan.c | 1990 +++ source/ir/self_store.c | 275 + source/ir/ssa.c | 298 +- source/ir/ssa.h | 12 +- source/ir/tccir.h | 135 +- source/ir/tccir_operand.c | 8 + source/ir/tccir_operand.h | 18 + source/machine/Makefile | 2 +- source/machine/tccabi.h | 2 - source/machine/tccls.c | 77 +- source/machine/tccls.h | 3 + source/memory/Makefile | 2 +- source/obj/Makefile | 8 +- source/obj/elf.h | 2609 +-- source/obj/tccdbg.c | 21 +- source/obj/tccelf.c | 6322 +------ source/obj/tccelf_layout.c | 991 ++ source/obj/tccelf_ld.c | 1098 ++ source/obj/tccelf_load.c | 969 ++ source/obj/tccelf_output.c | 1096 ++ source/obj/tccelf_priv.h | 211 + source/obj/tccelf_reloc.c | 1156 ++ source/obj/tccelf_sym.c | 1085 ++ source/obj/tccyaff.c | 370 +- source/obj/tccyaff.h | 61 +- source/opt/Makefile | 5 +- source/opt/analysis/du_chains.c | 2 +- source/opt/engine/pipeline_table.c | 8 + source/opt/flat/Makefile | 13 +- source/opt/flat/cfg/jump_thread.c | 130 + source/opt/flat/dce/body_essential.c | 1 + source/opt/flat/dce/dead_addrvar_elim.c | 36 +- source/opt/flat/dce/orphan_cmp_elim.c | 5 +- source/opt/flat/dce/redundant_cmp.c | 34 + source/opt/flat/fusion/add_deref_fold.c | 14 + source/opt/flat/fusion/cmp_hi_only.c | 151 + source/opt/flat/fusion/indexed_chain.c | 6 +- source/opt/flat/fusion/lea_fold.c | 184 +- source/opt/flat/fusion/lea_rmw_fold.c | 1 + source/opt/flat/fusion/pack64.c | 10 +- source/opt/flat/fusion/shift64_dead_half.c | 8 +- source/opt/flat/fusion/stackoff_addr_cse.c | 23 +- source/opt/flat/fusion/zero_half64.c | 33 +- source/opt/flat/ipa/tu_dead_statics.c | 34 +- source/opt/flat/ipa/tu_func_summary.c | 173 +- source/opt/flat/ipa/tu_summary.h | 4 + source/opt/flat/loop/backedge_phi_hoist.c | 2 +- source/opt/flat/loop/const_sim.c | 10 +- source/opt/flat/loop/func_purity.c | 5 + source/opt/flat/loop/iv_analysis.c | 369 +- source/opt/flat/loop/iv_strength_reduction.c | 169 +- source/opt/flat/loop/licm.c | 131 +- source/opt/flat/loop/loop_detect.c | 31 +- source/opt/flat/loop/loop_exit_analysis.c | 58 +- source/opt/flat/loop/loop_ir_mutate.c | 43 + source/opt/flat/loop/loop_unroll.c | 256 +- source/opt/flat/loop/reroll.c | 1 + source/opt/flat/memory/bitfield_unit_narrow.c | 5 +- source/opt/flat/memory/block_copy_init.c | 246 +- source/opt/flat/memory/const_local_table.c | 520 + source/opt/flat/memory/dead_static_store.c | 77 +- source/opt/flat/memory/dead_temp_local.c | 15 + source/opt/flat/memory/dse.c | 125 +- source/opt/flat/memory/entry_store_prop.c | 178 +- .../flat/memory/invariant_global_load_hoist.c | 26 +- .../flat/memory/invariant_temp_deref_hoist.c | 500 - source/opt/flat/memory/local_copy_prop.c | 377 - .../flat/memory/memmove_to_indexed_stores.c | 13 +- source/opt/flat/memory/param_copy_alias.c | 569 + source/opt/flat/memory/ptr_local_fwd.c | 128 +- source/opt/flat/memory/sl_forward.c | 152 +- .../opt/flat/memory/slot_const_store_fold.c | 209 + .../opt/flat/memory/small_memset_to_store.c | 3 + source/opt/flat/memory/sra.c | 2886 ++++ source/opt/flat/memory/sret_nrvo.c | 849 + .../flat/memory/struct_copy_roundtrip_elim.c | 65 +- source/opt/flat/scalar/add_reassoc.c | 6 + source/opt/flat/scalar/branch.c | 162 +- source/opt/flat/scalar/branch_fold.c | 309 - source/opt/flat/scalar/call_result.c | 12 +- source/opt/flat/scalar/cmp_expr_fold.c | 10 + .../flat/scalar/complex_const_param_fold.c | 161 - source/opt/flat/scalar/float_narrow.c | 276 - source/opt/flat/scalar/global_init_prop.c | 5 +- source/opt/flat/scalar/inline_param_copy.c | 158 +- source/opt/flat/scalar/neg_chain_cse.c | 236 - source/opt/flat/scalar/single_value_tmp.c | 161 - source/opt/flat/scalar/stack_addr_fold.c | 12 +- source/opt/flat/scalar/symref_const_prop.c | 27 +- source/opt/include/licm.h | 7 +- source/opt/include/opt.h | 60 +- source/opt/include/opt_loop_utils.h | 28 +- source/opt/include/opt_utils.h | 1 + source/opt/ra/umaal_fusion.c | 366 + source/opt/ssa/Makefile | 1 + source/opt/ssa/cfg/vrp.c | 13 + source/opt/ssa/dce/dce_common.c | 12 + source/opt/ssa/dce/dce_common.h | 8 + source/opt/ssa/dce/dead_global_stores.c | 7 +- source/opt/ssa/dce/dead_overwrite_stores.c | 7 +- source/opt/ssa/dce/dead_var_stores.c | 9 + source/opt/ssa/dce/ret_path_frame_store.c | 3 +- source/opt/ssa/dce/stackloc_stores.c | 2 +- source/opt/ssa/dce/temp_worklist.c | 5 + source/opt/ssa/engine/driver.c | 2 + source/opt/ssa/engine/stack_resolve.c | 125 + source/opt/ssa/engine/use_def.c | 27 +- .../opt/ssa/include/opt/ssa/ssa_opt_helpers.h | 2 +- source/opt/ssa/include/ssa_opt.h | 19 + source/opt/ssa/loop/dead_loop.c | 9 +- source/opt/ssa/loop/loop_bottom_test.c | 577 + source/opt/ssa/loop/loop_unroll.c | 282 + source/opt/ssa/memory/load_cse.c | 10 +- source/opt/ssa/scalar/cmp_offset_fold.c | 3 +- source/opt/ssa/scalar/cprop.c | 44 + source/opt/ssa/scalar/fold.c | 207 + source/opt/ssa/scalar/narrow.c | 60 +- source/opt/ssa/scalar/reassoc.c | 3 + source/opt/ssa/scalar/sccp.c | 65 +- source/opt/ssa/scalar/setif_or_taut.c | 3 +- source/opt/ssa/scalar/strength.c | 8 +- source/opt/util/const_eval.c | 12 +- source/opt/util/expr_equal.c | 12 + source/opt/util/volatile_access.c | 12 + source/opt/util/vreg_def_use.c | 12 +- source/opt/util/vreg_query.c | 2 +- source/support/Makefile | 2 +- source/unity/arm_01.c | 5 + source/unity/arm_02.c | 4 + source/unity/frontend_01.c | 11 + source/unity/frontend_02.c | 25 + source/unity/frontend_03.c | 12 + source/unity/frontend_04.c | 15 + source/unity/frontend_05.c | 16 + source/unity/frontend_06.c | 10 + source/unity/ir_01.c | 34 + source/unity/ir_02.c | 20 + source/unity/ir_03.c | 28 + source/unity/ir_04.c | 23 + source/unity/ir_05.c | 19 + source/unity/machine_01.c | 7 + source/unity/memory_01.c | 3 + source/unity/obj_01.c | 7 + source/unity/obj_02.c | 11 + source/unity/obj_03.c | 63 + source/unity/opt_01.c | 74 + source/unity/opt_02.c | 56 + source/unity/opt_03.c | 31 + source/unity/opt_04.c | 62 + source/unity/opt_05.c | 33 + source/unity/opt_06.c | 39 + source/unity/opt_07.c | 33 + source/unity/opt_08.c | 49 + source/unity/opt_09.c | 25 + source/unity/opt_10.c | 49 + source/unity/opt_11.c | 51 + source/unity/opt_12.c | 44 + source/unity/opt_13.c | 47 + source/unity/opt_14.c | 34 + source/unity/opt_15.c | 55 + source/unity/opt_16.c | 31 + source/unity/thumb_01.c | 23 + source/unity/thumb_02.c | 63 + source/unity/thumb_03.c | 3 + source/unity/thumb_04.c | 11 + source/unity/unity.mk | 422 + tests/fuzz/batch_sweep.py | 7 + tests/ir_tests/112_builtin_puts.c | 4 +- tests/ir_tests/112_builtin_puts.expect | 2 +- .../466_alive_share_boundary_coholder.c | 87 + .../466_alive_share_boundary_coholder.expect | 1 + tests/ir_tests/472_pragma_pack_saved_bodies.c | 75 + .../472_pragma_pack_saved_bodies.expect | 2 + .../473_reassoc_unsigned_into_signed.c | 29 + .../473_reassoc_unsigned_into_signed.expect | 2 + .../474_late_reopt_owed_inline_bodies.c | 28 + .../474_late_reopt_owed_inline_bodies.expect | 1 + tests/ir_tests/475_drop_unused_statics.c | 64 + tests/ir_tests/475_drop_unused_statics.expect | 5 + tests/ir_tests/476_write_only_statics.c | 60 + tests/ir_tests/476_write_only_statics.expect | 2 + tests/ir_tests/477_frame_copy_coalescing.c | 91 + .../ir_tests/477_frame_copy_coalescing.expect | 2 + tests/ir_tests/478_inline_atomic_load_store.c | 57 + .../478_inline_atomic_load_store.expect | 2 + tests/ir_tests/479_dse_ambiguous_addr_copy.c | 87 + .../479_dse_ambiguous_addr_copy.expect | 7 + tests/ir_tests/480_fputs_printf_result_used.c | 49 + .../480_fputs_printf_result_used.expect | 6 + tests/ir_tests/481_narrow_local_arg_width.c | 60 + .../481_narrow_local_arg_width.expect | 2 + tests/ir_tests/482_sret_into_struct_param.c | 54 + .../482_sret_into_struct_param.expect | 3 + tests/ir_tests/483_narrow_temp_spill_word.c | 90 + .../483_narrow_temp_spill_word.expect | 3 + tests/ir_tests/484_struct_ternary_arg_copy.c | 85 + .../484_struct_ternary_arg_copy.expect | 6 + tests/ir_tests/485_frame_colour_struct_far.c | 70 + .../485_frame_colour_struct_far.expect | 2 + tests/ir_tests/486_addrtaken_escape_slot.c | 79 + .../ir_tests/486_addrtaken_escape_slot.expect | 2 + tests/ir_tests/487_tail_call_early_return.c | 48 + .../487_tail_call_early_return.expect | 2 + tests/ir_tests/488_sccp_var_addr_not_slot.c | 261 + .../488_sccp_var_addr_not_slot.expect | 7 + tests/ir_tests/489_reload_elim_pins_regs.c | 117 + .../ir_tests/489_reload_elim_pins_regs.expect | 4 + tests/ir_tests/490_dse_addr_through_var_add.c | 195 + .../490_dse_addr_through_var_add.expect | 6 + .../ir_tests/491_block_copy_last_word_live.c | 52 + .../491_block_copy_last_word_live.expect | 1 + .../492_struct_roundtrip_addr_escape.c | 48 + .../492_struct_roundtrip_addr_escape.expect | 1 + tests/ir_tests/493_sccp_var_store_not_slot.c | 177 + .../493_sccp_var_store_not_slot.expect | 4 + .../ir_tests/494_dead_addrvar_store_indexed.c | 45 + .../494_dead_addrvar_store_indexed.expect | 5 + .../495_stack_struct_arg_base_in_lr.c | 43 + .../495_stack_struct_arg_base_in_lr.expect | 1 + .../496_frame_colour_ptr_via_var_load.c | 87 + .../496_frame_colour_ptr_via_var_load.expect | 1 + .../ir_tests/497_late_reopt_keeps_icf_alias.c | 41 + .../497_late_reopt_keeps_icf_alias.expect | 1 + tests/ir_tests/498_aeabi_mem_div_runtime.c | 101 + .../ir_tests/498_aeabi_mem_div_runtime.expect | 2 + .../499_redundant_cmp_cbz_i64_flags.c | 72 + .../499_redundant_cmp_cbz_i64_flags.expect | 5 + tests/ir_tests/500_asm_lr_clobber_is_a_call.c | 34 + .../500_asm_lr_clobber_is_a_call.expect | 2 + .../501_struct_copy_through_pointer_local.c | 97 + ...1_struct_copy_through_pointer_local.expect | 4 + .../502_dead_loop_vinfo_grow_def_instr.c | 46 + .../502_dead_loop_vinfo_grow_def_instr.expect | 16 + .../503_coalesce_swap_stale_allocation.c | 105 + .../503_coalesce_swap_stale_allocation.expect | 3 + .../ir_tests/504_sra_wide_doubleword_fields.c | 124 + .../504_sra_wide_doubleword_fields.expect | 2 + tests/ir_tests/505_const_store_var_forward.c | 100 + .../505_const_store_var_forward.expect | 8 + .../ir_tests/506_inline_aligned_struct_copy.c | 106 + .../506_inline_aligned_struct_copy.expect | 13 + tests/ir_tests/507_widening_mul_fold.c | 117 + tests/ir_tests/507_widening_mul_fold.expect | 4 + tests/ir_tests/508_umaal_fusion.c | 184 + tests/ir_tests/508_umaal_fusion.expect | 3 + tests/ir_tests/509_narrow_temp_store_def.c | 96 + .../ir_tests/509_narrow_temp_store_def.expect | 2 + tests/ir_tests/510_known_zero_extend_fold.c | 90 + .../510_known_zero_extend_fold.expect | 5 + tests/ir_tests/511_late_setif_branch_fuse.c | 67 + .../511_late_setif_branch_fuse.expect | 3 + tests/ir_tests/512_nested_inline_ptr_return.c | 66 + .../512_nested_inline_ptr_return.expect | 4 + tests/ir_tests/513_param_across_block_copy.c | 42 + .../513_param_across_block_copy.expect | 4 + tests/ir_tests/514_inline_large_copies.c | 115 + tests/ir_tests/514_inline_large_copies.expect | 8 + tests/ir_tests/515_const_local_table_rodata.c | 138 + .../515_const_local_table_rodata.expect | 7 + .../516_symref_struct_arg_from_table.c | 35 + .../516_symref_struct_arg_from_table.expect | 1 + tests/ir_tests/517_sra_pointer_var_roots.c | 160 + .../ir_tests/517_sra_pointer_var_roots.expect | 9 + .../518_ssa_promote_entry_and_sra_fields.c | 166 + ...18_ssa_promote_entry_and_sra_fields.expect | 7 + tests/ir_tests/519_sra_struct_arg_words.c | 96 + .../ir_tests/519_sra_struct_arg_words.expect | 2 + .../ir_tests/520_sra_copy_and_result_buffer.c | 165 + .../520_sra_copy_and_result_buffer.expect | 9 + .../ir_tests/521_sra_struct_arg_words_large.c | 102 + .../521_sra_struct_arg_words_large.expect | 2 + .../522_sra_narrow_field_read_wider.c | 114 + .../522_sra_narrow_field_read_wider.expect | 1 + tests/ir_tests/522_unroll_zig_loop_tables.c | 137 + .../522_unroll_zig_loop_tables.expect | 5 + .../523_zig_accessor_chain_collapse.c | 295 + .../523_zig_accessor_chain_collapse.expect | 10 + tests/ir_tests/524_sra_far_struct_arg.c | 100 + tests/ir_tests/524_sra_far_struct_arg.expect | 5 + tests/ir_tests/525_cmp_hi_only.c | 124 + tests/ir_tests/525_cmp_hi_only.expect | 15 + tests/ir_tests/526_bottom_test_ptr_walk.c | 115 + .../ir_tests/526_bottom_test_ptr_walk.expect | 4 + .../ir_tests/527_landor_const_last_operand.c | 102 + .../527_landor_const_last_operand.expect | 6 + tests/ir_tests/528_sra_partial_fields.c | 137 + tests/ir_tests/528_sra_partial_fields.expect | 9 + tests/ir_tests/529_sra_pair_copy.c | 77 + tests/ir_tests/529_sra_pair_copy.expect | 7 + .../530_cbz_over_aligned_branch_target.c | 66 + .../530_cbz_over_aligned_branch_target.expect | 5 + tests/ir_tests/531_dyn_arg_window.c | 73 + tests/ir_tests/531_dyn_arg_window.expect | 6 + tests/ir_tests/532_unreachable_branches.c | 94 + .../ir_tests/532_unreachable_branches.expect | 18 + .../ir_tests/533_inline_scope_caller_object.c | 52 + .../533_inline_scope_caller_object.expect | 2 + .../534_sccp_entry_store_escaped_call.c | 43 + .../534_sccp_entry_store_escaped_call.expect | 3 + tests/ir_tests/535_inline_big_struct_param.c | 65 + .../535_inline_big_struct_param.expect | 6 + .../536_frame_relayout_retry_keeps_copies.c | 85 + ...6_frame_relayout_retry_keeps_copies.expect | 3 + tests/ir_tests/537_indexed_chain_deref_base.c | 43 + .../537_indexed_chain_deref_base.expect | 2 + .../538_const_local_table_zero_fill.c | 75 + .../538_const_local_table_zero_fill.expect | 4 + .../539_cprop_store_copy_inline_helper.c | 74 + .../539_cprop_store_copy_inline_helper.expect | 3 + .../540_coalesce_high_pressure_latch.c | 60 + .../540_coalesce_high_pressure_latch.expect | 6 + tests/ir_tests/541_param_home_fwd.c | 65 + tests/ir_tests/541_param_home_fwd.expect | 4 + tests/ir_tests/542_sret_nrvo.c | 86 + tests/ir_tests/542_sret_nrvo.expect | 4 + tests/ir_tests/543_dead_def_cbz_narrow_ext.c | 109 + .../543_dead_def_cbz_narrow_ext.expect | 16 + .../ir_tests/544_frame_word_pair_slot_copy.c | 70 + .../544_frame_word_pair_slot_copy.expect | 3 + tests/ir_tests/545_cross_jump_alloc.c | 147 + tests/ir_tests/545_cross_jump_alloc.expect | 16 + tests/ir_tests/546_ssa_symref_store_def.c | 45 + .../ir_tests/546_ssa_symref_store_def.expect | 5 + .../ir_tests/548_store_imm_scratch_not_base.c | 49 + .../548_store_imm_scratch_not_base.expect | 1 + tests/ir_tests/549_copy_stub_calls.c | 134 + tests/ir_tests/549_copy_stub_calls.expect | 33 + tests/ir_tests/600_sra_padded_partial_store.c | 99 + .../600_sra_padded_partial_store.expect | 11 + tests/ir_tests/601_overflow_builtins_narrow.c | 272 + .../601_overflow_builtins_narrow.expect | 1 + tests/ir_tests/602_outline_shared_windows.c | 87 + .../602_outline_shared_windows.expect | 9 + .../603_inline_outparam_record_split.c | 113 + .../603_inline_outparam_record_split.expect | 4 + tests/ir_tests/604_toysh_wildcard_literal.c | 163 + .../604_toysh_wildcard_literal.expect | 7 + tests/ir_tests/605_strchr_nul_terminator.c | 11 + .../ir_tests/605_strchr_nul_terminator.expect | 1 + tests/ir_tests/606_param_ne_local_address.c | 56 + .../606_param_ne_local_address.expect | 2 + .../607_assign_not_of_call_writing_lhs.c | 55 + .../607_assign_not_of_call_writing_lhs.expect | 5 + tests/ir_tests/abi_mix.h | 140 + tests/ir_tests/abi_mix_a.c | 5 + tests/ir_tests/abi_mix_b.c | 5 + tests/ir_tests/abi_mix_main.c | 10 + tests/ir_tests/abi_mix_main.expect | 14 + tests/ir_tests/asm/branch_over_pool_window.c | 188 + tests/ir_tests/asm/cmp_neg_small.c | 14 + tests/ir_tests/asm/cmp_type_extreme.c | 12 + tests/ir_tests/asm/func_align_literal.c | 27 + tests/ir_tests/asm/has_builtin.c | 36 + tests/ir_tests/asm/hi_callee_pref.c | 22 + tests/ir_tests/asm/it_mov_narrow.c | 5 + tests/ir_tests/asm/shift64_var.c | 6 + tests/ir_tests/asm/size_tier.c | 51 + tests/ir_tests/asm/volatile_access.c | 10 + .../ir_tests/bug_addr_temp_reuse_before_def.c | 39 + .../bug_addr_temp_reuse_before_def.expect | 1 + .../ir_tests/bug_alive_share_evicted_owner.c | 83 + .../bug_alive_share_evicted_owner.expect | 1 + tests/ir_tests/bug_asm_clobber_live_values.c | 37 + .../bug_asm_clobber_live_values.expect | 2 + tests/ir_tests/bug_asm_regvar_local.c | 56 + tests/ir_tests/bug_asm_regvar_local.expect | 3 + tests/ir_tests/bug_block_copy_init_narrow.c | 49 + .../bug_block_copy_init_narrow.expect | 3 + tests/ir_tests/bug_branch_narrow_rehearsal.c | 101 + .../bug_branch_narrow_rehearsal.expect | 3 + tests/ir_tests/bug_dse_indexed_store_escape.c | 49 + .../bug_dse_indexed_store_escape.expect | 1 + tests/ir_tests/bug_modref_direct_block_copy.c | 37 + .../bug_modref_direct_block_copy.expect | 1 + .../ir_tests/bug_narrow_field_store_forward.c | 60 + .../bug_narrow_field_store_forward.expect | 2 + tests/ir_tests/bug_narrow_local_slot_width.c | 85 + .../bug_narrow_local_slot_width.expect | 3 + tests/ir_tests/bug_postinc_call_arg_deref.c | 47 + .../bug_postinc_call_arg_deref.expect | 4 + .../ir_tests/bug_sra_narrow_field_undefined.c | 88 + .../bug_sra_narrow_field_undefined.expect | 1 + tests/ir_tests/bug_vla_struct_array.c | 42 + tests/ir_tests/bug_vla_struct_array.expect | 2 + .../block_copy_init/clear_struct.expected | 13 +- .../dead_ternary_diamond.expected | 19 +- .../post_forward_arith_chain.expected | 4 +- .../inlined_struct_field_check.expected | 10 +- .../mask_shift_branch_fold.expected | 4 +- tests/ir_tests/nested_capture_64bit.c | 63 + tests/ir_tests/nested_capture_64bit.expect | 4 + tests/ir_tests/nested_capture_param_byref.c | 44 + .../nested_capture_param_byref.expect | 3 + tests/ir_tests/nested_capture_param_mixed.c | 30 + .../nested_capture_param_mixed.expect | 2 + .../nested_capture_param_reg_noinline.c | 22 + .../nested_capture_param_reg_noinline.expect | 1 + tests/ir_tests/nested_capture_param_stack.c | 23 + .../nested_capture_param_stack.expect | 1 + tests/ir_tests/nested_capture_two_level.c | 41 + .../ir_tests/nested_capture_two_level.expect | 2 + tests/ir_tests/qemu/mps2-an505/Makefile | 4 +- .../ir_tests/qemu/mps2-an505/linker_script.ld | 6 +- tests/ir_tests/qemu_run.py | 9 +- .../ir_tests/sl_forward_load_redefines_var.c | 46 + .../sl_forward_load_redefines_var.expect | 1 + tests/ir_tests/test_alloca_live_size.c | 35 + tests/ir_tests/test_alloca_live_size.expect | 1 + tests/ir_tests/test_call_buffer_copy_alias.c | 130 + .../test_call_buffer_copy_alias.expect | 4 + tests/ir_tests/test_codegen_asm.py | 156 +- tests/ir_tests/test_complex_param_parts.c | 38 + .../ir_tests/test_complex_param_parts.expect | 3 + tests/ir_tests/test_cross_jump.c | 116 + tests/ir_tests/test_cross_jump.expect | 4 + tests/ir_tests/test_drop_unused_statics.py | 198 + tests/ir_tests/test_dse_object_ranges.c | 59 + tests/ir_tests/test_dse_object_ranges.expect | 1 + tests/ir_tests/test_dse_wide.c | 44 + tests/ir_tests/test_dse_wide.expect | 1 + tests/ir_tests/test_entry_store_var_walk.c | 39 + .../ir_tests/test_entry_store_var_walk.expect | 1 + tests/ir_tests/test_errunion_bool_checks.c | 135 + .../ir_tests/test_errunion_bool_checks.expect | 3 + tests/ir_tests/test_fp_mode_selection.py | 10 +- tests/ir_tests/test_frame_block_copy.c | 66 + tests/ir_tests/test_frame_block_copy.expect | 2 + tests/ir_tests/test_frame_colour.c | 269 + tests/ir_tests/test_frame_colour.expect | 10 + tests/ir_tests/test_frame_loop_lifetimes.c | 90 + .../ir_tests/test_frame_loop_lifetimes.expect | 1 + tests/ir_tests/test_frame_relayout.c | 152 + tests/ir_tests/test_frame_relayout.expect | 7 + tests/ir_tests/test_icf_folding.c | 125 + tests/ir_tests/test_icf_folding.expect | 7 + tests/ir_tests/test_inline_called_once.c | 130 + tests/ir_tests/test_inline_called_once.expect | 4 + tests/ir_tests/test_ldm_block_copy.c | 163 + tests/ir_tests/test_ldm_block_copy.expect | 10 + tests/ir_tests/test_narrow_call_ext.c | 49 + tests/ir_tests/test_narrow_call_ext.expect | 2 + tests/ir_tests/test_param_copy_alias.c | 111 + tests/ir_tests/test_param_copy_alias.expect | 7 + tests/ir_tests/test_ptr_local_fwd.c | 141 + tests/ir_tests/test_ptr_local_fwd.expect | 7 + tests/ir_tests/test_pure_forward.c | 63 + tests/ir_tests/test_pure_forward.expect | 5 + tests/ir_tests/test_qemu.py | 398 +- tests/ir_tests/test_r9_static_forward_call.c | 52 + .../test_r9_static_forward_call.expect | 2 + tests/ir_tests/test_return_address.c | 168 + tests/ir_tests/test_return_address.expect | 2 + tests/ir_tests/test_self_store.c | 134 + tests/ir_tests/test_self_store.expect | 7 + tests/ir_tests/test_slot_const_store_fold.c | 59 + .../test_slot_const_store_fold.expect | 4 + tests/ir_tests/test_small_aggregate_copy.c | 114 + .../ir_tests/test_small_aggregate_copy.expect | 1 + tests/ir_tests/test_spill_slot_reuse.c | 51 + tests/ir_tests/test_spill_slot_reuse.expect | 2 + tests/ir_tests/test_sra.c | 155 + tests/ir_tests/test_sra.expect | 7 + tests/ir_tests/test_stack_deref_fold.c | 110 + tests/ir_tests/test_stack_deref_fold.expect | 3 + tests/ir_tests/test_stack_params_in_regs.c | 120 + .../ir_tests/test_stack_params_in_regs.expect | 11 + tests/ir_tests/test_struct_arg_stack.c | 76 + tests/ir_tests/test_struct_arg_stack.expect | 4 + tests/ir_tests/test_struct_far_frame.c | 47 + tests/ir_tests/test_struct_far_frame.expect | 2 + tests/ir_tests/test_struct_param_pass_on.c | 47 + .../ir_tests/test_struct_param_pass_on.expect | 2 + tests/ir_tests/test_struct_stack_ldm.c | 64 + tests/ir_tests/test_struct_stack_ldm.expect | 1 + tests/ir_tests/test_switch_case_merge.c | 221 + tests/ir_tests/test_switch_case_merge.expect | 10 + tests/ir_tests/test_tentative_definitions.c | 84 + .../test_tentative_definitions.expect | 5 + tests/ir_tests/test_yaff_section_padding.py | 15 +- .../symbols/01_binding_static_vs_global.c | 7 +- tests/linker/symbols/07_symtab_order.c | 9 +- tests/linker/test_linker.py | 14 +- tests/pp/has-attribute.c | 41 + tests/pp/has-attribute.expect | 10 + tests/selfhost/test_selfhost_compile.py | 37 +- tests/thumb/armv8m/data_processing_test.py | 3 + tests/thumb/armv8m/test_umaal.S | 15 + tests/unit/arm/armv8m/Makefile | 44 +- tests/unit/arm/armv8m/codegen_mop_stubs.c | 107 + tests/unit/arm/armv8m/libtcc_api_stubs.c | 1 + tests/unit/arm/armv8m/stubs.c | 2 +- .../arm/armv8m/stubs_gen_machine_fallback.c | 6 + tests/unit/arm/armv8m/tcc_state_stub.c | 44 + tests/unit/arm/armv8m/tccpp_stubs.c | 8 + tests/unit/arm/armv8m/test_arm_aapcs.c | 83 +- tests/unit/arm/armv8m/test_arm_thumb_asm.c | 6 +- .../unit/arm/armv8m/test_gen_dispatch_smoke.c | 28 +- tests/unit/arm/armv8m/test_gen_mem.c | 37 + tests/unit/arm/armv8m/test_metamorphic.c | 6 - tests/unit/arm/armv8m/test_opt_constfold.c | 309 +- tests/unit/arm/armv8m/test_opt_constprop.c | 148 - tests/unit/arm/armv8m/test_opt_float_branch.c | 367 - tests/unit/arm/armv8m/test_opt_loop_utils.c | 88 +- tests/unit/arm/armv8m/test_opt_memory_extra.c | 186 +- tests/unit/arm/armv8m/test_opt_neg_chain.c | 402 - .../arm/armv8m/test_opt_region_merge_case.c | 205 + tests/unit/arm/armv8m/test_opt_self_store.c | 152 + tests/unit/arm/armv8m/test_opt_utils.c | 5 +- tests/unit/arm/armv8m/test_ssa_opt_dce.c | 16 +- tests/unit/arm/armv8m/test_ssa_opt_load_cse.c | 39 +- tests/unit/arm/armv8m/test_tccgen.c | 50 +- tests/unit/arm/armv8m/test_tccls.c | 32 + tests/unit/arm/armv8m/test_tccyaff.c | 59 + 626 files changed, 71268 insertions(+), 34880 deletions(-) create mode 100644 docs/bugs/README.md create mode 100644 docs/bugs/ctags-entry-c-kconfig-choice-name.md delete mode 100644 docs/bugs/dce-dead-var-store-volatile-lval-src.md delete mode 100644 docs/bugs/load-cse-lval-store-src-tracked-as-value.md create mode 100644 docs/bugs/selfhost-const-compare-fold-breaks-native-tcc.md delete mode 100644 docs/bugs/volatile-local-folded-to-constant.md create mode 100644 docs/plan_switch_case_constant_merge.md create mode 100644 docs/zig_frame_promotion_plan.md create mode 100644 lib/arm_copystub.S create mode 100644 lib/arm_mem.S create mode 100755 scripts/gen_unity.py create mode 100644 source/backend/arch/arm/thumb/arm-thumb-alu.c create mode 100644 source/backend/arch/arm/thumb/arm-thumb-call.c create mode 100644 source/backend/arch/arm/thumb/arm-thumb-emit.c create mode 100644 source/backend/arch/arm/thumb/arm-thumb-fp.c create mode 100644 source/backend/arch/arm/thumb/arm-thumb-frame.c create mode 100644 source/backend/arch/arm/thumb/arm-thumb-gen.h create mode 100644 source/backend/arch/arm/thumb/arm-thumb-mem.c create mode 100644 source/backend/arch/arm/thumb/thumb-outline.c create mode 100644 source/backend/arch/unity.mk create mode 100644 source/backend/generators/icf.c create mode 100644 source/ir/cross_jump.c create mode 100644 source/ir/dead_def.c create mode 100644 source/ir/frame.c create mode 100644 source/ir/regalloc_entry.c create mode 100644 source/ir/regalloc_phi.c create mode 100644 source/ir/regalloc_post.c create mode 100644 source/ir/regalloc_priv.h create mode 100644 source/ir/regalloc_scan.c create mode 100644 source/ir/self_store.c create mode 100644 source/obj/tccelf_layout.c create mode 100644 source/obj/tccelf_ld.c create mode 100644 source/obj/tccelf_load.c create mode 100644 source/obj/tccelf_output.c create mode 100644 source/obj/tccelf_priv.h create mode 100644 source/obj/tccelf_reloc.c create mode 100644 source/obj/tccelf_sym.c create mode 100644 source/opt/flat/fusion/cmp_hi_only.c create mode 100644 source/opt/flat/memory/const_local_table.c delete mode 100644 source/opt/flat/memory/invariant_temp_deref_hoist.c delete mode 100644 source/opt/flat/memory/local_copy_prop.c create mode 100644 source/opt/flat/memory/param_copy_alias.c create mode 100644 source/opt/flat/memory/slot_const_store_fold.c create mode 100644 source/opt/flat/memory/sra.c create mode 100644 source/opt/flat/memory/sret_nrvo.c delete mode 100644 source/opt/flat/scalar/branch_fold.c delete mode 100644 source/opt/flat/scalar/complex_const_param_fold.c delete mode 100644 source/opt/flat/scalar/float_narrow.c delete mode 100644 source/opt/flat/scalar/neg_chain_cse.c delete mode 100644 source/opt/flat/scalar/single_value_tmp.c create mode 100644 source/opt/ra/umaal_fusion.c create mode 100644 source/opt/ssa/loop/loop_bottom_test.c create mode 100644 source/unity/arm_01.c create mode 100644 source/unity/arm_02.c create mode 100644 source/unity/frontend_01.c create mode 100644 source/unity/frontend_02.c create mode 100644 source/unity/frontend_03.c create mode 100644 source/unity/frontend_04.c create mode 100644 source/unity/frontend_05.c create mode 100644 source/unity/frontend_06.c create mode 100644 source/unity/ir_01.c create mode 100644 source/unity/ir_02.c create mode 100644 source/unity/ir_03.c create mode 100644 source/unity/ir_04.c create mode 100644 source/unity/ir_05.c create mode 100644 source/unity/machine_01.c create mode 100644 source/unity/memory_01.c create mode 100644 source/unity/obj_01.c create mode 100644 source/unity/obj_02.c create mode 100644 source/unity/obj_03.c create mode 100644 source/unity/opt_01.c create mode 100644 source/unity/opt_02.c create mode 100644 source/unity/opt_03.c create mode 100644 source/unity/opt_04.c create mode 100644 source/unity/opt_05.c create mode 100644 source/unity/opt_06.c create mode 100644 source/unity/opt_07.c create mode 100644 source/unity/opt_08.c create mode 100644 source/unity/opt_09.c create mode 100644 source/unity/opt_10.c create mode 100644 source/unity/opt_11.c create mode 100644 source/unity/opt_12.c create mode 100644 source/unity/opt_13.c create mode 100644 source/unity/opt_14.c create mode 100644 source/unity/opt_15.c create mode 100644 source/unity/opt_16.c create mode 100644 source/unity/thumb_01.c create mode 100644 source/unity/thumb_02.c create mode 100644 source/unity/thumb_03.c create mode 100644 source/unity/thumb_04.c create mode 100644 source/unity/unity.mk create mode 100644 tests/ir_tests/466_alive_share_boundary_coholder.c create mode 100644 tests/ir_tests/466_alive_share_boundary_coholder.expect create mode 100644 tests/ir_tests/472_pragma_pack_saved_bodies.c create mode 100644 tests/ir_tests/472_pragma_pack_saved_bodies.expect create mode 100644 tests/ir_tests/473_reassoc_unsigned_into_signed.c create mode 100644 tests/ir_tests/473_reassoc_unsigned_into_signed.expect create mode 100644 tests/ir_tests/474_late_reopt_owed_inline_bodies.c create mode 100644 tests/ir_tests/474_late_reopt_owed_inline_bodies.expect create mode 100644 tests/ir_tests/475_drop_unused_statics.c create mode 100644 tests/ir_tests/475_drop_unused_statics.expect create mode 100644 tests/ir_tests/476_write_only_statics.c create mode 100644 tests/ir_tests/476_write_only_statics.expect create mode 100644 tests/ir_tests/477_frame_copy_coalescing.c create mode 100644 tests/ir_tests/477_frame_copy_coalescing.expect create mode 100644 tests/ir_tests/478_inline_atomic_load_store.c create mode 100644 tests/ir_tests/478_inline_atomic_load_store.expect create mode 100644 tests/ir_tests/479_dse_ambiguous_addr_copy.c create mode 100644 tests/ir_tests/479_dse_ambiguous_addr_copy.expect create mode 100644 tests/ir_tests/480_fputs_printf_result_used.c create mode 100644 tests/ir_tests/480_fputs_printf_result_used.expect create mode 100644 tests/ir_tests/481_narrow_local_arg_width.c create mode 100644 tests/ir_tests/481_narrow_local_arg_width.expect create mode 100644 tests/ir_tests/482_sret_into_struct_param.c create mode 100644 tests/ir_tests/482_sret_into_struct_param.expect create mode 100644 tests/ir_tests/483_narrow_temp_spill_word.c create mode 100644 tests/ir_tests/483_narrow_temp_spill_word.expect create mode 100644 tests/ir_tests/484_struct_ternary_arg_copy.c create mode 100644 tests/ir_tests/484_struct_ternary_arg_copy.expect create mode 100644 tests/ir_tests/485_frame_colour_struct_far.c create mode 100644 tests/ir_tests/485_frame_colour_struct_far.expect create mode 100644 tests/ir_tests/486_addrtaken_escape_slot.c create mode 100644 tests/ir_tests/486_addrtaken_escape_slot.expect create mode 100644 tests/ir_tests/487_tail_call_early_return.c create mode 100644 tests/ir_tests/487_tail_call_early_return.expect create mode 100644 tests/ir_tests/488_sccp_var_addr_not_slot.c create mode 100644 tests/ir_tests/488_sccp_var_addr_not_slot.expect create mode 100644 tests/ir_tests/489_reload_elim_pins_regs.c create mode 100644 tests/ir_tests/489_reload_elim_pins_regs.expect create mode 100644 tests/ir_tests/490_dse_addr_through_var_add.c create mode 100644 tests/ir_tests/490_dse_addr_through_var_add.expect create mode 100644 tests/ir_tests/491_block_copy_last_word_live.c create mode 100644 tests/ir_tests/491_block_copy_last_word_live.expect create mode 100644 tests/ir_tests/492_struct_roundtrip_addr_escape.c create mode 100644 tests/ir_tests/492_struct_roundtrip_addr_escape.expect create mode 100644 tests/ir_tests/493_sccp_var_store_not_slot.c create mode 100644 tests/ir_tests/493_sccp_var_store_not_slot.expect create mode 100644 tests/ir_tests/494_dead_addrvar_store_indexed.c create mode 100644 tests/ir_tests/494_dead_addrvar_store_indexed.expect create mode 100644 tests/ir_tests/495_stack_struct_arg_base_in_lr.c create mode 100644 tests/ir_tests/495_stack_struct_arg_base_in_lr.expect create mode 100644 tests/ir_tests/496_frame_colour_ptr_via_var_load.c create mode 100644 tests/ir_tests/496_frame_colour_ptr_via_var_load.expect create mode 100644 tests/ir_tests/497_late_reopt_keeps_icf_alias.c create mode 100644 tests/ir_tests/497_late_reopt_keeps_icf_alias.expect create mode 100644 tests/ir_tests/498_aeabi_mem_div_runtime.c create mode 100644 tests/ir_tests/498_aeabi_mem_div_runtime.expect create mode 100644 tests/ir_tests/499_redundant_cmp_cbz_i64_flags.c create mode 100644 tests/ir_tests/499_redundant_cmp_cbz_i64_flags.expect create mode 100644 tests/ir_tests/500_asm_lr_clobber_is_a_call.c create mode 100644 tests/ir_tests/500_asm_lr_clobber_is_a_call.expect create mode 100644 tests/ir_tests/501_struct_copy_through_pointer_local.c create mode 100644 tests/ir_tests/501_struct_copy_through_pointer_local.expect create mode 100644 tests/ir_tests/502_dead_loop_vinfo_grow_def_instr.c create mode 100644 tests/ir_tests/502_dead_loop_vinfo_grow_def_instr.expect create mode 100644 tests/ir_tests/503_coalesce_swap_stale_allocation.c create mode 100644 tests/ir_tests/503_coalesce_swap_stale_allocation.expect create mode 100644 tests/ir_tests/504_sra_wide_doubleword_fields.c create mode 100644 tests/ir_tests/504_sra_wide_doubleword_fields.expect create mode 100644 tests/ir_tests/505_const_store_var_forward.c create mode 100644 tests/ir_tests/505_const_store_var_forward.expect create mode 100644 tests/ir_tests/506_inline_aligned_struct_copy.c create mode 100644 tests/ir_tests/506_inline_aligned_struct_copy.expect create mode 100644 tests/ir_tests/507_widening_mul_fold.c create mode 100644 tests/ir_tests/507_widening_mul_fold.expect create mode 100644 tests/ir_tests/508_umaal_fusion.c create mode 100644 tests/ir_tests/508_umaal_fusion.expect create mode 100644 tests/ir_tests/509_narrow_temp_store_def.c create mode 100644 tests/ir_tests/509_narrow_temp_store_def.expect create mode 100644 tests/ir_tests/510_known_zero_extend_fold.c create mode 100644 tests/ir_tests/510_known_zero_extend_fold.expect create mode 100644 tests/ir_tests/511_late_setif_branch_fuse.c create mode 100644 tests/ir_tests/511_late_setif_branch_fuse.expect create mode 100644 tests/ir_tests/512_nested_inline_ptr_return.c create mode 100644 tests/ir_tests/512_nested_inline_ptr_return.expect create mode 100644 tests/ir_tests/513_param_across_block_copy.c create mode 100644 tests/ir_tests/513_param_across_block_copy.expect create mode 100644 tests/ir_tests/514_inline_large_copies.c create mode 100644 tests/ir_tests/514_inline_large_copies.expect create mode 100644 tests/ir_tests/515_const_local_table_rodata.c create mode 100644 tests/ir_tests/515_const_local_table_rodata.expect create mode 100644 tests/ir_tests/516_symref_struct_arg_from_table.c create mode 100644 tests/ir_tests/516_symref_struct_arg_from_table.expect create mode 100644 tests/ir_tests/517_sra_pointer_var_roots.c create mode 100644 tests/ir_tests/517_sra_pointer_var_roots.expect create mode 100644 tests/ir_tests/518_ssa_promote_entry_and_sra_fields.c create mode 100644 tests/ir_tests/518_ssa_promote_entry_and_sra_fields.expect create mode 100644 tests/ir_tests/519_sra_struct_arg_words.c create mode 100644 tests/ir_tests/519_sra_struct_arg_words.expect create mode 100644 tests/ir_tests/520_sra_copy_and_result_buffer.c create mode 100644 tests/ir_tests/520_sra_copy_and_result_buffer.expect create mode 100644 tests/ir_tests/521_sra_struct_arg_words_large.c create mode 100644 tests/ir_tests/521_sra_struct_arg_words_large.expect create mode 100644 tests/ir_tests/522_sra_narrow_field_read_wider.c create mode 100644 tests/ir_tests/522_sra_narrow_field_read_wider.expect create mode 100644 tests/ir_tests/522_unroll_zig_loop_tables.c create mode 100644 tests/ir_tests/522_unroll_zig_loop_tables.expect create mode 100644 tests/ir_tests/523_zig_accessor_chain_collapse.c create mode 100644 tests/ir_tests/523_zig_accessor_chain_collapse.expect create mode 100644 tests/ir_tests/524_sra_far_struct_arg.c create mode 100644 tests/ir_tests/524_sra_far_struct_arg.expect create mode 100644 tests/ir_tests/525_cmp_hi_only.c create mode 100644 tests/ir_tests/525_cmp_hi_only.expect create mode 100644 tests/ir_tests/526_bottom_test_ptr_walk.c create mode 100644 tests/ir_tests/526_bottom_test_ptr_walk.expect create mode 100644 tests/ir_tests/527_landor_const_last_operand.c create mode 100644 tests/ir_tests/527_landor_const_last_operand.expect create mode 100644 tests/ir_tests/528_sra_partial_fields.c create mode 100644 tests/ir_tests/528_sra_partial_fields.expect create mode 100644 tests/ir_tests/529_sra_pair_copy.c create mode 100644 tests/ir_tests/529_sra_pair_copy.expect create mode 100644 tests/ir_tests/530_cbz_over_aligned_branch_target.c create mode 100644 tests/ir_tests/530_cbz_over_aligned_branch_target.expect create mode 100644 tests/ir_tests/531_dyn_arg_window.c create mode 100644 tests/ir_tests/531_dyn_arg_window.expect create mode 100644 tests/ir_tests/532_unreachable_branches.c create mode 100644 tests/ir_tests/532_unreachable_branches.expect create mode 100644 tests/ir_tests/533_inline_scope_caller_object.c create mode 100644 tests/ir_tests/533_inline_scope_caller_object.expect create mode 100644 tests/ir_tests/534_sccp_entry_store_escaped_call.c create mode 100644 tests/ir_tests/534_sccp_entry_store_escaped_call.expect create mode 100644 tests/ir_tests/535_inline_big_struct_param.c create mode 100644 tests/ir_tests/535_inline_big_struct_param.expect create mode 100644 tests/ir_tests/536_frame_relayout_retry_keeps_copies.c create mode 100644 tests/ir_tests/536_frame_relayout_retry_keeps_copies.expect create mode 100644 tests/ir_tests/537_indexed_chain_deref_base.c create mode 100644 tests/ir_tests/537_indexed_chain_deref_base.expect create mode 100644 tests/ir_tests/538_const_local_table_zero_fill.c create mode 100644 tests/ir_tests/538_const_local_table_zero_fill.expect create mode 100644 tests/ir_tests/539_cprop_store_copy_inline_helper.c create mode 100644 tests/ir_tests/539_cprop_store_copy_inline_helper.expect create mode 100644 tests/ir_tests/540_coalesce_high_pressure_latch.c create mode 100644 tests/ir_tests/540_coalesce_high_pressure_latch.expect create mode 100644 tests/ir_tests/541_param_home_fwd.c create mode 100644 tests/ir_tests/541_param_home_fwd.expect create mode 100644 tests/ir_tests/542_sret_nrvo.c create mode 100644 tests/ir_tests/542_sret_nrvo.expect create mode 100644 tests/ir_tests/543_dead_def_cbz_narrow_ext.c create mode 100644 tests/ir_tests/543_dead_def_cbz_narrow_ext.expect create mode 100644 tests/ir_tests/544_frame_word_pair_slot_copy.c create mode 100644 tests/ir_tests/544_frame_word_pair_slot_copy.expect create mode 100644 tests/ir_tests/545_cross_jump_alloc.c create mode 100644 tests/ir_tests/545_cross_jump_alloc.expect create mode 100644 tests/ir_tests/546_ssa_symref_store_def.c create mode 100644 tests/ir_tests/546_ssa_symref_store_def.expect create mode 100644 tests/ir_tests/548_store_imm_scratch_not_base.c create mode 100644 tests/ir_tests/548_store_imm_scratch_not_base.expect create mode 100644 tests/ir_tests/549_copy_stub_calls.c create mode 100644 tests/ir_tests/549_copy_stub_calls.expect create mode 100644 tests/ir_tests/600_sra_padded_partial_store.c create mode 100644 tests/ir_tests/600_sra_padded_partial_store.expect create mode 100644 tests/ir_tests/601_overflow_builtins_narrow.c create mode 100644 tests/ir_tests/601_overflow_builtins_narrow.expect create mode 100644 tests/ir_tests/602_outline_shared_windows.c create mode 100644 tests/ir_tests/602_outline_shared_windows.expect create mode 100644 tests/ir_tests/603_inline_outparam_record_split.c create mode 100644 tests/ir_tests/603_inline_outparam_record_split.expect create mode 100644 tests/ir_tests/604_toysh_wildcard_literal.c create mode 100644 tests/ir_tests/604_toysh_wildcard_literal.expect create mode 100644 tests/ir_tests/605_strchr_nul_terminator.c create mode 100644 tests/ir_tests/605_strchr_nul_terminator.expect create mode 100644 tests/ir_tests/606_param_ne_local_address.c create mode 100644 tests/ir_tests/606_param_ne_local_address.expect create mode 100644 tests/ir_tests/607_assign_not_of_call_writing_lhs.c create mode 100644 tests/ir_tests/607_assign_not_of_call_writing_lhs.expect create mode 100644 tests/ir_tests/abi_mix.h create mode 100644 tests/ir_tests/abi_mix_a.c create mode 100644 tests/ir_tests/abi_mix_b.c create mode 100644 tests/ir_tests/abi_mix_main.c create mode 100644 tests/ir_tests/abi_mix_main.expect create mode 100644 tests/ir_tests/asm/branch_over_pool_window.c create mode 100644 tests/ir_tests/asm/cmp_neg_small.c create mode 100644 tests/ir_tests/asm/cmp_type_extreme.c create mode 100644 tests/ir_tests/asm/func_align_literal.c create mode 100644 tests/ir_tests/asm/has_builtin.c create mode 100644 tests/ir_tests/asm/hi_callee_pref.c create mode 100644 tests/ir_tests/asm/it_mov_narrow.c create mode 100644 tests/ir_tests/asm/shift64_var.c create mode 100644 tests/ir_tests/asm/size_tier.c create mode 100644 tests/ir_tests/bug_addr_temp_reuse_before_def.c create mode 100644 tests/ir_tests/bug_addr_temp_reuse_before_def.expect create mode 100644 tests/ir_tests/bug_alive_share_evicted_owner.c create mode 100644 tests/ir_tests/bug_alive_share_evicted_owner.expect create mode 100644 tests/ir_tests/bug_asm_clobber_live_values.c create mode 100644 tests/ir_tests/bug_asm_clobber_live_values.expect create mode 100644 tests/ir_tests/bug_asm_regvar_local.c create mode 100644 tests/ir_tests/bug_asm_regvar_local.expect create mode 100644 tests/ir_tests/bug_block_copy_init_narrow.c create mode 100644 tests/ir_tests/bug_block_copy_init_narrow.expect create mode 100644 tests/ir_tests/bug_branch_narrow_rehearsal.c create mode 100644 tests/ir_tests/bug_branch_narrow_rehearsal.expect create mode 100644 tests/ir_tests/bug_dse_indexed_store_escape.c create mode 100644 tests/ir_tests/bug_dse_indexed_store_escape.expect create mode 100644 tests/ir_tests/bug_modref_direct_block_copy.c create mode 100644 tests/ir_tests/bug_modref_direct_block_copy.expect create mode 100644 tests/ir_tests/bug_narrow_field_store_forward.c create mode 100644 tests/ir_tests/bug_narrow_field_store_forward.expect create mode 100644 tests/ir_tests/bug_narrow_local_slot_width.c create mode 100644 tests/ir_tests/bug_narrow_local_slot_width.expect create mode 100644 tests/ir_tests/bug_postinc_call_arg_deref.c create mode 100644 tests/ir_tests/bug_postinc_call_arg_deref.expect create mode 100644 tests/ir_tests/bug_sra_narrow_field_undefined.c create mode 100644 tests/ir_tests/bug_sra_narrow_field_undefined.expect create mode 100644 tests/ir_tests/bug_vla_struct_array.c create mode 100644 tests/ir_tests/bug_vla_struct_array.expect create mode 100644 tests/ir_tests/nested_capture_64bit.c create mode 100644 tests/ir_tests/nested_capture_64bit.expect create mode 100644 tests/ir_tests/nested_capture_param_byref.c create mode 100644 tests/ir_tests/nested_capture_param_byref.expect create mode 100644 tests/ir_tests/nested_capture_param_mixed.c create mode 100644 tests/ir_tests/nested_capture_param_mixed.expect create mode 100644 tests/ir_tests/nested_capture_param_reg_noinline.c create mode 100644 tests/ir_tests/nested_capture_param_reg_noinline.expect create mode 100644 tests/ir_tests/nested_capture_param_stack.c create mode 100644 tests/ir_tests/nested_capture_param_stack.expect create mode 100644 tests/ir_tests/nested_capture_two_level.c create mode 100644 tests/ir_tests/nested_capture_two_level.expect create mode 100644 tests/ir_tests/sl_forward_load_redefines_var.c create mode 100644 tests/ir_tests/sl_forward_load_redefines_var.expect create mode 100644 tests/ir_tests/test_alloca_live_size.c create mode 100644 tests/ir_tests/test_alloca_live_size.expect create mode 100644 tests/ir_tests/test_call_buffer_copy_alias.c create mode 100644 tests/ir_tests/test_call_buffer_copy_alias.expect create mode 100644 tests/ir_tests/test_complex_param_parts.c create mode 100644 tests/ir_tests/test_complex_param_parts.expect create mode 100644 tests/ir_tests/test_cross_jump.c create mode 100644 tests/ir_tests/test_cross_jump.expect create mode 100644 tests/ir_tests/test_drop_unused_statics.py create mode 100644 tests/ir_tests/test_dse_object_ranges.c create mode 100644 tests/ir_tests/test_dse_object_ranges.expect create mode 100644 tests/ir_tests/test_dse_wide.c create mode 100644 tests/ir_tests/test_dse_wide.expect create mode 100644 tests/ir_tests/test_entry_store_var_walk.c create mode 100644 tests/ir_tests/test_entry_store_var_walk.expect create mode 100644 tests/ir_tests/test_errunion_bool_checks.c create mode 100644 tests/ir_tests/test_errunion_bool_checks.expect create mode 100644 tests/ir_tests/test_frame_block_copy.c create mode 100644 tests/ir_tests/test_frame_block_copy.expect create mode 100644 tests/ir_tests/test_frame_colour.c create mode 100644 tests/ir_tests/test_frame_colour.expect create mode 100644 tests/ir_tests/test_frame_loop_lifetimes.c create mode 100644 tests/ir_tests/test_frame_loop_lifetimes.expect create mode 100644 tests/ir_tests/test_frame_relayout.c create mode 100644 tests/ir_tests/test_frame_relayout.expect create mode 100644 tests/ir_tests/test_icf_folding.c create mode 100644 tests/ir_tests/test_icf_folding.expect create mode 100644 tests/ir_tests/test_inline_called_once.c create mode 100644 tests/ir_tests/test_inline_called_once.expect create mode 100644 tests/ir_tests/test_ldm_block_copy.c create mode 100644 tests/ir_tests/test_ldm_block_copy.expect create mode 100644 tests/ir_tests/test_narrow_call_ext.c create mode 100644 tests/ir_tests/test_narrow_call_ext.expect create mode 100644 tests/ir_tests/test_param_copy_alias.c create mode 100644 tests/ir_tests/test_param_copy_alias.expect create mode 100644 tests/ir_tests/test_ptr_local_fwd.c create mode 100644 tests/ir_tests/test_ptr_local_fwd.expect create mode 100644 tests/ir_tests/test_pure_forward.c create mode 100644 tests/ir_tests/test_pure_forward.expect create mode 100644 tests/ir_tests/test_r9_static_forward_call.c create mode 100644 tests/ir_tests/test_r9_static_forward_call.expect create mode 100644 tests/ir_tests/test_return_address.c create mode 100644 tests/ir_tests/test_return_address.expect create mode 100644 tests/ir_tests/test_self_store.c create mode 100644 tests/ir_tests/test_self_store.expect create mode 100644 tests/ir_tests/test_slot_const_store_fold.c create mode 100644 tests/ir_tests/test_slot_const_store_fold.expect create mode 100644 tests/ir_tests/test_small_aggregate_copy.c create mode 100644 tests/ir_tests/test_small_aggregate_copy.expect create mode 100644 tests/ir_tests/test_spill_slot_reuse.c create mode 100644 tests/ir_tests/test_spill_slot_reuse.expect create mode 100644 tests/ir_tests/test_sra.c create mode 100644 tests/ir_tests/test_sra.expect create mode 100644 tests/ir_tests/test_stack_deref_fold.c create mode 100644 tests/ir_tests/test_stack_deref_fold.expect create mode 100644 tests/ir_tests/test_stack_params_in_regs.c create mode 100644 tests/ir_tests/test_stack_params_in_regs.expect create mode 100644 tests/ir_tests/test_struct_arg_stack.c create mode 100644 tests/ir_tests/test_struct_arg_stack.expect create mode 100644 tests/ir_tests/test_struct_far_frame.c create mode 100644 tests/ir_tests/test_struct_far_frame.expect create mode 100644 tests/ir_tests/test_struct_param_pass_on.c create mode 100644 tests/ir_tests/test_struct_param_pass_on.expect create mode 100644 tests/ir_tests/test_struct_stack_ldm.c create mode 100644 tests/ir_tests/test_struct_stack_ldm.expect create mode 100644 tests/ir_tests/test_switch_case_merge.c create mode 100644 tests/ir_tests/test_switch_case_merge.expect create mode 100644 tests/ir_tests/test_tentative_definitions.c create mode 100644 tests/ir_tests/test_tentative_definitions.expect create mode 100644 tests/pp/has-attribute.c create mode 100644 tests/pp/has-attribute.expect create mode 100644 tests/thumb/armv8m/test_umaal.S delete mode 100644 tests/unit/arm/armv8m/test_opt_float_branch.c delete mode 100644 tests/unit/arm/armv8m/test_opt_neg_chain.c create mode 100644 tests/unit/arm/armv8m/test_opt_region_merge_case.c create mode 100644 tests/unit/arm/armv8m/test_opt_self_store.c diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 31909e8ae..6044ed4ae 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -47,7 +47,12 @@ jobs: - name: Configure # --debug enables CONFIG_TCC_DEBUG so the compiler supports -dump-ir; # without it the frontend types/ tests skip (they need IR dumps). - run: ./configure --enable-cross --enable-O2 --debug + run: ./configure --enable-cross --enable-O2 --debug --enable-asan + + - name: Unity groups are current + # source/unity/ must match the source lists (see scripts/gen_unity.py); + # a stale manifest still builds, but slower and without the checks. + run: python3 scripts/gen_unity.py --check - name: Build and test shell: bash @@ -148,7 +153,7 @@ jobs: - name: Build cross compiler # --enable-O2/--debug only affect tcc's own binary, not the code it # emits, so the codesize numbers are unaffected by this config. - run: ./configure --enable-cross --enable-O2 --debug && make cross -j$(nproc) + run: ./configure --enable-cross --enable-O2 --debug --enable-asan && make cross -j$(nproc) - name: Set up test venv # record.py's codesize child (regression_disasm.py --csv) imports the @@ -223,7 +228,13 @@ jobs: # the shell would fall back to interpreting it: "Syntax error: ) # unexpected"). Needed here because run_benchmark.py builds the RP2350 # perf firmware with this freshly built armv8m-tcc. - run: ./configure --enable-cross --enable-O2 --debug && make cross -j$(nproc) + # + # No --debug / --enable-asan here: the cloud jobs already run the tests + # under ASan, this compiler only builds benchmark firmware (its output + # does not depend on either), and on the Pi's four cores ASan + -g cost + # 2.3x the CPU of the build (182 s vs 79 s measured on x86) and slow + # every benchmark compile after it. + run: ./configure --enable-cross --enable-O2 && make cross -j$(nproc) - name: Download cloud codesize db uses: actions/download-artifact@v4 diff --git a/AGENTS.md b/AGENTS.md index 13027c6b3..02a17baff 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -17,7 +17,7 @@ library). ```bash # One-time setup -./configure # AddressSanitizer is ON by default; ./configure --disable-asan for fast/production builds +./configure # AddressSanitizer is off; ./configure --enable-asan for a sanitized build make download-gcc-tests # optional: sparse-fetch GCC torture tests (~16 MB, not the full gcc repo) # Build ARMv8-M cross compiler @@ -67,9 +67,10 @@ C Source → Preprocessor (source/frontend/tccpp.c) → Parser + type checker (source/frontend/gen/) → IR generation (source/ir/tccir.h, source/ir/gen/) → IR optimizations (source/opt/) - → Register allocation (source/machine/tccls.c + source/ir/regalloc.c) - → Thumb-2 code gen (source/backend/arch/arm/thumb/arm-thumb-gen.c) - → ELF output (source/obj/tccelf.c, source/obj/tccld.c) + → Register allocation (source/machine/tccls.c + source/ir/regalloc*.c) + → Thumb-2 code gen (source/backend/arch/arm/thumb/arm-thumb-gen.c and + arm-thumb-{emit,alu,mem,fp,frame,call}.c, sharing arm-thumb-gen.h) + → ELF output (source/obj/tccelf*.c, shared tccelf_priv.h; source/obj/tccld.c) ``` ## Source Tree @@ -96,18 +97,30 @@ libraries into `armv8m-tcc`. The repo root holds only build inputs | `source/backend/` | `libarm.a`, `libgenerators.a` | `arch/arm/` (+ `thumb/`, `fpu/`) and the generic `generators/` | The module libraries are linked `--whole-archive`. That is load-bearing, not -cosmetic: several TUs are referenced by nothing (the gdb pretty-printers, opt -passes not yet wired into the pipeline table), and an ordinary archive link -drops them silently — 53 symbols and 67 KB of `.text` on the first attempt. +cosmetic: several TUs are referenced by nothing (the gdb pretty-printers, the +pass registry), and an ordinary archive link drops them silently — 53 symbols and 67 KB of `.text` on the first attempt. `ARCH_LIB` deliberately stays outside the group so its orphaned members keep being dropped as before. Adding a module: create `source//` with a `Makefile` defining -`_INC` / `_SRC` / `_HDRS`, an object rule, and a +`_INC` / `_SRC` / `_HDRS`, an object rule, +`_OBJ = $(call lib-objs,,$(_SRC))`, and a `$(_LIB): $(_OBJ)` rule whose recipe is `$(ar-lib)`. Then add the `include` line and the `_INC` / `_LIB` / `_SRC` references to the top-level `Makefile`. Include order matters — see the comment above the include block. +**Unity build (default).** The library sources compile as ~30 grouped TUs +listed in `source/unity/` (`unity.mk` + one `.c` per group), not one TU per +file: every file includes `tcc.h`, and re-reading it 360 times was a fifth of an +-O2 self-compile. `scripts/gen_unity.py` writes those files; it keeps a file +on its own when grouping could change its meaning (it redefines a header macro, +or it is over 6000 lines) and never groups files whose file-scope names clash. +After adding, removing or renaming a `.c` file, or adding a file-scope +`static`/macro, run `make unity` (CI runs `scripts/gen_unity.py --check`). A +file the manifest does not know yet still builds, as its own TU; a file it +lists that no longer builds is a make error. `./configure --disable-unity` or +`make UNITY=no` goes back to one TU per file. + ## Code Architecture ### IR Subsystem (`source/ir/`) @@ -120,7 +133,7 @@ Public IR interface is `source/ir/tccir.h`. | `source/ir/gen/` | IR construction and manipulation (the former `ir/core.c`) | | `source/ir/gen/live.c` | Liveness analysis for register allocation | | `source/ir/codegen.c` | Central dispatch: unified two-pass loop (dry-run + real-run) routing IR ops to backend `_mop` handlers | -| `source/ir/regalloc.c` | SSA register allocator, parameterized by `RegAllocTarget` | +| `source/ir/regalloc*.c` | SSA register allocator, parameterized by `RegAllocTarget` (intervals, `_scan`, `_phi`, `_entry`, `_post`; shared `regalloc_priv.h`) | | `source/ir/vreg.c` | Virtual register management | | `source/ir/stack.c` | Stack frame layout | @@ -284,7 +297,7 @@ switch) for a real A/B. See "Re-checking only the seeds that failed" in **New IR instruction:** 1. Add opcode to `TccIrOp` in `tccir.h` -2. Add lowering in `arm-thumb-gen.c` +2. Add lowering in the matching `arm-thumb-{alu,mem,fp,frame,call}.c` 3. Add test in `tests/ir_tests/` **New assembly instruction:** diff --git a/Makefile b/Makefile index 959d5b143..39df04dc5 100644 --- a/Makefile +++ b/Makefile @@ -364,6 +364,34 @@ TCC_CORE_DEPS = $(wildcard *.h) \ $(foreach d,$(TCC_HDR_DIRS),$(wildcard $(d)/*.h)) \ config.mak +# Unity build: compile the sources as a few dozen grouped TUs instead of one TU +# per file. Every file includes tcc.h, which alone is ~560 KB / 14k lines of +# preprocessing and ~3 MB of symbol-table allocations, so building the 324 +# library sources one at a time spends 20% of an -O2 self-compile (65% at -O0) +# re-reading the same headers; grouped, a clean build is ~25% less CPU under both +# gcc and tcc. That matters on a Raspberry Pi and far more on the device. +# +# The groups live in source/unity/ (unity.mk plus one .c per group), generated by +# scripts/gen_unity.py so that no two members of a group can see each other's +# file-local names: see the script for the rules. Sources the manifest does not +# cover -- new files, or files it keeps apart on purpose -- still build as their +# own TU, so a stale manifest costs speed, never correctness. +# +# `./configure --disable-unity` or `make UNITY=no` builds one TU per file. +UNITY ?= $(if $(CONFIG_unity),$(CONFIG_unity),yes) +-include source/unity/unity.mk + +# $(call lib-objs,NAME,SOURCES): the objects a module library is built from. +# Also records SOURCES as UNITY_SRC_NAME for `make unity-info`, which is how +# gen_unity.py learns what each library holds. +unity-members = $(foreach g,$(UNITY_GROUPS_$1),$(UNITY_MEMBERS_$g)) +lib-objs = $(eval UNITY_SRC_$1 := $2)$(eval UNITY_LIB_NAMES := $(sort $(UNITY_LIB_NAMES) $1))$\ + $(if $(filter yes,$(UNITY)),$(call lib-objs-unity,$1,$2),$(patsubst %.c,$(X)%.o,$2)) +lib-objs-unity = $(if $(filter-out $2,$(call unity-members,$1)),$\ + $(error source/unity/unity.mk lists $(filter-out $2,$(call unity-members,$1)) under '$1', which no longer builds it; run scripts/gen_unity.py))$\ + $(foreach g,$(UNITY_GROUPS_$1),$(X)source/unity/$g.o) $\ + $(patsubst %.c,$(X)%.o,$(filter-out $(call unity-members,$1),$2)) + # Archive a module's objects into its library. Shared by every module Makefile # below so they all agree on how a lib is produced. define ar-lib @@ -425,11 +453,10 @@ ARCH_LIB = $($T_ARCH_LIB) # MODULE_LIBS is linked --whole-archive, which is not decoration. A normal # archive scan only pulls the members that resolve an undefined symbol, and # these modules hold TUs nothing references by design: the SValue/Sym -# pretty-printers exist to be called from gdb, and several opt passes are -# compiled but not yet wired into the pipeline table. Linking the archives -# a la carte dropped 53 symbols (all of tccmachine.c, the pass registry, eight -# tcc_ir_opt_* passes and their _ex thunks) and 67 KB of .text without a word -# of warning. --whole-archive reproduces exactly what listing the objects +# pretty-printers exist to be called from gdb. Linking the archives a la +# carte dropped 53 symbols (all of tccmachine.c, the pass registry, and eight +# tcc_ir_opt_* passes that were never wired into the pipeline, since deleted) +# and 67 KB of .text without a word of warning. --whole-archive reproduces exactly what listing the objects # individually used to do. Both GNU ld and tcc's own linker implement it # (libtcc.c "?whole-archive" -> AFF_WHOLE_ARCHIVE), which matters because the # YasOS self-host bootstrap relinks this with armv8m-tcc itself. @@ -480,6 +507,9 @@ tccdefs_table_.h : tccdefs_.h gen_predef_table.c $(addsuffix ,$(DEFINES) $(filter -D%,$(CFLAGS))) \ && ./gen_predef_table-$T$(EXESUF) $@ && rm -f gen_predef_table-$T$(EXESUF) +# Feature-test macros for every TU, not just those that include tcc.h before +# their first system header (tcc.h defines them too, for other builds). +DEFINES += -D_GNU_SOURCE -D_DARWIN_C_SOURCE DEFINES += -I$(TOP) $(CORE_INC) $(DRIVER_INC) $(FRONTEND_INC) $(IR_INC) \ $(MACHINE_INC) $(OBJ_INC) $(SUPPORT_INC) -I$(TOP)/source/opt/include \ $(OPT_DSL_INC) $(SSA_OPT_INC) $(FLAT_OPT_INC) $(MEMORY_INC) $(UTILS_INC) \ @@ -518,6 +548,27 @@ $(X)%.o : %.c $(LIBTCC_INC) config.mak @mkdir -p $(dir $@) $S$(CC) -o $@ -c $< $(addsuffix ,$(DEFINES) $(CFLAGS)) +# Unity TUs. A group sees every header its members do, so it depends on all of +# them, plus each member source (listed per group below). The extra -I's are +# the ones function_pipeline.c and backend/generators/ get from their own rules. +UNITY_HDRS = $(wildcard *.h $(addsuffix /*.h,source source/* source/*/* \ + source/*/*/* source/*/*/*/* source/*/*/*/*/* source/*/*/*/*/*/*)) +$(X)source/unity/%.o : source/unity/%.c $(UNITY_HDRS) config.mak + @mkdir -p $(dir $@) + $S$(CC) -o $@ -c $< $(addsuffix ,$(DEFINES) $(CFLAGS)) $(OPT_PIPELINE_INC) $(BACKEND_GENERATORS_INC) +$(foreach l,$(UNITY_LIB_NAMES),$(foreach g,$(UNITY_GROUPS_$l),$(eval $(X)source/unity/$g.o : $(UNITY_MEMBERS_$g)))) + +# Rewrite source/unity/ after adding, removing or renaming a library source. +unity: + python3 scripts/gen_unity.py +.PHONY: unity + +# What scripts/gen_unity.py needs: the compile line and each library's sources. +unity-info: + @echo 'CC $(CC) $(DEFINES) $(CFLAGS) $(OPT_PIPELINE_INC) $(BACKEND_GENERATORS_INC)' + @$(foreach l,$(UNITY_LIB_NAMES),echo 'LIB $l $(UNITY_SRC_$l)';) +.PHONY: unity-info + # Architecture library — built by nested Makefile TARGET_ARCH_NAME = $($T_ARCH) ARCH_DEFINES = $(subst -I.,-I$(CURDIR),$(DEFINES)) @@ -532,7 +583,7 @@ $(ARCH_LIB): FORCE fi $S$(MAKE) --no-print-directory -C source/backend ARCH=$(TARGET_ARCH_NAME) \ TOP=$(CURDIR) BUILD_DIR=$(CURDIR)/$(dir $(ARCH_LIB)) \ - CC="$(CC)" AR="$(AR)" CFLAGS="$(CFLAGS)" DEFINES="$(ARCH_DEFINES)" + CC="$(CC)" AR="$(AR)" CFLAGS="$(CFLAGS)" DEFINES="$(ARCH_DEFINES)" UNITY=$(UNITY) # additional dependencies # tcctools.c is #included by tcc.c rather than compiled on its own. @@ -865,14 +916,16 @@ warn-check: armv8m-tcc$(EXESUF) patch-newlib .PHONY: optflag-check optflag-check: armv8m-tcc$(EXESUF) @echo "------------ optflag-check: -finline-limit vs -O argument order ------------" + @# blend calls out so it is no small straight-line leaf: those inline at any limit. @d=$$(mktemp -d) ; \ trap 'rm -rf "$$d"' EXIT ; \ printf '%s\n' \ + 'int ext(int);' \ 'static int blend(int a, int b, int c) {' \ ' int t = a * 3 + b * 5;' \ ' int u = c ^ (a << 2);' \ ' int v = (t > u) ? (t - u) : (u - t);' \ - ' return v + t - u + (a & b) + (b | c);' \ + ' return v + t - u + (a & b) + ext(b | c);' \ '}' \ 'int sink1(int x) { return blend(x, x + 1, x + 2); }' \ 'int sink2(int x) { return blend(x * 2, x, x - 7); }' \ diff --git a/README b/README index cf57f96e4..59099d252 100644 --- a/README +++ b/README @@ -158,13 +158,13 @@ Frequently used `./configure` options: |------|-------------| | `--enable-cross` | Build the ARMv8-M cross compiler (`armv8m-tcc`) | | `--enable-O0` … `--enable-O3`, `--enable-Os`, `--enable-Og` | Optimization level TCC itself is built with (`-O2` default) | -| `--disable-asan` | **AddressSanitizer is ON by default**; disable it for fast or production builds | +| `--enable-asan` | Build TinyCC with AddressSanitizer (off by default) | | `--enable-ubsan`, `--enable-lsan` | Additional sanitizers | | `--debug` | Include debug info in the resulting binaries | Example: ```bash -./configure --enable-cross --disable-asan +./configure --enable-cross make cross ``` diff --git a/configure b/configure index 50e983fa7..0e151f15b 100755 --- a/configure +++ b/configure @@ -176,6 +176,10 @@ for opt do ;; --enable-ubsan) confvars_set ubsan ;; + --enable-unity) confvars_set unity + ;; + --disable-unity) confvars_set unity=no + ;; --enable-lsan) confvars_set lsan ;; --with-libgcc) confvars_set libgcc @@ -197,10 +201,10 @@ for opt do esac done -# AddressSanitizer is enabled by default for this fork; opt out with -# --disable-asan (e.g. for fast production / firmware builds). -if test "$asan_disabled" != "yes"; then - confvars_has asan || confvars_set asan +# AddressSanitizer is opt-in (--enable-asan); --disable-asan, the default, +# wins over an earlier --enable-asan. +if test "$asan_disabled" = "yes"; then + confvars=$(echo " $confvars " | sed 's/ asan / /g') fi show_help() { @@ -231,9 +235,12 @@ Advanced options (experts only): --extra-ldflags= specify linker options [$LDFLAGS] --debug include debug info with resulting binaries - --enable-asan enable AddressSanitizer (ASan) [default] - --disable-asan disable AddressSanitizer (ASan) + --enable-asan enable AddressSanitizer (ASan) + --disable-asan disable AddressSanitizer (ASan) [default] --enable-ubsan enable UndefinedBehaviorSanitizer (UBSan) + --enable-unity compile the tcc sources as the grouped TUs listed + in source/unity/unity.mk [default] + --disable-unity compile every tcc source file as its own TU --enable-lsan enable LeakSanitizer (LSan) --enable-O0 disable optimizations (GCC -O0) --enable-O1 basic optimizations (GCC -O1) diff --git a/docs/bugs/README.md b/docs/bugs/README.md new file mode 100644 index 000000000..320d37760 --- /dev/null +++ b/docs/bugs/README.md @@ -0,0 +1,22 @@ +# Open bug reports + +One file per known, unfixed tinycc bug: `docs/bugs/.md`. A bug +found but not fixed in the session that found it goes here; a session that +comes across a report here fixes it. + +Format: + +``` +# + +**Status:** open · **Severity:** correctness (miscompile) | performance | ... · **Found:** YYYY-MM-DD (how) + +## Summary +## Reproducer (C source + -O level + the wrong output or code) +## Root cause (file:function, what check is missing) +## Regression lock (unit test pinning the current behaviour, if any) +## Likely fix +``` + +When fixed: flip any `_bug` unit-test lock the report names, add an IR test, +and delete the report in the fixing commit (git history keeps it). diff --git a/docs/bugs/ctags-entry-c-kconfig-choice-name.md b/docs/bugs/ctags-entry-c-kconfig-choice-name.md new file mode 100644 index 000000000..067ee8dad --- /dev/null +++ b/docs/bugs/ctags-entry-c-kconfig-choice-name.md @@ -0,0 +1,63 @@ +# ctags main/entry.c at -O1/-O2 loses the name a Kconfig `choice` takes from its `prompt` + +**Status:** open · **Severity:** correctness (miscompile) · **Found:** 2026-09-28 (yasos ctags on QEMU, after the sccp fix 67aa75f6) + +## Summary + +universal-ctags (apps/ctags, pruned build) tagging this Kconfig: + +``` +menu "Kernel" +config SMP + bool "SMP support" +choice + prompt "Scheduler" +config SCHED_RR + bool "round robin" +endchoice +endmenu +``` + +should rename the anonymous choice after its prompt (host ctags 6.2, and the +yasos build with every TU at -O0): + +``` +Scheduler ... /^ prompt "Scheduler"$/;" C menu:Kernel +SCHED_RR ... c choice:Kernel""Scheduler typeref:typename:bool +``` + +With `main/entry.c` compiled by tcc at -O1 or -O2 (all other TUs either level) +the tag keeps its anonymous name: + +``` +choice3228b3730104 ... /^choice$/;" C menu:Kernel +``` + +## Reproducer + +TU bisection (every object -O0 except a subset at -O2, device run per step) +isolates `main/libctags_a-entry.o`: all -O0 + entry.o -O2 is BAD, all -O2 + +entry.o -O0 is OK. Still BAD with `-O1`, with `-O2 -fno-inline`, and with +**every** `TCC_DISABLE_PASS` knob of the 147 listed by `-dump-ir-passes=all` +disabled at once -- so the culprit is an ungated transform, codegen or the +register allocator, not one of the knobbed passes. + +Build the pruned ctags as build_rootfs.sh's build_ctags() does, run on QEMU +via the FAT drive (`--language-force=Kconfig -f - /mnt/KCONFIG`). + +The rename goes through the optscript DSL (optlib/kconfig.ctags: the prompt's +string replaces the choice tag's name); the entry.c side of that is presumably +the tag-name update on the cork queue entry -- a guess, not yet checked. + +## Root cause + +Not yet known. Next step: per-function mix of entry.c (-O0 vs -O2 bodies) to +find the function, then diff its -O0/-O2 code. + +## Regression lock + +None yet. + +## Likely fix + +Unknown until the function is found. diff --git a/docs/bugs/dce-dead-var-store-volatile-lval-src.md b/docs/bugs/dce-dead-var-store-volatile-lval-src.md deleted file mode 100644 index 5aaf575c0..000000000 --- a/docs/bugs/dce-dead-var-store-volatile-lval-src.md +++ /dev/null @@ -1,81 +0,0 @@ -# dce_dead_var_stores drops a store whose fused lval source is a volatile read - -**Status:** open · **Severity:** correctness (miscompile) · **Found:** 2026-07-19 -(unit tests for `source/opt/ssa/dce/`) - -## Summary - -`dce_dead_var_stores` eliminates a `STORE` to a dead VAR slot without checking -whether the stored **value** is itself a volatile memory read. For the fused -memory-to-memory copy form `STORE lval(V0slot) <- lval(V1)`, the store -instruction *embodies* the read of `V1`; when `V1` is volatile that read is a -mandated side effect (C11 5.1.2.3) and must survive even though `V0` is never -read. - -## Root cause - -The second scan loop in `dce_dead_var_stores` -([source/opt/ssa/dce/dead_var_stores.c](../../source/opt/ssa/dce/dead_var_stores.c) -lines ~164-187) NOPs any `STORE` whose dest is a non-volatile, non-address-taken, -unused VAR slot. The volatile pre-mark at lines 62-64 covers only the -**destination** slot's interval: - -```c -for (int p = 0; p < num_vars && p < ir->variables_live_intervals_size; p++) - if (ir->variables_live_intervals[p].is_volatile) - var_used[p / 8] |= (1 << (p % 8)); -``` - -Nothing inspects the source operands of the store being killed. The two -sibling passes have the guard this one is missing: - -- `vl_removable` in [var_liveness.c](../../source/opt/ssa/dce/var_liveness.c) - lines ~84-101 — rejects removal when any src's live interval `is_volatile`; -- `dce_temp_worklist` in [temp_worklist.c](../../source/opt/ssa/dce/temp_worklist.c) - lines ~42-50 — keeps a dead def whose dest slot is volatile. - -`dce_var_liveness` would keep the store (via `vl_removable`), but -`dce_dead_var_stores` runs earlier in `ssa_opt_dce` and kills it first. - -## Observable symptom - -Unit-level IR shape (confirmed in the UT11 harness): - -```c -/* intervals: V1.is_volatile = 1, V0 never read */ -STORE lval(local V0) <- lval(V1) /* fused copy: embodies a read of V1 */ -``` - -`ssa_opt_dce` at -O1 NOPs the store, deleting the volatile access to `V1`. - -**Caveat:** confirmed at pass level with a hand-built fixture; an end-to-end -C reproducer through `armv8m-tcc` has not been established yet. The fused -lval-source store form is real (see the ptr fuzz seed 7226 note in -[load-cse-lval-store-src-tracked-as-value.md](load-cse-lval-store-src-tracked-as-value.md)); -whether the frontend/fusion passes emit it with a volatile-qualified source -needs verification when fixing. - -## Regression lock - -`tests/unit/arm/armv8m/test_ssa_opt_dce.c`: - -- `test_dce_dead_var_store_volatile_lval_src_bug` pins the elimination. - -It asserts the CURRENT (buggy) result; flip its assertions to "store kept, -changed == 0" once fixed. Not yet fixed. - -## Likely fix - -In the elimination loop of `dce_dead_var_stores`, before NOP'ing a store, -reject it when any source operand is an lval whose vreg's live interval is -volatile — mirroring `vl_removable`: - -```c -if (q->op == TCCIR_OP_STORE && store_has_volatile_lval_src(ir, q)) - continue; -``` - -The sibling audit is worth doing at the same time: `dce_dead_stackloc_stores` -kills unread anonymous-slot stores and likewise ignores volatile sources, and -`dce_dead_overwrite_stores` / `dce_dead_global_stores` kill *earlier* stores — -a killed earlier store whose value was a volatile read has the same problem. diff --git a/docs/bugs/load-cse-lval-store-src-tracked-as-value.md b/docs/bugs/load-cse-lval-store-src-tracked-as-value.md deleted file mode 100644 index 18d258f60..000000000 --- a/docs/bugs/load-cse-lval-store-src-tracked-as-value.md +++ /dev/null @@ -1,90 +0,0 @@ -# load_cse: pointer-deref store source tracked as the stored value - -**Status:** open · **Severity:** correctness (miscompile) · **Found:** 2026-07-17 -(unit tests for `source/opt/ssa/memory/load_cse.c`) - -## Summary - -In `ssa_opt_load_cse`'s stack store-load forwarding, a `STORE` whose source is -a pointer dereference (the fused memory-to-memory copy form, e.g. -`StackLoc[0] <- *p` or `*q <- *p`) records the source **pointer** `p` — not -the pointee `*p` — as the slot's stored value. A later `LOAD` of that slot is -then rewritten to `ASSIGN p`, yielding the address where the loaded value was -expected. - -## Root cause - -Two `sstore_track_vr` call sites in `gload_process_block` -([source/opt/ssa/memory/load_cse.c](../../source/opt/ssa/memory/load_cse.c)) -take `svr = irop_get_vreg(src)` and track it whenever it is a TEMP, without -excluding `src.is_lval`: - -- Direct stack-slot tracker — load_cse.c:1031-1032 - (`dest.tag == IROP_TAG_STACKOFF` branch): - ```c - if (svr >= 0 && TCCIR_DECODE_VREG_TYPE(svr) == TCCIR_VREG_TYPE_TEMP) - sstore_track_vr(st, irop_get_stack_offset(dest), store_btype, svr, dest_base); - ``` -- TEMP-indir tracker (`*T <- val` resolving to `LEA(StackLoc[N])`) — - load_cse.c:1076-1077: - ```c - if (svr >= 0 && TCCIR_DECODE_VREG_TYPE(svr) == TCCIR_VREG_TYPE_TEMP) - sstore_track_vr(st, eff_off, store_btype, svr, store_base); - ``` - -All three sibling value trackers in the same function reject lval sources: - -- vslot tracker, load_cse.c:302 (`src.tag == IROP_TAG_VREG && !src.is_lval`) -- tvstore tracker, load_cse.c:1104 (`src.tag == IROP_TAG_VREG && !src.is_lval`) -- gstore tracker, load_cse.c:1181 (`sval.tag == IROP_TAG_VREG && !sval.is_lval`) - -The two `sstore_track_vr` sites are the odd ones out. - -## How the shape arises - -The forward-into-store-source block at load_cse.c:975-1008 exists precisely -for `STORE StackLoc <- ` (STACKOFF or VREG source — the comment -cites ptr fuzz seed 7226), so the fused mem-copy shape reaches this code in -real compiles. When the lval-VREG source resolves to a tracked slot holding an -immediate, the source is rewritten first and all is well; when it does **not** -resolve (arbitrary pointer) or the tracked entry holds a TEMP, the raw -`is_lval` VREG source falls through to the trackers above, which misrecord -the pointer as the value. - -## Observable symptom - -```c -/* IR shape (fused mem-to-mem copy, p not LEA-resolvable): */ -t1 = t0 + #0; /* some unresolvable pointer */ -StackLoc[0] <- *t1; /* STORE lval(stackoff 0), lval(t1) */ -t2 = LOAD(StackLoc[0]); -``` - -The pass rewrites the load to `t2 = ASSIGN t1` — the pointer, not `*t1`. -The same happens via the temp-indir site for -`t1 = &StackLoc[0]; *t1 = *t0; t2 = LOAD(StackLoc[0])` → `t2 = ASSIGN t0`. - -## Regression lock - -`tests/unit/arm/armv8m/test_ssa_opt_load_cse.c`: - -- `test_stack_store_lval_vreg_src_tracked_bug` pins the load_cse.c:1031 site -- `test_temp_indir_store_lval_src_tracked_bug` pins the load_cse.c:1076 site - -Both assert the CURRENT (buggy) result; flip their assertions to "no forward, -load preserved" once fixed. Not yet fixed. - -## Likely fix - -Add the sibling guard to both sites: - -```c -if (svr >= 0 && src.tag == IROP_TAG_VREG && !src.is_lval && - TCCIR_DECODE_VREG_TYPE(svr) == TCCIR_VREG_TYPE_TEMP) -``` - -so an lval source falls into the existing `else` branch -(`sstore_invalidate_overlap` + `sstore_remove_offset` for the direct site, -`sstore_remove_offset` for the temp-indir site), which is the conservative, -sound outcome. Note the forward-into-src block at :975-1008 already handles -the cases where forwarding is actually valid, so no optimization is lost. diff --git a/docs/bugs/selfhost-const-compare-fold-breaks-native-tcc.md b/docs/bugs/selfhost-const-compare-fold-breaks-native-tcc.md new file mode 100644 index 000000000..e09daa8e0 --- /dev/null +++ b/docs/bugs/selfhost-const-compare-fold-breaks-native-tcc.md @@ -0,0 +1,91 @@ +# Folding `x < INT32_MIN`-style compares in gen_opic breaks the self-hosted tcc + +**Status:** open · **Severity:** correctness (self-host miscompile, latent) · **Found:** 2026-09-29 (yasos QEMU smoke, zig-abi a9f76c02 + the hunk below) + +## Summary + +A frontend constant fold that is correct by itself -- in `gen_opic` +(source/frontend/gen/op/int.c), fold `x =u 0`, `x >u UINT_MAX`, +`x <=u UINT_MAX`, `x =s INT_MIN`, `x >s INT_MAX`, `x <=s INT_MAX` +(32- and 64-bit) to 0/1 when x is not a volatile read and not a pending +VT_CMP/VT_JMP -- makes the **native** tcc that the cross builds from tinycc's +own sources miscompile ordinary programs on the device: + +- `ir_tests/bug_ternary_switch.c` prints `test1: 1 FAIL` (expected `test1: 6 PASS`) + at **every** -O level of the native compiler, -O0 included; +- ~1,000 of the 14,104 QEMU smoke tests fail (IR tests at all levels), and the + run eventually hangs. + +The host cross with the same fold passes the whole IR suite (14,692) and GCC +torture (11,309), so the folded *code it produces for test programs* is right; +what breaks is the native tcc binary itself (built at -O1 by the cross). +Reverting just this hunk (keeping everything else) makes the device suite pass. + +## Suspicion + +The fold turns more conditions into compile-time constants. Constant +conditions go through tcc's nocode_wanted / CODE_OFF machinery (dead-branch +suppression); bug_ternary_switch itself is the regression test for a +CODE_OFF_BIT leak through expr_cond/backpatch. A tinycc source construct +where one of these compares is now constant probably reaches a path that +drops live code. Not investigated. + +## Repro + +1. Apply the hunk below to source/frontend/gen/op/int.c (it goes in + gen_opic, right after the relational-reversal block). +2. `./build_rootfs.sh -o rootfs.img && rm -rf .zig-cache && zig build -Doptimize=ReleaseSafe` +3. `./scripts/run_qemu_smoke.sh --no-build tcc_suite_test.py -k bug_ternary_switch` -> 3 failed. + +Next step: bisect which tinycc TU/function the fold changes (build the native +tcc with gcc objects mixed in, per docs/selfhost_miscompile_debugging.md), then +find the compare that became constant there. + +## The hunk + +```diff +@@ -260,7 +260,42 @@ void gen_opic(int op) + l1 = 0; + } + } +- if (c1 && ((l1 == 0 && (op == TOK_SHL || op == TOK_SHR || op == TOK_SAR)) || (l1 == -1 && op == TOK_SAR))) ++ /* x compared with an end of its own type's range -- `x < INT_MIN`, ++ * `x > UINT_MAX`, `x >= 0u` and their mirrors -- holds or fails for every ++ * x. zig.h's overflow wrappers test the wrapped result against ++ * zig_minInt/zig_maxInt of the type's full width that way after every ++ * __builtin_*_overflow, and left alone each is a CMP (one through the ++ * literal pool) and a branch. x is dropped only when that drops nothing: ++ * not a volatile read, not a pending compare or jump chain. */ ++ int taut = -1; ++ if (c2 && !c1 && t1 == t2 && (t1 == VT_INT || t1 == VT_LLONG) && !(v1->type.t & VT_VOLATILE) && ++ (v1->r & VT_VALMASK) != VT_CMP && (v1->r & VT_VALMASK) != VT_JMP && (v1->r & VT_VALMASK) != VT_JMPI) ++ { ++ const int is64 = t1 == VT_LLONG; ++ const uint64_t umax = is64 ? ~(uint64_t)0 : 0xffffffffull; ++ const uint64_t smin = is64 ? (uint64_t)1 << 63 : 0xffffffff80000000ull; ++ const uint64_t smax = is64 ? ~(uint64_t)0 >> 1 : 0x7fffffffull; ++ switch (op) ++ { ++ case TOK_ULT: taut = l2 == 0 ? 0 : -1; break; ++ case TOK_UGE: taut = l2 == 0 ? 1 : -1; break; ++ case TOK_UGT: taut = l2 == umax ? 0 : -1; break; ++ case TOK_ULE: taut = l2 == umax ? 1 : -1; break; ++ case TOK_LT: taut = l2 == smin ? 0 : -1; break; ++ case TOK_GE: taut = l2 == smin ? 1 : -1; break; ++ case TOK_GT: taut = l2 == smax ? 0 : -1; break; ++ case TOK_LE: taut = l2 == smax ? 1 : -1; break; ++ default: break; ++ } ++ } ++ if (taut >= 0) ++ { ++ vtop->c.i = taut; ++ vswap(); ++ vtop--; ++ vtop->r |= VT_NONCONST; ++ } ++ else if (c1 && ((l1 == 0 && (op == TOK_SHL || op == TOK_SHR || op == TOK_SAR)) || (l1 == -1 && op == TOK_SAR))) + { + /* treat (0 << x), (0 >> x) and (-1 >> x) as constant */ + vpop(); +``` diff --git a/docs/bugs/volatile-local-folded-to-constant.md b/docs/bugs/volatile-local-folded-to-constant.md deleted file mode 100644 index 5dd5637a7..000000000 --- a/docs/bugs/volatile-local-folded-to-constant.md +++ /dev/null @@ -1,90 +0,0 @@ -# volatile local reads folded to a constant - -**Status:** open · **Severity:** correctness (miscompile) · **Found:** 2026-07-09 -· **Pre-existing:** yes (reproduces with legacy `const_prop` on and off) - -## Summary - -Constant propagation folds reads of a `volatile`-qualified **local** variable to -the constant it was initialised with, eliminating the volatile memory accesses -that C requires (C11 6.7.3p7 / 5.1.2.3: every access to a volatile object is a -side effect that must not be optimised away). - -## Reproducer - -```c -int ext(int); -int f(int c) -{ - volatile int x = 5; - if (c) - return ext(x); /* must LOAD x from memory */ - return x; /* must LOAD x from memory */ -} -``` - -`armv8m-tcc -O1 -dump-ir -c` (IR after optimization): - -``` -0000: TEST_ZERO R0(P0) -0001: JMP to 5 if "==" -0002: PARAM0[call_0] #5 <-- x read folded to #5, no load -0003: CALL GlobalSym(1188) --> R0(T0) -0004: RETURNVALUE R0(T0) -0005: RETURNVALUE #5 <-- x read folded to #5, no load -``` - -Both the store `x = 5` and the two reads should survive as real memory accesses; -instead the reads become immediates and the slot store is dead-stored away. - -## Root cause - -VAR constant propagation treats a single-def local as a propagatable constant -without excluding `volatile`-qualified slots. The per-VAR "blocked" set only -excludes address-taken (and complex) locals, not volatile ones: - -- SSA `ssa_opt_var_imm_prop` — [ir/opt/ssa_opt_cprop.c](../../ir/opt/ssa_opt_cprop.c) - builds `blocked[]` from `variables_live_intervals[v].addrtaken` only - (~line 1912); no `is_volatile` check. Same gap applies to the `ASSIGN #imm` - def acceptance in `ssa_opt_var_const_fold`. -- Legacy `ssa`/flat `tcc_ir_opt_const_prop` — - [ir/opt_constprop.c](../../ir/opt_constprop.c) marks a def non-constant on - `interval->addrtaken || interval->is_complex` (~line 2032) but not - `interval->is_volatile`, then propagates the immediate (~line 2043 accepts - `ASSIGN|STORE` immediate defs). - -The interval already carries the needed flag: `IRLiveInterval.is_volatile` -([tccir.h](../../tccir.h) line ~317). The precedent guard exists in -[ir/opt/ssa_opt_dce.c](../../ir/opt/ssa_opt_dce.c) line ~1854 -(`if (!iv || iv->addrtaken || iv->is_volatile)`), which correctly excludes -volatile slots from dead-store elimination — the const-prop passes are missing -the symmetric check. - -**Scope is broader than the SSA passes.** With `const_prop` force-disabled, the -reproducer still folds to `#5` after individually disabling *each* of -`ssa:sccp`, `ssa:cprop`, `ssa:var_const_fold`, `ssa:var_imm_prop` -(`TCC_DISABLE_PASS=`), so the fold is also performed upstream — a flat -pre-SSA pass (`const_var_prop` / `value_tracking` / `copy_prop`) or IR -generation itself. A complete fix must guard `is_volatile` at every VAR-const -producer, not only the SSA ones. (The SSA `var_imm_prop` / `var_const_fold` -guards are already in place on `legacyOptRemoval` — they stop those paths from -folding volatile slots but do not by themselves fix the observable case.) - -## Proposed fix - -Exclude volatile VAR slots from constant propagation, mirroring the DSE guard: - -- SSA: when building `blocked[]` in `ssa_opt_var_imm_prop` (and the def gate in - `ssa_opt_var_const_fold`), also block when - `variables_live_intervals[v].is_volatile`. -- Legacy: in `tcc_ir_opt_const_prop`, treat `interval->is_volatile` like - `addrtaken`/`is_complex` (set `is_constant = 0`). - -Audit sibling VAR-const consumers (`sccp`, `var_const_fold`, `value_tracking`, -`known_bits`) for the same omission. - -## Regression test - -Add an IR test asserting the two reads of `x` remain memory loads (not `#5` -immediates) at `-O1`/`-O2` — e.g. the reproducer above, checking the disasm/IR -retains loads from the `x` slot. diff --git a/docs/coding_guidelines.md b/docs/coding_guidelines.md index acb04fded..c243d1e1d 100644 --- a/docs/coding_guidelines.md +++ b/docs/coding_guidelines.md @@ -71,7 +71,7 @@ Raw pointers are acceptable only at existing shared ABI/layout boundaries that c ## Conventions for Changes - **Never commit without a regression test** for a bug fix — verbatim or reduced repro under `tests/ir_tests/`, expected output in a `.expect` file. -- New IR opcode → lowering in `arm-thumb-gen.c` + test. +- New IR opcode → lowering in the matching `arm-thumb-{alu,mem,fp,frame,call}.c` + test. - New asm instruction → builder in `arm-thumb-opcodes.c` + token + parser + test. - Don't commit the temporary `TCC_SKIP_SSA*` env-var bisection gates — they are investigation-only scaffolding. diff --git a/docs/plan_switch_case_constant_merge.md b/docs/plan_switch_case_constant_merge.md new file mode 100644 index 000000000..a9a56aec3 --- /dev/null +++ b/docs/plan_switch_case_constant_merge.md @@ -0,0 +1,205 @@ +# Plan: merge switch case bodies that differ only in their own case constant + +Status: landed as 0512fba8 (2026-09-27). + +Corrections found while implementing: + +- The `SWITCH_TABLE` operand is `sel - min_val`, not the selector. Look through the + `idx <-- X SUB #min_val` right before the dispatch and use X. +- `alive_share` can give the selector's register to another value inside a body where + the selector is dead. A define scan of the selector vreg is not enough: scan every + operand of the kept copy for a register or slot overlapping the selector's post-RA + location, and require the selector in a callee-saved register (r4-r11) or a slot. +Owner: whoever picks up `source/ir/cross_jump.c` after e8501cf9. + +## Problem + +`tcc_ir_region_merge` (post-RA, `cross_jump.c`, landed e8501cf9) folds switch +case bodies that are byte-identical. In Zig-rendered C the dominant near-miss is + +```c +switch (x.tag) { + case 4: { ... r.tag = 4; ... } /* body identical to case 5 except the 4 */ + case 5: { ... r.tag = 5; ... } +} +``` + +The constant written in case k is k itself, i.e. the switch selector. The +matcher stops at that instruction (debug reason 2, `T <-- #4 [LOAD]` vs +`T <-- #5 [LOAD]`), so 79 bodies of `Value_interpret__anon_196470__49504` and +91 of `InternPool_PackedCallingConvention_pack__30991` stay separate. clang gets +this through SimplifyCFG sink-with-phi followed by `ForwardSwitchConditionToPHI`; +its merged body stores the tag from the selector register. + +Dry run (patch `.cache/zigmem/build/sw/rm_dry.patch`, `TCC_RM_DRY=1`, tally line +`RMDRY pairs= insns=`) on zig.c: 14 functions hit, ~30 KB of 5.24 MB: + +| function | est. removed | of | +|---|---|---| +| Value_interpret__anon_196470 | 16.6 KB | 24.8 KB | +| InternPool_PackedCallingConvention_unpack | ~7 KB | 7.6 KB | +| InternPool_PackedCallingConvention_pack | 7.3 KB | 7.6 KB | +| 11 others | < 300 B each | | + +The dry run leaves every other function byte-identical to the baseline (one +function, `zig_system_glibcVerFromRPath`, is 8 B larger because a virtually +retired body blocked a later exact merge; the real transform does not have +that problem since the retired body is really gone). + +## Design + +All work is inside `rm_match` / `tcc_ir_region_merge`. Nothing changes for +pairs that match exactly. + +### 1. Per-entry case value (setup, once per function) + +In the counting loop that already fills `st.trefs` from each `SWITCH_TABLE`, +also fill `int64_t caseval[n]`: + +- `RM_NOCASE` initially; +- `tab->min_val + e` for `tab->targets[e]` when the slot is still `RM_NOCASE`, + `RM_MULTI` if a second entry or the default lands on the same index; +- after the loop, `RM_MULTI` for any index with `st.jrefs[i] != 0` (a plain + jump also reaches it) or that the previous non-NOP instruction can fall into + (accept only `JUMP`, `RETURNVALUE`, `RETURNVOID`, `SWITCH_TABLE` before it). + +Also remember, per index, which `SWITCH_TABLE` instruction dispatches there +(`selsw[i]`), so the selector operand can be copied from it later. + +Only `SWITCH_TABLE` dispatch is covered. A compare chain (`CMP sel,#k; JUMPIF`) +also implies `sel == k` on the taken edge, but the target of a `JUMPIF` has +`jrefs`, and nothing in zig.c measured needs it. Leave it out of the first cut. + +### 2. Match rule (in `rm_match`) + +When `rm_insn_same(qa, qb)` fails, try `rm_insn_same_case(st, a, b, qa, qb)`: + +- same op, `rm_op_ok`, not a jump, not a call/param op; +- exactly one operand slot differs, both operands `IROP_TAG_IMM32`, not + struct-typed, every other operand flag equal (`btype`, `is_unsigned`, `aux`, + … the same list `cj_operand_same` checks); +- `imm_a == caseval[a]` and `imm_b == caseval[b]`, comparing both as int32 and + as uint32 against the int64 case value; +- the side tables `rm_insn_same` compares (barrel shift, shift64 dead half, + zero_half64, bfi params) equal. + +Record the hit: `st->param_hit = 1`, and push `(ia, slot)` onto a small list +`st->param_at[]` (several instructions in one body may carry the constant, for +instance a narrow store and a later reload; allow up to, say, 8 per region). + +The selector must be 32 bits or narrower. Reject when the `SWITCH_TABLE` +source operand is `IROP_BTYPE_I64` (a u64 selector cannot be copied into an +IMM32 slot). + +### 3. Selector availability at the use + +Before accepting a parameterised match, check that the selector still holds +the case value at each recorded instruction in **both** copies. Two shapes +occur: + +- **vreg selector** (`PackedCallingConvention_pack`: `T13807 <-- T4 UBFX #256`, + held in r4). tcc emits the dispatch after all bodies, so the selector's live + interval spans every body in linear order and no body redefines it. Check it + anyway: no instruction in `[entry, use]` of either copy defines that vreg + (`cj_defines`), and `tcc_ir_get_live_interval(ir, sel)` covers the use index + with a single location (register or spill slot). If the RA split the interval + and the location at the use differs from the location at the dispatch, skip. +- **VAR selector** (`Value_interpret`: `t26` is a local loaded with `ldrb` at + the dispatch). Value at the body entry equals the value at the dispatch, + because the dispatch jumps straight to the entry. Check that no `STORE` / + `STORE_INDEXED` / `ASSIGN` / `BLOCK_COPY` / memcpy-style call in + `[entry, use)` of either copy has that VAR as destination, and that the VAR + is not address-taken in the function (any `LEA`/`&V` of it anywhere → skip; + a pointer could alias it). + +If any check fails, treat the pair as a non-match (return 0), exactly as today. + +### 4. Rewrite (in the driver, before deleting copy B) + +For each recorded `(ia, slot)` in copy A: replace the IMM32 operand with a +verbatim copy of the `SWITCH_TABLE`'s source operand (`tcc_ir_op_get_src1` of +`selsw[a]`). The op stays what it was: + +- `T <-- #k [LOAD]` becomes `T <-- Tsel [LOAD]` (a register move; the same + form the RA already emits for register-passed PARAMs, `regalloc.c` ~1274) or + `T <-- Vsel [LOAD]` (a slot load with the dispatch's own width). +- `V <-- #k [STORE]` becomes `V <-- Tsel [STORE]` / `V <-- Vsel [STORE]`. + If the backend rejects a VAR as a STORE source post-RA, skip such pairs in + step 2 (only accept `LOAD` dests and register-source stores); both measured + functions carry the constant through a `LOAD` first. + +Then delete copy B and retarget its entry exactly as the exact-match path does. +Later candidates C against A compare `Tsel` (A) with `#k_c` (C): extend +`rm_insn_same_case` to accept "A's operand is already the selector operand +and C's constant equals `caseval[c]`" so the third and following bodies fold +into the same copy. + +Update `st.occ` for the selector's vreg key when a new use is added, since the +one-to-one value pairing counts occurrences. + +### 5. Knobs and debug + +- `TCC_DISABLE_PASS=region_merge:case_const` disables only the new rule. +- Keep the `TCC_RM_DBG=` reporter from `rm_debug.patch` (prints + `RM a= b= k= why= at=`), adding `why=14` for "constant differs but is not the + case value" so future near-misses are visible. + +## Implementation steps + +1. Land the dry-run scaffolding as real code: `caseval[]`, `selsw[]`, + `rm_no_fallthrough`, `rm_operand_same_but_imm`, `rm_insn_same_case` + (from `rm_dry.patch`, minus the tally). +2. Add the availability checks of §3 (vreg define scan, VAR store scan, + address-taken scan). +3. Add the rewrite of §4 and the "already the selector" comparison. +4. Unit test in `tests/unit/arm/armv8m/` (model: `test_opt_switch_to_data.c`, + builder `ir_build.h`): hand-build a `SWITCH_TABLE` with three targets whose + bodies are `T <-- #k; V <-- T [STORE]; JUMP end`. Assert: return value 2 + (two bodies removed), A's first instruction now reads the selector operand, + B's and C's entries are single `JUMP`s to A. Negative tests: a body whose + constant is `k+1` (no merge), an entry also reached by a plain `JUMP` (no + merge), a VAR selector stored inside the body (no merge), a u64 selector. +5. `tests/ir_tests/` program: a 6-case switch returning `(struct){.tag = k, + .payload = f(k)}` through a pointer, plus a default; prints the tags and + `PASS`. Same program compiled with the knob off must print the same. +6. Validation, in this order: `make test` IR suite (14,465 expected; 6 known + YAFF-v4 reds), torture (11,228/0), `check_zig` on the Linux build + (`.cache/zigmem/agg/check_zig_z0.sh`; compare on-vs-off C output, md5s of + different renders do not match), then the QEMU device suite via + `./scripts/run_qemu_smoke.sh --no-build tcc_suite_test.py` after the rootfs + rebuild (the native tcc is compiled by the cross, so a codegen bug here + shows up as a self-host miscompile; see + `docs/selfhost_miscompile_debugging.md`). +7. Measure on zig.c with `.cache/zigmem/build/sw` recipe: expect + `Value_interpret__anon_196470` ≈ 8 KB, both `PackedCallingConvention_*` + ≈ 1 KB, object −30 KB. `pack` should end up below clang -Oz (3.6 KB), + which does not merge these 91 bodies. + +## Risks + +- **Selector clobbered inside the body.** Covered by the define/store scans; + the dry run did not need them only because no rewrite happened. Do not skip + them. +- **Aliased VAR selector.** Zig renders the selector as a local loaded from an + error-union temp; if its address escapes, a call in the body could change it. + The address-taken scan is the guard. +- **Width.** The dispatch compares the full register value (`CMP sel,#max` + then table index), so in case k the 32-bit selector value is exactly k. The + constant may be typed narrower (u8 tag); copying the selector operand keeps + the dispatch's width, and the consuming instruction (narrow store, or a + 32-bit op) sees the same bits it saw with the immediate. A *signed* narrow + selector with negative case values is handled by comparing the immediate both + sign- and zero-extended; when in doubt reject `is_unsigned` mismatches. +- **Occurrence counts.** `rm_match` requires each paired value to have all its + occurrences inside the two regions; adding a use of the selector breaks that + for the selector itself. Exclude the selector vreg from that check (it is + live across the whole function by construction). + +## What this does not do + +It is a ~30 KB, three-function win. The remaining repeated text in zig.c +(1.09 MB within functions, 1.95 MB counting across functions, by 8-instruction +windows) is inlined callee bodies with private frame slots, block copies, +no-op same-slot copies and cross-function idioms; see the analysis recorded in +the agent memory note `tcc-zigc-size-vs-clang-oz-2026-09-27` (section +"repeated text decomposition"). diff --git a/docs/zig_frame_promotion_plan.md b/docs/zig_frame_promotion_plan.md new file mode 100644 index 000000000..8c2748329 --- /dev/null +++ b/docs/zig_frame_promotion_plan.md @@ -0,0 +1,236 @@ +# Promoting Zig's frame objects to registers — plan + +Written 2026-09-27 for the tcc size work on the Zig compiler (`zig.c`, the C +the Zig C backend emits). Work happens in the worktree +`.cache/zigmem/tccsize` (detached at `zig-abi` 17c4a873); finished steps are +committed on `zig-abi`. + +## Goal and metric + +The device compiler is 5.71 MB of code; clang `-Oz` builds the same C into +1.96 MB, the same as Zig's own LLVM build. Going through C costs nothing; the +whole gap is tcc's code generation, and 71% of it is spread evenly over +ordinary functions (1.5–3x clang each), not in outliers. + +| build of `zig.c` (filler-stripped), cortex-m33, data via r9 | `.text` | +|---|---| +| tcc -O2 (this worktree's cross) | 5,288,782 | +| gcc -Os | 2,875,448 | +| clang -Os | 2,464,126 | +| clang -Oz | 1,957,572 | + +Metric: `.text` of `zig.c` compiled by the worktree cross, measured by +`sizecmp.py` / `patterns.py` (in `.cache/zigmem`). Success for this plan is a +sound, default-on change set worth **≥ 500 KB** (10%); the stretch is 1 MB. +Nothing lands that makes plain C worse, or that is not proven on the Zig +compiler end to end. + +## Why memory is the lever + +A value in a frame slot is invisible to every value-level pass tcc has. Once +stored, it is not constant-propagated, not folded, not CSE'd, not +dead-code-eliminated, and its known-zero upper bits are lost so it is +re-extended on every reload. The register allocator never gets to decide about +it either: a slot is a spill the allocator did not choose. + +`Type_toUnsigned` (404 B vs clang 184 B) shows all of it in one function: a +parameter copied to a local and read back through a pointer local, an 8-byte +error union built in four slots and copied three times, a switch selector +reloaded before every compare, a 72-byte frame against clang's 8. + +The "small levers" measured earlier (extends 92 KB, store→reload 111 KB, +0xAA stores 83 KB, selector reloads, argument spills) are symptoms of this one +cause. Promotion gets them by construction, plus the cases a peephole cannot +reach. + +## The pipeline today + +From `-dump-ir-passes=all`: flat passes (`ptr_local_fwd`, `param_copy_alias`, +`sl_forward`, `dse`, …) → SSA loop passes → **`sra` → `ssa_promote` → +`ssa_rename` → `ssa_phi` → SSA value passes** (sccp, cprop, gvn, narrow, +dce, …) → `frame_relayout` (colouring) → `ra_linear_scan`. + +So: SRA (`source/opt/flat/memory/sra.c`) turns whole frame objects into +per-field VARs; SSA promotion (`source/ir/ssa.c`, `ssa_var_promotable`) turns +VARs into renamed SSA values with phis; the allocator sees what is left. The +phi machinery exists and works (multi-block defs are promoted with phis). The +gap is in what the two front stages accept. + +## Where the bytes are stuck (measured on `zig.c`) + +SRA rejections, by object bytes (`TCC_SRA_DBG=1`, with the escaping-op detail +added in this worktree): + +| reason | bytes | what it is | +|---|---|---| +| addr-escape via FUNCPARAMVAL | 664 KB | address passed to a call: sret buffers, out-pointers | +| address via FUNCPARAMVAL | 509 KB | same, direct slot address as the argument | +| address via LEA into a VAR | 524 KB | Zig's pointer locals: `t1 = &t0; … *t1` | +| struct-param | 395 KB | a struct passed **by value** from a frame object | +| narrow-store | 186 KB | a byte/halfword store whose value has unknown upper bits | +| BLOCK_COPY operand | 178 KB | objects moved by inline block copies | +| too-big | 95 KB | > 32 bytes | + +Promotion refusals: SRA's field VARs are defined by slot STOREs, and +`ssa_var_promotable` keeps an upward-exposed STORE-only VAR memory-resident +(a cycles rule from gcc-torture `loop-15`). In `main_cmdAstCheck`, 308 SRA-made +VARs survive to the final IR unpromoted. Not yet counted TU-wide. + +Rejected already, do not retry: **A1** (SRA across calls, materializing before +the call: +62 KB at best, because a Zig temp is used at exactly one call so +building it in registers then storing costs more than building it in place); +**A2** (coalescing word-pair copies in frame colouring: −12 KB, blocked by the +source still being used after the copy). Both are in this worktree behind env +knobs, uncommitted, for reference. + +## Phases + +Each phase: instrument → census → change behind a knob → measure → make +default → validate (below) → commit. Stop a phase when its census says the +prize is small. + +### Phase 1 — Let SSA promotion take what SRA produced — DONE + +Result (on zig-abi 3ab7386e, which already had Phase 2): zig.c `.text` +5,139,194 -> 5,047,910 (**-91 KB, -1.8%**); Zig compiler jobs astcheck +1545M -> 1510M, sema 1212M -> 1176M. Two widenings in `ssa_var_promotable`, +both default-on, off with `TCC_DISABLE_PASS=ssa_prom_sra,ssa_prom_single`: + +- SRA's field VARs skip the STORE-only rule (census: 22,979 refused for it). + Capped at 2,000 SRA VARs per function (`TCC_SSA_PROM_SRA_MAX`): + `main_buildOutputType` has more, and promoting all of them grew it + 141 -> 206 KB. Cap 1000 -30 KB, 2000 -33 KB, 4000 +32 KB (before rebase). +- A VAR whose every definition is in the entry block (entry has no + predecessors) is renamed even when no phi is needed (census: 20,614 + single-def refusals). The broader "empty dominance frontier" version is + -117 KB but UNSOUND: the frontier under-reports on the un-rotated loop + shape (gcc-torture pr78675). +- Both are gated on a VAR never being named at an offset other than its home, + because renaming clears the offset (`_Complex double`, ieee/cdivchk). + The SRA range travels in `ir->sra_var_lo/hi`, reset before the SRA call site. + +What the census showed first (kept for the record): + +1. Census: a `TCC_SSA_DBG` line per VAR the promoter refuses, with the reason + and whether SRA created it (`ir->next_local_variable` before `sra_objects` + is the boundary). Count TU-wide. +2. Experiment A: promote SRA-created VARs regardless of the STORE-only rule. + The `loop-15` cycles concern does not apply to them (they were never + upward-exposed loop variables in the source). +3. Experiment B: have SRA emit 32-bit field stores of a value as ASSIGN + definitions, as `sra_store_as_def` already does for narrow fields. Then + they are not STORE-only and promote under the existing rule. +4. Pick whichever is smaller code and sound; measure `zig.c`, then the IR + suite's benchmark cycles (the scorecard is cycles, not instructions) to + confirm plain C is unchanged. + +### Phase 2 — Address roots in pointer VARs (524 KB of objects) — DONE by yasos-zig-35 + +Landed as zig-abi 3ab7386e ("sra: follow object addresses held in pointer +VARs"): "address" rejections 47.8K -> 17.5K objects, .text -153 KB on that +session's build. Coordinated split: that session owns flat pre-SSA SRA +(sra.c, ptr_local_fwd.c), this plan owns ssa.c promotion and frame.c. + +`sra_addr_def` follows an object's address only through TEMPs defined once. +Zig writes the address into a VAR (`V0 <-- Addr[StackLoc]`), then copies it to +TEMPs and derefs those. Extend the address-root tracking to a VAR whose every +definition is the same `LEA` of the same object (the `ptr_local_fwd` +condition), with the same escape rules as for TEMPs: every use is an access +base at a constant displacement or feeds another such pointer. Fields reached +through it become SRA fields. `ptr_local_fwd` stays for scalars; check the two +do not fight (order: `ptr_local_fwd` is flat and early, SRA late). + +Census first: how many of the 524 KB have a single-LEA pointer VAR and no other +escape. + +### Phase 3 — Small structs by value through calls as words (395 KB + callee side) + +AAPCS passes a ≤16-byte struct in r0–r3; tcc builds it in the frame and +passes the address of the frame object to `FUNCPARAMVAL`, which is why the +object must stay in memory. Two halves, measured separately: + +- **Caller.** `FUNCPARAMVAL` of a word-only struct ≤ 16 B from a frame object + → N word parameters sourced from the field VARs (`sra_struct_is_word` does + this for 4-byte structs today; generalize to 8/12/16 with the AAPCS + even-register rule for 8-byte-aligned members — see the `_Alignas` fix + 2da73cc1). +- **Callee.** An incoming struct parameter is copied to a `FRAME_OBJ_ARG_COPY` + slot; Zig then does `t2 = a1; t3 = &t2`. Make the arg copy an SRA unit fed + from the parameter registers, so `t2` gets field VARs and Phase 2 handles + `t3`. + +This is an ABI-adjacent change in the caller's lowering, not in the ABI: the +bytes in r0–r3 are the same. Still, rebuild every tcc object (libc, apps) +before the device suite, as after any call-lowering change. + +### Phase 4 — sret and out-pointer buffers (664 + 509 KB): reassess, probably leave + +A callee writing through a pointer needs the buffer in memory; clang pays this +too (`sub sp, #8` in `Type_toUnsigned` is exactly one such buffer). The cost +tcc adds is around the buffer: copying it into the caller's own return object +(the helper-copy merge in `frame.c` already removes the memmove form) and +reloading fields that are then stored on. After Phases 1–3, re-measure the +`ldr [slot]; str [sret]` pattern; if it is still large, the fix is in copy +coalescing (A2's residue) or in the return lowering, not in promotion. + +### Phase 5 — Real mem2reg for the residue (too-big 95 KB, block-copied 178 KB) + +Objects over 32 bytes, or moved by inline block copies, are outside SRA. A +per-slot SSA with phis (the escape analysis already exists in `frame.c` as the +taint/lifetime machinery) is the general answer. Only worth designing if +Phases 1–3 leave more than ~300 KB of the gap; measure first. + +### Phase 6 — Allocation quality after promotion + +With values in registers the residue is the allocator's: spill order, +rematerializing `add rX, sp, #k` (268 KB today) instead of keeping addresses +live, callee-saved pressure (28,193 saved registers vs clang's 23,220). +Measure after Phase 3; not before. + +## Validation, every phase + +- Build: `make -j24 armv8m-tcc` in the worktree (configure was + `--enable-cross --enable-O2`; the four untracked test deps are copied in). +- Size: compile `zig.c` with `. ../agg/inc.sh` flags (2.5 min); `sizecmp.py` + against the previous object and against `clang_Oz.o`; `patterns.py` for the + class breakdown. +- Zig end to end: `agg/build_zig_linux.sh O2` (`OUT=` names the binary; the + zigcmp script overwrites a shared file, do not use it) then + `zigcmp/check_zig.sh ` at -O1 and -O2. **check_zig is the oracle;** a + size win that fails it is not a win. +- Plain C: `make test` in the worktree (green baseline 14,213 / 273 skipped at + the branch point; the two YAFF v4 tests are red at HEAD), then the abcorpus + object diff for anything the change is not meant to touch. +- Fuzz last, never chained after the suite. +- Device: the QEMU tcc suite (`run_qemu_smoke.sh --no-build tcc_suite_test.py`) + before the superproject bump; the native tcc is itself compiled by the cross, + so a cross bug can surface only there. +- When check_zig fails: `mix/globalize.sh` + `fbisectj.sh`; on "neither half", + `agg/ddmin.sh` for the minimal function set; then a per-function env knob + (`TCC_…_FN=`) and `TCC_DUMP_FUNC= -dump-ir-passes=all`. A1's + bug (argument-order: an object passed as PARAMn is written after PARAM0) was + found this way in about 15 minutes. + +## Risks + +- **Soundness of relaxed promotion.** The `ssa_var_promotable` rules exist for + reasons (pr78675, pr125291 in the comments); relax only for VARs SRA created, + whose every access SRA has already proven. +- **Performance.** Promotion raises register pressure; the allocator may spill + what it should keep. Measure cycles on the benchmark scorecard after each + phase, not just size. +- **Aliasing rules.** Everything here must stay C-correct: an object whose + address reaches a call, a store, or a return stays in memory. A1 tested the + alternative and it did not pay anyway. +- **Exclusions stay.** `sra_function_eligible` already refuses inline asm, + setjmp, VLAs, nested functions; keep them refused. +- **Two compilers in one tree.** Changes to call lowering (Phase 3) require + rebuilding libc and every app before the device suite means anything. + +## Non-goals + +A1 and A2 (measured, rejected/marginal). Pruning the Zig compiler itself +(separate ~0.5 MB, in the Zig fork). Moving data to the SD card (done, +patch pending). Switch lowering, extends and the other symptom-level levers +(B–F): revisit after Phase 3 with fresh `patterns.py` numbers, since promotion +should have taken most of them. diff --git a/include/stdatomic.h b/include/stdatomic.h index 2d423d1a9..79887e87e 100644 --- a/include/stdatomic.h +++ b/include/stdatomic.h @@ -102,12 +102,10 @@ typedef struct { #define atomic_store(object, desired) \ atomic_store_explicit (object, desired, __ATOMIC_SEQ_CST) +/* The value-returning builtin: `__atomic_load (ptr, &tmp, order)` would take + * the address of a local, which keeps it in memory for good. */ #define atomic_load_explicit(object, order) \ - ({ __typeof__ (object) ptr = (object); \ - __typeof__ (*ptr) tmp; \ - __atomic_load (ptr, &tmp, (order)); \ - tmp; \ - }) + __atomic_load_n ((object), (order)) #define atomic_load(object) atomic_load_explicit (object, __ATOMIC_SEQ_CST) #define atomic_exchange_explicit(object, desired, order) \ diff --git a/include/tccdefs.h b/include/tccdefs.h index fadf6058d..58c78cb22 100644 --- a/include/tccdefs.h +++ b/include/tccdefs.h @@ -258,9 +258,7 @@ #endif /* not implemented */ #define __PRETTY_FUNCTION__ __FUNCTION__ -#define __has_builtin(x) 0 #define __has_feature(x) 0 -#define __has_attribute(x) 0 /* C23 Keywords */ #define _Nonnull #define _Nullable diff --git a/lib/Makefile b/lib/Makefile index c41455537..a33a107b6 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -76,7 +76,7 @@ OBJ-arm-vfp = $(OBJ-arm) OBJ-arm-eabi = $(OBJ-arm) OBJ-arm-eabihf = $(OBJ-arm) OBJ-arm-wince = $(ARM_O) $(WIN_O) -OBJ-armv8m = libtcc1.o alloca.o armeabi.o armeabi_divmod.o va_list.o builtin.o arm_string.o +OBJ-armv8m = libtcc1.o alloca.o armeabi.o armeabi_divmod.o va_list.o builtin.o arm_string.o arm_mem.o arm_copystub.o OBJ-riscv64 = $(RISCV64_O) $(LIN_O) OBJ-extra = $(filter $(EXTRA_O),$(OBJ-$T)) @@ -100,6 +100,12 @@ $(TOP)/%.o : %.c $(TCC) $S$(XCC) -c $< -o $@ $(XFLAGS) $(TOP)/bcheck.o : XFLAGS += $(BFLAGS) -I$(TOP) +# libtcc1 is linked into modules of every GOT size and its slots come after the +# module's own, so its one global-reading member must not use the 4 KB-reach +# `ldr Rt,[r9,#imm12]` form (ctags' 13 KB GOT put chk_calls at offset 13040). +ifeq ($T,armv8m) +$(X)builtin.o : XFLAGS += -mno-sb-relative-got +endif # includes tccrun.c, $(TOP) for config.h (tccrun.c prerequisite via $(TCC)) $(TOP)/bt-exe.o : XFLAGS += -I$(TOP) diff --git a/lib/arm_copystub.S b/lib/arm_copystub.S new file mode 100644 index 000000000..f8013573c --- /dev/null +++ b/lib/arm_copystub.S @@ -0,0 +1,218 @@ +/* + * Fixed-size word copies the compiler calls instead of expanding a struct + * copy inline (arm-thumb-gen.c, thumb_copy_stub_call). + * + * __tcc_wcopy_N (r0 = dest, r1 = src), N = 3..32 words + * + * Both pointers are word aligned and the ranges are the same object or + * disjoint ones (C struct assignment, a by-value argument, an initialiser + * image), so the words go forwards, each group loaded before it is stored. + * Clobbers r0-r3, ip and lr -- what the inline copy clobbered -- and nothing + * else: no stack, no R9, no flags. + * + * Three chains, one per N mod 3. Each entry is the point in its chain with + * N/3 LDM/STM pairs of three words still to go; the chain ends in the 0-2 + * word remainder and the return. An N-register LDM/STM takes 1+N cycles on + * the M33, so three-register pairs cost what the inline four-register ones + * did per word; the call adds the BL and the BX. An inline 64-byte copy was + * 32 bytes of code at every site; the call is 4. + */ + +.syntax unified +.thumb + +.text +.align 2 + +/* N = 3k + 0 */ +.global __tcc_wcopy_30 +.type __tcc_wcopy_30, %function +.thumb_func +__tcc_wcopy_30: + ldmia r1!, {r2, r3, ip} + stmia r0!, {r2, r3, ip} +.global __tcc_wcopy_27 +.type __tcc_wcopy_27, %function +.thumb_func +__tcc_wcopy_27: + ldmia r1!, {r2, r3, ip} + stmia r0!, {r2, r3, ip} +.global __tcc_wcopy_24 +.type __tcc_wcopy_24, %function +.thumb_func +__tcc_wcopy_24: + ldmia r1!, {r2, r3, ip} + stmia r0!, {r2, r3, ip} +.global __tcc_wcopy_21 +.type __tcc_wcopy_21, %function +.thumb_func +__tcc_wcopy_21: + ldmia r1!, {r2, r3, ip} + stmia r0!, {r2, r3, ip} +.global __tcc_wcopy_18 +.type __tcc_wcopy_18, %function +.thumb_func +__tcc_wcopy_18: + ldmia r1!, {r2, r3, ip} + stmia r0!, {r2, r3, ip} +.global __tcc_wcopy_15 +.type __tcc_wcopy_15, %function +.thumb_func +__tcc_wcopy_15: + ldmia r1!, {r2, r3, ip} + stmia r0!, {r2, r3, ip} +.global __tcc_wcopy_12 +.type __tcc_wcopy_12, %function +.thumb_func +__tcc_wcopy_12: + ldmia r1!, {r2, r3, ip} + stmia r0!, {r2, r3, ip} +.global __tcc_wcopy_9 +.type __tcc_wcopy_9, %function +.thumb_func +__tcc_wcopy_9: + ldmia r1!, {r2, r3, ip} + stmia r0!, {r2, r3, ip} +.global __tcc_wcopy_6 +.type __tcc_wcopy_6, %function +.thumb_func +__tcc_wcopy_6: + ldmia r1!, {r2, r3, ip} + stmia r0!, {r2, r3, ip} +.global __tcc_wcopy_3 +.type __tcc_wcopy_3, %function +.thumb_func +__tcc_wcopy_3: + ldmia r1!, {r2, r3, ip} + stmia r0!, {r2, r3, ip} + bx lr + +/* N = 3k + 1 */ +.global __tcc_wcopy_31 +.type __tcc_wcopy_31, %function +.thumb_func +__tcc_wcopy_31: + ldmia r1!, {r2, r3, ip} + stmia r0!, {r2, r3, ip} +.global __tcc_wcopy_28 +.type __tcc_wcopy_28, %function +.thumb_func +__tcc_wcopy_28: + ldmia r1!, {r2, r3, ip} + stmia r0!, {r2, r3, ip} +.global __tcc_wcopy_25 +.type __tcc_wcopy_25, %function +.thumb_func +__tcc_wcopy_25: + ldmia r1!, {r2, r3, ip} + stmia r0!, {r2, r3, ip} +.global __tcc_wcopy_22 +.type __tcc_wcopy_22, %function +.thumb_func +__tcc_wcopy_22: + ldmia r1!, {r2, r3, ip} + stmia r0!, {r2, r3, ip} +.global __tcc_wcopy_19 +.type __tcc_wcopy_19, %function +.thumb_func +__tcc_wcopy_19: + ldmia r1!, {r2, r3, ip} + stmia r0!, {r2, r3, ip} +.global __tcc_wcopy_16 +.type __tcc_wcopy_16, %function +.thumb_func +__tcc_wcopy_16: + ldmia r1!, {r2, r3, ip} + stmia r0!, {r2, r3, ip} +.global __tcc_wcopy_13 +.type __tcc_wcopy_13, %function +.thumb_func +__tcc_wcopy_13: + ldmia r1!, {r2, r3, ip} + stmia r0!, {r2, r3, ip} +.global __tcc_wcopy_10 +.type __tcc_wcopy_10, %function +.thumb_func +__tcc_wcopy_10: + ldmia r1!, {r2, r3, ip} + stmia r0!, {r2, r3, ip} +.global __tcc_wcopy_7 +.type __tcc_wcopy_7, %function +.thumb_func +__tcc_wcopy_7: + ldmia r1!, {r2, r3, ip} + stmia r0!, {r2, r3, ip} +.global __tcc_wcopy_4 +.type __tcc_wcopy_4, %function +.thumb_func +__tcc_wcopy_4: + ldmia r1!, {r2, r3, ip} + stmia r0!, {r2, r3, ip} + ldr r2, [r1] + str r2, [r0] + bx lr + +/* N = 3k + 2 */ +.global __tcc_wcopy_32 +.type __tcc_wcopy_32, %function +.thumb_func +__tcc_wcopy_32: + ldmia r1!, {r2, r3, ip} + stmia r0!, {r2, r3, ip} +.global __tcc_wcopy_29 +.type __tcc_wcopy_29, %function +.thumb_func +__tcc_wcopy_29: + ldmia r1!, {r2, r3, ip} + stmia r0!, {r2, r3, ip} +.global __tcc_wcopy_26 +.type __tcc_wcopy_26, %function +.thumb_func +__tcc_wcopy_26: + ldmia r1!, {r2, r3, ip} + stmia r0!, {r2, r3, ip} +.global __tcc_wcopy_23 +.type __tcc_wcopy_23, %function +.thumb_func +__tcc_wcopy_23: + ldmia r1!, {r2, r3, ip} + stmia r0!, {r2, r3, ip} +.global __tcc_wcopy_20 +.type __tcc_wcopy_20, %function +.thumb_func +__tcc_wcopy_20: + ldmia r1!, {r2, r3, ip} + stmia r0!, {r2, r3, ip} +.global __tcc_wcopy_17 +.type __tcc_wcopy_17, %function +.thumb_func +__tcc_wcopy_17: + ldmia r1!, {r2, r3, ip} + stmia r0!, {r2, r3, ip} +.global __tcc_wcopy_14 +.type __tcc_wcopy_14, %function +.thumb_func +__tcc_wcopy_14: + ldmia r1!, {r2, r3, ip} + stmia r0!, {r2, r3, ip} +.global __tcc_wcopy_11 +.type __tcc_wcopy_11, %function +.thumb_func +__tcc_wcopy_11: + ldmia r1!, {r2, r3, ip} + stmia r0!, {r2, r3, ip} +.global __tcc_wcopy_8 +.type __tcc_wcopy_8, %function +.thumb_func +__tcc_wcopy_8: + ldmia r1!, {r2, r3, ip} + stmia r0!, {r2, r3, ip} +.global __tcc_wcopy_5 +.type __tcc_wcopy_5, %function +.thumb_func +__tcc_wcopy_5: + ldmia r1!, {r2, r3, ip} + stmia r0!, {r2, r3, ip} + ldrd r2, r3, [r1] + strd r2, r3, [r0] + bx lr diff --git a/lib/arm_mem.S b/lib/arm_mem.S new file mode 100644 index 000000000..a74091208 --- /dev/null +++ b/lib/arm_mem.S @@ -0,0 +1,336 @@ +/* + * ARM EABI memory helpers for ARMv8-M Mainline (Cortex-M33), Thumb-2. + * + * __aeabi_memcpy, __aeabi_memcpy_aligned (dest, src, n) + * __aeabi_memmove, __aeabi_memmove4, __aeabi_memmove8 (dest, src, n) + * __aeabi_memset (dest, n, c) -- EABI argument order, not ISO memset's + * __aeabi_memclr (dest, n) + * + * All of them return dest in r0 (the EABI says void; the C versions these + * replace returned dest and callers may rely on it). + * + * TCC lowers every struct copy it does not inline to __aeabi_memmove{,4,8}, + * so these run on hot paths: the C versions they replace moved one word in + * 13 instructions. Here a 16-byte block is one LDM + one STM. + * + * LDM/STM need word-aligned addresses, so the destination is aligned first + * and the block loop runs only when the source is then aligned too. A source + * that is not takes single LDRs, which ARMv8-M performs unaligned in Normal + * memory unless CCR.UNALIGN_TRP is set (YasOS leaves it clear, as does the + * reset value), and so do the halfword/word tails of short copies. + * + * memmove copies forward whenever the destination does not start inside the + * source ((unsigned)(dest - src) >= n); otherwise it copies from the top + * down. Each piece is loaded completely before it is stored, which is what + * makes both directions safe for overlapping ranges. + */ + +.syntax unified +.thumb + +.text +.align 2 + +/* ------------------------------------------------------------------------ + * Forward copy. r0 = dest, r1 = src, r2 = n. Returns dest. + * ------------------------------------------------------------------------ */ +.global __aeabi_memcpy +.type __aeabi_memcpy, %function +.global __aeabi_memcpy_aligned +.type __aeabi_memcpy_aligned, %function +.thumb_func +__aeabi_memcpy: +.thumb_func +__aeabi_memcpy_aligned: +.Lcpy: + mov ip, r0 + cmp r2, #8 + blo .Lcpy_small + push {r4, r5, r6, r7} + /* align dest (n >= 8, so the 1..3 bytes are there) */ + ands r3, r0, #3 + beq .Lcpy_dst_al + rsb r3, r3, #4 + subs r2, r2, r3 +.Lcpy_al: + ldrb r4, [r1], #1 + strb r4, [r0], #1 + subs r3, r3, #1 + bne .Lcpy_al +.Lcpy_dst_al: + tst r1, #3 + bne .Lcpy_usrc + subs r2, r2, #16 + blo .Lcpy_tail +.Lcpy_16: + ldmia r1!, {r3, r4, r5, r6} + stmia r0!, {r3, r4, r5, r6} + subs r2, r2, #16 + bhs .Lcpy_16 +.Lcpy_tail: + /* r2 = remaining - 16; its low four bits are the remaining count */ + lsls r3, r2, #29 /* C = bit 3, N = bit 2 */ + itt cs + ldmiacs r1!, {r4, r5} + stmiacs r0!, {r4, r5} + itt mi + ldrmi r4, [r1], #4 + strmi r4, [r0], #4 + lsls r3, r2, #31 /* C = bit 1, N = bit 0 */ + itt cs + ldrhcs r4, [r1], #2 + strhcs r4, [r0], #2 + itt mi + ldrbmi r4, [r1] + strbmi r4, [r0] + pop {r4, r5, r6, r7} + mov r0, ip + bx lr + +.Lcpy_usrc: + /* dest aligned, src not: unaligned single loads, aligned block stores */ + subs r2, r2, #16 + blo .Lcpy_utail +.Lcpy_u16: + ldr r3, [r1] + ldr r4, [r1, #4] + ldr r5, [r1, #8] + ldr r6, [r1, #12] + adds r1, r1, #16 + stmia r0!, {r3, r4, r5, r6} + subs r2, r2, #16 + bhs .Lcpy_u16 +.Lcpy_utail: + lsls r3, r2, #29 /* C = bit 3, N = bit 2 */ + bcc .Lcpy_u4 + ldr r4, [r1] + ldr r5, [r1, #4] + adds r1, r1, #8 + stmia r0!, {r4, r5} +.Lcpy_u4: + lsls r3, r2, #29 + itt mi + ldrmi r4, [r1], #4 + strmi r4, [r0], #4 + lsls r3, r2, #31 /* C = bit 1, N = bit 0 */ + itt cs + ldrhcs r4, [r1], #2 + strhcs r4, [r0], #2 + itt mi + ldrbmi r4, [r1] + strbmi r4, [r0] + pop {r4, r5, r6, r7} + mov r0, ip + bx lr + +.Lcpy_small: + /* n < 8: a word, a halfword and a byte as n's bits say */ + lsls r3, r2, #30 /* C = bit 2, N = bit 1 */ + itt cs + ldrcs r3, [r1], #4 + strcs r3, [r0], #4 + lsls r3, r2, #30 + itt mi + ldrhmi r3, [r1], #2 + strhmi r3, [r0], #2 + lsls r3, r2, #31 /* N = bit 0 */ + itt mi + ldrbmi r3, [r1] + strbmi r3, [r0] + mov r0, ip + bx lr +.size __aeabi_memcpy, .-__aeabi_memcpy +.size __aeabi_memcpy_aligned, .-__aeabi_memcpy_aligned + +/* ------------------------------------------------------------------------ + * memmove. r0 = dest, r1 = src, r2 = n. Returns dest. + * ------------------------------------------------------------------------ */ +.global __aeabi_memmove +.type __aeabi_memmove, %function +.global __aeabi_memmove4 +.type __aeabi_memmove4, %function +.global __aeabi_memmove8 +.type __aeabi_memmove8, %function +.thumb_func +__aeabi_memmove: +.thumb_func +__aeabi_memmove4: +.thumb_func +__aeabi_memmove8: + subs r3, r0, r1 + cmp r3, r2 + bhs .Lcpy /* dest outside [src, src + n): forward */ + cmp r3, #0 + beq .Lmov_ret /* dest == src (n > 0 here) */ + /* backward: dest - src in (0, n) */ + mov ip, r0 + adds r0, r0, r2 + adds r1, r1, r2 + cmp r2, #8 + blo .Lmov_small + push {r4, r5, r6, r7} + ands r3, r0, #3 /* align the dest END down */ + beq .Lmov_dst_al + subs r2, r2, r3 +.Lmov_al: + ldrb r4, [r1, #-1]! + strb r4, [r0, #-1]! + subs r3, r3, #1 + bne .Lmov_al +.Lmov_dst_al: + tst r1, #3 + bne .Lmov_usrc + subs r2, r2, #16 + blo .Lmov_tail +.Lmov_16: + ldmdb r1!, {r3, r4, r5, r6} + stmdb r0!, {r3, r4, r5, r6} + subs r2, r2, #16 + bhs .Lmov_16 +.Lmov_tail: + lsls r3, r2, #29 /* C = bit 3, N = bit 2 */ + itt cs + ldmdbcs r1!, {r4, r5} + stmdbcs r0!, {r4, r5} + itt mi + ldrmi r4, [r1, #-4]! + strmi r4, [r0, #-4]! + lsls r3, r2, #31 /* C = bit 1, N = bit 0 */ + itt cs + ldrhcs r4, [r1, #-2]! + strhcs r4, [r0, #-2]! + itt mi + ldrbmi r4, [r1, #-1] + strbmi r4, [r0, #-1] + pop {r4, r5, r6, r7} + mov r0, ip + bx lr + +.Lmov_usrc: + subs r2, r2, #16 + blo .Lmov_utail +.Lmov_u16: + ldr r3, [r1, #-16] + ldr r4, [r1, #-12] + ldr r5, [r1, #-8] + ldr r6, [r1, #-4] + subs r1, r1, #16 + stmdb r0!, {r3, r4, r5, r6} + subs r2, r2, #16 + bhs .Lmov_u16 +.Lmov_utail: + lsls r3, r2, #29 /* C = bit 3, N = bit 2 */ + bcc .Lmov_u4 + ldr r4, [r1, #-8] + ldr r5, [r1, #-4] + subs r1, r1, #8 + stmdb r0!, {r4, r5} +.Lmov_u4: + lsls r3, r2, #29 + itt mi + ldrmi r4, [r1, #-4]! + strmi r4, [r0, #-4]! + lsls r3, r2, #31 /* C = bit 1, N = bit 0 */ + itt cs + ldrhcs r4, [r1, #-2]! + strhcs r4, [r0, #-2]! + itt mi + ldrbmi r4, [r1, #-1] + strbmi r4, [r0, #-1] + pop {r4, r5, r6, r7} + mov r0, ip + bx lr + +.Lmov_small: + /* n < 8, top down */ + lsls r3, r2, #30 /* C = bit 2, N = bit 1 */ + itt cs + ldrcs r3, [r1, #-4]! + strcs r3, [r0, #-4]! + lsls r3, r2, #30 + itt mi + ldrhmi r3, [r1, #-2]! + strhmi r3, [r0, #-2]! + lsls r3, r2, #31 /* N = bit 0 */ + itt mi + ldrbmi r3, [r1, #-1] + strbmi r3, [r0, #-1] + mov r0, ip +.Lmov_ret: + bx lr +.size __aeabi_memmove, .-__aeabi_memmove +.size __aeabi_memmove4, .-__aeabi_memmove4 +.size __aeabi_memmove8, .-__aeabi_memmove8 + +/* ------------------------------------------------------------------------ + * ISO memset(dest, c, n). The compiler binds calls to memset here instead + * of libc's (a PLT call that also costs the caller an R9 reload): swap into + * the EABI order and fall into __aeabi_memset, which returns dest. + * ------------------------------------------------------------------------ */ +.global __tcc_memset +.type __tcc_memset, %function +.thumb_func +__tcc_memset: + mov r3, r1 + mov r1, r2 + mov r2, r3 + b __aeabi_memset +.size __tcc_memset, .-__tcc_memset + +/* ------------------------------------------------------------------------ + * memset. r0 = dest, r1 = n, r2 = c (EABI order). Returns dest. + * ------------------------------------------------------------------------ */ +.global __aeabi_memclr +.type __aeabi_memclr, %function +.global __aeabi_memset +.type __aeabi_memset, %function +.thumb_func +__aeabi_memclr: + movs r2, #0 +.thumb_func +__aeabi_memset: + mov ip, r0 + uxtb r2, r2 + orr r2, r2, r2, lsl #8 + orr r2, r2, r2, lsl #16 + cmp r1, #8 + blo .Lset_small + ands r3, r0, #3 + beq .Lset_dst_al + rsb r3, r3, #4 + subs r1, r1, r3 +.Lset_al: + strb r2, [r0], #1 + subs r3, r3, #1 + bne .Lset_al +.Lset_dst_al: + push {r4, r5} + mov r3, r2 + mov r4, r2 + mov r5, r2 + subs r1, r1, #16 + blo .Lset_tail +.Lset_16: + stmia r0!, {r2, r3, r4, r5} + subs r1, r1, #16 + bhs .Lset_16 +.Lset_tail: + pop {r4, r5} + lsls r3, r1, #29 /* C = bit 3; bits 2..0 go below */ + itt cs + strcs r2, [r0], #4 + strcs r2, [r0], #4 +.Lset_small: + /* the low three bits of r1 are what is left (n < 8, or the tail) */ + lsls r3, r1, #30 /* C = bit 2, N = bit 1 */ + it cs + strcs r2, [r0], #4 + it mi + strhmi r2, [r0], #2 + lsls r3, r1, #31 /* N = bit 0 */ + it mi + strbmi r2, [r0] + mov r0, ip + bx lr +.size __aeabi_memclr, .-__aeabi_memclr +.size __aeabi_memset, .-__aeabi_memset diff --git a/lib/armeabi.c b/lib/armeabi.c index 4b36d48f0..ffd77df7e 100644 --- a/lib/armeabi.c +++ b/lib/armeabi.c @@ -69,7 +69,9 @@ static double aeabi_dneg_impl(double a) /* ARM EABI required symbols for non-FP operations */ -/* Memory functions required by EABI */ +/* Memory functions required by EABI. ARMv8-M takes them from arm_mem.S + * (LDM/STM blocks); these byte/word loops remain for the other ARM targets. */ +#if !defined(__ARM_ARCH_8M__) /* NOTE: ARM EABI defines __aeabi_memset() argument order as (dest, n, c), * i.e. it differs from ISO C memset(dest, c, n). @@ -200,6 +202,8 @@ void __aeabi_memclr(void *dest, size_t n) (void)__aeabi_memset(dest, n, 0); } +#endif /* !__ARM_ARCH_8M__ */ + /* Division functions */ /* Unsigned 32-bit division */ @@ -256,31 +260,65 @@ typedef struct u32 remainder_high; } uint64_div_result; -static int u64_ge(u32 a_lo, u32 a_hi, u32 b_lo, u32 b_hi) +static int armeabi_clz32(u32 x); + +/* 32 x 32 -> 64 multiply from 16-bit halves (32-bit operations only). */ +static void umul32(u32 a, u32 b, u32 *lo, u32 *hi) { - /* Compare 64-bit values: return 1 if a >= b, else 0 */ - if (a_hi > b_hi) - return 1; - if (a_hi < b_hi) - return 0; - /* High words equal, compare low words (unsigned) */ - if (a_lo > b_lo) - return 1; - if (a_lo < b_lo) - return 0; - return 1; /* equal */ + u32 al = a & 0xFFFFu, ah = a >> 16, bl = b & 0xFFFFu, bh = b >> 16; + u32 ll = al * bl, lh = al * bh, hl = ah * bl, hh = ah * bh; + u32 mid = (ll >> 16) + (lh & 0xFFFFu) + (hl & 0xFFFFu); + *lo = (ll & 0xFFFFu) | (mid << 16); + *hi = hh + (lh >> 16) + (hl >> 16) + (mid >> 16); } -static inline void u64_sub(u32 *a_lo, u32 *a_hi, u32 b_lo, u32 b_hi) +/* (u1:u0) / v for u1 < v: a 32-bit quotient, the remainder in *r. + * Hacker's Delight divlu: normalise v, then two 16-bit quotient digits, each + * estimated by dividing by v's top 16 bits (UDIV on ARMv8-M) and corrected + * at most twice. */ +static u32 divlu(u32 u1, u32 u0, u32 v, u32 *r) { - const u32 old_lo = *a_lo; - *a_lo = old_lo - b_lo; - const u32 borrow = (old_lo < b_lo); - *a_hi = *a_hi - b_hi - borrow; + int s = armeabi_clz32(v); + u32 vn1, vn0, un32, un21, un10, un1, un0, q1, q0, rhat; + + v <<= s; + vn1 = v >> 16; + vn0 = v & 0xFFFFu; + un32 = s ? (u1 << s) | (u0 >> (32 - s)) : u1; + un10 = u0 << s; + un1 = un10 >> 16; + un0 = un10 & 0xFFFFu; + + q1 = un32 / vn1; + rhat = un32 - q1 * vn1; + while (q1 > 0xFFFFu || q1 * vn0 > ((rhat << 16) | un1)) + { + q1--; + rhat += vn1; + if (rhat > 0xFFFFu) + break; + } + un21 = (un32 << 16) + un1 - q1 * v; + + q0 = un21 / vn1; + rhat = un21 - q0 * vn1; + while (q0 > 0xFFFFu || q0 * vn0 > ((rhat << 16) | un0)) + { + q0--; + rhat += vn1; + if (rhat > 0xFFFFu) + break; + } + *r = ((un21 << 16) + un0 - q0 * v) >> s; + return (q1 << 16) | q0; } +/* 64 / 64 division (Hacker's Delight divdu), dividing by zero gives 0 rem 0. + * The bit-at-a-time loop this replaces took ~2,600 instructions a call. */ static void udivmod_u64(uint64_div_result *out, u32 n_lo, u32 n_hi, u32 d_lo, u32 d_hi) { + u32 s, v1, q0, junk, p_lo, p_hi, r_lo, r_hi; + out->quotient_low = 0; out->quotient_high = 0; out->remainder_low = 0; @@ -289,31 +327,45 @@ static void udivmod_u64(uint64_div_result *out, u32 n_lo, u32 n_hi, u32 d_lo, u3 if ((d_lo | d_hi) == 0) return; - int debug_count = 0; - - for (int i = 63; i >= 0; --i) + if (d_hi == 0) { - /* r <<= 1 */ - out->remainder_high = (out->remainder_high << 1) | (out->remainder_low >> 31); - out->remainder_low <<= 1; - - /* r |= (n >> i) & 1 */ - u32 bit; - if (i >= 32) - bit = (n_hi >> (i - 32)) & 1u; - else - bit = (n_lo >> i) & 1u; - out->remainder_low |= bit; - - if (u64_ge(out->remainder_low, out->remainder_high, d_lo, d_hi)) + u32 k = n_hi; + if (n_hi >= d_lo) { - u64_sub(&out->remainder_low, &out->remainder_high, d_lo, d_hi); - if (i >= 32) - out->quotient_high |= (1u << (i - 32)); - else - out->quotient_low |= (1u << i); + out->quotient_high = n_hi / d_lo; + k = n_hi - out->quotient_high * d_lo; } + out->quotient_low = divlu(k, n_lo, d_lo, &out->remainder_low); + return; } + + if (n_hi < d_hi || (n_hi == d_hi && n_lo < d_lo)) + { + out->remainder_low = n_lo; + out->remainder_high = n_hi; + return; + } + + /* d >= 2^32: the quotient fits 32 bits. Estimate it from n / 2 over the + * top 32 bits of d normalised; the estimate is exact or one too big. */ + s = armeabi_clz32(d_hi); + v1 = s ? (d_hi << s) | (d_lo >> (32 - s)) : d_hi; + q0 = divlu(n_hi >> 1, (n_lo >> 1) | (n_hi << 31), v1, &junk) >> (31 - s); + if (q0 != 0) + q0--; + umul32(q0, d_lo, &p_lo, &p_hi); + p_hi += q0 * d_hi; + r_lo = n_lo - p_lo; + r_hi = n_hi - p_hi - (n_lo < p_lo); + if (r_hi > d_hi || (r_hi == d_hi && r_lo >= d_lo)) + { + q0++; + r_hi = r_hi - d_hi - (r_lo < d_lo); + r_lo -= d_lo; + } + out->quotient_low = q0; + out->remainder_low = r_lo; + out->remainder_high = r_hi; } /* Helpers for __aeabi_{u,}ldivmod wrappers. diff --git a/lib/armeabi_divmod.S b/lib/armeabi_divmod.S index 9333324b4..975ae9410 100644 --- a/lib/armeabi_divmod.S +++ b/lib/armeabi_divmod.S @@ -10,6 +10,11 @@ * * Implement the public symbols in assembly and delegate the computation to C * helpers that write results to memory. + * + * When both high words are zero (and the divisor is not), the operands are + * 32-bit values that the hardware divider takes directly, signed or not: a + * zero high word means non-negative. That path returns without the call + * (only where the core has UDIV). */ .syntax unified @@ -26,6 +31,18 @@ .global __aeabi_uldivmod .type __aeabi_uldivmod, %function __aeabi_uldivmod: +#ifdef __ARM_ARCH_EXT_IDIV__ + orrs ip, r1, r3 + bne .Luldivmod_call + cmp r2, #0 + beq .Luldivmod_call + udiv ip, r0, r2 + mls r2, ip, r2, r0 /* remainder */ + mov r0, ip + movs r3, #0 + bx lr +#endif +.Luldivmod_call: push {r4, r5, r6, lr} sub sp, sp, #16 /* output slots: qlo,qhi,rlo,rhi */ mov r6, sp /* r6 = &out[0] */ @@ -59,6 +76,18 @@ __aeabi_uldivmod: .global __aeabi_ldivmod .type __aeabi_ldivmod, %function __aeabi_ldivmod: +#ifdef __ARM_ARCH_EXT_IDIV__ + orrs ip, r1, r3 + bne .Lldivmod_call + cmp r2, #0 + beq .Lldivmod_call + udiv ip, r0, r2 + mls r2, ip, r2, r0 /* remainder */ + mov r0, ip + movs r3, #0 + bx lr +#endif +.Lldivmod_call: push {r4, r5, r6, lr} sub sp, sp, #16 /* output slots: qlo,qhi,rlo,rhi */ mov r6, sp /* r6 = &out[0] */ @@ -92,6 +121,16 @@ __aeabi_ldivmod: .global __aeabi_ulmod .type __aeabi_ulmod, %function __aeabi_ulmod: +#ifdef __ARM_ARCH_EXT_IDIV__ + orrs ip, r1, r3 + bne .Lulmod_call + cmp r2, #0 + beq .Lulmod_call + udiv ip, r0, r2 + mls r0, ip, r2, r0 /* remainder */ + bx lr +#endif +.Lulmod_call: push {r4, r5, r6, lr} sub sp, sp, #16 /* output slots: qlo,qhi,rlo,rhi */ mov r6, sp /* r6 = &out[0] */ @@ -123,6 +162,16 @@ __aeabi_ulmod: .global __aeabi_lmod .type __aeabi_lmod, %function __aeabi_lmod: +#ifdef __ARM_ARCH_EXT_IDIV__ + orrs ip, r1, r3 + bne .Llmod_call + cmp r2, #0 + beq .Llmod_call + udiv ip, r0, r2 + mls r0, ip, r2, r0 /* remainder */ + bx lr +#endif +.Llmod_call: push {r4, r5, r6, lr} sub sp, sp, #16 /* output slots: qlo,qhi,rlo,rhi */ mov r6, sp /* r6 = &out[0] */ diff --git a/lib/builtin.c b/lib/builtin.c index f48f4b5f8..628e8938c 100644 --- a/lib/builtin.c +++ b/lib/builtin.c @@ -513,6 +513,27 @@ int __tcc_memcmp1(const void *lhs, const void *rhs) return (int)a[0] - (int)b[0]; } +/* memcmp for calls the compiler binds here instead of libc's (see + * thumb_local_libc_helper): word steps while both sides are word-aligned and + * equal, bytes to find the first difference. */ +int __tcc_memcmp(const void *lhs, const void *rhs, unsigned long n) +{ + const unsigned char *a = (const unsigned char *)lhs; + const unsigned char *b = (const unsigned char *)rhs; + + if ((((unsigned long)a | (unsigned long)b) & 3) == 0) + while (n >= 4 && *(const unsigned *)a == *(const unsigned *)b) + { + a += 4; + b += 4; + n -= 4; + } + for (; n > 0; n--, a++, b++) + if (*a != *b) + return (int)*a - (int)*b; + return 0; +} + int __tcc_strncmp(const char *lhs, const char *rhs, unsigned long n) { const unsigned char *a = (const unsigned char *)lhs; diff --git a/lib/fp/Makefile b/lib/fp/Makefile index 98112e374..d66870c7b 100644 --- a/lib/fp/Makefile +++ b/lib/fp/Makefile @@ -114,6 +114,7 @@ endif # armeabi.c is compiled and linked in because yasld symbol resolution only # searches self + children (not parent modules), so __aeabi_memset, # __aeabi_lcmp, etc. must be statically included in the shared library. +# arm_mem.S carries the __aeabi_mem* entry points armeabi.c leaves to it. build-shared: @echo "Building shared FP library for $(FPU)..." @mkdir -p $(BUILD_DIR)/$(FPU)-pic @@ -121,6 +122,8 @@ build-shared: FP_CC="$(FP_CC)" FP_CFLAGS="$(FP_CFLAGS) -I$(CURDIR)/../../include -fPIC -fpic" $(FP_CC) $(FP_CFLAGS) -I$(CURDIR)/../../include -fPIC -fpic \ -c $(CURDIR)/../armeabi.c -o $(BUILD_DIR)/$(FPU)-pic/armeabi.o + $(FP_CC) $(FP_CFLAGS) -I$(CURDIR)/../../include -fPIC -fpic \ + -c $(CURDIR)/../arm_mem.S -o $(BUILD_DIR)/$(FPU)-pic/arm_mem.o $(FP_CC) -shared -fPIC -nodefaultlibs -Wl,-Ttext=0x0 -Wl,-section-alignment=0x4 \ -Wl,-oformat=elf32-littlearm \ -o $(TARGET_SO).elf $(BUILD_DIR)/$(FPU)-pic/*.o diff --git a/scripts/bisect_pass.sh b/scripts/bisect_pass.sh index c941be084..e574b0f2b 100755 --- a/scripts/bisect_pass.sh +++ b/scripts/bisect_pass.sh @@ -11,7 +11,7 @@ run() { TCC_DISABLE_PASS="$1" python run.py -c "../../$SEED" --cflags="$OLEVEL" 2>/dev/null | grep -o 'checksum=[0-9a-f]*' } -PASSES="uninit_ub uninit_dom_ret dce const_prop const_var_prop global_init symref_prop global_sl_fwd const_prop_tmp const_agg_fold known_bits neg_chain_cse add_reassoc redundant_assign string_calls self_copy_elim value_tracking cmp_expr_fold self_arith cmp_offset_fold branch_fold switch_collapse stack_nonnull setif_fuse stack_bool or_bool setif_or_taut var_tmp_fwd var_to_tmp nonneg_fold float_branch vrp single_val_tmp float_narrow deref_fwd fusion_mla deref_indexed disp_fusion copy_prop chain_fold pair_reorder postinc bool_simplify sl_forward bf_insert_extract cmp_field_fuse const_cascade branch_fold_2x jump_thread elim_fallthru kb_cascade branch_cleanup dead_vla_struct alloca_load_fwd zero_vla byte_store_merge store_redundant dse dead_static_store dead_var_store dead_addrvar dead_trail_addrvar dead_alloca_vreg dead_local_slot dead_lea_store dead_temp_local inplace_arith global_base_share orphan_cmp inf_loop_simpl dead_pre_inf return_reuse entry_store esp_cleanup" +PASSES="uninit_ub uninit_dom_ret dce const_prop const_var_prop global_init symref_prop global_sl_fwd const_prop_tmp const_agg_fold known_bits add_reassoc redundant_assign string_calls self_copy_elim value_tracking cmp_expr_fold self_arith cmp_offset_fold branch_fold switch_collapse stack_nonnull setif_fuse stack_bool or_bool setif_or_taut var_tmp_fwd var_to_tmp nonneg_fold vrp deref_fwd fusion_mla deref_indexed disp_fusion copy_prop chain_fold pair_reorder postinc bool_simplify sl_forward bf_insert_extract cmp_field_fuse const_cascade branch_fold_2x jump_thread elim_fallthru kb_cascade branch_cleanup dead_vla_struct alloca_load_fwd zero_vla byte_store_merge store_redundant dse dead_static_store dead_var_store dead_addrvar dead_trail_addrvar dead_alloca_vreg dead_local_slot dead_lea_store dead_temp_local inplace_arith global_base_share orphan_cmp inf_loop_simpl dead_pre_inf return_reuse entry_store esp_cleanup" echo "=== baseline ($OLEVEL, no disable) ===" NONE=$(run "") diff --git a/scripts/gen_unity.py b/scripts/gen_unity.py new file mode 100755 index 000000000..f699dccab --- /dev/null +++ b/scripts/gen_unity.py @@ -0,0 +1,504 @@ +#!/usr/bin/env python3 +"""Generate the unity-build groups in source/unity/. + +A unity group is one TU that #includes several library sources, so the headers +every source shares (tcc.h alone is ~560 KB of preprocessing) are read once per +group instead of once per file. See the "Unity build" block in the Makefile. + +Putting two .c files in one TU is only safe if neither can see the other's +file-local names. This script guarantees that by construction, from each +source's own preprocessed text (gcc -E -dD, which says which file every #define +came from): + + * Macros a source defines are #undef'd right after it. A source that defines + or #undefs a macro one of its own headers also defines is kept out of every + group: the #undef would take the header's definition away from the members + after it, and the header, include-guarded, would not bring it back. Any + other source only shares a group whose members' headers do not define its + macros. + * Sources only share a group when their file-scope statics, tags, typedefs + and enumerators are disjoint, and none of them is a name that the headers + of another member declare (a local `static T name[]` next to some header's + `extern U name[]`). Two same-named functions would only fail to compile, + but two `static int x;` are both tentative definitions of ONE object in a + single TU, which would silently share state. + * tcc.h picks TCC_STATE_VAR / TCC_SET_STATE / _tcc_error by whether the + includer defined USING_GLOBALS first; a later member cannot change that, so + sources only share a group when they agree on it. + * Groups stay within one library and under --max-lines source lines, and a + source bigger than that stands alone. + +Usage (in a configured tree): + scripts/gen_unity.py # rewrite source/unity/ + scripts/gen_unity.py --check # exit 1 if source/unity/ is stale +""" + +import argparse +import collections +import concurrent.futures +import difflib +import os +import re +import shlex +import tempfile +import subprocess +import sys + +TOP = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +UNITY_DIR = os.path.join('source', 'unity') + +# Macros whose effect the grouping key already captures (see mode_key). tcc.h +# consumes USING_GLOBALS and #undefs it, so it is local to the includer. +MODE_MACROS = {'USING_GLOBALS'} +MODE_KEYS = ('TCC_STATE_VAR', 'TCC_SET_STATE', '_tcc_error') + +ARCH_DIRS = ('source/backend/arch/arm', 'source/backend/arch/arm/thumb') + +# The same sources also build the on-device compiler (build_rootfs.sh), whose +# #ifs can declare other names; analyse both and keep the union. +NATIVE_DEFINES = ['-DTCC_IS_NATIVE', '-DTARGETOS_YasOS=1', '-DTCC_DEBUG=0', + '-DCONFIG_TCC_DEBUG_ENV=0'] + +LINEMARK = re.compile(r'^#\s+\d+\s+"([^"]*)"((?:\s+\d)*)') +DEFINE = re.compile(r'^#define\s+(\w+)(.*)$') +UNDEF = re.compile(r'^#undef\s+(\w+)') +SRC_DEFINE = re.compile(r'^[ \t]*#[ \t]*(?:define|undef)[ \t]+(\w+)', re.M) + + +def preprocessor_flags(args): + """Only what decides the preprocessed text. -O, -g, -f and -W flags differ + between configurations (CI builds with ASan) and must not change the + groups, or --check would disagree from one tree to the next.""" + return [a for a in args if a[:2] in ('-D', '-U', '-I') or a.startswith('-std=')] + + +def make_info(target): + out = subprocess.run(['make', '--no-print-directory', 'unity-info', + 'CROSS_TARGET=' + target, 'UNITY=no'], + cwd=TOP, capture_output=True, text=True, check=True).stdout + flags, libs = None, collections.OrderedDict() + for line in out.splitlines(): + if line.startswith('CC '): + flags = preprocessor_flags(shlex.split(line[3:])[1:]) + elif line.startswith('LIB '): + f = line.split() + libs[f[1]] = f[2:] + if flags is None or not libs: + sys.exit('gen_unity: `make unity-info` gave nothing -- is the tree configured?') + lib_flags = {lib: flags for lib in libs} + # The arch sub-libraries are separate makes (source/backend/arch/unity.mk); + # they compile with the top-level flags plus these two -I's. + for d in ARCH_DIRS: + out = subprocess.run(['make', '--no-print-directory', '-C', d, 'unity-info', + 'TOP=' + TOP, 'UNITY=no'], + cwd=TOP, capture_output=True, text=True, check=True).stdout + for line in out.splitlines(): + if line.startswith('LIB '): + f = line.split() + libs[f[1]] = f[2:] + lib_flags[f[1]] = flags + ['-I.', '-Isource/ir'] + return lib_flags, libs + + +def is_header(path): + # and are shared by every source, like a header + return path.endswith('.h') or path.startswith('<') + + +def strip_literals(text): + """Blank string and char literals so their braces and words do not count.""" + return re.sub(r'"(?:\\.|[^"\\\n])*"|\'(?:\\.|[^\'\\\n])*\'', '""', text) + + +def file_scope_names(text, every=False): + """File-scope names a stretch of preprocessed C declares with internal + linkage or as types: statics, struct/union/enum tags, typedefs, enumerators; + with every=True, all declared names, externs and prototypes included. + Coarse, but it only ever errs towards reporting too much, which just keeps + two sources apart.""" + names = set() + text = strip_literals(text) + depth = 0 + stmt = [] + i, n = 0, len(text) + while i < n: + c = text[i] + if c == '{': + if depth == 0: + head = ''.join(stmt) + tag = re.search(r'\b(struct|union|enum)\s+(\w+)\s*$', head) + if tag: + names.add(tag.group(1) + ' ' + tag.group(2)) + if re.search(r'\benum\b[^;]*$', head): + j = text.find('}', i) + for e in text[i + 1:j].split(','): + m = re.match(r'\s*([A-Za-z_]\w*)', e) + if m: + names.add(m.group(1)) + if '=' in head or re.search(r'\b(struct|union|enum)\b[^()]*$', head): + # an initializer, or an aggregate body inside a declaration: + # skip it, keep the declaration going so its declarators + # are seen at the ';' + d = 0 + while i < n: + if text[i] == '{': + d += 1 + elif text[i] == '}': + d -= 1 + if d == 0: + break + i += 1 + stmt.append(' {} ') + i += 1 + continue + # a function body: the declaration ends here + declare(head, names, True, every) + stmt = [] + depth += 1 + elif c == '}': + depth -= 1 + elif depth == 0: + if c == ';': + declare(''.join(stmt), names, False, every) + stmt = [] + else: + stmt.append(c) + i += 1 + return names + + +ATTRIBUTE = re.compile(r'\b(?:__attribute__|__attribute|__asm__|__asm|asm)\s*\(') + + +def strip_attributes(decl): + """Drop __attribute__((...)) and asm("...") labels: their parentheses would + otherwise pass for a declarator's.""" + while True: + m = ATTRIBUTE.search(decl) + if not m: + return decl.replace('__extension__', ' ') + depth, j = 0, m.end() - 1 + while j < len(decl): + depth += {'(': 1, ')': -1}.get(decl[j], 0) + j += 1 + if depth == 0: + break + decl = decl[:m.start()] + ' ' + decl[j:] + + +def declare(decl, names, is_function, every): + decl = ' '.join(strip_attributes(decl).split()) + if not decl: + return + words = re.findall(r'[A-Za-z_]\w*', decl) + if every or 'typedef' in words or 'static' in words: + if is_function: + m = re.match(r'([^(]*)\(', decl) + ids = re.findall(r'[A-Za-z_]\w*', m.group(1)) if m else [] + if ids: + names.add(ids[-1]) + return + # split the declarators at top-level commas + depth, cur, parts = 0, '', [] + for ch in decl: + if ch in '([': + depth += 1 + elif ch in ')]': + depth -= 1 + if ch == ',' and depth == 0: + parts.append(cur) + cur = '' + else: + cur += ch + parts.append(cur) + for k, p in enumerate(parts): + p = p.split('=')[0] + # `(*name)(...)` declares name; otherwise the last word before '[' or '(' + m = re.search(r'\(\s*\*+\s*([A-Za-z_]\w*)\s*\)', p) + if m: + names.add(m.group(1)) + continue + p = re.split(r'[\[(]', p)[0] + ids = re.findall(r'[A-Za-z_]\w*', p) + if ids: + names.add(ids[-1]) + + +def analyse(src, flags): + pp = subprocess.run(['gcc', '-E', '-dD'] + flags + [src], cwd=TOP, + capture_output=True, text=True) + if pp.returncode != 0: + sys.exit('gen_unity: gcc -E failed on %s:\n%s' % (src, pp.stderr)) + # A region is the source's own when the source itself, or a non-header it + # includes, produced it; stab.def included from stab.h is a header's. + stack = [(src, True)] + local = True + hdr_macros = set() + local_defs, local_undefs = set(), set() + mode = {} + local_text, hdr_text = [], [] + code = [] + for line in pp.stdout.splitlines(): + m = LINEMARK.match(line) + if m: + path, fl = m.group(1), m.group(2).split() + if '1' in fl: + stack.append((path, stack[-1][1] and not is_header(path))) + elif '2' in fl: + while len(stack) > 1 and stack[-1][0] != path: + stack.pop() + elif len(stack) == 1: + stack[0] = (path, not is_header(path)) + local = stack[-1][1] + continue + m = DEFINE.match(line) + if m: + name = m.group(1) + if name in MODE_KEYS: + mode[name] = m.group(2).strip() + (local_defs if local else hdr_macros).add(name) + continue + m = UNDEF.match(line) + if m: + name = m.group(1) + if name in MODE_KEYS: + mode.pop(name, None) + (local_undefs if local else hdr_macros).add(name) + continue + if line.startswith('#'): + continue + (local_text if local else hdr_text).append(line) + if local: + code.extend(line.split()) + # Also every #define in the file itself, including ones this configuration's + # #ifs skip: the #undef list has to hold for every configuration. + with open(os.path.join(TOP, src), errors='replace') as f: + raw = f.read() + local_defs |= set(SRC_DEFINE.findall(raw)) - local_undefs + return dict( + src=src, + lines=raw.count('\n'), + mode=tuple(mode.get(k) for k in MODE_KEYS), + hdr_macros=hdr_macros, + local_defs=local_defs, + local_undefs=local_undefs, + code=code, + names=file_scope_names('\n'.join(local_text)), + hdr_names=file_scope_names('\n'.join(hdr_text), every=True), + ) + + +def plan(libs, info, max_lines, ungroupable=()): + kept_apart = {} + groups = collections.OrderedDict() + for lib, srcs in libs.items(): + chunks = collections.defaultdict(list) # mode -> [chunk] + for src in srcs: + i = info[src] + # A file that redefines a macro one of its own headers defines stays + # alone; otherwise it only has to avoid the headers of its group. + local = (i['local_defs'] | i['local_undefs']) - MODE_MACROS + clash = local & i['hdr_macros'] + if clash: + kept_apart[src] = 'redefines header macros ' + ' '.join(sorted(clash)) + continue + if src in ungroupable: + kept_apart[src] = 'expands differently inside a group' + continue + if i['lines'] > max_lines: + kept_apart[src] = '%d lines' % i['lines'] + continue + # A file-local name clashes with another member's file-local names, + # and with anything the headers of another member declare -- unless + # this file's own headers declare it too, which it already agrees with. + own, hdr = i['names'], i['hdr_names'] + loose = own - hdr + for ch in chunks[i['mode']]: + if (ch['lines'] + i['lines'] <= max_lines and not (own & ch['names']) + and not (loose & ch['hdr']) and not (ch['loose'] & hdr) + and not (local & ch['hdr_macros']) and not (ch['local'] & i['hdr_macros'])): + break + else: + ch = dict(lines=0, names=set(), hdr=set(), loose=set(), members=[], + local=set(), hdr_macros=set()) + chunks[i['mode']].append(ch) + ch['local'] |= local + ch['hdr_macros'] |= i['hdr_macros'] + ch['lines'] += i['lines'] + ch['names'] |= own + ch['hdr'] |= hdr + ch['loose'] |= loose + ch['members'].append(src) + # number the groups in source order, so the output is stable + found = [ch for cs in chunks.values() for ch in cs if len(ch['members']) > 1] + found.sort(key=lambda ch: srcs.index(ch['members'][0])) + for k, ch in enumerate(found): + groups['%s_%02d' % (lib, k + 1)] = (lib, ch['members']) + return groups, kept_apart + + +HEADER = '/* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */\n' + + +def unity_text(members, info): + body = [HEADER] + for src in members: + body.append('#include "../%s"\n' % os.path.relpath(src, 'source')) + for name in sorted(info[src]['local_defs']): + body.append('#undef %s\n' % name) + return ''.join(body) + + +def member_code(text, flags, members): + """Preprocess a unity TU and return each member's own tokens, the way + analyse() collects them from the member alone.""" + fd, tmp = tempfile.mkstemp(prefix='.gen_unity_', suffix='.c', dir=os.path.join(TOP, UNITY_DIR)) + try: + with os.fdopen(fd, 'w') as f: + f.write(text) + rel = os.path.relpath(tmp, TOP) + pp = subprocess.run(['gcc', '-E', '-dD'] + flags + [rel], cwd=TOP, + capture_output=True, text=True) + finally: + os.remove(tmp) + if pp.returncode != 0: + return None + out = pp.stdout + # __FILE__ says source/unity/../x.c inside a group; compare as source/x.c + for src in members: + out = out.replace('"' + os.path.join(UNITY_DIR, '..', os.path.relpath(src, 'source')) + '"', + '"' + src + '"') + codes = {src: [] for src in members} + stack = [] + for line in out.splitlines(): + m = LINEMARK.match(line) + if m: + path, fl = os.path.normpath(m.group(1)), m.group(2).split() + if '1' in fl: + local = (len(stack) == 1 and not is_header(path)) or \ + (len(stack) > 1 and stack[-1][1] and not is_header(path)) + stack.append((path, local)) + elif '2' in fl: + while len(stack) > 1 and stack[-1][0] != path: + stack.pop() + elif not stack: + stack.append((path, False)) + continue + if line.startswith('#') or len(stack) < 2 or not stack[-1][1]: + continue + if stack[1][0] in codes: + codes[stack[1][0]].extend(line.split()) + return codes + + +def verify(groups, info, lib_flags): + """Members whose own code does not preprocess to the same tokens inside + their group as alone (in either configuration).""" + jobs = [(members, lib_flags[lib] + extra, cfg) + for lib, members in groups.values() + for cfg, extra in (('cross', []), ('native', NATIVE_DEFINES))] + def run(job): + members, flags, cfg = job + codes = member_code(unity_text(members, info), flags, members) + if codes is None: + return set(members[1:]) # does not even preprocess: split it up + return {src for src in members if codes[src] != info[src]['code_' + cfg]} + with concurrent.futures.ThreadPoolExecutor(os.cpu_count()) as ex: + return set().union(*ex.map(run, jobs)) if jobs else set() + + +def render(groups, kept_apart, info, max_lines): + files = {} + mk = [HEADER.replace('/*', '#').replace(' */', ''), + '# Unity-build groups: see the "Unity build" block in the Makefile.\n', + '# Budget %d source lines per group.\n' % max_lines] + for src in sorted(kept_apart): + mk.append('# own TU: %s (%s)\n' % (src, kept_apart[src])) + by_lib = collections.OrderedDict() + for g, (lib, members) in groups.items(): + by_lib.setdefault(lib, []).append(g) + for lib, gs in by_lib.items(): + mk.append('\nUNITY_GROUPS_%s = %s\n' % (lib, ' '.join(gs))) + for g in gs: + members = groups[g][1] + mk.append('UNITY_MEMBERS_%s = \\\n\t%s\n' % (g, ' \\\n\t'.join(members))) + files[os.path.join(UNITY_DIR, g + '.c')] = unity_text(members, info) + files[os.path.join(UNITY_DIR, 'unity.mk')] = ''.join(mk) + return files + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.split('\n\n')[0]) + ap.add_argument('--check', action='store_true', + help='report whether source/unity/ is up to date; write nothing') + ap.add_argument('--max-lines', type=int, default=6000, + help='source-line budget per group (default %(default)s)') + ap.add_argument('--target', default='armv8m', help='CROSS_TARGET to ask make about') + ap.add_argument('-v', '--verbose', action='store_true') + args = ap.parse_args() + + lib_flags, libs = make_info(args.target) + jobs = [(s, lib_flags[lib]) for lib, ss in libs.items() for s in ss] + srcs = [s for s, _ in jobs] + with concurrent.futures.ThreadPoolExecutor(os.cpu_count()) as ex: + cross = list(ex.map(lambda j: analyse(*j), jobs)) + native = list(ex.map(lambda j: analyse(j[0], j[1] + NATIVE_DEFINES), jobs)) + info = {} + for a, b in zip(cross, native): + if a['mode'] != b['mode']: + sys.exit('gen_unity: %s picks USING_GLOBALS differently for the native build' + % a['src']) + info[a['src']] = dict(a, code_cross=a['code'], code_native=b['code'], + **{k: a[k] | b[k] for k in + ('hdr_macros', 'local_defs', 'local_undefs', + 'names', 'hdr_names')}) + # The rules above are the plan; this is the proof. Preprocess every group + # and keep out any member whose own code expands differently than alone. + ungroupable = set() + while True: + groups, kept_apart = plan(libs, info, args.max_lines, ungroupable) + bad = verify(groups, info, lib_flags) + if not bad: + break + ungroupable |= bad + files = render(groups, kept_apart, info, args.max_lines) + + old = {} + udir = os.path.join(TOP, UNITY_DIR) + if os.path.isdir(udir): + for f in os.listdir(udir): + p = os.path.join(UNITY_DIR, f) + with open(os.path.join(TOP, p)) as fh: + old[p] = fh.read() + stale = sorted(p for p in set(old) | set(files) if old.get(p) != files.get(p)) + + grouped = sum(len(m) for _, m in groups.values()) + summary = '%d sources -> %d groups holding %d, %d on their own' % ( + len(srcs), len(groups), grouped, len(srcs) - grouped) + if args.check: + if stale: + for p in stale: + sys.stdout.writelines(difflib.unified_diff( + old.get(p, '').splitlines(True), files.get(p, '').splitlines(True), + 'a/' + p, 'b/' + p)) + print('gen_unity: source/unity/ is stale (%d files); run scripts/gen_unity.py' % len(stale)) + return 1 + print('gen_unity: up to date: ' + summary) + return 0 + os.makedirs(udir, exist_ok=True) + for p in stale: + full = os.path.join(TOP, p) + if p in files: + with open(full, 'w') as fh: + fh.write(files[p]) + else: + os.remove(full) + print('gen_unity: ' + summary + ('; %d files changed' % len(stale))) + if args.verbose: + for src, why in sorted(kept_apart.items()): + print(' own TU: %s (%s)' % (src, why)) + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/source/backend/arch/arm/Makefile b/source/backend/arch/arm/Makefile index f41d9daf2..066ef5524 100644 --- a/source/backend/arch/arm/Makefile +++ b/source/backend/arch/arm/Makefile @@ -44,6 +44,10 @@ FPU_SRCS = fpv5-sp-d16.c fpv5-d16.c rp2350-dcp.c FPU_OBJS = $(addprefix $(BUILD_DIR)/fpu-, $(FPU_SRCS:.c=.o)) OBJS += $(FPU_OBJS) +UNITY_LIB = arm +UNITY_SRC_DIR = source/backend/arch/arm +include $(TOP)/source/backend/arch/unity.mk + $(BUILD_DIR)/fpu-%.o: $(TOP)/source/backend/arch/fpu/arm/%.c $(CORE_HDRS) @mkdir -p $(dir $@) $(CC) -o $@ -c $< $(CFLAGS) $(DEFINES) -I$(TOP) -I$(TOP)/source/ir diff --git a/source/backend/arch/arm/arm-link.c b/source/backend/arch/arm/arm-link.c index 73dcf3756..4ea346006 100644 --- a/source/backend/arch/arm/arm-link.c +++ b/source/backend/arch/arm/arm-link.c @@ -46,6 +46,7 @@ ST_FUNC int code_reloc(int reloc_type) case R_ARM_THM_JUMP6: case R_ARM_THM_PC12: case R_ARM_THM_PC8: + case R_ARM_YASOS_LOCAL_CALL: return 1; } return -1; @@ -62,6 +63,7 @@ ST_FUNC int gotplt_entry_type(int reloc_type) case R_ARM_COPY: case R_ARM_GLOB_DAT: case R_ARM_JUMP_SLOT: + case R_ARM_YASOS_LOCAL_CALL: /* a check on the BL beside it, which owns the PLT decision */ return NO_GOTPLT_ENTRY; case R_ARM_PC24: @@ -489,6 +491,23 @@ ST_FUNC void relocate(TCCState *s1, ElfW_Rel *rel, int type, unsigned char *ptr, (*(uint16_t *)(ptr + 2)) = (uint16_t)((lo & 0xd000) | (j1 << 13) | blx_bit | (j2 << 11) | imm11); } return; + case R_ARM_YASOS_LOCAL_CALL: + { + /* The compiler dropped the R9 reload after the BL at this offset because + -fmodule-local-calls classified the callee as module-local (no shared + library in the library paths exported it). If the call binds to the + PLT after all, the callee returns with its own R9 and this function + carries on addressing its data through the wrong GOT: refuse the link + rather than produce that. */ + int to_plt = s1->plt && val >= s1->plt->sh_addr && val < s1->plt->sh_addr + s1->plt->data_offset; + if (sym->st_shndx == SHN_UNDEF && ELFW(ST_BIND)(sym->st_info) == STB_WEAK) + return; /* an unresolved weak callee is the caller's problem, as for the BL */ + if (to_plt || sym->st_shndx == SHN_UNDEF) + tcc_error_noabort("call to '%s' at %x was compiled as module-local (its R9 reload dropped) but binds " + "to an import; compile with the library's -L path or with -fno-module-local-calls", + (char *)symtab_section->link->data + sym->st_name, (unsigned)addr); + } + return; case R_ARM_MOVT_ABS: case R_ARM_MOVW_ABS_NC: { diff --git a/source/backend/arch/arm/arm_aapcs.c b/source/backend/arch/arm/arm_aapcs.c index 41b55b03e..a7eedf8fc 100644 --- a/source/backend/arch/arm/arm_aapcs.c +++ b/source/backend/arch/arm/arm_aapcs.c @@ -138,46 +138,17 @@ TCCAbiArgLoc tcc_abi_classify_argument(TCCAbiCallLayout *layout, int arg_index, const int slot_sz = tcc_abi_align_up_int(size, 4); const int regs_needed = (slot_sz + 3) / 4; - /* Invisible reference for large composites (> 16 bytes). - * - * This is used only on the callee side (where arg_flags is allocated - * by tcc_abi_call_layout_ensure_capacity). On the caller/call-site - * side, arg_flags is NULL and large structs are classified as normal - * by-value composites — the frontend (gfunc_param_typed) handles the - * invisible-reference conversion for prototyped calls, while variadic - * anonymous arguments must be passed by value for va_arg to work. - * - * NOTE: The invisible-reference check must come BEFORE the 8-byte - * alignment padding below. When passed by invisible reference the - * argument is a 4-byte pointer, so the struct's natural alignment - * is irrelevant for register assignment and must not cause the NCRN - * to skip a register. */ - if (size > 16 && layout->arg_flags) - { - layout->arg_flags[arg_index] |= TCC_ABI_ARG_FLAG_INVISIBLE_REF; - /* Pass the pointer in a register (like a scalar) */ - if (layout->next_reg <= 3) - { - loc.kind = TCC_ABI_LOC_REG; - loc.reg_base = layout->next_reg; - loc.reg_count = 1; - loc.size = 4; /* pointer size */ - layout->next_reg++; - } - else - { - loc.kind = TCC_ABI_LOC_STACK; - loc.stack_off = layout->next_stack_off; - loc.size = 4; /* pointer size */ - layout->next_stack_off += 4; - } - } - else + /* AAPCS32 passes a composite of any size by value: in the core registers + * from the NCRN, then on the stack, split between the two when it + * straddles r3 (rule C.5). (Passing one larger than 16 bytes by a pointer + * to a copy is the AArch64 rule, not this one.) */ { /* AAPCS: Composite types with 8-byte natural alignment require * double-word alignment — the NCRN must be rounded up to the - * next even register number before allocation. This only applies - * to by-value composites, not invisible references (handled above). */ + * next even register number before allocation. A stack slot is + * aligned to at most 8 (C.4 caps the NSAA rounding at 8), whatever + * the composite's own alignment. */ + const int stack_align = align > 8 ? 8 : align; if (align >= 8 && (layout->next_reg & 1)) layout->next_reg++; @@ -197,7 +168,7 @@ TCCAbiArgLoc tcc_abi_classify_argument(TCCAbiCallLayout *layout, int arg_index, loc.kind = TCC_ABI_LOC_REG_STACK; loc.reg_base = layout->next_reg; loc.reg_count = (uint8_t)regs_avail; - layout->next_stack_off = tcc_abi_align_up_int(layout->next_stack_off, align); + layout->next_stack_off = tcc_abi_align_up_int(layout->next_stack_off, stack_align); loc.stack_off = layout->next_stack_off; loc.stack_size = (uint32_t)(words_on_stack * 4); layout->next_stack_off += words_on_stack * 4; @@ -205,7 +176,7 @@ TCCAbiArgLoc tcc_abi_classify_argument(TCCAbiCallLayout *layout, int arg_index, } else { - layout->next_stack_off = tcc_abi_align_up_int(layout->next_stack_off, align); + layout->next_stack_off = tcc_abi_align_up_int(layout->next_stack_off, stack_align); loc.kind = TCC_ABI_LOC_STACK; loc.stack_off = layout->next_stack_off; layout->next_stack_off += slot_sz; diff --git a/source/backend/arch/arm/ssa_opt_arm.c b/source/backend/arch/arm/ssa_opt_arm.c index ddb8787c9..7e551212d 100644 --- a/source/backend/arch/arm/ssa_opt_arm.c +++ b/source/backend/arch/arm/ssa_opt_arm.c @@ -85,7 +85,7 @@ int ssa_gen_arm_fuse_mul_add_to_mla(IRSSAOptCtx *ctx, int instr_idx) * original SHR def is now a NOP, so the def_op check above doesn't * fire — without this guard the MLA fusion would drop the shift. */ if (ir->barrel_shifts && add_q->orig_index >= 0 && - add_q->orig_index <= ir->max_orig_index && + add_q->orig_index < ir->barrel_shifts_len && ir->barrel_shifts[add_q->orig_index] != 0) return 0; diff --git a/source/backend/arch/arm/thumb/Makefile b/source/backend/arch/arm/thumb/Makefile index 15bf34eaf..46083fbca 100644 --- a/source/backend/arch/arm/thumb/Makefile +++ b/source/backend/arch/arm/thumb/Makefile @@ -12,7 +12,7 @@ AR ?= ar CFLAGS ?= DEFINES ?= -SRCS = thumb.c thop_alu_imm.c thop_alu_reg.c thop_cmp.c thop_shift_imm.c thop_shift_reg.c thop_mem_reg.c thop_mem_unpriv.c thop_mem_exclusive.c thop_mem_imm.c thop_extend.c thop_rev.c thop_bitfield.c thop_vfp.c thop_coproc.c thop_system.c thop_branch.c thop_block.c thop_mul.c thop_dsp.c thop_tbb.c thop_ldrd.c thop_ldrex.c thop_ldaex.c thop_mrs.c thop_pld.c thop_mov.c thop_mvn.c thop_adr.c thop_ldr_literal.c arm-thumb-gen.c arm-thumb-callsite.c arm-thumb-asm.c +SRCS = thumb.c thop_alu_imm.c thop_alu_reg.c thop_cmp.c thop_shift_imm.c thop_shift_reg.c thop_mem_reg.c thop_mem_unpriv.c thop_mem_exclusive.c thop_mem_imm.c thop_extend.c thop_rev.c thop_bitfield.c thop_vfp.c thop_coproc.c thop_system.c thop_branch.c thop_block.c thop_mul.c thop_dsp.c thop_tbb.c thop_ldrd.c thop_ldrex.c thop_ldaex.c thop_mrs.c thop_pld.c thop_mov.c thop_mvn.c thop_adr.c thop_ldr_literal.c arm-thumb-gen.c arm-thumb-emit.c arm-thumb-alu.c arm-thumb-mem.c arm-thumb-fp.c arm-thumb-frame.c arm-thumb-call.c arm-thumb-callsite.c arm-thumb-asm.c thumb-outline.c OBJS = $(addprefix $(BUILD_DIR)/, $(SRCS:.c=.o)) LIB = $(BUILD_DIR)/libthumb.a @@ -31,6 +31,10 @@ CORE_HDRS = $(wildcard $(TOP)/*.h) \ $(wildcard $(TOP)/source/backend/arch/arm/*.h) \ $(wildcard $(TOP)/source/backend/arch/arm/thumb/*.h) +UNITY_LIB = thumb +UNITY_SRC_DIR = source/backend/arch/arm/thumb +include $(TOP)/source/backend/arch/unity.mk + all: $(LIB) $(LIB): $(OBJS) diff --git a/source/backend/arch/arm/thumb/arm-thumb-alu.c b/source/backend/arch/arm/thumb/arm-thumb-alu.c new file mode 100644 index 000000000..78104bc0b --- /dev/null +++ b/source/backend/arch/arm/thumb/arm-thumb-alu.c @@ -0,0 +1,3484 @@ +/* + * ARMvX-m code generator for TCC + * Uses thumb instruction set + * + * Based on: + * ARM Thumb 2 instruction functions for TCC + * Copyright (c) 2020 Erlend J. Sveen + * from: + * https://git.erlendjs.no/erlendjs/tinycc/-/blob/arm-thumb/arm-thumb-gen.c + * https://git.erlendjs.no/erlendjs/tinycc/-/blob/arm-thumb/arm-thumb-instructions.c + * + * And + * + * ARMv4 code generator for TCC + * + * Copyright (c) 2003 Daniel Glöckner + * Copyright (c) 2012 Thomas Preud'homme + * + * Based on i386-gen.c by Fabrice Bellard + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +/* Thumb-2 code generator: 32- and 64-bit data processing, shifts, + * multiply/divide/modulo and compare lowering of MachineOperands. */ + +#include "arm-thumb-gen.h" + +/* tcc_gen_machine_data_processing_mop: MachineOperand-based entry point for + * arithmetic/logic operations. Called from ir/codegen.c when dest does not + * use a static chain register. + * Dispatches to thumb_emit_data_processing_mop64 / thumb_emit_shift64_mop for + * 64-bit pair destinations, or thumb_emit_data_processing_mop32 for 32-bit. + */ +static void data_processing_mop_impl(MachineOperand src1, MachineOperand src2, MachineOperand dest, TccIrOp op, + thumb_flags_behaviour flags_override, uint32_t barrel_shift); + +static ThumbAndImmForm thumb_and_imm_form(uint32_t mask) +{ + /* Zero-extends: a 2-byte encoding when both registers are low, against 4 + * bytes for the AND-immediate, which has no narrow form at all. */ + if (mask == 0xFFu) + return AND_IMM_UXTB; + if (mask == 0xFFFFu) + return AND_IMM_UXTH; + + /* A mask the AND-immediate already encodes is one instruction as it stands; + * these forms only pay off when they remove a constant materialization. */ + if (th_pack_const(mask) != 0) + return AND_IMM_NONE; + + /* A low-contiguous run of ones is exactly UBFX Rd, Rn, #0, #W. 0 and + * 0xFFFFFFFF are degenerate (the callers fold them to a load-zero and a + * copy) and UBFX cannot encode width 32 anyway. */ + if (mask != 0 && mask != 0xFFFFFFFFu && (mask & (mask + 1)) == 0) + return AND_IMM_UBFX; + + /* `x & ~m` is `BIC x, m`, and a clear-mask whose complement encodes is the + * usual shape of a bitfield read-modify-write. */ + if (~mask != 0 && th_pack_const(~mask) != 0) + return AND_IMM_BIC; + + return AND_IMM_NONE; +} + +static void thumb_emit_and_imm_special(ThumbAndImmForm form, int rd, int rn, uint32_t mask, + thumb_flags_behaviour flags) +{ + switch (form) + { + case AND_IMM_UXTB: + ot_check(th_uxtb((uint32_t)rd, (uint32_t)rn, THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); + return; + case AND_IMM_UXTH: + ot_check(th_uxth((uint32_t)rd, (uint32_t)rn, THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); + return; + case AND_IMM_UBFX: + { + int width = 0; + while ((mask >> width) & 1u) + width++; + thumb_opcode ubfx_op; + ubfx_op.size = 4; + /* 11110 0 11 1100 Rn | 0 imm3 Rd imm2 0 widthm1, with lsb 0 (imm3=imm2=0). */ + ubfx_op.opcode = 0xF3C00000u | ((uint32_t)rn << 16) | ((uint32_t)rd << 8) | (uint32_t)(width - 1); + ot(ubfx_op); + return; + } + case AND_IMM_BIC: + ot_check(th_bic_imm((uint32_t)rd, (uint32_t)rn, ~mask, flags, ENFORCE_ENCODING_NONE)); + return; + case AND_IMM_NONE: + default: + tcc_error("thumb_emit_and_imm_special: no form for mask 0x%x", (unsigned)mask); + return; + } +} + +/* ============================================================ + * thumb_try_orrs_zero64 + * ============================================================ + * A 64-bit value is zero exactly when the bitwise OR of its halves is, so + * `ORRS Rt, Rlo, Rhi` sets Z for the whole comparison in one instruction where + * the general form needs three (`CMP hi,#0` / `IT EQ` / `CMPEQ lo,#0`). Only + * Z is meaningful afterwards, which is all an equality comparison consumes -- + * the three-instruction form is no better in that respect, since which of its + * two CMPs ran last decides N/C/V. + * + * ORRS needs a destination the CMP form does not, so this fires only when the + * allocator has a register genuinely free at this point: buying one with a + * push/pop would cost more than the two instructions it saves. Rt may alias + * either source -- ORRS reads both operands before writing. + * + * `soft_common.h`'s classifiers are built out of this test (`double_mant(bits) + * != 0` and friends), 60 sites across the soft-float library. + * + * Returns 1 if it emitted the comparison, 0 if the caller must fall back. + */ +static int thumb_try_orrs_zero64(int rn_lo, int rn_hi) +{ + TCCIRState *ir = tcc_state->ir; + if (!ir) + return 0; + if (!thumb_is_hw_reg(rn_lo) || !thumb_is_hw_reg(rn_hi)) + return 0; + + const uint32_t excl = scratch_global_exclude; + int rt = tcc_ls_find_free_scratch_reg(&ir->ls, ir->codegen_instruction_idx, excl, ir->leaffunc); + if (rt == (int)PREG_NONE || !thumb_is_hw_reg(rt)) + return 0; + + thumb_opcode orrs = th_orr_reg((uint32_t)rt, (uint32_t)rn_lo, (uint32_t)rn_hi, FLAGS_BEHAVIOUR_SET, + THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE); + if (orrs.size == 0) + return 0; + ot_check(orrs); + return 1; +} + +static void thumb_emit_op_imm_fallback(int rd, int rn, uint32_t imm, thumb_flags_behaviour flags, + ThumbDataProcessingHandler handler) +{ + thumb_opcode sub_low = thumb_call_imm_handler(handler.imm_handler, rd, rn, imm, flags, ENFORCE_ENCODING_NONE); + if (sub_low.size == 0) + { + uint32_t exclude = 0; + if (rd >= 0 && rd <= 15) + exclude |= (1u << rd); + if (rn >= 0 && rn <= 15) + exclude |= (1u << rn); + ScratchRegAlloc scratch = get_scratch_reg_with_save(exclude); + tcc_machine_load_constant(scratch.reg, PREG_NONE, (int32_t)imm, 0, NULL); + ot_check(thumb_call_reg_handler(handler.reg_handler, rd, rn, scratch.reg, flags, THUMB_SHIFT_DEFAULT, + ENFORCE_ENCODING_NONE)); + restore_scratch_reg(&scratch); + } + else + { + ot_check(sub_low); + } +} + +static thumb_opcode thumb_mul_regonly(uint32_t rd, uint32_t rn, uint32_t rm) +{ + return th_mul(rd, rn, rm, flags_safe(), ENFORCE_ENCODING_NONE); +} + +static thumb_opcode thumb_sdiv_regonly(uint32_t rd, uint32_t rn, uint32_t rm) +{ + return th_sdiv((uint16_t)rd, (uint16_t)rn, (uint16_t)rm); +} + +static thumb_opcode thumb_udiv_regonly(uint32_t rd, uint32_t rn, uint32_t rm) +{ + return th_udiv((uint16_t)rd, (uint16_t)rn, (uint16_t)rm); +} + +/* A 64-bit operand whose needs_deref came from is_lval NAMES memory: the value + * sits at that address, it is not a pointer to follow. Two kinds are of that + * sort — PARAM_STACK (the caller's argument area) and CHAIN_REL (a variable + * captured from a parent frame, reached through the static chain). Splitting + * either into halves steps the OFFSET by 4, which is right; following it as a + * pointer reads the low word and dereferences that. MACH_OP_SPILL is not in + * the set: its needs_deref comes from is_llocal, which really is a pointer. */ +bool mach_op_64_names_memory(const MachineOperand *op) +{ + return op->kind == MACH_OP_PARAM_STACK || op->kind == MACH_OP_CHAIN_REL; +} + +/* ============================================================ + * mach_resolve_deref_64 + * ============================================================ + * When a 64-bit source has needs_deref=true, the operand holds a POINTER + * to a 64-bit value — not the value itself. Splitting such an operand + * via mach_make_lo_half / mach_make_hi_half is WRONG because + * mach_make_hi_half would increment the register number (e.g. R0 → R1) + * instead of the memory offset. + * + * This helper resolves the deref by loading both 32-bit halves from + * [base+0] and [base+4] into scratch registers, returning a clean + * MACH_OP_REG pair operand with needs_deref=false. The caller can + * then safely call mach_make_lo_half / mach_make_hi_half on the result. + * + * Returns *op unchanged if needs_deref is false. + */ +MachineOperand mach_resolve_deref_64(MachineCodegenContext *mctx, const MachineOperand *op, uint32_t *excl) +{ + if (!op->needs_deref) + return *op; + + /* The value is directly at [fp+offset] / [chain+offset]: clear needs_deref + * and let the normal mach_make_lo_half / mach_make_hi_half path handle it. */ + if (mach_op_64_names_memory(op)) + { + MachineOperand result = *op; + result.needs_deref = false; + return result; + } + + /* Strip deref to get the raw address into a register. */ + MachineOperand addr = *op; + addr.needs_deref = false; + addr.is_64bit = false; + addr.btype = IROP_BTYPE_INT32; + int base_reg = mach_ensure_in_reg(mctx, &addr, *excl); + if (thumb_is_hw_reg(base_reg)) + *excl |= (1u << (uint32_t)base_reg); + + /* Allocate two scratch registers for the loaded halves. */ + int lo_reg = mach_alloc_scratch(mctx, *excl); + *excl |= (1u << (uint32_t)lo_reg); + int hi_reg = mach_alloc_scratch(mctx, *excl); + *excl |= (1u << (uint32_t)hi_reg); + + /* Load [base+0] → lo, [base+4] → hi. Proven-aligned access (op->align4, + * from the frontend's packed-access tracking): one LDRD; otherwise the + * unaligned-safe pair of 32-bit loads. */ + if (!(op->align4 && try_ldrd_pair(lo_reg, hi_reg, base_reg, 0, 0))) + { + load_from_base(lo_reg, PREG_REG_NONE, IROP_BTYPE_INT32, 0, 0, 0, (uint32_t)base_reg); + load_from_base(hi_reg, PREG_REG_NONE, IROP_BTYPE_INT32, 0, 4, 0, (uint32_t)base_reg); + } + + /* Build a clean register-pair operand. */ + MachineOperand result = {0}; + result.kind = MACH_OP_REG; + result.is_64bit = true; + result.needs_deref = false; + result.btype = op->btype; + result.u.reg.r0 = lo_reg; + result.u.reg.r1 = hi_reg; + return result; +} + +/* ============================================================ + * mach_make_lo_half / mach_make_hi_half + * ============================================================ + * Split a 64-bit MachineOperand into its 32-bit low and high halves. + * The resulting operands have is_64bit=false and represent the individual + * 32-bit words, suitable for mach_ensure_in_reg / mach_writeback_dest. + * + * Only call mach_make_hi_half on a 64-bit operand (is_64bit=true or + * MACH_OP_SPILL); the result for 32-bit REG is the next register (r0+1). + */ +MachineOperand mach_make_lo_half(const MachineOperand *op) +{ + MachineOperand lo = *op; + lo.is_64bit = false; + if (lo.kind == MACH_OP_REG) + lo.u.reg.r1 = -1; + /* SPILL: keep the same offset — low word is at the base offset. */ + /* IMM: u.imm.val bits [31:0] are the low word (callers truncate). */ + /* CHAIN_REL: keep offset/chain_index — low word is at base offset. */ + return lo; +} +MachineOperand mach_make_hi_half(const MachineOperand *op) +{ + MachineOperand hi = *op; + hi.is_64bit = false; + switch (hi.kind) + { + case MACH_OP_REG: + /* r1 holds the high register for 64-bit pairs. If r1 is not a valid + * hardware register the allocator failed to produce a proper pair — + * error out instead of silently using r0+1 which can clobber reserved + * registers (e.g. R9 = GOT base). */ + if (!thumb_is_hw_reg(op->u.reg.r1)) + tcc_error("mach_make_hi_half: 64-bit REG operand has invalid r1=%d (r0=%d) — " + "register allocator must produce a valid pair", + op->u.reg.r1, op->u.reg.r0); + hi.u.reg.r0 = op->u.reg.r1; + hi.u.reg.r1 = -1; + break; + case MACH_OP_SPILL: + hi.u.spill.offset += 4; + break; + case MACH_OP_IMM: + hi.u.imm.val = (int64_t)(int32_t)(uint32_t)((uint64_t)op->u.imm.val >> 32); + break; + case MACH_OP_PARAM_STACK: + hi.u.param.offset += 4; + break; + case MACH_OP_CHAIN_REL: + hi.u.chain.offset += 4; /* high word is 4 bytes above low word */ + break; + case MACH_OP_SYMBOL: + hi.u.sym.addend += 4; /* high word at symbol + addend + 4 */ + break; + case MACH_OP_FRAME_ADDR: + hi.u.frame.offset += 4; /* high word at FP + offset + 4 */ + break; + default: + break; + } + return hi; +} + +/* Materialize both 32-bit halves of a 64-bit operand into registers. + * + * When the source is a plain, word-aligned stack slot the two halves sit at + * adjacent addresses — exactly LDRD's shape, so one instruction replaces the + * two LDRs the per-half `mach_ensure_in_reg` calls would emit. A `long long` + * comparison or arithmetic op reading a local is otherwise the one 64-bit + * memory shape that never pairs (STOREs already fuse to STRD, and derefs pair + * via mach_resolve_deref_64), which is why `if (x.a != y.a)` on 64-bit fields + * costs four loads instead of two. + * + * Every other operand kind, and any slot LDRD cannot reach, falls back to the + * original per-half path, so behaviour there is unchanged. `*excl` picks up + * both chosen registers, mirroring what the callers did by hand. */ +void mach_ensure_pair_in_regs(MachineCodegenContext *ctx, const MachineOperand *op64, + uint32_t *excl, int *out_lo, int *out_hi) +{ + MachineOperand lo = mach_make_lo_half(op64); + MachineOperand hi = mach_make_hi_half(op64); + lo.btype = IROP_BTYPE_INT32; + hi.btype = IROP_BTYPE_INT32; + + if (op64->is_64bit && lo.kind == MACH_OP_SPILL && !lo.needs_deref && (lo.u.spill.offset & 3) == 0) + { + int rlo = mach_alloc_scratch(ctx, *excl); + int rhi = mach_alloc_scratch(ctx, *excl | (1u << (uint32_t)rlo)); + if (rlo != rhi) + { + /* LDRD, or — when its offset/register constraints do not hold — the two + * single-word loads into the scratches we already own. */ + if (!tcc_gen_machine_try_ldrd_spill(rlo, lo.u.spill.offset, rhi, hi.u.spill.offset)) + { + tcc_machine_load_spill_slot(rlo, lo.u.spill.offset); + tcc_machine_load_spill_slot(rhi, hi.u.spill.offset); + } + *excl |= (1u << (uint32_t)rlo) | (1u << (uint32_t)rhi); + *out_lo = rlo; + *out_hi = rhi; + return; + } + tcc_machine_load_spill_slot(rlo, lo.u.spill.offset); + *excl |= (1u << (uint32_t)rlo); + *out_lo = rlo; + *out_hi = mach_ensure_in_reg(ctx, &hi, *excl); + if (thumb_is_hw_reg(*out_hi)) + *excl |= (1u << (uint32_t)*out_hi); + return; + } + + int rlo = mach_ensure_in_reg(ctx, &lo, *excl); + if (thumb_is_hw_reg(rlo)) + *excl |= (1u << (uint32_t)rlo); + int rhi = mach_ensure_in_reg(ctx, &hi, *excl); + if (thumb_is_hw_reg(rhi)) + *excl |= (1u << (uint32_t)rhi); + *out_lo = rlo; + *out_hi = rhi; +} + +/* ============================================================ + * thumb_emit_data_processing_mop64 + * ============================================================ + * 64-bit ADD / SUB / AND / OR / XOR via MachineOperand register pairs. + * Handles REG (r0:r1), SPILL (offset, offset+4) and IMM operands. + * + * uses_carry=true → low word uses FLAGS_BEHAVIOUR_SET, high word uses the + * carry handler (ADDS + ADC for ADD, SUBS + SBC for SUB). + * uses_carry=false → both halves use the same handler independently (AND/OR/XOR). + * + * If src1 is not 64-bit (e.g. int promoted to long long), its high half is + * zero-extended. Similarly for src2. + * + * `zh` carries source/opt/flat/fusion/zero_half64.c's verdict in its low six + * bits: which operand halves are provably the constant zero, and which halves + * of the result no consumer reads. + */ +static void thumb_emit_data_processing_mop64(const MachineOperand *src1, const MachineOperand *src2, + const MachineOperand *dest, TccIrOp op, ThumbDataProcessingHandler regular, + ThumbDataProcessingHandler carry_h, bool uses_carry, uint32_t zh) +{ + (void)op; + MachineCodegenContext mctx = {0}; + uint32_t excl = 0; + + /* 0. Determine destination register pair FIRST so that deref resolution + * never allocates scratch registers that overlap with the dest pair. + * Without this, mach_release_all would restore saved scratch regs + * and clobber the result sitting in rd_lo / rd_hi. */ + int rd_lo, rd_hi; + bool store_lo = false, store_hi = false; + if (dest->kind == MACH_OP_REG && !dest->needs_deref && dest->u.reg.r0 != (int)PREG_REG_NONE && dest->u.reg.r1 >= 0) + { + rd_lo = dest->u.reg.r0; + rd_hi = dest->u.reg.r1; + excl |= (1u << (uint32_t)rd_lo) | (1u << (uint32_t)rd_hi); + } + else + { + rd_lo = mach_alloc_scratch(&mctx, excl); + excl |= (1u << (uint32_t)rd_lo); + rd_hi = mach_alloc_scratch(&mctx, excl); + excl |= (1u << (uint32_t)rd_hi); + store_lo = store_hi = (dest->kind != MACH_OP_NONE); + } + + /* 0b. Pre-exclude register operands so that deref resolution of one + * source never steals the physical registers of another source. + * This must include needs_deref registers: they hold live pointers + * that will be consumed during their own deref resolution and must + * not be repurposed as scratch during the other source's deref. */ + if (src1->kind == MACH_OP_REG) + { + if (src1->u.reg.r0 != (int)PREG_REG_NONE) + excl |= (1u << (uint32_t)src1->u.reg.r0); + if (!src1->needs_deref && src1->is_64bit && src1->u.reg.r1 >= 0) + excl |= (1u << (uint32_t)src1->u.reg.r1); + } + if (src2->kind == MACH_OP_REG) + { + if (src2->u.reg.r0 != (int)PREG_REG_NONE) + excl |= (1u << (uint32_t)src2->u.reg.r0); + if (!src2->needs_deref && src2->is_64bit && src2->u.reg.r1 >= 0) + excl |= (1u << (uint32_t)src2->u.reg.r1); + } + + /* 1. Resolve deref'd source pointers before splitting into halves. */ + MachineOperand r_src1 = mach_resolve_deref_64(&mctx, src1, &excl); + src1 = &r_src1; + MachineOperand r_src2 = mach_resolve_deref_64(&mctx, src2, &excl); + src2 = &r_src2; + + /* 1b. Plan the HIGH half before loading anything. + * + * A source that is not 64-bit is zero-extended, and against a zero half a + * bitwise op has a constant answer: OR/XOR copy the other operand, AND give + * zero. Deciding that here rather than after the loads means the zero is + * never materialised AND the scratch register that would have held it is + * never allocated -- on a saturated function that scratch costs a push/pop + * of a callee-saved register too. + * + * `(uint64_t)x | ((uint64_t)y << 32)` and the make_double/make_float packing + * idiom soft float is built from are exactly this shape: before this, each + * such OR paid `movs rt,#0` plus an `orr rd,rt,rm` that could only ever + * return rm. */ + enum + { + HALF_NORMAL = 0, /* emit the handler over both registers of this half */ + HALF_ZERO, /* result half is the constant 0 */ + HALF_COPY_S1, /* result half is src1's */ + HALF_COPY_S2, /* result half is src2's */ + HALF_CARRY_IMM0, /* carry op, src2 half is zero: `adc/sbc rd, rn, #0` */ + HALF_DEAD /* nothing reads the result half; emit nothing at all */ + }; + /* A non-64-bit source is zero-extended by construction; the annotation adds + * the halves a 64-bit value is only *known* to hold zero in. */ + const bool s1_lo_zero = (zh & ZH64_S1_LO) != 0; + const bool s1_hi_zero = !src1->is_64bit || (zh & ZH64_S1_HI) != 0; + const bool s2_lo_zero = (zh & ZH64_S2_LO) != 0; + const bool s2_hi_zero = (src2->kind != MACH_OP_IMM && !src2->is_64bit) || (zh & ZH64_S2_HI) != 0; + int lo_plan = HALF_NORMAL, hi_plan = HALF_NORMAL; + /* Carry ops need the real words (the carry still has to propagate), and + * CMP's halves must set flags, so neither can take a shortcut. */ + if (!uses_carry && op != TCCIR_OP_CMP && src2->kind != MACH_OP_IMM) + { + for (int half = 0; half < 2; half++) + { + const bool z1 = half ? s1_hi_zero : s1_lo_zero; + const bool z2 = half ? s2_hi_zero : s2_lo_zero; + int plan = HALF_NORMAL; + if (!z1 && !z2) + plan = HALF_NORMAL; + else if (op == TCCIR_OP_AND) + plan = HALF_ZERO; + else if (op == TCCIR_OP_OR || op == TCCIR_OP_XOR) + plan = (z1 && z2) ? HALF_ZERO : (z1 ? HALF_COPY_S2 : HALF_COPY_S1); + if (half) + hi_plan = plan; + else + lo_plan = plan; + } + } + /* A carry op cannot skip a half -- the carry still has to propagate -- but a + * zero high word on src2 turns the high half into `adc rd, rn, #0` / + * `sbc rd, rn, #0`, which reads no second register and needs no `mov rt, #0` + * to feed it. This is most of what sfp_div_sig64 spends its zeros on: + * Knuth-D adds and subtracts 32-bit quantities from 64-bit ones throughout, + * and each one was materialising a zero purely to be added to a high word. + * src1 is deliberately not covered: SUB is not commutative, and mirroring + * this rule in zero_half64's producer analysis is only sound while the + * emitter's choice is unambiguous. */ + if (uses_carry && op != TCCIR_OP_CMP && src2->kind != MACH_OP_IMM && s2_hi_zero) + hi_plan = HALF_CARRY_IMM0; + + /* A 64-bit compare against a constant with a zero low half, read only by + * strict order tests: the low words cannot decide it, so drop that half and + * compare the high words with a real flag-setting CMP instead of the SBCS + * that would have folded in a borrow nothing can generate. Two instructions + * and a scratch register become one instruction and none. Proof and the + * conditions that make it exact: source/opt/flat/fusion/cmp_hi_only.c. */ + const bool cmp_hi_only = (op == TCCIR_OP_CMP) && (zh & ZH64_CMP_HI) != 0 && + src2->kind == MACH_OP_IMM && src1->is_64bit; + if (cmp_hi_only) + lo_plan = HALF_DEAD; + + /* A half no consumer reads outranks every plan above: skip it entirely. + * Only ever set for a register destination -- a memory one still owes its + * full eight bytes whatever the vreg uses do. */ + if ((zh & ZH64_D_LO) && !uses_carry && op != TCCIR_OP_CMP) + lo_plan = HALF_DEAD; + if ((zh & ZH64_D_HI) && !uses_carry && op != TCCIR_OP_CMP) + hi_plan = HALF_DEAD; + const bool need_rn_lo = (lo_plan == HALF_NORMAL || lo_plan == HALF_COPY_S1 || lo_plan == HALF_CARRY_IMM0); + const bool need_rn_hi = (hi_plan == HALF_NORMAL || hi_plan == HALF_COPY_S1 || hi_plan == HALF_CARRY_IMM0); + const bool need_rm_lo = (lo_plan == HALF_NORMAL || lo_plan == HALF_COPY_S1 || lo_plan == HALF_COPY_S2 || + lo_plan == HALF_CARRY_IMM0); + const bool need_rm_hi = (hi_plan == HALF_NORMAL || hi_plan == HALF_COPY_S2); + + /* 2. Load src1 low and high halves into registers. */ + int rn_lo = (int)PREG_REG_NONE; + if (need_rn_lo) + { + MachineOperand s1_lo = mach_make_lo_half(src1); + rn_lo = mach_ensure_in_reg(&mctx, &s1_lo, excl); + if (thumb_is_hw_reg(rn_lo)) + excl |= (1u << (uint32_t)rn_lo); + } + int rn_hi = (int)PREG_REG_NONE; + if (!need_rn_hi) + { + /* left unloaded on purpose -- nothing below reads it */ + } + else if (src1->is_64bit) + { + MachineOperand s1_hi = mach_make_hi_half(src1); + rn_hi = mach_ensure_in_reg(&mctx, &s1_hi, excl); + } + else + { + rn_hi = mach_alloc_scratch(&mctx, excl); + ot_check(th_mov_imm((uint32_t)rn_hi, 0, flags_safe(), ENFORCE_ENCODING_NONE)); + } + if (need_rn_hi && thumb_is_hw_reg(rn_hi)) + excl |= (1u << (uint32_t)rn_hi); + + /* 3. Load src2 and emit the 64-bit operation. */ + const thumb_flags_behaviour lo_flags = uses_carry ? FLAGS_BEHAVIOUR_SET : flags_safe(); + /* For CMP, the high-word SBCS must set flags (the following SETIF reads them). */ + const thumb_flags_behaviour hi_flags = (op == TCCIR_OP_CMP) ? FLAGS_BEHAVIOUR_SET : flags_safe(); + if (src2->kind == MACH_OP_IMM) + { + const uint32_t imm_lo = (uint32_t)((uint64_t)src2->u.imm.val & 0xffffffffu); + const uint32_t imm_hi = (uint32_t)((uint64_t)src2->u.imm.val >> 32); + /* Per-half peephole: when the immediate half makes the op a constant + * answer (OR/XOR with 0 → copy src; AND with 0 → load 0; AND with -1 → + * copy src), skip the data-processing op. Cuts dead `orr r, r, #0` and + * `and r, r, #0` halves left behind by 64-bit ops on 32-bit values. */ + const bool is_or = (op == TCCIR_OP_OR); + const bool is_xor = (op == TCCIR_OP_XOR); + const bool is_and = (op == TCCIR_OP_AND); + const bool can_simplify_lo = lo_flags == flags_safe(); + const bool can_simplify_hi = hi_flags == flags_safe(); + for (int half = 0; half < 2; half++) + { + const uint32_t imm = (half == 0) ? imm_lo : imm_hi; + const int rd = (half == 0) ? rd_lo : rd_hi; + const int rn = (half == 0) ? rn_lo : rn_hi; + const thumb_flags_behaviour fb = (half == 0) ? lo_flags : hi_flags; + const bool can_simplify = (half == 0) ? can_simplify_lo : can_simplify_hi; + const ThumbDataProcessingHandler *h = (half == 0 || cmp_hi_only) ? ®ular : &carry_h; + + if ((half == 0 ? lo_plan : hi_plan) == HALF_DEAD) + continue; + if (can_simplify && (is_or || is_xor) && imm == 0) + { + if (rd != rn) + ot_check_mov_reg((uint32_t)rd, (uint32_t)rn, flags_safe(), + THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE, false); + } + else if (can_simplify && is_and && imm == 0) + { + ot_check(th_mov_imm((uint32_t)rd, 0, flags_safe(), ENFORCE_ENCODING_NONE)); + } + else if (can_simplify && is_and && imm == 0xFFFFFFFFu) + { + if (rd != rn) + ot_check_mov_reg((uint32_t)rd, (uint32_t)rn, flags_safe(), + THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE, false); + } + /* A 64-bit AND against a bitfield mask splits into two half-masks, and + * the interesting half is never a modified immediate: DOUBLE_MANT_MASK's + * high half is 0x000FFFFF, which the fallback below materializes from the + * literal pool at every one of the dozen sites a soft-float classifier + * has. UBFX does the same job in one instruction with no constant. */ + else if (can_simplify && is_and && thumb_and_imm_form(imm) != AND_IMM_NONE) + { + thumb_emit_and_imm_special(thumb_and_imm_form(imm), rd, rn, imm, fb); + } + else + { + thumb_emit_op_imm_fallback(rd, rn, imm, fb, *h); + } + } + } + else + { + int rm_lo = (int)PREG_REG_NONE; + if (need_rm_lo) + { + MachineOperand s2_lo = mach_make_lo_half(src2); + rm_lo = mach_ensure_in_reg(&mctx, &s2_lo, excl); + if (thumb_is_hw_reg(rm_lo)) + excl |= (1u << (uint32_t)rm_lo); + } + int rm_hi = (int)PREG_REG_NONE; + if (!need_rm_hi) + { + /* left unloaded on purpose -- the high-half plan does not read it */ + } + else if (src2->is_64bit) + { + MachineOperand s2_hi = mach_make_hi_half(src2); + rm_hi = mach_ensure_in_reg(&mctx, &s2_hi, excl); + } + else + { + rm_hi = mach_alloc_scratch(&mctx, excl); + ot_check(th_mov_imm((uint32_t)rm_hi, 0, flags_safe(), ENFORCE_ENCODING_NONE)); + } + for (int half = 0; half < 2; half++) + { + const int plan = half ? hi_plan : lo_plan; + const int rd = half ? rd_hi : rd_lo; + const int rn = half ? rn_hi : rn_lo; + const int rm = half ? rm_hi : rm_lo; + const thumb_flags_behaviour fb = half ? hi_flags : lo_flags; + const ThumbDataProcessingHandler *h = half ? &carry_h : ®ular; + switch (plan) + { + case HALF_DEAD: + break; + case HALF_ZERO: + ot_check(th_mov_imm((uint32_t)rd, 0, flags_safe(), ENFORCE_ENCODING_NONE)); + break; + case HALF_COPY_S1: + if (rd != rn) + ot_check_mov_reg((uint32_t)rd, (uint32_t)rn, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE, + false); + break; + case HALF_COPY_S2: + if (rd != rm) + ot_check_mov_reg((uint32_t)rd, (uint32_t)rm, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE, + false); + break; + case HALF_CARRY_IMM0: + thumb_emit_op_imm_fallback(rd, rn, 0, fb, *h); + break; + default: + ot_check(thumb_call_reg_handler(h->reg_handler, (uint32_t)rd, (uint32_t)rn, (uint32_t)rm, fb, + THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); + break; + } + } + } + + /* 4. Write results back to spill/param slots if dest was not pre-allocated. + * A half nothing reads is not written back either -- the slot keeps + * whatever it held, which is exactly what "dead" asserts. */ + if (store_lo && lo_plan != HALF_DEAD) + { + MachineOperand dst_lo = mach_make_lo_half(dest); + dst_lo.btype = IROP_BTYPE_INT32; + mach_writeback_dest(&dst_lo, rd_lo); + } + if (store_hi && hi_plan != HALF_DEAD) + { + MachineOperand dst_hi = mach_make_hi_half(dest); + dst_hi.btype = IROP_BTYPE_INT32; + mach_writeback_dest(&dst_hi, rd_hi); + } + mach_release_all(&mctx); +} + +/* ============================================================ + * thumb_emit_shift64_reg_mop + * ============================================================ + * 64-bit SHL / SHR / SAR whose count is a value rather than a literal. + * + * Before this every such shift went to __aeabi_llsl / __aeabi_llsr / + * __aeabi_lasr. Those helpers are compiled from C like any other library + * routine, so each one costs a call, a frame and ~30 executed instructions; a + * dynamic profile of __aeabi_dadd put two of them -- the alignment shift and + * the `(1ULL << n) - 1` mask -- at 62 of its 267 instructions per call. + * Inline the same work is 8 instructions and touches no memory. + * + * The sequences lean on ARM's register-shift rule: LSL/LSR/ASR by a register + * read only the bottom BYTE of the count, and a count of 32 or more yields 0 + * (LSL/LSR) or a full sign fill (ASR). That makes the `32 - n` and `n - 32` + * cross terms self-disabling over the half of the range where they must not + * contribute, so the logical shifts need neither a branch nor a conditional: + * + * x << n : hi = (hi << n) | (lo >> (32-n)) | (lo << (n-32)); lo = lo << n + * x >> n : lo = (lo >> n) | (hi << (32-n)) | (hi >> (n-32)); hi = hi >> n + * + * SAR cannot borrow that trick for its cross term -- `hi ASR (n-32)` is a sign + * fill when n < 32, which would flood the low half with ones where it must + * contribute nothing -- so it selects between the two halves of the range with + * a mask taken from the sign of `n - 32`. A mask and not an IT/MOVGE pair + * because this runs wherever the allocator placed the shift, including with a + * pending CMP's flags still live. + * + * Counts of 64 and above are undefined in C; the sequences give 0 (or a sign + * fill for SAR), which is what GCC's inline expansion produces as well. + */ +static void thumb_emit_shift64_reg_mop(const MachineOperand *src1, const MachineOperand *src2, + const MachineOperand *dest, TccIrOp op, bool skip_lo, bool skip_hi) +{ + const bool is_left = (op == TCCIR_OP_SHL); + const bool arith_right = (op == TCCIR_OP_SAR); + + MachineCodegenContext mctx = {0}; + uint32_t excl = 0; + + /* Destination pair first, exactly as the immediate path does, so that deref + * resolution never hands out a scratch that overlaps it. */ + int dst_lo, dst_hi; + bool store_lo = false, store_hi = false; + if (dest->kind == MACH_OP_REG && !dest->needs_deref && dest->u.reg.r0 != (int)PREG_REG_NONE && dest->u.reg.r1 >= 0) + { + dst_lo = dest->u.reg.r0; + dst_hi = dest->u.reg.r1; + excl |= (1u << (uint32_t)dst_lo) | (1u << (uint32_t)dst_hi); + } + else + { + dst_lo = mach_alloc_scratch(&mctx, excl); + excl |= (1u << (uint32_t)dst_lo); + dst_hi = mach_alloc_scratch(&mctx, excl); + excl |= (1u << (uint32_t)dst_hi); + store_lo = store_hi = (dest->kind != MACH_OP_NONE); + } + + if (src1->kind == MACH_OP_REG) + { + if (src1->u.reg.r0 != (int)PREG_REG_NONE) + excl |= (1u << (uint32_t)src1->u.reg.r0); + if (!src1->needs_deref && src1->is_64bit && src1->u.reg.r1 >= 0) + excl |= (1u << (uint32_t)src1->u.reg.r1); + } + if (src2->kind == MACH_OP_REG) + { + if (src2->u.reg.r0 != (int)PREG_REG_NONE) + excl |= (1u << (uint32_t)src2->u.reg.r0); + if (!src2->needs_deref && src2->is_64bit && src2->u.reg.r1 >= 0) + excl |= (1u << (uint32_t)src2->u.reg.r1); + } + + MachineOperand r_src1 = mach_resolve_deref_64(&mctx, src1, &excl); + src1 = &r_src1; + + MachineOperand s1_lo = mach_make_lo_half(src1); + int src_lo = mach_ensure_in_reg(&mctx, &s1_lo, excl); + if (thumb_is_hw_reg(src_lo)) + excl |= (1u << (uint32_t)src_lo); + + /* A high word that is a known zero deletes every term that reads it -- and + * the register that would have held it. `(1ULL << n) - 1`, the mask soft + * float builds every alignment sticky-bit from, is exactly this shape: the + * generic sequence would materialize a zero just to shift and OR it in. */ + const int hi_is_zero = + !arith_right && (!src1->is_64bit || + (src1->kind == MACH_OP_IMM && ((uint64_t)src1->u.imm.val >> 32) == 0)); + + int src_hi = (int)PREG_REG_NONE; + if (!hi_is_zero) + { + if (src1->is_64bit) + { + MachineOperand s1_hi = mach_make_hi_half(src1); + src_hi = mach_ensure_in_reg(&mctx, &s1_hi, excl); + if (thumb_is_hw_reg(src_hi)) + excl |= (1u << (uint32_t)src_hi); + } + else + { + src_hi = mach_alloc_scratch(&mctx, excl); + excl |= (1u << (uint32_t)src_hi); + ot_check(th_asr_imm((uint32_t)src_hi, (uint32_t)src_lo, 31, flags_safe(), ENFORCE_ENCODING_NONE)); + } + } + + /* Only the low word of a 64-bit count can matter — the hardware reads one + * byte of it. */ + MachineOperand cnt_op = src2->is_64bit ? mach_make_lo_half(src2) : *src2; + int cnt = mach_ensure_in_reg(&mctx, &cnt_op, excl); + if (thumb_is_hw_reg(cnt)) + excl |= (1u << (uint32_t)cnt); + + /* The half that collects the cross terms. Writing it straight into the + * destination saves a MOV, but only where that register is neither a source + * half still to be read nor the count. */ + /* A right shift of a zero-high-word value needs no accumulator and no + * temporaries; allocating them anyway can cost a scratch push/pop. */ + const bool need_acc = (!is_left && hi_is_zero) ? false : (is_left ? !skip_hi : !skip_lo); + const int acc_pref = is_left ? dst_hi : dst_lo; + const int acc_conflict = is_left ? src_lo : src_hi; + int acc = -1; + if (need_acc) + { + acc = (acc_pref != acc_conflict && acc_pref != cnt) ? acc_pref : -1; + if (acc < 0) + { + acc = mach_alloc_scratch(&mctx, excl); + excl |= (1u << (uint32_t)acc); + } + } + + int t0 = -1, t1 = -1; + if (need_acc) + { + t0 = mach_alloc_scratch(&mctx, excl); + excl |= (1u << (uint32_t)t0); + if (arith_right) + { + t1 = mach_alloc_scratch(&mctx, excl); + excl |= (1u << (uint32_t)t1); + } + } + + if (is_left) + { + if (need_acc) + { + ot_check(th_rsb_imm((uint32_t)t0, (uint32_t)cnt, 32, flags_safe(), ENFORCE_ENCODING_NONE)); + if (hi_is_zero) + { + /* (0 << n) | (lo >> (32-n)) is just the cross term. */ + ot_check(th_lsr_reg((uint32_t)acc, (uint32_t)src_lo, (uint32_t)t0, flags_safe(), THUMB_SHIFT_DEFAULT, + ENFORCE_ENCODING_NONE)); + } + else + { + ot_check(th_lsl_reg((uint32_t)acc, (uint32_t)src_hi, (uint32_t)cnt, flags_safe(), THUMB_SHIFT_DEFAULT, + ENFORCE_ENCODING_NONE)); + ot_check(th_lsr_reg((uint32_t)t0, (uint32_t)src_lo, (uint32_t)t0, flags_safe(), THUMB_SHIFT_DEFAULT, + ENFORCE_ENCODING_NONE)); + ot_check(th_orr_reg((uint32_t)acc, (uint32_t)acc, (uint32_t)t0, flags_safe(), THUMB_SHIFT_DEFAULT, + ENFORCE_ENCODING_NONE)); + } + ot_check(th_sub_imm((uint32_t)t0, (uint32_t)cnt, 32, flags_safe(), ENFORCE_ENCODING_NONE)); + ot_check(th_lsl_reg((uint32_t)t0, (uint32_t)src_lo, (uint32_t)t0, flags_safe(), THUMB_SHIFT_DEFAULT, + ENFORCE_ENCODING_NONE)); + ot_check(th_orr_reg((uint32_t)acc, (uint32_t)acc, (uint32_t)t0, flags_safe(), THUMB_SHIFT_DEFAULT, + ENFORCE_ENCODING_NONE)); + } + /* dst_lo last: it is the final read of src_lo and of the count. */ + if (!skip_lo) + ot_check(th_lsl_reg((uint32_t)dst_lo, (uint32_t)src_lo, (uint32_t)cnt, flags_safe(), THUMB_SHIFT_DEFAULT, + ENFORCE_ENCODING_NONE)); + if (need_acc && acc != dst_hi) + ot_check_mov_reg((uint32_t)dst_hi, (uint32_t)acc, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE, + false); + } + else if (hi_is_zero) + { + /* Every cross term reads the zero high word, so the whole shift is one + * instruction and a zero fill. */ + if (!skip_lo) + ot_check(th_lsr_reg((uint32_t)dst_lo, (uint32_t)src_lo, (uint32_t)cnt, flags_safe(), THUMB_SHIFT_DEFAULT, + ENFORCE_ENCODING_NONE)); + if (!skip_hi) + ot_check(th_mov_imm((uint32_t)dst_hi, 0, flags_safe(), ENFORCE_ENCODING_NONE)); + } + else + { + if (need_acc) + { + ot_check(th_rsb_imm((uint32_t)t0, (uint32_t)cnt, 32, flags_safe(), ENFORCE_ENCODING_NONE)); + ot_check(th_lsr_reg((uint32_t)acc, (uint32_t)src_lo, (uint32_t)cnt, flags_safe(), THUMB_SHIFT_DEFAULT, + ENFORCE_ENCODING_NONE)); + ot_check(th_lsl_reg((uint32_t)t0, (uint32_t)src_hi, (uint32_t)t0, flags_safe(), THUMB_SHIFT_DEFAULT, + ENFORCE_ENCODING_NONE)); + ot_check(th_orr_reg((uint32_t)acc, (uint32_t)acc, (uint32_t)t0, flags_safe(), THUMB_SHIFT_DEFAULT, + ENFORCE_ENCODING_NONE)); + ot_check(th_sub_imm((uint32_t)t0, (uint32_t)cnt, 32, flags_safe(), ENFORCE_ENCODING_NONE)); + if (arith_right) + { + /* `hi ASR (n-32)` is only the answer for n >= 32; below that it is a + * sign fill that must contribute nothing. `(n-32) ASR 31` is all-ones + * exactly on that range, so it selects between the two candidates + * without touching the flags. */ + ot_check(th_asr_reg((uint32_t)t1, (uint32_t)src_hi, (uint32_t)t0, flags_safe(), THUMB_SHIFT_DEFAULT, + ENFORCE_ENCODING_NONE)); + ot_check(th_asr_imm((uint32_t)t0, (uint32_t)t0, 31, flags_safe(), ENFORCE_ENCODING_NONE)); + ot_check(th_and_reg((uint32_t)acc, (uint32_t)acc, (uint32_t)t0, flags_safe(), THUMB_SHIFT_DEFAULT, + ENFORCE_ENCODING_NONE)); + ot_check(th_bic_reg((uint32_t)t1, (uint32_t)t1, (uint32_t)t0, flags_safe(), THUMB_SHIFT_DEFAULT, + ENFORCE_ENCODING_NONE)); + ot_check(th_orr_reg((uint32_t)acc, (uint32_t)acc, (uint32_t)t1, flags_safe(), THUMB_SHIFT_DEFAULT, + ENFORCE_ENCODING_NONE)); + } + else + { + ot_check(th_lsr_reg((uint32_t)t0, (uint32_t)src_hi, (uint32_t)t0, flags_safe(), THUMB_SHIFT_DEFAULT, + ENFORCE_ENCODING_NONE)); + ot_check(th_orr_reg((uint32_t)acc, (uint32_t)acc, (uint32_t)t0, flags_safe(), THUMB_SHIFT_DEFAULT, + ENFORCE_ENCODING_NONE)); + } + } + /* dst_hi last: it is the final read of src_hi and of the count. */ + if (!skip_hi) + { + if (arith_right) + ot_check(th_asr_reg((uint32_t)dst_hi, (uint32_t)src_hi, (uint32_t)cnt, flags_safe(), THUMB_SHIFT_DEFAULT, + ENFORCE_ENCODING_NONE)); + else + ot_check(th_lsr_reg((uint32_t)dst_hi, (uint32_t)src_hi, (uint32_t)cnt, flags_safe(), THUMB_SHIFT_DEFAULT, + ENFORCE_ENCODING_NONE)); + } + if (need_acc && acc != dst_lo) + ot_check_mov_reg((uint32_t)dst_lo, (uint32_t)acc, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE, + false); + } + + if (store_lo && !skip_lo) + { + MachineOperand dst_lo_op = mach_make_lo_half(dest); + dst_lo_op.btype = IROP_BTYPE_INT32; + mach_writeback_dest(&dst_lo_op, dst_lo); + } + if (store_hi && !skip_hi) + { + MachineOperand dst_hi_op = mach_make_hi_half(dest); + dst_hi_op.btype = IROP_BTYPE_INT32; + mach_writeback_dest(&dst_hi_op, dst_hi); + } + mach_release_all(&mctx); +} + +/* ============================================================ + * thumb_emit_shift64_mop + * ============================================================ + * 64-bit SHL / SHR / SAR via MachineOperand register pairs. + * Shift amount (src2) must be a 32-bit immediate (MACH_OP_IMM). + * Logic mirrors thumb_emit_shift64_imm but operates on register numbers + * extracted from MachineOperand rather than IROperand fields. + */ +static void thumb_emit_shift64_mop(const MachineOperand *src1, const MachineOperand *src2, const MachineOperand *dest, + TccIrOp op, bool skip_lo, bool skip_hi) +{ + if (src2->kind != MACH_OP_IMM) + { + thumb_emit_shift64_reg_mop(src1, src2, dest, op, skip_lo, skip_hi); + return; + } + const uint32_t sh = (uint32_t)(uint64_t)src2->u.imm.val; + const bool is_left = (op == TCCIR_OP_SHL); + const bool arith_right = (op == TCCIR_OP_SAR); + + thumb_imm_handler_t dst_lo_shift, dst_hi_shift, cross_shift; + if (is_left) + { + dst_lo_shift = th_lsl_imm; + dst_hi_shift = th_lsl_imm; + cross_shift = th_lsr_imm; + } + else if (arith_right) + { + dst_lo_shift = th_lsr_imm; + dst_hi_shift = th_asr_imm; + cross_shift = th_lsl_imm; + } + else + { + dst_lo_shift = th_lsr_imm; + dst_hi_shift = th_lsr_imm; + cross_shift = th_lsl_imm; + } + + MachineCodegenContext mctx = {0}; + uint32_t excl = 0; + + /* Determine destination register pair FIRST so that deref resolution + * never allocates scratch registers that overlap with the dest pair. */ + int dst_lo, dst_hi; + bool store_lo = false, store_hi = false; + if (dest->kind == MACH_OP_REG && !dest->needs_deref && dest->u.reg.r0 != (int)PREG_REG_NONE && dest->u.reg.r1 >= 0) + { + dst_lo = dest->u.reg.r0; + dst_hi = dest->u.reg.r1; + excl |= (1u << (uint32_t)dst_lo) | (1u << (uint32_t)dst_hi); + } + else + { + dst_lo = mach_alloc_scratch(&mctx, excl); + excl |= (1u << (uint32_t)dst_lo); + dst_hi = mach_alloc_scratch(&mctx, excl); + excl |= (1u << (uint32_t)dst_hi); + store_lo = store_hi = (dest->kind != MACH_OP_NONE); + } + + /* Pre-exclude register operands so that deref resolution does not + * steal the physical registers already holding src1 values. + * Include needs_deref registers: they hold live pointers needed + * during their own deref resolution. */ + if (src1->kind == MACH_OP_REG) + { + if (src1->u.reg.r0 != (int)PREG_REG_NONE) + excl |= (1u << (uint32_t)src1->u.reg.r0); + if (!src1->needs_deref && src1->is_64bit && src1->u.reg.r1 >= 0) + excl |= (1u << (uint32_t)src1->u.reg.r1); + } + + /* Resolve deref'd source pointer before splitting into halves. */ + MachineOperand r_src1 = mach_resolve_deref_64(&mctx, src1, &excl); + src1 = &r_src1; + + /* Load src1 low half. */ + MachineOperand s1_lo = mach_make_lo_half(src1); + int src_lo = mach_ensure_in_reg(&mctx, &s1_lo, excl); + if (thumb_is_hw_reg(src_lo)) + excl |= (1u << (uint32_t)src_lo); + + /* Skip src1 high-half materialization when the shift will not read it. + * SHL with sh >= 32 only uses src_lo (everything shifts up out of view). + * SHR/SAR with sh >= 64 produces a 0/sign-fill that the emit tail + * generates directly without referencing src_hi. */ + int hi_needed = 1; + if (is_left && sh >= 32) + hi_needed = 0; + else if (!is_left && sh >= 64 && !arith_right) + hi_needed = 0; + /* SAR by >= 64 is the exception: its tail fills BOTH halves from the sign of + * src_hi, so it reads the very half the count would suggest is irrelevant. + * Declaring it unneeded left src_hi as PREG_REG_NONE and the tail encoded a + * shift off a register that was never loaded. */ + + /* Load src1 high half or compute by extension. `hi_needed` gates the + * ALREADY-64-bit case too: materializing a half nothing below reads is a + * whole instruction, and `(uint64_t)sign << 63` in make_double is exactly + * that -- the widening puts a zero in a register purely to shift it out of + * existence. */ + int src_hi = (int)PREG_REG_NONE; + if (src1->is_64bit) + { + if (hi_needed) + { + MachineOperand s1_hi = mach_make_hi_half(src1); + src_hi = mach_ensure_in_reg(&mctx, &s1_hi, excl); + if (thumb_is_hw_reg(src_hi)) + excl |= (1u << (uint32_t)src_hi); + } + } + else if (hi_needed) + { + src_hi = mach_alloc_scratch(&mctx, excl); + excl |= (1u << (uint32_t)src_hi); + if (arith_right) + ot_check( + th_asr_imm((uint32_t)src_hi, (uint32_t)src_lo, 31, flags_safe(), ENFORCE_ENCODING_NONE)); + else + ot_check(th_mov_imm((uint32_t)src_hi, 0, flags_safe(), ENFORCE_ENCODING_NONE)); + } + + /* `x <<= 1` is ADDS/ADC: two instructions where the generic sh<32 path below + * needs four (cross-shift into a scratch, a shift per half, then an OR). + * This is the step every restoring-division and normalization loop is built + * from -- a dynamic profile of ddiv puts 280,000 iterations through it for + * the benchmark's double kernels, against 5,000 through everything else. + * + * Two guards. ADDS/ADC write the flags, so it must not run while a pending + * CMP's flags are still live. And ADC has to read src_hi AFTER ADDS has + * written dst_lo, so dst_lo aliasing src_hi rules the pair out; every other + * aliasing case is safe, since each instruction reads its sources before + * writing its destination. */ + if (is_left && sh == 1 && !skip_lo && !skip_hi && + thumb_is_hw_reg(src_lo) && thumb_is_hw_reg(src_hi) && dst_lo != src_hi && + !(tcc_state->ir && tcc_state->ir->codegen_flags_live)) + { + ot_check(th_add_reg((uint32_t)dst_lo, (uint32_t)src_lo, (uint32_t)src_lo, FLAGS_BEHAVIOUR_SET, + THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); + ot_check(th_adc_reg((uint32_t)dst_hi, (uint32_t)src_hi, (uint32_t)src_hi, flags_safe(), + THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); + } + /* Emit the shift — logic identical to thumb_emit_shift64_imm core. */ + else if (sh == 0) + { + ot_check_mov_reg((uint32_t)dst_lo, (uint32_t)src_lo, flags_safe(), THUMB_SHIFT_DEFAULT, + ENFORCE_ENCODING_NONE, false); + ot_check_mov_reg((uint32_t)dst_hi, (uint32_t)src_hi, flags_safe(), THUMB_SHIFT_DEFAULT, + ENFORCE_ENCODING_NONE, false); + } + else if (sh < 32) + { + /* The cross term is an ORR away from being free: T2 ORR takes a shifted + * REGISTER operand, so `orr rd, rd, src, LSR #(32-sh)` does in one + * instruction what a separate shift into a scratch plus a plain ORR did in + * two -- and it needs no scratch at all, which on a saturated function is + * a push/pop as well. Every 64-bit shift by a literal in soft float is + * this shape (`mant <<= 3`, `mant >> 3`), and __aeabi_dadd runs two per + * call. + * + * Only the aliasing case the fallback below reorders for is excluded: the + * destination half written FIRST must not be the source half the ORR still + * has to read. */ + const uint32_t cross_amount = (uint32_t)(32 - sh); + const thumb_shift cross_fold = {is_left ? THUMB_SHIFT_LSR : THUMB_SHIFT_LSL, cross_amount, + THUMB_SHIFT_IMMEDIATE}; + if (is_left && dst_hi != src_lo) + { + ot_check(thumb_call_imm_handler(dst_hi_shift, (uint32_t)dst_hi, (uint32_t)src_hi, sh, flags_safe(), + ENFORCE_ENCODING_NONE)); + ot_check(th_orr_reg((uint32_t)dst_hi, (uint32_t)dst_hi, (uint32_t)src_lo, flags_safe(), cross_fold, + ENFORCE_ENCODING_32BIT)); + if (!skip_lo) + ot_check(thumb_call_imm_handler(dst_lo_shift, (uint32_t)dst_lo, (uint32_t)src_lo, sh, flags_safe(), + ENFORCE_ENCODING_NONE)); + goto shift64_imm_done; + } + if (!is_left && dst_lo != src_hi) + { + ot_check(th_lsr_imm((uint32_t)dst_lo, (uint32_t)src_lo, sh, flags_safe(), ENFORCE_ENCODING_NONE)); + ot_check(th_orr_reg((uint32_t)dst_lo, (uint32_t)dst_lo, (uint32_t)src_hi, flags_safe(), cross_fold, + ENFORCE_ENCODING_32BIT)); + ot_check(thumb_call_imm_handler(dst_hi_shift, (uint32_t)dst_hi, (uint32_t)src_hi, sh, flags_safe(), + ENFORCE_ENCODING_NONE)); + goto shift64_imm_done; + } + + const int regs[] = {dst_lo, dst_hi, src_lo, src_hi}; + ScratchRegAlloc tmp = get_scratch_reg_with_save(thumb_exclude_mask_for_regs(4, regs) | excl); + if (is_left) + { + /* Compute the cross-shift into tmp BEFORE any destination is written, + * because dst_lo/dst_hi may alias src_lo/src_hi. */ + ot_check(thumb_call_imm_handler(cross_shift, (uint32_t)tmp.reg, (uint32_t)src_lo, 32 - sh, flags_safe(), + ENFORCE_ENCODING_NONE)); + if (dst_hi == src_lo) + { + /* dst_hi aliases src_lo — compute dst_lo first (needs src_lo). */ + if (!skip_lo) + ot_check( + thumb_call_imm_handler(dst_lo_shift, (uint32_t)dst_lo, (uint32_t)src_lo, sh, flags_safe(), ENFORCE_ENCODING_NONE)); + ot_check( + thumb_call_imm_handler(dst_hi_shift, (uint32_t)dst_hi, (uint32_t)src_hi, sh, flags_safe(), ENFORCE_ENCODING_NONE)); + } + else + { + /* Default order: dst_hi first to avoid clobbering src_hi via dst_lo. */ + ot_check( + thumb_call_imm_handler(dst_hi_shift, (uint32_t)dst_hi, (uint32_t)src_hi, sh, flags_safe(), ENFORCE_ENCODING_NONE)); + if (!skip_lo) + ot_check( + thumb_call_imm_handler(dst_lo_shift, (uint32_t)dst_lo, (uint32_t)src_lo, sh, flags_safe(), ENFORCE_ENCODING_NONE)); + } + ot_check(th_orr_reg((uint32_t)dst_hi, (uint32_t)dst_hi, (uint32_t)tmp.reg, flags_safe(), + THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); + } + else + { + /* Compute the cross-shift into tmp BEFORE any destination is written, + * because dst_lo/dst_hi may alias src_lo/src_hi. */ + ot_check(thumb_call_imm_handler(cross_shift, (uint32_t)tmp.reg, (uint32_t)src_hi, 32 - sh, flags_safe(), + ENFORCE_ENCODING_NONE)); + if (dst_lo == src_hi) + { + /* dst_lo aliases src_hi — compute dst_hi first (needs src_hi). */ + ot_check( + thumb_call_imm_handler(dst_hi_shift, (uint32_t)dst_hi, (uint32_t)src_hi, sh, flags_safe(), ENFORCE_ENCODING_NONE)); + ot_check( + th_lsr_imm((uint32_t)dst_lo, (uint32_t)src_lo, sh, flags_safe(), ENFORCE_ENCODING_NONE)); + } + else + { + /* Default order: dst_lo first to avoid clobbering src_lo via dst_hi. */ + ot_check( + th_lsr_imm((uint32_t)dst_lo, (uint32_t)src_lo, sh, flags_safe(), ENFORCE_ENCODING_NONE)); + ot_check( + thumb_call_imm_handler(dst_hi_shift, (uint32_t)dst_hi, (uint32_t)src_hi, sh, flags_safe(), ENFORCE_ENCODING_NONE)); + } + ot_check(th_orr_reg((uint32_t)dst_lo, (uint32_t)dst_lo, (uint32_t)tmp.reg, flags_safe(), + THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); + } + restore_scratch_reg(&tmp); + } + else if (sh == 32) + { + if (is_left) + { + /* Emit MOV dst_hi first: dst_lo may alias src_lo. */ + ot_check_mov_reg((uint32_t)dst_hi, (uint32_t)src_lo, flags_safe(), THUMB_SHIFT_DEFAULT, + ENFORCE_ENCODING_NONE, false); + if (!skip_lo) + ot_check(th_mov_imm((uint32_t)dst_lo, 0, flags_safe(), ENFORCE_ENCODING_NONE)); + } + else + { + /* Emit MOV dst_lo first: dst_hi may alias src_hi. */ + ot_check_mov_reg((uint32_t)dst_lo, (uint32_t)src_hi, flags_safe(), THUMB_SHIFT_DEFAULT, + ENFORCE_ENCODING_NONE, false); + if (!skip_hi) + { + if (arith_right) + ot_check( + th_asr_imm((uint32_t)dst_hi, (uint32_t)src_hi, 31, flags_safe(), ENFORCE_ENCODING_NONE)); + else + ot_check(th_mov_imm((uint32_t)dst_hi, 0, flags_safe(), ENFORCE_ENCODING_NONE)); + } + } + } + else if (sh < 64) + { + if (is_left) + { + /* Emit shift into dst_hi first: dst_lo may alias src_lo. */ + ot_check(thumb_call_imm_handler(dst_hi_shift, (uint32_t)dst_hi, (uint32_t)src_lo, sh - 32, flags_safe(), + ENFORCE_ENCODING_NONE)); + if (!skip_lo) + ot_check(th_mov_imm((uint32_t)dst_lo, 0, flags_safe(), ENFORCE_ENCODING_NONE)); + } + else + { + if (arith_right && dst_lo == src_hi) + { + /* dst_lo aliases src_hi — compute dst_hi (sign extension) first + * while src_hi is still intact, then shift into dst_lo. */ + if (!skip_hi) + ot_check( + th_asr_imm((uint32_t)dst_hi, (uint32_t)src_hi, 31, flags_safe(), ENFORCE_ENCODING_NONE)); + ot_check(thumb_call_imm_handler(dst_hi_shift, (uint32_t)dst_lo, (uint32_t)src_hi, sh - 32, flags_safe(), + ENFORCE_ENCODING_NONE)); + } + else + { + ot_check(thumb_call_imm_handler(dst_hi_shift, (uint32_t)dst_lo, (uint32_t)src_hi, sh - 32, flags_safe(), + ENFORCE_ENCODING_NONE)); + if (!skip_hi) + { + if (arith_right) + ot_check( + th_asr_imm((uint32_t)dst_hi, (uint32_t)src_hi, 31, flags_safe(), ENFORCE_ENCODING_NONE)); + else + ot_check(th_mov_imm((uint32_t)dst_hi, 0, flags_safe(), ENFORCE_ENCODING_NONE)); + } + } + } + } + else /* sh >= 64 */ + { + if (is_left) + { + if (!skip_lo) + ot_check(th_mov_imm((uint32_t)dst_lo, 0, flags_safe(), ENFORCE_ENCODING_NONE)); + if (!skip_hi) + ot_check(th_mov_imm((uint32_t)dst_hi, 0, flags_safe(), ENFORCE_ENCODING_NONE)); + } + else if (arith_right) + { + /* Both halves are the sign of src_hi; dst_lo copies dst_hi, so leave + * this degenerate path intact rather than risk the inter-half dep. */ + ot_check( + th_asr_imm((uint32_t)dst_hi, (uint32_t)src_hi, 31, flags_safe(), ENFORCE_ENCODING_NONE)); + ot_check_mov_reg((uint32_t)dst_lo, (uint32_t)dst_hi, flags_safe(), THUMB_SHIFT_DEFAULT, + ENFORCE_ENCODING_NONE, false); + } + else + { + if (!skip_lo) + ot_check(th_mov_imm((uint32_t)dst_lo, 0, flags_safe(), ENFORCE_ENCODING_NONE)); + if (!skip_hi) + ot_check(th_mov_imm((uint32_t)dst_hi, 0, flags_safe(), ENFORCE_ENCODING_NONE)); + } + } + +shift64_imm_done: + /* Write back. A dead half was never materialized, so skip its store. */ + if (store_lo && !skip_lo) + { + MachineOperand dst_lo_op = mach_make_lo_half(dest); + dst_lo_op.btype = IROP_BTYPE_INT32; + mach_writeback_dest(&dst_lo_op, dst_lo); + } + if (store_hi && !skip_hi) + { + MachineOperand dst_hi_op = mach_make_hi_half(dest); + dst_hi_op.btype = IROP_BTYPE_INT32; + mach_writeback_dest(&dst_hi_op, dst_hi); + } + mach_release_all(&mctx); +} + +/* ============================================================ + * MachineOperand-based data processing (_mop path) + * ============================================================ + * thumb_emit_data_processing_mop32: simplified version of + * thumb_emit_data_processing_op32 using MachineOperand instead of IROperand. + * Handles 32-bit non-complex arithmetic/logic ops via the mach_* helpers, + * eliminating the two-layer materialization present in the old path. + */ +/* TCC_NO_CMP_ADDS=1 keeps CMP Rn, #-k as CMP.W (thumb_emit_data_processing_mop32). */ +TCC_DBG_ENV_FLAG(cmp_adds_off, "TCC_NO_CMP_ADDS") +static int cmp_adds_enabled(void) +{ + return !cmp_adds_off(); +} + +static void thumb_emit_data_processing_mop32(const MachineOperand *src1, const MachineOperand *src2, + const MachineOperand *dest, TccIrOp op, ThumbDataProcessingHandler handler, + thumb_flags_behaviour flags, uint32_t barrel_shift) +{ + const bool dest_sets_flags = (op == TCCIR_OP_CMP); + MachineCodegenContext mctx = {0}; + + /* Barrel-shifted immediate src2: the ALU immediate form has no shift field, + * so if src2 carries a barrel-shift annotation but has been lowered to an + * immediate (a rematerialized constant substituted for the shift's source + * register — see ra_mark_rematerializable), fold the shift into the constant + * and clear the annotation. Without this the shift is silently dropped: the + * immediate path below emits ` Rd, Rn, #imm` and the `if (!imm_emitted)` + * shift block never runs (fuzz seed longlong:6393 — `y ^ (u5 LSR #31)` with + * u5 remat'd to #1 wrongly emitted `eor r,r,#1` instead of `eor r,r,#0`). + * The fusion pass keeps amount in 0..31 (never the ARM "0 means 32" LSR/ASR/ + * ROR case), so these C shifts are well-defined. */ + MachineOperand src2_folded; + if (barrel_shift != 0 && src2->kind == MACH_OP_IMM && !src2->needs_deref && !src2->is_64bit) + { + uint32_t stype = (barrel_shift >> 5) & 7; + uint32_t samt = barrel_shift & 31; + uint32_t v = (uint32_t)src2->u.imm.val; + uint32_t r; + switch (stype) + { + case 1: r = v << samt; break; /* LSL */ + case 2: r = v >> samt; break; /* LSR */ + case 3: r = (uint32_t)((int32_t)v >> samt); break; /* ASR */ + case 4: r = samt ? ((v >> samt) | (v << (32 - samt))) : v; break; /* ROR */ + default: r = v; break; + } + src2_folded = *src2; + src2_folded.u.imm.val = (int64_t)(int32_t)r; + src2 = &src2_folded; + barrel_shift = 0; + } + + /* RSB fast path: SUB with immediate src1 → RSB Rd, src2, #imm. + * Avoids materializing the immediate into a register. + * Only attempt when the immediate is encodable as a Thumb-2 modified + * constant (th_pack_const returns non-zero, or imm==0). */ + if (op == TCCIR_OP_SUB && !dest_sets_flags && barrel_shift == 0 && + src1->kind == MACH_OP_IMM && !src1->needs_deref && !src1->is_64bit) + { + uint32_t imm = (uint32_t)src1->u.imm.val; + if (imm == 0 || th_pack_const(imm) != 0) + { + int dest_reg = mach_get_dest_reg(&mctx, dest, 0); + uint32_t excl = thumb_is_hw_reg(dest_reg) ? (1u << (uint32_t)dest_reg) : 0; + int src2_reg = mach_ensure_in_reg(&mctx, src2, excl); + ot_check(th_rsb_imm((uint32_t)dest_reg, (uint32_t)src2_reg, imm, flags, ENFORCE_ENCODING_NONE)); + if (dest->kind != MACH_OP_NONE) + { + const bool needs_wb = dest->kind == MACH_OP_SPILL || dest->kind == MACH_OP_PARAM_STACK || + (dest->kind == MACH_OP_REG && (dest->needs_deref || dest->u.reg.r0 == (int)PREG_REG_NONE)); + if (needs_wb) + mach_writeback_dest(dest, dest_reg); + } + mach_release_all(&mctx); + return; + } + } + + /* Shift-by-0 identity: on ARM, LSR/ASR with immediate field 0 means + * shift-by-32 (yielding 0 / sign-extend), NOT shift-by-0. Fold x >> 0 + * to a plain MOV Rd, Rm so the semantics are correct regardless of + * whether the optimizer managed to simplify the IR. */ + if (!dest_sets_flags && barrel_shift == 0 && + (op == TCCIR_OP_SHR || op == TCCIR_OP_SAR || op == TCCIR_OP_ROR) && + src2->kind == MACH_OP_IMM && !src2->needs_deref && !src2->is_64bit && + (uint32_t)src2->u.imm.val == 0) + { + int dest_reg = mach_get_dest_reg(&mctx, dest, 0); + uint32_t excl = thumb_is_hw_reg(dest_reg) ? (1u << (uint32_t)dest_reg) : 0; + int src1_reg = mach_ensure_in_reg(&mctx, src1, excl); + ot_check_mov_reg((uint32_t)dest_reg, (uint32_t)src1_reg, flags, + THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE, false); + if (dest->kind != MACH_OP_NONE) + { + const bool needs_wb = dest->kind == MACH_OP_SPILL || dest->kind == MACH_OP_PARAM_STACK || + (dest->kind == MACH_OP_REG && (dest->needs_deref || dest->u.reg.r0 == (int)PREG_REG_NONE)); + if (needs_wb) + mach_writeback_dest(dest, dest_reg); + } + mach_release_all(&mctx); + return; + } + + /* Mask fast paths: AND with an immediate the AND-immediate encoding cannot + * hold, lowered as UXTB/UXTH/UBFX/BIC instead of materializing a constant. + * See thumb_try_and_imm_special; the 64-bit half-lowering shares it. */ + if (op == TCCIR_OP_AND && !dest_sets_flags && barrel_shift == 0 && + src2->kind == MACH_OP_IMM && !src2->needs_deref && !src2->is_64bit && + flags != FLAGS_BEHAVIOUR_SET) + { + const uint32_t mask = (uint32_t)src2->u.imm.val; + const ThumbAndImmForm form = thumb_and_imm_form(mask); + /* Classify before resolving: on AND_IMM_NONE nothing has been emitted and + * the general path below resolves both operands itself. */ + if (form != AND_IMM_NONE) + { + int dest_reg = mach_get_dest_reg(&mctx, dest, 0); + uint32_t excl = thumb_is_hw_reg(dest_reg) ? (1u << (uint32_t)dest_reg) : 0; + int src1_reg = mach_ensure_in_reg(&mctx, src1, excl); + thumb_emit_and_imm_special(form, dest_reg, src1_reg, mask, flags); + if (dest->kind != MACH_OP_NONE) + { + const bool needs_wb = dest->kind == MACH_OP_SPILL || dest->kind == MACH_OP_PARAM_STACK || + (dest->kind == MACH_OP_REG && (dest->needs_deref || dest->u.reg.r0 == (int)PREG_REG_NONE)); + if (needs_wb) + mach_writeback_dest(dest, dest_reg); + } + mach_release_all(&mctx); + return; + } + } + + /* 1. Determine dest register (allocate scratch for spills/param/no-reg). + * CMP and other flag-setting ops don't write a result register, so we + * use R0 as a dummy (Rd field is architecturally ignored). */ + int dest_reg; + if (dest_sets_flags) + dest_reg = R0; + else + dest_reg = mach_get_dest_reg(&mctx, dest, 0); + + uint32_t excl = thumb_is_hw_reg(dest_reg) ? (1u << (uint32_t)dest_reg) : 0; + + /* Exclude src2's register from scratch allocation for src1. + * Without this, materializing an immediate for src1 could pick src2's + * register, clobbering it before src2 is read. This applies whether + * src2 is a plain register or a dereferenced one (the address register + * must survive until the load). */ + if (src2->kind == MACH_OP_REG && thumb_is_hw_reg(src2->u.reg.r0)) + excl |= (1u << (uint32_t)src2->u.reg.r0); + + /* 2. Ensure src1 is in a register; add it to the exclusion mask. */ + int src1_reg = mach_ensure_in_reg(&mctx, src1, excl); + if (thumb_is_hw_reg(src1_reg)) + excl |= (1u << (uint32_t)src1_reg); + + /* CMP Rn, #-k (k = 1..7, Rn low) as ADDS Rt, Rn, #k into a low register + * free here: Rn - (-k) and Rn + k are one sum with one carry out and one + * overflow, so N, Z, C and V are the same, in 16 bits where the compare + * needs CMP.W (or CMN.W). Zig compares against maxInt(u32), its `none` + * index, 1,800 times in zig.c. The free register comes from liveness alone + * (not the prolog's pushed set), so the rehearsal decides the same way; + * never inside an IT block, where the 16-bit ADDS does not set flags. */ + if (dest_sets_flags && src2->kind == MACH_OP_IMM && !src2->needs_deref && !src2->is_64bit && + thumb_is_hw_reg(src1_reg) && src1_reg < 8 && pool_flush_it_pending == 0 && tcc_state->ir && cmp_adds_enabled()) + { + const int32_t v = (int32_t)src2->u.imm.val; + if (v >= -7 && v <= -1) + { + TCCIRState *ir = tcc_state->ir; + int rt = tcc_ls_find_free_scratch_reg(&ir->ls, ir->codegen_instruction_idx, + excl | scratch_global_exclude | 0xFF00u | (1u << (uint32_t)src1_reg), + ir->leaffunc); + if (rt != PREG_NONE && rt >= 0 && rt < 8) + { + ot_check(th_add_imm((uint32_t)rt, (uint32_t)src1_reg, (uint32_t)-v, FLAGS_BEHAVIOUR_SET, + ENFORCE_ENCODING_16BIT)); + mach_release_all(&mctx); + return; + } + } + } + + /* 3. Try immediate form for src2; fall back to register if needed. */ + bool imm_emitted = false; + int src2_reg = + mach_ensure_imm_or_reg(&mctx, src2, excl, handler.imm_handler, dest_reg, src1_reg, flags, &imm_emitted); + if (!imm_emitted) + { + /* Decode barrel shift annotation (0=none, else type<<5|amount). */ + thumb_shift sh = THUMB_SHIFT_DEFAULT; + if (barrel_shift != 0) + { + static const thumb_shift_type bs_map[] = { + [1] = THUMB_SHIFT_LSL, [2] = THUMB_SHIFT_LSR, + [3] = THUMB_SHIFT_ASR, [4] = THUMB_SHIFT_ROR, + }; + uint32_t stype = (barrel_shift >> 5) & 7; + uint32_t samt = barrel_shift & 31; + sh.type = bs_map[stype]; + sh.value = samt; + sh.mode = THUMB_SHIFT_IMMEDIATE; + } + thumb_enforce_encoding enc = (barrel_shift != 0) ? ENFORCE_ENCODING_32BIT : ENFORCE_ENCODING_NONE; + ot_check(thumb_call_reg_handler(handler.reg_handler, (uint32_t)dest_reg, (uint32_t)src1_reg, (uint32_t)src2_reg, + flags, sh, enc)); + } + + /* 4. Write result back to spill slot / stack param / pointer-dest. */ + if (!dest_sets_flags && dest && dest->kind != MACH_OP_NONE) + { + const bool needs_wb = dest->kind == MACH_OP_SPILL || dest->kind == MACH_OP_PARAM_STACK || + (dest->kind == MACH_OP_REG && (dest->needs_deref || dest->u.reg.r0 == (int)PREG_REG_NONE)); + if (needs_wb) + mach_writeback_dest(dest, dest_reg); + } + + /* 5. Release all scratches in LIFO order. */ + mach_release_all(&mctx); +} + +void tcc_gen_machine_data_processing_mop(MachineOperand src1, MachineOperand src2, MachineOperand dest, TccIrOp op, + uint32_t barrel_shift) +{ + data_processing_mop_impl(src1, src2, dest, op, flags_safe(), barrel_shift); +} + +void tcc_gen_machine_data_processing_mop_flags(MachineOperand src1, MachineOperand src2, MachineOperand dest, + TccIrOp op, uint32_t barrel_shift) +{ + data_processing_mop_impl(src1, src2, dest, op, FLAGS_BEHAVIOUR_SET, barrel_shift); +} + +static void data_processing_mop_impl(MachineOperand src1, MachineOperand src2, MachineOperand dest, TccIrOp op, + thumb_flags_behaviour flags_override, uint32_t barrel_shift) +{ + ThumbDataProcessingHandler handler; + ThumbDataProcessingHandler carry_handler; /* used for hi word of 64-bit ops */ + bool uses_carry = false; + /* CMP always sets flags — it has no non-flag-setting variant. + * Ignore FLAGS_BEHAVIOUR_BLOCK for CMP; it must always use SET. */ + thumb_flags_behaviour flags = (op == TCCIR_OP_CMP) ? FLAGS_BEHAVIOUR_SET : flags_override; + + switch (op) + { + case TCCIR_OP_ADD: + handler.imm_handler = th_add_imm; + handler.reg_handler = th_add_reg; + carry_handler.imm_handler = th_adc_imm; + carry_handler.reg_handler = th_adc_reg; + uses_carry = true; + break; + case TCCIR_OP_SUB: + handler.imm_handler = th_sub_imm; + handler.reg_handler = th_sub_reg; + carry_handler.imm_handler = th_sbc_imm; + carry_handler.reg_handler = th_sbc_reg; + uses_carry = true; + break; + case TCCIR_OP_CMP: + handler.imm_handler = th_cmp_imm_handler; + handler.reg_handler = th_cmp_reg; + carry_handler.imm_handler = th_sbc_imm; + carry_handler.reg_handler = th_sbc_reg; + uses_carry = true; + break; + case TCCIR_OP_SHL: + handler.imm_handler = th_lsl_imm; + handler.reg_handler = th_lsl_reg; + carry_handler = handler; + break; + case TCCIR_OP_SHR: + handler.imm_handler = th_lsr_imm; + handler.reg_handler = th_lsr_reg; + carry_handler = handler; + break; + case TCCIR_OP_SAR: + handler.imm_handler = th_asr_imm; + handler.reg_handler = th_asr_reg; + carry_handler = handler; + break; + case TCCIR_OP_ROR: + handler.imm_handler = th_ror_imm; + handler.reg_handler = th_ror_reg; + carry_handler = handler; + break; + case TCCIR_OP_OR: + handler.imm_handler = th_orr_imm; + handler.reg_handler = th_orr_reg; + carry_handler = handler; + break; + case TCCIR_OP_AND: + handler.imm_handler = th_and_imm; + handler.reg_handler = th_and_reg; + carry_handler = handler; + break; + case TCCIR_OP_XOR: + handler.imm_handler = th_eor_imm; + handler.reg_handler = th_eor_reg; + carry_handler = handler; + break; + case TCCIR_OP_ADC_GEN: + flags = FLAGS_BEHAVIOUR_SET; + /* fall through */ + case TCCIR_OP_ADC_USE: + handler.imm_handler = th_adc_imm; + handler.reg_handler = th_adc_reg; + carry_handler = handler; + break; + default: + tcc_error("compiler_error: tcc_gen_machine_data_processing_mop: unhandled op %d", (int)op); + return; + } + + /* Dispatch 64-bit pair destinations to the mop64 path. + * CMP has no dest (MACH_OP_NONE), so also check src1 for 64-bit. */ + if (dest.is_64bit || (op == TCCIR_OP_CMP && src1.is_64bit)) + { + if (op == TCCIR_OP_SHL || op == TCCIR_OP_SHR || op == TCCIR_OP_SAR) + { + /* Two independent reasons a half need not be written: shift64_dead_half's + * bitfield-extract rule (bits 16-17), and zero_half64 finding the half a + * provable zero that every consumer now ignores (bit 22/23). */ + const uint32_t zh = (barrel_shift >> 18) & 0x7Fu; + bool skip_lo = ((barrel_shift >> 16) & 1) || (zh & ZH64_D_LO) != 0; + bool skip_hi = ((barrel_shift >> 17) & 1) || (zh & ZH64_D_HI) != 0; + thumb_emit_shift64_mop(&src1, &src2, &dest, op, skip_lo, skip_hi); + } + else + thumb_emit_data_processing_mop64(&src1, &src2, &dest, op, handler, carry_handler, uses_carry, + (barrel_shift >> 18) & 0x7Fu); + return; + } + + thumb_emit_data_processing_mop32(&src1, &src2, &dest, op, handler, flags, barrel_shift & 0xFFFFu); +} + +/* tcc_gen_machine_ubfx_mop: emit UBFX Rd, Rn, #lsb, #width. + * src2 encodes lsb (bits 0-4) and width (bits 5-9). */ +void tcc_gen_machine_ubfx_mop(MachineOperand src1, MachineOperand src2, MachineOperand dest) +{ + MachineCodegenContext ctx = {0}; + int rd = mach_get_dest_reg(&ctx, &dest, 0); + uint32_t excl = (1u << (uint32_t)rd); + int param = (src2.kind == MACH_OP_IMM) ? (int)src2.u.imm.val : 0; + /* The field may live in the HIGH word of a 64-bit source: `(v >> 52) & 0x7FF` + * is one UBFX on that word, with no shift in front of it. */ + if ((param & UBFX_HI_HALF) && src1.is_64bit && !src1.needs_deref) + src1 = mach_make_hi_half(&src1); + int rn = mach_ensure_in_reg(&ctx, &src1, excl); + int lsb = param & 0x1F; + int width = (param >> 5) & 0x1F; + if (width == 0) + width = 8; + /* A low byte or halfword is UXTB / UXTH, which has a 16-bit form for low + * registers; UBFX is always 32 bits. */ + if (lsb == 0 && (width == 8 || width == 16)) + { + if (width == 8) + ot_check(th_uxtb((uint32_t)rd, (uint32_t)rn, THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); + else + ot_check(th_uxth((uint32_t)rd, (uint32_t)rn, THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); + mach_writeback_dest(&dest, rd); + mach_release_all(&ctx); + return; + } + int widthm1 = width - 1; + int imm3 = (lsb >> 2) & 0x7; + int imm2 = lsb & 0x3; + /* Thumb-2 UBFX encoding: 11110 0 11 1100 Rn | 0 imm3 Rd imm2 0 widthm1 */ + thumb_opcode op; + op.size = 4; + op.opcode = 0xF3C00000 | ((uint32_t)rn << 16) | ((uint32_t)imm3 << 12) | ((uint32_t)rd << 8) | ((uint32_t)imm2 << 6) | (uint32_t)widthm1; + ot(op); + mach_writeback_dest(&dest, rd); + mach_release_all(&ctx); +} + +/* tcc_machine_has_bit_ops: does the active core encode clz/rbit/rev/rev16? + * All four live in the Thumb-2 main extension (rev/rev16 also have a T16 form, + * but the front end needs one answer for the whole family, so require T32 + + * clz_rbit). */ +ST_FUNC int tcc_machine_has_bit_ops(void) +{ + return arm_target_dependent.feat.t32 && arm_target_dependent.feat.clz_rbit; +} + +/* tcc_gen_machine_bitop1_mop: emit a single-operand bit manipulation, + * dest = (src1), for TCCIR_OP_CLZ / RBIT / REV / REV16. */ +ST_FUNC void tcc_gen_machine_bitop1_mop(MachineOperand src1, MachineOperand dest, TccIrOp op) +{ + MachineCodegenContext ctx = {0}; + int rd = mach_get_dest_reg(&ctx, &dest, 0); + int rm = mach_ensure_in_reg(&ctx, &src1, (1u << (uint32_t)rd)); + switch (op) + { + case TCCIR_OP_CLZ: + ot_check(th_clz((uint32_t)rd, (uint32_t)rm)); + break; + case TCCIR_OP_RBIT: + ot_check(th_rbit((uint32_t)rd, (uint32_t)rm)); + break; + case TCCIR_OP_REV: + ot_check(th_rev((uint32_t)rd, (uint32_t)rm, ENFORCE_ENCODING_NONE)); + break; + case TCCIR_OP_REV16: + ot_check(th_rev16((uint32_t)rd, (uint32_t)rm, ENFORCE_ENCODING_NONE)); + break; + default: + tcc_error("compiler_error: tcc_gen_machine_bitop1_mop: unhandled op %d", (int)op); + break; + } + mach_writeback_dest(&dest, rd); + mach_release_all(&ctx); +} + +/* tcc_gen_machine_sbfx_mop: emit SBFX Rd, Rn, #lsb, #width. + * src2 encodes lsb (bits 0-4) and width (bits 5-9). Same field layout as UBFX; + * only the base opcode differs (0xF3400000 vs 0xF3C00000). */ +void tcc_gen_machine_sbfx_mop(MachineOperand src1, MachineOperand src2, MachineOperand dest) +{ + MachineCodegenContext ctx = {0}; + int rd = mach_get_dest_reg(&ctx, &dest, 0); + uint32_t excl = (1u << (uint32_t)rd); + int rn = mach_ensure_in_reg(&ctx, &src1, excl); + int param = (src2.kind == MACH_OP_IMM) ? (int)src2.u.imm.val : 0; + int lsb = param & 0x1F; + int width = (param >> 5) & 0x1F; + if (width == 0) + width = 8; + /* SXTB / SXTH: 16 bits for low registers, like UXTB / UXTH above. */ + if (lsb == 0 && (width == 8 || width == 16)) + { + if (width == 8) + ot_check(th_sxtb((uint32_t)rd, (uint32_t)rn, THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); + else + ot_check(th_sxth((uint32_t)rd, (uint32_t)rn, THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); + mach_writeback_dest(&dest, rd); + mach_release_all(&ctx); + return; + } + int widthm1 = width - 1; + int imm3 = (lsb >> 2) & 0x7; + int imm2 = lsb & 0x3; + /* Thumb-2 SBFX encoding: 11110 0 11 0100 Rn | 0 imm3 Rd imm2 0 widthm1 */ + thumb_opcode op; + op.size = 4; + op.opcode = 0xF3400000 | ((uint32_t)rn << 16) | ((uint32_t)imm3 << 12) | ((uint32_t)rd << 8) | ((uint32_t)imm2 << 6) | (uint32_t)widthm1; + ot(op); + mach_writeback_dest(&dest, rd); + mach_release_all(&ctx); +} + +/* tcc_gen_machine_bfi_mop: emit BFI Rd, Rn, #lsb, #width. + * src1 = host word (moved into Rd, the BFI base, if not already there), + * src2 = value supplying the field bits (only its low `width` bits are used), + * dest = result. params packs lsb (bits 0-7) and width (bits 8-15). */ +void tcc_gen_machine_bfi_mop(MachineOperand src1, MachineOperand src2, MachineOperand dest, uint32_t params) +{ + MachineCodegenContext ctx = {0}; + int rd = mach_get_dest_reg(&ctx, &dest, 0); + int rn = mach_ensure_in_reg(&ctx, &src2, 0); /* value (Rn) */ + int rword = mach_ensure_in_reg(&ctx, &src1, (1u << (uint32_t)rn)); /* host word */ + /* Establish Rd = host word. If the value happens to live in Rd (RA coalesced + * the result onto src2), preserve it in a scratch before clobbering Rd. */ + if (rd != rword) + { + if (rd == rn) + { + int tmp = mach_alloc_scratch(&ctx, (1u << (uint32_t)rd) | (1u << (uint32_t)rword)); + ot_check_mov_reg((uint32_t)tmp, (uint32_t)rd, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE, false); + rn = tmp; + } + ot_check_mov_reg((uint32_t)rd, (uint32_t)rword, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE, false); + } + int lsb = (int)(params & 0xFF); + int width = (int)((params >> 8) & 0xFF); + if (width < 1) + width = 1; + int msb = lsb + width - 1; + if (msb > 31) + msb = 31; + int imm3 = (lsb >> 2) & 0x7; + int imm2 = lsb & 0x3; + /* Thumb-2 BFI: 11110 0 11 0110 Rn | 0 imm3 Rd imm2 0 msb */ + thumb_opcode op; + op.size = 4; + op.opcode = 0xF3600000 | ((uint32_t)rn << 16) | ((uint32_t)imm3 << 12) | ((uint32_t)rd << 8) | ((uint32_t)imm2 << 6) | (uint32_t)msb; + ot(op); + mach_writeback_dest(&dest, rd); + mach_release_all(&ctx); +} + +/* ============================================================ + * MachineOperand-based mul/div/mod/test-zero (_mop path) + * ============================================================ + * Internal helpers and public entry point for 32-bit register-only ops: + * MUL, DIV, UDIV — simple rd = rn OP rm + * IMOD, UMOD — dest = src1 - (src1/src2)*src2 + * TEST_ZERO — CMP src, #0 (flags only, no dest) + * MLA (accumulator) and UMULL (64-bit pair) remain on the old IR path. + */ + +/* Emit rd = emitter(src1, src2) for register-only 3-operand ops. */ +static void mach_regonly_binop_mop(MachineCodegenContext *ctx, const MachineOperand *src1, const MachineOperand *src2, + const MachineOperand *dest, thumb_regonly3_handler_t emitter) +{ + /* 1. Get dest register (scratch if spill/param). */ + int dest_reg = mach_get_dest_reg(ctx, dest, 0); + uint32_t excl = thumb_is_hw_reg(dest_reg) ? (1u << (uint32_t)dest_reg) : 0; + + /* Pre-exclude src2's physical register so that loading src1 (which may + * need a scratch for deref) does not clobber src2's value. This must apply + * even when src2 is a DEREF operand: its register holds the pointer, and the + * step-3 load reads through that same register, so it has to survive src1's + * materialization. Omitting the deref case let src1's scratch land on the + * pointer register at high pressure -> `u6 * arr10[3]` dereferenced u6 as a + * pointer -> bus fault (fuzz switch 219754). Matches the data-processing + * guard in thumb_emit_data_processing_mop32. */ + if (src2->kind == MACH_OP_REG && thumb_is_hw_reg(src2->u.reg.r0)) + excl |= (1u << (uint32_t)src2->u.reg.r0); + + /* 2. Ensure src1 in a register; extend exclusion mask. */ + int src1_reg = mach_ensure_in_reg(ctx, src1, excl); + if (thumb_is_hw_reg(src1_reg)) + excl |= (1u << (uint32_t)src1_reg); + + /* 3. Ensure src2 in a register. */ + int src2_reg = mach_ensure_in_reg(ctx, src2, excl); + + /* 4. Emit instruction. */ + ot_check(emitter((uint32_t)dest_reg, (uint32_t)src1_reg, (uint32_t)src2_reg)); + + /* 5. Write result back to spill slot / stack param if needed. */ + mach_writeback_dest(dest, dest_reg); +} + +/* Emit dest = src1 - (src1/src2)*src2 for IMOD/UMOD. */ +static void mach_mod_mop(MachineCodegenContext *ctx, const MachineOperand *src1, const MachineOperand *src2, + const MachineOperand *dest, thumb_regonly3_handler_t div_emitter) +{ + /* 1. Get dest register. */ + int dest_reg = mach_get_dest_reg(ctx, dest, 0); + uint32_t excl = thumb_is_hw_reg(dest_reg) ? (1u << (uint32_t)dest_reg) : 0; + + /* Pre-exclude src2's physical register so that materializing src1 (which may + * need a scratch when it is an immediate or a deref) does not clobber src2's + * value before the divide reads it — same guard as mach_regonly_binop_mop. + * Without it an immediate dividend's scratch load could land on the divisor's + * register (random-C O1 wrong-code, seed 151: `K % (lr|1)` divisor clobbered). + * Applies to the deref case too (pointer register must survive src1's + * materialization, same as mach_regonly_binop_mop / fuzz switch 219754). */ + if (src2->kind == MACH_OP_REG && thumb_is_hw_reg(src2->u.reg.r0)) + excl |= (1u << (uint32_t)src2->u.reg.r0); + + /* 2. Ensure src1 in a register. */ + int src1_reg = mach_ensure_in_reg(ctx, src1, excl); + if (thumb_is_hw_reg(src1_reg)) + excl |= (1u << (uint32_t)src1_reg); + + /* 3. Ensure src2 in a register. */ + int src2_reg = mach_ensure_in_reg(ctx, src2, excl); + if (thumb_is_hw_reg(src2_reg)) + excl |= (1u << (uint32_t)src2_reg); + + /* 4. Scratch register for quotient. */ + int quotient_reg = mach_alloc_scratch(ctx, excl); + + /* 5. quotient = src1 / src2 */ + ot_check(div_emitter((uint32_t)quotient_reg, (uint32_t)src1_reg, (uint32_t)src2_reg)); + + /* 6. quotient = quotient * src2 */ + ot_check(thumb_mul_regonly((uint32_t)quotient_reg, (uint32_t)quotient_reg, (uint32_t)src2_reg)); + + /* 7. dest = src1 - quotient */ + ot_check(th_sub_reg(dest_reg, src1_reg, quotient_reg, flags_safe(), THUMB_SHIFT_DEFAULT, + ENFORCE_ENCODING_NONE)); + + /* 8. Write result back. */ + mach_writeback_dest(dest, dest_reg); +} + +/* thumb_emit_mul64_mop + * ============================================================ + * Emit a 64-bit multiply (lower 64 bits of the result) using MachineOperands. + * + * For a 64-bit result (dest->is_64bit): + * UMULL r_c_lo, r_c_hi, r_a_lo, r_b_lo // a_lo * b_lo → 64-bit unsigned + * MLA r_c_hi, r_a_hi, r_b_lo, r_c_hi // cross product (when src1 is 64-bit) + * MLA r_c_hi, r_a_lo, r_b_hi, r_c_hi // cross product (when src2 is 64-bit) + * + * For a 32-bit result with 64-bit source(s): + * MUL r_c, r_a_lo, r_b_lo // upper bits don't contribute + * + * The lower 64 bits of the signed / unsigned 128-bit product are identical + * (i.e. UMULL is correct for both signed and unsigned long long mul). + * The caller must call mach_release_all() after this function returns. + */ +static void thumb_emit_mul64_mop(MachineCodegenContext *ctx, const MachineOperand *src1, const MachineOperand *src2, + const MachineOperand *dest) +{ + uint32_t excl = 0; + + /* Resolve deref'd 64-bit sources before splitting into halves. */ + MachineOperand r_s1 = mach_resolve_deref_64(ctx, src1, &excl); + MachineOperand r_s2 = mach_resolve_deref_64(ctx, src2, &excl); + + /* Load lo halves (always needed). */ + MachineOperand a_lo_op = r_s1.is_64bit ? mach_make_lo_half(&r_s1) : r_s1; + a_lo_op.btype = IROP_BTYPE_INT32; + a_lo_op.is_64bit = false; + MachineOperand b_lo_op = r_s2.is_64bit ? mach_make_lo_half(&r_s2) : r_s2; + b_lo_op.btype = IROP_BTYPE_INT32; + b_lo_op.is_64bit = false; + + int r_a_lo = mach_ensure_in_reg(ctx, &a_lo_op, excl); + if (thumb_is_hw_reg(r_a_lo)) + excl |= (1u << (uint32_t)r_a_lo); + int r_b_lo = mach_ensure_in_reg(ctx, &b_lo_op, excl); + if (thumb_is_hw_reg(r_b_lo)) + excl |= (1u << (uint32_t)r_b_lo); + + if (dest->is_64bit) + { + /* Load hi halves for cross-product MLA terms. */ + int r_a_hi = PREG_REG_NONE, r_b_hi = PREG_REG_NONE; + if (r_s1.is_64bit) + { + MachineOperand a_hi_op = mach_make_hi_half(&r_s1); + a_hi_op.btype = IROP_BTYPE_INT32; + r_a_hi = mach_ensure_in_reg(ctx, &a_hi_op, excl); + if (thumb_is_hw_reg(r_a_hi)) + excl |= (1u << (uint32_t)r_a_hi); + } + if (r_s2.is_64bit) + { + MachineOperand b_hi_op = mach_make_hi_half(&r_s2); + b_hi_op.btype = IROP_BTYPE_INT32; + r_b_hi = mach_ensure_in_reg(ctx, &b_hi_op, excl); + if (thumb_is_hw_reg(r_b_hi)) + excl |= (1u << (uint32_t)r_b_hi); + } + + /* Allocate 64-bit destination pair — must not overlap sources for UMULL. */ + MachineOperand dst_lo_op = mach_make_lo_half(dest); + dst_lo_op.btype = IROP_BTYPE_INT32; + MachineOperand dst_hi_op = mach_make_hi_half(dest); + dst_hi_op.btype = IROP_BTYPE_INT32; + + int r_c_lo = mach_get_dest_reg(ctx, &dst_lo_op, excl); + if (thumb_is_hw_reg(r_c_lo)) + excl |= (1u << (uint32_t)r_c_lo); + int r_c_hi = mach_get_dest_reg(ctx, &dst_hi_op, excl); + + /* UMULL: r_c_lo:r_c_hi = r_a_lo * r_b_lo (unsigned 64-bit product) */ + ot_check(th_umull((uint32_t)r_c_lo, (uint32_t)r_c_hi, (uint32_t)r_a_lo, (uint32_t)r_b_lo)); + + /* Add cross products to high half. */ + if (thumb_is_hw_reg(r_a_hi)) + ot_check(th_mla((uint32_t)r_c_hi, (uint32_t)r_a_hi, (uint32_t)r_b_lo, (uint32_t)r_c_hi)); + if (thumb_is_hw_reg(r_b_hi)) + ot_check(th_mla((uint32_t)r_c_hi, (uint32_t)r_a_lo, (uint32_t)r_b_hi, (uint32_t)r_c_hi)); + + mach_writeback_dest(&dst_lo_op, r_c_lo); + mach_writeback_dest(&dst_hi_op, r_c_hi); + } + else + { + /* 32-bit result with 64-bit source(s): only the low bits matter. */ + MachineOperand dest32 = *dest; + dest32.is_64bit = false; + int r_c = mach_get_dest_reg(ctx, &dest32, excl); + ot_check(thumb_mul_regonly((uint32_t)r_c, (uint32_t)r_a_lo, (uint32_t)r_b_lo)); + mach_writeback_dest(&dest32, r_c); + } +} + +/* tcc_gen_machine_muldiv_mop: MachineOperand-based entry point for multiply, + * divide, modulo, and test-zero operations. Called from ir/codegen.c when + * use_mop_muldiv is true for: + * MUL — 32-bit or 64-bit multiply + * DIV, UDIV, IMOD, UMOD — 32-bit divide/modulo + * TEST_ZERO — 32-bit or 64-bit compare against zero (flags only) + * MLA (accumulator; 4-operand) uses tcc_gen_machine_mla_mop. + * UMULL (64-bit output from 32-bit inputs) uses tcc_gen_machine_umull_mop. + */ +/* Decompose multiply-by-constant into shift+add sequences. + * Returns 1 if handled, 0 to fall back to hardware MUL. */ +static int thumb_try_mul_by_const_mop(MachineCodegenContext *ctx, MachineOperand *src1, MachineOperand *src2, + MachineOperand *dest) +{ + /* Identify which operand is the immediate and which is the variable. */ + const MachineOperand *imm_op, *var_op; + if (src2->kind == MACH_OP_IMM) + { + imm_op = src2; + var_op = src1; + } + else if (src1->kind == MACH_OP_IMM) + { + imm_op = src1; + var_op = src2; + } + else + return 0; + + int64_t c = imm_op->u.imm.val; + if (c <= 0) + return 0; + + /* Determine the decomposition pattern. + * We handle: powers of 2, (2^n ± 1), and products thereof. + * + * Pattern Insns Example + * ─────────────────── ───── ─────── + * 2^n 1 LSL Rd, Rn, #n + * 2^n + 1 1 ADD Rd, Rn, Rn LSL #n + * 2^n - 1 1 SUB Rd, Rn LSL #n, Rn (RSB-like via SUB) + * (2^a + 1) * 2^b 2 ADD Rd, Rn, Rn LSL #a; LSL Rd, Rd, #b + * (2^a - 1) * 2^b 2 SUB Rd, Rn LSL #a, Rn; LSL Rd, Rd, #b + * (2^a + 1)(2^b + 1) 2 ADD Rd, Rn, Rn LSL #a; ADD Rd, Rd, Rn LSL #(a+b) + * — only some cases, handled via table + */ + + int shift1 = 0, shift2 = 0; + enum + { + MUL_NONE, + MUL_POWER_OF_2, /* c = 2^n : LSL #n */ + MUL_TWO_N_PLUS_1, /* c = 2^n+1 : ADD Rd, Rn, Rn LSL #n */ + MUL_TWO_N_MINUS_1, /* c = 2^n-1 : SUB Rd, Rn LSL #n, Rn */ + MUL_TWO_N_PLUS_1_SHIFT, /* c = (2^a+1)*2^b : ADD; LSL */ + MUL_TWO_N_MINUS_1_SHIFT, /* c = (2^a-1)*2^b : SUB; LSL */ + } pattern = MUL_NONE; + + /* Check for power of 2 */ + if (c > 0 && (c & (c - 1)) == 0) + { + int n = 0; + int64_t v = c; + while (v > 1) + { + n++; + v >>= 1; + } + if (n >= 1 && n <= 31) + { + shift1 = n; + pattern = MUL_POWER_OF_2; + } + } + + /* Check for 2^n + 1 (3, 5, 9, 17, ...) */ + if (pattern == MUL_NONE && c >= 3) + { + int64_t v = c - 1; + if (v > 0 && (v & (v - 1)) == 0) + { + int n = 0; + while (v > 1) + { + n++; + v >>= 1; + } + if (n >= 1 && n <= 31) + { + shift1 = n; + pattern = MUL_TWO_N_PLUS_1; + } + } + } + + /* Check for 2^n - 1 (7, 15, 31, ...) */ + if (pattern == MUL_NONE && c >= 7) + { + int64_t v = c + 1; + if (v > 0 && (v & (v - 1)) == 0) + { + int n = 0; + while (v > 1) + { + n++; + v >>= 1; + } + if (n >= 2 && n <= 31) + { + shift1 = n; + pattern = MUL_TWO_N_MINUS_1; + } + } + } + + /* Check for (2^a + 1) * 2^b (6, 10, 12, 20, 24, 40, 48, ...) */ + if (pattern == MUL_NONE && c >= 6) + { + int64_t v = c; + int b = 0; + while ((v & 1) == 0) + { + b++; + v >>= 1; + } + if (b >= 1 && b <= 31) + { + int64_t inner = v - 1; + if (inner > 0 && (inner & (inner - 1)) == 0) + { + int a = 0; + while (inner > 1) + { + a++; + inner >>= 1; + } + if (a >= 1 && a <= 31) + { + shift1 = a; + shift2 = b; + pattern = MUL_TWO_N_PLUS_1_SHIFT; + } + } + } + } + + /* Check for (2^a - 1) * 2^b (14, 28, 30, 56, 60, 62, ...) */ + if (pattern == MUL_NONE && c >= 14) + { + int64_t v = c; + int b = 0; + while ((v & 1) == 0) + { + b++; + v >>= 1; + } + if (b >= 1 && b <= 31) + { + int64_t inner = v + 1; + if (inner > 0 && (inner & (inner - 1)) == 0) + { + int a = 0; + while (inner > 1) + { + a++; + inner >>= 1; + } + if (a >= 2 && a <= 31) + { + shift1 = a; + shift2 = b; + pattern = MUL_TWO_N_MINUS_1_SHIFT; + } + } + } + } + + if (pattern == MUL_NONE) + return 0; + + /* Emit the decomposed sequence. */ + int dest_reg = mach_get_dest_reg(ctx, dest, 0); + uint32_t excl = thumb_is_hw_reg(dest_reg) ? (1u << (uint32_t)dest_reg) : 0; + int var_reg = mach_ensure_in_reg(ctx, var_op, excl); + thumb_flags_behaviour fl = flags_safe(); + thumb_shift sh; + + switch (pattern) + { + case MUL_POWER_OF_2: + ot_check(th_lsl_imm((uint32_t)dest_reg, (uint32_t)var_reg, (uint32_t)shift1, fl, ENFORCE_ENCODING_NONE)); + break; + + case MUL_TWO_N_PLUS_1: + sh = (thumb_shift){THUMB_SHIFT_LSL, (uint16_t)shift1, THUMB_SHIFT_IMMEDIATE}; + ot_check(th_add_reg((uint32_t)dest_reg, (uint32_t)var_reg, (uint32_t)var_reg, fl, sh, ENFORCE_ENCODING_NONE)); + break; + + case MUL_TWO_N_MINUS_1: + { + /* Thumb-2 SUB Rd, Rn, Rm LSL #n = Rn - (Rm << n). + * We need (var << n) - var, which is the reverse. No RSB with shift + * exists in Thumb-2, so we do: LSL tmp, var, #n; SUB Rd, tmp, var. + * The LSL destination must differ from var_reg, otherwise it destroys + * var before the SUB reads it (mach_ensure_in_reg returns an already- + * resident var in dest_reg's register, ignoring the exclusion mask, so + * dest_reg == var_reg is reachable). Shift straight into dest when they + * differ; otherwise borrow a scratch. */ + int tmp = (dest_reg == var_reg) ? mach_alloc_scratch(ctx, (1u << (uint32_t)var_reg)) : dest_reg; + ot_check(th_lsl_imm((uint32_t)tmp, (uint32_t)var_reg, (uint32_t)shift1, fl, ENFORCE_ENCODING_NONE)); + sh = (thumb_shift){THUMB_SHIFT_NONE, 0, THUMB_SHIFT_IMMEDIATE}; + ot_check(th_sub_reg((uint32_t)dest_reg, (uint32_t)tmp, (uint32_t)var_reg, fl, sh, ENFORCE_ENCODING_NONE)); + break; + } + + case MUL_TWO_N_PLUS_1_SHIFT: + sh = (thumb_shift){THUMB_SHIFT_LSL, (uint16_t)shift1, THUMB_SHIFT_IMMEDIATE}; + ot_check(th_add_reg((uint32_t)dest_reg, (uint32_t)var_reg, (uint32_t)var_reg, fl, sh, ENFORCE_ENCODING_NONE)); + ot_check(th_lsl_imm((uint32_t)dest_reg, (uint32_t)dest_reg, (uint32_t)shift2, fl, ENFORCE_ENCODING_NONE)); + break; + + case MUL_TWO_N_MINUS_1_SHIFT: + { + /* (2^a - 1) * 2^b: LSL tmp, var, #a; SUB tmp, tmp, var; LSL Rd, tmp, #b. + * As in MUL_TWO_N_MINUS_1, the first LSL must not target var_reg, or it + * destroys var before the SUB reads it. */ + int tmp = (dest_reg == var_reg) ? mach_alloc_scratch(ctx, (1u << (uint32_t)var_reg)) : dest_reg; + ot_check(th_lsl_imm((uint32_t)tmp, (uint32_t)var_reg, (uint32_t)shift1, fl, ENFORCE_ENCODING_NONE)); + sh = (thumb_shift){THUMB_SHIFT_NONE, 0, THUMB_SHIFT_IMMEDIATE}; + ot_check(th_sub_reg((uint32_t)tmp, (uint32_t)tmp, (uint32_t)var_reg, fl, sh, ENFORCE_ENCODING_NONE)); + ot_check(th_lsl_imm((uint32_t)dest_reg, (uint32_t)tmp, (uint32_t)shift2, fl, ENFORCE_ENCODING_NONE)); + break; + } + + default: + return 0; + } + + mach_writeback_dest(dest, dest_reg); + mach_release_all(ctx); + return 1; +} + +/* Fused MUL-by-const + ADD peephole. + * Transforms: tmp = var * C; dest = base + tmp + * Into a shorter sequence using ARM shifted-add (ADD Rd, Rn, Rm LSL #imm): + * C = 2^n: ADD dest, base, var LSL #n (1 insn vs 2) + * C = (2^a+1)*2^b: ADD t, var, var LSL #a; + * ADD dest, base, t LSL #b (2 insn vs 3) + * C = (2^a-1)*2^b: LSL t, var, #a; SUB t, t, var; + * ADD dest, base, t LSL #b (3 insn vs 4) + * Returns 1 if fused, 0 to fall back to separate MUL + ADD. */ +ST_FUNC int tcc_gen_machine_mul_const_add_fused_mop(MachineOperand mul_var, int64_t mul_const, + MachineOperand mul_dest, MachineOperand add_base, + MachineOperand add_dest) +{ + if (mul_const <= 0) + return 0; + + int shift1 = 0, shift2 = 0; + enum + { + FUSE_NONE, + FUSE_POW2, + FUSE_TWO_N_PLUS_1_SHIFT, + FUSE_TWO_N_MINUS_1_SHIFT, + } pattern = FUSE_NONE; + + /* Power of 2: C = 2^n */ + if (mul_const > 1 && (mul_const & (mul_const - 1)) == 0) + { + int n = 0; + int64_t v = mul_const; + while (v > 1) { n++; v >>= 1; } + if (n >= 1 && n <= 31) + { + shift1 = n; + pattern = FUSE_POW2; + } + } + + /* (2^a + 1) * 2^b: e.g. 12 = 3*4 = (2^1+1)*2^2 */ + if (pattern == FUSE_NONE && mul_const >= 6) + { + int64_t v = mul_const; + int b = 0; + while ((v & 1) == 0) { b++; v >>= 1; } + if (b >= 1 && b <= 31) + { + int64_t inner = v - 1; + if (inner > 0 && (inner & (inner - 1)) == 0) + { + int a = 0; + while (inner > 1) { a++; inner >>= 1; } + if (a >= 1 && a <= 31) + { + shift1 = a; + shift2 = b; + pattern = FUSE_TWO_N_PLUS_1_SHIFT; + } + } + } + } + + /* (2^a - 1) * 2^b: e.g. 28 = 7*4 = (2^3-1)*2^2 */ + if (pattern == FUSE_NONE && mul_const >= 14) + { + int64_t v = mul_const; + int b = 0; + while ((v & 1) == 0) { b++; v >>= 1; } + if (b >= 1 && b <= 31) + { + int64_t inner = v + 1; + if (inner > 0 && (inner & (inner - 1)) == 0) + { + int a = 0; + while (inner > 1) { a++; inner >>= 1; } + if (a >= 2 && a <= 31) + { + shift1 = a; + shift2 = b; + pattern = FUSE_TWO_N_MINUS_1_SHIFT; + } + } + } + } + + if (pattern == FUSE_NONE) + return 0; + + MachineCodegenContext ctx = {0}; + thumb_flags_behaviour fl = flags_safe(); + thumb_shift sh; + + /* Allocate registers: dest first (may hint to the ADD dest's phys reg), + * then base and var, using exclusion masks to prevent conflicts. */ + int dest_reg = mach_get_dest_reg(&ctx, &add_dest, 0); + uint32_t excl = thumb_is_hw_reg(dest_reg) ? (1u << (uint32_t)dest_reg) : 0; + int base_reg = mach_ensure_in_reg(&ctx, &add_base, excl); + if (thumb_is_hw_reg(base_reg)) + excl |= (1u << (uint32_t)base_reg); + int var_reg = mach_ensure_in_reg(&ctx, &mul_var, excl); + + switch (pattern) + { + case FUSE_POW2: + /* ADD dest, base, var LSL #n */ + sh = (thumb_shift){THUMB_SHIFT_LSL, (uint16_t)shift1, THUMB_SHIFT_IMMEDIATE}; + ot_check(th_add_reg((uint32_t)dest_reg, (uint32_t)base_reg, (uint32_t)var_reg, fl, sh, ENFORCE_ENCODING_NONE)); + break; + + case FUSE_TWO_N_PLUS_1_SHIFT: + { + /* Step 1: ADD tmp, var, var LSL #a */ + if (thumb_is_hw_reg(var_reg)) + excl |= (1u << (uint32_t)var_reg); + int tmp_reg = mach_get_dest_reg(&ctx, &mul_dest, excl); + sh = (thumb_shift){THUMB_SHIFT_LSL, (uint16_t)shift1, THUMB_SHIFT_IMMEDIATE}; + ot_check(th_add_reg((uint32_t)tmp_reg, (uint32_t)var_reg, (uint32_t)var_reg, fl, sh, ENFORCE_ENCODING_NONE)); + /* Step 2: ADD dest, base, tmp LSL #b */ + sh = (thumb_shift){THUMB_SHIFT_LSL, (uint16_t)shift2, THUMB_SHIFT_IMMEDIATE}; + ot_check(th_add_reg((uint32_t)dest_reg, (uint32_t)base_reg, (uint32_t)tmp_reg, fl, sh, ENFORCE_ENCODING_NONE)); + break; + } + + case FUSE_TWO_N_MINUS_1_SHIFT: + { + /* Step 1: LSL tmp, var, #a */ + if (thumb_is_hw_reg(var_reg)) + excl |= (1u << (uint32_t)var_reg); + int tmp_reg = mach_get_dest_reg(&ctx, &mul_dest, excl); + ot_check(th_lsl_imm((uint32_t)tmp_reg, (uint32_t)var_reg, (uint32_t)shift1, fl, ENFORCE_ENCODING_NONE)); + /* Step 2: SUB tmp, tmp, var */ + sh = (thumb_shift){THUMB_SHIFT_NONE, 0, THUMB_SHIFT_IMMEDIATE}; + ot_check(th_sub_reg((uint32_t)tmp_reg, (uint32_t)tmp_reg, (uint32_t)var_reg, fl, sh, ENFORCE_ENCODING_NONE)); + /* Step 3: ADD dest, base, tmp LSL #b */ + sh = (thumb_shift){THUMB_SHIFT_LSL, (uint16_t)shift2, THUMB_SHIFT_IMMEDIATE}; + ot_check(th_add_reg((uint32_t)dest_reg, (uint32_t)base_reg, (uint32_t)tmp_reg, fl, sh, ENFORCE_ENCODING_NONE)); + break; + } + + default: + mach_release_all(&ctx); + return 0; + } + + mach_writeback_dest(&add_dest, dest_reg); + mach_release_all(&ctx); + return 1; +} + +ST_FUNC void tcc_gen_machine_muldiv_mop(MachineOperand src1, MachineOperand src2, MachineOperand dest, TccIrOp op) +{ + MachineCodegenContext ctx = {0}; + switch (op) + { + case TCCIR_OP_MUL: + if (src1.is_64bit || src2.is_64bit || dest.is_64bit) + thumb_emit_mul64_mop(&ctx, &src1, &src2, &dest); + else if (!thumb_try_mul_by_const_mop(&ctx, &src1, &src2, &dest)) + mach_regonly_binop_mop(&ctx, &src1, &src2, &dest, thumb_mul_regonly); + break; + case TCCIR_OP_DIV: + mach_regonly_binop_mop(&ctx, &src1, &src2, &dest, thumb_sdiv_regonly); + break; + case TCCIR_OP_UDIV: + mach_regonly_binop_mop(&ctx, &src1, &src2, &dest, thumb_udiv_regonly); + break; + case TCCIR_OP_IMOD: + mach_mod_mop(&ctx, &src1, &src2, &dest, thumb_sdiv_regonly); + break; + case TCCIR_OP_UMOD: + mach_mod_mop(&ctx, &src1, &src2, &dest, thumb_udiv_regonly); + break; + case TCCIR_OP_TEST_ZERO: + { + if (src1.is_64bit) + { + /* 64-bit: Z set iff (lo == 0 && hi == 0). + * Use CMP lo,#0; IT EQ; CMPEQ hi,#0 to avoid clobbering source registers. */ + uint32_t excl = 0; + MachineOperand resolved = mach_resolve_deref_64(&ctx, &src1, &excl); + MachineOperand lo = mach_make_lo_half(&resolved); + lo.btype = IROP_BTYPE_INT32; + MachineOperand hi = mach_make_hi_half(&resolved); + hi.btype = IROP_BTYPE_INT32; + int r_lo = mach_ensure_in_reg(&ctx, &lo, excl); + if (thumb_is_hw_reg(r_lo)) + excl |= (1u << (uint32_t)r_lo); + int r_hi = mach_ensure_in_reg(&ctx, &hi, excl); + if (!thumb_try_orrs_zero64(r_lo, r_hi)) + { + ot_check(th_cmp_imm(r_lo, 0, FLAGS_BEHAVIOUR_SET, ENFORCE_ENCODING_NONE)); + th_literal_pool_reserve_upcoming_bytes(6); + ot_check(th_it(mapcc(TOK_EQ), 0x8)); /* IT EQ (single instruction) */ + ot_check(th_cmp_imm(r_hi, 0, FLAGS_BEHAVIOUR_SET, ENFORCE_ENCODING_NONE)); + } + } + else + { + /* 32-bit: CMP src, #0 — no destination, only flags. */ + int src_reg = mach_ensure_in_reg(&ctx, &src1, 0); + ot_check(th_cmp_imm(src_reg, 0, FLAGS_BEHAVIOUR_SET, ENFORCE_ENCODING_NONE)); + } + break; + } + default: + tcc_error("compiler_error: tcc_gen_machine_muldiv_mop: unhandled op %d", (int)op); + break; + } + mach_release_all(&ctx); +} + +/* tcc_gen_machine_cmp_eq64_mop: 64-bit equality comparison. + * Emits CMP hi1,hi2; IT EQ; CMPEQ lo1,lo2 which correctly sets + * the Z flag for full 64-bit equality (used by SETIF/JUMPIF EQ/NE). */ +ST_FUNC void tcc_gen_machine_cmp_eq64_mop(MachineOperand src1, MachineOperand src2) +{ + MachineCodegenContext ctx = {0}; + uint32_t excl = 0; + + if (src1.kind == MACH_OP_REG) + { + if (src1.u.reg.r0 != (int)PREG_REG_NONE) + excl |= (1u << (uint32_t)src1.u.reg.r0); + if (!src1.needs_deref && src1.is_64bit && src1.u.reg.r1 >= 0) + excl |= (1u << (uint32_t)src1.u.reg.r1); + } + if (src2.kind == MACH_OP_REG) + { + if (src2.u.reg.r0 != (int)PREG_REG_NONE) + excl |= (1u << (uint32_t)src2.u.reg.r0); + if (!src2.needs_deref && src2.is_64bit && src2.u.reg.r1 >= 0) + excl |= (1u << (uint32_t)src2.u.reg.r1); + } + + MachineOperand r_src1 = mach_resolve_deref_64(&ctx, &src1, &excl); + MachineOperand r_src2 = mach_resolve_deref_64(&ctx, &src2, &excl); + + int rn_lo, rn_hi; + mach_ensure_pair_in_regs(&ctx, &r_src1, &excl, &rn_lo, &rn_hi); + + /* Immediate-CMP fast path: if src2 is a u64 immediate, try the cmp-imm + * form (`cmp.w Rn, #imm`) for each half — avoids loading the constant + * into a scratch reg. Probe encodability before allocating scratches: + * `mach_ensure_in_reg` on a MACH_OP_IMM would unconditionally emit a + * `movs Rscratch, #imm`, which is exactly the instruction we're trying + * to avoid here. */ + thumb_opcode hi_imm_op = {0}; + thumb_opcode lo_imm_op = {0}; + if (r_src2.kind == MACH_OP_IMM) + { + const uint64_t imm = (uint64_t)r_src2.u.imm.val; + const uint32_t imm_lo = (uint32_t)(imm & 0xffffffffu); + const uint32_t imm_hi = (uint32_t)(imm >> 32); + hi_imm_op = th_cmp_imm((uint32_t)rn_hi, imm_hi, FLAGS_BEHAVIOUR_SET, ENFORCE_ENCODING_NONE); + lo_imm_op = th_cmp_imm((uint32_t)rn_lo, imm_lo, FLAGS_BEHAVIOUR_SET, ENFORCE_ENCODING_NONE); + } + + /* Use cmp-imm for whichever halves fit; only allocate scratch + * registers for halves that need them. */ + int hi_uses_imm = (r_src2.kind == MACH_OP_IMM && hi_imm_op.size); + int lo_uses_imm = (r_src2.kind == MACH_OP_IMM && lo_imm_op.size); + + /* Comparing against literal zero is `(lo | hi) == 0`, one ORRS. */ + if (r_src2.kind == MACH_OP_IMM && (uint64_t)r_src2.u.imm.val == 0 && + thumb_try_orrs_zero64(rn_lo, rn_hi)) + { + mach_release_all(&ctx); + return; + } + + int rm_lo = 0, rm_hi = 0; + if (!lo_uses_imm && !hi_uses_imm) + { + /* Both halves are needed in registers — pair them (the immediate fast path + * above is the only case that wants one half loaded and not the other). */ + mach_ensure_pair_in_regs(&ctx, &r_src2, &excl, &rm_lo, &rm_hi); + } + else + { + if (!lo_uses_imm) + { + MachineOperand s2_lo = mach_make_lo_half(&r_src2); + s2_lo.btype = IROP_BTYPE_INT32; + rm_lo = mach_ensure_in_reg(&ctx, &s2_lo, excl); + if (thumb_is_hw_reg(rm_lo)) + excl |= (1u << (uint32_t)rm_lo); + } + if (!hi_uses_imm) + { + MachineOperand s2_hi = mach_make_hi_half(&r_src2); + s2_hi.btype = IROP_BTYPE_INT32; + rm_hi = mach_ensure_in_reg(&ctx, &s2_hi, excl); + } + } + + if (hi_uses_imm) + ot_check(hi_imm_op); + else + ot_check(th_cmp_reg(0, (uint32_t)rn_hi, (uint32_t)rm_hi, FLAGS_BEHAVIOUR_SET, THUMB_SHIFT_DEFAULT, + ENFORCE_ENCODING_NONE)); + th_literal_pool_reserve_upcoming_bytes(6); + ot_check(th_it(mapcc(TOK_EQ), 0x8)); + if (lo_uses_imm) + ot_check(lo_imm_op); + else + ot_check(th_cmp_reg(0, (uint32_t)rn_lo, (uint32_t)rm_lo, FLAGS_BEHAVIOUR_SET, THUMB_SHIFT_DEFAULT, + ENFORCE_ENCODING_NONE)); + + mach_release_all(&ctx); +} + +/* tcc_gen_machine_subs_eq_select_01: emit + * SUBS dest, src1, #K + * IT NE + * MOVNE dest, #1 + * for the CMP src1,#K + SELECT(#1,#0,NE) / SELECT(#0,#1,EQ) peephole. + * Returns 1 if emitted, 0 if the SUBS immediate didn't encode (caller falls back). */ +ST_FUNC int tcc_gen_machine_subs_eq_select_01(MachineOperand src1, MachineOperand src2, MachineOperand dest) +{ + if (src2.kind != MACH_OP_IMM) + return 0; + if (src1.kind != MACH_OP_REG || src1.needs_deref || src1.u.reg.r0 < 0) + return 0; + if (dest.kind != MACH_OP_REG || dest.needs_deref || dest.u.reg.r0 < 0) + return 0; + + uint32_t src_reg = (uint32_t)src1.u.reg.r0; + uint32_t dst_reg = (uint32_t)dest.u.reg.r0; + uint32_t Ku = (uint32_t)src2.u.imm.val; + + thumb_opcode subs = th_sub_imm(dst_reg, src_reg, Ku, FLAGS_BEHAVIOUR_SET, ENFORCE_ENCODING_NONE); + if (subs.size == 0) + return 0; + + /* Reserve so a literal-pool flush can't split the IT/MOV pair. */ + th_literal_pool_reserve_upcoming_bytes(10); + ot_check(subs); + ot_check(th_it(mapcc(TOK_NE), 0x8u)); + thumb_opcode movne = th_generic_mov_imm(dst_reg, 1); + if (movne.size != 0) { + ot_check(movne); + } else { + /* mov #1 always encodes on ARM Thumb-2, but be safe. */ + load_full_const((int)dst_reg, PREG_NONE, 1u, 0u); + } + return 1; +} + +/* tcc_gen_machine_mla_mop: MachineOperand-based entry point for MLA. + * dest = src1 * src2 + accum (all operands are 32-bit) + * + * All four operands are loaded into hardware registers via mach_ensure_in_reg + * before emitting a single MLA instruction. No fallback path is needed + * because mach_ensure_in_reg always returns a valid register. + * + * Note: th_mla(rd, rn, rm, ra) → rd = rn * rm + ra + */ +ST_FUNC void tcc_gen_machine_mla_mop(MachineOperand src1, MachineOperand src2, MachineOperand dest, + MachineOperand accum) +{ + MachineCodegenContext ctx = {0}; + + /* Pre-exclude registers directly referenced by REG operands so that scratch + * allocations for other operands (e.g. immediates) cannot clobber them. + * A dereferenced operand's r0 is its POINTER register — it must survive + * until that operand's load is emitted, so it is excluded exactly like a + * plain value register (ptr fuzz seed 59549: src2's spill reload picked the + * deref-accumulator's pointer register as scratch, and the accumulator then + * dereferenced the just-loaded multiplicand value → wild-address fault). + * The pre-allocated DEST register must be excluded too: if a source load + * grabs it as a saved scratch (push/pop), the restoring pop after the MLA + * overwrites the just-computed result. */ + uint32_t live_regs = 0; + if (src1.kind == MACH_OP_REG && src1.u.reg.r0 >= 0 && src1.u.reg.r0 < 16) + live_regs |= (1u << (uint32_t)src1.u.reg.r0); + if (src2.kind == MACH_OP_REG && src2.u.reg.r0 >= 0 && src2.u.reg.r0 < 16) + live_regs |= (1u << (uint32_t)src2.u.reg.r0); + if (accum.kind == MACH_OP_REG && accum.u.reg.r0 >= 0 && accum.u.reg.r0 < 16) + live_regs |= (1u << (uint32_t)accum.u.reg.r0); + if (dest.kind == MACH_OP_REG && + dest.u.reg.r0 != (int)PREG_REG_NONE && dest.u.reg.r0 >= 0 && dest.u.reg.r0 < 16) + live_regs |= (1u << (uint32_t)dest.u.reg.r0); + + int src1_reg = mach_ensure_in_reg(&ctx, &src1, live_regs); + uint32_t excl = live_regs; + if (thumb_is_hw_reg(src1_reg)) + excl |= (1u << (uint32_t)src1_reg); + + int src2_reg = mach_ensure_in_reg(&ctx, &src2, excl); + if (thumb_is_hw_reg(src2_reg)) + excl |= (1u << (uint32_t)src2_reg); + + int accum_reg = mach_ensure_in_reg(&ctx, &accum, excl); + if (thumb_is_hw_reg(accum_reg)) + excl |= (1u << (uint32_t)accum_reg); + + int dest_reg = mach_get_dest_reg(&ctx, &dest, excl); + + /* th_mla(rd, rn, rm, ra): rd = rn * rm + ra */ + ot_check(th_mla((uint32_t)dest_reg, (uint32_t)src1_reg, (uint32_t)src2_reg, (uint32_t)accum_reg)); + + mach_writeback_dest(&dest, dest_reg); + mach_release_all(&ctx); +} + +/* tcc_gen_machine_umull_mop: MachineOperand-based entry point for UMULL. + * {dest_hi:dest_lo} = (uint32_t)src1 * (uint32_t)src2 (64-bit unsigned result) + * + * src1 and src2 are 32-bit inputs (is_64bit is cleared before loading). + * dest must be a 64-bit pair; it is split via mach_make_lo/hi_half. + * Each half is allocated independently via mach_get_dest_reg, with the + * exclusion mask preventing rdlo==rdhi and preventing overlap with rn/rm. + * + * Note: th_umull(rdlo, rdhi, rn, rm) → {rdhi:rdlo} = rn * rm (unsigned) + */ +ST_FUNC void tcc_gen_machine_umull_mop(MachineOperand src1, MachineOperand src2, MachineOperand dest) +{ + MachineCodegenContext ctx = {0}; + + /* UMULL takes 32-bit inputs — drop any 64-bit flag the src may carry. */ + MachineOperand s1 = src1; + s1.is_64bit = false; + MachineOperand s2 = src2; + s2.is_64bit = false; + + /* Pre-exclude the pre-allocated dest pair: a saved-scratch (push/pop) on a + * dest register would have its restoring pop clobber the result. */ + uint32_t dest_excl = 0; + if (dest.kind == MACH_OP_REG && !dest.needs_deref) + { + if (dest.u.reg.r0 != (int)PREG_REG_NONE) + dest_excl |= (1u << (uint32_t)dest.u.reg.r0); + if (dest.is_64bit && dest.u.reg.r1 >= 0 && dest.u.reg.r1 != (int)PREG_REG_NONE) + dest_excl |= (1u << (uint32_t)dest.u.reg.r1); + } + + int rn = mach_ensure_in_reg(&ctx, &s1, dest_excl); + uint32_t excl = dest_excl | (thumb_is_hw_reg(rn) ? (1u << (uint32_t)rn) : 0u); + + int rm = mach_ensure_in_reg(&ctx, &s2, excl); + if (thumb_is_hw_reg(rm)) + excl |= (1u << (uint32_t)rm); + + /* Split 64-bit destination into lo (bits [31:0]) and hi (bits [63:32]). */ + MachineOperand dst_lo = mach_make_lo_half(&dest); + MachineOperand dst_hi = mach_make_hi_half(&dest); + dst_lo.btype = IROP_BTYPE_INT32; + dst_hi.btype = IROP_BTYPE_INT32; + + int rd_lo = mach_get_dest_reg(&ctx, &dst_lo, excl); + if (thumb_is_hw_reg(rd_lo)) + excl |= (1u << (uint32_t)rd_lo); + int rd_hi = mach_get_dest_reg(&ctx, &dst_hi, excl); + + /* th_umull(rdlo, rdhi, rn, rm): {rdhi:rdlo} = rn * rm */ + ot_check(th_umull((uint32_t)rd_lo, (uint32_t)rd_hi, (uint32_t)rn, (uint32_t)rm)); + + mach_writeback_dest(&dst_lo, rd_lo); + mach_writeback_dest(&dst_hi, rd_hi); + mach_release_all(&ctx); +} + +/* tcc_gen_machine_smull_mop: MachineOperand-based entry point for SMULL. + * {dest_hi:dest_lo} = (int32_t)src1 * (int32_t)src2 (64-bit signed result). + * Mirrors umull_mop but emits th_smull. */ +ST_FUNC void tcc_gen_machine_smull_mop(MachineOperand src1, MachineOperand src2, MachineOperand dest) +{ + MachineCodegenContext ctx = {0}; + + MachineOperand s1 = src1; + s1.is_64bit = false; + MachineOperand s2 = src2; + s2.is_64bit = false; + + /* Pre-exclude the pre-allocated dest pair (see umull_mop). */ + uint32_t dest_excl = 0; + if (dest.kind == MACH_OP_REG && !dest.needs_deref) + { + if (dest.u.reg.r0 != (int)PREG_REG_NONE) + dest_excl |= (1u << (uint32_t)dest.u.reg.r0); + if (dest.is_64bit && dest.u.reg.r1 >= 0 && dest.u.reg.r1 != (int)PREG_REG_NONE) + dest_excl |= (1u << (uint32_t)dest.u.reg.r1); + } + + int rn = mach_ensure_in_reg(&ctx, &s1, dest_excl); + uint32_t excl = dest_excl | (thumb_is_hw_reg(rn) ? (1u << (uint32_t)rn) : 0u); + + int rm = mach_ensure_in_reg(&ctx, &s2, excl); + if (thumb_is_hw_reg(rm)) + excl |= (1u << (uint32_t)rm); + + MachineOperand dst_lo = mach_make_lo_half(&dest); + MachineOperand dst_hi = mach_make_hi_half(&dest); + dst_lo.btype = IROP_BTYPE_INT32; + dst_hi.btype = IROP_BTYPE_INT32; + + int rd_lo = mach_get_dest_reg(&ctx, &dst_lo, excl); + if (thumb_is_hw_reg(rd_lo)) + excl |= (1u << (uint32_t)rd_lo); + int rd_hi = mach_get_dest_reg(&ctx, &dst_hi, excl); + + /* th_smull(rdlo, rdhi, rn, rm): {rdhi:rdlo} = (signed)rn * (signed)rm */ + ot_check(th_smull((uint32_t)rd_lo, (uint32_t)rd_hi, (uint32_t)rn, (uint32_t)rm)); + + mach_writeback_dest(&dst_lo, rd_lo); + mach_writeback_dest(&dst_hi, rd_hi); + mach_release_all(&ctx); +} + +/* tcc_gen_machine_mlal_accum_mop: emit SMLAL/UMLAL for + * dest = accum + (int32/uint32)src1 * (int32/uint32)src2 + * + * This narrow helper is used by codegen peepholes after register allocation. + * It only handles the cheap in-place accumulate form, where the ADD destination + * already holds the accumulator pair. Other forms fall back to SMULL/UMULL + * plus the normal 64-bit ADD so we do not risk clobbering multiply sources. */ +ST_FUNC int tcc_gen_machine_mlal_accum_mop(MachineOperand src1, MachineOperand src2, MachineOperand accum, + MachineOperand dest, int is_signed) +{ + if (!dest.is_64bit || !accum.is_64bit) + return 0; + if (dest.kind != MACH_OP_REG || accum.kind != MACH_OP_REG) + return 0; + if (dest.needs_deref || accum.needs_deref) + return 0; + if (dest.u.reg.r0 != accum.u.reg.r0 || dest.u.reg.r1 != accum.u.reg.r1) + return 0; + + int rd_lo = dest.u.reg.r0; + int rd_hi = dest.u.reg.r1; + /* Inline the hw-reg range checks rather than calling thumb_is_hw_reg(): the + * self-host cross drops the argument move into the inlined helper here and + * tests a stale register (the dest pointer) instead of rd_lo/rd_hi, so the + * native compiler wrongly bails out of every in-place 64-bit MLA with + * "unable to lower 64-bit MLA". Direct comparisons on rd_lo/rd_hi (as the + * adjacent rd_lo == rd_hi check already does) compile correctly. */ + if (rd_lo < 0 || rd_lo > 15 || rd_hi < 0 || rd_hi > 15 || rd_lo == rd_hi) + return 0; + + MachineCodegenContext ctx = {0}; + MachineOperand s1 = src1; + s1.is_64bit = false; + MachineOperand s2 = src2; + s2.is_64bit = false; + + uint32_t excl = (1u << (uint32_t)rd_lo) | (1u << (uint32_t)rd_hi); + /* Pre-exclude both sources' registers (a deref operand's r0 is its pointer + * register) so ensuring one source cannot grab the other's register as a + * spill-reload scratch — same clobber class as tcc_gen_machine_mla_mop. */ + if (s1.kind == MACH_OP_REG && s1.u.reg.r0 >= 0 && s1.u.reg.r0 < 16) + excl |= (1u << (uint32_t)s1.u.reg.r0); + if (s2.kind == MACH_OP_REG && s2.u.reg.r0 >= 0 && s2.u.reg.r0 < 16) + excl |= (1u << (uint32_t)s2.u.reg.r0); + int rn = mach_ensure_in_reg(&ctx, &s1, excl); + if (thumb_is_hw_reg(rn)) + excl |= (1u << (uint32_t)rn); + + int rm = mach_ensure_in_reg(&ctx, &s2, excl); + if (thumb_is_hw_reg(rm)) + excl |= (1u << (uint32_t)rm); + + if (is_signed) + ot_check(th_smlal((uint32_t)rd_lo, (uint32_t)rd_hi, (uint32_t)rn, (uint32_t)rm)); + else + ot_check(th_umlal((uint32_t)rd_lo, (uint32_t)rd_hi, (uint32_t)rn, (uint32_t)rm)); + + MachineOperand dst_lo = mach_make_lo_half(&dest); + MachineOperand dst_hi = mach_make_hi_half(&dest); + dst_lo.btype = IROP_BTYPE_INT32; + dst_hi.btype = IROP_BTYPE_INT32; + mach_writeback_dest(&dst_lo, rd_lo); + mach_writeback_dest(&dst_hi, rd_hi); + mach_release_all(&ctx); + return 1; +} + +/* The plain register a 32-bit operand lives in, or -1 (spilled, immediate, + * register-indirect, ...). */ +static int mach_plain_reg(const MachineOperand *op) +{ + if (op->kind == MACH_OP_REG && !op->needs_deref && op->u.reg.r0 >= 0 && op->u.reg.r0 < 16) + return op->u.reg.r0; + return -1; +} + +/* Put a 32-bit operand's value in register `target`: a move, or a direct load + * for a spill slot or an immediate. */ +static void mach_move_into(MachineCodegenContext *ctx, int target, const MachineOperand *op, uint32_t excl) +{ + int r = mach_plain_reg(op); + if (r == target) + return; + if (r < 0 && op->kind == MACH_OP_SPILL && !op->needs_deref) + { + mach_load_slot(target, op); + return; + } + if (r < 0 && op->kind == MACH_OP_IMM && !op->needs_deref) + { + tcc_machine_load_constant(target, PREG_REG_NONE, op->u.imm.val, 0, NULL); + return; + } + if (r < 0) + r = mach_ensure_in_reg(ctx, op, excl | (1u << (uint32_t)target)); + ot_check_mov_reg((uint32_t)target, (uint32_t)r, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE, false); +} + +/* tcc_machine_has_umaal: UMAAL is in the DSP extension (ARMv7E-M, ARMv8-M + * Mainline with DSP), which v8-M Baseline and plain v7-M lack. */ +ST_FUNC int tcc_machine_has_umaal(void) +{ + return arm_target_dependent.feat.t32 && arm_target_dependent.feat.dsp; +} + +/* tcc_gen_machine_umaal_mop: TCCIR_OP_UMAAL, + * dest = (uint32)src1 * (uint32)src2 + lo32(accum) + hi32(accum) + * UMAAL accumulates in place ({RdHi:RdLo} = Rn * Rm + RdLo + RdHi), so the + * accumulator's words go to the destination pair first -- no move at all when + * the allocator put them there. The pair the moves overwrite must not hold a + * multiplicand; a destination that overlaps one is replaced by a scratch pair + * and written back after. */ +ST_FUNC void tcc_gen_machine_umaal_mop(MachineOperand src1, MachineOperand src2, MachineOperand accum, + MachineOperand dest) +{ + MachineCodegenContext ctx = {0}; + MachineOperand s1 = src1; + s1.is_64bit = false; + MachineOperand s2 = src2; + s2.is_64bit = false; + MachineOperand acc_lo = mach_make_lo_half(&accum); + MachineOperand acc_hi = mach_make_hi_half(&accum); + acc_lo.btype = IROP_BTYPE_INT32; + acc_hi.btype = IROP_BTYPE_INT32; + MachineOperand dst_lo = mach_make_lo_half(&dest); + MachineOperand dst_hi = mach_make_hi_half(&dest); + dst_lo.btype = IROP_BTYPE_INT32; + dst_hi.btype = IROP_BTYPE_INT32; + + const int al = mach_plain_reg(&acc_lo), ah = mach_plain_reg(&acc_hi); + const int dl = mach_plain_reg(&dst_lo), dh = mach_plain_reg(&dst_hi); + uint32_t excl = 0; + if (al >= 0) excl |= 1u << (uint32_t)al; + if (ah >= 0) excl |= 1u << (uint32_t)ah; + if (dl >= 0) excl |= 1u << (uint32_t)dl; + if (dh >= 0) excl |= 1u << (uint32_t)dh; + if (mach_plain_reg(&s1) >= 0) excl |= 1u << (uint32_t)mach_plain_reg(&s1); + if (mach_plain_reg(&s2) >= 0) excl |= 1u << (uint32_t)mach_plain_reg(&s2); + + int rn = mach_ensure_in_reg(&ctx, &s1, excl); + if (thumb_is_hw_reg(rn)) + excl |= 1u << (uint32_t)rn; + int rm = mach_ensure_in_reg(&ctx, &s2, excl); + if (thumb_is_hw_reg(rm)) + excl |= 1u << (uint32_t)rm; + + int rd_lo = dl, rd_hi = dh; + if (rd_lo < 0 || rd_lo == rn || rd_lo == rm || rd_lo == rd_hi) + { + rd_lo = mach_alloc_scratch(&ctx, excl); + excl |= 1u << (uint32_t)rd_lo; + } + if (rd_hi < 0 || rd_hi == rn || rd_hi == rm || rd_hi == rd_lo) + { + rd_hi = mach_alloc_scratch(&ctx, excl); + excl |= 1u << (uint32_t)rd_hi; + } + + /* The two addends are summed symmetrically, so either may go in either + * register: keep whichever is already in place, which also means neither + * move can overwrite the other addend before it is read. */ + if (al == rd_hi || ah == rd_lo) + { + mach_move_into(&ctx, rd_lo, &acc_hi, excl); + mach_move_into(&ctx, rd_hi, &acc_lo, excl); + } + else + { + mach_move_into(&ctx, rd_lo, &acc_lo, excl); + mach_move_into(&ctx, rd_hi, &acc_hi, excl); + } + + ot_check(th_umaal((uint32_t)rd_lo, (uint32_t)rd_hi, (uint32_t)rn, (uint32_t)rm)); + + mach_writeback_dest(&dst_lo, rd_lo); + mach_writeback_dest(&dst_hi, rd_hi); + mach_release_all(&ctx); +} + +/* tcc_gen_machine_pack64_mop: lower TCCIR_OP_PACK64 by emitting two + * 32-bit assigns into the dest's halves. src_lo and src_hi are u32 + * operands; dest is a u64 register pair / spill / param slot. + * + * The two sub-assigns delegate to tcc_gen_machine_assign_mop, so they + * benefit from its existing handling of every dest kind (REG/SPILL/...). + * Often regalloc has already aligned the registers (e.g. dest.r0 = src_lo + * register), in which case the sub-assigns degrade to a no-op MOV that + * the encoder can skip. */ +ST_FUNC void tcc_gen_machine_pack64_mop(MachineOperand src_lo, MachineOperand src_hi, MachineOperand dest) +{ + if (!dest.is_64bit) + { + tcc_error("compiler_error: tcc_gen_machine_pack64_mop: dest not 64-bit"); + return; + } + MachineOperand dst_lo = mach_make_lo_half(&dest); + MachineOperand dst_hi = mach_make_hi_half(&dest); + dst_lo.btype = IROP_BTYPE_INT32; + dst_hi.btype = IROP_BTYPE_INT32; + + /* Detect register-swap aliasing: dst_lo == src_hi AND dst_hi == src_lo. + * Neither write order can preserve both source values; we must stage one + * side through a scratch register. */ + int swap_alias = 0; + if (src_lo.kind == MACH_OP_REG && !src_lo.needs_deref && + src_hi.kind == MACH_OP_REG && !src_hi.needs_deref && + dst_lo.kind == MACH_OP_REG && !dst_lo.needs_deref && + dst_hi.kind == MACH_OP_REG && !dst_hi.needs_deref && + src_hi.u.reg.r0 == dst_lo.u.reg.r0 && src_lo.u.reg.r0 == dst_hi.u.reg.r0 && + src_lo.u.reg.r0 != src_hi.u.reg.r0) + swap_alias = 1; + + if (swap_alias) + { + /* Save src_lo to a scratch before overwriting it via dst_hi. */ + uint32_t excl = (1u << (uint32_t)dst_lo.u.reg.r0) | (1u << (uint32_t)dst_hi.u.reg.r0); + ScratchRegAlloc scratch = get_scratch_reg_with_save(excl); + ot_check_mov_reg((uint32_t)scratch.reg, (uint32_t)src_lo.u.reg.r0, flags_safe(), + THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE, false); + /* Now dst_hi = src_hi (still live), then dst_lo = scratch (=old src_lo). */ + tcc_gen_machine_assign_mop(src_hi, dst_hi, TCCIR_OP_ASSIGN); + MachineOperand scratch_op = src_lo; + scratch_op.u.reg.r0 = scratch.reg; + tcc_gen_machine_assign_mop(scratch_op, dst_lo, TCCIR_OP_ASSIGN); + restore_scratch_reg(&scratch); + } + else if (src_hi.kind == MACH_OP_REG && !src_hi.needs_deref && + dst_lo.kind == MACH_OP_REG && !dst_lo.needs_deref && + src_hi.u.reg.r0 == dst_lo.u.reg.r0) + { + /* dst_lo == src_hi register: write hi first to free src_hi's slot. */ + tcc_gen_machine_assign_mop(src_hi, dst_hi, TCCIR_OP_ASSIGN); + tcc_gen_machine_assign_mop(src_lo, dst_lo, TCCIR_OP_ASSIGN); + } + else + { + tcc_gen_machine_assign_mop(src_lo, dst_lo, TCCIR_OP_ASSIGN); + tcc_gen_machine_assign_mop(src_hi, dst_hi, TCCIR_OP_ASSIGN); + } +} + +/* tcc_gen_machine_assign_mop: MachineOperand-based entry point for simple + * 32-bit value assignment. Called from ir/codegen.c instead of + * tcc_gen_machine_assign_op when: + * - Neither dest nor src requires a 64-bit or complex register pair, AND + * - The function does not use a static chain. + * + * Handles all destination kinds: MACH_OP_REG (direct), MACH_OP_SPILL + * (via mach_get_dest_reg + mach_writeback_dest → tcc_machine_store_spill_slot), + * and MACH_OP_PARAM_STACK (via mach_writeback_dest → tcc_machine_store_param_slot). + * + * Strategy: load src directly into dest_reg; use mach_ensure_in_reg only + * as a fallback for unhandled source kinds. + */ +ST_FUNC void tcc_gen_machine_assign_mop(MachineOperand src, MachineOperand dest, TccIrOp op) +{ + tcc_gen_machine_assign_mop_ex(src, dest, op, 0); +} + +ST_FUNC void tcc_gen_machine_assign_mop_ex(MachineOperand src, MachineOperand dest, TccIrOp op, uint32_t zh) +{ + (void)op; + /* Only the top-level 64-bit destination consults `zh`; the per-half recursive + * calls below have already split the pair and pass 0. */ + const bool dst_lo_dead = (zh & ZH64_D_LO) != 0; + const bool dst_hi_dead = (zh & ZH64_D_HI) != 0; + + /* 64-bit pair assignment: handle each 32-bit half independently. + * mach_make_lo/hi_half splits MACH_OP_REG (r0:r1), MACH_OP_SPILL (offset, + * offset+4) and MACH_OP_IMM into separate 32-bit MachineOperands. + * We then recursively assign each half (is_64bit=false prevents recursion). + * + * Special care: when src has needs_deref=true, the operand is a POINTER + * to a 64-bit value. The address is in one register (or spill slot); + * splitting registers via mach_make_hi_half would create a bogus base + * address. Instead, load both halves from [base+0] and [base+4]. + * + * Exception: PARAM_STACK and CHAIN_REL with needs_deref mean the 64-bit + * value IS directly at [fp+offset] / [chain+offset], not a pointer to + * follow (mach_op_64_names_memory). Clear needs_deref so it falls through + * to the normal lo/hi split path. */ + if (src.needs_deref && src.is_64bit && mach_op_64_names_memory(&src)) + src.needs_deref = false; + + if (dest.is_64bit) + { + MachineOperand dst_lo = mach_make_lo_half(&dest); + MachineOperand dst_hi = mach_make_hi_half(&dest); + dst_lo.btype = IROP_BTYPE_INT32; + dst_hi.btype = IROP_BTYPE_INT32; + + if (src.needs_deref && src.is_64bit) + { + /* Source is a 64-bit lvalue: a pointer to a 64-bit value (e.g. R0 + * holding address of an unsigned long long). Load both 32-bit halves + * from [base+0] and [base+4] using the same base address register. */ + MachineCodegenContext mctx = {0}; + + /* Strip deref to get the raw address into a register. */ + MachineOperand addr = src; + addr.needs_deref = false; + addr.is_64bit = false; + addr.btype = IROP_BTYPE_INT32; + int base_reg = mach_ensure_in_reg(&mctx, &addr, 0); + uint32_t excl = (1u << (uint32_t)base_reg); + + /* Determine destination registers for lo and hi halves. */ + int lo_reg = mach_get_dest_reg(&mctx, &dst_lo, excl); + if (thumb_is_hw_reg(lo_reg)) + excl |= (1u << (uint32_t)lo_reg); + int hi_reg = mach_get_dest_reg(&mctx, &dst_hi, excl); + + /* Load [base+0] → lo, [base+4] → hi (32-bit loads). */ + load_from_base(lo_reg, PREG_REG_NONE, IROP_BTYPE_INT32, 0, 0, 0, (uint32_t)base_reg); + load_from_base(hi_reg, PREG_REG_NONE, IROP_BTYPE_INT32, 0, 4, 0, (uint32_t)base_reg); + + mach_writeback_dest(&dst_lo, lo_reg); + mach_writeback_dest(&dst_hi, hi_reg); + mach_release_all(&mctx); + return; + } + + if (src.is_64bit) + { + /* Whole 64-bit local → register pair: one LDRD instead of the two LDRs + * the per-half recursion below would emit. */ + /* LDRD moves both halves at once, so it is only the right choice while + * both are wanted. */ + if (!dst_lo_dead && !dst_hi_dead && src.kind == MACH_OP_SPILL && !src.needs_deref && + (src.u.spill.offset & 3) == 0 && dest.kind == MACH_OP_REG && !dest.needs_deref && + thumb_is_hw_reg(dest.u.reg.r0) && thumb_is_hw_reg(dest.u.reg.r1) && + dest.u.reg.r0 != dest.u.reg.r1 && + tcc_gen_machine_try_ldrd_spill(dest.u.reg.r0, src.u.spill.offset, dest.u.reg.r1, + src.u.spill.offset + 4)) + return; + + MachineOperand src_lo = mach_make_lo_half(&src); + MachineOperand src_hi = mach_make_hi_half(&src); + src_lo.btype = IROP_BTYPE_INT32; + src_hi.btype = IROP_BTYPE_INT32; + if (!dst_lo_dead) + tcc_gen_machine_assign_mop(src_lo, dst_lo, op); + if (!dst_hi_dead) + tcc_gen_machine_assign_mop(src_hi, dst_hi, op); + } + else + { + /* 32-bit source into 64-bit dest: assign lo half, zero the high half. + * That zero is the single commonest instruction in soft float -- every + * `(uint64_t)x` before a shift-and-OR pays one -- so skipping it when no + * consumer reads the half is the point of the whole annotation. */ + MachineOperand zero = {0}; + zero.kind = MACH_OP_IMM; + zero.u.imm.val = 0; + zero.btype = IROP_BTYPE_INT32; + if (!dst_lo_dead) + tcc_gen_machine_assign_mop(src, dst_lo, op); + if (!dst_hi_dead) + tcc_gen_machine_assign_mop(zero, dst_hi, op); + } + return; + } + + if (src.is_64bit && !dest.is_64bit) + { + /* Truncation: extract and assign only the low half of the 64-bit source. */ + MachineOperand src_lo = mach_make_lo_half(&src); + src_lo.btype = IROP_BTYPE_INT32; + tcc_gen_machine_assign_mop(src_lo, dest, op); + return; + } + + /* Single-precision float in a VFP register (hard-float): vmov between s-regs, + * or bridge through a GPR for the GPR/imm/spill/memory endpoint. */ + if (src.kind == MACH_OP_VFP_REG || dest.kind == MACH_OP_VFP_REG) + { + if (src.kind == MACH_OP_VFP_REG && dest.kind == MACH_OP_VFP_REG) + { + if (src.u.reg.r0 != dest.u.reg.r0) + ot_check(th_vmov_register((uint16_t)dest.u.reg.r0, (uint16_t)src.u.reg.r0, 0)); + return; + } + MachineCodegenContext vctx = {0}; + if (dest.kind == MACH_OP_VFP_REG) + { + int gpr = mach_ensure_in_reg(&vctx, &src, 0); /* GPR/imm/spill/... -> GPR */ + ot_check(th_vmov_gp_sp((uint16_t)gpr, (uint16_t)dest.u.reg.r0, 0)); /* sN = gpr */ + } + else + { + int gpr = mach_ensure_in_reg(&vctx, &src, 0); /* VFP src bridges to a GPR */ + mach_writeback_dest(&dest, gpr); + } + mach_release_all(&vctx); + return; + } + + /* A copy between two values the allocator spilled to the SAME slot, read + * and written at the same width, leaves memory as it was. The store half + * was already dropped by the encoder (strldr_cache_str_is_redundant), which + * left the load behind as a dead `ldr rX,[sp,#N]`: a phi copy whose two + * sides share a slot. Knob: TCC_DISABLE_PASS=codegen:slot_self_copy. */ + if (!slot_self_copy_off && src.kind == MACH_OP_SPILL && dest.kind == MACH_OP_SPILL && !src.needs_deref && + !dest.needs_deref && src.u.spill.offset == dest.u.spill.offset && + mach_slot_load_width(&src) == mach_slot_store_width(&dest)) + return; + + MachineCodegenContext mctx = {0}; + + /* --- Fast path: source is already in a register (no dereference) --- + * Write it directly to the destination via mach_writeback_dest without + * allocating any scratch. This covers REG→REG (MOV or NOP) and + * REG→SPILL/PARAM_STACK (direct store from src register). */ + if (src.kind == MACH_OP_REG && !src.needs_deref) + { + mach_writeback_dest(&dest, src.u.reg.r0); + return; + } + + /* --- Determine destination register --- + * For REG destinations, reuse the pre-allocated register (0 scratch). + * For SPILL/PARAM_STACK/REG(deref) destinations, allocate a scratch. */ + int dest_reg; + bool need_writeback; + if (dest.kind == MACH_OP_REG && !dest.needs_deref && dest.u.reg.r0 != (int)PREG_REG_NONE) + { + dest_reg = dest.u.reg.r0; + need_writeback = false; + } + else + { + dest_reg = mach_get_dest_reg(&mctx, &dest, 0); + need_writeback = true; + } + + /* --- Load source value directly into dest_reg --- */ + switch (src.kind) + { + case MACH_OP_REG: + /* Only the needs_deref case reaches here (non-deref handled above). + * Load from [src_reg] directly into dest_reg. */ + load_from_base(dest_reg, PREG_REG_NONE, src.btype, (int)src.is_unsigned, 0, 0, (uint32_t)src.u.reg.r0); + break; + + case MACH_OP_IMM: + tcc_machine_load_constant(dest_reg, PREG_REG_NONE, src.u.imm.val, 0, NULL); + break; + + case MACH_OP_SPILL: + mach_load_slot(dest_reg, &src); + if (src.needs_deref) + { + /* Double indirection: dest_reg now holds a pointer; dereference it. */ + load_from_base(dest_reg, PREG_REG_NONE, src.btype, (int)src.is_unsigned, 0, 0, (uint32_t)dest_reg); + } + break; + + case MACH_OP_SYMBOL: + { + Sym *sym = src.u.sym.sym ? validate_sym_for_reloc(src.u.sym.sym) : NULL; + if (!src.needs_deref) + { + tcc_machine_load_constant(dest_reg, PREG_REG_NONE, src.u.sym.addend, 0, sym); + } + else + { + const int32_t addend = src.u.sym.addend; + const int abs_off = addend < 0 ? (int)(-addend) : (int)addend; + const int sign = addend < 0 ? 1 : 0; + /* Prefer a base register OTHER than dest_reg: `ldr rD,[rD,#off]` + * overwrites the address with the loaded value, so imm_cache loses it + * and the store that follows a read-modify-write has to re-load the + * literal. A separate base keeps the address cached (and an imm_cache + * hit removes this materialization entirely). Only when no register is + * free do we fall back to routing through dest_reg - taking a + * push/pop there would cost more than the reload it saves. */ + ScratchRegAlloc base; + if (try_scratch_reg_for_sym_addr(sym, 0, (1u << (uint32_t)dest_reg), &base)) + { + tcc_machine_load_constant(base.reg, PREG_REG_NONE, 0, 0, sym); + load_from_base(dest_reg, PREG_REG_NONE, src.btype, (int)src.is_unsigned, abs_off, sign, + (uint32_t)base.reg); + restore_scratch_reg(&base); + } + else + { + tcc_machine_load_constant(dest_reg, PREG_REG_NONE, 0, 0, sym); + load_from_base(dest_reg, PREG_REG_NONE, src.btype, (int)src.is_unsigned, abs_off, sign, + (uint32_t)dest_reg); + } + } + break; + } + + case MACH_OP_FRAME_ADDR: + tcc_machine_addr_of_stack_slot(dest_reg, src.u.frame.offset, 0); + break; + + case MACH_OP_PARAM_STACK: + { + const int adjusted = param_frame_offset(src.u.param.offset); + const int base_reg = tcc_state->need_frame_pointer ? R_FP : R_SP; + const int sign = (adjusted < 0); + const int abs_off = sign ? -adjusted : adjusted; + load_from_base(dest_reg, PREG_REG_NONE, src.btype, (int)src.is_unsigned, abs_off, sign, (uint32_t)base_reg); + break; + } + + default: + { + /* Fallback: generic mach_ensure_in_reg + MOV. */ + uint32_t excl = thumb_is_hw_reg(dest_reg) ? (1u << (uint32_t)dest_reg) : 0; + int src_reg = mach_ensure_in_reg(&mctx, &src, excl); + if (src_reg != dest_reg) + ot_check_mov_reg((uint32_t)dest_reg, (uint32_t)src_reg, flags_safe(), THUMB_SHIFT_DEFAULT, + ENFORCE_ENCODING_NONE, false); + break; + } + } + + if (need_writeback) + mach_writeback_dest(&dest, dest_reg); + + mach_release_all(&mctx); +} + +/* tcc_gen_machine_setif_mop: MachineOperand-based entry point for SETIF. + * src must be MACH_OP_IMM carrying the raw condition code in u.imm.val. + * + * 32-bit dest: + * ITE + * MOV dest, #1 (T: cond met) + * MOV dest, #0 (E: cond not met) + * + * 64-bit dest pair (e.g. long long result = (x > y)): + * The boolean result 0 or 1 fits in 32 bits, so hi word is always 0. + * ITE + * MOV dest_lo, #1 + * MOV dest_lo, #0 + * MOV dest_hi, #0 (unconditional, outside IT block — hi is always 0) + * + * Inner MOVs use NOT_IMPORTANT for flags: SETIF is the consumer of the CMP + * flags; once the ITE captures the condition, no subsequent code in this + * lowering depends on CMP's flag state, so the 16-bit T1 encoding (which + * implicitly sets flags) is safe. This shrinks each conditional MOV from + * 4 bytes (mov.w) to 2 bytes (movs). + */ +ST_FUNC void tcc_gen_machine_setif_mop(MachineOperand src, MachineOperand dest, TccIrOp op) +{ + (void)op; + MachineCodegenContext mctx = {0}; + + const int cond = mapcc((int)src.u.imm.val); + /* ITE mask: 2nd instruction has opposite condition. + * mask[3] = 1 if it should be the 'else' bit (opposite of cond[0]). + * For the T-then-E pattern, mask = ((!cond[0]) << 3) | 0x4. */ + const uint16_t ite_mask = (uint16_t)(((cond ^ 1) & 1) << 3) | 0x4u; + + if (dest.is_64bit) + { + /* Split 64-bit destination into two independent 32-bit halves. */ + MachineOperand dst_lo = mach_make_lo_half(&dest); + MachineOperand dst_hi = mach_make_hi_half(&dest); + dst_lo.btype = IROP_BTYPE_INT32; + dst_hi.btype = IROP_BTYPE_INT32; + + int lo_reg = mach_get_dest_reg(&mctx, &dst_lo, 0); + uint32_t excl = thumb_is_hw_reg(lo_reg) ? (1u << (uint32_t)lo_reg) : 0u; + int hi_reg = mach_get_dest_reg(&mctx, &dst_hi, excl); + + /* Emit ITE sequence for lo word. Reserve the WHOLE atomic ITE+movs block so + * a literal-pool flush never lands between the IT and its conditioned movs. + * A high register (R8-R12) dest forces the 4-byte mov.w (T2) encoding, so the + * worst case is ITE(2) + 3*mov.w(4) = 14 bytes — NOT 6 (which only covers the + * 2-byte movs of a low-reg dest). Under-reserving split the ITE and ran the + * fall-through into the literal pool (seed 89 O1 HardFault). */ + th_literal_pool_reserve_upcoming_bytes(14); + ot_check(th_it(cond, ite_mask)); /* ITE — two conditioned instructions */ + ot_check(th_mov_imm(lo_reg, 1, FLAGS_BEHAVIOUR_NOT_IMPORTANT, ENFORCE_ENCODING_NONE)); + ot_check(th_mov_imm(lo_reg, 0, FLAGS_BEHAVIOUR_NOT_IMPORTANT, ENFORCE_ENCODING_NONE)); + /* Hi word is always 0 — boolean result never exceeds 1 (i.e. fits in 32-bit lo). */ + ot_check(th_mov_imm(hi_reg, 0, FLAGS_BEHAVIOUR_NOT_IMPORTANT, ENFORCE_ENCODING_NONE)); + + mach_writeback_dest(&dst_lo, lo_reg); + mach_writeback_dest(&dst_hi, hi_reg); + } + else + { + int dest_reg = mach_get_dest_reg(&mctx, &dest, 0); + + /* Reserve the whole ITE+2-movs block: a high-register dest (R8-R12) uses the + * 4-byte mov.w (T2) encoding, so the worst case is ITE(2) + 2*mov.w(4) = 10 + * bytes, not 6. Under-reserving let a literal-pool flush split the ITE and + * run the fall-through into the pool (seed 89 O1 HardFault). */ + th_literal_pool_reserve_upcoming_bytes(10); + ot_check(th_it(cond, ite_mask)); /* ITE — two conditioned instructions */ + ot_check(th_mov_imm(dest_reg, 1, FLAGS_BEHAVIOUR_NOT_IMPORTANT, ENFORCE_ENCODING_NONE)); + ot_check(th_mov_imm(dest_reg, 0, FLAGS_BEHAVIOUR_NOT_IMPORTANT, ENFORCE_ENCODING_NONE)); + + mach_writeback_dest(&dest, dest_reg); + } + + mach_release_all(&mctx); +} + +/* tcc_gen_machine_bool_mop: MachineOperand-based entry point for + * BOOL_OR / BOOL_AND. Called from ir/codegen.c for simple 32-bit + * non-complex boolean operations. + * + * BOOL_OR: ORRS dest, src1, src2 (sets Z flag) + * MOV dest, #0 (flag-preserving) + * IT NE + * MOV dest, #1 + * + * BOOL_AND: CMP src1, #0 + * IT NE + * CMP src2, #0 (only if src1 != 0) + * MOV dest, #0 (flag-preserving) + * IT NE + * MOV dest, #1 + */ +/* Flags for a MOV #imm8 that is the conditioned instruction of an IT block: + * there the 16-bit encoding sets no flags, so a low register need not pay for + * the flag-preserving MOV.W. */ +static thumb_flags_behaviour it_mov_flags(int reg) +{ + return reg < 8 ? FLAGS_BEHAVIOUR_NOT_IMPORTANT : flags_safe(); +} + +ST_FUNC void tcc_gen_machine_bool_mop(MachineOperand src1, MachineOperand src2, MachineOperand dest, TccIrOp op) +{ + MachineCodegenContext mctx = {0}; + + /* 64-bit operands: reduce each to a 32-bit "is non-zero" value by OR-ing + * its low and high halves, then apply the standard 32-bit BOOL logic. */ + if (src1.is_64bit || src2.is_64bit) + { + uint32_t excl = 0; + int r1, r2; + + if (src1.is_64bit) + { + MachineOperand lo1 = mach_make_lo_half(&src1); + lo1.btype = IROP_BTYPE_INT32; + MachineOperand hi1 = mach_make_hi_half(&src1); + hi1.btype = IROP_BTYPE_INT32; + r1 = mach_ensure_in_reg(&mctx, &lo1, excl); + excl |= thumb_is_hw_reg(r1) ? (1u << (uint32_t)r1) : 0; + int hi1_reg = mach_ensure_in_reg(&mctx, &hi1, excl); + excl |= thumb_is_hw_reg(hi1_reg) ? (1u << (uint32_t)hi1_reg) : 0; + /* r1 = lo1 | hi1 — is src1 non-zero? */ + ot_check(th_orr_reg(r1, r1, hi1_reg, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); + } + else + { + r1 = mach_ensure_in_reg(&mctx, &src1, excl); + excl |= thumb_is_hw_reg(r1) ? (1u << (uint32_t)r1) : 0; + } + + if (src2.is_64bit) + { + MachineOperand lo2 = mach_make_lo_half(&src2); + lo2.btype = IROP_BTYPE_INT32; + MachineOperand hi2 = mach_make_hi_half(&src2); + hi2.btype = IROP_BTYPE_INT32; + r2 = mach_ensure_in_reg(&mctx, &lo2, excl); + excl |= thumb_is_hw_reg(r2) ? (1u << (uint32_t)r2) : 0; + int hi2_reg = mach_ensure_in_reg(&mctx, &hi2, excl); + excl |= thumb_is_hw_reg(hi2_reg) ? (1u << (uint32_t)hi2_reg) : 0; + /* r2 = lo2 | hi2 — is src2 non-zero? */ + ot_check(th_orr_reg(r2, r2, hi2_reg, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); + } + else + { + r2 = mach_ensure_in_reg(&mctx, &src2, excl); + excl |= thumb_is_hw_reg(r2) ? (1u << (uint32_t)r2) : 0; + } + + int dest_reg = mach_get_dest_reg(&mctx, &dest, excl); + + if (op == TCCIR_OP_BOOL_OR) + { + ot_check(th_orr_reg(dest_reg, r1, r2, FLAGS_BEHAVIOUR_SET, THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); + ot_check(th_mov_imm(dest_reg, 0, FLAGS_BEHAVIOUR_BLOCK, ENFORCE_ENCODING_NONE)); + th_literal_pool_reserve_upcoming_bytes(6); + ot_check(th_it(0x1, 0x8)); /* IT NE */ + ot_check(th_mov_imm(dest_reg, 1, it_mov_flags(dest_reg), ENFORCE_ENCODING_NONE)); + } + else /* TCCIR_OP_BOOL_AND */ + { + ot_check(th_cmp_imm(r1, 0, FLAGS_BEHAVIOUR_SET, ENFORCE_ENCODING_NONE)); + th_literal_pool_reserve_upcoming_bytes(6); + ot_check(th_it(0x1, 0x8)); /* IT NE */ + ot_check(th_cmp_imm(r2, 0, FLAGS_BEHAVIOUR_SET, ENFORCE_ENCODING_NONE)); /* CMPne r2, #0 */ + ot_check(th_mov_imm(dest_reg, 0, FLAGS_BEHAVIOUR_BLOCK, ENFORCE_ENCODING_NONE)); + th_literal_pool_reserve_upcoming_bytes(6); + ot_check(th_it(0x1, 0x8)); /* IT NE */ + ot_check(th_mov_imm(dest_reg, 1, it_mov_flags(dest_reg), ENFORCE_ENCODING_NONE)); + } + + mach_writeback_dest(&dest, dest_reg); + mach_release_all(&mctx); + return; + } + + int dest_reg = mach_get_dest_reg(&mctx, &dest, 0); + uint32_t excl = thumb_is_hw_reg(dest_reg) ? (1u << (uint32_t)dest_reg) : 0; + + int src1_reg = mach_ensure_in_reg(&mctx, &src1, excl); + if (thumb_is_hw_reg(src1_reg)) + excl |= (1u << (uint32_t)src1_reg); + + int src2_reg = mach_ensure_in_reg(&mctx, &src2, excl); + + if (op == TCCIR_OP_BOOL_OR) + { + ot_check(th_orr_reg(dest_reg, src1_reg, src2_reg, FLAGS_BEHAVIOUR_SET, THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); + ot_check(th_mov_imm(dest_reg, 0, FLAGS_BEHAVIOUR_BLOCK, ENFORCE_ENCODING_NONE)); + th_literal_pool_reserve_upcoming_bytes(6); + ot_check(th_it(0x1, 0x8)); /* IT NE */ + ot_check(th_mov_imm(dest_reg, 1, it_mov_flags(dest_reg), ENFORCE_ENCODING_NONE)); + } + else /* TCCIR_OP_BOOL_AND */ + { + ot_check(th_cmp_imm(src1_reg, 0, FLAGS_BEHAVIOUR_SET, ENFORCE_ENCODING_NONE)); + th_literal_pool_reserve_upcoming_bytes(6); + ot_check(th_it(0x1, 0x8)); /* IT NE */ + ot_check(th_cmp_imm(src2_reg, 0, FLAGS_BEHAVIOUR_SET, ENFORCE_ENCODING_NONE)); /* CMPne src2, #0 */ + ot_check(th_mov_imm(dest_reg, 0, FLAGS_BEHAVIOUR_BLOCK, ENFORCE_ENCODING_NONE)); + th_literal_pool_reserve_upcoming_bytes(6); + ot_check(th_it(0x1, 0x8)); /* IT NE */ + ot_check(th_mov_imm(dest_reg, 1, it_mov_flags(dest_reg), ENFORCE_ENCODING_NONE)); + } + + mach_writeback_dest(&dest, dest_reg); + mach_release_all(&mctx); +} diff --git a/source/backend/arch/arm/thumb/arm-thumb-asm.c b/source/backend/arch/arm/thumb/arm-thumb-asm.c index aa40f21d7..21ded133d 100644 --- a/source/backend/arch/arm/thumb/arm-thumb-asm.c +++ b/source/backend/arch/arm/thumb/arm-thumb-asm.c @@ -456,6 +456,15 @@ ST_FUNC void asm_gen_code(ASMOperand *operands, int nb_operands, int nb_outputs, } else { // epilog + /* The asm body was emitted without passing through ot(), so no register- + * content cache saw what it wrote. Forget them before storing outputs: + * with a stale `mov` equivalence the store of a "+r" operand back to its + * variable (`mov r4, r0` after the body incremented r0) is elided as a + * copy between registers the cache still believes equal. */ + tcc_gen_machine_imm_cache_reset(); + tcc_gen_machine_mov_equiv_reset(); + tcc_gen_machine_strldr_cache_reset(); + /* generate save code */ for (i = 0; i < nb_outputs; i++) { @@ -641,10 +650,10 @@ instruction operands[k].input_index = i; op->priority = 5; } - else if ((op->vt->r & VT_VALMASK) == VT_LOCAL && op->vt->sym && (reg = op->vt->sym->r & VT_VALMASK) < VT_CONST) + else if (op->regvar) { op->priority = 1; - op->reg = reg; + op->reg = op->regvar - 1; } else { @@ -2998,6 +3007,8 @@ static thumb_opcode thumb_math_opcode(TCCState *s1, int token) return th_smull(ops[0].reg, ops[1].reg, ops[2].reg, ops[3].reg); case TOK_ASM_umlal: return th_umlal(ops[0].reg, ops[1].reg, ops[2].reg, ops[3].reg); + case TOK_ASM_umaal: + return th_umaal(ops[0].reg, ops[1].reg, ops[2].reg, ops[3].reg); case TOK_ASM_umull: return th_umull(ops[0].reg, ops[1].reg, ops[2].reg, ops[3].reg); } @@ -3772,6 +3783,7 @@ ST_FUNC void asm_opcode(TCCState *s1, int token) case TOK_ASM_smlal: case TOK_ASM_smull: case TOK_ASM_umlal: + case TOK_ASM_umaal: case TOK_ASM_mls: case TOK_ASM_umull: return thumb_emit_opcode(thumb_math_opcode(s1, token)); diff --git a/source/backend/arch/arm/thumb/arm-thumb-call.c b/source/backend/arch/arm/thumb/arm-thumb-call.c new file mode 100644 index 000000000..8e1605ce2 --- /dev/null +++ b/source/backend/arch/arm/thumb/arm-thumb-call.c @@ -0,0 +1,4833 @@ +/* + * ARMvX-m code generator for TCC + * Uses thumb instruction set + * + * Based on: + * ARM Thumb 2 instruction functions for TCC + * Copyright (c) 2020 Erlend J. Sveen + * from: + * https://git.erlendjs.no/erlendjs/tinycc/-/blob/arm-thumb/arm-thumb-gen.c + * https://git.erlendjs.no/erlendjs/tinycc/-/blob/arm-thumb/arm-thumb-instructions.c + * + * And + * + * ARMv4 code generator for TCC + * + * Copyright (c) 2003 Daniel Glöckner + * Copyright (c) 2012 Thomas Preud'homme + * + * Based on i386-gen.c by Fabrice Bellard + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +/* Thumb-2 code generator: call lowering -- argument moves and stack + * arguments, the function-call generator, block copies, VLA, setjmp/longjmp, + * and __builtin_apply. */ + +#include "arm-thumb-gen.h" + +static int get_struct_base_addr_mop(const MachineOperand *mop, int default_reg); +static int find_call_scratch(uint32_t extra_exclude, uint32_t arg_move_dst_mask); +static int find_call_scratch_free(uint32_t extra_exclude, uint32_t arg_move_dst_mask); + +/* Is the by-value struct source ADDRESS provably 4-byte aligned? + * + * `struct_src_align` is the C type's alignment, and it is 1 for every packed + * aggregate — tinycc's own 9-byte IROperand, which dominates these call sites, + * being the obvious one. What LDRD and LDM actually require is an aligned + * address, so for a frame-resident source ask the offset instead of the type. + * + * Only the low two bits matter, and they survive the frame bias: SP is 8-byte + * aligned and FP 4-byte aligned at steady state, and every term + * fp_adjust_local_offset() adds (allocated_stack_size, scratch_push_sp_bias) + * is a multiple of 4. Testing the RAW offset therefore + * gives the same answer as testing the adjusted one, and — unlike the adjusted + * one, whose scratch_push_sp_bias() term is dry-run gated — it reads the same + * in the rehearsal and the real pass, so it cannot desync the two code sizes. + * + * Incoming stack parameters are aligned by construction (arm_aapcs.c floors + * every argument slot at 4 and offset_to_args is a multiple of 4), but the + * check is written out rather than assumed. */ +static bool struct_src_addr_aligned4(const ThumbArgMove *m) +{ + if (m->struct_src_align >= 4) + return true; + if (m->mop.needs_deref) + return false; /* address comes from memory — nothing provable */ + switch (m->mop.kind) + { + case MACH_OP_SPILL: + return (m->mop.u.spill.offset & 3) == 0; + case MACH_OP_FRAME_ADDR: + return (m->mop.u.frame.offset & 3) == 0; + case MACH_OP_PARAM_STACK: + return ((param_frame_offset(m->mop.u.param.offset)) & 3) == 0; + default: + return false; + } +} + +static void thumb_emit_arg_move(const ThumbArgMove *m) +{ + if (m->kind == THUMB_ARG_MOVE_REG) + { + if (m->src_reg == m->dst_reg) + return; + ot_check_mov_reg(m->dst_reg, m->src_reg, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE, + false); + return; + } + + if (m->kind == THUMB_ARG_MOVE_LOCAL_ADDR) + { + /* Compute address of local variable: dst = fp + offset */ + tcc_machine_addr_of_stack_slot(m->dst_reg, m->local_offset, m->local_is_param); + return; + } + + if (m->kind == THUMB_ARG_MOVE_STRUCT) + { + /* Load struct words into consecutive registers. + * The mop contains the struct operand; get its base address. */ + int word_count = m->struct_word_count; + int base_dst = m->dst_reg; + + /* LDRD fast path for the common 2-word (8-byte) aggregate case sourced + * directly from a stack-backed location. Mirrors the LDRD path used by + * THUMB_ARG_MOVE_MOP for 64-bit scalars — Thumb-2 LDRD requires natural + * 4-byte alignment, which spill slots and the caller param stack both + * provide. Skips the scratch + per-word loads that would otherwise + * emit `add.w ip, sp, #N; ldr lo, [ip]; ldr hi, [ip, #4]` (3 insts). + * + * LDRD writes Rt before Rt2, so Rt2 (dst+1) must not equal the base + * register, otherwise the 2nd half reads from a clobbered base. */ + if (word_count == 2 && !m->mop.needs_deref && + (m->mop.kind == MACH_OP_SPILL || m->mop.kind == MACH_OP_PARAM_STACK)) + { + int raw_off = + (m->mop.kind == MACH_OP_SPILL) ? m->mop.u.spill.offset : param_frame_offset(m->mop.u.param.offset); + int adjusted = (m->mop.kind == MACH_OP_SPILL) ? fp_adjust_local_offset(raw_off, 0) : raw_off; + int ldrd_base = tcc_state->need_frame_pointer ? R_FP : R_SP; + int ldrd_sign = (adjusted < 0); + int ldrd_abs_off = ldrd_sign ? -adjusted : adjusted; + int dst_hi = base_dst + 1; + if (dst_hi != ldrd_base && base_dst != ldrd_base && + try_ldrd_pair(base_dst, dst_hi, ldrd_base, ldrd_abs_off, ldrd_sign)) + { + return; + } + } + + /* A struct in the frame loads word by word straight off sp/fp, without + * first building its address in a scratch register + * (`add.w ip, sp, #N; ldr r1, [ip]`). The sources and offsets are the + * ones get_struct_base_addr_mop would address: only a spill slot's + * needs_deref means the slot holds a pointer. Taken when every word encodes + * and the loads are no larger than the address + LDM form (8 bytes). */ + if (base_dst >= 0 && ((m->mop.kind == MACH_OP_SPILL && !m->mop.needs_deref) || + m->mop.kind == MACH_OP_PARAM_STACK || m->mop.kind == MACH_OP_FRAME_ADDR)) + { + int off; + if (m->mop.kind == MACH_OP_PARAM_STACK) + off = param_frame_offset(m->mop.u.param.offset); + else + off = fp_adjust_local_offset(m->mop.kind == MACH_OP_SPILL ? m->mop.u.spill.offset : m->mop.u.frame.offset, 0); + int base = tcc_state->need_frame_pointer ? R_FP : R_SP; + int bytes = 0; + for (int w = 0; w < word_count && bytes >= 0; ++w) + { + int o = off + 4 * w; + int size = base_dst + w == base ? 0 + : th_ldr_imm(base_dst + w, base, o < 0 ? -o : o, o < 0 ? 4 : 6, + ENFORCE_ENCODING_NONE) + .size; + bytes = size ? bytes + size : -1; + } + /* Three or more low registers: build the address in the last of them + * and load the run with a 16-bit LDM, which does no writeback when its + * base is in the list -- `add r3, sp, #N; ldm r3, {r0-r3}`, 4 bytes where + * four LDRs take 8. */ + const int last = base_dst + word_count - 1; + if (word_count >= 3 && last <= R7 && !(base >= base_dst && base <= last) && (off & 3) == 0 && + struct_src_addr_aligned4(m)) + { + thumb_opcode addr = off < 0 ? th_sub_imm(last, base, -off, flags_safe(), ENFORCE_ENCODING_NONE) + : th_add_imm(last, base, off, flags_safe(), ENFORCE_ENCODING_NONE); + if (addr.size && (bytes < 0 || addr.size + 2 < bytes)) + { + ot_check(addr); + const uint32_t regs = ((1u << word_count) - 1u) << base_dst; + ot_check((thumb_opcode){.size = 2, .opcode = 0xC800u | ((uint32_t)last << 8) | regs}); + return; + } + } + if (bytes > 0 && (word_count <= 2 || bytes <= 8)) + { + for (int w = 0; w < word_count; ++w) + { + int o = off + 4 * w; + load_word_from_base(base_dst + w, base, o < 0 ? -o : o, o < 0); + } + return; + } + } + + /* Get the struct base address into a scratch register */ + ScratchRegAlloc struct_scratch = get_scratch_reg_with_save(0); + int base_addr_reg = get_struct_base_addr_mop(&m->mop, struct_scratch.reg); + + /* Load each word from the struct into consecutive target registers. + * Adjacent word pairs use LDRD when the source ADDRESS is 4-byte aligned + * (LDRD faults otherwise) and neither destination register aliases the base + * (LDRD writes Rt then Rt2; an alias would read a clobbered base on the + * fallback path / be unsafe). */ + bool src_aligned = struct_src_addr_aligned4(m); + + /* A single LDMIA.W replaces the whole per-word load sequence. It only pays + * from three words up: at two, the LDRD in the loop below is already one + * 4-byte instruction. + * + * th_ldm() silently strips the base register from the list when there is no + * writeback, so a base sitting inside the destination range would quietly + * load one register too few — reject that here instead of relying on the + * helper. PC and SP must stay out of the list too: PC would turn the load + * into a branch. */ + int last_dst = base_dst + word_count - 1; + if (word_count >= 3 && src_aligned && base_addr_reg != R_SP && base_dst >= 0 && last_dst <= R_IP && + !(base_addr_reg >= base_dst && base_addr_reg <= last_dst)) + { + uint32_t regset = 0; + for (int k = 0; k < word_count; ++k) + regset |= 1u << (unsigned)(base_dst + k); + ot_check(th_ldm((uint32_t)base_addr_reg, regset, 0 /* no writeback */, ENFORCE_ENCODING_NONE)); + restore_scratch_reg(&struct_scratch); + return; + } + + int w = 0; + for (; w + 1 < word_count; ) + { + int dst = base_dst + w; + int dst_hi = base_dst + w + 1; + int offset = w * 4; + if (src_aligned && dst != base_addr_reg && dst_hi != base_addr_reg && + tcc_gen_machine_try_ldrd_base(dst, dst_hi, base_addr_reg, offset)) + { + w += 2; + continue; + } + /* Single-word load of this word; the next iteration handles w+1. */ + if (!load_word_from_base(dst, base_addr_reg, offset, 0)) + { + ScratchRegAlloc off_scratch = get_scratch_reg_with_save((1u << base_addr_reg) | (1u << dst)); + load_immediate(off_scratch.reg, offset, NULL, false); + ot_check(th_ldr_reg(dst, base_addr_reg, off_scratch.reg, THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); + restore_scratch_reg(&off_scratch); + } + w += 1; + } + /* Trailing odd word. */ + for (; w < word_count; ++w) + { + int dst = base_dst + w; + int offset = w * 4; + if (!load_word_from_base(dst, base_addr_reg, offset, 0)) + { + ScratchRegAlloc off_scratch = get_scratch_reg_with_save((1u << base_addr_reg) | (1u << dst)); + load_immediate(off_scratch.reg, offset, NULL, false); + ot_check(th_ldr_reg(dst, base_addr_reg, off_scratch.reg, THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); + restore_scratch_reg(&off_scratch); + } + } + restore_scratch_reg(&struct_scratch); + return; + } + + if (m->kind == THUMB_ARG_MOVE_MOP) + { + /* Generic MachineOperand → register load. + * Handles all MOP kinds (REG+deref, SPILL, PARAM_STACK, CHAIN_REL, + * SYMBOL+deref, etc.) via mach_ensure_in_reg. */ + MachineCodegenContext mctx = {0}; + if (m->mop.is_64bit && m->dst_reg_hi != 0 && m->dst_reg_hi != PREG_REG_NONE) + { + if (m->mop.needs_deref && !mach_op_64_names_memory(&m->mop)) + { + /* The operand holds a pointer (in reg, spill, etc.). Load the + * pointer into a register, then fetch lo/hi from [ptr+0]/[ptr+4]. + * mach_make_hi_half cannot handle this because it adjusts the + * storage location (e.g. spill offset) instead of the deref offset. + * + * PARAM_STACK and CHAIN_REL are excluded (mach_op_64_names_memory): + * mach_ensure_in_reg loads those directly from the caller's argument + * area / the parent frame, so the mach_make_lo/hi_half path handles + * them correctly. */ + int base; + if (m->mop.kind == MACH_OP_REG) + { + base = m->mop.u.reg.r0; + } + else + { + MachineOperand addr = m->mop; + addr.needs_deref = false; + addr.is_64bit = false; + addr.btype = IROP_BTYPE_INT32; + uint32_t excl = (1u << m->dst_reg) | (1u << m->dst_reg_hi); + base = mach_ensure_in_reg(&mctx, &addr, excl); + } + /* Proven-aligned access (mop.align4): LDRD straight into the pair — + * Rt == Rn is fine without writeback, so no base preservation needed. + * Otherwise the 64-bit load_from_base path, which preserves the base + * when base == dst_reg (the lo-load would clobber it before the + * hi-load can use it) and stays unaligned-safe. */ + if (!(m->mop.align4 && try_ldrd_pair(m->dst_reg, m->dst_reg_hi, base, 0, 0))) + load_from_base(m->dst_reg, m->dst_reg_hi, IROP_BTYPE_INT64, 0, 0, 0, (uint32_t)base); + } + else + { + /* Fast path: when the source is a stack-backed 64-bit value (spill + * slot or caller's param stack area) and the destination is a valid + * AAPCS register pair, emit a single LDRD straight into dst_reg / + * dst_reg_hi, skipping the scratch + MOV sequence that the generic + * lo/hi lowering below would produce. + * + * Stack spill slots and the caller-argument frame are guaranteed + * 8-byte aligned (AAPCS stack_align = 8, spill slots obey type + * alignment), so LDRD's 4-byte alignment requirement is satisfied. */ + int ldrd_base = -1; + int ldrd_abs_off = 0; + int ldrd_sign = 0; + int ldrd_ok = 0; + if (!m->mop.needs_deref && (m->mop.kind == MACH_OP_SPILL || m->mop.kind == MACH_OP_PARAM_STACK)) + { + int raw_off = (m->mop.kind == MACH_OP_SPILL) ? m->mop.u.spill.offset : param_frame_offset(m->mop.u.param.offset); + int adjusted = (m->mop.kind == MACH_OP_SPILL) ? fp_adjust_local_offset(raw_off, 0) : raw_off; + ldrd_base = tcc_state->need_frame_pointer ? R_FP : R_SP; + ldrd_sign = (adjusted < 0); + ldrd_abs_off = ldrd_sign ? -adjusted : adjusted; + ldrd_ok = 1; + } + if (ldrd_ok && try_ldrd_pair(m->dst_reg, m->dst_reg_hi, ldrd_base, ldrd_abs_off, ldrd_sign)) + { + /* LDRD emitted. */ + } + else + { + /* 64-bit: load lo and hi halves separately. */ + MachineOperand lo = mach_make_lo_half(&m->mop); + MachineOperand hi = mach_make_hi_half(&m->mop); + uint32_t excl = (1u << m->dst_reg) | (1u << m->dst_reg_hi); + int r_lo = mach_ensure_in_reg(&mctx, &lo, excl); + int r_hi = mach_ensure_in_reg(&mctx, &hi, excl | (1u << (uint32_t)r_lo)); + if (r_lo != m->dst_reg) + ot_check_mov_reg(m->dst_reg, r_lo, flags_safe(), THUMB_SHIFT_DEFAULT, + ENFORCE_ENCODING_NONE, false); + if (r_hi != m->dst_reg_hi) + ot_check_mov_reg(m->dst_reg_hi, r_hi, flags_safe(), THUMB_SHIFT_DEFAULT, + ENFORCE_ENCODING_NONE, false); + } + } + } + else + { + /* 32-bit: prefer loading directly into dst_reg when the operand kind + * permits it, bypassing the scratch + MOV sequence that + * mach_ensure_in_reg would emit. Kinds that need an extra + * pointer-chain scratch beyond dst_reg (CHAIN_REL) fall through to + * the generic path. */ + const MachineOperand *mop = &m->mop; + const int dst = m->dst_reg; + int handled = 0; + + switch (mop->kind) + { + case MACH_OP_NONE: + tcc_machine_load_constant(dst, PREG_REG_NONE, 0, 0, NULL); + handled = 1; + break; + + case MACH_OP_REG: + if (mop->needs_deref) + { + /* LDR dst, [r0]; legal even when r0 == dst (loaded value + * just replaces the base). */ + load_from_base(dst, PREG_REG_NONE, mop->btype, (int)mop->is_unsigned, 0, 0, + (uint32_t)mop->u.reg.r0); + } + else if (mop->u.reg.r0 != dst) + { + ot_check_mov_reg(dst, mop->u.reg.r0, flags_safe(), THUMB_SHIFT_DEFAULT, + ENFORCE_ENCODING_NONE, false); + } + handled = 1; + break; + + case MACH_OP_SPILL: + if (!mop->needs_deref) + { + mach_load_slot(dst, mop); + } + else + { + /* LLOCAL: load pointer into dst, then dereference into dst. */ + tcc_machine_load_spill_slot(dst, mop->u.spill.offset); + load_from_base(dst, PREG_REG_NONE, mop->btype, (int)mop->is_unsigned, 0, 0, + (uint32_t)dst); + } + handled = 1; + break; + + case MACH_OP_PARAM_STACK: + { + const int adjusted = param_frame_offset(mop->u.param.offset); + const int base_reg = tcc_state->need_frame_pointer ? R_FP : R_SP; + const int sign = (adjusted < 0); + const int abs_off = sign ? -adjusted : adjusted; + load_from_base(dst, PREG_REG_NONE, mop->btype, (int)mop->is_unsigned, abs_off, sign, + (uint32_t)base_reg); + handled = 1; + break; + } + + case MACH_OP_IMM: + tcc_machine_load_constant(dst, PREG_REG_NONE, mop->u.imm.val, 0, NULL); + handled = 1; + break; + + case MACH_OP_FRAME_ADDR: + if (!mop->needs_deref) + { + tcc_machine_addr_of_stack_slot(dst, mop->u.frame.offset, 0); + } + else + { + tcc_machine_addr_of_stack_slot(dst, mop->u.frame.offset, 0); + load_from_base(dst, PREG_REG_NONE, mop->btype, (int)mop->is_unsigned, 0, 0, + (uint32_t)dst); + } + handled = 1; + break; + + case MACH_OP_SYMBOL: + { + Sym *raw_sym = mop->u.sym.sym; + Sym *sym = raw_sym ? validate_sym_for_reloc(raw_sym) : NULL; + if (!mop->needs_deref) + { + tcc_machine_load_constant(dst, PREG_REG_NONE, mop->u.sym.addend, 0, sym); + } + else + { + tcc_machine_load_constant(dst, PREG_REG_NONE, 0, 0, sym); + const int32_t addend = mop->u.sym.addend; + const int sign = (addend < 0); + const int abs_off = sign ? (int)(-addend) : (int)addend; + load_from_base(dst, PREG_REG_NONE, mop->btype, (int)mop->is_unsigned, abs_off, sign, + (uint32_t)dst); + } + handled = 1; + break; + } + + default: + /* CHAIN_REL etc.: fall through to generic scratch + MOV. */ + break; + } + + if (!handled) + { + uint32_t excl = (1u << dst); + int r = mach_ensure_in_reg(&mctx, mop, excl); + if (r != dst) + ot_check_mov_reg(dst, r, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE, + false); + } + } + mach_release_all(&mctx); + return; + } + + if (m->kind == THUMB_ARG_MOVE_IMM64) + { + /* Load 64-bit immediate into register pair */ + uint32_t lo = (uint32_t)(m->imm64 & 0xFFFFFFFF); + uint32_t hi = (uint32_t)(m->imm64 >> 32); + load_immediate(m->dst_reg, lo, NULL, false); + load_immediate(m->dst_reg_hi, hi, NULL, false); + return; + } + + /* THUMB_ARG_MOVE_IMM */ + load_immediate(m->dst_reg, m->imm, m->sym, false); +} + +/* Compute the full set of destination registers written by an arg move. + * Multi-register moves (IMM64, 64-bit MOP, STRUCT) write more than dst_reg. + * The parallel move scheduler must check ALL written registers against + * pending source registers to avoid clobbering. */ +static uint32_t arg_move_write_set(const ThumbArgMove *m) +{ + uint32_t set = (1u << m->dst_reg); + switch (m->kind) + { + case THUMB_ARG_MOVE_IMM64: + set |= (1u << m->dst_reg_hi); + break; + case THUMB_ARG_MOVE_MOP: + if (m->dst_reg_hi > 0 && m->dst_reg_hi < 16) + set |= (1u << m->dst_reg_hi); + break; + case THUMB_ARG_MOVE_STRUCT: + for (int w = 1; w < m->struct_word_count; w++) + set |= (1u << (m->dst_reg + w)); + break; + default: + break; + } + return set; +} + +/* Schedule register argument setup as a parallel assignment. + * This avoids clobbering a source register needed for another argument. + * Example: r0 <- r6, r1 <- r0 must be emitted as: + * mov r1, r0 + * mov r0, r6 + */ +static void thumb_emit_parallel_arg_moves(ThumbArgMove *moves, int move_count) +{ + if (move_count <= 0) + return; + + uint8_t done[16]; + memset(done, 0, sizeof(done)); + + ScratchRegAlloc tmp_alloc = (ScratchRegAlloc){0}; + int have_tmp = 0; + + for (int remaining = move_count; remaining > 0;) + { + uint32_t src_set = 0; + for (int i = 0; i < move_count; ++i) + { + if (done[i]) + continue; + if (moves[i].kind == THUMB_ARG_MOVE_REG) + src_set |= (1u << moves[i].src_reg); + } + + int chosen = -1; + for (int i = 0; i < move_count; ++i) + { + if (done[i]) + continue; + /* Check ALL destination registers of this move against pending sources. + * Multi-reg writes (IMM64, 64-bit MOP, STRUCT) must not clobber any + * register that a pending REG move still needs to read. */ + if ((src_set & arg_move_write_set(&moves[i])) == 0) + { + chosen = i; + break; + } + } + + if (chosen < 0) + { + /* Cycle among register moves. Break it with a scratch temp. */ + int cyc = -1; + for (int i = 0; i < move_count; ++i) + { + if (!done[i] && moves[i].kind == THUMB_ARG_MOVE_REG) + { + cyc = i; + break; + } + } + if (cyc < 0) + tcc_error("compiler_error: arg move cycle without reg sources"); + + if (!have_tmp) + { + /* Exclude all regs involved in the parallel move. */ + uint32_t exclude = 0; + for (int i = 0; i < move_count; ++i) + { + if (done[i]) + continue; + exclude |= arg_move_write_set(&moves[i]); + if (moves[i].kind == THUMB_ARG_MOVE_REG) + exclude |= (1u << moves[i].src_reg); + } + /* Also exclude SP/PC. */ + exclude |= (1u << ARM_SP) | (1u << ARM_PC); + tmp_alloc = get_scratch_reg_with_save(exclude); + have_tmp = 1; + } + + thumb_require_materialized_reg("thumb_emit_parallel_arg_moves", "tmp", tmp_alloc.reg); + ot_check_mov_reg(tmp_alloc.reg, moves[cyc].src_reg, flags_safe(), THUMB_SHIFT_DEFAULT, + ENFORCE_ENCODING_NONE, false); + moves[cyc].src_reg = tmp_alloc.reg; + continue; + } + + thumb_emit_arg_move(&moves[chosen]); + done[chosen] = 1; + --remaining; + } + + if (have_tmp && tmp_alloc.saved) + ot_check(th_pop(1u << tmp_alloc.reg)); +} + +/* ======================================================================== + * Helper functions for call argument handling + * ======================================================================== */ + +/* Store a word to stack with large offset fallback. An offset out of the + * immediate's reach goes through a scratch register, which must not be one + * in `keep`: the other half of a pair still to be stored, a struct base. */ +static void store_word_to_stack_keep(int src_reg, int stack_offset, uint32_t keep) +{ + if (!store_word_to_base(src_reg, ARM_SP, stack_offset, 0)) + { + ScratchRegAlloc sc = get_scratch_reg_with_save((1u << src_reg) | keep); + /* A scratch saved by PUSH has moved SP down a word under the store. */ + load_immediate(sc.reg, stack_offset + (sc.saved == 1 ? 4 : 0), NULL, false); + ot_check(th_str_reg(src_reg, ARM_SP, sc.reg, THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); + restore_scratch_reg(&sc); + } +} + +static void store_word_to_stack(int src_reg, int stack_offset) +{ + store_word_to_stack_keep(src_reg, stack_offset, 0); +} + +/* Store a word to stack, keeping the struct base register intact */ +static void store_word_to_stack_safe(int src_reg, int stack_offset, int base_addr_reg) +{ + store_word_to_stack_keep(src_reg, stack_offset, 1u << base_addr_reg); +} + +/* Get struct base address into a register (MOP path). + * For struct arguments, we want the ADDRESS of the struct, not a word from it. + * The MOP from machine_op_from_ir encodes the "value-level" view, so we + * convert / strip one level of indirection to obtain the address instead. */ +static int get_struct_base_addr_mop(const MachineOperand *mop, int default_reg) +{ + switch (mop->kind) + { + case MACH_OP_REG: + /* Register holds the struct address (for both needs_deref=true and false, + * the register value IS the address we want for struct copying). */ + return mop->u.reg.r0; + + case MACH_OP_FRAME_ADDR: + /* Address-of local struct: compute FP + offset. */ + tcc_machine_addr_of_stack_slot(default_reg, mop->u.frame.offset, 0); + return default_reg; + + case MACH_OP_SPILL: + if (mop->needs_deref) + { + /* llocal: spill slot holds pointer to struct. Load just the pointer. */ + tcc_machine_load_spill_slot(default_reg, mop->u.spill.offset); + } + else + { + /* Local struct on stack: compute address FP + offset. */ + tcc_machine_addr_of_stack_slot(default_reg, mop->u.spill.offset, 0); + } + return default_reg; + + case MACH_OP_PARAM_STACK: + /* Struct in caller's argument area: compute address with param adjustment. */ + tcc_machine_addr_of_stack_slot(default_reg, mop->u.param.offset, 1 /* is_param */); + return default_reg; + + case MACH_OP_SYMBOL: + { + Sym *sym = mop->u.sym.sym ? validate_sym_for_reloc(mop->u.sym.sym) : NULL; + load_immediate(default_reg, (uint32_t)mop->u.sym.addend, sym, false); + return default_reg; + } + + default: + { + /* CHAIN_REL, etc: generic path with needs_deref stripped. */ + MachineOperand addr_mop = *mop; + addr_mop.needs_deref = false; + MachineCodegenContext mctx = {0}; + int r = mach_ensure_in_reg(&mctx, &addr_mop, 0); + mach_release_all(&mctx); + return r; + } + } +} + +/* Build register move for a struct argument (MOP path) */ +static int build_reg_move_struct(ThumbArgMove *moves, int move_count, const MachineOperand *mop, + const TCCAbiArgLoc *loc, int base_reg, ThumbGenCallSite *call_site, + int src_align) +{ + int words = loc->reg_count; + if (words > 0 && words <= 4) + { + moves[move_count++] = (ThumbArgMove){ + .kind = THUMB_ARG_MOVE_STRUCT, + .dst_reg = base_reg, + .mop = *mop, + .struct_word_count = words, + .struct_src_align = src_align, + }; + } + for (int w = 0; w < words && w < loc->reg_count; w++) + call_site->registers_map |= (1 << (base_reg + w)); + return move_count; +} + +/* Build register move for a 64-bit argument (MOP path) */ +static int build_reg_move_64bit(ThumbArgMove *moves, int move_count, const MachineOperand *mop, const IROperand *arg, + int base_reg, ThumbGenCallSite *call_site, TCCIRState *ir) +{ + if (mop->kind == MACH_OP_REG && !mop->needs_deref && thumb_is_hw_reg(mop->u.reg.r0) && thumb_is_hw_reg(mop->u.reg.r1)) + { + /* Both halves in registers — emit up to two REG moves. */ + if (mop->u.reg.r0 != base_reg) + moves[move_count++] = (ThumbArgMove){.kind = THUMB_ARG_MOVE_REG, .dst_reg = base_reg, .src_reg = mop->u.reg.r0}; + if (mop->u.reg.r1 != (base_reg + 1)) + moves[move_count++] = + (ThumbArgMove){.kind = THUMB_ARG_MOVE_REG, .dst_reg = base_reg + 1, .src_reg = mop->u.reg.r1}; + } + else if (mop->kind == MACH_OP_IMM) + { + const uint64_t imm64 = (uint64_t)mop->u.imm.val; + moves[move_count++] = + (ThumbArgMove){.kind = THUMB_ARG_MOVE_IMM64, .dst_reg = base_reg, .dst_reg_hi = base_reg + 1, .imm64 = imm64}; + } + else + { + /* Generic: load MOP value into register pair at emit time. + * Covers SPILL, PARAM_STACK, CHAIN_REL, REG+needs_deref, SYMBOL, etc. */ + MachineOperand m = *mop; + m.is_64bit = true; + moves[move_count++] = + (ThumbArgMove){.kind = THUMB_ARG_MOVE_MOP, .dst_reg = base_reg, .dst_reg_hi = base_reg + 1, .mop = m}; + } + + call_site->registers_map |= (1 << base_reg) | (1 << (base_reg + 1)); + return move_count; +} + +/* Build register move for a 32-bit argument (MOP path) */ +static int build_reg_move_32bit(ThumbArgMove *moves, int move_count, const MachineOperand *mop, const IROperand *arg, + int base_reg, ThumbGenCallSite *call_site, TCCIRState *ir) +{ + switch (mop->kind) + { + case MACH_OP_REG: + if (mop->needs_deref) + { + /* Register-indirect: needs dereference at emit time. */ + moves[move_count++] = (ThumbArgMove){.kind = THUMB_ARG_MOVE_MOP, .dst_reg = base_reg, .mop = *mop}; + } + else if (mop->u.reg.r0 != base_reg) + { + moves[move_count++] = (ThumbArgMove){.kind = THUMB_ARG_MOVE_REG, .dst_reg = base_reg, .src_reg = mop->u.reg.r0}; + } + break; + + case MACH_OP_IMM: + moves[move_count++] = + (ThumbArgMove){.kind = THUMB_ARG_MOVE_IMM, .dst_reg = base_reg, .imm = (uint32_t)mop->u.imm.val, .sym = NULL}; + break; + + case MACH_OP_SYMBOL: + if (mop->needs_deref) + { + /* Load value from global symbol — emit at emit time. */ + moves[move_count++] = (ThumbArgMove){.kind = THUMB_ARG_MOVE_MOP, .dst_reg = base_reg, .mop = *mop}; + } + else + { + /* Load symbol address (with addend). */ + moves[move_count++] = (ThumbArgMove){ + .kind = THUMB_ARG_MOVE_IMM, .dst_reg = base_reg, .imm = (uint32_t)mop->u.sym.addend, .sym = mop->u.sym.sym}; + } + break; + + case MACH_OP_FRAME_ADDR: + moves[move_count++] = (ThumbArgMove){.kind = THUMB_ARG_MOVE_LOCAL_ADDR, + .dst_reg = base_reg, + .local_offset = mop->u.frame.offset, + .local_is_param = 0}; + break; + + case MACH_OP_PARAM_STACK: + /* `&P` of a parameter in the caller's argument area -- the sret pointer of + * `s = f(s)` when `s` is a by-value struct parameter split across r0-r3 + * and the stack. The generic MOP load below always reads the slot, so it + * passed the struct's first word as the pointer. A value read (is_lval, + * or a PARAM vreg) still loads. */ + if (arg && arg->is_local && !arg->is_lval && !arg->is_llocal && !mop->needs_deref) + { + moves[move_count++] = (ThumbArgMove){.kind = THUMB_ARG_MOVE_LOCAL_ADDR, + .dst_reg = base_reg, + .local_offset = mop->u.param.offset, + .local_is_param = 1}; + break; + } + moves[move_count++] = (ThumbArgMove){.kind = THUMB_ARG_MOVE_MOP, .dst_reg = base_reg, .mop = *mop}; + break; + + default: + /* SPILL, PARAM_STACK, CHAIN_REL, etc.: generic MOP load at emit time. */ + moves[move_count++] = (ThumbArgMove){.kind = THUMB_ARG_MOVE_MOP, .dst_reg = base_reg, .mop = *mop}; + break; + } + + call_site->registers_map |= (1 << base_reg); + return move_count; +} + +/* Place a struct argument on stack (MOP path) */ +/* Load one struct word at [base_addr_reg + off] into `reg`, falling back to a + * register-offset load when `off` exceeds the LDR immediate range. */ +static void load_struct_word_into(int reg, int base_addr_reg, int off) +{ + if (!load_word_from_base(reg, base_addr_reg, off, 0)) + { + load_immediate(reg, off, NULL, false); + ot_check(th_ldr_reg(reg, base_addr_reg, reg, THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); + } +} + +/* Word copies at least this many words long call libtcc1's __tcc_wcopy_N + * (lib/arm_copystub.S) instead of expanding inline; 0 keeps them all inline. */ +TCC_DBG_ENV_INT(copy_stub_min_words, "TCC_COPY_STUB_MIN", 3) +#define COPY_STUB_MAX_WORDS 32 + +/* A word-aligned copy of `size` bytes from [r1] to [r0] as a call to the + * libtcc1 routine of its size: 4 bytes of code where the inline LDM/STM pairs + * took 8 per 16 bytes copied (Zig's C passes, returns and copies 40-200 byte + * structs by value everywhere). The routine clobbers exactly what the inline + * copy did -- r0-r3, IP, LR -- and neither R9 nor the stack. Only when the + * libtcc1.a this module links has it: -nodefaultlibs links none. */ +static int thumb_copy_stub_call(int size) +{ + const int words = size / 4, min = copy_stub_min_words(); + if (min <= 0 || (size & 3) || words < (min > 3 ? min : 3) || words > COPY_STUB_MAX_WORDS) + return 0; + char name[32]; + snprintf(name, sizeof name, "__tcc_wcopy_%d", words); + if (!tcc_yaff_libtcc1_has(tcc_state, name)) + return 0; + MachineOperand target = {0}; + target.kind = MACH_OP_SYMBOL; + target.u.sym.sym = external_helper_sym(tok_alloc_const(name)); + /* A struct argument of a tail call is copied before its B.W: this BL + * returns. */ + const int saved_tail = tail_call_pending; + tail_call_pending = 0; + gcall_or_jump_mop(0, target); + tail_call_pending = saved_tail; + return 1; +} + +/* Copy `size` bytes from [r1] to [r0], both word-aligned, clobbering r0-r3, + * IP and LR (what a memcpy call would). A call to __tcc_wcopy_N when there + * is one for the size (thumb_copy_stub_call); else up to + * STACK_ARG_UNROLL_MAX_WORDS, `ldmia r1!, {r2,r3,ip,lr}` / `stmia r0!` per 16 + * bytes; above, a loop moving 12 bytes a turn with LR counting. A 1-3 byte + * tail moves by bytes. */ +static void thumb_emit_word_copy_r0_r1(int size) +{ + const uint32_t set4 = (1u << R2) | (1u << R3) | (1u << R12) | (1u << ARM_LR); + const uint32_t set3 = (1u << R2) | (1u << R3) | (1u << R12); + if (thumb_copy_stub_call(size)) + return; + int left = size / 4; + if (left <= STACK_ARG_UNROLL_MAX_WORDS) + { + for (; left >= 4; left -= 4) + { + ot_check(th_ldm(R1, set4, 1, ENFORCE_ENCODING_NONE)); + ot_check(th_stm(R0, set4, 1, ENFORCE_ENCODING_NONE)); + } + } + else + { + load_immediate(ARM_LR, left / 3, NULL, false); + th_literal_pool_reserve_upcoming_bytes(14); + const int top = ind; + ot_check(th_ldm(R1, set3, 1, ENFORCE_ENCODING_NONE)); + ot_check(th_stm(R0, set3, 1, ENFORCE_ENCODING_NONE)); + ot_check(th_sub_imm(ARM_LR, ARM_LR, 1, FLAGS_BEHAVIOUR_SET, ENFORCE_ENCODING_NONE)); + const int off = top - (ind + 4); + if (off < -256) + tcc_error("compiler_error: word copy loop out of branch range"); + ot_check(th_b_t1(1 /* NE */, (uint32_t)(off >> 1))); + left %= 3; + } + if (left == 3) + { + ot_check(th_ldm(R1, set3, 0, ENFORCE_ENCODING_NONE)); + ot_check(th_stm(R0, set3, 0, ENFORCE_ENCODING_NONE)); + } + else if (left == 2) + { + ot_check(th_ldrd_imm(R2, R3, R1, 0, 6)); + ot_check(th_strd_imm(R2, R3, R0, 0, 6)); + } + else if (left == 1) + { + ot_check_ldr_imm(R2, R1, 0, 6, ENFORCE_ENCODING_NONE); + ot_check_str_imm(R2, R0, 0, 6, ENFORCE_ENCODING_NONE); + } + for (int b = 0; b < (size & 3); b++) + { + ot_check(th_ldrb_imm(R2, R1, left * 4 + b, 6, ENFORCE_ENCODING_NONE)); + ot_check(th_strb_imm(R2, R0, left * 4 + b, 6, ENFORCE_ENCODING_NONE)); + } +} + +static void place_stack_arg_struct(const MachineOperand *mop, const TCCAbiArgLoc *loc, int stack_offset, + int src_align, uint32_t arg_move_dst_mask) +{ + int words_in_regs = (loc->kind == TCC_ABI_LOC_REG_STACK) ? loc->reg_count : 0; + int struct_src_offset = words_in_regs * 4; + int struct_size = (loc->kind == TCC_ABI_LOC_REG_STACK) ? loc->stack_size : loc->size; + int words = (struct_size + 3) / 4; + + /* A large struct (AAPCS32 passes any size by value) is copied in the + * argument registers, IP and LR rather than a word per load/store pair. + * They may hold values the rest of the call setup still needs, so they are + * saved around the copy; the 24 bytes pushed enter every SP-relative address + * computed meanwhile. */ + if (words >= STACK_ARG_MEMCPY_MIN_WORDS) + { + const uint16_t saved = (uint16_t)((1u << R0) | (1u << R1) | (1u << R2) | (1u << R3) | (1u << R12) | (1u << ARM_LR)); + ot_check(th_push(saved)); + helper_call_sp_bias += 24; + int src = get_struct_base_addr_mop(mop, R1); + if (src != R1) + ot_check_mov_reg(R1, (uint32_t)src, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE, false); + if (struct_src_offset) + ot_check(th_add_imm(R1, R1, (uint32_t)struct_src_offset, flags_safe(), ENFORCE_ENCODING_NONE)); + load_immediate(R0, stack_offset + scratch_push_sp_bias() - call_args_sp_bias, NULL, false); + ot_check(th_add_reg(R0, R0, ARM_SP, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); + if (src_align >= 4) + { + /* A word-aligned source copies inline: the saved registers are all free + * now. A memcpy call cost ~75 instructions for the 48-256 byte structs + * the Zig C backend passes by value. */ + thumb_emit_word_copy_r0_r1(words * 4); + ot_check(th_pop(saved)); + helper_call_sp_bias -= 24; + return; + } + load_immediate(R2, words * 4, NULL, false); + Sym *memcpy_sym = external_global_sym(tok_alloc_const("memcpy"), &func_old_type); + MachineOperand func_mop = {0}; + func_mop.kind = MACH_OP_SYMBOL; + func_mop.u.sym.sym = memcpy_sym; + if (text_and_data_separation) + ot_check(th_push((uint16_t)((1 << R9) | (1 << R12)))); + gcall_or_jump_mop(0, func_mop); + if (text_and_data_separation) + ot_check(th_pop((uint16_t)((1 << R9) | (1 << R12)))); + ot_check(th_pop(saved)); + helper_call_sp_bias -= 24; + return; + } + + /* A frame-resident source needs no base register: LDR/LDRD carry the frame + * offset in their own immediate field, so the `add rX, sp, #off` that + * get_struct_base_addr_mop emits is pure overhead — one instruction per + * marshaled by-value struct argument. Only take the direct form while every + * source word stays inside the offset range both LDR and LDRD encode + * (word-aligned, 0..1020); outside it the per-word fallback would cost more + * than the single ADD. Nothing in the loop below pushes a scratch across a + * source load, so the SP bias folded in here stays valid throughout. */ + int frame_raw = 0; + bool src_is_frame = false; + if (mop->kind == MACH_OP_SPILL && !mop->needs_deref) + { + frame_raw = mop->u.spill.offset; + src_is_frame = true; + } + else if (mop->kind == MACH_OP_FRAME_ADDR) + { + frame_raw = mop->u.frame.offset; + src_is_frame = true; + } + else if (mop->kind == MACH_OP_PARAM_STACK) + { + /* A by-value parameter passed on: in the incoming argument area. */ + src_is_frame = true; + } + + int base_bias = 0; + int base_addr_reg; + ScratchRegAlloc struct_sc = {0}; + bool have_struct_sc = false; + + if (src_is_frame) + { + const int adj = mop->kind == MACH_OP_PARAM_STACK ? param_frame_offset(mop->u.param.offset) + : fp_adjust_local_offset(frame_raw, 0); + const int first = adj + struct_src_offset; + const int last = first + (words > 0 ? (words - 1) * 4 : 0); + if ((adj & 3) == 0 && first >= 0 && last <= 1020) + { + base_addr_reg = tcc_state->need_frame_pointer ? R_FP : R_SP; + base_bias = adj; + } + else + src_is_frame = false; + } + if (!src_is_frame) + { + struct_sc = get_scratch_reg_with_save(0); + have_struct_sc = true; + base_addr_reg = get_struct_base_addr_mop(mop, struct_sc.reg); + } + + /* The data registers. The first is LR, free at a call's argument setup + * since the call clobbers it -- unless the struct base is in LR: with nothing + * else free, get_scratch_reg_with_save hands out LR itself. Loading the + * first pair into LR then overwrote the base, and every later word was read + * through the loaded value (Zig's AstGen.containerDecl passing a 20-byte + * ResultInfo split across r2-r3 and the stack). The second data register, + * for paired LDRD/STRD, comes from find_call_scratch too. It never pushes + * (SP-relative store offsets must stay valid), and its R_IP last resort is + * refused while IP holds a live value -- a later stack argument, say. */ + int data1 = ARM_LR; + if (base_addr_reg == ARM_LR) + data1 = find_call_scratch_free(1u << ARM_LR, 0); + if (data1 < 0) + { + /* No register but the base itself: rebuild the address into LR for every + * word and load through it. Only a scratch-built base can be rebuilt; a + * struct whose address lives in LR as a register operand cannot. */ + if (mop->kind == MACH_OP_REG) + tcc_error("internal error: struct argument addressed through lr with no free register"); + for (int k = 0; k < words; ++k) + { + int b = get_struct_base_addr_mop(mop, ARM_LR); + load_struct_word_into(ARM_LR, b, struct_src_offset + k * 4); + store_word_to_stack_keep(ARM_LR, stack_offset + k * 4, 0); + } + if (have_struct_sc) + restore_scratch_reg(&struct_sc); + return; + } + int data2 = find_call_scratch_free((1u << ARM_LR) | (1u << (uint32_t)data1) | (1u << (uint32_t)base_addr_reg), 0); + bool can_pair = (words >= 2 && data2 != data1 && data2 != base_addr_reg && data2 >= 0 && + data2 <= R_LR && data2 != R_SP); + bool src_aligned = (src_align >= 4); + + int w = 0; + + /* Runs of three or four words from a frame source move by LDM/STM: + * `add rA, sp, #src; ldm rA, {..}` then `stmia.w sp, {..}` for the bottom of + * the outgoing area, else `add rB, sp, #dst; stmia rB!, {..}` -- 8 or 10 + * bytes where LDRD/STRD pairs take 12-16 -- as long as it comes out shorter + * with the registers free here. The LDM's base is one of its own + * destinations (no writeback then). Registers load and store in ascending + * order, so any set keeps the words in place. */ + if (src_is_frame && src_aligned && !((base_bias + struct_src_offset) & 3)) + { + while (words - w >= 3) + { + const int src_off = base_bias + struct_src_offset + w * 4, dst_off = stack_offset + w * 4; + const int direct_sp = dst_off == 0; + uint32_t excl = (1u << (uint32_t)base_addr_reg) | (1u << R_SP) | (1u << R_PC); + int regs[6], got = 0; + int want = (words - w >= 4 ? 4 : 3) + !direct_sp; + /* LR is free at a call's argument setup: the call clobbers it. */ + regs[got++] = ARM_LR; + excl |= 1u << ARM_LR; + while (got < want) + { + int r = find_call_scratch_free(excl, arg_move_dst_mask); + if (r < 0 || r == R_SP || r == R_PC || (excl & (1u << (uint32_t)r)) || + (r == R_FP && tcc_state->need_frame_pointer)) + break; + regs[got++] = r; + excl |= 1u << (uint32_t)r; + } + int rb = -1; + if (!direct_sp) + { + /* The STM base: the lowest register, left out of the list. */ + int bi = 0; + for (int i = 1; i < got; i++) + if (regs[i] < regs[bi]) + bi = i; + if (got < 4) + break; + rb = regs[bi]; + regs[bi] = regs[--got]; + } + const int k = got < 4 ? got : 4; + if (k < 3) + break; + uint32_t list = 0; + int ra = -1, all_low = 1; + for (int i = 0; i < k; i++) + { + list |= 1u << (uint32_t)regs[i]; + all_low &= regs[i] <= R7; + if (regs[i] <= R7 && regs[i] > ra) + ra = regs[i]; + } + if (ra < 0) + ra = regs[0]; + thumb_opcode addr_a = src_off < 0 + ? th_sub_imm(ra, base_addr_reg, -src_off, flags_safe(), ENFORCE_ENCODING_NONE) + : th_add_imm(ra, base_addr_reg, src_off, flags_safe(), ENFORCE_ENCODING_NONE); + thumb_opcode addr_b = {0}; + if (!direct_sp) + addr_b = th_add_imm(rb, ARM_SP, dst_off, flags_safe(), ENFORCE_ENCODING_NONE); + const int ldm_size = all_low && ra <= R7 ? 2 : 4; + const int stm_size = !direct_sp && all_low && rb <= R7 ? 2 : 4; + if (!addr_a.size || (!direct_sp && !addr_b.size) || + addr_a.size + ldm_size + addr_b.size + stm_size >= 4 * k) + break; + ot_check(addr_a); + if (ldm_size == 2) + ot_check((thumb_opcode){.size = 2, .opcode = 0xC800u | ((uint32_t)ra << 8) | list}); + else + ot_check((thumb_opcode){.size = 4, .opcode = ((0xE890u | (uint32_t)ra) << 16) | list}); + if (direct_sp) + ot_check((thumb_opcode){.size = 4, .opcode = ((0xE880u | (uint32_t)R_SP) << 16) | list}); + else + { + ot_check(addr_b); + if (stm_size == 2) + ot_check((thumb_opcode){.size = 2, .opcode = 0xC000u | ((uint32_t)rb << 8) | list}); + else + ot_check((thumb_opcode){.size = 4, .opcode = ((0xE8A0u | (uint32_t)rb) << 16) | list}); + } + w += k; + } + } + + if (can_pair) + { + for (; w + 1 < words; w += 2) + { + int src_off = base_bias + struct_src_offset + w * 4; + int dst_off = stack_offset + w * 4; + + if (!(src_aligned && tcc_gen_machine_try_ldrd_base(data1, data2, base_addr_reg, src_off))) + { + load_struct_word_into(data1, base_addr_reg, src_off); + load_struct_word_into(data2, base_addr_reg, src_off + 4); + } + if (!tcc_gen_machine_try_strd_base(data1, data2, ARM_SP, dst_off)) + { + store_word_to_stack_keep(data1, dst_off, (1u << base_addr_reg) | (1u << data2)); + store_word_to_stack_safe(data2, dst_off + 4, base_addr_reg); + } + } + } + + /* Trailing odd word, or every word when pairing was unavailable. */ + for (; w < words; ++w) + { + int src_off = base_bias + struct_src_offset + w * 4; + int dst_off = stack_offset + w * 4; + load_struct_word_into(data1, base_addr_reg, src_off); + store_word_to_stack_safe(data1, dst_off, base_addr_reg); + } + if (have_struct_sc) + restore_scratch_reg(&struct_sc); +} + +/* Find a free scratch register via liveness (no push/pop). + * Returns the register number, or R_IP as last resort. + * Must not push/pop since SP-relative offsets for stack args would shift. + * + * Unlike tcc_ls_find_free_scratch_reg (which refuses callee-saved regs), + * this also considers callee-saved registers already pushed in the prologue. + * Those are safe to clobber because the epilogue will restore them. + * + * arg_move_dst_mask: registers that will be explicitly written by register + * arg moves AFTER stack arg placement. These are safe to clobber even if + * currently live, because the subsequent moves will overwrite them. + * Pass 0 when not in a pre-move stack arg placement context. */ +static int find_call_scratch(uint32_t extra_exclude, uint32_t arg_move_dst_mask) +{ + TCCIRState *ir = tcc_state->ir; + uint32_t exclude = scratch_global_exclude | extra_exclude; + if (ir) + { + /* Standard path: try caller-saved regs via liveness */ + int reg = tcc_ls_find_free_scratch_reg(&ir->ls, ir->codegen_instruction_idx, exclude, ir->leaffunc); + if (reg != PREG_NONE && reg >= 0 && reg < 16 && reg != R_SP && reg != R_PC) + return reg; + + /* Extended path: try callee-saved regs that are already pushed in prologue + * AND not live at this instruction (so we won't clobber active values). */ + if (ir->ls.live_regs_by_instruction && ir->codegen_instruction_idx >= 0 && + ir->codegen_instruction_idx < ir->ls.live_regs_by_instruction_size) + { + uint32_t live = ir->ls.live_regs_by_instruction[ir->codegen_instruction_idx]; + uint32_t callee_pushed = pushed_registers & 0x0FF0u; /* R4-R11 that were pushed */ + /* R7 is the frame base: liveness never models it (it is not an + * interval), so it ALWAYS looks dead here — handing it out clobbered + * the FP mid-marshal and every later FP-relative address computed + * from garbage (lac_slot_key: `ldr r7,[ip,#8]` for a struct stack + * arg, then `sub ip, r7, #12` for the next arg). R9 is the GOT + * base under text_and_data_separation. Mirrors the reserved set in + * scratch_pushed_dead_reg. */ + uint32_t reserved = (1u << R_FP) | scratch_exclude_baseline(); + if (tcc_state->text_and_data_separation) + reserved |= (1u << 9); + uint32_t candidates = callee_pushed & ~live & ~exclude & ~reserved; + if (candidates) + { + /* Prefer low registers (R4-R7) for 16-bit encoding */ + int r = (int)__builtin_ctz(candidates); + return r; + } + } + + /* Pre-move path: registers that are destinations of explicit (non-identity) + * register arg moves can be used as scratch — the moves will overwrite them. + * Prefer low registers for 16-bit encoding. */ + if (arg_move_dst_mask) + { + uint32_t candidates = arg_move_dst_mask & ~exclude; + if (candidates) + { + int r = (int)__builtin_ctz(candidates); + if (r >= 0 && r < 16 && r != R_SP && r != R_PC) + return r; + } + } + } + return R_IP; +} + +/* find_call_scratch without its last resort: -1 instead of R_IP while IP is + * taken -- live in the allocation (an argument still to be stored, say), or + * handed out as a scratch already. find_call_scratch returns R_IP then too, + * which a caller about to load into it cannot use. */ +static int find_call_scratch_free(uint32_t extra_exclude, uint32_t arg_move_dst_mask) +{ + const int r = find_call_scratch(extra_exclude, arg_move_dst_mask); + TCCIRState *ir = tcc_state->ir; + if (r != R_IP || !ir || (arg_move_dst_mask & (1u << R_IP))) + return r; + const uint32_t exclude = scratch_global_exclude | extra_exclude; + if ((exclude & (1u << R_IP)) || + tcc_ls_find_free_scratch_reg(&ir->ls, ir->codegen_instruction_idx, ~(1u << R_IP), ir->leaffunc) != R_IP) + return -1; + return r; +} + +/* Place a 64-bit argument on stack (MOP path) */ +static void place_stack_arg_64bit(const MachineOperand *mop, int stack_offset, TCCIRState *ir, + uint32_t arg_move_dst_mask) +{ + int lo_offset = stack_offset; + int hi_offset = stack_offset + 4; + + if (mop->kind == MACH_OP_REG && !mop->needs_deref && thumb_is_hw_reg(mop->u.reg.r0) && thumb_is_hw_reg(mop->u.reg.r1)) + { + /* If either register is R0-R3, the value was already stored by + * presave_stack_args_from_arg_regs before the register shuffle. */ + if (mop->u.reg.r0 <= ARM_R3 || mop->u.reg.r1 <= ARM_R3) + return; + /* The two halves land on adjacent, word-aligned SP slots — exactly STRD's + * shape. (STRD to the outgoing argument area is always safe: SP-relative, + * word-multiple offset, SP 8-aligned.) */ + if (!tcc_gen_machine_try_strd_base(mop->u.reg.r0, mop->u.reg.r1, ARM_SP, lo_offset)) + { + store_word_to_stack_keep(mop->u.reg.r0, lo_offset, 1u << mop->u.reg.r1); + store_word_to_stack(mop->u.reg.r1, hi_offset); + } + } + else if (mop->kind == MACH_OP_IMM) + { + uint64_t imm64 = (uint64_t)mop->u.imm.val; + uint32_t lo_val = (uint32_t)imm64; + uint32_t hi_val = (uint32_t)(imm64 >> 32); + int scr = find_call_scratch(0, arg_move_dst_mask); + load_immediate(scr, lo_val, NULL, false); + + /* Pair the two halves into one STRD. When both halves are equal (every + * `0.0`/`0LL` variadic argument) the single scratch is stored twice — + * try_strd_pair permits Rt == Rt2 (only LDRD forbids it). Otherwise take a + * second scratch for the high half; find_call_scratch falls back to R_IP + * without honouring the exclude mask, so a collision means no pair. */ + int hi_scr = scr; + if (hi_val != lo_val) + { + hi_scr = find_call_scratch(1u << (uint32_t)scr, arg_move_dst_mask); + if (hi_scr == scr) + { + /* No second scratch: fall back to the sequential two-store form. */ + store_word_to_stack(scr, lo_offset); + load_immediate(scr, hi_val, NULL, false); + store_word_to_stack(scr, hi_offset); + return; + } + load_immediate(hi_scr, hi_val, NULL, false); + } + + if (!tcc_gen_machine_try_strd_base(scr, hi_scr, ARM_SP, lo_offset)) + { + store_word_to_stack_keep(scr, lo_offset, 1u << hi_scr); + store_word_to_stack(hi_scr, hi_offset); + } + } + else if (mop->needs_deref && !mach_op_64_names_memory(mop)) + { + /* The operand holds a pointer (in reg, spill, etc.), not the 64-bit + * value itself. Load the pointer into a register, then fetch the + * lo/hi halves from [ptr+0] and [ptr+4]. Splitting via + * mach_make_hi_half would incorrectly adjust the storage location + * (e.g. spill offset) instead of the dereference offset. + * + * PARAM_STACK and CHAIN_REL are excluded (mach_op_64_names_memory): + * mach_ensure_in_reg loads those directly from the caller's argument + * area / the parent frame (ignoring needs_deref), so the else path with + * mach_make_lo/hi_half handles them correctly. + * + * The base register must NOT be the scratch because both halves are + * loaded into the scratch. If base == scratch the first load would + * clobber the pointer before the second load can use it. */ + int scr = find_call_scratch(0, arg_move_dst_mask); + int base; + MachineCodegenContext mctx = {0}; + bool need_release = false; + if (mop->kind == MACH_OP_REG && mop->u.reg.r0 != scr) + { + base = mop->u.reg.r0; + } + else + { + MachineOperand addr = *mop; + addr.needs_deref = false; + addr.is_64bit = false; + addr.btype = IROP_BTYPE_INT32; + base = mach_ensure_in_reg(&mctx, &addr, (1u << scr)); + need_release = true; + } + load_from_base(scr, PREG_REG_NONE, IROP_BTYPE_INT32, 0, 0, 0, (uint32_t)base); + store_word_to_stack_keep(scr, lo_offset, 1u << base); + load_from_base(scr, PREG_REG_NONE, IROP_BTYPE_INT32, 0, 4, 0, (uint32_t)base); + store_word_to_stack(scr, hi_offset); + if (need_release) + mach_release_all(&mctx); + } + else + { + /* Load each 32-bit half individually. Override btype to INT32 so that + * mach_ensure_in_reg → load_from_base does a single-word LDR instead + * of a 64-bit pair load (which would allocate an extra scratch via push, + * shift SP, and corrupt the SP-relative store offsets below). */ + MachineOperand lo = mach_make_lo_half(mop); + MachineOperand hi = mach_make_hi_half(mop); + lo.btype = IROP_BTYPE_INT32; + hi.btype = IROP_BTYPE_INT32; + MachineCodegenContext mctx = {0}; + int r_lo = mach_ensure_in_reg(&mctx, &lo, 0); + store_word_to_stack_safe(r_lo, lo_offset, r_lo); + mach_release_all(&mctx); + mctx = (MachineCodegenContext){0}; + int r_hi = mach_ensure_in_reg(&mctx, &hi, 0); + store_word_to_stack_safe(r_hi, hi_offset, r_hi); + mach_release_all(&mctx); + } +} + +/* Place a 32-bit argument on stack (MOP path) */ +static void place_stack_arg_32bit(const MachineOperand *mop, int stack_offset, CallGenContext *ctx) +{ + switch (mop->kind) + { + case MACH_OP_REG: + if (!mop->needs_deref) + { + /* Skip R0-R3 sources — handled in pre-shuffle save. */ + if (mop->u.reg.r0 <= ARM_R3) + return; + store_word_to_stack(mop->u.reg.r0, stack_offset); + } + else + { + /* Register-indirect: load through the register, then store to stack. + * Must use btype-aware load so that byte/short values are properly + * zero/sign-extended (LDRB/LDRH) instead of always doing a word LDR. */ + int scr = find_call_scratch(1u << mop->u.reg.r0, ctx->arg_move_dst_mask); + load_from_base(scr, PREG_REG_NONE, mop->btype, mop->is_unsigned, 0, 0, mop->u.reg.r0); + store_word_to_stack(scr, stack_offset); + } + break; + + case MACH_OP_IMM: + { + int scr = find_call_scratch(0, ctx->arg_move_dst_mask); + load_immediate(scr, (uint32_t)mop->u.imm.val, NULL, false); + store_word_to_stack(scr, stack_offset); + break; + } + + case MACH_OP_SYMBOL: + { + int scr = find_call_scratch(0, ctx->arg_move_dst_mask); + Sym *sym = mop->u.sym.sym ? validate_sym_for_reloc(mop->u.sym.sym) : NULL; + if (mop->needs_deref) + { + /* Load value from global symbol address. */ + load_immediate(scr, 0, sym, false); + int32_t addend = mop->u.sym.addend; + int sign = (addend < 0); + int abs_off = sign ? -addend : addend; + load_from_base(scr, PREG_REG_NONE, mop->btype, mop->is_unsigned, abs_off, sign, scr); + } + else + { + load_immediate(scr, (uint32_t)mop->u.sym.addend, sym, false); + } + store_word_to_stack(scr, stack_offset); + break; + } + + default: + { + /* SPILL, PARAM_STACK, FRAME_ADDR, CHAIN_REL: generic MOP load. */ + MachineCodegenContext mctx = {0}; + int r = mach_ensure_in_reg(&mctx, mop, 0); + store_word_to_stack(r, stack_offset); + mach_release_all(&mctx); + break; + } + } +} + +/* Build all register argument moves */ +static int build_register_arg_moves(CallGenContext *ctx, ThumbArgMove *reg_moves) +{ + int move_count = 0; + + for (int i = 0; i < ctx->argc; ++i) + { + const TCCAbiArgLoc *loc = &ctx->layout->locs[i]; + const IROperand *arg = &ctx->args[i]; + const MachineOperand *mop = &ctx->mops[i]; + const int bt = irop_get_btype(*arg); + const int is_64bit = mop->is_64bit; + + if (loc->kind != TCC_ABI_LOC_REG && loc->kind != TCC_ABI_LOC_REG_STACK) + continue; + + int base_reg = ARM_R0 + loc->reg_base; + + if (bt == IROP_BTYPE_STRUCT || arg->is_complex) + { + /* Complex values already in a register pair hold the actual value, + * not a pointer to it. Route through individual register moves + * instead of the struct-copy path (which dereferences as an address). */ + if (arg->is_complex && mop->kind == MACH_OP_REG && !mop->needs_deref && mop->is_64bit) + { + int words = loc->reg_count; + if (words >= 1 && mop->u.reg.r0 != base_reg) + reg_moves[move_count++] = + (ThumbArgMove){.kind = THUMB_ARG_MOVE_REG, .dst_reg = base_reg, .src_reg = mop->u.reg.r0}; + if (words >= 2 && mop->u.reg.r1 != (base_reg + 1)) + reg_moves[move_count++] = + (ThumbArgMove){.kind = THUMB_ARG_MOVE_REG, .dst_reg = base_reg + 1, .src_reg = mop->u.reg.r1}; + for (int w = 0; w < words; w++) + ctx->call_site->registers_map |= (1 << (base_reg + w)); + } + else if (arg->is_complex && mop->kind == MACH_OP_IMM) + { + /* Complex immediate: split 64-bit packed value (real_lo | imag_hi) + * into individual 32-bit register moves. */ + const uint64_t imm64 = (uint64_t)mop->u.imm.val; + int words = loc->reg_count; + if (words >= 1) + reg_moves[move_count++] = + (ThumbArgMove){.kind = THUMB_ARG_MOVE_IMM, .dst_reg = base_reg, .imm = (uint32_t)imm64, .sym = NULL}; + if (words >= 2) + reg_moves[move_count++] = (ThumbArgMove){ + .kind = THUMB_ARG_MOVE_IMM, .dst_reg = base_reg + 1, .imm = (uint32_t)(imm64 >> 32), .sym = NULL}; + for (int w = 0; w < words; w++) + ctx->call_site->registers_map |= (1 << (base_reg + w)); + } + else + { + int src_align = 0; + irop_type_size_align(*arg, &src_align); + move_count = build_reg_move_struct(reg_moves, move_count, mop, loc, base_reg, ctx->call_site, src_align); + } + } + else if (is_64bit) + { + if (loc->reg_count < 2) + tcc_error("compiler_error: 64-bit register argument has insufficient registers"); + move_count = build_reg_move_64bit(reg_moves, move_count, mop, arg, base_reg, ctx->call_site, tcc_state->ir); + } + else + { + move_count = build_reg_move_32bit(reg_moves, move_count, mop, arg, base_reg, ctx->call_site, tcc_state->ir); + } + } + + return move_count; +} + +/* Place hard-float single-precision arguments into their VFP argument registers + * (s0..s15). Emitted AFTER the GPR argument moves so r0-r3 already hold their + * final values. VFP-resident sources form a permutation among the s-registers, + * resolved as a parallel move with VFP_SCRATCH0 (s14) breaking any cycle; other + * sources (imm/memory) are materialized through a GPR scratch that excludes the + * GPR argument registers. */ +static void emit_vfp_arg_moves(CallGenContext *ctx) +{ + int dst[16], src[16]; + const MachineOperand *mop[16]; + int n = 0; + const uint32_t gpr_excl_all = (uint32_t)(ctx->call_site->registers_map & 0xffffu); + + for (int i = 0; i < ctx->argc && n < 16; ++i) + { + const TCCAbiArgLoc *loc = &ctx->layout->locs[i]; + if (loc->kind != TCC_ABI_LOC_VFP_REG) + continue; + + /* Double: the value lives in a GPR pair (doubles are not VFP-resident — + * there is no double-precision arithmetic on this FPU), so pack the two + * halves straight into the d-register. Emitted immediately: d-registers + * are not sources for any other argument move. */ + if (loc->reg_count == 2) + { + MachineCodegenContext mctx = {0}; + MachineOperand m = ctx->mops[i]; + MachineOperand lo = mach_make_lo_half(&m); + MachineOperand hi = mach_make_hi_half(&m); + lo.btype = IROP_BTYPE_INT32; + hi.btype = IROP_BTYPE_INT32; + int rlo = mach_ensure_in_reg(&mctx, &lo, gpr_excl_all); + int rhi = mach_ensure_in_reg(&mctx, &hi, gpr_excl_all | (1u << (uint32_t)rlo)); + ot_check(th_vmov_2gp_dp((uint16_t)rlo, (uint16_t)rhi, (uint16_t)(loc->reg_base / 2), 0)); + mach_release_all(&mctx); + continue; + } + + dst[n] = loc->reg_base; + const MachineOperand *m = &ctx->mops[i]; + src[n] = (m->kind == MACH_OP_VFP_REG) ? m->u.reg.r0 : -1; /* -1 = materialize */ + mop[n] = m; + n++; + } + if (n == 0) + return; + + const uint32_t gpr_excl = gpr_excl_all; + int remaining = n; + while (remaining > 0) + { + int progressed = 0; + for (int i = 0; i < n; ++i) + { + if (dst[i] < 0) + continue; + /* Safe to emit when this destination is not still needed as a source. */ + int needed = 0; + for (int j = 0; j < n; ++j) + if (dst[j] >= 0 && j != i && src[j] == dst[i]) + { + needed = 1; + break; + } + if (needed) + continue; + if (src[i] >= 0) + { + if (src[i] != dst[i]) + ot_check(th_vmov_register((uint16_t)dst[i], (uint16_t)src[i], 0)); /* vmov.f32 sd, ss */ + } + else + { + MachineCodegenContext mctx = {0}; + int gpr = mach_ensure_in_reg(&mctx, mop[i], gpr_excl); + ot_check(th_vmov_gp_sp((uint16_t)gpr, (uint16_t)dst[i], 0)); /* sd = gpr */ + mach_release_all(&mctx); + } + dst[i] = -1; + remaining--; + progressed = 1; + } + if (!progressed) + { + /* Only cycles of VFP-resident moves remain: park one source in s14. */ + for (int i = 0; i < n; ++i) + if (dst[i] >= 0 && src[i] >= 0) + { + ot_check(th_vmov_register(VFP_SCRATCH0, (uint16_t)src[i], 0)); + src[i] = VFP_SCRATCH0; + break; + } + } + } +} + +/* Pre-save stack arguments that source from R0-R3 before register shuffle */ +static void presave_stack_args_from_arg_regs(CallGenContext *ctx) +{ + for (int i = 0; i < ctx->argc; ++i) + { + const TCCAbiArgLoc *loc = &ctx->layout->locs[i]; + const MachineOperand *mop = &ctx->mops[i]; + const int bt = mop->btype; + + if (loc->kind == TCC_ABI_LOC_REG || loc->kind == TCC_ABI_LOC_VFP_REG) + continue; + if (bt == IROP_BTYPE_STRUCT || mop->is_complex) + continue; + if (mop->kind != MACH_OP_REG || mop->needs_deref) + continue; + + if (mop->is_64bit) + { + /* Pre-save 64-bit register pair if either register is in R0-R3. + * The register arg shuffle will overwrite R0-R3, so both halves + * must be stored to the stack before that happens. */ + int r0 = mop->u.reg.r0; + int r1 = mop->u.reg.r1; + if ((thumb_is_hw_reg(r0) && r0 <= ARM_R3) || (thumb_is_hw_reg(r1) && r1 <= ARM_R3)) + { + int stack_offset = loc->stack_off; + if (thumb_is_hw_reg(r0)) + store_word_to_stack(r0, stack_offset); + if (thumb_is_hw_reg(r1)) + store_word_to_stack(r1, stack_offset + 4); + } + } + else + { + /* Only pre-save if operand is in R0-R3 (arg registers that get overwritten). */ + if (mop->u.reg.r0 <= ARM_R3) + { + store_word_to_stack(mop->u.reg.r0, loc->stack_off); + } + } + } +} + +/* True for a plain 32-bit immediate argument destined for a stack slot. */ +static int is_simple_imm_stack_arg(const TCCAbiArgLoc *loc, const MachineOperand *mop) +{ + return loc->kind != TCC_ABI_LOC_REG && loc->kind != TCC_ABI_LOC_VFP_REG && mop->kind == MACH_OP_IMM && + !mop->is_64bit && mop->btype != IROP_BTYPE_STRUCT && !mop->is_complex; +} + +/* Order by 4 KB window, then value, then offset. Grouping equal values within a + * window lets each distinct value be materialized once per window instead of once + * per argument; the window ordering bounds base-register re-materialization. */ +static int stack_imm_arg_cmp(const void *a, const void *b) +{ + const StackImmArg *x = (const StackImmArg *)a; + const StackImmArg *y = (const StackImmArg *)b; + int wx = x->off & ~0xFFF, wy = y->off & ~0xFFF; + if (wx != wy) + return wx < wy ? -1 : 1; + if (x->val != y->val) + return x->val < y->val ? -1 : 1; + if (x->off != y->off) + return x->off < y->off ? -1 : 1; + return 0; +} + +/* Emit a single non-simple-immediate stack argument (struct/complex/64-bit, or a + * non-immediate 32-bit source). Extracted from place_stack_arguments so both the + * inline and the grouped emission paths share identical handling. */ +static void place_one_stack_arg(CallGenContext *ctx, const TCCAbiArgLoc *loc, const MachineOperand *mop, + int stack_offset, int arg_index) +{ + if (mop->btype == IROP_BTYPE_STRUCT || mop->is_complex) + { + /* Complex values in a register pair: store the stack portion directly + * from registers instead of treating the pair as a memory pointer. */ + if (mop->is_complex && mop->kind == MACH_OP_REG && !mop->needs_deref && mop->is_64bit) + { + int words_in_regs = (loc->kind == TCC_ABI_LOC_REG_STACK) ? loc->reg_count : 0; + int stack_bytes = (loc->kind == TCC_ABI_LOC_REG_STACK) ? loc->stack_size : loc->size; + int stack_words = (stack_bytes + 3) / 4; + int pair_regs[2] = {mop->u.reg.r0, mop->u.reg.r1}; + for (int w = 0; w < stack_words; w++) + { + int reg_idx = words_in_regs + w; + if (reg_idx < 2) + store_word_to_stack(pair_regs[reg_idx], stack_offset + w * 4); + } + } + else if (mop->is_complex && mop->kind == MACH_OP_IMM) + { + /* Complex immediate on stack: split 64-bit packed value into words. */ + const uint64_t imm64 = (uint64_t)mop->u.imm.val; + int words_in_regs = (loc->kind == TCC_ABI_LOC_REG_STACK) ? loc->reg_count : 0; + int stack_bytes = (loc->kind == TCC_ABI_LOC_REG_STACK) ? loc->stack_size : loc->size; + int stack_words = (stack_bytes + 3) / 4; + int scr = find_call_scratch(0, ctx->arg_move_dst_mask); + for (int w = 0; w < stack_words; w++) + { + int word_idx = words_in_regs + w; + uint32_t word_val = (uint32_t)(imm64 >> (word_idx * 32)); + load_immediate(scr, word_val, NULL, false); + store_word_to_stack(scr, stack_offset + w * 4); + } + } + else + { + /* Struct's natural alignment gates source-side LDRD (see + * place_stack_arg_struct). Default conservatively to 1 (no LDRD) when + * the originating IR operand is unavailable. */ + int src_align = 1; + if (ctx->args && arg_index >= 0 && arg_index < ctx->argc) + { + int a = 0; + irop_type_size_align(ctx->args[arg_index], &a); + if (a > 0) + src_align = a; + } + place_stack_arg_struct(mop, loc, stack_offset, src_align, ctx->arg_move_dst_mask); + } + } + else if (mop->is_64bit) + place_stack_arg_64bit(mop, stack_offset, tcc_state->ir, ctx->arg_move_dst_mask); + else + place_stack_arg_32bit(mop, stack_offset, ctx); +} + +/* Inline (original-order) emission of every stack argument. Used for the common + * case where stack args stay within the immediate-offset store range. */ +static void place_stack_arguments_inline(CallGenContext *ctx) +{ + int cached_imm_reg = -1; + uint32_t cached_imm_val = 0; + + for (int i = 0; i < ctx->argc; ++i) + { + const TCCAbiArgLoc *loc = &ctx->layout->locs[i]; + const MachineOperand *mop = &ctx->mops[i]; + + if (loc->kind == TCC_ABI_LOC_REG || loc->kind == TCC_ABI_LOC_VFP_REG) + continue; + + int stack_offset = loc->stack_off; + + if (is_simple_imm_stack_arg(loc, mop)) + { + uint32_t val = (uint32_t)mop->u.imm.val; + int scr = find_call_scratch(0, ctx->arg_move_dst_mask); + if (cached_imm_reg != scr || cached_imm_val != val) + { + load_immediate(scr, val, NULL, false); + cached_imm_reg = scr; + cached_imm_val = val; + } + store_word_to_stack(scr, stack_offset); + continue; + } + + cached_imm_reg = -1; + place_one_stack_arg(ctx, loc, mop, stack_offset, i); + } +} + +/* Place all stack arguments. + * + * For the common case the inline path is byte-identical to before. When simple + * 32-bit immediate stack args spill beyond the immediate-offset store range + * (offset > 4092) — exactly where the naive path emits movw+indexed (3 instr/arg) + * — a windowed/grouped path is used instead: + * - a base register holds sp+window so each store is a single str.w [rb,#disp] + * (re-materialized only when crossing a 4 KB window, ~once / 1024 stores); + * - the immediate stores are reordered by (window, value) so each distinct + * value is loaded once per window rather than once per argument. + * Reordering pure-immediate stores to distinct, non-aliasing stack slots leaves + * the pre-call stack image unchanged, so it is observationally identical. */ +/* TCC_NO_STACK_ARG_GROUP restores the ungrouped inline path, the control arm + * of the windowed stack-argument A/B. */ +TCC_DBG_ENV_FLAG(th_no_stack_arg_group, "TCC_NO_STACK_ARG_GROUP") + +static void place_stack_arguments(CallGenContext *ctx) +{ + int max_imm_off = -1; + int imm_count = 0; + for (int i = 0; i < ctx->argc; ++i) + { + const TCCAbiArgLoc *loc = &ctx->layout->locs[i]; + const MachineOperand *mop = &ctx->mops[i]; + if (is_simple_imm_stack_arg(loc, mop)) + { + imm_count++; + if (loc->stack_off > max_imm_off) + max_imm_off = loc->stack_off; + } + } + + if (!(max_imm_off > 4092 && imm_count >= 2) || th_no_stack_arg_group()) + { + place_stack_arguments_inline(ctx); + return; + } + + /* --- Windowed/grouped path --- */ + + /* Pass 1: emit every non-simple-immediate stack arg first, in original order. */ + for (int i = 0; i < ctx->argc; ++i) + { + const TCCAbiArgLoc *loc = &ctx->layout->locs[i]; + const MachineOperand *mop = &ctx->mops[i]; + if (loc->kind == TCC_ABI_LOC_REG || loc->kind == TCC_ABI_LOC_VFP_REG || is_simple_imm_stack_arg(loc, mop)) + continue; + place_one_stack_arg(ctx, loc, mop, loc->stack_off, i); + } + + /* Reserve two stable scratch registers: rv (holds the value) and rb (base + * address). Both are free across the whole argument-setup region — the call's + * register args are moved in afterwards, and find_call_scratch only returns + * registers that are dead here or are arg-move destinations (overwritten + * later). Prefer the lower-numbered register for rv so value materialization + * can use the 16-bit MOVS encoding. */ + int s0 = find_call_scratch(0, ctx->arg_move_dst_mask); + int s1 = find_call_scratch(1u << s0, ctx->arg_move_dst_mask); + if (s1 < s0) + { + int t = s0; + s0 = s1; + s1 = t; + } + int rv = s0, rb = s1; + int regs_ok = (rv != rb && rv >= 0 && rv < 16 && rb >= 0 && rb < 16 && rv != ARM_SP && rv != ARM_PC && + rb != ARM_SP && rb != ARM_PC); + + StackImmArg *items = regs_ok ? tcc_malloc(sizeof(StackImmArg) * imm_count) : NULL; + if (!items) + { + /* Out of stable registers (or alloc failure): emit the immediate args inline. */ + int cached_imm_reg = -1; + uint32_t cached_imm_val = 0; + for (int i = 0; i < ctx->argc; ++i) + { + const TCCAbiArgLoc *loc = &ctx->layout->locs[i]; + const MachineOperand *mop = &ctx->mops[i]; + if (!is_simple_imm_stack_arg(loc, mop)) + continue; + uint32_t val = (uint32_t)mop->u.imm.val; + int scr = find_call_scratch(0, ctx->arg_move_dst_mask); + if (cached_imm_reg != scr || cached_imm_val != val) + { + load_immediate(scr, val, NULL, false); + cached_imm_reg = scr; + cached_imm_val = val; + } + store_word_to_stack(scr, loc->stack_off); + } + return; + } + + int n = 0; + for (int i = 0; i < ctx->argc; ++i) + { + const TCCAbiArgLoc *loc = &ctx->layout->locs[i]; + const MachineOperand *mop = &ctx->mops[i]; + if (!is_simple_imm_stack_arg(loc, mop)) + continue; + items[n].off = loc->stack_off; + items[n].val = (uint32_t)mop->u.imm.val; + n++; + } + qsort(items, n, sizeof(StackImmArg), stack_imm_arg_cmp); + + uint32_t saved_excl = scratch_global_exclude; + scratch_global_exclude |= (1u << rv) | (1u << rb); + + int cur_window = -1; /* base offset of the window currently in rb */ + int have_val = 0; + uint32_t cur_val = 0; + for (int k = 0; k < n; ++k) + { + int off = items[k].off; + uint32_t val = items[k].val; + int window = off & ~0xFFF; + int disp = off & 0xFFF; + int base_reg; + + if (window == 0) + { + base_reg = ARM_SP; /* sp+0 — store directly off sp, no base register needed */ + } + else + { + if (window != cur_window) + { + thumb_opcode op = th_add_imm(rb, ARM_SP, (uint32_t)window, flags_safe(), ENFORCE_ENCODING_NONE); + if (is_valid_opcode(op)) + ot(op); + else + { + load_full_const(rb, PREG_NONE, (uint32_t)window, 0); + ot_check(th_add_reg(rb, ARM_SP, rb, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); + } + } + base_reg = rb; + } + cur_window = window; + + if (!have_val || cur_val != val) + { + load_immediate(rv, val, NULL, false); + have_val = 1; + cur_val = val; + } + + if (!store_word_to_base(rv, base_reg, disp, 0)) + { + /* disp <= 4092 always encodes via str.w; keep a correct fallback regardless. */ + ScratchRegAlloc sc = get_scratch_reg_with_save((1u << rv) | (1u << base_reg)); + load_immediate(sc.reg, (uint32_t)off, NULL, false); + ot_check(th_str_reg(rv, ARM_SP, sc.reg, THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); + restore_scratch_reg(&sc); + } + } + + scratch_global_exclude = saved_excl; + tcc_free(items); +} + +/* Handle return value after call (MOP path). + * The 'dest_mop' describes where the return value must be written. + * mach_writeback_dest() handles all destination kinds: + * MACH_OP_REG — emit MOV dest.r0, ARM_R0 when needed + * MACH_OP_SPILL — emit STR R0 to the spill slot + * MACH_OP_PARAM_STACK — emit STR R0 to the param stack slot + * MACH_OP_NONE — no-op (void return or drop_value) + * 64-bit pairs (int64, double, complex float) are split into lo/hi halves + * via mach_make_lo_half / mach_make_hi_half (R0 → lo, R1 → hi). */ +/* True when a call targets a soft-float __aeabi_* runtime helper, which returns + * a float in R0 (soft ABI) even under hard-float — as opposed to a user + * function, which returns it in s0. The __aeabi_ namespace is reserved, so the + * name is an unambiguous signal. */ +static int mach_callee_is_aeabi(const MachineOperand *func_mop) +{ + if (func_mop->kind != MACH_OP_SYMBOL || !func_mop->u.sym.sym) + return 0; + const char *name = get_tok_str(func_mop->u.sym.sym->v, NULL); + return name && strncmp(name, "__aeabi_", 8) == 0; +} + +/* True when the callee is a variadic (FUNC_ELLIPSIS) function. Such a callee + * receives every argument by the base (GPR) standard, so FP arguments are not + * placed in VFP registers. Indirect calls (no symbol) are assumed non-variadic + * — function pointers to variadic functions are rare. */ +static int mach_callee_is_variadic(const MachineOperand *func_mop) +{ + if (func_mop->kind != MACH_OP_SYMBOL || !func_mop->u.sym.sym) + return 0; + Sym *s = func_mop->u.sym.sym; + return s->type.ref && s->type.ref->f.func_type == FUNC_ELLIPSIS; +} + +/* True when the callee is a function this translation unit has already + * emitted, so the call cannot leave the module and R9 survives it. + * + * Only an already-defined symbol qualifies: the test reads the ELF symbol, + * and a forward reference (callee defined later in the TU, or extern) is + * still SHN_UNDEF here, so it conservatively keeps the reload. That also + * makes the predicate stable across the dry run and the real pass, which it + * must be or literal-pool windows desync -- the dry run skips + * put_extern_sym, but that only ever creates UNDEF entries, so a symbol + * reads defined in both passes or neither. + * + * STT_FUNC is required so a same-named data symbol cannot qualify, and the + * target section must be executable: an SHN_ABS or data-section symbol is + * not a function body we compiled under this R9 contract. */ +/* -fmodule-local-calls: a global function with no definition in this + * translation unit binds inside the module unless a shared library in the + * library paths exports it (tcc_yaff_import_set_has). Per-TU builds of + * toybox, GNU make and tcc itself made every call to a function in another of + * their own files reload R9 -- 20,792 of the native tcc's 22,035 reloads. The + * assumption is recorded on the BL (R_ARM_YASOS_LOCAL_CALL) and the linker + * rejects the link if the call binds to the PLT after all. */ +static int thumb_callee_assumed_local(Sym *sym) +{ + /* An asm symbol is a helper the compiler calls on its own (__aeabi_memmove, + * __tcc_strcmp); libtcc1.a defines most of them, so they bind locally. */ + if (!tcc_state->module_local_calls || ((sym->type.t & VT_BTYPE) != VT_FUNC && !IS_ASM_SYM(sym))) + return 0; + /* A weak callee may bind to nothing at all; dllimport is the per-declaration + * opt-out for a function a library outside the library paths provides. */ + if (sym->a.weak || sym->a.dllimport) + return 0; + /* The link name, not the spelling: __builtin_printf binds as `printf`. */ + return tcc_yaff_import_set_has(tcc_state, get_tok_str(sym->asm_label ? sym->asm_label : sym->v, NULL)) == 0; +} + +/* Only an undefined callee has anything to verify at link time. */ +int thumb_callee_needs_local_call_marker(Sym *sym) +{ + ElfSym *esym; + if (!text_and_data_separation || !sym || (sym->v & SYM_FIELD)) + return 0; + if ((sym->type.t & VT_BTYPE) == VT_FUNC && (sym->type.t & VT_STATIC)) + return 0; + esym = elfsym(sym); + if (esym && esym->st_shndx != SHN_UNDEF) + return 0; + return thumb_callee_assumed_local(sym); +} + +/* libc's memcpy/memset/memcmp are PLT calls, which also cost the caller an R9 + * reload, and byte loops besides; libtcc1 has local versions with the same + * contract (its __aeabi_memcpy returns dest, see arm_mem.S). A direct call to + * one of them binds to the libtcc1 copy instead: gcall_or_jump_mop emits the + * helper, and thumb_sym_in_this_module counts the call as module-local, so it + * keeps R9. Not under -nostdlib (libc itself, which defines them) nor when this + * TU defines the function; only when the libtcc1.a in the library paths has + * the helper. Returns the helper symbol, or NULL to call `sym` as written. */ +Sym *thumb_local_libc_helper(Sym *sym) +{ + static const char *const map[][2] = { + {"memcpy", "__aeabi_memcpy"}, + {"memset", "__tcc_memset"}, + {"memcmp", "__tcc_memcmp"}, + }; + ElfSym *esym; + const char *name; + + if (!text_and_data_separation || !tcc_state->module_local_calls || tcc_state->nostdlib || !sym || + (sym->v & SYM_FIELD) || (sym->type.t & VT_BTYPE) != VT_FUNC || (sym->type.t & VT_STATIC) || + sym->a.weak || sym->a.dllimport) + return NULL; + esym = elfsym(sym); + if (esym && esym->st_shndx != SHN_UNDEF) + return NULL; + name = get_tok_str(sym->asm_label ? sym->asm_label : sym->v, NULL); + for (unsigned i = 0; i < sizeof(map) / sizeof(map[0]); i++) + if (!strcmp(name, map[i][0])) + return tcc_yaff_libtcc1_has(tcc_state, map[i][1]) ? external_helper_sym(tok_alloc_const(map[i][1])) : NULL; + return NULL; +} + +static int thumb_sym_in_this_module(Sym *sym) +{ + ElfSym *esym; + Section *sec; + + if (!text_and_data_separation || !sym) + return 0; + if (sym->v & SYM_FIELD) + return 0; + if (thumb_local_libc_helper(sym)) + return 1; + /* A static function is this translation unit's own, so this module's, + * whether or not its definition has been seen yet: calls to functions + * defined further down would otherwise reload R9 for nothing -- in the Zig C + * backend's output, where every function is static, 83,487 reloads. */ + if ((sym->type.t & VT_BTYPE) == VT_FUNC && (sym->type.t & VT_STATIC)) + return 1; + esym = elfsym(sym); + if (!esym || esym->st_shndx == SHN_UNDEF) + return thumb_callee_assumed_local(sym); + if (esym->st_shndx == SHN_ABS || esym->st_shndx >= tcc_state->nb_sections) + return 0; + if (ELFW(ST_TYPE)(esym->st_info) != STT_FUNC) + return 0; + sec = tcc_state->sections[esym->st_shndx]; + return sec && (sec->sh_flags & SHF_EXECINSTR) != 0; +} +int thumb_callee_in_this_module(const MachineOperand *func_mop) +{ + if (func_mop->kind != MACH_OP_SYMBOL) + return 0; + return thumb_sym_in_this_module(func_mop->u.sym.sym); +} + +/* A direct call that never returns needs no R9 reload after it: nothing after + * it runs. (A longjmp out of the callee comes back through setjmp's own call + * site, which reloads.) */ +int thumb_callee_noreturn(const MachineOperand *func_mop) +{ + Sym *sym; + if (func_mop->kind != MACH_OP_SYMBOL || !(sym = func_mop->u.sym.sym) || (sym->v & SYM_FIELD)) + return 0; + return (sym->type.t & VT_BTYPE) == VT_FUNC && sym->type.ref && sym->type.ref->f.func_noreturn; +} + +ST_FUNC int tcc_gen_machine_callee_in_this_module(Sym *sym) +{ + return thumb_sym_in_this_module(sym); +} + +static void handle_return_value_mop(const MachineOperand *dest_mop, int drop_value, int soft_float_return) +{ + if (drop_value) + return; + + /* Hard-float single-precision return arrives in s0 (AAPCS VFP) — unless the + * callee is a soft __aeabi_* helper, which returns it in R0. */ + if (tcc_state && tcc_state->float_abi == ARM_HARD_FLOAT && !soft_float_return && !dest_mop->is_64bit && + dest_mop->btype == IROP_BTYPE_FLOAT32) + { + if (dest_mop->kind == MACH_OP_VFP_REG) + { + if (dest_mop->u.reg.r0 != 0) + ot_check(th_vmov_register((uint16_t)dest_mop->u.reg.r0, 0, 0)); /* s_dest = s0 */ + } + else + { + MachineCodegenContext ctx = {0}; + int r = mach_get_dest_reg(&ctx, dest_mop, 0); + ot_check(th_vmov_gp_sp((uint16_t)r, 0, 1)); /* r = s0 */ + mach_writeback_dest(dest_mop, r); + mach_release_all(&ctx); + } + return; + } + + /* Hard-float double return arrives in d0 (AAPCS VFP) — unless the callee is a + * soft __aeabi_* helper, whose result comes back in the R0:R1 pair. */ + if (tcc_state && tcc_state->float_abi == ARM_HARD_FLOAT && !soft_float_return && dest_mop->is_64bit && + dest_mop->btype == IROP_BTYPE_FLOAT64 && !dest_mop->is_complex) + { + MachineCodegenContext ctx = {0}; + MachineOperand lo = mach_make_lo_half(dest_mop); + MachineOperand hi = mach_make_hi_half(dest_mop); + lo.btype = IROP_BTYPE_INT32; + hi.btype = IROP_BTYPE_INT32; + int rlo = mach_get_dest_reg(&ctx, &lo, 0); + int rhi = mach_get_dest_reg(&ctx, &hi, (1u << (uint32_t)rlo)); + ot_check(th_vmov_2gp_dp((uint16_t)rlo, (uint16_t)rhi, 0, 1)); /* rlo,rhi = d0 */ + mach_writeback_dest(&lo, rlo); + mach_writeback_dest(&hi, rhi); + mach_release_all(&ctx); + return; + } + + if (dest_mop->is_64bit) + { + /* 64-bit return value: R0 = low word, R1 = high word (AAPCS). */ + MachineOperand lo = mach_make_lo_half(dest_mop); + lo.btype = IROP_BTYPE_INT32; + MachineOperand hi = mach_make_hi_half(dest_mop); + hi.btype = IROP_BTYPE_INT32; + mach_writeback_dest(&lo, ARM_R0); + mach_writeback_dest(&hi, ARM_R1); + return; + } + mach_writeback_dest(dest_mop, ARM_R0); +} + +/* ======================================================================== */ + +/* tcc_gen_machine_func_call_mop — MOP-path function call code generator. + * + * The function target and return-value destination are passed as MachineOperands. + * The call_id_op is always an immediate IROperand (no fill needed). + * + * Phase 5g: func_mop replaces the old filled IROperand func_target. + * gcall_or_jump_mop() replaces gcall_or_jump_ir(). + */ +/* Largest copy an aligned __aeabi_memmove4/8 or __aeabi_memcpy4/8 call is + * unrolled inline for; larger ones become a copy loop, and 0 keeps every + * call. Each such call is a struct assignment (vstore.c): the Zig C backend + * copies 20-128 byte aggregates millions of times, and every call ran ~45 + * instructions in the helper for what four registers of LDM/STM do in a few. */ +TCC_DBG_ENV_INT(inline_copy_max, "TCC_INLINE_COPY_MAX", 128) + +/* True when a pointer argument is provably word aligned: the address of a + * frame slot at a word offset (the frame base is 8-aligned), or of a symbol + * defined here in a section aligned to 4 at a word offset -- a string + * literal's or an initialiser template's .rodata copy. */ +static int thumb_copy_arg_word_aligned(TCCIRState *ir, IROperand op) +{ + if (op.is_lval || op.is_llocal) + return 0; + const int tag = irop_get_tag(op); + if (tag == IROP_TAG_STACKOFF) + return op.is_local && irop_get_vreg(op) < 0 && !(irop_get_stack_offset(op) & 3); + if (tag != IROP_TAG_SYMREF || !ir) + return 0; + IRPoolSymref *sr = irop_get_symref_ex(ir, op); + if (!sr || !sr->sym) + return 0; + ElfSym *esym = elfsym(sr->sym); + if (!esym || esym->st_shndx == SHN_UNDEF || esym->st_shndx >= SHN_LORESERVE || + esym->st_shndx >= (unsigned)tcc_state->nb_sections || ELFW(ST_BIND)(esym->st_info) != STB_LOCAL) + return 0; + Section *sec = tcc_state->sections[esym->st_shndx]; + return sec && sec->sh_addralign >= 4 && !((esym->st_value + (addr_t)sr->addend) & 3); +} + +/* Two frame slots a memmove could be shifting within one buffer: the forward + * word copy is only right when they coincide or are disjoint. */ +static int thumb_copy_args_overlap(const IROperand *args, int n) +{ + if (irop_get_tag(args[0]) != IROP_TAG_STACKOFF || irop_get_tag(args[1]) != IROP_TAG_STACKOFF) + return 0; + const int d = irop_get_stack_offset(args[0]) - irop_get_stack_offset(args[1]); + return d && d > -n && d < n; +} + +/* A call to an aligned block copy with a constant size, emitted as the copy + * itself: dst in R0, src in R1 (the argument moves are done), 16 bytes per + * `ldmia r1!, {r2,r3,r12,lr}` / `stmia r0!` pair, the rest in one smaller + * pair or a single word. Every register it touches is one the call would + * have clobbered -- the IR still sees a call, so the allocator keeps nothing + * live in R0-R3/R12, and the function saves LR. Both pointers are word + * aligned (the helper's contract, or thumb_copy_arg_word_aligned's proof for + * the plain __aeabi_memcpy/memmove), which LDM/STM need. Source and + * destination are the same object or disjoint objects (C struct assignment + * allows no partial overlap), and each chunk is loaded before it is stored, so + * the memmove variants are safe copied forwards. */ +/* How thumb_inline_aligned_copy_call will emit this call: 0 as a call, 1 by + * thumb_emit_word_copy_r0_r1, 2 as unrolled 16-byte LDM/STM pairs. Decided + * before the argument moves, so the size never goes to R2 for an expansion + * that does not read it (`movs r2, #72` right before the `ldmia r1!, + * {r2, r3, ip, lr}` overwriting it: 7k dead moves in zig.c). */ +static int thumb_inline_copy_mode(TCCIRState *ir, MachineOperand func_mop, const IROperand *args, int argc) +{ + if (func_mop.kind != MACH_OP_SYMBOL || !func_mop.u.sym.sym || func_mop.u.sym.addend || argc != 3) + return 0; + const char *nm = get_tok_str(func_mop.u.sym.sym->v, NULL); + IROperand n_op = args[2]; + if (irop_get_tag(n_op) != IROP_TAG_IMM32 || n_op.is_lval) + return 0; + int n = irop_get_imm32(n_op); + /* -Os: a copy longer than the largest copy stub stays the call (a MOVS and + * a BL) -- inline it is a 14-byte LDM/STM loop plus its count and tail. */ + if (tcc_state->optimize_size && n > COPY_STUB_MAX_WORDS * 4) + return 0; + /* An unaligned-contract copy whose pointers are word aligned after all: a + * string literal copied into a char array field of a word-aligned local + * (Zig's `undefined` fill, 48 bytes in every Wyhash.init). */ + if ((!strcmp(nm, "__aeabi_memcpy") || !strcmp(nm, "__aeabi_memmove")) && n >= 8 && inline_copy_max() && + thumb_copy_arg_word_aligned(ir, args[0]) && thumb_copy_arg_word_aligned(ir, args[1]) && + !thumb_copy_args_overlap(args, n)) + return 1; + if (strcmp(nm, "__aeabi_memmove4") && strcmp(nm, "__aeabi_memmove8") && strcmp(nm, "__aeabi_memcpy4") && + strcmp(nm, "__aeabi_memcpy8")) + return 0; + if (n <= 0 || (n & 3) || !inline_copy_max()) + return 0; + return n > inline_copy_max() ? 1 : 2; +} + +/* A call to an aligned block copy with a constant size, emitted as the copy + * itself: dst in R0, src in R1 (the argument moves are done), 16 bytes per + * `ldmia r1!, {r2,r3,r12,lr}` / `stmia r0!` pair, the rest in one smaller + * pair or a single word. Every register it touches is one the call would + * have clobbered -- the IR still sees a call, so the allocator keeps nothing + * live in R0-R3/R12, and the function saves LR. Both pointers are word + * aligned (the helper's contract, or thumb_copy_arg_word_aligned's proof for + * the plain __aeabi_memcpy/memmove), which LDM/STM need. Source and + * destination are the same object or disjoint objects (C struct assignment + * allows no partial overlap), and each chunk is loaded before it is stored, so + * the memmove variants are safe copied forwards. The size is not read: it + * is not moved to R2 (thumb_inline_copy_mode). */ +static int thumb_inline_aligned_copy_call(TCCIRState *ir, MachineOperand func_mop, const IROperand *args, int argc) +{ + const int mode = thumb_inline_copy_mode(ir, func_mop, args, argc); + if (!mode) + return 0; + int n = irop_get_imm32(args[2]); + if (mode == 1) + { + thumb_emit_word_copy_r0_r1(n); + return 1; + } + if (thumb_copy_stub_call(n)) + return 1; + const uint32_t quad = (1u << R2) | (1u << R3) | (1u << R_IP) | (1u << R_LR); + for (; n >= 16; n -= 16) + { + ot_check(th_ldm(R1, quad, 1, ENFORCE_ENCODING_NONE)); + ot_check(th_stm(R0, quad, 1, ENFORCE_ENCODING_NONE)); + } + if (n == 4) + { + ot_check(th_ldr_imm(R2, R1, 0, 6, ENFORCE_ENCODING_NONE)); + ot_check(th_str_imm(R2, R0, 0, 6, ENFORCE_ENCODING_NONE)); + } + else if (n) + { + uint32_t regs = n == 8 ? (1u << R2) | (1u << R3) : (1u << R2) | (1u << R3) | (1u << R_IP); + ot_check(th_ldm(R1, regs, 1, ENFORCE_ENCODING_NONE)); + ot_check(th_stm(R0, regs, 1, ENFORCE_ENCODING_NONE)); + } + return 1; +} + +ST_FUNC void tcc_gen_machine_func_call_mop(MachineOperand func_mop, IROperand call_id_op, MachineOperand dest_mop, + int drop_value, TCCIRState *ir, int call_idx) +{ + /* === Validation === */ + if (irop_is_none(call_id_op) || !ir) + tcc_error("compiler_error: func_call_op requires call_id+ir"); + + const int call_id = TCCIR_DECODE_CALL_ID(call_id_op.u.imm32); + const int argc_hint = TCCIR_DECODE_CALL_ARGC(call_id_op.u.imm32); + + /* The memory barrier of an inline atomic (parse_atomic): a call everywhere + * before this point, so every pass treats it as touching all memory, and + * one DMB here. On M-profile every DMB option behaves as SY. */ + if (func_mop.kind == MACH_OP_SYMBOL && func_mop.u.sym.sym && + !strcmp(get_tok_str(func_mop.u.sym.sym->v, NULL), "__tcc_dmb")) + { + ot_check(th_dmb(0xf)); + return; + } + + ThumbGenCallSite *call_site = thumb_get_call_site_for_id(call_id); + if (!call_site) + tcc_error("compiler_error: no call site found for call_id=%d", call_id); + + /* === Build ABI layout === */ + TCCAbiCallLayout layout; + memset(&layout, 0, sizeof(layout)); + + /* Hard-float VFP argument passing applies to non-variadic user functions. + * Soft __aeabi_* helpers take their float arguments in GPRs regardless, and a + * variadic callee passes every argument by the base (GPR) standard. */ + layout.hard_float = + (tcc_state && tcc_state->float_abi == ARM_HARD_FLOAT) && !mach_callee_is_aeabi(&func_mop); + layout.is_variadic = mach_callee_is_variadic(&func_mop); + + small_sequence(ThumbIROperandSequence) args_owner = {0}; + small_sequence(ThumbMachineOperandSequence) mops_owner = {0}; + const int argc = thumb_build_call_layout_from_ir(ir, call_idx, call_id, argc_hint, &layout, &args_owner, &mops_owner); + if (argc < 0) + tcc_error("compiler_error: failed to build call layout for call_id=%d", call_id); + + int stack_size = (argc > 0) ? (int)layout.stack_size : 0; + + /* === Setup call context === */ + CallGenContext ctx = { + .call_site = call_site, + .layout = &layout, + .args = ThumbIROperandSequence_data(&args_owner), + .mops = ThumbMachineOperandSequence_data(&mops_owner), + .argc = argc, + .stack_size = stack_size, + }; + + /* Set tail_call_pending if this is a tail-call-only function. */ + if (ir->tail_call_only) + tail_call_pending = 1; + + /* === Preserve nested call registers (R0-R3, R9) via STR to frame === + * Instead of PUSH/POP (which moves SP), store to the pre-reserved + * nested-call save area in the frame. SP stays fixed. */ + int arg_regs_in_use = call_site->registers_map & 0x0F; + int arg_regs_save_mask = tail_call_pending ? 0 : (arg_regs_in_use); + + /* On yasos with no-pic-data-is-text-relative, R9 holds the GOT base and is + * caller-saved. Save it alongside the nested-call argument registers so it + * is restored after the callee returns. */ + if (!tail_call_pending && text_and_data_separation) + arg_regs_save_mask |= (1 << ARM_R9); + + /* ...but a callee in THIS module shares our GOT base and hands it back + * intact, so its reload is dead. R9 is not allocatable under + * text_and_data_separation, tail calls are disabled outright in this mode + * (ir/codegen.c), and every call site below reloads R9, so a function here + * either never touches R9 or returns with its own module's base in it -- + * which for a same-module callee is the value we already hold. Only the + * cross-module path breaks that: an import goes through a loader thunk that + * swaps in the callee's base and tail-jumps, so nothing restores ours. + * + * Note this suppresses the RELOAD only, never the mask bit: the mask also + * assigns save-area slots, and clearing R9 from it would slide R0-R3 down + * onto slot 0 and overwrite the prologue-stored GOT base for every other + * call site in the function. */ + int restore_r9 = 1; + if (arg_regs_save_mask & (1 << ARM_R9)) + restore_r9 = !thumb_callee_in_this_module(&func_mop) && !thumb_callee_noreturn(&func_mop); + + /* Save nested-call registers to pre-reserved frame area via STR. + * The nested save area is at [SP + ir->call_outgoing_size]. + * + * In functions with VLA/alloca the runtime SP has moved below the static + * frame, so [SP + off] would land inside the dynamically allocated memory + * (the callee then overwrites the saved R9/GOT base with user data). + * Address the slots FP-relative instead, at the same offsets: the frame + * pointer is the static SP. */ + int nested_save_sp_offset = ir ? ir->call_outgoing_size : 0; + int nested_save_count = 0; + if (arg_regs_save_mask) + { + /* R9 takes slot 0 so that its frame offset is the same at every call site + * in the function. That is what lets the store be hoisted: the GOT base + * is function-invariant, so the single store in the prologue dominates + * every reload below and the per-call-site store is redundant. It used to + * be emitted here, and because R9 was slotted after whichever of R0-R3 + * happened to be live the offset moved from call to call, which is what + * hid the redundancy. The argument registers, whose live set genuinely + * does vary per call site, are packed above it. + * + * Emitting the store here again would be harmless but wasteful: it was + * 45,329 instructions (165 KiB) across the compiler's own build. */ + if (arg_regs_save_mask & (1 << ARM_R9)) + nested_save_count = 1; + for (int r = 0; r < 16; r++) + { + if (r == ARM_R9) + continue; + if (arg_regs_save_mask & (1 << r)) + { + if (tcc_state->func_dynamic_sp) + tcc_gen_machine_store_to_stack_ex( + r, nested_save_sp_offset + nested_save_count * 4, + arg_regs_save_mask); + else + store_word_to_stack(r, nested_save_sp_offset + nested_save_count * 4); + nested_save_count++; + } + } + } + if (nested_save_count > dry_run_state.max_nested_saves) + dry_run_state.max_nested_saves = nested_save_count; + + /* Stack args go to the pre-reserved outgoing area at [SP+0] -- unless this + * call passes more than the area holds (ir->call_dyn_extra): then SP drops + * by the difference for the call alone, the arguments land at the new SP, + * and every frame-relative access meanwhile adds the drop + * (call_args_sp_bias). The nested-call saves above are already done + * against the steady SP and are restored after the window closes. */ + stack_size = (stack_size + 7) & ~7; /* 8-byte align */ + const int dyn_extra = + ir->call_dyn_extra && call_id >= 0 && call_id < ir->call_dyn_extra_size ? ir->call_dyn_extra[call_id] : 0; + const int dyn_push_mark = scratch_push_count; + if (dyn_extra) + { + gadd_sp_ex(-dyn_extra, -1); + call_args_sp_bias += dyn_extra; + } + + /* === Save scratch exclusion state === */ + uint32_t saved_scratch_exclude = scratch_global_exclude; + + /* === Pre-save indirect call target if it resides in an argument register === + * + * When a function pointer is allocated to R0-R3 by the register allocator, + * the argument placement phase will overwrite those registers. Pre-move the + * pointer to a safe register before argument setup. + * + * Phase 5g: operates on MachineOperand func_mop instead of filled IROperand. + */ + { + const int is_direct = (func_mop.kind == MACH_OP_SYMBOL || func_mop.kind == MACH_OP_IMM); + if (!is_direct && func_mop.kind == MACH_OP_REG && !func_mop.needs_deref && func_mop.u.reg.r0 >= 0 && + func_mop.u.reg.r0 <= 3) + { + /* Find a free register outside R0-R3, R12 (stack-arg scratch), SP, PC. */ + uint32_t exclude = scratch_global_exclude | 0x0Fu | (1u << R_IP) | (1u << R_SP) | (1u << R_PC); + int safe_reg = PREG_NONE; + if (ir) + safe_reg = tcc_ls_find_free_scratch_reg(&ir->ls, ir->codegen_instruction_idx, exclude, ir->leaffunc); + + if (safe_reg == PREG_NONE || safe_reg < 0 || safe_reg >= 16 || safe_reg == R_SP || safe_reg == R_PC) + tcc_error("compiler_error: func_call_mop: cannot find safe register " + "to pre-save indirect call target (R%d)", + func_mop.u.reg.r0); + + /* Move function pointer from arg reg to safe reg. */ + thumb_shift no_shift = {THUMB_SHIFT_NONE, 0, THUMB_SHIFT_IMMEDIATE}; + ot_check_mov_reg(safe_reg, func_mop.u.reg.r0, flags_safe(), no_shift, ENFORCE_ENCODING_NONE, + false); + + /* Rewrite func_mop to point to the safe register. */ + func_mop.kind = MACH_OP_REG; + func_mop.u.reg.r0 = safe_reg; + func_mop.u.reg.r1 = -1; + func_mop.needs_deref = false; + + /* Protect the safe register from scratch allocation during arg setup. */ + scratch_global_exclude |= (1u << safe_reg); + } + } + + /* === Build register argument moves === */ + ThumbArgMove reg_moves[8]; + int reg_move_count = build_register_arg_moves(&ctx, reg_moves); + /* An inline-expanded copy never reads its size: no move into R2. */ + if (!tail_call_pending && thumb_inline_copy_mode(ir, func_mop, ctx.args, ctx.argc)) + { + int w = 0; + for (int i = 0; i < reg_move_count; i++) + if (!(reg_moves[i].dst_reg == R2 && reg_moves[i].kind != THUMB_ARG_MOVE_IMM64 && + reg_moves[i].kind != THUMB_ARG_MOVE_STRUCT)) + reg_moves[w++] = reg_moves[i]; + reg_move_count = w; + } + + /* === Compute arg_move_dst_mask and identity-move protection === + * + * Stack arguments are placed BEFORE register argument moves so that + * R0-R3 (non-identity move destinations) can serve as scratch registers + * for stack arg stores, saving 2 bytes per store (16-bit vs 32-bit encoding). + * + * arg_move_dst_mask: registers written by explicit (non-identity) reg moves. + * These will be overwritten by the moves, so they're safe as scratch. + * identity_mask: registers where the reg allocator already placed the correct + * value (no move entry created). These MUST be protected from clobbering. */ + { + uint32_t arg_move_dst_mask = 0; + /* Registers the pending moves still have to READ. A register can be both: + * `mov r1, r0` reads r0 while another move writes r0 (`add r0, sp, #52`), + * which put r0 in arg_move_dst_mask and so offered it as scratch -- and + * the stack-arg placement that took it destroyed the value the first move + * was about to copy. The parallel-move scheduler below already respects + * this (see the src_set it builds); placement, which runs first, did not. */ + uint32_t arg_move_src_mask = 0; + for (int i = 0; i < reg_move_count; i++) + { + arg_move_dst_mask |= arg_move_write_set(®_moves[i]); + if (reg_moves[i].kind == THUMB_ARG_MOVE_REG) + arg_move_src_mask |= (1u << reg_moves[i].src_reg); + } + + /* Compute all register-arg destination registers from the ABI layout. */ + uint32_t all_reg_arg_dst = 0; + for (int i = 0; i < ctx.argc; i++) + { + const TCCAbiArgLoc *loc = &ctx.layout->locs[i]; + if (loc->kind == TCC_ABI_LOC_REG || loc->kind == TCC_ABI_LOC_REG_STACK) + { + int base = ARM_R0 + loc->reg_base; + for (int w = 0; w < loc->reg_count; w++) + all_reg_arg_dst |= (1u << (base + w)); + } + } + + /* Protect identity-move registers (value already in place, no move entry). */ + uint32_t identity_mask = all_reg_arg_dst & ~arg_move_dst_mask; + scratch_global_exclude |= identity_mask; + + /* ...and every register a pending move reads, on every scratch path, not + * just the arg_move_dst_mask one: the value sitting there may have been + * materialised by the argument lowering itself and so have no live + * interval for the liveness query to find. */ + scratch_global_exclude |= arg_move_src_mask; + + ctx.arg_move_dst_mask = arg_move_dst_mask & ~arg_move_src_mask; + } + + /* Pre-save stack args sourcing from R0-R3 before register shuffle */ + presave_stack_args_from_arg_regs(&ctx); + + /* === Place stack arguments FIRST === + * R0-R3 that are non-identity move destinations can be used as scratch + * via arg_move_dst_mask in find_call_scratch, yielding 16-bit STR + * encodings instead of 32-bit STR.W with R12. */ + place_stack_arguments(&ctx); + + /* === Now block all R0-R3 and emit register argument moves === */ + scratch_global_exclude |= 0x0F; + thumb_emit_parallel_arg_moves(reg_moves, reg_move_count); + emit_vfp_arg_moves(&ctx); + + /* === Tail call: tear down frame before branching === */ + if (tail_call_pending) + { + /* For indirect calls, the target may be in a callee-saved register that + * will be popped. Move it to R_IP (R12) before frame teardown. */ + if (func_mop.kind == MACH_OP_REG && !func_mop.needs_deref && + func_mop.u.reg.r0 >= R4 && func_mop.u.reg.r0 <= R11) + { + ot_check_mov_reg(R_IP, func_mop.u.reg.r0, flags_safe(), + THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE, false); + func_mop.u.reg.r0 = R_IP; + } + if (epilogue_stack_dealloc > 0) + gadd_sp_ex(epilogue_stack_dealloc, R_IP); + /* Only pop true callee-saved registers (R4-R11), and LR when the prologue + * saved it (__builtin_return_address, or LR taken as a scratch): the + * callee returns straight to our caller, so LR must hold our return + * address again and SP must be back where our caller left it. R0-R3 may + * be pushed for alignment but now hold call arguments — popping them + * would clobber the prepared args. Skip non-callee slots FIRST (they sit + * at lower addresses after push), then pop callee-saved from correct + * position. */ + uint32_t callee_pop = pushed_registers & (0x0FF0u | (1u << R_LR)); + uint32_t non_callee = pushed_registers & ~callee_pop & ~(1u << R_LR) & ~(1u << R_PC); + int non_callee_bytes = __builtin_popcount(non_callee) * 4; + if (non_callee_bytes > 0) + gadd_sp_ex(non_callee_bytes, R_IP); + if (callee_pop) + ot_check(th_pop(callee_pop)); + } + + /* === Emit call === */ + if (tail_call_pending || !thumb_inline_aligned_copy_call(ir, func_mop, ctx.args, ctx.argc)) + gcall_or_jump_mop(0, func_mop); + if (dyn_extra) + { + /* A scratch PUSHed inside the window would be popped after SP moved back + * -- from the wrong address. The window is only opened where the scratch + * save slots stay in reach (ir/codegen.c, DYN_ARGS_MAX), so this is a + * compiler bug, not a limit. */ + for (int k = dyn_push_mark; k < scratch_push_count; k++) + if (scratch_push_type[k] == 1 && !dry_run_state.active) + tcc_error("compiler_error: scratch PUSH inside a call's argument window"); + call_args_sp_bias -= dyn_extra; + gadd_sp_ex(dyn_extra, -1); + } + /* Restore scratch register exclusion */ + scratch_global_exclude = saved_scratch_exclude; + + if (tail_call_pending) + { + tail_call_pending = 0; + goto call_cleanup; + } + + handle_return_value_mop(&dest_mop, drop_value, mach_callee_is_aeabi(&func_mop)); + + /* === Cleanup: restore nested-call saved registers via LDR === */ + if (arg_regs_save_mask) + { + /* Match the FP-relative addressing used by the save side in functions + * with VLA/alloca (runtime SP has moved; see the save block above). */ + const int restore_base = tcc_state->func_dynamic_sp ? R_FP : ARM_SP; + /* Mirror the slot assignment of the save block: R9 is at slot 0 (stored + * once in the prologue), the argument registers start above it. */ + int restore_idx = (arg_regs_save_mask & (1 << ARM_R9)) ? 1 : 0; + for (int r = 0; r < 16; r++) + { + if (r == ARM_R9 && !restore_r9) + continue; /* same-module callee: our GOT base is still live in R9 */ + if (arg_regs_save_mask & (1 << r)) + { + int slot = (r == ARM_R9) ? 0 : restore_idx; + int off = nested_save_sp_offset + slot * 4; + int sign = (off < 0); + int abs_off = sign ? -off : off; + /* R9 restore in text_and_data_separation mode needs the write guard + * temporarily lifted — the safety check blocks all R9 writes, but + * we are legitimately restoring it after a call. */ + if (r == ARM_R9 && text_and_data_separation) + allow_r9_write = 1; + if (!load_word_from_base(r, restore_base, abs_off, sign)) + { + ScratchRegAlloc osc = get_scratch_reg_with_save((1u << r)); + load_immediate(osc.reg, off, NULL, false); + ot_check(th_ldr_reg(r, restore_base, osc.reg, THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); + restore_scratch_reg(&osc); + } + if (r == ARM_R9 && text_and_data_separation) + allow_r9_write = 0; + if (r != ARM_R9) + restore_idx++; + } + } + } + + call_site->registers_map &= ~0x0F; /* Clear R0-R3 */ + +call_cleanup: + if (layout.locs) + tcc_free(layout.locs); +} + +/* Check if a backward branch to target_ir can use a narrow encoding. + * For backward branches, the target code address is already known in + * ir_to_code_mapping (it was emitted earlier in this pass). + * current_ir_idx is the IR index of the branch instruction itself. + * Returns 1 if narrow encoding fits, 0 otherwise. */ +static int can_narrow_backward_branch(int32_t target_ir, int is_conditional, int current_ir_idx) +{ + TCCIRState *ir = tcc_state->ir; + if (!ir || !ir->ir_to_code_mapping) + return 0; + if (target_ir < 0 || target_ir >= ir->ir_to_code_mapping_size) + return 0; + + /* Forward branches have uninitialized ir_to_code_mapping[target_ir] (still 0). + * Only narrow genuinely backward branches where target was already emitted. */ + if (target_ir >= current_ir_idx) + return 0; + + int target_addr = (int)ir->ir_to_code_mapping[target_ir]; + /* ind is the current code address where the branch will be emitted. + * offset = target - (source + 4) for Thumb pipeline. */ + int offset = target_addr - ind - 4; + + /* Only backward branches (negative offset) are safe to narrow here */ + if (offset >= 0) + return 0; + + /* If emitting the narrow branch would first flush a pending literal pool, + * the branch source moves forward after this range check. A borderline + * T1/T2 branch can become out of range by the time backpatching runs, and + * th_patch_call() cannot widen an already-emitted 16-bit branch in place. */ + if (th_literal_pool_would_flush_for(2)) + return 0; + + return is_conditional ? branch_fits_t1(offset) : branch_fits_t2(offset); +} + +/* Check if a FORWARD branch to target_ir can use a narrow encoding. + * + * The target address is not known yet in the real pass — it is backpatched + * later — so the decision comes from the rehearsal pass, which laid the + * function out with the same registers and frame the real pass uses and + * differs from it only by emitting every branch wide. The real pass is + * therefore never larger, so the real distance between two IR instructions is + * <= their rehearsal distance, and "fits in range in the rehearsal" implies + * "fits in range for real" -- except for word-alignment pads, which follow the + * parity the narrowed branches change; rehearsal_max_growth_between adds + * them back. + * + * The one way that breaks is a literal pool flush landing between the branch + * and its target in the real pass but not in the rehearsal — it inserts up to + * ~1 KB. Since the real pass emits fewer bytes, code_size grows more slowly + * and a flush can only move LATER, so one sitting before the branch in the + * rehearsal could drift into the range. Rather than pin flush points, this + * refuses to narrow unless no flush is possible at all: the range adds at most + * dry_dist bytes of code and (entries in range * 4) bytes of pool, and if that + * upper bound stays below the flush threshold the monotonicity argument holds + * unconditionally. */ +/* Upper bound on how many bytes the real pass can ADD, relative to the + * rehearsal, to the distance from from_ir to to_ir's address. The rehearsal + * emits every branch wide, so without alignment the real pass is never + * larger -- but a word-alignment pad (IRQuadCompact.align_target) depends on + * the parity of everything before it, and narrowing a branch flips it: a + * target the rehearsal found aligned needs the pad for real. Each aligned + * instruction in (from_ir, to_ir] can add one pad; to_ir's own pad sits before + * its recorded address, so it counts. Found as a CBZ over two such targets + * that measured 126 bytes in the rehearsal and landed at 128 (Zig's + * InternPool.getOrPutTrailingString). */ +static int rehearsal_max_growth_between(TCCIRState *ir, int from_ir, int to_ir) +{ + if (!align_pad_max) + return 0; + int pads = 0; + for (int k = from_ir + 1; k <= to_ir && k < ir->next_instruction_index; k++) + if (ir->compact_instructions[k].align_target) + pads++; + return pads * align_pad_max; +} + +/* Upper bound on how many bytes the real pass can shed, relative to the + * rehearsal, strictly between two IR instructions. The rehearsal emits every + * branch wide and never fuses CBZ, so the only shrink sources are: + * - a wide branch narrowed to its 16-bit form (4 -> 2, saves 2) + * - a CMP+B.W pair fused into a single CBZ/CBNZ (6 -> 2, saves 4) + * - a word-alignment pad the rehearsal needed and the real pass does not + * Only a JUMPIF right after its CMP or TEST_ZERO can be fused, which is the + * shape the codegen.c peepholes look for (NOPs between are skipped there too); + * every other branch, a return's jump to the epilogue included, can at most + * be narrowed. Charging 4 bytes to every branch refused CBZs whose target + * sat just past a `return` (Zig's `if (err) return err;`, ~19k of them in + * the Zig compiler built as C). The pads come from + * rehearsal_max_growth_between. */ +static int rehearsal_max_shrink_between(TCCIRState *ir, int from_ir, int to_ir) +{ + int shrink = 0; + int prev_op = TCCIR_OP_NOP; + for (int k = from_ir; k >= 0 && prev_op == TCCIR_OP_NOP; k--) + prev_op = ir->compact_instructions[k].op; + for (int k = from_ir + 1; k < to_ir; k++) + { + TccIrOp op = (TccIrOp)ir->compact_instructions[k].op; + if (op == TCCIR_OP_JUMPIF) + shrink += (prev_op == TCCIR_OP_CMP || prev_op == TCCIR_OP_TEST_ZERO) ? 4 : 2; + else if (op == TCCIR_OP_JUMP || op == TCCIR_OP_RETURNVALUE || op == TCCIR_OP_RETURNVOID) + shrink += 2; + if (op != TCCIR_OP_NOP) + prev_op = op; + } + /* ... a window the -Os outliner calls instead, and a pad the rehearsal + * needed can be gone (parity flipped back). */ + return shrink + tcc_gen_machine_outline_shrink_between(from_ir, to_ir) + + rehearsal_max_growth_between(ir, from_ir, to_ir); +} + +/* Does the range between two IR instructions (exclusive of from_ir, inclusive + * of to_ir) hold an inline asm statement? The dry passes never assemble the + * body, so the rehearsal lays it out as zero bytes and no distance measured + * across it bounds the real one: a CBZ over 144 bytes of asm was fused at a + * modelled offset of a few bytes. Narrowing checks refuse such ranges. */ +static int rehearsal_range_has_asm(TCCIRState *ir, int from_ir, int to_ir) +{ + if (ir->inline_asm_count == 0) + return 0; + for (int k = from_ir + 1; k <= to_ir && k < ir->next_instruction_index; k++) + if (ir->compact_instructions[k].op == TCCIR_OP_INLINE_ASM) + return 1; + return 0; +} + +/* Can `CMP rN,#0; B target` at current_ir_idx be fused into a single + * 16-bit CBZ/CBNZ? CBZ is forward-only with a 0..126 byte range and cannot be + * widened in place once committed (th_patch_call errors), so the check has to + * be sound in BOTH directions, unlike plain narrowing which only needs an + * upper bound: + * + * upper — the real offset is at most the rehearsal-derived one, because the + * real pass only ever shrinks; + * lower — the real offset is at least that minus the maximum shrink the + * range can undergo, and it must not go below 0. + * + * The fusion itself removes 6 bytes (CMP + wide branch) and puts back 2 at the + * branch site, which is why the base offset is dry_dist - 8 rather than the + * dry_dist - 4 that plain narrowing uses. */ +ST_FUNC int tcc_gen_machine_cbz_forward_ok(int32_t target_ir, int current_ir_idx) +{ + TCCIRState *ir = tcc_state->ir; + if (!ir || !ir->codegen_cbz_dry_mapping || !ir->codegen_dry_pool_entries) + return 0; + if (current_ir_idx < 0 || target_ir <= current_ir_idx) + return 0; + if (target_ir >= ir->ir_to_code_mapping_size || target_ir >= ir->next_instruction_index) + return 0; + + const uint32_t *dry = ir->codegen_cbz_dry_mapping; + int dry_dist = (int)dry[target_ir] - (int)dry[current_ir_idx]; + if (dry_dist < 0) + return 0; + + /* The JUMPIF this fusion consumes is part of the 8 already; count what can + * shrink after it. */ + int jump_ir = current_ir_idx + 1; + while (jump_ir < target_ir && ir->compact_instructions[jump_ir].op == TCCIR_OP_NOP) + jump_ir++; + if (jump_ir >= target_ir || ir->compact_instructions[jump_ir].op != TCCIR_OP_JUMPIF) + return 0; + int max_offset = dry_dist - 8 + rehearsal_max_growth_between(ir, current_ir_idx, target_ir); + int min_offset = dry_dist - 8 - rehearsal_max_shrink_between(ir, jump_ir, target_ir); + if (min_offset < 0 || max_offset > 126) + return 0; + if (rehearsal_range_has_asm(ir, current_ir_idx, target_ir)) + return 0; + + /* A pool flush anywhere in the range would push the branch out of its 126-byte + * reach, and unlike a wide branch it cannot be repaired. Same bound as + * can_narrow_forward_branch. */ + if (th_literal_pool_would_flush_for(2)) + return 0; + int entries_in_range = + (int)ir->codegen_dry_pool_entries[target_ir] - (int)ir->codegen_dry_pool_entries[current_ir_idx]; + if (entries_in_range < 0) + return 0; + /* The emitter flushes when th_pool_span_after() reaches the range limit, + * evaluated continuously while the range's instructions emit. This + * one-shot estimate bounds the span at any point in the range: the current + * span plus the range's code bytes plus the pool bytes its entries can + * add. The real pass can still drift past it — per-emit upcoming bytes, + * dry-vs-real sizing differences. Fuzz seed float:8061 passed a hairline + * check and then took a 188-byte in-range flush (COMPILE_FAIL at + * th_patch_call, offset 204). Require an extra 64-byte cushion so a + * hairline pass can't commit an unrepairable CBZ. */ + int pressure = th_pool_span_after(8) + dry_dist + entries_in_range * 4; + if (pressure >= THUMB_POOL_RANGE_LIMIT - THUMB_POOL_FLUSH_SLACK - 64) + return 0; + + return 1; +} + +/* The most a range of code can grow by literal-pool flushes landing inside + * it: every entry pending now and every one the range adds (at most 8 bytes + * each) dumped, and each dump preceded by its B.W over the pool and an + * alignment halfword. A dump needs a pending entry, so there are at most + * one more of them than entries added. Plus the 64-byte cushion + * tcc_gen_machine_cbz_forward_ok keeps for dry-vs-real drift. A 16-bit + * branch whose offset still fits after all of that is safe wherever the + * flushes fall -- the "no flush can happen in the range" rule refused almost + * every branch longer than a pool window (~950 bytes), 7,900 in zig.c. */ +static int pool_flush_growth_bound(int entries_in_range) +{ + return thumb_gen_state.pool_bytes + entries_in_range * (8 + 6) + 6 + 64; +} + +/* Return jumps pass target_ir == -1: they go to the epilogue, which has no IR + * index, so can_narrow_forward_branch cannot size them and they were always + * emitted wide. The rehearsal recorded where the body ends — which is exactly + * where the epilogue begins — so the same monotonicity argument applies: the + * real pass is never larger, so a distance that fits in the rehearsal fits for + * real. Same pool-flush bound as the forward case, using the running entry + * total at the end of the function. */ +static int can_narrow_epilogue_branch(int32_t target_ir, int current_ir_idx) +{ + TCCIRState *ir = tcc_state->ir; + /* A jump to the end label (target == next_instruction_index) lands on the + * epilogue too. */ + if (target_ir != -1 && !(ir && target_ir == ir->next_instruction_index)) + return 0; + if (!ir || !ir->codegen_cbz_dry_mapping || !ir->codegen_dry_pool_entries) + return 0; + if (current_ir_idx < 0 || current_ir_idx >= ir->next_instruction_index) + return 0; + if (ir->codegen_rehearsal_end == 0) + return 0; + + int dry_dist = (int)ir->codegen_rehearsal_end - (int)ir->codegen_cbz_dry_mapping[current_ir_idx]; + if (dry_dist < 0) + return 0; + const int est_offset = + dry_dist - 4 + rehearsal_max_growth_between(ir, current_ir_idx, ir->next_instruction_index - 1); + if (!branch_fits_t2(est_offset)) + return 0; + if (rehearsal_range_has_asm(ir, current_ir_idx, ir->next_instruction_index - 1)) + return 0; + + if (th_literal_pool_would_flush_for(2)) + return 0; + + /* Bound the pool pressure over the whole remaining body: entries added from + * here to the last instruction is the most that can accumulate before the + * epilogue. */ + int last = ir->next_instruction_index - 1; + int entries_in_range = (int)ir->codegen_dry_pool_entries[last] - + (int)ir->codegen_dry_pool_entries[current_ir_idx]; + if (entries_in_range < 0) + return 0; + int pressure = th_pool_span_after(8) + dry_dist + entries_in_range * 4; + if (pressure >= THUMB_POOL_RANGE_LIMIT - THUMB_POOL_FLUSH_SLACK && + !branch_fits_t2(est_offset + pool_flush_growth_bound(entries_in_range))) + return 0; + + return 1; +} + +static int can_narrow_forward_branch(int32_t target_ir, int is_conditional, int current_ir_idx) +{ + TCCIRState *ir = tcc_state->ir; + if (!ir || !ir->codegen_cbz_dry_mapping || !ir->codegen_dry_pool_entries) + return 0; + if (current_ir_idx < 0 || target_ir <= current_ir_idx) + return 0; + if (target_ir >= ir->ir_to_code_mapping_size || target_ir >= ir->next_instruction_index) + return 0; + + const uint32_t *dry = ir->codegen_cbz_dry_mapping; + int dry_dist = (int)dry[target_ir] - (int)dry[current_ir_idx]; + if (dry_dist < 0) + return 0; + + /* dry[current_ir_idx] is the START of the IR instruction; the branch may sit + * a few bytes into it, which only makes the true offset smaller. Dropping + * the 4-byte Thumb pipeline bias keeps the estimate conservative. */ + int est_offset = dry_dist - 4 + rehearsal_max_growth_between(ir, current_ir_idx, target_ir); + if (!(is_conditional ? branch_fits_t1(est_offset) : branch_fits_t2(est_offset))) + return 0; + if (rehearsal_range_has_asm(ir, current_ir_idx, target_ir)) + return 0; + + /* A flush scheduled at this very point would move the branch itself. */ + if (th_literal_pool_would_flush_for(2)) + return 0; + + int entries_in_range = + (int)ir->codegen_dry_pool_entries[target_ir] - (int)ir->codegen_dry_pool_entries[current_ir_idx]; + if (entries_in_range < 0) + return 0; + /* Same bound as tcc_gen_machine_cbz_forward_ok: no flush can trigger in + * the range while this stays under the flush threshold. Past it, the + * branch still narrows if it fits with every flush the range can take. */ + int pressure = th_pool_span_after(8) + dry_dist + entries_in_range * 4; + if (pressure >= THUMB_POOL_RANGE_LIMIT - THUMB_POOL_FLUSH_SLACK && + !(is_conditional ? branch_fits_t1 : branch_fits_t2)(est_offset + pool_flush_growth_bound(entries_in_range))) + return 0; + + return 1; +} + +ST_FUNC int tcc_gen_machine_jump_mop(TccIrOp op, int32_t target_ir, int ir_idx) +{ + + if (dry_run_state.active) + { + /* Emit 32-bit placeholder for code size tracking */ + ot_check(th_b_t4(0)); + return 4; + } + + /* Real pass: try narrow encoding, backward first then forward */ + if (can_narrow_backward_branch(target_ir, 0, ir_idx) || can_narrow_forward_branch(target_ir, 0, ir_idx) || + can_narrow_epilogue_branch(target_ir, ir_idx)) + { + ot_check(th_b_t2(0)); /* 16-bit unconditional */ + return 2; + } + else + { + ot_check(th_b_t4(0)); /* 32-bit unconditional */ + return 4; + } +} + +ST_FUNC int tcc_gen_machine_conditional_jump_mop(int32_t condition, TccIrOp op, int32_t target_ir, int ir_idx) +{ + int cond = mapcc(condition); + + if (dry_run_state.active) + { + /* Emit 32-bit placeholder for code size tracking */ + ot_check(th_b_t3(cond, 0)); + return 4; + } + + /* Real pass: try narrow encoding, backward first then forward */ + if (can_narrow_backward_branch(target_ir, 1, ir_idx) || can_narrow_forward_branch(target_ir, 1, ir_idx)) + { + ot_check(th_b_t1(cond, 0)); /* 16-bit conditional */ + return 2; + } + else + { + ot_check(th_b_t3(cond, 0)); /* 32-bit conditional */ + return 4; + } +} + +/* Return the maximum bytes a pending literal pool dump could insert. + * Used for CBZ/CBNZ distance safety checks. */ +ST_FUNC int tcc_gen_machine_pending_pool_size(void) +{ + int count = dry_run_state.active ? dry_run_literal_pool_count : thumb_gen_state.literal_pool_count; + return count * 4 + (count > 0 ? 2 : 0); /* entries + possible alignment padding */ +} + +/* Emit CBZ/CBNZ: combined compare-zero + branch in a single 16-bit instruction. + * rn must be r0-r7, target must be forward within 126 bytes. + * Returns the instruction size (always 2). */ +ST_FUNC int tcc_gen_machine_cbz_jump_mop(int rn, int nonzero, int32_t target_ir, int ir_idx) +{ + ot_check(th_cbz((uint16_t)rn, 0, (uint32_t)nonzero)); + return 2; +} + +/* reg = this frame's top: the frame pointer plus the frame size. That is + * the static chain a nested function receives: it addresses its parent's + * captured variables at their IR frame offsets from it (MACH_OP_CHAIN_REL, + * negative, counted down from the top), and the parent's own saved chain at + * -4 (resolve_chain_base) -- the frame pointer itself is the frame's bottom. */ +static void emit_frame_top(int reg) +{ + if (allocated_stack_size == 0) + { + ot_check_mov_reg(reg, R_FP, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE, false); + return; + } + if (!ot(th_add_imm(reg, R_FP, allocated_stack_size, flags_safe(), ENFORCE_ENCODING_NONE))) + { + load_full_const(reg, PREG_NONE, LFC_SPLIT(allocated_stack_size)); + ot_check(th_add_reg(reg, reg, R_FP, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); + } +} + +/* Offset of this frame's top above the frame pointer: DWARF's frame base for + * a frame-pointer function, where variables sit at their IR frame offsets. */ +ST_FUNC int tcc_gen_machine_frame_top_offset(void) +{ + return allocated_stack_size; +} + +/* Set static chain register: R10 = this frame's top (emit_frame_top). */ +ST_FUNC void tcc_gen_machine_set_chain(void) +{ + emit_frame_top(architecture_config.static_chain_reg); +} + +/* Reload static chain register from the chain save slot (frame offset -4). + * Called after function calls in nested functions with has_static_chain, + * because trampoline calls can clobber R10. */ +ST_FUNC void tcc_gen_machine_restore_chain(void) +{ + int chain_reg = architecture_config.static_chain_reg; + const int off = fp_adjust_local_offset(-4, 0); + const int sign = off < 0; + const int abs_off = sign ? -off : off; + if (!load_word_from_base(chain_reg, R_FP, abs_off, sign)) + { + ScratchRegAlloc rr_alloc = th_offset_to_reg_ex(abs_off, sign, (1u << chain_reg) | (1u << R_FP)); + int rr = rr_alloc.reg; + ot_check(th_ldr_reg(chain_reg, R_FP, rr, THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); + restore_scratch_reg(&rr_alloc); + } +} + +/* Store this frame's top into chain slot in .data for nested function trampoline. + * src1 carries the chain slot symbol via SYMREF so we can emit a relocation. */ +ST_FUNC void tcc_gen_machine_init_chain_slot(IROperand src1) +{ + /* Extract the chain slot Sym* from the IROperand */ + Sym *chain_sym = irop_get_sym(src1); + if (!chain_sym) + tcc_error("internal error: INIT_CHAIN_SLOT without chain slot symbol"); + + /* Get a scratch register to hold the chain slot address */ + ScratchRegAlloc scratch = get_scratch_reg_with_save(0); + + /* Load chain slot address into scratch register via literal pool. */ + _lfc_sym = chain_sym; + load_full_const(scratch.reg, PREG_NONE, 0, 0); + + /* STR top, [scratch, #0] — store this frame's top (the static chain the + * trampoline hands the nested function, emit_frame_top) into the slot */ + ScratchRegAlloc top = get_scratch_reg_with_save(1u << scratch.reg); + emit_frame_top(top.reg); + ot_check_str_imm(top.reg, scratch.reg, 0, 6, ENFORCE_ENCODING_NONE); + + /* Restore scratch registers */ + restore_scratch_reg(&top); + restore_scratch_reg(&scratch); +} + +/* Called at end of each IR instruction to clean up scratch register state. + * - Restores any pushed scratch registers (POP in reverse push order) + * - Resets global exclusion mask for next instruction */ +ST_FUNC void tcc_gen_machine_end_instruction(void) +{ + restore_all_pushed_scratch_regs(); +} + +/* Bytes of outgoing-argument area that have to stay below the live stack top. + * + * Stack arguments are written at [SP, SP+call_outgoing_size) right before the + * BL, because that is where AAPCS says the callee looks for them, so SP cannot + * also be the block alloca just handed back -- the next call with a stack + * argument writes straight over it. GNU make's pattern_search does exactly + * that: `int_file = alloca (sizeof (struct file))` followed by a five-argument + * recursive call, and the fifth argument landed on int_file->name, so the + * callee dereferenced a NULL name. + * + * SP is therefore kept this many bytes below the *logical* stack top: + * VLA_ALLOC carves its block off that top and then drops SP past a fresh + * outgoing area, and VLA_SP_SAVE / VLA_SP_RESTORE convert between the two. + * The conversion is symmetric, so the SAVE/RESTORE pairs that bracket a VLA + * scope still round-trip, and the SAVE that captures an alloca's result now + * yields the block rather than the argument area beneath it. + * + * The nested-call R9/argument save slots need no equivalent: in a dynamic-SP + * function they are addressed FP-relative (the frame pointer is the static + * SP; see the call-site save block). */ +static int vla_outgoing_reserve(void) +{ + TCCIRState *ir = tcc_state->ir; + int k = (ir && ir->call_outgoing_size > 0) ? ir->call_outgoing_size : 0; + return (k + 7) & ~7; +} + +/* Whether the value in register `reg` is last used by the current + * instruction, so the instruction may overwrite it. Without liveness (no IR) + * nothing is known to outlive it. */ +static int reg_value_dies_here(int reg) +{ + TCCIRState *ir = tcc_state->ir; + if (!ir) + return 1; + int holder = tcc_ls_find_int_reg_holder(&ir->ls, reg, ir->codegen_instruction_idx); + return holder >= 0 && (int)ir->ls.intervals[holder].end <= ir->codegen_instruction_idx; +} + +/* tcc_gen_machine_vla_mop: MachineOperand-based entry point for VLA operations. + * + * VLA_ALLOC: src1=size(bytes), src2=alignment(IMM bytes), dest unused + * VLA_SP_SAVE: dest=save slot, src1/src2 unused + * VLA_SP_RESTORE: src1=save slot, dest/src2 unused + * + * Gate: !ir->has_static_chain (VLA ops are always 32-bit pointer/int sized). + */ +ST_FUNC void tcc_gen_machine_vla_mop(MachineOperand dest, MachineOperand src1, MachineOperand src2, TccIrOp op) +{ + MachineCodegenContext ctx = {0}; + switch (op) + { + case TCCIR_OP_VLA_ALLOC: + { + /* src1=size (may be register or spilled); src2=alignment (IMM or NONE). */ + int align = (src2.kind == MACH_OP_IMM) ? (int)src2.u.imm.val : 8; + if (align < 8) + align = 8; + if (align & (align - 1)) + tcc_error("alignment is not a power of 2: %i", align); + + /* The new top is built in a working register: the size's own register + * when this is its last use (a VLA's byte count), else a scratch. A size + * that stays live -- alloca(n) with n used afterwards -- had its register + * overwritten by the new SP. The scratch must not be one saved on the + * stack: its restore would read from the moved SP. The dry run records + * such a save and the fixup frees a register for the real run. */ + int size_reg = mach_ensure_in_reg(&ctx, &src1, 0); + int r = size_reg; + if (src1.kind == MACH_OP_REG && !src1.needs_deref && !reg_value_dies_here(size_reg)) + { + r = mach_alloc_scratch(&ctx, 1u << (uint32_t)size_reg); + if (ctx.scratches[ctx.n_scratch - 1].would_save && !dry_run_state.active) + tcc_error("compiler_error: alloca/VLA size stays live and no register is free " + "to compute the new stack top"); + } + if (r == R_SP) + tcc_error("compiler_error: VLA alloc picked SP as temp"); + + const int vla_reserve = vla_outgoing_reserve(); + + /* r = SP - size */ + ot_check(th_sub_reg(r, R_SP, size_reg, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); + + /* ... and back up to the logical top, so the block is carved above the + * outgoing-argument area rather than on top of it. */ + if (vla_reserve) + { + thumb_opcode add_res = th_add_imm(r, r, (uint32_t)vla_reserve, flags_safe(), ENFORCE_ENCODING_NONE); + if (is_valid_opcode(add_res)) + { + ot(add_res); + } + else + { + int res_reg = mach_alloc_scratch(&ctx, 1u << (uint32_t)r); + if (!ot(th_generic_mov_imm(res_reg, vla_reserve))) + load_full_const(res_reg, PREG_NONE, LFC_SPLIT(vla_reserve)); + ot_check(th_add_reg(r, r, res_reg, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); + } + } + + if (align > 1) + { + /* Align down: r &= ~(align-1). Try immediate BIC first. */ + if (!ot(th_bic_imm(r, r, (uint32_t)(align - 1), flags_safe(), ENFORCE_ENCODING_NONE))) + { + /* Fallback: materialize mask in a scratch register. */ + int mask_reg = mach_alloc_scratch(&ctx, 1u << (uint32_t)r); + if (!ot(th_generic_mov_imm(mask_reg, align - 1))) + load_full_const(mask_reg, PREG_NONE, LFC_SPLIT(align - 1)); + ot_check(th_bic_reg(r, r, mask_reg, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); + } + } + + ot_check_mov_reg(R_SP, r, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE, false); + /* r is the new logical top; SP sits a fresh outgoing area below it. */ + gadd_sp(-vla_reserve); + break; + } + case TCCIR_OP_VLA_SP_SAVE: + { + /* Fast path: when dest is a register-allocated vreg, copy SP directly into + * its register — saves the scratch-mov + writeback-mov pair that the + * generic path would emit. Triggered by the alloca-load-fwd IR pass + * which rewrites a `VLA_SP_SAVE slot; LOAD vreg <- slot` pair into a + * single `VLA_SP_SAVE vreg`. */ + const int save_reserve = vla_outgoing_reserve(); + if (dest.kind == MACH_OP_REG && !dest.needs_deref && + dest.u.reg.r0 != (int)PREG_REG_NONE) + { + ot_check_mov_reg((uint32_t)dest.u.reg.r0, R_SP, flags_safe(), + THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE, false); + if (save_reserve) + ot_check(th_add_imm((uint32_t)dest.u.reg.r0, (uint32_t)dest.u.reg.r0, (uint32_t)save_reserve, + flags_safe(), ENFORCE_ENCODING_NONE)); + break; + } + /* Save the logical stack top to the destination save slot via a scratch. */ + ScratchRegAlloc sp_scratch = get_scratch_reg_with_save(0); + ot_check_mov_reg(sp_scratch.reg, R_SP, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE, + false); + if (save_reserve) + ot_check(th_add_imm((uint32_t)sp_scratch.reg, (uint32_t)sp_scratch.reg, (uint32_t)save_reserve, + flags_safe(), ENFORCE_ENCODING_NONE)); + mach_writeback_dest(&dest, sp_scratch.reg); + restore_scratch_reg(&sp_scratch); + break; + } + case TCCIR_OP_VLA_SP_RESTORE: + { + /* Load the saved logical top from src1, then put SP back below it. The + * SUB goes through SP rather than the loaded register: src1 may be a live + * vreg (alloca_load_fwd hands one over) that this must not clobber. */ + int saved_sp = mach_ensure_in_reg(&ctx, &src1, 0); + ot_check_mov_reg(R_SP, saved_sp, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE, false); + gadd_sp(-vla_outgoing_reserve()); + break; + } + default: + tcc_error("compiler_error: tcc_gen_machine_vla_mop unsupported op %d", op); + } + mach_release_all(&ctx); +} + +/* Block copy from const data section to stack using LDM/STM. + * dest = STACKOFF (destination stack offset, is_local=1) + * src = SYMREF (anonymous symbol in rodata) + * size = number of bytes to copy (must be multiple of 4) + * + * Generated code for 20 bytes (5 words): + * LDR r_src, [PC, #lit_pool] ; load rodata address + * ADD r_dst, FP/SP, #stack_off ; compute stack dest + * LDMIA r_src!, {r0, r1, r2, r3} ; load 4 words from rodata + * STMIA r_dst!, {r0, r1, r2, r3} ; store 4 words to stack + * LDR r0, [r_src] ; load remaining word + * STR r0, [r_dst] ; store remaining word + */ +/* tcc_gen_machine_select_mop: Conditional select using ITE block. + * Emits: ITE ; MOV dest, then_val; MOV dest, else_val + * + * For simple register/immediate operands, this is 3 instructions (ITE + 2 MOVs) + * instead of 5+ (B.cond + MOV + B + MOV + ...) with branching. + */ +/* Check if a MachineOperand can be materialized in exactly one instruction. + * Returns 1 for: IMM (any value), REG (no deref), SYMBOL (no deref), SPILL (no deref). + * Returns 0 for: multi-instruction sequences (deref, chain_rel, etc). */ +static int select_can_inline(const MachineOperand *op) +{ + switch (op->kind) + { + case MACH_OP_IMM: + return 1; /* MOV/MOVW/MVN or literal pool LDR — always 1 instruction */ + case MACH_OP_REG: + return !op->needs_deref; /* MOV reg is 1 instr; deref needs LDR too */ + case MACH_OP_SYMBOL: + /* A symbol address is a single literal-pool LDR only in the plain, + * non-PIC, non-separated layout. Under PIC/PIE or text+data separation it + * expands to a multi-instruction GOT/GOTOFF sequence (ldr GOT-slot; add r9; + * ldr; ...). Emitting that "inline" inside an IT block predicates only the + * FIRST instruction and lets the remaining ones run unconditionally, which + * clobbers the select result with the else-operand's address. Force + * pre-materialization into a scratch register in those modes. */ + return !op->needs_deref && !pic && !text_and_data_separation; + case MACH_OP_SPILL: + return !op->needs_deref; /* LDR from stack is 1 instr; deref (VT_LLOCAL) needs 2 */ + case MACH_OP_FRAME_ADDR: + return 1; /* ADD reg, FP, #off is 1 instr */ + default: + return 0; + } +} + +/* Emit a single-instruction materialization of 'op' into 'reg'. + * Caller must ensure select_can_inline(op) returned 1. */ +/* Every caller emits this as an instruction of an IT block, where the 16-bit + * MOV #imm8 (MOVS outside one) sets no flags: a low register takes it, not the + * flag-preserving 4-byte MOV.W -- `ite eq; moveq r2, #1; movne r2, #0`. */ +static void select_emit_inline(MachineCodegenContext *ctx, const MachineOperand *op, int reg) +{ + switch (op->kind) + { + case MACH_OP_IMM: + { + if (reg < 8 && op->u.imm.val >= 0 && op->u.imm.val <= 255) + { + ot_check(th_mov_imm((uint32_t)reg, (uint32_t)op->u.imm.val, FLAGS_BEHAVIOUR_NOT_IMPORTANT, + ENFORCE_ENCODING_16BIT)); + break; + } + thumb_opcode imm_op = th_generic_mov_imm((uint32_t)reg, (int)op->u.imm.val); + if (imm_op.size != 0) + ot(imm_op); + else + load_full_const(reg, PREG_NONE, LFC_SPLIT(op->u.imm.val)); + break; + } + case MACH_OP_REG: + ot_check_mov_reg(reg, op->u.reg.r0, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE, + true); + break; + case MACH_OP_SYMBOL: + { + Sym *raw_sym = op->u.sym.sym; + Sym *sym = raw_sym ? validate_sym_for_reloc(raw_sym) : NULL; + tcc_machine_load_constant(reg, PREG_REG_NONE, op->u.sym.addend, 0, sym); + break; + } + case MACH_OP_SPILL: + mach_load_slot(reg, op); + break; + case MACH_OP_FRAME_ADDR: + tcc_machine_addr_of_stack_slot(reg, op->u.frame.offset, 0); + break; + default: + tcc_error("compiler_error: select_emit_inline: unhandled kind %d", (int)op->kind); + break; + } +} + +ST_FUNC void tcc_gen_machine_select_mop(MachineOperand then_val, MachineOperand else_val, MachineOperand dest, + int cond_code) +{ + MachineCodegenContext mctx = {0}; + + int cond = mapcc(cond_code); + + /* Get destination register */ + int dest_reg = mach_get_dest_reg(&mctx, &dest, 0); + uint32_t excl = (1u << (uint32_t)dest_reg); + + /* Determine if each operand can be materialized in exactly one instruction. + * If so, we can emit it directly inside the ITE block into dest_reg, + * saving scratch registers and pre-materialization instructions. + * + * Emitting inside the IT block is preferred because: + * - It avoids flag clobber (MOVS before ITE would destroy CMP flags) + * - It saves scratch registers (no pre-materialization needed) + * - It produces smaller code */ + int then_inline = select_can_inline(&then_val); + int else_inline = select_can_inline(&else_val); + + int then_reg = -1, else_reg = -1; + + /* Pre-materialize operands that need multi-instruction sequences. + * These are loaded into scratch registers BEFORE the ITE block. */ + if (!then_inline) + { + then_reg = mach_ensure_in_reg(&mctx, &then_val, excl); + excl |= (1u << (uint32_t)then_reg); + } + if (!else_inline) + { + else_reg = mach_ensure_in_reg(&mctx, &else_val, excl); + excl |= (1u << (uint32_t)else_reg); + } + + /* Identity-then shortcut: if the then-value is already in dest_reg, the + * predicated mov would be `movXX dest, dest` — a real instruction inside an + * IT block (the usual elision in ot_check_mov_reg is suppressed by in_it). + * Emit `IT ` + the else mov instead. Saves one instruction. */ + int then_is_identity = 0; + if (then_inline && then_val.kind == MACH_OP_REG && !then_val.needs_deref && + (int)then_val.u.reg.r0 == dest_reg) + then_is_identity = 1; + else if (!then_inline && then_reg == dest_reg) + then_is_identity = 1; + + if (then_is_identity) + { + int inv_cond = cond ^ 1; + th_literal_pool_reserve_upcoming_bytes(8); /* IT(2) + instr(2-4) */ + ot_check(th_it((uint16_t)inv_cond, 0x8u)); /* IT , single insn */ + if (else_inline) + select_emit_inline(&mctx, &else_val, dest_reg); + else + ot_check_mov_reg(dest_reg, else_reg, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE, true); + mach_writeback_dest(&dest, dest_reg); + mach_release_all(&mctx); + return; + } + + /* Identity-else shortcut: mirror of the above. If the else-value is already + * in dest_reg, emit `IT ` + the then mov only, dropping the + * `movXX dest, dest` for the else arm. Common for `cond ? f(x) : x` shapes + * (e.g. abs `x<0?-x:x`) where the else arm is the unmodified input. */ + int else_is_identity = 0; + if (else_inline && else_val.kind == MACH_OP_REG && !else_val.needs_deref && + (int)else_val.u.reg.r0 == dest_reg) + else_is_identity = 1; + else if (!else_inline && else_reg == dest_reg) + else_is_identity = 1; + + if (else_is_identity) + { + th_literal_pool_reserve_upcoming_bytes(8); /* IT(2) + instr(2-4) */ + ot_check(th_it((uint16_t)cond, 0x8u)); /* IT , single insn */ + if (then_inline) + select_emit_inline(&mctx, &then_val, dest_reg); + else + ot_check_mov_reg(dest_reg, then_reg, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE, true); + mach_writeback_dest(&dest, dest_reg); + mach_release_all(&mctx); + return; + } + + /* ITE mask: the second instruction uses the opposite condition. + * mask encoding: bit3 = E_flag for 2nd instr, bit2 = end marker. + * E_flag = opposite of cond[0], so: mask = ((cond[0]^1) << 3) | (1 << 2) */ + uint32_t ite_mask = (uint32_t)(((cond & 1) ^ 1) << 3) | 0x4u; + + /* Reserve literal pool space to prevent pool dumps inside the IT block */ + th_literal_pool_reserve_upcoming_bytes(10); /* ITE(2) + instr(2-4) + instr(2-4) */ + + ot_check(th_it((uint16_t)cond, (uint16_t)ite_mask)); + + /* Emit the Then instruction inside IT block */ + if (then_inline) + select_emit_inline(&mctx, &then_val, dest_reg); + else + ot_check_mov_reg(dest_reg, then_reg, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE, + true); + + /* Emit the Else instruction inside IT block */ + if (else_inline) + select_emit_inline(&mctx, &else_val, dest_reg); + else + ot_check_mov_reg(dest_reg, else_reg, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE, + true); + + mach_writeback_dest(&dest, dest_reg); + mach_release_all(&mctx); +} + +/* Predicated-compute fusion: when a single-use ALU op feeds an else-identity + * SELECT (else == SELECT dest) whose condition flags are still live from a + * preceding CMP, the op can be emitted predicated inside an IT block directly + * into the SELECT dest — `cmp; it ; rsb.w dest, src, #0` — instead of + * `rsb Tc,src,#0; it ; mov dest,Tc`. gcc-parity abs `x<0?-x:x`. + * + * Only a reverse-subtract negate (`dest = 0 - src2`) is folded: it lowers to a + * single 32-bit rsb.w, which inside the IT is conditional and, emitted + * flags-safe, leaves NZCV untouched. can_predicate answers before emission so + * the caller keeps the two-pass scratch accounting consistent. */ +ST_FUNC int tcc_gen_machine_can_predicate_alu(MachineOperand src1, MachineOperand src2, + MachineOperand dest, TccIrOp op) +{ + if (op != TCCIR_OP_SUB) + return 0; + if (!(src1.kind == MACH_OP_IMM && src1.u.imm.val == 0)) + return 0; + if (dest.is_64bit || src2.is_64bit || dest.needs_deref) + return 0; + return 1; +} + +ST_FUNC void tcc_gen_machine_predicated_alu_mop(MachineOperand src1, MachineOperand src2, + MachineOperand dest, TccIrOp op, int cond_code) +{ + (void)src1; + (void)op; + MachineCodegenContext mctx = {0}; + int cond = mapcc(cond_code); + int dest_reg = mach_get_dest_reg(&mctx, &dest, 0); + uint32_t excl = (1u << (uint32_t)dest_reg); + int src_reg = mach_ensure_in_reg(&mctx, &src2, excl); + + th_literal_pool_reserve_upcoming_bytes(8); /* IT(2) + rsb.w(4) */ + ot_check(th_it((uint16_t)cond, 0x8u)); /* IT , single insn */ + ot_check(th_rsb_imm((uint32_t)dest_reg, (uint32_t)src_reg, 0, flags_safe(), ENFORCE_ENCODING_NONE)); + + mach_writeback_dest(&dest, dest_reg); + mach_release_all(&mctx); +} + +ST_FUNC void tcc_gen_machine_block_copy_mop(TCCIRState *ir, IROperand dest, IROperand src, int size) +{ + if (size <= 0) + tcc_error("compiler_error: block_copy size must be positive, got %d", size); + + /* Get the source symbol from the SYMREF operand */ + IRPoolSymref *symref = irop_get_symref_ex(ir, src); + if (!symref || !symref->sym) + tcc_error("compiler_error: block_copy source is not a valid symbol reference"); + Sym *sym = validate_sym_for_reloc(symref->sym); + + /* Get the destination stack offset */ + int frame_offset = (int)irop_get_imm64_ex(ir, dest); + + /* Large copies run in r0-r3/IP/LR, which the allocator keeps free across + * them as across a memcpy call (ra_build_call_prefix), instead of in saved + * scratch registers. Compute dest address into r0 BEFORE pushing lr, since + * the address is sp-relative and pushing changes sp. The copy clobbers lr, + * so it is saved/restored for leaf functions whose prologue didn't. */ + if (size >= TCCIR_BLOCK_COPY_MEMCPY_MIN_BYTES) + { + tcc_machine_addr_of_stack_slot(R0, frame_offset, 0 /* not param */); + tcc_machine_load_constant(R1, PREG_REG_NONE, symref->addend, 0, sym); + int need_lr_save = ir->leaffunc; + if (need_lr_save) + ot_check(th_push(1u << ARM_LR)); + if (!(frame_offset & 3) && !(symref->addend & 3)) + thumb_emit_word_copy_r0_r1(size); + else + { + /* LDM/STM fault on an unaligned base: a byte buffer's slot, say. */ + tcc_machine_load_constant(R2, PREG_REG_NONE, size, 0, NULL); + Sym *memcpy_sym = external_global_sym(tok_alloc_const("memcpy"), &func_old_type); + MachineOperand func_mop = {0}; + func_mop.kind = MACH_OP_SYMBOL; + func_mop.u.sym.sym = memcpy_sym; + if (text_and_data_separation) + ot_check(th_push((uint16_t)((1 << R9) | (1 << R12)))); + gcall_or_jump_mop(0, func_mop); + if (text_and_data_separation) + ot_check(th_pop((uint16_t)((1 << R9) | (1 << R12)))); + } + if (need_lr_save) + ot_check(th_pop(1u << ARM_LR)); + return; + } + + int nwords = size / 4; + + /* Allocate pointer registers first and compute addresses BEFORE allocating + * data registers. Data register saves may use PUSH which modifies SP, + * so all SP-relative address computation must happen before that. */ + ScratchRegAlloc src_scratch = get_scratch_reg_with_save(0); + int r_src = src_scratch.reg; + ScratchRegAlloc dst_scratch = get_scratch_reg_with_save(1u << (uint32_t)r_src); + int r_dst = dst_scratch.reg; + + /* Load source address (rodata symbol) into r_src */ + tcc_machine_load_constant(r_src, PREG_REG_NONE, symref->addend, 0, sym); + + /* Compute destination stack address into r_dst BEFORE any data reg saves + * that might change SP via PUSH */ + tcc_machine_addr_of_stack_slot(r_dst, frame_offset, 0 /* not param */); + + /* Now allocate data registers for LDM/STM. Even if these saves use PUSH + * and modify SP, we've already captured the destination address in r_dst. */ + int max_data = nwords < 4 ? nwords : 4; + if (max_data < 1) + max_data = 1; + + ScratchRegAlloc data_scratches[4]; + int data_regs[4]; + int ndata = 0; + uint32_t exclude = (1u << (uint32_t)r_src) | (1u << (uint32_t)r_dst); + for (int k = 0; k < max_data; k++) + { + data_scratches[k] = get_scratch_reg_with_save(exclude); + data_regs[k] = data_scratches[k].reg; + exclude |= (1u << (uint32_t)data_regs[k]); + ndata++; + } + + int remaining_words = nwords; + + /* Process in chunks of ndata words using LDM/STM with writeback */ + while (remaining_words >= ndata && ndata >= 2) + { + uint32_t regset = 0; + for (int j = 0; j < ndata; j++) + regset |= (1u << (uint32_t)data_regs[j]); + + ot_check(th_ldm(r_src, regset, 1 /* writeback */, ENFORCE_ENCODING_NONE)); + ot_check(th_stm(r_dst, regset, 1 /* writeback */, ENFORCE_ENCODING_NONE)); + remaining_words -= ndata; + } + + /* Handle remaining words individually */ + int dr = data_regs[0]; /* first data register */ + int single_words = remaining_words; + while (remaining_words > 0) + { + ot_check_ldr_imm(dr, r_src, 0, 6, ENFORCE_ENCODING_NONE); + ot_check_str_imm(dr, r_dst, 0, 6, ENFORCE_ENCODING_NONE); + if (remaining_words > 1) + { + if (!ot(th_add_imm(r_src, r_src, 4, flags_safe(), ENFORCE_ENCODING_NONE))) + tcc_error("compiler_error: block_copy cannot advance source pointer"); + if (!ot(th_add_imm(r_dst, r_dst, 4, flags_safe(), ENFORCE_ENCODING_NONE))) + tcc_error("compiler_error: block_copy cannot advance dest pointer"); + } + remaining_words--; + } + + /* Copy the 1..3 trailing bytes of a size that is not a whole number of words. + * Writing them as bytes matters: a strcpy fold must not touch dst past the + * terminating NUL, so the copy may not be rounded up to a word. The LDM/STM + * chunks advanced the pointers by writeback; the single-word loop above stops + * ON its last word rather than past it, so the tail sits 4 bytes further on + * whenever that loop ran at all. */ + int tail = size & 3; + if (tail) + { + int tail_off = single_words > 0 ? 4 : 0; + for (int b = 0; b < tail; b++) + { + ot_check(th_ldrb_imm(dr, r_src, tail_off + b, 6, ENFORCE_ENCODING_NONE)); + ot_check(th_strb_imm(dr, r_dst, tail_off + b, 6, ENFORCE_ENCODING_NONE)); + } + } + + /* Restore all scratch registers in reverse order: data regs first, then ptrs */ + for (int k = ndata - 1; k >= 0; k--) + restore_scratch_reg(&data_scratches[k]); + restore_scratch_reg(&dst_scratch); + restore_scratch_reg(&src_scratch); +} + +/* Size of copying `nwords` words base+src -> base+dst in chunks through data + * registers regs[0..k) (the LDM's base the highest low one) with STM base rb; + * emits the code when `emit`. -1 when some chunk would hold a lone word or an + * address does not encode. */ +static int spill_block_copy_plan(int base, int src, int dst, int nwords, const int *regs, int k, int rb, int emit) +{ + int all_low = rb <= R7; + for (int i = 0; i < k; i++) + all_low &= regs[i] <= R7; + /* Words spread evenly over the chunks, so none is left with a lone word. */ + const int nchunks = (nwords + k - 1) / k; + int size = 0, w = 0; + for (int chunk = 0; chunk < nchunks; chunk++) + { + const int n = (nwords - w) / (nchunks - chunk); + if (n < 2) + return -1; + uint32_t l = 0; + int la = -1; + for (int i = 0; i < n; i++) + { + l |= 1u << (uint32_t)regs[i]; + if (regs[i] <= R7 && regs[i] > la) + la = regs[i]; + } + if (la < 0) + la = regs[0]; + const int so = src + 4 * w; + thumb_opcode a = so < 0 ? th_sub_imm(la, base, -so, flags_safe(), ENFORCE_ENCODING_NONE) + : th_add_imm(la, base, so, flags_safe(), ENFORCE_ENCODING_NONE); + thumb_opcode b = {0}; + if (chunk == 0) + b = dst < 0 ? th_sub_imm(rb, base, -dst, flags_safe(), ENFORCE_ENCODING_NONE) + : th_add_imm(rb, base, dst, flags_safe(), ENFORCE_ENCODING_NONE); + if (!a.size || (chunk == 0 && !b.size)) + return -1; + const thumb_opcode ldm = all_low ? (thumb_opcode){.size = 2, .opcode = 0xC800u | ((uint32_t)la << 8) | l} + : (thumb_opcode){.size = 4, .opcode = ((0xE890u | (uint32_t)la) << 16) | l}; + const thumb_opcode stm = all_low ? (thumb_opcode){.size = 2, .opcode = 0xC000u | ((uint32_t)rb << 8) | l} + : (thumb_opcode){.size = 4, .opcode = ((0xE8A0u | (uint32_t)rb) << 16) | l}; + size += a.size + ldm.size + b.size + stm.size; + if (emit) + { + ot_check(a); + ot_check(ldm); + if (chunk == 0) + ot_check(b); + ot_check(stm); + } + w += n; + } + return size; +} + +/* Copy `nwords` words between two frame slots with registers that are free at + * this instruction (plus `also_free`), never pushing: chunks of + * add rA, sp, #src; ldm rA, {..} (rA one of the loaded registers) + * add rB, sp, #dst; stmia rB!, {..} (rB outside the list; first chunk) + * Registers load and store in ascending order, so any set keeps the words in + * place. Of the all-low (16-bit) and the widest plan, the shorter is taken; + * returns 0, having emitted nothing, when neither beats one LDR/STR pair per + * word. */ +ST_FUNC int tcc_gen_machine_spill_block_copy_free(int32_t src_spill_off, int32_t dst_spill_off, int nwords, + uint32_t also_free) +{ + TCCIRState *ir = tcc_state->ir; + if (!ir || nwords < 3) + return 0; + const int base = tcc_state->need_frame_pointer ? R_FP : R_SP; + const int src = fp_adjust_local_offset(src_spill_off, 0), dst = fp_adjust_local_offset(dst_spill_off, 0); + if ((src & 3) || (dst & 3)) + return 0; + uint32_t excl = (1u << (uint32_t)base) | (1u << R_SP) | (1u << R_PC) | scratch_global_exclude; + uint32_t freeset = also_free & ~excl & 0x100Fu; /* r0-r3, ip */ + for (;;) + { + int r = tcc_ls_find_free_scratch_reg(&ir->ls, ir->codegen_instruction_idx, excl | freeset, ir->leaffunc); + if (r < 0 || r == PREG_NONE || r >= 16 || ((excl | freeset) & (1u << (uint32_t)r))) + break; + freeset |= 1u << (uint32_t)r; + } + if (__builtin_popcount(freeset) < 3) + return 0; + /* The STM base: the lowest free register; data registers low first. */ + const int rb = __builtin_ctz(freeset); + int all[6], nall = 0, low[6], nlow = 0; + for (int r = 0; r < 16 && nall < 5; r++) + if (r != rb && (freeset & (1u << (uint32_t)r))) + { + all[nall++] = r; + if (r <= R7) + low[nlow++] = r; + } + + int plain = 0; + for (int w = 0; w < nwords; w++) + { + const int so = src + 4 * w, d = dst + 4 * w; + thumb_opcode l = th_ldr_imm(R0, base, so < 0 ? -so : so, so < 0 ? 4 : 6, ENFORCE_ENCODING_NONE); + thumb_opcode st = th_str_imm(R0, base, d < 0 ? -d : d, d < 0 ? 4 : 6, ENFORCE_ENCODING_NONE); + if (!l.size || !st.size) + return 0; + plain += l.size + st.size; + } + const int kl = nlow < 4 ? nlow : 4, ka = nall < 4 ? nall : 4; + const int cost_low = kl >= 2 ? spill_block_copy_plan(base, src, dst, nwords, low, kl, rb, 0) : -1; + const int cost_all = ka >= 2 ? spill_block_copy_plan(base, src, dst, nwords, all, ka, rb, 0) : -1; + if (cost_low > 0 && cost_low < plain && (cost_all < 0 || cost_low <= cost_all)) + spill_block_copy_plan(base, src, dst, nwords, low, kl, rb, 1); + else if (cost_all > 0 && cost_all < plain) + spill_block_copy_plan(base, src, dst, nwords, all, ka, rb, 1); + else + return 0; + return 1; +} + +/* Size of one word access `reg <- [base, #off]` (or the store), as the encoder + * would lay it out; 0 when the offset does not encode. */ +static int block_copy_word_size(int reg, int base, int32_t off, int is_store) +{ + const int neg = off < 0; + const uint32_t a = neg ? (uint32_t)-off : (uint32_t)off; + const thumb_opcode o = is_store ? th_str_imm(reg, base, a, neg ? 4 : 6, ENFORCE_ENCODING_NONE) + : th_ldr_imm(reg, base, a, neg ? 4 : 6, ENFORCE_ENCODING_NONE); + return o.size; +} + +/* Copy `nwords` words from src_base+src_off to dst_base+dst_off as one LDM and + * one STM through `regs`, which the caller owns: each holds one word of the + * copy and dies at its store. Which register takes which word does not matter + * -- LDM and STM both run in ascending register order, so the same list on both + * sides keeps the words in place. Unlike the frame-slot copier above, either + * side may be a general base register (a pointer deref), and the whole copy is + * one LDM followed by one STM, so it holds for overlapping ranges exactly as + * the loads-then-stores it replaces does. Returns 0, having emitted nothing, + * when an address does not encode, no register is free to hold a destination + * address that needs computing, or the pair is no smaller than the individual + * accesses. */ +ST_FUNC int tcc_gen_machine_reg_block_copy(int src_base, int32_t src_off, int dst_base, int32_t dst_off, + const int *regs, int nwords, int end_idx) +{ + TCCIRState *ir = tcc_state->ir; + if (!ir || nwords < 3 || nwords > 8) + return 0; + + const int frame = tcc_state->need_frame_pointer ? R_FP : R_SP; + if (src_base == MACH_BLOCK_COPY_FRAME) + { + src_base = frame; + src_off = fp_adjust_local_offset(src_off, 0); + } + if (dst_base == MACH_BLOCK_COPY_FRAME) + { + dst_base = frame; + dst_off = fp_adjust_local_offset(dst_off, 0); + } + if (src_base < 0 || src_base >= 16 || dst_base < 0 || dst_base >= 16) + return 0; + if ((src_off & 3) || (dst_off & 3)) + return 0; + + uint32_t list = 0; + int all_low = 1, top_low = -1; + for (int k = 0; k < nwords; k++) + { + const int r = regs[k]; + if (r < 0 || r >= 16 || r == R_SP || r == R_PC || (list & (1u << (uint32_t)r))) + return 0; + list |= 1u << (uint32_t)r; + if (r > R7) + all_low = 0; + else if (r > top_low) + top_low = r; + } + /* The LDM would overwrite the destination address before the STM reads it. */ + if (list & (1u << (uint32_t)dst_base)) + return 0; + + /* The LDM's base. With an offset it goes into one of the loaded registers, + * which the LDM then overwrites -- that is what makes it free. At offset 0 + * the source register itself is the base, and since it is then not + * necessarily in the list the wide (no-writeback) LDM has to carry it: the + * narrow one writes back whenever its base is outside the list. */ + int ra = src_base; + thumb_opcode add_src = {0}; + if (src_off != 0) + { + ra = top_low >= 0 ? top_low : regs[0]; + add_src = src_off < 0 ? th_sub_imm(ra, src_base, (uint32_t)-src_off, flags_safe(), ENFORCE_ENCODING_NONE) + : th_add_imm(ra, src_base, (uint32_t)src_off, flags_safe(), ENFORCE_ENCODING_NONE); + if (!add_src.size) + return 0; + } + const int ldm_narrow = all_low && ra <= R7 && (list & (1u << (uint32_t)ra)); + const thumb_opcode ldm = ldm_narrow ? (thumb_opcode){.size = 2, .opcode = 0xC800u | ((uint32_t)ra << 8) | list} + : (thumb_opcode){.size = 4, .opcode = ((0xE890u | (uint32_t)ra) << 16) | list}; + + /* The STM's base. A computed destination address is ours to destroy, so it + * may take the narrow (writeback) form; the destination register itself must + * survive, so it takes the wide one with the writeback bit clear. */ + int rb = dst_base; + thumb_opcode add_dst = {0}; + if (dst_off != 0) + { + /* Free at the run's LAST instruction, not its first: the destination + * address is computed after the LDM has read everything it needs, so a + * register the copy itself was using -- the source pointer above all -- is + * as good as one that was free all along. Nothing inside the run reads any + * other register, so no value can end between the two points. */ + const uint32_t excl = list | (1u << (uint32_t)R_SP) | (1u << R_PC) | (1u << (uint32_t)dst_base) | + scratch_global_exclude; + rb = tcc_ls_find_free_scratch_reg(&ir->ls, end_idx, excl, ir->leaffunc); + if (rb < 0 || rb == PREG_NONE || rb >= 16 || (excl & (1u << (uint32_t)rb))) + return 0; + add_dst = dst_off < 0 ? th_sub_imm(rb, dst_base, (uint32_t)-dst_off, flags_safe(), ENFORCE_ENCODING_NONE) + : th_add_imm(rb, dst_base, (uint32_t)dst_off, flags_safe(), ENFORCE_ENCODING_NONE); + if (!add_dst.size) + return 0; + } + const int stm_narrow = all_low && rb <= R7 && dst_off != 0; + const thumb_opcode stm = stm_narrow ? (thumb_opcode){.size = 2, .opcode = 0xC000u | ((uint32_t)rb << 8) | list} + : (thumb_opcode){.size = 4, .opcode = ((0xE880u | (uint32_t)rb) << 16) | list}; + + /* Against the loads and stores themselves, paired into LDRD/STRD where the + * peepholes downstream would pair them. */ + int plain = 0; + for (int k = 0; k < nwords; k++) + { + const int l = block_copy_word_size(regs[k], src_base, src_off + 4 * k, 0); + const int s = block_copy_word_size(regs[k], dst_base, dst_off + 4 * k, 1); + if (!l || !s) + return 0; + plain += l + s; + } + for (int k = 0; k + 1 < nwords; k += 2) + { + const int l = block_copy_word_size(regs[k], src_base, src_off + 4 * k, 0) + + block_copy_word_size(regs[k + 1], src_base, src_off + 4 * k + 4, 0); + const int s = block_copy_word_size(regs[k], dst_base, dst_off + 4 * k, 1) + + block_copy_word_size(regs[k + 1], dst_base, dst_off + 4 * k + 4, 1); + plain -= (l > 4 ? l - 4 : 0) + (s > 4 ? s - 4 : 0); + } + if (add_src.size + ldm.size + add_dst.size + stm.size >= plain) + return 0; + + if (add_src.size) + ot_check(add_src); + ot_check(ldm); + if (add_dst.size) + ot_check(add_dst); + ot_check(stm); + return 1; +} + +ST_FUNC void tcc_gen_machine_spill_block_copy(int32_t src_spill_off, int32_t dst_spill_off, int nwords) +{ + ScratchRegAlloc src_scratch = get_scratch_reg_with_save(0); + int r_src = src_scratch.reg; + ScratchRegAlloc dst_scratch = get_scratch_reg_with_save(1u << (uint32_t)r_src); + int r_dst = dst_scratch.reg; + + tcc_machine_addr_of_stack_slot(r_src, src_spill_off, 0); + tcc_machine_addr_of_stack_slot(r_dst, dst_spill_off, 0); + + int max_data = nwords < 4 ? nwords : 4; + if (max_data < 1) + max_data = 1; + + ScratchRegAlloc data_scratches[4]; + int data_regs[4]; + int ndata = 0; + uint32_t exclude = (1u << (uint32_t)r_src) | (1u << (uint32_t)r_dst); + for (int k = 0; k < max_data; k++) + { + data_scratches[k] = get_scratch_reg_with_save(exclude); + data_regs[k] = data_scratches[k].reg; + exclude |= (1u << (uint32_t)data_regs[k]); + ndata++; + } + + int remaining = nwords; + + while (remaining >= ndata && ndata >= 2) + { + uint32_t regset = 0; + for (int j = 0; j < ndata; j++) + regset |= (1u << (uint32_t)data_regs[j]); + ot_check(th_ldm(r_src, regset, 1 /* writeback */, ENFORCE_ENCODING_NONE)); + ot_check(th_stm(r_dst, regset, 1 /* writeback */, ENFORCE_ENCODING_NONE)); + remaining -= ndata; + } + + int dr = data_regs[0]; + while (remaining > 0) + { + ot_check_ldr_imm(dr, r_src, 0, 6, ENFORCE_ENCODING_NONE); + ot_check_str_imm(dr, r_dst, 0, 6, ENFORCE_ENCODING_NONE); + if (remaining > 1) + { + if (!ot(th_add_imm(r_src, r_src, 4, flags_safe(), ENFORCE_ENCODING_NONE))) + tcc_error("compiler_error: spill_block_copy cannot advance source pointer"); + if (!ot(th_add_imm(r_dst, r_dst, 4, flags_safe(), ENFORCE_ENCODING_NONE))) + tcc_error("compiler_error: spill_block_copy cannot advance dest pointer"); + } + remaining--; + } + + for (int k = ndata - 1; k >= 0; k--) + restore_scratch_reg(&data_scratches[k]); + restore_scratch_reg(&dst_scratch); + restore_scratch_reg(&src_scratch); +} + +ST_FUNC void tcc_gen_machine_trap_mop(void) +{ + /* Emit UDF #0xfe - Undefined instruction for trap */ + ot_check(th_udf(0xfe, ENFORCE_ENCODING_NONE)); +} + +ST_FUNC void tcc_gen_machine_prefetch_mop(MachineOperand addr, int rw) +{ + /* Emit PLD (Preload Data) or PLDW (Preload Data with intent to Write) + * based on the rw hint. + * + * PLD/PLDW are hints to the memory system that data may be needed soon. + * They don't wait for the data and don't fault if the address is invalid. + * + * We support several addressing modes: + * - Register indirect: [Rn] -> use th_pld_imm with offset 0 + * - Register + immediate offset: [Rn, #imm] + * - Literal (PC-relative): label + */ + (void)rw; /* PLD/PLDW distinction may not be supported on all ARM variants */ + + switch (addr.kind) + { + case MACH_OP_REG: + { + /* Register indirect: PLD [Rn] */ + int reg = addr.u.reg.r0; + ot_check(th_pld_imm((uint32_t)reg, 0, 0)); + break; + } + case MACH_OP_SPILL: + { + /* Spill slot: compute address (FP + offset) then PLD */ + int32_t offset = addr.u.spill.offset; + if (offset != 0) + { + ScratchRegAlloc scr = get_scratch_reg_with_save(0); + load_full_const(scr.reg, PREG_NONE, LFC_SPLIT(offset)); + ot_check(th_add_reg(scr.reg, R_FP, scr.reg, flags_safe(), THUMB_SHIFT_DEFAULT, + ENFORCE_ENCODING_NONE)); + ot_check(th_pld_imm(scr.reg, 0, 0)); + restore_scratch_reg(&scr); + } + else + { + ot_check(th_pld_imm(R_FP, 0, 0)); + } + break; + } + case MACH_OP_IMM: + { + /* For immediate addresses, load into a register first */ + ScratchRegAlloc scr = get_scratch_reg_with_save(0); + load_full_const(scr.reg, PREG_NONE, LFC_SPLIT(addr.u.imm.val)); + ot_check(th_pld_imm(scr.reg, 0, 0)); + restore_scratch_reg(&scr); + break; + } + case MACH_OP_SYMBOL: + { + /* For symbol addresses, load into a register first */ + ScratchRegAlloc scr = get_scratch_reg_with_save(0); + _lfc_sym = addr.u.sym.sym; + load_full_const(scr.reg, PREG_NONE, LFC_SPLIT(addr.u.sym.addend)); + ot_check(th_pld_imm(scr.reg, 0, 0)); + restore_scratch_reg(&scr); + break; + } + case MACH_OP_FRAME_ADDR: + { + /* Frame address: frame base + the frame offset, adjusted like any local + * (this used to add the raw IR offset to r7, frame pointer or not). */ + int32_t offset = fp_adjust_local_offset(addr.u.frame.offset, 0); + const int base = tcc_state->need_frame_pointer ? R_FP : R_SP; + if (offset != 0) + { + ScratchRegAlloc scr = get_scratch_reg_with_save(0); + load_full_const(scr.reg, PREG_NONE, LFC_SPLIT(offset)); + ot_check(th_add_reg(scr.reg, base, scr.reg, flags_safe(), THUMB_SHIFT_DEFAULT, + ENFORCE_ENCODING_NONE)); + ot_check(th_pld_imm(scr.reg, 0, 0)); + restore_scratch_reg(&scr); + } + else + { + ot_check(th_pld_imm(base, 0, 0)); + } + break; + } + default: + tcc_error("unsupported operand type for __builtin_prefetch"); + } +} + +/* __builtin_setjmp implementation for ARM Thumb-2. + * + * GCC's documented ABI gives __builtin_setjmp a 5-word buffer; callers + * (e.g. gcc.c-torture pr84521) really do pass `void *buf[5]`, so nothing + * larger may be written through the buffer pointer. The callee-saved + * register file (r4-r11) still must be restored on longjmp — the register + * allocator keeps VARs and the R9 GOT base in r4-r11 across the setjmp — + * so those 8 words live in a hidden, compiler-allocated save area in the + * setjmp-containing function's frame (src2/area), which stays valid for + * as long as a longjmp to this buffer is legal. + * + * Jump buffer layout (4 words used, fits the standard 5-word buffer): + * buf[0] = frame pointer (R7/FP) + * buf[1] = resume address (Thumb-bit set) + * buf[2] = stack pointer (SP) + * buf[3] = address of the hidden r4-r11 save area (32 bytes) + * + * Returns 0 on initial call, 1 when returning via longjmp. + */ +ST_FUNC void tcc_gen_machine_setjmp_mop(MachineOperand buf, MachineOperand area, MachineOperand dest) +{ + MachineCodegenContext ctx = {0}; + int buf_reg; + + if (buf.kind == MACH_OP_NONE) + { + buf_reg = mach_alloc_scratch(&ctx, 0); + ot_check(th_mov_imm(buf_reg, 0, flags_safe(), ENFORCE_ENCODING_NONE)); + } + else + { + /* Exclude r4-r11 as scratch candidates: a saved-scratch there would + * hold the buffer pointer when the area stores below run, corrupting + * the saved register file (same class as the MLA scratch-pop bug). */ + buf_reg = mach_ensure_in_reg(&ctx, &buf, 0x0FF0); + } + + /* ---- save callee-saved r4-r11 into the hidden frame area ---- + * The area address is computed in IP (caller-saved) so the r4-r11 + * values stored are the untouched setjmp-time ones; a scratch from + * mach_alloc_scratch could pick a callee-saved register. */ + if (area.kind == MACH_OP_FRAME_ADDR) + { + tcc_machine_addr_of_stack_slot(R_IP, area.u.frame.offset, 0 /* not param */); + } + else + { + tcc_error("compiler_error: setjmp save area must be a frame slot (kind %d)", (int)area.kind); + } + ot_check_str_imm(4, R_IP, 0, 6, ENFORCE_ENCODING_NONE); /* r4 -> area[0] */ + ot_check_str_imm(5, R_IP, 4, 6, ENFORCE_ENCODING_NONE); /* r5 -> area[1] */ + ot_check_str_imm(6, R_IP, 8, 6, ENFORCE_ENCODING_NONE); /* r6 -> area[2] */ + ot_check_str_imm(R_FP, R_IP, 12, 6, ENFORCE_ENCODING_NONE); /* r7 -> area[3] */ + ot_check_str_imm(8, R_IP, 16, 6, ENFORCE_ENCODING_NONE); /* r8 -> area[4] */ + ot_check_str_imm(9, R_IP, 20, 6, ENFORCE_ENCODING_NONE); /* r9 -> area[5] */ + ot_check_str_imm(10, R_IP, 24, 6, ENFORCE_ENCODING_NONE); /* r10 -> area[6] */ + ot_check_str_imm(11, R_IP, 28, 6, ENFORCE_ENCODING_NONE); /* r11 -> area[7] */ + ot_check_str_imm(R_IP, buf_reg, 12, 6, ENFORCE_ENCODING_NONE); /* &area -> buf[3] */ + + /* ---- save frame pointer ---- */ + ot_check_str_imm(R_FP, buf_reg, 0, 6, ENFORCE_ENCODING_NONE); /* r7 -> buf[0] */ + + /* ---- save SP ---- */ + ot_check_mov_reg(R_IP, R_SP, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE, false); + ot_check_str_imm(R_IP, buf_reg, 8, 6, ENFORCE_ENCODING_NONE); /* SP -> buf[2] */ + + /* ---- save resume address (ADR IP, resume_label) ---- */ + int adr_addr = ind; + int adr_pc = adr_addr + 4; + int adr_base = adr_pc & ~3; + int resume_label_addr = adr_addr + 20; /* 4(ORR)+4(STR)+4(MOV)+4(B) after ADR */ + int adr_imm = resume_label_addr - adr_base; + ot_check(th_adr_imm(R_IP, adr_imm, ENFORCE_ENCODING_32BIT)); + + ot_check(th_orr_imm(R_IP, R_IP, 1, flags_safe(), ENFORCE_ENCODING_NONE)); /* Thumb bit */ + ot_check_str_imm(R_IP, buf_reg, 4, 6, ENFORCE_ENCODING_NONE); /* -> buf[1] */ + + /* ---- normal path: return 0 ---- */ + int dest_reg = mach_get_dest_reg(&ctx, &dest, 0); + ot_check(th_mov_imm(dest_reg, 0, flags_safe(), ENFORCE_ENCODING_32BIT)); /* dest = 0 */ + ot_check(th_b_t4(4)); /* B.W +4 (skip resume) */ + codegen_internal_merge_point(); /* longjmp lands below */ + + /* ---- resume_label: longjmp lands here ---- */ + ot_check(th_mov_imm(dest_reg, 1, flags_safe(), ENFORCE_ENCODING_32BIT)); /* dest = 1 */ + /* ---- end_label ---- */ + + mach_writeback_dest(&dest, dest_reg); + mach_release_all(&ctx); +} + +/* Non-local goto setjmp: saves ALL callee-saved registers (r4-r11), SP, + * and resume address in a 40-byte buffer. Used for __label__ + nested + * function goto support. + * + * Buffer layout (10 words = 40 bytes): + * buf[0-7] = r4-r11 + * buf[8] = SP + * buf[9] = resume address (Thumb-bit set) + */ +ST_FUNC void tcc_gen_machine_nl_setjmp_mop(MachineOperand buf, MachineOperand dest) +{ + MachineCodegenContext ctx = {0}; + int buf_reg; + + if (buf.kind == MACH_OP_NONE) + { + buf_reg = mach_alloc_scratch(&ctx, 0); + ot_check(th_mov_imm(buf_reg, 0, flags_safe(), ENFORCE_ENCODING_NONE)); + } + else + { + buf_reg = mach_ensure_in_reg(&ctx, &buf, 0); + } + + /* ---- save callee-saved registers r4-r11 ---- */ + ot_check_str_imm(4, buf_reg, 0, 6, ENFORCE_ENCODING_NONE); /* r4 -> buf[0] */ + ot_check_str_imm(5, buf_reg, 4, 6, ENFORCE_ENCODING_NONE); /* r5 -> buf[1] */ + ot_check_str_imm(6, buf_reg, 8, 6, ENFORCE_ENCODING_NONE); /* r6 -> buf[2] */ + ot_check_str_imm(R_FP, buf_reg, 12, 6, ENFORCE_ENCODING_NONE); /* r7 -> buf[3] */ + ot_check_str_imm(8, buf_reg, 16, 6, ENFORCE_ENCODING_NONE); /* r8 -> buf[4] */ + ot_check_str_imm(9, buf_reg, 20, 6, ENFORCE_ENCODING_NONE); /* r9 -> buf[5] */ + ot_check_str_imm(10, buf_reg, 24, 6, ENFORCE_ENCODING_NONE); /* r10 -> buf[6] */ + ot_check_str_imm(11, buf_reg, 28, 6, ENFORCE_ENCODING_NONE); /* r11 -> buf[7] */ + + /* ---- save SP ---- */ + ot_check_mov_reg(R_IP, R_SP, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE, false); + ot_check_str_imm(R_IP, buf_reg, 32, 6, ENFORCE_ENCODING_NONE); /* SP -> buf[8] */ + + /* ---- save resume address (ADR IP, resume_label) ---- */ + int adr_addr = ind; + int adr_pc = adr_addr + 4; + int adr_base = adr_pc & ~3; + int resume_label_addr = adr_addr + 20; /* 4(ORR)+4(STR)+4(MOV)+4(B) after ADR */ + int adr_imm = resume_label_addr - adr_base; + ot_check(th_adr_imm(R_IP, adr_imm, ENFORCE_ENCODING_32BIT)); + + ot_check(th_orr_imm(R_IP, R_IP, 1, flags_safe(), ENFORCE_ENCODING_NONE)); /* Thumb bit */ + ot_check_str_imm(R_IP, buf_reg, 36, 6, ENFORCE_ENCODING_NONE); /* -> buf[9] */ + + /* ---- normal path: return 0 ---- */ + int dest_reg = mach_get_dest_reg(&ctx, &dest, 0); + ot_check(th_mov_imm(dest_reg, 0, flags_safe(), ENFORCE_ENCODING_32BIT)); /* dest = 0 */ + ot_check(th_b_t4(4)); /* B.W +4 (skip resume) */ + codegen_internal_merge_point(); /* longjmp lands below */ + + /* ---- resume_label: longjmp lands here ---- */ + ot_check(th_mov_imm(dest_reg, 1, flags_safe(), ENFORCE_ENCODING_32BIT)); /* dest = 1 */ + /* ---- end_label ---- */ + + mach_writeback_dest(&dest, dest_reg); + mach_release_all(&ctx); +} + +/* __builtin_longjmp implementation for ARM Thumb-2. + * + * Restores the callee-saved register file (r4-r11, from the hidden save + * area whose address setjmp left in buf[3]) and SP, then jumps to the + * resume address. This function does not return, so every caller-saved + * register is fair game as a temporary. + * + * Buffer layout (must match tcc_gen_machine_setjmp_mop): + * buf[0] = FP, buf[1] = resume_addr, buf[2] = SP, buf[3] = &save_area + */ +ST_FUNC void tcc_gen_machine_longjmp_mop(MachineOperand buf) +{ + MachineCodegenContext ctx = {0}; + int buf_reg; + + if (buf.kind == MACH_OP_NONE) + { + tcc_error("__builtin_longjmp: invalid buffer operand"); + return; + } + + buf_reg = mach_ensure_in_reg(&ctx, &buf, 0); + + /* Copy buf pointer to IP so it survives the register restores */ + ot_check_mov_reg(R_IP, buf_reg, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE, false); + + /* Read resume address, saved SP and save-area pointer into caller-saved + * regs before clobbering anything callee-saved. */ + ot_check_ldr_imm(0, R_IP, 4, 6, ENFORCE_ENCODING_NONE); /* r0 = resume addr */ + ot_check_ldr_imm(1, R_IP, 8, 6, ENFORCE_ENCODING_NONE); /* r1 = saved SP */ + ot_check_ldr_imm(2, R_IP, 12, 6, ENFORCE_ENCODING_NONE); /* r2 = &save_area */ + + /* Restore callee-saved r4-r11 (r7/FP comes from the area too; the copy + * in buf[0] is identical). */ + ot_check_ldr_imm(4, 2, 0, 6, ENFORCE_ENCODING_NONE); /* r4 */ + ot_check_ldr_imm(5, 2, 4, 6, ENFORCE_ENCODING_NONE); /* r5 */ + ot_check_ldr_imm(6, 2, 8, 6, ENFORCE_ENCODING_NONE); /* r6 */ + ot_check_ldr_imm(R_FP, 2, 12, 6, ENFORCE_ENCODING_NONE); /* r7 */ + ot_check_ldr_imm(8, 2, 16, 6, ENFORCE_ENCODING_NONE); /* r8 */ + allow_r9_write = 1; /* restoring the setjmp-time GOT base is the point */ + ot_check_ldr_imm(9, 2, 20, 6, ENFORCE_ENCODING_NONE); /* r9 */ + allow_r9_write = 0; + ot_check_ldr_imm(10, 2, 24, 6, ENFORCE_ENCODING_NONE); /* r10 */ + ot_check_ldr_imm(11, 2, 28, 6, ENFORCE_ENCODING_NONE); /* r11 */ + + /* Restore SP */ + ot_check_mov_reg(R_SP, 1, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE, false); + + /* Jump to resume address (Thumb bit already set by setjmp code) */ + ot_check(th_bx_reg(0)); + + mach_release_all(&ctx); +} + +/* Non-local goto longjmp: restores ALL callee-saved registers (r4-r11), SP, + * then jumps to the resume address. Used for __label__ + nested function goto. + * + * Buffer layout (must match nl_setjmp): + * buf[0-7] = r4-r11, buf[8] = SP, buf[9] = resume_addr + */ +ST_FUNC void tcc_gen_machine_nl_longjmp_mop(MachineOperand buf) +{ + MachineCodegenContext ctx = {0}; + int buf_reg; + + if (buf.kind == MACH_OP_NONE) + { + tcc_error("nl_longjmp: invalid buffer operand"); + return; + } + + if (buf.kind == MACH_OP_CHAIN_REL) + { + /* For chain-relative buffers (non-local goto from nested function), + * we need the ADDRESS of the buffer in the parent frame, not the value. + * mach_ensure_in_reg would load the value; use LEA logic instead. */ + ScratchRegAlloc chain_scratch = {0}; + int chain_used = 0; + uint32_t excl = 0; + int base = resolve_chain_base(tcc_state->ir, buf.u.chain.chain_index, excl, &chain_scratch, &chain_used); + buf_reg = mach_alloc_scratch(&ctx, excl | (1u << (uint32_t)base)); + int32_t off = buf.u.chain.offset; + int sign = (off < 0); + int abs_off = sign ? (int)(-off) : (int)off; + if (abs_off == 0) + { + if (buf_reg != base) + ot_check_mov_reg((uint32_t)buf_reg, (uint32_t)base, flags_safe(), THUMB_SHIFT_DEFAULT, + ENFORCE_ENCODING_NONE, false); + } + else + { + thumb_opcode ins = sign + ? th_sub_imm(buf_reg, base, abs_off, flags_safe(), ENFORCE_ENCODING_NONE) + : th_add_imm(buf_reg, base, abs_off, flags_safe(), ENFORCE_ENCODING_NONE); + if (ins.size != 0) + { + ot_check(ins); + } + else + { + ScratchRegAlloc off_sc = get_scratch_reg_with_save(excl | (1u << (uint32_t)buf_reg) | (1u << (uint32_t)base)); + load_full_const(off_sc.reg, PREG_NONE, LFC_SPLIT(abs_off)); + ot_check(sign ? th_sub_reg(buf_reg, base, off_sc.reg, flags_safe(), THUMB_SHIFT_DEFAULT, + ENFORCE_ENCODING_NONE) + : th_add_reg(buf_reg, base, off_sc.reg, flags_safe(), THUMB_SHIFT_DEFAULT, + ENFORCE_ENCODING_NONE)); + restore_scratch_reg(&off_sc); + } + } + if (chain_used) + restore_scratch_reg(&chain_scratch); + } + else + { + buf_reg = mach_ensure_in_reg(&ctx, &buf, 0); + } + + /* Copy buf pointer to IP so it survives register restores */ + ot_check_mov_reg(R_IP, buf_reg, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE, false); + + /* Load resume address and saved SP into caller-saved regs first + * (before we clobber r4+ with the restore) */ + ot_check_ldr_imm(0, R_IP, 36, 6, ENFORCE_ENCODING_NONE); /* r0 = resume addr */ + ot_check_ldr_imm(1, R_IP, 32, 6, ENFORCE_ENCODING_NONE); /* r1 = saved SP */ + + /* Restore callee-saved registers r4-r11 */ + ot_check_ldr_imm(4, R_IP, 0, 6, ENFORCE_ENCODING_NONE); /* r4 = buf[0] */ + ot_check_ldr_imm(5, R_IP, 4, 6, ENFORCE_ENCODING_NONE); /* r5 = buf[1] */ + ot_check_ldr_imm(6, R_IP, 8, 6, ENFORCE_ENCODING_NONE); /* r6 = buf[2] */ + ot_check_ldr_imm(R_FP, R_IP, 12, 6, ENFORCE_ENCODING_NONE); /* r7 = buf[3] (FP) */ + ot_check_ldr_imm(8, R_IP, 16, 6, ENFORCE_ENCODING_NONE); /* r8 = buf[4] */ + allow_r9_write = 1; + ot_check_ldr_imm(9, R_IP, 20, 6, ENFORCE_ENCODING_NONE); /* r9 = buf[5] */ + allow_r9_write = 0; + ot_check_ldr_imm(10, R_IP, 24, 6, ENFORCE_ENCODING_NONE); /* r10 = buf[6] */ + ot_check_ldr_imm(11, R_IP, 28, 6, ENFORCE_ENCODING_NONE); /* r11 = buf[7] */ + + /* Restore SP */ + ot_check_mov_reg(R_SP, 1, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE, false); + + /* Jump to resume address (Thumb bit already set by setjmp code) */ + ot_check(th_bx_reg(0)); + + mach_release_all(&ctx); +} + +/* ============================================================================ + * __builtin_apply_args / __builtin_apply implementation for ARM Thumb-2 + * ============================================================================ + * + * __builtin_apply_args() returns a pointer to a saved argument block: + * [0] pointer to incoming stack arguments (above saved register area) + * [4] saved r0 + * [8] saved r1 + * [12] saved r2 + * [16] saved r3 + * + * The prologue stores r0-r3 and the stack args pointer when + * func_save_apply_args is set. This handler just computes the address. + * + * __builtin_apply(fn, args, size) restores r0-r3 from the args block, + * calls fn via BLX, and returns the result in dest (r0). + */ + +ST_FUNC void tcc_gen_machine_builtin_apply_args_mop(MachineOperand dest) +{ + MachineCodegenContext ctx = {0}; + + /* The apply_args block lives at tcc_state->apply_args_offset relative to FP. + * Compute FP + adjusted_offset into the dest register. */ + int dest_reg = mach_get_dest_reg(&ctx, &dest, 0); + int offset = tcc_state->apply_args_offset; + tcc_machine_addr_of_stack_slot(dest_reg, offset, 0 /* not param */); + + mach_writeback_dest(&dest, dest_reg); + mach_release_all(&ctx); +} + +/* __builtin_return_address(0). force_lr_save makes every prologue save LR, + * and LR is the highest register of its register push, so it sits in the word + * just below the argument base: param offset 0 is the pushed r0 of a function + * whose r0-r3 are saved first (split struct), else the first stack argument. + * A variadic function's offset_to_args also spans its r0-r3 save. The same + * holds for the two-phase {r7, lr} frame record. Reading the slot off SP + * needs no frame pointer, which used to be forced here: a Thumb r7 frame + * addresses locals at -255..0, so every other access of a large function + * cost movw + rsb + ldr (zig's allocator helpers read @returnAddress()). */ +ST_FUNC void tcc_gen_machine_return_address_mop(MachineOperand dest) +{ + MachineCodegenContext ctx = {0}; + int rd = mach_get_dest_reg(&ctx, &dest, 0); + const int adjusted = param_frame_offset(-4 - (func_var ? vararg_push_size : 0)); + const int base_reg = tcc_state->need_frame_pointer ? R_FP : R_SP; + const int sign = adjusted < 0; + load_from_base(rd, PREG_REG_NONE, IROP_BTYPE_INT32, 0, sign ? -adjusted : adjusted, sign, (uint32_t)base_reg); + mach_writeback_dest(&dest, rd); + mach_release_all(&ctx); +} + +ST_FUNC void tcc_gen_machine_builtin_apply_mop(MachineOperand fn, MachineOperand args, MachineOperand dest) +{ + MachineCodegenContext ctx = {0}; + + /* Registers destroyed by the restore-and-call sequence below: r0-r3 are + * reloaded with the saved argument values, and ip(r12)+lr are clobbered by + * the BLX. The args-block base pointer (used by all four restore loads) and + * the callee address must therefore live OUTSIDE this set until used. */ + const uint32_t clobbered = + (1u << R0) | (1u << R1) | (1u << R2) | (1u << R3) | (1u << (uint32_t)R_IP); + + /* Step 1: Materialize the args block pointer, then guarantee it is in a + * register the restore loads won't overwrite. mach_ensure_in_reg returns an + * already-allocated operand register verbatim (ignoring the exclusion mask), + * so when the value already lives in r0-r3 / ip we must relocate it to a + * safe scratch — otherwise the very first load (r0 <- [base+4]) destroys the + * base pointer and the remaining loads read from garbage addresses. */ + int args_reg = mach_ensure_in_reg(&ctx, &args, clobbered); + if (clobbered & (1u << (uint32_t)args_reg)) + { + int safe = mach_alloc_scratch(&ctx, clobbered); + ot_check_mov_reg(safe, args_reg, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE, false); + args_reg = safe; + } + + /* Step 2: Load the function pointer into R12 (IP), which survives the + * register loads below because IP is not one of r0-r3. */ + int fn_reg = mach_ensure_in_reg(&ctx, &fn, (1u << (uint32_t)args_reg)); + if (fn_reg != R_IP) + { + ot_check_mov_reg(R_IP, fn_reg, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE, false); + } + + /* Step 3: Restore r0-r3 from the args block. + * Layout: [+0]=stack_args_ptr, [+4]=r0, [+8]=r1, [+12]=r2, [+16]=r3. */ + ot_check_ldr_imm(R0, args_reg, 4, 6, ENFORCE_ENCODING_NONE); + ot_check_ldr_imm(R1, args_reg, 8, 6, ENFORCE_ENCODING_NONE); + ot_check_ldr_imm(R2, args_reg, 12, 6, ENFORCE_ENCODING_NONE); + ot_check_ldr_imm(R3, args_reg, 16, 6, ENFORCE_ENCODING_NONE); + + /* Step 4: Call the function via BLX R12. + * This clobbers LR and r0-r3 (caller-saved). */ + ot_check(th_blx_reg(R_IP)); + + /* Step 5: Move return value (r0) to dest register. */ + int dest_reg = mach_get_dest_reg(&ctx, &dest, 0); + if (dest_reg != R0) + { + ot_check_mov_reg(dest_reg, R0, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE, false); + } + + mach_writeback_dest(&dest, dest_reg); + mach_release_all(&ctx); +} + +ST_FUNC void tcc_gen_machine_backpatch_jump(int address, int offset) +{ + th_patch_call(address, offset); +} + +static int tcc_get_type_size(CType *type) +{ + switch (type->t & VT_BTYPE) + { + case VT_BYTE: + return 1; + case VT_SHORT: + return 2; + case VT_INT: + case VT_LONG: + return 4; + case VT_LLONG: + return 8; + case VT_FLOAT: + return 4; + case VT_DOUBLE: + return 8; + case VT_LDOUBLE: + return 8; // treat long double as double for ARM EABI softcalls + default: + return 0; + } +} + +ST_FUNC const char *tcc_get_abi_softcall_name(SValue *src1, SValue *src2, SValue *dest, TccIrOp op) +{ + const int src1_64bit = tcc_is_64bit_operand(src1); + const int src2_64bit = src2 ? tcc_is_64bit_operand(src2) : 0; + const int dest_64bit = dest ? tcc_is_64bit_type(dest->type.t) : 0; + const int src1_size = tcc_get_type_size(&src1->type); + const int dest_size = dest ? tcc_get_type_size(&dest->type) : 0; + + if (src1_64bit || src2_64bit || dest_64bit) + { + switch (op) + { + case TCCIR_OP_FADD: + return "__aeabi_dadd"; + case TCCIR_OP_FSUB: + return "__aeabi_dsub"; + case TCCIR_OP_FMUL: + return "__aeabi_dmul"; + case TCCIR_OP_FDIV: + return "__aeabi_ddiv"; + case TCCIR_OP_FNEG: + return "__aeabi_dneg"; + default: + break; + } + } + else + { + switch (op) + { + case TCCIR_OP_FADD: + return "__aeabi_fadd"; + case TCCIR_OP_FSUB: + return "__aeabi_fsub"; + case TCCIR_OP_FMUL: + return "__aeabi_fmul"; + case TCCIR_OP_FDIV: + return "__aeabi_fdiv"; + case TCCIR_OP_FNEG: + return "__aeabi_fneg"; + default: + break; + } + } + + switch (op) + { + case TCCIR_OP_CVT_FTOF: + { + if (src1_size == 4 && dest_size == 8) + { + return "__aeabi_f2d"; + } + else if (src1_size == 8 && dest_size == 4) + { + return "__aeabi_d2f"; + } + /* Same size conversion is a no-op, no function needed */ + return NULL; + } + break; + case TCCIR_OP_CVT_FTOI: + { + /* Float/double to integer conversion. + * Map based on destination width, not just VT_BTYPE (since VT_LONG is 32-bit on ARM). + * Use the standard ARM EABI helpers: + * - 32-bit: __aeabi_{f,d}2iz / __aeabi_{f,d}2uiz + * - 64-bit: __aeabi_{f,d}2lz / __aeabi_{f,d}2ulz + */ + const int is_float = (src1_size == 4); + const int is_unsigned = (dest && (dest->type.t & VT_UNSIGNED)) ? 1 : 0; + + if (dest_size == 8) + { + return is_unsigned ? (is_float ? "__aeabi_f2ulz" : "__aeabi_d2ulz") + : (is_float ? "__aeabi_f2lz" : "__aeabi_d2lz"); + } + + return is_unsigned ? (is_float ? "__aeabi_f2uiz" : "__aeabi_d2uiz") : (is_float ? "__aeabi_f2iz" : "__aeabi_d2iz"); + } + break; + case TCCIR_OP_FCMP: + { + /* Get comparison operation from src2.c.i (stored during IR generation) */ + int cmp_op = src2->c.i; + int is_float = (src1_size == 4); + + switch (cmp_op) + { + case TOK_EQ: + return is_float ? "__aeabi_fcmpeq" : "__aeabi_dcmpeq"; + case TOK_NE: + /* NE uses cmpeq and inverts the result */ + return is_float ? "__aeabi_fcmpeq" : "__aeabi_dcmpeq"; + case TOK_LT: + case TOK_ULT: + return is_float ? "__aeabi_fcmplt" : "__aeabi_dcmplt"; + case TOK_LE: + case TOK_ULE: + return is_float ? "__aeabi_fcmple" : "__aeabi_dcmple"; + case TOK_GT: + case TOK_UGT: + return is_float ? "__aeabi_fcmpgt" : "__aeabi_dcmpgt"; + case TOK_GE: + case TOK_UGE: + return is_float ? "__aeabi_fcmpge" : "__aeabi_dcmpge"; + default: + /* Fallback to cfcmple/cdcmple which sets flags */ + return is_float ? "__aeabi_cfcmple" : "__aeabi_cdcmple"; + } + } + break; + case TCCIR_OP_CVT_ITOF: + { + /* Integer to float/double conversion. + * Need to distinguish 32-bit int vs 64-bit long long sources: + * - 32-bit: __aeabi_{ui,i}2{d,f} + * - 64-bit: __aeabi_{ul,l}2{d,f} + */ + int is_unsigned = (src1->type.t & VT_UNSIGNED) ? 1 : 0; + if (src1_size == 8) + { + /* 64-bit integer source (long long / unsigned long long) */ + if (is_unsigned) + return dest_64bit ? "__aeabi_ul2d" : "__aeabi_ul2f"; + return dest_64bit ? "__aeabi_l2d" : "__aeabi_l2f"; + } + /* 32-bit integer source (int / unsigned int) */ + if (is_unsigned) + return dest_64bit ? "__aeabi_ui2d" : "__aeabi_ui2f"; + return dest_64bit ? "__aeabi_i2d" : "__aeabi_i2f"; + } + break; + default: + break; + } + + return NULL; +} + +/* tcc_gen_machine_func_parameter_mop: MachineOperand-based entry point for + * FUNCPARAMVAL / FUNCPARAMVOID. src2_enc must be MACH_OP_IMM holding the + * packed call_id / param_idx value (same encoding as irop_get_imm64_ex). + * src1 is the value being passed (unused here — handled by the call-site ABI). + */ +ST_FUNC void tcc_gen_machine_func_parameter_mop(MachineOperand src1, MachineOperand src2_enc, TccIrOp op) +{ + (void)src1; + + const uint32_t encoded = (uint32_t)src2_enc.u.imm.val; + int call_id = TCCIR_DECODE_CALL_ID(encoded); + int param_index = TCCIR_DECODE_PARAM_IDX(encoded); + + /* Find or create call site for this call_id */ + ThumbGenCallSite *call_site = thumb_get_or_create_call_site(call_id); + if (call_site == NULL) + { + tcc_error("compiler_error: failed to allocate call site for call_id=%d", call_id); + return; + } + + /* FUNCPARAMVOID is a marker for a 0-argument call. + * Ensure the call site exists, but do not create a fake argument entry. */ + if (op == TCCIR_OP_FUNCPARAMVOID) + return; + + /* During dry-run, don't modify the argument list - it causes memory leaks + * when we restore the call sites after dry-run. The argument list is not + * needed for scratch register tracking anyway. */ + if (dry_run_state.active) + return; + + /* Expand argument list if needed */ + if (param_index >= call_site->function_argument_count) + { + int new_count = param_index + 1; + call_site->function_argument_list = (int *)tcc_realloc(call_site->function_argument_list, new_count * sizeof(int)); + /* Initialize new slots */ + for (int i = call_site->function_argument_count; i < new_count; i++) + { + call_site->function_argument_list[i] = -1; + } + call_site->function_argument_count = new_count; + } + + /* Store parameter information - for now just mark as present */ + call_site->function_argument_list[param_index] = 1; /* Mark parameter as present */ +} +/* Emit a nested-function trampoline into the current text section. + * chain_slot_sym: TCC symbol for the chain slot in .data + * func_sym: TCC symbol for the nested function in .text + * + * The trampoline loads the parent frame pointer from the chain slot + * into R10 (the static-chain register) and tail-calls the nested function. + * + * Two variants: + * - GOT-indirect (text_and_data_separation): uses R9-relative GOT loads, + * relocations are R_ARM_GOT32 (linker-resolved, no absolute addresses + * in the code section). + * - Direct: inline literal pool with R_ARM_ABS32 relocations. + */ +ST_FUNC addr_t gen_nested_func_trampoline(Sym *chain_slot_sym, Sym *func_sym) +{ + Section *text_sec = cur_text_section; + int use_got = tcc_state->text_and_data_separation; + + section_prealloc(text_sec, use_got ? 36 : 24); + + /* Align ind to 4-byte boundary */ + while (ind & 3) + text_sec->data[ind++] = 0x00; + + addr_t tramp_start = ind; + + if (use_got) + { + /* GOT-indirect trampoline (32 bytes): + * +0: LDR r12, [pc, #20] ; GOT offset of chain_slot (from +24) + * +4: LDR r10, [r9, r12] ; chain_slot address via GOT + * +8: LDR r10, [r10, #0] ; *chain_slot = parent FP + * +12: LDR r12, [pc, #12] ; GOT offset of function (from +28) + * +16: LDR r12, [r9, r12] ; function address via GOT + * +20: BX r12 ; tail-call + * +22: NOP + * +24: .word 0 ; R_ARM_GOT32 chain_slot + * +28: .word 0 ; R_ARM_GOT32 function + */ + + /* +0: LDR R12, [PC, #20] - F8DF C014 */ + text_sec->data[ind++] = 0xDF; + text_sec->data[ind++] = 0xF8; + text_sec->data[ind++] = 0x14; + text_sec->data[ind++] = 0xC0; + + /* +4: LDR R10, [R9, R12] - F859 A00C */ + text_sec->data[ind++] = 0x59; + text_sec->data[ind++] = 0xF8; + text_sec->data[ind++] = 0x0C; + text_sec->data[ind++] = 0xA0; + + /* +8: LDR R10, [R10, #0] - F8DA A000 */ + text_sec->data[ind++] = 0xDA; + text_sec->data[ind++] = 0xF8; + text_sec->data[ind++] = 0x00; + text_sec->data[ind++] = 0xA0; + + /* +12: LDR R12, [PC, #12] - F8DF C00C */ + text_sec->data[ind++] = 0xDF; + text_sec->data[ind++] = 0xF8; + text_sec->data[ind++] = 0x0C; + text_sec->data[ind++] = 0xC0; + + /* +16: LDR R12, [R9, R12] - F859 C00C */ + text_sec->data[ind++] = 0x59; + text_sec->data[ind++] = 0xF8; + text_sec->data[ind++] = 0x0C; + text_sec->data[ind++] = 0xC0; + + /* +20: BX R12 - 4760 */ + text_sec->data[ind++] = 0x60; + text_sec->data[ind++] = 0x47; + + /* +22: NOP - BF00 */ + text_sec->data[ind++] = 0x00; + text_sec->data[ind++] = 0xBF; + + /* +24: chain slot GOT offset */ + greloc(text_sec, chain_slot_sym, ind, R_ARM_GOT32); + text_sec->data[ind++] = 0x00; + text_sec->data[ind++] = 0x00; + text_sec->data[ind++] = 0x00; + text_sec->data[ind++] = 0x00; + + /* +28: function GOT offset */ + greloc(text_sec, func_sym, ind, R_ARM_GOT32); + text_sec->data[ind++] = 0x00; + text_sec->data[ind++] = 0x00; + text_sec->data[ind++] = 0x00; + text_sec->data[ind++] = 0x00; + } + else + { + /* Direct trampoline (20 bytes): + * +0: LDR r10, [pc, #8] ; chain_slot address (from +12) + * +4: LDR r10, [r10, #0] ; *chain_slot = parent FP + * +8: LDR pc, [pc, #4] ; function address (from +16), tail call + * +12: .word chain_slot ; R_ARM_ABS32 + * +16: .word function ; R_ARM_ABS32 + */ + + /* LDR R10, [PC, #8] - F8DF A008 */ + text_sec->data[ind++] = 0xDF; + text_sec->data[ind++] = 0xF8; + text_sec->data[ind++] = 0x08; + text_sec->data[ind++] = 0xA0; + + /* LDR R10, [R10, #0] - F8DA A000 */ + text_sec->data[ind++] = 0xDA; + text_sec->data[ind++] = 0xF8; + text_sec->data[ind++] = 0x00; + text_sec->data[ind++] = 0xA0; + + /* LDR PC, [PC, #4] - F8DF F004 */ + text_sec->data[ind++] = 0xDF; + text_sec->data[ind++] = 0xF8; + text_sec->data[ind++] = 0x04; + text_sec->data[ind++] = 0xF0; + + /* chain slot address */ + greloc(text_sec, chain_slot_sym, ind, R_ARM_ABS32); + text_sec->data[ind++] = 0x00; + text_sec->data[ind++] = 0x00; + text_sec->data[ind++] = 0x00; + text_sec->data[ind++] = 0x00; + + /* function address */ + greloc(text_sec, func_sym, ind, R_ARM_ABS32); + text_sec->data[ind++] = 0x00; + text_sec->data[ind++] = 0x00; + text_sec->data[ind++] = 0x00; + text_sec->data[ind++] = 0x00; + } + + text_sec->data_offset = ind; + return tramp_start + 1; /* +1 for Thumb interworking bit */ +} diff --git a/source/backend/arch/arm/thumb/arm-thumb-callsite.c b/source/backend/arch/arm/thumb/arm-thumb-callsite.c index 3c86f54b5..57a9ca75b 100644 --- a/source/backend/arch/arm/thumb/arm-thumb-callsite.c +++ b/source/backend/arch/arm/thumb/arm-thumb-callsite.c @@ -206,7 +206,7 @@ int thumb_build_call_layout_from_ir(TCCIRState *ir, int call_idx, int call_id, i if (align < 1) align = 1; arg_descs[param_idx].kind = TCC_ABI_ARG_STRUCT_BYVAL; - arg_descs[param_idx].size = (uint16_t)size; + arg_descs[param_idx].size = (uint32_t)size; /* Use AAPCS natural alignment (based on member types, not * __attribute__((aligned)) on the struct). This determines * register alignment (even-register rule for 8-byte aligned). */ diff --git a/source/backend/arch/arm/thumb/arm-thumb-emit.c b/source/backend/arch/arm/thumb/arm-thumb-emit.c new file mode 100644 index 000000000..49b86b0e8 --- /dev/null +++ b/source/backend/arch/arm/thumb/arm-thumb-emit.c @@ -0,0 +1,2251 @@ +/* + * ARMvX-m code generator for TCC + * Uses thumb instruction set + * + * Based on: + * ARM Thumb 2 instruction functions for TCC + * Copyright (c) 2020 Erlend J. Sveen + * from: + * https://git.erlendjs.no/erlendjs/tinycc/-/blob/arm-thumb/arm-thumb-gen.c + * https://git.erlendjs.no/erlendjs/tinycc/-/blob/arm-thumb/arm-thumb-instructions.c + * + * And + * + * ARMv4 code generator for TCC + * + * Copyright (c) 2003 Daniel Glöckner + * Copyright (c) 2012 Thomas Preud'homme + * + * Based on i386-gen.c by Fabrice Bellard + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +/* Thumb-2 code generator: instruction emission (ot), branch patching, SP + * adjustment, frame-word pairing into LDRD/STRD, and loading constants and + * addresses into registers. */ + +#include "arm-thumb-gen.h" + +static int unalias_ldbl(int btype); +static int is_hgen_float_aggr(CType *type); + +int ot(thumb_opcode op) +{ + if (op.size == 0) + return op.size; + + /* Detect instructions that write to R9 when it's reserved for GOT pointer. + * Exclude push/pop/stmdb/ldmia which legitimately save/restore R9. */ + if (text_and_data_separation && !allow_r9_write) + { + int dest = thumb_decode_dest_reg(op); + if (dest == R9) + { + tcc_error("instruction 0x%0*x (size=%d) writes to R9 (GOT pointer) at ind=0x%x ir_op=%d", op.size == 4 ? 8 : 4, + op.opcode, op.size, (unsigned)ind, g_debug_current_op); + } + } + + /* Update the MOV-coalescing register-equivalence cache and the STR->LDR + * redundant-reload cache based on what is about to be emitted. This only + * tracks state — no elision happens here; elision is performed at the + * call sites via ot_check_mov_reg / ot_check_ldr_imm so that ot()'s + * return value remains the real emitted size and downstream jump/offset + * accounting never sees a phantom emission. + * + * IT blocks: instructions inside an IT/ITx/ITxy/ITxyz are conditionally + * executed. Their writes are therefore not guaranteed, so destination + * registers must be invalidated rather than recorded as equivalences. */ + if (thumb_gen_state.generating_function) + { + if (mov_equiv_it_pending > 0) + { + /* Conditional instruction: pessimistically drop anything this op + * might write, and never record new equivalences. Treat STR/LDR + * the same way — their effect is gated on the IT condition. */ + int mv_rd = -1, mv_rm = -1; + if (decode_mov_reg_plain(op, &mv_rd, &mv_rm)) + { + mov_equiv_invalidate_reg(mv_rd); + strldr_cache_invalidate_reg(mv_rd); + imm_cache_invalidate_reg(mv_rd); + } + else if (thumb_op_is_pure_flag_setter(op)) + { + /* CMP/CMN/TST/TEQ — no GPR clobber even under predication. */ + } + else if (thumb_op_is_plain_branch(op)) + { + /* Writes neither a GPR nor memory; see the unconditioned path. */ + } + else + { + int dest = thumb_decode_dest_reg(op); + if (dest >= 0) + { + mov_equiv_invalidate_reg(dest); + strldr_cache_invalidate_reg(dest); + imm_cache_invalidate_reg(dest); + } + else + { + mov_equiv_reset_all(); + tcc_gen_machine_strldr_cache_reset(); + imm_cache_reset_all(); + } + } + mov_equiv_it_pending--; + } + else + { + int it_len = mov_equiv_it_block_length(op); + if (it_len > 0) + { + /* IT itself writes no GPR; start the conditional window. */ + mov_equiv_it_pending = it_len; + } + else + { + int mv_rd = -1, mv_rm = -1; + int sl_is_str = 0, sl_rt = 0, sl_rn = 0, sl_imm = 0, sl_width = 0; + uint32_t sl_puw = 0; + int sd_is_str = 0, sd_rn = 0, sd_imm = 0; + if (decode_str_ldr_imm(op, &sl_is_str, &sl_rt, &sl_rn, &sl_imm, &sl_puw, &sl_width)) + { + if (!sl_is_str) + { + /* LDR writes Rt: invalidate both caches for that register FIRST. + * If the call-site helper ran the match it would have + * elided without reaching ot(); so if we get here, this LDR + * is actually emitting and genuinely clobbers Rt. */ + mov_equiv_invalidate_reg(sl_rt); + strldr_cache_invalidate_reg(sl_rt); + imm_cache_invalidate_reg(sl_rt); + } + /* Either way Rt now holds [Rn+imm], so both directions are worth + * recording; a store additionally kills what it overwrote. */ + strldr_cache_record_access(sl_rt, sl_rn, sl_imm, sl_puw, op.size, sl_width, sl_is_str); + if (sl_puw & 1) + { + /* Write-back form (post/pre-indexed): Rn now holds the UPDATED + * address. record_access already threw the whole strldr cache + * away (puw != 6); the register-equivalence and known-immediate + * caches must drop Rn too, or a later `mov rn,rx` / immediate + * rematerialization gets elided against the pre-increment value. */ + mov_equiv_invalidate_reg(sl_rn); + imm_cache_invalidate_reg(sl_rn); + } + } + else if (decode_mov_reg_plain(op, &mv_rd, &mv_rm)) + { + mov_equiv_record_mov(mv_rd, mv_rm); + strldr_cache_invalidate_reg(mv_rd); + imm_cache_invalidate_reg(mv_rd); + } + else if (op.size == 4 && + (((op.opcode >> 16) & 0xFE40) == 0xE840)) + { + /* LDRD/STRD (Thumb-2): encoded as 1110 100P U1W0 nnnn (STRD) or + * 1110 100P U1W1 nnnn (LDRD). Bit 20 (high-halfword bit 4) + * distinguishes load (1) vs store (0). + * + * STRD writes no GPR — only memory. LDRD writes both Rt and Rt2 + * (low-halfword bits [15:12] and [11:8] respectively). Either way + * the rest of the GPR-equivalence cache is unaffected, so don't + * fall through to the "unknown opcode → reset everything" path + * which destroys upstream coalescing wins. */ + if ((op.opcode >> 20) & 1) + { + /* LDRD: invalidate Rt and Rt2 (writeback to Rn is rare here and + * already covered by the writeback handling — for the typical + * STRD imm with W=0 used by the codegen we don't touch Rn). */ + int rt = (int)((op.opcode >> 12) & 0xF); + int rt2 = (int)((op.opcode >> 8) & 0xF); + mov_equiv_invalidate_reg(rt); + mov_equiv_invalidate_reg(rt2); + strldr_cache_invalidate_reg(rt); + strldr_cache_invalidate_reg(rt2); + imm_cache_invalidate_reg(rt); + imm_cache_invalidate_reg(rt2); + } + else if (decode_strd_ldrd_imm(op, &sd_is_str, &sd_rn, &sd_imm)) + { + /* STRD writes no GPR, but it writes EIGHT bytes of memory: with + * the cache living across IR ops, missing that leaves a stale + * entry for either half. Undecodable form -> assume the worst. */ + strldr_cache_invalidate_mem(sd_rn, sd_imm, 8); + /* Then record what it just put there, as the two 4-byte slots the + * rest of the cache is made of. That is what lets the LDRD which + * reads the pair straight back -- the return-value slot round trip + * every 64-bit helper ends with -- be skipped entirely. */ + if (sd_imm >= 0) + { + /* Only the U=1 form: every entry in this cache is a puw==6 + * access, and handing a subtract-offset one to the recorder + * would make it throw the whole cache away. */ + int sd_rt = (int)((op.opcode >> 12) & 0xF); + int sd_rt2 = (int)((op.opcode >> 8) & 0xF); + strldr_cache_record_access(sd_rt, sd_rn, sd_imm, 6, 4, 4, 1); + strldr_cache_record_access(sd_rt2, sd_rn, sd_imm + 4, 6, 4, 4, 1); + } + } + else + { + tcc_gen_machine_strldr_cache_reset(); + } + } + else if (thumb_op_is_pure_flag_setter(op)) + { + /* CMP/CMN/TST/TEQ write only the flags — no GPR clobber, no + * cache invalidation needed. */ + } + else if (thumb_op_is_plain_branch(op)) + { + /* B / B / CBZ / CBNZ write neither a GPR nor memory, so the + * FALL-THROUGH path's register and memory state is exactly what it + * was before -- and the taken path lands on an IR-level jump target, + * where codegen.c resets. Without this a conditional branch inside + * a basic block would drop every entry, which is the whole shape + * this cache exists for: reload four spilled halves, compare, branch, + * and use the same four in the not-taken arm. BL/BLX are NOT here: + * they write LR and clobber memory, and fall into the reset below. */ + } + else + { + int dest = thumb_decode_dest_reg(op); + if (dest >= 0) + { + mov_equiv_invalidate_reg(dest); + strldr_cache_invalidate_reg(dest); + imm_cache_invalidate_reg(dest); + } + else + { + mov_equiv_reset_all(); + tcc_gen_machine_strldr_cache_reset(); + imm_cache_reset_all(); + } + } + } + } + } + else + { + mov_equiv_reset_all(); + tcc_gen_machine_strldr_cache_reset(); + imm_cache_reset_all(); + } + + /* -Os outliner: an instruction of a window the real pass calls instead is + * recorded, not written, after the caches above saw it -- the BL runs + * exactly this code. `ind` advances as if it had been emitted, so the + * frame-word pair peephole can still rewind over it. */ + if (tcc_gen_machine_outline_suppress(op.opcode, op.size)) + { + thumb_gen_state.code_size += op.size; + ind += op.size; + return op.size; + } + + /* Literal-pool flush safety around IT blocks. Call-site reservations + * (th_literal_pool_reserve_upcoming_bytes) cover the block's CODE bytes, + * but a conditioned arm that materializes a large constant + * (load_full_const) grows the pool AFTER the reservation was checked, so + * the threshold can still trip mid-block. Track the architectural IT + * window here and (a) never flush while an op is conditioned, (b) flush + * BEFORE the IT opcode itself if the worst-case block — 4 code bytes plus + * an 8-byte pool entry per conditioned instruction — could hit the + * threshold, so the deferred flush of (a) never overshoots the LDR-literal + * range. Runs in both passes so dry-run and real layouts stay identical. */ + int op_in_it_block = 0; + if (thumb_gen_state.generating_function) + { + if (pool_flush_it_pending > 0) + { + op_in_it_block = 1; + pool_flush_it_pending--; + } + else + { + int it_len = mov_equiv_it_block_length(op); + if (it_len > 0) + { + /* 12 * it_len: worst case per conditioned insn is 4 code bytes plus + * an 8-byte pool entry. */ + if (th_pool_span_after(op.size + 12 * it_len) >= THUMB_POOL_RANGE_LIMIT - THUMB_POOL_FLUSH_SLACK) + th_literal_pool_generate(); + pool_flush_it_pending = it_len; + } + } + } + + tcc_gen_machine_outline_capture(op.opcode, op.size, ind); + + /* Dry run: don't emit actual opcodes, but still track code size and + * handle literal pool generation to ensure code addresses match real pass. */ + if (dry_run_state.active) + { + if (thumb_gen_state.generating_function) + { + thumb_gen_state.code_size += op.size; + /* Check if literal pool needs to be generated during dry-run. + * We need to call th_literal_pool_generate to properly track the + * code size including the literal pool, so that ind matches + * between dry-run and real pass. */ + if (th_pool_span_after(op.size) >= THUMB_POOL_RANGE_LIMIT - THUMB_POOL_FLUSH_SLACK && !op_in_it_block) + { + th_literal_pool_generate(); + } + } + /* Increment ind as if we emitted the instruction, but don't write to section */ + ind += op.size; + return op.size; + } + + if (thumb_gen_state.generating_function) + { + thumb_gen_state.code_size += op.size; + if (th_pool_span_after(op.size) >= THUMB_POOL_RANGE_LIMIT - THUMB_POOL_FLUSH_SLACK && !op_in_it_block) + { + th_literal_pool_generate(); + } + } + + if (op.size == 4) + o(op.opcode >> 16); + o(op.opcode & 0xffff); + return op.size; +} + +// TODO: this is armv7-m code +int decbranch(int pos) +{ + int xa = *(uint16_t *)(cur_text_section->data + pos); + int xb = *(uint16_t *)(cur_text_section->data + pos + 2); + + TRACE(" decbranch ins at pos 0x%.8x, target inst 0x%x 0x%x", pos, xa, xb); + + if ((xa & 0xf000) == 0xd000) + { + // Branch encoding t1 + xa &= 0x00ff; + if (xa & 0x0080) + xa -= 0x100; + xa = (xa * 2) + pos + 4; + } + else if ((xa & 0xf800) == 0xe000) + { + // Branch encoding t2 + xa &= 0x7ff; + if (xa & 0x400) + xa -= 0x800; + xa = (xa * 2) + pos + 4; + } + else if ((xa & 0xf800) == 0xf000 && (xb & 0xd000) == 0x8000) + { + // Branch encoding t3 + uint32_t s = (xa >> 10) & 1; + uint32_t imm6 = (xa & 0x3f); + uint32_t j1 = (xb >> 13) & 1; + uint32_t j2 = (xb >> 11) & 1; + uint32_t imm11 = xb & 0x7ff; + + // 10 9876543210 9876543210 9876543210 + // IMM: s 21bbbbbbaa aaaaaaaaa0 + // IMM: s21bbbbbba aaaaaaaaaa + uint32_t ret = (j2 << 19) | (j1 << 18) | (imm6 << 12) | (imm11 << 1); + if (s) + ret |= 0xfff00000; + + xa = ret + pos + 4; + } + else if ((xa & 0xf800) == 0xf000 && (xb & 0xd000) == 0x9000) + { + // Branch encoding t4 + uint32_t s = (xa >> 10) & 1; + uint32_t imm10 = (xa & 0x3ff); + uint32_t j1 = (xb >> 13) & 1; + uint32_t j2 = (xb >> 11) & 1; + uint32_t imm11 = xb & 0x7ff; + + uint32_t i1 = ~(j1 ^ s) & 1; + uint32_t i2 = ~(j2 ^ s) & 1; + + // 10 9876543210 9876543210 9876543210 + // IMM: s21bb bbbbbbbbaa aaaaaaaaa0 + uint32_t ret = (i2 << 23) | (i1 << 22) | (imm10 << 12) | (imm11 << 1); + if (s) + ret |= 0xff000000; + + xa = ret + pos + 4; + } + else if ((xa & 0xf500) == 0xb100) + { + /* CBZ/CBNZ encoding: offset = (i:imm5) * 2, forward only */ + uint32_t i_bit = (xa >> 9) & 1; + uint32_t imm5 = (xa >> 3) & 0x1f; + uint32_t imm6 = (i_bit << 5) | imm5; + xa = (int)(imm6 * 2) + pos + 4; + } + else + { + tcc_error("internal error: decbranch unknown encoding pos 0x%x, inst: 0x%x\n", pos, xa); + return 0; + } + + return xa; +} +thumb_opcode th_generic_mov_imm(uint32_t r, int imm) +{ + if (imm < 0) + { + return th_mvn_imm(r, 0, -imm - 1, flags_safe(), ENFORCE_ENCODING_NONE); + } + return th_mov_imm(r, imm, flags_safe(), ENFORCE_ENCODING_NONE); +} + +/* Pure query for the IR-level constant hoist: true when a 32-bit constant has + * no single-instruction materialization (all MOV/MOVW/MVN encodings fail) and + * tcc_machine_load_constant would fall through to a literal-pool load. Probes + * with r0 so the 16-bit MOVS form is reachable — encodability is the same for + * every register from there (MOV.W covers imm8 for high registers). */ +ST_FUNC int tcc_gen_machine_const_needs_pool(int32_t value) +{ + return th_generic_mov_imm(0, value).size == 0; +} +ScratchRegAlloc th_offset_to_reg_ex(int off, int sign, uint32_t exclude_regs) +{ + /* Find a free scratch register (must not clobber excluded regs). + * Returns ScratchRegAlloc struct so caller can manage cleanup. + * Caller MUST call restore_scratch_reg() when done with the register. */ + ScratchRegAlloc alloc = get_scratch_reg_with_save(exclude_regs); + int rr = alloc.reg; + + /* If mov is not possible then load from data */ + if (!ot(th_generic_mov_imm(rr, off))) + { + load_full_const(rr, PREG_NONE, LFC_SPLIT(sign ? -off : off)); + return alloc; + } + + if (sign) + ot_check(th_rsb_imm(rr, rr, 0, flags_safe(), ENFORCE_ENCODING_NONE)); + return alloc; +} + +int th_patch_call(int t, int a) +{ + uint16_t *x = (uint16_t *)(cur_text_section->data + t); + int lt = t; + + TRACE("'th_patch_call' t: %.8x, a: %.8x\n", t, a); + + t = decbranch(t); + TRACE("t: %.8x\n", t); + if (a == lt + 2) + *x = 0xbf00; + else if ((*x & 0xf000) == 0xd000) + { + *x &= 0xff00; + *x |= th_encbranch_8(lt, a); + } + else if ((*x & 0xf800) == 0xe000) + { + *x &= 0xf800; + *x |= th_encbranch_11(lt, a); + } + else if ((x[0] & 0xf800) == 0xf000 && (x[1] & 0xd000) == 0x8000) + { + uint32_t enc = 0; + x[0] &= 0xfbc0; + x[1] &= 0xd000; + enc = th_encbranch_b_t3(th_encbranch_20(lt, a)); + x[0] |= enc >> 16; + x[1] |= enc; + } + else if ((x[0] & 0xf800) == 0xf000 && (x[1] & 0xd000) == 0x9000) + { + uint32_t enc = 0; + x[0] &= 0xf800; + x[1] &= 0xd000; + enc = th_packimm_10_11_0(th_encbranch_20(lt, a) << 1); + x[0] |= enc >> 16; + x[1] |= enc; + } + else if ((*x & 0xf500) == 0xb100) + { + /* CBZ/CBNZ: 16-bit, forward-only, range 0-126 bytes. + * CBZ base = 0xb100, CBNZ base = 0xb900; both match (x & 0xf500) == 0xb100 + * since bit 11 (0x0800) is not in the mask. + * Encoding: op | (i << 9) | (imm5 << 3) | Rn + * where offset = (i:imm5) * 2 */ + int offset = a - (lt + 4); /* PC-relative, Thumb PC = insn + 4 */ + if (offset < 0 || offset > 126 || (offset & 1)) + tcc_error("compiler_error: CBZ/CBNZ target out of range: offset=%d in %s", offset, funcname ? funcname : "?"); + uint32_t imm6 = (uint32_t)offset >> 1; + uint32_t i_bit = (imm6 >> 5) & 1; + uint32_t imm5 = imm6 & 0x1f; + *x &= 0xfd07; /* Keep base opcode, NZ bit, and Rn */ + *x |= (uint16_t)((i_bit << 9) | (imm5 << 3)); + } + else + tcc_error("compiler_error: unhandled branch type in th_patch_call for: t: " + "0x%x, a: 0x%x, x: 0x%x 0x%x\n", + t, a, x[0], x[1]); + + return t; +} + +/* Add a value to SP. When the immediate doesn't fit the ADD/SUB SP encoding, + * a scratch register is needed. scratch_reg selects which one: + * >= 0 : use that specific physical register (caller guarantees it's free) + * < 0 : default to R_IP (safe in prologue/epilogue where R0-R3 hold args) + */ +void gadd_sp_ex(int val, int scratch_reg) +{ + if (val == 0) + return; + + if (scratch_reg < 0) + scratch_reg = R_IP; + + if (val > 0) + { + thumb_opcode add_imm = th_add_imm(R_SP, R_SP, (uint32_t)val, flags_safe(), ENFORCE_ENCODING_NONE); + if (is_valid_opcode(add_imm)) + { + ot(add_imm); + return; + } + + load_full_const(scratch_reg, PREG_NONE, (uint32_t)val, 0); + ot_check( + th_add_reg(R_SP, R_SP, scratch_reg, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); + return; + } + + /* val < 0 */ + const uint32_t sub = (uint32_t)(-val); + thumb_opcode sub_imm = th_sub_imm(R_SP, R_SP, sub, flags_safe(), ENFORCE_ENCODING_NONE); + if (is_valid_opcode(sub_imm)) + { + ot(sub_imm); + return; + } + + load_full_const(scratch_reg, PREG_NONE, (uint32_t)sub, 0); + ot_check(th_sub_reg(R_SP, R_SP, scratch_reg, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); +} +void gadd_sp(int val) +{ + gadd_sp_ex(val, -1); +} + +void ggoto(void) +{ + TRACE("'ggoto'"); + { + SValue target = *vtop; + tcc_ir_put(tcc_state->ir, TCCIR_OP_IJUMP, &target, NULL, NULL); + } + vtop--; + print_vstack("ggoto"); +} + +ST_FUNC void tcc_gen_machine_indirect_jump_mop(MachineOperand src, TccIrOp op) +{ + (void)op; + MachineCodegenContext ctx = {0}; + int target = mach_ensure_in_reg(&ctx, &src, 0); + ot_check(th_bx_reg((uint16_t)target)); + mach_release_all(&ctx); +} + +/* Returns the number of bytes emitted by tcc_gen_machine_switch_table_mop for + * a table with the given number of entries. Used by the dry-run pass in + * codegen.c so that branch-offset analysis is accurate without the backend + * having to emit any real instructions. */ +ST_FUNC int tcc_gen_machine_switch_table_dry_run_size(int num_entries) +{ + /* Layout: LSL.W(4) + ADD(2) + LDR.W(4) + ADD(2) + BX(2) = 14 bytes preamble + * + 4 bytes per table entry (32-bit signed PC-relative offsets). */ + return 14 + num_entries * 4; +} + +/* Force any pending literal pool to be flushed before a region of + * `upcoming_bytes` is emitted, if leaving the pool pending that long would + * push its load out of range. Public wrapper so codegen.c can reserve + * space symmetrically in both the dry-run and real-run passes. + * + * The SWITCH_TABLE dispatch needs this: its preamble (LSL/ADD/LDR/ADD/BX) + * must be emitted atomically — a literal-pool flush in the middle relocates + * the terminal `ADD Rt, PC; BX Rt` past the pool (bridged by a B.W), which + * invalidates the `ref_point == table_start` assumption that the switch- + * table offset backpatch in codegen.c relies on, producing a wild jump. + * Flushing the pool up front (in both passes, so dry-run size estimates and + * real-run addresses stay consistent) keeps the preamble + table contiguous. */ +ST_FUNC void tcc_gen_machine_reserve_pool_bytes(int upcoming_bytes) +{ + th_literal_pool_reserve_upcoming_bytes(upcoming_bytes); +} + +/* MOP variant: accepts a MachineOperand for the index register. */ +ST_FUNC void tcc_gen_machine_switch_table_mop(MachineOperand src, TCCIRSwitchTable *table, TCCIRState *ir, int ir_idx) +{ + (void)ir_idx; + + TRACE("'tcc_gen_machine_switch_table_mop' table_id=%d entries=%d\n", table - ir->switch_tables, table->num_entries); + + MachineCodegenContext ctx = {0}; + /* The index value must be in a register at this point. */ + int index_reg = mach_ensure_in_reg(&ctx, &src, 0); + if (!thumb_is_hw_reg(index_reg)) + tcc_error("internal error: SWITCH_TABLE index not in a hardware register (mop)"); + + /* Use R_IP as scratch to avoid clobbering index_reg, which may still be + live at the switch targets (SSA can place the loop counter directly here). */ + int rt = R_IP; + + ot_check(th_lsl_imm(rt, index_reg, 2, flags_safe(), ENFORCE_ENCODING_32BIT)); + ot_check(th_add_reg(rt, rt, R_PC, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); + ot_check_ldr_imm(rt, rt, 6, 6, ENFORCE_ENCODING_32BIT); + ot_check(th_add_reg(rt, rt, R_PC, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); + ot_check(th_bx_reg(rt)); + + int table_start = ind; + /* The offset table is data in .text — mark it for disassemblers. */ + map_sym_d(); + for (int i = 0; i < table->num_entries; i++) + { + g(0); + g(0); + g(0); + g(0); + } + map_sym_t(); + table->table_code_addr = table_start; + mach_release_all(&ctx); +} + +/* SWITCH_LOAD: data-table dispatch that loads values[index] into dest. + * + * Layout (uniform 14-byte preamble): + * + * LSL.W rt, index, #2 (4 bytes) + * ADD rt, rt, pc (2 bytes) ; PC=preamble_start+8 + * LDR.W ip, [rt, #6] (4 bytes) ; load table[index] -> ip + * B.W skip (4 bytes) ; jump past the table + * (4*N bytes) + * skip: + * [optional STR/MOV ip -> dest] ; only if dest is spilled, + * ; emitted by the IR-level + * ; ASSIGN that follows. + * + * The fixed loaded register is R_IP (same as SWITCH_TABLE's scratch); the + * IR-level optimization wraps SWITCH_LOAD with an ASSIGN that places IP into + * the real dest, so we don't need a separate spill path here. + * + * SYMREF entries emit R_ARM_ABS32 relocations at their table slots; the + * linker fills in the absolute symbol address. + */ +/* SWITCH_LOAD dispatch size: literal-pool LDR (4 bytes, T2 encoding for + * R_IP) + indexed shifted LDR.W (4 bytes). The table itself lives in + * .rodata and contributes no .text bytes. */ +ST_FUNC int tcc_gen_machine_switch_load_dry_run_size(int num_entries) +{ + (void)num_entries; + return 8; +} + +ST_FUNC void tcc_gen_machine_switch_load_mop(MachineOperand src, MachineOperand dest, TCCIRSwitchValueTable *vtab, + TCCIRState *ir, int ir_idx) +{ + (void)ir_idx; + (void)ir; + + TRACE("'tcc_gen_machine_switch_load_mop' vt_id=%d entries=%d\n", (int)(vtab - ir->switch_value_tables), + vtab->num_entries); + + if (!vtab->rodata_sym) + tcc_error("internal error: SWITCH_LOAD table has no rodata symbol (switch_to_data should have allocated it)"); + + MachineCodegenContext ctx = {0}; + /* Keep the index out of R_IP, which we clobber with the table base below. */ + int index_reg = mach_ensure_in_reg(&ctx, &src, (1u << (uint32_t)R_IP)); + if (!thumb_is_hw_reg(index_reg)) + tcc_error("internal error: SWITCH_LOAD index not in a hardware register"); + + /* Resolve the destination register. The switch_to_data optimization tries to + * keep the SWITCH_LOAD dest in a hardware register, but under high register + * pressure the allocator can spill it (or it may be an lvalue store). Rather + * than bail out, allocate a scratch via mach_get_dest_reg() and store it back + * with mach_writeback_dest() afterwards. Exclude index_reg and R_IP — both + * are read by the indexed load below. */ + uint32_t dest_excl = (1u << (uint32_t)index_reg) | (1u << (uint32_t)R_IP); + int dest_reg = mach_get_dest_reg(&ctx, &dest, dest_excl); + + /* Load the table's base address from the literal pool into IP. */ + _lfc_sym = vtab->rodata_sym; + load_full_const(R_IP, PREG_NONE, 0, 0); + + /* dest = table[index] via LDR.W dest, [ip, index, LSL #2]. */ + thumb_shift shift = {THUMB_SHIFT_LSL, 2, THUMB_SHIFT_IMMEDIATE}; + ot_check(th_ldr_reg((uint32_t)dest_reg, (uint32_t)R_IP, (uint32_t)index_reg, shift, ENFORCE_ENCODING_32BIT)); + + /* If the dest was a spill slot or lvalue, write the loaded value back. */ + mach_writeback_dest(&dest, dest_reg); + + mach_release_all(&ctx); +} + +void gsym_addr(int t, int a) +{ + TRACE("'gsym_addr' %.8x branch target: %.8x\n", t, a); + + while (t > 0) /* -1 or 0 means end of chain / no chain */ + t = th_patch_call(t, a); +} + +ST_FUNC void gen_vla_alloc(CType *type, int align) +{ + /* vtop holds the allocation size in bytes. Adjust SP down by that runtime + * size and align it to at least 8 bytes. + * + * This follows the classic TCC scheme: + * r = sp - size + * r = r & ~(align-1) + * sp = r + * + * The size expression is consumed from the value stack. + */ + (void)type; + + int r = gv(RC_INT); + + /* r = SP - r */ + ot_check(th_sub_reg(r, R_SP, r, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); + + if (align < 8) + align = 8; + if (align & (align - 1)) + tcc_error("alignment is not a power of 2: %i", align); + + if (align > 1) + { + /* Try immediate BIC first; if it doesn't encode, fall back to register mask. */ + if (!ot(th_bic_imm(r, r, (uint32_t)(align - 1), flags_safe(), ENFORCE_ENCODING_NONE))) + { + ScratchRegAlloc mask_alloc = get_scratch_reg_with_save(1u << r); + int mask_reg = mask_alloc.reg; + if (!ot(th_generic_mov_imm(mask_reg, align - 1))) + { + load_full_const(mask_reg, PREG_NONE, LFC_SPLIT(align - 1)); + } + ot_check(th_bic_reg(r, r, mask_reg, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); + if (mask_alloc.saved) + { + ot_check(th_pop(1u << mask_reg)); + } + } + } + + /* SP = r */ + ot_check_mov_reg(R_SP, r, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE, false); + + vpop(); +} + +ST_FUNC void gen_vla_sp_save(int addr) +{ + if (nocode_wanted) + return; + + /* Store SP to the local stack slot at frame offset `addr`. */ + int off = fp_adjust_local_offset(addr, 0 /* not param */); + int sign = (off < 0) ? 1 : 0; + int abs_off = sign ? -off : off; + const int base_reg = tcc_state->need_frame_pointer ? R_FP : R_SP; + + ScratchRegAlloc vla_sc = get_scratch_reg_with_save(0); + ot_check_mov_reg(vla_sc.reg, R_SP, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE, false); + th_store32_imm_or_reg_ex(vla_sc.reg, base_reg, abs_off, sign, 0); + restore_scratch_reg(&vla_sc); +} + +ST_FUNC void gen_vla_sp_restore(int addr) +{ + if (nocode_wanted) + return; + + /* Load SP from the local stack slot at frame offset `addr`. */ + int off = fp_adjust_local_offset(addr, 0 /* not param */); + int sign = (off < 0) ? 1 : 0; + int abs_off = sign ? -off : off; + const int base_reg = tcc_state->need_frame_pointer ? R_FP : R_SP; + + ScratchRegAlloc vla_sc = get_scratch_reg_with_save(0); + load_from_base(vla_sc.reg, PREG_REG_NONE, IROP_BTYPE_INT32, 0, abs_off, sign, base_reg); + ot_check_mov_reg(R_SP, vla_sc.reg, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE, false); + restore_scratch_reg(&vla_sc); +} + +int load_ushort_from_base(int ir, int base, int fc, int sign) +{ + const thumb_opcode ins = th_ldrh_imm(ir, base, fc, sign ? 4 : 6, ENFORCE_ENCODING_NONE); + TRACE("Load ushort sign: %d, r %d, base: %d, fc: %d\n", sign, ir, base, fc, sign); + return ot(ins); +} + +int load_byte_from_base(int ir, int base, int fc, int sign) +{ + const thumb_opcode ins = th_ldrsb_imm(ir, base, fc, sign ? 4 : 6, ENFORCE_ENCODING_NONE); + TRACE("Load byte sign: %d, r %d, base: %d, fc: %d\n", sign, ir, base, fc, sign); + return ot(ins); +} + +int load_ubyte_from_base(int ir, int base, int fc, int sign) +{ + const thumb_opcode ins = th_ldrb_imm(ir, base, fc, sign ? 4 : 6, ENFORCE_ENCODING_NONE); + TRACE("Load ubyte sign: %d, r %d, base: %d, fc: %d\n", sign, ir, base, fc, sign); + return ot(ins); +} + +/* Knob for frame_word_pair_rewind: TCC_DISABLE_PASS=codegen:frame_word_pair. + * Off in a function with a volatile access, like the other memory caches. */ +static int frame_word_pair_off; + +ST_FUNC void tcc_gen_machine_frame_word_pair_set_enabled(int enabled) +{ + frame_word_pair_off = !enabled; + frame_word_last.kind = 0; +} + +/* Two word accesses to neighbouring frame words, one right after the other -- + * `ldr r7,[sp,#12]; ldr.w r8,[sp,#16]`, 60k pairs in the Zig compiler built as + * C -- are one LDRD/STRD. The word helpers are asked for one word at a time, + * so the pair is formed after the fact: when this access is the word next to + * the previous one, which was a single immediate-offset LDR/STR of the same + * kind and the last thing emitted, rewind over it and emit both as one + * instruction. Every pass takes the same decision from the same `ind` + * arithmetic, so the dry runs lay out what the real pass emits. + * + * Only frame bases (frame_word_base): LDRD/STRD fault on an address that is + * not word aligned, which only a frame slot at a multiple-of-4 offset + * guarantees. Nothing else + * can have captured the rewound bytes: the word helpers record no relocation, + * literal or branch there, and every label resets the pending access (see + * tcc_gen_machine_strldr_cache_reset). LDRD may not load one register twice; + * STRD may store it. Returns 1 when the pair was emitted. */ +static int frame_word_pair_rewind(int is_load, int reg, int base, int off) +{ + if (frame_word_pair_off || !thumb_gen_state.generating_function || nocode_wanted) + return 0; + if (frame_word_last.kind != (is_load ? 2 : 1) || frame_word_last.end != ind || frame_word_last.base != base) + return 0; + if (mov_equiv_it_pending != 0 || pool_flush_it_pending != 0) + return 0; + const int prev_reg = frame_word_last.reg, prev_off = frame_word_last.off; + if (prev_off + 4 != off && off + 4 != prev_off) + return 0; + if (is_load && prev_reg == reg) + return 0; + const int lo_reg = prev_off < off ? prev_reg : reg; + const int hi_reg = prev_off < off ? reg : prev_reg; + const int lo_off = prev_off < off ? prev_off : off; + if (lo_reg < 0 || lo_reg >= R_SP || hi_reg < 0 || hi_reg >= R_SP || lo_reg == base || hi_reg == base) + return 0; + const int neg = (lo_off < 0); + const int abs_off = neg ? -lo_off : lo_off; + if ((abs_off & 3) != 0 || abs_off > 1020) + return 0; + const uint32_t puw = neg ? 4u : 6u; + const thumb_opcode pair = is_load ? th_ldrd_imm((uint32_t)lo_reg, (uint32_t)hi_reg, (uint32_t)base, abs_off, puw) + : th_strd_imm((uint32_t)lo_reg, (uint32_t)hi_reg, (uint32_t)base, abs_off, puw); + if (pair.size != 4) + return 0; + thumb_gen_state.code_size -= ind - frame_word_last.start; + ind = frame_word_last.start; + frame_word_last.kind = 0; + ot_check(pair); + return 1; +} + +/* A base register whose word-aligned offsets are word-aligned addresses: SP, + * or R7 while it is the frame pointer. Without a frame pointer R7 is an + * ordinary register holding any pointer -- a packed field's address, say -- + * and an LDRD through it took an UNALIGNED usage fault in the native tcc. */ +static int frame_word_base(int base) +{ + return base == R_SP || (base == R_FP && tcc_state->need_frame_pointer); +} + +/* Remember a word access to the frame that was emitted as exactly `ins`. */ +static void frame_word_note(int is_load, int start, thumb_opcode ins, int reg, int base, int fc, int sign) +{ + frame_word_last.kind = 0; + if (!frame_word_base(base) || ins.size == 0 || ind - start != ins.size) + return; + frame_word_last.kind = is_load ? 2 : 1; + frame_word_last.start = start; + frame_word_last.end = ind; + frame_word_last.reg = reg; + frame_word_last.base = base; + frame_word_last.off = sign ? -fc : fc; +} + +int load_word_from_base(int ir, int base, int fc, int sign) +{ + const thumb_opcode ins = th_ldr_imm(ir, base, fc, sign ? 4 : 6, ENFORCE_ENCODING_NONE); + TRACE("Load word sign: %d, r %d, base: %d, fc: %d\n", sign, ir, base, fc, sign); + /* This is the path every spill reload takes, so it -- not just the + * ot_check_ldr_imm call sites -- is where the memory cache has to be + * consulted. Returning the encoded size without emitting keeps the + * caller's "did it encode?" contract intact; only `ind` stands still. */ + if (ins.size != 0 && thumb_gen_state.generating_function && !sign && + strldr_cache_try_match_ldr(ir, base, fc, 6, ins.size, 4)) + return ins.size; + /* Satisfying the load from ANOTHER register holding the slot (`mov rt,rh`) + * was built and MEASURED on the rig, and it is a small LOSS -- rijndael and + * double_mul improve, double_add/cmp/mixed/div all give more back, total + * -0.09% against -0.12% for this same-register form alone. A `mov` and a + * cached-line `ldr` from the frame cost about the same on M33, so all it + * really does is move code. Do not re-propose without a measurement. */ + if (ins.size != 0 && frame_word_base(base) && frame_word_pair_rewind(1, ir, base, sign ? -fc : fc)) + return ins.size; + const int start = ind; + const int size = ot(ins); + frame_word_note(1, start, ins, ir, base, fc, sign); + return size; +} + +int store_word_to_base(int ir, int base, int fc, int sign) +{ + const thumb_opcode ins = th_str_imm(ir, base, fc, sign ? 4 : 6, ENFORCE_ENCODING_NONE); + TRACE("Store word sign: %d, r %d, base: %d, fc: %d\n", sign, ir, base, fc, sign); + if (!sign && strldr_cache_str_is_redundant(ir, base, fc, 6, ins.size)) + return ins.size; + if (ins.size != 0 && frame_word_base(base) && frame_word_pair_rewind(0, ir, base, sign ? -fc : fc)) + return ins.size; + const int start = ind; + const int size = ot(ins); + frame_word_note(0, start, ins, ir, base, fc, sign); + return size; +} + +/* Returns 1 if a 64-bit access at (sym + addend) is guaranteed 4-byte aligned + * (so LDRD/STRD is safe). Conservative: only allows natural alignment for + * non-struct, non-packed symbols, plus any explicit alignment >= 4. */ +int sym_is_4_byte_aligned_for_64bit(Sym *sym, int32_t addend) +{ + if (!sym) + return 0; + if ((addend & 3) != 0) + return 0; + if (sym->a.packed) + return 0; + if (sym->a.aligned >= 3) /* explicit alignment 2^(n-1) >= 4 */ + return 1; + if (sym->a.aligned > 0) /* explicit 1 or 2 byte alignment — not safe */ + return 0; + /* sym->a.aligned == 0: rely on the declared type's natural alignment. + * Structs/unions may be packed-wrapped; reject conservatively. Native + * scalars (long long, double, pointer) have natural alignment >= 4. */ + int btype = sym->type.t & VT_BTYPE; + if (btype == VT_STRUCT) + return 0; + return 1; +} + +/* Try to emit STRD Rt, Rt2, [base, #±abs_off] for a 64-bit paired store. + * Constraints (Thumb-2 STRD imm T1): + * - Rt != Rt2 + * - Rt, Rt2 in r0..r12 or r14 (not SP, not PC) + * - abs_off 4-byte aligned and <= 1020 + * Returns 1 on success, 0 if the caller must fall back to two 32-bit stores. */ +int try_strd_pair(int lo_reg, int hi_reg, int base, int abs_off, int sign) +{ + if ((unsigned)base > 15) + return 0; + if ((abs_off & 3) != 0 || abs_off > 1020) + return 0; + if (lo_reg < 0 || lo_reg > R_LR || lo_reg == R_SP) + return 0; + if (hi_reg < 0 || hi_reg > R_LR || hi_reg == R_SP) + return 0; + const uint32_t puw = sign ? 4u : 6u; + ot_check(th_strd_imm((uint32_t)lo_reg, (uint32_t)hi_reg, (uint32_t)base, abs_off, puw)); + return 1; +} + +/* Mirror of try_strd_pair for LDRD. Same register and offset constraints; + * the caller is responsible for guaranteeing 4-byte alignment of the target + * address (stack, or a symbol that passes sym_is_4_byte_aligned_for_64bit). */ +int try_ldrd_pair(int lo_reg, int hi_reg, int base, int abs_off, int sign) +{ + if ((abs_off & 3) != 0 || abs_off > 1020) + return 0; + if (lo_reg < 0 || lo_reg > R_LR || lo_reg == R_SP) + return 0; + if (hi_reg < 0 || hi_reg > R_LR || hi_reg == R_SP) + return 0; + if (lo_reg == hi_reg) + return 0; + const uint32_t puw = sign ? 4u : 6u; + if (strldr_cache_ldrd_is_redundant(lo_reg, hi_reg, base, abs_off, puw)) + return 1; + ot_check(th_ldrd_imm((uint32_t)lo_reg, (uint32_t)hi_reg, (uint32_t)base, abs_off, puw)); + return 1; +} + +/* Emit a single STR to a spill slot. Used by the codegen STRD pairing logic + * to flush a pending store when pairing wasn't possible. */ +ST_FUNC void tcc_gen_machine_store_spill(int src_reg, int32_t spill_offset) +{ + const int base_reg = tcc_state->need_frame_pointer ? R_FP : R_SP; + int adj = fp_adjust_local_offset(spill_offset, 0); + int sign = (adj < 0); + int abs_off = sign ? -adj : adj; + ot_check_str_imm((uint32_t)src_reg, (uint32_t)base_reg, + abs_off, sign ? 4u : 6u, ENFORCE_ENCODING_NONE); +} + +/* Try to emit STRD for two 32-bit values to adjacent spill slots. + * off1 must be the lower offset (off1 + 4 == off2). + * Returns 1 on success, 0 if STRD constraints not met. */ +ST_FUNC int tcc_gen_machine_try_strd_spill(int reg1, int32_t off1, int reg2, int32_t off2) +{ + if (off1 + 4 != off2) + return 0; + const int base_reg = tcc_state->need_frame_pointer ? R_FP : R_SP; + int adj = fp_adjust_local_offset(off1, 0); + int sign = (adj < 0); + int abs_off = sign ? -adj : adj; + return try_strd_pair(reg1, reg2, base_reg, abs_off, sign); +} + +/* Try to emit LDRD for two 32-bit values from adjacent spill slots. + * off1 must be the lower offset (off1 + 4 == off2). + * Returns 1 on success, 0 if LDRD constraints not met. */ +ST_FUNC int tcc_gen_machine_try_ldrd_spill(int reg1, int32_t off1, int reg2, int32_t off2) +{ + if (off1 + 4 != off2) + return 0; + const int base_reg = tcc_state->need_frame_pointer ? R_FP : R_SP; + int adj = fp_adjust_local_offset(off1, 0); + int sign = (adj < 0); + int abs_off = sign ? -adj : adj; + return try_ldrd_pair(reg1, reg2, base_reg, abs_off, sign); +} + +/* Try to emit LDRD/STRD for two 32-bit values from adjacent offsets off a + * generic base register (not FP/SP). Used by the LOAD_INDEXED/STORE_INDEXED + * pairing peephole. `off` is the lower offset (caller has verified + * off + 4 fits within the same access range). Returns 1 on success. */ +ST_FUNC int tcc_gen_machine_try_ldrd_base(int reg1, int reg2, int base_reg, int32_t off) +{ + int sign = (off < 0); + int abs_off = sign ? -off : off; + return try_ldrd_pair(reg1, reg2, base_reg, abs_off, sign); +} + +ST_FUNC int tcc_gen_machine_try_strd_base(int reg1, int reg2, int base_reg, int32_t off) +{ + int sign = (off < 0); + int abs_off = sign ? -off : off; + return try_strd_pair(reg1, reg2, base_reg, abs_off, sign); +} + +ST_FUNC int tcc_gen_machine_try_strd_imm_spill(int64_t val1, int64_t val2, + int32_t off1, int32_t off2) +{ + if (off1 + 4 != off2) + return 0; + const int base_reg = tcc_state->need_frame_pointer ? R_FP : R_SP; + int adj = fp_adjust_local_offset(off1, 0); + int sign = (adj < 0); + int abs_off = sign ? -adj : adj; + if ((abs_off & 3) != 0 || abs_off > 1020) + return 0; + + MachineCodegenContext ctx = {0}; + /* Materializing the immediates may PUSH the scratch register(s) when FP is + * omitted and no scratch-save area is reserved, lowering SP by 4 per push. + * The STRD destination is SP-relative, so an uncompensated offset would write + * the pair 4*pushes bytes below the intended slot — the array/struct + * initializer then lands at the wrong offset and later reads return stale + * data (fuzz seed 12057). Snapshot the push stack so we can measure the SP + * shift after acquiring the registers and fold it into the offset. */ + int spc_before = scratch_push_count; + MachineOperand op1 = {.kind = MACH_OP_IMM, .u.imm.val = val1}; + int r1 = mach_ensure_in_reg(&ctx, &op1, 0); + int r2; + if (val1 == val2) { + r2 = r1; + } else { + MachineOperand op2 = {.kind = MACH_OP_IMM, .u.imm.val = val2}; + r2 = mach_ensure_in_reg(&ctx, &op2, (1u << (uint32_t)r1)); + } + if (r1 == R_SP || r2 == R_SP) { + mach_release_all(&ctx); + return 0; + } + /* Account for any real SP-lowering pushes (type 1) done above. Saves routed + * to a reserved scratch area (type 2) keep SP stable and need no adjustment. + * The shift only affects an SP-relative base; an FP base is unperturbed. */ + if (base_reg == R_SP) { + int sp_shift = 0; + for (int s = spc_before; s < scratch_push_count && s < 128; s++) + if (scratch_push_type[s] == 1) + sp_shift += 4; + if (sp_shift) { + /* Only the positive (above-SP) local case is safe to compensate by simple + * addition; a negative (below-SP) offset combined with the shift is rare + * and not worth special-casing — fall back to per-element stores. */ + if (sign || abs_off + sp_shift > 1020) { + mach_release_all(&ctx); + return 0; + } + abs_off += sp_shift; + } + } + const uint32_t puw = sign ? 4u : 6u; + ot_check(th_strd_imm((uint32_t)r1, (uint32_t)r2, (uint32_t)base_reg, abs_off, puw)); + mach_release_all(&ctx); + return 1; +} + +ST_FUNC int tcc_gen_machine_try_strd_imm_base(int64_t val1, int64_t val2, + int base_reg, int32_t off) +{ + int sign = (off < 0); + int abs_off = sign ? -off : off; + if ((unsigned)base_reg > 15) + return 0; + if ((abs_off & 3) != 0 || abs_off > 1020) + return 0; + + uint32_t excl = (1u << (uint32_t)base_reg); + MachineCodegenContext ctx = {0}; + MachineOperand op1 = {.kind = MACH_OP_IMM, .u.imm.val = val1}; + int r1 = mach_ensure_in_reg(&ctx, &op1, excl); + int r2; + if (val1 == val2) { + r2 = r1; + } else { + MachineOperand op2 = {.kind = MACH_OP_IMM, .u.imm.val = val2}; + r2 = mach_ensure_in_reg(&ctx, &op2, excl | (1u << (uint32_t)r1)); + } + if (r1 == R_SP || r2 == R_SP) { + mach_release_all(&ctx); + return 0; + } + const uint32_t puw = sign ? 4u : 6u; + ot_check(th_strd_imm((uint32_t)r1, (uint32_t)r2, (uint32_t)base_reg, abs_off, puw)); + mach_release_all(&ctx); + return 1; +} + +ST_FUNC int tcc_machine_can_encode_stack_offset_for_reg(int frame_offset, int dest_reg) +{ + /* Check if frame_offset can be directly encoded in ldr/str instructions + * without requiring a scratch register. This is used to avoid wasteful + * address materialization when the backend can handle the offset directly. + * Tests with dest_reg since encoding availability depends on the register. */ + /* Adjust for callee-saved gap below FP (spill offsets are always locals) */ + frame_offset = fp_adjust_local_offset(frame_offset, 0); + const int base_reg = tcc_state->need_frame_pointer ? R_FP : R_SP; + const int sign = (frame_offset < 0); + const int abs_offset = sign ? -frame_offset : frame_offset; + + /* Try to encode as ldr instruction with the actual destination register. + * Some encodings (e.g., Thumb-1 T1) only work with low registers (r0-r7). */ + const thumb_opcode ins = th_ldr_imm(dest_reg, base_reg, abs_offset, sign ? 4 : 6, ENFORCE_ENCODING_NONE); + return (ins.size != 0); +} + +ST_FUNC int tcc_machine_can_encode_stack_offset_with_param_adj(int frame_offset, int is_param, int dest_reg) +{ + /* Like tcc_machine_can_encode_stack_offset_for_reg, but applies offset_to_args for VT_PARAM. + * Stack parameters need offset_to_args adjustment (prologue push size). */ + int offset = frame_offset; + if (is_param) + offset = param_frame_offset(offset); + return tcc_machine_can_encode_stack_offset_for_reg(offset, dest_reg); +} + +ST_FUNC void tcc_machine_load_spill_slot(int dest_reg, int frame_offset) +{ + if (dest_reg == PREG_REG_NONE) + tcc_error("compiler_error: load_spill_slot requires a destination register"); + + /* Adjust for callee-saved gap below FP (spill slots are always locals) */ + frame_offset = fp_adjust_local_offset(frame_offset, 0); + + /* Peephole: if the previous emit was a STR or LDR of the same register to/from + * the same slot AND no other instruction has been emitted since, the value is + * already in dest_reg — skip the redundant load. */ + TCCIRState *ir = tcc_state ? tcc_state->ir : NULL; + if (ir && ir->spill_cache.last_emit_kind != 0 && + ir->spill_cache.last_emit_ind == ind && + ir->spill_cache.last_emit_reg == dest_reg && + ir->spill_cache.last_emit_offset == frame_offset) + { + return; + } + + const int base_reg = tcc_state->need_frame_pointer ? R_FP : R_SP; + const int sign = (frame_offset < 0); + const int abs_offset = sign ? -frame_offset : frame_offset; + if (!load_word_from_base(dest_reg, base_reg, abs_offset, sign)) + { + ScratchRegAlloc rr_alloc = th_offset_to_reg_ex(abs_offset, sign, (1u << dest_reg) | (1u << base_reg)); + int rr = rr_alloc.reg; + ot_check(th_ldr_reg(dest_reg, base_reg, rr, THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); + restore_scratch_reg(&rr_alloc); + } + + if (ir) + { + ir->spill_cache.last_emit_kind = 2; /* LDR */ + ir->spill_cache.last_emit_ind = ind; + ir->spill_cache.last_emit_reg = (int8_t)dest_reg; + ir->spill_cache.last_emit_offset = frame_offset; + } +} + +ST_FUNC void tcc_machine_store_spill_slot(int src_reg, int frame_offset) +{ + if (src_reg == PREG_REG_NONE) + tcc_error("compiler_error: store_spill_slot requires a source register"); + + /* Adjust for callee-saved gap below FP (spill slots are always locals) */ + frame_offset = fp_adjust_local_offset(frame_offset, 0); + const int base_reg = tcc_state->need_frame_pointer ? R_FP : R_SP; + const int sign = (frame_offset < 0); + const int abs_offset = sign ? -frame_offset : frame_offset; + if (!store_word_to_base(src_reg, base_reg, abs_offset, sign)) + { + /* Avoid clobbering the other half of a 64-bit value when storing + * paired registers. The allocator uses adjacent register pairs for + * 64-bit values (e.g. r0/r1, r2/r3, r4/r5). When storing one half, + * do not use the adjacent register as the scratch offset register. + */ + uint32_t extra_exclude = 0; + if (src_reg >= ARM_R0 && src_reg <= ARM_R12) + { + int adj = (src_reg & 1) ? (src_reg - 1) : (src_reg + 1); + if (adj >= ARM_R0 && adj <= ARM_R12 && adj != ARM_SP && adj != ARM_PC) + extra_exclude |= (1u << adj); + } + + ScratchRegAlloc rr_alloc = + th_offset_to_reg_ex(abs_offset, sign, (1u << src_reg) | (1u << base_reg) | extra_exclude); + int rr = rr_alloc.reg; + ot_check(th_str_reg(src_reg, base_reg, rr, THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); + restore_scratch_reg(&rr_alloc); + } + + TCCIRState *ir = tcc_state ? tcc_state->ir : NULL; + if (ir) + { + ir->spill_cache.last_emit_kind = 1; /* STR */ + ir->spill_cache.last_emit_ind = ind; + ir->spill_cache.last_emit_reg = (int8_t)src_reg; + ir->spill_cache.last_emit_offset = frame_offset; + } +} + +/* Like tcc_machine_store_spill_slot, but for stack-passed parameters. + * Adds offset_to_args (prologue push size) to the frame offset so that + * the store targets the correct caller-stack location above FP. */ +ST_FUNC void tcc_machine_store_param_slot(int src_reg, int frame_offset) +{ + tcc_machine_store_spill_slot(src_reg, param_frame_offset(frame_offset)); +} + +static int unalias_ldbl(int btype) +{ +#if LDOUBLE_SIZE == 8 + if (btype == VT_LDOUBLE) + btype = VT_DOUBLE; +#endif + return btype; +} + +/* Return whether a structure is an homogeneous float aggregate or not. + The answer is true if all the elements of the structure are of the same + primitive float type and there is less than 4 elements. + + type: the type corresponding to the structure to be tested */ +static int is_hgen_float_aggr(CType *type) +{ + if ((type->t & VT_BTYPE) == VT_STRUCT) + { + struct Sym *ref; + int btype, nb_fields = 0; + + ref = type->ref->next; + if (ref) + { + btype = unalias_ldbl(ref->type.t & VT_BTYPE); + if (btype == VT_FLOAT || btype == VT_DOUBLE) + { + for (; ref && btype == unalias_ldbl(ref->type.t & VT_BTYPE); ref = ref->next, nb_fields++) + ; + return !ref && nb_fields <= 4; + } + } + } + return 0; +} + +// How many registers are necessary to return struct via registers +// if not possible, then 0 means return via struct pointer +ST_FUNC int gfunc_sret(CType *vt, int variadic, CType *ret, int *ret_align, int *regsize) +{ + int align; + const int size = type_size(vt, &align); + + TRACE("'gfunc_sret'"); + if (float_abi == ARM_HARD_FLOAT && !variadic && (is_float(vt->t) || is_hgen_float_aggr(vt))) + { + *ret_align = 8; + *regsize = 8; + ret->ref = NULL; + ret->t = VT_DOUBLE; + return ceil_div(size, 8); + } + else if (size > 0 && size <= 4) + { + *ret_align = 4; + *regsize = 4; + ret->ref = NULL; + ret->t = VT_INT; + return 1; + } + return 0; +} + +// are those offsets to allow TREG_R0 start from other register than r0? +// not sure + +void th_store32_imm_or_reg_ex(int src_reg, uint32_t base_reg, int abs_off, int sign, uint32_t extra_exclude) +{ + const thumb_opcode st = th_str_imm(src_reg, base_reg, abs_off, sign ? 4 : 6, ENFORCE_ENCODING_NONE); + if (!sign && strldr_cache_str_is_redundant(src_reg, (int)base_reg, abs_off, 6, st.size)) + return; + if (!ot(st)) + { + ScratchRegAlloc rr_alloc = th_offset_to_reg_ex(abs_off, sign, (1u << src_reg) | (1u << base_reg) | extra_exclude); + int rr = rr_alloc.reg; + ot_check(th_str_reg(src_reg, base_reg, rr, THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); + restore_scratch_reg(&rr_alloc); + } +} +void th_store16_imm_or_reg(int src_reg, uint32_t base_reg, int abs_off, int sign) +{ + if (!ot(th_strh_imm(src_reg, base_reg, abs_off, sign ? 4 : 6, ENFORCE_ENCODING_NONE))) + { + ScratchRegAlloc rr_alloc = th_offset_to_reg_ex(abs_off, sign, (1u << src_reg) | (1u << base_reg)); + int rr = rr_alloc.reg; + ot_check(th_strh_reg(src_reg, base_reg, rr, THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); + restore_scratch_reg(&rr_alloc); + } +} +void th_store8_imm_or_reg(int src_reg, uint32_t base_reg, int abs_off, int sign) +{ + if (!ot(th_strb_imm(src_reg, base_reg, abs_off, sign ? 4 : 6, ENFORCE_ENCODING_NONE))) + { + ScratchRegAlloc rr_alloc = th_offset_to_reg_ex(abs_off, sign, (1u << src_reg) | (1u << base_reg)); + int rr = rr_alloc.reg; + ot_check(th_strb_reg(src_reg, base_reg, rr, THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); + restore_scratch_reg(&rr_alloc); + } +} + +static ThumbLiteralPoolEntry *th_literal_pool_allocate() +{ + ThumbLiteralPoolEntry *entry; + + pool_entries_total++; + + /* During dry-run, use separate pool to avoid modifying the real pool. + * This prevents memory corruption when restoring state after dry-run. */ + if (dry_run_state.active) + { + if (dry_run_literal_pool_count >= dry_run_literal_pool_size) + { + dry_run_literal_pool_size <<= 1; + dry_run_literal_pool = + tcc_realloc(dry_run_literal_pool, dry_run_literal_pool_size * sizeof(ThumbLiteralPoolEntry)); + tcc_chained_hash_reserve(&literal_pool_hash, dry_run_literal_pool_size); + } + entry = &dry_run_literal_pool[dry_run_literal_pool_count++]; + entry->sym = NULL; + entry->relocation = -1; + entry->shared_index = -1; + /* Track the count in the main state for code size calculations */ + thumb_gen_state.literal_pool_count++; + return entry; + } + + if (thumb_gen_state.literal_pool_count >= thumb_gen_state.literal_pool_size) + { + const int new_size = thumb_gen_state.literal_pool_size << 1; + thumb_gen_state.literal_pool = tcc_realloc(thumb_gen_state.literal_pool, new_size * sizeof(ThumbLiteralPoolEntry)); + thumb_gen_state.literal_pool_size = new_size; + tcc_chained_hash_reserve(&literal_pool_hash, new_size); + } + entry = &thumb_gen_state.literal_pool[thumb_gen_state.literal_pool_count++]; + entry->sym = NULL; + entry->relocation = -1; + entry->shared_index = -1; + return entry; +} + +/* Find existing literal pool entry with same sym and imm, and allocate new + entry that shares its literal value. + Uses hash table for O(1) lookup instead of O(n) linear search. */ +static ThumbLiteralPoolEntry *th_literal_pool_find_or_allocate(Sym *sym, int64_t imm) +{ + int found_index; + uint32_t full_hash; + TCCChainedHash *hash; + LiteralPoolLookupCache *cache; + ThumbLiteralPoolEntry *pool; + int new_index; + + if (dry_run_state.active) + { + hash = &literal_pool_hash; + cache = &literal_pool_last_lookup; + pool = dry_run_literal_pool; + new_index = dry_run_literal_pool_count; + } + else + { + hash = &literal_pool_hash; + cache = &literal_pool_last_lookup; + pool = thumb_gen_state.literal_pool; + new_index = thumb_gen_state.literal_pool_count; + } + + full_hash = literal_pool_hash_func(sym, imm); + found_index = literal_pool_lookup_cache_find(cache, full_hash, sym, imm); + if (found_index < 0) + { + found_index = literal_pool_hash_find(hash, pool, full_hash, sym, imm); + } + + /* Allocate new entry */ + ThumbLiteralPoolEntry *entry = th_literal_pool_allocate(); + if (found_index >= 0) + { + /* Mark as sharing with the found entry */ + entry->shared_index = found_index; + } + else + { + literal_pool_hash_insert(hash, full_hash, new_index); + found_index = new_index; + } + literal_pool_lookup_cache_insert(cache, full_hash, sym, imm, found_index); + return entry; +} + +/* PIC + text/data-separation relocation choice. Single source of truth: the + * SB-relative fast path and the literal-pool path must agree, or the dry-run + * and real passes emit different sizes. See docs/sb_relative_got.md. */ +static int th_pic_reloc_for_sym(Sym *sym, int sym_off) +{ + int sym_in_code_section = 0; + int sym_in_rodata = 0; + if (sym_off > 0 && sym_off < tcc_state->nb_sections) + { + Section *sym_sec = tcc_state->sections[sym_off]; + if (sym_sec && (sym_sec->sh_flags & SHF_EXECINSTR)) + sym_in_code_section = 1; + /* Exact pointer match: only the main .rodata is anchor-addressed. */ + if (sym_sec && sym_sec == rodata_section) + sym_in_rodata = 1; + } + if (tcc_state->share_rodata && (sym->type.t & VT_STATIC) && sym_off != SHN_UNDEF && sym_in_rodata) + return R_ARM_RODATA_OFF; + /* sym_off == SHN_UNDEF: forward-declared, section unknown — GOT32 is safe. + * So is a constant still waiting in COMMON (a tentative `static const T x;` + * whose definition follows): the definition may place it in .rodata, which + * r9 does not reach -- a variadic body reading it where it stands loaded + * zeros through GOTOFF. Anything else in COMMON ends in .data or .bss. */ + if (sym_off == SHN_COMMON && tcc_state->share_rodata) + { + const CType *tp = &sym->type; + while ((tp->t & (VT_BTYPE | VT_ARRAY)) == (VT_PTR | VT_ARRAY)) + tp = &tp->ref->type; + if (tp->t & VT_CONSTANT) + return R_ARM_GOT32; + } + if ((sym->type.t & VT_STATIC) && sym_off != SHN_UNDEF && sym_off != cur_text_section->sh_num && + !sym_in_code_section) + return R_ARM_GOTOFF; + return R_ARM_GOT32; +} + +/* References in this body that a hoisted .rodata anchor would serve, counted + * from the IR so the register allocator can decide whether to leave a register + * for one. An over- or under-count only moves bytes: the backend still emits + * whatever form the register situation allows. Symbols reach codegen from + * places with no IR operand behind them (libcall names, switch tables), so + * this is a floor, not the exact site count. */ +ST_FUNC int tcc_gen_machine_rodata_anchor_ir_sites(const TCCIRState *ir, int stop_at) +{ + if (!ir || !tcc_state->share_rodata || !text_and_data_separation) + return 0; + + int sites = 0; + for (int i = 0; i < ir->next_instruction_index && sites < stop_at; i++) + { + IRQuadCompact *q = &ir->compact_instructions[i]; + const IROperand ops[3] = {tcc_ir_op_get_dest(ir, q), tcc_ir_op_get_src1(ir, q), + tcc_ir_op_get_src2(ir, q)}; + for (int k = 0; k < 3; k++) + { + Sym *sym = irop_get_sym_ex(ir, ops[k]); + if (!sym) + continue; + ElfSym *esym = elfsym(sym); + if (th_pic_reloc_for_sym(sym, esym ? esym->st_shndx : 0) == R_ARM_RODATA_OFF) + sites++; + } + } + return sites; +} + +/* Add a symbol addend to an already-materialised address in r. imm == 0 is the + * common case and is skipped: it cost 9,733 dead instructions (19 KiB) across + * the compiler's own build. imm is pass-invariant, so sizes still agree. */ +static void th_emit_sym_addend(int r, int64_t imm) +{ + thumb_opcode ot; + if (imm == 0) + return; + if ((ot = th_add_imm(r, r, imm, flags_safe(), ENFORCE_ENCODING_NONE)).size != 0) + { + ot_check(ot); + return; + } + uint32_t exclude_regs = (1 << r); + ScratchRegAlloc scratch_alloc = get_scratch_reg_with_save(exclude_regs); + int scratch = scratch_alloc.reg; + + thumb_opcode ldr = th_ldr_literal(scratch, 0, 1); + ot_check(ldr); + + ThumbLiteralPoolEntry *entry2 = th_literal_pool_allocate(); + entry2->sym = NULL; + entry2->imm = imm; + entry2->patch_position = ind - ldr.size; + entry2->relocation = -1; + entry2->data_size = 4; + entry2->short_instruction = (ldr.size == 2); + th_literal_pool_note_entry(entry2); + ot_check(th_add_reg(r, r, scratch, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); + restore_scratch_reg(&scratch_alloc); +} +void load_full_const(int r, int r1, uint32_t imm_lo, uint32_t imm_hi) +{ + struct Sym *sym = _lfc_sym; + _lfc_sym = NULL; + int64_t imm = (int64_t)((uint64_t)imm_hi << 32 | (uint64_t)imm_lo); + ThumbLiteralPoolEntry *entry; + thumb_opcode load_ins; + int patch_pos; + + /* Validate symbol - only use symbols that can be externalized */ + sym = validate_sym_for_reloc(sym); + + /* Stable cache key: the validated symbol *before* the registration block + * below may NULL it. Registration is skipped during dry-run, so using the + * post-registration `sym` would make the dry and real passes disagree on + * cache hits and desynchronise code size. `reuse_sym` is identical in both + * passes (validate_sym_for_reloc does not depend on dry-run state). */ + Sym *reuse_sym = sym; + + /* Symbol-address reuse: when a register already holds &sym+imm, skip the + * redundant literal-pool load. Uses the same per-register imm_cache that + * is invalidated on every clobbering emit and at IR boundaries, so the + * decision is deterministic across the dry-run and real passes. Only the + * single-register (non-LDRD) form participates. */ + if (reuse_sym && thumb_gen_state.generating_function && r1 == PREG_NONE && r >= 0 && r < 16) + { + if (imm_cache[r].valid && imm_cache[r].sym == reuse_sym && imm_cache[r].value == imm) + return; /* r already holds &sym+imm */ + for (int rr = 0; rr < 16; rr++) + { + if (rr != r && imm_cache[rr].valid && imm_cache[rr].sym == reuse_sym && imm_cache[rr].value == imm) + { + /* Another register holds it: copy instead of reloading from the + * literal pool (saves a memory access and a pool word). */ + ot_check_mov_reg((uint32_t)r, (uint32_t)rr, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE, false); + imm_cache[r].value = imm; + imm_cache[r].sym = reuse_sym; + imm_cache[r].valid = 1; + return; + } + } + } + + /* During dry-run, skip symbol registration and literal pool allocation. + * We just emit the instruction (ot_check handles dry-run mode) to track + * code size and scratch register usage, without creating side effects. */ + if (!dry_run_state.active) + { + if (sym && sym->c == 0) + { + /* Symbol not yet registered - try to register it */ + put_extern_sym(sym, NULL, 0, 0); + if (sym->c <= 0) + { + /* Registration failed - symbol can't be externalized */ + sym = NULL; + } + } + } + + /* SB-relative GOT: a single `ldr.w Rt,[r9,#imm12]` replaces literal + add R9 + * + load and the pool word. Goes through ot_check rather than + * ot_check_ldr_imm on purpose — the latter's redundant-reload cache is keyed + * on the pre-relocation immediate (0), which would conflate distinct + * symbols. See docs/sb_relative_got.md. */ + if (sym && pic && text_and_data_separation && sb_relative_got && r1 == PREG_NONE) + { + ElfSym *sb_esym = elfsym(sym); + int sb_sym_off = sb_esym ? sb_esym->st_shndx : 0; + if (th_pic_reloc_for_sym(sym, sb_sym_off) == R_ARM_GOT32) + { + thumb_opcode sb_ins = th_ldr_imm(r, R9, 0, 6, ENFORCE_ENCODING_32BIT); + if (sb_ins.size != 0) + { + ot_check(sb_ins); + tcc_gen_machine_outline_capture_reloc(sym, ind - sb_ins.size); + if (!dry_run_state.active && !tcc_gen_machine_outline_suppressing()) + greloc(cur_text_section, sym, ind - sb_ins.size, R_ARM_GOT_SBREL12); + th_emit_sym_addend(r, imm); + return; + } + } + } + + TRACE("'load_full_const' to register: %d, with imm: %d\n", r, imm); + + /* Emit the instruction first. + * ot() may flush the current literal pool BEFORE emitting this op. + * If patch_position is captured before ot_check(), it can end up pointing + * at the pool skip-branch and later patching would clobber it. + */ + if (r1 == PREG_NONE) + { + load_ins = th_ldr_literal(r, 0, 1); + } + else + { + load_ins = th_ldrd_imm(r, r1, R_PC, 0, 4); + } + ot_check(load_ins); + patch_pos = ind - load_ins.size; + + /* Record that r now holds &sym+imm so a later reference to the same global + * address can be elided. Must run after ot_check(), whose emit-level + * invalidation cleared imm_cache[r] for the LDR we just produced. Keyed on + * the pre-registration `reuse_sym` for dry/real-pass consistency. */ + if (reuse_sym && thumb_gen_state.generating_function && r1 == PREG_NONE && r >= 0 && r < 16) + { + imm_cache[r].value = imm; + imm_cache[r].sym = reuse_sym; + imm_cache[r].valid = 1; + } + + /* During dry-run, we still need to create the literal pool entry to ensure + * the literal pool behavior (threshold checks, sharing, etc.) matches the real pass. + * We still set sym so that find_or_allocate can match entries correctly. + * We just skip symbol registration and relocation setup. */ + entry = th_literal_pool_find_or_allocate(sym, imm); + entry->sym = sym; + entry->patch_position = patch_pos; + entry->relocation = -1; /* No relocation by default */ + entry->data_size = (r1 == PREG_NONE) ? 4 : 8; + entry->short_instruction = (r1 == PREG_NONE && load_ins.size == 2); + th_literal_pool_note_entry(entry); + + if (!sym) + { + entry->imm = imm; + return; + } + + /* Re-derive esym after ot_check(): literal pool generation during ot_check + * can call put_elf_sym → section_ptr_add → section_realloc, which may + * free and reallocate the symtab section buffer, invalidating any + * earlier ElfSym pointer. */ + ElfSym *esym = elfsym(sym); + int sym_off = 0; + if (esym) + { + sym_off = esym->st_shndx; + } + if (!pic) + { + entry->relocation = R_ARM_ABS32; + /* The imm value is the addend (offset from symbol base). + For arr[i], imm = i * sizeof(element). + The linker will add the symbol's address to this addend. */ + entry->imm = imm; + } + else + { + /* For PIC without a symbol, the literal is a plain constant (e.g. -1). + * Must still store the value so the pool emits it correctly. */ + entry->imm = imm; + if (sym) + { + if (text_and_data_separation) + { + /* Relocation strategy for text_and_data_separation + PIC: + * + * R_ARM_GOTOFF computes (symbol - GOT_addr) and at runtime adds R9. + * This only works when symbol and GOT are in the same loadable + * segment (i.e. both in data). With text/data separation, code + * (.text) and data (.got) are loaded at independent addresses. + * + * Static symbols in *data* sections (no SHF_EXECINSTR): + * GOTOFF is fine — symbol and GOT are both in the data segment. + * + * Everything else (including static functions in other .text.* + * sections from -ffunction-sections): + * Use R_ARM_GOT32 — indirect through a GOT slot. The linker + * creates a GOT entry (put_got_entry → R_RELATIVE for locals), + * fill_local_got_entries writes sym->st_value into the slot, + * and the YAFF writer emits a data relocation so the dynamic + * loader patches the slot to the runtime code address. + */ + entry->relocation = th_pic_reloc_for_sym(sym, sym_off); + } + else + { + if (sym->type.t & VT_STATIC) + { + entry->relocation = R_ARM_REL32; + } + else + { + entry->relocation = R_ARM_GOT_PREL; + } + } + } + } + + if (pic) + { + if (sym) + { + if (text_and_data_separation) + { + /* Dispatch on the relocation chosen above — GOTOFF adds R9, GOT32 + * adds R9 then loads through the slot. */ + if (entry->relocation == R_ARM_RODATA_OFF) + { + /* Shared .rodata anchor: r holds (sym - rodata_base) from the + * R_ARM_RODATA_OFF literal; add the rodata runtime base. */ + if (DRY_RUN_MODELLING) + dry_run_state.rodata_anchor_sites++; + if (rodata_anchor_reg >= 0) + { + /* The prologue parked the base in a register reserved for the + * whole body (tcc_gen_machine_rodata_anchor_claim). */ + if (r == rodata_anchor_reg) + tcc_error("compiler_error: rodata anchor r%d handed out as a value register", + rodata_anchor_reg); + ot_check(th_add_reg(r, r, rodata_anchor_reg, flags_safe(), THUMB_SHIFT_DEFAULT, + ENFORCE_ENCODING_NONE)); + } + else + { + /* push {tmp}; ldr tmp, [R9, #24]; add r, r, tmp; pop {tmp} + * Use a DETERMINISTIC fixed scratch (a low register other than r, + * saved by push/pop), NOT get_scratch_reg_with_save: the latter's + * callee-saved fallback is gated on !dry_run_state.active, so under + * register pressure (e.g. ps's larger functions) it can pick a + * different register in the dry-run vs real pass, desync instruction + * sizes, and corrupt literal-pool offsets — yielding a near-NULL + * rodata address. A fixed push/pop emits identically in both passes. */ + int anchor_tmp = (r == 0) ? 1 : 0; + ot_check(th_push((uint16_t)(1u << anchor_tmp))); + ot_check_ldr_imm(anchor_tmp, R9, YAFF_RODATA_ANCHOR_GOT_OFFSET, 6, ENFORCE_ENCODING_NONE); + ot_check( + th_add_reg(r, r, anchor_tmp, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); + ot_check(th_pop((uint16_t)(1u << anchor_tmp))); + } + } + else if (entry->relocation == R_ARM_GOTOFF) + { + /* Static data symbol — GOTOFF (add R9) */ + ot_check(th_add_reg(r, r, R9, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); + } + else + { + ot_check(th_add_reg(r, r, R9, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); + + ot_check_ldr_imm(r, r, 0, 6, ENFORCE_ENCODING_NONE); + th_emit_sym_addend(r, imm); + } + } + else + { + if (sym->type.t & VT_STATIC) + { + ot_check(th_add_reg(r, r, R_PC, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); + ot_check(th_sub_imm(r, r, 8, flags_safe(), ENFORCE_ENCODING_NONE)); + } + else + { + thumb_opcode ot; + ot_check(th_add_reg(r, r, R_PC, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); + ot_check_ldr_imm(r, r, 4, 6, ENFORCE_ENCODING_NONE); + /* Same zero-addend skip as the GOT-relative path above. */ + if (imm == 0) + { + /* no addend: nothing to add */ + } + else if ((ot = th_add_imm(r, r, imm, flags_safe(), ENFORCE_ENCODING_NONE)).size != 0) + { + ot_check(ot); + } + else + { + /* Find a free scratch register for literal pool entry */ + uint32_t exclude_regs = (1 << r); /* Exclude destination register */ + ScratchRegAlloc scratch_alloc = get_scratch_reg_with_save(exclude_regs); + int scratch = scratch_alloc.reg; + + thumb_opcode ldr = th_ldr_literal(scratch, 0, 1); + ot_check(ldr); + + ThumbLiteralPoolEntry *entry2 = th_literal_pool_allocate(); + entry2->sym = NULL; + entry2->imm = imm; + entry2->patch_position = ind - ldr.size; + entry2->relocation = -1; + entry2->data_size = 4; + entry2->short_instruction = (ldr.size == 2); + th_literal_pool_note_entry(entry2); + ot_check( + th_add_reg(r, r, scratch, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); + restore_scratch_reg(&scratch_alloc); + } + } + } + } + } +} + +int load_short_from_base(int ir, int base, int fc, int sign) +{ + const thumb_opcode ins = th_ldrsh_imm(ir, base, fc, sign ? 4 : 6, ENFORCE_ENCODING_NONE); + TRACE("Load short sign: %d, r %d, base: %d, fc: %d\n", sign, ir, base, fc); + return ot(ins); +} + +ST_FUNC void tcc_machine_addr_of_stack_slot(int dest_reg, int frame_offset, int is_param) +{ + if (dest_reg == PREG_REG_NONE) + tcc_error("compiler_error: addr_of_stack_slot requires a destination register"); + + /* Stack parameters live above the saved-register area. + * When computing their address, fold in offset_to_args (prologue push size). + * Locals/spills need callee-saved gap adjustment. */ + if (is_param) + frame_offset = param_frame_offset(frame_offset); + else + frame_offset = fp_adjust_local_offset(frame_offset, 0); + + const int base_reg = tcc_state->need_frame_pointer ? R_FP : R_SP; + + if (frame_offset == 0) + { + if (dest_reg != base_reg) + { + ot_check_mov_reg(dest_reg, base_reg, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE, + false); + } + return; + } + + /* Check FP offset cache for existing computation + * Only use cache for callee-saved registers (r4-r11) since scratch registers + * like ip (r12) can be overwritten at any time without invalidating the cache. */ + TCCIRState *ir = tcc_state->ir; + int cached_reg = -1; + int is_callee_saved = (dest_reg >= R4 && dest_reg <= R11); + + if (ir && is_callee_saved && tcc_ir_opt_fp_cache_lookup(ir, frame_offset, &cached_reg)) + { + /* Cache hit! Verify the cached register is also callee-saved */ + if (cached_reg >= R4 && cached_reg <= R11) + { + if (cached_reg != dest_reg) + { + ot_check_mov_reg(dest_reg, cached_reg, flags_safe(), THUMB_SHIFT_DEFAULT, + ENFORCE_ENCODING_NONE, false); + } + return; + } + /* Cached in scratch register - don't use it */ + } + + const int neg = (frame_offset < 0); + int abs_off = neg ? -frame_offset : frame_offset; + thumb_opcode op = neg ? th_sub_imm(dest_reg, base_reg, abs_off, flags_safe(), ENFORCE_ENCODING_NONE) + : th_add_imm(dest_reg, base_reg, abs_off, flags_safe(), ENFORCE_ENCODING_NONE); + + if (op.size != 0) + { + ot_check(op); + /* Record in cache for future reuse - only for callee-saved registers + * which won't be clobbered unexpectedly */ + if (ir && is_callee_saved) + tcc_ir_opt_fp_cache_record(ir, frame_offset, dest_reg); + return; + } + + ScratchRegAlloc offset_alloc = {0}; + int offset_reg = dest_reg; + + if (dest_reg == base_reg) + { + offset_alloc = get_scratch_reg_with_save(1u << base_reg); + if (offset_alloc.reg == PREG_NONE) + tcc_error("compiler_error: unable to allocate scratch register for stack address"); + offset_reg = offset_alloc.reg; + } + + load_full_const(offset_reg, PREG_NONE, LFC_SPLIT(frame_offset)); + ot_check(th_add_reg(dest_reg, base_reg, offset_reg, flags_safe(), THUMB_SHIFT_DEFAULT, + ENFORCE_ENCODING_NONE)); + + if (dest_reg == base_reg) + { + restore_scratch_reg(&offset_alloc); + } + + /* Record complex computation in cache - only for callee-saved registers */ + if (ir && is_callee_saved) + tcc_ir_opt_fp_cache_record(ir, frame_offset, dest_reg); +} + +/* Load a constant value into a register (or register pair for 64-bit). + * This is a simplified wrapper around load_full_const/th_generic_mov_imm + * that doesn't require an SValue. Used by IR-level materialization. + * If sym is non-NULL, a relocation will be generated for symbol-relative constants. */ +ST_FUNC void tcc_machine_load_constant(int dest_reg, int dest_reg_high, int64_t value, int is_64bit, Sym *sym) +{ + if (dest_reg == PREG_REG_NONE) + tcc_error("compiler_error: load_constant requires a destination register"); + + /* Symbol-relative constants always need the literal pool for relocations */ + if (sym) + { + Sym *validated_sym = validate_sym_for_reloc(sym); + if (validated_sym) + { + _lfc_sym = validated_sym; + load_full_const(dest_reg, dest_reg_high, LFC_SPLIT(value)); + return; + } + /* Invalid or missing sym - fall through to treat as plain constant */ + } + + if (!sym && !is_64bit && dest_reg >= 0 && dest_reg < 16 && + imm_cache[dest_reg].valid && imm_cache[dest_reg].sym == NULL && + imm_cache[dest_reg].value == value) + return; + + if (is_64bit) + { + const uint32_t lo = (uint32_t)(value & 0xFFFFFFFF); + const uint32_t hi = (uint32_t)((uint64_t)value >> 32); + + /* Try immediate encoding for both halves */ + thumb_opcode o1 = th_generic_mov_imm(dest_reg, (int)lo); + thumb_opcode o2 = th_generic_mov_imm(dest_reg_high, (int)hi); + + if (o1.size != 0 && o2.size != 0) + { + /* Both can be encoded as immediates */ + ot(o1); + ot(o2); + return; + } + + /* At least one half needs literal pool - use combined 64-bit load */ + load_full_const(dest_reg, dest_reg_high, LFC_SPLIT(value)); + return; + } + + /* 32-bit constant */ + if (!ot(th_generic_mov_imm(dest_reg, (uint32_t)value))) + load_full_const(dest_reg, PREG_NONE, LFC_SPLIT(value)); + + if (!sym && !is_64bit && dest_reg >= 0 && dest_reg < 16) + { + imm_cache[dest_reg].value = value; + imm_cache[dest_reg].sym = NULL; + imm_cache[dest_reg].valid = 1; + } +} + +/* Load comparison result (0 or 1) based on condition flags. + * Used by IR-level materialization for VT_CMP values. */ +ST_FUNC void tcc_machine_load_cmp_result(int dest_reg, int condition_code) +{ + if (dest_reg == PREG_REG_NONE) + tcc_error("compiler_error: load_cmp_result requires a destination register"); + if (dest_reg == R_SP || dest_reg == R_PC) + tcc_error("compiler_error: load_cmp_result cannot use SP or PC"); + + const uint32_t firstcond = mapcc(condition_code); + /* IT block: if cond then mov 1, else mov 0 */ + o(0xbf00 | (firstcond << 4) | 0x4 | ((~firstcond & 1) << 3)); + ot_check(th_generic_mov_imm(dest_reg, 1)); + ot_check(th_generic_mov_imm(dest_reg, 0)); +} + +/* Load jump condition result (0 or 1) based on a pending jump target. + * Used by IR-level materialization for VT_JMP/VT_JMPI values. */ +ST_FUNC void tcc_machine_load_jmp_result(int dest_reg, int jmp_addr, int invert) +{ + if (dest_reg == PREG_REG_NONE) + tcc_error("compiler_error: load_jmp_result requires a destination register"); + +#ifdef TCC_TARGET_ARM_ARCHV6M + if (dest_reg > 7) + tcc_error("compiler_error: implement load_jmp_result for armv6m with high register"); +#endif + + /* Load the "true" branch value, then unconditionally branch over the "false" value, + * then patch the jump target to land on the "false" value */ + ot_check(th_generic_mov_imm(dest_reg, invert ? 0 : 1)); + ot_check(th_b_t4(2)); + gsym(jmp_addr); + codegen_internal_merge_point(); + ot_check(th_generic_mov_imm(dest_reg, invert ? 1 : 0)); +} + +/* Load value from memory at base+offset into register(s). + * Uses IROP_BTYPE_* constants directly, no VT_* conversion needed. + */ +void load_from_base(int r, int r1, int irop_btype, int is_unsigned, int fc, int sign, uint32_t base) +{ + int success = 0; + const int is_64bit = + (irop_btype == IROP_BTYPE_INT64 || irop_btype == IROP_BTYPE_FLOAT64 || (r1 >= 0 && r1 != PREG_REG_NONE)); + + TRACE("load_from_base: r=%d, r1=%d, irop_btype=%d, is_unsigned=%d, fc=%d, sign=%d, base=%d", r, r1, irop_btype, + is_unsigned, fc, sign, base); + + if (is_64bit) + { + /* 64-bit value (double float or long long) - load to register pair */ + int ir_high = r1; + ScratchRegAlloc ir_high_alloc = {0}; + if (ir_high < 0 || ir_high == PREG_REG_NONE) + { + /* No explicit high register — always use scratch to avoid clobbering + * r+1 which may be allocated to another live variable. The old r+1 + * fallback was only safe when mat.c pre-materialized into scratch + * registers (ip:lr pair) before the handler. */ + ir_high_alloc = get_scratch_reg_with_save((1u << r) | (1u << base)); + ir_high = ir_high_alloc.reg; + } + + /* If base overlaps with destination, preserve it */ + ScratchRegAlloc base_alloc = {0}; + uint32_t base_reg = base; + if (base_reg == (uint32_t)r || base_reg == (uint32_t)ir_high) + { + uint32_t exclude = (1u << r) | (1u << ir_high); + base_alloc = get_scratch_reg_with_save(exclude); + base_reg = (uint32_t)base_alloc.reg; + ot_check_mov_reg((int)base_reg, (int)base, flags_safe(), THUMB_SHIFT_DEFAULT, + ENFORCE_ENCODING_NONE, false); + } + + /* Try LDRD Rt, Rt2, [Rn, #±imm] when both halves share one base. + * T1 encoding requires: Rt != Rt2, Rt/Rt2 not SP/PC, offset 4-byte + * aligned and |offset| <= 1020. LDRD also requires the target address + * to be 4-byte aligned on ARMv7-M/v8-M (faults otherwise, regardless of + * UNALIGN_TRP). Restrict to SP/FP-relative bases where TCC's stack + * allocator guarantees 4-byte alignment of 64-bit slots; arbitrary + * pointers (e.g. into a packed struct) may be unaligned. */ + const int base_is_stack = (base_reg == (uint32_t)R_SP || base_reg == (uint32_t)R_FP); + if (base_is_stack && (fc & 3) == 0 && fc <= 1020 && r >= 0 && r <= R_LR && r != R_SP && ir_high >= 0 && + ir_high <= R_LR && ir_high != R_SP && r != ir_high) + { + uint32_t puw = sign ? 4 : 6; + if (!strldr_cache_ldrd_is_redundant(r, ir_high, (int)base_reg, fc, puw)) + ot_check(th_ldrd_imm((uint32_t)r, (uint32_t)ir_high, base_reg, fc, puw)); + if (base_alloc.saved) + restore_scratch_reg(&base_alloc); + if (ir_high_alloc.saved) + restore_scratch_reg(&ir_high_alloc); + return; + } + + /* Load low word */ + success = load_word_from_base(r, base_reg, fc, sign); + if (!success) + { + ScratchRegAlloc rr_alloc = th_offset_to_reg_ex(fc, sign, (1u << r) | (1u << base_reg) | (1u << ir_high)); + int rr = rr_alloc.reg; + ot_check(th_ldr_reg(r, base_reg, rr, THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); + restore_scratch_reg(&rr_alloc); + } + + /* Load high word */ + int fc_high = sign ? (fc - 4) : (fc + 4); + success = load_word_from_base(ir_high, base_reg, fc_high, sign); + if (!success) + { + ScratchRegAlloc rr_alloc = th_offset_to_reg_ex(fc_high, sign, (1u << r) | (1u << base_reg) | (1u << ir_high)); + int rr = rr_alloc.reg; + ot_check(th_ldr_reg(ir_high, base_reg, rr, THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); + restore_scratch_reg(&rr_alloc); + } + + if (base_alloc.saved) + restore_scratch_reg(&base_alloc); + if (ir_high_alloc.saved) + restore_scratch_reg(&ir_high_alloc); + return; + } + + if (irop_btype == IROP_BTYPE_INT16) + { + if (!is_unsigned) + success = load_short_from_base(r, base, fc, sign); + else + success = load_ushort_from_base(r, base, fc, sign); + } + else if (irop_btype == IROP_BTYPE_INT8) + { + if (!is_unsigned) + success = load_byte_from_base(r, base, fc, sign); + else + success = load_ubyte_from_base(r, base, fc, sign); + } + else + { + /* IROP_BTYPE_INT32, IROP_BTYPE_FLOAT32, IROP_BTYPE_STRUCT, IROP_BTYPE_FUNC: load as word */ + success = load_word_from_base(r, base, fc, sign); + } + + if (!success) + { + ScratchRegAlloc rr_alloc = th_offset_to_reg_ex(fc, sign, (1u << r) | (1u << base)); + int rr = rr_alloc.reg; + if (irop_btype == IROP_BTYPE_INT16) + { + if (is_unsigned) + ot_check(th_ldrh_reg(r, base, rr, THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); + else + ot_check(th_ldrsh_reg(r, base, rr, THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); + } + else if (irop_btype == IROP_BTYPE_INT8) + { + if (is_unsigned) + ot_check(th_ldrb_reg(r, base, rr, THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); + else + ot_check(th_ldrsb_reg(r, base, rr, THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); + } + else + ot_check(th_ldr_reg(r, base, rr, THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); + restore_scratch_reg(&rr_alloc); + } +} + +ST_FUNC void gen_increment_tcov(SValue *sv) +{ + TRACE("'gen_increment_tcov'"); +} + +/* Dispatch a reg_handler call through a direct call instead of an indirect + * (function pointer) call. This works around a code-generation bug where + * struct-by-value arguments (thumb_shift) get corrupted when passed through + * indirect calls that also use sret return (thumb_opcode is 8 bytes). + * By comparing the function pointer and branching to a direct call, the + * cross-compiler generates correct struct passing code. */ +thumb_opcode thumb_call_reg_handler(thumb_reg_handler_t fn, uint32_t rd, uint32_t rn, uint32_t rm, + thumb_flags_behaviour flags, thumb_shift shift, + thumb_enforce_encoding encoding) +{ + if (fn == th_add_reg) + return th_add_reg(rd, rn, rm, flags, shift, encoding); + if (fn == th_sub_reg) + return th_sub_reg(rd, rn, rm, flags, shift, encoding); + if (fn == th_adc_reg) + return th_adc_reg(rd, rn, rm, flags, shift, encoding); + if (fn == th_sbc_reg) + return th_sbc_reg(rd, rn, rm, flags, shift, encoding); + if (fn == th_cmp_reg) + return th_cmp_reg(rd, rn, rm, flags, shift, encoding); + if (fn == th_lsl_reg) + return th_lsl_reg(rd, rn, rm, flags, shift, encoding); + if (fn == th_lsr_reg) + return th_lsr_reg(rd, rn, rm, flags, shift, encoding); + if (fn == th_asr_reg) + return th_asr_reg(rd, rn, rm, flags, shift, encoding); + if (fn == th_orr_reg) + return th_orr_reg(rd, rn, rm, flags, shift, encoding); + if (fn == th_and_reg) + return th_and_reg(rd, rn, rm, flags, shift, encoding); + if (fn == th_eor_reg) + return th_eor_reg(rd, rn, rm, flags, shift, encoding); + if (fn == th_bic_reg) + return th_bic_reg(rd, rn, rm, flags, shift, encoding); + /* Unreachable for known handlers — fallback to direct call. */ + return fn(rd, rn, rm, flags, shift, encoding); +} +void thumb_require_materialized_reg(const char *ctx, const char *operand, int reg) +{ + const bool reg_is_hw = (reg >= 0) && (reg <= 15); + if (reg == PREG_REG_NONE || !reg_is_hw) + { + tcc_error("compiler_error: %s expects %s in a physical register (pr=%d)", ctx, operand, reg); + } +} +uint32_t thumb_exclude_mask_for_regs(int count, const int *regs) +{ + uint32_t mask = 0; + for (int i = 0; i < count; ++i) + { + const int reg = regs[i]; + if (reg >= 0 && reg <= 15) + mask |= (1u << reg); + } + return mask; +} +bool thumb_is_hw_reg(int reg) +{ + return reg >= 0 && reg <= 15; +} diff --git a/source/backend/arch/arm/thumb/arm-thumb-fp.c b/source/backend/arch/arm/thumb/arm-thumb-fp.c new file mode 100644 index 000000000..47901f754 --- /dev/null +++ b/source/backend/arch/arm/thumb/arm-thumb-fp.c @@ -0,0 +1,1310 @@ +/* + * ARMvX-m code generator for TCC + * Uses thumb instruction set + * + * Based on: + * ARM Thumb 2 instruction functions for TCC + * Copyright (c) 2020 Erlend J. Sveen + * from: + * https://git.erlendjs.no/erlendjs/tinycc/-/blob/arm-thumb/arm-thumb-gen.c + * https://git.erlendjs.no/erlendjs/tinycc/-/blob/arm-thumb/arm-thumb-instructions.c + * + * And + * + * ARMv4 code generator for TCC + * + * Copyright (c) 2003 Daniel Glöckner + * Copyright (c) 2012 Thomas Preud'homme + * + * Based on i386-gen.c by Fabrice Bellard + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +/* Thumb-2 code generator: floating point -- soft-float helper calls, VFP and + * RP2350 DCP arithmetic, and complex arithmetic. */ + +#include "arm-thumb-gen.h" + +/* Get the soft float library function name for an FP operation */ +static const char *get_softfp_func_name(TccIrOp op, int is_double) +{ + switch (op) + { + case TCCIR_OP_FADD: + return is_double ? "__aeabi_dadd" : "__aeabi_fadd"; + case TCCIR_OP_FSUB: + return is_double ? "__aeabi_dsub" : "__aeabi_fsub"; + case TCCIR_OP_FMUL: + return is_double ? "__aeabi_dmul" : "__aeabi_fmul"; + case TCCIR_OP_FDIV: + return is_double ? "__aeabi_ddiv" : "__aeabi_fdiv"; + case TCCIR_OP_FNEG: + /* For negation, we can XOR the sign bit - handled separately */ + return NULL; + default: + return NULL; + } +} + +/* fp_mop_load_arg: Load a MachineOperand value into a fixed argument register + * (R0, R1, etc.) for a soft-float ABI call. Unlike mach_ensure_in_reg, this + * writes to a caller-specified register without scratch allocation bookkeeping. + * Used by tcc_gen_machine_fp_mop to set up R0/R1 before BL __aeabi_f*. */ +static void fp_mop_load_arg(int target_reg, const MachineOperand *op) +{ + switch (op->kind) + { + case MACH_OP_NONE: + return; + case MACH_OP_VFP_REG: + ot_check(th_vmov_gp_sp((uint16_t)target_reg, (uint16_t)op->u.reg.r0, 1)); /* target = sN */ + return; + case MACH_OP_REG: + if (!op->needs_deref) + { + if (op->u.reg.r0 != target_reg) + ot_check_mov_reg((uint32_t)target_reg, (uint32_t)op->u.reg.r0, flags_safe(), + THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE, false); + } + else + { + load_from_base(target_reg, PREG_REG_NONE, op->btype, (int)op->is_unsigned, 0, 0, (uint32_t)op->u.reg.r0); + } + return; + case MACH_OP_SPILL: + tcc_machine_load_spill_slot(target_reg, op->u.spill.offset); + if (op->needs_deref) + load_from_base(target_reg, PREG_REG_NONE, op->btype, (int)op->is_unsigned, 0, 0, (uint32_t)target_reg); + return; + case MACH_OP_PARAM_STACK: + { + const int adjusted = param_frame_offset(op->u.param.offset); + const int base_reg = tcc_state->need_frame_pointer ? R_FP : R_SP; + const int sign = (adjusted < 0); + const int abs_off = sign ? -adjusted : adjusted; + load_from_base(target_reg, PREG_REG_NONE, op->btype, (int)op->is_unsigned, abs_off, sign, (uint32_t)base_reg); + return; + } + case MACH_OP_IMM: + tcc_machine_load_constant(target_reg, PREG_REG_NONE, op->u.imm.val, 0, NULL); + return; + case MACH_OP_SYMBOL: + { + Sym *sym = op->u.sym.sym ? validate_sym_for_reloc(op->u.sym.sym) : NULL; + if (!op->needs_deref) + { + tcc_machine_load_constant(target_reg, PREG_REG_NONE, op->u.sym.addend, 0, sym); + } + else + { + /* Load symbol address into target_reg, then dereference through it. */ + tcc_machine_load_constant(target_reg, PREG_REG_NONE, 0, 0, sym); + const int32_t addend = op->u.sym.addend; + load_from_base(target_reg, PREG_REG_NONE, op->btype, (int)op->is_unsigned, + addend < 0 ? (int)(-addend) : (int)addend, addend < 0 ? 1 : 0, (uint32_t)target_reg); + } + return; + } + case MACH_OP_FRAME_ADDR: + tcc_machine_addr_of_stack_slot(target_reg, op->u.frame.offset, 0); + if (op->needs_deref) + load_from_base(target_reg, PREG_REG_NONE, op->btype, (int)op->is_unsigned, 0, 0, (uint32_t)target_reg); + return; + case MACH_OP_CHAIN_REL: + { + /* Captured variable: load from parent frame via static chain. */ + ScratchRegAlloc chain_scratch = {0}; + int chain_used = 0; + int base = resolve_chain_base(tcc_state->ir, op->u.chain.chain_index, (1u << (uint32_t)target_reg), &chain_scratch, + &chain_used); + int32_t off = op->u.chain.offset; + int sign = (off < 0); + int abs_off = sign ? (int)(-off) : (int)off; + load_from_base(target_reg, PREG_REG_NONE, op->btype, (int)op->is_unsigned, abs_off, sign, (uint32_t)base); + if (chain_used) + restore_scratch_reg(&chain_scratch); + return; + } + default: + tcc_error("compiler_error: fp_mop_load_arg: unhandled kind %d", (int)op->kind); + } +} + +/* Load a 64-bit (double-precision) MachineOperand into two consecutive argument + * registers (lo_reg = low 32 bits, hi_reg = high 32 bits). + * Handles REG pair, SPILL pair, PARAM_STACK, and deref'd REG. */ +static void fp_mop_load_double_arg(int lo_reg, int hi_reg, const MachineOperand *op) +{ + if (op->needs_deref && op->kind == MACH_OP_REG) + { + /* Pointer in register: resolve [r0] and [r0+4] into physical regs first. */ + MachineCodegenContext mctx; + memset(&mctx, 0, sizeof(mctx)); + uint32_t excl = (1u << (uint32_t)lo_reg) | (1u << (uint32_t)hi_reg); + MachineOperand resolved = mach_resolve_deref_64(&mctx, op, &excl); + mach_release_all(&mctx); + MachineOperand lo_op = mach_make_lo_half(&resolved); + MachineOperand hi_op = mach_make_hi_half(&resolved); + fp_mop_load_arg(lo_reg, &lo_op); + fp_mop_load_arg(hi_reg, &hi_op); + return; + } + if (op->kind == MACH_OP_PARAM_STACK) + { + /* Stack parameter: low word at op->offset, high word at op->offset + 4. */ + fp_mop_load_arg(lo_reg, op); + MachineOperand hi_op = *op; + hi_op.u.param.offset += 4; + fp_mop_load_arg(hi_reg, &hi_op); + return; + } + /* REG (non-deref) or SPILL: split with lo/hi helpers. */ + { + MachineOperand lo_op = mach_make_lo_half(op); + MachineOperand hi_op = mach_make_hi_half(op); + fp_mop_load_arg(lo_reg, &lo_op); + fp_mop_load_arg(hi_reg, &hi_op); + } +} + +/* Issue a BL to a soft-float library function. Under text+data separation a + * helper in a shared FP runtime is reached through a stub that swaps in its + * own R9, so R9 (with R12, keeping SP 8-aligned) is saved around the call; a + * helper linked into this module (static FP runtime, libtcc1) leaves R9 alone + * and needs neither. */ +static void fp_mop_do_bl(const char *func_name) +{ + Sym *sym = external_global_sym(tok_alloc_const(func_name), &func_old_type); + MachineOperand func_mop = {0}; + func_mop.kind = MACH_OP_SYMBOL; + func_mop.u.sym.sym = sym; + func_mop.u.sym.addend = 0; + const int save_r9 = text_and_data_separation && !thumb_callee_in_this_module(&func_mop); + if (save_r9) + ot_check(th_push((uint16_t)((1 << R9) | (1 << R12)))); + gcall_or_jump_mop(0, func_mop); + if (save_r9) + ot_check(th_pop((uint16_t)((1 << R9) | (1 << R12)))); +} + +/* Write a soft-float call result back to dest. + * Single-precision result is in R0; double-precision is in R0 (lo) : R1 (hi). */ +static void fp_mop_writeback_result(const MachineOperand *dest, int is_double) +{ + if (is_double) + { + MachineOperand lo_dest = mach_make_lo_half(dest); + MachineOperand hi_dest = mach_make_hi_half(dest); + mach_writeback_dest(&lo_dest, R0); + mach_writeback_dest(&hi_dest, R1); + } + else + mach_writeback_dest(dest, R0); +} + +/* ============================================================ + * Complex float MOP path — Phase 5k + * ============================================================ + * + * Complex floats are 64-bit register pairs: lo = real, hi = imaginary. + * Complex doubles are 128-bit values (always spilled): real at offset+0, imag at offset+8. + * These functions use the MOP infrastructure (fp_mop_load_arg, fp_mop_do_bl, + * mach_writeback_dest) to handle any operand kind (REG, SPILL, PARAM_STACK, + * CHAIN_REL, etc.) without requiring fill_registers_ir. + * + * Strategy: save all inputs to a stack frame, call __aeabi_f* / __aeabi_d* + * library functions, write results back to dest via mach_writeback_dest. + */ + +/* Split a complex MachineOperand into its real component. + * For complex float: real is the 32-bit lo half (same as mach_make_lo_half). + * For complex double: real is the 64-bit double at the base offset. */ +MachineOperand mach_make_complex_real(const MachineOperand *op) +{ + if (op->btype == IROP_BTYPE_FLOAT64) + { + /* Complex double: real part is a 64-bit double at the base offset. */ + MachineOperand real = *op; + real.is_complex = false; + real.is_64bit = true; /* each component is 64-bit double */ + if (real.kind == MACH_OP_REG) + ; /* keep r0:r1 pair — only valid for register-allocated complex floats */ + return real; + } + /* Complex float: fall back to lo half. */ + return mach_make_lo_half(op); +} + +/* Split a complex MachineOperand into its imaginary component. + * For complex float: imag is the 32-bit hi half (same as mach_make_hi_half). + * For complex double: imag is the 64-bit double at base offset + 8. */ +MachineOperand mach_make_complex_imag(const MachineOperand *op) +{ + if (op->btype == IROP_BTYPE_FLOAT64) + { + /* Complex double: imag part is a 64-bit double at offset + 8. */ + MachineOperand imag = *op; + imag.is_complex = false; + imag.is_64bit = true; + switch (imag.kind) + { + case MACH_OP_SPILL: + imag.u.spill.offset += 8; + break; + case MACH_OP_FRAME_ADDR: + imag.u.frame.offset += 8; + break; + case MACH_OP_PARAM_STACK: + imag.u.param.offset += 8; + break; + case MACH_OP_CHAIN_REL: + imag.u.chain.offset += 8; + break; + case MACH_OP_SYMBOL: + imag.u.sym.addend += 8; + break; + case MACH_OP_REG: + /* Register-based complex double shouldn't happen (force-spilled), + * but handle gracefully: imaginary part is not representable. */ + break; + default: + break; + } + return imag; + } + /* Complex float: fall back to hi half. */ + return mach_make_hi_half(op); +} + +/* Helper: save a double from R0:R1 to SP-relative stack offset. */ +static void fp_mop_save_double_to_sp(int off) +{ + ot_check_str_imm(R0, R_SP, off, 6, ENFORCE_ENCODING_NONE); + ot_check_str_imm(R1, R_SP, off + 4, 6, ENFORCE_ENCODING_NONE); +} + +/* Helper: load a double from SP-relative stack offset into (lo_reg, hi_reg). */ +static void fp_mop_load_double_from_sp(int lo_reg, int hi_reg, int off) +{ + ot_check_ldr_imm(lo_reg, R_SP, off, 6, ENFORCE_ENCODING_NONE); + ot_check_ldr_imm(hi_reg, R_SP, off + 4, 6, ENFORCE_ENCODING_NONE); +} + +/* Process complex double multiplication via MachineOperands. + * Handles all cases: + * scalar × complex: a * (c+di) = ac + (ad)i + * complex × scalar: (a+bi) * c = ac + (bc)i + * complex × complex: (a+bi) * (c+di) = (ac-bd) + (ad+bc)i + * + * Uses __aeabi_dmul, __aeabi_dadd, __aeabi_dsub for double-precision. + * Double AEABI calling convention: R0:R1 = arg1, R2:R3 = arg2, result in R0:R1. + */ +static void thumb_process_complex_mul_double_mop(MachineOperand src1, MachineOperand src2, MachineOperand dest) +{ + int s1_complex = src1.is_complex; + int s2_complex = src2.is_complex; + + MachineOperand d_real = mach_make_complex_real(&dest); + MachineOperand d_imag = mach_make_complex_imag(&dest); + + if (!s1_complex && s2_complex) + { + /* scalar double × complex double: a * (c+di) = ac + (ad)i */ + MachineOperand s2_real = mach_make_complex_real(&src2); + MachineOperand s2_imag = mach_make_complex_imag(&src2); + + /* Allocate 8 bytes to save the scalar 'a'. */ + ot_check(th_sub_imm(R_SP, R_SP, 8, flags_safe(), ENFORCE_ENCODING_NONE)); + + /* Load scalar 'a' into R0:R1 and save to stack. */ + fp_mop_load_double_arg(R0, R1, &src1); + fp_mop_save_double_to_sp(0); + + /* Compute a * c: load 'c' into R2:R3. R0:R1 already = 'a'. */ + fp_mop_load_double_arg(R2, R3, &s2_real); + fp_mop_do_bl("__aeabi_dmul"); + /* R0:R1 = a*c → write to dest real. */ + fp_mop_writeback_result(&d_real, 1); + + /* Compute a * d: reload 'a' from stack, load 'd' into R2:R3. */ + fp_mop_load_double_from_sp(R0, R1, 0); + fp_mop_load_double_arg(R2, R3, &s2_imag); + fp_mop_do_bl("__aeabi_dmul"); + /* R0:R1 = a*d → write to dest imag. */ + fp_mop_writeback_result(&d_imag, 1); + + ot_check(th_add_imm(R_SP, R_SP, 8, flags_safe(), ENFORCE_ENCODING_NONE)); + } + else if (s1_complex && !s2_complex) + { + /* complex double × scalar double: (a+bi) * c = ac + (bc)i */ + MachineOperand s1_real = mach_make_complex_real(&src1); + MachineOperand s1_imag = mach_make_complex_imag(&src1); + + /* Allocate 8 bytes to save the scalar 'c'. */ + ot_check(th_sub_imm(R_SP, R_SP, 8, flags_safe(), ENFORCE_ENCODING_NONE)); + + /* Load scalar 'c' into R0:R1 and save to stack. */ + fp_mop_load_double_arg(R0, R1, &src2); + fp_mop_save_double_to_sp(0); + + /* Compute a * c. */ + fp_mop_load_double_arg(R0, R1, &s1_real); + fp_mop_load_double_arg(R2, R3, &src2); + fp_mop_do_bl("__aeabi_dmul"); + fp_mop_writeback_result(&d_real, 1); + + /* Compute b * c: reload 'c', load 'b'. */ + fp_mop_load_double_arg(R0, R1, &s1_imag); + fp_mop_load_double_from_sp(R2, R3, 0); + fp_mop_do_bl("__aeabi_dmul"); + fp_mop_writeback_result(&d_imag, 1); + + ot_check(th_add_imm(R_SP, R_SP, 8, flags_safe(), ENFORCE_ENCODING_NONE)); + } + else + { + /* complex × complex: (a+bi)*(c+di) = (ac-bd) + (ad+bc)i + * + * Stack layout (48 bytes): + * [sp+40] = d (imag of src2, 8 bytes) + * [sp+32] = c (real of src2, 8 bytes) + * [sp+24] = b (imag of src1, 8 bytes) + * [sp+16] = a (real of src1, 8 bytes) + * [sp+8] = scratch1 (8 bytes) + * [sp+0] = scratch0 (8 bytes) + */ + MachineOperand s1_real = mach_make_complex_real(&src1); + MachineOperand s1_imag = mach_make_complex_imag(&src1); + MachineOperand s2_real = mach_make_complex_real(&src2); + MachineOperand s2_imag = mach_make_complex_imag(&src2); + + const int off_scratch0 = 0, off_scratch1 = 8; + const int off_a = 16, off_b = 24, off_c = 32, off_d = 40; + + ot_check(th_sub_imm(R_SP, R_SP, 48, flags_safe(), ENFORCE_ENCODING_NONE)); + + /* Save all 4 components to stack. */ + fp_mop_load_double_arg(R0, R1, &s1_real); + fp_mop_save_double_to_sp(off_a); + fp_mop_load_double_arg(R0, R1, &s1_imag); + fp_mop_save_double_to_sp(off_b); + fp_mop_load_double_arg(R0, R1, &s2_real); + fp_mop_save_double_to_sp(off_c); + fp_mop_load_double_arg(R0, R1, &s2_imag); + fp_mop_save_double_to_sp(off_d); + + /* Step 1: ac → scratch0. */ + fp_mop_load_double_from_sp(R0, R1, off_a); + fp_mop_load_double_from_sp(R2, R3, off_c); + fp_mop_do_bl("__aeabi_dmul"); + fp_mop_save_double_to_sp(off_scratch0); + + /* Step 2: bd → scratch1. */ + fp_mop_load_double_from_sp(R0, R1, off_b); + fp_mop_load_double_from_sp(R2, R3, off_d); + fp_mop_do_bl("__aeabi_dmul"); + fp_mop_save_double_to_sp(off_scratch1); + + /* Step 3: real = ac - bd → scratch0. */ + fp_mop_load_double_from_sp(R0, R1, off_scratch0); + fp_mop_load_double_from_sp(R2, R3, off_scratch1); + fp_mop_do_bl("__aeabi_dsub"); + fp_mop_save_double_to_sp(off_scratch0); + + /* Step 4: ad → scratch1. */ + fp_mop_load_double_from_sp(R0, R1, off_a); + fp_mop_load_double_from_sp(R2, R3, off_d); + fp_mop_do_bl("__aeabi_dmul"); + fp_mop_save_double_to_sp(off_scratch1); + + /* Step 5: bc → off_a (reuse slot). */ + fp_mop_load_double_from_sp(R0, R1, off_b); + fp_mop_load_double_from_sp(R2, R3, off_c); + fp_mop_do_bl("__aeabi_dmul"); + fp_mop_save_double_to_sp(off_a); + + /* Step 6: imag = ad + bc → scratch1. */ + fp_mop_load_double_from_sp(R0, R1, off_scratch1); + fp_mop_load_double_from_sp(R2, R3, off_a); + fp_mop_do_bl("__aeabi_dadd"); + fp_mop_save_double_to_sp(off_scratch1); + + /* Write results back to dest. */ + fp_mop_load_double_from_sp(R0, R1, off_scratch0); + fp_mop_writeback_result(&d_real, 1); + fp_mop_load_double_from_sp(R0, R1, off_scratch1); + fp_mop_writeback_result(&d_imag, 1); + + ot_check(th_add_imm(R_SP, R_SP, 48, flags_safe(), ENFORCE_ENCODING_NONE)); + } +} + +/* complex_pair_writeback: Write a (real, imag) pair from two physical registers + * into a split MachineOperand pair without clobbering. + * Handles the case where d_lo's target register overlaps hi_reg (or vice versa) + * by saving the clobbered value to R2 or R3 first. */ +static void complex_pair_writeback(MachineOperand *d_lo, int lo_reg, MachineOperand *d_hi, int hi_reg) +{ + int lo_clobbers_hi = (d_lo->kind == MACH_OP_REG && d_lo->u.reg.r0 == hi_reg); + int hi_clobbers_lo = (d_hi->kind == MACH_OP_REG && d_hi->u.reg.r0 == lo_reg); + + if (lo_clobbers_hi && hi_clobbers_lo) + { + /* Total swap: save hi to temp, then write both */ + int tmp = (lo_reg != R2 && hi_reg != R2) ? R2 : R3; + ot_check_mov_reg((uint32_t)tmp, (uint32_t)hi_reg, flags_safe(), THUMB_SHIFT_DEFAULT, + ENFORCE_ENCODING_NONE, false); + mach_writeback_dest(d_lo, lo_reg); + mach_writeback_dest(d_hi, tmp); + } + else if (lo_clobbers_hi) + { + /* Lo writeback would clobber hi value; save hi first */ + int tmp = (lo_reg != R2 && hi_reg != R2) ? R2 : R3; + ot_check_mov_reg((uint32_t)tmp, (uint32_t)hi_reg, flags_safe(), THUMB_SHIFT_DEFAULT, + ENFORCE_ENCODING_NONE, false); + mach_writeback_dest(d_lo, lo_reg); + mach_writeback_dest(d_hi, tmp); + } + else if (hi_clobbers_lo) + { + /* Hi writeback would clobber lo value; write lo first */ + mach_writeback_dest(d_lo, lo_reg); + mach_writeback_dest(d_hi, hi_reg); + } + else + { + mach_writeback_dest(d_lo, lo_reg); + mach_writeback_dest(d_hi, hi_reg); + } +} + +/* thumb_emit_dcp_addsub_mop: inline double add/subtract on RP2350's DCP. + * + * The sequence is the one in lib/fp/arm/rp2350/dcp_aeabi.S, itself transcribed + * from pico-sdk's dcp_canned.inc.S -- keep the two in step: + * + * WXUP a ; WYUP b ; ADD0 ; ADD1|SUB1 ; NRDD ; RDDA|RDDS + * + * Six instructions, no scratch registers, and -- unlike the library form -- + * on whichever registers the allocator already chose. `mcrr`/`mrrc` move a + * GPR *pair* but place no consecutiveness constraint on it (that is LDRD's + * rule, not theirs), so any two distinct registers work and nothing has to be + * shuffled into r0-r3 first. + * + * Overwriting a source register with the result is safe: WXUP/WYUP have + * already copied both operands into the coprocessor by the time RDDA/RDDS + * writes back. The destination is still excluded from source allocation + * because the shared 64-bit idiom does that, not because it is required. + */ +static void thumb_emit_dcp_addsub_mop(MachineOperand src1, MachineOperand src2, MachineOperand dest, int is_sub) +{ + MachineCodegenContext mctx = {0}; + uint32_t excl = 0; + + /* Destination pair first, so deref resolution never allocates a scratch that + * overlaps it and then restores over the result (same reason as the integer + * 64-bit path). */ + int rd_lo, rd_hi; + bool store_lo = false, store_hi = false; + if (dest.kind == MACH_OP_REG && !dest.needs_deref && dest.u.reg.r0 != (int)PREG_REG_NONE && dest.u.reg.r1 >= 0) + { + rd_lo = dest.u.reg.r0; + rd_hi = dest.u.reg.r1; + excl |= (1u << (uint32_t)rd_lo) | (1u << (uint32_t)rd_hi); + } + else + { + rd_lo = mach_alloc_scratch(&mctx, excl); + excl |= (1u << (uint32_t)rd_lo); + rd_hi = mach_alloc_scratch(&mctx, excl); + excl |= (1u << (uint32_t)rd_hi); + store_lo = store_hi = (dest.kind != MACH_OP_NONE); + } + + /* Pre-exclude both sources' registers so resolving one deref cannot steal + * the physical registers of the other. */ + if (src1.kind == MACH_OP_REG) + { + if (src1.u.reg.r0 != (int)PREG_REG_NONE) + excl |= (1u << (uint32_t)src1.u.reg.r0); + if (!src1.needs_deref && src1.is_64bit && src1.u.reg.r1 >= 0) + excl |= (1u << (uint32_t)src1.u.reg.r1); + } + if (src2.kind == MACH_OP_REG) + { + if (src2.u.reg.r0 != (int)PREG_REG_NONE) + excl |= (1u << (uint32_t)src2.u.reg.r0); + if (!src2.needs_deref && src2.is_64bit && src2.u.reg.r1 >= 0) + excl |= (1u << (uint32_t)src2.u.reg.r1); + } + + MachineOperand a = mach_resolve_deref_64(&mctx, &src1, &excl); + MachineOperand b = mach_resolve_deref_64(&mctx, &src2, &excl); + + int a_lo, a_hi, b_lo, b_hi; + mach_ensure_pair_in_regs(&mctx, &a, &excl, &a_lo, &a_hi); + mach_ensure_pair_in_regs(&mctx, &b, &excl, &b_lo, &b_hi); + + ot_check(th_mcrr(4, 1, (uint32_t)a_lo, (uint32_t)a_hi, 0, 0)); /* WXUP a */ + ot_check(th_mcrr(4, 1, (uint32_t)b_lo, (uint32_t)b_hi, 1, 0)); /* WYUP b */ + ot_check(th_cdp(4, 0, 0, 0, 1, 0, 0)); /* ADD0 */ + ot_check(th_cdp(4, 1, 0, 0, 1, is_sub ? 1 : 0, 0)); /* SUB1 : ADD1 */ + ot_check(th_cdp(4, 8, 0, 0, 0, 1, 0)); /* NRDD */ + ot_check(th_mrrc(4, is_sub ? 3 : 1, (uint32_t)rd_lo, (uint32_t)rd_hi, 0, 0)); /* RDDS : RDDA */ + + if (store_lo) + { + MachineOperand dst_lo = mach_make_lo_half(&dest); + dst_lo.btype = IROP_BTYPE_INT32; + mach_writeback_dest(&dst_lo, rd_lo); + } + if (store_hi) + { + MachineOperand dst_hi = mach_make_hi_half(&dest); + dst_hi.btype = IROP_BTYPE_INT32; + mach_writeback_dest(&dst_hi, rd_hi); + } + mach_release_all(&mctx); +} + +/* thumb_emit_vfp_arith_mop: inline single-precision arithmetic on the FPU. + * + * vmov s0, rn ; vmov s1, rm ; v.f32 s0, s0, s1 ; vmov rd, s0 + * + * Four instructions against a BL plus the library's own five. s0/s1 are used + * as fixed scratch rather than allocated: this is the softfp ABI, so no float + * value ever lives in a VFP register between operations and s0-s15 are free at + * every point. That is also what keeps this independent of Phase 5 proper -- + * making floats *live* in s0-s15 needs a register class and th_vldr/th_vstr + * (which do not exist yet) for spill and reload. + * + * The values move through GPRs on both sides, so nothing here depends on the + * float ABI and fast-mode objects stay link-compatible with portable ones. + */ +/* Get a float operand into a single-precision register: use it directly when it + * already lives in one (MACH_OP_VFP_REG), else materialize it through a GPR into + * the caller-provided scratch s-register. */ +static int vfp_operand_to_sreg(MachineCodegenContext *ctx, const MachineOperand *op, int scratch_s) +{ + if (op->kind == MACH_OP_VFP_REG) + return op->u.reg.r0; + int gpr = mach_ensure_in_reg(ctx, op, 0); + ot_check(th_vmov_gp_sp((uint16_t)gpr, (uint16_t)scratch_s, 0)); /* scratch_s = gpr */ + return scratch_s; +} + +static void thumb_emit_vfp_arith_mop(MachineOperand src1, MachineOperand src2, MachineOperand dest, TccIrOp op) +{ + MachineCodegenContext ctx = {0}; + + /* Hard-float: operate on real single-precision registers. VFP-resident + * operands/dest are used directly (v.f32 sd, sn, sm); GPR/imm/spill + * endpoints are materialized through the reserved s14/s15 scratch. */ + if (tcc_state && tcc_state->float_abi == ARM_HARD_FLOAT) + { + int sn = vfp_operand_to_sreg(&ctx, &src1, VFP_SCRATCH0); + int sm = vfp_operand_to_sreg(&ctx, &src2, VFP_SCRATCH1); + int sd = (dest.kind == MACH_OP_VFP_REG) ? dest.u.reg.r0 : VFP_SCRATCH0; + switch (op) + { + case TCCIR_OP_FADD: ot_check(th_vadd_f((uint16_t)sd, (uint16_t)sn, (uint16_t)sm, 0)); break; + case TCCIR_OP_FSUB: ot_check(th_vsub_f((uint16_t)sd, (uint16_t)sn, (uint16_t)sm, 0)); break; + case TCCIR_OP_FMUL: ot_check(th_vmul_f((uint16_t)sd, (uint16_t)sn, (uint16_t)sm, 0)); break; + case TCCIR_OP_FDIV: ot_check(th_vdiv_f((uint16_t)sd, (uint16_t)sn, (uint16_t)sm, 0)); break; + default: tcc_error("compiler_error: thumb_emit_vfp_arith_mop: unhandled op %d", (int)op); break; + } + if (dest.kind != MACH_OP_VFP_REG) + { + int rd = mach_get_dest_reg(&ctx, &dest, 0); + ot_check(th_vmov_gp_sp((uint16_t)rd, (uint16_t)sd, 1)); /* rd = sd */ + mach_writeback_dest(&dest, rd); + } + mach_release_all(&ctx); + return; + } + + /* Softfp: no float ever lives in a VFP register, so s0/s1 are free fixed + * scratch. Values move through GPRs on both sides — kept byte-identical. */ + int rd = mach_get_dest_reg(&ctx, &dest, 0); + uint32_t excl = (1u << (uint32_t)rd); + int rn = mach_ensure_in_reg(&ctx, &src1, excl); + if (thumb_is_hw_reg(rn)) + excl |= (1u << (uint32_t)rn); + int rm = mach_ensure_in_reg(&ctx, &src2, excl); + + ot_check(th_vmov_gp_sp((uint16_t)rn, 0, 0)); /* s0 = rn */ + ot_check(th_vmov_gp_sp((uint16_t)rm, 1, 0)); /* s1 = rm */ + switch (op) + { + case TCCIR_OP_FADD: + ot_check(th_vadd_f(0, 0, 1, 0)); + break; + case TCCIR_OP_FSUB: + ot_check(th_vsub_f(0, 0, 1, 0)); + break; + case TCCIR_OP_FMUL: + ot_check(th_vmul_f(0, 0, 1, 0)); + break; + case TCCIR_OP_FDIV: + ot_check(th_vdiv_f(0, 0, 1, 0)); + break; + default: + tcc_error("compiler_error: thumb_emit_vfp_arith_mop: unhandled op %d", (int)op); + break; + } + ot_check(th_vmov_gp_sp((uint16_t)rd, 0, 1)); /* rd = s0 */ + + mach_writeback_dest(&dest, rd); + mach_release_all(&ctx); +} + +/* thumb_emit_dcp_cmp_mop: inline double compare on RP2350's DCP. + * + * WXUP a ; WYUP b ; ADD0 ; RCMP apsr_nzcv + * + * Four instructions and no result register: RCMP with Rt == PC writes the + * relation directly into the flags, so the compare feeds an ordinary + * conditional branch instead of a call plus flag decode. + * + * The flag encoding is the AEABI's -- C set means "ordered and >=", Z set + * means equal, V set means unordered -- and is read with *unsigned* + * conditions. ir/gen/float.c already mirrored the operands and recorded + * TOK_UGT / TOK_UGE accordingly, so this only has to compare in operand + * order. + * + * The RCMP is emitted LAST, after mach_release_all(): releasing can restore + * saved scratch registers, and the flags must be the final thing written + * before the branch consumes them. Splitting ADD0 from RCMP is safe because + * the DCP holds its state across unrelated core instructions -- that is the + * same property __rp2350_dcp_save/_restore rely on. + */ +static void thumb_emit_dcp_cmp_mop(MachineOperand src1, MachineOperand src2) +{ + MachineCodegenContext mctx = {0}; + uint32_t excl = 0; + + if (src1.kind == MACH_OP_REG) + { + if (src1.u.reg.r0 != (int)PREG_REG_NONE) + excl |= (1u << (uint32_t)src1.u.reg.r0); + if (!src1.needs_deref && src1.is_64bit && src1.u.reg.r1 >= 0) + excl |= (1u << (uint32_t)src1.u.reg.r1); + } + if (src2.kind == MACH_OP_REG) + { + if (src2.u.reg.r0 != (int)PREG_REG_NONE) + excl |= (1u << (uint32_t)src2.u.reg.r0); + if (!src2.needs_deref && src2.is_64bit && src2.u.reg.r1 >= 0) + excl |= (1u << (uint32_t)src2.u.reg.r1); + } + + MachineOperand a = mach_resolve_deref_64(&mctx, &src1, &excl); + MachineOperand b = mach_resolve_deref_64(&mctx, &src2, &excl); + + int a_lo, a_hi, b_lo, b_hi; + mach_ensure_pair_in_regs(&mctx, &a, &excl, &a_lo, &a_hi); + mach_ensure_pair_in_regs(&mctx, &b, &excl, &b_lo, &b_hi); + + ot_check(th_mcrr(4, 1, (uint32_t)a_lo, (uint32_t)a_hi, 0, 0)); /* WXUP a */ + ot_check(th_mcrr(4, 1, (uint32_t)b_lo, (uint32_t)b_hi, 1, 0)); /* WYUP b */ + ot_check(th_cdp(4, 0, 0, 0, 1, 0, 0)); /* ADD0 */ + + mach_release_all(&mctx); + + ot_check(th_mrc(4, 0, R_PC, 0, 0, 1, 0)); /* RCMP apsr_nzcv */ +} + +/* Process complex double addition/subtraction via MachineOperands. + * (a+bi) + (c+di) = (a+c) + (b+d)i + * (a+bi) - (c+di) = (a-c) + (b-d)i + * Uses __aeabi_dadd/__aeabi_dsub for double-precision. + * Double AEABI calling convention: R0:R1 = arg1, R2:R3 = arg2, result in R0:R1. + */ +static void thumb_process_complex_op_double_mop(MachineOperand src1, MachineOperand src2, MachineOperand dest, + TccIrOp op) +{ + const int is_add = (op == TCCIR_OP_ADD); + const char *func_name = is_add ? "__aeabi_dadd" : "__aeabi_dsub"; + + MachineOperand s1_real = mach_make_complex_real(&src1); + MachineOperand s1_imag = mach_make_complex_imag(&src1); + MachineOperand s2_real = mach_make_complex_real(&src2); + MachineOperand s2_imag = mach_make_complex_imag(&src2); + MachineOperand d_real = mach_make_complex_real(&dest); + MachineOperand d_imag = mach_make_complex_imag(&dest); + + /* Stack layout (32 bytes): + * [sp+24] = s2_imag (8 bytes) + * [sp+16] = s2_real (8 bytes) + * [sp+8] = s1_imag (8 bytes) + * [sp+0] = s1_real (8 bytes) + */ + ot_check(th_sub_imm(R_SP, R_SP, 32, flags_safe(), ENFORCE_ENCODING_NONE)); + + /* Save all 4 components to stack. */ + fp_mop_load_double_arg(R0, R1, &s1_real); + fp_mop_save_double_to_sp(0); + fp_mop_load_double_arg(R0, R1, &s1_imag); + fp_mop_save_double_to_sp(8); + fp_mop_load_double_arg(R0, R1, &s2_real); + fp_mop_save_double_to_sp(16); + fp_mop_load_double_arg(R0, R1, &s2_imag); + fp_mop_save_double_to_sp(24); + + /* Compute real part: func(a.real, b.real) */ + fp_mop_load_double_from_sp(R0, R1, 0); + fp_mop_load_double_from_sp(R2, R3, 16); + fp_mop_do_bl(func_name); + /* Save real result to stack slot 0 */ + fp_mop_save_double_to_sp(0); + + /* Compute imag part: func(a.imag, b.imag) */ + fp_mop_load_double_from_sp(R0, R1, 8); + fp_mop_load_double_from_sp(R2, R3, 24); + fp_mop_do_bl(func_name); + /* R0:R1 = imag result. Load real result from stack. */ + fp_mop_save_double_to_sp(8); /* save imag to slot 8 */ + + /* Write results back to dest. */ + fp_mop_load_double_from_sp(R0, R1, 0); + fp_mop_writeback_result(&d_real, 1); + fp_mop_load_double_from_sp(R0, R1, 8); + fp_mop_writeback_result(&d_imag, 1); + + ot_check(th_add_imm(R_SP, R_SP, 32, flags_safe(), ENFORCE_ENCODING_NONE)); +} + +/* Process complex addition/subtraction via MachineOperands. + * (a+bi) + (c+di) = (a+c) + (b+d)i + * (a+bi) - (c+di) = (a-c) + (b-d)i + */ +static void thumb_process_complex_op_mop(MachineOperand src1, MachineOperand src2, MachineOperand dest, TccIrOp op) +{ + const int is_add = (op == TCCIR_OP_ADD); + + /* Complex float: each component is a 32-bit float. */ + const char *func_name = is_add ? "__aeabi_fadd" : "__aeabi_fsub"; + + /* Split into real/imag components. */ + MachineOperand s1_real = mach_make_lo_half(&src1); + MachineOperand s1_imag = mach_make_hi_half(&src1); + MachineOperand s2_real = mach_make_lo_half(&src2); + MachineOperand s2_imag = mach_make_hi_half(&src2); + + /* Stack-based: save all 4 inputs, do calls, write results back. + * Stack layout (16 bytes): + * [sp+12] = s2_imag + * [sp+8] = s2_real + * [sp+4] = s1_imag + * [sp+0] = s1_real + */ + ot_check(th_sub_imm(R_SP, R_SP, 16, flags_safe(), ENFORCE_ENCODING_NONE)); + + /* Load and save each component to stack. */ + fp_mop_load_arg(R0, &s1_real); + ot_check_str_imm(R0, R_SP, 0, 6, ENFORCE_ENCODING_NONE); + fp_mop_load_arg(R0, &s1_imag); + ot_check_str_imm(R0, R_SP, 4, 6, ENFORCE_ENCODING_NONE); + fp_mop_load_arg(R0, &s2_real); + ot_check_str_imm(R0, R_SP, 8, 6, ENFORCE_ENCODING_NONE); + fp_mop_load_arg(R0, &s2_imag); + ot_check_str_imm(R0, R_SP, 12, 6, ENFORCE_ENCODING_NONE); + + /* Compute real part: func(a.real, b.real) */ + ot_check_ldr_imm(R0, R_SP, 0, 6, ENFORCE_ENCODING_NONE); + ot_check_ldr_imm(R1, R_SP, 8, 6, ENFORCE_ENCODING_NONE); + fp_mop_do_bl(func_name); + /* Save real result to stack slot 0 */ + ot_check_str_imm(R0, R_SP, 0, 6, ENFORCE_ENCODING_NONE); + + /* Compute imag part: func(a.imag, b.imag) */ + ot_check_ldr_imm(R0, R_SP, 4, 6, ENFORCE_ENCODING_NONE); + ot_check_ldr_imm(R1, R_SP, 12, 6, ENFORCE_ENCODING_NONE); + fp_mop_do_bl(func_name); + /* R0 = imag result */ + + /* Load real result from stack, deallocate, write back. */ + MachineOperand d_real = mach_make_lo_half(&dest); + MachineOperand d_imag = mach_make_hi_half(&dest); + + /* R0 = imag result. Load real result from stack into R1. */ + ot_check_ldr_imm(R1, R_SP, 0, 6, ENFORCE_ENCODING_NONE); + ot_check(th_add_imm(R_SP, R_SP, 16, flags_safe(), ENFORCE_ENCODING_NONE)); + + /* Write back: R1 = real part, R0 = imag part. + * Use safe writeback to avoid clobbering when dest overlaps R0/R1. */ + complex_pair_writeback(&d_real, R1, &d_imag, R0); +} + +/* Process complex multiplication via MachineOperands. + * (a+bi) * (c+di) = (ac-bd) + (ad+bc)i + * + * Stack layout (24 bytes): + * [sp+20] = d (imag of src2) + * [sp+16] = c (real of src2) + * [sp+12] = b (imag of src1) + * [sp+8] = a (real of src1) + * [sp+4] = scratch1 + * [sp+0] = scratch0 + */ +static void thumb_process_complex_mul_mop(MachineOperand src1, MachineOperand src2, MachineOperand dest) +{ + MachineOperand s1_real = mach_make_lo_half(&src1); + MachineOperand s1_imag = mach_make_hi_half(&src1); + MachineOperand s2_real = mach_make_lo_half(&src2); + MachineOperand s2_imag = mach_make_hi_half(&src2); + + /* Allocate 24 bytes on stack */ + ot_check(th_sub_imm(R_SP, R_SP, 24, flags_safe(), ENFORCE_ENCODING_NONE)); + + /* Save inputs to stack */ + fp_mop_load_arg(R0, &s1_real); + ot_check_str_imm(R0, R_SP, 8, 6, ENFORCE_ENCODING_NONE); /* a */ + fp_mop_load_arg(R0, &s1_imag); + ot_check_str_imm(R0, R_SP, 12, 6, ENFORCE_ENCODING_NONE); /* b */ + fp_mop_load_arg(R0, &s2_real); + ot_check_str_imm(R0, R_SP, 16, 6, ENFORCE_ENCODING_NONE); /* c */ + fp_mop_load_arg(R0, &s2_imag); + ot_check_str_imm(R0, R_SP, 20, 6, ENFORCE_ENCODING_NONE); /* d */ + + const int off_scratch0 = 0; + const int off_scratch1 = 4; + const int off_a = 8; + const int off_b = 12; + const int off_c = 16; + const int off_d = 20; + + /* Step 1: ac = a * c → scratch0 */ + ot_check_ldr_imm(R0, R_SP, off_a, 6, ENFORCE_ENCODING_NONE); + ot_check_ldr_imm(R1, R_SP, off_c, 6, ENFORCE_ENCODING_NONE); + fp_mop_do_bl("__aeabi_fmul"); + ot_check_str_imm(R0, R_SP, off_scratch0, 6, ENFORCE_ENCODING_NONE); + + /* Step 2: bd = b * d → scratch1 */ + ot_check_ldr_imm(R0, R_SP, off_b, 6, ENFORCE_ENCODING_NONE); + ot_check_ldr_imm(R1, R_SP, off_d, 6, ENFORCE_ENCODING_NONE); + fp_mop_do_bl("__aeabi_fmul"); + ot_check_str_imm(R0, R_SP, off_scratch1, 6, ENFORCE_ENCODING_NONE); + + /* Step 3: real = ac - bd → scratch0 */ + ot_check_ldr_imm(R0, R_SP, off_scratch0, 6, ENFORCE_ENCODING_NONE); + ot_check_ldr_imm(R1, R_SP, off_scratch1, 6, ENFORCE_ENCODING_NONE); + fp_mop_do_bl("__aeabi_fsub"); + ot_check_str_imm(R0, R_SP, off_scratch0, 6, ENFORCE_ENCODING_NONE); + + /* Step 4: ad = a * d → scratch1 */ + ot_check_ldr_imm(R0, R_SP, off_a, 6, ENFORCE_ENCODING_NONE); + ot_check_ldr_imm(R1, R_SP, off_d, 6, ENFORCE_ENCODING_NONE); + fp_mop_do_bl("__aeabi_fmul"); + ot_check_str_imm(R0, R_SP, off_scratch1, 6, ENFORCE_ENCODING_NONE); + + /* Step 5: bc = b * c → off_a (no longer needed) */ + ot_check_ldr_imm(R0, R_SP, off_b, 6, ENFORCE_ENCODING_NONE); + ot_check_ldr_imm(R1, R_SP, off_c, 6, ENFORCE_ENCODING_NONE); + fp_mop_do_bl("__aeabi_fmul"); + ot_check_str_imm(R0, R_SP, off_a, 6, ENFORCE_ENCODING_NONE); + + /* Step 6: imag = ad + bc → scratch1 */ + ot_check_ldr_imm(R0, R_SP, off_scratch1, 6, ENFORCE_ENCODING_NONE); + ot_check_ldr_imm(R1, R_SP, off_a, 6, ENFORCE_ENCODING_NONE); + fp_mop_do_bl("__aeabi_fadd"); + ot_check_str_imm(R0, R_SP, off_scratch1, 6, ENFORCE_ENCODING_NONE); + + /* Load results and write back */ + MachineOperand d_real = mach_make_lo_half(&dest); + MachineOperand d_imag = mach_make_hi_half(&dest); + ot_check_ldr_imm(R0, R_SP, off_scratch0, 6, ENFORCE_ENCODING_NONE); /* real */ + ot_check_ldr_imm(R1, R_SP, off_scratch1, 6, ENFORCE_ENCODING_NONE); /* imag */ + ot_check(th_add_imm(R_SP, R_SP, 24, flags_safe(), ENFORCE_ENCODING_NONE)); + + complex_pair_writeback(&d_real, R0, &d_imag, R1); +} + +/* Process complex float division via MachineOperands. + * Calls __divsc3 from libgcc for numerically robust division. + * + * __divsc3 calling convention (soft-float AAPCS, hidden return pointer): + * R0 = hidden return pointer (8-byte buffer for result) + * R1 = a_real (first float arg) + * R2 = a_imag (second float arg) + * R3 = b_real (third float arg) + * [sp+0] = b_imag (fourth float arg, on stack) + * Result written to [R0+0..3] = real, [R0+4..7] = imag + * + * Stack layout (16 bytes): + * [sp+0] = b_imag for __divsc3 stack arg (4 bytes) + * [sp+4] = padding (4 bytes) + * [sp+8] = result buffer: real part (4 bytes) + * [sp+12] = result buffer: imag part (4 bytes) + */ +static void thumb_process_complex_div_mop(MachineOperand src1, MachineOperand src2, MachineOperand dest) +{ + MachineOperand s1_real = mach_make_lo_half(&src1); + MachineOperand s1_imag = mach_make_hi_half(&src1); + MachineOperand s2_real = mach_make_lo_half(&src2); + MachineOperand s2_imag = mach_make_hi_half(&src2); + + /* In PIC mode, save {r9, r12} BEFORE allocating the call frame so that + * SP-relative offsets within the 16-byte area remain correct when + * __divsc3 reads its stack arg at [sp+0]. */ + if (text_and_data_separation) + ot_check(th_push((uint16_t)((1 << R9) | (1 << R12)))); + + /* Allocate 16 bytes on stack. */ + ot_check(th_sub_imm(R_SP, R_SP, 16, flags_safe(), ENFORCE_ENCODING_NONE)); + + /* Save all inputs to stack first to avoid register clobbering. */ + fp_mop_load_arg(R0, &s1_real); + ot_check_str_imm(R0, R_SP, 0, 6, ENFORCE_ENCODING_NONE); /* a_real */ + fp_mop_load_arg(R0, &s1_imag); + ot_check_str_imm(R0, R_SP, 4, 6, ENFORCE_ENCODING_NONE); /* a_imag */ + fp_mop_load_arg(R0, &s2_real); + ot_check_str_imm(R0, R_SP, 8, 6, ENFORCE_ENCODING_NONE); /* b_real */ + fp_mop_load_arg(R0, &s2_imag); + ot_check_str_imm(R0, R_SP, 12, 6, ENFORCE_ENCODING_NONE); /* b_imag */ + + /* Rearrange stack for __divsc3 call: + * Need [sp+0] = b_imag, [sp+8..15] = result buffer. + * Currently [sp+0]=a_real, [sp+4]=a_imag, [sp+8]=b_real, [sp+12]=b_imag. + * Load R1-R3 from stack, then rearrange. */ + ot_check_ldr_imm(R1, R_SP, 0, 6, ENFORCE_ENCODING_NONE); /* R1 = a_real */ + ot_check_ldr_imm(R2, R_SP, 4, 6, ENFORCE_ENCODING_NONE); /* R2 = a_imag */ + ot_check_ldr_imm(R3, R_SP, 8, 6, ENFORCE_ENCODING_NONE); /* R3 = b_real */ + ot_check_ldr_imm(R0, R_SP, 12, 6, ENFORCE_ENCODING_NONE); /* R0 = b_imag */ + ot_check_str_imm(R0, R_SP, 0, 6, ENFORCE_ENCODING_NONE); /* [sp+0] = b_imag (stack arg) */ + + /* R0 = pointer to result buffer at [sp+8]. */ + ot_check(th_add_imm(R0, R_SP, 8, flags_safe(), ENFORCE_ENCODING_NONE)); + + /* Call __divsc3 (directly, not via fp_mop_do_bl which would add another + * push/pop of {r9, r12} and corrupt the stack arg layout). */ + { + Sym *sym = external_global_sym(tok_alloc_const("__divsc3"), &func_old_type); + MachineOperand func_mop = {0}; + func_mop.kind = MACH_OP_SYMBOL; + func_mop.u.sym.sym = sym; + func_mop.u.sym.addend = 0; + gcall_or_jump_mop(0, func_mop); + } + + /* Read result from buffer and write back to dest. */ + MachineOperand d_real = mach_make_lo_half(&dest); + MachineOperand d_imag = mach_make_hi_half(&dest); + ot_check_ldr_imm(R0, R_SP, 8, 6, ENFORCE_ENCODING_NONE); /* real */ + ot_check_ldr_imm(R1, R_SP, 12, 6, ENFORCE_ENCODING_NONE); /* imag */ + ot_check(th_add_imm(R_SP, R_SP, 16, flags_safe(), ENFORCE_ENCODING_NONE)); + + if (text_and_data_separation) + ot_check(th_pop((uint16_t)((1 << R9) | (1 << R12)))); + + complex_pair_writeback(&d_real, R0, &d_imag, R1); +} + +/* Process complex double division via MachineOperands. + * Calls __divdc3 from libgcc for numerically robust division. + * + * __divdc3 calling convention (soft-float AAPCS, hidden return pointer): + * R0 = hidden return pointer (16-byte buffer for result) + * R2:R3 = a_re (first double, even-aligned) + * [sp+0] = a_im (second double, on stack) + * [sp+8] = b_re (third double, on stack) + * [sp+16] = b_im (fourth double, on stack) + * Result written to [R0+0..7] = real, [R0+8..15] = imag + * + * Stack layout (40 bytes, 8-byte aligned): + * [sp+0] = a_im for __divdc3 stack arg (8 bytes) + * [sp+8] = b_re for __divdc3 stack arg (8 bytes) + * [sp+16] = b_im for __divdc3 stack arg (8 bytes) + * [sp+24] = result buffer: real part (8 bytes) + * [sp+32] = result buffer: imag part (8 bytes) + */ +static void thumb_process_complex_div_double_mop(MachineOperand src1, MachineOperand src2, MachineOperand dest) +{ + MachineOperand s1_real = mach_make_complex_real(&src1); + MachineOperand s1_imag = mach_make_complex_imag(&src1); + MachineOperand s2_real = mach_make_complex_real(&src2); + MachineOperand s2_imag = mach_make_complex_imag(&src2); + MachineOperand d_real = mach_make_complex_real(&dest); + MachineOperand d_imag = mach_make_complex_imag(&dest); + + /* In PIC mode, save {r9, r12} BEFORE allocating the call frame so that + * SP-relative offsets within the 40-byte area remain correct when + * __divdc3 reads its stack args at [sp+0..23]. */ + if (text_and_data_separation) + ot_check(th_push((uint16_t)((1 << R9) | (1 << R12)))); + + /* Allocate 40 bytes (8-byte aligned). */ + ot_check(th_sub_imm(R_SP, R_SP, 40, flags_safe(), ENFORCE_ENCODING_NONE)); + + /* Set up __divdc3 stack args (must be at lowest sp offsets). */ + /* [sp+16] = b_im (src2 imag). */ + fp_mop_load_double_arg(R0, R1, &s2_imag); + fp_mop_save_double_to_sp(16); + /* [sp+8] = b_re (src2 real). */ + fp_mop_load_double_arg(R0, R1, &s2_real); + fp_mop_save_double_to_sp(8); + /* [sp+0] = a_im (src1 imag). */ + fp_mop_load_double_arg(R0, R1, &s1_imag); + fp_mop_save_double_to_sp(0); + + /* R2:R3 = a_re (src1 real) — first double arg in even register pair. */ + fp_mop_load_double_arg(R2, R3, &s1_real); + + /* R0 = pointer to result buffer at [sp+24]. */ + ot_check(th_add_imm(R0, R_SP, 24, flags_safe(), ENFORCE_ENCODING_NONE)); + + /* Call __divdc3 (directly, not via fp_mop_do_bl which would add another + * push/pop of {r9, r12} and corrupt the stack arg layout). */ + { + Sym *sym = external_global_sym(tok_alloc_const("__divdc3"), &func_old_type); + MachineOperand func_mop = {0}; + func_mop.kind = MACH_OP_SYMBOL; + func_mop.u.sym.sym = sym; + func_mop.u.sym.addend = 0; + gcall_or_jump_mop(0, func_mop); + } + + /* Read result from buffer and write back to dest. */ + fp_mop_load_double_from_sp(R0, R1, 24); + fp_mop_writeback_result(&d_real, 1); + fp_mop_load_double_from_sp(R0, R1, 32); + fp_mop_writeback_result(&d_imag, 1); + + ot_check(th_add_imm(R_SP, R_SP, 40, flags_safe(), ENFORCE_ENCODING_NONE)); + + if (text_and_data_separation) + ot_check(th_pop((uint16_t)((1 << R9) | (1 << R12)))); +} + +/* tcc_gen_machine_fp_mop: MachineOperand-based entry point for floating-point + * operations via soft-float EABI library calls. + * Handles single-precision, double-precision, and complex float operations. + * + * Soft-float EABI calling convention (single-precision): + * binary arithmetic: src1 → R0, src2 → R1, result ← R0 + * comparison: src1 → R0, src2 → R1, result ← CPSR flags + * negation: src1 → R0, XOR sign bit, result ← R0 + * conversion: src1 → R0, result ← R0 + * CVT_FTOF identity: float32→float32, src1 → R0, dest ← R0 + */ +ST_FUNC void tcc_gen_machine_fp_mop(MachineOperand src1, MachineOperand src2, MachineOperand dest, TccIrOp op, + int is_complex) +{ + /* Phase 5k: handle complex float operations via MOP path. */ + if (is_complex) + { + /* Detect double-precision complex: any operand has FLOAT64 btype. */ + const int complex_is_double = + (src1.btype == IROP_BTYPE_FLOAT64 || src2.btype == IROP_BTYPE_FLOAT64 || dest.btype == IROP_BTYPE_FLOAT64); + if (op == TCCIR_OP_FADD || op == TCCIR_OP_FSUB) + { + if (complex_is_double) + return thumb_process_complex_op_double_mop(src1, src2, dest, op == TCCIR_OP_FADD ? TCCIR_OP_ADD : TCCIR_OP_SUB); + return thumb_process_complex_op_mop(src1, src2, dest, op == TCCIR_OP_FADD ? TCCIR_OP_ADD : TCCIR_OP_SUB); + } + else if (op == TCCIR_OP_FMUL) + { + if (complex_is_double) + return thumb_process_complex_mul_double_mop(src1, src2, dest); + return thumb_process_complex_mul_mop(src1, src2, dest); + } + else if (op == TCCIR_OP_FDIV) + { + if (complex_is_double) + return thumb_process_complex_div_double_mop(src1, src2, dest); + return thumb_process_complex_div_mop(src1, src2, dest); + } + /* Other ops (FNEG, FCMP, CVT_*) on complex types: fall through to + * scalar path — they operate componentwise on the lo (real) half only, + * same as regular scalars. TODO: extend if needed. */ + } + + /* is_double: true when the primary operand is a 64-bit float (double). + * Note: complex float has is_64bit=true (register pair), but its btype + * is FLOAT32, so it is NOT double. Only FLOAT64 btype is true double. */ + const int is_double = (src1.btype == IROP_BTYPE_FLOAT64) || (dest.btype == IROP_BTYPE_FLOAT64); + const char *func_name = NULL; + + /* --- Inline double lowering, where the target has it --- + * + * Reaching here with a has_d* bit set means ir_put_soft_call_fpu_if_needed() + * deliberately did NOT rewrite this operation into an __aeabi_ call, so the + * backend owes an inline sequence. Falling through to the call path would + * still produce working code, but ir_op_is_implicit_call_ra() keys its + * clobber model on the same bits -- so the two must agree or the allocator + * stops modelling r0-r3 as dead across an operation that still calls. + * Enable a has_d* bit and its emitter in the same commit. */ + { + const FloatingPointConfig *fpu = architecture_config.fpu; + if (is_double && fpu && fpu->double_impl == FP_DOUBLE_IMPL_DCP) + { + if (op == TCCIR_OP_FADD && fpu->has_dadd) + return thumb_emit_dcp_addsub_mop(src1, src2, dest, 0); + if (op == TCCIR_OP_FSUB && fpu->has_dsub) + return thumb_emit_dcp_addsub_mop(src1, src2, dest, 1); + if (op == TCCIR_OP_FCMP && fpu->has_dcmp) + return thumb_emit_dcp_cmp_mop(src1, src2); + } + + /* Single precision is plain VFP wherever it is inline at all -- there is + * no second single-precision implementation on ARM to discriminate + * between, so has_f* alone selects the sequence. + * + * -mfloat-abi=soft is the one mode that forbids it: soft means "emit no + * FPU instructions at all", as distinct from softfp, which is FPU + * instructions with GPR argument passing (the default here, and what + * -mfpu=rp2350 keeps). has_f* describes the silicon, not the ABI, so the + * ABI has to be checked separately or `-mfloat-abi=soft` starts emitting + * vadd.f32. */ + if (!is_double && fpu && tcc_state && tcc_state->float_abi != ARM_SOFT_FLOAT) + { + if ((op == TCCIR_OP_FADD && fpu->has_fadd) || (op == TCCIR_OP_FSUB && fpu->has_fsub) || + (op == TCCIR_OP_FMUL && fpu->has_fmul) || (op == TCCIR_OP_FDIV && fpu->has_fdiv)) + return thumb_emit_vfp_arith_mop(src1, src2, dest, op); + } + } + + /* --- FNEG: XOR sign bit, no BL needed --- */ + if (op == TCCIR_OP_FNEG) + { + ScratchRegAlloc scr; + if (is_double) + { + /* f64: load pair into R0:R1, flip sign bit of hi word (R1) only */ + fp_mop_load_double_arg(R0, R1, &src1); + scr = get_scratch_reg_with_save((1u << R0) | (1u << R1)); + load_full_const(scr.reg, PREG_NONE, 0x80000000, 0); + ot_check(th_eor_reg(R1, R1, scr.reg, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); + restore_scratch_reg(&scr); + fp_mop_writeback_result(&dest, 1); + } + else + { + /* f32: R0 ^= 0x80000000 */ + fp_mop_load_arg(R0, &src1); + scr = get_scratch_reg_with_save(1u << R0); + load_full_const(scr.reg, PREG_NONE, 0x80000000, 0); + ot_check(th_eor_reg(R0, R0, scr.reg, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); + restore_scratch_reg(&scr); + mach_writeback_dest(&dest, R0); + } + return; + } + + /* --- CVT_FTOF: identity or f32<->f64 conversion via BL --- */ + if (op == TCCIR_OP_CVT_FTOF) + { + const int src_double = src1.is_64bit; + const int dst_double = dest.is_64bit; + if (!src_double && !dst_double) + { + /* f32 -> f32 identity: direct copy without going through R0 */ + tcc_gen_machine_assign_mop(src1, dest, TCCIR_OP_ASSIGN); + return; + } + if (src_double && dst_double) + { + /* f64 -> f64 identity: direct copy without going through R0:R1. + * Using R0:R1 as intermediaries would clobber live values in those + * registers (e.g. function parameters in soft-float ABI). */ + tcc_gen_machine_assign_mop(src1, dest, TCCIR_OP_ASSIGN); + return; + } + /* f32 -> f64: __aeabi_f2d; f64 -> f32: __aeabi_d2f */ + { + const char *cvt_func = src_double ? "__aeabi_d2f" : "__aeabi_f2d"; + if (src_double) + fp_mop_load_double_arg(R0, R1, &src1); + else + fp_mop_load_arg(R0, &src1); + fp_mop_do_bl(cvt_func); + fp_mop_writeback_result(&dest, dst_double); + } + return; + } + + /* --- Load operands into argument registers --- */ + if (op == TCCIR_OP_FCMP || op == TCCIR_OP_FADD || op == TCCIR_OP_FSUB || op == TCCIR_OP_FMUL || op == TCCIR_OP_FDIV) + { + /* Binary: src1 first arg, src2 second arg */ + if (is_double) + { + fp_mop_load_double_arg(R0, R1, &src1); + fp_mop_load_double_arg(R2, R3, &src2); + } + else + { + fp_mop_load_arg(R0, &src1); + fp_mop_load_arg(R1, &src2); + } + } + else + { + /* Unary conversion (CVT_ITOF, CVT_FTOI): load src1 into R0 or R0:R1 */ + if (src1.is_64bit) + fp_mop_load_double_arg(R0, R1, &src1); + else + fp_mop_load_arg(R0, &src1); + } + + /* --- Determine soft-float function name --- */ + if (op == TCCIR_OP_FCMP) + { + func_name = is_double ? "__aeabi_cdcmple" : "__aeabi_cfcmple"; + } + else if (op == TCCIR_OP_CVT_ITOF) + { + const int src64 = src1.is_64bit; + const int dst64 = dest.is_64bit; + if (src64 && dst64) + func_name = src1.is_unsigned ? "__aeabi_ul2d" : "__aeabi_l2d"; + else if (src64) + func_name = src1.is_unsigned ? "__aeabi_ul2f" : "__aeabi_l2f"; + else if (dst64) + func_name = src1.is_unsigned ? "__aeabi_ui2d" : "__aeabi_i2d"; + else + func_name = src1.is_unsigned ? "__aeabi_ui2f" : "__aeabi_i2f"; + } + else if (op == TCCIR_OP_CVT_FTOI) + { + const int src64 = src1.is_64bit; + const int dst64 = dest.is_64bit; + if (src64 && dst64) + func_name = dest.is_unsigned ? "__aeabi_d2ulz" : "__aeabi_d2lz"; + else if (src64) + func_name = dest.is_unsigned ? "__aeabi_d2uiz" : "__aeabi_d2iz"; + else if (dst64) + func_name = dest.is_unsigned ? "__aeabi_f2ulz" : "__aeabi_f2lz"; + else + func_name = dest.is_unsigned ? "__aeabi_f2uiz" : "__aeabi_f2iz"; + } + else + { + /* FADD, FSUB, FMUL, FDIV */ + func_name = get_softfp_func_name(op, is_double); + } + + if (!func_name) + tcc_error("compiler_error: tcc_gen_machine_fp_mop: no func_name for op %d", (int)op); + + fp_mop_do_bl(func_name); + + /* Write result back (FCMP sets CPSR flags only -- no register result) */ + if (op != TCCIR_OP_FCMP) + fp_mop_writeback_result(&dest, dest.is_64bit); +} diff --git a/source/backend/arch/arm/thumb/arm-thumb-frame.c b/source/backend/arch/arm/thumb/arm-thumb-frame.c new file mode 100644 index 000000000..190b947a2 --- /dev/null +++ b/source/backend/arch/arm/thumb/arm-thumb-frame.c @@ -0,0 +1,1283 @@ +/* + * ARMvX-m code generator for TCC + * Uses thumb instruction set + * + * Based on: + * ARM Thumb 2 instruction functions for TCC + * Copyright (c) 2020 Erlend J. Sveen + * from: + * https://git.erlendjs.no/erlendjs/tinycc/-/blob/arm-thumb/arm-thumb-gen.c + * https://git.erlendjs.no/erlendjs/tinycc/-/blob/arm-thumb/arm-thumb-instructions.c + * + * And + * + * ARMv4 code generator for TCC + * + * Copyright (c) 2003 Daniel Glöckner + * Copyright (c) 2012 Thomas Preud'homme + * + * Based on i386-gen.c by Fabrice Bellard + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +/* Thumb-2 code generator: return values, prologue and epilogue, LEA, stack + * stores and direct/indirect call emission. */ + +#include "arm-thumb-gen.h" + +/* tcc_gen_machine_return_value_mop: MachineOperand-based entry point for + * function return. Moves src into the return register(s) using the most + * efficient sequence available: + * 32-bit: src → R0 + * - Already R0 (common after ASSIGN peephole): NOP, 0 scratch. + * - IMM / SYMBOL: load directly into R0, 0 scratch. + * - REG / SPILL / FRAME_ADDR / PARAM_STACK: mach_ensure_in_reg + optional MOV. + * 64-bit: lo → R0 (REG_IRET), hi → R1 (REG_IRE2) + * - Delegates to tcc_gen_machine_assign_mop with a synthetic R0:R1 dest. + * - Safe ordering guaranteed: AAPCS ensures hi is never in R0. + */ +ST_FUNC void tcc_gen_machine_return_value_mop(MachineOperand src, TccIrOp op) +{ + (void)op; + + /* Hard-float single-precision return: the value goes in s0 (AAPCS VFP). */ + if (tcc_state && tcc_state->float_abi == ARM_HARD_FLOAT && !src.is_64bit && src.btype == IROP_BTYPE_FLOAT32) + { + if (src.kind == MACH_OP_VFP_REG) + { + if (src.u.reg.r0 != 0) + ot_check(th_vmov_register(0, (uint16_t)src.u.reg.r0, 0)); /* s0 = s_src */ + } + else + { + MachineCodegenContext ctx = {0}; + int r = mach_ensure_in_reg(&ctx, &src, 0); + ot_check(th_vmov_gp_sp((uint16_t)r, 0, 0)); /* s0 = r */ + mach_release_all(&ctx); + } + return; + } + + /* Hard-float double return: the value goes in d0 (AAPCS VFP). Doubles are + * not VFP-resident here, so materialize the GPR pair and pack it into d0. */ + if (tcc_state && tcc_state->float_abi == ARM_HARD_FLOAT && src.is_64bit && src.btype == IROP_BTYPE_FLOAT64 && + !src.is_complex) + { + MachineCodegenContext ctx = {0}; + MachineOperand lo = mach_make_lo_half(&src); + MachineOperand hi = mach_make_hi_half(&src); + lo.btype = IROP_BTYPE_INT32; + hi.btype = IROP_BTYPE_INT32; + int rlo = mach_ensure_in_reg(&ctx, &lo, 0); + int rhi = mach_ensure_in_reg(&ctx, &hi, (1u << (uint32_t)rlo)); + ot_check(th_vmov_2gp_dp((uint16_t)rlo, (uint16_t)rhi, 0, 0)); /* d0 = rlo,rhi */ + mach_release_all(&ctx); + return; + } + + /* 64-bit return: lo word → R0 (REG_IRET), hi word → R1 (REG_IRE2). + * AAPCS guarantees that for a 64-bit pair src.u.reg.r1 = src.u.reg.r0 + 1 ≥ R1, + * so hi is never in R0. Moving lo→R0 first is always safe. + * delegate to assign_mop which handles REG/SPILL/IMM/SYMBOL src kinds. */ + if (src.is_64bit) + { + MachineOperand ret_pair; + memset(&ret_pair, 0, sizeof(ret_pair)); + ret_pair.kind = MACH_OP_REG; + ret_pair.u.reg.r0 = TREG_R0; /* REG_IRET */ + ret_pair.u.reg.r1 = TREG_R1; /* REG_IRE2 */ + ret_pair.is_64bit = true; + ret_pair.btype = src.btype; + tcc_gen_machine_assign_mop(src, ret_pair, op); + return; + } + + /* Fast path: value already in R0 (ASSIGN peephole sets dest→R0) */ + if (src.kind == MACH_OP_REG && !src.needs_deref && src.u.reg.r0 == R0) + return; + + /* Immediate: materialize directly into R0 (no scratch register needed) */ + if (src.kind == MACH_OP_IMM) + { + tcc_machine_load_constant(R0, PREG_NONE, src.u.imm.val, 0, NULL); + return; + } + + /* Symbol: load address (+ optional deref) directly into R0 */ + if (src.kind == MACH_OP_SYMBOL) + { + Sym *sym = src.u.sym.sym ? validate_sym_for_reloc(src.u.sym.sym) : NULL; + if (!src.needs_deref) + { + tcc_machine_load_constant(R0, PREG_NONE, src.u.sym.addend, 0, sym); + } + else + { + tcc_machine_load_constant(R0, PREG_NONE, 0, 0, sym); + const int32_t addend = src.u.sym.addend; + load_from_base(R0, PREG_REG_NONE, src.btype, (int)src.is_unsigned, addend < 0 ? (int)(-addend) : (int)addend, + addend < 0 ? 1 : 0, (uint32_t)R0); + } + return; + } + + /* General case: materialize into any available register, then MOV to R0 */ + MachineCodegenContext ctx = {0}; + int src_reg = mach_ensure_in_reg(&ctx, &src, 0); + if (src_reg != R0) + ot_check_mov_reg(R0, src_reg, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE, false); + mach_release_all(&ctx); +} + +/* Does some call of the function reload R9 from its save slot afterwards -- + * the only reader of the slot the prologue stores it to: one through a + * pointer, a __builtin_apply, or a direct call to a callee not known to be in + * this module (restore_r9 at the call site). */ +ST_FUNC int tcc_gen_machine_calls_reload_r9(TCCIRState *ir) +{ + for (int i = 0; i < ir->next_instruction_index; i++) + { + IRQuadCompact *q = &ir->compact_instructions[i]; + if (q->op == TCCIR_OP_BUILTIN_APPLY) + return 1; + if (q->op != TCCIR_OP_FUNCCALLVAL && q->op != TCCIR_OP_FUNCCALLVOID) + continue; + IROperand fn = tcc_ir_op_get_src1(ir, q); + MachineOperand m = machine_op_from_ir(ir, &fn); + if (!thumb_callee_in_this_module(&m) && !thumb_callee_noreturn(&m)) + return 1; + } + return 0; +} + +/* Load the stack-passed parameter at parameter offset `off` into the core + * register(s) the allocator kept it in. An offset beyond the immediate forms + * is built in the destination itself: in the prologue no other register is + * known to be free, as any may hold a parameter. */ +static void prologue_load_stack_param(int lo, int hi, int off) +{ + const int base = tcc_state->need_frame_pointer ? R_FP : R_SP; + const int adj = param_frame_offset(off); + if (hi < 0) + { + if (load_word_from_base(lo, base, adj, 0)) + return; + if (!ot(th_generic_mov_imm(lo, adj))) + load_full_const(lo, PREG_NONE, LFC_SPLIT(adj)); + ot_check(th_ldr_reg(lo, base, lo, THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); + return; + } + if (adj + 4 <= 4095) + { + load_from_base(lo, hi, IROP_BTYPE_INT64, 0, adj, 0, base); + return; + } + if (!ot(th_generic_mov_imm(lo, adj))) + load_full_const(lo, PREG_NONE, LFC_SPLIT(adj)); + ot_check(th_add_reg(lo, base, lo, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); + ot_check(th_ldr_imm(hi, lo, 4, 6, ENFORCE_ENCODING_NONE)); + ot_check(th_ldr_imm(lo, lo, 0, 6, ENFORCE_ENCODING_NONE)); +} + +/* Whether the last prologue saved LR: the body may then BL. */ +static int prolog_saved_lr; +ST_FUNC int tcc_gen_machine_prolog_saved_lr(void) +{ + return prolog_saved_lr; +} + +/* -Os outliner: a window the real pass calls instead begins. Nothing pairs + * across its edges: the rehearsal would have put such a pair's halves in two + * ops, and neither can be in a window. */ +ST_FUNC void tcc_gen_machine_outline_window_open(void) +{ + frame_word_last.kind = 0; +} + +/* The window's ops are done, and `bytes` of them were recorded, not emitted: + * the BL to the shared body replaces them. */ +ST_FUNC void tcc_gen_machine_outline_window_closed(int bytes) +{ + thumb_gen_state.code_size -= bytes; + frame_word_last.kind = 0; +} + +/* The BL to a shared body. Not through ot(): the caches saw the window's own + * instructions, which is what the body runs, not a call. Same pool check as + * ot(). */ +ST_FUNC void tcc_gen_machine_outline_emit_bl(int esym) +{ + thumb_gen_state.code_size += 4; + if (th_pool_span_after(4) >= THUMB_POOL_RANGE_LIMIT - THUMB_POOL_FLUSH_SLACK) + th_literal_pool_generate(); + int pos = ind; + thumb_opcode bl = th_bl_t1((uint32_t)-4); + o(bl.opcode >> 16); + o(bl.opcode & 0xffff); + put_elf_reloc(symtab_section, cur_text_section, pos, R_ARM_THM_JUMP24, esym); +} + +/* One halfword of a window put back where it was generated (already counted + * in code_size when ot() recorded it). */ +ST_FUNC void tcc_gen_machine_outline_emit_raw(uint16_t hw) +{ + o(hw); +} + +ST_FUNC void tcc_gen_machine_prolog(int leaffunc, uint64_t used_registers, int stack_size, uint32_t extra_prologue_regs) +{ + thumb_gen_state.function_argument_count = 0; + /* call_id -1 is reserved for function prolog metadata - but that doesn't + * need to be stored in call_sites_by_id (which uses non-negative IDs). + * If needed, handle it separately or skip. */ + + thumb_gen_state.generating_function = 1; + thumb_gen_state.code_size = 0; + /* Clear global symbol cache at function start */ + thumb_gen_state.cached_global_sym = NULL; + thumb_gen_state.cached_global_reg = PREG_NONE; + /* MOV-coalescing cache is per-function: register live ranges don't + * cross function boundaries. */ + mov_equiv_reset_all(); + pool_flush_it_pending = 0; + TCCIRState *ir = tcc_state->ir; + + /* Determine if LR needs saving */ + int save_lr = !leaffunc || tcc_state->force_lr_save; + if (extra_prologue_regs & (1u << R_LR)) + save_lr = 1; + prolog_saved_lr = save_lr; + + /* Variadic functions need a stable FP for va_list setup. */ + if (func_var) + tcc_state->need_frame_pointer = 1; + + const int need_fp = (tcc_state->force_frame_pointer || tcc_state->need_frame_pointer); + tcc_state->need_frame_pointer = need_fp; + + /* The allocator hands out R7 only when ra_may_need_frame_pointer predicted + * no frame pointer; a function that reaches here needing FP with R7 + * allocated means the prediction missed a forcing condition, and silently + * continuing would use one register as frame base and value at once. */ + if (need_fp && (used_registers & (1ULL << R_FP))) + tcc_error("compiler_error: R7 allocated in a frame-pointer function " + "(ra_may_need_frame_pointer out of sync with a forcing site)"); + + /* Collect callee-saved registers */ + uint16_t callee_regs_local = 0; + int callee_count = 0; + for (int i = R4; i <= R11; ++i) + { + if (tcc_state->text_and_data_separation && i == R9) + continue; + if (i == R_FP && need_fp) + continue; /* r7 is the frame base; pushed below with LR */ + if (used_registers & (1ULL << i)) + { + callee_regs_local |= (1 << i); + callee_count++; + } + } + + /* Registers codegen asks to be saved on top of the allocation: the static + * chain register (R10) for nested functions, and the rodata anchor. */ + for (int i = R4; i <= R11; ++i) + { + if (!(extra_prologue_regs & (1u << i))) + continue; + if (tcc_state->text_and_data_separation && i == R9) + continue; + if (i == R_FP && need_fp) + continue; + if (callee_regs_local & (1u << i)) + continue; + callee_regs_local |= (1u << i); + callee_count++; + } + + int push_align_pad = 0; /* 4 if push count is odd, absorbed into SUB SP */ + + uint16_t registers_to_push = callee_regs_local; + int registers_count = callee_count; + + if (save_lr) + { + registers_to_push |= (1 << R_LR); + registers_count++; + } + if (need_fp) + { + registers_to_push |= (1 << R_FP); + registers_count++; + } + + /* Keep the total push size 8-byte aligned (AAPCS). + * When there are no locals (stack_size == 0), pad by pushing a dummy low + * register (R3) — avoids SUB SP + ADD SP, saving 2 insns. When locals + * exist, absorb the gap into SUB SP instead, keeping PUSH in 16-bit + * encoding (no high regs like R12). */ + if (registers_count % 2 != 0) + { + if (stack_size == 0) + { + registers_to_push |= (1 << R3); + registers_count++; + } + else + { + push_align_pad = 4; + } + } + + map_sym_t(); + + /* Variadic, or a parameter straddling r3 and the stack: push r0-r3 FIRST + * so they are contiguous with the stack args. */ + vararg_push_size = 0; + if (func_var || (ir && ir->push_arg_regs)) + { + ot_check(th_push((1 << R0) | (1 << R1) | (1 << R2) | (1 << R3))); + vararg_push_size = 16; + } + + offset_to_args = registers_count * 4 + (func_var ? vararg_push_size : 0); + + if (registers_count > 0) + ot_check(th_push(registers_to_push)); + + pushed_registers = registers_to_push; + + // allocate stack space for local variables + /* Variadic save area is reserved in the IR stack layout (loc bias). */ + + /* Keep SP 8-byte aligned (AAPCS). tccir normally pre-aligns stack_size, but + * be defensive here because other codepaths may call into the backend. + */ + if (stack_size & 7) + stack_size = (stack_size + 7) & ~7; + + /* allocated_stack_size is the portion of the stack used for locals/spills. + * It must NOT include the alignment pad — local offsets are computed + * relative to this value, and the pad sits below all addressable locals. */ + allocated_stack_size = stack_size; + + /* total_stack_dealloc is the full amount to restore in the epilogue, + * including the alignment pad that sits below addressable locals. */ + int total_stack_dealloc = stack_size + push_align_pad; + epilogue_stack_dealloc = total_stack_dealloc; + stack_size = total_stack_dealloc; + if (stack_size > 0) + { + gadd_sp(-stack_size); + } + + /* The frame pointer is the SP left here, the bottom of the frame: locals + * and incoming parameters sit at the same positive offsets from it as from + * SP (see fp_adjust_local_offset). SP moves only for VLA/alloca, and for + * the soft-float helpers' and scratch pushes' balanced windows. */ + if (need_fp) + { + if (!ot(th_add_imm(R_FP, R_SP, 0, flags_safe(), ENFORCE_ENCODING_NONE))) + { + fprintf(stderr, "compiler_error: prolog frame pointer setup failed\n"); + exit(1); + } + } + + /* SP (and the frame pointer) is lower by the full SUB SP amount (locals + + * alignment pad): incoming stack parameters are that much further up. + * Local addressing uses allocated_stack_size (without pad): the pad sits at + * the top of the SUB SP region, right below the pushed registers, so + * locals occupy base+0 .. base+allocated_stack_size-1. */ + offset_to_args += epilogue_stack_dealloc; + + /* Everything from here to the parameter shuffle below runs before a single + * incoming value has been moved to its allocated home, so these registers + * are all still live and none of them may be taken as a scratch register: + * R0-R3 (the argument registers -- all four, not just the named ones, since + * a variadic function saves the anonymous ones too), R9 (the GOT base under + * text_and_data_separation) and the static chain register. + * + * A store whose offset does not fit the STR immediate needs a register to + * hold the offset, and the scratch picker's idea of "free" comes from the + * live intervals, which do not model the incoming registers at prologue + * position. It therefore handed out R0. In a function with alloca and a + * frame deeper than the 255-byte negative STR immediate -- glob() and + * fnmatch() in GNU make's gnulib, among others -- the R9 spill below then + * emitted + * movw r0,#0x131c ; rsb r0,r0,#0 ; str.w r9,[r7,r0] + * movw r4,#0x1018 ; rsb r4,r4,#0 ; str r0,[r7,r4] + * and the first parameter, which the second store was meant to spill, was + * the offset constant by the time it got there. */ + uint32_t prologue_incoming_mask = (1u << R0) | (1u << R1) | (1u << R2) | (1u << R3); + if (tcc_state->text_and_data_separation) + prologue_incoming_mask |= (1u << ARM_R9); + if (ir && ir->has_static_chain) + prologue_incoming_mask |= (1u << architecture_config.static_chain_reg); + + /* Save the PIC GOT base (R9) once, into slot 0 of the nested-call save area. + * + * R9 is caller-saved under text_and_data_separation, so every call reloads + * it from the frame; but the value never changes within a function, so the + * store only has to happen once. It used to be emitted at each call site + * next to the reload, which cost 45,329 instructions (165 KiB) in the + * compiler's own build and produced runs of literally + * ldr.w r9, [sp] ; str.w r9, [sp] + * where the reload was immediately followed by writing the same value back. + * + * call_nested_save_size is non-zero exactly when the function has at least + * one call under text_and_data_separation (ir/codegen.c, where the area is + * sized), so it is the same guard in both codegen passes — and R9 is saved + * at every one of those call sites, so slot 0 is always reserved for it. */ + if (tcc_state->text_and_data_separation && ir && ir->call_nested_save_size > 0 && tcc_gen_machine_calls_reload_r9(ir)) + { + const int r9_slot_offset = ir->call_outgoing_size; + /* SP-relative, like the store_word_to_stack() of the call sites. A + * VLA/alloca function addresses the slots off the frame pointer instead, + * at the same offset: here, before any dynamic allocation, both are the + * bottom of the frame. */ + tcc_gen_machine_store_to_sp_ex(ARM_R9, r9_slot_offset, prologue_incoming_mask); + } + + /* Save incoming static chain (R10) at its fixed slot, frame offset -4 + * (CHAIN_SLOT_OFFSET), where the body and tcc_gen_machine_restore_chain + * read it back. */ + if (ir && ir->has_static_chain) + { + tcc_gen_machine_store_to_stack_ex(architecture_config.static_chain_reg, fp_adjust_local_offset(-4, 0), + prologue_incoming_mask); + } + + /* For variadic functions, save incoming r0-r3 in a fixed area at frame + * offsets -16..-4 (for named parameter access). + * The PUSH {r0-r3} at function entry already creates a contiguous register + * save area above the callee-saved pushes, adjacent to the stack arguments. + * __gr_top points to the end of that area (= start of stack args). + */ + if (func_var) + { + /* Store r0-r3 at frame offsets -16..-4 for named parameter access. + * (The contiguous PUSH'd copy is at FP+offset_to_args-16..FP+offset_to_args-4 + * and is used by va_arg for anonymous argument traversal.) */ + tcc_gen_machine_store_to_stack_ex(R0, fp_adjust_local_offset(-16, 0), prologue_incoming_mask); + tcc_gen_machine_store_to_stack_ex(R1, fp_adjust_local_offset(-12, 0), prologue_incoming_mask); + tcc_gen_machine_store_to_stack_ex(R2, fp_adjust_local_offset(-8, 0), prologue_incoming_mask); + tcc_gen_machine_store_to_stack_ex(R3, fp_adjust_local_offset(-4, 0), prologue_incoming_mask); + + /* The frame-metadata triple that used to live at FP-20/-24/-28 (__gr_top, + * named_arg_reg_bytes, named_arg_stack_bytes) is gone: it existed purely so + * the runtime __tcc_va_start could rediscover the address of the first + * anonymous argument. TOK_builtin_va_start now materializes that address + * directly (a PARAM-relative LEA), so those six prologue instructions were + * dead in every variadic function. */ + } + + /* __builtin_apply_args: save incoming r0-r3 and stack args pointer + * to the reserved apply_args block so __builtin_apply can replay them. + * Layout at apply_args_offset: [stack_args_ptr, r0, r1, r2, r3]. */ + if (tcc_state->func_save_apply_args && ir) + { + int adj = fp_adjust_local_offset(tcc_state->apply_args_offset, 0); + + /* Store stack args pointer (FP + offset_to_args = start of stack args area) */ + { + const int fp_or_sp = tcc_state->need_frame_pointer ? R_FP : R_SP; + ot_check(th_add_imm(R_IP, fp_or_sp, offset_to_args, flags_safe(), ENFORCE_ENCODING_NONE)); + } + tcc_gen_machine_store_to_stack_ex(R_IP, adj, prologue_incoming_mask); + + /* Store r0-r3 at offsets +4, +8, +12, +16 from the block start */ + tcc_gen_machine_store_to_stack_ex(R0, adj + 4, prologue_incoming_mask); + tcc_gen_machine_store_to_stack_ex(R1, adj + 8, prologue_incoming_mask); + tcc_gen_machine_store_to_stack_ex(R2, adj + 12, prologue_incoming_mask); + tcc_gen_machine_store_to_stack_ex(R3, adj + 16, prologue_incoming_mask); + } + + /* Move parameters from incoming registers to their allocated locations. + * For non-leaf functions or parameters that cross calls: + * - If allocated to callee-saved register: move from R0-R3 to allocated reg + * - If spilled: store from R0-R3 to stack location + * For leaf functions with params staying in R0-R3: no move needed */ + if (ir) + { + /* Parameter shuffling must not clobber still-needed incoming registers. + * Example (4 args): if z is assigned to R3 and w is assigned to R6, a naive + * sequence "mov r3,r2; mov r6,r3" destroys w. + * + * Strategy: + * 1) Handle register-passed params first: store spills, collect reg->reg moves. + * 2) Execute reg->reg moves as a parallel move with cycle breaking. + * 3) Then load stack-passed params into their allocated registers. + */ + + /* Stack-passed parameters the allocator kept in core registers + * (tcc_ir_avoid_spilling_stack_passed_params). Loaded last: a load may + * target a register an incoming argument still occupies until step 2. */ + typedef struct StackParamLoad + { + int lo, hi, off; + } StackParamLoad; + StackParamLoad *stack_loads = tcc_malloc(sizeof(StackParamLoad) * (ir->next_parameter + 1)); + int stack_load_count = 0; + + typedef struct ParamMove + { + int dst; + int src; + } ParamMove; + + /* NOTE: Do not hard-code small fixed arrays here. + * Functions can legally have >32 parameters (e.g. sum40 in tests), and + * overflowing these buffers corrupts prolog codegen and breaks calls. + * Worst-case: a 64-bit param can contribute up to 2 reg moves. + */ + const int max_param_moves = ir->next_parameter * 2 + 8; + ParamMove *moves = tcc_malloc(sizeof(ParamMove) * max_param_moves); + int move_count = 0; + + /* Hard-float double parameters arrive in d0-d7 but are not VFP-resident, so + * each is unpacked into its allocated GPR pair. Collected here and emitted + * after the GPR parallel move: the unpack overwrites GPRs that may still + * hold incoming GPR arguments. */ + typedef struct DoubleUnpack + { + int dreg, lo, hi; + } DoubleUnpack; + DoubleUnpack dbl_unpack[8]; + int dbl_unpack_count = 0; + + /* Build a bitmask of ALL incoming argument registers (R0-R3) that need + * to be saved. When storing a spilled parameter to the stack at a large + * offset, the scratch register allocator must NOT pick any of these + * registers — they still hold incoming parameter values. + * + * Without this mask, storing e.g. R0 to [FP-1028] can use R1 as scratch + * for the offset constant, destroying the parameter value in R1. + */ + uint32_t incoming_arg_regs_mask = 0; + for (int vreg = 0; vreg < ir->next_parameter; ++vreg) + { + const int encoded_vreg = (TCCIR_VREG_TYPE_PARAM << 28) | vreg; + IRLiveInterval *interval = tcc_ir_get_live_interval(ir, encoded_vreg); + if (!interval) + continue; + if (interval->incoming_reg0 >= 0 && interval->incoming_reg0 < 16) + incoming_arg_regs_mask |= (1u << interval->incoming_reg0); + if (interval->incoming_reg1 >= 0 && interval->incoming_reg1 < 16) + incoming_arg_regs_mask |= (1u << interval->incoming_reg1); + } + + for (int vreg = 0; vreg < ir->next_parameter; ++vreg) + { + const int encoded_vreg = (TCCIR_VREG_TYPE_PARAM << 28) | vreg; + IRLiveInterval *interval = tcc_ir_get_live_interval(ir, encoded_vreg); + + if (!interval) + continue; + + const int incoming_r0 = interval->incoming_reg0; + const int incoming_r1 = interval->incoming_reg1; + const int alloc_r0 = interval->allocation.r0; + const int alloc_r1 = interval->allocation.r1; + const int is_64bit = interval->is_double || interval->is_llong; + + if (incoming_r0 < 0) + { + if (interval->allocation.offset == 0 && alloc_r0 <= R12) + { + stack_loads[stack_load_count++] = + (StackParamLoad){.lo = alloc_r0, .hi = is_64bit ? alloc_r1 : -1, .off = interval->original_offset}; + continue; + } + /* Captured by a nested function: the caller's argument area is not a + * home the child can name, so this parameter was given a slot in OUR + * frame. Copy the incoming value into it — the child reads the slot + * chain-relative, and every parent access goes through it too, which + * is what keeps the by-reference semantics. */ + if (interval->nested_home && interval->allocation.offset != 0) + { + const int dst = fp_adjust_local_offset(interval->allocation.offset, 0); + ScratchRegAlloc sc = get_scratch_reg_with_save(incoming_arg_regs_mask); + prologue_load_stack_param(sc.reg, -1, interval->original_offset); + tcc_gen_machine_store_to_stack_ex(sc.reg, dst, incoming_arg_regs_mask); + if (is_64bit) + { + prologue_load_stack_param(sc.reg, -1, interval->original_offset + 4); + tcc_gen_machine_store_to_stack_ex(sc.reg, dst + 4, incoming_arg_regs_mask); + } + restore_scratch_reg(&sc); + continue; + } + /* Otherwise it lives in the caller's argument area. Leave the + * allocation empty so IR materialization treats it as a VT_PARAM + * lvalue and loads it where it is used. */ + interval->allocation.r0 = PREG_NONE; + interval->allocation.r1 = PREG_NONE; + interval->allocation.offset = 0; + continue; + } + + /* Stack-home parameters: store incoming regs to their stack slots. + * IR owns spills; avoid inspecting PREG_SPILLED sentinels here. + * Use the incoming_arg_regs_mask to prevent the scratch allocator + * from clobbering other incoming argument registers. + */ + if (interval->allocation.offset != 0) + { + /* Adjust for callee-saved gap below FP in two-phase push. */ + const int stack_offset = fp_adjust_local_offset(interval->allocation.offset, 0); + /* Float/double param arriving in a VFP register but spilled to the + * stack: bridge it through GPR scratch to reuse the ordinary store. A + * double occupies d and needs both halves written. */ + if (is_vfp_reg(incoming_r0)) + { + ScratchRegAlloc sc = get_scratch_reg_with_save(incoming_arg_regs_mask); + if (is_64bit) + { + ScratchRegAlloc sc2 = get_scratch_reg_with_save(incoming_arg_regs_mask | (1u << (uint32_t)sc.reg)); + ot_check(th_vmov_2gp_dp((uint16_t)sc.reg, (uint16_t)sc2.reg, + (uint16_t)(vfp_num(incoming_r0) / 2), 1)); /* lo,hi = d */ + tcc_gen_machine_store_to_stack_ex(sc.reg, stack_offset, incoming_arg_regs_mask); + tcc_gen_machine_store_to_stack_ex(sc2.reg, stack_offset + 4, incoming_arg_regs_mask); + restore_scratch_reg(&sc2); + } + else + { + ot_check(th_vmov_gp_sp((uint16_t)sc.reg, (uint16_t)vfp_num(incoming_r0), 1)); /* gpr = s_incoming */ + tcc_gen_machine_store_to_stack_ex(sc.reg, stack_offset, incoming_arg_regs_mask); + } + restore_scratch_reg(&sc); + continue; + } + if (is_64bit && incoming_r1 >= 0) + { + tcc_gen_machine_store_to_stack_ex(incoming_r0, stack_offset, incoming_arg_regs_mask); + /* R0 is now saved; remove it from the protection mask */ + incoming_arg_regs_mask &= ~(1u << incoming_r0); + tcc_gen_machine_store_to_stack_ex(incoming_r1, stack_offset + 4, incoming_arg_regs_mask); + incoming_arg_regs_mask &= ~(1u << incoming_r1); + } + else + { + tcc_gen_machine_store_to_stack_ex(incoming_r0, stack_offset, incoming_arg_regs_mask); + incoming_arg_regs_mask &= ~(1u << incoming_r0); + } + continue; + } + + /* Double parameter arriving in a VFP register (hard-float): it is not + * VFP-resident — there is no double-precision arithmetic here — so unpack + * d into the GPR pair the allocator gave it. Emitted before the GPR + * parallel move, whose sources are all still live. */ + if (is_64bit && is_vfp_reg(incoming_r0) && alloc_r0 != PREG_NONE && alloc_r0 >= 0 && alloc_r1 >= 0) + { + /* Deferred: the unpack writes GPRs that may still hold incoming GPR + * arguments (a leading int parameter sits in r0, and d0 commonly unpacks + * into r0:r1), so it must run only after the GPR parallel move below has + * consumed them. */ + if (dbl_unpack_count < (int)(sizeof(dbl_unpack) / sizeof(dbl_unpack[0]))) + dbl_unpack[dbl_unpack_count++] = + (DoubleUnpack){.dreg = vfp_num(incoming_r0) / 2, .lo = alloc_r0, .hi = alloc_r1}; + continue; + } + + /* Register-allocated parameters: record reg->reg moves (parallel move). + * + * IMPORTANT: Do NOT remove incoming_r0/r1 from incoming_arg_regs_mask + * here! The reg->reg moves are only COLLECTED now and executed LATER + * as a parallel move. The incoming register still holds the live + * parameter value at this point. If we removed it from the mask, a + * subsequent spill-store for another parameter could use it as scratch + * and destroy the value before the parallel move consumes it. + */ + if (alloc_r0 != PREG_NONE && is_vfp_reg(alloc_r0)) + { + /* Float parameter homed to a VFP register. It arrives either in a VFP + * register (hard-float argument passing) or in a GPR (soft layout); + * either way move it into its VFP home now, while the incoming register + * is still live. VFP destinations never participate in the GPR parallel + * move (different register file), so emitting here is safe. */ + if (is_vfp_reg(incoming_r0)) + { + if (incoming_r0 != alloc_r0) + ot_check(th_vmov_register((uint16_t)vfp_num(alloc_r0), (uint16_t)vfp_num(incoming_r0), 0)); + } + else + { + ot_check(th_vmov_gp_sp((uint16_t)incoming_r0, (uint16_t)vfp_num(alloc_r0), 0)); + } + continue; + } + if (alloc_r0 != PREG_NONE && alloc_r0 >= 0 && alloc_r0 <= R12 && alloc_r0 != incoming_r0) + { + moves[move_count++] = (ParamMove){.dst = alloc_r0, .src = incoming_r0}; + } + if (is_64bit && incoming_r1 >= 0 && alloc_r1 != PREG_NONE && alloc_r1 >= 0 && alloc_r1 <= R12 && + alloc_r1 != incoming_r1) + { + moves[move_count++] = (ParamMove){.dst = alloc_r1, .src = incoming_r1}; + } + } + + /* Execute collected register moves as a true parallel move. + * + * - Emit any move whose source is not a destination. + * - If only cycles remain, rotate a cycle using a temporary register. + * + * This is required for correct swaps like (r1<-r2, r2<-r1) without + * clobbering one of the incoming argument registers. + */ + while (move_count > 0) + { + uint32_t dst_mask = 0; + uint32_t src_mask = 0; + for (int i = 0; i < move_count; ++i) + { + if (moves[i].dst >= 0 && moves[i].dst < 32) + dst_mask |= (1u << moves[i].dst); + if (moves[i].src >= 0 && moves[i].src < 32) + src_mask |= (1u << moves[i].src); + } + + /* First: emit all acyclic moves. + * + * A move is safe to emit if its destination is not used as a source by + * any remaining move. This prevents clobbering values needed later. + * + * Example chain that must be ordered correctly: + * r1 <- r2 + * r2 <- r3 + * Here r2 is both a source and a destination. We must emit r1<-r2 first. + */ + int progressed = 0; + for (int i = 0; i < move_count; ++i) + { + const int dst = moves[i].dst; + const int src = moves[i].src; + + if (dst == src) + { + moves[i] = moves[--move_count]; + --i; + progressed = 1; + continue; + } + + if (dst >= 0 && dst < 32 && (src_mask & (1u << dst))) + continue; /* dst's current value is still needed as a source somewhere */ + + ot_check_mov_reg(dst, src, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE, false); + moves[i] = moves[--move_count]; + --i; + progressed = 1; + } + if (progressed) + continue; + + /* Cycle: rotate it using a temporary register (prefer IP). */ + int temp = R_IP; + if (dst_mask & (1u << temp)) + { + for (int r = R4; r <= R11; ++r) + { + if (!(dst_mask & (1u << r))) + { + temp = r; + break; + } + } + } + if (dst_mask & (1u << temp)) + { + tcc_error("compiler_error: prolog param shuffle has no temp register"); + } + + /* Pick any destination in the remaining cycle, save its original value, + * then walk dst<-src edges until we return to the start. + */ + const int start = moves[0].dst; + ot_check_mov_reg(temp, start, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE, false); + + int cur = start; + for (;;) + { + int idx = -1; + for (int i = 0; i < move_count; ++i) + { + if (moves[i].dst == cur) + { + idx = i; + break; + } + } + if (idx < 0) + { + tcc_error("compiler_error: broken prolog param shuffle cycle"); + } + + const int src = moves[idx].src; + moves[idx] = moves[--move_count]; + + if (src == start) + { + ot_check_mov_reg(cur, temp, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE, false); + break; + } + + ot_check_mov_reg(cur, src, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE, false); + cur = src; + } + } + + /* Now that every incoming GPR argument has reached its home, it is safe to + * overwrite the argument registers: unpack the hard-float doubles from + * d0-d7 into their allocated GPR pairs. */ + for (int i = 0; i < dbl_unpack_count; ++i) + ot_check(th_vmov_2gp_dp((uint16_t)dbl_unpack[i].lo, (uint16_t)dbl_unpack[i].hi, + (uint16_t)dbl_unpack[i].dreg, 1)); + + for (int i = 0; i < stack_load_count; ++i) + prologue_load_stack_param(stack_loads[i].lo, stack_loads[i].hi, stack_loads[i].off); + + tcc_free(stack_loads); + tcc_free(moves); + } + + /* Materialise the .rodata base for the body. Last in the prologue: the + * parameter shuffle above may borrow any callee-saved register that holds + * no incoming argument, which includes this one until it is loaded. R9 is + * the incoming GOT base and nothing in the prologue writes it. */ + if (rodata_anchor_reg >= 0) + ot_check_ldr_imm(rodata_anchor_reg, R9, YAFF_RODATA_ANCHOR_GOT_OFFSET, 6, ENFORCE_ENCODING_NONE); +} + +/* Return through the saved LR while dropping the r0-r3 pushed below the saved + * registers (variadic, push_arg_regs): pop the rest, then one post-indexed LDR + * loads LR's slot into PC and steps SP past it and the argument area. Two + * bytes shorter than `pop {.., lr}; add sp; bx lr`, as a 16-bit POP cannot + * name LR. */ +static void epilogue_return_over_pushed_args(uint32_t saved_with_lr) +{ + uint32_t rest = saved_with_lr & ~(1u << R_LR); + if (rest) + ot_check(th_pop(rest)); + ot_check(th_ldr_imm(R_PC, R_SP, 4 + vararg_push_size, 3 /* post-indexed, add, writeback */, + ENFORCE_ENCODING_32BIT)); +} + +ST_FUNC void tcc_gen_machine_epilog(int leaffunc) +{ + TRACE("'tcc_gen_machine_epilog'"); + + int lr_saved = pushed_registers & (1 << R_LR); + + /* Where SP may have moved at run time -- VLA, alloca, a call to the + * library alloca() -- restore it from the frame pointer, which is the SP + * the prologue left; then leave as a frame without one does (the pushed + * registers include r7). */ + if (tcc_state->need_frame_pointer && (tcc_state->func_dynamic_sp || tcc_state->force_frame_pointer)) + ot_check_mov_reg(R_SP, R_FP, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE, false); + if (epilogue_stack_dealloc > 0) + gadd_sp_ex(epilogue_stack_dealloc, R3); + if (vararg_push_size > 0 && lr_saved) + { + /* r0-r3 were pushed below the stack arguments (push_arg_regs). */ + epilogue_return_over_pushed_args(pushed_registers); + } + else if (vararg_push_size > 0) + { + if (pushed_registers > 0) + ot_check(th_pop(pushed_registers)); + gadd_sp_ex(vararg_push_size, R3); + ot_check(th_bx_reg(R_LR)); + } + else if (lr_saved) + { + pushed_registers |= 1 << R_PC; + pushed_registers &= ~(1 << R_LR); + ot_check(th_pop(pushed_registers)); + } + else + { + if (pushed_registers > 0) + ot_check(th_pop(pushed_registers)); + ot_check(th_bx_reg(R_LR)); + } + + thumb_gen_state.generating_function = 0; + /* The anchor register holds this function's .rodata base and the epilogue + * has just restored the caller's value in it. */ + rodata_anchor_reg = -1; + th_literal_pool_generate(); + thumb_free_call_sites(); +} + +ST_FUNC void tcc_gen_machine_finish_noreturn(void) +{ + thumb_gen_state.generating_function = 0; + rodata_anchor_reg = -1; + th_literal_pool_generate(); + thumb_free_call_sites(); +} + +/* Load Effective Address: compute the address of src1 into dest. + * This is the explicit "address-of" operation for local variables/arrays. + * Unlike LOAD which dereferences, LEA computes FP+offset into a register. + */ + +/* MachineOperand-based LEA. Computes the address of the source operand + * into the destination. + * + * Most operand kinds are handled directly by mach_ensure_in_reg: + * MACH_OP_FRAME_ADDR → ADD dest, FP, #offset (local variable address) + * MACH_OP_SYMBOL → LDR dest, =symbol (global variable address) + * MACH_OP_REG → MOV dest, src_reg (address already computed) + * + * PARAM_STACK and CHAIN_REL need special handling because mach_ensure_in_reg + * always loads the VALUE from the stack. For LEA, we need the ADDRESS instead: + * MACH_OP_PARAM_STACK → ADD dest, FP, #(offset + offset_to_args) + * MACH_OP_CHAIN_REL → ADD/SUB dest, chain_base, #offset + */ +ST_FUNC void tcc_gen_machine_lea_mop(MachineOperand dest, MachineOperand src) +{ + MachineCodegenContext ctx = {0}; + int r; + + /* When dest is a writable register, compute the address directly into it + * to avoid a scratch + redundant mov. mach_alloc_scratch is driven by the + * per-instruction live_regs bitmap which already marks the dest reg "live" + * (it's the def target), so it would otherwise pick a different reg and + * writeback would emit `mov dest_reg, scratch`. */ + int dest_reg = -1; + if (dest.kind == MACH_OP_REG && !dest.needs_deref && + dest.u.reg.r0 != (int)PREG_REG_NONE) + dest_reg = dest.u.reg.r0; + + switch (src.kind) + { + case MACH_OP_PARAM_STACK: + { + /* Compute address of caller's argument slot. */ + r = (dest_reg >= 0) ? dest_reg : mach_alloc_scratch(&ctx, 0); + tcc_machine_addr_of_stack_slot(r, src.u.param.offset, 1 /* is_param */); + break; + } + case MACH_OP_CHAIN_REL: + { + /* Compute address in parent frame via static chain. */ + ScratchRegAlloc chain_scratch = {0}; + int chain_used = 0; + uint32_t excl = 0; + int base = resolve_chain_base(tcc_state->ir, src.u.chain.chain_index, excl, &chain_scratch, &chain_used); + /* dest_reg only usable if it doesn't collide with the chain base */ + if (dest_reg >= 0 && dest_reg != base && !(excl & (1u << (uint32_t)dest_reg))) + r = dest_reg; + else + r = mach_alloc_scratch(&ctx, excl | (1u << (uint32_t)base)); + int32_t off = src.u.chain.offset; + int sign = (off < 0); + int abs_off = sign ? (int)(-off) : (int)off; + if (abs_off == 0) + { + if (r != base) + ot_check_mov_reg((uint32_t)r, (uint32_t)base, flags_safe(), THUMB_SHIFT_DEFAULT, + ENFORCE_ENCODING_NONE, false); + } + else + { + thumb_opcode ins = sign ? th_sub_imm(r, base, abs_off, flags_safe(), ENFORCE_ENCODING_NONE) + : th_add_imm(r, base, abs_off, flags_safe(), ENFORCE_ENCODING_NONE); + if (ins.size != 0) + { + ot_check(ins); + } + else + { + /* Large offset: load into a scratch and use register ADD/SUB */ + ScratchRegAlloc off_sc = get_scratch_reg_with_save(excl | (1u << (uint32_t)r) | (1u << (uint32_t)base)); + load_full_const(off_sc.reg, PREG_NONE, LFC_SPLIT(abs_off)); + ot_check(sign ? th_sub_reg(r, base, off_sc.reg, flags_safe(), THUMB_SHIFT_DEFAULT, + ENFORCE_ENCODING_NONE) + : th_add_reg(r, base, off_sc.reg, flags_safe(), THUMB_SHIFT_DEFAULT, + ENFORCE_ENCODING_NONE)); + restore_scratch_reg(&off_sc); + } + } + if (chain_used) + restore_scratch_reg(&chain_scratch); + break; + } + case MACH_OP_FRAME_ADDR: + /* Address of a local stack slot: ADD dest, sp, #offset. */ + r = (dest_reg >= 0) ? dest_reg : mach_alloc_scratch(&ctx, 0); + tcc_machine_addr_of_stack_slot(r, src.u.frame.offset, 0); + break; + case MACH_OP_SYMBOL: + if (!src.needs_deref) + { + Sym *raw_sym = src.u.sym.sym; + Sym *sym = raw_sym ? validate_sym_for_reloc(raw_sym) : NULL; + r = (dest_reg >= 0) ? dest_reg : mach_alloc_scratch(&ctx, 0); + tcc_machine_load_constant(r, PREG_REG_NONE, src.u.sym.addend, 0, sym); + break; + } + /* fallthrough for needs_deref */ + default: + /* REG and other lvalue-y forms: mach_ensure_in_reg already computes the address. */ + r = mach_ensure_in_reg(&ctx, &src, 0); + break; + } + + mach_writeback_dest(&dest, r); + mach_release_all(&ctx); +} + +// r0 - function +// r1 - function +// r2 - function +// r3 - function + +// r4 - lrsa +// r5 - lrsa +// r6 - lrsa +// r7 - lrsa +// r8 - lrsa +// r9 - PIC +// r10 - lrsa + +ST_FUNC int tcc_gen_machine_number_of_registers(void) +{ + return 11; +} + +/* Store a register to a stack slot relative to FP. + * offset is typically negative (local variables below FP). */ +ST_FUNC void tcc_gen_machine_store_to_stack(int reg, int offset) +{ + tcc_gen_machine_store_to_stack_ex(reg, offset, 0); +} + +/* Store a register to a FP-relative stack slot, with additional register + * exclusions for the scratch allocator. The extra_exclude mask prevents + * the scratch register allocator from picking registers that still hold + * live values (e.g. incoming argument registers during the prologue). + */ +ST_FUNC void tcc_gen_machine_store_to_stack_ex(int reg, int offset, uint32_t extra_exclude) +{ + const int base_reg = tcc_state->need_frame_pointer ? R_FP : R_SP; + int sign = (offset < 0); + int abs_offset = sign ? -offset : offset; + + /* Try direct STR with immediate offset */ + if (!store_word_to_base(reg, base_reg, abs_offset, sign)) + { + /* Offset too large, use scratch register */ + /* Don't reuse the source register as offset scratch, otherwise we'd + * clobber the value before the STR (e.g. store -offset instead of value). */ + ScratchRegAlloc rr_alloc = th_offset_to_reg_ex(abs_offset, sign, (1u << reg) | (1u << base_reg) | extra_exclude); + int rr = rr_alloc.reg; + ot_check(th_str_reg(reg, base_reg, rr, THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); + restore_scratch_reg(&rr_alloc); + } +} + +/* Store a register to a stack slot relative to SP. + * Used for outgoing call arguments (stack args must be located at SP at call time). + * offset is expected to be non-negative. + */ +ST_FUNC void tcc_gen_machine_store_to_sp(int reg, int offset) +{ + tcc_gen_machine_store_to_sp_ex(reg, offset, 0); +} + +/* SP-relative store with extra scratch exclusions; see the _ex comment on the + * FP-relative variant above. */ +ST_FUNC void tcc_gen_machine_store_to_sp_ex(int reg, int offset, uint32_t extra_exclude) +{ + int sign = (offset < 0); + int abs_offset = sign ? -offset : offset; + + if (!store_word_to_base(reg, R_SP, abs_offset, sign)) + { + ScratchRegAlloc rr_alloc = th_offset_to_reg_ex(abs_offset, sign, (1u << reg) | (1u << R_SP) | extra_exclude); + int rr = rr_alloc.reg; + ot_check(th_str_reg(reg, R_SP, rr, THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); + restore_scratch_reg(&rr_alloc); + } +} + +/* MachineOperand variant of gcall_or_jump. + * Called from func_call_mop after argument setup is complete. + * + * MACH_OP_SYMBOL → direct call via BL + relocation + * MACH_OP_IMM → relative call (rare) + * MACH_OP_REG → indirect call: BLX through register + * Other kinds → load to scratch via mach_ensure_in_reg, then BLX + * + * For btype=FUNC + needs_deref: the register already holds the function + * pointer value (not an address to load through), so needs_deref is cleared. + */ +void gcall_or_jump_mop(int is_jmp, MachineOperand target) +{ + /* Tail-call: promote is_jmp so we emit B/BX instead of BL/BLX. */ + if (tail_call_pending) + is_jmp = 1; + + if (target.kind == MACH_OP_SYMBOL) + { + /* Direct call via BL (or B.W for tail call) with relocation. */ + Sym *sym = target.u.sym.sym; + Sym *helper = thumb_local_libc_helper(sym); + if (helper) + sym = helper; + int32_t addend = target.u.sym.addend; + Sym *validated_sym = sym ? validate_sym_for_reloc(sym) : NULL; + Sym *reloc_sym = NULL; + + if (!dry_run_state.active) + { + if (sym && !validated_sym && !(sym->v & SYM_FIELD)) + { + put_extern_sym(sym, NULL, 0, 0); + validated_sym = validate_sym_for_reloc(sym); + } + if (sym && !(sym->v & SYM_FIELD)) + reloc_sym = validated_sym ? validated_sym : sym; + } + + uint32_t imm; + if (reloc_sym) + imm = (uint32_t)-4; /* placeholder for linker */ + else + imm = th_encbranch(ind, ind + addend); + + TRACE("gcall_or_jmp_mop: %d, ind: 0x%x, 0x%x", is_jmp, ind, imm); + if (imm) + { + if (is_jmp) + ot_check(th_b_t4((int32_t)imm)); + else + ot_check(th_bl_t1(imm)); + if (!dry_run_state.active && reloc_sym) + { + int call_pos = ind - 4; + greloc(cur_text_section, reloc_sym, call_pos, R_ARM_THM_JUMP24); + /* No R9 reload follows this call (or this B.W never comes back) on + * the assumption that the callee is module-local: have the linker + * check it. */ + if (thumb_callee_needs_local_call_marker(sym)) + greloc(cur_text_section, reloc_sym, call_pos, R_ARM_YASOS_LOCAL_CALL); + } + } + return; + } + + if (target.kind == MACH_OP_IMM) + { + /* Relative call (rare). */ + uint32_t imm = th_encbranch(ind, ind + (int32_t)target.u.imm.val); + TRACE("gcall_or_jmp_mop(imm): %d, ind: 0x%x, 0x%x", is_jmp, ind, imm); + if (imm) + { + if (is_jmp) + ot_check(th_b_t4((int32_t)imm)); + else + ot_check(th_bl_t1(imm)); + } + return; + } + + /* Indirect call through register/spill/frame/param. + * + * For btype=FUNC with needs_deref: the register already holds the function + * pointer value, not an address to dereference. Clear needs_deref to avoid + * a spurious LDR before BLX. */ + MachineOperand adjusted = target; + if (adjusted.btype == IROP_BTYPE_FUNC && adjusted.needs_deref && adjusted.kind == MACH_OP_REG) + adjusted.needs_deref = false; + + /* Keep R0-R3 safe during target materialization. */ + const uint32_t arg_regs = (1u << R0) | (1u << R1) | (1u << R2) | (1u << R3); + MachineCodegenContext mctx = {0}; + + if (is_jmp) + { + int r = mach_ensure_in_reg(&mctx, &adjusted, arg_regs); + ot_check(th_bx_reg(r)); + } + else + { + /* For calls, allocate scratch excluding R0-R3 so args are preserved. */ + uint32_t old_exclude = scratch_global_exclude; + scratch_global_exclude |= arg_regs; + + int r = mach_ensure_in_reg(&mctx, &adjusted, arg_regs); + + scratch_global_exclude = old_exclude; + ot_check(th_blx_reg(r)); + } +} + +/* Load a 32-bit immediate value or symbol address into a register. + * Uses th_generic_mov_imm if possible (pure immediates only), otherwise loads from literal pool. + * reg: target register + * imm: 32-bit immediate value or offset to load + * sym: symbol reference (NULL for pure immediates) + * update_flags: whether the load should update condition flags (currently unused) + */ +void load_immediate(int reg, uint32_t imm, Sym *sym, int update_flags) +{ + (void)update_flags; /* Currently not used, reserved for future use */ + + /* If there's a symbol, always use literal pool for relocations */ + if (sym) + { + _lfc_sym = sym; + load_full_const(reg, PREG_NONE, imm, 0); + return; + } + + /* Plain-constant reuse, same contract as tcc_machine_load_constant: when the + * per-register materialisation cache still says `reg` holds this exact + * constant, the MOV is dead. Call-argument marshalling is where this pays: + * every stack-passed literal takes a fresh find_call_scratch()+load_immediate + * pair, and a variadic call with a run of zero arguments therefore emitted + * `movs rN,#0; str rN,[sp,#k]` once per word instead of hoisting the zero. + * imm_cache is invalidated by ot()'s emit-level decode and reset at IR + * boundaries/calls in both the dry and real passes, so the decision (and the + * resulting code size) stays identical between them. */ + const int cacheable = thumb_gen_state.generating_function && reg >= 0 && reg < 16; + const int64_t key = (int64_t)(uint32_t)imm; + + if (cacheable && imm_cache[reg].valid && imm_cache[reg].sym == NULL && imm_cache[reg].value == key) + return; + + /* Try to encode as ARM immediate (supports various rotated 8-bit patterns) */ + if (!ot(th_generic_mov_imm(reg, imm))) + { + /* Value doesn't fit in immediate encoding, use literal pool */ + load_full_const(reg, PREG_NONE, imm, 0); + } + + /* Record after the emit: ot()/load_full_const just invalidated imm_cache[reg] + * for the instruction they produced. */ + if (cacheable) + { + imm_cache[reg].value = key; + imm_cache[reg].sym = NULL; + imm_cache[reg].valid = 1; + } +} diff --git a/source/backend/arch/arm/thumb/arm-thumb-gen.c b/source/backend/arch/arm/thumb/arm-thumb-gen.c index 64dab62fb..ef8c9345e 100644 --- a/source/backend/arch/arm/thumb/arm-thumb-gen.c +++ b/source/backend/arch/arm/thumb/arm-thumb-gen.c @@ -33,69 +33,27 @@ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ -#if defined(TCC_ARM_EABI) && !defined(TCC_ARM_VFP) -#error "Currently TinyCC only supports float computation with VFP instructions" -#endif - -#ifndef CONFIG_TCC_CPUVER -#define CONFIG_TCC_CPUVER 5 -#endif - -#include "source/backend/arch/arm/arm.h" -#include "source/backend/arch/arm/ssa_opt_arm.h" -#include "arm-thumb-defs.h" -#include "source/opt/include/opt.h" -#include "tcc-chained-hash.h" -#include "tcc.h" -#include "tccir.h" -#include "tccls.h" -#include "tcctype.h" -#include "arm-thumb-callsite.h" -#include "memory/small_sequence.h" +/* Thumb-2 code generator: target state and ABI tables, the scratch-register + * tracker, dry-run state, branch optimisation, the MOV-coalescing and + * STR->LDR peephole caches, and the literal pool. The rest of the generator + * is in arm-thumb-{emit,alu,mem,fp,frame,call}.c; see arm-thumb-gen.h. */ -/* Inline-first literal-pool position scratch (heap only past the inline cap). */ -TCC_SMALL_SEQUENCE_DEFINE(ThumbLitPosSeq, int, 128) +#include "arm-thumb-gen.h" -static void load_full_const(int r, int r1, uint32_t imm_lo, uint32_t imm_hi); +static ScratchRegAlloc get_scratch_reg_for_sym_addr(Sym *raw_sym, int64_t imm, uint32_t exclude_regs); +static ScratchRegAlloc get_scratch_reg_for_const(int64_t value, uint32_t exclude_regs); +/* Forward declarations */ +static void branch_opt_init(void); +static void branch_opt_analyze(uint32_t *ir_to_code_mapping, int mapping_size); /* Workaround for TCC ARM ABI bugs: * 1. int64_t args miscount register pairs 2. 5th+ args not correctly pushed to stack * By passing sym through a file-scope global, load_full_const stays at 4 register args. * Set _lfc_sym before calling load_full_const; it is consumed and reset to NULL inside. */ -static struct Sym *_lfc_sym; - -/* Helper macro: split a 64-bit value into (lo, hi) uint32_t pair for load_full_const. - * Avoids int64_t in function signatures — TCC ARM codegen miscounts int64_t register pairs. */ -#define LFC_SPLIT(v) (uint32_t)((uint64_t)(v)), (uint32_t)((uint64_t)(v) >> 32) +struct Sym *_lfc_sym; ThumbGeneratorState thumb_gen_state; -enum Armv8mRegisters -{ - ARM_R0 = 0, - ARM_R1 = 1, - ARM_R2 = 2, - ARM_R3 = 3, - ARM_R4 = 4, - ARM_R5 = 5, - ARM_R6 = 6, - ARM_R7 = 7, - ARM_R8 = 8, - ARM_R9 = 9, - ARM_R10 = 10, - ARM_R11 = 11, - ARM_R12 = 12, - ARM_SP = 13, - ARM_LR = 14, - ARM_PC = 15 -}; - -#define USING_GLOBALS -#include "tcc.h" - -#include -#include - /* Target ABI hook: AAPCS-like argument assignment for ARM (R0-R3 + stack). * * This is a pure layout function: it does not materialize values and does not @@ -131,38 +89,7 @@ ST_FUNC int tcc_gen_machine_abi_assign_call_args(const TCCAbiArgDesc *args, int out_layout->stack_align = call_layout.stack_align; return 0; } - -#include "source/backend/arch/fpu/arm/fpv5-sp-d16.h" -#include "source/backend/arch/fpu/arm/rp2350-dcp.h" -#include "source/backend/arch/fpu/arm/fpv5-d16.h" -#include "source/backend/arch/arm/thumb/thumb.h" -#include "source/backend/arch/arm/thumb/thop_adr.h" -#include "source/backend/arch/arm/thumb/thop_alu_imm.h" -#include "source/backend/arch/arm/thumb/thop_alu_reg.h" -#include "source/backend/arch/arm/thumb/thop_block.h" -#include "source/backend/arch/arm/thumb/thop_branch.h" -#include "source/backend/arch/arm/thumb/thop_cmp.h" -#include "source/backend/arch/arm/thumb/thop_extend.h" -#include "source/backend/arch/arm/thumb/thop_coproc.h" -#include "source/backend/arch/arm/thumb/thop_vfp.h" -#include "source/backend/arch/arm/thumb/thop_ldr_literal.h" -#include "source/backend/arch/arm/thumb/thop_ldrd.h" -#include "source/backend/arch/arm/thumb/thop_mem_imm.h" -#include "source/backend/arch/arm/thumb/thop_mem_reg.h" -#include "source/backend/arch/arm/thumb/thop_mov.h" -#include "source/backend/arch/arm/thumb/thop_mul.h" -#include "source/backend/arch/arm/thumb/thop_mvn.h" -#include "source/backend/arch/arm/thumb/thop_pld.h" -#include "source/backend/arch/arm/thumb/thop_shift_imm.h" -#include "source/backend/arch/arm/thumb/thop_rev.h" -#include "source/backend/arch/arm/thumb/thop_shift_reg.h" -#include "source/backend/arch/arm/thumb/thop_system.h" - -#include - -int load_word_from_base(int ir, int base, int fc, int sign); - -static inline thumb_flags_behaviour flags_safe(void) +thumb_flags_behaviour flags_safe(void) { if (tcc_state->ir && tcc_state->ir->codegen_flags_live) return FLAGS_BEHAVIOUR_BLOCK; @@ -170,7 +97,7 @@ static inline thumb_flags_behaviour flags_safe(void) } /* Helper to validate a Sym pointer - returns NULL if invalid/unusable for relocation */ -static inline Sym *validate_sym_for_reloc(Sym *sym) +Sym *validate_sym_for_reloc(Sym *sym) { if (!sym) return NULL; @@ -227,62 +154,62 @@ uint32_t caller_saved_registers; uint32_t pushed_registers; int allocated_stack_size; int epilogue_stack_dealloc; /* total SUB SP amount to restore in epilogue (includes alignment pad) */ -int callee_push_size = 0; /* bytes pushed BELOW FP in two-phase push */ -uint32_t callee_saved_regs = 0; /* register mask for second push (below FP) */ int vararg_push_size = 0; /* bytes pushed for variadic r0-r3 save (16 or 0) */ -/* Adjust a local/spill frame offset. - * - * When FP is used with two-phase push: adjusts by callee_push_size (regs - * pushed below FP). - * - * When FP is omitted: converts FP-relative negative offsets to SP-relative - * positive offsets. The alignment pad sits at the top of the SUB SP region - * (right below pushed regs), so locals are addressed relative to - * allocated_stack_size (without pad): - * FP + frame_offset = SP + allocated_stack_size + frame_offset. */ -/* Bytes the real run's scratch PUSHes have currently moved SP below its - * steady-state position (see get_scratch_reg_with_save). Defined after the - * scratch bookkeeping state below. */ -static int scratch_push_sp_bias(void); - -static inline int fp_adjust_local_offset(int frame_offset, int is_param) -{ - if (is_param) +/* Convert a local/spill frame offset (IR view: negative, counted down from + * the top of the frame) into an offset from the frame base register. + * + * The base is SP, or the frame pointer, which the prologue sets to the SP it + * leaves (the bottom of the frame, as gcc's r7 does) -- so the two agree: + * locals occupy base+0 .. base+allocated_stack_size-1, and the alignment pad + * sits above them, below the pushed registers: + * top + frame_offset = base + allocated_stack_size + frame_offset. + * Offsets off the frame pointer are positive, so a Thumb load/store encodes + * them directly up to 4095 (16-bit to 124). A frame pointer at the top of + * the frame, as before, only reached -255 without movw + rsb. Off SP, a + * scratch PUSH inside the current instruction has moved SP down: without the + * bias every access in the push window reads/writes 4 bytes low per active + * push (struct_byval fuzz seed 6105: LDR of a by-value field between + * push {r0} and pop {r0}). */ +int fp_adjust_local_offset(int frame_offset, int is_param) +{ + if (is_param || frame_offset > 0) return frame_offset; + int off = allocated_stack_size + frame_offset; + if (!tcc_state->need_frame_pointer) + off += scratch_push_sp_bias(); + return off; +} - if (!tcc_state->need_frame_pointer && frame_offset <= 0) - { - /* Convert FP-relative (negative) to SP-relative (positive). - * FP + frame_offset = SP + allocated_stack_size + frame_offset. - * A scratch PUSH inside the current instruction has moved SP down; - * without the bias every access in the push window reads/writes 4 - * bytes low per active push (struct_byval fuzz seed 6105: LDR of a - * by-value field between push {r0} and pop {r0}). */ - return allocated_stack_size + scratch_push_sp_bias() + frame_offset; - } - - if (frame_offset < 0 && callee_push_size > 0) - return frame_offset - callee_push_size; - - return frame_offset; +/* Offset from the frame base register (FP, or SP when there is none) of byte + * `param_off` of the incoming stack parameters, which sit above the saved + * registers (offset_to_args). Off SP, a scratch PUSH active in the current + * instruction has moved SP down, exactly as for a local (fp_adjust_local_offset): + * without the bias a by-value parameter passed on through the memcpy path of + * place_stack_arg_struct was copied from 24 bytes too low. */ +int param_frame_offset(int param_off) +{ + int off = param_off + offset_to_args; + if (!tcc_state->need_frame_pointer) + off += scratch_push_sp_bias(); + return off; } /* Additional scratch register exclusions (e.g. to protect argument registers * while materializing an indirect call target). Applied on top of per-call * exclude masks. */ -static uint32_t scratch_global_exclude = 0; +uint32_t scratch_global_exclude = 0; /* Callee-saved register holding this function's .rodata runtime base for the * whole body, or -1 when the function addresses shared .rodata the long way. * Claimed per function by tcc_gen_machine_rodata_anchor_claim(); see the * commentary there. */ -static int rodata_anchor_reg = -1; +int rodata_anchor_reg = -1; /* Registers no scratch picker may ever hand out in this function: R9 is the * GOT base under text/data separation, and the rodata anchor holds a value * with no interval behind it, so liveness always reports it dead. */ -static uint32_t scratch_exclude_baseline(void) +uint32_t scratch_exclude_baseline(void) { uint32_t mask = text_and_data_separation ? (1u << R9) : 0; if (rodata_anchor_reg >= 0) @@ -296,8 +223,8 @@ static uint32_t scratch_exclude_baseline(void) * Size 128 since same register can be pushed multiple times for complex ops like * function calls with many arguments. */ static int scratch_push_stack[128]; -static int scratch_push_type[128]; /* 1 = PUSH, 2 = STR to scratch area */ -static int scratch_push_count = 0; +int scratch_push_type[128]; /* 1 = PUSH, 2 = STR to scratch area */ +int scratch_push_count = 0; /* Flag: set to 1 when a real-run (non-dry-run) scratch PUSH is emitted. * Used by codegen to detect when FP omission caused SP-corrupting pushes @@ -306,7 +233,7 @@ static int real_run_scratch_push_detected = 0; /* Tail-call flag: when set, the next gcall_or_jump_mop emits B (branch) * instead of BL (branch-with-link), and post-call cleanup is skipped. */ -static int tail_call_pending = 0; +int tail_call_pending = 0; /* Current slot index within the scratch save area (0-based). * Incremented on save, decremented on restore. */ @@ -315,47 +242,12 @@ static int scratch_save_slot = 0; /* Debug tracking: current IR opcode being processed (set by codegen.c) */ int g_debug_current_op = -1; -int is_valid_opcode(thumb_opcode op); -int ot(thumb_opcode op); -int ot_check(thumb_opcode op); -static int ot_check_mov_reg(uint32_t rd, uint32_t rm, thumb_flags_behaviour flags, thumb_shift shift, - thumb_enforce_encoding enc, bool in_it); -static int ot_check_ldr_imm(uint32_t rt, uint32_t rn, int imm, uint32_t puw, thumb_enforce_encoding enc); -static int ot_check_str_imm(uint32_t rt, uint32_t rn, int imm, uint32_t puw, thumb_enforce_encoding enc); -static void mov_equiv_reset_all(void); -static void imm_cache_reset_all(void); -static void imm_cache_invalidate_reg(int reg); -ST_FUNC void tcc_gen_machine_strldr_cache_reset(void); -ST_FUNC void tcc_gen_machine_strldr_cache_set_enabled(int enabled); -ST_FUNC void tcc_gen_machine_imm_cache_reset(void); -static void thumb_require_materialized_reg(const char *ctx, const char *operand, int reg); -static bool thumb_is_hw_reg(int reg); -static int get_struct_base_addr_mop(const MachineOperand *mop, int default_reg); -static int find_call_scratch(uint32_t extra_exclude, uint32_t arg_move_dst_mask); -int load_word_from_base(int ir, int base, int fc, int sign); -int th_patch_call(int t, int a); -/* Structure to track scratch register allocation with potential save/restore */ -typedef struct ScratchRegAlloc -{ - int reg : 29; /* The allocated scratch register (range 0-15 for ARM) */ - uint32_t saved : 2; /* 0=not saved, 1=PUSH to stack, 2=STR to scratch area */ - uint32_t would_save : 1; /* Whether a push was needed (set in both dry-run and real emit) */ -} ScratchRegAlloc; - -/* Forward declarations needed by multi-scratch helpers. */ -static ScratchRegAlloc get_scratch_reg_with_save(uint32_t exclude_regs); -static ScratchRegAlloc get_scratch_reg_for_sym_addr(Sym *raw_sym, int64_t imm, uint32_t exclude_regs); -static ScratchRegAlloc get_scratch_reg_for_const(int64_t value, uint32_t exclude_regs); -static void restore_scratch_reg(ScratchRegAlloc *alloc); -static void load_from_base(int r, int r1, int irop_btype, int is_unsigned, int fc, int sign, uint32_t base); -static void th_store32_imm_or_reg_ex(int src_reg, uint32_t base_reg, int abs_off, int sign, uint32_t extra_exclude); - /* Resolve the base register for a captured variable access. * For depth 1, returns R10 directly. * For depth > 1, emits LDR chain to follow ancestor frame pointers * and returns a scratch register holding the target ancestor's FP. * Caller must restore scratch via *out_scratch when done. */ -static int resolve_chain_base(TCCIRState *ir, int ci, uint32_t exclude_regs, ScratchRegAlloc *out_scratch, +int resolve_chain_base(TCCIRState *ir, int ci, uint32_t exclude_regs, ScratchRegAlloc *out_scratch, int *used_scratch) { int depth = ir->captured_chain_depths[ci]; @@ -366,11 +258,12 @@ static int resolve_chain_base(TCCIRState *ir, int ci, uint32_t exclude_regs, Scr } /* Multi-hop: follow chain through (depth - 1) intermediate frames. - * Each frame saves its incoming R10 at [FP - 4] (CHAIN_SLOT_OFFSET). */ + * Each frame saves its incoming R10 at frame offset -4 (CHAIN_SLOT_OFFSET), + * i.e. 4 below the frame top that the chain points at. */ *out_scratch = get_scratch_reg_with_save(exclude_regs); *used_scratch = 1; - /* Start from R10 (points to immediate parent's FP) */ + /* Start from R10 (points to immediate parent's frame top) */ thumb_shift no_shift = {THUMB_SHIFT_NONE, 0, THUMB_SHIFT_IMMEDIATE}; ot_check_mov_reg(out_scratch->reg, architecture_config.static_chain_reg, flags_safe(), no_shift, ENFORCE_ENCODING_NONE, false); @@ -384,26 +277,6 @@ static int resolve_chain_base(TCCIRState *ir, int ci, uint32_t exclude_regs, Scr return out_scratch->reg; } -typedef struct ScratchRegAllocs -{ - int regs[8]; /* The allocated scratch registers */ - int count; /* Number of registers allocated */ - uint32_t saved_mask; /* Bitmask of registers that were saved (pushed) */ -} ScratchRegAllocs; - -/* ============================================================ - * MachineCodegenContext — per-instruction scratch-register tracker - * ============================================================ - * Used by the MachineOperand-based (_mop) code-generation path. - * Callers allocate scratches via mach_alloc_scratch(), then call - * mach_release_all() at the end of the instruction to pop them in LIFO order. - */ - -/* Forward declarations needed by the mach_* helpers (defined later in this file). */ -typedef thumb_opcode (*thumb_imm_handler_t)(uint32_t rd, uint32_t rn, uint32_t imm, - thumb_flags_behaviour flags_behaviour, - thumb_enforce_encoding enforce_encoding); - /* Dispatch an imm_handler call through a direct call instead of an indirect * (function pointer) call. Same workaround as thumb_call_reg_handler: the * cross-compiler miscompiles indirect calls that combine an sret return @@ -411,7 +284,7 @@ typedef thumb_opcode (*thumb_imm_handler_t)(uint32_t rd, uint32_t rn, uint32_t i * garbage for the 5th/6th parameters (flags/enc), so e.g. the high-half SBCS * of a 64-bit CMP silently loses its S bit. Comparing the pointer and * branching to a direct call makes the cross emit correct argument passing. */ -static thumb_opcode thumb_call_imm_handler(thumb_imm_handler_t fn, uint32_t rd, uint32_t rn, uint32_t imm, +thumb_opcode thumb_call_imm_handler(thumb_imm_handler_t fn, uint32_t rd, uint32_t rn, uint32_t imm, thumb_flags_behaviour flags, thumb_enforce_encoding encoding) { if (fn == th_add_imm) @@ -445,16 +318,6 @@ static thumb_opcode thumb_call_imm_handler(thumb_imm_handler_t fn, uint32_t rd, /* Unreachable for known handlers — fallback to direct call. */ return fn(rd, rn, imm, flags, encoding); } -int store_word_to_base(int ir, int base, int fc, int sign); -static ScratchRegAlloc th_offset_to_reg_ex(int off, int sign, uint32_t exclude_regs); - -#define MACH_CTX_MAX_SCRATCH 12 - -typedef struct MachineCodegenContext -{ - ScratchRegAlloc scratches[MACH_CTX_MAX_SCRATCH]; - int n_scratch; -} MachineCodegenContext; /* Phase-3 per-instruction scratch constraint counters. * Incremented/set by mach_alloc_scratch(); reset and read via the @@ -463,23 +326,13 @@ typedef struct MachineCodegenContext * dry_run_state which is defined later in the file. */ static int g_insn_scratch_allocs = 0; /* total scratch allocs this instruction */ static uint16_t g_insn_scratch_saves = 0; /* registers that required PUSH this instruction */ - -/* --- Hard-float single-precision VFP support --- - * Float operands reach codegen as MACH_OP_VFP_REG (u.reg.r0 = s-register 0-31), - * a distinct kind from GPR MACH_OP_REG. s14/s15 are held out of the allocator - * (arm_init) as fixed VFP scratch so the FP emitters can shuttle GPR/imm/spill - * operands through the FPU without clobbering a live float. is_vfp_reg tests a - * raw allocation register number (LS_VFP_REG_BASE + n) as written by the RA. */ -#define VFP_SCRATCH0 14 -#define VFP_SCRATCH1 15 -static inline int is_vfp_reg(int r) { return LS_IS_VFP_REG(r); } -static inline int vfp_num(int r) { return LS_VFP_REG_NUM(r); } - +int is_vfp_reg(int r) { return LS_IS_VFP_REG(r); } +int vfp_num(int r) { return LS_VFP_REG_NUM(r); } /* Allocate a scratch register for the current instruction. * excl: bitmask of registers that must not be chosen. * The allocation is recorded in ctx so mach_release_all() can free it. */ -static int mach_alloc_scratch(MachineCodegenContext *ctx, uint32_t excl) +int mach_alloc_scratch(MachineCodegenContext *ctx, uint32_t excl) { if (ctx->n_scratch >= MACH_CTX_MAX_SCRATCH) tcc_error("compiler_error: mach_alloc_scratch: per-instruction scratch limit exceeded"); @@ -499,7 +352,7 @@ static int mach_alloc_scratch(MachineCodegenContext *ctx, uint32_t excl) * goes through get_scratch_reg_for_sym_addr so a repeat materialization of * the same symbol address reuses the register still holding it (the * subsequent load then elides to zero instructions). */ -static int mach_alloc_scratch_for_sym(MachineCodegenContext *ctx, uint32_t excl, Sym *sym, int64_t imm) +int mach_alloc_scratch_for_sym(MachineCodegenContext *ctx, uint32_t excl, Sym *sym, int64_t imm) { if (ctx->n_scratch >= MACH_CTX_MAX_SCRATCH) tcc_error("compiler_error: mach_alloc_scratch: per-instruction scratch limit exceeded"); @@ -530,7 +383,7 @@ static int mach_alloc_scratch_for_const(MachineCodegenContext *ctx, uint32_t exc /* Release all scratch registers allocated for the current instruction in * reverse (LIFO) order — required because ARM push/pop works by register * number, so the last-pushed register must be popped first. */ -static void mach_release_all(MachineCodegenContext *ctx) +void mach_release_all(MachineCodegenContext *ctx) { for (int i = ctx->n_scratch - 1; i >= 0; i--) restore_scratch_reg(&ctx->scratches[i]); @@ -545,7 +398,93 @@ static void mach_release_all(MachineCodegenContext *ctx) * necessary load instructions, and returns the scratch register. * * excl: bitmask of registers that must not be used for any scratch. */ -static int mach_ensure_in_reg(MachineCodegenContext *ctx, const MachineOperand *op, uint32_t excl) +/* A MACH_OP_SPILL operand with vreg < 0 is a local's own stack slot + * (machine_op_from_ir, "concrete stack slots"), holding an object exactly as + * wide as its btype; with a vreg it is that vreg's spill slot, a full register + * image a word store wrote. lea_fold turns `T = &local; ... *T` into such a + * local slot, so a byte or halfword local reaches the value paths below. Read + * as a word, a u16 picked up its neighbour's bytes (Zig's `if (err.error)` on a + * 2-byte error union took the wrong branch); written as a word, it would + * clobber them. */ +static int mach_is_narrow_local_slot(const MachineOperand *op) +{ + return op->kind == MACH_OP_SPILL && !op->needs_deref && op->vreg < 0 && + (op->btype == IROP_BTYPE_INT8 || op->btype == IROP_BTYPE_INT16); +} + +/* A VAR the allocator spilled to a slot of its own: an integer spill slot is a + * whole word nobody else shares, and its reads take the width of whichever + * operand names it -- `CMP V,#0` reads a word even when V's def stored a byte. + * Its narrow stores extend to a word like a TEMP's. An address-taken VAR lives + * in its home, which pointer stores write narrow, so it keeps narrow stores + * (and mach_load_slot's narrow reads). */ +int mach_var_owns_spill_slot(int vreg) +{ + if (vreg < 0 || TCCIR_DECODE_VREG_TYPE(vreg) != TCCIR_VREG_TYPE_VAR || !tcc_state->ir) + return 0; + IRLiveInterval *li = tcc_ir_get_live_interval(tcc_state->ir, vreg); + return li && !li->addrtaken && !li->is_lvalue && !li->is_volatile && !li->is_struct; +} + +/* Load a MACH_OP_SPILL slot's value (the slot itself, not through it). + * A narrow local slot, or a VAR's, is read at its width, as the LOAD handler + * reads it: a VAR whose address is taken keeps its vreg but lives in its + * home, which a store through the pointer wrote a byte or halfword of -- + * read as a word, the rest came from whatever was there before (a u8 passed + * to printf printed as aaaaaaa5). A TEMP's or PARAM's slot only ever holds + * a register image written as a word, and a word load is exact for it; its + * operand's narrow type need not say how that image was extended. A copy + * into a loop temp first defined from a plain `char` keeps the char type + * after `(unsigned)c` (a no-op in the IR), and reading the spilled + * `cond * (unsigned)c` back with LDRSB turned 0xaf into 0xffffffaf + * (212_fuzz_cprop_copy_into_loop_phi at -O0). */ +void mach_load_slot(int dest_reg, const MachineOperand *op) +{ + if (op->needs_deref || (op->btype != IROP_BTYPE_INT8 && op->btype != IROP_BTYPE_INT16) || + (op->vreg >= 0 && TCCIR_DECODE_VREG_TYPE(op->vreg) != TCCIR_VREG_TYPE_VAR)) + { + tcc_machine_load_spill_slot(dest_reg, op->u.spill.offset); + return; + } + const int adj = fp_adjust_local_offset(op->u.spill.offset, 0); + const int sign = (adj < 0), abs_off = sign ? -adj : adj; + const uint32_t base = (uint32_t)(tcc_state->need_frame_pointer ? R_FP : R_SP); + load_from_base(dest_reg, PREG_REG_NONE, op->btype, (int)op->is_unsigned, abs_off, sign, base); +} + +/* Bytes mach_load_slot reads and mach_store_slot writes for a MACH_OP_SPILL + * operand. Kept next to the two functions whose choices they mirror. */ +int mach_slot_load_width(const MachineOperand *op) +{ + if (op->needs_deref || (op->btype != IROP_BTYPE_INT8 && op->btype != IROP_BTYPE_INT16) || + (op->vreg >= 0 && TCCIR_DECODE_VREG_TYPE(op->vreg) != TCCIR_VREG_TYPE_VAR)) + return 4; + return op->btype == IROP_BTYPE_INT8 ? 1 : 2; +} +int mach_slot_store_width(const MachineOperand *op) +{ + if (!mach_is_narrow_local_slot(op)) + return 4; + return op->btype == IROP_BTYPE_INT8 ? 1 : 2; +} + +/* Store a value into a MACH_OP_SPILL slot (the slot itself, not through it). */ +static void mach_store_slot(int src_reg, const MachineOperand *op) +{ + if (!mach_is_narrow_local_slot(op)) + { + tcc_machine_store_spill_slot(src_reg, op->u.spill.offset); + return; + } + const int adj = fp_adjust_local_offset(op->u.spill.offset, 0); + const int sign = (adj < 0), abs_off = sign ? -adj : adj; + const uint32_t base = (uint32_t)(tcc_state->need_frame_pointer ? R_FP : R_SP); + if (op->btype == IROP_BTYPE_INT8) + th_store8_imm_or_reg(src_reg, base, abs_off, sign); + else + th_store16_imm_or_reg(src_reg, base, abs_off, sign); +} +int mach_ensure_in_reg(MachineCodegenContext *ctx, const MachineOperand *op, uint32_t excl) { switch (op->kind) { @@ -580,9 +519,9 @@ static int mach_ensure_in_reg(MachineCodegenContext *ctx, const MachineOperand * case MACH_OP_SPILL: if (!op->needs_deref) { - /* Simple spill: load the word-sized register value from the spill slot. */ + /* Simple spill (or a local's own slot): load the value. */ int r = mach_alloc_scratch(ctx, excl); - tcc_machine_load_spill_slot(r, op->u.spill.offset); + mach_load_slot(r, op); return r; } else @@ -653,7 +592,7 @@ static int mach_ensure_in_reg(MachineCodegenContext *ctx, const MachineOperand * * for 64-bit split operands), but the load is still required — needs_deref=false * in this context means "not a pointer-to-follow", not "compute address". */ int r = mach_alloc_scratch(ctx, excl); - const int adjusted = op->u.param.offset + offset_to_args; + const int adjusted = param_frame_offset(op->u.param.offset); const int base_reg = tcc_state->need_frame_pointer ? R_FP : R_SP; const int sign = (adjusted < 0); const int abs_off = sign ? -adjusted : adjusted; @@ -683,10 +622,28 @@ static int mach_ensure_in_reg(MachineCodegenContext *ctx, const MachineOperand * } } +/* The registers a register operand reads where it sits: exclude them when + * another operand of the same instruction is materialised first. A scratch + * is taken from the live registers when none is free (saved around the use), + * so a scratch not told about the base could be the base itself: + * `str r0,[sp]; movs r0,#0; str r0,[r0,#64]` for `*(P0 + 64) = 0` in a + * function whose r0-r3 all hold parameters (pr108498-1 after sret_nrvo). */ +uint32_t mach_reg_excl(const MachineOperand *op) +{ + if (op->kind != MACH_OP_REG) + return 0; + uint32_t m = 0; + if (thumb_is_hw_reg(op->u.reg.r0)) + m |= 1u << (uint32_t)op->u.reg.r0; + if (op->is_64bit && thumb_is_hw_reg(op->u.reg.r1)) + m |= 1u << (uint32_t)op->u.reg.r1; + return m; +} + /* Try to emit an immediate-form instruction for src2; if the encoding succeeds, * sets *imm_emitted=true and returns PREG_REG_NONE. Otherwise loads src2 into * a scratch register and returns it (like mach_ensure_in_reg). */ -static int mach_ensure_imm_or_reg(MachineCodegenContext *ctx, const MachineOperand *op, uint32_t excl, +int mach_ensure_imm_or_reg(MachineCodegenContext *ctx, const MachineOperand *op, uint32_t excl, thumb_imm_handler_t imm_handler, int dest_reg, int src1_reg, thumb_flags_behaviour flags, bool *imm_emitted) { @@ -709,7 +666,7 @@ static int mach_ensure_imm_or_reg(MachineCodegenContext *ctx, const MachineOpera * Returns the physical register that should hold the result. * If the destination is a spill slot or needs pointer write-back, allocates a * scratch; call mach_writeback_dest() after emitting the instruction. */ -static int mach_get_dest_reg(MachineCodegenContext *ctx, const MachineOperand *op, uint32_t excl) +int mach_get_dest_reg(MachineCodegenContext *ctx, const MachineOperand *op, uint32_t excl) { if (!op || op->kind == MACH_OP_NONE) return R0; /* CMP / flag-setting ops: Rd is ignored. */ @@ -741,7 +698,7 @@ static int mach_get_dest_reg(MachineCodegenContext *ctx, const MachineOperand *o * Only needed when the destination was a spill slot, stack parameter, or an * lvalue (store-through-pointer). Must be called after mach_get_dest_reg() * allocated a scratch for those cases. */ -static void mach_writeback_dest(const MachineOperand *op, int reg) +void mach_writeback_dest(const MachineOperand *op, int reg) { if (!op || op->kind == MACH_OP_NONE) return; @@ -774,7 +731,7 @@ static void mach_writeback_dest(const MachineOperand *op, int reg) break; case MACH_OP_SPILL: - tcc_machine_store_spill_slot(reg, op->u.spill.offset); + mach_store_slot(reg, op); break; case MACH_OP_FRAME_ADDR: @@ -860,7 +817,7 @@ void tcc_gen_mach_load_to_reg(int dest_reg, const MachineOperand *op) case MACH_OP_SPILL: if (!op->needs_deref) { - tcc_machine_load_spill_slot(dest_reg, op->u.spill.offset); + mach_load_slot(dest_reg, op); return; } else @@ -905,7 +862,7 @@ void tcc_gen_mach_load_to_reg(int dest_reg, const MachineOperand *op) case MACH_OP_PARAM_STACK: { - const int adjusted = op->u.param.offset + offset_to_args; + const int adjusted = param_frame_offset(op->u.param.offset); const int base_reg = tcc_state->need_frame_pointer ? R_FP : R_SP; const int sign = (adjusted < 0); const int abs_off = sign ? -adjusted : adjusted; @@ -946,66 +903,52 @@ void tcc_gen_mach_store_from_reg(int src_reg, const MachineOperand *op) { mach_writeback_dest(op, src_reg); } - -/* ============================================================ - * Dry-Run Code Generation State - * ============================================================ - * Two-pass code generation system for optimal register allocation. - * Pass 1 (Dry Run): Analyze register needs without emitting code - * Pass 2 (Real Emit): Generate code with optimal prologue based on Pass 1 - */ - -typedef struct CodeGenDryRunState -{ - int active; /* 1 = dry run, 0 = real emit */ - uint32_t scratch_regs_pushed; /* Bitmap of regs pushed as scratch */ - int scratch_push_count; /* Total scratch push operations */ - int lr_push_count; /* Times LR specifically was pushed */ - int instruction_count; /* IR instructions processed */ - int max_nested_saves; /* Max nested-call save slots used at any call site */ - int rodata_anchor_sites; /* Shared-.rodata addresses materialised in the body */ -} CodeGenDryRunState; - -static CodeGenDryRunState dry_run_state; +CodeGenDryRunState dry_run_state; /* Rehearsal mode: still a dry run (ot() writes no bytes) but every decision is * made exactly as the real pass would make it, so the resulting address map is * a faithful size model. The discovery dry run cannot be one: where it finds * no free scratch register it merely RECORDS the push and emits nothing, and * its finalisation then reassigns registers and resizes the frame — so its * layout is not the layout the real pass produces. */ -static int dry_run_rehearsal = 0; +int dry_run_rehearsal = 0; ST_FUNC void tcc_gen_machine_dry_run_set_rehearsal(int on) { dry_run_rehearsal = on; } /* Mapping-symbol emission, gated to the real pass. Dry passes advance `ind` * with drifted offsets (branches emit wide), so a symbol emitted there points * mid-pool or mid-code and actively misleads objdump — that is exactly the * bug that made the ungated th_sym_* calls worse than useless. */ -static void map_sym_d(void) +void map_sym_d(void) { if (dry_run_state.active || nocode_wanted) return; th_sym_d(); } - -static void map_sym_t(void) +void map_sym_t(void) { if (dry_run_state.active || nocode_wanted) return; th_sym_t(); } -/* True only for the discovery dry run: "model what would happen" rather than - * "emit what the real pass emits". */ -#define DRY_RUN_MODELLING (dry_run_state.active && !dry_run_rehearsal) /* Bytes the real run's scratch PUSHes have currently moved SP below its * steady-state position. Derived from the push bookkeeping so it can never * drift from the actual PUSH/POP pairing (including deferred pops). The dry * run never emits pushes, so its bias is always 0. */ -static int scratch_push_sp_bias(void) -{ +/* Bytes an explicit register save around a helper call has moved SP (see + * place_stack_arg_struct's memcpy path). Emitted, and so counted, in the dry + * run as in the real one. */ +int helper_call_sp_bias; + +/* Bytes a call's own argument window has dropped SP below the frame's + * reserved outgoing area (tcc_gen_machine_func_call_mop). Part of every + * frame-relative SP offset, like the pushes; NOT of an offset into the + * argument area, which is relative to the moved SP. */ +int call_args_sp_bias; +int scratch_push_sp_bias(void) +{ + int bias = helper_call_sp_bias + call_args_sp_bias; if (dry_run_state.active) - return 0; - int bias = 0; + return bias; for (int i = 0; i < scratch_push_count; i++) if (scratch_push_type[i] == 1) bias += 4; @@ -1014,43 +957,39 @@ static int scratch_push_sp_bias(void) /* Separate literal pool for dry-run mode to avoid modifying the real pool. * This allows accurate code size tracking without affecting the real pass. */ -static ThumbLiteralPoolEntry *dry_run_literal_pool = NULL; -static int dry_run_literal_pool_count = 0; +ThumbLiteralPoolEntry *dry_run_literal_pool = NULL; +int dry_run_literal_pool_count = 0; /* Monotonic per-function totals (never reset by a pool flush) used by the * forward-branch narrowing safety check: the rehearsal records them per IR * instruction so the real pass can bound the pool pressure over a branch's * range and prove no flush can land inside it. */ static int pool_flushes_total = 0; -static int pool_entries_total = 0; +int pool_entries_total = 0; ST_FUNC int tcc_gen_machine_pool_flushes_total(void) { return pool_flushes_total; } ST_FUNC int tcc_gen_machine_pool_entries_total(void) { return pool_entries_total; } -static int dry_run_literal_pool_size = 0; -/* Literal pool dedup uses the same bucket+chain scheme as TinyCC's ELF hashes. - * We only hash entries created through th_literal_pool_find_or_allocate(), so - * plain th_literal_pool_allocate() users stay distinct. */ -#define LITERAL_POOL_HASH_BUCKET_COUNT 512 -#define LITERAL_POOL_LOOKUP_CACHE_SIZE 16 +/* A 16-bit NOP so the following instruction starts on a word boundary. `o()` + * carries the literal-pool window along (the window is measured from `ind`), + * and a NOP writes no register and touches no memory, so none of ot()'s + * encoder caches need invalidating -- the loop head resets them anyway. */ +/* The largest pad tcc_gen_machine_align_branch_target can emit (align - 2 + * bytes; 2 at the default word alignment), for the branch-narrowing bounds. */ +int align_pad_max; -typedef struct LiteralPoolLookupCacheEntry -{ - Sym *sym; - int64_t imm; - int pool_index; - uint32_t hash; - int valid; -} LiteralPoolLookupCacheEntry; - -typedef struct LiteralPoolLookupCache +ST_FUNC void tcc_gen_machine_align_branch_target(int align) { - LiteralPoolLookupCacheEntry entries[LITERAL_POOL_LOOKUP_CACHE_SIZE]; -} LiteralPoolLookupCache; - -static TCCChainedHash literal_pool_hash; -static LiteralPoolLookupCache literal_pool_last_lookup; - -static inline uint32_t literal_pool_hash_func(Sym *sym, int64_t imm) + if (align < 2) + return; + if (align - 2 > align_pad_max) + align_pad_max = align - 2; + while (ind & (align - 1)) + o(0xbf00); +} +int dry_run_literal_pool_size = 0; +TCCChainedHash literal_pool_hash; +LiteralPoolLookupCache literal_pool_last_lookup; +uint32_t literal_pool_hash_func(Sym *sym, int64_t imm) { /* 32-bit hash to avoid expensive 64-bit multiply on Cortex-M */ uint32_t h = (uint32_t)(uintptr_t)sym; @@ -1071,8 +1010,7 @@ static void literal_pool_lookup_cache_clear(LiteralPoolLookupCache *cache) { memset(cache, 0, sizeof(*cache)); } - -static inline int literal_pool_lookup_cache_find(LiteralPoolLookupCache *cache, uint32_t full_hash, Sym *sym, +int literal_pool_lookup_cache_find(LiteralPoolLookupCache *cache, uint32_t full_hash, Sym *sym, int64_t imm) { LiteralPoolLookupCacheEntry *entry = &cache->entries[full_hash & (LITERAL_POOL_LOOKUP_CACHE_SIZE - 1)]; @@ -1080,8 +1018,7 @@ static inline int literal_pool_lookup_cache_find(LiteralPoolLookupCache *cache, return entry->pool_index; return -1; } - -static inline void literal_pool_lookup_cache_insert(LiteralPoolLookupCache *cache, uint32_t full_hash, Sym *sym, +void literal_pool_lookup_cache_insert(LiteralPoolLookupCache *cache, uint32_t full_hash, Sym *sym, int64_t imm, int pool_index) { LiteralPoolLookupCacheEntry *entry = &cache->entries[full_hash & (LITERAL_POOL_LOOKUP_CACHE_SIZE - 1)]; @@ -1091,8 +1028,7 @@ static inline void literal_pool_lookup_cache_insert(LiteralPoolLookupCache *cach entry->hash = full_hash; entry->valid = 1; } - -static inline int literal_pool_hash_find(TCCChainedHash *hash, ThumbLiteralPoolEntry *pool, uint32_t full_hash, +int literal_pool_hash_find(TCCChainedHash *hash, ThumbLiteralPoolEntry *pool, uint32_t full_hash, Sym *sym, int64_t imm) { uint32_t slot = tcc_chained_hash_bucket_head(hash, full_hash); @@ -1106,8 +1042,7 @@ static inline int literal_pool_hash_find(TCCChainedHash *hash, ThumbLiteralPoolE } return -1; } - -static inline void literal_pool_hash_insert(TCCChainedHash *hash, uint32_t full_hash, int pool_index) +void literal_pool_hash_insert(TCCChainedHash *hash, uint32_t full_hash, int pool_index) { tcc_chained_hash_insert_head(hash, full_hash, (uint32_t)pool_index); } @@ -1125,22 +1060,6 @@ static void dry_run_record_push(int reg) dry_run_state.lr_push_count++; } -/* Structure to save/restore thumb_gen_state for dry-run isolation */ -typedef struct ThumbGenStateSnapshot -{ - int code_size; - int literal_pool_count; - int literal_pool_size; - int pool_window_first; - int pool_bytes; - ThumbLiteralPoolEntry *literal_pool; - Sym *cached_global_sym; - int cached_global_reg; - int function_argument_count; - int call_sites_by_id_size; - ThumbGenCallSite *call_sites_by_id; -} ThumbGenStateSnapshot; - static ThumbGenStateSnapshot dry_run_snapshot; static void thumb_gen_state_snapshot_save(ThumbGenStateSnapshot *snap) @@ -1184,56 +1103,18 @@ static void thumb_gen_state_snapshot_restore(ThumbGenStateSnapshot *snap) thumb_gen_state.call_sites_by_id_size = snap->call_sites_by_id_size; } -/* ============================================================ - * Branch Instruction Optimization State - * ============================================================ - * Tracks branch instructions during dry-run to select optimal - * 16-bit vs 32-bit encodings based on actual jump distances. - */ - -typedef enum -{ - BRANCH_ENC_UNKNOWN = 0, - BRANCH_ENC_16BIT = 16, - BRANCH_ENC_32BIT = 32 -} BranchEncoding; - -typedef struct BranchInfo -{ - int ir_index; /* IR instruction index of the branch */ - int source_addr; /* Code address where branch is emitted */ - int target_ir; /* Target IR instruction index */ - int target_addr; /* Target code address (computed after dry-run) */ - int offset; /* Computed offset = target - source - 4 */ - int is_conditional; /* 1 = conditional (JUMPIF), 0 = unconditional (JUMP) */ - BranchEncoding encoding; /* Selected encoding after analysis */ -} BranchInfo; - -typedef struct BranchOptState -{ - BranchInfo *branches; /* Array of branch info */ - int branch_count; /* Number of branches */ - int branch_capacity; /* Allocated capacity */ - int optimization_enabled; /* Flag to enable/disable */ - int code_size_reduction; /* Total bytes saved */ -} BranchOptState; - static BranchOptState branch_opt_state; -/* Forward declarations */ -static void branch_opt_init(void); -static void branch_opt_analyze(uint32_t *ir_to_code_mapping, int mapping_size); - /* Check if offset fits in 16-bit conditional branch (T1 encoding) * Range: -256 to +254 bytes (imm8 * 2), must be even */ -static int branch_fits_t1(int offset) +int branch_fits_t1(int offset) { return (offset >= -256 && offset <= 254 && (offset & 1) == 0); } /* Check if offset fits in 16-bit unconditional branch (T2 encoding) * Range: -2048 to +2046 bytes (imm11 * 2), must be even */ -static int branch_fits_t2(int offset) +int branch_fits_t2(int offset) { return (offset >= -2048 && offset <= 2046 && (offset & 1) == 0); } @@ -1631,7 +1512,7 @@ static int scratch_pushed_dead_reg(TCCIRState *ir, uint32_t exclude_regs, uint32 * If no free register is found, saves R_IP to stack and returns it. * Returns ScratchRegAlloc with the register and whether it was saved. */ -static ScratchRegAlloc get_scratch_reg_with_save(uint32_t exclude_regs) +ScratchRegAlloc get_scratch_reg_with_save(uint32_t exclude_regs) { ScratchRegAlloc result = {0}; TCCIRState *ir = tcc_state->ir; @@ -1764,12 +1645,19 @@ static ScratchRegAlloc get_scratch_reg_with_save(uint32_t exclude_regs) } /* When FP is omitted, use STR to the pre-reserved scratch save area instead - * of PUSH, to avoid moving SP (which would break SP-relative addressing). */ + * of PUSH, to avoid moving SP (which would break SP-relative addressing). + * Only while the slot is in STR's immediate range: in a frame of more than + * 4 KB it may not be, and with no free register there is nothing to build + * the address in -- then PUSH, which scratch_push_sp_bias accounts for + * (pr28982b's 256 KB frame at -O1/-O2). */ + int save_sp_offset = 0; if (!tcc_state->need_frame_pointer && ir && ir->scratch_save_size > 0 && scratch_save_slot < (ir->scratch_save_size / 4)) + save_sp_offset = allocated_stack_size + scratch_push_sp_bias() + ir->scratch_save_base + (scratch_save_slot * 4); + if (!tcc_state->need_frame_pointer && ir && ir->scratch_save_size > 0 && + scratch_save_slot < (ir->scratch_save_size / 4) && save_sp_offset >= 0 && save_sp_offset <= 4095) { - int frame_offset = ir->scratch_save_base + (scratch_save_slot * 4); - int sp_offset = allocated_stack_size + scratch_push_sp_bias() + frame_offset; + int sp_offset = save_sp_offset; if (!store_word_to_base(reg_to_save, R_SP, sp_offset, 0)) tcc_error("compiler_error: scratch save STR failed (offset %d)", sp_offset); result.reg = reg_to_save; @@ -1807,7 +1695,7 @@ static ScratchRegAlloc get_scratch_reg_with_save(uint32_t exclude_regs) } /* Restore a scratch register if it was saved */ -static void restore_scratch_reg(ScratchRegAlloc *alloc) +void restore_scratch_reg(ScratchRegAlloc *alloc) { if (alloc->saved) imm_cache_invalidate_reg(alloc->reg); @@ -1887,7 +1775,7 @@ static void restore_scratch_reg(ScratchRegAlloc *alloc) /* Restore all scratch registers that were pushed but not explicitly restored. * Call this at the end of each IR instruction to clean up after callers that * used .reg and discarded the saved flag. POP in reverse order of PUSH! */ -static void restore_all_pushed_scratch_regs(void) +void restore_all_pushed_scratch_regs(void) { /* Dry run: don't emit pops, just reset tracking */ if (DRY_RUN_MODELLING) @@ -2014,12 +1902,6 @@ int ot_check(thumb_opcode op) return ot(op); } -/* Forward declarations for helpers used by spill preloading. */ -int load_short_from_base(int ir, int base, int fc, int sign); -int load_ushort_from_base(int ir, int base, int fc, int sign); -int load_byte_from_base(int ir, int base, int fc, int sign); -int load_ubyte_from_base(int ir, int base, int fc, int sign); - /* Spill cache management functions for avoiding redundant loads */ void tcc_ir_spill_cache_clear(SpillCache *cache) @@ -2131,21 +2013,9 @@ static uint8_t mov_equiv[16]; * block. While this is non-zero the opcode stream seen by ot() is * conditionally executed; cache updates must treat destinations as "may or * may not be written", not as guaranteed assignments. */ -static int mov_equiv_it_pending; - -/* Immediate-value cache: tracks the last pure-integer constant loaded into - * each register by tcc_machine_load_constant (no symbol involved). Persists - * across IR instruction boundaries so consecutive STORE instructions that - * materialise the same constant can skip the redundant MOV. Reset at jump - * targets and function calls. */ -/* Per-register materialisation cache. `sym == NULL` means the register holds - * the plain constant `value`; `sym != NULL` means it holds the address of that - * symbol plus addend `value` (so a later reference to the same global address - * can skip the redundant literal-pool load). Invalidated per-register on every - * clobbering emit and at IR boundaries, just like the constant cache. */ -static struct { int64_t value; Sym *sym; uint8_t valid; } imm_cache[16]; - -static void imm_cache_reset_all(void) +int mov_equiv_it_pending; +ImmCacheEntry imm_cache[16]; +void imm_cache_reset_all(void) { for (int i = 0; i < 16; i++) { @@ -2153,8 +2023,7 @@ static void imm_cache_reset_all(void) imm_cache[i].sym = NULL; } } - -static void imm_cache_invalidate_reg(int reg) +void imm_cache_invalidate_reg(int reg) { if (reg >= 0 && reg < 16) imm_cache[reg].valid = 0; @@ -2181,7 +2050,7 @@ static void imm_cache_invalidate_reg(int reg) * identically in the dry and real passes (the invariant the elide path in * load_full_const already relies on), so dry/real code sizes stay in sync. * Every other case falls back to get_scratch_reg_with_save. */ -static int try_scratch_reg_for_sym_addr(Sym *raw_sym, int64_t imm, uint32_t exclude_regs, ScratchRegAlloc *out) +int try_scratch_reg_for_sym_addr(Sym *raw_sym, int64_t imm, uint32_t exclude_regs, ScratchRegAlloc *out) { TCCIRState *ir = tcc_state->ir; Sym *vsym = raw_sym ? validate_sym_for_reloc(raw_sym) : NULL; @@ -2295,8 +2164,7 @@ static ScratchRegAlloc get_scratch_reg_for_const(int64_t value, uint32_t exclude } return get_scratch_reg_with_save(exclude_regs); } - -static void mov_equiv_reset_all(void) +void mov_equiv_reset_all(void) { for (int i = 0; i < 16; i++) mov_equiv[i] = (uint8_t)i; @@ -2307,7 +2175,7 @@ static void mov_equiv_reset_all(void) * return the number of instructions that will execute conditionally after * it — 1..4 depending on which bit of the mask is lowest-set. Returns 0 * when the opcode is not an IT (mask == 0 is a plain NOP/hint). */ -static int mov_equiv_it_block_length(thumb_opcode op) +int mov_equiv_it_block_length(thumb_opcode op) { if (op.size != 2) return 0; @@ -2325,8 +2193,7 @@ static int mov_equiv_it_block_length(thumb_opcode op) return 2; return 1; /* mask & 0x8 */ } - -static void mov_equiv_invalidate_reg(int reg) +void mov_equiv_invalidate_reg(int reg) { if (reg < 0 || reg >= 16) return; @@ -2340,8 +2207,7 @@ static void mov_equiv_invalidate_reg(int reg) } mov_equiv[reg] = (uint8_t)reg; } - -static void mov_equiv_record_mov(int rd, int rm) +void mov_equiv_record_mov(int rd, int rm) { if (rd < 0 || rd >= 16 || rm < 0 || rm >= 16) { @@ -2379,9 +2245,8 @@ static void mov_equiv_record_mov(int rd, int rm) * ot() decrements it as each conditioned op is emitted, so on entry to the * helpers below a non-zero value means "the op about to be emitted is inside * an IT block". */ -static int pool_flush_it_pending; - -static int ot_check_mov_reg(uint32_t rd, uint32_t rm, thumb_flags_behaviour flags, thumb_shift shift, +int pool_flush_it_pending; +int ot_check_mov_reg(uint32_t rd, uint32_t rm, thumb_flags_behaviour flags, thumb_shift shift, thumb_enforce_encoding enc, bool in_it) { /* `in_it` is what the caller believes; pool_flush_it_pending is what ot() @@ -2404,7 +2269,7 @@ static int ot_check_mov_reg(uint32_t rd, uint32_t rm, thumb_flags_behaviour flag /* Return 1 if `op` is a plain MOV Rd, Rm with no shift and no flag set, * filling *rd_out / *rm_out. Accepts the 16-bit T1 high-register form and * the 32-bit T2 form when shift/flags are zero. */ -static int decode_mov_reg_plain(thumb_opcode op, int *rd_out, int *rm_out) +int decode_mov_reg_plain(thumb_opcode op, int *rd_out, int *rm_out) { if (op.size == 2) { @@ -2440,44 +2305,6 @@ static int decode_mov_reg_plain(thumb_opcode op, int *rd_out, int *rm_out) } return 0; } - -/* Bit layout of tcc_ir_zero_half64_at's byte, as it arrives in bits 18-23 of - * `barrel_shift`. Mirrors source/opt/flat/fusion/zero_half64.c. */ -#define ZH64_S1_LO 0x01u -#define ZH64_S1_HI 0x02u -#define ZH64_S2_LO 0x04u -#define ZH64_S2_HI 0x08u -#define ZH64_D_LO 0x10u -#define ZH64_D_HI 0x20u - -/* --------------------------------------------------------------------------- - * STR -> LDR redundant-reload peephole - * - * Tracks recent immediate-offset STR Rt, [Rn, #imm] emissions and skips the - * subsequent LDR Rt, [Rn, #imm] at the call site when Rt is still known to - * hold the stored value. The cache is reset at every IR instruction - * boundary (via tcc_gen_machine_mov_coalesce_reset, same hook as plan C) - * so that cross-IR equivalences cannot be exploited — any IR op may be a - * branch target, and the runtime register/memory state on an entry-by-jump - * path is not what the emission-order state predicts. - * - * Only puw == 6 (P=1, U=1, W=0 — no writeback) STR/LDR forms are tracked. - * The classifier below recognises the T1 16-bit, T2 16-bit SP-relative, and - * T3 32-bit encodings (those that cover the common stack-spill path). - * --------------------------------------------------------------------------- */ - -typedef struct StrLdrCacheEntry -{ - uint8_t valid; - uint8_t rt; - uint8_t rn; - uint8_t size; /* encoding size, 2 or 4 */ - uint8_t width; /* bytes the access moves; only 4 is ever recorded */ - int imm; - uint32_t puw; -} StrLdrCacheEntry; - -#define STRLDR_CACHE_CAPACITY 8 static StrLdrCacheEntry strldr_cache[STRLDR_CACHE_CAPACITY]; static int strldr_cache_count; /* Off for a function that touches volatile ANYWHERE. Resetting per IR op is @@ -2485,10 +2312,34 @@ static int strldr_cache_count; * op, and both have to reach memory. Volatile bodies are rare, so the whole * function pays rather than the analysis getting clever. */ static int strldr_cache_off; +FrameWordAccess frame_word_last; ST_FUNC void tcc_gen_machine_strldr_cache_reset(void) { strldr_cache_count = 0; + /* Every point where another path can arrive (IR jump targets, backend + * labels, asm, setjmp) resets this cache, so it is also where a pending + * frame-word pair has to be forgotten. */ + frame_word_last.kind = 0; +} + +/* Knob for strldr_cache_str_is_redundant alone: TCC_DISABLE_PASS=codegen:str_elide. */ +static int strldr_str_elide_off; + +ST_FUNC void tcc_gen_machine_str_elide_set_enabled(int enabled) +{ + strldr_str_elide_off = !enabled; +} + +/* Knob for the same-slot ASSIGN skip in tcc_gen_machine_assign_mop_ex: + * TCC_DISABLE_PASS=codegen:slot_self_copy. Also off in a function with a + * volatile access, where a copy of a value onto itself may still have to + * touch memory. */ +int slot_self_copy_off; + +ST_FUNC void tcc_gen_machine_slot_self_copy_set_enabled(int enabled) +{ + slot_self_copy_off = !enabled; } ST_FUNC void tcc_gen_machine_strldr_cache_set_enabled(int enabled) @@ -2503,7 +2354,7 @@ ST_FUNC void tcc_gen_machine_strldr_cache_set_enabled(int enabled) * labels); every other merge is an IR-level jump target that codegen.c * already resets at. Needed because a plain branch no longer resets the * memory cache on its own -- see thumb_op_is_plain_branch. */ -static void codegen_internal_merge_point(void) +void codegen_internal_merge_point(void) { mov_equiv_reset_all(); tcc_gen_machine_strldr_cache_reset(); @@ -2526,7 +2377,7 @@ ST_FUNC void tcc_gen_machine_imm_cache_invalidate_live(uint32_t live_mask) /* Invalidate entries where the given register is either the stored value * (Rt) or the base register (Rn). Called when a subsequent instruction * writes to that register. */ -static void strldr_cache_invalidate_reg(int reg) +void strldr_cache_invalidate_reg(int reg) { for (int i = 0; i < strldr_cache_count; i++) { @@ -2544,7 +2395,7 @@ static void strldr_cache_invalidate_reg(int reg) * through one of them has to clear the other's entries. A write to `rn` * itself is handled separately by strldr_cache_invalidate_reg, which is what * makes "same base register" mean "same address" here. */ -static void strldr_cache_invalidate_mem(int rn, int imm, int width) +void strldr_cache_invalidate_mem(int rn, int imm, int width) { for (int i = 0; i < strldr_cache_count; i++) { @@ -2567,7 +2418,7 @@ static void strldr_cache_invalidate_mem(int rn, int imm, int width) * `ldrb rt,[rn,#4]` brings it back zero-extended, and the T1 STRB imm5 is * UNSCALED where the T1 STR imm5 is word-scaled, so the two decode to the same * (rt, rn, imm, size) triple and would match each other. */ -static void strldr_cache_record_access(int rt, int rn, int imm, uint32_t puw, int size, int width, int is_store) +void strldr_cache_record_access(int rt, int rn, int imm, uint32_t puw, int size, int width, int is_store) { if (puw != 6) { @@ -2603,7 +2454,7 @@ static void strldr_cache_record_access(int rt, int rn, int imm, uint32_t puw, in * LDR redundant. Matches on all fields so a 16-bit LDR won't be elided * against a 32-bit STR (and vice versa) — the encodings might pick * different scale semantics. */ -static int strldr_cache_try_match_ldr(int rt, int rn, int imm, uint32_t puw, int size, int width) +int strldr_cache_try_match_ldr(int rt, int rn, int imm, uint32_t puw, int size, int width) { if (strldr_cache_off || puw != 6 || width != 4) return 0; @@ -2637,12 +2488,25 @@ static int strldr_cache_try_match_ldr(int rt, int rn, int imm, uint32_t puw, int return 0; } +/* A word STR of a register that already holds [rn + imm]: the memory has that + * value, so the store writes nothing new. This is the tail of a copy between + * two values frame layout or the allocator put on the same slot -- + * `ldr r0,[sp,#N]; str r0,[sp,#N]`. Same entry rules as the reload check + * (4-byte, no writeback, same encoding size, not inside an IT block, cache + * off in functions with volatile accesses). Knob: TCC_DISABLE_PASS=codegen:str_elide. */ +int strldr_cache_str_is_redundant(int rt, int rn, int imm, uint32_t puw, int size) +{ + if (strldr_str_elide_off || !thumb_gen_state.generating_function || puw != 6 || size == 0) + return 0; + return strldr_cache_try_match_ldr(rt, rn, imm, puw, size, 4); +} + /* Both halves of an LDRD already sitting in the registers it would load them * into: the instruction moves nothing. Asked per half, so a pair written by * two single STRs counts as much as one written by a STRD. Neither register * may be the base -- the load would otherwise overwrite the address it reads * from, and the entries describe a slot that no longer exists. */ -static int strldr_cache_ldrd_is_redundant(int rt, int rt2, int rn, int imm, uint32_t puw) +int strldr_cache_ldrd_is_redundant(int rt, int rt2, int rn, int imm, uint32_t puw) { if (!thumb_gen_state.generating_function || puw != 6) return 0; @@ -2655,7 +2519,7 @@ static int strldr_cache_ldrd_is_redundant(int rt, int rt2, int rn, int imm, uint /* Decode T1/T2/T3 STR/LDR immediate-offset forms with no writeback. * Returns 1 and fills outputs when the opcode matches, 0 otherwise. * *is_str_out is 1 for STR, 0 for LDR. */ -static int decode_str_ldr_imm(thumb_opcode op, int *is_str_out, int *rt_out, int *rn_out, int *imm_out, +int decode_str_ldr_imm(thumb_opcode op, int *is_str_out, int *rt_out, int *rn_out, int *imm_out, uint32_t *puw_out, int *width_out) { if (op.size == 2) @@ -2763,7 +2627,7 @@ static int decode_str_ldr_imm(thumb_opcode op, int *is_str_out, int *rt_out, int * 1110 100P U1W1 nnnn (LDRD), imm8 word-scaled in the low halfword. These * move EIGHT bytes, so a cache that only understands 4-byte accesses has to * see them or a `strd` would silently leave a stale entry for either half. */ -static int decode_strd_ldrd_imm(thumb_opcode op, int *is_str_out, int *rn_out, int *imm_out) +int decode_strd_ldrd_imm(thumb_opcode op, int *is_str_out, int *rn_out, int *imm_out) { if (op.size != 4) return 0; @@ -2786,7 +2650,7 @@ static int decode_strd_ldrd_imm(thumb_opcode op, int *is_str_out, int *rn_out, i * is skipped entirely. ot() is never called in the elided path, so `ind` * and code_size only advance for real emissions — same contract as the * MOV coalescing helper. */ -static int ot_check_ldr_imm(uint32_t rt, uint32_t rn, int imm, uint32_t puw, thumb_enforce_encoding enc) +int ot_check_ldr_imm(uint32_t rt, uint32_t rn, int imm, uint32_t puw, thumb_enforce_encoding enc) { thumb_opcode ins = th_ldr_imm(rt, rn, imm, puw, enc); if (thumb_gen_state.generating_function && puw == 6 && ins.size != 0 && @@ -2800,20 +2664,18 @@ static int ot_check_ldr_imm(uint32_t rt, uint32_t rn, int imm, uint32_t puw, thu return ot_check(ins); } -/* Emit STR Rt, [Rn, #imm]. Always emits (STR cannot be elided); the - * cache-record side effect happens inside ot() once the opcode is - * classified, so there is nothing extra to do here other than go through - * the standard ot_check path. Kept as a dedicated helper only for - * symmetry with ot_check_ldr_imm — callers use it so future refinements - * (e.g. dropping a dead store that follows another store to the same - * slot) can land in one place. */ -static int ot_check_str_imm(uint32_t rt, uint32_t rn, int imm, uint32_t puw, thumb_enforce_encoding enc) +/* Emit STR Rt, [Rn, #imm], unless the frame-slot cache knows Rt already + * holds [Rn+imm] (strldr_cache_str_is_redundant). The cache-record side + * effect of an emitted store happens inside ot() once the opcode is + * classified. */ +int ot_check_str_imm(uint32_t rt, uint32_t rn, int imm, uint32_t puw, thumb_enforce_encoding enc) { thumb_opcode ins = th_str_imm(rt, rn, imm, puw, enc); + if (strldr_cache_str_is_redundant((int)rt, (int)rn, imm, puw, ins.size)) + return 1; return ot_check(ins); } - -static uint32_t mapcc(int cc) +uint32_t mapcc(int cc) { /* In most places we carry high-level TOK_* comparisons (TOK_EQ, TOK_LT, ...). * Some IR lowering paths may already store an ARM condition code nibble @@ -2867,9 +2729,6 @@ const char *default_elfinterp(struct TCCState *s) static CType float_type, double_type, func_float_type, func_double_type; -static int unalias_ldbl(int btype); -static int is_hgen_float_aggr(CType *type); - static void th_literal_pool_init() { thumb_gen_state.literal_pool_size = 64; @@ -3159,9 +3018,12 @@ void o(unsigned int i) cur_text_section->data[ind++] = i & 255; cur_text_section->data[ind++] = i >> 8; } - -static void th_literal_pool_generate(void) +void th_literal_pool_generate(void) { + /* Not between the virtual instructions of an outlined window: the real pass + * is shorter than the rehearsal, which placed no pool there. */ + if (tcc_gen_machine_outline_suppressing()) + return; static int generating_pool = 0; /* Prevent recursive calls */ static int pool_seq = 0; @@ -3402,17 +3264,6 @@ static void th_literal_pool_generate(void) literal_pool_lookup_cache_clear(&literal_pool_last_lookup); } -/* Architectural ceiling for a pool window: the patch in - * th_literal_pool_generate() encodes (aligned_position - 4), and both the T1 - * LDR-literal and the LDRD-literal forms carry an 8-bit word-scaled offset, - * i.e. at most 1020 bytes from the load to its literal. */ -#define THUMB_POOL_RANGE_LIMIT 1020 -/* Slack under the ceiling for what the span measurement cannot see coming: a - * flush suppressed by op_in_it_block is deferred to the end of the IT block - * (up to 4 conditioned insns, each able to add a pool entry), and multi-op - * sequences reserve only an estimate of their size. */ -#define THUMB_POOL_FLUSH_SLACK 64 - /* Exact span from the earliest pending literal load to the projected end of * the pool if it were flushed after `upcoming_bytes` more code: measured from * `ind` (ground truth — includes raw o() emissions such as switch_to_data @@ -3421,7 +3272,7 @@ static void th_literal_pool_generate(void) * the B.W over the pool, worst-case alignment pad, and the exact pool bytes. * Runs identically in the dry and real passes: both advance `ind` and both * update the window fields through th_literal_pool_note_entry(). */ -static int th_pool_span_after(int upcoming_bytes) +int th_pool_span_after(int upcoming_bytes) { int pool_count = dry_run_state.active ? dry_run_literal_pool_count : thumb_gen_state.literal_pool_count; if (pool_count == 0 || thumb_gen_state.pool_window_first < 0) @@ -3433,15 +3284,14 @@ static int th_pool_span_after(int upcoming_bytes) /* Fold a freshly initialized entry into the window bookkeeping. Must run * after the caller has set patch_position/data_size (find_or_allocate cannot: * those fields are filled in afterwards). */ -static void th_literal_pool_note_entry(const ThumbLiteralPoolEntry *entry) +void th_literal_pool_note_entry(const ThumbLiteralPoolEntry *entry) { if (thumb_gen_state.pool_window_first < 0 || entry->patch_position < thumb_gen_state.pool_window_first) thumb_gen_state.pool_window_first = entry->patch_position; if (entry->shared_index == -1) thumb_gen_state.pool_bytes += (entry->data_size == 8) ? 8 : 4; } - -static void th_literal_pool_reserve_upcoming_bytes(int upcoming_bytes) +void th_literal_pool_reserve_upcoming_bytes(int upcoming_bytes) { if (!thumb_gen_state.generating_function) return; @@ -3449,8 +3299,7 @@ static void th_literal_pool_reserve_upcoming_bytes(int upcoming_bytes) if (th_pool_span_after(upcoming_bytes) >= THUMB_POOL_RANGE_LIMIT - THUMB_POOL_FLUSH_SLACK) th_literal_pool_generate(); } - -static int th_literal_pool_would_flush_for(int upcoming_bytes) +int th_literal_pool_would_flush_for(int upcoming_bytes) { if (!thumb_gen_state.generating_function) return 0; @@ -3476,7 +3325,7 @@ int is_valid_opcode(thumb_opcode op) /* B / B / CBZ / CBNZ: transfer control and write nothing else. BL, BLX and * BX are deliberately NOT here -- BL/BLX write LR and clobber caller-saved * registers and memory, and BX ends the block. */ -static int thumb_op_is_plain_branch(thumb_opcode op) +int thumb_op_is_plain_branch(thumb_opcode op) { if (op.size == 2) { @@ -3509,8 +3358,7 @@ static int thumb_op_is_plain_branch(thumb_opcode op) } return 0; } - -static int thumb_op_is_pure_flag_setter(thumb_opcode op) +int thumb_op_is_pure_flag_setter(thumb_opcode op) { uint32_t w = op.opcode; if (op.size == 2) @@ -3552,8 +3400,7 @@ static int thumb_op_is_pure_flag_setter(thumb_opcode op) } return 0; } - -static int thumb_decode_dest_reg(thumb_opcode op) +int thumb_decode_dest_reg(thumb_opcode op) { uint32_t w = op.opcode; @@ -3660,12784 +3507,3 @@ static int thumb_decode_dest_reg(thumb_opcode op) return -1; } - -int ot(thumb_opcode op) -{ - if (op.size == 0) - return op.size; - - /* Detect instructions that write to R9 when it's reserved for GOT pointer. - * Exclude push/pop/stmdb/ldmia which legitimately save/restore R9. */ - if (text_and_data_separation && !allow_r9_write) - { - int dest = thumb_decode_dest_reg(op); - if (dest == R9) - { - tcc_error("instruction 0x%0*x (size=%d) writes to R9 (GOT pointer) at ind=0x%x ir_op=%d", op.size == 4 ? 8 : 4, - op.opcode, op.size, (unsigned)ind, g_debug_current_op); - } - } - - /* Update the MOV-coalescing register-equivalence cache and the STR->LDR - * redundant-reload cache based on what is about to be emitted. This only - * tracks state — no elision happens here; elision is performed at the - * call sites via ot_check_mov_reg / ot_check_ldr_imm so that ot()'s - * return value remains the real emitted size and downstream jump/offset - * accounting never sees a phantom emission. - * - * IT blocks: instructions inside an IT/ITx/ITxy/ITxyz are conditionally - * executed. Their writes are therefore not guaranteed, so destination - * registers must be invalidated rather than recorded as equivalences. */ - if (thumb_gen_state.generating_function) - { - if (mov_equiv_it_pending > 0) - { - /* Conditional instruction: pessimistically drop anything this op - * might write, and never record new equivalences. Treat STR/LDR - * the same way — their effect is gated on the IT condition. */ - int mv_rd = -1, mv_rm = -1; - if (decode_mov_reg_plain(op, &mv_rd, &mv_rm)) - { - mov_equiv_invalidate_reg(mv_rd); - strldr_cache_invalidate_reg(mv_rd); - imm_cache_invalidate_reg(mv_rd); - } - else if (thumb_op_is_pure_flag_setter(op)) - { - /* CMP/CMN/TST/TEQ — no GPR clobber even under predication. */ - } - else if (thumb_op_is_plain_branch(op)) - { - /* Writes neither a GPR nor memory; see the unconditioned path. */ - } - else - { - int dest = thumb_decode_dest_reg(op); - if (dest >= 0) - { - mov_equiv_invalidate_reg(dest); - strldr_cache_invalidate_reg(dest); - imm_cache_invalidate_reg(dest); - } - else - { - mov_equiv_reset_all(); - tcc_gen_machine_strldr_cache_reset(); - imm_cache_reset_all(); - } - } - mov_equiv_it_pending--; - } - else - { - int it_len = mov_equiv_it_block_length(op); - if (it_len > 0) - { - /* IT itself writes no GPR; start the conditional window. */ - mov_equiv_it_pending = it_len; - } - else - { - int mv_rd = -1, mv_rm = -1; - int sl_is_str = 0, sl_rt = 0, sl_rn = 0, sl_imm = 0, sl_width = 0; - uint32_t sl_puw = 0; - int sd_is_str = 0, sd_rn = 0, sd_imm = 0; - if (decode_str_ldr_imm(op, &sl_is_str, &sl_rt, &sl_rn, &sl_imm, &sl_puw, &sl_width)) - { - if (!sl_is_str) - { - /* LDR writes Rt: invalidate both caches for that register FIRST. - * If the call-site helper ran the match it would have - * elided without reaching ot(); so if we get here, this LDR - * is actually emitting and genuinely clobbers Rt. */ - mov_equiv_invalidate_reg(sl_rt); - strldr_cache_invalidate_reg(sl_rt); - imm_cache_invalidate_reg(sl_rt); - } - /* Either way Rt now holds [Rn+imm], so both directions are worth - * recording; a store additionally kills what it overwrote. */ - strldr_cache_record_access(sl_rt, sl_rn, sl_imm, sl_puw, op.size, sl_width, sl_is_str); - if (sl_puw & 1) - { - /* Write-back form (post/pre-indexed): Rn now holds the UPDATED - * address. record_access already threw the whole strldr cache - * away (puw != 6); the register-equivalence and known-immediate - * caches must drop Rn too, or a later `mov rn,rx` / immediate - * rematerialization gets elided against the pre-increment value. */ - mov_equiv_invalidate_reg(sl_rn); - imm_cache_invalidate_reg(sl_rn); - } - } - else if (decode_mov_reg_plain(op, &mv_rd, &mv_rm)) - { - mov_equiv_record_mov(mv_rd, mv_rm); - strldr_cache_invalidate_reg(mv_rd); - imm_cache_invalidate_reg(mv_rd); - } - else if (op.size == 4 && - (((op.opcode >> 16) & 0xFE40) == 0xE840)) - { - /* LDRD/STRD (Thumb-2): encoded as 1110 100P U1W0 nnnn (STRD) or - * 1110 100P U1W1 nnnn (LDRD). Bit 20 (high-halfword bit 4) - * distinguishes load (1) vs store (0). - * - * STRD writes no GPR — only memory. LDRD writes both Rt and Rt2 - * (low-halfword bits [15:12] and [11:8] respectively). Either way - * the rest of the GPR-equivalence cache is unaffected, so don't - * fall through to the "unknown opcode → reset everything" path - * which destroys upstream coalescing wins. */ - if ((op.opcode >> 20) & 1) - { - /* LDRD: invalidate Rt and Rt2 (writeback to Rn is rare here and - * already covered by the writeback handling — for the typical - * STRD imm with W=0 used by the codegen we don't touch Rn). */ - int rt = (int)((op.opcode >> 12) & 0xF); - int rt2 = (int)((op.opcode >> 8) & 0xF); - mov_equiv_invalidate_reg(rt); - mov_equiv_invalidate_reg(rt2); - strldr_cache_invalidate_reg(rt); - strldr_cache_invalidate_reg(rt2); - imm_cache_invalidate_reg(rt); - imm_cache_invalidate_reg(rt2); - } - else if (decode_strd_ldrd_imm(op, &sd_is_str, &sd_rn, &sd_imm)) - { - /* STRD writes no GPR, but it writes EIGHT bytes of memory: with - * the cache living across IR ops, missing that leaves a stale - * entry for either half. Undecodable form -> assume the worst. */ - strldr_cache_invalidate_mem(sd_rn, sd_imm, 8); - /* Then record what it just put there, as the two 4-byte slots the - * rest of the cache is made of. That is what lets the LDRD which - * reads the pair straight back -- the return-value slot round trip - * every 64-bit helper ends with -- be skipped entirely. */ - if (sd_imm >= 0) - { - /* Only the U=1 form: every entry in this cache is a puw==6 - * access, and handing a subtract-offset one to the recorder - * would make it throw the whole cache away. */ - int sd_rt = (int)((op.opcode >> 12) & 0xF); - int sd_rt2 = (int)((op.opcode >> 8) & 0xF); - strldr_cache_record_access(sd_rt, sd_rn, sd_imm, 6, 4, 4, 1); - strldr_cache_record_access(sd_rt2, sd_rn, sd_imm + 4, 6, 4, 4, 1); - } - } - else - { - tcc_gen_machine_strldr_cache_reset(); - } - } - else if (thumb_op_is_pure_flag_setter(op)) - { - /* CMP/CMN/TST/TEQ write only the flags — no GPR clobber, no - * cache invalidation needed. */ - } - else if (thumb_op_is_plain_branch(op)) - { - /* B / B / CBZ / CBNZ write neither a GPR nor memory, so the - * FALL-THROUGH path's register and memory state is exactly what it - * was before -- and the taken path lands on an IR-level jump target, - * where codegen.c resets. Without this a conditional branch inside - * a basic block would drop every entry, which is the whole shape - * this cache exists for: reload four spilled halves, compare, branch, - * and use the same four in the not-taken arm. BL/BLX are NOT here: - * they write LR and clobber memory, and fall into the reset below. */ - } - else - { - int dest = thumb_decode_dest_reg(op); - if (dest >= 0) - { - mov_equiv_invalidate_reg(dest); - strldr_cache_invalidate_reg(dest); - imm_cache_invalidate_reg(dest); - } - else - { - mov_equiv_reset_all(); - tcc_gen_machine_strldr_cache_reset(); - imm_cache_reset_all(); - } - } - } - } - } - else - { - mov_equiv_reset_all(); - tcc_gen_machine_strldr_cache_reset(); - imm_cache_reset_all(); - } - - /* Literal-pool flush safety around IT blocks. Call-site reservations - * (th_literal_pool_reserve_upcoming_bytes) cover the block's CODE bytes, - * but a conditioned arm that materializes a large constant - * (load_full_const) grows the pool AFTER the reservation was checked, so - * the threshold can still trip mid-block. Track the architectural IT - * window here and (a) never flush while an op is conditioned, (b) flush - * BEFORE the IT opcode itself if the worst-case block — 4 code bytes plus - * an 8-byte pool entry per conditioned instruction — could hit the - * threshold, so the deferred flush of (a) never overshoots the LDR-literal - * range. Runs in both passes so dry-run and real layouts stay identical. */ - int op_in_it_block = 0; - if (thumb_gen_state.generating_function) - { - if (pool_flush_it_pending > 0) - { - op_in_it_block = 1; - pool_flush_it_pending--; - } - else - { - int it_len = mov_equiv_it_block_length(op); - if (it_len > 0) - { - /* 12 * it_len: worst case per conditioned insn is 4 code bytes plus - * an 8-byte pool entry. */ - if (th_pool_span_after(op.size + 12 * it_len) >= THUMB_POOL_RANGE_LIMIT - THUMB_POOL_FLUSH_SLACK) - th_literal_pool_generate(); - pool_flush_it_pending = it_len; - } - } - } - - /* Dry run: don't emit actual opcodes, but still track code size and - * handle literal pool generation to ensure code addresses match real pass. */ - if (dry_run_state.active) - { - if (thumb_gen_state.generating_function) - { - thumb_gen_state.code_size += op.size; - /* Check if literal pool needs to be generated during dry-run. - * We need to call th_literal_pool_generate to properly track the - * code size including the literal pool, so that ind matches - * between dry-run and real pass. */ - if (th_pool_span_after(op.size) >= THUMB_POOL_RANGE_LIMIT - THUMB_POOL_FLUSH_SLACK && !op_in_it_block) - { - th_literal_pool_generate(); - } - } - /* Increment ind as if we emitted the instruction, but don't write to section */ - ind += op.size; - return op.size; - } - - if (thumb_gen_state.generating_function) - { - thumb_gen_state.code_size += op.size; - if (th_pool_span_after(op.size) >= THUMB_POOL_RANGE_LIMIT - THUMB_POOL_FLUSH_SLACK && !op_in_it_block) - { - th_literal_pool_generate(); - } - } - - if (op.size == 4) - o(op.opcode >> 16); - o(op.opcode & 0xffff); - return op.size; -} - -static void gcall_or_jump_mop(int is_jmp, MachineOperand target); - -// TODO: this is armv7-m code -int decbranch(int pos) -{ - int xa = *(uint16_t *)(cur_text_section->data + pos); - int xb = *(uint16_t *)(cur_text_section->data + pos + 2); - - TRACE(" decbranch ins at pos 0x%.8x, target inst 0x%x 0x%x", pos, xa, xb); - - if ((xa & 0xf000) == 0xd000) - { - // Branch encoding t1 - xa &= 0x00ff; - if (xa & 0x0080) - xa -= 0x100; - xa = (xa * 2) + pos + 4; - } - else if ((xa & 0xf800) == 0xe000) - { - // Branch encoding t2 - xa &= 0x7ff; - if (xa & 0x400) - xa -= 0x800; - xa = (xa * 2) + pos + 4; - } - else if ((xa & 0xf800) == 0xf000 && (xb & 0xd000) == 0x8000) - { - // Branch encoding t3 - uint32_t s = (xa >> 10) & 1; - uint32_t imm6 = (xa & 0x3f); - uint32_t j1 = (xb >> 13) & 1; - uint32_t j2 = (xb >> 11) & 1; - uint32_t imm11 = xb & 0x7ff; - - // 10 9876543210 9876543210 9876543210 - // IMM: s 21bbbbbbaa aaaaaaaaa0 - // IMM: s21bbbbbba aaaaaaaaaa - uint32_t ret = (j2 << 19) | (j1 << 18) | (imm6 << 12) | (imm11 << 1); - if (s) - ret |= 0xfff00000; - - xa = ret + pos + 4; - } - else if ((xa & 0xf800) == 0xf000 && (xb & 0xd000) == 0x9000) - { - // Branch encoding t4 - uint32_t s = (xa >> 10) & 1; - uint32_t imm10 = (xa & 0x3ff); - uint32_t j1 = (xb >> 13) & 1; - uint32_t j2 = (xb >> 11) & 1; - uint32_t imm11 = xb & 0x7ff; - - uint32_t i1 = ~(j1 ^ s) & 1; - uint32_t i2 = ~(j2 ^ s) & 1; - - // 10 9876543210 9876543210 9876543210 - // IMM: s21bb bbbbbbbbaa aaaaaaaaa0 - uint32_t ret = (i2 << 23) | (i1 << 22) | (imm10 << 12) | (imm11 << 1); - if (s) - ret |= 0xff000000; - - xa = ret + pos + 4; - } - else if ((xa & 0xf500) == 0xb100) - { - /* CBZ/CBNZ encoding: offset = (i:imm5) * 2, forward only */ - uint32_t i_bit = (xa >> 9) & 1; - uint32_t imm5 = (xa >> 3) & 0x1f; - uint32_t imm6 = (i_bit << 5) | imm5; - xa = (int)(imm6 * 2) + pos + 4; - } - else - { - tcc_error("internal error: decbranch unknown encoding pos 0x%x, inst: 0x%x\n", pos, xa); - return 0; - } - - return xa; -} - -static thumb_opcode th_generic_mov_imm(uint32_t r, int imm) -{ - if (imm < 0) - { - return th_mvn_imm(r, 0, -imm - 1, flags_safe(), ENFORCE_ENCODING_NONE); - } - return th_mov_imm(r, imm, flags_safe(), ENFORCE_ENCODING_NONE); -} - -/* Pure query for the IR-level constant hoist: true when a 32-bit constant has - * no single-instruction materialization (all MOV/MOVW/MVN encodings fail) and - * tcc_machine_load_constant would fall through to a literal-pool load. Probes - * with r0 so the 16-bit MOVS form is reachable — encodability is the same for - * every register from there (MOV.W covers imm8 for high registers). */ -ST_FUNC int tcc_gen_machine_const_needs_pool(int32_t value) -{ - return th_generic_mov_imm(0, value).size == 0; -} -static ScratchRegAlloc th_offset_to_reg_ex(int off, int sign, uint32_t exclude_regs) -{ - /* Find a free scratch register (must not clobber excluded regs). - * Returns ScratchRegAlloc struct so caller can manage cleanup. - * Caller MUST call restore_scratch_reg() when done with the register. */ - ScratchRegAlloc alloc = get_scratch_reg_with_save(exclude_regs); - int rr = alloc.reg; - - /* If mov is not possible then load from data */ - if (!ot(th_generic_mov_imm(rr, off))) - { - load_full_const(rr, PREG_NONE, LFC_SPLIT(sign ? -off : off)); - return alloc; - } - - if (sign) - ot_check(th_rsb_imm(rr, rr, 0, flags_safe(), ENFORCE_ENCODING_NONE)); - return alloc; -} - -int th_patch_call(int t, int a) -{ - uint16_t *x = (uint16_t *)(cur_text_section->data + t); - int lt = t; - - TRACE("'th_patch_call' t: %.8x, a: %.8x\n", t, a); - - t = decbranch(t); - TRACE("t: %.8x\n", t); - if (a == lt + 2) - *x = 0xbf00; - else if ((*x & 0xf000) == 0xd000) - { - *x &= 0xff00; - *x |= th_encbranch_8(lt, a); - } - else if ((*x & 0xf800) == 0xe000) - { - *x &= 0xf800; - *x |= th_encbranch_11(lt, a); - } - else if ((x[0] & 0xf800) == 0xf000 && (x[1] & 0xd000) == 0x8000) - { - uint32_t enc = 0; - x[0] &= 0xfbc0; - x[1] &= 0xd000; - enc = th_encbranch_b_t3(th_encbranch_20(lt, a)); - x[0] |= enc >> 16; - x[1] |= enc; - } - else if ((x[0] & 0xf800) == 0xf000 && (x[1] & 0xd000) == 0x9000) - { - uint32_t enc = 0; - x[0] &= 0xf800; - x[1] &= 0xd000; - enc = th_packimm_10_11_0(th_encbranch_20(lt, a) << 1); - x[0] |= enc >> 16; - x[1] |= enc; - } - else if ((*x & 0xf500) == 0xb100) - { - /* CBZ/CBNZ: 16-bit, forward-only, range 0-126 bytes. - * CBZ base = 0xb100, CBNZ base = 0xb900; both match (x & 0xf500) == 0xb100 - * since bit 11 (0x0800) is not in the mask. - * Encoding: op | (i << 9) | (imm5 << 3) | Rn - * where offset = (i:imm5) * 2 */ - int offset = a - (lt + 4); /* PC-relative, Thumb PC = insn + 4 */ - if (offset < 0 || offset > 126 || (offset & 1)) - tcc_error("compiler_error: CBZ/CBNZ target out of range: offset=%d", offset); - uint32_t imm6 = (uint32_t)offset >> 1; - uint32_t i_bit = (imm6 >> 5) & 1; - uint32_t imm5 = imm6 & 0x1f; - *x &= 0xfd07; /* Keep base opcode, NZ bit, and Rn */ - *x |= (uint16_t)((i_bit << 9) | (imm5 << 3)); - } - else - tcc_error("compiler_error: unhandled branch type in th_patch_call for: t: " - "0x%x, a: 0x%x, x: 0x%x 0x%x\n", - t, a, x[0], x[1]); - - return t; -} - -/* Add a value to SP. When the immediate doesn't fit the ADD/SUB SP encoding, - * a scratch register is needed. scratch_reg selects which one: - * >= 0 : use that specific physical register (caller guarantees it's free) - * < 0 : default to R_IP (safe in prologue/epilogue where R0-R3 hold args) - */ -static void gadd_sp_ex(int val, int scratch_reg) -{ - if (val == 0) - return; - - if (scratch_reg < 0) - scratch_reg = R_IP; - - if (val > 0) - { - thumb_opcode add_imm = th_add_imm(R_SP, R_SP, (uint32_t)val, flags_safe(), ENFORCE_ENCODING_NONE); - if (is_valid_opcode(add_imm)) - { - ot(add_imm); - return; - } - - load_full_const(scratch_reg, PREG_NONE, (uint32_t)val, 0); - ot_check( - th_add_reg(R_SP, R_SP, scratch_reg, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); - return; - } - - /* val < 0 */ - const uint32_t sub = (uint32_t)(-val); - thumb_opcode sub_imm = th_sub_imm(R_SP, R_SP, sub, flags_safe(), ENFORCE_ENCODING_NONE); - if (is_valid_opcode(sub_imm)) - { - ot(sub_imm); - return; - } - - load_full_const(scratch_reg, PREG_NONE, (uint32_t)sub, 0); - ot_check(th_sub_reg(R_SP, R_SP, scratch_reg, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); -} - -static void gadd_sp(int val) -{ - gadd_sp_ex(val, -1); -} - -void ggoto(void) -{ - TRACE("'ggoto'"); - { - SValue target = *vtop; - tcc_ir_put(tcc_state->ir, TCCIR_OP_IJUMP, &target, NULL, NULL); - } - vtop--; - print_vstack("ggoto"); -} - -ST_FUNC void tcc_gen_machine_indirect_jump_mop(MachineOperand src, TccIrOp op) -{ - (void)op; - MachineCodegenContext ctx = {0}; - int target = mach_ensure_in_reg(&ctx, &src, 0); - ot_check(th_bx_reg((uint16_t)target)); - mach_release_all(&ctx); -} - -/* Returns the number of bytes emitted by tcc_gen_machine_switch_table_mop for - * a table with the given number of entries. Used by the dry-run pass in - * codegen.c so that branch-offset analysis is accurate without the backend - * having to emit any real instructions. */ -ST_FUNC int tcc_gen_machine_switch_table_dry_run_size(int num_entries) -{ - /* Layout: LSL.W(4) + ADD(2) + LDR.W(4) + ADD(2) + BX(2) = 14 bytes preamble - * + 4 bytes per table entry (32-bit signed PC-relative offsets). */ - return 14 + num_entries * 4; -} - -/* Force any pending literal pool to be flushed before a region of - * `upcoming_bytes` is emitted, if leaving the pool pending that long would - * push its load out of range. Public wrapper so codegen.c can reserve - * space symmetrically in both the dry-run and real-run passes. - * - * The SWITCH_TABLE dispatch needs this: its preamble (LSL/ADD/LDR/ADD/BX) - * must be emitted atomically — a literal-pool flush in the middle relocates - * the terminal `ADD Rt, PC; BX Rt` past the pool (bridged by a B.W), which - * invalidates the `ref_point == table_start` assumption that the switch- - * table offset backpatch in codegen.c relies on, producing a wild jump. - * Flushing the pool up front (in both passes, so dry-run size estimates and - * real-run addresses stay consistent) keeps the preamble + table contiguous. */ -ST_FUNC void tcc_gen_machine_reserve_pool_bytes(int upcoming_bytes) -{ - th_literal_pool_reserve_upcoming_bytes(upcoming_bytes); -} - -/* MOP variant: accepts a MachineOperand for the index register. */ -ST_FUNC void tcc_gen_machine_switch_table_mop(MachineOperand src, TCCIRSwitchTable *table, TCCIRState *ir, int ir_idx) -{ - (void)ir_idx; - - TRACE("'tcc_gen_machine_switch_table_mop' table_id=%d entries=%d\n", table - ir->switch_tables, table->num_entries); - - MachineCodegenContext ctx = {0}; - /* The index value must be in a register at this point. */ - int index_reg = mach_ensure_in_reg(&ctx, &src, 0); - if (!thumb_is_hw_reg(index_reg)) - tcc_error("internal error: SWITCH_TABLE index not in a hardware register (mop)"); - - /* Use R_IP as scratch to avoid clobbering index_reg, which may still be - live at the switch targets (SSA can place the loop counter directly here). */ - int rt = R_IP; - - ot_check(th_lsl_imm(rt, index_reg, 2, flags_safe(), ENFORCE_ENCODING_32BIT)); - ot_check(th_add_reg(rt, rt, R_PC, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); - ot_check_ldr_imm(rt, rt, 6, 6, ENFORCE_ENCODING_32BIT); - ot_check(th_add_reg(rt, rt, R_PC, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); - ot_check(th_bx_reg(rt)); - - int table_start = ind; - /* The offset table is data in .text — mark it for disassemblers. */ - map_sym_d(); - for (int i = 0; i < table->num_entries; i++) - { - g(0); - g(0); - g(0); - g(0); - } - map_sym_t(); - table->table_code_addr = table_start; - mach_release_all(&ctx); -} - -/* SWITCH_LOAD: data-table dispatch that loads values[index] into dest. - * - * Layout (uniform 14-byte preamble): - * - * LSL.W rt, index, #2 (4 bytes) - * ADD rt, rt, pc (2 bytes) ; PC=preamble_start+8 - * LDR.W ip, [rt, #6] (4 bytes) ; load table[index] -> ip - * B.W skip (4 bytes) ; jump past the table - *
(4*N bytes) - * skip: - * [optional STR/MOV ip -> dest] ; only if dest is spilled, - * ; emitted by the IR-level - * ; ASSIGN that follows. - * - * The fixed loaded register is R_IP (same as SWITCH_TABLE's scratch); the - * IR-level optimization wraps SWITCH_LOAD with an ASSIGN that places IP into - * the real dest, so we don't need a separate spill path here. - * - * SYMREF entries emit R_ARM_ABS32 relocations at their table slots; the - * linker fills in the absolute symbol address. - */ -/* SWITCH_LOAD dispatch size: literal-pool LDR (4 bytes, T2 encoding for - * R_IP) + indexed shifted LDR.W (4 bytes). The table itself lives in - * .rodata and contributes no .text bytes. */ -ST_FUNC int tcc_gen_machine_switch_load_dry_run_size(int num_entries) -{ - (void)num_entries; - return 8; -} - -ST_FUNC void tcc_gen_machine_switch_load_mop(MachineOperand src, MachineOperand dest, TCCIRSwitchValueTable *vtab, - TCCIRState *ir, int ir_idx) -{ - (void)ir_idx; - (void)ir; - - TRACE("'tcc_gen_machine_switch_load_mop' vt_id=%d entries=%d\n", (int)(vtab - ir->switch_value_tables), - vtab->num_entries); - - if (!vtab->rodata_sym) - tcc_error("internal error: SWITCH_LOAD table has no rodata symbol (switch_to_data should have allocated it)"); - - MachineCodegenContext ctx = {0}; - /* Keep the index out of R_IP, which we clobber with the table base below. */ - int index_reg = mach_ensure_in_reg(&ctx, &src, (1u << (uint32_t)R_IP)); - if (!thumb_is_hw_reg(index_reg)) - tcc_error("internal error: SWITCH_LOAD index not in a hardware register"); - - /* Resolve the destination register. The switch_to_data optimization tries to - * keep the SWITCH_LOAD dest in a hardware register, but under high register - * pressure the allocator can spill it (or it may be an lvalue store). Rather - * than bail out, allocate a scratch via mach_get_dest_reg() and store it back - * with mach_writeback_dest() afterwards. Exclude index_reg and R_IP — both - * are read by the indexed load below. */ - uint32_t dest_excl = (1u << (uint32_t)index_reg) | (1u << (uint32_t)R_IP); - int dest_reg = mach_get_dest_reg(&ctx, &dest, dest_excl); - - /* Load the table's base address from the literal pool into IP. */ - _lfc_sym = vtab->rodata_sym; - load_full_const(R_IP, PREG_NONE, 0, 0); - - /* dest = table[index] via LDR.W dest, [ip, index, LSL #2]. */ - thumb_shift shift = {THUMB_SHIFT_LSL, 2, THUMB_SHIFT_IMMEDIATE}; - ot_check(th_ldr_reg((uint32_t)dest_reg, (uint32_t)R_IP, (uint32_t)index_reg, shift, ENFORCE_ENCODING_32BIT)); - - /* If the dest was a spill slot or lvalue, write the loaded value back. */ - mach_writeback_dest(&dest, dest_reg); - - mach_release_all(&ctx); -} - -void gsym_addr(int t, int a) -{ - TRACE("'gsym_addr' %.8x branch target: %.8x\n", t, a); - - while (t > 0) /* -1 or 0 means end of chain / no chain */ - t = th_patch_call(t, a); -} - -ST_FUNC void gen_vla_alloc(CType *type, int align) -{ - /* vtop holds the allocation size in bytes. Adjust SP down by that runtime - * size and align it to at least 8 bytes. - * - * This follows the classic TCC scheme: - * r = sp - size - * r = r & ~(align-1) - * sp = r - * - * The size expression is consumed from the value stack. - */ - (void)type; - - int r = gv(RC_INT); - - /* r = SP - r */ - ot_check(th_sub_reg(r, R_SP, r, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); - - if (align < 8) - align = 8; - if (align & (align - 1)) - tcc_error("alignment is not a power of 2: %i", align); - - if (align > 1) - { - /* Try immediate BIC first; if it doesn't encode, fall back to register mask. */ - if (!ot(th_bic_imm(r, r, (uint32_t)(align - 1), flags_safe(), ENFORCE_ENCODING_NONE))) - { - ScratchRegAlloc mask_alloc = get_scratch_reg_with_save(1u << r); - int mask_reg = mask_alloc.reg; - if (!ot(th_generic_mov_imm(mask_reg, align - 1))) - { - load_full_const(mask_reg, PREG_NONE, LFC_SPLIT(align - 1)); - } - ot_check(th_bic_reg(r, r, mask_reg, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); - if (mask_alloc.saved) - { - ot_check(th_pop(1u << mask_reg)); - } - } - } - - /* SP = r */ - ot_check_mov_reg(R_SP, r, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE, false); - - vpop(); -} - -ST_FUNC void gen_vla_sp_save(int addr) -{ - if (nocode_wanted) - return; - - /* Store SP to the local stack slot at frame offset `addr`. */ - int off = fp_adjust_local_offset(addr, 0 /* not param */); - int sign = (off < 0) ? 1 : 0; - int abs_off = sign ? -off : off; - const int base_reg = tcc_state->need_frame_pointer ? R_FP : R_SP; - - ScratchRegAlloc vla_sc = get_scratch_reg_with_save(0); - ot_check_mov_reg(vla_sc.reg, R_SP, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE, false); - th_store32_imm_or_reg_ex(vla_sc.reg, base_reg, abs_off, sign, 0); - restore_scratch_reg(&vla_sc); -} - -ST_FUNC void gen_vla_sp_restore(int addr) -{ - if (nocode_wanted) - return; - - /* Load SP from the local stack slot at frame offset `addr`. */ - int off = fp_adjust_local_offset(addr, 0 /* not param */); - int sign = (off < 0) ? 1 : 0; - int abs_off = sign ? -off : off; - const int base_reg = tcc_state->need_frame_pointer ? R_FP : R_SP; - - ScratchRegAlloc vla_sc = get_scratch_reg_with_save(0); - load_from_base(vla_sc.reg, PREG_REG_NONE, IROP_BTYPE_INT32, 0, abs_off, sign, base_reg); - ot_check_mov_reg(R_SP, vla_sc.reg, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE, false); - restore_scratch_reg(&vla_sc); -} - -int load_ushort_from_base(int ir, int base, int fc, int sign) -{ - const thumb_opcode ins = th_ldrh_imm(ir, base, fc, sign ? 4 : 6, ENFORCE_ENCODING_NONE); - TRACE("Load ushort sign: %d, r %d, base: %d, fc: %d\n", sign, ir, base, fc, sign); - return ot(ins); -} - -int load_byte_from_base(int ir, int base, int fc, int sign) -{ - const thumb_opcode ins = th_ldrsb_imm(ir, base, fc, sign ? 4 : 6, ENFORCE_ENCODING_NONE); - TRACE("Load byte sign: %d, r %d, base: %d, fc: %d\n", sign, ir, base, fc, sign); - return ot(ins); -} - -int load_ubyte_from_base(int ir, int base, int fc, int sign) -{ - const thumb_opcode ins = th_ldrb_imm(ir, base, fc, sign ? 4 : 6, ENFORCE_ENCODING_NONE); - TRACE("Load ubyte sign: %d, r %d, base: %d, fc: %d\n", sign, ir, base, fc, sign); - return ot(ins); -} - -int load_word_from_base(int ir, int base, int fc, int sign) -{ - const thumb_opcode ins = th_ldr_imm(ir, base, fc, sign ? 4 : 6, ENFORCE_ENCODING_NONE); - TRACE("Load word sign: %d, r %d, base: %d, fc: %d\n", sign, ir, base, fc, sign); - /* This is the path every spill reload takes, so it -- not just the - * ot_check_ldr_imm call sites -- is where the memory cache has to be - * consulted. Returning the encoded size without emitting keeps the - * caller's "did it encode?" contract intact; only `ind` stands still. */ - if (ins.size != 0 && thumb_gen_state.generating_function && !sign && - strldr_cache_try_match_ldr(ir, base, fc, 6, ins.size, 4)) - return ins.size; - /* Satisfying the load from ANOTHER register holding the slot (`mov rt,rh`) - * was built and MEASURED on the rig, and it is a small LOSS -- rijndael and - * double_mul improve, double_add/cmp/mixed/div all give more back, total - * -0.09% against -0.12% for this same-register form alone. A `mov` and a - * cached-line `ldr` from the frame cost about the same on M33, so all it - * really does is move code. Do not re-propose without a measurement. */ - return ot(ins); -} - -int store_word_to_base(int ir, int base, int fc, int sign) -{ - const thumb_opcode ins = th_str_imm(ir, base, fc, sign ? 4 : 6, ENFORCE_ENCODING_NONE); - TRACE("Store word sign: %d, r %d, base: %d, fc: %d\n", sign, ir, base, fc, sign); - return ot(ins); -} - -/* Returns 1 if a 64-bit access at (sym + addend) is guaranteed 4-byte aligned - * (so LDRD/STRD is safe). Conservative: only allows natural alignment for - * non-struct, non-packed symbols, plus any explicit alignment >= 4. */ -static int sym_is_4_byte_aligned_for_64bit(Sym *sym, int32_t addend) -{ - if (!sym) - return 0; - if ((addend & 3) != 0) - return 0; - if (sym->a.packed) - return 0; - if (sym->a.aligned >= 3) /* explicit alignment 2^(n-1) >= 4 */ - return 1; - if (sym->a.aligned > 0) /* explicit 1 or 2 byte alignment — not safe */ - return 0; - /* sym->a.aligned == 0: rely on the declared type's natural alignment. - * Structs/unions may be packed-wrapped; reject conservatively. Native - * scalars (long long, double, pointer) have natural alignment >= 4. */ - int btype = sym->type.t & VT_BTYPE; - if (btype == VT_STRUCT) - return 0; - return 1; -} - -/* Try to emit STRD Rt, Rt2, [base, #±abs_off] for a 64-bit paired store. - * Constraints (Thumb-2 STRD imm T1): - * - Rt != Rt2 - * - Rt, Rt2 in r0..r12 or r14 (not SP, not PC) - * - abs_off 4-byte aligned and <= 1020 - * Returns 1 on success, 0 if the caller must fall back to two 32-bit stores. */ -static int try_strd_pair(int lo_reg, int hi_reg, int base, int abs_off, int sign) -{ - if ((unsigned)base > 15) - return 0; - if ((abs_off & 3) != 0 || abs_off > 1020) - return 0; - if (lo_reg < 0 || lo_reg > R_LR || lo_reg == R_SP) - return 0; - if (hi_reg < 0 || hi_reg > R_LR || hi_reg == R_SP) - return 0; - const uint32_t puw = sign ? 4u : 6u; - ot_check(th_strd_imm((uint32_t)lo_reg, (uint32_t)hi_reg, (uint32_t)base, abs_off, puw)); - return 1; -} - -/* Mirror of try_strd_pair for LDRD. Same register and offset constraints; - * the caller is responsible for guaranteeing 4-byte alignment of the target - * address (stack, or a symbol that passes sym_is_4_byte_aligned_for_64bit). */ -static int try_ldrd_pair(int lo_reg, int hi_reg, int base, int abs_off, int sign) -{ - if ((abs_off & 3) != 0 || abs_off > 1020) - return 0; - if (lo_reg < 0 || lo_reg > R_LR || lo_reg == R_SP) - return 0; - if (hi_reg < 0 || hi_reg > R_LR || hi_reg == R_SP) - return 0; - if (lo_reg == hi_reg) - return 0; - const uint32_t puw = sign ? 4u : 6u; - if (strldr_cache_ldrd_is_redundant(lo_reg, hi_reg, base, abs_off, puw)) - return 1; - ot_check(th_ldrd_imm((uint32_t)lo_reg, (uint32_t)hi_reg, (uint32_t)base, abs_off, puw)); - return 1; -} - -/* Emit a single STR to a spill slot. Used by the codegen STRD pairing logic - * to flush a pending store when pairing wasn't possible. */ -ST_FUNC void tcc_gen_machine_store_spill(int src_reg, int32_t spill_offset) -{ - const int base_reg = tcc_state->need_frame_pointer ? R_FP : R_SP; - int adj = fp_adjust_local_offset(spill_offset, 0); - int sign = (adj < 0); - int abs_off = sign ? -adj : adj; - ot_check_str_imm((uint32_t)src_reg, (uint32_t)base_reg, - abs_off, sign ? 4u : 6u, ENFORCE_ENCODING_NONE); -} - -/* Try to emit STRD for two 32-bit values to adjacent spill slots. - * off1 must be the lower offset (off1 + 4 == off2). - * Returns 1 on success, 0 if STRD constraints not met. */ -ST_FUNC int tcc_gen_machine_try_strd_spill(int reg1, int32_t off1, int reg2, int32_t off2) -{ - if (off1 + 4 != off2) - return 0; - const int base_reg = tcc_state->need_frame_pointer ? R_FP : R_SP; - int adj = fp_adjust_local_offset(off1, 0); - int sign = (adj < 0); - int abs_off = sign ? -adj : adj; - return try_strd_pair(reg1, reg2, base_reg, abs_off, sign); -} - -/* Try to emit LDRD for two 32-bit values from adjacent spill slots. - * off1 must be the lower offset (off1 + 4 == off2). - * Returns 1 on success, 0 if LDRD constraints not met. */ -ST_FUNC int tcc_gen_machine_try_ldrd_spill(int reg1, int32_t off1, int reg2, int32_t off2) -{ - if (off1 + 4 != off2) - return 0; - const int base_reg = tcc_state->need_frame_pointer ? R_FP : R_SP; - int adj = fp_adjust_local_offset(off1, 0); - int sign = (adj < 0); - int abs_off = sign ? -adj : adj; - return try_ldrd_pair(reg1, reg2, base_reg, abs_off, sign); -} - -/* Try to emit LDRD/STRD for two 32-bit values from adjacent offsets off a - * generic base register (not FP/SP). Used by the LOAD_INDEXED/STORE_INDEXED - * pairing peephole. `off` is the lower offset (caller has verified - * off + 4 fits within the same access range). Returns 1 on success. */ -ST_FUNC int tcc_gen_machine_try_ldrd_base(int reg1, int reg2, int base_reg, int32_t off) -{ - int sign = (off < 0); - int abs_off = sign ? -off : off; - return try_ldrd_pair(reg1, reg2, base_reg, abs_off, sign); -} - -ST_FUNC int tcc_gen_machine_try_strd_base(int reg1, int reg2, int base_reg, int32_t off) -{ - int sign = (off < 0); - int abs_off = sign ? -off : off; - return try_strd_pair(reg1, reg2, base_reg, abs_off, sign); -} - -ST_FUNC int tcc_gen_machine_try_strd_imm_spill(int64_t val1, int64_t val2, - int32_t off1, int32_t off2) -{ - if (off1 + 4 != off2) - return 0; - const int base_reg = tcc_state->need_frame_pointer ? R_FP : R_SP; - int adj = fp_adjust_local_offset(off1, 0); - int sign = (adj < 0); - int abs_off = sign ? -adj : adj; - if ((abs_off & 3) != 0 || abs_off > 1020) - return 0; - - MachineCodegenContext ctx = {0}; - /* Materializing the immediates may PUSH the scratch register(s) when FP is - * omitted and no scratch-save area is reserved, lowering SP by 4 per push. - * The STRD destination is SP-relative, so an uncompensated offset would write - * the pair 4*pushes bytes below the intended slot — the array/struct - * initializer then lands at the wrong offset and later reads return stale - * data (fuzz seed 12057). Snapshot the push stack so we can measure the SP - * shift after acquiring the registers and fold it into the offset. */ - int spc_before = scratch_push_count; - MachineOperand op1 = {.kind = MACH_OP_IMM, .u.imm.val = val1}; - int r1 = mach_ensure_in_reg(&ctx, &op1, 0); - int r2; - if (val1 == val2) { - r2 = r1; - } else { - MachineOperand op2 = {.kind = MACH_OP_IMM, .u.imm.val = val2}; - r2 = mach_ensure_in_reg(&ctx, &op2, (1u << (uint32_t)r1)); - } - if (r1 == R_SP || r2 == R_SP) { - mach_release_all(&ctx); - return 0; - } - /* Account for any real SP-lowering pushes (type 1) done above. Saves routed - * to a reserved scratch area (type 2) keep SP stable and need no adjustment. - * The shift only affects an SP-relative base; an FP base is unperturbed. */ - if (base_reg == R_SP) { - int sp_shift = 0; - for (int s = spc_before; s < scratch_push_count && s < 128; s++) - if (scratch_push_type[s] == 1) - sp_shift += 4; - if (sp_shift) { - /* Only the positive (above-SP) local case is safe to compensate by simple - * addition; a negative (below-SP) offset combined with the shift is rare - * and not worth special-casing — fall back to per-element stores. */ - if (sign || abs_off + sp_shift > 1020) { - mach_release_all(&ctx); - return 0; - } - abs_off += sp_shift; - } - } - const uint32_t puw = sign ? 4u : 6u; - ot_check(th_strd_imm((uint32_t)r1, (uint32_t)r2, (uint32_t)base_reg, abs_off, puw)); - mach_release_all(&ctx); - return 1; -} - -ST_FUNC int tcc_gen_machine_try_strd_imm_base(int64_t val1, int64_t val2, - int base_reg, int32_t off) -{ - int sign = (off < 0); - int abs_off = sign ? -off : off; - if ((unsigned)base_reg > 15) - return 0; - if ((abs_off & 3) != 0 || abs_off > 1020) - return 0; - - uint32_t excl = (1u << (uint32_t)base_reg); - MachineCodegenContext ctx = {0}; - MachineOperand op1 = {.kind = MACH_OP_IMM, .u.imm.val = val1}; - int r1 = mach_ensure_in_reg(&ctx, &op1, excl); - int r2; - if (val1 == val2) { - r2 = r1; - } else { - MachineOperand op2 = {.kind = MACH_OP_IMM, .u.imm.val = val2}; - r2 = mach_ensure_in_reg(&ctx, &op2, excl | (1u << (uint32_t)r1)); - } - if (r1 == R_SP || r2 == R_SP) { - mach_release_all(&ctx); - return 0; - } - const uint32_t puw = sign ? 4u : 6u; - ot_check(th_strd_imm((uint32_t)r1, (uint32_t)r2, (uint32_t)base_reg, abs_off, puw)); - mach_release_all(&ctx); - return 1; -} - -ST_FUNC int tcc_machine_can_encode_stack_offset_for_reg(int frame_offset, int dest_reg) -{ - /* Check if frame_offset can be directly encoded in ldr/str instructions - * without requiring a scratch register. This is used to avoid wasteful - * address materialization when the backend can handle the offset directly. - * Tests with dest_reg since encoding availability depends on the register. */ - /* Adjust for callee-saved gap below FP (spill offsets are always locals) */ - frame_offset = fp_adjust_local_offset(frame_offset, 0); - const int base_reg = tcc_state->need_frame_pointer ? R_FP : R_SP; - const int sign = (frame_offset < 0); - const int abs_offset = sign ? -frame_offset : frame_offset; - - /* Try to encode as ldr instruction with the actual destination register. - * Some encodings (e.g., Thumb-1 T1) only work with low registers (r0-r7). */ - const thumb_opcode ins = th_ldr_imm(dest_reg, base_reg, abs_offset, sign ? 4 : 6, ENFORCE_ENCODING_NONE); - return (ins.size != 0); -} - -ST_FUNC int tcc_machine_can_encode_stack_offset_with_param_adj(int frame_offset, int is_param, int dest_reg) -{ - /* Like tcc_machine_can_encode_stack_offset_for_reg, but applies offset_to_args for VT_PARAM. - * Stack parameters need offset_to_args adjustment (prologue push size). */ - int offset = frame_offset; - if (is_param) - offset += offset_to_args; - return tcc_machine_can_encode_stack_offset_for_reg(offset, dest_reg); -} - -ST_FUNC void tcc_machine_load_spill_slot(int dest_reg, int frame_offset) -{ - if (dest_reg == PREG_REG_NONE) - tcc_error("compiler_error: load_spill_slot requires a destination register"); - - /* Adjust for callee-saved gap below FP (spill slots are always locals) */ - frame_offset = fp_adjust_local_offset(frame_offset, 0); - - /* Peephole: if the previous emit was a STR or LDR of the same register to/from - * the same slot AND no other instruction has been emitted since, the value is - * already in dest_reg — skip the redundant load. */ - TCCIRState *ir = tcc_state ? tcc_state->ir : NULL; - if (ir && ir->spill_cache.last_emit_kind != 0 && - ir->spill_cache.last_emit_ind == ind && - ir->spill_cache.last_emit_reg == dest_reg && - ir->spill_cache.last_emit_offset == frame_offset) - { - return; - } - - const int base_reg = tcc_state->need_frame_pointer ? R_FP : R_SP; - const int sign = (frame_offset < 0); - const int abs_offset = sign ? -frame_offset : frame_offset; - - if (!load_word_from_base(dest_reg, base_reg, abs_offset, sign)) - { - ScratchRegAlloc rr_alloc = th_offset_to_reg_ex(abs_offset, sign, (1u << dest_reg) | (1u << base_reg)); - int rr = rr_alloc.reg; - ot_check(th_ldr_reg(dest_reg, base_reg, rr, THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); - restore_scratch_reg(&rr_alloc); - } - - if (ir) - { - ir->spill_cache.last_emit_kind = 2; /* LDR */ - ir->spill_cache.last_emit_ind = ind; - ir->spill_cache.last_emit_reg = (int8_t)dest_reg; - ir->spill_cache.last_emit_offset = frame_offset; - } -} - -ST_FUNC void tcc_machine_store_spill_slot(int src_reg, int frame_offset) -{ - if (src_reg == PREG_REG_NONE) - tcc_error("compiler_error: store_spill_slot requires a source register"); - - /* Adjust for callee-saved gap below FP (spill slots are always locals) */ - frame_offset = fp_adjust_local_offset(frame_offset, 0); - const int base_reg = tcc_state->need_frame_pointer ? R_FP : R_SP; - const int sign = (frame_offset < 0); - const int abs_offset = sign ? -frame_offset : frame_offset; - - if (!store_word_to_base(src_reg, base_reg, abs_offset, sign)) - { - /* Avoid clobbering the other half of a 64-bit value when storing - * paired registers. The allocator uses adjacent register pairs for - * 64-bit values (e.g. r0/r1, r2/r3, r4/r5). When storing one half, - * do not use the adjacent register as the scratch offset register. - */ - uint32_t extra_exclude = 0; - if (src_reg >= ARM_R0 && src_reg <= ARM_R12) - { - int adj = (src_reg & 1) ? (src_reg - 1) : (src_reg + 1); - if (adj >= ARM_R0 && adj <= ARM_R12 && adj != ARM_SP && adj != ARM_PC) - extra_exclude |= (1u << adj); - } - - ScratchRegAlloc rr_alloc = - th_offset_to_reg_ex(abs_offset, sign, (1u << src_reg) | (1u << base_reg) | extra_exclude); - int rr = rr_alloc.reg; - ot_check(th_str_reg(src_reg, base_reg, rr, THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); - restore_scratch_reg(&rr_alloc); - } - - TCCIRState *ir = tcc_state ? tcc_state->ir : NULL; - if (ir) - { - ir->spill_cache.last_emit_kind = 1; /* STR */ - ir->spill_cache.last_emit_ind = ind; - ir->spill_cache.last_emit_reg = (int8_t)src_reg; - ir->spill_cache.last_emit_offset = frame_offset; - } -} - -/* Like tcc_machine_store_spill_slot, but for stack-passed parameters. - * Adds offset_to_args (prologue push size) to the frame offset so that - * the store targets the correct caller-stack location above FP. */ -ST_FUNC void tcc_machine_store_param_slot(int src_reg, int frame_offset) -{ - tcc_machine_store_spill_slot(src_reg, frame_offset + offset_to_args); -} - -static int unalias_ldbl(int btype) -{ -#if LDOUBLE_SIZE == 8 - if (btype == VT_LDOUBLE) - btype = VT_DOUBLE; -#endif - return btype; -} - -/* Return whether a structure is an homogeneous float aggregate or not. - The answer is true if all the elements of the structure are of the same - primitive float type and there is less than 4 elements. - - type: the type corresponding to the structure to be tested */ -static int is_hgen_float_aggr(CType *type) -{ - if ((type->t & VT_BTYPE) == VT_STRUCT) - { - struct Sym *ref; - int btype, nb_fields = 0; - - ref = type->ref->next; - if (ref) - { - btype = unalias_ldbl(ref->type.t & VT_BTYPE); - if (btype == VT_FLOAT || btype == VT_DOUBLE) - { - for (; ref && btype == unalias_ldbl(ref->type.t & VT_BTYPE); ref = ref->next, nb_fields++) - ; - return !ref && nb_fields <= 4; - } - } - } - return 0; -} - -// How many registers are necessary to return struct via registers -// if not possible, then 0 means return via struct pointer -ST_FUNC int gfunc_sret(CType *vt, int variadic, CType *ret, int *ret_align, int *regsize) -{ - int align; - const int size = type_size(vt, &align); - - TRACE("'gfunc_sret'"); - if (float_abi == ARM_HARD_FLOAT && !variadic && (is_float(vt->t) || is_hgen_float_aggr(vt))) - { - *ret_align = 8; - *regsize = 8; - ret->ref = NULL; - ret->t = VT_DOUBLE; - return ceil_div(size, 8); - } - else if (size > 0 && size <= 4) - { - *ret_align = 4; - *regsize = 4; - ret->ref = NULL; - ret->t = VT_INT; - return 1; - } - return 0; -} - -// are those offsets to allow TREG_R0 start from other register than r0? -// not sure - -static void th_store32_imm_or_reg_ex(int src_reg, uint32_t base_reg, int abs_off, int sign, uint32_t extra_exclude) -{ - if (!ot(th_str_imm(src_reg, base_reg, abs_off, sign ? 4 : 6, ENFORCE_ENCODING_NONE))) - { - ScratchRegAlloc rr_alloc = th_offset_to_reg_ex(abs_off, sign, (1u << src_reg) | (1u << base_reg) | extra_exclude); - int rr = rr_alloc.reg; - ot_check(th_str_reg(src_reg, base_reg, rr, THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); - restore_scratch_reg(&rr_alloc); - } -} - -static void th_store16_imm_or_reg(int src_reg, uint32_t base_reg, int abs_off, int sign) -{ - if (!ot(th_strh_imm(src_reg, base_reg, abs_off, sign ? 4 : 6, ENFORCE_ENCODING_NONE))) - { - ScratchRegAlloc rr_alloc = th_offset_to_reg_ex(abs_off, sign, (1u << src_reg) | (1u << base_reg)); - int rr = rr_alloc.reg; - ot_check(th_strh_reg(src_reg, base_reg, rr, THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); - restore_scratch_reg(&rr_alloc); - } -} - -static void th_store8_imm_or_reg(int src_reg, uint32_t base_reg, int abs_off, int sign) -{ - if (!ot(th_strb_imm(src_reg, base_reg, abs_off, sign ? 4 : 6, ENFORCE_ENCODING_NONE))) - { - ScratchRegAlloc rr_alloc = th_offset_to_reg_ex(abs_off, sign, (1u << src_reg) | (1u << base_reg)); - int rr = rr_alloc.reg; - ot_check(th_strb_reg(src_reg, base_reg, rr, THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); - restore_scratch_reg(&rr_alloc); - } -} - -static ThumbLiteralPoolEntry *th_literal_pool_allocate() -{ - ThumbLiteralPoolEntry *entry; - - pool_entries_total++; - - /* During dry-run, use separate pool to avoid modifying the real pool. - * This prevents memory corruption when restoring state after dry-run. */ - if (dry_run_state.active) - { - if (dry_run_literal_pool_count >= dry_run_literal_pool_size) - { - dry_run_literal_pool_size <<= 1; - dry_run_literal_pool = - tcc_realloc(dry_run_literal_pool, dry_run_literal_pool_size * sizeof(ThumbLiteralPoolEntry)); - tcc_chained_hash_reserve(&literal_pool_hash, dry_run_literal_pool_size); - } - entry = &dry_run_literal_pool[dry_run_literal_pool_count++]; - entry->sym = NULL; - entry->relocation = -1; - entry->shared_index = -1; - /* Track the count in the main state for code size calculations */ - thumb_gen_state.literal_pool_count++; - return entry; - } - - if (thumb_gen_state.literal_pool_count >= thumb_gen_state.literal_pool_size) - { - const int new_size = thumb_gen_state.literal_pool_size << 1; - thumb_gen_state.literal_pool = tcc_realloc(thumb_gen_state.literal_pool, new_size * sizeof(ThumbLiteralPoolEntry)); - thumb_gen_state.literal_pool_size = new_size; - tcc_chained_hash_reserve(&literal_pool_hash, new_size); - } - entry = &thumb_gen_state.literal_pool[thumb_gen_state.literal_pool_count++]; - entry->sym = NULL; - entry->relocation = -1; - entry->shared_index = -1; - return entry; -} - -/* Find existing literal pool entry with same sym and imm, and allocate new - entry that shares its literal value. - Uses hash table for O(1) lookup instead of O(n) linear search. */ -static ThumbLiteralPoolEntry *th_literal_pool_find_or_allocate(Sym *sym, int64_t imm) -{ - int found_index; - uint32_t full_hash; - TCCChainedHash *hash; - LiteralPoolLookupCache *cache; - ThumbLiteralPoolEntry *pool; - int new_index; - - if (dry_run_state.active) - { - hash = &literal_pool_hash; - cache = &literal_pool_last_lookup; - pool = dry_run_literal_pool; - new_index = dry_run_literal_pool_count; - } - else - { - hash = &literal_pool_hash; - cache = &literal_pool_last_lookup; - pool = thumb_gen_state.literal_pool; - new_index = thumb_gen_state.literal_pool_count; - } - - full_hash = literal_pool_hash_func(sym, imm); - found_index = literal_pool_lookup_cache_find(cache, full_hash, sym, imm); - if (found_index < 0) - { - found_index = literal_pool_hash_find(hash, pool, full_hash, sym, imm); - } - - /* Allocate new entry */ - ThumbLiteralPoolEntry *entry = th_literal_pool_allocate(); - if (found_index >= 0) - { - /* Mark as sharing with the found entry */ - entry->shared_index = found_index; - } - else - { - literal_pool_hash_insert(hash, full_hash, new_index); - found_index = new_index; - } - literal_pool_lookup_cache_insert(cache, full_hash, sym, imm, found_index); - return entry; -} - -/* PIC + text/data-separation relocation choice. Single source of truth: the - * SB-relative fast path and the literal-pool path must agree, or the dry-run - * and real passes emit different sizes. See docs/sb_relative_got.md. */ -static int th_pic_reloc_for_sym(Sym *sym, int sym_off) -{ - int sym_in_code_section = 0; - int sym_in_rodata = 0; - if (sym_off > 0 && sym_off < tcc_state->nb_sections) - { - Section *sym_sec = tcc_state->sections[sym_off]; - if (sym_sec && (sym_sec->sh_flags & SHF_EXECINSTR)) - sym_in_code_section = 1; - /* Exact pointer match: only the main .rodata is anchor-addressed. */ - if (sym_sec && sym_sec == rodata_section) - sym_in_rodata = 1; - } - if (tcc_state->share_rodata && (sym->type.t & VT_STATIC) && sym_off != SHN_UNDEF && sym_in_rodata) - return R_ARM_RODATA_OFF; - /* sym_off == SHN_UNDEF: forward-declared, section unknown — GOT32 is safe. */ - if ((sym->type.t & VT_STATIC) && sym_off != SHN_UNDEF && sym_off != cur_text_section->sh_num && - !sym_in_code_section) - return R_ARM_GOTOFF; - return R_ARM_GOT32; -} - -/* References in this body that a hoisted .rodata anchor would serve, counted - * from the IR so the register allocator can decide whether to leave a register - * for one. An over- or under-count only moves bytes: the backend still emits - * whatever form the register situation allows. Symbols reach codegen from - * places with no IR operand behind them (libcall names, switch tables), so - * this is a floor, not the exact site count. */ -ST_FUNC int tcc_gen_machine_rodata_anchor_ir_sites(const TCCIRState *ir, int stop_at) -{ - if (!ir || !tcc_state->share_rodata || !text_and_data_separation) - return 0; - - int sites = 0; - for (int i = 0; i < ir->next_instruction_index && sites < stop_at; i++) - { - IRQuadCompact *q = &ir->compact_instructions[i]; - const IROperand ops[3] = {tcc_ir_op_get_dest(ir, q), tcc_ir_op_get_src1(ir, q), - tcc_ir_op_get_src2(ir, q)}; - for (int k = 0; k < 3; k++) - { - Sym *sym = irop_get_sym_ex(ir, ops[k]); - if (!sym) - continue; - ElfSym *esym = elfsym(sym); - if (th_pic_reloc_for_sym(sym, esym ? esym->st_shndx : 0) == R_ARM_RODATA_OFF) - sites++; - } - } - return sites; -} - -/* Add a symbol addend to an already-materialised address in r. imm == 0 is the - * common case and is skipped: it cost 9,733 dead instructions (19 KiB) across - * the compiler's own build. imm is pass-invariant, so sizes still agree. */ -static void th_emit_sym_addend(int r, int64_t imm) -{ - thumb_opcode ot; - if (imm == 0) - return; - if ((ot = th_add_imm(r, r, imm, flags_safe(), ENFORCE_ENCODING_NONE)).size != 0) - { - ot_check(ot); - return; - } - uint32_t exclude_regs = (1 << r); - ScratchRegAlloc scratch_alloc = get_scratch_reg_with_save(exclude_regs); - int scratch = scratch_alloc.reg; - - thumb_opcode ldr = th_ldr_literal(scratch, 0, 1); - ot_check(ldr); - - ThumbLiteralPoolEntry *entry2 = th_literal_pool_allocate(); - entry2->sym = NULL; - entry2->imm = imm; - entry2->patch_position = ind - ldr.size; - entry2->relocation = -1; - entry2->data_size = 4; - entry2->short_instruction = (ldr.size == 2); - th_literal_pool_note_entry(entry2); - ot_check(th_add_reg(r, r, scratch, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); - restore_scratch_reg(&scratch_alloc); -} - -static void load_full_const(int r, int r1, uint32_t imm_lo, uint32_t imm_hi) -{ - struct Sym *sym = _lfc_sym; - _lfc_sym = NULL; - int64_t imm = (int64_t)((uint64_t)imm_hi << 32 | (uint64_t)imm_lo); - ThumbLiteralPoolEntry *entry; - thumb_opcode load_ins; - int patch_pos; - - /* Validate symbol - only use symbols that can be externalized */ - sym = validate_sym_for_reloc(sym); - - /* Stable cache key: the validated symbol *before* the registration block - * below may NULL it. Registration is skipped during dry-run, so using the - * post-registration `sym` would make the dry and real passes disagree on - * cache hits and desynchronise code size. `reuse_sym` is identical in both - * passes (validate_sym_for_reloc does not depend on dry-run state). */ - Sym *reuse_sym = sym; - - /* Symbol-address reuse: when a register already holds &sym+imm, skip the - * redundant literal-pool load. Uses the same per-register imm_cache that - * is invalidated on every clobbering emit and at IR boundaries, so the - * decision is deterministic across the dry-run and real passes. Only the - * single-register (non-LDRD) form participates. */ - if (reuse_sym && thumb_gen_state.generating_function && r1 == PREG_NONE && r >= 0 && r < 16) - { - if (imm_cache[r].valid && imm_cache[r].sym == reuse_sym && imm_cache[r].value == imm) - return; /* r already holds &sym+imm */ - for (int rr = 0; rr < 16; rr++) - { - if (rr != r && imm_cache[rr].valid && imm_cache[rr].sym == reuse_sym && imm_cache[rr].value == imm) - { - /* Another register holds it: copy instead of reloading from the - * literal pool (saves a memory access and a pool word). */ - ot_check_mov_reg((uint32_t)r, (uint32_t)rr, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE, false); - imm_cache[r].value = imm; - imm_cache[r].sym = reuse_sym; - imm_cache[r].valid = 1; - return; - } - } - } - - /* During dry-run, skip symbol registration and literal pool allocation. - * We just emit the instruction (ot_check handles dry-run mode) to track - * code size and scratch register usage, without creating side effects. */ - if (!dry_run_state.active) - { - if (sym && sym->c == 0) - { - /* Symbol not yet registered - try to register it */ - put_extern_sym(sym, NULL, 0, 0); - if (sym->c <= 0) - { - /* Registration failed - symbol can't be externalized */ - sym = NULL; - } - } - } - - /* SB-relative GOT: a single `ldr.w Rt,[r9,#imm12]` replaces literal + add R9 - * + load and the pool word. Goes through ot_check rather than - * ot_check_ldr_imm on purpose — the latter's redundant-reload cache is keyed - * on the pre-relocation immediate (0), which would conflate distinct - * symbols. See docs/sb_relative_got.md. */ - if (sym && pic && text_and_data_separation && sb_relative_got && r1 == PREG_NONE) - { - ElfSym *sb_esym = elfsym(sym); - int sb_sym_off = sb_esym ? sb_esym->st_shndx : 0; - if (th_pic_reloc_for_sym(sym, sb_sym_off) == R_ARM_GOT32) - { - thumb_opcode sb_ins = th_ldr_imm(r, R9, 0, 6, ENFORCE_ENCODING_32BIT); - if (sb_ins.size != 0) - { - ot_check(sb_ins); - if (!dry_run_state.active) - greloc(cur_text_section, sym, ind - sb_ins.size, R_ARM_GOT_SBREL12); - th_emit_sym_addend(r, imm); - return; - } - } - } - - TRACE("'load_full_const' to register: %d, with imm: %d\n", r, imm); - - /* Emit the instruction first. - * ot() may flush the current literal pool BEFORE emitting this op. - * If patch_position is captured before ot_check(), it can end up pointing - * at the pool skip-branch and later patching would clobber it. - */ - if (r1 == PREG_NONE) - { - load_ins = th_ldr_literal(r, 0, 1); - } - else - { - load_ins = th_ldrd_imm(r, r1, R_PC, 0, 4); - } - ot_check(load_ins); - patch_pos = ind - load_ins.size; - - /* Record that r now holds &sym+imm so a later reference to the same global - * address can be elided. Must run after ot_check(), whose emit-level - * invalidation cleared imm_cache[r] for the LDR we just produced. Keyed on - * the pre-registration `reuse_sym` for dry/real-pass consistency. */ - if (reuse_sym && thumb_gen_state.generating_function && r1 == PREG_NONE && r >= 0 && r < 16) - { - imm_cache[r].value = imm; - imm_cache[r].sym = reuse_sym; - imm_cache[r].valid = 1; - } - - /* During dry-run, we still need to create the literal pool entry to ensure - * the literal pool behavior (threshold checks, sharing, etc.) matches the real pass. - * We still set sym so that find_or_allocate can match entries correctly. - * We just skip symbol registration and relocation setup. */ - entry = th_literal_pool_find_or_allocate(sym, imm); - entry->sym = sym; - entry->patch_position = patch_pos; - entry->relocation = -1; /* No relocation by default */ - entry->data_size = (r1 == PREG_NONE) ? 4 : 8; - entry->short_instruction = (r1 == PREG_NONE && load_ins.size == 2); - th_literal_pool_note_entry(entry); - - if (!sym) - { - entry->imm = imm; - return; - } - - /* Re-derive esym after ot_check(): literal pool generation during ot_check - * can call put_elf_sym → section_ptr_add → section_realloc, which may - * free and reallocate the symtab section buffer, invalidating any - * earlier ElfSym pointer. */ - ElfSym *esym = elfsym(sym); - int sym_off = 0; - if (esym) - { - sym_off = esym->st_shndx; - } - if (!pic) - { - entry->relocation = R_ARM_ABS32; - /* The imm value is the addend (offset from symbol base). - For arr[i], imm = i * sizeof(element). - The linker will add the symbol's address to this addend. */ - entry->imm = imm; - } - else - { - /* For PIC without a symbol, the literal is a plain constant (e.g. -1). - * Must still store the value so the pool emits it correctly. */ - entry->imm = imm; - if (sym) - { - if (text_and_data_separation) - { - /* Relocation strategy for text_and_data_separation + PIC: - * - * R_ARM_GOTOFF computes (symbol - GOT_addr) and at runtime adds R9. - * This only works when symbol and GOT are in the same loadable - * segment (i.e. both in data). With text/data separation, code - * (.text) and data (.got) are loaded at independent addresses. - * - * Static symbols in *data* sections (no SHF_EXECINSTR): - * GOTOFF is fine — symbol and GOT are both in the data segment. - * - * Everything else (including static functions in other .text.* - * sections from -ffunction-sections): - * Use R_ARM_GOT32 — indirect through a GOT slot. The linker - * creates a GOT entry (put_got_entry → R_RELATIVE for locals), - * fill_local_got_entries writes sym->st_value into the slot, - * and the YAFF writer emits a data relocation so the dynamic - * loader patches the slot to the runtime code address. - */ - entry->relocation = th_pic_reloc_for_sym(sym, sym_off); - } - else - { - if (sym->type.t & VT_STATIC) - { - entry->relocation = R_ARM_REL32; - } - else - { - entry->relocation = R_ARM_GOT_PREL; - } - } - } - } - - if (pic) - { - if (sym) - { - if (text_and_data_separation) - { - /* Dispatch on the relocation chosen above — GOTOFF adds R9, GOT32 - * adds R9 then loads through the slot. */ - if (entry->relocation == R_ARM_RODATA_OFF) - { - /* Shared .rodata anchor: r holds (sym - rodata_base) from the - * R_ARM_RODATA_OFF literal; add the rodata runtime base. */ - if (DRY_RUN_MODELLING) - dry_run_state.rodata_anchor_sites++; - if (rodata_anchor_reg >= 0) - { - /* The prologue parked the base in a register reserved for the - * whole body (tcc_gen_machine_rodata_anchor_claim). */ - if (r == rodata_anchor_reg) - tcc_error("compiler_error: rodata anchor r%d handed out as a value register", - rodata_anchor_reg); - ot_check(th_add_reg(r, r, rodata_anchor_reg, flags_safe(), THUMB_SHIFT_DEFAULT, - ENFORCE_ENCODING_NONE)); - } - else - { - /* push {tmp}; ldr tmp, [R9, #24]; add r, r, tmp; pop {tmp} - * Use a DETERMINISTIC fixed scratch (a low register other than r, - * saved by push/pop), NOT get_scratch_reg_with_save: the latter's - * callee-saved fallback is gated on !dry_run_state.active, so under - * register pressure (e.g. ps's larger functions) it can pick a - * different register in the dry-run vs real pass, desync instruction - * sizes, and corrupt literal-pool offsets — yielding a near-NULL - * rodata address. A fixed push/pop emits identically in both passes. */ - int anchor_tmp = (r == 0) ? 1 : 0; - ot_check(th_push((uint16_t)(1u << anchor_tmp))); - ot_check_ldr_imm(anchor_tmp, R9, YAFF_RODATA_ANCHOR_GOT_OFFSET, 6, ENFORCE_ENCODING_NONE); - ot_check( - th_add_reg(r, r, anchor_tmp, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); - ot_check(th_pop((uint16_t)(1u << anchor_tmp))); - } - } - else if (entry->relocation == R_ARM_GOTOFF) - { - /* Static data symbol — GOTOFF (add R9) */ - ot_check(th_add_reg(r, r, R9, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); - } - else - { - ot_check(th_add_reg(r, r, R9, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); - - ot_check_ldr_imm(r, r, 0, 6, ENFORCE_ENCODING_NONE); - th_emit_sym_addend(r, imm); - } - } - else - { - if (sym->type.t & VT_STATIC) - { - ot_check(th_add_reg(r, r, R_PC, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); - ot_check(th_sub_imm(r, r, 8, flags_safe(), ENFORCE_ENCODING_NONE)); - } - else - { - thumb_opcode ot; - ot_check(th_add_reg(r, r, R_PC, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); - ot_check_ldr_imm(r, r, 4, 6, ENFORCE_ENCODING_NONE); - /* Same zero-addend skip as the GOT-relative path above. */ - if (imm == 0) - { - /* no addend: nothing to add */ - } - else if ((ot = th_add_imm(r, r, imm, flags_safe(), ENFORCE_ENCODING_NONE)).size != 0) - { - ot_check(ot); - } - else - { - /* Find a free scratch register for literal pool entry */ - uint32_t exclude_regs = (1 << r); /* Exclude destination register */ - ScratchRegAlloc scratch_alloc = get_scratch_reg_with_save(exclude_regs); - int scratch = scratch_alloc.reg; - - thumb_opcode ldr = th_ldr_literal(scratch, 0, 1); - ot_check(ldr); - - ThumbLiteralPoolEntry *entry2 = th_literal_pool_allocate(); - entry2->sym = NULL; - entry2->imm = imm; - entry2->patch_position = ind - ldr.size; - entry2->relocation = -1; - entry2->data_size = 4; - entry2->short_instruction = (ldr.size == 2); - th_literal_pool_note_entry(entry2); - ot_check( - th_add_reg(r, r, scratch, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); - restore_scratch_reg(&scratch_alloc); - } - } - } - } - } -} - -int load_short_from_base(int ir, int base, int fc, int sign) -{ - const thumb_opcode ins = th_ldrsh_imm(ir, base, fc, sign ? 4 : 6, ENFORCE_ENCODING_NONE); - TRACE("Load short sign: %d, r %d, base: %d, fc: %d\n", sign, ir, base, fc); - return ot(ins); -} - -ST_FUNC void tcc_machine_addr_of_stack_slot(int dest_reg, int frame_offset, int is_param) -{ - if (dest_reg == PREG_REG_NONE) - tcc_error("compiler_error: addr_of_stack_slot requires a destination register"); - - /* Stack parameters live above the saved-register area. - * When computing their address, fold in offset_to_args (prologue push size). - * Locals/spills need callee-saved gap adjustment. */ - if (is_param) - frame_offset += offset_to_args; - else - frame_offset = fp_adjust_local_offset(frame_offset, 0); - - const int base_reg = tcc_state->need_frame_pointer ? R_FP : R_SP; - - if (frame_offset == 0) - { - if (dest_reg != base_reg) - { - ot_check_mov_reg(dest_reg, base_reg, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE, - false); - } - return; - } - - /* Check FP offset cache for existing computation - * Only use cache for callee-saved registers (r4-r11) since scratch registers - * like ip (r12) can be overwritten at any time without invalidating the cache. */ - TCCIRState *ir = tcc_state->ir; - int cached_reg = -1; - int is_callee_saved = (dest_reg >= R4 && dest_reg <= R11); - - if (ir && is_callee_saved && tcc_ir_opt_fp_cache_lookup(ir, frame_offset, &cached_reg)) - { - /* Cache hit! Verify the cached register is also callee-saved */ - if (cached_reg >= R4 && cached_reg <= R11) - { - if (cached_reg != dest_reg) - { - ot_check_mov_reg(dest_reg, cached_reg, flags_safe(), THUMB_SHIFT_DEFAULT, - ENFORCE_ENCODING_NONE, false); - } - return; - } - /* Cached in scratch register - don't use it */ - } - - const int neg = (frame_offset < 0); - int abs_off = neg ? -frame_offset : frame_offset; - thumb_opcode op = neg ? th_sub_imm(dest_reg, base_reg, abs_off, flags_safe(), ENFORCE_ENCODING_NONE) - : th_add_imm(dest_reg, base_reg, abs_off, flags_safe(), ENFORCE_ENCODING_NONE); - - if (op.size != 0) - { - ot_check(op); - /* Record in cache for future reuse - only for callee-saved registers - * which won't be clobbered unexpectedly */ - if (ir && is_callee_saved) - tcc_ir_opt_fp_cache_record(ir, frame_offset, dest_reg); - return; - } - - ScratchRegAlloc offset_alloc = {0}; - int offset_reg = dest_reg; - - if (dest_reg == base_reg) - { - offset_alloc = get_scratch_reg_with_save(1u << base_reg); - if (offset_alloc.reg == PREG_NONE) - tcc_error("compiler_error: unable to allocate scratch register for stack address"); - offset_reg = offset_alloc.reg; - } - - load_full_const(offset_reg, PREG_NONE, LFC_SPLIT(frame_offset)); - ot_check(th_add_reg(dest_reg, base_reg, offset_reg, flags_safe(), THUMB_SHIFT_DEFAULT, - ENFORCE_ENCODING_NONE)); - - if (dest_reg == base_reg) - { - restore_scratch_reg(&offset_alloc); - } - - /* Record complex computation in cache - only for callee-saved registers */ - if (ir && is_callee_saved) - tcc_ir_opt_fp_cache_record(ir, frame_offset, dest_reg); -} - -/* Load a constant value into a register (or register pair for 64-bit). - * This is a simplified wrapper around load_full_const/th_generic_mov_imm - * that doesn't require an SValue. Used by IR-level materialization. - * If sym is non-NULL, a relocation will be generated for symbol-relative constants. */ -ST_FUNC void tcc_machine_load_constant(int dest_reg, int dest_reg_high, int64_t value, int is_64bit, Sym *sym) -{ - if (dest_reg == PREG_REG_NONE) - tcc_error("compiler_error: load_constant requires a destination register"); - - /* Symbol-relative constants always need the literal pool for relocations */ - if (sym) - { - Sym *validated_sym = validate_sym_for_reloc(sym); - if (validated_sym) - { - _lfc_sym = validated_sym; - load_full_const(dest_reg, dest_reg_high, LFC_SPLIT(value)); - return; - } - /* Invalid or missing sym - fall through to treat as plain constant */ - } - - if (!sym && !is_64bit && dest_reg >= 0 && dest_reg < 16 && - imm_cache[dest_reg].valid && imm_cache[dest_reg].sym == NULL && - imm_cache[dest_reg].value == value) - return; - - if (is_64bit) - { - const uint32_t lo = (uint32_t)(value & 0xFFFFFFFF); - const uint32_t hi = (uint32_t)((uint64_t)value >> 32); - - /* Try immediate encoding for both halves */ - thumb_opcode o1 = th_generic_mov_imm(dest_reg, (int)lo); - thumb_opcode o2 = th_generic_mov_imm(dest_reg_high, (int)hi); - - if (o1.size != 0 && o2.size != 0) - { - /* Both can be encoded as immediates */ - ot(o1); - ot(o2); - return; - } - - /* At least one half needs literal pool - use combined 64-bit load */ - load_full_const(dest_reg, dest_reg_high, LFC_SPLIT(value)); - return; - } - - /* 32-bit constant */ - if (!ot(th_generic_mov_imm(dest_reg, (uint32_t)value))) - load_full_const(dest_reg, PREG_NONE, LFC_SPLIT(value)); - - if (!sym && !is_64bit && dest_reg >= 0 && dest_reg < 16) - { - imm_cache[dest_reg].value = value; - imm_cache[dest_reg].sym = NULL; - imm_cache[dest_reg].valid = 1; - } -} - -/* Load comparison result (0 or 1) based on condition flags. - * Used by IR-level materialization for VT_CMP values. */ -ST_FUNC void tcc_machine_load_cmp_result(int dest_reg, int condition_code) -{ - if (dest_reg == PREG_REG_NONE) - tcc_error("compiler_error: load_cmp_result requires a destination register"); - if (dest_reg == R_SP || dest_reg == R_PC) - tcc_error("compiler_error: load_cmp_result cannot use SP or PC"); - - const uint32_t firstcond = mapcc(condition_code); - /* IT block: if cond then mov 1, else mov 0 */ - o(0xbf00 | (firstcond << 4) | 0x4 | ((~firstcond & 1) << 3)); - ot_check(th_generic_mov_imm(dest_reg, 1)); - ot_check(th_generic_mov_imm(dest_reg, 0)); -} - -/* Load jump condition result (0 or 1) based on a pending jump target. - * Used by IR-level materialization for VT_JMP/VT_JMPI values. */ -ST_FUNC void tcc_machine_load_jmp_result(int dest_reg, int jmp_addr, int invert) -{ - if (dest_reg == PREG_REG_NONE) - tcc_error("compiler_error: load_jmp_result requires a destination register"); - -#ifdef TCC_TARGET_ARM_ARCHV6M - if (dest_reg > 7) - tcc_error("compiler_error: implement load_jmp_result for armv6m with high register"); -#endif - - /* Load the "true" branch value, then unconditionally branch over the "false" value, - * then patch the jump target to land on the "false" value */ - ot_check(th_generic_mov_imm(dest_reg, invert ? 0 : 1)); - ot_check(th_b_t4(2)); - gsym(jmp_addr); - codegen_internal_merge_point(); - ot_check(th_generic_mov_imm(dest_reg, invert ? 1 : 0)); -} - -/* Load value from memory at base+offset into register(s). - * Uses IROP_BTYPE_* constants directly, no VT_* conversion needed. - */ -static void load_from_base(int r, int r1, int irop_btype, int is_unsigned, int fc, int sign, uint32_t base) -{ - int success = 0; - const int is_64bit = - (irop_btype == IROP_BTYPE_INT64 || irop_btype == IROP_BTYPE_FLOAT64 || (r1 >= 0 && r1 != PREG_REG_NONE)); - - TRACE("load_from_base: r=%d, r1=%d, irop_btype=%d, is_unsigned=%d, fc=%d, sign=%d, base=%d", r, r1, irop_btype, - is_unsigned, fc, sign, base); - - if (is_64bit) - { - /* 64-bit value (double float or long long) - load to register pair */ - int ir_high = r1; - ScratchRegAlloc ir_high_alloc = {0}; - if (ir_high < 0 || ir_high == PREG_REG_NONE) - { - /* No explicit high register — always use scratch to avoid clobbering - * r+1 which may be allocated to another live variable. The old r+1 - * fallback was only safe when mat.c pre-materialized into scratch - * registers (ip:lr pair) before the handler. */ - ir_high_alloc = get_scratch_reg_with_save((1u << r) | (1u << base)); - ir_high = ir_high_alloc.reg; - } - - /* If base overlaps with destination, preserve it */ - ScratchRegAlloc base_alloc = {0}; - uint32_t base_reg = base; - if (base_reg == (uint32_t)r || base_reg == (uint32_t)ir_high) - { - uint32_t exclude = (1u << r) | (1u << ir_high); - base_alloc = get_scratch_reg_with_save(exclude); - base_reg = (uint32_t)base_alloc.reg; - ot_check_mov_reg((int)base_reg, (int)base, flags_safe(), THUMB_SHIFT_DEFAULT, - ENFORCE_ENCODING_NONE, false); - } - - /* Try LDRD Rt, Rt2, [Rn, #±imm] when both halves share one base. - * T1 encoding requires: Rt != Rt2, Rt/Rt2 not SP/PC, offset 4-byte - * aligned and |offset| <= 1020. LDRD also requires the target address - * to be 4-byte aligned on ARMv7-M/v8-M (faults otherwise, regardless of - * UNALIGN_TRP). Restrict to SP/FP-relative bases where TCC's stack - * allocator guarantees 4-byte alignment of 64-bit slots; arbitrary - * pointers (e.g. into a packed struct) may be unaligned. */ - const int base_is_stack = (base_reg == (uint32_t)R_SP || base_reg == (uint32_t)R_FP); - if (base_is_stack && (fc & 3) == 0 && fc <= 1020 && r >= 0 && r <= R_LR && r != R_SP && ir_high >= 0 && - ir_high <= R_LR && ir_high != R_SP && r != ir_high) - { - uint32_t puw = sign ? 4 : 6; - if (!strldr_cache_ldrd_is_redundant(r, ir_high, (int)base_reg, fc, puw)) - ot_check(th_ldrd_imm((uint32_t)r, (uint32_t)ir_high, base_reg, fc, puw)); - if (base_alloc.saved) - restore_scratch_reg(&base_alloc); - if (ir_high_alloc.saved) - restore_scratch_reg(&ir_high_alloc); - return; - } - - /* Load low word */ - success = load_word_from_base(r, base_reg, fc, sign); - if (!success) - { - ScratchRegAlloc rr_alloc = th_offset_to_reg_ex(fc, sign, (1u << r) | (1u << base_reg) | (1u << ir_high)); - int rr = rr_alloc.reg; - ot_check(th_ldr_reg(r, base_reg, rr, THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); - restore_scratch_reg(&rr_alloc); - } - - /* Load high word */ - int fc_high = sign ? (fc - 4) : (fc + 4); - success = load_word_from_base(ir_high, base_reg, fc_high, sign); - if (!success) - { - ScratchRegAlloc rr_alloc = th_offset_to_reg_ex(fc_high, sign, (1u << r) | (1u << base_reg) | (1u << ir_high)); - int rr = rr_alloc.reg; - ot_check(th_ldr_reg(ir_high, base_reg, rr, THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); - restore_scratch_reg(&rr_alloc); - } - - if (base_alloc.saved) - restore_scratch_reg(&base_alloc); - if (ir_high_alloc.saved) - restore_scratch_reg(&ir_high_alloc); - return; - } - - if (irop_btype == IROP_BTYPE_INT16) - { - if (!is_unsigned) - success = load_short_from_base(r, base, fc, sign); - else - success = load_ushort_from_base(r, base, fc, sign); - } - else if (irop_btype == IROP_BTYPE_INT8) - { - if (!is_unsigned) - success = load_byte_from_base(r, base, fc, sign); - else - success = load_ubyte_from_base(r, base, fc, sign); - } - else - { - /* IROP_BTYPE_INT32, IROP_BTYPE_FLOAT32, IROP_BTYPE_STRUCT, IROP_BTYPE_FUNC: load as word */ - success = load_word_from_base(r, base, fc, sign); - } - - if (!success) - { - ScratchRegAlloc rr_alloc = th_offset_to_reg_ex(fc, sign, (1u << r) | (1u << base)); - int rr = rr_alloc.reg; - if (irop_btype == IROP_BTYPE_INT16) - { - if (is_unsigned) - ot_check(th_ldrh_reg(r, base, rr, THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); - else - ot_check(th_ldrsh_reg(r, base, rr, THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); - } - else if (irop_btype == IROP_BTYPE_INT8) - { - if (is_unsigned) - ot_check(th_ldrb_reg(r, base, rr, THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); - else - ot_check(th_ldrsb_reg(r, base, rr, THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); - } - else - ot_check(th_ldr_reg(r, base, rr, THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); - restore_scratch_reg(&rr_alloc); - } -} - -ST_FUNC void gen_increment_tcov(SValue *sv) -{ - TRACE("'gen_increment_tcov'"); -} - -typedef thumb_opcode (*thumb_reg_handler_t)(uint32_t rd, uint32_t rn, uint32_t rm, - thumb_flags_behaviour flags_behaviour, thumb_shift shift_type, - thumb_enforce_encoding enforce_encoding); -typedef struct ThumbDataProcessingHandler -{ - thumb_imm_handler_t imm_handler; - thumb_reg_handler_t reg_handler; -} ThumbDataProcessingHandler; - -/* Dispatch a reg_handler call through a direct call instead of an indirect - * (function pointer) call. This works around a code-generation bug where - * struct-by-value arguments (thumb_shift) get corrupted when passed through - * indirect calls that also use sret return (thumb_opcode is 8 bytes). - * By comparing the function pointer and branching to a direct call, the - * cross-compiler generates correct struct passing code. */ -static thumb_opcode thumb_call_reg_handler(thumb_reg_handler_t fn, uint32_t rd, uint32_t rn, uint32_t rm, - thumb_flags_behaviour flags, thumb_shift shift, - thumb_enforce_encoding encoding) -{ - if (fn == th_add_reg) - return th_add_reg(rd, rn, rm, flags, shift, encoding); - if (fn == th_sub_reg) - return th_sub_reg(rd, rn, rm, flags, shift, encoding); - if (fn == th_adc_reg) - return th_adc_reg(rd, rn, rm, flags, shift, encoding); - if (fn == th_sbc_reg) - return th_sbc_reg(rd, rn, rm, flags, shift, encoding); - if (fn == th_cmp_reg) - return th_cmp_reg(rd, rn, rm, flags, shift, encoding); - if (fn == th_lsl_reg) - return th_lsl_reg(rd, rn, rm, flags, shift, encoding); - if (fn == th_lsr_reg) - return th_lsr_reg(rd, rn, rm, flags, shift, encoding); - if (fn == th_asr_reg) - return th_asr_reg(rd, rn, rm, flags, shift, encoding); - if (fn == th_orr_reg) - return th_orr_reg(rd, rn, rm, flags, shift, encoding); - if (fn == th_and_reg) - return th_and_reg(rd, rn, rm, flags, shift, encoding); - if (fn == th_eor_reg) - return th_eor_reg(rd, rn, rm, flags, shift, encoding); - if (fn == th_bic_reg) - return th_bic_reg(rd, rn, rm, flags, shift, encoding); - /* Unreachable for known handlers — fallback to direct call. */ - return fn(rd, rn, rm, flags, shift, encoding); -} - -static void thumb_require_materialized_reg(const char *ctx, const char *operand, int reg) -{ - const bool reg_is_hw = (reg >= 0) && (reg <= 15); - if (reg == PREG_REG_NONE || !reg_is_hw) - { - tcc_error("compiler_error: %s expects %s in a physical register (pr=%d)", ctx, operand, reg); - } -} - -static uint32_t thumb_exclude_mask_for_regs(int count, const int *regs) -{ - uint32_t mask = 0; - for (int i = 0; i < count; ++i) - { - const int reg = regs[i]; - if (reg >= 0 && reg <= 15) - mask |= (1u << reg); - } - return mask; -} - -static bool thumb_is_hw_reg(int reg) -{ - return reg >= 0 && reg <= 15; -} - -/* ============================================================ - * thumb_and_imm_form / thumb_emit_and_imm_special - * ============================================================ - * One-instruction lowerings of `AND Rd, Rn, #mask` that never materialize the - * mask. Thumb-2's AND-immediate takes only a modified immediate (8 bits - * rotated, or a byte replicated across the word), so an ordinary bitfield mask - * -- 0x7FF, 0x000FFFFF, 0x007FFFFF -- costs a movw/movt pair or a literal-pool - * load on top of the AND. In the soft-float library that pair is the single - * most common shape there is: every classifier opens with - * `bits & DOUBLE_MANT_MASK`. - * - * The classification is a pure function of the mask so that a caller can - * decide BEFORE resolving operands into registers -- resolving can itself emit - * (a spilled source becomes a load), so a caller that resolved first and then - * fell through to the general path would emit that load twice. - * - * None of these forms writes the flags, so the caller must already have - * established that no flag result is wanted. `flags` is passed through for - * BIC, which does have a flag-setting encoding. - */ -typedef enum ThumbAndImmForm -{ - AND_IMM_NONE = 0, /* nothing better than the general path */ - AND_IMM_UXTB, /* mask 0xFF */ - AND_IMM_UXTH, /* mask 0xFFFF */ - AND_IMM_UBFX, /* low-contiguous run of ones */ - AND_IMM_BIC, /* complement encodes as a modified immediate */ -} ThumbAndImmForm; - -static ThumbAndImmForm thumb_and_imm_form(uint32_t mask) -{ - /* Zero-extends: a 2-byte encoding when both registers are low, against 4 - * bytes for the AND-immediate, which has no narrow form at all. */ - if (mask == 0xFFu) - return AND_IMM_UXTB; - if (mask == 0xFFFFu) - return AND_IMM_UXTH; - - /* A mask the AND-immediate already encodes is one instruction as it stands; - * these forms only pay off when they remove a constant materialization. */ - if (th_pack_const(mask) != 0) - return AND_IMM_NONE; - - /* A low-contiguous run of ones is exactly UBFX Rd, Rn, #0, #W. 0 and - * 0xFFFFFFFF are degenerate (the callers fold them to a load-zero and a - * copy) and UBFX cannot encode width 32 anyway. */ - if (mask != 0 && mask != 0xFFFFFFFFu && (mask & (mask + 1)) == 0) - return AND_IMM_UBFX; - - /* `x & ~m` is `BIC x, m`, and a clear-mask whose complement encodes is the - * usual shape of a bitfield read-modify-write. */ - if (~mask != 0 && th_pack_const(~mask) != 0) - return AND_IMM_BIC; - - return AND_IMM_NONE; -} - -static void thumb_emit_and_imm_special(ThumbAndImmForm form, int rd, int rn, uint32_t mask, - thumb_flags_behaviour flags) -{ - switch (form) - { - case AND_IMM_UXTB: - ot_check(th_uxtb((uint32_t)rd, (uint32_t)rn, THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); - return; - case AND_IMM_UXTH: - ot_check(th_uxth((uint32_t)rd, (uint32_t)rn, THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); - return; - case AND_IMM_UBFX: - { - int width = 0; - while ((mask >> width) & 1u) - width++; - thumb_opcode ubfx_op; - ubfx_op.size = 4; - /* 11110 0 11 1100 Rn | 0 imm3 Rd imm2 0 widthm1, with lsb 0 (imm3=imm2=0). */ - ubfx_op.opcode = 0xF3C00000u | ((uint32_t)rn << 16) | ((uint32_t)rd << 8) | (uint32_t)(width - 1); - ot(ubfx_op); - return; - } - case AND_IMM_BIC: - ot_check(th_bic_imm((uint32_t)rd, (uint32_t)rn, ~mask, flags, ENFORCE_ENCODING_NONE)); - return; - case AND_IMM_NONE: - default: - tcc_error("thumb_emit_and_imm_special: no form for mask 0x%x", (unsigned)mask); - return; - } -} - -/* ============================================================ - * thumb_try_orrs_zero64 - * ============================================================ - * A 64-bit value is zero exactly when the bitwise OR of its halves is, so - * `ORRS Rt, Rlo, Rhi` sets Z for the whole comparison in one instruction where - * the general form needs three (`CMP hi,#0` / `IT EQ` / `CMPEQ lo,#0`). Only - * Z is meaningful afterwards, which is all an equality comparison consumes -- - * the three-instruction form is no better in that respect, since which of its - * two CMPs ran last decides N/C/V. - * - * ORRS needs a destination the CMP form does not, so this fires only when the - * allocator has a register genuinely free at this point: buying one with a - * push/pop would cost more than the two instructions it saves. Rt may alias - * either source -- ORRS reads both operands before writing. - * - * `soft_common.h`'s classifiers are built out of this test (`double_mant(bits) - * != 0` and friends), 60 sites across the soft-float library. - * - * Returns 1 if it emitted the comparison, 0 if the caller must fall back. - */ -static int thumb_try_orrs_zero64(int rn_lo, int rn_hi) -{ - TCCIRState *ir = tcc_state->ir; - if (!ir) - return 0; - if (!thumb_is_hw_reg(rn_lo) || !thumb_is_hw_reg(rn_hi)) - return 0; - - const uint32_t excl = scratch_global_exclude; - int rt = tcc_ls_find_free_scratch_reg(&ir->ls, ir->codegen_instruction_idx, excl, ir->leaffunc); - if (rt == (int)PREG_NONE || !thumb_is_hw_reg(rt)) - return 0; - - thumb_opcode orrs = th_orr_reg((uint32_t)rt, (uint32_t)rn_lo, (uint32_t)rn_hi, FLAGS_BEHAVIOUR_SET, - THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE); - if (orrs.size == 0) - return 0; - ot_check(orrs); - return 1; -} - -static void thumb_emit_op_imm_fallback(int rd, int rn, uint32_t imm, thumb_flags_behaviour flags, - ThumbDataProcessingHandler handler) -{ - thumb_opcode sub_low = thumb_call_imm_handler(handler.imm_handler, rd, rn, imm, flags, ENFORCE_ENCODING_NONE); - if (sub_low.size == 0) - { - uint32_t exclude = 0; - if (rd >= 0 && rd <= 15) - exclude |= (1u << rd); - if (rn >= 0 && rn <= 15) - exclude |= (1u << rn); - ScratchRegAlloc scratch = get_scratch_reg_with_save(exclude); - tcc_machine_load_constant(scratch.reg, PREG_NONE, (int32_t)imm, 0, NULL); - ot_check(thumb_call_reg_handler(handler.reg_handler, rd, rn, scratch.reg, flags, THUMB_SHIFT_DEFAULT, - ENFORCE_ENCODING_NONE)); - restore_scratch_reg(&scratch); - } - else - { - ot_check(sub_low); - } -} - -typedef thumb_opcode (*thumb_regonly3_handler_t)(uint32_t rd, uint32_t rn, uint32_t rm); - -static thumb_opcode thumb_mul_regonly(uint32_t rd, uint32_t rn, uint32_t rm) -{ - return th_mul(rd, rn, rm, flags_safe(), ENFORCE_ENCODING_NONE); -} - -static thumb_opcode thumb_sdiv_regonly(uint32_t rd, uint32_t rn, uint32_t rm) -{ - return th_sdiv((uint16_t)rd, (uint16_t)rn, (uint16_t)rm); -} - -static thumb_opcode thumb_udiv_regonly(uint32_t rd, uint32_t rn, uint32_t rm) -{ - return th_udiv((uint16_t)rd, (uint16_t)rn, (uint16_t)rm); -} - -typedef thumb_opcode (*thumb_longmul_handler_t)(uint32_t rdlo, uint32_t rdhi, uint32_t rn, uint32_t rm); - -/* ============================================================ - * mach_resolve_deref_64 - * ============================================================ - * When a 64-bit source has needs_deref=true, the operand holds a POINTER - * to a 64-bit value — not the value itself. Splitting such an operand - * via mach_make_lo_half / mach_make_hi_half is WRONG because - * mach_make_hi_half would increment the register number (e.g. R0 → R1) - * instead of the memory offset. - * - * This helper resolves the deref by loading both 32-bit halves from - * [base+0] and [base+4] into scratch registers, returning a clean - * MACH_OP_REG pair operand with needs_deref=false. The caller can - * then safely call mach_make_lo_half / mach_make_hi_half on the result. - * - * Returns *op unchanged if needs_deref is false. - */ -static MachineOperand mach_resolve_deref_64(MachineCodegenContext *mctx, const MachineOperand *op, uint32_t *excl) -{ - if (!op->needs_deref) - return *op; - - /* PARAM_STACK with needs_deref (is_lval): the 64-bit value IS directly - * at [fp+offset], NOT a pointer to follow. Clear needs_deref and let - * the normal mach_make_lo_half / mach_make_hi_half path handle it. */ - if (op->kind == MACH_OP_PARAM_STACK) - { - MachineOperand result = *op; - result.needs_deref = false; - return result; - } - - /* Strip deref to get the raw address into a register. */ - MachineOperand addr = *op; - addr.needs_deref = false; - addr.is_64bit = false; - addr.btype = IROP_BTYPE_INT32; - int base_reg = mach_ensure_in_reg(mctx, &addr, *excl); - if (thumb_is_hw_reg(base_reg)) - *excl |= (1u << (uint32_t)base_reg); - - /* Allocate two scratch registers for the loaded halves. */ - int lo_reg = mach_alloc_scratch(mctx, *excl); - *excl |= (1u << (uint32_t)lo_reg); - int hi_reg = mach_alloc_scratch(mctx, *excl); - *excl |= (1u << (uint32_t)hi_reg); - - /* Load [base+0] → lo, [base+4] → hi. Proven-aligned access (op->align4, - * from the frontend's packed-access tracking): one LDRD; otherwise the - * unaligned-safe pair of 32-bit loads. */ - if (!(op->align4 && try_ldrd_pair(lo_reg, hi_reg, base_reg, 0, 0))) - { - load_from_base(lo_reg, PREG_REG_NONE, IROP_BTYPE_INT32, 0, 0, 0, (uint32_t)base_reg); - load_from_base(hi_reg, PREG_REG_NONE, IROP_BTYPE_INT32, 0, 4, 0, (uint32_t)base_reg); - } - - /* Build a clean register-pair operand. */ - MachineOperand result = {0}; - result.kind = MACH_OP_REG; - result.is_64bit = true; - result.needs_deref = false; - result.btype = op->btype; - result.u.reg.r0 = lo_reg; - result.u.reg.r1 = hi_reg; - return result; -} - -/* ============================================================ - * mach_make_lo_half / mach_make_hi_half - * ============================================================ - * Split a 64-bit MachineOperand into its 32-bit low and high halves. - * The resulting operands have is_64bit=false and represent the individual - * 32-bit words, suitable for mach_ensure_in_reg / mach_writeback_dest. - * - * Only call mach_make_hi_half on a 64-bit operand (is_64bit=true or - * MACH_OP_SPILL); the result for 32-bit REG is the next register (r0+1). - */ -static MachineOperand mach_make_lo_half(const MachineOperand *op) -{ - MachineOperand lo = *op; - lo.is_64bit = false; - if (lo.kind == MACH_OP_REG) - lo.u.reg.r1 = -1; - /* SPILL: keep the same offset — low word is at the base offset. */ - /* IMM: u.imm.val bits [31:0] are the low word (callers truncate). */ - /* CHAIN_REL: keep offset/chain_index — low word is at base offset. */ - return lo; -} - -static MachineOperand mach_make_hi_half(const MachineOperand *op) -{ - MachineOperand hi = *op; - hi.is_64bit = false; - switch (hi.kind) - { - case MACH_OP_REG: - /* r1 holds the high register for 64-bit pairs. If r1 is not a valid - * hardware register the allocator failed to produce a proper pair — - * error out instead of silently using r0+1 which can clobber reserved - * registers (e.g. R9 = GOT base). */ - if (!thumb_is_hw_reg(op->u.reg.r1)) - tcc_error("mach_make_hi_half: 64-bit REG operand has invalid r1=%d (r0=%d) — " - "register allocator must produce a valid pair", - op->u.reg.r1, op->u.reg.r0); - hi.u.reg.r0 = op->u.reg.r1; - hi.u.reg.r1 = -1; - break; - case MACH_OP_SPILL: - hi.u.spill.offset += 4; - break; - case MACH_OP_IMM: - hi.u.imm.val = (int64_t)(int32_t)(uint32_t)((uint64_t)op->u.imm.val >> 32); - break; - case MACH_OP_PARAM_STACK: - hi.u.param.offset += 4; - break; - case MACH_OP_CHAIN_REL: - hi.u.chain.offset += 4; /* high word is 4 bytes above low word */ - break; - case MACH_OP_SYMBOL: - hi.u.sym.addend += 4; /* high word at symbol + addend + 4 */ - break; - case MACH_OP_FRAME_ADDR: - hi.u.frame.offset += 4; /* high word at FP + offset + 4 */ - break; - default: - break; - } - return hi; -} - -/* Materialize both 32-bit halves of a 64-bit operand into registers. - * - * When the source is a plain, word-aligned stack slot the two halves sit at - * adjacent addresses — exactly LDRD's shape, so one instruction replaces the - * two LDRs the per-half `mach_ensure_in_reg` calls would emit. A `long long` - * comparison or arithmetic op reading a local is otherwise the one 64-bit - * memory shape that never pairs (STOREs already fuse to STRD, and derefs pair - * via mach_resolve_deref_64), which is why `if (x.a != y.a)` on 64-bit fields - * costs four loads instead of two. - * - * Every other operand kind, and any slot LDRD cannot reach, falls back to the - * original per-half path, so behaviour there is unchanged. `*excl` picks up - * both chosen registers, mirroring what the callers did by hand. */ -static void mach_ensure_pair_in_regs(MachineCodegenContext *ctx, const MachineOperand *op64, - uint32_t *excl, int *out_lo, int *out_hi) -{ - MachineOperand lo = mach_make_lo_half(op64); - MachineOperand hi = mach_make_hi_half(op64); - lo.btype = IROP_BTYPE_INT32; - hi.btype = IROP_BTYPE_INT32; - - if (op64->is_64bit && lo.kind == MACH_OP_SPILL && !lo.needs_deref && (lo.u.spill.offset & 3) == 0) - { - int rlo = mach_alloc_scratch(ctx, *excl); - int rhi = mach_alloc_scratch(ctx, *excl | (1u << (uint32_t)rlo)); - if (rlo != rhi) - { - /* LDRD, or — when its offset/register constraints do not hold — the two - * single-word loads into the scratches we already own. */ - if (!tcc_gen_machine_try_ldrd_spill(rlo, lo.u.spill.offset, rhi, hi.u.spill.offset)) - { - tcc_machine_load_spill_slot(rlo, lo.u.spill.offset); - tcc_machine_load_spill_slot(rhi, hi.u.spill.offset); - } - *excl |= (1u << (uint32_t)rlo) | (1u << (uint32_t)rhi); - *out_lo = rlo; - *out_hi = rhi; - return; - } - tcc_machine_load_spill_slot(rlo, lo.u.spill.offset); - *excl |= (1u << (uint32_t)rlo); - *out_lo = rlo; - *out_hi = mach_ensure_in_reg(ctx, &hi, *excl); - if (thumb_is_hw_reg(*out_hi)) - *excl |= (1u << (uint32_t)*out_hi); - return; - } - - int rlo = mach_ensure_in_reg(ctx, &lo, *excl); - if (thumb_is_hw_reg(rlo)) - *excl |= (1u << (uint32_t)rlo); - int rhi = mach_ensure_in_reg(ctx, &hi, *excl); - if (thumb_is_hw_reg(rhi)) - *excl |= (1u << (uint32_t)rhi); - *out_lo = rlo; - *out_hi = rhi; -} - -/* ============================================================ - * thumb_emit_data_processing_mop64 - * ============================================================ - * 64-bit ADD / SUB / AND / OR / XOR via MachineOperand register pairs. - * Handles REG (r0:r1), SPILL (offset, offset+4) and IMM operands. - * - * uses_carry=true → low word uses FLAGS_BEHAVIOUR_SET, high word uses the - * carry handler (ADDS + ADC for ADD, SUBS + SBC for SUB). - * uses_carry=false → both halves use the same handler independently (AND/OR/XOR). - * - * If src1 is not 64-bit (e.g. int promoted to long long), its high half is - * zero-extended. Similarly for src2. - * - * `zh` carries source/opt/flat/fusion/zero_half64.c's verdict in its low six - * bits: which operand halves are provably the constant zero, and which halves - * of the result no consumer reads. - */ -static void thumb_emit_data_processing_mop64(const MachineOperand *src1, const MachineOperand *src2, - const MachineOperand *dest, TccIrOp op, ThumbDataProcessingHandler regular, - ThumbDataProcessingHandler carry_h, bool uses_carry, uint32_t zh) -{ - (void)op; - MachineCodegenContext mctx = {0}; - uint32_t excl = 0; - - /* 0. Determine destination register pair FIRST so that deref resolution - * never allocates scratch registers that overlap with the dest pair. - * Without this, mach_release_all would restore saved scratch regs - * and clobber the result sitting in rd_lo / rd_hi. */ - int rd_lo, rd_hi; - bool store_lo = false, store_hi = false; - if (dest->kind == MACH_OP_REG && !dest->needs_deref && dest->u.reg.r0 != (int)PREG_REG_NONE && dest->u.reg.r1 >= 0) - { - rd_lo = dest->u.reg.r0; - rd_hi = dest->u.reg.r1; - excl |= (1u << (uint32_t)rd_lo) | (1u << (uint32_t)rd_hi); - } - else - { - rd_lo = mach_alloc_scratch(&mctx, excl); - excl |= (1u << (uint32_t)rd_lo); - rd_hi = mach_alloc_scratch(&mctx, excl); - excl |= (1u << (uint32_t)rd_hi); - store_lo = store_hi = (dest->kind != MACH_OP_NONE); - } - - /* 0b. Pre-exclude register operands so that deref resolution of one - * source never steals the physical registers of another source. - * This must include needs_deref registers: they hold live pointers - * that will be consumed during their own deref resolution and must - * not be repurposed as scratch during the other source's deref. */ - if (src1->kind == MACH_OP_REG) - { - if (src1->u.reg.r0 != (int)PREG_REG_NONE) - excl |= (1u << (uint32_t)src1->u.reg.r0); - if (!src1->needs_deref && src1->is_64bit && src1->u.reg.r1 >= 0) - excl |= (1u << (uint32_t)src1->u.reg.r1); - } - if (src2->kind == MACH_OP_REG) - { - if (src2->u.reg.r0 != (int)PREG_REG_NONE) - excl |= (1u << (uint32_t)src2->u.reg.r0); - if (!src2->needs_deref && src2->is_64bit && src2->u.reg.r1 >= 0) - excl |= (1u << (uint32_t)src2->u.reg.r1); - } - - /* 1. Resolve deref'd source pointers before splitting into halves. */ - MachineOperand r_src1 = mach_resolve_deref_64(&mctx, src1, &excl); - src1 = &r_src1; - MachineOperand r_src2 = mach_resolve_deref_64(&mctx, src2, &excl); - src2 = &r_src2; - - /* 1b. Plan the HIGH half before loading anything. - * - * A source that is not 64-bit is zero-extended, and against a zero half a - * bitwise op has a constant answer: OR/XOR copy the other operand, AND give - * zero. Deciding that here rather than after the loads means the zero is - * never materialised AND the scratch register that would have held it is - * never allocated -- on a saturated function that scratch costs a push/pop - * of a callee-saved register too. - * - * `(uint64_t)x | ((uint64_t)y << 32)` and the make_double/make_float packing - * idiom soft float is built from are exactly this shape: before this, each - * such OR paid `movs rt,#0` plus an `orr rd,rt,rm` that could only ever - * return rm. */ - enum - { - HALF_NORMAL = 0, /* emit the handler over both registers of this half */ - HALF_ZERO, /* result half is the constant 0 */ - HALF_COPY_S1, /* result half is src1's */ - HALF_COPY_S2, /* result half is src2's */ - HALF_CARRY_IMM0, /* carry op, src2 half is zero: `adc/sbc rd, rn, #0` */ - HALF_DEAD /* nothing reads the result half; emit nothing at all */ - }; - /* A non-64-bit source is zero-extended by construction; the annotation adds - * the halves a 64-bit value is only *known* to hold zero in. */ - const bool s1_lo_zero = (zh & ZH64_S1_LO) != 0; - const bool s1_hi_zero = !src1->is_64bit || (zh & ZH64_S1_HI) != 0; - const bool s2_lo_zero = (zh & ZH64_S2_LO) != 0; - const bool s2_hi_zero = (src2->kind != MACH_OP_IMM && !src2->is_64bit) || (zh & ZH64_S2_HI) != 0; - int lo_plan = HALF_NORMAL, hi_plan = HALF_NORMAL; - /* Carry ops need the real words (the carry still has to propagate), and - * CMP's halves must set flags, so neither can take a shortcut. */ - if (!uses_carry && op != TCCIR_OP_CMP && src2->kind != MACH_OP_IMM) - { - for (int half = 0; half < 2; half++) - { - const bool z1 = half ? s1_hi_zero : s1_lo_zero; - const bool z2 = half ? s2_hi_zero : s2_lo_zero; - int plan = HALF_NORMAL; - if (!z1 && !z2) - plan = HALF_NORMAL; - else if (op == TCCIR_OP_AND) - plan = HALF_ZERO; - else if (op == TCCIR_OP_OR || op == TCCIR_OP_XOR) - plan = (z1 && z2) ? HALF_ZERO : (z1 ? HALF_COPY_S2 : HALF_COPY_S1); - if (half) - hi_plan = plan; - else - lo_plan = plan; - } - } - /* A carry op cannot skip a half -- the carry still has to propagate -- but a - * zero high word on src2 turns the high half into `adc rd, rn, #0` / - * `sbc rd, rn, #0`, which reads no second register and needs no `mov rt, #0` - * to feed it. This is most of what sfp_div_sig64 spends its zeros on: - * Knuth-D adds and subtracts 32-bit quantities from 64-bit ones throughout, - * and each one was materialising a zero purely to be added to a high word. - * src1 is deliberately not covered: SUB is not commutative, and mirroring - * this rule in zero_half64's producer analysis is only sound while the - * emitter's choice is unambiguous. */ - if (uses_carry && op != TCCIR_OP_CMP && src2->kind != MACH_OP_IMM && s2_hi_zero) - hi_plan = HALF_CARRY_IMM0; - - /* A half no consumer reads outranks every plan above: skip it entirely. - * Only ever set for a register destination -- a memory one still owes its - * full eight bytes whatever the vreg uses do. */ - if ((zh & ZH64_D_LO) && !uses_carry && op != TCCIR_OP_CMP) - lo_plan = HALF_DEAD; - if ((zh & ZH64_D_HI) && !uses_carry && op != TCCIR_OP_CMP) - hi_plan = HALF_DEAD; - const bool need_rn_lo = (lo_plan == HALF_NORMAL || lo_plan == HALF_COPY_S1 || lo_plan == HALF_CARRY_IMM0); - const bool need_rn_hi = (hi_plan == HALF_NORMAL || hi_plan == HALF_COPY_S1 || hi_plan == HALF_CARRY_IMM0); - const bool need_rm_lo = (lo_plan == HALF_NORMAL || lo_plan == HALF_COPY_S1 || lo_plan == HALF_COPY_S2 || - lo_plan == HALF_CARRY_IMM0); - const bool need_rm_hi = (hi_plan == HALF_NORMAL || hi_plan == HALF_COPY_S2); - - /* 2. Load src1 low and high halves into registers. */ - int rn_lo = (int)PREG_REG_NONE; - if (need_rn_lo) - { - MachineOperand s1_lo = mach_make_lo_half(src1); - rn_lo = mach_ensure_in_reg(&mctx, &s1_lo, excl); - if (thumb_is_hw_reg(rn_lo)) - excl |= (1u << (uint32_t)rn_lo); - } - int rn_hi = (int)PREG_REG_NONE; - if (!need_rn_hi) - { - /* left unloaded on purpose -- nothing below reads it */ - } - else if (src1->is_64bit) - { - MachineOperand s1_hi = mach_make_hi_half(src1); - rn_hi = mach_ensure_in_reg(&mctx, &s1_hi, excl); - } - else - { - rn_hi = mach_alloc_scratch(&mctx, excl); - ot_check(th_mov_imm((uint32_t)rn_hi, 0, flags_safe(), ENFORCE_ENCODING_NONE)); - } - if (need_rn_hi && thumb_is_hw_reg(rn_hi)) - excl |= (1u << (uint32_t)rn_hi); - - /* 3. Load src2 and emit the 64-bit operation. */ - const thumb_flags_behaviour lo_flags = uses_carry ? FLAGS_BEHAVIOUR_SET : flags_safe(); - /* For CMP, the high-word SBCS must set flags (the following SETIF reads them). */ - const thumb_flags_behaviour hi_flags = (op == TCCIR_OP_CMP) ? FLAGS_BEHAVIOUR_SET : flags_safe(); - if (src2->kind == MACH_OP_IMM) - { - const uint32_t imm_lo = (uint32_t)((uint64_t)src2->u.imm.val & 0xffffffffu); - const uint32_t imm_hi = (uint32_t)((uint64_t)src2->u.imm.val >> 32); - /* Per-half peephole: when the immediate half makes the op a constant - * answer (OR/XOR with 0 → copy src; AND with 0 → load 0; AND with -1 → - * copy src), skip the data-processing op. Cuts dead `orr r, r, #0` and - * `and r, r, #0` halves left behind by 64-bit ops on 32-bit values. */ - const bool is_or = (op == TCCIR_OP_OR); - const bool is_xor = (op == TCCIR_OP_XOR); - const bool is_and = (op == TCCIR_OP_AND); - const bool can_simplify_lo = lo_flags == flags_safe(); - const bool can_simplify_hi = hi_flags == flags_safe(); - for (int half = 0; half < 2; half++) - { - const uint32_t imm = (half == 0) ? imm_lo : imm_hi; - const int rd = (half == 0) ? rd_lo : rd_hi; - const int rn = (half == 0) ? rn_lo : rn_hi; - const thumb_flags_behaviour fb = (half == 0) ? lo_flags : hi_flags; - const bool can_simplify = (half == 0) ? can_simplify_lo : can_simplify_hi; - const ThumbDataProcessingHandler *h = (half == 0) ? ®ular : &carry_h; - - if ((half == 0 ? lo_plan : hi_plan) == HALF_DEAD) - continue; - if (can_simplify && (is_or || is_xor) && imm == 0) - { - if (rd != rn) - ot_check_mov_reg((uint32_t)rd, (uint32_t)rn, flags_safe(), - THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE, false); - } - else if (can_simplify && is_and && imm == 0) - { - ot_check(th_mov_imm((uint32_t)rd, 0, flags_safe(), ENFORCE_ENCODING_NONE)); - } - else if (can_simplify && is_and && imm == 0xFFFFFFFFu) - { - if (rd != rn) - ot_check_mov_reg((uint32_t)rd, (uint32_t)rn, flags_safe(), - THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE, false); - } - /* A 64-bit AND against a bitfield mask splits into two half-masks, and - * the interesting half is never a modified immediate: DOUBLE_MANT_MASK's - * high half is 0x000FFFFF, which the fallback below materializes from the - * literal pool at every one of the dozen sites a soft-float classifier - * has. UBFX does the same job in one instruction with no constant. */ - else if (can_simplify && is_and && thumb_and_imm_form(imm) != AND_IMM_NONE) - { - thumb_emit_and_imm_special(thumb_and_imm_form(imm), rd, rn, imm, fb); - } - else - { - thumb_emit_op_imm_fallback(rd, rn, imm, fb, *h); - } - } - } - else - { - int rm_lo = (int)PREG_REG_NONE; - if (need_rm_lo) - { - MachineOperand s2_lo = mach_make_lo_half(src2); - rm_lo = mach_ensure_in_reg(&mctx, &s2_lo, excl); - if (thumb_is_hw_reg(rm_lo)) - excl |= (1u << (uint32_t)rm_lo); - } - int rm_hi = (int)PREG_REG_NONE; - if (!need_rm_hi) - { - /* left unloaded on purpose -- the high-half plan does not read it */ - } - else if (src2->is_64bit) - { - MachineOperand s2_hi = mach_make_hi_half(src2); - rm_hi = mach_ensure_in_reg(&mctx, &s2_hi, excl); - } - else - { - rm_hi = mach_alloc_scratch(&mctx, excl); - ot_check(th_mov_imm((uint32_t)rm_hi, 0, flags_safe(), ENFORCE_ENCODING_NONE)); - } - for (int half = 0; half < 2; half++) - { - const int plan = half ? hi_plan : lo_plan; - const int rd = half ? rd_hi : rd_lo; - const int rn = half ? rn_hi : rn_lo; - const int rm = half ? rm_hi : rm_lo; - const thumb_flags_behaviour fb = half ? hi_flags : lo_flags; - const ThumbDataProcessingHandler *h = half ? &carry_h : ®ular; - switch (plan) - { - case HALF_DEAD: - break; - case HALF_ZERO: - ot_check(th_mov_imm((uint32_t)rd, 0, flags_safe(), ENFORCE_ENCODING_NONE)); - break; - case HALF_COPY_S1: - if (rd != rn) - ot_check_mov_reg((uint32_t)rd, (uint32_t)rn, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE, - false); - break; - case HALF_COPY_S2: - if (rd != rm) - ot_check_mov_reg((uint32_t)rd, (uint32_t)rm, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE, - false); - break; - case HALF_CARRY_IMM0: - thumb_emit_op_imm_fallback(rd, rn, 0, fb, *h); - break; - default: - ot_check(thumb_call_reg_handler(h->reg_handler, (uint32_t)rd, (uint32_t)rn, (uint32_t)rm, fb, - THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); - break; - } - } - } - - /* 4. Write results back to spill/param slots if dest was not pre-allocated. - * A half nothing reads is not written back either -- the slot keeps - * whatever it held, which is exactly what "dead" asserts. */ - if (store_lo && lo_plan != HALF_DEAD) - { - MachineOperand dst_lo = mach_make_lo_half(dest); - dst_lo.btype = IROP_BTYPE_INT32; - mach_writeback_dest(&dst_lo, rd_lo); - } - if (store_hi && hi_plan != HALF_DEAD) - { - MachineOperand dst_hi = mach_make_hi_half(dest); - dst_hi.btype = IROP_BTYPE_INT32; - mach_writeback_dest(&dst_hi, rd_hi); - } - mach_release_all(&mctx); -} - -/* ============================================================ - * thumb_emit_shift64_reg_mop - * ============================================================ - * 64-bit SHL / SHR / SAR whose count is a value rather than a literal. - * - * Before this every such shift went to __aeabi_llsl / __aeabi_llsr / - * __aeabi_lasr. Those helpers are compiled from C like any other library - * routine, so each one costs a call, a frame and ~30 executed instructions; a - * dynamic profile of __aeabi_dadd put two of them -- the alignment shift and - * the `(1ULL << n) - 1` mask -- at 62 of its 267 instructions per call. - * Inline the same work is 8 instructions and touches no memory. - * - * The sequences lean on ARM's register-shift rule: LSL/LSR/ASR by a register - * read only the bottom BYTE of the count, and a count of 32 or more yields 0 - * (LSL/LSR) or a full sign fill (ASR). That makes the `32 - n` and `n - 32` - * cross terms self-disabling over the half of the range where they must not - * contribute, so the logical shifts need neither a branch nor a conditional: - * - * x << n : hi = (hi << n) | (lo >> (32-n)) | (lo << (n-32)); lo = lo << n - * x >> n : lo = (lo >> n) | (hi << (32-n)) | (hi >> (n-32)); hi = hi >> n - * - * SAR cannot borrow that trick for its cross term -- `hi ASR (n-32)` is a sign - * fill when n < 32, which would flood the low half with ones where it must - * contribute nothing -- so it selects between the two halves of the range with - * a mask taken from the sign of `n - 32`. A mask and not an IT/MOVGE pair - * because this runs wherever the allocator placed the shift, including with a - * pending CMP's flags still live. - * - * Counts of 64 and above are undefined in C; the sequences give 0 (or a sign - * fill for SAR), which is what GCC's inline expansion produces as well. - */ -static void thumb_emit_shift64_reg_mop(const MachineOperand *src1, const MachineOperand *src2, - const MachineOperand *dest, TccIrOp op, bool skip_lo, bool skip_hi) -{ - const bool is_left = (op == TCCIR_OP_SHL); - const bool arith_right = (op == TCCIR_OP_SAR); - - MachineCodegenContext mctx = {0}; - uint32_t excl = 0; - - /* Destination pair first, exactly as the immediate path does, so that deref - * resolution never hands out a scratch that overlaps it. */ - int dst_lo, dst_hi; - bool store_lo = false, store_hi = false; - if (dest->kind == MACH_OP_REG && !dest->needs_deref && dest->u.reg.r0 != (int)PREG_REG_NONE && dest->u.reg.r1 >= 0) - { - dst_lo = dest->u.reg.r0; - dst_hi = dest->u.reg.r1; - excl |= (1u << (uint32_t)dst_lo) | (1u << (uint32_t)dst_hi); - } - else - { - dst_lo = mach_alloc_scratch(&mctx, excl); - excl |= (1u << (uint32_t)dst_lo); - dst_hi = mach_alloc_scratch(&mctx, excl); - excl |= (1u << (uint32_t)dst_hi); - store_lo = store_hi = (dest->kind != MACH_OP_NONE); - } - - if (src1->kind == MACH_OP_REG) - { - if (src1->u.reg.r0 != (int)PREG_REG_NONE) - excl |= (1u << (uint32_t)src1->u.reg.r0); - if (!src1->needs_deref && src1->is_64bit && src1->u.reg.r1 >= 0) - excl |= (1u << (uint32_t)src1->u.reg.r1); - } - if (src2->kind == MACH_OP_REG) - { - if (src2->u.reg.r0 != (int)PREG_REG_NONE) - excl |= (1u << (uint32_t)src2->u.reg.r0); - if (!src2->needs_deref && src2->is_64bit && src2->u.reg.r1 >= 0) - excl |= (1u << (uint32_t)src2->u.reg.r1); - } - - MachineOperand r_src1 = mach_resolve_deref_64(&mctx, src1, &excl); - src1 = &r_src1; - - MachineOperand s1_lo = mach_make_lo_half(src1); - int src_lo = mach_ensure_in_reg(&mctx, &s1_lo, excl); - if (thumb_is_hw_reg(src_lo)) - excl |= (1u << (uint32_t)src_lo); - - /* A high word that is a known zero deletes every term that reads it -- and - * the register that would have held it. `(1ULL << n) - 1`, the mask soft - * float builds every alignment sticky-bit from, is exactly this shape: the - * generic sequence would materialize a zero just to shift and OR it in. */ - const int hi_is_zero = - !arith_right && (!src1->is_64bit || - (src1->kind == MACH_OP_IMM && ((uint64_t)src1->u.imm.val >> 32) == 0)); - - int src_hi = (int)PREG_REG_NONE; - if (!hi_is_zero) - { - if (src1->is_64bit) - { - MachineOperand s1_hi = mach_make_hi_half(src1); - src_hi = mach_ensure_in_reg(&mctx, &s1_hi, excl); - if (thumb_is_hw_reg(src_hi)) - excl |= (1u << (uint32_t)src_hi); - } - else - { - src_hi = mach_alloc_scratch(&mctx, excl); - excl |= (1u << (uint32_t)src_hi); - ot_check(th_asr_imm((uint32_t)src_hi, (uint32_t)src_lo, 31, flags_safe(), ENFORCE_ENCODING_NONE)); - } - } - - /* Only the low word of a 64-bit count can matter — the hardware reads one - * byte of it. */ - MachineOperand cnt_op = src2->is_64bit ? mach_make_lo_half(src2) : *src2; - int cnt = mach_ensure_in_reg(&mctx, &cnt_op, excl); - if (thumb_is_hw_reg(cnt)) - excl |= (1u << (uint32_t)cnt); - - /* The half that collects the cross terms. Writing it straight into the - * destination saves a MOV, but only where that register is neither a source - * half still to be read nor the count. */ - /* A right shift of a zero-high-word value needs no accumulator and no - * temporaries; allocating them anyway can cost a scratch push/pop. */ - const bool need_acc = (!is_left && hi_is_zero) ? false : (is_left ? !skip_hi : !skip_lo); - const int acc_pref = is_left ? dst_hi : dst_lo; - const int acc_conflict = is_left ? src_lo : src_hi; - int acc = -1; - if (need_acc) - { - acc = (acc_pref != acc_conflict && acc_pref != cnt) ? acc_pref : -1; - if (acc < 0) - { - acc = mach_alloc_scratch(&mctx, excl); - excl |= (1u << (uint32_t)acc); - } - } - - int t0 = -1, t1 = -1; - if (need_acc) - { - t0 = mach_alloc_scratch(&mctx, excl); - excl |= (1u << (uint32_t)t0); - if (arith_right) - { - t1 = mach_alloc_scratch(&mctx, excl); - excl |= (1u << (uint32_t)t1); - } - } - - if (is_left) - { - if (need_acc) - { - ot_check(th_rsb_imm((uint32_t)t0, (uint32_t)cnt, 32, flags_safe(), ENFORCE_ENCODING_NONE)); - if (hi_is_zero) - { - /* (0 << n) | (lo >> (32-n)) is just the cross term. */ - ot_check(th_lsr_reg((uint32_t)acc, (uint32_t)src_lo, (uint32_t)t0, flags_safe(), THUMB_SHIFT_DEFAULT, - ENFORCE_ENCODING_NONE)); - } - else - { - ot_check(th_lsl_reg((uint32_t)acc, (uint32_t)src_hi, (uint32_t)cnt, flags_safe(), THUMB_SHIFT_DEFAULT, - ENFORCE_ENCODING_NONE)); - ot_check(th_lsr_reg((uint32_t)t0, (uint32_t)src_lo, (uint32_t)t0, flags_safe(), THUMB_SHIFT_DEFAULT, - ENFORCE_ENCODING_NONE)); - ot_check(th_orr_reg((uint32_t)acc, (uint32_t)acc, (uint32_t)t0, flags_safe(), THUMB_SHIFT_DEFAULT, - ENFORCE_ENCODING_NONE)); - } - ot_check(th_sub_imm((uint32_t)t0, (uint32_t)cnt, 32, flags_safe(), ENFORCE_ENCODING_NONE)); - ot_check(th_lsl_reg((uint32_t)t0, (uint32_t)src_lo, (uint32_t)t0, flags_safe(), THUMB_SHIFT_DEFAULT, - ENFORCE_ENCODING_NONE)); - ot_check(th_orr_reg((uint32_t)acc, (uint32_t)acc, (uint32_t)t0, flags_safe(), THUMB_SHIFT_DEFAULT, - ENFORCE_ENCODING_NONE)); - } - /* dst_lo last: it is the final read of src_lo and of the count. */ - if (!skip_lo) - ot_check(th_lsl_reg((uint32_t)dst_lo, (uint32_t)src_lo, (uint32_t)cnt, flags_safe(), THUMB_SHIFT_DEFAULT, - ENFORCE_ENCODING_NONE)); - if (need_acc && acc != dst_hi) - ot_check_mov_reg((uint32_t)dst_hi, (uint32_t)acc, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE, - false); - } - else if (hi_is_zero) - { - /* Every cross term reads the zero high word, so the whole shift is one - * instruction and a zero fill. */ - if (!skip_lo) - ot_check(th_lsr_reg((uint32_t)dst_lo, (uint32_t)src_lo, (uint32_t)cnt, flags_safe(), THUMB_SHIFT_DEFAULT, - ENFORCE_ENCODING_NONE)); - if (!skip_hi) - ot_check(th_mov_imm((uint32_t)dst_hi, 0, flags_safe(), ENFORCE_ENCODING_NONE)); - } - else - { - if (need_acc) - { - ot_check(th_rsb_imm((uint32_t)t0, (uint32_t)cnt, 32, flags_safe(), ENFORCE_ENCODING_NONE)); - ot_check(th_lsr_reg((uint32_t)acc, (uint32_t)src_lo, (uint32_t)cnt, flags_safe(), THUMB_SHIFT_DEFAULT, - ENFORCE_ENCODING_NONE)); - ot_check(th_lsl_reg((uint32_t)t0, (uint32_t)src_hi, (uint32_t)t0, flags_safe(), THUMB_SHIFT_DEFAULT, - ENFORCE_ENCODING_NONE)); - ot_check(th_orr_reg((uint32_t)acc, (uint32_t)acc, (uint32_t)t0, flags_safe(), THUMB_SHIFT_DEFAULT, - ENFORCE_ENCODING_NONE)); - ot_check(th_sub_imm((uint32_t)t0, (uint32_t)cnt, 32, flags_safe(), ENFORCE_ENCODING_NONE)); - if (arith_right) - { - /* `hi ASR (n-32)` is only the answer for n >= 32; below that it is a - * sign fill that must contribute nothing. `(n-32) ASR 31` is all-ones - * exactly on that range, so it selects between the two candidates - * without touching the flags. */ - ot_check(th_asr_reg((uint32_t)t1, (uint32_t)src_hi, (uint32_t)t0, flags_safe(), THUMB_SHIFT_DEFAULT, - ENFORCE_ENCODING_NONE)); - ot_check(th_asr_imm((uint32_t)t0, (uint32_t)t0, 31, flags_safe(), ENFORCE_ENCODING_NONE)); - ot_check(th_and_reg((uint32_t)acc, (uint32_t)acc, (uint32_t)t0, flags_safe(), THUMB_SHIFT_DEFAULT, - ENFORCE_ENCODING_NONE)); - ot_check(th_bic_reg((uint32_t)t1, (uint32_t)t1, (uint32_t)t0, flags_safe(), THUMB_SHIFT_DEFAULT, - ENFORCE_ENCODING_NONE)); - ot_check(th_orr_reg((uint32_t)acc, (uint32_t)acc, (uint32_t)t1, flags_safe(), THUMB_SHIFT_DEFAULT, - ENFORCE_ENCODING_NONE)); - } - else - { - ot_check(th_lsr_reg((uint32_t)t0, (uint32_t)src_hi, (uint32_t)t0, flags_safe(), THUMB_SHIFT_DEFAULT, - ENFORCE_ENCODING_NONE)); - ot_check(th_orr_reg((uint32_t)acc, (uint32_t)acc, (uint32_t)t0, flags_safe(), THUMB_SHIFT_DEFAULT, - ENFORCE_ENCODING_NONE)); - } - } - /* dst_hi last: it is the final read of src_hi and of the count. */ - if (!skip_hi) - { - if (arith_right) - ot_check(th_asr_reg((uint32_t)dst_hi, (uint32_t)src_hi, (uint32_t)cnt, flags_safe(), THUMB_SHIFT_DEFAULT, - ENFORCE_ENCODING_NONE)); - else - ot_check(th_lsr_reg((uint32_t)dst_hi, (uint32_t)src_hi, (uint32_t)cnt, flags_safe(), THUMB_SHIFT_DEFAULT, - ENFORCE_ENCODING_NONE)); - } - if (need_acc && acc != dst_lo) - ot_check_mov_reg((uint32_t)dst_lo, (uint32_t)acc, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE, - false); - } - - if (store_lo && !skip_lo) - { - MachineOperand dst_lo_op = mach_make_lo_half(dest); - dst_lo_op.btype = IROP_BTYPE_INT32; - mach_writeback_dest(&dst_lo_op, dst_lo); - } - if (store_hi && !skip_hi) - { - MachineOperand dst_hi_op = mach_make_hi_half(dest); - dst_hi_op.btype = IROP_BTYPE_INT32; - mach_writeback_dest(&dst_hi_op, dst_hi); - } - mach_release_all(&mctx); -} - -/* ============================================================ - * thumb_emit_shift64_mop - * ============================================================ - * 64-bit SHL / SHR / SAR via MachineOperand register pairs. - * Shift amount (src2) must be a 32-bit immediate (MACH_OP_IMM). - * Logic mirrors thumb_emit_shift64_imm but operates on register numbers - * extracted from MachineOperand rather than IROperand fields. - */ -static void thumb_emit_shift64_mop(const MachineOperand *src1, const MachineOperand *src2, const MachineOperand *dest, - TccIrOp op, bool skip_lo, bool skip_hi) -{ - if (src2->kind != MACH_OP_IMM) - { - thumb_emit_shift64_reg_mop(src1, src2, dest, op, skip_lo, skip_hi); - return; - } - const uint32_t sh = (uint32_t)(uint64_t)src2->u.imm.val; - const bool is_left = (op == TCCIR_OP_SHL); - const bool arith_right = (op == TCCIR_OP_SAR); - - thumb_imm_handler_t dst_lo_shift, dst_hi_shift, cross_shift; - if (is_left) - { - dst_lo_shift = th_lsl_imm; - dst_hi_shift = th_lsl_imm; - cross_shift = th_lsr_imm; - } - else if (arith_right) - { - dst_lo_shift = th_lsr_imm; - dst_hi_shift = th_asr_imm; - cross_shift = th_lsl_imm; - } - else - { - dst_lo_shift = th_lsr_imm; - dst_hi_shift = th_lsr_imm; - cross_shift = th_lsl_imm; - } - - MachineCodegenContext mctx = {0}; - uint32_t excl = 0; - - /* Determine destination register pair FIRST so that deref resolution - * never allocates scratch registers that overlap with the dest pair. */ - int dst_lo, dst_hi; - bool store_lo = false, store_hi = false; - if (dest->kind == MACH_OP_REG && !dest->needs_deref && dest->u.reg.r0 != (int)PREG_REG_NONE && dest->u.reg.r1 >= 0) - { - dst_lo = dest->u.reg.r0; - dst_hi = dest->u.reg.r1; - excl |= (1u << (uint32_t)dst_lo) | (1u << (uint32_t)dst_hi); - } - else - { - dst_lo = mach_alloc_scratch(&mctx, excl); - excl |= (1u << (uint32_t)dst_lo); - dst_hi = mach_alloc_scratch(&mctx, excl); - excl |= (1u << (uint32_t)dst_hi); - store_lo = store_hi = (dest->kind != MACH_OP_NONE); - } - - /* Pre-exclude register operands so that deref resolution does not - * steal the physical registers already holding src1 values. - * Include needs_deref registers: they hold live pointers needed - * during their own deref resolution. */ - if (src1->kind == MACH_OP_REG) - { - if (src1->u.reg.r0 != (int)PREG_REG_NONE) - excl |= (1u << (uint32_t)src1->u.reg.r0); - if (!src1->needs_deref && src1->is_64bit && src1->u.reg.r1 >= 0) - excl |= (1u << (uint32_t)src1->u.reg.r1); - } - - /* Resolve deref'd source pointer before splitting into halves. */ - MachineOperand r_src1 = mach_resolve_deref_64(&mctx, src1, &excl); - src1 = &r_src1; - - /* Load src1 low half. */ - MachineOperand s1_lo = mach_make_lo_half(src1); - int src_lo = mach_ensure_in_reg(&mctx, &s1_lo, excl); - if (thumb_is_hw_reg(src_lo)) - excl |= (1u << (uint32_t)src_lo); - - /* Skip src1 high-half materialization when the shift will not read it. - * SHL with sh >= 32 only uses src_lo (everything shifts up out of view). - * SHR/SAR with sh >= 64 produces a 0/sign-fill that the emit tail - * generates directly without referencing src_hi. */ - int hi_needed = 1; - if (is_left && sh >= 32) - hi_needed = 0; - else if (!is_left && sh >= 64 && !arith_right) - hi_needed = 0; - /* SAR by >= 64 is the exception: its tail fills BOTH halves from the sign of - * src_hi, so it reads the very half the count would suggest is irrelevant. - * Declaring it unneeded left src_hi as PREG_REG_NONE and the tail encoded a - * shift off a register that was never loaded. */ - - /* Load src1 high half or compute by extension. `hi_needed` gates the - * ALREADY-64-bit case too: materializing a half nothing below reads is a - * whole instruction, and `(uint64_t)sign << 63` in make_double is exactly - * that -- the widening puts a zero in a register purely to shift it out of - * existence. */ - int src_hi = (int)PREG_REG_NONE; - if (src1->is_64bit) - { - if (hi_needed) - { - MachineOperand s1_hi = mach_make_hi_half(src1); - src_hi = mach_ensure_in_reg(&mctx, &s1_hi, excl); - if (thumb_is_hw_reg(src_hi)) - excl |= (1u << (uint32_t)src_hi); - } - } - else if (hi_needed) - { - src_hi = mach_alloc_scratch(&mctx, excl); - excl |= (1u << (uint32_t)src_hi); - if (arith_right) - ot_check( - th_asr_imm((uint32_t)src_hi, (uint32_t)src_lo, 31, flags_safe(), ENFORCE_ENCODING_NONE)); - else - ot_check(th_mov_imm((uint32_t)src_hi, 0, flags_safe(), ENFORCE_ENCODING_NONE)); - } - - /* `x <<= 1` is ADDS/ADC: two instructions where the generic sh<32 path below - * needs four (cross-shift into a scratch, a shift per half, then an OR). - * This is the step every restoring-division and normalization loop is built - * from -- a dynamic profile of ddiv puts 280,000 iterations through it for - * the benchmark's double kernels, against 5,000 through everything else. - * - * Two guards. ADDS/ADC write the flags, so it must not run while a pending - * CMP's flags are still live. And ADC has to read src_hi AFTER ADDS has - * written dst_lo, so dst_lo aliasing src_hi rules the pair out; every other - * aliasing case is safe, since each instruction reads its sources before - * writing its destination. */ - if (is_left && sh == 1 && !skip_lo && !skip_hi && - thumb_is_hw_reg(src_lo) && thumb_is_hw_reg(src_hi) && dst_lo != src_hi && - !(tcc_state->ir && tcc_state->ir->codegen_flags_live)) - { - ot_check(th_add_reg((uint32_t)dst_lo, (uint32_t)src_lo, (uint32_t)src_lo, FLAGS_BEHAVIOUR_SET, - THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); - ot_check(th_adc_reg((uint32_t)dst_hi, (uint32_t)src_hi, (uint32_t)src_hi, flags_safe(), - THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); - } - /* Emit the shift — logic identical to thumb_emit_shift64_imm core. */ - else if (sh == 0) - { - ot_check_mov_reg((uint32_t)dst_lo, (uint32_t)src_lo, flags_safe(), THUMB_SHIFT_DEFAULT, - ENFORCE_ENCODING_NONE, false); - ot_check_mov_reg((uint32_t)dst_hi, (uint32_t)src_hi, flags_safe(), THUMB_SHIFT_DEFAULT, - ENFORCE_ENCODING_NONE, false); - } - else if (sh < 32) - { - /* The cross term is an ORR away from being free: T2 ORR takes a shifted - * REGISTER operand, so `orr rd, rd, src, LSR #(32-sh)` does in one - * instruction what a separate shift into a scratch plus a plain ORR did in - * two -- and it needs no scratch at all, which on a saturated function is - * a push/pop as well. Every 64-bit shift by a literal in soft float is - * this shape (`mant <<= 3`, `mant >> 3`), and __aeabi_dadd runs two per - * call. - * - * Only the aliasing case the fallback below reorders for is excluded: the - * destination half written FIRST must not be the source half the ORR still - * has to read. */ - const uint32_t cross_amount = (uint32_t)(32 - sh); - const thumb_shift cross_fold = {is_left ? THUMB_SHIFT_LSR : THUMB_SHIFT_LSL, cross_amount, - THUMB_SHIFT_IMMEDIATE}; - if (is_left && dst_hi != src_lo) - { - ot_check(thumb_call_imm_handler(dst_hi_shift, (uint32_t)dst_hi, (uint32_t)src_hi, sh, flags_safe(), - ENFORCE_ENCODING_NONE)); - ot_check(th_orr_reg((uint32_t)dst_hi, (uint32_t)dst_hi, (uint32_t)src_lo, flags_safe(), cross_fold, - ENFORCE_ENCODING_32BIT)); - if (!skip_lo) - ot_check(thumb_call_imm_handler(dst_lo_shift, (uint32_t)dst_lo, (uint32_t)src_lo, sh, flags_safe(), - ENFORCE_ENCODING_NONE)); - goto shift64_imm_done; - } - if (!is_left && dst_lo != src_hi) - { - ot_check(th_lsr_imm((uint32_t)dst_lo, (uint32_t)src_lo, sh, flags_safe(), ENFORCE_ENCODING_NONE)); - ot_check(th_orr_reg((uint32_t)dst_lo, (uint32_t)dst_lo, (uint32_t)src_hi, flags_safe(), cross_fold, - ENFORCE_ENCODING_32BIT)); - ot_check(thumb_call_imm_handler(dst_hi_shift, (uint32_t)dst_hi, (uint32_t)src_hi, sh, flags_safe(), - ENFORCE_ENCODING_NONE)); - goto shift64_imm_done; - } - - const int regs[] = {dst_lo, dst_hi, src_lo, src_hi}; - ScratchRegAlloc tmp = get_scratch_reg_with_save(thumb_exclude_mask_for_regs(4, regs) | excl); - if (is_left) - { - /* Compute the cross-shift into tmp BEFORE any destination is written, - * because dst_lo/dst_hi may alias src_lo/src_hi. */ - ot_check(thumb_call_imm_handler(cross_shift, (uint32_t)tmp.reg, (uint32_t)src_lo, 32 - sh, flags_safe(), - ENFORCE_ENCODING_NONE)); - if (dst_hi == src_lo) - { - /* dst_hi aliases src_lo — compute dst_lo first (needs src_lo). */ - if (!skip_lo) - ot_check( - thumb_call_imm_handler(dst_lo_shift, (uint32_t)dst_lo, (uint32_t)src_lo, sh, flags_safe(), ENFORCE_ENCODING_NONE)); - ot_check( - thumb_call_imm_handler(dst_hi_shift, (uint32_t)dst_hi, (uint32_t)src_hi, sh, flags_safe(), ENFORCE_ENCODING_NONE)); - } - else - { - /* Default order: dst_hi first to avoid clobbering src_hi via dst_lo. */ - ot_check( - thumb_call_imm_handler(dst_hi_shift, (uint32_t)dst_hi, (uint32_t)src_hi, sh, flags_safe(), ENFORCE_ENCODING_NONE)); - if (!skip_lo) - ot_check( - thumb_call_imm_handler(dst_lo_shift, (uint32_t)dst_lo, (uint32_t)src_lo, sh, flags_safe(), ENFORCE_ENCODING_NONE)); - } - ot_check(th_orr_reg((uint32_t)dst_hi, (uint32_t)dst_hi, (uint32_t)tmp.reg, flags_safe(), - THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); - } - else - { - /* Compute the cross-shift into tmp BEFORE any destination is written, - * because dst_lo/dst_hi may alias src_lo/src_hi. */ - ot_check(thumb_call_imm_handler(cross_shift, (uint32_t)tmp.reg, (uint32_t)src_hi, 32 - sh, flags_safe(), - ENFORCE_ENCODING_NONE)); - if (dst_lo == src_hi) - { - /* dst_lo aliases src_hi — compute dst_hi first (needs src_hi). */ - ot_check( - thumb_call_imm_handler(dst_hi_shift, (uint32_t)dst_hi, (uint32_t)src_hi, sh, flags_safe(), ENFORCE_ENCODING_NONE)); - ot_check( - th_lsr_imm((uint32_t)dst_lo, (uint32_t)src_lo, sh, flags_safe(), ENFORCE_ENCODING_NONE)); - } - else - { - /* Default order: dst_lo first to avoid clobbering src_lo via dst_hi. */ - ot_check( - th_lsr_imm((uint32_t)dst_lo, (uint32_t)src_lo, sh, flags_safe(), ENFORCE_ENCODING_NONE)); - ot_check( - thumb_call_imm_handler(dst_hi_shift, (uint32_t)dst_hi, (uint32_t)src_hi, sh, flags_safe(), ENFORCE_ENCODING_NONE)); - } - ot_check(th_orr_reg((uint32_t)dst_lo, (uint32_t)dst_lo, (uint32_t)tmp.reg, flags_safe(), - THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); - } - restore_scratch_reg(&tmp); - } - else if (sh == 32) - { - if (is_left) - { - /* Emit MOV dst_hi first: dst_lo may alias src_lo. */ - ot_check_mov_reg((uint32_t)dst_hi, (uint32_t)src_lo, flags_safe(), THUMB_SHIFT_DEFAULT, - ENFORCE_ENCODING_NONE, false); - if (!skip_lo) - ot_check(th_mov_imm((uint32_t)dst_lo, 0, flags_safe(), ENFORCE_ENCODING_NONE)); - } - else - { - /* Emit MOV dst_lo first: dst_hi may alias src_hi. */ - ot_check_mov_reg((uint32_t)dst_lo, (uint32_t)src_hi, flags_safe(), THUMB_SHIFT_DEFAULT, - ENFORCE_ENCODING_NONE, false); - if (!skip_hi) - { - if (arith_right) - ot_check( - th_asr_imm((uint32_t)dst_hi, (uint32_t)src_hi, 31, flags_safe(), ENFORCE_ENCODING_NONE)); - else - ot_check(th_mov_imm((uint32_t)dst_hi, 0, flags_safe(), ENFORCE_ENCODING_NONE)); - } - } - } - else if (sh < 64) - { - if (is_left) - { - /* Emit shift into dst_hi first: dst_lo may alias src_lo. */ - ot_check(thumb_call_imm_handler(dst_hi_shift, (uint32_t)dst_hi, (uint32_t)src_lo, sh - 32, flags_safe(), - ENFORCE_ENCODING_NONE)); - if (!skip_lo) - ot_check(th_mov_imm((uint32_t)dst_lo, 0, flags_safe(), ENFORCE_ENCODING_NONE)); - } - else - { - if (arith_right && dst_lo == src_hi) - { - /* dst_lo aliases src_hi — compute dst_hi (sign extension) first - * while src_hi is still intact, then shift into dst_lo. */ - if (!skip_hi) - ot_check( - th_asr_imm((uint32_t)dst_hi, (uint32_t)src_hi, 31, flags_safe(), ENFORCE_ENCODING_NONE)); - ot_check(thumb_call_imm_handler(dst_hi_shift, (uint32_t)dst_lo, (uint32_t)src_hi, sh - 32, flags_safe(), - ENFORCE_ENCODING_NONE)); - } - else - { - ot_check(thumb_call_imm_handler(dst_hi_shift, (uint32_t)dst_lo, (uint32_t)src_hi, sh - 32, flags_safe(), - ENFORCE_ENCODING_NONE)); - if (!skip_hi) - { - if (arith_right) - ot_check( - th_asr_imm((uint32_t)dst_hi, (uint32_t)src_hi, 31, flags_safe(), ENFORCE_ENCODING_NONE)); - else - ot_check(th_mov_imm((uint32_t)dst_hi, 0, flags_safe(), ENFORCE_ENCODING_NONE)); - } - } - } - } - else /* sh >= 64 */ - { - if (is_left) - { - if (!skip_lo) - ot_check(th_mov_imm((uint32_t)dst_lo, 0, flags_safe(), ENFORCE_ENCODING_NONE)); - if (!skip_hi) - ot_check(th_mov_imm((uint32_t)dst_hi, 0, flags_safe(), ENFORCE_ENCODING_NONE)); - } - else if (arith_right) - { - /* Both halves are the sign of src_hi; dst_lo copies dst_hi, so leave - * this degenerate path intact rather than risk the inter-half dep. */ - ot_check( - th_asr_imm((uint32_t)dst_hi, (uint32_t)src_hi, 31, flags_safe(), ENFORCE_ENCODING_NONE)); - ot_check_mov_reg((uint32_t)dst_lo, (uint32_t)dst_hi, flags_safe(), THUMB_SHIFT_DEFAULT, - ENFORCE_ENCODING_NONE, false); - } - else - { - if (!skip_lo) - ot_check(th_mov_imm((uint32_t)dst_lo, 0, flags_safe(), ENFORCE_ENCODING_NONE)); - if (!skip_hi) - ot_check(th_mov_imm((uint32_t)dst_hi, 0, flags_safe(), ENFORCE_ENCODING_NONE)); - } - } - -shift64_imm_done: - /* Write back. A dead half was never materialized, so skip its store. */ - if (store_lo && !skip_lo) - { - MachineOperand dst_lo_op = mach_make_lo_half(dest); - dst_lo_op.btype = IROP_BTYPE_INT32; - mach_writeback_dest(&dst_lo_op, dst_lo); - } - if (store_hi && !skip_hi) - { - MachineOperand dst_hi_op = mach_make_hi_half(dest); - dst_hi_op.btype = IROP_BTYPE_INT32; - mach_writeback_dest(&dst_hi_op, dst_hi); - } - mach_release_all(&mctx); -} - -/* ============================================================ - * MachineOperand-based data processing (_mop path) - * ============================================================ - * thumb_emit_data_processing_mop32: simplified version of - * thumb_emit_data_processing_op32 using MachineOperand instead of IROperand. - * Handles 32-bit non-complex arithmetic/logic ops via the mach_* helpers, - * eliminating the two-layer materialization present in the old path. - */ -static void thumb_emit_data_processing_mop32(const MachineOperand *src1, const MachineOperand *src2, - const MachineOperand *dest, TccIrOp op, ThumbDataProcessingHandler handler, - thumb_flags_behaviour flags, uint32_t barrel_shift) -{ - const bool dest_sets_flags = (op == TCCIR_OP_CMP); - MachineCodegenContext mctx = {0}; - - /* Barrel-shifted immediate src2: the ALU immediate form has no shift field, - * so if src2 carries a barrel-shift annotation but has been lowered to an - * immediate (a rematerialized constant substituted for the shift's source - * register — see ra_mark_rematerializable), fold the shift into the constant - * and clear the annotation. Without this the shift is silently dropped: the - * immediate path below emits ` Rd, Rn, #imm` and the `if (!imm_emitted)` - * shift block never runs (fuzz seed longlong:6393 — `y ^ (u5 LSR #31)` with - * u5 remat'd to #1 wrongly emitted `eor r,r,#1` instead of `eor r,r,#0`). - * The fusion pass keeps amount in 0..31 (never the ARM "0 means 32" LSR/ASR/ - * ROR case), so these C shifts are well-defined. */ - MachineOperand src2_folded; - if (barrel_shift != 0 && src2->kind == MACH_OP_IMM && !src2->needs_deref && !src2->is_64bit) - { - uint32_t stype = (barrel_shift >> 5) & 7; - uint32_t samt = barrel_shift & 31; - uint32_t v = (uint32_t)src2->u.imm.val; - uint32_t r; - switch (stype) - { - case 1: r = v << samt; break; /* LSL */ - case 2: r = v >> samt; break; /* LSR */ - case 3: r = (uint32_t)((int32_t)v >> samt); break; /* ASR */ - case 4: r = samt ? ((v >> samt) | (v << (32 - samt))) : v; break; /* ROR */ - default: r = v; break; - } - src2_folded = *src2; - src2_folded.u.imm.val = (int64_t)(int32_t)r; - src2 = &src2_folded; - barrel_shift = 0; - } - - /* RSB fast path: SUB with immediate src1 → RSB Rd, src2, #imm. - * Avoids materializing the immediate into a register. - * Only attempt when the immediate is encodable as a Thumb-2 modified - * constant (th_pack_const returns non-zero, or imm==0). */ - if (op == TCCIR_OP_SUB && !dest_sets_flags && barrel_shift == 0 && - src1->kind == MACH_OP_IMM && !src1->needs_deref && !src1->is_64bit) - { - uint32_t imm = (uint32_t)src1->u.imm.val; - if (imm == 0 || th_pack_const(imm) != 0) - { - int dest_reg = mach_get_dest_reg(&mctx, dest, 0); - uint32_t excl = thumb_is_hw_reg(dest_reg) ? (1u << (uint32_t)dest_reg) : 0; - int src2_reg = mach_ensure_in_reg(&mctx, src2, excl); - ot_check(th_rsb_imm((uint32_t)dest_reg, (uint32_t)src2_reg, imm, flags, ENFORCE_ENCODING_NONE)); - if (dest->kind != MACH_OP_NONE) - { - const bool needs_wb = dest->kind == MACH_OP_SPILL || dest->kind == MACH_OP_PARAM_STACK || - (dest->kind == MACH_OP_REG && (dest->needs_deref || dest->u.reg.r0 == (int)PREG_REG_NONE)); - if (needs_wb) - mach_writeback_dest(dest, dest_reg); - } - mach_release_all(&mctx); - return; - } - } - - /* Shift-by-0 identity: on ARM, LSR/ASR with immediate field 0 means - * shift-by-32 (yielding 0 / sign-extend), NOT shift-by-0. Fold x >> 0 - * to a plain MOV Rd, Rm so the semantics are correct regardless of - * whether the optimizer managed to simplify the IR. */ - if (!dest_sets_flags && barrel_shift == 0 && - (op == TCCIR_OP_SHR || op == TCCIR_OP_SAR || op == TCCIR_OP_ROR) && - src2->kind == MACH_OP_IMM && !src2->needs_deref && !src2->is_64bit && - (uint32_t)src2->u.imm.val == 0) - { - int dest_reg = mach_get_dest_reg(&mctx, dest, 0); - uint32_t excl = thumb_is_hw_reg(dest_reg) ? (1u << (uint32_t)dest_reg) : 0; - int src1_reg = mach_ensure_in_reg(&mctx, src1, excl); - ot_check_mov_reg((uint32_t)dest_reg, (uint32_t)src1_reg, flags, - THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE, false); - if (dest->kind != MACH_OP_NONE) - { - const bool needs_wb = dest->kind == MACH_OP_SPILL || dest->kind == MACH_OP_PARAM_STACK || - (dest->kind == MACH_OP_REG && (dest->needs_deref || dest->u.reg.r0 == (int)PREG_REG_NONE)); - if (needs_wb) - mach_writeback_dest(dest, dest_reg); - } - mach_release_all(&mctx); - return; - } - - /* Mask fast paths: AND with an immediate the AND-immediate encoding cannot - * hold, lowered as UXTB/UXTH/UBFX/BIC instead of materializing a constant. - * See thumb_try_and_imm_special; the 64-bit half-lowering shares it. */ - if (op == TCCIR_OP_AND && !dest_sets_flags && barrel_shift == 0 && - src2->kind == MACH_OP_IMM && !src2->needs_deref && !src2->is_64bit && - flags != FLAGS_BEHAVIOUR_SET) - { - const uint32_t mask = (uint32_t)src2->u.imm.val; - const ThumbAndImmForm form = thumb_and_imm_form(mask); - /* Classify before resolving: on AND_IMM_NONE nothing has been emitted and - * the general path below resolves both operands itself. */ - if (form != AND_IMM_NONE) - { - int dest_reg = mach_get_dest_reg(&mctx, dest, 0); - uint32_t excl = thumb_is_hw_reg(dest_reg) ? (1u << (uint32_t)dest_reg) : 0; - int src1_reg = mach_ensure_in_reg(&mctx, src1, excl); - thumb_emit_and_imm_special(form, dest_reg, src1_reg, mask, flags); - if (dest->kind != MACH_OP_NONE) - { - const bool needs_wb = dest->kind == MACH_OP_SPILL || dest->kind == MACH_OP_PARAM_STACK || - (dest->kind == MACH_OP_REG && (dest->needs_deref || dest->u.reg.r0 == (int)PREG_REG_NONE)); - if (needs_wb) - mach_writeback_dest(dest, dest_reg); - } - mach_release_all(&mctx); - return; - } - } - - /* 1. Determine dest register (allocate scratch for spills/param/no-reg). - * CMP and other flag-setting ops don't write a result register, so we - * use R0 as a dummy (Rd field is architecturally ignored). */ - int dest_reg; - if (dest_sets_flags) - dest_reg = R0; - else - dest_reg = mach_get_dest_reg(&mctx, dest, 0); - - uint32_t excl = thumb_is_hw_reg(dest_reg) ? (1u << (uint32_t)dest_reg) : 0; - - /* Exclude src2's register from scratch allocation for src1. - * Without this, materializing an immediate for src1 could pick src2's - * register, clobbering it before src2 is read. This applies whether - * src2 is a plain register or a dereferenced one (the address register - * must survive until the load). */ - if (src2->kind == MACH_OP_REG && thumb_is_hw_reg(src2->u.reg.r0)) - excl |= (1u << (uint32_t)src2->u.reg.r0); - - /* 2. Ensure src1 is in a register; add it to the exclusion mask. */ - int src1_reg = mach_ensure_in_reg(&mctx, src1, excl); - if (thumb_is_hw_reg(src1_reg)) - excl |= (1u << (uint32_t)src1_reg); - - /* 3. Try immediate form for src2; fall back to register if needed. */ - bool imm_emitted = false; - int src2_reg = - mach_ensure_imm_or_reg(&mctx, src2, excl, handler.imm_handler, dest_reg, src1_reg, flags, &imm_emitted); - if (!imm_emitted) - { - /* Decode barrel shift annotation (0=none, else type<<5|amount). */ - thumb_shift sh = THUMB_SHIFT_DEFAULT; - if (barrel_shift != 0) - { - static const thumb_shift_type bs_map[] = { - [1] = THUMB_SHIFT_LSL, [2] = THUMB_SHIFT_LSR, - [3] = THUMB_SHIFT_ASR, [4] = THUMB_SHIFT_ROR, - }; - uint32_t stype = (barrel_shift >> 5) & 7; - uint32_t samt = barrel_shift & 31; - sh.type = bs_map[stype]; - sh.value = samt; - sh.mode = THUMB_SHIFT_IMMEDIATE; - } - thumb_enforce_encoding enc = (barrel_shift != 0) ? ENFORCE_ENCODING_32BIT : ENFORCE_ENCODING_NONE; - ot_check(thumb_call_reg_handler(handler.reg_handler, (uint32_t)dest_reg, (uint32_t)src1_reg, (uint32_t)src2_reg, - flags, sh, enc)); - } - - /* 4. Write result back to spill slot / stack param / pointer-dest. */ - if (!dest_sets_flags && dest && dest->kind != MACH_OP_NONE) - { - const bool needs_wb = dest->kind == MACH_OP_SPILL || dest->kind == MACH_OP_PARAM_STACK || - (dest->kind == MACH_OP_REG && (dest->needs_deref || dest->u.reg.r0 == (int)PREG_REG_NONE)); - if (needs_wb) - mach_writeback_dest(dest, dest_reg); - } - - /* 5. Release all scratches in LIFO order. */ - mach_release_all(&mctx); -} - -/* tcc_gen_machine_data_processing_mop: MachineOperand-based entry point for - * arithmetic/logic operations. Called from ir/codegen.c when dest does not - * use a static chain register. - * Dispatches to thumb_emit_data_processing_mop64 / thumb_emit_shift64_mop for - * 64-bit pair destinations, or thumb_emit_data_processing_mop32 for 32-bit. - */ -static void data_processing_mop_impl(MachineOperand src1, MachineOperand src2, MachineOperand dest, TccIrOp op, - thumb_flags_behaviour flags_override, uint32_t barrel_shift); - -void tcc_gen_machine_data_processing_mop(MachineOperand src1, MachineOperand src2, MachineOperand dest, TccIrOp op, - uint32_t barrel_shift) -{ - data_processing_mop_impl(src1, src2, dest, op, flags_safe(), barrel_shift); -} - -void tcc_gen_machine_data_processing_mop_flags(MachineOperand src1, MachineOperand src2, MachineOperand dest, - TccIrOp op, uint32_t barrel_shift) -{ - data_processing_mop_impl(src1, src2, dest, op, FLAGS_BEHAVIOUR_SET, barrel_shift); -} - -static void data_processing_mop_impl(MachineOperand src1, MachineOperand src2, MachineOperand dest, TccIrOp op, - thumb_flags_behaviour flags_override, uint32_t barrel_shift) -{ - ThumbDataProcessingHandler handler; - ThumbDataProcessingHandler carry_handler; /* used for hi word of 64-bit ops */ - bool uses_carry = false; - /* CMP always sets flags — it has no non-flag-setting variant. - * Ignore FLAGS_BEHAVIOUR_BLOCK for CMP; it must always use SET. */ - thumb_flags_behaviour flags = (op == TCCIR_OP_CMP) ? FLAGS_BEHAVIOUR_SET : flags_override; - - switch (op) - { - case TCCIR_OP_ADD: - handler.imm_handler = th_add_imm; - handler.reg_handler = th_add_reg; - carry_handler.imm_handler = th_adc_imm; - carry_handler.reg_handler = th_adc_reg; - uses_carry = true; - break; - case TCCIR_OP_SUB: - handler.imm_handler = th_sub_imm; - handler.reg_handler = th_sub_reg; - carry_handler.imm_handler = th_sbc_imm; - carry_handler.reg_handler = th_sbc_reg; - uses_carry = true; - break; - case TCCIR_OP_CMP: - handler.imm_handler = th_cmp_imm_handler; - handler.reg_handler = th_cmp_reg; - carry_handler.imm_handler = th_sbc_imm; - carry_handler.reg_handler = th_sbc_reg; - uses_carry = true; - break; - case TCCIR_OP_SHL: - handler.imm_handler = th_lsl_imm; - handler.reg_handler = th_lsl_reg; - carry_handler = handler; - break; - case TCCIR_OP_SHR: - handler.imm_handler = th_lsr_imm; - handler.reg_handler = th_lsr_reg; - carry_handler = handler; - break; - case TCCIR_OP_SAR: - handler.imm_handler = th_asr_imm; - handler.reg_handler = th_asr_reg; - carry_handler = handler; - break; - case TCCIR_OP_ROR: - handler.imm_handler = th_ror_imm; - handler.reg_handler = th_ror_reg; - carry_handler = handler; - break; - case TCCIR_OP_OR: - handler.imm_handler = th_orr_imm; - handler.reg_handler = th_orr_reg; - carry_handler = handler; - break; - case TCCIR_OP_AND: - handler.imm_handler = th_and_imm; - handler.reg_handler = th_and_reg; - carry_handler = handler; - break; - case TCCIR_OP_XOR: - handler.imm_handler = th_eor_imm; - handler.reg_handler = th_eor_reg; - carry_handler = handler; - break; - case TCCIR_OP_ADC_GEN: - flags = FLAGS_BEHAVIOUR_SET; - /* fall through */ - case TCCIR_OP_ADC_USE: - handler.imm_handler = th_adc_imm; - handler.reg_handler = th_adc_reg; - carry_handler = handler; - break; - default: - tcc_error("compiler_error: tcc_gen_machine_data_processing_mop: unhandled op %d", (int)op); - return; - } - - /* Dispatch 64-bit pair destinations to the mop64 path. - * CMP has no dest (MACH_OP_NONE), so also check src1 for 64-bit. */ - if (dest.is_64bit || (op == TCCIR_OP_CMP && src1.is_64bit)) - { - if (op == TCCIR_OP_SHL || op == TCCIR_OP_SHR || op == TCCIR_OP_SAR) - { - /* Two independent reasons a half need not be written: shift64_dead_half's - * bitfield-extract rule (bits 16-17), and zero_half64 finding the half a - * provable zero that every consumer now ignores (bit 22/23). */ - const uint32_t zh = (barrel_shift >> 18) & 0x3Fu; - bool skip_lo = ((barrel_shift >> 16) & 1) || (zh & ZH64_D_LO) != 0; - bool skip_hi = ((barrel_shift >> 17) & 1) || (zh & ZH64_D_HI) != 0; - thumb_emit_shift64_mop(&src1, &src2, &dest, op, skip_lo, skip_hi); - } - else - thumb_emit_data_processing_mop64(&src1, &src2, &dest, op, handler, carry_handler, uses_carry, - (barrel_shift >> 18) & 0x3Fu); - return; - } - - thumb_emit_data_processing_mop32(&src1, &src2, &dest, op, handler, flags, barrel_shift & 0xFFFFu); -} - -/* tcc_gen_machine_ubfx_mop: emit UBFX Rd, Rn, #lsb, #width. - * src2 encodes lsb (bits 0-4) and width (bits 5-9). */ -void tcc_gen_machine_ubfx_mop(MachineOperand src1, MachineOperand src2, MachineOperand dest) -{ - MachineCodegenContext ctx = {0}; - int rd = mach_get_dest_reg(&ctx, &dest, 0); - uint32_t excl = (1u << (uint32_t)rd); - int param = (src2.kind == MACH_OP_IMM) ? (int)src2.u.imm.val : 0; - /* The field may live in the HIGH word of a 64-bit source: `(v >> 52) & 0x7FF` - * is one UBFX on that word, with no shift in front of it. */ - if ((param & UBFX_HI_HALF) && src1.is_64bit && !src1.needs_deref) - src1 = mach_make_hi_half(&src1); - int rn = mach_ensure_in_reg(&ctx, &src1, excl); - int lsb = param & 0x1F; - int width = (param >> 5) & 0x1F; - if (width == 0) - width = 8; - int widthm1 = width - 1; - int imm3 = (lsb >> 2) & 0x7; - int imm2 = lsb & 0x3; - /* Thumb-2 UBFX encoding: 11110 0 11 1100 Rn | 0 imm3 Rd imm2 0 widthm1 */ - thumb_opcode op; - op.size = 4; - op.opcode = 0xF3C00000 | ((uint32_t)rn << 16) | ((uint32_t)imm3 << 12) | ((uint32_t)rd << 8) | ((uint32_t)imm2 << 6) | (uint32_t)widthm1; - ot(op); - mach_writeback_dest(&dest, rd); - mach_release_all(&ctx); -} - -/* tcc_machine_has_bit_ops: does the active core encode clz/rbit/rev/rev16? - * All four live in the Thumb-2 main extension (rev/rev16 also have a T16 form, - * but the front end needs one answer for the whole family, so require T32 + - * clz_rbit). */ -ST_FUNC int tcc_machine_has_bit_ops(void) -{ - return arm_target_dependent.feat.t32 && arm_target_dependent.feat.clz_rbit; -} - -/* tcc_gen_machine_bitop1_mop: emit a single-operand bit manipulation, - * dest = (src1), for TCCIR_OP_CLZ / RBIT / REV / REV16. */ -ST_FUNC void tcc_gen_machine_bitop1_mop(MachineOperand src1, MachineOperand dest, TccIrOp op) -{ - MachineCodegenContext ctx = {0}; - int rd = mach_get_dest_reg(&ctx, &dest, 0); - int rm = mach_ensure_in_reg(&ctx, &src1, (1u << (uint32_t)rd)); - switch (op) - { - case TCCIR_OP_CLZ: - ot_check(th_clz((uint32_t)rd, (uint32_t)rm)); - break; - case TCCIR_OP_RBIT: - ot_check(th_rbit((uint32_t)rd, (uint32_t)rm)); - break; - case TCCIR_OP_REV: - ot_check(th_rev((uint32_t)rd, (uint32_t)rm, ENFORCE_ENCODING_NONE)); - break; - case TCCIR_OP_REV16: - ot_check(th_rev16((uint32_t)rd, (uint32_t)rm, ENFORCE_ENCODING_NONE)); - break; - default: - tcc_error("compiler_error: tcc_gen_machine_bitop1_mop: unhandled op %d", (int)op); - break; - } - mach_writeback_dest(&dest, rd); - mach_release_all(&ctx); -} - -/* tcc_gen_machine_sbfx_mop: emit SBFX Rd, Rn, #lsb, #width. - * src2 encodes lsb (bits 0-4) and width (bits 5-9). Same field layout as UBFX; - * only the base opcode differs (0xF3400000 vs 0xF3C00000). */ -void tcc_gen_machine_sbfx_mop(MachineOperand src1, MachineOperand src2, MachineOperand dest) -{ - MachineCodegenContext ctx = {0}; - int rd = mach_get_dest_reg(&ctx, &dest, 0); - uint32_t excl = (1u << (uint32_t)rd); - int rn = mach_ensure_in_reg(&ctx, &src1, excl); - int param = (src2.kind == MACH_OP_IMM) ? (int)src2.u.imm.val : 0; - int lsb = param & 0x1F; - int width = (param >> 5) & 0x1F; - if (width == 0) - width = 8; - int widthm1 = width - 1; - int imm3 = (lsb >> 2) & 0x7; - int imm2 = lsb & 0x3; - /* Thumb-2 SBFX encoding: 11110 0 11 0100 Rn | 0 imm3 Rd imm2 0 widthm1 */ - thumb_opcode op; - op.size = 4; - op.opcode = 0xF3400000 | ((uint32_t)rn << 16) | ((uint32_t)imm3 << 12) | ((uint32_t)rd << 8) | ((uint32_t)imm2 << 6) | (uint32_t)widthm1; - ot(op); - mach_writeback_dest(&dest, rd); - mach_release_all(&ctx); -} - -/* tcc_gen_machine_bfi_mop: emit BFI Rd, Rn, #lsb, #width. - * src1 = host word (moved into Rd, the BFI base, if not already there), - * src2 = value supplying the field bits (only its low `width` bits are used), - * dest = result. params packs lsb (bits 0-7) and width (bits 8-15). */ -void tcc_gen_machine_bfi_mop(MachineOperand src1, MachineOperand src2, MachineOperand dest, uint32_t params) -{ - MachineCodegenContext ctx = {0}; - int rd = mach_get_dest_reg(&ctx, &dest, 0); - int rn = mach_ensure_in_reg(&ctx, &src2, 0); /* value (Rn) */ - int rword = mach_ensure_in_reg(&ctx, &src1, (1u << (uint32_t)rn)); /* host word */ - /* Establish Rd = host word. If the value happens to live in Rd (RA coalesced - * the result onto src2), preserve it in a scratch before clobbering Rd. */ - if (rd != rword) - { - if (rd == rn) - { - int tmp = mach_alloc_scratch(&ctx, (1u << (uint32_t)rd) | (1u << (uint32_t)rword)); - ot_check_mov_reg((uint32_t)tmp, (uint32_t)rd, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE, false); - rn = tmp; - } - ot_check_mov_reg((uint32_t)rd, (uint32_t)rword, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE, false); - } - int lsb = (int)(params & 0xFF); - int width = (int)((params >> 8) & 0xFF); - if (width < 1) - width = 1; - int msb = lsb + width - 1; - if (msb > 31) - msb = 31; - int imm3 = (lsb >> 2) & 0x7; - int imm2 = lsb & 0x3; - /* Thumb-2 BFI: 11110 0 11 0110 Rn | 0 imm3 Rd imm2 0 msb */ - thumb_opcode op; - op.size = 4; - op.opcode = 0xF3600000 | ((uint32_t)rn << 16) | ((uint32_t)imm3 << 12) | ((uint32_t)rd << 8) | ((uint32_t)imm2 << 6) | (uint32_t)msb; - ot(op); - mach_writeback_dest(&dest, rd); - mach_release_all(&ctx); -} - -/* ============================================================ - * MachineOperand-based mul/div/mod/test-zero (_mop path) - * ============================================================ - * Internal helpers and public entry point for 32-bit register-only ops: - * MUL, DIV, UDIV — simple rd = rn OP rm - * IMOD, UMOD — dest = src1 - (src1/src2)*src2 - * TEST_ZERO — CMP src, #0 (flags only, no dest) - * MLA (accumulator) and UMULL (64-bit pair) remain on the old IR path. - */ - -/* Emit rd = emitter(src1, src2) for register-only 3-operand ops. */ -static void mach_regonly_binop_mop(MachineCodegenContext *ctx, const MachineOperand *src1, const MachineOperand *src2, - const MachineOperand *dest, thumb_regonly3_handler_t emitter) -{ - /* 1. Get dest register (scratch if spill/param). */ - int dest_reg = mach_get_dest_reg(ctx, dest, 0); - uint32_t excl = thumb_is_hw_reg(dest_reg) ? (1u << (uint32_t)dest_reg) : 0; - - /* Pre-exclude src2's physical register so that loading src1 (which may - * need a scratch for deref) does not clobber src2's value. This must apply - * even when src2 is a DEREF operand: its register holds the pointer, and the - * step-3 load reads through that same register, so it has to survive src1's - * materialization. Omitting the deref case let src1's scratch land on the - * pointer register at high pressure -> `u6 * arr10[3]` dereferenced u6 as a - * pointer -> bus fault (fuzz switch 219754). Matches the data-processing - * guard in thumb_emit_data_processing_mop32. */ - if (src2->kind == MACH_OP_REG && thumb_is_hw_reg(src2->u.reg.r0)) - excl |= (1u << (uint32_t)src2->u.reg.r0); - - /* 2. Ensure src1 in a register; extend exclusion mask. */ - int src1_reg = mach_ensure_in_reg(ctx, src1, excl); - if (thumb_is_hw_reg(src1_reg)) - excl |= (1u << (uint32_t)src1_reg); - - /* 3. Ensure src2 in a register. */ - int src2_reg = mach_ensure_in_reg(ctx, src2, excl); - - /* 4. Emit instruction. */ - ot_check(emitter((uint32_t)dest_reg, (uint32_t)src1_reg, (uint32_t)src2_reg)); - - /* 5. Write result back to spill slot / stack param if needed. */ - mach_writeback_dest(dest, dest_reg); -} - -/* Emit dest = src1 - (src1/src2)*src2 for IMOD/UMOD. */ -static void mach_mod_mop(MachineCodegenContext *ctx, const MachineOperand *src1, const MachineOperand *src2, - const MachineOperand *dest, thumb_regonly3_handler_t div_emitter) -{ - /* 1. Get dest register. */ - int dest_reg = mach_get_dest_reg(ctx, dest, 0); - uint32_t excl = thumb_is_hw_reg(dest_reg) ? (1u << (uint32_t)dest_reg) : 0; - - /* Pre-exclude src2's physical register so that materializing src1 (which may - * need a scratch when it is an immediate or a deref) does not clobber src2's - * value before the divide reads it — same guard as mach_regonly_binop_mop. - * Without it an immediate dividend's scratch load could land on the divisor's - * register (random-C O1 wrong-code, seed 151: `K % (lr|1)` divisor clobbered). - * Applies to the deref case too (pointer register must survive src1's - * materialization, same as mach_regonly_binop_mop / fuzz switch 219754). */ - if (src2->kind == MACH_OP_REG && thumb_is_hw_reg(src2->u.reg.r0)) - excl |= (1u << (uint32_t)src2->u.reg.r0); - - /* 2. Ensure src1 in a register. */ - int src1_reg = mach_ensure_in_reg(ctx, src1, excl); - if (thumb_is_hw_reg(src1_reg)) - excl |= (1u << (uint32_t)src1_reg); - - /* 3. Ensure src2 in a register. */ - int src2_reg = mach_ensure_in_reg(ctx, src2, excl); - if (thumb_is_hw_reg(src2_reg)) - excl |= (1u << (uint32_t)src2_reg); - - /* 4. Scratch register for quotient. */ - int quotient_reg = mach_alloc_scratch(ctx, excl); - - /* 5. quotient = src1 / src2 */ - ot_check(div_emitter((uint32_t)quotient_reg, (uint32_t)src1_reg, (uint32_t)src2_reg)); - - /* 6. quotient = quotient * src2 */ - ot_check(thumb_mul_regonly((uint32_t)quotient_reg, (uint32_t)quotient_reg, (uint32_t)src2_reg)); - - /* 7. dest = src1 - quotient */ - ot_check(th_sub_reg(dest_reg, src1_reg, quotient_reg, flags_safe(), THUMB_SHIFT_DEFAULT, - ENFORCE_ENCODING_NONE)); - - /* 8. Write result back. */ - mach_writeback_dest(dest, dest_reg); -} - -/* thumb_emit_mul64_mop - * ============================================================ - * Emit a 64-bit multiply (lower 64 bits of the result) using MachineOperands. - * - * For a 64-bit result (dest->is_64bit): - * UMULL r_c_lo, r_c_hi, r_a_lo, r_b_lo // a_lo * b_lo → 64-bit unsigned - * MLA r_c_hi, r_a_hi, r_b_lo, r_c_hi // cross product (when src1 is 64-bit) - * MLA r_c_hi, r_a_lo, r_b_hi, r_c_hi // cross product (when src2 is 64-bit) - * - * For a 32-bit result with 64-bit source(s): - * MUL r_c, r_a_lo, r_b_lo // upper bits don't contribute - * - * The lower 64 bits of the signed / unsigned 128-bit product are identical - * (i.e. UMULL is correct for both signed and unsigned long long mul). - * The caller must call mach_release_all() after this function returns. - */ -static void thumb_emit_mul64_mop(MachineCodegenContext *ctx, const MachineOperand *src1, const MachineOperand *src2, - const MachineOperand *dest) -{ - uint32_t excl = 0; - - /* Resolve deref'd 64-bit sources before splitting into halves. */ - MachineOperand r_s1 = mach_resolve_deref_64(ctx, src1, &excl); - MachineOperand r_s2 = mach_resolve_deref_64(ctx, src2, &excl); - - /* Load lo halves (always needed). */ - MachineOperand a_lo_op = r_s1.is_64bit ? mach_make_lo_half(&r_s1) : r_s1; - a_lo_op.btype = IROP_BTYPE_INT32; - a_lo_op.is_64bit = false; - MachineOperand b_lo_op = r_s2.is_64bit ? mach_make_lo_half(&r_s2) : r_s2; - b_lo_op.btype = IROP_BTYPE_INT32; - b_lo_op.is_64bit = false; - - int r_a_lo = mach_ensure_in_reg(ctx, &a_lo_op, excl); - if (thumb_is_hw_reg(r_a_lo)) - excl |= (1u << (uint32_t)r_a_lo); - int r_b_lo = mach_ensure_in_reg(ctx, &b_lo_op, excl); - if (thumb_is_hw_reg(r_b_lo)) - excl |= (1u << (uint32_t)r_b_lo); - - if (dest->is_64bit) - { - /* Load hi halves for cross-product MLA terms. */ - int r_a_hi = PREG_REG_NONE, r_b_hi = PREG_REG_NONE; - if (r_s1.is_64bit) - { - MachineOperand a_hi_op = mach_make_hi_half(&r_s1); - a_hi_op.btype = IROP_BTYPE_INT32; - r_a_hi = mach_ensure_in_reg(ctx, &a_hi_op, excl); - if (thumb_is_hw_reg(r_a_hi)) - excl |= (1u << (uint32_t)r_a_hi); - } - if (r_s2.is_64bit) - { - MachineOperand b_hi_op = mach_make_hi_half(&r_s2); - b_hi_op.btype = IROP_BTYPE_INT32; - r_b_hi = mach_ensure_in_reg(ctx, &b_hi_op, excl); - if (thumb_is_hw_reg(r_b_hi)) - excl |= (1u << (uint32_t)r_b_hi); - } - - /* Allocate 64-bit destination pair — must not overlap sources for UMULL. */ - MachineOperand dst_lo_op = mach_make_lo_half(dest); - dst_lo_op.btype = IROP_BTYPE_INT32; - MachineOperand dst_hi_op = mach_make_hi_half(dest); - dst_hi_op.btype = IROP_BTYPE_INT32; - - int r_c_lo = mach_get_dest_reg(ctx, &dst_lo_op, excl); - if (thumb_is_hw_reg(r_c_lo)) - excl |= (1u << (uint32_t)r_c_lo); - int r_c_hi = mach_get_dest_reg(ctx, &dst_hi_op, excl); - - /* UMULL: r_c_lo:r_c_hi = r_a_lo * r_b_lo (unsigned 64-bit product) */ - ot_check(th_umull((uint32_t)r_c_lo, (uint32_t)r_c_hi, (uint32_t)r_a_lo, (uint32_t)r_b_lo)); - - /* Add cross products to high half. */ - if (thumb_is_hw_reg(r_a_hi)) - ot_check(th_mla((uint32_t)r_c_hi, (uint32_t)r_a_hi, (uint32_t)r_b_lo, (uint32_t)r_c_hi)); - if (thumb_is_hw_reg(r_b_hi)) - ot_check(th_mla((uint32_t)r_c_hi, (uint32_t)r_a_lo, (uint32_t)r_b_hi, (uint32_t)r_c_hi)); - - mach_writeback_dest(&dst_lo_op, r_c_lo); - mach_writeback_dest(&dst_hi_op, r_c_hi); - } - else - { - /* 32-bit result with 64-bit source(s): only the low bits matter. */ - MachineOperand dest32 = *dest; - dest32.is_64bit = false; - int r_c = mach_get_dest_reg(ctx, &dest32, excl); - ot_check(thumb_mul_regonly((uint32_t)r_c, (uint32_t)r_a_lo, (uint32_t)r_b_lo)); - mach_writeback_dest(&dest32, r_c); - } -} - -/* tcc_gen_machine_muldiv_mop: MachineOperand-based entry point for multiply, - * divide, modulo, and test-zero operations. Called from ir/codegen.c when - * use_mop_muldiv is true for: - * MUL — 32-bit or 64-bit multiply - * DIV, UDIV, IMOD, UMOD — 32-bit divide/modulo - * TEST_ZERO — 32-bit or 64-bit compare against zero (flags only) - * MLA (accumulator; 4-operand) uses tcc_gen_machine_mla_mop. - * UMULL (64-bit output from 32-bit inputs) uses tcc_gen_machine_umull_mop. - */ -/* Decompose multiply-by-constant into shift+add sequences. - * Returns 1 if handled, 0 to fall back to hardware MUL. */ -static int thumb_try_mul_by_const_mop(MachineCodegenContext *ctx, MachineOperand *src1, MachineOperand *src2, - MachineOperand *dest) -{ - /* Identify which operand is the immediate and which is the variable. */ - const MachineOperand *imm_op, *var_op; - if (src2->kind == MACH_OP_IMM) - { - imm_op = src2; - var_op = src1; - } - else if (src1->kind == MACH_OP_IMM) - { - imm_op = src1; - var_op = src2; - } - else - return 0; - - int64_t c = imm_op->u.imm.val; - if (c <= 0) - return 0; - - /* Determine the decomposition pattern. - * We handle: powers of 2, (2^n ± 1), and products thereof. - * - * Pattern Insns Example - * ─────────────────── ───── ─────── - * 2^n 1 LSL Rd, Rn, #n - * 2^n + 1 1 ADD Rd, Rn, Rn LSL #n - * 2^n - 1 1 SUB Rd, Rn LSL #n, Rn (RSB-like via SUB) - * (2^a + 1) * 2^b 2 ADD Rd, Rn, Rn LSL #a; LSL Rd, Rd, #b - * (2^a - 1) * 2^b 2 SUB Rd, Rn LSL #a, Rn; LSL Rd, Rd, #b - * (2^a + 1)(2^b + 1) 2 ADD Rd, Rn, Rn LSL #a; ADD Rd, Rd, Rn LSL #(a+b) - * — only some cases, handled via table - */ - - int shift1 = 0, shift2 = 0; - enum - { - MUL_NONE, - MUL_POWER_OF_2, /* c = 2^n : LSL #n */ - MUL_TWO_N_PLUS_1, /* c = 2^n+1 : ADD Rd, Rn, Rn LSL #n */ - MUL_TWO_N_MINUS_1, /* c = 2^n-1 : SUB Rd, Rn LSL #n, Rn */ - MUL_TWO_N_PLUS_1_SHIFT, /* c = (2^a+1)*2^b : ADD; LSL */ - MUL_TWO_N_MINUS_1_SHIFT, /* c = (2^a-1)*2^b : SUB; LSL */ - } pattern = MUL_NONE; - - /* Check for power of 2 */ - if (c > 0 && (c & (c - 1)) == 0) - { - int n = 0; - int64_t v = c; - while (v > 1) - { - n++; - v >>= 1; - } - if (n >= 1 && n <= 31) - { - shift1 = n; - pattern = MUL_POWER_OF_2; - } - } - - /* Check for 2^n + 1 (3, 5, 9, 17, ...) */ - if (pattern == MUL_NONE && c >= 3) - { - int64_t v = c - 1; - if (v > 0 && (v & (v - 1)) == 0) - { - int n = 0; - while (v > 1) - { - n++; - v >>= 1; - } - if (n >= 1 && n <= 31) - { - shift1 = n; - pattern = MUL_TWO_N_PLUS_1; - } - } - } - - /* Check for 2^n - 1 (7, 15, 31, ...) */ - if (pattern == MUL_NONE && c >= 7) - { - int64_t v = c + 1; - if (v > 0 && (v & (v - 1)) == 0) - { - int n = 0; - while (v > 1) - { - n++; - v >>= 1; - } - if (n >= 2 && n <= 31) - { - shift1 = n; - pattern = MUL_TWO_N_MINUS_1; - } - } - } - - /* Check for (2^a + 1) * 2^b (6, 10, 12, 20, 24, 40, 48, ...) */ - if (pattern == MUL_NONE && c >= 6) - { - int64_t v = c; - int b = 0; - while ((v & 1) == 0) - { - b++; - v >>= 1; - } - if (b >= 1 && b <= 31) - { - int64_t inner = v - 1; - if (inner > 0 && (inner & (inner - 1)) == 0) - { - int a = 0; - while (inner > 1) - { - a++; - inner >>= 1; - } - if (a >= 1 && a <= 31) - { - shift1 = a; - shift2 = b; - pattern = MUL_TWO_N_PLUS_1_SHIFT; - } - } - } - } - - /* Check for (2^a - 1) * 2^b (14, 28, 30, 56, 60, 62, ...) */ - if (pattern == MUL_NONE && c >= 14) - { - int64_t v = c; - int b = 0; - while ((v & 1) == 0) - { - b++; - v >>= 1; - } - if (b >= 1 && b <= 31) - { - int64_t inner = v + 1; - if (inner > 0 && (inner & (inner - 1)) == 0) - { - int a = 0; - while (inner > 1) - { - a++; - inner >>= 1; - } - if (a >= 2 && a <= 31) - { - shift1 = a; - shift2 = b; - pattern = MUL_TWO_N_MINUS_1_SHIFT; - } - } - } - } - - if (pattern == MUL_NONE) - return 0; - - /* Emit the decomposed sequence. */ - int dest_reg = mach_get_dest_reg(ctx, dest, 0); - uint32_t excl = thumb_is_hw_reg(dest_reg) ? (1u << (uint32_t)dest_reg) : 0; - int var_reg = mach_ensure_in_reg(ctx, var_op, excl); - thumb_flags_behaviour fl = flags_safe(); - thumb_shift sh; - - switch (pattern) - { - case MUL_POWER_OF_2: - ot_check(th_lsl_imm((uint32_t)dest_reg, (uint32_t)var_reg, (uint32_t)shift1, fl, ENFORCE_ENCODING_NONE)); - break; - - case MUL_TWO_N_PLUS_1: - sh = (thumb_shift){THUMB_SHIFT_LSL, (uint16_t)shift1, THUMB_SHIFT_IMMEDIATE}; - ot_check(th_add_reg((uint32_t)dest_reg, (uint32_t)var_reg, (uint32_t)var_reg, fl, sh, ENFORCE_ENCODING_NONE)); - break; - - case MUL_TWO_N_MINUS_1: - { - /* Thumb-2 SUB Rd, Rn, Rm LSL #n = Rn - (Rm << n). - * We need (var << n) - var, which is the reverse. No RSB with shift - * exists in Thumb-2, so we do: LSL tmp, var, #n; SUB Rd, tmp, var. - * The LSL destination must differ from var_reg, otherwise it destroys - * var before the SUB reads it (mach_ensure_in_reg returns an already- - * resident var in dest_reg's register, ignoring the exclusion mask, so - * dest_reg == var_reg is reachable). Shift straight into dest when they - * differ; otherwise borrow a scratch. */ - int tmp = (dest_reg == var_reg) ? mach_alloc_scratch(ctx, (1u << (uint32_t)var_reg)) : dest_reg; - ot_check(th_lsl_imm((uint32_t)tmp, (uint32_t)var_reg, (uint32_t)shift1, fl, ENFORCE_ENCODING_NONE)); - sh = (thumb_shift){THUMB_SHIFT_NONE, 0, THUMB_SHIFT_IMMEDIATE}; - ot_check(th_sub_reg((uint32_t)dest_reg, (uint32_t)tmp, (uint32_t)var_reg, fl, sh, ENFORCE_ENCODING_NONE)); - break; - } - - case MUL_TWO_N_PLUS_1_SHIFT: - sh = (thumb_shift){THUMB_SHIFT_LSL, (uint16_t)shift1, THUMB_SHIFT_IMMEDIATE}; - ot_check(th_add_reg((uint32_t)dest_reg, (uint32_t)var_reg, (uint32_t)var_reg, fl, sh, ENFORCE_ENCODING_NONE)); - ot_check(th_lsl_imm((uint32_t)dest_reg, (uint32_t)dest_reg, (uint32_t)shift2, fl, ENFORCE_ENCODING_NONE)); - break; - - case MUL_TWO_N_MINUS_1_SHIFT: - { - /* (2^a - 1) * 2^b: LSL tmp, var, #a; SUB tmp, tmp, var; LSL Rd, tmp, #b. - * As in MUL_TWO_N_MINUS_1, the first LSL must not target var_reg, or it - * destroys var before the SUB reads it. */ - int tmp = (dest_reg == var_reg) ? mach_alloc_scratch(ctx, (1u << (uint32_t)var_reg)) : dest_reg; - ot_check(th_lsl_imm((uint32_t)tmp, (uint32_t)var_reg, (uint32_t)shift1, fl, ENFORCE_ENCODING_NONE)); - sh = (thumb_shift){THUMB_SHIFT_NONE, 0, THUMB_SHIFT_IMMEDIATE}; - ot_check(th_sub_reg((uint32_t)tmp, (uint32_t)tmp, (uint32_t)var_reg, fl, sh, ENFORCE_ENCODING_NONE)); - ot_check(th_lsl_imm((uint32_t)dest_reg, (uint32_t)tmp, (uint32_t)shift2, fl, ENFORCE_ENCODING_NONE)); - break; - } - - default: - return 0; - } - - mach_writeback_dest(dest, dest_reg); - mach_release_all(ctx); - return 1; -} - -/* Fused MUL-by-const + ADD peephole. - * Transforms: tmp = var * C; dest = base + tmp - * Into a shorter sequence using ARM shifted-add (ADD Rd, Rn, Rm LSL #imm): - * C = 2^n: ADD dest, base, var LSL #n (1 insn vs 2) - * C = (2^a+1)*2^b: ADD t, var, var LSL #a; - * ADD dest, base, t LSL #b (2 insn vs 3) - * C = (2^a-1)*2^b: LSL t, var, #a; SUB t, t, var; - * ADD dest, base, t LSL #b (3 insn vs 4) - * Returns 1 if fused, 0 to fall back to separate MUL + ADD. */ -ST_FUNC int tcc_gen_machine_mul_const_add_fused_mop(MachineOperand mul_var, int64_t mul_const, - MachineOperand mul_dest, MachineOperand add_base, - MachineOperand add_dest) -{ - if (mul_const <= 0) - return 0; - - int shift1 = 0, shift2 = 0; - enum - { - FUSE_NONE, - FUSE_POW2, - FUSE_TWO_N_PLUS_1_SHIFT, - FUSE_TWO_N_MINUS_1_SHIFT, - } pattern = FUSE_NONE; - - /* Power of 2: C = 2^n */ - if (mul_const > 1 && (mul_const & (mul_const - 1)) == 0) - { - int n = 0; - int64_t v = mul_const; - while (v > 1) { n++; v >>= 1; } - if (n >= 1 && n <= 31) - { - shift1 = n; - pattern = FUSE_POW2; - } - } - - /* (2^a + 1) * 2^b: e.g. 12 = 3*4 = (2^1+1)*2^2 */ - if (pattern == FUSE_NONE && mul_const >= 6) - { - int64_t v = mul_const; - int b = 0; - while ((v & 1) == 0) { b++; v >>= 1; } - if (b >= 1 && b <= 31) - { - int64_t inner = v - 1; - if (inner > 0 && (inner & (inner - 1)) == 0) - { - int a = 0; - while (inner > 1) { a++; inner >>= 1; } - if (a >= 1 && a <= 31) - { - shift1 = a; - shift2 = b; - pattern = FUSE_TWO_N_PLUS_1_SHIFT; - } - } - } - } - - /* (2^a - 1) * 2^b: e.g. 28 = 7*4 = (2^3-1)*2^2 */ - if (pattern == FUSE_NONE && mul_const >= 14) - { - int64_t v = mul_const; - int b = 0; - while ((v & 1) == 0) { b++; v >>= 1; } - if (b >= 1 && b <= 31) - { - int64_t inner = v + 1; - if (inner > 0 && (inner & (inner - 1)) == 0) - { - int a = 0; - while (inner > 1) { a++; inner >>= 1; } - if (a >= 2 && a <= 31) - { - shift1 = a; - shift2 = b; - pattern = FUSE_TWO_N_MINUS_1_SHIFT; - } - } - } - } - - if (pattern == FUSE_NONE) - return 0; - - MachineCodegenContext ctx = {0}; - thumb_flags_behaviour fl = flags_safe(); - thumb_shift sh; - - /* Allocate registers: dest first (may hint to the ADD dest's phys reg), - * then base and var, using exclusion masks to prevent conflicts. */ - int dest_reg = mach_get_dest_reg(&ctx, &add_dest, 0); - uint32_t excl = thumb_is_hw_reg(dest_reg) ? (1u << (uint32_t)dest_reg) : 0; - int base_reg = mach_ensure_in_reg(&ctx, &add_base, excl); - if (thumb_is_hw_reg(base_reg)) - excl |= (1u << (uint32_t)base_reg); - int var_reg = mach_ensure_in_reg(&ctx, &mul_var, excl); - - switch (pattern) - { - case FUSE_POW2: - /* ADD dest, base, var LSL #n */ - sh = (thumb_shift){THUMB_SHIFT_LSL, (uint16_t)shift1, THUMB_SHIFT_IMMEDIATE}; - ot_check(th_add_reg((uint32_t)dest_reg, (uint32_t)base_reg, (uint32_t)var_reg, fl, sh, ENFORCE_ENCODING_NONE)); - break; - - case FUSE_TWO_N_PLUS_1_SHIFT: - { - /* Step 1: ADD tmp, var, var LSL #a */ - if (thumb_is_hw_reg(var_reg)) - excl |= (1u << (uint32_t)var_reg); - int tmp_reg = mach_get_dest_reg(&ctx, &mul_dest, excl); - sh = (thumb_shift){THUMB_SHIFT_LSL, (uint16_t)shift1, THUMB_SHIFT_IMMEDIATE}; - ot_check(th_add_reg((uint32_t)tmp_reg, (uint32_t)var_reg, (uint32_t)var_reg, fl, sh, ENFORCE_ENCODING_NONE)); - /* Step 2: ADD dest, base, tmp LSL #b */ - sh = (thumb_shift){THUMB_SHIFT_LSL, (uint16_t)shift2, THUMB_SHIFT_IMMEDIATE}; - ot_check(th_add_reg((uint32_t)dest_reg, (uint32_t)base_reg, (uint32_t)tmp_reg, fl, sh, ENFORCE_ENCODING_NONE)); - break; - } - - case FUSE_TWO_N_MINUS_1_SHIFT: - { - /* Step 1: LSL tmp, var, #a */ - if (thumb_is_hw_reg(var_reg)) - excl |= (1u << (uint32_t)var_reg); - int tmp_reg = mach_get_dest_reg(&ctx, &mul_dest, excl); - ot_check(th_lsl_imm((uint32_t)tmp_reg, (uint32_t)var_reg, (uint32_t)shift1, fl, ENFORCE_ENCODING_NONE)); - /* Step 2: SUB tmp, tmp, var */ - sh = (thumb_shift){THUMB_SHIFT_NONE, 0, THUMB_SHIFT_IMMEDIATE}; - ot_check(th_sub_reg((uint32_t)tmp_reg, (uint32_t)tmp_reg, (uint32_t)var_reg, fl, sh, ENFORCE_ENCODING_NONE)); - /* Step 3: ADD dest, base, tmp LSL #b */ - sh = (thumb_shift){THUMB_SHIFT_LSL, (uint16_t)shift2, THUMB_SHIFT_IMMEDIATE}; - ot_check(th_add_reg((uint32_t)dest_reg, (uint32_t)base_reg, (uint32_t)tmp_reg, fl, sh, ENFORCE_ENCODING_NONE)); - break; - } - - default: - mach_release_all(&ctx); - return 0; - } - - mach_writeback_dest(&add_dest, dest_reg); - mach_release_all(&ctx); - return 1; -} - -ST_FUNC void tcc_gen_machine_muldiv_mop(MachineOperand src1, MachineOperand src2, MachineOperand dest, TccIrOp op) -{ - MachineCodegenContext ctx = {0}; - switch (op) - { - case TCCIR_OP_MUL: - if (src1.is_64bit || src2.is_64bit || dest.is_64bit) - thumb_emit_mul64_mop(&ctx, &src1, &src2, &dest); - else if (!thumb_try_mul_by_const_mop(&ctx, &src1, &src2, &dest)) - mach_regonly_binop_mop(&ctx, &src1, &src2, &dest, thumb_mul_regonly); - break; - case TCCIR_OP_DIV: - mach_regonly_binop_mop(&ctx, &src1, &src2, &dest, thumb_sdiv_regonly); - break; - case TCCIR_OP_UDIV: - mach_regonly_binop_mop(&ctx, &src1, &src2, &dest, thumb_udiv_regonly); - break; - case TCCIR_OP_IMOD: - mach_mod_mop(&ctx, &src1, &src2, &dest, thumb_sdiv_regonly); - break; - case TCCIR_OP_UMOD: - mach_mod_mop(&ctx, &src1, &src2, &dest, thumb_udiv_regonly); - break; - case TCCIR_OP_TEST_ZERO: - { - if (src1.is_64bit) - { - /* 64-bit: Z set iff (lo == 0 && hi == 0). - * Use CMP lo,#0; IT EQ; CMPEQ hi,#0 to avoid clobbering source registers. */ - uint32_t excl = 0; - MachineOperand resolved = mach_resolve_deref_64(&ctx, &src1, &excl); - MachineOperand lo = mach_make_lo_half(&resolved); - lo.btype = IROP_BTYPE_INT32; - MachineOperand hi = mach_make_hi_half(&resolved); - hi.btype = IROP_BTYPE_INT32; - int r_lo = mach_ensure_in_reg(&ctx, &lo, excl); - if (thumb_is_hw_reg(r_lo)) - excl |= (1u << (uint32_t)r_lo); - int r_hi = mach_ensure_in_reg(&ctx, &hi, excl); - if (!thumb_try_orrs_zero64(r_lo, r_hi)) - { - ot_check(th_cmp_imm(r_lo, 0, FLAGS_BEHAVIOUR_SET, ENFORCE_ENCODING_NONE)); - th_literal_pool_reserve_upcoming_bytes(6); - ot_check(th_it(mapcc(TOK_EQ), 0x8)); /* IT EQ (single instruction) */ - ot_check(th_cmp_imm(r_hi, 0, FLAGS_BEHAVIOUR_SET, ENFORCE_ENCODING_NONE)); - } - } - else - { - /* 32-bit: CMP src, #0 — no destination, only flags. */ - int src_reg = mach_ensure_in_reg(&ctx, &src1, 0); - ot_check(th_cmp_imm(src_reg, 0, FLAGS_BEHAVIOUR_SET, ENFORCE_ENCODING_NONE)); - } - break; - } - default: - tcc_error("compiler_error: tcc_gen_machine_muldiv_mop: unhandled op %d", (int)op); - break; - } - mach_release_all(&ctx); -} - -/* tcc_gen_machine_cmp_eq64_mop: 64-bit equality comparison. - * Emits CMP hi1,hi2; IT EQ; CMPEQ lo1,lo2 which correctly sets - * the Z flag for full 64-bit equality (used by SETIF/JUMPIF EQ/NE). */ -ST_FUNC void tcc_gen_machine_cmp_eq64_mop(MachineOperand src1, MachineOperand src2) -{ - MachineCodegenContext ctx = {0}; - uint32_t excl = 0; - - if (src1.kind == MACH_OP_REG) - { - if (src1.u.reg.r0 != (int)PREG_REG_NONE) - excl |= (1u << (uint32_t)src1.u.reg.r0); - if (!src1.needs_deref && src1.is_64bit && src1.u.reg.r1 >= 0) - excl |= (1u << (uint32_t)src1.u.reg.r1); - } - if (src2.kind == MACH_OP_REG) - { - if (src2.u.reg.r0 != (int)PREG_REG_NONE) - excl |= (1u << (uint32_t)src2.u.reg.r0); - if (!src2.needs_deref && src2.is_64bit && src2.u.reg.r1 >= 0) - excl |= (1u << (uint32_t)src2.u.reg.r1); - } - - MachineOperand r_src1 = mach_resolve_deref_64(&ctx, &src1, &excl); - MachineOperand r_src2 = mach_resolve_deref_64(&ctx, &src2, &excl); - - int rn_lo, rn_hi; - mach_ensure_pair_in_regs(&ctx, &r_src1, &excl, &rn_lo, &rn_hi); - - /* Immediate-CMP fast path: if src2 is a u64 immediate, try the cmp-imm - * form (`cmp.w Rn, #imm`) for each half — avoids loading the constant - * into a scratch reg. Probe encodability before allocating scratches: - * `mach_ensure_in_reg` on a MACH_OP_IMM would unconditionally emit a - * `movs Rscratch, #imm`, which is exactly the instruction we're trying - * to avoid here. */ - thumb_opcode hi_imm_op = {0}; - thumb_opcode lo_imm_op = {0}; - if (r_src2.kind == MACH_OP_IMM) - { - const uint64_t imm = (uint64_t)r_src2.u.imm.val; - const uint32_t imm_lo = (uint32_t)(imm & 0xffffffffu); - const uint32_t imm_hi = (uint32_t)(imm >> 32); - hi_imm_op = th_cmp_imm((uint32_t)rn_hi, imm_hi, FLAGS_BEHAVIOUR_SET, ENFORCE_ENCODING_NONE); - lo_imm_op = th_cmp_imm((uint32_t)rn_lo, imm_lo, FLAGS_BEHAVIOUR_SET, ENFORCE_ENCODING_NONE); - } - - /* Use cmp-imm for whichever halves fit; only allocate scratch - * registers for halves that need them. */ - int hi_uses_imm = (r_src2.kind == MACH_OP_IMM && hi_imm_op.size); - int lo_uses_imm = (r_src2.kind == MACH_OP_IMM && lo_imm_op.size); - - /* Comparing against literal zero is `(lo | hi) == 0`, one ORRS. */ - if (r_src2.kind == MACH_OP_IMM && (uint64_t)r_src2.u.imm.val == 0 && - thumb_try_orrs_zero64(rn_lo, rn_hi)) - { - mach_release_all(&ctx); - return; - } - - int rm_lo = 0, rm_hi = 0; - if (!lo_uses_imm && !hi_uses_imm) - { - /* Both halves are needed in registers — pair them (the immediate fast path - * above is the only case that wants one half loaded and not the other). */ - mach_ensure_pair_in_regs(&ctx, &r_src2, &excl, &rm_lo, &rm_hi); - } - else - { - if (!lo_uses_imm) - { - MachineOperand s2_lo = mach_make_lo_half(&r_src2); - s2_lo.btype = IROP_BTYPE_INT32; - rm_lo = mach_ensure_in_reg(&ctx, &s2_lo, excl); - if (thumb_is_hw_reg(rm_lo)) - excl |= (1u << (uint32_t)rm_lo); - } - if (!hi_uses_imm) - { - MachineOperand s2_hi = mach_make_hi_half(&r_src2); - s2_hi.btype = IROP_BTYPE_INT32; - rm_hi = mach_ensure_in_reg(&ctx, &s2_hi, excl); - } - } - - if (hi_uses_imm) - ot_check(hi_imm_op); - else - ot_check(th_cmp_reg(0, (uint32_t)rn_hi, (uint32_t)rm_hi, FLAGS_BEHAVIOUR_SET, THUMB_SHIFT_DEFAULT, - ENFORCE_ENCODING_NONE)); - th_literal_pool_reserve_upcoming_bytes(6); - ot_check(th_it(mapcc(TOK_EQ), 0x8)); - if (lo_uses_imm) - ot_check(lo_imm_op); - else - ot_check(th_cmp_reg(0, (uint32_t)rn_lo, (uint32_t)rm_lo, FLAGS_BEHAVIOUR_SET, THUMB_SHIFT_DEFAULT, - ENFORCE_ENCODING_NONE)); - - mach_release_all(&ctx); -} - -/* tcc_gen_machine_subs_eq_select_01: emit - * SUBS dest, src1, #K - * IT NE - * MOVNE dest, #1 - * for the CMP src1,#K + SELECT(#1,#0,NE) / SELECT(#0,#1,EQ) peephole. - * Returns 1 if emitted, 0 if the SUBS immediate didn't encode (caller falls back). */ -ST_FUNC int tcc_gen_machine_subs_eq_select_01(MachineOperand src1, MachineOperand src2, MachineOperand dest) -{ - if (src2.kind != MACH_OP_IMM) - return 0; - if (src1.kind != MACH_OP_REG || src1.needs_deref || src1.u.reg.r0 < 0) - return 0; - if (dest.kind != MACH_OP_REG || dest.needs_deref || dest.u.reg.r0 < 0) - return 0; - - uint32_t src_reg = (uint32_t)src1.u.reg.r0; - uint32_t dst_reg = (uint32_t)dest.u.reg.r0; - uint32_t Ku = (uint32_t)src2.u.imm.val; - - thumb_opcode subs = th_sub_imm(dst_reg, src_reg, Ku, FLAGS_BEHAVIOUR_SET, ENFORCE_ENCODING_NONE); - if (subs.size == 0) - return 0; - - /* Reserve so a literal-pool flush can't split the IT/MOV pair. */ - th_literal_pool_reserve_upcoming_bytes(10); - ot_check(subs); - ot_check(th_it(mapcc(TOK_NE), 0x8u)); - thumb_opcode movne = th_generic_mov_imm(dst_reg, 1); - if (movne.size != 0) { - ot_check(movne); - } else { - /* mov #1 always encodes on ARM Thumb-2, but be safe. */ - load_full_const((int)dst_reg, PREG_NONE, 1u, 0u); - } - return 1; -} - -/* tcc_gen_machine_mla_mop: MachineOperand-based entry point for MLA. - * dest = src1 * src2 + accum (all operands are 32-bit) - * - * All four operands are loaded into hardware registers via mach_ensure_in_reg - * before emitting a single MLA instruction. No fallback path is needed - * because mach_ensure_in_reg always returns a valid register. - * - * Note: th_mla(rd, rn, rm, ra) → rd = rn * rm + ra - */ -ST_FUNC void tcc_gen_machine_mla_mop(MachineOperand src1, MachineOperand src2, MachineOperand dest, - MachineOperand accum) -{ - MachineCodegenContext ctx = {0}; - - /* Pre-exclude registers directly referenced by REG operands so that scratch - * allocations for other operands (e.g. immediates) cannot clobber them. - * A dereferenced operand's r0 is its POINTER register — it must survive - * until that operand's load is emitted, so it is excluded exactly like a - * plain value register (ptr fuzz seed 59549: src2's spill reload picked the - * deref-accumulator's pointer register as scratch, and the accumulator then - * dereferenced the just-loaded multiplicand value → wild-address fault). - * The pre-allocated DEST register must be excluded too: if a source load - * grabs it as a saved scratch (push/pop), the restoring pop after the MLA - * overwrites the just-computed result. */ - uint32_t live_regs = 0; - if (src1.kind == MACH_OP_REG && src1.u.reg.r0 >= 0 && src1.u.reg.r0 < 16) - live_regs |= (1u << (uint32_t)src1.u.reg.r0); - if (src2.kind == MACH_OP_REG && src2.u.reg.r0 >= 0 && src2.u.reg.r0 < 16) - live_regs |= (1u << (uint32_t)src2.u.reg.r0); - if (accum.kind == MACH_OP_REG && accum.u.reg.r0 >= 0 && accum.u.reg.r0 < 16) - live_regs |= (1u << (uint32_t)accum.u.reg.r0); - if (dest.kind == MACH_OP_REG && - dest.u.reg.r0 != (int)PREG_REG_NONE && dest.u.reg.r0 >= 0 && dest.u.reg.r0 < 16) - live_regs |= (1u << (uint32_t)dest.u.reg.r0); - - int src1_reg = mach_ensure_in_reg(&ctx, &src1, live_regs); - uint32_t excl = live_regs; - if (thumb_is_hw_reg(src1_reg)) - excl |= (1u << (uint32_t)src1_reg); - - int src2_reg = mach_ensure_in_reg(&ctx, &src2, excl); - if (thumb_is_hw_reg(src2_reg)) - excl |= (1u << (uint32_t)src2_reg); - - int accum_reg = mach_ensure_in_reg(&ctx, &accum, excl); - if (thumb_is_hw_reg(accum_reg)) - excl |= (1u << (uint32_t)accum_reg); - - int dest_reg = mach_get_dest_reg(&ctx, &dest, excl); - - /* th_mla(rd, rn, rm, ra): rd = rn * rm + ra */ - ot_check(th_mla((uint32_t)dest_reg, (uint32_t)src1_reg, (uint32_t)src2_reg, (uint32_t)accum_reg)); - - mach_writeback_dest(&dest, dest_reg); - mach_release_all(&ctx); -} - -/* tcc_gen_machine_umull_mop: MachineOperand-based entry point for UMULL. - * {dest_hi:dest_lo} = (uint32_t)src1 * (uint32_t)src2 (64-bit unsigned result) - * - * src1 and src2 are 32-bit inputs (is_64bit is cleared before loading). - * dest must be a 64-bit pair; it is split via mach_make_lo/hi_half. - * Each half is allocated independently via mach_get_dest_reg, with the - * exclusion mask preventing rdlo==rdhi and preventing overlap with rn/rm. - * - * Note: th_umull(rdlo, rdhi, rn, rm) → {rdhi:rdlo} = rn * rm (unsigned) - */ -ST_FUNC void tcc_gen_machine_umull_mop(MachineOperand src1, MachineOperand src2, MachineOperand dest) -{ - MachineCodegenContext ctx = {0}; - - /* UMULL takes 32-bit inputs — drop any 64-bit flag the src may carry. */ - MachineOperand s1 = src1; - s1.is_64bit = false; - MachineOperand s2 = src2; - s2.is_64bit = false; - - /* Pre-exclude the pre-allocated dest pair: a saved-scratch (push/pop) on a - * dest register would have its restoring pop clobber the result. */ - uint32_t dest_excl = 0; - if (dest.kind == MACH_OP_REG && !dest.needs_deref) - { - if (dest.u.reg.r0 != (int)PREG_REG_NONE) - dest_excl |= (1u << (uint32_t)dest.u.reg.r0); - if (dest.is_64bit && dest.u.reg.r1 >= 0 && dest.u.reg.r1 != (int)PREG_REG_NONE) - dest_excl |= (1u << (uint32_t)dest.u.reg.r1); - } - - int rn = mach_ensure_in_reg(&ctx, &s1, dest_excl); - uint32_t excl = dest_excl | (thumb_is_hw_reg(rn) ? (1u << (uint32_t)rn) : 0u); - - int rm = mach_ensure_in_reg(&ctx, &s2, excl); - if (thumb_is_hw_reg(rm)) - excl |= (1u << (uint32_t)rm); - - /* Split 64-bit destination into lo (bits [31:0]) and hi (bits [63:32]). */ - MachineOperand dst_lo = mach_make_lo_half(&dest); - MachineOperand dst_hi = mach_make_hi_half(&dest); - dst_lo.btype = IROP_BTYPE_INT32; - dst_hi.btype = IROP_BTYPE_INT32; - - int rd_lo = mach_get_dest_reg(&ctx, &dst_lo, excl); - if (thumb_is_hw_reg(rd_lo)) - excl |= (1u << (uint32_t)rd_lo); - int rd_hi = mach_get_dest_reg(&ctx, &dst_hi, excl); - - /* th_umull(rdlo, rdhi, rn, rm): {rdhi:rdlo} = rn * rm */ - ot_check(th_umull((uint32_t)rd_lo, (uint32_t)rd_hi, (uint32_t)rn, (uint32_t)rm)); - - mach_writeback_dest(&dst_lo, rd_lo); - mach_writeback_dest(&dst_hi, rd_hi); - mach_release_all(&ctx); -} - -/* tcc_gen_machine_smull_mop: MachineOperand-based entry point for SMULL. - * {dest_hi:dest_lo} = (int32_t)src1 * (int32_t)src2 (64-bit signed result). - * Mirrors umull_mop but emits th_smull. */ -ST_FUNC void tcc_gen_machine_smull_mop(MachineOperand src1, MachineOperand src2, MachineOperand dest) -{ - MachineCodegenContext ctx = {0}; - - MachineOperand s1 = src1; - s1.is_64bit = false; - MachineOperand s2 = src2; - s2.is_64bit = false; - - /* Pre-exclude the pre-allocated dest pair (see umull_mop). */ - uint32_t dest_excl = 0; - if (dest.kind == MACH_OP_REG && !dest.needs_deref) - { - if (dest.u.reg.r0 != (int)PREG_REG_NONE) - dest_excl |= (1u << (uint32_t)dest.u.reg.r0); - if (dest.is_64bit && dest.u.reg.r1 >= 0 && dest.u.reg.r1 != (int)PREG_REG_NONE) - dest_excl |= (1u << (uint32_t)dest.u.reg.r1); - } - - int rn = mach_ensure_in_reg(&ctx, &s1, dest_excl); - uint32_t excl = dest_excl | (thumb_is_hw_reg(rn) ? (1u << (uint32_t)rn) : 0u); - - int rm = mach_ensure_in_reg(&ctx, &s2, excl); - if (thumb_is_hw_reg(rm)) - excl |= (1u << (uint32_t)rm); - - MachineOperand dst_lo = mach_make_lo_half(&dest); - MachineOperand dst_hi = mach_make_hi_half(&dest); - dst_lo.btype = IROP_BTYPE_INT32; - dst_hi.btype = IROP_BTYPE_INT32; - - int rd_lo = mach_get_dest_reg(&ctx, &dst_lo, excl); - if (thumb_is_hw_reg(rd_lo)) - excl |= (1u << (uint32_t)rd_lo); - int rd_hi = mach_get_dest_reg(&ctx, &dst_hi, excl); - - /* th_smull(rdlo, rdhi, rn, rm): {rdhi:rdlo} = (signed)rn * (signed)rm */ - ot_check(th_smull((uint32_t)rd_lo, (uint32_t)rd_hi, (uint32_t)rn, (uint32_t)rm)); - - mach_writeback_dest(&dst_lo, rd_lo); - mach_writeback_dest(&dst_hi, rd_hi); - mach_release_all(&ctx); -} - -/* tcc_gen_machine_mlal_accum_mop: emit SMLAL/UMLAL for - * dest = accum + (int32/uint32)src1 * (int32/uint32)src2 - * - * This narrow helper is used by codegen peepholes after register allocation. - * It only handles the cheap in-place accumulate form, where the ADD destination - * already holds the accumulator pair. Other forms fall back to SMULL/UMULL - * plus the normal 64-bit ADD so we do not risk clobbering multiply sources. */ -ST_FUNC int tcc_gen_machine_mlal_accum_mop(MachineOperand src1, MachineOperand src2, MachineOperand accum, - MachineOperand dest, int is_signed) -{ - if (!dest.is_64bit || !accum.is_64bit) - return 0; - if (dest.kind != MACH_OP_REG || accum.kind != MACH_OP_REG) - return 0; - if (dest.needs_deref || accum.needs_deref) - return 0; - if (dest.u.reg.r0 != accum.u.reg.r0 || dest.u.reg.r1 != accum.u.reg.r1) - return 0; - - int rd_lo = dest.u.reg.r0; - int rd_hi = dest.u.reg.r1; - /* Inline the hw-reg range checks rather than calling thumb_is_hw_reg(): the - * self-host cross drops the argument move into the inlined helper here and - * tests a stale register (the dest pointer) instead of rd_lo/rd_hi, so the - * native compiler wrongly bails out of every in-place 64-bit MLA with - * "unable to lower 64-bit MLA". Direct comparisons on rd_lo/rd_hi (as the - * adjacent rd_lo == rd_hi check already does) compile correctly. */ - if (rd_lo < 0 || rd_lo > 15 || rd_hi < 0 || rd_hi > 15 || rd_lo == rd_hi) - return 0; - - MachineCodegenContext ctx = {0}; - MachineOperand s1 = src1; - s1.is_64bit = false; - MachineOperand s2 = src2; - s2.is_64bit = false; - - uint32_t excl = (1u << (uint32_t)rd_lo) | (1u << (uint32_t)rd_hi); - /* Pre-exclude both sources' registers (a deref operand's r0 is its pointer - * register) so ensuring one source cannot grab the other's register as a - * spill-reload scratch — same clobber class as tcc_gen_machine_mla_mop. */ - if (s1.kind == MACH_OP_REG && s1.u.reg.r0 >= 0 && s1.u.reg.r0 < 16) - excl |= (1u << (uint32_t)s1.u.reg.r0); - if (s2.kind == MACH_OP_REG && s2.u.reg.r0 >= 0 && s2.u.reg.r0 < 16) - excl |= (1u << (uint32_t)s2.u.reg.r0); - int rn = mach_ensure_in_reg(&ctx, &s1, excl); - if (thumb_is_hw_reg(rn)) - excl |= (1u << (uint32_t)rn); - - int rm = mach_ensure_in_reg(&ctx, &s2, excl); - if (thumb_is_hw_reg(rm)) - excl |= (1u << (uint32_t)rm); - - if (is_signed) - ot_check(th_smlal((uint32_t)rd_lo, (uint32_t)rd_hi, (uint32_t)rn, (uint32_t)rm)); - else - ot_check(th_umlal((uint32_t)rd_lo, (uint32_t)rd_hi, (uint32_t)rn, (uint32_t)rm)); - - MachineOperand dst_lo = mach_make_lo_half(&dest); - MachineOperand dst_hi = mach_make_hi_half(&dest); - dst_lo.btype = IROP_BTYPE_INT32; - dst_hi.btype = IROP_BTYPE_INT32; - mach_writeback_dest(&dst_lo, rd_lo); - mach_writeback_dest(&dst_hi, rd_hi); - mach_release_all(&ctx); - return 1; -} - -/* tcc_gen_machine_pack64_mop: lower TCCIR_OP_PACK64 by emitting two - * 32-bit assigns into the dest's halves. src_lo and src_hi are u32 - * operands; dest is a u64 register pair / spill / param slot. - * - * The two sub-assigns delegate to tcc_gen_machine_assign_mop, so they - * benefit from its existing handling of every dest kind (REG/SPILL/...). - * Often regalloc has already aligned the registers (e.g. dest.r0 = src_lo - * register), in which case the sub-assigns degrade to a no-op MOV that - * the encoder can skip. */ -ST_FUNC void tcc_gen_machine_pack64_mop(MachineOperand src_lo, MachineOperand src_hi, MachineOperand dest) -{ - if (!dest.is_64bit) - { - tcc_error("compiler_error: tcc_gen_machine_pack64_mop: dest not 64-bit"); - return; - } - MachineOperand dst_lo = mach_make_lo_half(&dest); - MachineOperand dst_hi = mach_make_hi_half(&dest); - dst_lo.btype = IROP_BTYPE_INT32; - dst_hi.btype = IROP_BTYPE_INT32; - - /* Detect register-swap aliasing: dst_lo == src_hi AND dst_hi == src_lo. - * Neither write order can preserve both source values; we must stage one - * side through a scratch register. */ - int swap_alias = 0; - if (src_lo.kind == MACH_OP_REG && !src_lo.needs_deref && - src_hi.kind == MACH_OP_REG && !src_hi.needs_deref && - dst_lo.kind == MACH_OP_REG && !dst_lo.needs_deref && - dst_hi.kind == MACH_OP_REG && !dst_hi.needs_deref && - src_hi.u.reg.r0 == dst_lo.u.reg.r0 && src_lo.u.reg.r0 == dst_hi.u.reg.r0 && - src_lo.u.reg.r0 != src_hi.u.reg.r0) - swap_alias = 1; - - if (swap_alias) - { - /* Save src_lo to a scratch before overwriting it via dst_hi. */ - uint32_t excl = (1u << (uint32_t)dst_lo.u.reg.r0) | (1u << (uint32_t)dst_hi.u.reg.r0); - ScratchRegAlloc scratch = get_scratch_reg_with_save(excl); - ot_check_mov_reg((uint32_t)scratch.reg, (uint32_t)src_lo.u.reg.r0, flags_safe(), - THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE, false); - /* Now dst_hi = src_hi (still live), then dst_lo = scratch (=old src_lo). */ - tcc_gen_machine_assign_mop(src_hi, dst_hi, TCCIR_OP_ASSIGN); - MachineOperand scratch_op = src_lo; - scratch_op.u.reg.r0 = scratch.reg; - tcc_gen_machine_assign_mop(scratch_op, dst_lo, TCCIR_OP_ASSIGN); - restore_scratch_reg(&scratch); - } - else if (src_hi.kind == MACH_OP_REG && !src_hi.needs_deref && - dst_lo.kind == MACH_OP_REG && !dst_lo.needs_deref && - src_hi.u.reg.r0 == dst_lo.u.reg.r0) - { - /* dst_lo == src_hi register: write hi first to free src_hi's slot. */ - tcc_gen_machine_assign_mop(src_hi, dst_hi, TCCIR_OP_ASSIGN); - tcc_gen_machine_assign_mop(src_lo, dst_lo, TCCIR_OP_ASSIGN); - } - else - { - tcc_gen_machine_assign_mop(src_lo, dst_lo, TCCIR_OP_ASSIGN); - tcc_gen_machine_assign_mop(src_hi, dst_hi, TCCIR_OP_ASSIGN); - } -} - -/* tcc_gen_machine_assign_mop: MachineOperand-based entry point for simple - * 32-bit value assignment. Called from ir/codegen.c instead of - * tcc_gen_machine_assign_op when: - * - Neither dest nor src requires a 64-bit or complex register pair, AND - * - The function does not use a static chain. - * - * Handles all destination kinds: MACH_OP_REG (direct), MACH_OP_SPILL - * (via mach_get_dest_reg + mach_writeback_dest → tcc_machine_store_spill_slot), - * and MACH_OP_PARAM_STACK (via mach_writeback_dest → tcc_machine_store_param_slot). - * - * Strategy: load src directly into dest_reg; use mach_ensure_in_reg only - * as a fallback for unhandled source kinds. - */ -ST_FUNC void tcc_gen_machine_assign_mop(MachineOperand src, MachineOperand dest, TccIrOp op) -{ - tcc_gen_machine_assign_mop_ex(src, dest, op, 0); -} - -ST_FUNC void tcc_gen_machine_assign_mop_ex(MachineOperand src, MachineOperand dest, TccIrOp op, uint32_t zh) -{ - (void)op; - /* Only the top-level 64-bit destination consults `zh`; the per-half recursive - * calls below have already split the pair and pass 0. */ - const bool dst_lo_dead = (zh & ZH64_D_LO) != 0; - const bool dst_hi_dead = (zh & ZH64_D_HI) != 0; - - /* 64-bit pair assignment: handle each 32-bit half independently. - * mach_make_lo/hi_half splits MACH_OP_REG (r0:r1), MACH_OP_SPILL (offset, - * offset+4) and MACH_OP_IMM into separate 32-bit MachineOperands. - * We then recursively assign each half (is_64bit=false prevents recursion). - * - * Special care: when src has needs_deref=true, the operand is a POINTER - * to a 64-bit value. The address is in one register (or spill slot); - * splitting registers via mach_make_hi_half would create a bogus base - * address. Instead, load both halves from [base+0] and [base+4]. - * - * Exception: PARAM_STACK with needs_deref means the 64-bit value IS - * directly at [fp+offset], not a pointer to follow. Clear needs_deref - * so it falls through to the normal lo/hi split path. */ - if (src.needs_deref && src.is_64bit && src.kind == MACH_OP_PARAM_STACK) - src.needs_deref = false; - - if (dest.is_64bit) - { - MachineOperand dst_lo = mach_make_lo_half(&dest); - MachineOperand dst_hi = mach_make_hi_half(&dest); - dst_lo.btype = IROP_BTYPE_INT32; - dst_hi.btype = IROP_BTYPE_INT32; - - if (src.needs_deref && src.is_64bit) - { - /* Source is a 64-bit lvalue: a pointer to a 64-bit value (e.g. R0 - * holding address of an unsigned long long). Load both 32-bit halves - * from [base+0] and [base+4] using the same base address register. */ - MachineCodegenContext mctx = {0}; - - /* Strip deref to get the raw address into a register. */ - MachineOperand addr = src; - addr.needs_deref = false; - addr.is_64bit = false; - addr.btype = IROP_BTYPE_INT32; - int base_reg = mach_ensure_in_reg(&mctx, &addr, 0); - uint32_t excl = (1u << (uint32_t)base_reg); - - /* Determine destination registers for lo and hi halves. */ - int lo_reg = mach_get_dest_reg(&mctx, &dst_lo, excl); - if (thumb_is_hw_reg(lo_reg)) - excl |= (1u << (uint32_t)lo_reg); - int hi_reg = mach_get_dest_reg(&mctx, &dst_hi, excl); - - /* Load [base+0] → lo, [base+4] → hi (32-bit loads). */ - load_from_base(lo_reg, PREG_REG_NONE, IROP_BTYPE_INT32, 0, 0, 0, (uint32_t)base_reg); - load_from_base(hi_reg, PREG_REG_NONE, IROP_BTYPE_INT32, 0, 4, 0, (uint32_t)base_reg); - - mach_writeback_dest(&dst_lo, lo_reg); - mach_writeback_dest(&dst_hi, hi_reg); - mach_release_all(&mctx); - return; - } - - if (src.is_64bit) - { - /* Whole 64-bit local → register pair: one LDRD instead of the two LDRs - * the per-half recursion below would emit. */ - /* LDRD moves both halves at once, so it is only the right choice while - * both are wanted. */ - if (!dst_lo_dead && !dst_hi_dead && src.kind == MACH_OP_SPILL && !src.needs_deref && - (src.u.spill.offset & 3) == 0 && dest.kind == MACH_OP_REG && !dest.needs_deref && - thumb_is_hw_reg(dest.u.reg.r0) && thumb_is_hw_reg(dest.u.reg.r1) && - dest.u.reg.r0 != dest.u.reg.r1 && - tcc_gen_machine_try_ldrd_spill(dest.u.reg.r0, src.u.spill.offset, dest.u.reg.r1, - src.u.spill.offset + 4)) - return; - - MachineOperand src_lo = mach_make_lo_half(&src); - MachineOperand src_hi = mach_make_hi_half(&src); - src_lo.btype = IROP_BTYPE_INT32; - src_hi.btype = IROP_BTYPE_INT32; - if (!dst_lo_dead) - tcc_gen_machine_assign_mop(src_lo, dst_lo, op); - if (!dst_hi_dead) - tcc_gen_machine_assign_mop(src_hi, dst_hi, op); - } - else - { - /* 32-bit source into 64-bit dest: assign lo half, zero the high half. - * That zero is the single commonest instruction in soft float -- every - * `(uint64_t)x` before a shift-and-OR pays one -- so skipping it when no - * consumer reads the half is the point of the whole annotation. */ - MachineOperand zero = {0}; - zero.kind = MACH_OP_IMM; - zero.u.imm.val = 0; - zero.btype = IROP_BTYPE_INT32; - if (!dst_lo_dead) - tcc_gen_machine_assign_mop(src, dst_lo, op); - if (!dst_hi_dead) - tcc_gen_machine_assign_mop(zero, dst_hi, op); - } - return; - } - - if (src.is_64bit && !dest.is_64bit) - { - /* Truncation: extract and assign only the low half of the 64-bit source. */ - MachineOperand src_lo = mach_make_lo_half(&src); - src_lo.btype = IROP_BTYPE_INT32; - tcc_gen_machine_assign_mop(src_lo, dest, op); - return; - } - - /* Single-precision float in a VFP register (hard-float): vmov between s-regs, - * or bridge through a GPR for the GPR/imm/spill/memory endpoint. */ - if (src.kind == MACH_OP_VFP_REG || dest.kind == MACH_OP_VFP_REG) - { - if (src.kind == MACH_OP_VFP_REG && dest.kind == MACH_OP_VFP_REG) - { - if (src.u.reg.r0 != dest.u.reg.r0) - ot_check(th_vmov_register((uint16_t)dest.u.reg.r0, (uint16_t)src.u.reg.r0, 0)); - return; - } - MachineCodegenContext vctx = {0}; - if (dest.kind == MACH_OP_VFP_REG) - { - int gpr = mach_ensure_in_reg(&vctx, &src, 0); /* GPR/imm/spill/... -> GPR */ - ot_check(th_vmov_gp_sp((uint16_t)gpr, (uint16_t)dest.u.reg.r0, 0)); /* sN = gpr */ - } - else - { - int gpr = mach_ensure_in_reg(&vctx, &src, 0); /* VFP src bridges to a GPR */ - mach_writeback_dest(&dest, gpr); - } - mach_release_all(&vctx); - return; - } - - MachineCodegenContext mctx = {0}; - - /* --- Fast path: source is already in a register (no dereference) --- - * Write it directly to the destination via mach_writeback_dest without - * allocating any scratch. This covers REG→REG (MOV or NOP) and - * REG→SPILL/PARAM_STACK (direct store from src register). */ - if (src.kind == MACH_OP_REG && !src.needs_deref) - { - mach_writeback_dest(&dest, src.u.reg.r0); - return; - } - - /* --- Determine destination register --- - * For REG destinations, reuse the pre-allocated register (0 scratch). - * For SPILL/PARAM_STACK/REG(deref) destinations, allocate a scratch. */ - int dest_reg; - bool need_writeback; - if (dest.kind == MACH_OP_REG && !dest.needs_deref && dest.u.reg.r0 != (int)PREG_REG_NONE) - { - dest_reg = dest.u.reg.r0; - need_writeback = false; - } - else - { - dest_reg = mach_get_dest_reg(&mctx, &dest, 0); - need_writeback = true; - } - - /* --- Load source value directly into dest_reg --- */ - switch (src.kind) - { - case MACH_OP_REG: - /* Only the needs_deref case reaches here (non-deref handled above). - * Load from [src_reg] directly into dest_reg. */ - load_from_base(dest_reg, PREG_REG_NONE, src.btype, (int)src.is_unsigned, 0, 0, (uint32_t)src.u.reg.r0); - break; - - case MACH_OP_IMM: - tcc_machine_load_constant(dest_reg, PREG_REG_NONE, src.u.imm.val, 0, NULL); - break; - - case MACH_OP_SPILL: - tcc_machine_load_spill_slot(dest_reg, src.u.spill.offset); - if (src.needs_deref) - { - /* Double indirection: dest_reg now holds a pointer; dereference it. */ - load_from_base(dest_reg, PREG_REG_NONE, src.btype, (int)src.is_unsigned, 0, 0, (uint32_t)dest_reg); - } - break; - - case MACH_OP_SYMBOL: - { - Sym *sym = src.u.sym.sym ? validate_sym_for_reloc(src.u.sym.sym) : NULL; - if (!src.needs_deref) - { - tcc_machine_load_constant(dest_reg, PREG_REG_NONE, src.u.sym.addend, 0, sym); - } - else - { - const int32_t addend = src.u.sym.addend; - const int abs_off = addend < 0 ? (int)(-addend) : (int)addend; - const int sign = addend < 0 ? 1 : 0; - /* Prefer a base register OTHER than dest_reg: `ldr rD,[rD,#off]` - * overwrites the address with the loaded value, so imm_cache loses it - * and the store that follows a read-modify-write has to re-load the - * literal. A separate base keeps the address cached (and an imm_cache - * hit removes this materialization entirely). Only when no register is - * free do we fall back to routing through dest_reg - taking a - * push/pop there would cost more than the reload it saves. */ - ScratchRegAlloc base; - if (try_scratch_reg_for_sym_addr(sym, 0, (1u << (uint32_t)dest_reg), &base)) - { - tcc_machine_load_constant(base.reg, PREG_REG_NONE, 0, 0, sym); - load_from_base(dest_reg, PREG_REG_NONE, src.btype, (int)src.is_unsigned, abs_off, sign, - (uint32_t)base.reg); - restore_scratch_reg(&base); - } - else - { - tcc_machine_load_constant(dest_reg, PREG_REG_NONE, 0, 0, sym); - load_from_base(dest_reg, PREG_REG_NONE, src.btype, (int)src.is_unsigned, abs_off, sign, - (uint32_t)dest_reg); - } - } - break; - } - - case MACH_OP_FRAME_ADDR: - tcc_machine_addr_of_stack_slot(dest_reg, src.u.frame.offset, 0); - break; - - case MACH_OP_PARAM_STACK: - { - const int adjusted = src.u.param.offset + offset_to_args; - const int base_reg = tcc_state->need_frame_pointer ? R_FP : R_SP; - const int sign = (adjusted < 0); - const int abs_off = sign ? -adjusted : adjusted; - load_from_base(dest_reg, PREG_REG_NONE, src.btype, (int)src.is_unsigned, abs_off, sign, (uint32_t)base_reg); - break; - } - - default: - { - /* Fallback: generic mach_ensure_in_reg + MOV. */ - uint32_t excl = thumb_is_hw_reg(dest_reg) ? (1u << (uint32_t)dest_reg) : 0; - int src_reg = mach_ensure_in_reg(&mctx, &src, excl); - if (src_reg != dest_reg) - ot_check_mov_reg((uint32_t)dest_reg, (uint32_t)src_reg, flags_safe(), THUMB_SHIFT_DEFAULT, - ENFORCE_ENCODING_NONE, false); - break; - } - } - - if (need_writeback) - mach_writeback_dest(&dest, dest_reg); - - mach_release_all(&mctx); -} - -/* tcc_gen_machine_setif_mop: MachineOperand-based entry point for SETIF. - * src must be MACH_OP_IMM carrying the raw condition code in u.imm.val. - * - * 32-bit dest: - * ITE - * MOV dest, #1 (T: cond met) - * MOV dest, #0 (E: cond not met) - * - * 64-bit dest pair (e.g. long long result = (x > y)): - * The boolean result 0 or 1 fits in 32 bits, so hi word is always 0. - * ITE - * MOV dest_lo, #1 - * MOV dest_lo, #0 - * MOV dest_hi, #0 (unconditional, outside IT block — hi is always 0) - * - * Inner MOVs use NOT_IMPORTANT for flags: SETIF is the consumer of the CMP - * flags; once the ITE captures the condition, no subsequent code in this - * lowering depends on CMP's flag state, so the 16-bit T1 encoding (which - * implicitly sets flags) is safe. This shrinks each conditional MOV from - * 4 bytes (mov.w) to 2 bytes (movs). - */ -ST_FUNC void tcc_gen_machine_setif_mop(MachineOperand src, MachineOperand dest, TccIrOp op) -{ - (void)op; - MachineCodegenContext mctx = {0}; - - const int cond = mapcc((int)src.u.imm.val); - /* ITE mask: 2nd instruction has opposite condition. - * mask[3] = 1 if it should be the 'else' bit (opposite of cond[0]). - * For the T-then-E pattern, mask = ((!cond[0]) << 3) | 0x4. */ - const uint16_t ite_mask = (uint16_t)(((cond ^ 1) & 1) << 3) | 0x4u; - - if (dest.is_64bit) - { - /* Split 64-bit destination into two independent 32-bit halves. */ - MachineOperand dst_lo = mach_make_lo_half(&dest); - MachineOperand dst_hi = mach_make_hi_half(&dest); - dst_lo.btype = IROP_BTYPE_INT32; - dst_hi.btype = IROP_BTYPE_INT32; - - int lo_reg = mach_get_dest_reg(&mctx, &dst_lo, 0); - uint32_t excl = thumb_is_hw_reg(lo_reg) ? (1u << (uint32_t)lo_reg) : 0u; - int hi_reg = mach_get_dest_reg(&mctx, &dst_hi, excl); - - /* Emit ITE sequence for lo word. Reserve the WHOLE atomic ITE+movs block so - * a literal-pool flush never lands between the IT and its conditioned movs. - * A high register (R8-R12) dest forces the 4-byte mov.w (T2) encoding, so the - * worst case is ITE(2) + 3*mov.w(4) = 14 bytes — NOT 6 (which only covers the - * 2-byte movs of a low-reg dest). Under-reserving split the ITE and ran the - * fall-through into the literal pool (seed 89 O1 HardFault). */ - th_literal_pool_reserve_upcoming_bytes(14); - ot_check(th_it(cond, ite_mask)); /* ITE — two conditioned instructions */ - ot_check(th_mov_imm(lo_reg, 1, FLAGS_BEHAVIOUR_NOT_IMPORTANT, ENFORCE_ENCODING_NONE)); - ot_check(th_mov_imm(lo_reg, 0, FLAGS_BEHAVIOUR_NOT_IMPORTANT, ENFORCE_ENCODING_NONE)); - /* Hi word is always 0 — boolean result never exceeds 1 (i.e. fits in 32-bit lo). */ - ot_check(th_mov_imm(hi_reg, 0, FLAGS_BEHAVIOUR_NOT_IMPORTANT, ENFORCE_ENCODING_NONE)); - - mach_writeback_dest(&dst_lo, lo_reg); - mach_writeback_dest(&dst_hi, hi_reg); - } - else - { - int dest_reg = mach_get_dest_reg(&mctx, &dest, 0); - - /* Reserve the whole ITE+2-movs block: a high-register dest (R8-R12) uses the - * 4-byte mov.w (T2) encoding, so the worst case is ITE(2) + 2*mov.w(4) = 10 - * bytes, not 6. Under-reserving let a literal-pool flush split the ITE and - * run the fall-through into the pool (seed 89 O1 HardFault). */ - th_literal_pool_reserve_upcoming_bytes(10); - ot_check(th_it(cond, ite_mask)); /* ITE — two conditioned instructions */ - ot_check(th_mov_imm(dest_reg, 1, FLAGS_BEHAVIOUR_NOT_IMPORTANT, ENFORCE_ENCODING_NONE)); - ot_check(th_mov_imm(dest_reg, 0, FLAGS_BEHAVIOUR_NOT_IMPORTANT, ENFORCE_ENCODING_NONE)); - - mach_writeback_dest(&dest, dest_reg); - } - - mach_release_all(&mctx); -} - -/* tcc_gen_machine_bool_mop: MachineOperand-based entry point for - * BOOL_OR / BOOL_AND. Called from ir/codegen.c for simple 32-bit - * non-complex boolean operations. - * - * BOOL_OR: ORRS dest, src1, src2 (sets Z flag) - * MOV dest, #0 (flag-preserving) - * IT NE - * MOV dest, #1 - * - * BOOL_AND: CMP src1, #0 - * IT NE - * CMP src2, #0 (only if src1 != 0) - * MOV dest, #0 (flag-preserving) - * IT NE - * MOV dest, #1 - */ -ST_FUNC void tcc_gen_machine_bool_mop(MachineOperand src1, MachineOperand src2, MachineOperand dest, TccIrOp op) -{ - MachineCodegenContext mctx = {0}; - - /* 64-bit operands: reduce each to a 32-bit "is non-zero" value by OR-ing - * its low and high halves, then apply the standard 32-bit BOOL logic. */ - if (src1.is_64bit || src2.is_64bit) - { - uint32_t excl = 0; - int r1, r2; - - if (src1.is_64bit) - { - MachineOperand lo1 = mach_make_lo_half(&src1); - lo1.btype = IROP_BTYPE_INT32; - MachineOperand hi1 = mach_make_hi_half(&src1); - hi1.btype = IROP_BTYPE_INT32; - r1 = mach_ensure_in_reg(&mctx, &lo1, excl); - excl |= thumb_is_hw_reg(r1) ? (1u << (uint32_t)r1) : 0; - int hi1_reg = mach_ensure_in_reg(&mctx, &hi1, excl); - excl |= thumb_is_hw_reg(hi1_reg) ? (1u << (uint32_t)hi1_reg) : 0; - /* r1 = lo1 | hi1 — is src1 non-zero? */ - ot_check(th_orr_reg(r1, r1, hi1_reg, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); - } - else - { - r1 = mach_ensure_in_reg(&mctx, &src1, excl); - excl |= thumb_is_hw_reg(r1) ? (1u << (uint32_t)r1) : 0; - } - - if (src2.is_64bit) - { - MachineOperand lo2 = mach_make_lo_half(&src2); - lo2.btype = IROP_BTYPE_INT32; - MachineOperand hi2 = mach_make_hi_half(&src2); - hi2.btype = IROP_BTYPE_INT32; - r2 = mach_ensure_in_reg(&mctx, &lo2, excl); - excl |= thumb_is_hw_reg(r2) ? (1u << (uint32_t)r2) : 0; - int hi2_reg = mach_ensure_in_reg(&mctx, &hi2, excl); - excl |= thumb_is_hw_reg(hi2_reg) ? (1u << (uint32_t)hi2_reg) : 0; - /* r2 = lo2 | hi2 — is src2 non-zero? */ - ot_check(th_orr_reg(r2, r2, hi2_reg, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); - } - else - { - r2 = mach_ensure_in_reg(&mctx, &src2, excl); - excl |= thumb_is_hw_reg(r2) ? (1u << (uint32_t)r2) : 0; - } - - int dest_reg = mach_get_dest_reg(&mctx, &dest, excl); - - if (op == TCCIR_OP_BOOL_OR) - { - ot_check(th_orr_reg(dest_reg, r1, r2, FLAGS_BEHAVIOUR_SET, THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); - ot_check(th_mov_imm(dest_reg, 0, FLAGS_BEHAVIOUR_BLOCK, ENFORCE_ENCODING_NONE)); - th_literal_pool_reserve_upcoming_bytes(6); - ot_check(th_it(0x1, 0x8)); /* IT NE */ - ot_check(th_mov_imm(dest_reg, 1, flags_safe(), ENFORCE_ENCODING_NONE)); - } - else /* TCCIR_OP_BOOL_AND */ - { - ot_check(th_cmp_imm(r1, 0, FLAGS_BEHAVIOUR_SET, ENFORCE_ENCODING_NONE)); - th_literal_pool_reserve_upcoming_bytes(6); - ot_check(th_it(0x1, 0x8)); /* IT NE */ - ot_check(th_cmp_imm(r2, 0, FLAGS_BEHAVIOUR_SET, ENFORCE_ENCODING_NONE)); /* CMPne r2, #0 */ - ot_check(th_mov_imm(dest_reg, 0, FLAGS_BEHAVIOUR_BLOCK, ENFORCE_ENCODING_NONE)); - th_literal_pool_reserve_upcoming_bytes(6); - ot_check(th_it(0x1, 0x8)); /* IT NE */ - ot_check(th_mov_imm(dest_reg, 1, flags_safe(), ENFORCE_ENCODING_NONE)); - } - - mach_writeback_dest(&dest, dest_reg); - mach_release_all(&mctx); - return; - } - - int dest_reg = mach_get_dest_reg(&mctx, &dest, 0); - uint32_t excl = thumb_is_hw_reg(dest_reg) ? (1u << (uint32_t)dest_reg) : 0; - - int src1_reg = mach_ensure_in_reg(&mctx, &src1, excl); - if (thumb_is_hw_reg(src1_reg)) - excl |= (1u << (uint32_t)src1_reg); - - int src2_reg = mach_ensure_in_reg(&mctx, &src2, excl); - - if (op == TCCIR_OP_BOOL_OR) - { - ot_check(th_orr_reg(dest_reg, src1_reg, src2_reg, FLAGS_BEHAVIOUR_SET, THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); - ot_check(th_mov_imm(dest_reg, 0, FLAGS_BEHAVIOUR_BLOCK, ENFORCE_ENCODING_NONE)); - th_literal_pool_reserve_upcoming_bytes(6); - ot_check(th_it(0x1, 0x8)); /* IT NE */ - ot_check(th_mov_imm(dest_reg, 1, flags_safe(), ENFORCE_ENCODING_NONE)); - } - else /* TCCIR_OP_BOOL_AND */ - { - ot_check(th_cmp_imm(src1_reg, 0, FLAGS_BEHAVIOUR_SET, ENFORCE_ENCODING_NONE)); - th_literal_pool_reserve_upcoming_bytes(6); - ot_check(th_it(0x1, 0x8)); /* IT NE */ - ot_check(th_cmp_imm(src2_reg, 0, FLAGS_BEHAVIOUR_SET, ENFORCE_ENCODING_NONE)); /* CMPne src2, #0 */ - ot_check(th_mov_imm(dest_reg, 0, FLAGS_BEHAVIOUR_BLOCK, ENFORCE_ENCODING_NONE)); - th_literal_pool_reserve_upcoming_bytes(6); - ot_check(th_it(0x1, 0x8)); /* IT NE */ - ot_check(th_mov_imm(dest_reg, 1, flags_safe(), ENFORCE_ENCODING_NONE)); - } - - mach_writeback_dest(&dest, dest_reg); - mach_release_all(&mctx); -} - -/* tcc_gen_machine_load_mop: MachineOperand-based entry point for TCCIR_OP_LOAD. - * - * dest can be MACH_OP_REG, MACH_OP_SPILL, or MACH_OP_PARAM_STACK. - * For spilled destinations, a scratch register is allocated and the result - * is written back to the spill slot after the load completes. - * 64-bit dest is supported: for MACH_OP_REG dest.u.reg.r1 holds the hi - * register; for spilled dests, a second scratch is allocated for hi-half. - * - * src encodes the memory address: - * MACH_OP_REG + needs_deref=true → LDR dest, [src_reg] - * MACH_OP_SPILL → LDR dest, [FP + fp_adjust(offset)] - * MACH_OP_SPILL + needs_deref=true → LLOCAL: LDR ptr,[FP+off]; LDR dest,[ptr] - * MACH_OP_PARAM_STACK → LDR dest, [FP + param_off + offset_to_args] - * MACH_OP_SYMBOL → LDR_literal addr; LDR dest, [addr] - * MACH_OP_IMM → tcc_machine_load_constant (constant load) - */ -ST_FUNC void tcc_gen_machine_load_mop(MachineOperand src, MachineOperand dest, TccIrOp op) -{ - MachineCodegenContext ctx = {0}; - (void)op; - - /* A load whose source is a VFP register is a read out of that s-register (a - * register-promoted float), not a memory load — delegate to the VFP-aware - * assign path, which handles every destination kind. */ - if (src.kind == MACH_OP_VFP_REG) - { - tcc_gen_machine_assign_mop(src, dest, TCCIR_OP_ASSIGN); - return; - } - - /* Determine dest register — allocates scratch if dest is SPILL/PARAM_STACK. */ - const bool dest_is_simple_reg = - (dest.kind == MACH_OP_REG && !dest.needs_deref && dest.u.reg.r0 != (int)PREG_REG_NONE); - const int dest_reg = mach_get_dest_reg(&ctx, &dest, 0); - - /* For 64-bit pairs: get hi-half dest register. */ - int dest_r1 = PREG_REG_NONE; - MachineOperand dest_hi_mop = {0}; - if (dest.is_64bit) - { - if (dest_is_simple_reg) - { - dest_r1 = dest.u.reg.r1; - } - else - { - dest_hi_mop = mach_make_hi_half(&dest); - dest_r1 = mach_get_dest_reg(&ctx, &dest_hi_mop, (1u << (uint32_t)dest_reg)); - } - } - - const int btype = src.btype; - const int is_unsigned = (int)src.is_unsigned; - - switch (src.kind) - { - case MACH_OP_REG: - if (src.needs_deref) - { - /* Register-indirect: LDR dest, [src_reg]. - * 64-bit + proven >= 4-byte alignment (src.align4, from the frontend's - * packed-access tracking): use LDRD directly. LDRD Rt==Rn is fine - * without writeback, so no base-preservation dance is needed. Without - * the proof, load_from_base emits the unaligned-safe LDR pair. */ - if (dest.is_64bit && !dest.is_complex && src.align4 && dest_r1 != (int)PREG_REG_NONE && - try_ldrd_pair(dest_reg, dest_r1, src.u.reg.r0, 0, 0)) - break; - load_from_base(dest_reg, dest_r1, btype, is_unsigned, 0, 0, (uint32_t)src.u.reg.r0); - } - else - { - /* Direct register-to-register (treat as MOV — should be ASSIGN, not LOAD) */ - if (dest_reg != src.u.reg.r0) - ot_check_mov_reg((uint32_t)dest_reg, (uint32_t)src.u.reg.r0, flags_safe(), THUMB_SHIFT_DEFAULT, - ENFORCE_ENCODING_NONE, false); - /* Narrow sub-word parameter values: when a parameter is declared as - * char/short but arrives in a full 32-bit register (AAPCS default - * argument promotion), the upper bits may contain garbage. Emit - * UXTB/SXTB/UXTH/SXTH to truncate to the declared type width. */ - if (btype == IROP_BTYPE_INT8) - { - if (is_unsigned) - ot_check(th_uxtb((uint32_t)dest_reg, (uint32_t)dest_reg, THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); - else - ot_check(th_sxtb((uint32_t)dest_reg, (uint32_t)dest_reg, THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); - } - else if (btype == IROP_BTYPE_INT16) - { - if (is_unsigned) - ot_check(th_uxth((uint32_t)dest_reg, (uint32_t)dest_reg, THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); - else - ot_check(th_sxth((uint32_t)dest_reg, (uint32_t)dest_reg, THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); - } - /* 64-bit pair: also copy the hi-half register */ - if (dest_r1 != PREG_REG_NONE && src.u.reg.r1 >= 0 && dest_r1 != src.u.reg.r1) - ot_check_mov_reg((uint32_t)dest_r1, (uint32_t)src.u.reg.r1, flags_safe(), THUMB_SHIFT_DEFAULT, - ENFORCE_ENCODING_NONE, false); - } - break; - - case MACH_OP_SPILL: - { - const int adj = fp_adjust_local_offset(src.u.spill.offset, 0); - const int sign = (adj < 0), abs_off = sign ? -adj : adj; - const uint32_t base = (uint32_t)(tcc_state->need_frame_pointer ? R_FP : R_SP); - if (!src.needs_deref) - { - /* Load value directly from spill/local slot */ - load_from_base(dest_reg, dest_r1, btype, is_unsigned, abs_off, sign, base); - } - else - { - /* LLOCAL: spill slot holds a pointer; load ptr, then dereference */ - int ptr_r = mach_alloc_scratch(&ctx, ((uint32_t)1u << (uint32_t)dest_reg) | ((uint32_t)1u << base)); - if (!load_word_from_base(ptr_r, (int)base, abs_off, sign)) - { - ScratchRegAlloc rr = th_offset_to_reg_ex(abs_off, sign, ((uint32_t)1u << ptr_r) | ((uint32_t)1u << base)); - ot_check(th_ldr_reg((uint32_t)ptr_r, base, (uint32_t)rr.reg, THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); - restore_scratch_reg(&rr); - } - /* 64-bit + proven alignment: LDRD through the loaded pointer. */ - if (dest.is_64bit && !dest.is_complex && src.align4 && dest_r1 != (int)PREG_REG_NONE && - try_ldrd_pair(dest_reg, dest_r1, ptr_r, 0, 0)) - break; - load_from_base(dest_reg, dest_r1, btype, is_unsigned, 0, 0, (uint32_t)ptr_r); - } - break; - } - - case MACH_OP_PARAM_STACK: - { - const int adj = src.u.param.offset + offset_to_args; - const int sign = (adj < 0), abs_off = sign ? -adj : adj; - const uint32_t base = (uint32_t)(tcc_state->need_frame_pointer ? R_FP : R_SP); - load_from_base(dest_reg, dest_r1, btype, is_unsigned, abs_off, sign, base); - break; - } - - case MACH_OP_SYMBOL: - { - Sym *sym = src.u.sym.sym ? validate_sym_for_reloc(src.u.sym.sym) : NULL; - const int32_t addend = src.u.sym.addend; - if (!src.needs_deref) - { - /* Load symbol address (+ addend) into dest — no dereference. - * Load symbol address (+ addend) — no dereference. */ - tcc_machine_load_constant(dest_reg, dest_r1, (int64_t)addend, (int)dest.is_64bit, sym); - break; - } - /* needs_deref: load symbol address into scratch, then dereference. */ - int addr_r = mach_alloc_scratch_for_sym(&ctx, (uint32_t)1u << (uint32_t)dest_reg, sym, 0); - tcc_machine_load_constant(addr_r, PREG_REG_NONE, 0, 0, sym); - /* For a 64-bit deref, try LDRD when we can prove the symbol's address - * at `addend` is 4-byte aligned. Otherwise fall back to the pair of - * 32-bit loads via load_from_base. */ - const int sym_sign = (addend < 0), sym_abs = sym_sign ? (int)(-addend) : (int)addend; - if (dest.is_64bit && dest_r1 != PREG_REG_NONE && sym_is_4_byte_aligned_for_64bit(sym, addend) && - try_ldrd_pair(dest_reg, dest_r1, addr_r, sym_abs, sym_sign)) - { - break; - } - load_from_base(dest_reg, dest_r1, btype, is_unsigned, sym_abs, sym_sign, (uint32_t)addr_r); - break; - } - - case MACH_OP_IMM: - if (src.needs_deref) - { - /* Read through an absolute address: `*(volatile T *)0x40000000`, the - * MMIO register idiom. Materialize the address, then load from it — - * the same shape as the MACH_OP_SYMBOL deref path above, and the mirror - * of the store side, which has always done this. Alignment of an - * arbitrary constant address is unknown, so no LDRD. */ - int addr_r = mach_alloc_scratch(&ctx, (uint32_t)1u << (uint32_t)dest_reg); - tcc_machine_load_constant(addr_r, PREG_REG_NONE, src.u.imm.val, 0, NULL); - load_from_base(dest_reg, dest_r1, btype, is_unsigned, 0, 0, (uint32_t)addr_r); - break; - } - /* Not an lvalue: the immediate IS the value. */ - tcc_machine_load_constant(dest_reg, dest_r1, src.u.imm.val, (int)dest.is_64bit, NULL); - break; - - case MACH_OP_FRAME_ADDR: - { - if (!src.needs_deref) - { - /* Load the frame-slot address itself (LEA semantics). */ - tcc_machine_addr_of_stack_slot(dest_reg, src.u.frame.offset, 0); - } - else - { - /* Frame address is a pointer to data — compute addr, then dereference. */ - int addr_r = mach_alloc_scratch(&ctx, (uint32_t)1u << (uint32_t)dest_reg); - tcc_machine_addr_of_stack_slot(addr_r, src.u.frame.offset, 0); - load_from_base(dest_reg, dest_r1, btype, is_unsigned, 0, 0, (uint32_t)addr_r); - } - break; - } - - case MACH_OP_CHAIN_REL: - { - /* Captured variable: load from parent frame via static chain. */ - ScratchRegAlloc chain_scratch = {0}; - int chain_used = 0; - int base = resolve_chain_base(tcc_state->ir, src.u.chain.chain_index, (1u << (uint32_t)dest_reg), &chain_scratch, - &chain_used); - int32_t off = src.u.chain.offset; - int sign = (off < 0), abs_off = sign ? (int)(-off) : (int)off; - load_from_base(dest_reg, dest_r1, btype, is_unsigned, abs_off, sign, (uint32_t)base); - if (chain_used) - restore_scratch_reg(&chain_scratch); - break; - } - - default: - tcc_error("compiler_error: load_mop: unhandled src kind %d", (int)src.kind); - } - - /* Write back result to spill/param slot if dest was not a plain register. */ - if (!dest_is_simple_reg) - { - if (dest.is_64bit) - { - MachineOperand dest_lo_mop = mach_make_lo_half(&dest); - mach_writeback_dest(&dest_lo_mop, dest_reg); - mach_writeback_dest(&dest_hi_mop, dest_r1); - } - else - { - mach_writeback_dest(&dest, dest_reg); - } - } - - mach_release_all(&ctx); -} - -/* Forward declarations for complex splitting helpers (defined in complex MOP section below). */ -static MachineOperand mach_make_complex_real(const MachineOperand *op); -static MachineOperand mach_make_complex_imag(const MachineOperand *op); - -/* tcc_gen_machine_store_mop: MachineOperand-based entry point for TCCIR_OP_STORE. - * - * dest encodes the destination address (memory location to write to). - * src encodes the value to store. - * Store width is determined by dest.btype. - * - * dest kinds handled: - * MACH_OP_REG + needs_deref=true → STR src, [dest_reg] - * MACH_OP_REG (no deref) → MOV dest_reg, src (reg-to-reg) - * MACH_OP_SPILL → STR src, [FP + fp_adjust(offset)] - * MACH_OP_PARAM_STACK → STR src, [FP + param_off + offset_to_args] - * MACH_OP_SYMBOL → load addr, STR src, [addr + addend] - * - * 64-bit src: emits two 32-bit stores at [dest+0] (lo) and [dest+4] (hi) - * for all dest kinds above, plus MACH_OP_IMM and MACH_OP_FRAME_ADDR. - */ -ST_FUNC void tcc_gen_machine_store_mop(MachineOperand dest, MachineOperand src, TccIrOp op) -{ - MachineCodegenContext ctx = {0}; - (void)op; - - /* A store whose destination is a VFP register is a write into that s-register - * (a register-promoted float), not a memory store — delegate to the VFP-aware - * assign path. */ - if (dest.kind == MACH_OP_VFP_REG) - { - tcc_gen_machine_assign_mop(src, dest, TCCIR_OP_ASSIGN); - return; - } - - /* 128-bit complex double store: emit four 32-bit stores for real lo/hi + imag lo/hi. - * Complex double values are 16 bytes: real (8 bytes) at base, imag (8 bytes) at base+8. - * The source is always spilled (force-spilled by the register allocator). */ - if (src.is_64bit && src.is_complex && src.btype == IROP_BTYPE_FLOAT64) - { - /* Split into four 32-bit words using complex then lo/hi splitting. */ - MachineOperand real_part = mach_make_complex_real(&src); - MachineOperand imag_part = mach_make_complex_imag(&src); - MachineOperand w0 = mach_make_lo_half(&real_part); - w0.btype = IROP_BTYPE_INT32; - MachineOperand w1 = mach_make_hi_half(&real_part); - w1.btype = IROP_BTYPE_INT32; - MachineOperand w2 = mach_make_lo_half(&imag_part); - w2.btype = IROP_BTYPE_INT32; - MachineOperand w3 = mach_make_hi_half(&imag_part); - w3.btype = IROP_BTYPE_INT32; - - /* Load all 4 words into registers. */ - const int r0 = mach_ensure_in_reg(&ctx, &w0, 0); - uint32_t excl = (1u << (uint32_t)r0); - const int r1 = mach_ensure_in_reg(&ctx, &w1, excl); - excl |= (1u << (uint32_t)r1); - const int r2 = mach_ensure_in_reg(&ctx, &w2, excl); - excl |= (1u << (uint32_t)r2); - const int r3 = mach_ensure_in_reg(&ctx, &w3, excl); - excl |= (1u << (uint32_t)r3); - - /* Store through dest: 4 × 32-bit stores at [dest+0], [dest+4], [dest+8], [dest+12]. */ - if (dest.kind == MACH_OP_REG && dest.needs_deref) - { - const uint32_t base = (uint32_t)dest.u.reg.r0; - th_store32_imm_or_reg_ex(r0, base, 0, 0, excl | (1u << base)); - th_store32_imm_or_reg_ex(r1, base, 4, 0, excl | (1u << base)); - th_store32_imm_or_reg_ex(r2, base, 8, 0, excl | (1u << base)); - th_store32_imm_or_reg_ex(r3, base, 12, 0, excl | (1u << base)); - } - else if (dest.kind == MACH_OP_SPILL) - { - const int adj = fp_adjust_local_offset(dest.u.spill.offset, 0); - const uint32_t base = (uint32_t)(tcc_state->need_frame_pointer ? R_FP : R_SP); - th_store32_imm_or_reg_ex(r0, base, adj < 0 ? -adj : adj, adj < 0 ? 1 : 0, excl | (1u << base)); - int a1 = adj + 4; - th_store32_imm_or_reg_ex(r1, base, a1 < 0 ? -a1 : a1, a1 < 0 ? 1 : 0, excl | (1u << base)); - int a2 = adj + 8; - th_store32_imm_or_reg_ex(r2, base, a2 < 0 ? -a2 : a2, a2 < 0 ? 1 : 0, excl | (1u << base)); - int a3 = adj + 12; - th_store32_imm_or_reg_ex(r3, base, a3 < 0 ? -a3 : a3, a3 < 0 ? 1 : 0, excl | (1u << base)); - } - else - { - tcc_error("compiler_error: store_mop: unhandled dest kind %d for complex double store", (int)dest.kind); - } - mach_release_all(&ctx); - return; - } - - /* 64-bit store: emit two 32-bit stores for lo and hi halves */ - if (src.is_64bit) - { - MachineOperand src_lo = mach_make_lo_half(&src); - src_lo.btype = IROP_BTYPE_INT32; - MachineOperand src_hi = mach_make_hi_half(&src); - src_hi.btype = IROP_BTYPE_INT32; - - uint32_t dest_excl = 0; - if (dest.kind == MACH_OP_REG && dest.needs_deref && - dest.u.reg.r0 >= 0 && dest.u.reg.r0 < 16) - dest_excl = (1u << (uint32_t)dest.u.reg.r0); - - const int lo_reg = mach_ensure_in_reg(&ctx, &src_lo, dest_excl); - uint32_t excl = dest_excl | (thumb_is_hw_reg(lo_reg) ? (1u << (uint32_t)lo_reg) : 0u); - const int hi_reg = mach_ensure_in_reg(&ctx, &src_hi, excl); - excl |= thumb_is_hw_reg(hi_reg) ? (1u << (uint32_t)hi_reg) : 0u; - - switch (dest.kind) - { - case MACH_OP_REG: - if (dest.needs_deref) - { - /* 64-bit pointer-store through a register-held address. STRD needs - * a 4-byte-aligned address on ARMv7-M/v8-M (faults otherwise, - * regardless of UNALIGN_TRP); use it only when the frontend proved - * alignment (dest.align4 — no packed member in the access chain). - * Otherwise the pointer may target packed-struct memory that is only - * 1- or 2-byte aligned, so two plain STRs stay the safe fallback. */ - const uint32_t base = (uint32_t)dest.u.reg.r0; - if (dest.align4 && try_strd_pair(lo_reg, hi_reg, (int)base, 0, 0)) - break; - th_store32_imm_or_reg_ex(lo_reg, base, 0, 0, excl | (1u << base)); - th_store32_imm_or_reg_ex(hi_reg, base, 4, 0, excl | (1u << base)); - } - else - { - /* Reg-pair dst: emit hi first unless lo_reg == dest.r1 (safe-ordering) */ - const int dreg_lo = dest.u.reg.r0; - const int dreg_hi = dest.u.reg.r1; - if (lo_reg == dreg_hi) - { - if (dreg_lo != lo_reg && dreg_lo != (int)PREG_REG_NONE) - ot_check_mov_reg((uint32_t)dreg_lo, (uint32_t)lo_reg, flags_safe(), THUMB_SHIFT_DEFAULT, - ENFORCE_ENCODING_NONE, false); - if (dreg_hi != hi_reg && dreg_hi != (int)PREG_REG_NONE) - ot_check_mov_reg((uint32_t)dreg_hi, (uint32_t)hi_reg, flags_safe(), THUMB_SHIFT_DEFAULT, - ENFORCE_ENCODING_NONE, false); - } - else - { - if (dreg_hi != hi_reg && dreg_hi != (int)PREG_REG_NONE) - ot_check_mov_reg((uint32_t)dreg_hi, (uint32_t)hi_reg, flags_safe(), THUMB_SHIFT_DEFAULT, - ENFORCE_ENCODING_NONE, false); - if (dreg_lo != lo_reg && dreg_lo != (int)PREG_REG_NONE) - ot_check_mov_reg((uint32_t)dreg_lo, (uint32_t)lo_reg, flags_safe(), THUMB_SHIFT_DEFAULT, - ENFORCE_ENCODING_NONE, false); - } - } - break; - - case MACH_OP_SPILL: - { - const int adj = fp_adjust_local_offset(dest.u.spill.offset, 0); - const uint32_t base = (uint32_t)(tcc_state->need_frame_pointer ? R_FP : R_SP); - if (dest.needs_deref) - { - /* LLOCAL: spill slot holds a pointer; load ptr, then store through it */ - int ptr_r = mach_alloc_scratch(&ctx, excl | (1u << base)); - if (!load_word_from_base(ptr_r, (int)base, adj < 0 ? -adj : adj, adj < 0 ? 1 : 0)) - { - ScratchRegAlloc rr = - th_offset_to_reg_ex(adj < 0 ? -adj : adj, adj < 0 ? 1 : 0, (uint32_t)(1u << ptr_r) | (1u << base)); - ot_check(th_ldr_reg((uint32_t)ptr_r, base, (uint32_t)rr.reg, THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); - restore_scratch_reg(&rr); - } - /* Pointer-through store from an LLOCAL spill slot: STRD only when - * the frontend proved >= 4-byte alignment of the target (dest.align4); - * an arbitrary pointer may reference unaligned packed-struct memory. */ - if (dest.align4 && try_strd_pair(lo_reg, hi_reg, ptr_r, 0, 0)) - break; - th_store32_imm_or_reg_ex(lo_reg, (uint32_t)ptr_r, 0, 0, excl | (1u << (uint32_t)ptr_r)); - th_store32_imm_or_reg_ex(hi_reg, (uint32_t)ptr_r, 4, 0, excl | (1u << (uint32_t)ptr_r)); - } - else - { - const int adj_hi = adj + 4; - const int sign = (adj < 0), abs_off = sign ? -adj : adj; - if (!try_strd_pair(lo_reg, hi_reg, (int)base, abs_off, sign)) - { - th_store32_imm_or_reg_ex(lo_reg, base, abs_off, sign, excl | (1u << base)); - th_store32_imm_or_reg_ex(hi_reg, base, adj_hi < 0 ? -adj_hi : adj_hi, adj_hi < 0 ? 1 : 0, - excl | (1u << base)); - } - } - break; - } - - case MACH_OP_PARAM_STACK: - { - const int adj = dest.u.param.offset + offset_to_args; - const int adj_hi = adj + 4; - const uint32_t base = (uint32_t)(tcc_state->need_frame_pointer ? R_FP : R_SP); - const int sign = (adj < 0), abs_off = sign ? -adj : adj; - if (!try_strd_pair(lo_reg, hi_reg, (int)base, abs_off, sign)) - { - th_store32_imm_or_reg_ex(lo_reg, base, abs_off, sign, excl | (1u << base)); - th_store32_imm_or_reg_ex(hi_reg, base, adj_hi < 0 ? -adj_hi : adj_hi, adj_hi < 0 ? 1 : 0, excl | (1u << base)); - } - break; - } - - case MACH_OP_SYMBOL: - { - /* Global symbol store. STRD needs 4-byte alignment; allow it only when - * the symbol's declared type guarantees natural alignment >= 4 (regular - * scalar globals) or the symbol was explicitly aligned. Packed structs - * and struct-typed globals stay on the STR-pair path. */ - Sym *sym = dest.u.sym.sym ? validate_sym_for_reloc(dest.u.sym.sym) : NULL; - int addr_r = mach_alloc_scratch_for_sym(&ctx, excl, sym, 0); - tcc_machine_load_constant(addr_r, PREG_REG_NONE, 0, 0, sym); - const int32_t addend = dest.u.sym.addend; - const int32_t addend_hi = addend + 4; - const int sign = (addend < 0), abs_off = sign ? (int)(-addend) : (int)addend; - if (sym_is_4_byte_aligned_for_64bit(sym, addend) && try_strd_pair(lo_reg, hi_reg, addr_r, abs_off, sign)) - { - break; - } - th_store32_imm_or_reg_ex(lo_reg, (uint32_t)addr_r, abs_off, sign, excl | (1u << addr_r)); - th_store32_imm_or_reg_ex(hi_reg, (uint32_t)addr_r, addend_hi < 0 ? (int)(-addend_hi) : (int)addend_hi, - addend_hi < 0 ? 1 : 0, excl | (1u << addr_r)); - break; - } - - case MACH_OP_IMM: - { - /* Store to a constant address — alignment unknown, skip STRD. */ - int addr_r = mach_alloc_scratch(&ctx, excl); - tcc_machine_load_constant(addr_r, PREG_REG_NONE, dest.u.imm.val, 0, NULL); - th_store32_imm_or_reg_ex(lo_reg, (uint32_t)addr_r, 0, 0, excl | (1u << addr_r)); - th_store32_imm_or_reg_ex(hi_reg, (uint32_t)addr_r, 4, 0, excl | (1u << addr_r)); - break; - } - - case MACH_OP_FRAME_ADDR: - { - int addr_r = mach_alloc_scratch(&ctx, excl); - tcc_machine_addr_of_stack_slot(addr_r, dest.u.frame.offset, 0 /* not param */); - if (!try_strd_pair(lo_reg, hi_reg, addr_r, 0, 0)) - { - th_store32_imm_or_reg_ex(lo_reg, (uint32_t)addr_r, 0, 0, excl | (1u << addr_r)); - th_store32_imm_or_reg_ex(hi_reg, (uint32_t)addr_r, 4, 0, excl | (1u << addr_r)); - } - break; - } - - case MACH_OP_CHAIN_REL: - { - /* 64-bit captured variable: store lo+hi words to parent frame. */ - ScratchRegAlloc chain_scratch = {0}; - int chain_used = 0; - int base = resolve_chain_base(tcc_state->ir, dest.u.chain.chain_index, excl, &chain_scratch, &chain_used); - int32_t off = dest.u.chain.offset; - int sign = (off < 0), abs_off = sign ? (int)(-off) : (int)off; - int32_t off_hi = off + 4; - int sign_hi = (off_hi < 0), abs_off_hi = sign_hi ? (int)(-off_hi) : (int)off_hi; - if (!try_strd_pair(lo_reg, hi_reg, base, abs_off, sign)) - { - th_store32_imm_or_reg_ex(lo_reg, (uint32_t)base, abs_off, sign, excl | (1u << (uint32_t)base)); - th_store32_imm_or_reg_ex(hi_reg, (uint32_t)base, abs_off_hi, sign_hi, excl | (1u << (uint32_t)base)); - } - if (chain_used) - restore_scratch_reg(&chain_scratch); - break; - } - - default: - tcc_error("compiler_error: store_mop: unhandled dest kind %d for 64-bit src", (int)dest.kind); - } - mach_release_all(&ctx); - return; - } - - const int btype = dest.btype; /* Store width from destination type */ - - /* Fast path: plain-register dest (no deref) — load src directly into dest, - * skipping the intermediate scratch + MOV that the generic path emits. - * Covers IMM, SYMBOL, SPILL, FRAME_ADDR, PARAM_STACK, CHAIN_REL, and - * REG-with-or-without-deref src kinds. */ - if (dest.kind == MACH_OP_REG && !dest.needs_deref && dest.u.reg.r0 != (int)PREG_REG_NONE) - { - tcc_gen_mach_load_to_reg(dest.u.reg.r0, &src); - mach_release_all(&ctx); - return; - } - - /* Get source value register — may allocate a scratch if spilled/const. - * When storing through a register-held pointer, protect the base register - * before materializing immediates or spilled values. */ - uint32_t src_excl = 0; - if (dest.kind == MACH_OP_REG && dest.needs_deref && - dest.u.reg.r0 >= 0 && dest.u.reg.r0 < 16) - src_excl |= (1u << (uint32_t)dest.u.reg.r0); - const int src_reg = mach_ensure_in_reg(&ctx, &src, src_excl); - - switch (dest.kind) - { - case MACH_OP_REG: - if (dest.needs_deref) - { - /* Store through pointer: STR src, [dest_reg] */ - const uint32_t base = (uint32_t)dest.u.reg.r0; - if (btype == IROP_BTYPE_INT8) - th_store8_imm_or_reg(src_reg, base, 0, 0); - else if (btype == IROP_BTYPE_INT16) - th_store16_imm_or_reg(src_reg, base, 0, 0); - else - th_store32_imm_or_reg_ex(src_reg, base, 0, 0, (uint32_t)1u << (uint32_t)src_reg); - } - else - { - /* Register-to-register store (MOV) */ - const int dreg = dest.u.reg.r0; - if (dreg != src_reg && dreg != (int)PREG_REG_NONE) - ot_check_mov_reg((uint32_t)dreg, (uint32_t)src_reg, flags_safe(), THUMB_SHIFT_DEFAULT, - ENFORCE_ENCODING_NONE, false); - } - break; - - case MACH_OP_SPILL: - { - const int adj = fp_adjust_local_offset(dest.u.spill.offset, 0); - const int sign = (adj < 0), abs_off = sign ? -adj : adj; - const uint32_t base = (uint32_t)(tcc_state->need_frame_pointer ? R_FP : R_SP); - if (dest.needs_deref) - { - /* LLOCAL: spill slot holds a pointer; load ptr, then store through it */ - int ptr_r = mach_alloc_scratch(&ctx, (uint32_t)1u << (uint32_t)src_reg | (1u << base)); - if (!load_word_from_base(ptr_r, (int)base, abs_off, sign)) - { - ScratchRegAlloc rr = - th_offset_to_reg_ex(abs_off, sign, (uint32_t)(1u << ptr_r) | (1u << base) | (1u << (uint32_t)src_reg)); - ot_check(th_ldr_reg((uint32_t)ptr_r, base, (uint32_t)rr.reg, THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); - restore_scratch_reg(&rr); - } - if (btype == IROP_BTYPE_INT8) - th_store8_imm_or_reg(src_reg, (uint32_t)ptr_r, 0, 0); - else if (btype == IROP_BTYPE_INT16) - th_store16_imm_or_reg(src_reg, (uint32_t)ptr_r, 0, 0); - else - th_store32_imm_or_reg_ex(src_reg, (uint32_t)ptr_r, 0, 0, - (uint32_t)1u << (uint32_t)src_reg | (1u << (uint32_t)ptr_r)); - } - else - { - if (btype == IROP_BTYPE_INT8) - th_store8_imm_or_reg(src_reg, base, abs_off, sign); - else if (btype == IROP_BTYPE_INT16) - th_store16_imm_or_reg(src_reg, base, abs_off, sign); - else - th_store32_imm_or_reg_ex(src_reg, base, abs_off, sign, (uint32_t)1u << (uint32_t)src_reg); - } - break; - } - - case MACH_OP_PARAM_STACK: - { - const int adj = dest.u.param.offset + offset_to_args; - const int sign = (adj < 0), abs_off = sign ? -adj : adj; - const uint32_t base = (uint32_t)(tcc_state->need_frame_pointer ? R_FP : R_SP); - if (btype == IROP_BTYPE_INT8) - th_store8_imm_or_reg(src_reg, base, abs_off, sign); - else if (btype == IROP_BTYPE_INT16) - th_store16_imm_or_reg(src_reg, base, abs_off, sign); - else - th_store32_imm_or_reg_ex(src_reg, base, abs_off, sign, (uint32_t)1u << (uint32_t)src_reg); - break; - } - - case MACH_OP_SYMBOL: - { - Sym *sym = dest.u.sym.sym ? validate_sym_for_reloc(dest.u.sym.sym) : NULL; - int addr_r = mach_alloc_scratch_for_sym(&ctx, (uint32_t)1u << (uint32_t)src_reg, sym, 0); - tcc_machine_load_constant(addr_r, PREG_REG_NONE, 0, 0, sym); - const int32_t addend = dest.u.sym.addend; - const int abs_off = addend < 0 ? (int)(-addend) : (int)addend; - const int sign = addend < 0 ? 1 : 0; - if (btype == IROP_BTYPE_INT8) - th_store8_imm_or_reg(src_reg, (uint32_t)addr_r, abs_off, sign); - else if (btype == IROP_BTYPE_INT16) - th_store16_imm_or_reg(src_reg, (uint32_t)addr_r, abs_off, sign); - else - th_store32_imm_or_reg_ex(src_reg, (uint32_t)addr_r, abs_off, sign, (uint32_t)1u << (uint32_t)src_reg); - break; - } - - case MACH_OP_IMM: - { - /* Store to an absolute address — e.g. *(volatile uint32_t*)0xABCD = val */ - int addr_r = mach_alloc_scratch(&ctx, (uint32_t)1u << (uint32_t)src_reg); - tcc_machine_load_constant(addr_r, PREG_REG_NONE, dest.u.imm.val, 0, NULL); - if (btype == IROP_BTYPE_INT8) - th_store8_imm_or_reg(src_reg, (uint32_t)addr_r, 0, 0); - else if (btype == IROP_BTYPE_INT16) - th_store16_imm_or_reg(src_reg, (uint32_t)addr_r, 0, 0); - else - th_store32_imm_or_reg_ex(src_reg, (uint32_t)addr_r, 0, 0, (uint32_t)1u << (uint32_t)src_reg); - break; - } - - case MACH_OP_FRAME_ADDR: - { - /* Store to a frame-relative address; equivalent to MACH_OP_SPILL but via addr computation */ - int addr_r = mach_alloc_scratch(&ctx, (uint32_t)1u << (uint32_t)src_reg); - tcc_machine_addr_of_stack_slot(addr_r, dest.u.frame.offset, 0 /* not param */); - if (btype == IROP_BTYPE_INT8) - th_store8_imm_or_reg(src_reg, (uint32_t)addr_r, 0, 0); - else if (btype == IROP_BTYPE_INT16) - th_store16_imm_or_reg(src_reg, (uint32_t)addr_r, 0, 0); - else - th_store32_imm_or_reg_ex(src_reg, (uint32_t)addr_r, 0, 0, (uint32_t)1u << (uint32_t)src_reg); - break; - } - - case MACH_OP_CHAIN_REL: - { - /* Captured variable: store to parent frame via static chain. */ - ScratchRegAlloc chain_scratch = {0}; - int chain_used = 0; - int base = resolve_chain_base(tcc_state->ir, dest.u.chain.chain_index, (1u << (uint32_t)src_reg), &chain_scratch, - &chain_used); - int32_t off = dest.u.chain.offset; - int sign = (off < 0), abs_off = sign ? (int)(-off) : (int)off; - if (btype == IROP_BTYPE_INT8) - th_store8_imm_or_reg(src_reg, (uint32_t)base, abs_off, sign); - else if (btype == IROP_BTYPE_INT16) - th_store16_imm_or_reg(src_reg, (uint32_t)base, abs_off, sign); - else - th_store32_imm_or_reg_ex(src_reg, (uint32_t)base, abs_off, sign, - (1u << (uint32_t)src_reg) | (1u << (uint32_t)base)); - if (chain_used) - restore_scratch_reg(&chain_scratch); - break; - } - - default: - tcc_error("compiler_error: store_mop: unhandled dest kind %d", (int)dest.kind); - } - - mach_release_all(&ctx); -} - -/* Indexed load: dest = *(base + (index << scale)) - * Generates: LDR dest, [base, index, LSL #scale] - */ -ST_FUNC void tcc_gen_machine_load_indexed_mop(MachineOperand dest, MachineOperand base, MachineOperand index, - MachineOperand scale, TccIrOp op) -{ - MachineCodegenContext ctx = {0}; - (void)op; - - int shift_amount = (scale.kind == MACH_OP_IMM) ? (int)scale.u.imm.val : 2; - if (shift_amount < 0 || shift_amount > 31) - shift_amount = 2; - - /* Fast path: base is &local + constant index — fold into SP/FP-relative load. - * Mirrors the store_indexed FRAME_ADDR fast path. */ - if (!dest.is_64bit && shift_amount == 0 && index.kind == MACH_OP_IMM && - base.kind == MACH_OP_FRAME_ADDR && !base.needs_deref) - { - int combined = base.u.frame.offset + (int)index.u.imm.val; - int adjusted = fp_adjust_local_offset(combined, 0); - int sign = (adjusted < 0); - int abs_off = sign ? -adjusted : adjusted; - if (abs_off <= 4095) - { - const int dest_reg = mach_get_dest_reg(&ctx, &dest, 0); - const int base_reg = tcc_state->need_frame_pointer ? R_FP : R_SP; - load_from_base(dest_reg, PREG_REG_NONE, dest.btype, (int)dest.is_unsigned, abs_off, sign, (uint32_t)base_reg); - mach_writeback_dest(&dest, dest_reg); - mach_release_all(&ctx); - return; - } - } - - /* Fast path: constant-displacement load (scale == 0 and index is an immediate). - * Generated by the displacement-fusion pass when folding `ADD base,#imm; LOAD *` - * into a single `LDR dest,[base,#imm]`, matching GCC's addressing-mode output. */ - if (!dest.is_64bit && shift_amount == 0 && index.kind == MACH_OP_IMM) - { - int imm = (int)index.u.imm.val; - int sign = (imm < 0); - int abs_off = sign ? -imm : imm; - if (abs_off <= 4095) - { - const int dest_reg = mach_get_dest_reg(&ctx, &dest, 0); - uint32_t excl = (1u << (uint32_t)dest_reg); - int base_reg = mach_ensure_in_reg(&ctx, &base, excl); - load_from_base(dest_reg, PREG_REG_NONE, dest.btype, (int)dest.is_unsigned, abs_off, sign, (uint32_t)base_reg); - mach_writeback_dest(&dest, dest_reg); - mach_release_all(&ctx); - return; - } - } - - /* scale 0 → no shift: use THUMB_SHIFT_NONE so the 16-bit T1 register-offset - * encoding (all-low regs) can be selected instead of the wide T32 form. */ - thumb_shift shift = (shift_amount == 0) - ? (thumb_shift){.type = THUMB_SHIFT_NONE, .value = 0, .mode = THUMB_SHIFT_IMMEDIATE} - : (thumb_shift){.type = THUMB_SHIFT_LSL, .value = (uint32_t)shift_amount, .mode = THUMB_SHIFT_IMMEDIATE}; - - /* Fast path: 64-bit constant-displacement load using LDRD [base, #imm]. - * LDRD supports word-aligned offsets in range [-1020, 1020]. - * base.underalign_hint (packed-derived address): skip — LDRD faults on - * unaligned addresses; the generic path below emits an LDR pair instead. */ - if (dest.is_64bit && shift_amount == 0 && index.kind == MACH_OP_IMM && !base.underalign_hint) - { - int imm = (int)index.u.imm.val; - int sign = (imm < 0); - int abs_off = sign ? -imm : imm; - if (abs_off <= 1020 && (abs_off & 3) == 0) - { - const bool dest_is_reg = (dest.kind == MACH_OP_REG && !dest.needs_deref); - int dest_lo, dest_hi; - MachineOperand dest_hi_mop = {0}; - uint32_t excl = 0; - - if (dest_is_reg) - { - dest_lo = dest.u.reg.r0; - if (!thumb_is_hw_reg(dest.u.reg.r1)) - tcc_error("load_indexed_mop: 64-bit dest has invalid r1=%d (r0=%d) — " - "register allocator must produce a valid pair", - dest.u.reg.r1, dest.u.reg.r0); - dest_hi = dest.u.reg.r1; - excl = (1u << (uint32_t)dest_lo) | (1u << (uint32_t)dest_hi); - } - else - { - dest_lo = mach_get_dest_reg(&ctx, &dest, 0); - excl = (1u << (uint32_t)dest_lo); - dest_hi_mop = mach_make_hi_half(&dest); - dest_hi = mach_get_dest_reg(&ctx, &dest_hi_mop, excl); - excl |= (1u << (uint32_t)dest_hi); - } - - int base_reg = mach_ensure_in_reg(&ctx, &base, excl); - uint32_t puw = sign ? 4u : 6u; - ot_check(th_ldrd_imm((uint32_t)dest_lo, (uint32_t)dest_hi, (uint32_t)base_reg, abs_off, puw)); - if (!dest_is_reg) - { - MachineOperand dest_lo_mop = mach_make_lo_half(&dest); - mach_writeback_dest(&dest_lo_mop, dest_lo); - mach_writeback_dest(&dest_hi_mop, dest_hi); - } - mach_release_all(&ctx); - return; - } - } - - /* 64-bit indexed load: compute EA = base + index< 31) - shift_amount = 2; - - /* Fast path: base is &local + constant index — fold into SP/FP-relative store. - * Avoids emitting a separate `ADD base, sp, #frame_off` LEA before the STR, - * cutting one instruction per access in dense local-array initialization. */ - if (!value.is_64bit && shift_amount == 0 && index.kind == MACH_OP_IMM && - base.kind == MACH_OP_FRAME_ADDR && !base.needs_deref) - { - int combined = base.u.frame.offset + (int)index.u.imm.val; - int adjusted = fp_adjust_local_offset(combined, 0); - int sign = (adjusted < 0); - int abs_off = sign ? -adjusted : adjusted; - if (abs_off <= 4095) - { - const int btype = value.btype; - const int base_reg = tcc_state->need_frame_pointer ? R_FP : R_SP; - int value_reg = mach_ensure_in_reg(&ctx, &value, 0); - if (btype == IROP_BTYPE_INT8) - th_store8_imm_or_reg(value_reg, (uint32_t)base_reg, abs_off, sign); - else if (btype == IROP_BTYPE_INT16) - th_store16_imm_or_reg(value_reg, (uint32_t)base_reg, abs_off, sign); - else - th_store32_imm_or_reg_ex(value_reg, (uint32_t)base_reg, abs_off, sign, - (1u << (uint32_t)value_reg) | (1u << (uint32_t)base_reg)); - mach_release_all(&ctx); - return; - } - } - - /* Fast path: constant-displacement store (scale == 0 and index is an immediate). - * Mirrors the load_indexed fast path; emits `STR value,[base,#imm]`. */ - if (!value.is_64bit && shift_amount == 0 && index.kind == MACH_OP_IMM) - { - int imm = (int)index.u.imm.val; - int sign = (imm < 0); - int abs_off = sign ? -imm : imm; - if (abs_off <= 4095) - { - const int btype = value.btype; - int value_reg = mach_ensure_in_reg(&ctx, &value, 0); - uint32_t excl = (1u << (uint32_t)value_reg); - int base_reg = mach_ensure_in_reg(&ctx, &base, excl); - if (btype == IROP_BTYPE_INT8) - th_store8_imm_or_reg(value_reg, (uint32_t)base_reg, abs_off, sign); - else if (btype == IROP_BTYPE_INT16) - th_store16_imm_or_reg(value_reg, (uint32_t)base_reg, abs_off, sign); - else - th_store32_imm_or_reg_ex(value_reg, (uint32_t)base_reg, abs_off, sign, - (1u << (uint32_t)value_reg) | (1u << (uint32_t)base_reg)); - mach_release_all(&ctx); - return; - } - } - - /* scale 0 → no shift: use THUMB_SHIFT_NONE so the 16-bit T1 register-offset - * encoding (all-low regs) can be selected instead of the wide T32 form. */ - thumb_shift shift = (shift_amount == 0) - ? (thumb_shift){.type = THUMB_SHIFT_NONE, .value = 0, .mode = THUMB_SHIFT_IMMEDIATE} - : (thumb_shift){.type = THUMB_SHIFT_LSL, .value = (uint32_t)shift_amount, .mode = THUMB_SHIFT_IMMEDIATE}; - - /* Fast path: 64-bit constant-displacement store using STRD [base, #imm]. - * base.underalign_hint (packed-derived address): skip — STRD faults on - * unaligned addresses; the generic path below emits an STR pair instead. */ - if (value.is_64bit && shift_amount == 0 && index.kind == MACH_OP_IMM && !base.underalign_hint) - { - int imm = (int)index.u.imm.val; - int sign = (imm < 0); - int abs_off = sign ? -imm : imm; - if (abs_off <= 1020 && (abs_off & 3) == 0) - { - MachineOperand val_lo = mach_make_lo_half(&value); - val_lo.btype = IROP_BTYPE_INT32; - MachineOperand val_hi = mach_make_hi_half(&value); - val_hi.btype = IROP_BTYPE_INT32; - const int lo_reg = mach_ensure_in_reg(&ctx, &val_lo, 0); - uint32_t excl = (1u << (uint32_t)lo_reg); - const int hi_reg = mach_ensure_in_reg(&ctx, &val_hi, excl); - excl |= (1u << (uint32_t)hi_reg); - int base_reg = mach_ensure_in_reg(&ctx, &base, excl); - uint32_t puw = sign ? 4u : 6u; - ot_check(th_strd_imm((uint32_t)lo_reg, (uint32_t)hi_reg, (uint32_t)base_reg, abs_off, puw)); - mach_release_all(&ctx); - return; - } - } - - /* 64-bit indexed store: compute EA = base + index< 255) - { - mach_release_all(&ctx); - tcc_error("compiler_error: post-increment offset %d out of range (0-255)", offset_imm); - return; - } - const uint32_t puw = 3; /* post-index (p=0), add (u=1), writeback (w=1) */ - - /* 64-bit post-increment load: LDRD dest_lo, dest_hi, [ptr], #offset */ - if (dest.is_64bit) - { - const int dest_lo = mach_get_dest_reg(&ctx, &dest, 0); - MachineOperand dest_hi_mop = mach_make_hi_half(&dest); - const int dest_hi = mach_get_dest_reg(&ctx, &dest_hi_mop, (1u << (uint32_t)dest_lo)); - uint32_t excl = (1u << (uint32_t)dest_lo) | (1u << (uint32_t)dest_hi); - int ptr_reg = mach_ensure_in_reg(&ctx, &ptr, excl); - ot_check( - th_ldrd_imm((uint32_t)dest_lo, (uint32_t)dest_hi, (uint32_t)ptr_reg, offset_imm, puw)); - mach_writeback_dest(&dest_hi_mop, dest_hi); - mach_writeback_dest(&dest, dest_lo); - mach_release_all(&ctx); - return; - } - - const int dest_reg = mach_get_dest_reg(&ctx, &dest, 0); - const int btype = dest.btype; - const int is_unsigned = (int)dest.is_unsigned; - - uint32_t excl = (1u << (uint32_t)dest_reg); - int ptr_reg = mach_ensure_in_reg(&ctx, &ptr, excl); - - if (btype == IROP_BTYPE_INT8) - { - if (is_unsigned) - ot_check(th_ldrb_imm(dest_reg, ptr_reg, offset_imm, puw, ENFORCE_ENCODING_NONE)); - else - ot_check(th_ldrsb_imm(dest_reg, ptr_reg, offset_imm, puw, ENFORCE_ENCODING_NONE)); - } - else if (btype == IROP_BTYPE_INT16) - { - if (is_unsigned) - ot_check(th_ldrh_imm(dest_reg, ptr_reg, offset_imm, puw, ENFORCE_ENCODING_NONE)); - else - ot_check(th_ldrsh_imm(dest_reg, ptr_reg, offset_imm, puw, ENFORCE_ENCODING_NONE)); - } - else - { - ot_check_ldr_imm(dest_reg, ptr_reg, offset_imm, puw, ENFORCE_ENCODING_NONE); - } - mach_writeback_dest(&dest, dest_reg); - mach_release_all(&ctx); -} - -/* Post-increment store: *ptr = value; ptr += offset - * Generates: STR value, [ptr], #offset (puw=3: post-index, add, writeback) - */ -ST_FUNC void tcc_gen_machine_store_postinc_mop(MachineOperand ptr, MachineOperand value, MachineOperand offset, - TccIrOp op) -{ - MachineCodegenContext ctx = {0}; - (void)op; - - int offset_imm = (offset.kind == MACH_OP_IMM) ? (int)offset.u.imm.val : 4; - if (offset_imm < 0 || offset_imm > 255) - { - mach_release_all(&ctx); - tcc_error("compiler_error: post-increment offset %d out of range (0-255)", offset_imm); - return; - } - const uint32_t puw = 3; /* post-index (p=0), add (u=1), writeback (w=1) */ - - /* 64-bit post-increment store: STRD lo, hi, [ptr], #offset */ - if (value.is_64bit) - { - MachineOperand val_lo = mach_make_lo_half(&value); - val_lo.btype = IROP_BTYPE_INT32; - MachineOperand val_hi = mach_make_hi_half(&value); - val_hi.btype = IROP_BTYPE_INT32; - const int lo_reg = mach_ensure_in_reg(&ctx, &val_lo, 0); - uint32_t excl = (1u << (uint32_t)lo_reg); - const int hi_reg = mach_ensure_in_reg(&ctx, &val_hi, excl); - excl |= (1u << (uint32_t)hi_reg); - int ptr_reg = mach_ensure_in_reg(&ctx, &ptr, excl); - ot_check( - th_strd_imm((uint32_t)lo_reg, (uint32_t)hi_reg, (uint32_t)ptr_reg, offset_imm, puw)); - mach_release_all(&ctx); - return; - } - - const int btype = value.btype; - - int value_reg = mach_ensure_in_reg(&ctx, &value, 0); - uint32_t excl = (1u << (uint32_t)value_reg); - int ptr_reg = mach_ensure_in_reg(&ctx, &ptr, excl); - - if (btype == IROP_BTYPE_INT8) - ot_check(th_strb_imm(value_reg, ptr_reg, offset_imm, puw, ENFORCE_ENCODING_NONE)); - else if (btype == IROP_BTYPE_INT16) - ot_check(th_strh_imm(value_reg, ptr_reg, offset_imm, puw, ENFORCE_ENCODING_NONE)); - else - ot_check_str_imm(value_reg, ptr_reg, offset_imm, puw, ENFORCE_ENCODING_NONE); - - mach_release_all(&ctx); -} - -/* Get the soft float library function name for an FP operation */ -static const char *get_softfp_func_name(TccIrOp op, int is_double) -{ - switch (op) - { - case TCCIR_OP_FADD: - return is_double ? "__aeabi_dadd" : "__aeabi_fadd"; - case TCCIR_OP_FSUB: - return is_double ? "__aeabi_dsub" : "__aeabi_fsub"; - case TCCIR_OP_FMUL: - return is_double ? "__aeabi_dmul" : "__aeabi_fmul"; - case TCCIR_OP_FDIV: - return is_double ? "__aeabi_ddiv" : "__aeabi_fdiv"; - case TCCIR_OP_FNEG: - /* For negation, we can XOR the sign bit - handled separately */ - return NULL; - default: - return NULL; - } -} - -/* fp_mop_load_arg: Load a MachineOperand value into a fixed argument register - * (R0, R1, etc.) for a soft-float ABI call. Unlike mach_ensure_in_reg, this - * writes to a caller-specified register without scratch allocation bookkeeping. - * Used by tcc_gen_machine_fp_mop to set up R0/R1 before BL __aeabi_f*. */ -static void fp_mop_load_arg(int target_reg, const MachineOperand *op) -{ - switch (op->kind) - { - case MACH_OP_NONE: - return; - case MACH_OP_VFP_REG: - ot_check(th_vmov_gp_sp((uint16_t)target_reg, (uint16_t)op->u.reg.r0, 1)); /* target = sN */ - return; - case MACH_OP_REG: - if (!op->needs_deref) - { - if (op->u.reg.r0 != target_reg) - ot_check_mov_reg((uint32_t)target_reg, (uint32_t)op->u.reg.r0, flags_safe(), - THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE, false); - } - else - { - load_from_base(target_reg, PREG_REG_NONE, op->btype, (int)op->is_unsigned, 0, 0, (uint32_t)op->u.reg.r0); - } - return; - case MACH_OP_SPILL: - tcc_machine_load_spill_slot(target_reg, op->u.spill.offset); - if (op->needs_deref) - load_from_base(target_reg, PREG_REG_NONE, op->btype, (int)op->is_unsigned, 0, 0, (uint32_t)target_reg); - return; - case MACH_OP_PARAM_STACK: - { - const int adjusted = op->u.param.offset + offset_to_args; - const int base_reg = tcc_state->need_frame_pointer ? R_FP : R_SP; - const int sign = (adjusted < 0); - const int abs_off = sign ? -adjusted : adjusted; - load_from_base(target_reg, PREG_REG_NONE, op->btype, (int)op->is_unsigned, abs_off, sign, (uint32_t)base_reg); - return; - } - case MACH_OP_IMM: - tcc_machine_load_constant(target_reg, PREG_REG_NONE, op->u.imm.val, 0, NULL); - return; - case MACH_OP_SYMBOL: - { - Sym *sym = op->u.sym.sym ? validate_sym_for_reloc(op->u.sym.sym) : NULL; - if (!op->needs_deref) - { - tcc_machine_load_constant(target_reg, PREG_REG_NONE, op->u.sym.addend, 0, sym); - } - else - { - /* Load symbol address into target_reg, then dereference through it. */ - tcc_machine_load_constant(target_reg, PREG_REG_NONE, 0, 0, sym); - const int32_t addend = op->u.sym.addend; - load_from_base(target_reg, PREG_REG_NONE, op->btype, (int)op->is_unsigned, - addend < 0 ? (int)(-addend) : (int)addend, addend < 0 ? 1 : 0, (uint32_t)target_reg); - } - return; - } - case MACH_OP_FRAME_ADDR: - tcc_machine_addr_of_stack_slot(target_reg, op->u.frame.offset, 0); - if (op->needs_deref) - load_from_base(target_reg, PREG_REG_NONE, op->btype, (int)op->is_unsigned, 0, 0, (uint32_t)target_reg); - return; - case MACH_OP_CHAIN_REL: - { - /* Captured variable: load from parent frame via static chain. */ - ScratchRegAlloc chain_scratch = {0}; - int chain_used = 0; - int base = resolve_chain_base(tcc_state->ir, op->u.chain.chain_index, (1u << (uint32_t)target_reg), &chain_scratch, - &chain_used); - int32_t off = op->u.chain.offset; - int sign = (off < 0); - int abs_off = sign ? (int)(-off) : (int)off; - load_from_base(target_reg, PREG_REG_NONE, op->btype, (int)op->is_unsigned, abs_off, sign, (uint32_t)base); - if (chain_used) - restore_scratch_reg(&chain_scratch); - return; - } - default: - tcc_error("compiler_error: fp_mop_load_arg: unhandled kind %d", (int)op->kind); - } -} - -/* Load a 64-bit (double-precision) MachineOperand into two consecutive argument - * registers (lo_reg = low 32 bits, hi_reg = high 32 bits). - * Handles REG pair, SPILL pair, PARAM_STACK, and deref'd REG. */ -static void fp_mop_load_double_arg(int lo_reg, int hi_reg, const MachineOperand *op) -{ - if (op->needs_deref && op->kind == MACH_OP_REG) - { - /* Pointer in register: resolve [r0] and [r0+4] into physical regs first. */ - MachineCodegenContext mctx; - memset(&mctx, 0, sizeof(mctx)); - uint32_t excl = (1u << (uint32_t)lo_reg) | (1u << (uint32_t)hi_reg); - MachineOperand resolved = mach_resolve_deref_64(&mctx, op, &excl); - mach_release_all(&mctx); - MachineOperand lo_op = mach_make_lo_half(&resolved); - MachineOperand hi_op = mach_make_hi_half(&resolved); - fp_mop_load_arg(lo_reg, &lo_op); - fp_mop_load_arg(hi_reg, &hi_op); - return; - } - if (op->kind == MACH_OP_PARAM_STACK) - { - /* Stack parameter: low word at op->offset, high word at op->offset + 4. */ - fp_mop_load_arg(lo_reg, op); - MachineOperand hi_op = *op; - hi_op.u.param.offset += 4; - fp_mop_load_arg(hi_reg, &hi_op); - return; - } - /* REG (non-deref) or SPILL: split with lo/hi helpers. */ - { - MachineOperand lo_op = mach_make_lo_half(op); - MachineOperand hi_op = mach_make_hi_half(op); - fp_mop_load_arg(lo_reg, &lo_op); - fp_mop_load_arg(hi_reg, &hi_op); - } -} - -/* Issue a BL to a soft-float library function, saving/restoring R9+R12 in - * text+data-separation (PIC) mode. */ -static void fp_mop_do_bl(const char *func_name) -{ - Sym *sym = external_global_sym(tok_alloc_const(func_name), &func_old_type); - MachineOperand func_mop = {0}; - func_mop.kind = MACH_OP_SYMBOL; - func_mop.u.sym.sym = sym; - func_mop.u.sym.addend = 0; - if (text_and_data_separation) - ot_check(th_push((uint16_t)((1 << R9) | (1 << R12)))); - gcall_or_jump_mop(0, func_mop); - if (text_and_data_separation) - ot_check(th_pop((uint16_t)((1 << R9) | (1 << R12)))); -} - -/* Write a soft-float call result back to dest. - * Single-precision result is in R0; double-precision is in R0 (lo) : R1 (hi). */ -static void fp_mop_writeback_result(const MachineOperand *dest, int is_double) -{ - if (is_double) - { - MachineOperand lo_dest = mach_make_lo_half(dest); - MachineOperand hi_dest = mach_make_hi_half(dest); - mach_writeback_dest(&lo_dest, R0); - mach_writeback_dest(&hi_dest, R1); - } - else - mach_writeback_dest(dest, R0); -} - -/* ============================================================ - * Complex float MOP path — Phase 5k - * ============================================================ - * - * Complex floats are 64-bit register pairs: lo = real, hi = imaginary. - * Complex doubles are 128-bit values (always spilled): real at offset+0, imag at offset+8. - * These functions use the MOP infrastructure (fp_mop_load_arg, fp_mop_do_bl, - * mach_writeback_dest) to handle any operand kind (REG, SPILL, PARAM_STACK, - * CHAIN_REL, etc.) without requiring fill_registers_ir. - * - * Strategy: save all inputs to a stack frame, call __aeabi_f* / __aeabi_d* - * library functions, write results back to dest via mach_writeback_dest. - */ - -/* Split a complex MachineOperand into its real component. - * For complex float: real is the 32-bit lo half (same as mach_make_lo_half). - * For complex double: real is the 64-bit double at the base offset. */ -static MachineOperand mach_make_complex_real(const MachineOperand *op) -{ - if (op->btype == IROP_BTYPE_FLOAT64) - { - /* Complex double: real part is a 64-bit double at the base offset. */ - MachineOperand real = *op; - real.is_complex = false; - real.is_64bit = true; /* each component is 64-bit double */ - if (real.kind == MACH_OP_REG) - ; /* keep r0:r1 pair — only valid for register-allocated complex floats */ - return real; - } - /* Complex float: fall back to lo half. */ - return mach_make_lo_half(op); -} - -/* Split a complex MachineOperand into its imaginary component. - * For complex float: imag is the 32-bit hi half (same as mach_make_hi_half). - * For complex double: imag is the 64-bit double at base offset + 8. */ -static MachineOperand mach_make_complex_imag(const MachineOperand *op) -{ - if (op->btype == IROP_BTYPE_FLOAT64) - { - /* Complex double: imag part is a 64-bit double at offset + 8. */ - MachineOperand imag = *op; - imag.is_complex = false; - imag.is_64bit = true; - switch (imag.kind) - { - case MACH_OP_SPILL: - imag.u.spill.offset += 8; - break; - case MACH_OP_FRAME_ADDR: - imag.u.frame.offset += 8; - break; - case MACH_OP_PARAM_STACK: - imag.u.param.offset += 8; - break; - case MACH_OP_CHAIN_REL: - imag.u.chain.offset += 8; - break; - case MACH_OP_SYMBOL: - imag.u.sym.addend += 8; - break; - case MACH_OP_REG: - /* Register-based complex double shouldn't happen (force-spilled), - * but handle gracefully: imaginary part is not representable. */ - break; - default: - break; - } - return imag; - } - /* Complex float: fall back to hi half. */ - return mach_make_hi_half(op); -} - -/* Helper: save a double from R0:R1 to SP-relative stack offset. */ -static void fp_mop_save_double_to_sp(int off) -{ - ot_check_str_imm(R0, R_SP, off, 6, ENFORCE_ENCODING_NONE); - ot_check_str_imm(R1, R_SP, off + 4, 6, ENFORCE_ENCODING_NONE); -} - -/* Helper: load a double from SP-relative stack offset into (lo_reg, hi_reg). */ -static void fp_mop_load_double_from_sp(int lo_reg, int hi_reg, int off) -{ - ot_check_ldr_imm(lo_reg, R_SP, off, 6, ENFORCE_ENCODING_NONE); - ot_check_ldr_imm(hi_reg, R_SP, off + 4, 6, ENFORCE_ENCODING_NONE); -} - -/* Process complex double multiplication via MachineOperands. - * Handles all cases: - * scalar × complex: a * (c+di) = ac + (ad)i - * complex × scalar: (a+bi) * c = ac + (bc)i - * complex × complex: (a+bi) * (c+di) = (ac-bd) + (ad+bc)i - * - * Uses __aeabi_dmul, __aeabi_dadd, __aeabi_dsub for double-precision. - * Double AEABI calling convention: R0:R1 = arg1, R2:R3 = arg2, result in R0:R1. - */ -static void thumb_process_complex_mul_double_mop(MachineOperand src1, MachineOperand src2, MachineOperand dest) -{ - int s1_complex = src1.is_complex; - int s2_complex = src2.is_complex; - - MachineOperand d_real = mach_make_complex_real(&dest); - MachineOperand d_imag = mach_make_complex_imag(&dest); - - if (!s1_complex && s2_complex) - { - /* scalar double × complex double: a * (c+di) = ac + (ad)i */ - MachineOperand s2_real = mach_make_complex_real(&src2); - MachineOperand s2_imag = mach_make_complex_imag(&src2); - - /* Allocate 8 bytes to save the scalar 'a'. */ - ot_check(th_sub_imm(R_SP, R_SP, 8, flags_safe(), ENFORCE_ENCODING_NONE)); - - /* Load scalar 'a' into R0:R1 and save to stack. */ - fp_mop_load_double_arg(R0, R1, &src1); - fp_mop_save_double_to_sp(0); - - /* Compute a * c: load 'c' into R2:R3. R0:R1 already = 'a'. */ - fp_mop_load_double_arg(R2, R3, &s2_real); - fp_mop_do_bl("__aeabi_dmul"); - /* R0:R1 = a*c → write to dest real. */ - fp_mop_writeback_result(&d_real, 1); - - /* Compute a * d: reload 'a' from stack, load 'd' into R2:R3. */ - fp_mop_load_double_from_sp(R0, R1, 0); - fp_mop_load_double_arg(R2, R3, &s2_imag); - fp_mop_do_bl("__aeabi_dmul"); - /* R0:R1 = a*d → write to dest imag. */ - fp_mop_writeback_result(&d_imag, 1); - - ot_check(th_add_imm(R_SP, R_SP, 8, flags_safe(), ENFORCE_ENCODING_NONE)); - } - else if (s1_complex && !s2_complex) - { - /* complex double × scalar double: (a+bi) * c = ac + (bc)i */ - MachineOperand s1_real = mach_make_complex_real(&src1); - MachineOperand s1_imag = mach_make_complex_imag(&src1); - - /* Allocate 8 bytes to save the scalar 'c'. */ - ot_check(th_sub_imm(R_SP, R_SP, 8, flags_safe(), ENFORCE_ENCODING_NONE)); - - /* Load scalar 'c' into R0:R1 and save to stack. */ - fp_mop_load_double_arg(R0, R1, &src2); - fp_mop_save_double_to_sp(0); - - /* Compute a * c. */ - fp_mop_load_double_arg(R0, R1, &s1_real); - fp_mop_load_double_arg(R2, R3, &src2); - fp_mop_do_bl("__aeabi_dmul"); - fp_mop_writeback_result(&d_real, 1); - - /* Compute b * c: reload 'c', load 'b'. */ - fp_mop_load_double_arg(R0, R1, &s1_imag); - fp_mop_load_double_from_sp(R2, R3, 0); - fp_mop_do_bl("__aeabi_dmul"); - fp_mop_writeback_result(&d_imag, 1); - - ot_check(th_add_imm(R_SP, R_SP, 8, flags_safe(), ENFORCE_ENCODING_NONE)); - } - else - { - /* complex × complex: (a+bi)*(c+di) = (ac-bd) + (ad+bc)i - * - * Stack layout (48 bytes): - * [sp+40] = d (imag of src2, 8 bytes) - * [sp+32] = c (real of src2, 8 bytes) - * [sp+24] = b (imag of src1, 8 bytes) - * [sp+16] = a (real of src1, 8 bytes) - * [sp+8] = scratch1 (8 bytes) - * [sp+0] = scratch0 (8 bytes) - */ - MachineOperand s1_real = mach_make_complex_real(&src1); - MachineOperand s1_imag = mach_make_complex_imag(&src1); - MachineOperand s2_real = mach_make_complex_real(&src2); - MachineOperand s2_imag = mach_make_complex_imag(&src2); - - const int off_scratch0 = 0, off_scratch1 = 8; - const int off_a = 16, off_b = 24, off_c = 32, off_d = 40; - - ot_check(th_sub_imm(R_SP, R_SP, 48, flags_safe(), ENFORCE_ENCODING_NONE)); - - /* Save all 4 components to stack. */ - fp_mop_load_double_arg(R0, R1, &s1_real); - fp_mop_save_double_to_sp(off_a); - fp_mop_load_double_arg(R0, R1, &s1_imag); - fp_mop_save_double_to_sp(off_b); - fp_mop_load_double_arg(R0, R1, &s2_real); - fp_mop_save_double_to_sp(off_c); - fp_mop_load_double_arg(R0, R1, &s2_imag); - fp_mop_save_double_to_sp(off_d); - - /* Step 1: ac → scratch0. */ - fp_mop_load_double_from_sp(R0, R1, off_a); - fp_mop_load_double_from_sp(R2, R3, off_c); - fp_mop_do_bl("__aeabi_dmul"); - fp_mop_save_double_to_sp(off_scratch0); - - /* Step 2: bd → scratch1. */ - fp_mop_load_double_from_sp(R0, R1, off_b); - fp_mop_load_double_from_sp(R2, R3, off_d); - fp_mop_do_bl("__aeabi_dmul"); - fp_mop_save_double_to_sp(off_scratch1); - - /* Step 3: real = ac - bd → scratch0. */ - fp_mop_load_double_from_sp(R0, R1, off_scratch0); - fp_mop_load_double_from_sp(R2, R3, off_scratch1); - fp_mop_do_bl("__aeabi_dsub"); - fp_mop_save_double_to_sp(off_scratch0); - - /* Step 4: ad → scratch1. */ - fp_mop_load_double_from_sp(R0, R1, off_a); - fp_mop_load_double_from_sp(R2, R3, off_d); - fp_mop_do_bl("__aeabi_dmul"); - fp_mop_save_double_to_sp(off_scratch1); - - /* Step 5: bc → off_a (reuse slot). */ - fp_mop_load_double_from_sp(R0, R1, off_b); - fp_mop_load_double_from_sp(R2, R3, off_c); - fp_mop_do_bl("__aeabi_dmul"); - fp_mop_save_double_to_sp(off_a); - - /* Step 6: imag = ad + bc → scratch1. */ - fp_mop_load_double_from_sp(R0, R1, off_scratch1); - fp_mop_load_double_from_sp(R2, R3, off_a); - fp_mop_do_bl("__aeabi_dadd"); - fp_mop_save_double_to_sp(off_scratch1); - - /* Write results back to dest. */ - fp_mop_load_double_from_sp(R0, R1, off_scratch0); - fp_mop_writeback_result(&d_real, 1); - fp_mop_load_double_from_sp(R0, R1, off_scratch1); - fp_mop_writeback_result(&d_imag, 1); - - ot_check(th_add_imm(R_SP, R_SP, 48, flags_safe(), ENFORCE_ENCODING_NONE)); - } -} - -/* complex_pair_writeback: Write a (real, imag) pair from two physical registers - * into a split MachineOperand pair without clobbering. - * Handles the case where d_lo's target register overlaps hi_reg (or vice versa) - * by saving the clobbered value to R2 or R3 first. */ -static void complex_pair_writeback(MachineOperand *d_lo, int lo_reg, MachineOperand *d_hi, int hi_reg) -{ - int lo_clobbers_hi = (d_lo->kind == MACH_OP_REG && d_lo->u.reg.r0 == hi_reg); - int hi_clobbers_lo = (d_hi->kind == MACH_OP_REG && d_hi->u.reg.r0 == lo_reg); - - if (lo_clobbers_hi && hi_clobbers_lo) - { - /* Total swap: save hi to temp, then write both */ - int tmp = (lo_reg != R2 && hi_reg != R2) ? R2 : R3; - ot_check_mov_reg((uint32_t)tmp, (uint32_t)hi_reg, flags_safe(), THUMB_SHIFT_DEFAULT, - ENFORCE_ENCODING_NONE, false); - mach_writeback_dest(d_lo, lo_reg); - mach_writeback_dest(d_hi, tmp); - } - else if (lo_clobbers_hi) - { - /* Lo writeback would clobber hi value; save hi first */ - int tmp = (lo_reg != R2 && hi_reg != R2) ? R2 : R3; - ot_check_mov_reg((uint32_t)tmp, (uint32_t)hi_reg, flags_safe(), THUMB_SHIFT_DEFAULT, - ENFORCE_ENCODING_NONE, false); - mach_writeback_dest(d_lo, lo_reg); - mach_writeback_dest(d_hi, tmp); - } - else if (hi_clobbers_lo) - { - /* Hi writeback would clobber lo value; write lo first */ - mach_writeback_dest(d_lo, lo_reg); - mach_writeback_dest(d_hi, hi_reg); - } - else - { - mach_writeback_dest(d_lo, lo_reg); - mach_writeback_dest(d_hi, hi_reg); - } -} - -/* thumb_emit_dcp_addsub_mop: inline double add/subtract on RP2350's DCP. - * - * The sequence is the one in lib/fp/arm/rp2350/dcp_aeabi.S, itself transcribed - * from pico-sdk's dcp_canned.inc.S -- keep the two in step: - * - * WXUP a ; WYUP b ; ADD0 ; ADD1|SUB1 ; NRDD ; RDDA|RDDS - * - * Six instructions, no scratch registers, and -- unlike the library form -- - * on whichever registers the allocator already chose. `mcrr`/`mrrc` move a - * GPR *pair* but place no consecutiveness constraint on it (that is LDRD's - * rule, not theirs), so any two distinct registers work and nothing has to be - * shuffled into r0-r3 first. - * - * Overwriting a source register with the result is safe: WXUP/WYUP have - * already copied both operands into the coprocessor by the time RDDA/RDDS - * writes back. The destination is still excluded from source allocation - * because the shared 64-bit idiom does that, not because it is required. - */ -static void thumb_emit_dcp_addsub_mop(MachineOperand src1, MachineOperand src2, MachineOperand dest, int is_sub) -{ - MachineCodegenContext mctx = {0}; - uint32_t excl = 0; - - /* Destination pair first, so deref resolution never allocates a scratch that - * overlaps it and then restores over the result (same reason as the integer - * 64-bit path). */ - int rd_lo, rd_hi; - bool store_lo = false, store_hi = false; - if (dest.kind == MACH_OP_REG && !dest.needs_deref && dest.u.reg.r0 != (int)PREG_REG_NONE && dest.u.reg.r1 >= 0) - { - rd_lo = dest.u.reg.r0; - rd_hi = dest.u.reg.r1; - excl |= (1u << (uint32_t)rd_lo) | (1u << (uint32_t)rd_hi); - } - else - { - rd_lo = mach_alloc_scratch(&mctx, excl); - excl |= (1u << (uint32_t)rd_lo); - rd_hi = mach_alloc_scratch(&mctx, excl); - excl |= (1u << (uint32_t)rd_hi); - store_lo = store_hi = (dest.kind != MACH_OP_NONE); - } - - /* Pre-exclude both sources' registers so resolving one deref cannot steal - * the physical registers of the other. */ - if (src1.kind == MACH_OP_REG) - { - if (src1.u.reg.r0 != (int)PREG_REG_NONE) - excl |= (1u << (uint32_t)src1.u.reg.r0); - if (!src1.needs_deref && src1.is_64bit && src1.u.reg.r1 >= 0) - excl |= (1u << (uint32_t)src1.u.reg.r1); - } - if (src2.kind == MACH_OP_REG) - { - if (src2.u.reg.r0 != (int)PREG_REG_NONE) - excl |= (1u << (uint32_t)src2.u.reg.r0); - if (!src2.needs_deref && src2.is_64bit && src2.u.reg.r1 >= 0) - excl |= (1u << (uint32_t)src2.u.reg.r1); - } - - MachineOperand a = mach_resolve_deref_64(&mctx, &src1, &excl); - MachineOperand b = mach_resolve_deref_64(&mctx, &src2, &excl); - - int a_lo, a_hi, b_lo, b_hi; - mach_ensure_pair_in_regs(&mctx, &a, &excl, &a_lo, &a_hi); - mach_ensure_pair_in_regs(&mctx, &b, &excl, &b_lo, &b_hi); - - ot_check(th_mcrr(4, 1, (uint32_t)a_lo, (uint32_t)a_hi, 0, 0)); /* WXUP a */ - ot_check(th_mcrr(4, 1, (uint32_t)b_lo, (uint32_t)b_hi, 1, 0)); /* WYUP b */ - ot_check(th_cdp(4, 0, 0, 0, 1, 0, 0)); /* ADD0 */ - ot_check(th_cdp(4, 1, 0, 0, 1, is_sub ? 1 : 0, 0)); /* SUB1 : ADD1 */ - ot_check(th_cdp(4, 8, 0, 0, 0, 1, 0)); /* NRDD */ - ot_check(th_mrrc(4, is_sub ? 3 : 1, (uint32_t)rd_lo, (uint32_t)rd_hi, 0, 0)); /* RDDS : RDDA */ - - if (store_lo) - { - MachineOperand dst_lo = mach_make_lo_half(&dest); - dst_lo.btype = IROP_BTYPE_INT32; - mach_writeback_dest(&dst_lo, rd_lo); - } - if (store_hi) - { - MachineOperand dst_hi = mach_make_hi_half(&dest); - dst_hi.btype = IROP_BTYPE_INT32; - mach_writeback_dest(&dst_hi, rd_hi); - } - mach_release_all(&mctx); -} - -/* thumb_emit_vfp_arith_mop: inline single-precision arithmetic on the FPU. - * - * vmov s0, rn ; vmov s1, rm ; v.f32 s0, s0, s1 ; vmov rd, s0 - * - * Four instructions against a BL plus the library's own five. s0/s1 are used - * as fixed scratch rather than allocated: this is the softfp ABI, so no float - * value ever lives in a VFP register between operations and s0-s15 are free at - * every point. That is also what keeps this independent of Phase 5 proper -- - * making floats *live* in s0-s15 needs a register class and th_vldr/th_vstr - * (which do not exist yet) for spill and reload. - * - * The values move through GPRs on both sides, so nothing here depends on the - * float ABI and fast-mode objects stay link-compatible with portable ones. - */ -/* Get a float operand into a single-precision register: use it directly when it - * already lives in one (MACH_OP_VFP_REG), else materialize it through a GPR into - * the caller-provided scratch s-register. */ -static int vfp_operand_to_sreg(MachineCodegenContext *ctx, const MachineOperand *op, int scratch_s) -{ - if (op->kind == MACH_OP_VFP_REG) - return op->u.reg.r0; - int gpr = mach_ensure_in_reg(ctx, op, 0); - ot_check(th_vmov_gp_sp((uint16_t)gpr, (uint16_t)scratch_s, 0)); /* scratch_s = gpr */ - return scratch_s; -} - -static void thumb_emit_vfp_arith_mop(MachineOperand src1, MachineOperand src2, MachineOperand dest, TccIrOp op) -{ - MachineCodegenContext ctx = {0}; - - /* Hard-float: operate on real single-precision registers. VFP-resident - * operands/dest are used directly (v.f32 sd, sn, sm); GPR/imm/spill - * endpoints are materialized through the reserved s14/s15 scratch. */ - if (tcc_state && tcc_state->float_abi == ARM_HARD_FLOAT) - { - int sn = vfp_operand_to_sreg(&ctx, &src1, VFP_SCRATCH0); - int sm = vfp_operand_to_sreg(&ctx, &src2, VFP_SCRATCH1); - int sd = (dest.kind == MACH_OP_VFP_REG) ? dest.u.reg.r0 : VFP_SCRATCH0; - switch (op) - { - case TCCIR_OP_FADD: ot_check(th_vadd_f((uint16_t)sd, (uint16_t)sn, (uint16_t)sm, 0)); break; - case TCCIR_OP_FSUB: ot_check(th_vsub_f((uint16_t)sd, (uint16_t)sn, (uint16_t)sm, 0)); break; - case TCCIR_OP_FMUL: ot_check(th_vmul_f((uint16_t)sd, (uint16_t)sn, (uint16_t)sm, 0)); break; - case TCCIR_OP_FDIV: ot_check(th_vdiv_f((uint16_t)sd, (uint16_t)sn, (uint16_t)sm, 0)); break; - default: tcc_error("compiler_error: thumb_emit_vfp_arith_mop: unhandled op %d", (int)op); break; - } - if (dest.kind != MACH_OP_VFP_REG) - { - int rd = mach_get_dest_reg(&ctx, &dest, 0); - ot_check(th_vmov_gp_sp((uint16_t)rd, (uint16_t)sd, 1)); /* rd = sd */ - mach_writeback_dest(&dest, rd); - } - mach_release_all(&ctx); - return; - } - - /* Softfp: no float ever lives in a VFP register, so s0/s1 are free fixed - * scratch. Values move through GPRs on both sides — kept byte-identical. */ - int rd = mach_get_dest_reg(&ctx, &dest, 0); - uint32_t excl = (1u << (uint32_t)rd); - int rn = mach_ensure_in_reg(&ctx, &src1, excl); - if (thumb_is_hw_reg(rn)) - excl |= (1u << (uint32_t)rn); - int rm = mach_ensure_in_reg(&ctx, &src2, excl); - - ot_check(th_vmov_gp_sp((uint16_t)rn, 0, 0)); /* s0 = rn */ - ot_check(th_vmov_gp_sp((uint16_t)rm, 1, 0)); /* s1 = rm */ - switch (op) - { - case TCCIR_OP_FADD: - ot_check(th_vadd_f(0, 0, 1, 0)); - break; - case TCCIR_OP_FSUB: - ot_check(th_vsub_f(0, 0, 1, 0)); - break; - case TCCIR_OP_FMUL: - ot_check(th_vmul_f(0, 0, 1, 0)); - break; - case TCCIR_OP_FDIV: - ot_check(th_vdiv_f(0, 0, 1, 0)); - break; - default: - tcc_error("compiler_error: thumb_emit_vfp_arith_mop: unhandled op %d", (int)op); - break; - } - ot_check(th_vmov_gp_sp((uint16_t)rd, 0, 1)); /* rd = s0 */ - - mach_writeback_dest(&dest, rd); - mach_release_all(&ctx); -} - -/* thumb_emit_dcp_cmp_mop: inline double compare on RP2350's DCP. - * - * WXUP a ; WYUP b ; ADD0 ; RCMP apsr_nzcv - * - * Four instructions and no result register: RCMP with Rt == PC writes the - * relation directly into the flags, so the compare feeds an ordinary - * conditional branch instead of a call plus flag decode. - * - * The flag encoding is the AEABI's -- C set means "ordered and >=", Z set - * means equal, V set means unordered -- and is read with *unsigned* - * conditions. ir/gen/float.c already mirrored the operands and recorded - * TOK_UGT / TOK_UGE accordingly, so this only has to compare in operand - * order. - * - * The RCMP is emitted LAST, after mach_release_all(): releasing can restore - * saved scratch registers, and the flags must be the final thing written - * before the branch consumes them. Splitting ADD0 from RCMP is safe because - * the DCP holds its state across unrelated core instructions -- that is the - * same property __rp2350_dcp_save/_restore rely on. - */ -static void thumb_emit_dcp_cmp_mop(MachineOperand src1, MachineOperand src2) -{ - MachineCodegenContext mctx = {0}; - uint32_t excl = 0; - - if (src1.kind == MACH_OP_REG) - { - if (src1.u.reg.r0 != (int)PREG_REG_NONE) - excl |= (1u << (uint32_t)src1.u.reg.r0); - if (!src1.needs_deref && src1.is_64bit && src1.u.reg.r1 >= 0) - excl |= (1u << (uint32_t)src1.u.reg.r1); - } - if (src2.kind == MACH_OP_REG) - { - if (src2.u.reg.r0 != (int)PREG_REG_NONE) - excl |= (1u << (uint32_t)src2.u.reg.r0); - if (!src2.needs_deref && src2.is_64bit && src2.u.reg.r1 >= 0) - excl |= (1u << (uint32_t)src2.u.reg.r1); - } - - MachineOperand a = mach_resolve_deref_64(&mctx, &src1, &excl); - MachineOperand b = mach_resolve_deref_64(&mctx, &src2, &excl); - - int a_lo, a_hi, b_lo, b_hi; - mach_ensure_pair_in_regs(&mctx, &a, &excl, &a_lo, &a_hi); - mach_ensure_pair_in_regs(&mctx, &b, &excl, &b_lo, &b_hi); - - ot_check(th_mcrr(4, 1, (uint32_t)a_lo, (uint32_t)a_hi, 0, 0)); /* WXUP a */ - ot_check(th_mcrr(4, 1, (uint32_t)b_lo, (uint32_t)b_hi, 1, 0)); /* WYUP b */ - ot_check(th_cdp(4, 0, 0, 0, 1, 0, 0)); /* ADD0 */ - - mach_release_all(&mctx); - - ot_check(th_mrc(4, 0, R_PC, 0, 0, 1, 0)); /* RCMP apsr_nzcv */ -} - -/* Process complex double addition/subtraction via MachineOperands. - * (a+bi) + (c+di) = (a+c) + (b+d)i - * (a+bi) - (c+di) = (a-c) + (b-d)i - * Uses __aeabi_dadd/__aeabi_dsub for double-precision. - * Double AEABI calling convention: R0:R1 = arg1, R2:R3 = arg2, result in R0:R1. - */ -static void thumb_process_complex_op_double_mop(MachineOperand src1, MachineOperand src2, MachineOperand dest, - TccIrOp op) -{ - const int is_add = (op == TCCIR_OP_ADD); - const char *func_name = is_add ? "__aeabi_dadd" : "__aeabi_dsub"; - - MachineOperand s1_real = mach_make_complex_real(&src1); - MachineOperand s1_imag = mach_make_complex_imag(&src1); - MachineOperand s2_real = mach_make_complex_real(&src2); - MachineOperand s2_imag = mach_make_complex_imag(&src2); - MachineOperand d_real = mach_make_complex_real(&dest); - MachineOperand d_imag = mach_make_complex_imag(&dest); - - /* Stack layout (32 bytes): - * [sp+24] = s2_imag (8 bytes) - * [sp+16] = s2_real (8 bytes) - * [sp+8] = s1_imag (8 bytes) - * [sp+0] = s1_real (8 bytes) - */ - ot_check(th_sub_imm(R_SP, R_SP, 32, flags_safe(), ENFORCE_ENCODING_NONE)); - - /* Save all 4 components to stack. */ - fp_mop_load_double_arg(R0, R1, &s1_real); - fp_mop_save_double_to_sp(0); - fp_mop_load_double_arg(R0, R1, &s1_imag); - fp_mop_save_double_to_sp(8); - fp_mop_load_double_arg(R0, R1, &s2_real); - fp_mop_save_double_to_sp(16); - fp_mop_load_double_arg(R0, R1, &s2_imag); - fp_mop_save_double_to_sp(24); - - /* Compute real part: func(a.real, b.real) */ - fp_mop_load_double_from_sp(R0, R1, 0); - fp_mop_load_double_from_sp(R2, R3, 16); - fp_mop_do_bl(func_name); - /* Save real result to stack slot 0 */ - fp_mop_save_double_to_sp(0); - - /* Compute imag part: func(a.imag, b.imag) */ - fp_mop_load_double_from_sp(R0, R1, 8); - fp_mop_load_double_from_sp(R2, R3, 24); - fp_mop_do_bl(func_name); - /* R0:R1 = imag result. Load real result from stack. */ - fp_mop_save_double_to_sp(8); /* save imag to slot 8 */ - - /* Write results back to dest. */ - fp_mop_load_double_from_sp(R0, R1, 0); - fp_mop_writeback_result(&d_real, 1); - fp_mop_load_double_from_sp(R0, R1, 8); - fp_mop_writeback_result(&d_imag, 1); - - ot_check(th_add_imm(R_SP, R_SP, 32, flags_safe(), ENFORCE_ENCODING_NONE)); -} - -/* Process complex addition/subtraction via MachineOperands. - * (a+bi) + (c+di) = (a+c) + (b+d)i - * (a+bi) - (c+di) = (a-c) + (b-d)i - */ -static void thumb_process_complex_op_mop(MachineOperand src1, MachineOperand src2, MachineOperand dest, TccIrOp op) -{ - const int is_add = (op == TCCIR_OP_ADD); - - /* Complex float: each component is a 32-bit float. */ - const char *func_name = is_add ? "__aeabi_fadd" : "__aeabi_fsub"; - - /* Split into real/imag components. */ - MachineOperand s1_real = mach_make_lo_half(&src1); - MachineOperand s1_imag = mach_make_hi_half(&src1); - MachineOperand s2_real = mach_make_lo_half(&src2); - MachineOperand s2_imag = mach_make_hi_half(&src2); - - /* Stack-based: save all 4 inputs, do calls, write results back. - * Stack layout (16 bytes): - * [sp+12] = s2_imag - * [sp+8] = s2_real - * [sp+4] = s1_imag - * [sp+0] = s1_real - */ - ot_check(th_sub_imm(R_SP, R_SP, 16, flags_safe(), ENFORCE_ENCODING_NONE)); - - /* Load and save each component to stack. */ - fp_mop_load_arg(R0, &s1_real); - ot_check_str_imm(R0, R_SP, 0, 6, ENFORCE_ENCODING_NONE); - fp_mop_load_arg(R0, &s1_imag); - ot_check_str_imm(R0, R_SP, 4, 6, ENFORCE_ENCODING_NONE); - fp_mop_load_arg(R0, &s2_real); - ot_check_str_imm(R0, R_SP, 8, 6, ENFORCE_ENCODING_NONE); - fp_mop_load_arg(R0, &s2_imag); - ot_check_str_imm(R0, R_SP, 12, 6, ENFORCE_ENCODING_NONE); - - /* Compute real part: func(a.real, b.real) */ - ot_check_ldr_imm(R0, R_SP, 0, 6, ENFORCE_ENCODING_NONE); - ot_check_ldr_imm(R1, R_SP, 8, 6, ENFORCE_ENCODING_NONE); - fp_mop_do_bl(func_name); - /* Save real result to stack slot 0 */ - ot_check_str_imm(R0, R_SP, 0, 6, ENFORCE_ENCODING_NONE); - - /* Compute imag part: func(a.imag, b.imag) */ - ot_check_ldr_imm(R0, R_SP, 4, 6, ENFORCE_ENCODING_NONE); - ot_check_ldr_imm(R1, R_SP, 12, 6, ENFORCE_ENCODING_NONE); - fp_mop_do_bl(func_name); - /* R0 = imag result */ - - /* Load real result from stack, deallocate, write back. */ - MachineOperand d_real = mach_make_lo_half(&dest); - MachineOperand d_imag = mach_make_hi_half(&dest); - - /* R0 = imag result. Load real result from stack into R1. */ - ot_check_ldr_imm(R1, R_SP, 0, 6, ENFORCE_ENCODING_NONE); - ot_check(th_add_imm(R_SP, R_SP, 16, flags_safe(), ENFORCE_ENCODING_NONE)); - - /* Write back: R1 = real part, R0 = imag part. - * Use safe writeback to avoid clobbering when dest overlaps R0/R1. */ - complex_pair_writeback(&d_real, R1, &d_imag, R0); -} - -/* Process complex multiplication via MachineOperands. - * (a+bi) * (c+di) = (ac-bd) + (ad+bc)i - * - * Stack layout (24 bytes): - * [sp+20] = d (imag of src2) - * [sp+16] = c (real of src2) - * [sp+12] = b (imag of src1) - * [sp+8] = a (real of src1) - * [sp+4] = scratch1 - * [sp+0] = scratch0 - */ -static void thumb_process_complex_mul_mop(MachineOperand src1, MachineOperand src2, MachineOperand dest) -{ - MachineOperand s1_real = mach_make_lo_half(&src1); - MachineOperand s1_imag = mach_make_hi_half(&src1); - MachineOperand s2_real = mach_make_lo_half(&src2); - MachineOperand s2_imag = mach_make_hi_half(&src2); - - /* Allocate 24 bytes on stack */ - ot_check(th_sub_imm(R_SP, R_SP, 24, flags_safe(), ENFORCE_ENCODING_NONE)); - - /* Save inputs to stack */ - fp_mop_load_arg(R0, &s1_real); - ot_check_str_imm(R0, R_SP, 8, 6, ENFORCE_ENCODING_NONE); /* a */ - fp_mop_load_arg(R0, &s1_imag); - ot_check_str_imm(R0, R_SP, 12, 6, ENFORCE_ENCODING_NONE); /* b */ - fp_mop_load_arg(R0, &s2_real); - ot_check_str_imm(R0, R_SP, 16, 6, ENFORCE_ENCODING_NONE); /* c */ - fp_mop_load_arg(R0, &s2_imag); - ot_check_str_imm(R0, R_SP, 20, 6, ENFORCE_ENCODING_NONE); /* d */ - - const int off_scratch0 = 0; - const int off_scratch1 = 4; - const int off_a = 8; - const int off_b = 12; - const int off_c = 16; - const int off_d = 20; - - /* Step 1: ac = a * c → scratch0 */ - ot_check_ldr_imm(R0, R_SP, off_a, 6, ENFORCE_ENCODING_NONE); - ot_check_ldr_imm(R1, R_SP, off_c, 6, ENFORCE_ENCODING_NONE); - fp_mop_do_bl("__aeabi_fmul"); - ot_check_str_imm(R0, R_SP, off_scratch0, 6, ENFORCE_ENCODING_NONE); - - /* Step 2: bd = b * d → scratch1 */ - ot_check_ldr_imm(R0, R_SP, off_b, 6, ENFORCE_ENCODING_NONE); - ot_check_ldr_imm(R1, R_SP, off_d, 6, ENFORCE_ENCODING_NONE); - fp_mop_do_bl("__aeabi_fmul"); - ot_check_str_imm(R0, R_SP, off_scratch1, 6, ENFORCE_ENCODING_NONE); - - /* Step 3: real = ac - bd → scratch0 */ - ot_check_ldr_imm(R0, R_SP, off_scratch0, 6, ENFORCE_ENCODING_NONE); - ot_check_ldr_imm(R1, R_SP, off_scratch1, 6, ENFORCE_ENCODING_NONE); - fp_mop_do_bl("__aeabi_fsub"); - ot_check_str_imm(R0, R_SP, off_scratch0, 6, ENFORCE_ENCODING_NONE); - - /* Step 4: ad = a * d → scratch1 */ - ot_check_ldr_imm(R0, R_SP, off_a, 6, ENFORCE_ENCODING_NONE); - ot_check_ldr_imm(R1, R_SP, off_d, 6, ENFORCE_ENCODING_NONE); - fp_mop_do_bl("__aeabi_fmul"); - ot_check_str_imm(R0, R_SP, off_scratch1, 6, ENFORCE_ENCODING_NONE); - - /* Step 5: bc = b * c → off_a (no longer needed) */ - ot_check_ldr_imm(R0, R_SP, off_b, 6, ENFORCE_ENCODING_NONE); - ot_check_ldr_imm(R1, R_SP, off_c, 6, ENFORCE_ENCODING_NONE); - fp_mop_do_bl("__aeabi_fmul"); - ot_check_str_imm(R0, R_SP, off_a, 6, ENFORCE_ENCODING_NONE); - - /* Step 6: imag = ad + bc → scratch1 */ - ot_check_ldr_imm(R0, R_SP, off_scratch1, 6, ENFORCE_ENCODING_NONE); - ot_check_ldr_imm(R1, R_SP, off_a, 6, ENFORCE_ENCODING_NONE); - fp_mop_do_bl("__aeabi_fadd"); - ot_check_str_imm(R0, R_SP, off_scratch1, 6, ENFORCE_ENCODING_NONE); - - /* Load results and write back */ - MachineOperand d_real = mach_make_lo_half(&dest); - MachineOperand d_imag = mach_make_hi_half(&dest); - ot_check_ldr_imm(R0, R_SP, off_scratch0, 6, ENFORCE_ENCODING_NONE); /* real */ - ot_check_ldr_imm(R1, R_SP, off_scratch1, 6, ENFORCE_ENCODING_NONE); /* imag */ - ot_check(th_add_imm(R_SP, R_SP, 24, flags_safe(), ENFORCE_ENCODING_NONE)); - - complex_pair_writeback(&d_real, R0, &d_imag, R1); -} - -/* Process complex float division via MachineOperands. - * Calls __divsc3 from libgcc for numerically robust division. - * - * __divsc3 calling convention (soft-float AAPCS, hidden return pointer): - * R0 = hidden return pointer (8-byte buffer for result) - * R1 = a_real (first float arg) - * R2 = a_imag (second float arg) - * R3 = b_real (third float arg) - * [sp+0] = b_imag (fourth float arg, on stack) - * Result written to [R0+0..3] = real, [R0+4..7] = imag - * - * Stack layout (16 bytes): - * [sp+0] = b_imag for __divsc3 stack arg (4 bytes) - * [sp+4] = padding (4 bytes) - * [sp+8] = result buffer: real part (4 bytes) - * [sp+12] = result buffer: imag part (4 bytes) - */ -static void thumb_process_complex_div_mop(MachineOperand src1, MachineOperand src2, MachineOperand dest) -{ - MachineOperand s1_real = mach_make_lo_half(&src1); - MachineOperand s1_imag = mach_make_hi_half(&src1); - MachineOperand s2_real = mach_make_lo_half(&src2); - MachineOperand s2_imag = mach_make_hi_half(&src2); - - /* In PIC mode, save {r9, r12} BEFORE allocating the call frame so that - * SP-relative offsets within the 16-byte area remain correct when - * __divsc3 reads its stack arg at [sp+0]. */ - if (text_and_data_separation) - ot_check(th_push((uint16_t)((1 << R9) | (1 << R12)))); - - /* Allocate 16 bytes on stack. */ - ot_check(th_sub_imm(R_SP, R_SP, 16, flags_safe(), ENFORCE_ENCODING_NONE)); - - /* Save all inputs to stack first to avoid register clobbering. */ - fp_mop_load_arg(R0, &s1_real); - ot_check_str_imm(R0, R_SP, 0, 6, ENFORCE_ENCODING_NONE); /* a_real */ - fp_mop_load_arg(R0, &s1_imag); - ot_check_str_imm(R0, R_SP, 4, 6, ENFORCE_ENCODING_NONE); /* a_imag */ - fp_mop_load_arg(R0, &s2_real); - ot_check_str_imm(R0, R_SP, 8, 6, ENFORCE_ENCODING_NONE); /* b_real */ - fp_mop_load_arg(R0, &s2_imag); - ot_check_str_imm(R0, R_SP, 12, 6, ENFORCE_ENCODING_NONE); /* b_imag */ - - /* Rearrange stack for __divsc3 call: - * Need [sp+0] = b_imag, [sp+8..15] = result buffer. - * Currently [sp+0]=a_real, [sp+4]=a_imag, [sp+8]=b_real, [sp+12]=b_imag. - * Load R1-R3 from stack, then rearrange. */ - ot_check_ldr_imm(R1, R_SP, 0, 6, ENFORCE_ENCODING_NONE); /* R1 = a_real */ - ot_check_ldr_imm(R2, R_SP, 4, 6, ENFORCE_ENCODING_NONE); /* R2 = a_imag */ - ot_check_ldr_imm(R3, R_SP, 8, 6, ENFORCE_ENCODING_NONE); /* R3 = b_real */ - ot_check_ldr_imm(R0, R_SP, 12, 6, ENFORCE_ENCODING_NONE); /* R0 = b_imag */ - ot_check_str_imm(R0, R_SP, 0, 6, ENFORCE_ENCODING_NONE); /* [sp+0] = b_imag (stack arg) */ - - /* R0 = pointer to result buffer at [sp+8]. */ - ot_check(th_add_imm(R0, R_SP, 8, flags_safe(), ENFORCE_ENCODING_NONE)); - - /* Call __divsc3 (directly, not via fp_mop_do_bl which would add another - * push/pop of {r9, r12} and corrupt the stack arg layout). */ - { - Sym *sym = external_global_sym(tok_alloc_const("__divsc3"), &func_old_type); - MachineOperand func_mop = {0}; - func_mop.kind = MACH_OP_SYMBOL; - func_mop.u.sym.sym = sym; - func_mop.u.sym.addend = 0; - gcall_or_jump_mop(0, func_mop); - } - - /* Read result from buffer and write back to dest. */ - MachineOperand d_real = mach_make_lo_half(&dest); - MachineOperand d_imag = mach_make_hi_half(&dest); - ot_check_ldr_imm(R0, R_SP, 8, 6, ENFORCE_ENCODING_NONE); /* real */ - ot_check_ldr_imm(R1, R_SP, 12, 6, ENFORCE_ENCODING_NONE); /* imag */ - ot_check(th_add_imm(R_SP, R_SP, 16, flags_safe(), ENFORCE_ENCODING_NONE)); - - if (text_and_data_separation) - ot_check(th_pop((uint16_t)((1 << R9) | (1 << R12)))); - - complex_pair_writeback(&d_real, R0, &d_imag, R1); -} - -/* Process complex double division via MachineOperands. - * Calls __divdc3 from libgcc for numerically robust division. - * - * __divdc3 calling convention (soft-float AAPCS, hidden return pointer): - * R0 = hidden return pointer (16-byte buffer for result) - * R2:R3 = a_re (first double, even-aligned) - * [sp+0] = a_im (second double, on stack) - * [sp+8] = b_re (third double, on stack) - * [sp+16] = b_im (fourth double, on stack) - * Result written to [R0+0..7] = real, [R0+8..15] = imag - * - * Stack layout (40 bytes, 8-byte aligned): - * [sp+0] = a_im for __divdc3 stack arg (8 bytes) - * [sp+8] = b_re for __divdc3 stack arg (8 bytes) - * [sp+16] = b_im for __divdc3 stack arg (8 bytes) - * [sp+24] = result buffer: real part (8 bytes) - * [sp+32] = result buffer: imag part (8 bytes) - */ -static void thumb_process_complex_div_double_mop(MachineOperand src1, MachineOperand src2, MachineOperand dest) -{ - MachineOperand s1_real = mach_make_complex_real(&src1); - MachineOperand s1_imag = mach_make_complex_imag(&src1); - MachineOperand s2_real = mach_make_complex_real(&src2); - MachineOperand s2_imag = mach_make_complex_imag(&src2); - MachineOperand d_real = mach_make_complex_real(&dest); - MachineOperand d_imag = mach_make_complex_imag(&dest); - - /* In PIC mode, save {r9, r12} BEFORE allocating the call frame so that - * SP-relative offsets within the 40-byte area remain correct when - * __divdc3 reads its stack args at [sp+0..23]. */ - if (text_and_data_separation) - ot_check(th_push((uint16_t)((1 << R9) | (1 << R12)))); - - /* Allocate 40 bytes (8-byte aligned). */ - ot_check(th_sub_imm(R_SP, R_SP, 40, flags_safe(), ENFORCE_ENCODING_NONE)); - - /* Set up __divdc3 stack args (must be at lowest sp offsets). */ - /* [sp+16] = b_im (src2 imag). */ - fp_mop_load_double_arg(R0, R1, &s2_imag); - fp_mop_save_double_to_sp(16); - /* [sp+8] = b_re (src2 real). */ - fp_mop_load_double_arg(R0, R1, &s2_real); - fp_mop_save_double_to_sp(8); - /* [sp+0] = a_im (src1 imag). */ - fp_mop_load_double_arg(R0, R1, &s1_imag); - fp_mop_save_double_to_sp(0); - - /* R2:R3 = a_re (src1 real) — first double arg in even register pair. */ - fp_mop_load_double_arg(R2, R3, &s1_real); - - /* R0 = pointer to result buffer at [sp+24]. */ - ot_check(th_add_imm(R0, R_SP, 24, flags_safe(), ENFORCE_ENCODING_NONE)); - - /* Call __divdc3 (directly, not via fp_mop_do_bl which would add another - * push/pop of {r9, r12} and corrupt the stack arg layout). */ - { - Sym *sym = external_global_sym(tok_alloc_const("__divdc3"), &func_old_type); - MachineOperand func_mop = {0}; - func_mop.kind = MACH_OP_SYMBOL; - func_mop.u.sym.sym = sym; - func_mop.u.sym.addend = 0; - gcall_or_jump_mop(0, func_mop); - } - - /* Read result from buffer and write back to dest. */ - fp_mop_load_double_from_sp(R0, R1, 24); - fp_mop_writeback_result(&d_real, 1); - fp_mop_load_double_from_sp(R0, R1, 32); - fp_mop_writeback_result(&d_imag, 1); - - ot_check(th_add_imm(R_SP, R_SP, 40, flags_safe(), ENFORCE_ENCODING_NONE)); - - if (text_and_data_separation) - ot_check(th_pop((uint16_t)((1 << R9) | (1 << R12)))); -} - -/* tcc_gen_machine_fp_mop: MachineOperand-based entry point for floating-point - * operations via soft-float EABI library calls. - * Handles single-precision, double-precision, and complex float operations. - * - * Soft-float EABI calling convention (single-precision): - * binary arithmetic: src1 → R0, src2 → R1, result ← R0 - * comparison: src1 → R0, src2 → R1, result ← CPSR flags - * negation: src1 → R0, XOR sign bit, result ← R0 - * conversion: src1 → R0, result ← R0 - * CVT_FTOF identity: float32→float32, src1 → R0, dest ← R0 - */ -ST_FUNC void tcc_gen_machine_fp_mop(MachineOperand src1, MachineOperand src2, MachineOperand dest, TccIrOp op, - int is_complex) -{ - /* Phase 5k: handle complex float operations via MOP path. */ - if (is_complex) - { - /* Detect double-precision complex: any operand has FLOAT64 btype. */ - const int complex_is_double = - (src1.btype == IROP_BTYPE_FLOAT64 || src2.btype == IROP_BTYPE_FLOAT64 || dest.btype == IROP_BTYPE_FLOAT64); - if (op == TCCIR_OP_FADD || op == TCCIR_OP_FSUB) - { - if (complex_is_double) - return thumb_process_complex_op_double_mop(src1, src2, dest, op == TCCIR_OP_FADD ? TCCIR_OP_ADD : TCCIR_OP_SUB); - return thumb_process_complex_op_mop(src1, src2, dest, op == TCCIR_OP_FADD ? TCCIR_OP_ADD : TCCIR_OP_SUB); - } - else if (op == TCCIR_OP_FMUL) - { - if (complex_is_double) - return thumb_process_complex_mul_double_mop(src1, src2, dest); - return thumb_process_complex_mul_mop(src1, src2, dest); - } - else if (op == TCCIR_OP_FDIV) - { - if (complex_is_double) - return thumb_process_complex_div_double_mop(src1, src2, dest); - return thumb_process_complex_div_mop(src1, src2, dest); - } - /* Other ops (FNEG, FCMP, CVT_*) on complex types: fall through to - * scalar path — they operate componentwise on the lo (real) half only, - * same as regular scalars. TODO: extend if needed. */ - } - - /* is_double: true when the primary operand is a 64-bit float (double). - * Note: complex float has is_64bit=true (register pair), but its btype - * is FLOAT32, so it is NOT double. Only FLOAT64 btype is true double. */ - const int is_double = (src1.btype == IROP_BTYPE_FLOAT64) || (dest.btype == IROP_BTYPE_FLOAT64); - const char *func_name = NULL; - - /* --- Inline double lowering, where the target has it --- - * - * Reaching here with a has_d* bit set means ir_put_soft_call_fpu_if_needed() - * deliberately did NOT rewrite this operation into an __aeabi_ call, so the - * backend owes an inline sequence. Falling through to the call path would - * still produce working code, but ir_op_is_implicit_call_ra() keys its - * clobber model on the same bits -- so the two must agree or the allocator - * stops modelling r0-r3 as dead across an operation that still calls. - * Enable a has_d* bit and its emitter in the same commit. */ - { - const FloatingPointConfig *fpu = architecture_config.fpu; - if (is_double && fpu && fpu->double_impl == FP_DOUBLE_IMPL_DCP) - { - if (op == TCCIR_OP_FADD && fpu->has_dadd) - return thumb_emit_dcp_addsub_mop(src1, src2, dest, 0); - if (op == TCCIR_OP_FSUB && fpu->has_dsub) - return thumb_emit_dcp_addsub_mop(src1, src2, dest, 1); - if (op == TCCIR_OP_FCMP && fpu->has_dcmp) - return thumb_emit_dcp_cmp_mop(src1, src2); - } - - /* Single precision is plain VFP wherever it is inline at all -- there is - * no second single-precision implementation on ARM to discriminate - * between, so has_f* alone selects the sequence. - * - * -mfloat-abi=soft is the one mode that forbids it: soft means "emit no - * FPU instructions at all", as distinct from softfp, which is FPU - * instructions with GPR argument passing (the default here, and what - * -mfpu=rp2350 keeps). has_f* describes the silicon, not the ABI, so the - * ABI has to be checked separately or `-mfloat-abi=soft` starts emitting - * vadd.f32. */ - if (!is_double && fpu && tcc_state && tcc_state->float_abi != ARM_SOFT_FLOAT) - { - if ((op == TCCIR_OP_FADD && fpu->has_fadd) || (op == TCCIR_OP_FSUB && fpu->has_fsub) || - (op == TCCIR_OP_FMUL && fpu->has_fmul) || (op == TCCIR_OP_FDIV && fpu->has_fdiv)) - return thumb_emit_vfp_arith_mop(src1, src2, dest, op); - } - } - - /* --- FNEG: XOR sign bit, no BL needed --- */ - if (op == TCCIR_OP_FNEG) - { - ScratchRegAlloc scr; - if (is_double) - { - /* f64: load pair into R0:R1, flip sign bit of hi word (R1) only */ - fp_mop_load_double_arg(R0, R1, &src1); - scr = get_scratch_reg_with_save((1u << R0) | (1u << R1)); - load_full_const(scr.reg, PREG_NONE, 0x80000000, 0); - ot_check(th_eor_reg(R1, R1, scr.reg, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); - restore_scratch_reg(&scr); - fp_mop_writeback_result(&dest, 1); - } - else - { - /* f32: R0 ^= 0x80000000 */ - fp_mop_load_arg(R0, &src1); - scr = get_scratch_reg_with_save(1u << R0); - load_full_const(scr.reg, PREG_NONE, 0x80000000, 0); - ot_check(th_eor_reg(R0, R0, scr.reg, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); - restore_scratch_reg(&scr); - mach_writeback_dest(&dest, R0); - } - return; - } - - /* --- CVT_FTOF: identity or f32<->f64 conversion via BL --- */ - if (op == TCCIR_OP_CVT_FTOF) - { - const int src_double = src1.is_64bit; - const int dst_double = dest.is_64bit; - if (!src_double && !dst_double) - { - /* f32 -> f32 identity: direct copy without going through R0 */ - tcc_gen_machine_assign_mop(src1, dest, TCCIR_OP_ASSIGN); - return; - } - if (src_double && dst_double) - { - /* f64 -> f64 identity: direct copy without going through R0:R1. - * Using R0:R1 as intermediaries would clobber live values in those - * registers (e.g. function parameters in soft-float ABI). */ - tcc_gen_machine_assign_mop(src1, dest, TCCIR_OP_ASSIGN); - return; - } - /* f32 -> f64: __aeabi_f2d; f64 -> f32: __aeabi_d2f */ - { - const char *cvt_func = src_double ? "__aeabi_d2f" : "__aeabi_f2d"; - if (src_double) - fp_mop_load_double_arg(R0, R1, &src1); - else - fp_mop_load_arg(R0, &src1); - fp_mop_do_bl(cvt_func); - fp_mop_writeback_result(&dest, dst_double); - } - return; - } - - /* --- Load operands into argument registers --- */ - if (op == TCCIR_OP_FCMP || op == TCCIR_OP_FADD || op == TCCIR_OP_FSUB || op == TCCIR_OP_FMUL || op == TCCIR_OP_FDIV) - { - /* Binary: src1 first arg, src2 second arg */ - if (is_double) - { - fp_mop_load_double_arg(R0, R1, &src1); - fp_mop_load_double_arg(R2, R3, &src2); - } - else - { - fp_mop_load_arg(R0, &src1); - fp_mop_load_arg(R1, &src2); - } - } - else - { - /* Unary conversion (CVT_ITOF, CVT_FTOI): load src1 into R0 or R0:R1 */ - if (src1.is_64bit) - fp_mop_load_double_arg(R0, R1, &src1); - else - fp_mop_load_arg(R0, &src1); - } - - /* --- Determine soft-float function name --- */ - if (op == TCCIR_OP_FCMP) - { - func_name = is_double ? "__aeabi_cdcmple" : "__aeabi_cfcmple"; - } - else if (op == TCCIR_OP_CVT_ITOF) - { - const int src64 = src1.is_64bit; - const int dst64 = dest.is_64bit; - if (src64 && dst64) - func_name = src1.is_unsigned ? "__aeabi_ul2d" : "__aeabi_l2d"; - else if (src64) - func_name = src1.is_unsigned ? "__aeabi_ul2f" : "__aeabi_l2f"; - else if (dst64) - func_name = src1.is_unsigned ? "__aeabi_ui2d" : "__aeabi_i2d"; - else - func_name = src1.is_unsigned ? "__aeabi_ui2f" : "__aeabi_i2f"; - } - else if (op == TCCIR_OP_CVT_FTOI) - { - const int src64 = src1.is_64bit; - const int dst64 = dest.is_64bit; - if (src64 && dst64) - func_name = dest.is_unsigned ? "__aeabi_d2ulz" : "__aeabi_d2lz"; - else if (src64) - func_name = dest.is_unsigned ? "__aeabi_d2uiz" : "__aeabi_d2iz"; - else if (dst64) - func_name = dest.is_unsigned ? "__aeabi_f2ulz" : "__aeabi_f2lz"; - else - func_name = dest.is_unsigned ? "__aeabi_f2uiz" : "__aeabi_f2iz"; - } - else - { - /* FADD, FSUB, FMUL, FDIV */ - func_name = get_softfp_func_name(op, is_double); - } - - if (!func_name) - tcc_error("compiler_error: tcc_gen_machine_fp_mop: no func_name for op %d", (int)op); - - fp_mop_do_bl(func_name); - - /* Write result back (FCMP sets CPSR flags only -- no register result) */ - if (op != TCCIR_OP_FCMP) - fp_mop_writeback_result(&dest, dest.is_64bit); -} - -/* tcc_gen_machine_return_value_mop: MachineOperand-based entry point for - * function return. Moves src into the return register(s) using the most - * efficient sequence available: - * 32-bit: src → R0 - * - Already R0 (common after ASSIGN peephole): NOP, 0 scratch. - * - IMM / SYMBOL: load directly into R0, 0 scratch. - * - REG / SPILL / FRAME_ADDR / PARAM_STACK: mach_ensure_in_reg + optional MOV. - * 64-bit: lo → R0 (REG_IRET), hi → R1 (REG_IRE2) - * - Delegates to tcc_gen_machine_assign_mop with a synthetic R0:R1 dest. - * - Safe ordering guaranteed: AAPCS ensures hi is never in R0. - */ -ST_FUNC void tcc_gen_machine_return_value_mop(MachineOperand src, TccIrOp op) -{ - (void)op; - - /* Hard-float single-precision return: the value goes in s0 (AAPCS VFP). */ - if (tcc_state && tcc_state->float_abi == ARM_HARD_FLOAT && !src.is_64bit && src.btype == IROP_BTYPE_FLOAT32) - { - if (src.kind == MACH_OP_VFP_REG) - { - if (src.u.reg.r0 != 0) - ot_check(th_vmov_register(0, (uint16_t)src.u.reg.r0, 0)); /* s0 = s_src */ - } - else - { - MachineCodegenContext ctx = {0}; - int r = mach_ensure_in_reg(&ctx, &src, 0); - ot_check(th_vmov_gp_sp((uint16_t)r, 0, 0)); /* s0 = r */ - mach_release_all(&ctx); - } - return; - } - - /* Hard-float double return: the value goes in d0 (AAPCS VFP). Doubles are - * not VFP-resident here, so materialize the GPR pair and pack it into d0. */ - if (tcc_state && tcc_state->float_abi == ARM_HARD_FLOAT && src.is_64bit && src.btype == IROP_BTYPE_FLOAT64 && - !src.is_complex) - { - MachineCodegenContext ctx = {0}; - MachineOperand lo = mach_make_lo_half(&src); - MachineOperand hi = mach_make_hi_half(&src); - lo.btype = IROP_BTYPE_INT32; - hi.btype = IROP_BTYPE_INT32; - int rlo = mach_ensure_in_reg(&ctx, &lo, 0); - int rhi = mach_ensure_in_reg(&ctx, &hi, (1u << (uint32_t)rlo)); - ot_check(th_vmov_2gp_dp((uint16_t)rlo, (uint16_t)rhi, 0, 0)); /* d0 = rlo,rhi */ - mach_release_all(&ctx); - return; - } - - /* 64-bit return: lo word → R0 (REG_IRET), hi word → R1 (REG_IRE2). - * AAPCS guarantees that for a 64-bit pair src.u.reg.r1 = src.u.reg.r0 + 1 ≥ R1, - * so hi is never in R0. Moving lo→R0 first is always safe. - * delegate to assign_mop which handles REG/SPILL/IMM/SYMBOL src kinds. */ - if (src.is_64bit) - { - MachineOperand ret_pair; - memset(&ret_pair, 0, sizeof(ret_pair)); - ret_pair.kind = MACH_OP_REG; - ret_pair.u.reg.r0 = TREG_R0; /* REG_IRET */ - ret_pair.u.reg.r1 = TREG_R1; /* REG_IRE2 */ - ret_pair.is_64bit = true; - ret_pair.btype = src.btype; - tcc_gen_machine_assign_mop(src, ret_pair, op); - return; - } - - /* Fast path: value already in R0 (ASSIGN peephole sets dest→R0) */ - if (src.kind == MACH_OP_REG && !src.needs_deref && src.u.reg.r0 == R0) - return; - - /* Immediate: materialize directly into R0 (no scratch register needed) */ - if (src.kind == MACH_OP_IMM) - { - tcc_machine_load_constant(R0, PREG_NONE, src.u.imm.val, 0, NULL); - return; - } - - /* Symbol: load address (+ optional deref) directly into R0 */ - if (src.kind == MACH_OP_SYMBOL) - { - Sym *sym = src.u.sym.sym ? validate_sym_for_reloc(src.u.sym.sym) : NULL; - if (!src.needs_deref) - { - tcc_machine_load_constant(R0, PREG_NONE, src.u.sym.addend, 0, sym); - } - else - { - tcc_machine_load_constant(R0, PREG_NONE, 0, 0, sym); - const int32_t addend = src.u.sym.addend; - load_from_base(R0, PREG_REG_NONE, src.btype, (int)src.is_unsigned, addend < 0 ? (int)(-addend) : (int)addend, - addend < 0 ? 1 : 0, (uint32_t)R0); - } - return; - } - - /* General case: materialize into any available register, then MOV to R0 */ - MachineCodegenContext ctx = {0}; - int src_reg = mach_ensure_in_reg(&ctx, &src, 0); - if (src_reg != R0) - ot_check_mov_reg(R0, src_reg, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE, false); - mach_release_all(&ctx); -} - -ST_FUNC void tcc_gen_machine_prolog(int leaffunc, uint64_t used_registers, int stack_size, uint32_t extra_prologue_regs) -{ - thumb_gen_state.function_argument_count = 0; - /* call_id -1 is reserved for function prolog metadata - but that doesn't - * need to be stored in call_sites_by_id (which uses non-negative IDs). - * If needed, handle it separately or skip. */ - - thumb_gen_state.generating_function = 1; - thumb_gen_state.code_size = 0; - /* Clear global symbol cache at function start */ - thumb_gen_state.cached_global_sym = NULL; - thumb_gen_state.cached_global_reg = PREG_NONE; - /* MOV-coalescing cache is per-function: register live ranges don't - * cross function boundaries. */ - mov_equiv_reset_all(); - pool_flush_it_pending = 0; - TCCIRState *ir = tcc_state->ir; - - /* Determine if LR needs saving */ - int save_lr = !leaffunc || tcc_state->force_lr_save; - if (extra_prologue_regs & (1u << R_LR)) - save_lr = 1; - - /* Variadic functions need a stable FP for va_list setup. */ - if (func_var) - tcc_state->need_frame_pointer = 1; - - /* Also force FP when force_lr_save is set (builtin_return_address). */ - if (tcc_state->force_lr_save) - tcc_state->need_frame_pointer = 1; - - const int need_fp = (tcc_state->force_frame_pointer || tcc_state->need_frame_pointer); - tcc_state->need_frame_pointer = need_fp; - - /* Use two-phase push (standard frame record) when __builtin_return_address - * needs a predictable {FP, LR} layout at [FP+0] and [FP+4]. */ - const int standard_frame_record = need_fp && tcc_state->force_lr_save; - - /* The allocator hands out R7 only when ra_may_need_frame_pointer predicted - * no frame pointer; a function that reaches here needing FP with R7 - * allocated means the prediction missed a forcing condition, and silently - * continuing would use one register as frame base and value at once. */ - if (need_fp && (used_registers & (1ULL << R_FP))) - tcc_error("compiler_error: R7 allocated in a frame-pointer function " - "(ra_may_need_frame_pointer out of sync with a forcing site)"); - - /* Collect callee-saved registers */ - uint16_t callee_regs_local = 0; - int callee_count = 0; - for (int i = R4; i <= R11; ++i) - { - if (tcc_state->text_and_data_separation && i == R9) - continue; - if (i == R_FP && need_fp) - continue; /* r7 is the frame base; pushed via frame_regs instead */ - if (used_registers & (1ULL << i)) - { - callee_regs_local |= (1 << i); - callee_count++; - } - } - - /* Registers codegen asks to be saved on top of the allocation: the static - * chain register (R10) for nested functions, and the rodata anchor. */ - for (int i = R4; i <= R11; ++i) - { - if (!(extra_prologue_regs & (1u << i))) - continue; - if (tcc_state->text_and_data_separation && i == R9) - continue; - if (i == R_FP && need_fp) - continue; - if (callee_regs_local & (1u << i)) - continue; - callee_regs_local |= (1u << i); - callee_count++; - } - - int push_align_pad = 0; /* 4 if push count is odd, absorbed into SUB SP */ - - if (standard_frame_record) - { - /* ── Two-phase push: frame record {r7, lr} then callee-saved ── - * Layout: [FP+0]=old_FP, [FP+4]=LR, callee-saved below FP. */ - uint16_t frame_regs = (1 << R_FP); - int frame_count = 1; - if (save_lr) - { - frame_regs |= (1 << R_LR); - frame_count++; - } - - /* Pad total to even count for 8-byte alignment (AAPCS). - * Standard frame record uses FP-relative negative offsets (e.g. static - * chain at [FP-4]), so the alignment gap must stay as SUB SP space - * below FP — cannot use a dummy push register here. */ - int total = frame_count + callee_count; - if (total % 2 != 0) - { - push_align_pad = 4; - } - - map_sym_t(); - - /* Variadic: push r0-r3 FIRST so they are contiguous with stack args */ - vararg_push_size = 0; - if (func_var) - { - ot_check(th_push((1 << R0) | (1 << R1) | (1 << R2) | (1 << R3))); - vararg_push_size = 16; - } - - /* Phase A: push frame record */ - ot_check(th_push(frame_regs)); - - /* MOV r7, sp — FP points at the frame record */ - if (!ot(th_add_imm(R_FP, R_SP, 0, flags_safe(), ENFORCE_ENCODING_NONE))) - { - fprintf(stderr, "compiler_error: prolog frame pointer setup failed\n"); - exit(1); - } - - /* Phase B: push callee-saved regs (below FP) */ - if (callee_count > 0) - ot_check(th_push(callee_regs_local)); - - callee_push_size = callee_count * 4; - callee_saved_regs = callee_regs_local; - offset_to_args = frame_count * 4 + vararg_push_size; - pushed_registers = frame_regs | callee_regs_local; - } - else - { - /* ── Original single-push layout ── */ - uint16_t registers_to_push = callee_regs_local; - int registers_count = callee_count; - - if (save_lr) - { - registers_to_push |= (1 << R_LR); - registers_count++; - } - if (need_fp) - { - registers_to_push |= (1 << R_FP); - registers_count++; - } - - /* Keep the total push size 8-byte aligned (AAPCS). - * When no locals/FP (stack_size == 0, no frame pointer), pad by pushing - * a dummy low register (R3) — avoids SUB SP + ADD SP, saving 2 insns. - * When FP is used, alignment pad must stay as SUB SP space because - * FP-relative negative offsets may address that area. - * When locals exist, absorb the gap into SUB SP instead, keeping - * PUSH in 16-bit encoding (no high regs like R12). */ - if (registers_count % 2 != 0) - { - if (stack_size == 0 && !need_fp) - { - registers_to_push |= (1 << R3); - registers_count++; - } - else - { - push_align_pad = 4; - } - } - - map_sym_t(); - - /* Variadic: push r0-r3 FIRST so they are contiguous with stack args */ - vararg_push_size = 0; - if (func_var) - { - ot_check(th_push((1 << R0) | (1 << R1) | (1 << R2) | (1 << R3))); - vararg_push_size = 16; - } - - offset_to_args = registers_count * 4 + vararg_push_size; - - if (registers_count > 0) - ot_check(th_push(registers_to_push)); - - pushed_registers = registers_to_push; - callee_push_size = 0; - callee_saved_regs = 0; - } - - // allocate stack space for local variables - /* Variadic save area is reserved in the IR stack layout (loc bias). */ - - /* Keep SP 8-byte aligned (AAPCS). tccir normally pre-aligns stack_size, but - * be defensive here because other codepaths may call into the backend. - */ - if (stack_size & 7) - stack_size = (stack_size + 7) & ~7; - - /* allocated_stack_size is the portion of the stack used for locals/spills. - * It must NOT include the alignment pad — local offsets are computed - * relative to this value, and the pad sits below all addressable locals. */ - allocated_stack_size = stack_size; - - /* total_stack_dealloc is the full amount to restore in the epilogue, - * including the alignment pad that sits below addressable locals. */ - int total_stack_dealloc = stack_size + push_align_pad; - epilogue_stack_dealloc = total_stack_dealloc; - stack_size = total_stack_dealloc; - if (tcc_state->need_frame_pointer && !standard_frame_record) - { - if (!ot(th_add_imm(R_FP, R_SP, 0, flags_safe(), ENFORCE_ENCODING_NONE))) - { - // todo mov fp, sp - // load r12 immediate - // add fp, sp, r12 - fprintf(stderr, "compiler_error: prolog frame pointer setup failed\n"); - exit(1); - } - } - if (stack_size > 0) - { - gadd_sp(-stack_size); - } - - /* When FP is omitted, SP is lower by the full SUB SP amount (locals + - * alignment pad). Adjust offset_to_args so incoming stack parameters - * are found at the correct SP-relative position. */ - if (!need_fp) - offset_to_args += epilogue_stack_dealloc; - - /* However, local addressing uses allocated_stack_size (without pad). - * The pad sits at the top of the SUB SP region, right below pushed regs, - * so locals occupy SP+0 .. SP+allocated_stack_size-1, matching the same - * addresses as the old push-IP-for-alignment approach. */ - - /* Everything from here to the parameter shuffle below runs before a single - * incoming value has been moved to its allocated home, so these registers - * are all still live and none of them may be taken as a scratch register: - * R0-R3 (the argument registers -- all four, not just the named ones, since - * a variadic function saves the anonymous ones too), R9 (the GOT base under - * text_and_data_separation) and the static chain register. - * - * A store whose offset does not fit the STR immediate needs a register to - * hold the offset, and the scratch picker's idea of "free" comes from the - * live intervals, which do not model the incoming registers at prologue - * position. It therefore handed out R0. In a function with alloca and a - * frame deeper than the 255-byte negative STR immediate -- glob() and - * fnmatch() in GNU make's gnulib, among others -- the R9 spill below then - * emitted - * movw r0,#0x131c ; rsb r0,r0,#0 ; str.w r9,[r7,r0] - * movw r4,#0x1018 ; rsb r4,r4,#0 ; str r0,[r7,r4] - * and the first parameter, which the second store was meant to spill, was - * the offset constant by the time it got there. */ - uint32_t prologue_incoming_mask = (1u << R0) | (1u << R1) | (1u << R2) | (1u << R3); - if (tcc_state->text_and_data_separation) - prologue_incoming_mask |= (1u << ARM_R9); - if (ir && ir->has_static_chain) - prologue_incoming_mask |= (1u << architecture_config.static_chain_reg); - - /* Save the PIC GOT base (R9) once, into slot 0 of the nested-call save area. - * - * R9 is caller-saved under text_and_data_separation, so every call reloads - * it from the frame; but the value never changes within a function, so the - * store only has to happen once. It used to be emitted at each call site - * next to the reload, which cost 45,329 instructions (165 KiB) in the - * compiler's own build and produced runs of literally - * ldr.w r9, [sp] ; str.w r9, [sp] - * where the reload was immediately followed by writing the same value back. - * - * call_nested_save_size is non-zero exactly when the function has at least - * one call under text_and_data_separation (ir/codegen.c, where the area is - * sized), so it is the same guard in both codegen passes — and R9 is saved - * at every one of those call sites, so slot 0 is always reserved for it. */ - if (tcc_state->text_and_data_separation && ir && ir->call_nested_save_size > 0) - { - const int r9_slot_offset = ir->call_outgoing_size; - if (tcc_state->func_dynamic_sp) - /* VLA/alloca: the runtime SP has moved, so the call sites address these - * slots FP-relative with this bias. Match it. */ - tcc_gen_machine_store_to_stack_ex(ARM_R9, r9_slot_offset - (callee_push_size + epilogue_stack_dealloc), - prologue_incoming_mask); - else - /* Plain SP-relative, matching the store_word_to_stack() the call sites - * used: these slots are always addressed off SP when the frame is - * static, even in functions that also keep a frame pointer. */ - tcc_gen_machine_store_to_sp_ex(ARM_R9, r9_slot_offset, prologue_incoming_mask); - } - - /* Save incoming static chain (R10) at fixed chain slot. - * With two-phase push, callee-saved regs are below FP, so the chain - * slot is at [FP - callee_push_size - 4] instead of [FP - 4]. - * The body reads via offset -4 which gets fp_adjust_local_offset applied. */ - if (ir && ir->has_static_chain) - { - tcc_gen_machine_store_to_stack_ex(architecture_config.static_chain_reg, -(callee_push_size + 4), - prologue_incoming_mask); - } - - /* For variadic functions, save incoming r0-r3 in a fixed area at FP-16..FP-4 - * (for named parameter access) and store __gr_top at FP-20. - * The PUSH {r0-r3} at function entry already creates a contiguous register - * save area above the callee-saved pushes, adjacent to the stack arguments. - * __gr_top points to the end of that area (= start of stack args). - */ - if (func_var) - { - /* Store r0-r3 at FP-16..FP-4 for named parameter access. - * (The contiguous PUSH'd copy is at FP+offset_to_args-16..FP+offset_to_args-4 - * and is used by va_arg for anonymous argument traversal.) */ - tcc_gen_machine_store_to_stack_ex(R0, -(callee_push_size + 16), prologue_incoming_mask); - tcc_gen_machine_store_to_stack_ex(R1, -(callee_push_size + 12), prologue_incoming_mask); - tcc_gen_machine_store_to_stack_ex(R2, -(callee_push_size + 8), prologue_incoming_mask); - tcc_gen_machine_store_to_stack_ex(R3, -(callee_push_size + 4), prologue_incoming_mask); - - /* The frame-metadata triple that used to live at FP-20/-24/-28 (__gr_top, - * named_arg_reg_bytes, named_arg_stack_bytes) is gone: it existed purely so - * the runtime __tcc_va_start could rediscover the address of the first - * anonymous argument. TOK_builtin_va_start now materializes that address - * directly (a PARAM-relative LEA), so those six prologue instructions were - * dead in every variadic function. */ - } - - /* __builtin_apply_args: save incoming r0-r3 and stack args pointer - * to the reserved apply_args block so __builtin_apply can replay them. - * Layout at apply_args_offset: [stack_args_ptr, r0, r1, r2, r3]. */ - if (tcc_state->func_save_apply_args && ir) - { - int base_off = tcc_state->apply_args_offset; - /* Adjust for callee push gap (same adjustment as fp_adjust_local_offset) */ - int adj = base_off; - if (adj < 0 && callee_push_size > 0) - adj -= callee_push_size; - - /* Store stack args pointer (FP + offset_to_args = start of stack args area) */ - { - const int fp_or_sp = tcc_state->need_frame_pointer ? R_FP : R_SP; - ot_check(th_add_imm(R_IP, fp_or_sp, offset_to_args, flags_safe(), ENFORCE_ENCODING_NONE)); - } - tcc_gen_machine_store_to_stack_ex(R_IP, adj, prologue_incoming_mask); - - /* Store r0-r3 at offsets +4, +8, +12, +16 from the block start */ - tcc_gen_machine_store_to_stack_ex(R0, adj + 4, prologue_incoming_mask); - tcc_gen_machine_store_to_stack_ex(R1, adj + 8, prologue_incoming_mask); - tcc_gen_machine_store_to_stack_ex(R2, adj + 12, prologue_incoming_mask); - tcc_gen_machine_store_to_stack_ex(R3, adj + 16, prologue_incoming_mask); - } - - /* Move parameters from incoming registers to their allocated locations. - * For non-leaf functions or parameters that cross calls: - * - If allocated to callee-saved register: move from R0-R3 to allocated reg - * - If spilled: store from R0-R3 to stack location - * For leaf functions with params staying in R0-R3: no move needed */ - if (ir) - { - /* Parameter shuffling must not clobber still-needed incoming registers. - * Example (4 args): if z is assigned to R3 and w is assigned to R6, a naive - * sequence "mov r3,r2; mov r6,r3" destroys w. - * - * Strategy: - * 1) Handle register-passed params first: store spills, collect reg->reg moves. - * 2) Execute reg->reg moves as a parallel move with cycle breaking. - * 3) Then load stack-passed params into their allocated registers. - */ - - typedef struct ParamMove - { - int dst; - int src; - } ParamMove; - - /* NOTE: Do not hard-code small fixed arrays here. - * Functions can legally have >32 parameters (e.g. sum40 in tests), and - * overflowing these buffers corrupts prolog codegen and breaks calls. - * Worst-case: a 64-bit param can contribute up to 2 reg moves. - */ - const int max_param_moves = ir->next_parameter * 2 + 8; - ParamMove *moves = tcc_malloc(sizeof(ParamMove) * max_param_moves); - int move_count = 0; - - /* Hard-float double parameters arrive in d0-d7 but are not VFP-resident, so - * each is unpacked into its allocated GPR pair. Collected here and emitted - * after the GPR parallel move: the unpack overwrites GPRs that may still - * hold incoming GPR arguments. */ - typedef struct DoubleUnpack - { - int dreg, lo, hi; - } DoubleUnpack; - DoubleUnpack dbl_unpack[8]; - int dbl_unpack_count = 0; - - /* Build a bitmask of ALL incoming argument registers (R0-R3) that need - * to be saved. When storing a spilled parameter to the stack at a large - * offset, the scratch register allocator must NOT pick any of these - * registers — they still hold incoming parameter values. - * - * Without this mask, storing e.g. R0 to [FP-1028] can use R1 as scratch - * for the offset constant, destroying the parameter value in R1. - */ - uint32_t incoming_arg_regs_mask = 0; - for (int vreg = 0; vreg < ir->next_parameter; ++vreg) - { - const int encoded_vreg = (TCCIR_VREG_TYPE_PARAM << 28) | vreg; - IRLiveInterval *interval = tcc_ir_get_live_interval(ir, encoded_vreg); - if (!interval) - continue; - if (interval->incoming_reg0 >= 0 && interval->incoming_reg0 < 16) - incoming_arg_regs_mask |= (1u << interval->incoming_reg0); - if (interval->incoming_reg1 >= 0 && interval->incoming_reg1 < 16) - incoming_arg_regs_mask |= (1u << interval->incoming_reg1); - } - - for (int vreg = 0; vreg < ir->next_parameter; ++vreg) - { - const int encoded_vreg = (TCCIR_VREG_TYPE_PARAM << 28) | vreg; - IRLiveInterval *interval = tcc_ir_get_live_interval(ir, encoded_vreg); - - if (!interval) - continue; - - const int incoming_r0 = interval->incoming_reg0; - const int incoming_r1 = interval->incoming_reg1; - const int alloc_r0 = interval->allocation.r0; - const int alloc_r1 = interval->allocation.r1; - const int is_64bit = interval->is_double || interval->is_llong; - - if (incoming_r0 < 0) - { - /* Stack-passed parameters live permanently in the caller's argument - * area. Leave their allocations empty so IR materialization can treat - * them as VT_PARAM lvalues and load directly when needed. */ - interval->allocation.r0 = PREG_NONE; - interval->allocation.r1 = PREG_NONE; - interval->allocation.offset = 0; - continue; - } - - /* Stack-home parameters: store incoming regs to their stack slots. - * IR owns spills; avoid inspecting PREG_SPILLED sentinels here. - * Use the incoming_arg_regs_mask to prevent the scratch allocator - * from clobbering other incoming argument registers. - */ - if (interval->allocation.offset != 0) - { - /* Adjust for callee-saved gap below FP in two-phase push. */ - const int stack_offset = fp_adjust_local_offset(interval->allocation.offset, 0); - /* Float/double param arriving in a VFP register but spilled to the - * stack: bridge it through GPR scratch to reuse the ordinary store. A - * double occupies d and needs both halves written. */ - if (is_vfp_reg(incoming_r0)) - { - ScratchRegAlloc sc = get_scratch_reg_with_save(incoming_arg_regs_mask); - if (is_64bit) - { - ScratchRegAlloc sc2 = get_scratch_reg_with_save(incoming_arg_regs_mask | (1u << (uint32_t)sc.reg)); - ot_check(th_vmov_2gp_dp((uint16_t)sc.reg, (uint16_t)sc2.reg, - (uint16_t)(vfp_num(incoming_r0) / 2), 1)); /* lo,hi = d */ - tcc_gen_machine_store_to_stack_ex(sc.reg, stack_offset, incoming_arg_regs_mask); - tcc_gen_machine_store_to_stack_ex(sc2.reg, stack_offset + 4, incoming_arg_regs_mask); - restore_scratch_reg(&sc2); - } - else - { - ot_check(th_vmov_gp_sp((uint16_t)sc.reg, (uint16_t)vfp_num(incoming_r0), 1)); /* gpr = s_incoming */ - tcc_gen_machine_store_to_stack_ex(sc.reg, stack_offset, incoming_arg_regs_mask); - } - restore_scratch_reg(&sc); - continue; - } - if (is_64bit && incoming_r1 >= 0) - { - tcc_gen_machine_store_to_stack_ex(incoming_r0, stack_offset, incoming_arg_regs_mask); - /* R0 is now saved; remove it from the protection mask */ - incoming_arg_regs_mask &= ~(1u << incoming_r0); - tcc_gen_machine_store_to_stack_ex(incoming_r1, stack_offset + 4, incoming_arg_regs_mask); - incoming_arg_regs_mask &= ~(1u << incoming_r1); - } - else - { - tcc_gen_machine_store_to_stack_ex(incoming_r0, stack_offset, incoming_arg_regs_mask); - incoming_arg_regs_mask &= ~(1u << incoming_r0); - } - continue; - } - - /* Double parameter arriving in a VFP register (hard-float): it is not - * VFP-resident — there is no double-precision arithmetic here — so unpack - * d into the GPR pair the allocator gave it. Emitted before the GPR - * parallel move, whose sources are all still live. */ - if (is_64bit && is_vfp_reg(incoming_r0) && alloc_r0 != PREG_NONE && alloc_r0 >= 0 && alloc_r1 >= 0) - { - /* Deferred: the unpack writes GPRs that may still hold incoming GPR - * arguments (a leading int parameter sits in r0, and d0 commonly unpacks - * into r0:r1), so it must run only after the GPR parallel move below has - * consumed them. */ - if (dbl_unpack_count < (int)(sizeof(dbl_unpack) / sizeof(dbl_unpack[0]))) - dbl_unpack[dbl_unpack_count++] = - (DoubleUnpack){.dreg = vfp_num(incoming_r0) / 2, .lo = alloc_r0, .hi = alloc_r1}; - continue; - } - - /* Register-allocated parameters: record reg->reg moves (parallel move). - * - * IMPORTANT: Do NOT remove incoming_r0/r1 from incoming_arg_regs_mask - * here! The reg->reg moves are only COLLECTED now and executed LATER - * as a parallel move. The incoming register still holds the live - * parameter value at this point. If we removed it from the mask, a - * subsequent spill-store for another parameter could use it as scratch - * and destroy the value before the parallel move consumes it. - */ - if (alloc_r0 != PREG_NONE && is_vfp_reg(alloc_r0)) - { - /* Float parameter homed to a VFP register. It arrives either in a VFP - * register (hard-float argument passing) or in a GPR (soft layout); - * either way move it into its VFP home now, while the incoming register - * is still live. VFP destinations never participate in the GPR parallel - * move (different register file), so emitting here is safe. */ - if (is_vfp_reg(incoming_r0)) - { - if (incoming_r0 != alloc_r0) - ot_check(th_vmov_register((uint16_t)vfp_num(alloc_r0), (uint16_t)vfp_num(incoming_r0), 0)); - } - else - { - ot_check(th_vmov_gp_sp((uint16_t)incoming_r0, (uint16_t)vfp_num(alloc_r0), 0)); - } - continue; - } - if (alloc_r0 != PREG_NONE && alloc_r0 >= 0 && alloc_r0 <= R12 && alloc_r0 != incoming_r0) - { - moves[move_count++] = (ParamMove){.dst = alloc_r0, .src = incoming_r0}; - } - if (is_64bit && incoming_r1 >= 0 && alloc_r1 != PREG_NONE && alloc_r1 >= 0 && alloc_r1 <= R12 && - alloc_r1 != incoming_r1) - { - moves[move_count++] = (ParamMove){.dst = alloc_r1, .src = incoming_r1}; - } - } - - /* Execute collected register moves as a true parallel move. - * - * - Emit any move whose source is not a destination. - * - If only cycles remain, rotate a cycle using a temporary register. - * - * This is required for correct swaps like (r1<-r2, r2<-r1) without - * clobbering one of the incoming argument registers. - */ - while (move_count > 0) - { - uint32_t dst_mask = 0; - uint32_t src_mask = 0; - for (int i = 0; i < move_count; ++i) - { - if (moves[i].dst >= 0 && moves[i].dst < 32) - dst_mask |= (1u << moves[i].dst); - if (moves[i].src >= 0 && moves[i].src < 32) - src_mask |= (1u << moves[i].src); - } - - /* First: emit all acyclic moves. - * - * A move is safe to emit if its destination is not used as a source by - * any remaining move. This prevents clobbering values needed later. - * - * Example chain that must be ordered correctly: - * r1 <- r2 - * r2 <- r3 - * Here r2 is both a source and a destination. We must emit r1<-r2 first. - */ - int progressed = 0; - for (int i = 0; i < move_count; ++i) - { - const int dst = moves[i].dst; - const int src = moves[i].src; - - if (dst == src) - { - moves[i] = moves[--move_count]; - --i; - progressed = 1; - continue; - } - - if (dst >= 0 && dst < 32 && (src_mask & (1u << dst))) - continue; /* dst's current value is still needed as a source somewhere */ - - ot_check_mov_reg(dst, src, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE, false); - moves[i] = moves[--move_count]; - --i; - progressed = 1; - } - if (progressed) - continue; - - /* Cycle: rotate it using a temporary register (prefer IP). */ - int temp = R_IP; - if (dst_mask & (1u << temp)) - { - for (int r = R4; r <= R11; ++r) - { - if (!(dst_mask & (1u << r))) - { - temp = r; - break; - } - } - } - if (dst_mask & (1u << temp)) - { - tcc_error("compiler_error: prolog param shuffle has no temp register"); - } - - /* Pick any destination in the remaining cycle, save its original value, - * then walk dst<-src edges until we return to the start. - */ - const int start = moves[0].dst; - ot_check_mov_reg(temp, start, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE, false); - - int cur = start; - for (;;) - { - int idx = -1; - for (int i = 0; i < move_count; ++i) - { - if (moves[i].dst == cur) - { - idx = i; - break; - } - } - if (idx < 0) - { - tcc_error("compiler_error: broken prolog param shuffle cycle"); - } - - const int src = moves[idx].src; - moves[idx] = moves[--move_count]; - - if (src == start) - { - ot_check_mov_reg(cur, temp, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE, false); - break; - } - - ot_check_mov_reg(cur, src, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE, false); - cur = src; - } - } - - /* Now that every incoming GPR argument has reached its home, it is safe to - * overwrite the argument registers: unpack the hard-float doubles from - * d0-d7 into their allocated GPR pairs. */ - for (int i = 0; i < dbl_unpack_count; ++i) - ot_check(th_vmov_2gp_dp((uint16_t)dbl_unpack[i].lo, (uint16_t)dbl_unpack[i].hi, - (uint16_t)dbl_unpack[i].dreg, 1)); - - tcc_free(moves); - } - - /* Materialise the .rodata base for the body. Last in the prologue: the - * parameter shuffle above may borrow any callee-saved register that holds - * no incoming argument, which includes this one until it is loaded. R9 is - * the incoming GOT base and nothing in the prologue writes it. */ - if (rodata_anchor_reg >= 0) - ot_check_ldr_imm(rodata_anchor_reg, R9, YAFF_RODATA_ANCHOR_GOT_OFFSET, 6, ENFORCE_ENCODING_NONE); -} - -ST_FUNC void tcc_gen_machine_epilog(int leaffunc) -{ - TRACE("'tcc_gen_machine_epilog'"); - - int lr_saved = pushed_registers & (1 << R_LR); - - if (tcc_state->need_frame_pointer && callee_saved_regs) - { - /* ── Two-phase pop (mirrors two-phase push) ── */ - /* Restore SP from FP (works even with alloca/VLA since FP is stable) */ - ot_check_mov_reg(R_SP, R_FP, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE, false); - /* SP = FP; callee-saved regs are below FP. Adjust SP down. - * R3 is free in the epilogue (not used for return values). */ - gadd_sp_ex(-callee_push_size, R3); - ot_check(th_pop(callee_saved_regs)); - /* SP is now at FP (pointing at frame record {r7, [lr]}) */ - if (vararg_push_size > 0 && lr_saved) - { - /* Variadic: pop FP+LR, then skip over the pushed r0-r3 area */ - ot_check(th_pop((1 << R_FP) | (1 << R_LR))); - gadd_sp_ex(vararg_push_size, R3); - ot_check(th_bx_reg(R_LR)); - } - else if (lr_saved) - { - ot_check(th_pop((1 << R_FP) | (1 << R_PC))); - } - else - { - ot_check(th_pop(1 << R_FP)); - if (vararg_push_size > 0) - gadd_sp_ex(vararg_push_size, R3); - ot_check(th_bx_reg(R_LR)); - } - } - else if (tcc_state->need_frame_pointer) - { - /* ── Original single-push with FP: restore SP from FP, then pop all ── */ - ot_check_mov_reg(R_SP, R_FP, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE, false); - if (vararg_push_size > 0 && lr_saved) - { - /* Variadic: pop all regs with LR (not PC), then skip pushed r0-r3 */ - ot_check(th_pop(pushed_registers)); - gadd_sp_ex(vararg_push_size, R3); - ot_check(th_bx_reg(R_LR)); - } - else if (lr_saved) - { - pushed_registers |= 1 << R_PC; - pushed_registers &= ~(1 << R_LR); - ot_check(th_pop(pushed_registers)); - } - else - { - if (pushed_registers > 0) - ot_check(th_pop(pushed_registers)); - if (vararg_push_size > 0) - gadd_sp_ex(vararg_push_size, R3); - ot_check(th_bx_reg(R_LR)); - } - } - else - { - /* ── No frame pointer ── */ - if (epilogue_stack_dealloc > 0) - gadd_sp_ex(epilogue_stack_dealloc, R3); - if (lr_saved) - { - pushed_registers |= 1 << R_PC; - pushed_registers &= ~(1 << R_LR); - ot_check(th_pop(pushed_registers)); - } - else - { - if (pushed_registers > 0) - ot_check(th_pop(pushed_registers)); - ot_check(th_bx_reg(R_LR)); - } - } - - thumb_gen_state.generating_function = 0; - /* The anchor register holds this function's .rodata base and the epilogue - * has just restored the caller's value in it. */ - rodata_anchor_reg = -1; - th_literal_pool_generate(); - thumb_free_call_sites(); -} - -ST_FUNC void tcc_gen_machine_finish_noreturn(void) -{ - thumb_gen_state.generating_function = 0; - rodata_anchor_reg = -1; - th_literal_pool_generate(); - thumb_free_call_sites(); -} - -/* Load Effective Address: compute the address of src1 into dest. - * This is the explicit "address-of" operation for local variables/arrays. - * Unlike LOAD which dereferences, LEA computes FP+offset into a register. - */ - -/* MachineOperand-based LEA. Computes the address of the source operand - * into the destination. - * - * Most operand kinds are handled directly by mach_ensure_in_reg: - * MACH_OP_FRAME_ADDR → ADD dest, FP, #offset (local variable address) - * MACH_OP_SYMBOL → LDR dest, =symbol (global variable address) - * MACH_OP_REG → MOV dest, src_reg (address already computed) - * - * PARAM_STACK and CHAIN_REL need special handling because mach_ensure_in_reg - * always loads the VALUE from the stack. For LEA, we need the ADDRESS instead: - * MACH_OP_PARAM_STACK → ADD dest, FP, #(offset + offset_to_args) - * MACH_OP_CHAIN_REL → ADD/SUB dest, chain_base, #offset - */ -ST_FUNC void tcc_gen_machine_lea_mop(MachineOperand dest, MachineOperand src) -{ - MachineCodegenContext ctx = {0}; - int r; - - /* When dest is a writable register, compute the address directly into it - * to avoid a scratch + redundant mov. mach_alloc_scratch is driven by the - * per-instruction live_regs bitmap which already marks the dest reg "live" - * (it's the def target), so it would otherwise pick a different reg and - * writeback would emit `mov dest_reg, scratch`. */ - int dest_reg = -1; - if (dest.kind == MACH_OP_REG && !dest.needs_deref && - dest.u.reg.r0 != (int)PREG_REG_NONE) - dest_reg = dest.u.reg.r0; - - switch (src.kind) - { - case MACH_OP_PARAM_STACK: - { - /* Compute address of caller's argument slot. */ - r = (dest_reg >= 0) ? dest_reg : mach_alloc_scratch(&ctx, 0); - tcc_machine_addr_of_stack_slot(r, src.u.param.offset, 1 /* is_param */); - break; - } - case MACH_OP_CHAIN_REL: - { - /* Compute address in parent frame via static chain. */ - ScratchRegAlloc chain_scratch = {0}; - int chain_used = 0; - uint32_t excl = 0; - int base = resolve_chain_base(tcc_state->ir, src.u.chain.chain_index, excl, &chain_scratch, &chain_used); - /* dest_reg only usable if it doesn't collide with the chain base */ - if (dest_reg >= 0 && dest_reg != base && !(excl & (1u << (uint32_t)dest_reg))) - r = dest_reg; - else - r = mach_alloc_scratch(&ctx, excl | (1u << (uint32_t)base)); - int32_t off = src.u.chain.offset; - int sign = (off < 0); - int abs_off = sign ? (int)(-off) : (int)off; - if (abs_off == 0) - { - if (r != base) - ot_check_mov_reg((uint32_t)r, (uint32_t)base, flags_safe(), THUMB_SHIFT_DEFAULT, - ENFORCE_ENCODING_NONE, false); - } - else - { - thumb_opcode ins = sign ? th_sub_imm(r, base, abs_off, flags_safe(), ENFORCE_ENCODING_NONE) - : th_add_imm(r, base, abs_off, flags_safe(), ENFORCE_ENCODING_NONE); - if (ins.size != 0) - { - ot_check(ins); - } - else - { - /* Large offset: load into a scratch and use register ADD/SUB */ - ScratchRegAlloc off_sc = get_scratch_reg_with_save(excl | (1u << (uint32_t)r) | (1u << (uint32_t)base)); - load_full_const(off_sc.reg, PREG_NONE, LFC_SPLIT(abs_off)); - ot_check(sign ? th_sub_reg(r, base, off_sc.reg, flags_safe(), THUMB_SHIFT_DEFAULT, - ENFORCE_ENCODING_NONE) - : th_add_reg(r, base, off_sc.reg, flags_safe(), THUMB_SHIFT_DEFAULT, - ENFORCE_ENCODING_NONE)); - restore_scratch_reg(&off_sc); - } - } - if (chain_used) - restore_scratch_reg(&chain_scratch); - break; - } - case MACH_OP_FRAME_ADDR: - /* Address of a local stack slot: ADD dest, sp, #offset. */ - r = (dest_reg >= 0) ? dest_reg : mach_alloc_scratch(&ctx, 0); - tcc_machine_addr_of_stack_slot(r, src.u.frame.offset, 0); - break; - case MACH_OP_SYMBOL: - if (!src.needs_deref) - { - Sym *raw_sym = src.u.sym.sym; - Sym *sym = raw_sym ? validate_sym_for_reloc(raw_sym) : NULL; - r = (dest_reg >= 0) ? dest_reg : mach_alloc_scratch(&ctx, 0); - tcc_machine_load_constant(r, PREG_REG_NONE, src.u.sym.addend, 0, sym); - break; - } - /* fallthrough for needs_deref */ - default: - /* REG and other lvalue-y forms: mach_ensure_in_reg already computes the address. */ - r = mach_ensure_in_reg(&ctx, &src, 0); - break; - } - - mach_writeback_dest(&dest, r); - mach_release_all(&ctx); -} - -// r0 - function -// r1 - function -// r2 - function -// r3 - function - -// r4 - lrsa -// r5 - lrsa -// r6 - lrsa -// r7 - lrsa -// r8 - lrsa -// r9 - PIC -// r10 - lrsa - -ST_FUNC int tcc_gen_machine_number_of_registers(void) -{ - return 11; -} - -/* Store a register to a stack slot relative to FP. - * offset is typically negative (local variables below FP). */ -ST_FUNC void tcc_gen_machine_store_to_stack(int reg, int offset) -{ - tcc_gen_machine_store_to_stack_ex(reg, offset, 0); -} - -/* Store a register to a FP-relative stack slot, with additional register - * exclusions for the scratch allocator. The extra_exclude mask prevents - * the scratch register allocator from picking registers that still hold - * live values (e.g. incoming argument registers during the prologue). - */ -ST_FUNC void tcc_gen_machine_store_to_stack_ex(int reg, int offset, uint32_t extra_exclude) -{ - const int base_reg = tcc_state->need_frame_pointer ? R_FP : R_SP; - int sign = (offset < 0); - int abs_offset = sign ? -offset : offset; - - /* Try direct STR with immediate offset */ - if (!store_word_to_base(reg, base_reg, abs_offset, sign)) - { - /* Offset too large, use scratch register */ - /* Don't reuse the source register as offset scratch, otherwise we'd - * clobber the value before the STR (e.g. store -offset instead of value). */ - ScratchRegAlloc rr_alloc = th_offset_to_reg_ex(abs_offset, sign, (1u << reg) | (1u << base_reg) | extra_exclude); - int rr = rr_alloc.reg; - ot_check(th_str_reg(reg, base_reg, rr, THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); - restore_scratch_reg(&rr_alloc); - } -} - -/* Store a register to a stack slot relative to SP. - * Used for outgoing call arguments (stack args must be located at SP at call time). - * offset is expected to be non-negative. - */ -ST_FUNC void tcc_gen_machine_store_to_sp(int reg, int offset) -{ - tcc_gen_machine_store_to_sp_ex(reg, offset, 0); -} - -/* SP-relative store with extra scratch exclusions; see the _ex comment on the - * FP-relative variant above. */ -ST_FUNC void tcc_gen_machine_store_to_sp_ex(int reg, int offset, uint32_t extra_exclude) -{ - int sign = (offset < 0); - int abs_offset = sign ? -offset : offset; - - if (!store_word_to_base(reg, R_SP, abs_offset, sign)) - { - ScratchRegAlloc rr_alloc = th_offset_to_reg_ex(abs_offset, sign, (1u << reg) | (1u << R_SP) | extra_exclude); - int rr = rr_alloc.reg; - ot_check(th_str_reg(reg, R_SP, rr, THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); - restore_scratch_reg(&rr_alloc); - } -} - -/* MachineOperand variant of gcall_or_jump. - * Called from func_call_mop after argument setup is complete. - * - * MACH_OP_SYMBOL → direct call via BL + relocation - * MACH_OP_IMM → relative call (rare) - * MACH_OP_REG → indirect call: BLX through register - * Other kinds → load to scratch via mach_ensure_in_reg, then BLX - * - * For btype=FUNC + needs_deref: the register already holds the function - * pointer value (not an address to load through), so needs_deref is cleared. - */ -static void gcall_or_jump_mop(int is_jmp, MachineOperand target) -{ - /* Tail-call: promote is_jmp so we emit B/BX instead of BL/BLX. */ - if (tail_call_pending) - is_jmp = 1; - - if (target.kind == MACH_OP_SYMBOL) - { - /* Direct call via BL (or B.W for tail call) with relocation. */ - Sym *sym = target.u.sym.sym; - int32_t addend = target.u.sym.addend; - Sym *validated_sym = sym ? validate_sym_for_reloc(sym) : NULL; - Sym *reloc_sym = NULL; - - if (!dry_run_state.active) - { - if (sym && !validated_sym && !(sym->v & SYM_FIELD)) - { - put_extern_sym(sym, NULL, 0, 0); - validated_sym = validate_sym_for_reloc(sym); - } - if (sym && !(sym->v & SYM_FIELD)) - reloc_sym = validated_sym ? validated_sym : sym; - } - - uint32_t imm; - if (reloc_sym) - imm = (uint32_t)-4; /* placeholder for linker */ - else - imm = th_encbranch(ind, ind + addend); - - TRACE("gcall_or_jmp_mop: %d, ind: 0x%x, 0x%x", is_jmp, ind, imm); - if (imm) - { - if (is_jmp) - ot_check(th_b_t4((int32_t)imm)); - else - ot_check(th_bl_t1(imm)); - if (!dry_run_state.active && reloc_sym) - { - int call_pos = ind - 4; - greloc(cur_text_section, reloc_sym, call_pos, R_ARM_THM_JUMP24); - } - } - return; - } - - if (target.kind == MACH_OP_IMM) - { - /* Relative call (rare). */ - uint32_t imm = th_encbranch(ind, ind + (int32_t)target.u.imm.val); - TRACE("gcall_or_jmp_mop(imm): %d, ind: 0x%x, 0x%x", is_jmp, ind, imm); - if (imm) - { - if (is_jmp) - ot_check(th_b_t4((int32_t)imm)); - else - ot_check(th_bl_t1(imm)); - } - return; - } - - /* Indirect call through register/spill/frame/param. - * - * For btype=FUNC with needs_deref: the register already holds the function - * pointer value, not an address to dereference. Clear needs_deref to avoid - * a spurious LDR before BLX. */ - MachineOperand adjusted = target; - if (adjusted.btype == IROP_BTYPE_FUNC && adjusted.needs_deref && adjusted.kind == MACH_OP_REG) - adjusted.needs_deref = false; - - /* Keep R0-R3 safe during target materialization. */ - const uint32_t arg_regs = (1u << R0) | (1u << R1) | (1u << R2) | (1u << R3); - MachineCodegenContext mctx = {0}; - - if (is_jmp) - { - int r = mach_ensure_in_reg(&mctx, &adjusted, arg_regs); - ot_check(th_bx_reg(r)); - } - else - { - /* For calls, allocate scratch excluding R0-R3 so args are preserved. */ - uint32_t old_exclude = scratch_global_exclude; - scratch_global_exclude |= arg_regs; - - int r = mach_ensure_in_reg(&mctx, &adjusted, arg_regs); - - scratch_global_exclude = old_exclude; - ot_check(th_blx_reg(r)); - } -} - -/* Load a 32-bit immediate value or symbol address into a register. - * Uses th_generic_mov_imm if possible (pure immediates only), otherwise loads from literal pool. - * reg: target register - * imm: 32-bit immediate value or offset to load - * sym: symbol reference (NULL for pure immediates) - * update_flags: whether the load should update condition flags (currently unused) - */ -static void load_immediate(int reg, uint32_t imm, Sym *sym, int update_flags) -{ - (void)update_flags; /* Currently not used, reserved for future use */ - - /* If there's a symbol, always use literal pool for relocations */ - if (sym) - { - _lfc_sym = sym; - load_full_const(reg, PREG_NONE, imm, 0); - return; - } - - /* Plain-constant reuse, same contract as tcc_machine_load_constant: when the - * per-register materialisation cache still says `reg` holds this exact - * constant, the MOV is dead. Call-argument marshalling is where this pays: - * every stack-passed literal takes a fresh find_call_scratch()+load_immediate - * pair, and a variadic call with a run of zero arguments therefore emitted - * `movs rN,#0; str rN,[sp,#k]` once per word instead of hoisting the zero. - * imm_cache is invalidated by ot()'s emit-level decode and reset at IR - * boundaries/calls in both the dry and real passes, so the decision (and the - * resulting code size) stays identical between them. */ - const int cacheable = thumb_gen_state.generating_function && reg >= 0 && reg < 16; - const int64_t key = (int64_t)(uint32_t)imm; - - if (cacheable && imm_cache[reg].valid && imm_cache[reg].sym == NULL && imm_cache[reg].value == key) - return; - - /* Try to encode as ARM immediate (supports various rotated 8-bit patterns) */ - if (!ot(th_generic_mov_imm(reg, imm))) - { - /* Value doesn't fit in immediate encoding, use literal pool */ - load_full_const(reg, PREG_NONE, imm, 0); - } - - /* Record after the emit: ot()/load_full_const just invalidated imm_cache[reg] - * for the instruction they produced. */ - if (cacheable) - { - imm_cache[reg].value = key; - imm_cache[reg].sym = NULL; - imm_cache[reg].valid = 1; - } -} - -typedef enum ThumbArgMoveKind -{ - THUMB_ARG_MOVE_REG, - THUMB_ARG_MOVE_IMM, - THUMB_ARG_MOVE_IMM64, /* load 64-bit immediate into register pair */ - THUMB_ARG_MOVE_LOCAL_ADDR, /* compute address of local: fp + offset */ - THUMB_ARG_MOVE_STRUCT, /* load struct words into consecutive registers */ - THUMB_ARG_MOVE_MOP, /* generic: load MachineOperand into dst_reg (+ dst_reg_hi for 64-bit) */ -} ThumbArgMoveKind; - -typedef struct ThumbArgMove -{ - ThumbArgMoveKind kind; - int dst_reg; - int dst_reg_hi; /* valid when kind==THUMB_ARG_MOVE_IMM64 */ - int src_reg; /* valid when kind==THUMB_ARG_MOVE_REG */ - uint32_t imm; /* valid when kind==THUMB_ARG_MOVE_IMM */ - uint64_t imm64; /* valid when kind==THUMB_ARG_MOVE_IMM64 */ - Sym *sym; /* valid when kind==THUMB_ARG_MOVE_IMM */ - int local_offset; /* valid when kind==THUMB_ARG_MOVE_LOCAL_ADDR */ - int local_is_param; /* valid when kind==THUMB_ARG_MOVE_LOCAL_ADDR - if true, add offset_to_args */ - int struct_word_count; /* valid when kind==THUMB_ARG_MOVE_STRUCT */ - int struct_src_align; /* struct natural alignment (bytes); gates source LDRD */ - MachineOperand mop; /* valid when kind==THUMB_ARG_MOVE_MOP */ -} ThumbArgMove; - -/* Context for function call generation - reduces parameter passing */ -typedef struct CallGenContext -{ - ThumbGenCallSite *call_site; - TCCAbiCallLayout *layout; - IROperand *args; - MachineOperand *mops; - int argc; - int stack_size; - uint32_t arg_move_dst_mask; /* Registers that will be explicitly written by register arg moves. - * These are safe to clobber as scratch during stack arg placement - * because the subsequent register moves will overwrite them. */ -} CallGenContext; - -/* Is the by-value struct source ADDRESS provably 4-byte aligned? - * - * `struct_src_align` is the C type's alignment, and it is 1 for every packed - * aggregate — tinycc's own 9-byte IROperand, which dominates these call sites, - * being the obvious one. What LDRD and LDM actually require is an aligned - * address, so for a frame-resident source ask the offset instead of the type. - * - * Only the low two bits matter, and they survive the frame bias: SP is 8-byte - * aligned and FP 4-byte aligned at steady state, and every term - * fp_adjust_local_offset() adds (allocated_stack_size, callee_push_size, - * scratch_push_sp_bias) is a multiple of 4. Testing the RAW offset therefore - * gives the same answer as testing the adjusted one, and — unlike the adjusted - * one, whose scratch_push_sp_bias() term is dry-run gated — it reads the same - * in the rehearsal and the real pass, so it cannot desync the two code sizes. - * - * Incoming stack parameters are aligned by construction (arm_aapcs.c floors - * every argument slot at 4 and offset_to_args is a multiple of 4), but the - * check is written out rather than assumed. */ -static bool struct_src_addr_aligned4(const ThumbArgMove *m) -{ - if (m->struct_src_align >= 4) - return true; - if (m->mop.needs_deref) - return false; /* address comes from memory — nothing provable */ - switch (m->mop.kind) - { - case MACH_OP_SPILL: - return (m->mop.u.spill.offset & 3) == 0; - case MACH_OP_FRAME_ADDR: - return (m->mop.u.frame.offset & 3) == 0; - case MACH_OP_PARAM_STACK: - return ((m->mop.u.param.offset + offset_to_args) & 3) == 0; - default: - return false; - } -} - -static void thumb_emit_arg_move(const ThumbArgMove *m) -{ - if (m->kind == THUMB_ARG_MOVE_REG) - { - if (m->src_reg == m->dst_reg) - return; - ot_check_mov_reg(m->dst_reg, m->src_reg, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE, - false); - return; - } - - if (m->kind == THUMB_ARG_MOVE_LOCAL_ADDR) - { - /* Compute address of local variable: dst = fp + offset */ - tcc_machine_addr_of_stack_slot(m->dst_reg, m->local_offset, m->local_is_param); - return; - } - - if (m->kind == THUMB_ARG_MOVE_STRUCT) - { - /* Load struct words into consecutive registers. - * The mop contains the struct operand; get its base address. */ - int word_count = m->struct_word_count; - int base_dst = m->dst_reg; - - /* LDRD fast path for the common 2-word (8-byte) aggregate case sourced - * directly from a stack-backed location. Mirrors the LDRD path used by - * THUMB_ARG_MOVE_MOP for 64-bit scalars — Thumb-2 LDRD requires natural - * 4-byte alignment, which spill slots and the caller param stack both - * provide. Skips the scratch + per-word loads that would otherwise - * emit `add.w ip, sp, #N; ldr lo, [ip]; ldr hi, [ip, #4]` (3 insts). - * - * LDRD writes Rt before Rt2, so Rt2 (dst+1) must not equal the base - * register, otherwise the 2nd half reads from a clobbered base. */ - if (word_count == 2 && !m->mop.needs_deref && - (m->mop.kind == MACH_OP_SPILL || m->mop.kind == MACH_OP_PARAM_STACK)) - { - int raw_off = - (m->mop.kind == MACH_OP_SPILL) ? m->mop.u.spill.offset : m->mop.u.param.offset + offset_to_args; - int adjusted = (m->mop.kind == MACH_OP_SPILL) ? fp_adjust_local_offset(raw_off, 0) : raw_off; - int ldrd_base = tcc_state->need_frame_pointer ? R_FP : R_SP; - int ldrd_sign = (adjusted < 0); - int ldrd_abs_off = ldrd_sign ? -adjusted : adjusted; - int dst_hi = base_dst + 1; - if (dst_hi != ldrd_base && base_dst != ldrd_base && - try_ldrd_pair(base_dst, dst_hi, ldrd_base, ldrd_abs_off, ldrd_sign)) - { - return; - } - } - - /* Get the struct base address into a scratch register */ - ScratchRegAlloc struct_scratch = get_scratch_reg_with_save(0); - int base_addr_reg = get_struct_base_addr_mop(&m->mop, struct_scratch.reg); - - /* Load each word from the struct into consecutive target registers. - * Adjacent word pairs use LDRD when the source ADDRESS is 4-byte aligned - * (LDRD faults otherwise) and neither destination register aliases the base - * (LDRD writes Rt then Rt2; an alias would read a clobbered base on the - * fallback path / be unsafe). */ - bool src_aligned = struct_src_addr_aligned4(m); - - /* A single LDMIA.W replaces the whole per-word load sequence. It only pays - * from three words up: at two, the LDRD in the loop below is already one - * 4-byte instruction. - * - * th_ldm() silently strips the base register from the list when there is no - * writeback, so a base sitting inside the destination range would quietly - * load one register too few — reject that here instead of relying on the - * helper. PC and SP must stay out of the list too: PC would turn the load - * into a branch. */ - int last_dst = base_dst + word_count - 1; - if (word_count >= 3 && src_aligned && base_addr_reg != R_SP && base_dst >= 0 && last_dst <= R_IP && - !(base_addr_reg >= base_dst && base_addr_reg <= last_dst)) - { - uint32_t regset = 0; - for (int k = 0; k < word_count; ++k) - regset |= 1u << (unsigned)(base_dst + k); - ot_check(th_ldm((uint32_t)base_addr_reg, regset, 0 /* no writeback */, ENFORCE_ENCODING_NONE)); - restore_scratch_reg(&struct_scratch); - return; - } - - int w = 0; - for (; w + 1 < word_count; ) - { - int dst = base_dst + w; - int dst_hi = base_dst + w + 1; - int offset = w * 4; - if (src_aligned && dst != base_addr_reg && dst_hi != base_addr_reg && - tcc_gen_machine_try_ldrd_base(dst, dst_hi, base_addr_reg, offset)) - { - w += 2; - continue; - } - /* Single-word load of this word; the next iteration handles w+1. */ - if (!load_word_from_base(dst, base_addr_reg, offset, 0)) - { - ScratchRegAlloc off_scratch = get_scratch_reg_with_save((1u << base_addr_reg) | (1u << dst)); - load_immediate(off_scratch.reg, offset, NULL, false); - ot_check(th_ldr_reg(dst, base_addr_reg, off_scratch.reg, THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); - restore_scratch_reg(&off_scratch); - } - w += 1; - } - /* Trailing odd word. */ - for (; w < word_count; ++w) - { - int dst = base_dst + w; - int offset = w * 4; - if (!load_word_from_base(dst, base_addr_reg, offset, 0)) - { - ScratchRegAlloc off_scratch = get_scratch_reg_with_save((1u << base_addr_reg) | (1u << dst)); - load_immediate(off_scratch.reg, offset, NULL, false); - ot_check(th_ldr_reg(dst, base_addr_reg, off_scratch.reg, THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); - restore_scratch_reg(&off_scratch); - } - } - restore_scratch_reg(&struct_scratch); - return; - } - - if (m->kind == THUMB_ARG_MOVE_MOP) - { - /* Generic MachineOperand → register load. - * Handles all MOP kinds (REG+deref, SPILL, PARAM_STACK, CHAIN_REL, - * SYMBOL+deref, etc.) via mach_ensure_in_reg. */ - MachineCodegenContext mctx = {0}; - if (m->mop.is_64bit && m->dst_reg_hi != 0 && m->dst_reg_hi != PREG_REG_NONE) - { - if (m->mop.needs_deref && m->mop.kind != MACH_OP_PARAM_STACK) - { - /* The operand holds a pointer (in reg, spill, etc.). Load the - * pointer into a register, then fetch lo/hi from [ptr+0]/[ptr+4]. - * mach_make_hi_half cannot handle this because it adjusts the - * storage location (e.g. spill offset) instead of the deref offset. - * - * PARAM_STACK is excluded: mach_ensure_in_reg for PARAM_STACK - * always loads directly from the caller's argument area, so the - * mach_make_lo/hi_half path handles it correctly. */ - int base; - if (m->mop.kind == MACH_OP_REG) - { - base = m->mop.u.reg.r0; - } - else - { - MachineOperand addr = m->mop; - addr.needs_deref = false; - addr.is_64bit = false; - addr.btype = IROP_BTYPE_INT32; - uint32_t excl = (1u << m->dst_reg) | (1u << m->dst_reg_hi); - base = mach_ensure_in_reg(&mctx, &addr, excl); - } - /* Proven-aligned access (mop.align4): LDRD straight into the pair — - * Rt == Rn is fine without writeback, so no base preservation needed. - * Otherwise the 64-bit load_from_base path, which preserves the base - * when base == dst_reg (the lo-load would clobber it before the - * hi-load can use it) and stays unaligned-safe. */ - if (!(m->mop.align4 && try_ldrd_pair(m->dst_reg, m->dst_reg_hi, base, 0, 0))) - load_from_base(m->dst_reg, m->dst_reg_hi, IROP_BTYPE_INT64, 0, 0, 0, (uint32_t)base); - } - else - { - /* Fast path: when the source is a stack-backed 64-bit value (spill - * slot or caller's param stack area) and the destination is a valid - * AAPCS register pair, emit a single LDRD straight into dst_reg / - * dst_reg_hi, skipping the scratch + MOV sequence that the generic - * lo/hi lowering below would produce. - * - * Stack spill slots and the caller-argument frame are guaranteed - * 8-byte aligned (AAPCS stack_align = 8, spill slots obey type - * alignment), so LDRD's 4-byte alignment requirement is satisfied. */ - int ldrd_base = -1; - int ldrd_abs_off = 0; - int ldrd_sign = 0; - int ldrd_ok = 0; - if (!m->mop.needs_deref && (m->mop.kind == MACH_OP_SPILL || m->mop.kind == MACH_OP_PARAM_STACK)) - { - int raw_off = (m->mop.kind == MACH_OP_SPILL) ? m->mop.u.spill.offset : m->mop.u.param.offset + offset_to_args; - int adjusted = (m->mop.kind == MACH_OP_SPILL) ? fp_adjust_local_offset(raw_off, 0) : raw_off; - ldrd_base = tcc_state->need_frame_pointer ? R_FP : R_SP; - ldrd_sign = (adjusted < 0); - ldrd_abs_off = ldrd_sign ? -adjusted : adjusted; - ldrd_ok = 1; - } - if (ldrd_ok && try_ldrd_pair(m->dst_reg, m->dst_reg_hi, ldrd_base, ldrd_abs_off, ldrd_sign)) - { - /* LDRD emitted. */ - } - else - { - /* 64-bit: load lo and hi halves separately. */ - MachineOperand lo = mach_make_lo_half(&m->mop); - MachineOperand hi = mach_make_hi_half(&m->mop); - uint32_t excl = (1u << m->dst_reg) | (1u << m->dst_reg_hi); - int r_lo = mach_ensure_in_reg(&mctx, &lo, excl); - int r_hi = mach_ensure_in_reg(&mctx, &hi, excl | (1u << (uint32_t)r_lo)); - if (r_lo != m->dst_reg) - ot_check_mov_reg(m->dst_reg, r_lo, flags_safe(), THUMB_SHIFT_DEFAULT, - ENFORCE_ENCODING_NONE, false); - if (r_hi != m->dst_reg_hi) - ot_check_mov_reg(m->dst_reg_hi, r_hi, flags_safe(), THUMB_SHIFT_DEFAULT, - ENFORCE_ENCODING_NONE, false); - } - } - } - else - { - /* 32-bit: prefer loading directly into dst_reg when the operand kind - * permits it, bypassing the scratch + MOV sequence that - * mach_ensure_in_reg would emit. Kinds that need an extra - * pointer-chain scratch beyond dst_reg (CHAIN_REL) fall through to - * the generic path. */ - const MachineOperand *mop = &m->mop; - const int dst = m->dst_reg; - int handled = 0; - - switch (mop->kind) - { - case MACH_OP_NONE: - tcc_machine_load_constant(dst, PREG_REG_NONE, 0, 0, NULL); - handled = 1; - break; - - case MACH_OP_REG: - if (mop->needs_deref) - { - /* LDR dst, [r0]; legal even when r0 == dst (loaded value - * just replaces the base). */ - load_from_base(dst, PREG_REG_NONE, mop->btype, (int)mop->is_unsigned, 0, 0, - (uint32_t)mop->u.reg.r0); - } - else if (mop->u.reg.r0 != dst) - { - ot_check_mov_reg(dst, mop->u.reg.r0, flags_safe(), THUMB_SHIFT_DEFAULT, - ENFORCE_ENCODING_NONE, false); - } - handled = 1; - break; - - case MACH_OP_SPILL: - if (!mop->needs_deref) - { - tcc_machine_load_spill_slot(dst, mop->u.spill.offset); - } - else - { - /* LLOCAL: load pointer into dst, then dereference into dst. */ - tcc_machine_load_spill_slot(dst, mop->u.spill.offset); - load_from_base(dst, PREG_REG_NONE, mop->btype, (int)mop->is_unsigned, 0, 0, - (uint32_t)dst); - } - handled = 1; - break; - - case MACH_OP_PARAM_STACK: - { - const int adjusted = mop->u.param.offset + offset_to_args; - const int base_reg = tcc_state->need_frame_pointer ? R_FP : R_SP; - const int sign = (adjusted < 0); - const int abs_off = sign ? -adjusted : adjusted; - load_from_base(dst, PREG_REG_NONE, mop->btype, (int)mop->is_unsigned, abs_off, sign, - (uint32_t)base_reg); - handled = 1; - break; - } - - case MACH_OP_IMM: - tcc_machine_load_constant(dst, PREG_REG_NONE, mop->u.imm.val, 0, NULL); - handled = 1; - break; - - case MACH_OP_FRAME_ADDR: - if (!mop->needs_deref) - { - tcc_machine_addr_of_stack_slot(dst, mop->u.frame.offset, 0); - } - else - { - tcc_machine_addr_of_stack_slot(dst, mop->u.frame.offset, 0); - load_from_base(dst, PREG_REG_NONE, mop->btype, (int)mop->is_unsigned, 0, 0, - (uint32_t)dst); - } - handled = 1; - break; - - case MACH_OP_SYMBOL: - { - Sym *raw_sym = mop->u.sym.sym; - Sym *sym = raw_sym ? validate_sym_for_reloc(raw_sym) : NULL; - if (!mop->needs_deref) - { - tcc_machine_load_constant(dst, PREG_REG_NONE, mop->u.sym.addend, 0, sym); - } - else - { - tcc_machine_load_constant(dst, PREG_REG_NONE, 0, 0, sym); - const int32_t addend = mop->u.sym.addend; - const int sign = (addend < 0); - const int abs_off = sign ? (int)(-addend) : (int)addend; - load_from_base(dst, PREG_REG_NONE, mop->btype, (int)mop->is_unsigned, abs_off, sign, - (uint32_t)dst); - } - handled = 1; - break; - } - - default: - /* CHAIN_REL etc.: fall through to generic scratch + MOV. */ - break; - } - - if (!handled) - { - uint32_t excl = (1u << dst); - int r = mach_ensure_in_reg(&mctx, mop, excl); - if (r != dst) - ot_check_mov_reg(dst, r, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE, - false); - } - } - mach_release_all(&mctx); - return; - } - - if (m->kind == THUMB_ARG_MOVE_IMM64) - { - /* Load 64-bit immediate into register pair */ - uint32_t lo = (uint32_t)(m->imm64 & 0xFFFFFFFF); - uint32_t hi = (uint32_t)(m->imm64 >> 32); - load_immediate(m->dst_reg, lo, NULL, false); - load_immediate(m->dst_reg_hi, hi, NULL, false); - return; - } - - /* THUMB_ARG_MOVE_IMM */ - load_immediate(m->dst_reg, m->imm, m->sym, false); -} - -/* Compute the full set of destination registers written by an arg move. - * Multi-register moves (IMM64, 64-bit MOP, STRUCT) write more than dst_reg. - * The parallel move scheduler must check ALL written registers against - * pending source registers to avoid clobbering. */ -static uint32_t arg_move_write_set(const ThumbArgMove *m) -{ - uint32_t set = (1u << m->dst_reg); - switch (m->kind) - { - case THUMB_ARG_MOVE_IMM64: - set |= (1u << m->dst_reg_hi); - break; - case THUMB_ARG_MOVE_MOP: - if (m->dst_reg_hi > 0 && m->dst_reg_hi < 16) - set |= (1u << m->dst_reg_hi); - break; - case THUMB_ARG_MOVE_STRUCT: - for (int w = 1; w < m->struct_word_count; w++) - set |= (1u << (m->dst_reg + w)); - break; - default: - break; - } - return set; -} - -/* Schedule register argument setup as a parallel assignment. - * This avoids clobbering a source register needed for another argument. - * Example: r0 <- r6, r1 <- r0 must be emitted as: - * mov r1, r0 - * mov r0, r6 - */ -static void thumb_emit_parallel_arg_moves(ThumbArgMove *moves, int move_count) -{ - if (move_count <= 0) - return; - - uint8_t done[16]; - memset(done, 0, sizeof(done)); - - ScratchRegAlloc tmp_alloc = (ScratchRegAlloc){0}; - int have_tmp = 0; - - for (int remaining = move_count; remaining > 0;) - { - uint32_t src_set = 0; - for (int i = 0; i < move_count; ++i) - { - if (done[i]) - continue; - if (moves[i].kind == THUMB_ARG_MOVE_REG) - src_set |= (1u << moves[i].src_reg); - } - - int chosen = -1; - for (int i = 0; i < move_count; ++i) - { - if (done[i]) - continue; - /* Check ALL destination registers of this move against pending sources. - * Multi-reg writes (IMM64, 64-bit MOP, STRUCT) must not clobber any - * register that a pending REG move still needs to read. */ - if ((src_set & arg_move_write_set(&moves[i])) == 0) - { - chosen = i; - break; - } - } - - if (chosen < 0) - { - /* Cycle among register moves. Break it with a scratch temp. */ - int cyc = -1; - for (int i = 0; i < move_count; ++i) - { - if (!done[i] && moves[i].kind == THUMB_ARG_MOVE_REG) - { - cyc = i; - break; - } - } - if (cyc < 0) - tcc_error("compiler_error: arg move cycle without reg sources"); - - if (!have_tmp) - { - /* Exclude all regs involved in the parallel move. */ - uint32_t exclude = 0; - for (int i = 0; i < move_count; ++i) - { - if (done[i]) - continue; - exclude |= arg_move_write_set(&moves[i]); - if (moves[i].kind == THUMB_ARG_MOVE_REG) - exclude |= (1u << moves[i].src_reg); - } - /* Also exclude SP/PC. */ - exclude |= (1u << ARM_SP) | (1u << ARM_PC); - tmp_alloc = get_scratch_reg_with_save(exclude); - have_tmp = 1; - } - - thumb_require_materialized_reg("thumb_emit_parallel_arg_moves", "tmp", tmp_alloc.reg); - ot_check_mov_reg(tmp_alloc.reg, moves[cyc].src_reg, flags_safe(), THUMB_SHIFT_DEFAULT, - ENFORCE_ENCODING_NONE, false); - moves[cyc].src_reg = tmp_alloc.reg; - continue; - } - - thumb_emit_arg_move(&moves[chosen]); - done[chosen] = 1; - --remaining; - } - - if (have_tmp && tmp_alloc.saved) - ot_check(th_pop(1u << tmp_alloc.reg)); -} - -/* ======================================================================== - * Helper functions for call argument handling - * ======================================================================== */ - -/* Store a word to stack with large offset fallback */ -static void store_word_to_stack(int src_reg, int stack_offset) -{ - if (!store_word_to_base(src_reg, ARM_SP, stack_offset, 0)) - { - ScratchRegAlloc sc = get_scratch_reg_with_save((1u << src_reg)); - load_immediate(sc.reg, stack_offset, NULL, false); - ot_check(th_str_reg(src_reg, ARM_SP, sc.reg, THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); - restore_scratch_reg(&sc); - } -} - -/* Store a word to stack, preserving R0 if needed as scratch */ -static void store_word_to_stack_safe(int src_reg, int stack_offset, int base_addr_reg) -{ - if (!store_word_to_base(src_reg, ARM_SP, stack_offset, 0)) - { - ScratchRegAlloc sc = get_scratch_reg_with_save((1u << src_reg) | (1u << base_addr_reg)); - load_immediate(sc.reg, stack_offset, NULL, false); - ot_check(th_str_reg(src_reg, ARM_SP, sc.reg, THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); - restore_scratch_reg(&sc); - } -} - -/* Get struct base address into a register (MOP path). - * For struct arguments, we want the ADDRESS of the struct, not a word from it. - * The MOP from machine_op_from_ir encodes the "value-level" view, so we - * convert / strip one level of indirection to obtain the address instead. */ -static int get_struct_base_addr_mop(const MachineOperand *mop, int default_reg) -{ - switch (mop->kind) - { - case MACH_OP_REG: - /* Register holds the struct address (for both needs_deref=true and false, - * the register value IS the address we want for struct copying). */ - return mop->u.reg.r0; - - case MACH_OP_FRAME_ADDR: - /* Address-of local struct: compute FP + offset. */ - tcc_machine_addr_of_stack_slot(default_reg, mop->u.frame.offset, 0); - return default_reg; - - case MACH_OP_SPILL: - if (mop->needs_deref) - { - /* llocal: spill slot holds pointer to struct. Load just the pointer. */ - tcc_machine_load_spill_slot(default_reg, mop->u.spill.offset); - } - else - { - /* Local struct on stack: compute address FP + offset. */ - tcc_machine_addr_of_stack_slot(default_reg, mop->u.spill.offset, 0); - } - return default_reg; - - case MACH_OP_PARAM_STACK: - /* Struct in caller's argument area: compute address with param adjustment. */ - tcc_machine_addr_of_stack_slot(default_reg, mop->u.param.offset, 1 /* is_param */); - return default_reg; - - case MACH_OP_SYMBOL: - { - Sym *sym = mop->u.sym.sym ? validate_sym_for_reloc(mop->u.sym.sym) : NULL; - load_immediate(default_reg, (uint32_t)mop->u.sym.addend, sym, false); - return default_reg; - } - - default: - { - /* CHAIN_REL, etc: generic path with needs_deref stripped. */ - MachineOperand addr_mop = *mop; - addr_mop.needs_deref = false; - MachineCodegenContext mctx = {0}; - int r = mach_ensure_in_reg(&mctx, &addr_mop, 0); - mach_release_all(&mctx); - return r; - } - } -} - -/* Build register move for a struct argument (MOP path) */ -static int build_reg_move_struct(ThumbArgMove *moves, int move_count, const MachineOperand *mop, - const TCCAbiArgLoc *loc, int base_reg, ThumbGenCallSite *call_site, - int src_align) -{ - int words = loc->reg_count; - if (words > 0 && words <= 4) - { - moves[move_count++] = (ThumbArgMove){ - .kind = THUMB_ARG_MOVE_STRUCT, - .dst_reg = base_reg, - .mop = *mop, - .struct_word_count = words, - .struct_src_align = src_align, - }; - } - for (int w = 0; w < words && w < loc->reg_count; w++) - call_site->registers_map |= (1 << (base_reg + w)); - return move_count; -} - -/* Build register move for a 64-bit argument (MOP path) */ -static int build_reg_move_64bit(ThumbArgMove *moves, int move_count, const MachineOperand *mop, const IROperand *arg, - int base_reg, ThumbGenCallSite *call_site, TCCIRState *ir) -{ - if (mop->kind == MACH_OP_REG && !mop->needs_deref && thumb_is_hw_reg(mop->u.reg.r0) && thumb_is_hw_reg(mop->u.reg.r1)) - { - /* Both halves in registers — emit up to two REG moves. */ - if (mop->u.reg.r0 != base_reg) - moves[move_count++] = (ThumbArgMove){.kind = THUMB_ARG_MOVE_REG, .dst_reg = base_reg, .src_reg = mop->u.reg.r0}; - if (mop->u.reg.r1 != (base_reg + 1)) - moves[move_count++] = - (ThumbArgMove){.kind = THUMB_ARG_MOVE_REG, .dst_reg = base_reg + 1, .src_reg = mop->u.reg.r1}; - } - else if (mop->kind == MACH_OP_IMM) - { - const uint64_t imm64 = (uint64_t)mop->u.imm.val; - moves[move_count++] = - (ThumbArgMove){.kind = THUMB_ARG_MOVE_IMM64, .dst_reg = base_reg, .dst_reg_hi = base_reg + 1, .imm64 = imm64}; - } - else - { - /* Generic: load MOP value into register pair at emit time. - * Covers SPILL, PARAM_STACK, CHAIN_REL, REG+needs_deref, SYMBOL, etc. */ - MachineOperand m = *mop; - m.is_64bit = true; - moves[move_count++] = - (ThumbArgMove){.kind = THUMB_ARG_MOVE_MOP, .dst_reg = base_reg, .dst_reg_hi = base_reg + 1, .mop = m}; - } - - call_site->registers_map |= (1 << base_reg) | (1 << (base_reg + 1)); - return move_count; -} - -/* Build register move for a 32-bit argument (MOP path) */ -static int build_reg_move_32bit(ThumbArgMove *moves, int move_count, const MachineOperand *mop, const IROperand *arg, - int base_reg, ThumbGenCallSite *call_site, TCCIRState *ir) -{ - switch (mop->kind) - { - case MACH_OP_REG: - if (mop->needs_deref) - { - /* Register-indirect: needs dereference at emit time. */ - moves[move_count++] = (ThumbArgMove){.kind = THUMB_ARG_MOVE_MOP, .dst_reg = base_reg, .mop = *mop}; - } - else if (mop->u.reg.r0 != base_reg) - { - moves[move_count++] = (ThumbArgMove){.kind = THUMB_ARG_MOVE_REG, .dst_reg = base_reg, .src_reg = mop->u.reg.r0}; - } - break; - - case MACH_OP_IMM: - moves[move_count++] = - (ThumbArgMove){.kind = THUMB_ARG_MOVE_IMM, .dst_reg = base_reg, .imm = (uint32_t)mop->u.imm.val, .sym = NULL}; - break; - - case MACH_OP_SYMBOL: - if (mop->needs_deref) - { - /* Load value from global symbol — emit at emit time. */ - moves[move_count++] = (ThumbArgMove){.kind = THUMB_ARG_MOVE_MOP, .dst_reg = base_reg, .mop = *mop}; - } - else - { - /* Load symbol address (with addend). */ - moves[move_count++] = (ThumbArgMove){ - .kind = THUMB_ARG_MOVE_IMM, .dst_reg = base_reg, .imm = (uint32_t)mop->u.sym.addend, .sym = mop->u.sym.sym}; - } - break; - - case MACH_OP_FRAME_ADDR: - moves[move_count++] = (ThumbArgMove){.kind = THUMB_ARG_MOVE_LOCAL_ADDR, - .dst_reg = base_reg, - .local_offset = mop->u.frame.offset, - .local_is_param = 0}; - break; - - default: - /* SPILL, PARAM_STACK, CHAIN_REL, etc.: generic MOP load at emit time. */ - moves[move_count++] = (ThumbArgMove){.kind = THUMB_ARG_MOVE_MOP, .dst_reg = base_reg, .mop = *mop}; - break; - } - - call_site->registers_map |= (1 << base_reg); - return move_count; -} - -/* Place a struct argument on stack (MOP path) */ -/* Load one struct word at [base_addr_reg + off] into `reg`, falling back to a - * register-offset load when `off` exceeds the LDR immediate range. */ -static void load_struct_word_into(int reg, int base_addr_reg, int off) -{ - if (!load_word_from_base(reg, base_addr_reg, off, 0)) - { - load_immediate(reg, off, NULL, false); - ot_check(th_ldr_reg(reg, base_addr_reg, reg, THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); - } -} - -/* Copy a (possibly split) struct argument's stack portion into the outgoing - * argument area. `src_align` is the struct's natural alignment in bytes. - * - * Adjacent word pairs are copied with LDRD/STRD instead of two LDR/STR. The - * destination is the outgoing arg area — SP-relative with a word-multiple - * offset and SP 8-byte aligned at the call boundary — so STRD is always - * alignment-safe. LDRD additionally requires the *source* address to be - * 4-byte aligned, which holds exactly when the struct's natural alignment is - * >= 4 (the stack portion starts at base + words_in_regs*4, a word multiple). */ -static void place_stack_arg_struct(const MachineOperand *mop, const TCCAbiArgLoc *loc, int stack_offset, - int src_align) -{ - int words_in_regs = (loc->kind == TCC_ABI_LOC_REG_STACK) ? loc->reg_count : 0; - int struct_src_offset = words_in_regs * 4; - int struct_size = (loc->kind == TCC_ABI_LOC_REG_STACK) ? loc->stack_size : loc->size; - int words = (struct_size + 3) / 4; - - /* A frame-resident source needs no base register: LDR/LDRD carry the frame - * offset in their own immediate field, so the `add rX, sp, #off` that - * get_struct_base_addr_mop emits is pure overhead — one instruction per - * marshaled by-value struct argument. Only take the direct form while every - * source word stays inside the offset range both LDR and LDRD encode - * (word-aligned, 0..1020); outside it the per-word fallback would cost more - * than the single ADD. Nothing in the loop below pushes a scratch across a - * source load, so the SP bias folded in here stays valid throughout. */ - int frame_raw = 0; - bool src_is_frame = false; - if (mop->kind == MACH_OP_SPILL && !mop->needs_deref) - { - frame_raw = mop->u.spill.offset; - src_is_frame = true; - } - else if (mop->kind == MACH_OP_FRAME_ADDR) - { - frame_raw = mop->u.frame.offset; - src_is_frame = true; - } - - int base_bias = 0; - int base_addr_reg; - ScratchRegAlloc struct_sc = {0}; - bool have_struct_sc = false; - - if (src_is_frame) - { - const int adj = fp_adjust_local_offset(frame_raw, 0); - const int first = adj + struct_src_offset; - const int last = first + (words > 0 ? (words - 1) * 4 : 0); - if ((adj & 3) == 0 && first >= 0 && last <= 1020) - { - base_addr_reg = tcc_state->need_frame_pointer ? R_FP : R_SP; - base_bias = adj; - } - else - src_is_frame = false; - } - if (!src_is_frame) - { - struct_sc = get_scratch_reg_with_save(0); - have_struct_sc = true; - base_addr_reg = get_struct_base_addr_mop(mop, struct_sc.reg); - } - - /* Second data register (besides LR) for paired LDRD/STRD. find_call_scratch - * never pushes (SP-relative store offsets must stay valid) and we exclude LR - * and the struct base; an R_IP last-resort result is a permanent scratch and - * safe to clobber. */ - int data2 = find_call_scratch((1u << ARM_LR) | (1u << (uint32_t)base_addr_reg), 0); - bool can_pair = (words >= 2 && data2 != ARM_LR && data2 != base_addr_reg && data2 >= 0 && - data2 <= R_LR && data2 != R_SP); - bool src_aligned = (src_align >= 4); - - int w = 0; - if (can_pair) - { - for (; w + 1 < words; w += 2) - { - int src_off = base_bias + struct_src_offset + w * 4; - int dst_off = stack_offset + w * 4; - - if (!(src_aligned && tcc_gen_machine_try_ldrd_base(ARM_LR, data2, base_addr_reg, src_off))) - { - load_struct_word_into(ARM_LR, base_addr_reg, src_off); - load_struct_word_into(data2, base_addr_reg, src_off + 4); - } - if (!tcc_gen_machine_try_strd_base(ARM_LR, data2, ARM_SP, dst_off)) - { - store_word_to_stack_safe(ARM_LR, dst_off, base_addr_reg); - store_word_to_stack_safe(data2, dst_off + 4, base_addr_reg); - } - } - } - - /* Trailing odd word, or every word when pairing was unavailable. */ - for (; w < words; ++w) - { - int src_off = base_bias + struct_src_offset + w * 4; - int dst_off = stack_offset + w * 4; - load_struct_word_into(ARM_LR, base_addr_reg, src_off); - store_word_to_stack_safe(ARM_LR, dst_off, base_addr_reg); - } - if (have_struct_sc) - restore_scratch_reg(&struct_sc); -} - -/* Find a free scratch register via liveness (no push/pop). - * Returns the register number, or R_IP as last resort. - * Must not push/pop since SP-relative offsets for stack args would shift. - * - * Unlike tcc_ls_find_free_scratch_reg (which refuses callee-saved regs), - * this also considers callee-saved registers already pushed in the prologue. - * Those are safe to clobber because the epilogue will restore them. - * - * arg_move_dst_mask: registers that will be explicitly written by register - * arg moves AFTER stack arg placement. These are safe to clobber even if - * currently live, because the subsequent moves will overwrite them. - * Pass 0 when not in a pre-move stack arg placement context. */ -static int find_call_scratch(uint32_t extra_exclude, uint32_t arg_move_dst_mask) -{ - TCCIRState *ir = tcc_state->ir; - uint32_t exclude = scratch_global_exclude | extra_exclude; - if (ir) - { - /* Standard path: try caller-saved regs via liveness */ - int reg = tcc_ls_find_free_scratch_reg(&ir->ls, ir->codegen_instruction_idx, exclude, ir->leaffunc); - if (reg != PREG_NONE && reg >= 0 && reg < 16 && reg != R_SP && reg != R_PC) - return reg; - - /* Extended path: try callee-saved regs that are already pushed in prologue - * AND not live at this instruction (so we won't clobber active values). */ - if (ir->ls.live_regs_by_instruction && ir->codegen_instruction_idx >= 0 && - ir->codegen_instruction_idx < ir->ls.live_regs_by_instruction_size) - { - uint32_t live = ir->ls.live_regs_by_instruction[ir->codegen_instruction_idx]; - uint32_t callee_pushed = pushed_registers & 0x0FF0u; /* R4-R11 that were pushed */ - /* R7 is the frame base: liveness never models it (it is not an - * interval), so it ALWAYS looks dead here — handing it out clobbered - * the FP mid-marshal and every later FP-relative address computed - * from garbage (lac_slot_key: `ldr r7,[ip,#8]` for a struct stack - * arg, then `sub ip, r7, #12` for the next arg). R9 is the GOT - * base under text_and_data_separation. Mirrors the reserved set in - * scratch_pushed_dead_reg. */ - uint32_t reserved = (1u << R_FP) | scratch_exclude_baseline(); - if (tcc_state->text_and_data_separation) - reserved |= (1u << 9); - uint32_t candidates = callee_pushed & ~live & ~exclude & ~reserved; - if (candidates) - { - /* Prefer low registers (R4-R7) for 16-bit encoding */ - int r = (int)__builtin_ctz(candidates); - return r; - } - } - - /* Pre-move path: registers that are destinations of explicit (non-identity) - * register arg moves can be used as scratch — the moves will overwrite them. - * Prefer low registers for 16-bit encoding. */ - if (arg_move_dst_mask) - { - uint32_t candidates = arg_move_dst_mask & ~exclude; - if (candidates) - { - int r = (int)__builtin_ctz(candidates); - if (r >= 0 && r < 16 && r != R_SP && r != R_PC) - return r; - } - } - } - return R_IP; -} - -/* Place a 64-bit argument on stack (MOP path) */ -static void place_stack_arg_64bit(const MachineOperand *mop, int stack_offset, TCCIRState *ir, - uint32_t arg_move_dst_mask) -{ - int lo_offset = stack_offset; - int hi_offset = stack_offset + 4; - - if (mop->kind == MACH_OP_REG && !mop->needs_deref && thumb_is_hw_reg(mop->u.reg.r0) && thumb_is_hw_reg(mop->u.reg.r1)) - { - /* If either register is R0-R3, the value was already stored by - * presave_stack_args_from_arg_regs before the register shuffle. */ - if (mop->u.reg.r0 <= ARM_R3 || mop->u.reg.r1 <= ARM_R3) - return; - /* The two halves land on adjacent, word-aligned SP slots — exactly STRD's - * shape. (STRD to the outgoing argument area is always safe: SP-relative, - * word-multiple offset, SP 8-aligned.) */ - if (!tcc_gen_machine_try_strd_base(mop->u.reg.r0, mop->u.reg.r1, ARM_SP, lo_offset)) - { - store_word_to_stack(mop->u.reg.r0, lo_offset); - store_word_to_stack(mop->u.reg.r1, hi_offset); - } - } - else if (mop->kind == MACH_OP_IMM) - { - uint64_t imm64 = (uint64_t)mop->u.imm.val; - uint32_t lo_val = (uint32_t)imm64; - uint32_t hi_val = (uint32_t)(imm64 >> 32); - int scr = find_call_scratch(0, arg_move_dst_mask); - load_immediate(scr, lo_val, NULL, false); - - /* Pair the two halves into one STRD. When both halves are equal (every - * `0.0`/`0LL` variadic argument) the single scratch is stored twice — - * try_strd_pair permits Rt == Rt2 (only LDRD forbids it). Otherwise take a - * second scratch for the high half; find_call_scratch falls back to R_IP - * without honouring the exclude mask, so a collision means no pair. */ - int hi_scr = scr; - if (hi_val != lo_val) - { - hi_scr = find_call_scratch(1u << (uint32_t)scr, arg_move_dst_mask); - if (hi_scr == scr) - { - /* No second scratch: fall back to the sequential two-store form. */ - store_word_to_stack(scr, lo_offset); - load_immediate(scr, hi_val, NULL, false); - store_word_to_stack(scr, hi_offset); - return; - } - load_immediate(hi_scr, hi_val, NULL, false); - } - - if (!tcc_gen_machine_try_strd_base(scr, hi_scr, ARM_SP, lo_offset)) - { - store_word_to_stack(scr, lo_offset); - store_word_to_stack(hi_scr, hi_offset); - } - } - else if (mop->needs_deref && mop->kind != MACH_OP_PARAM_STACK) - { - /* The operand holds a pointer (in reg, spill, etc.), not the 64-bit - * value itself. Load the pointer into a register, then fetch the - * lo/hi halves from [ptr+0] and [ptr+4]. Splitting via - * mach_make_hi_half would incorrectly adjust the storage location - * (e.g. spill offset) instead of the dereference offset. - * - * PARAM_STACK is excluded: mach_ensure_in_reg for PARAM_STACK always - * loads directly from the caller's argument area (ignores needs_deref), - * so the else path with mach_make_lo/hi_half handles it correctly. - * - * The base register must NOT be the scratch because both halves are - * loaded into the scratch. If base == scratch the first load would - * clobber the pointer before the second load can use it. */ - int scr = find_call_scratch(0, arg_move_dst_mask); - int base; - MachineCodegenContext mctx = {0}; - bool need_release = false; - if (mop->kind == MACH_OP_REG && mop->u.reg.r0 != scr) - { - base = mop->u.reg.r0; - } - else - { - MachineOperand addr = *mop; - addr.needs_deref = false; - addr.is_64bit = false; - addr.btype = IROP_BTYPE_INT32; - base = mach_ensure_in_reg(&mctx, &addr, (1u << scr)); - need_release = true; - } - load_from_base(scr, PREG_REG_NONE, IROP_BTYPE_INT32, 0, 0, 0, (uint32_t)base); - store_word_to_stack(scr, lo_offset); - load_from_base(scr, PREG_REG_NONE, IROP_BTYPE_INT32, 0, 4, 0, (uint32_t)base); - store_word_to_stack(scr, hi_offset); - if (need_release) - mach_release_all(&mctx); - } - else - { - /* Load each 32-bit half individually. Override btype to INT32 so that - * mach_ensure_in_reg → load_from_base does a single-word LDR instead - * of a 64-bit pair load (which would allocate an extra scratch via push, - * shift SP, and corrupt the SP-relative store offsets below). */ - MachineOperand lo = mach_make_lo_half(mop); - MachineOperand hi = mach_make_hi_half(mop); - lo.btype = IROP_BTYPE_INT32; - hi.btype = IROP_BTYPE_INT32; - MachineCodegenContext mctx = {0}; - int r_lo = mach_ensure_in_reg(&mctx, &lo, 0); - store_word_to_stack_safe(r_lo, lo_offset, r_lo); - mach_release_all(&mctx); - mctx = (MachineCodegenContext){0}; - int r_hi = mach_ensure_in_reg(&mctx, &hi, 0); - store_word_to_stack_safe(r_hi, hi_offset, r_hi); - mach_release_all(&mctx); - } -} - -/* Place a 32-bit argument on stack (MOP path) */ -static void place_stack_arg_32bit(const MachineOperand *mop, int stack_offset, CallGenContext *ctx) -{ - switch (mop->kind) - { - case MACH_OP_REG: - if (!mop->needs_deref) - { - /* Skip R0-R3 sources — handled in pre-shuffle save. */ - if (mop->u.reg.r0 <= ARM_R3) - return; - store_word_to_stack(mop->u.reg.r0, stack_offset); - } - else - { - /* Register-indirect: load through the register, then store to stack. - * Must use btype-aware load so that byte/short values are properly - * zero/sign-extended (LDRB/LDRH) instead of always doing a word LDR. */ - int scr = find_call_scratch(1u << mop->u.reg.r0, ctx->arg_move_dst_mask); - load_from_base(scr, PREG_REG_NONE, mop->btype, mop->is_unsigned, 0, 0, mop->u.reg.r0); - store_word_to_stack(scr, stack_offset); - } - break; - - case MACH_OP_IMM: - { - int scr = find_call_scratch(0, ctx->arg_move_dst_mask); - load_immediate(scr, (uint32_t)mop->u.imm.val, NULL, false); - store_word_to_stack(scr, stack_offset); - break; - } - - case MACH_OP_SYMBOL: - { - int scr = find_call_scratch(0, ctx->arg_move_dst_mask); - Sym *sym = mop->u.sym.sym ? validate_sym_for_reloc(mop->u.sym.sym) : NULL; - if (mop->needs_deref) - { - /* Load value from global symbol address. */ - load_immediate(scr, 0, sym, false); - int32_t addend = mop->u.sym.addend; - int sign = (addend < 0); - int abs_off = sign ? -addend : addend; - load_from_base(scr, PREG_REG_NONE, mop->btype, mop->is_unsigned, abs_off, sign, scr); - } - else - { - load_immediate(scr, (uint32_t)mop->u.sym.addend, sym, false); - } - store_word_to_stack(scr, stack_offset); - break; - } - - default: - { - /* SPILL, PARAM_STACK, FRAME_ADDR, CHAIN_REL: generic MOP load. */ - MachineCodegenContext mctx = {0}; - int r = mach_ensure_in_reg(&mctx, mop, 0); - store_word_to_stack(r, stack_offset); - mach_release_all(&mctx); - break; - } - } -} - -/* Build all register argument moves */ -static int build_register_arg_moves(CallGenContext *ctx, ThumbArgMove *reg_moves) -{ - int move_count = 0; - - for (int i = 0; i < ctx->argc; ++i) - { - const TCCAbiArgLoc *loc = &ctx->layout->locs[i]; - const IROperand *arg = &ctx->args[i]; - const MachineOperand *mop = &ctx->mops[i]; - const int bt = irop_get_btype(*arg); - const int is_64bit = mop->is_64bit; - - if (loc->kind != TCC_ABI_LOC_REG && loc->kind != TCC_ABI_LOC_REG_STACK) - continue; - - int base_reg = ARM_R0 + loc->reg_base; - - if (bt == IROP_BTYPE_STRUCT || arg->is_complex) - { - /* Complex values already in a register pair hold the actual value, - * not a pointer to it. Route through individual register moves - * instead of the struct-copy path (which dereferences as an address). */ - if (arg->is_complex && mop->kind == MACH_OP_REG && !mop->needs_deref && mop->is_64bit) - { - int words = loc->reg_count; - if (words >= 1 && mop->u.reg.r0 != base_reg) - reg_moves[move_count++] = - (ThumbArgMove){.kind = THUMB_ARG_MOVE_REG, .dst_reg = base_reg, .src_reg = mop->u.reg.r0}; - if (words >= 2 && mop->u.reg.r1 != (base_reg + 1)) - reg_moves[move_count++] = - (ThumbArgMove){.kind = THUMB_ARG_MOVE_REG, .dst_reg = base_reg + 1, .src_reg = mop->u.reg.r1}; - for (int w = 0; w < words; w++) - ctx->call_site->registers_map |= (1 << (base_reg + w)); - } - else if (arg->is_complex && mop->kind == MACH_OP_IMM) - { - /* Complex immediate: split 64-bit packed value (real_lo | imag_hi) - * into individual 32-bit register moves. */ - const uint64_t imm64 = (uint64_t)mop->u.imm.val; - int words = loc->reg_count; - if (words >= 1) - reg_moves[move_count++] = - (ThumbArgMove){.kind = THUMB_ARG_MOVE_IMM, .dst_reg = base_reg, .imm = (uint32_t)imm64, .sym = NULL}; - if (words >= 2) - reg_moves[move_count++] = (ThumbArgMove){ - .kind = THUMB_ARG_MOVE_IMM, .dst_reg = base_reg + 1, .imm = (uint32_t)(imm64 >> 32), .sym = NULL}; - for (int w = 0; w < words; w++) - ctx->call_site->registers_map |= (1 << (base_reg + w)); - } - else - { - int src_align = 0; - irop_type_size_align(*arg, &src_align); - move_count = build_reg_move_struct(reg_moves, move_count, mop, loc, base_reg, ctx->call_site, src_align); - } - } - else if (is_64bit) - { - if (loc->reg_count < 2) - tcc_error("compiler_error: 64-bit register argument has insufficient registers"); - move_count = build_reg_move_64bit(reg_moves, move_count, mop, arg, base_reg, ctx->call_site, tcc_state->ir); - } - else - { - move_count = build_reg_move_32bit(reg_moves, move_count, mop, arg, base_reg, ctx->call_site, tcc_state->ir); - } - } - - return move_count; -} - -/* Place hard-float single-precision arguments into their VFP argument registers - * (s0..s15). Emitted AFTER the GPR argument moves so r0-r3 already hold their - * final values. VFP-resident sources form a permutation among the s-registers, - * resolved as a parallel move with VFP_SCRATCH0 (s14) breaking any cycle; other - * sources (imm/memory) are materialized through a GPR scratch that excludes the - * GPR argument registers. */ -static void emit_vfp_arg_moves(CallGenContext *ctx) -{ - int dst[16], src[16]; - const MachineOperand *mop[16]; - int n = 0; - const uint32_t gpr_excl_all = (uint32_t)(ctx->call_site->registers_map & 0xffffu); - - for (int i = 0; i < ctx->argc && n < 16; ++i) - { - const TCCAbiArgLoc *loc = &ctx->layout->locs[i]; - if (loc->kind != TCC_ABI_LOC_VFP_REG) - continue; - - /* Double: the value lives in a GPR pair (doubles are not VFP-resident — - * there is no double-precision arithmetic on this FPU), so pack the two - * halves straight into the d-register. Emitted immediately: d-registers - * are not sources for any other argument move. */ - if (loc->reg_count == 2) - { - MachineCodegenContext mctx = {0}; - MachineOperand m = ctx->mops[i]; - MachineOperand lo = mach_make_lo_half(&m); - MachineOperand hi = mach_make_hi_half(&m); - lo.btype = IROP_BTYPE_INT32; - hi.btype = IROP_BTYPE_INT32; - int rlo = mach_ensure_in_reg(&mctx, &lo, gpr_excl_all); - int rhi = mach_ensure_in_reg(&mctx, &hi, gpr_excl_all | (1u << (uint32_t)rlo)); - ot_check(th_vmov_2gp_dp((uint16_t)rlo, (uint16_t)rhi, (uint16_t)(loc->reg_base / 2), 0)); - mach_release_all(&mctx); - continue; - } - - dst[n] = loc->reg_base; - const MachineOperand *m = &ctx->mops[i]; - src[n] = (m->kind == MACH_OP_VFP_REG) ? m->u.reg.r0 : -1; /* -1 = materialize */ - mop[n] = m; - n++; - } - if (n == 0) - return; - - const uint32_t gpr_excl = gpr_excl_all; - int remaining = n; - while (remaining > 0) - { - int progressed = 0; - for (int i = 0; i < n; ++i) - { - if (dst[i] < 0) - continue; - /* Safe to emit when this destination is not still needed as a source. */ - int needed = 0; - for (int j = 0; j < n; ++j) - if (dst[j] >= 0 && j != i && src[j] == dst[i]) - { - needed = 1; - break; - } - if (needed) - continue; - if (src[i] >= 0) - { - if (src[i] != dst[i]) - ot_check(th_vmov_register((uint16_t)dst[i], (uint16_t)src[i], 0)); /* vmov.f32 sd, ss */ - } - else - { - MachineCodegenContext mctx = {0}; - int gpr = mach_ensure_in_reg(&mctx, mop[i], gpr_excl); - ot_check(th_vmov_gp_sp((uint16_t)gpr, (uint16_t)dst[i], 0)); /* sd = gpr */ - mach_release_all(&mctx); - } - dst[i] = -1; - remaining--; - progressed = 1; - } - if (!progressed) - { - /* Only cycles of VFP-resident moves remain: park one source in s14. */ - for (int i = 0; i < n; ++i) - if (dst[i] >= 0 && src[i] >= 0) - { - ot_check(th_vmov_register(VFP_SCRATCH0, (uint16_t)src[i], 0)); - src[i] = VFP_SCRATCH0; - break; - } - } - } -} - -/* Pre-save stack arguments that source from R0-R3 before register shuffle */ -static void presave_stack_args_from_arg_regs(CallGenContext *ctx) -{ - for (int i = 0; i < ctx->argc; ++i) - { - const TCCAbiArgLoc *loc = &ctx->layout->locs[i]; - const MachineOperand *mop = &ctx->mops[i]; - const int bt = mop->btype; - - if (loc->kind == TCC_ABI_LOC_REG || loc->kind == TCC_ABI_LOC_VFP_REG) - continue; - if (bt == IROP_BTYPE_STRUCT || mop->is_complex) - continue; - if (mop->kind != MACH_OP_REG || mop->needs_deref) - continue; - - if (mop->is_64bit) - { - /* Pre-save 64-bit register pair if either register is in R0-R3. - * The register arg shuffle will overwrite R0-R3, so both halves - * must be stored to the stack before that happens. */ - int r0 = mop->u.reg.r0; - int r1 = mop->u.reg.r1; - if ((thumb_is_hw_reg(r0) && r0 <= ARM_R3) || (thumb_is_hw_reg(r1) && r1 <= ARM_R3)) - { - int stack_offset = loc->stack_off; - if (thumb_is_hw_reg(r0)) - store_word_to_stack(r0, stack_offset); - if (thumb_is_hw_reg(r1)) - store_word_to_stack(r1, stack_offset + 4); - } - } - else - { - /* Only pre-save if operand is in R0-R3 (arg registers that get overwritten). */ - if (mop->u.reg.r0 <= ARM_R3) - { - store_word_to_stack(mop->u.reg.r0, loc->stack_off); - } - } - } -} - -/* True for a plain 32-bit immediate argument destined for a stack slot. */ -static int is_simple_imm_stack_arg(const TCCAbiArgLoc *loc, const MachineOperand *mop) -{ - return loc->kind != TCC_ABI_LOC_REG && loc->kind != TCC_ABI_LOC_VFP_REG && mop->kind == MACH_OP_IMM && - !mop->is_64bit && mop->btype != IROP_BTYPE_STRUCT && !mop->is_complex; -} - -/* One collected immediate stack store, for the grouped/windowed emission path. */ -typedef struct StackImmArg -{ - int off; - uint32_t val; -} StackImmArg; - -/* Order by 4 KB window, then value, then offset. Grouping equal values within a - * window lets each distinct value be materialized once per window instead of once - * per argument; the window ordering bounds base-register re-materialization. */ -static int stack_imm_arg_cmp(const void *a, const void *b) -{ - const StackImmArg *x = (const StackImmArg *)a; - const StackImmArg *y = (const StackImmArg *)b; - int wx = x->off & ~0xFFF, wy = y->off & ~0xFFF; - if (wx != wy) - return wx < wy ? -1 : 1; - if (x->val != y->val) - return x->val < y->val ? -1 : 1; - if (x->off != y->off) - return x->off < y->off ? -1 : 1; - return 0; -} - -/* Emit a single non-simple-immediate stack argument (struct/complex/64-bit, or a - * non-immediate 32-bit source). Extracted from place_stack_arguments so both the - * inline and the grouped emission paths share identical handling. */ -static void place_one_stack_arg(CallGenContext *ctx, const TCCAbiArgLoc *loc, const MachineOperand *mop, - int stack_offset, int arg_index) -{ - if (mop->btype == IROP_BTYPE_STRUCT || mop->is_complex) - { - /* Complex values in a register pair: store the stack portion directly - * from registers instead of treating the pair as a memory pointer. */ - if (mop->is_complex && mop->kind == MACH_OP_REG && !mop->needs_deref && mop->is_64bit) - { - int words_in_regs = (loc->kind == TCC_ABI_LOC_REG_STACK) ? loc->reg_count : 0; - int stack_bytes = (loc->kind == TCC_ABI_LOC_REG_STACK) ? loc->stack_size : loc->size; - int stack_words = (stack_bytes + 3) / 4; - int pair_regs[2] = {mop->u.reg.r0, mop->u.reg.r1}; - for (int w = 0; w < stack_words; w++) - { - int reg_idx = words_in_regs + w; - if (reg_idx < 2) - store_word_to_stack(pair_regs[reg_idx], stack_offset + w * 4); - } - } - else if (mop->is_complex && mop->kind == MACH_OP_IMM) - { - /* Complex immediate on stack: split 64-bit packed value into words. */ - const uint64_t imm64 = (uint64_t)mop->u.imm.val; - int words_in_regs = (loc->kind == TCC_ABI_LOC_REG_STACK) ? loc->reg_count : 0; - int stack_bytes = (loc->kind == TCC_ABI_LOC_REG_STACK) ? loc->stack_size : loc->size; - int stack_words = (stack_bytes + 3) / 4; - int scr = find_call_scratch(0, ctx->arg_move_dst_mask); - for (int w = 0; w < stack_words; w++) - { - int word_idx = words_in_regs + w; - uint32_t word_val = (uint32_t)(imm64 >> (word_idx * 32)); - load_immediate(scr, word_val, NULL, false); - store_word_to_stack(scr, stack_offset + w * 4); - } - } - else - { - /* Struct's natural alignment gates source-side LDRD (see - * place_stack_arg_struct). Default conservatively to 1 (no LDRD) when - * the originating IR operand is unavailable. */ - int src_align = 1; - if (ctx->args && arg_index >= 0 && arg_index < ctx->argc) - { - int a = 0; - irop_type_size_align(ctx->args[arg_index], &a); - if (a > 0) - src_align = a; - } - place_stack_arg_struct(mop, loc, stack_offset, src_align); - } - } - else if (mop->is_64bit) - place_stack_arg_64bit(mop, stack_offset, tcc_state->ir, ctx->arg_move_dst_mask); - else - place_stack_arg_32bit(mop, stack_offset, ctx); -} - -/* Inline (original-order) emission of every stack argument. Used for the common - * case where stack args stay within the immediate-offset store range. */ -static void place_stack_arguments_inline(CallGenContext *ctx) -{ - int cached_imm_reg = -1; - uint32_t cached_imm_val = 0; - - for (int i = 0; i < ctx->argc; ++i) - { - const TCCAbiArgLoc *loc = &ctx->layout->locs[i]; - const MachineOperand *mop = &ctx->mops[i]; - - if (loc->kind == TCC_ABI_LOC_REG || loc->kind == TCC_ABI_LOC_VFP_REG) - continue; - - int stack_offset = loc->stack_off; - - if (is_simple_imm_stack_arg(loc, mop)) - { - uint32_t val = (uint32_t)mop->u.imm.val; - int scr = find_call_scratch(0, ctx->arg_move_dst_mask); - if (cached_imm_reg != scr || cached_imm_val != val) - { - load_immediate(scr, val, NULL, false); - cached_imm_reg = scr; - cached_imm_val = val; - } - store_word_to_stack(scr, stack_offset); - continue; - } - - cached_imm_reg = -1; - place_one_stack_arg(ctx, loc, mop, stack_offset, i); - } -} - -/* Place all stack arguments. - * - * For the common case the inline path is byte-identical to before. When simple - * 32-bit immediate stack args spill beyond the immediate-offset store range - * (offset > 4092) — exactly where the naive path emits movw+indexed (3 instr/arg) - * — a windowed/grouped path is used instead: - * - a base register holds sp+window so each store is a single str.w [rb,#disp] - * (re-materialized only when crossing a 4 KB window, ~once / 1024 stores); - * - the immediate stores are reordered by (window, value) so each distinct - * value is loaded once per window rather than once per argument. - * Reordering pure-immediate stores to distinct, non-aliasing stack slots leaves - * the pre-call stack image unchanged, so it is observationally identical. */ -/* TCC_NO_STACK_ARG_GROUP restores the ungrouped inline path, the control arm - * of the windowed stack-argument A/B. */ -TCC_DBG_ENV_FLAG(th_no_stack_arg_group, "TCC_NO_STACK_ARG_GROUP") - -static void place_stack_arguments(CallGenContext *ctx) -{ - int max_imm_off = -1; - int imm_count = 0; - for (int i = 0; i < ctx->argc; ++i) - { - const TCCAbiArgLoc *loc = &ctx->layout->locs[i]; - const MachineOperand *mop = &ctx->mops[i]; - if (is_simple_imm_stack_arg(loc, mop)) - { - imm_count++; - if (loc->stack_off > max_imm_off) - max_imm_off = loc->stack_off; - } - } - - if (!(max_imm_off > 4092 && imm_count >= 2) || th_no_stack_arg_group()) - { - place_stack_arguments_inline(ctx); - return; - } - - /* --- Windowed/grouped path --- */ - - /* Pass 1: emit every non-simple-immediate stack arg first, in original order. */ - for (int i = 0; i < ctx->argc; ++i) - { - const TCCAbiArgLoc *loc = &ctx->layout->locs[i]; - const MachineOperand *mop = &ctx->mops[i]; - if (loc->kind == TCC_ABI_LOC_REG || loc->kind == TCC_ABI_LOC_VFP_REG || is_simple_imm_stack_arg(loc, mop)) - continue; - place_one_stack_arg(ctx, loc, mop, loc->stack_off, i); - } - - /* Reserve two stable scratch registers: rv (holds the value) and rb (base - * address). Both are free across the whole argument-setup region — the call's - * register args are moved in afterwards, and find_call_scratch only returns - * registers that are dead here or are arg-move destinations (overwritten - * later). Prefer the lower-numbered register for rv so value materialization - * can use the 16-bit MOVS encoding. */ - int s0 = find_call_scratch(0, ctx->arg_move_dst_mask); - int s1 = find_call_scratch(1u << s0, ctx->arg_move_dst_mask); - if (s1 < s0) - { - int t = s0; - s0 = s1; - s1 = t; - } - int rv = s0, rb = s1; - int regs_ok = (rv != rb && rv >= 0 && rv < 16 && rb >= 0 && rb < 16 && rv != ARM_SP && rv != ARM_PC && - rb != ARM_SP && rb != ARM_PC); - - StackImmArg *items = regs_ok ? tcc_malloc(sizeof(StackImmArg) * imm_count) : NULL; - if (!items) - { - /* Out of stable registers (or alloc failure): emit the immediate args inline. */ - int cached_imm_reg = -1; - uint32_t cached_imm_val = 0; - for (int i = 0; i < ctx->argc; ++i) - { - const TCCAbiArgLoc *loc = &ctx->layout->locs[i]; - const MachineOperand *mop = &ctx->mops[i]; - if (!is_simple_imm_stack_arg(loc, mop)) - continue; - uint32_t val = (uint32_t)mop->u.imm.val; - int scr = find_call_scratch(0, ctx->arg_move_dst_mask); - if (cached_imm_reg != scr || cached_imm_val != val) - { - load_immediate(scr, val, NULL, false); - cached_imm_reg = scr; - cached_imm_val = val; - } - store_word_to_stack(scr, loc->stack_off); - } - return; - } - - int n = 0; - for (int i = 0; i < ctx->argc; ++i) - { - const TCCAbiArgLoc *loc = &ctx->layout->locs[i]; - const MachineOperand *mop = &ctx->mops[i]; - if (!is_simple_imm_stack_arg(loc, mop)) - continue; - items[n].off = loc->stack_off; - items[n].val = (uint32_t)mop->u.imm.val; - n++; - } - qsort(items, n, sizeof(StackImmArg), stack_imm_arg_cmp); - - uint32_t saved_excl = scratch_global_exclude; - scratch_global_exclude |= (1u << rv) | (1u << rb); - - int cur_window = -1; /* base offset of the window currently in rb */ - int have_val = 0; - uint32_t cur_val = 0; - for (int k = 0; k < n; ++k) - { - int off = items[k].off; - uint32_t val = items[k].val; - int window = off & ~0xFFF; - int disp = off & 0xFFF; - int base_reg; - - if (window == 0) - { - base_reg = ARM_SP; /* sp+0 — store directly off sp, no base register needed */ - } - else - { - if (window != cur_window) - { - thumb_opcode op = th_add_imm(rb, ARM_SP, (uint32_t)window, flags_safe(), ENFORCE_ENCODING_NONE); - if (is_valid_opcode(op)) - ot(op); - else - { - load_full_const(rb, PREG_NONE, (uint32_t)window, 0); - ot_check(th_add_reg(rb, ARM_SP, rb, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); - } - } - base_reg = rb; - } - cur_window = window; - - if (!have_val || cur_val != val) - { - load_immediate(rv, val, NULL, false); - have_val = 1; - cur_val = val; - } - - if (!store_word_to_base(rv, base_reg, disp, 0)) - { - /* disp <= 4092 always encodes via str.w; keep a correct fallback regardless. */ - ScratchRegAlloc sc = get_scratch_reg_with_save((1u << rv) | (1u << base_reg)); - load_immediate(sc.reg, (uint32_t)off, NULL, false); - ot_check(th_str_reg(rv, ARM_SP, sc.reg, THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); - restore_scratch_reg(&sc); - } - } - - scratch_global_exclude = saved_excl; - tcc_free(items); -} - -/* Handle return value after call (MOP path). - * The 'dest_mop' describes where the return value must be written. - * mach_writeback_dest() handles all destination kinds: - * MACH_OP_REG — emit MOV dest.r0, ARM_R0 when needed - * MACH_OP_SPILL — emit STR R0 to the spill slot - * MACH_OP_PARAM_STACK — emit STR R0 to the param stack slot - * MACH_OP_NONE — no-op (void return or drop_value) - * 64-bit pairs (int64, double, complex float) are split into lo/hi halves - * via mach_make_lo_half / mach_make_hi_half (R0 → lo, R1 → hi). */ -/* True when a call targets a soft-float __aeabi_* runtime helper, which returns - * a float in R0 (soft ABI) even under hard-float — as opposed to a user - * function, which returns it in s0. The __aeabi_ namespace is reserved, so the - * name is an unambiguous signal. */ -static int mach_callee_is_aeabi(const MachineOperand *func_mop) -{ - if (func_mop->kind != MACH_OP_SYMBOL || !func_mop->u.sym.sym) - return 0; - const char *name = get_tok_str(func_mop->u.sym.sym->v, NULL); - return name && strncmp(name, "__aeabi_", 8) == 0; -} - -/* True when the callee is a variadic (FUNC_ELLIPSIS) function. Such a callee - * receives every argument by the base (GPR) standard, so FP arguments are not - * placed in VFP registers. Indirect calls (no symbol) are assumed non-variadic - * — function pointers to variadic functions are rare. */ -static int mach_callee_is_variadic(const MachineOperand *func_mop) -{ - if (func_mop->kind != MACH_OP_SYMBOL || !func_mop->u.sym.sym) - return 0; - Sym *s = func_mop->u.sym.sym; - return s->type.ref && s->type.ref->f.func_type == FUNC_ELLIPSIS; -} - -/* True when the callee is a function this translation unit has already - * emitted, so the call cannot leave the module and R9 survives it. - * - * Only an already-defined symbol qualifies: the test reads the ELF symbol, - * and a forward reference (callee defined later in the TU, or extern) is - * still SHN_UNDEF here, so it conservatively keeps the reload. That also - * makes the predicate stable across the dry run and the real pass, which it - * must be or literal-pool windows desync -- the dry run skips - * put_extern_sym, but that only ever creates UNDEF entries, so a symbol - * reads defined in both passes or neither. - * - * STT_FUNC is required so a same-named data symbol cannot qualify, and the - * target section must be executable: an SHN_ABS or data-section symbol is - * not a function body we compiled under this R9 contract. */ -static int thumb_callee_in_this_module(const MachineOperand *func_mop) -{ - Sym *sym; - ElfSym *esym; - Section *sec; - - if (!text_and_data_separation) - return 0; - if (func_mop->kind != MACH_OP_SYMBOL || !func_mop->u.sym.sym) - return 0; - sym = func_mop->u.sym.sym; - if (sym->v & SYM_FIELD) - return 0; - esym = elfsym(sym); - if (!esym || esym->st_shndx == SHN_UNDEF || esym->st_shndx == SHN_ABS || - esym->st_shndx >= tcc_state->nb_sections) - return 0; - if (ELFW(ST_TYPE)(esym->st_info) != STT_FUNC) - return 0; - sec = tcc_state->sections[esym->st_shndx]; - return sec && (sec->sh_flags & SHF_EXECINSTR) != 0; -} - -static void handle_return_value_mop(const MachineOperand *dest_mop, int drop_value, int soft_float_return) -{ - if (drop_value) - return; - - /* Hard-float single-precision return arrives in s0 (AAPCS VFP) — unless the - * callee is a soft __aeabi_* helper, which returns it in R0. */ - if (tcc_state && tcc_state->float_abi == ARM_HARD_FLOAT && !soft_float_return && !dest_mop->is_64bit && - dest_mop->btype == IROP_BTYPE_FLOAT32) - { - if (dest_mop->kind == MACH_OP_VFP_REG) - { - if (dest_mop->u.reg.r0 != 0) - ot_check(th_vmov_register((uint16_t)dest_mop->u.reg.r0, 0, 0)); /* s_dest = s0 */ - } - else - { - MachineCodegenContext ctx = {0}; - int r = mach_get_dest_reg(&ctx, dest_mop, 0); - ot_check(th_vmov_gp_sp((uint16_t)r, 0, 1)); /* r = s0 */ - mach_writeback_dest(dest_mop, r); - mach_release_all(&ctx); - } - return; - } - - /* Hard-float double return arrives in d0 (AAPCS VFP) — unless the callee is a - * soft __aeabi_* helper, whose result comes back in the R0:R1 pair. */ - if (tcc_state && tcc_state->float_abi == ARM_HARD_FLOAT && !soft_float_return && dest_mop->is_64bit && - dest_mop->btype == IROP_BTYPE_FLOAT64 && !dest_mop->is_complex) - { - MachineCodegenContext ctx = {0}; - MachineOperand lo = mach_make_lo_half(dest_mop); - MachineOperand hi = mach_make_hi_half(dest_mop); - lo.btype = IROP_BTYPE_INT32; - hi.btype = IROP_BTYPE_INT32; - int rlo = mach_get_dest_reg(&ctx, &lo, 0); - int rhi = mach_get_dest_reg(&ctx, &hi, (1u << (uint32_t)rlo)); - ot_check(th_vmov_2gp_dp((uint16_t)rlo, (uint16_t)rhi, 0, 1)); /* rlo,rhi = d0 */ - mach_writeback_dest(&lo, rlo); - mach_writeback_dest(&hi, rhi); - mach_release_all(&ctx); - return; - } - - if (dest_mop->is_64bit) - { - /* 64-bit return value: R0 = low word, R1 = high word (AAPCS). */ - MachineOperand lo = mach_make_lo_half(dest_mop); - lo.btype = IROP_BTYPE_INT32; - MachineOperand hi = mach_make_hi_half(dest_mop); - hi.btype = IROP_BTYPE_INT32; - mach_writeback_dest(&lo, ARM_R0); - mach_writeback_dest(&hi, ARM_R1); - return; - } - mach_writeback_dest(dest_mop, ARM_R0); -} - -/* ======================================================================== */ - -/* tcc_gen_machine_func_call_mop — MOP-path function call code generator. - * - * The function target and return-value destination are passed as MachineOperands. - * The call_id_op is always an immediate IROperand (no fill needed). - * - * Phase 5g: func_mop replaces the old filled IROperand func_target. - * gcall_or_jump_mop() replaces gcall_or_jump_ir(). - */ -ST_FUNC void tcc_gen_machine_func_call_mop(MachineOperand func_mop, IROperand call_id_op, MachineOperand dest_mop, - int drop_value, TCCIRState *ir, int call_idx) -{ - /* === Validation === */ - if (irop_is_none(call_id_op) || !ir) - tcc_error("compiler_error: func_call_op requires call_id+ir"); - - const int call_id = TCCIR_DECODE_CALL_ID(call_id_op.u.imm32); - const int argc_hint = TCCIR_DECODE_CALL_ARGC(call_id_op.u.imm32); - - ThumbGenCallSite *call_site = thumb_get_call_site_for_id(call_id); - if (!call_site) - tcc_error("compiler_error: no call site found for call_id=%d", call_id); - - /* === Build ABI layout === */ - TCCAbiCallLayout layout; - memset(&layout, 0, sizeof(layout)); - - /* Hard-float VFP argument passing applies to non-variadic user functions. - * Soft __aeabi_* helpers take their float arguments in GPRs regardless, and a - * variadic callee passes every argument by the base (GPR) standard. */ - layout.hard_float = - (tcc_state && tcc_state->float_abi == ARM_HARD_FLOAT) && !mach_callee_is_aeabi(&func_mop); - layout.is_variadic = mach_callee_is_variadic(&func_mop); - - small_sequence(ThumbIROperandSequence) args_owner = {0}; - small_sequence(ThumbMachineOperandSequence) mops_owner = {0}; - const int argc = thumb_build_call_layout_from_ir(ir, call_idx, call_id, argc_hint, &layout, &args_owner, &mops_owner); - if (argc < 0) - tcc_error("compiler_error: failed to build call layout for call_id=%d", call_id); - - int stack_size = (argc > 0) ? (int)layout.stack_size : 0; - - /* === Setup call context === */ - CallGenContext ctx = { - .call_site = call_site, - .layout = &layout, - .args = ThumbIROperandSequence_data(&args_owner), - .mops = ThumbMachineOperandSequence_data(&mops_owner), - .argc = argc, - .stack_size = stack_size, - }; - - /* Set tail_call_pending if this is a tail-call-only function. */ - if (ir->tail_call_only) - tail_call_pending = 1; - - /* === Preserve nested call registers (R0-R3, R9) via STR to frame === - * Instead of PUSH/POP (which moves SP), store to the pre-reserved - * nested-call save area in the frame. SP stays fixed. */ - int arg_regs_in_use = call_site->registers_map & 0x0F; - int arg_regs_save_mask = tail_call_pending ? 0 : (arg_regs_in_use); - - /* On yasos with no-pic-data-is-text-relative, R9 holds the GOT base and is - * caller-saved. Save it alongside the nested-call argument registers so it - * is restored after the callee returns. */ - if (!tail_call_pending && text_and_data_separation) - arg_regs_save_mask |= (1 << ARM_R9); - - /* ...but a callee in THIS module shares our GOT base and hands it back - * intact, so its reload is dead. R9 is not allocatable under - * text_and_data_separation, tail calls are disabled outright in this mode - * (ir/codegen.c), and every call site below reloads R9, so a function here - * either never touches R9 or returns with its own module's base in it -- - * which for a same-module callee is the value we already hold. Only the - * cross-module path breaks that: an import goes through a loader thunk that - * swaps in the callee's base and tail-jumps, so nothing restores ours. - * - * Note this suppresses the RELOAD only, never the mask bit: the mask also - * assigns save-area slots, and clearing R9 from it would slide R0-R3 down - * onto slot 0 and overwrite the prologue-stored GOT base for every other - * call site in the function. */ - int restore_r9 = 1; - if (arg_regs_save_mask & (1 << ARM_R9)) - restore_r9 = !thumb_callee_in_this_module(&func_mop); - - /* Save nested-call registers to pre-reserved frame area via STR. - * The nested save area is at [SP + ir->call_outgoing_size]. - * - * In functions with VLA/alloca the runtime SP has moved below the static - * frame, so [SP + off] would land inside the dynamically allocated memory - * (the callee then overwrites the saved R9/GOT base with user data). - * Address the slots FP-relative instead: the static SP equals - * FP - callee_push_size - epilogue_stack_dealloc. */ - int nested_save_sp_offset = ir ? ir->call_outgoing_size : 0; - int nested_save_fp_bias = tcc_state->func_dynamic_sp - ? -(callee_push_size + epilogue_stack_dealloc) - : 0; - int nested_save_count = 0; - if (arg_regs_save_mask) - { - /* R9 takes slot 0 so that its frame offset is the same at every call site - * in the function. That is what lets the store be hoisted: the GOT base - * is function-invariant, so the single store in the prologue dominates - * every reload below and the per-call-site store is redundant. It used to - * be emitted here, and because R9 was slotted after whichever of R0-R3 - * happened to be live the offset moved from call to call, which is what - * hid the redundancy. The argument registers, whose live set genuinely - * does vary per call site, are packed above it. - * - * Emitting the store here again would be harmless but wasteful: it was - * 45,329 instructions (165 KiB) across the compiler's own build. */ - if (arg_regs_save_mask & (1 << ARM_R9)) - nested_save_count = 1; - for (int r = 0; r < 16; r++) - { - if (r == ARM_R9) - continue; - if (arg_regs_save_mask & (1 << r)) - { - if (tcc_state->func_dynamic_sp) - tcc_gen_machine_store_to_stack_ex( - r, nested_save_sp_offset + nested_save_count * 4 + nested_save_fp_bias, - arg_regs_save_mask); - else - store_word_to_stack(r, nested_save_sp_offset + nested_save_count * 4); - nested_save_count++; - } - } - } - if (nested_save_count > dry_run_state.max_nested_saves) - dry_run_state.max_nested_saves = nested_save_count; - - /* Stack args are already placed in the pre-reserved outgoing area at [SP+0]. - * No need to adjust SP — the area was allocated in the prologue. */ - stack_size = (stack_size + 7) & ~7; /* 8-byte align */ - - /* === Save scratch exclusion state === */ - uint32_t saved_scratch_exclude = scratch_global_exclude; - - /* === Pre-save indirect call target if it resides in an argument register === - * - * When a function pointer is allocated to R0-R3 by the register allocator, - * the argument placement phase will overwrite those registers. Pre-move the - * pointer to a safe register before argument setup. - * - * Phase 5g: operates on MachineOperand func_mop instead of filled IROperand. - */ - { - const int is_direct = (func_mop.kind == MACH_OP_SYMBOL || func_mop.kind == MACH_OP_IMM); - if (!is_direct && func_mop.kind == MACH_OP_REG && !func_mop.needs_deref && func_mop.u.reg.r0 >= 0 && - func_mop.u.reg.r0 <= 3) - { - /* Find a free register outside R0-R3, R12 (stack-arg scratch), SP, PC. */ - uint32_t exclude = scratch_global_exclude | 0x0Fu | (1u << R_IP) | (1u << R_SP) | (1u << R_PC); - int safe_reg = PREG_NONE; - if (ir) - safe_reg = tcc_ls_find_free_scratch_reg(&ir->ls, ir->codegen_instruction_idx, exclude, ir->leaffunc); - - if (safe_reg == PREG_NONE || safe_reg < 0 || safe_reg >= 16 || safe_reg == R_SP || safe_reg == R_PC) - tcc_error("compiler_error: func_call_mop: cannot find safe register " - "to pre-save indirect call target (R%d)", - func_mop.u.reg.r0); - - /* Move function pointer from arg reg to safe reg. */ - thumb_shift no_shift = {THUMB_SHIFT_NONE, 0, THUMB_SHIFT_IMMEDIATE}; - ot_check_mov_reg(safe_reg, func_mop.u.reg.r0, flags_safe(), no_shift, ENFORCE_ENCODING_NONE, - false); - - /* Rewrite func_mop to point to the safe register. */ - func_mop.kind = MACH_OP_REG; - func_mop.u.reg.r0 = safe_reg; - func_mop.u.reg.r1 = -1; - func_mop.needs_deref = false; - - /* Protect the safe register from scratch allocation during arg setup. */ - scratch_global_exclude |= (1u << safe_reg); - } - } - - /* === Build register argument moves === */ - ThumbArgMove reg_moves[8]; - int reg_move_count = build_register_arg_moves(&ctx, reg_moves); - - /* === Compute arg_move_dst_mask and identity-move protection === - * - * Stack arguments are placed BEFORE register argument moves so that - * R0-R3 (non-identity move destinations) can serve as scratch registers - * for stack arg stores, saving 2 bytes per store (16-bit vs 32-bit encoding). - * - * arg_move_dst_mask: registers written by explicit (non-identity) reg moves. - * These will be overwritten by the moves, so they're safe as scratch. - * identity_mask: registers where the reg allocator already placed the correct - * value (no move entry created). These MUST be protected from clobbering. */ - { - uint32_t arg_move_dst_mask = 0; - for (int i = 0; i < reg_move_count; i++) - arg_move_dst_mask |= arg_move_write_set(®_moves[i]); - - /* Compute all register-arg destination registers from the ABI layout. */ - uint32_t all_reg_arg_dst = 0; - for (int i = 0; i < ctx.argc; i++) - { - const TCCAbiArgLoc *loc = &ctx.layout->locs[i]; - if (loc->kind == TCC_ABI_LOC_REG || loc->kind == TCC_ABI_LOC_REG_STACK) - { - int base = ARM_R0 + loc->reg_base; - for (int w = 0; w < loc->reg_count; w++) - all_reg_arg_dst |= (1u << (base + w)); - } - } - - /* Protect identity-move registers (value already in place, no move entry). */ - uint32_t identity_mask = all_reg_arg_dst & ~arg_move_dst_mask; - scratch_global_exclude |= identity_mask; - - ctx.arg_move_dst_mask = arg_move_dst_mask; - } - - /* Pre-save stack args sourcing from R0-R3 before register shuffle */ - presave_stack_args_from_arg_regs(&ctx); - - /* === Place stack arguments FIRST === - * R0-R3 that are non-identity move destinations can be used as scratch - * via arg_move_dst_mask in find_call_scratch, yielding 16-bit STR - * encodings instead of 32-bit STR.W with R12. */ - place_stack_arguments(&ctx); - - /* === Now block all R0-R3 and emit register argument moves === */ - scratch_global_exclude |= 0x0F; - thumb_emit_parallel_arg_moves(reg_moves, reg_move_count); - emit_vfp_arg_moves(&ctx); - - /* === Tail call: tear down frame before branching === */ - if (tail_call_pending) - { - /* For indirect calls, the target may be in a callee-saved register that - * will be popped. Move it to R_IP (R12) before frame teardown. */ - if (func_mop.kind == MACH_OP_REG && !func_mop.needs_deref && - func_mop.u.reg.r0 >= R4 && func_mop.u.reg.r0 <= R11) - { - ot_check_mov_reg(R_IP, func_mop.u.reg.r0, flags_safe(), - THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE, false); - func_mop.u.reg.r0 = R_IP; - } - if (epilogue_stack_dealloc > 0) - gadd_sp_ex(epilogue_stack_dealloc, R_IP); - /* Only pop true callee-saved registers (R4-R11). R0-R3 may be pushed - * for alignment but now hold call arguments — popping them would clobber - * the prepared args. Skip non-callee slots FIRST (they sit at lower - * addresses after push), then pop callee-saved from correct position. */ - uint32_t callee_pop = pushed_registers & 0x0FF0u; /* R4-R11 only */ - uint32_t non_callee = pushed_registers & ~callee_pop & ~(1u << R_LR) & ~(1u << R_PC); - int non_callee_bytes = __builtin_popcount(non_callee) * 4; - if (non_callee_bytes > 0) - gadd_sp_ex(non_callee_bytes, R_IP); - if (callee_pop) - ot_check(th_pop(callee_pop)); - } - - /* === Emit call === */ - gcall_or_jump_mop(0, func_mop); - /* Restore scratch register exclusion */ - scratch_global_exclude = saved_scratch_exclude; - - if (tail_call_pending) - { - tail_call_pending = 0; - goto call_cleanup; - } - - handle_return_value_mop(&dest_mop, drop_value, mach_callee_is_aeabi(&func_mop)); - - /* === Cleanup: restore nested-call saved registers via LDR === */ - if (arg_regs_save_mask) - { - /* Match the FP-relative addressing used by the save side in functions - * with VLA/alloca (runtime SP has moved; see the save block above). */ - const int restore_base = tcc_state->func_dynamic_sp ? R_FP : ARM_SP; - /* Mirror the slot assignment of the save block: R9 is at slot 0 (stored - * once in the prologue), the argument registers start above it. */ - int restore_idx = (arg_regs_save_mask & (1 << ARM_R9)) ? 1 : 0; - for (int r = 0; r < 16; r++) - { - if (r == ARM_R9 && !restore_r9) - continue; /* same-module callee: our GOT base is still live in R9 */ - if (arg_regs_save_mask & (1 << r)) - { - int slot = (r == ARM_R9) ? 0 : restore_idx; - int off = nested_save_sp_offset + slot * 4 + nested_save_fp_bias; - int sign = (off < 0); - int abs_off = sign ? -off : off; - /* R9 restore in text_and_data_separation mode needs the write guard - * temporarily lifted — the safety check blocks all R9 writes, but - * we are legitimately restoring it after a call. */ - if (r == ARM_R9 && text_and_data_separation) - allow_r9_write = 1; - if (!load_word_from_base(r, restore_base, abs_off, sign)) - { - ScratchRegAlloc osc = get_scratch_reg_with_save((1u << r)); - load_immediate(osc.reg, off, NULL, false); - ot_check(th_ldr_reg(r, restore_base, osc.reg, THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); - restore_scratch_reg(&osc); - } - if (r == ARM_R9 && text_and_data_separation) - allow_r9_write = 0; - if (r != ARM_R9) - restore_idx++; - } - } - } - - call_site->registers_map &= ~0x0F; /* Clear R0-R3 */ - -call_cleanup: - if (layout.locs) - tcc_free(layout.locs); -} - -/* Check if a backward branch to target_ir can use a narrow encoding. - * For backward branches, the target code address is already known in - * ir_to_code_mapping (it was emitted earlier in this pass). - * current_ir_idx is the IR index of the branch instruction itself. - * Returns 1 if narrow encoding fits, 0 otherwise. */ -static int can_narrow_backward_branch(int32_t target_ir, int is_conditional, int current_ir_idx) -{ - TCCIRState *ir = tcc_state->ir; - if (!ir || !ir->ir_to_code_mapping) - return 0; - if (target_ir < 0 || target_ir >= ir->ir_to_code_mapping_size) - return 0; - - /* Forward branches have uninitialized ir_to_code_mapping[target_ir] (still 0). - * Only narrow genuinely backward branches where target was already emitted. */ - if (target_ir >= current_ir_idx) - return 0; - - int target_addr = (int)ir->ir_to_code_mapping[target_ir]; - /* ind is the current code address where the branch will be emitted. - * offset = target - (source + 4) for Thumb pipeline. */ - int offset = target_addr - ind - 4; - - /* Only backward branches (negative offset) are safe to narrow here */ - if (offset >= 0) - return 0; - - /* If emitting the narrow branch would first flush a pending literal pool, - * the branch source moves forward after this range check. A borderline - * T1/T2 branch can become out of range by the time backpatching runs, and - * th_patch_call() cannot widen an already-emitted 16-bit branch in place. */ - if (th_literal_pool_would_flush_for(2)) - return 0; - - return is_conditional ? branch_fits_t1(offset) : branch_fits_t2(offset); -} - -/* Check if a FORWARD branch to target_ir can use a narrow encoding. - * - * The target address is not known yet in the real pass — it is backpatched - * later — so the decision comes from the rehearsal pass, which laid the - * function out with the same registers and frame the real pass uses and - * differs from it only by emitting every branch wide. The real pass is - * therefore never larger, so the real distance between two IR instructions is - * <= their rehearsal distance, and "fits in range in the rehearsal" implies - * "fits in range for real". - * - * The one way that breaks is a literal pool flush landing between the branch - * and its target in the real pass but not in the rehearsal — it inserts up to - * ~1 KB. Since the real pass emits fewer bytes, code_size grows more slowly - * and a flush can only move LATER, so one sitting before the branch in the - * rehearsal could drift into the range. Rather than pin flush points, this - * refuses to narrow unless no flush is possible at all: the range adds at most - * dry_dist bytes of code and (entries in range * 4) bytes of pool, and if that - * upper bound stays below the flush threshold the monotonicity argument holds - * unconditionally. */ -/* Upper bound on how many bytes the real pass can shed, relative to the - * rehearsal, strictly between two IR instructions. The rehearsal emits every - * branch wide and never fuses CBZ, so the only shrink sources are: - * - a wide branch narrowed to its 16-bit form (4 -> 2, saves 2) - * - a CMP+B.W pair fused into a single CBZ/CBNZ (6 -> 2, saves 4) - * Both are bounded by 4 bytes per branch instruction in the range, so counting - * branches is a safe over-estimate. */ -static int rehearsal_max_shrink_between(TCCIRState *ir, int from_ir, int to_ir) -{ - int branches = 0; - for (int k = from_ir + 1; k < to_ir; k++) - { - TccIrOp op = (TccIrOp)ir->compact_instructions[k].op; - if (op == TCCIR_OP_JUMP || op == TCCIR_OP_JUMPIF) - branches++; - } - return branches * 4; -} - -/* Can `CMP rN,#0; B target` at current_ir_idx be fused into a single - * 16-bit CBZ/CBNZ? CBZ is forward-only with a 0..126 byte range and cannot be - * widened in place once committed (th_patch_call errors), so the check has to - * be sound in BOTH directions, unlike plain narrowing which only needs an - * upper bound: - * - * upper — the real offset is at most the rehearsal-derived one, because the - * real pass only ever shrinks; - * lower — the real offset is at least that minus the maximum shrink the - * range can undergo, and it must not go below 0. - * - * The fusion itself removes 6 bytes (CMP + wide branch) and puts back 2 at the - * branch site, which is why the base offset is dry_dist - 8 rather than the - * dry_dist - 4 that plain narrowing uses. */ -ST_FUNC int tcc_gen_machine_cbz_forward_ok(int32_t target_ir, int current_ir_idx) -{ - TCCIRState *ir = tcc_state->ir; - if (!ir || !ir->codegen_cbz_dry_mapping || !ir->codegen_dry_pool_entries) - return 0; - if (current_ir_idx < 0 || target_ir <= current_ir_idx) - return 0; - if (target_ir >= ir->ir_to_code_mapping_size || target_ir >= ir->next_instruction_index) - return 0; - - const uint32_t *dry = ir->codegen_cbz_dry_mapping; - int dry_dist = (int)dry[target_ir] - (int)dry[current_ir_idx]; - if (dry_dist < 0) - return 0; - - int max_offset = dry_dist - 8; - int min_offset = max_offset - rehearsal_max_shrink_between(ir, current_ir_idx, target_ir); - if (min_offset < 0 || max_offset > 126) - return 0; - - /* A pool flush anywhere in the range would push the branch out of its 126-byte - * reach, and unlike a wide branch it cannot be repaired. Same bound as - * can_narrow_forward_branch. */ - if (th_literal_pool_would_flush_for(2)) - return 0; - int entries_in_range = - (int)ir->codegen_dry_pool_entries[target_ir] - (int)ir->codegen_dry_pool_entries[current_ir_idx]; - if (entries_in_range < 0) - return 0; - /* The emitter flushes when th_pool_span_after() reaches the range limit, - * evaluated continuously while the range's instructions emit. This - * one-shot estimate bounds the span at any point in the range: the current - * span plus the range's code bytes plus the pool bytes its entries can - * add. The real pass can still drift past it — per-emit upcoming bytes, - * dry-vs-real sizing differences. Fuzz seed float:8061 passed a hairline - * check and then took a 188-byte in-range flush (COMPILE_FAIL at - * th_patch_call, offset 204). Require an extra 64-byte cushion so a - * hairline pass can't commit an unrepairable CBZ. */ - int pressure = th_pool_span_after(8) + dry_dist + entries_in_range * 4; - if (pressure >= THUMB_POOL_RANGE_LIMIT - THUMB_POOL_FLUSH_SLACK - 64) - return 0; - - return 1; -} - -/* Return jumps pass target_ir == -1: they go to the epilogue, which has no IR - * index, so can_narrow_forward_branch cannot size them and they were always - * emitted wide. The rehearsal recorded where the body ends — which is exactly - * where the epilogue begins — so the same monotonicity argument applies: the - * real pass is never larger, so a distance that fits in the rehearsal fits for - * real. Same pool-flush bound as the forward case, using the running entry - * total at the end of the function. */ -static int can_narrow_epilogue_branch(int32_t target_ir, int current_ir_idx) -{ - TCCIRState *ir = tcc_state->ir; - if (target_ir != -1) - return 0; - if (!ir || !ir->codegen_cbz_dry_mapping || !ir->codegen_dry_pool_entries) - return 0; - if (current_ir_idx < 0 || current_ir_idx >= ir->next_instruction_index) - return 0; - if (ir->codegen_rehearsal_end == 0) - return 0; - - int dry_dist = (int)ir->codegen_rehearsal_end - (int)ir->codegen_cbz_dry_mapping[current_ir_idx]; - if (dry_dist < 0) - return 0; - if (!branch_fits_t2(dry_dist - 4)) - return 0; - - if (th_literal_pool_would_flush_for(2)) - return 0; - - /* Bound the pool pressure over the whole remaining body: entries added from - * here to the last instruction is the most that can accumulate before the - * epilogue. */ - int last = ir->next_instruction_index - 1; - int entries_in_range = (int)ir->codegen_dry_pool_entries[last] - - (int)ir->codegen_dry_pool_entries[current_ir_idx]; - if (entries_in_range < 0) - return 0; - int pressure = th_pool_span_after(8) + dry_dist + entries_in_range * 4; - if (pressure >= THUMB_POOL_RANGE_LIMIT - THUMB_POOL_FLUSH_SLACK) - return 0; - - return 1; -} - -static int can_narrow_forward_branch(int32_t target_ir, int is_conditional, int current_ir_idx) -{ - TCCIRState *ir = tcc_state->ir; - if (!ir || !ir->codegen_cbz_dry_mapping || !ir->codegen_dry_pool_entries) - return 0; - if (current_ir_idx < 0 || target_ir <= current_ir_idx) - return 0; - if (target_ir >= ir->ir_to_code_mapping_size || target_ir >= ir->next_instruction_index) - return 0; - - const uint32_t *dry = ir->codegen_cbz_dry_mapping; - int dry_dist = (int)dry[target_ir] - (int)dry[current_ir_idx]; - if (dry_dist < 0) - return 0; - - /* dry[current_ir_idx] is the START of the IR instruction; the branch may sit - * a few bytes into it, which only makes the true offset smaller. Dropping - * the 4-byte Thumb pipeline bias keeps the estimate conservative. */ - int est_offset = dry_dist - 4; - if (!(is_conditional ? branch_fits_t1(est_offset) : branch_fits_t2(est_offset))) - return 0; - - /* A flush scheduled at this very point would move the branch itself. */ - if (th_literal_pool_would_flush_for(2)) - return 0; - - int entries_in_range = - (int)ir->codegen_dry_pool_entries[target_ir] - (int)ir->codegen_dry_pool_entries[current_ir_idx]; - if (entries_in_range < 0) - return 0; - /* Same bound as tcc_gen_machine_cbz_forward_ok: no flush can trigger in - * the range while this stays under the flush threshold. */ - int pressure = th_pool_span_after(8) + dry_dist + entries_in_range * 4; - if (pressure >= THUMB_POOL_RANGE_LIMIT - THUMB_POOL_FLUSH_SLACK) - return 0; - - return 1; -} - -ST_FUNC int tcc_gen_machine_jump_mop(TccIrOp op, int32_t target_ir, int ir_idx) -{ - - if (dry_run_state.active) - { - /* Emit 32-bit placeholder for code size tracking */ - ot_check(th_b_t4(0)); - return 4; - } - - /* Real pass: try narrow encoding, backward first then forward */ - if (can_narrow_backward_branch(target_ir, 0, ir_idx) || can_narrow_forward_branch(target_ir, 0, ir_idx) || - can_narrow_epilogue_branch(target_ir, ir_idx)) - { - ot_check(th_b_t2(0)); /* 16-bit unconditional */ - return 2; - } - else - { - ot_check(th_b_t4(0)); /* 32-bit unconditional */ - return 4; - } -} - -ST_FUNC int tcc_gen_machine_conditional_jump_mop(int32_t condition, TccIrOp op, int32_t target_ir, int ir_idx) -{ - int cond = mapcc(condition); - - if (dry_run_state.active) - { - /* Emit 32-bit placeholder for code size tracking */ - ot_check(th_b_t3(cond, 0)); - return 4; - } - - /* Real pass: try narrow encoding, backward first then forward */ - if (can_narrow_backward_branch(target_ir, 1, ir_idx) || can_narrow_forward_branch(target_ir, 1, ir_idx)) - { - ot_check(th_b_t1(cond, 0)); /* 16-bit conditional */ - return 2; - } - else - { - ot_check(th_b_t3(cond, 0)); /* 32-bit conditional */ - return 4; - } -} - -/* Return the maximum bytes a pending literal pool dump could insert. - * Used for CBZ/CBNZ distance safety checks. */ -ST_FUNC int tcc_gen_machine_pending_pool_size(void) -{ - int count = dry_run_state.active ? dry_run_literal_pool_count : thumb_gen_state.literal_pool_count; - return count * 4 + (count > 0 ? 2 : 0); /* entries + possible alignment padding */ -} - -/* Emit CBZ/CBNZ: combined compare-zero + branch in a single 16-bit instruction. - * rn must be r0-r7, target must be forward within 126 bytes. - * Returns the instruction size (always 2). */ -ST_FUNC int tcc_gen_machine_cbz_jump_mop(int rn, int nonzero, int32_t target_ir, int ir_idx) -{ - ot_check(th_cbz((uint16_t)rn, 0, (uint32_t)nonzero)); - return 2; -} - -/* Set static chain register: MOV R10, R7 (FP) */ -ST_FUNC void tcc_gen_machine_set_chain(void) -{ - int chain_reg = architecture_config.static_chain_reg; - thumb_shift no_shift = {THUMB_SHIFT_NONE, 0, THUMB_SHIFT_IMMEDIATE}; - /* MOV chain_reg, R_FP (R7 on ARM Thumb) */ - ot_check_mov_reg(chain_reg, R_FP, flags_safe(), no_shift, ENFORCE_ENCODING_NONE, false); -} - -/* Reload static chain register from the chain save slot at [FP - 4]. - * Called after function calls in nested functions with has_static_chain, - * because trampoline calls can clobber R10. */ -ST_FUNC void tcc_gen_machine_restore_chain(void) -{ - int chain_reg = architecture_config.static_chain_reg; - /* LDR chain_reg, [FP, #-4] */ - if (!load_word_from_base(chain_reg, R_FP, 4, 1)) - { - /* Fallback for large offset (should not happen for -4) */ - ScratchRegAlloc rr_alloc = th_offset_to_reg_ex(4, 1, (1u << chain_reg) | (1u << R_FP)); - int rr = rr_alloc.reg; - ot_check(th_ldr_reg(chain_reg, R_FP, rr, THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); - restore_scratch_reg(&rr_alloc); - } -} - -/* Store parent FP (R7) into chain slot in .data for nested function trampoline. - * src1 carries the chain slot symbol via SYMREF so we can emit a relocation. */ -ST_FUNC void tcc_gen_machine_init_chain_slot(IROperand src1) -{ - /* Extract the chain slot Sym* from the IROperand */ - Sym *chain_sym = irop_get_sym(src1); - if (!chain_sym) - tcc_error("internal error: INIT_CHAIN_SLOT without chain slot symbol"); - - /* Get a scratch register to hold the chain slot address */ - ScratchRegAlloc scratch = get_scratch_reg_with_save(0); - - /* Load chain slot address into scratch register via literal pool. */ - _lfc_sym = chain_sym; - load_full_const(scratch.reg, PREG_NONE, 0, 0); - - /* STR R7, [scratch, #0] — store frame pointer into chain slot */ - ot_check_str_imm(R_FP, scratch.reg, 0, 6, ENFORCE_ENCODING_NONE); - - /* Restore scratch register */ - restore_scratch_reg(&scratch); -} - -/* Called at end of each IR instruction to clean up scratch register state. - * - Restores any pushed scratch registers (POP in reverse push order) - * - Resets global exclusion mask for next instruction */ -ST_FUNC void tcc_gen_machine_end_instruction(void) -{ - restore_all_pushed_scratch_regs(); -} - -/* Bytes of outgoing-argument area that have to stay below the live stack top. - * - * Stack arguments are written at [SP, SP+call_outgoing_size) right before the - * BL, because that is where AAPCS says the callee looks for them, so SP cannot - * also be the block alloca just handed back -- the next call with a stack - * argument writes straight over it. GNU make's pattern_search does exactly - * that: `int_file = alloca (sizeof (struct file))` followed by a five-argument - * recursive call, and the fifth argument landed on int_file->name, so the - * callee dereferenced a NULL name. - * - * SP is therefore kept this many bytes below the *logical* stack top: - * VLA_ALLOC carves its block off that top and then drops SP past a fresh - * outgoing area, and VLA_SP_SAVE / VLA_SP_RESTORE convert between the two. - * The conversion is symmetric, so the SAVE/RESTORE pairs that bracket a VLA - * scope still round-trip, and the SAVE that captures an alloca's result now - * yields the block rather than the argument area beneath it. - * - * The nested-call R9/argument save slots need no equivalent: in a dynamic-SP - * function they are already addressed FP-relative (see gfunc_prolog and the - * call-site nested_save_fp_bias). */ -static int vla_outgoing_reserve(void) -{ - TCCIRState *ir = tcc_state->ir; - int k = (ir && ir->call_outgoing_size > 0) ? ir->call_outgoing_size : 0; - return (k + 7) & ~7; -} - -/* tcc_gen_machine_vla_mop: MachineOperand-based entry point for VLA operations. - * - * VLA_ALLOC: src1=size(bytes), src2=alignment(IMM bytes), dest unused - * VLA_SP_SAVE: dest=save slot, src1/src2 unused - * VLA_SP_RESTORE: src1=save slot, dest/src2 unused - * - * Gate: !ir->has_static_chain (VLA ops are always 32-bit pointer/int sized). - */ -ST_FUNC void tcc_gen_machine_vla_mop(MachineOperand dest, MachineOperand src1, MachineOperand src2, TccIrOp op) -{ - MachineCodegenContext ctx = {0}; - switch (op) - { - case TCCIR_OP_VLA_ALLOC: - { - /* src1=size (may be register or spilled); src2=alignment (IMM or NONE). */ - int align = (src2.kind == MACH_OP_IMM) ? (int)src2.u.imm.val : 8; - if (align < 8) - align = 8; - if (align & (align - 1)) - tcc_error("alignment is not a power of 2: %i", align); - - /* Load size into a working register — it's dead after this op. */ - int r = mach_ensure_in_reg(&ctx, &src1, 0); - if (r == R_SP) - tcc_error("compiler_error: VLA alloc picked SP as temp"); - - const int vla_reserve = vla_outgoing_reserve(); - - /* r = SP - r (subtract size from stack pointer) */ - ot_check(th_sub_reg(r, R_SP, r, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); - - /* ... and back up to the logical top, so the block is carved above the - * outgoing-argument area rather than on top of it. */ - if (vla_reserve) - { - thumb_opcode add_res = th_add_imm(r, r, (uint32_t)vla_reserve, flags_safe(), ENFORCE_ENCODING_NONE); - if (is_valid_opcode(add_res)) - { - ot(add_res); - } - else - { - int res_reg = mach_alloc_scratch(&ctx, 1u << (uint32_t)r); - if (!ot(th_generic_mov_imm(res_reg, vla_reserve))) - load_full_const(res_reg, PREG_NONE, LFC_SPLIT(vla_reserve)); - ot_check(th_add_reg(r, r, res_reg, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); - } - } - - if (align > 1) - { - /* Align down: r &= ~(align-1). Try immediate BIC first. */ - if (!ot(th_bic_imm(r, r, (uint32_t)(align - 1), flags_safe(), ENFORCE_ENCODING_NONE))) - { - /* Fallback: materialize mask in a scratch register. */ - int mask_reg = mach_alloc_scratch(&ctx, 1u << (uint32_t)r); - if (!ot(th_generic_mov_imm(mask_reg, align - 1))) - load_full_const(mask_reg, PREG_NONE, LFC_SPLIT(align - 1)); - ot_check(th_bic_reg(r, r, mask_reg, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); - } - } - - ot_check_mov_reg(R_SP, r, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE, false); - /* r is the new logical top; SP sits a fresh outgoing area below it. */ - gadd_sp(-vla_reserve); - break; - } - case TCCIR_OP_VLA_SP_SAVE: - { - /* Fast path: when dest is a register-allocated vreg, copy SP directly into - * its register — saves the scratch-mov + writeback-mov pair that the - * generic path would emit. Triggered by the alloca-load-fwd IR pass - * which rewrites a `VLA_SP_SAVE slot; LOAD vreg <- slot` pair into a - * single `VLA_SP_SAVE vreg`. */ - const int save_reserve = vla_outgoing_reserve(); - if (dest.kind == MACH_OP_REG && !dest.needs_deref && - dest.u.reg.r0 != (int)PREG_REG_NONE) - { - ot_check_mov_reg((uint32_t)dest.u.reg.r0, R_SP, flags_safe(), - THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE, false); - if (save_reserve) - ot_check(th_add_imm((uint32_t)dest.u.reg.r0, (uint32_t)dest.u.reg.r0, (uint32_t)save_reserve, - flags_safe(), ENFORCE_ENCODING_NONE)); - break; - } - /* Save the logical stack top to the destination save slot via a scratch. */ - ScratchRegAlloc sp_scratch = get_scratch_reg_with_save(0); - ot_check_mov_reg(sp_scratch.reg, R_SP, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE, - false); - if (save_reserve) - ot_check(th_add_imm((uint32_t)sp_scratch.reg, (uint32_t)sp_scratch.reg, (uint32_t)save_reserve, - flags_safe(), ENFORCE_ENCODING_NONE)); - mach_writeback_dest(&dest, sp_scratch.reg); - restore_scratch_reg(&sp_scratch); - break; - } - case TCCIR_OP_VLA_SP_RESTORE: - { - /* Load the saved logical top from src1, then put SP back below it. The - * SUB goes through SP rather than the loaded register: src1 may be a live - * vreg (alloca_load_fwd hands one over) that this must not clobber. */ - int saved_sp = mach_ensure_in_reg(&ctx, &src1, 0); - ot_check_mov_reg(R_SP, saved_sp, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE, false); - gadd_sp(-vla_outgoing_reserve()); - break; - } - default: - tcc_error("compiler_error: tcc_gen_machine_vla_mop unsupported op %d", op); - } - mach_release_all(&ctx); -} - -/* Block copy from const data section to stack using LDM/STM. - * dest = STACKOFF (destination stack offset, is_local=1) - * src = SYMREF (anonymous symbol in rodata) - * size = number of bytes to copy (must be multiple of 4) - * - * Generated code for 20 bytes (5 words): - * LDR r_src, [PC, #lit_pool] ; load rodata address - * ADD r_dst, FP/SP, #stack_off ; compute stack dest - * LDMIA r_src!, {r0, r1, r2, r3} ; load 4 words from rodata - * STMIA r_dst!, {r0, r1, r2, r3} ; store 4 words to stack - * LDR r0, [r_src] ; load remaining word - * STR r0, [r_dst] ; store remaining word - */ -/* tcc_gen_machine_select_mop: Conditional select using ITE block. - * Emits: ITE ; MOV dest, then_val; MOV dest, else_val - * - * For simple register/immediate operands, this is 3 instructions (ITE + 2 MOVs) - * instead of 5+ (B.cond + MOV + B + MOV + ...) with branching. - */ -/* Check if a MachineOperand can be materialized in exactly one instruction. - * Returns 1 for: IMM (any value), REG (no deref), SYMBOL (no deref), SPILL (no deref). - * Returns 0 for: multi-instruction sequences (deref, chain_rel, etc). */ -static int select_can_inline(const MachineOperand *op) -{ - switch (op->kind) - { - case MACH_OP_IMM: - return 1; /* MOV/MOVW/MVN or literal pool LDR — always 1 instruction */ - case MACH_OP_REG: - return !op->needs_deref; /* MOV reg is 1 instr; deref needs LDR too */ - case MACH_OP_SYMBOL: - /* A symbol address is a single literal-pool LDR only in the plain, - * non-PIC, non-separated layout. Under PIC/PIE or text+data separation it - * expands to a multi-instruction GOT/GOTOFF sequence (ldr GOT-slot; add r9; - * ldr; ...). Emitting that "inline" inside an IT block predicates only the - * FIRST instruction and lets the remaining ones run unconditionally, which - * clobbers the select result with the else-operand's address. Force - * pre-materialization into a scratch register in those modes. */ - return !op->needs_deref && !pic && !text_and_data_separation; - case MACH_OP_SPILL: - return !op->needs_deref; /* LDR from stack is 1 instr; deref (VT_LLOCAL) needs 2 */ - case MACH_OP_FRAME_ADDR: - return 1; /* ADD reg, FP, #off is 1 instr */ - default: - return 0; - } -} - -/* Emit a single-instruction materialization of 'op' into 'reg'. - * Caller must ensure select_can_inline(op) returned 1. */ -static void select_emit_inline(MachineCodegenContext *ctx, const MachineOperand *op, int reg) -{ - switch (op->kind) - { - case MACH_OP_IMM: - { - thumb_opcode imm_op = th_generic_mov_imm((uint32_t)reg, (int)op->u.imm.val); - if (imm_op.size != 0) - ot(imm_op); - else - load_full_const(reg, PREG_NONE, LFC_SPLIT(op->u.imm.val)); - break; - } - case MACH_OP_REG: - ot_check_mov_reg(reg, op->u.reg.r0, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE, - true); - break; - case MACH_OP_SYMBOL: - { - Sym *raw_sym = op->u.sym.sym; - Sym *sym = raw_sym ? validate_sym_for_reloc(raw_sym) : NULL; - tcc_machine_load_constant(reg, PREG_REG_NONE, op->u.sym.addend, 0, sym); - break; - } - case MACH_OP_SPILL: - tcc_machine_load_spill_slot(reg, op->u.spill.offset); - break; - case MACH_OP_FRAME_ADDR: - tcc_machine_addr_of_stack_slot(reg, op->u.frame.offset, 0); - break; - default: - tcc_error("compiler_error: select_emit_inline: unhandled kind %d", (int)op->kind); - break; - } -} - -ST_FUNC void tcc_gen_machine_select_mop(MachineOperand then_val, MachineOperand else_val, MachineOperand dest, - int cond_code) -{ - MachineCodegenContext mctx = {0}; - - int cond = mapcc(cond_code); - - /* Get destination register */ - int dest_reg = mach_get_dest_reg(&mctx, &dest, 0); - uint32_t excl = (1u << (uint32_t)dest_reg); - - /* Determine if each operand can be materialized in exactly one instruction. - * If so, we can emit it directly inside the ITE block into dest_reg, - * saving scratch registers and pre-materialization instructions. - * - * Emitting inside the IT block is preferred because: - * - It avoids flag clobber (MOVS before ITE would destroy CMP flags) - * - It saves scratch registers (no pre-materialization needed) - * - It produces smaller code */ - int then_inline = select_can_inline(&then_val); - int else_inline = select_can_inline(&else_val); - - int then_reg = -1, else_reg = -1; - - /* Pre-materialize operands that need multi-instruction sequences. - * These are loaded into scratch registers BEFORE the ITE block. */ - if (!then_inline) - { - then_reg = mach_ensure_in_reg(&mctx, &then_val, excl); - excl |= (1u << (uint32_t)then_reg); - } - if (!else_inline) - { - else_reg = mach_ensure_in_reg(&mctx, &else_val, excl); - excl |= (1u << (uint32_t)else_reg); - } - - /* Identity-then shortcut: if the then-value is already in dest_reg, the - * predicated mov would be `movXX dest, dest` — a real instruction inside an - * IT block (the usual elision in ot_check_mov_reg is suppressed by in_it). - * Emit `IT ` + the else mov instead. Saves one instruction. */ - int then_is_identity = 0; - if (then_inline && then_val.kind == MACH_OP_REG && !then_val.needs_deref && - (int)then_val.u.reg.r0 == dest_reg) - then_is_identity = 1; - else if (!then_inline && then_reg == dest_reg) - then_is_identity = 1; - - if (then_is_identity) - { - int inv_cond = cond ^ 1; - th_literal_pool_reserve_upcoming_bytes(8); /* IT(2) + instr(2-4) */ - ot_check(th_it((uint16_t)inv_cond, 0x8u)); /* IT , single insn */ - if (else_inline) - select_emit_inline(&mctx, &else_val, dest_reg); - else - ot_check_mov_reg(dest_reg, else_reg, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE, true); - mach_writeback_dest(&dest, dest_reg); - mach_release_all(&mctx); - return; - } - - /* Identity-else shortcut: mirror of the above. If the else-value is already - * in dest_reg, emit `IT ` + the then mov only, dropping the - * `movXX dest, dest` for the else arm. Common for `cond ? f(x) : x` shapes - * (e.g. abs `x<0?-x:x`) where the else arm is the unmodified input. */ - int else_is_identity = 0; - if (else_inline && else_val.kind == MACH_OP_REG && !else_val.needs_deref && - (int)else_val.u.reg.r0 == dest_reg) - else_is_identity = 1; - else if (!else_inline && else_reg == dest_reg) - else_is_identity = 1; - - if (else_is_identity) - { - th_literal_pool_reserve_upcoming_bytes(8); /* IT(2) + instr(2-4) */ - ot_check(th_it((uint16_t)cond, 0x8u)); /* IT , single insn */ - if (then_inline) - select_emit_inline(&mctx, &then_val, dest_reg); - else - ot_check_mov_reg(dest_reg, then_reg, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE, true); - mach_writeback_dest(&dest, dest_reg); - mach_release_all(&mctx); - return; - } - - /* ITE mask: the second instruction uses the opposite condition. - * mask encoding: bit3 = E_flag for 2nd instr, bit2 = end marker. - * E_flag = opposite of cond[0], so: mask = ((cond[0]^1) << 3) | (1 << 2) */ - uint32_t ite_mask = (uint32_t)(((cond & 1) ^ 1) << 3) | 0x4u; - - /* Reserve literal pool space to prevent pool dumps inside the IT block */ - th_literal_pool_reserve_upcoming_bytes(10); /* ITE(2) + instr(2-4) + instr(2-4) */ - - ot_check(th_it((uint16_t)cond, (uint16_t)ite_mask)); - - /* Emit the Then instruction inside IT block */ - if (then_inline) - select_emit_inline(&mctx, &then_val, dest_reg); - else - ot_check_mov_reg(dest_reg, then_reg, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE, - true); - - /* Emit the Else instruction inside IT block */ - if (else_inline) - select_emit_inline(&mctx, &else_val, dest_reg); - else - ot_check_mov_reg(dest_reg, else_reg, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE, - true); - - mach_writeback_dest(&dest, dest_reg); - mach_release_all(&mctx); -} - -/* Predicated-compute fusion: when a single-use ALU op feeds an else-identity - * SELECT (else == SELECT dest) whose condition flags are still live from a - * preceding CMP, the op can be emitted predicated inside an IT block directly - * into the SELECT dest — `cmp; it ; rsb.w dest, src, #0` — instead of - * `rsb Tc,src,#0; it ; mov dest,Tc`. gcc-parity abs `x<0?-x:x`. - * - * Only a reverse-subtract negate (`dest = 0 - src2`) is folded: it lowers to a - * single 32-bit rsb.w, which inside the IT is conditional and, emitted - * flags-safe, leaves NZCV untouched. can_predicate answers before emission so - * the caller keeps the two-pass scratch accounting consistent. */ -ST_FUNC int tcc_gen_machine_can_predicate_alu(MachineOperand src1, MachineOperand src2, - MachineOperand dest, TccIrOp op) -{ - if (op != TCCIR_OP_SUB) - return 0; - if (!(src1.kind == MACH_OP_IMM && src1.u.imm.val == 0)) - return 0; - if (dest.is_64bit || src2.is_64bit || dest.needs_deref) - return 0; - return 1; -} - -ST_FUNC void tcc_gen_machine_predicated_alu_mop(MachineOperand src1, MachineOperand src2, - MachineOperand dest, TccIrOp op, int cond_code) -{ - (void)src1; - (void)op; - MachineCodegenContext mctx = {0}; - int cond = mapcc(cond_code); - int dest_reg = mach_get_dest_reg(&mctx, &dest, 0); - uint32_t excl = (1u << (uint32_t)dest_reg); - int src_reg = mach_ensure_in_reg(&mctx, &src2, excl); - - th_literal_pool_reserve_upcoming_bytes(8); /* IT(2) + rsb.w(4) */ - ot_check(th_it((uint16_t)cond, 0x8u)); /* IT , single insn */ - ot_check(th_rsb_imm((uint32_t)dest_reg, (uint32_t)src_reg, 0, flags_safe(), ENFORCE_ENCODING_NONE)); - - mach_writeback_dest(&dest, dest_reg); - mach_release_all(&mctx); -} - -ST_FUNC void tcc_gen_machine_block_copy_mop(TCCIRState *ir, IROperand dest, IROperand src, int size) -{ - if (size <= 0) - tcc_error("compiler_error: block_copy size must be positive, got %d", size); - - /* Get the source symbol from the SYMREF operand */ - IRPoolSymref *symref = irop_get_symref_ex(ir, src); - if (!symref || !symref->sym) - tcc_error("compiler_error: block_copy source is not a valid symbol reference"); - Sym *sym = validate_sym_for_reloc(symref->sym); - - /* Get the destination stack offset */ - int frame_offset = (int)irop_get_imm64_ex(ir, dest); - - /* For large copies, call memcpy instead of inline LDM/STM. - * Compute dest address into r0 BEFORE pushing lr, since the address is - * sp-relative and pushing changes sp. The BL to memcpy clobbers lr, - * so we must save/restore it for leaf functions whose prologue didn't. */ - if (size >= TCCIR_BLOCK_COPY_MEMCPY_MIN_BYTES) - { - tcc_machine_addr_of_stack_slot(R0, frame_offset, 0 /* not param */); - tcc_machine_load_constant(R1, PREG_REG_NONE, symref->addend, 0, sym); - tcc_machine_load_constant(R2, PREG_REG_NONE, size, 0, NULL); - int need_lr_save = ir->leaffunc; - if (need_lr_save) - ot_check(th_push(1u << ARM_LR)); - Sym *memcpy_sym = external_global_sym(tok_alloc_const("memcpy"), &func_old_type); - MachineOperand func_mop = {0}; - func_mop.kind = MACH_OP_SYMBOL; - func_mop.u.sym.sym = memcpy_sym; - func_mop.u.sym.addend = 0; - if (text_and_data_separation) - ot_check(th_push((uint16_t)((1 << R9) | (1 << R12)))); - gcall_or_jump_mop(0, func_mop); - if (text_and_data_separation) - ot_check(th_pop((uint16_t)((1 << R9) | (1 << R12)))); - if (need_lr_save) - ot_check(th_pop(1u << ARM_LR)); - return; - } - - int nwords = size / 4; - - /* Allocate pointer registers first and compute addresses BEFORE allocating - * data registers. Data register saves may use PUSH which modifies SP, - * so all SP-relative address computation must happen before that. */ - ScratchRegAlloc src_scratch = get_scratch_reg_with_save(0); - int r_src = src_scratch.reg; - ScratchRegAlloc dst_scratch = get_scratch_reg_with_save(1u << (uint32_t)r_src); - int r_dst = dst_scratch.reg; - - /* Load source address (rodata symbol) into r_src */ - tcc_machine_load_constant(r_src, PREG_REG_NONE, symref->addend, 0, sym); - - /* Compute destination stack address into r_dst BEFORE any data reg saves - * that might change SP via PUSH */ - tcc_machine_addr_of_stack_slot(r_dst, frame_offset, 0 /* not param */); - - /* Now allocate data registers for LDM/STM. Even if these saves use PUSH - * and modify SP, we've already captured the destination address in r_dst. */ - int max_data = nwords < 4 ? nwords : 4; - if (max_data < 1) - max_data = 1; - - ScratchRegAlloc data_scratches[4]; - int data_regs[4]; - int ndata = 0; - uint32_t exclude = (1u << (uint32_t)r_src) | (1u << (uint32_t)r_dst); - for (int k = 0; k < max_data; k++) - { - data_scratches[k] = get_scratch_reg_with_save(exclude); - data_regs[k] = data_scratches[k].reg; - exclude |= (1u << (uint32_t)data_regs[k]); - ndata++; - } - - int remaining_words = nwords; - - /* Process in chunks of ndata words using LDM/STM with writeback */ - while (remaining_words >= ndata && ndata >= 2) - { - uint32_t regset = 0; - for (int j = 0; j < ndata; j++) - regset |= (1u << (uint32_t)data_regs[j]); - - ot_check(th_ldm(r_src, regset, 1 /* writeback */, ENFORCE_ENCODING_NONE)); - ot_check(th_stm(r_dst, regset, 1 /* writeback */, ENFORCE_ENCODING_NONE)); - remaining_words -= ndata; - } - - /* Handle remaining words individually */ - int dr = data_regs[0]; /* first data register */ - int single_words = remaining_words; - while (remaining_words > 0) - { - ot_check_ldr_imm(dr, r_src, 0, 6, ENFORCE_ENCODING_NONE); - ot_check_str_imm(dr, r_dst, 0, 6, ENFORCE_ENCODING_NONE); - if (remaining_words > 1) - { - if (!ot(th_add_imm(r_src, r_src, 4, flags_safe(), ENFORCE_ENCODING_NONE))) - tcc_error("compiler_error: block_copy cannot advance source pointer"); - if (!ot(th_add_imm(r_dst, r_dst, 4, flags_safe(), ENFORCE_ENCODING_NONE))) - tcc_error("compiler_error: block_copy cannot advance dest pointer"); - } - remaining_words--; - } - - /* Copy the 1..3 trailing bytes of a size that is not a whole number of words. - * Writing them as bytes matters: a strcpy fold must not touch dst past the - * terminating NUL, so the copy may not be rounded up to a word. The LDM/STM - * chunks advanced the pointers by writeback; the single-word loop above stops - * ON its last word rather than past it, so the tail sits 4 bytes further on - * whenever that loop ran at all. */ - int tail = size & 3; - if (tail) - { - int tail_off = single_words > 0 ? 4 : 0; - for (int b = 0; b < tail; b++) - { - ot_check(th_ldrb_imm(dr, r_src, tail_off + b, 6, ENFORCE_ENCODING_NONE)); - ot_check(th_strb_imm(dr, r_dst, tail_off + b, 6, ENFORCE_ENCODING_NONE)); - } - } - - /* Restore all scratch registers in reverse order: data regs first, then ptrs */ - for (int k = ndata - 1; k >= 0; k--) - restore_scratch_reg(&data_scratches[k]); - restore_scratch_reg(&dst_scratch); - restore_scratch_reg(&src_scratch); -} - -ST_FUNC void tcc_gen_machine_spill_block_copy(int32_t src_spill_off, int32_t dst_spill_off, int nwords) -{ - ScratchRegAlloc src_scratch = get_scratch_reg_with_save(0); - int r_src = src_scratch.reg; - ScratchRegAlloc dst_scratch = get_scratch_reg_with_save(1u << (uint32_t)r_src); - int r_dst = dst_scratch.reg; - - tcc_machine_addr_of_stack_slot(r_src, src_spill_off, 0); - tcc_machine_addr_of_stack_slot(r_dst, dst_spill_off, 0); - - int max_data = nwords < 4 ? nwords : 4; - if (max_data < 1) - max_data = 1; - - ScratchRegAlloc data_scratches[4]; - int data_regs[4]; - int ndata = 0; - uint32_t exclude = (1u << (uint32_t)r_src) | (1u << (uint32_t)r_dst); - for (int k = 0; k < max_data; k++) - { - data_scratches[k] = get_scratch_reg_with_save(exclude); - data_regs[k] = data_scratches[k].reg; - exclude |= (1u << (uint32_t)data_regs[k]); - ndata++; - } - - int remaining = nwords; - - while (remaining >= ndata && ndata >= 2) - { - uint32_t regset = 0; - for (int j = 0; j < ndata; j++) - regset |= (1u << (uint32_t)data_regs[j]); - ot_check(th_ldm(r_src, regset, 1 /* writeback */, ENFORCE_ENCODING_NONE)); - ot_check(th_stm(r_dst, regset, 1 /* writeback */, ENFORCE_ENCODING_NONE)); - remaining -= ndata; - } - - int dr = data_regs[0]; - while (remaining > 0) - { - ot_check_ldr_imm(dr, r_src, 0, 6, ENFORCE_ENCODING_NONE); - ot_check_str_imm(dr, r_dst, 0, 6, ENFORCE_ENCODING_NONE); - if (remaining > 1) - { - if (!ot(th_add_imm(r_src, r_src, 4, flags_safe(), ENFORCE_ENCODING_NONE))) - tcc_error("compiler_error: spill_block_copy cannot advance source pointer"); - if (!ot(th_add_imm(r_dst, r_dst, 4, flags_safe(), ENFORCE_ENCODING_NONE))) - tcc_error("compiler_error: spill_block_copy cannot advance dest pointer"); - } - remaining--; - } - - for (int k = ndata - 1; k >= 0; k--) - restore_scratch_reg(&data_scratches[k]); - restore_scratch_reg(&dst_scratch); - restore_scratch_reg(&src_scratch); -} - -ST_FUNC void tcc_gen_machine_trap_mop(void) -{ - /* Emit UDF #0xfe - Undefined instruction for trap */ - ot_check(th_udf(0xfe, ENFORCE_ENCODING_NONE)); -} - -ST_FUNC void tcc_gen_machine_prefetch_mop(MachineOperand addr, int rw) -{ - /* Emit PLD (Preload Data) or PLDW (Preload Data with intent to Write) - * based on the rw hint. - * - * PLD/PLDW are hints to the memory system that data may be needed soon. - * They don't wait for the data and don't fault if the address is invalid. - * - * We support several addressing modes: - * - Register indirect: [Rn] -> use th_pld_imm with offset 0 - * - Register + immediate offset: [Rn, #imm] - * - Literal (PC-relative): label - */ - (void)rw; /* PLD/PLDW distinction may not be supported on all ARM variants */ - - switch (addr.kind) - { - case MACH_OP_REG: - { - /* Register indirect: PLD [Rn] */ - int reg = addr.u.reg.r0; - ot_check(th_pld_imm((uint32_t)reg, 0, 0)); - break; - } - case MACH_OP_SPILL: - { - /* Spill slot: compute address (FP + offset) then PLD */ - int32_t offset = addr.u.spill.offset; - if (offset != 0) - { - ScratchRegAlloc scr = get_scratch_reg_with_save(0); - load_full_const(scr.reg, PREG_NONE, LFC_SPLIT(offset)); - ot_check(th_add_reg(scr.reg, R_FP, scr.reg, flags_safe(), THUMB_SHIFT_DEFAULT, - ENFORCE_ENCODING_NONE)); - ot_check(th_pld_imm(scr.reg, 0, 0)); - restore_scratch_reg(&scr); - } - else - { - ot_check(th_pld_imm(R_FP, 0, 0)); - } - break; - } - case MACH_OP_IMM: - { - /* For immediate addresses, load into a register first */ - ScratchRegAlloc scr = get_scratch_reg_with_save(0); - load_full_const(scr.reg, PREG_NONE, LFC_SPLIT(addr.u.imm.val)); - ot_check(th_pld_imm(scr.reg, 0, 0)); - restore_scratch_reg(&scr); - break; - } - case MACH_OP_SYMBOL: - { - /* For symbol addresses, load into a register first */ - ScratchRegAlloc scr = get_scratch_reg_with_save(0); - _lfc_sym = addr.u.sym.sym; - load_full_const(scr.reg, PREG_NONE, LFC_SPLIT(addr.u.sym.addend)); - ot_check(th_pld_imm(scr.reg, 0, 0)); - restore_scratch_reg(&scr); - break; - } - case MACH_OP_FRAME_ADDR: - { - /* Frame address: FP + offset */ - int32_t offset = addr.u.frame.offset; - if (offset != 0) - { - ScratchRegAlloc scr = get_scratch_reg_with_save(0); - load_full_const(scr.reg, PREG_NONE, LFC_SPLIT(offset)); - ot_check(th_add_reg(scr.reg, R_FP, scr.reg, flags_safe(), THUMB_SHIFT_DEFAULT, - ENFORCE_ENCODING_NONE)); - ot_check(th_pld_imm(scr.reg, 0, 0)); - restore_scratch_reg(&scr); - } - else - { - ot_check(th_pld_imm(R_FP, 0, 0)); - } - break; - } - default: - tcc_error("unsupported operand type for __builtin_prefetch"); - } -} - -/* __builtin_setjmp implementation for ARM Thumb-2. - * - * GCC's documented ABI gives __builtin_setjmp a 5-word buffer; callers - * (e.g. gcc.c-torture pr84521) really do pass `void *buf[5]`, so nothing - * larger may be written through the buffer pointer. The callee-saved - * register file (r4-r11) still must be restored on longjmp — the register - * allocator keeps VARs and the R9 GOT base in r4-r11 across the setjmp — - * so those 8 words live in a hidden, compiler-allocated save area in the - * setjmp-containing function's frame (src2/area), which stays valid for - * as long as a longjmp to this buffer is legal. - * - * Jump buffer layout (4 words used, fits the standard 5-word buffer): - * buf[0] = frame pointer (R7/FP) - * buf[1] = resume address (Thumb-bit set) - * buf[2] = stack pointer (SP) - * buf[3] = address of the hidden r4-r11 save area (32 bytes) - * - * Returns 0 on initial call, 1 when returning via longjmp. - */ -ST_FUNC void tcc_gen_machine_setjmp_mop(MachineOperand buf, MachineOperand area, MachineOperand dest) -{ - MachineCodegenContext ctx = {0}; - int buf_reg; - - if (buf.kind == MACH_OP_NONE) - { - buf_reg = mach_alloc_scratch(&ctx, 0); - ot_check(th_mov_imm(buf_reg, 0, flags_safe(), ENFORCE_ENCODING_NONE)); - } - else - { - /* Exclude r4-r11 as scratch candidates: a saved-scratch there would - * hold the buffer pointer when the area stores below run, corrupting - * the saved register file (same class as the MLA scratch-pop bug). */ - buf_reg = mach_ensure_in_reg(&ctx, &buf, 0x0FF0); - } - - /* ---- save callee-saved r4-r11 into the hidden frame area ---- - * The area address is computed in IP (caller-saved) so the r4-r11 - * values stored are the untouched setjmp-time ones; a scratch from - * mach_alloc_scratch could pick a callee-saved register. */ - if (area.kind == MACH_OP_FRAME_ADDR) - { - tcc_machine_addr_of_stack_slot(R_IP, area.u.frame.offset, 0 /* not param */); - } - else - { - tcc_error("compiler_error: setjmp save area must be a frame slot (kind %d)", (int)area.kind); - } - ot_check_str_imm(4, R_IP, 0, 6, ENFORCE_ENCODING_NONE); /* r4 -> area[0] */ - ot_check_str_imm(5, R_IP, 4, 6, ENFORCE_ENCODING_NONE); /* r5 -> area[1] */ - ot_check_str_imm(6, R_IP, 8, 6, ENFORCE_ENCODING_NONE); /* r6 -> area[2] */ - ot_check_str_imm(R_FP, R_IP, 12, 6, ENFORCE_ENCODING_NONE); /* r7 -> area[3] */ - ot_check_str_imm(8, R_IP, 16, 6, ENFORCE_ENCODING_NONE); /* r8 -> area[4] */ - ot_check_str_imm(9, R_IP, 20, 6, ENFORCE_ENCODING_NONE); /* r9 -> area[5] */ - ot_check_str_imm(10, R_IP, 24, 6, ENFORCE_ENCODING_NONE); /* r10 -> area[6] */ - ot_check_str_imm(11, R_IP, 28, 6, ENFORCE_ENCODING_NONE); /* r11 -> area[7] */ - ot_check_str_imm(R_IP, buf_reg, 12, 6, ENFORCE_ENCODING_NONE); /* &area -> buf[3] */ - - /* ---- save frame pointer ---- */ - ot_check_str_imm(R_FP, buf_reg, 0, 6, ENFORCE_ENCODING_NONE); /* r7 -> buf[0] */ - - /* ---- save SP ---- */ - ot_check_mov_reg(R_IP, R_SP, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE, false); - ot_check_str_imm(R_IP, buf_reg, 8, 6, ENFORCE_ENCODING_NONE); /* SP -> buf[2] */ - - /* ---- save resume address (ADR IP, resume_label) ---- */ - int adr_addr = ind; - int adr_pc = adr_addr + 4; - int adr_base = adr_pc & ~3; - int resume_label_addr = adr_addr + 20; /* 4(ORR)+4(STR)+4(MOV)+4(B) after ADR */ - int adr_imm = resume_label_addr - adr_base; - ot_check(th_adr_imm(R_IP, adr_imm, ENFORCE_ENCODING_32BIT)); - - ot_check(th_orr_imm(R_IP, R_IP, 1, flags_safe(), ENFORCE_ENCODING_NONE)); /* Thumb bit */ - ot_check_str_imm(R_IP, buf_reg, 4, 6, ENFORCE_ENCODING_NONE); /* -> buf[1] */ - - /* ---- normal path: return 0 ---- */ - int dest_reg = mach_get_dest_reg(&ctx, &dest, 0); - ot_check(th_mov_imm(dest_reg, 0, flags_safe(), ENFORCE_ENCODING_32BIT)); /* dest = 0 */ - ot_check(th_b_t4(4)); /* B.W +4 (skip resume) */ - codegen_internal_merge_point(); /* longjmp lands below */ - - /* ---- resume_label: longjmp lands here ---- */ - ot_check(th_mov_imm(dest_reg, 1, flags_safe(), ENFORCE_ENCODING_32BIT)); /* dest = 1 */ - /* ---- end_label ---- */ - - mach_writeback_dest(&dest, dest_reg); - mach_release_all(&ctx); -} - -/* Non-local goto setjmp: saves ALL callee-saved registers (r4-r11), SP, - * and resume address in a 40-byte buffer. Used for __label__ + nested - * function goto support. - * - * Buffer layout (10 words = 40 bytes): - * buf[0-7] = r4-r11 - * buf[8] = SP - * buf[9] = resume address (Thumb-bit set) - */ -ST_FUNC void tcc_gen_machine_nl_setjmp_mop(MachineOperand buf, MachineOperand dest) -{ - MachineCodegenContext ctx = {0}; - int buf_reg; - - if (buf.kind == MACH_OP_NONE) - { - buf_reg = mach_alloc_scratch(&ctx, 0); - ot_check(th_mov_imm(buf_reg, 0, flags_safe(), ENFORCE_ENCODING_NONE)); - } - else - { - buf_reg = mach_ensure_in_reg(&ctx, &buf, 0); - } - - /* ---- save callee-saved registers r4-r11 ---- */ - ot_check_str_imm(4, buf_reg, 0, 6, ENFORCE_ENCODING_NONE); /* r4 -> buf[0] */ - ot_check_str_imm(5, buf_reg, 4, 6, ENFORCE_ENCODING_NONE); /* r5 -> buf[1] */ - ot_check_str_imm(6, buf_reg, 8, 6, ENFORCE_ENCODING_NONE); /* r6 -> buf[2] */ - ot_check_str_imm(R_FP, buf_reg, 12, 6, ENFORCE_ENCODING_NONE); /* r7 -> buf[3] */ - ot_check_str_imm(8, buf_reg, 16, 6, ENFORCE_ENCODING_NONE); /* r8 -> buf[4] */ - ot_check_str_imm(9, buf_reg, 20, 6, ENFORCE_ENCODING_NONE); /* r9 -> buf[5] */ - ot_check_str_imm(10, buf_reg, 24, 6, ENFORCE_ENCODING_NONE); /* r10 -> buf[6] */ - ot_check_str_imm(11, buf_reg, 28, 6, ENFORCE_ENCODING_NONE); /* r11 -> buf[7] */ - - /* ---- save SP ---- */ - ot_check_mov_reg(R_IP, R_SP, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE, false); - ot_check_str_imm(R_IP, buf_reg, 32, 6, ENFORCE_ENCODING_NONE); /* SP -> buf[8] */ - - /* ---- save resume address (ADR IP, resume_label) ---- */ - int adr_addr = ind; - int adr_pc = adr_addr + 4; - int adr_base = adr_pc & ~3; - int resume_label_addr = adr_addr + 20; /* 4(ORR)+4(STR)+4(MOV)+4(B) after ADR */ - int adr_imm = resume_label_addr - adr_base; - ot_check(th_adr_imm(R_IP, adr_imm, ENFORCE_ENCODING_32BIT)); - - ot_check(th_orr_imm(R_IP, R_IP, 1, flags_safe(), ENFORCE_ENCODING_NONE)); /* Thumb bit */ - ot_check_str_imm(R_IP, buf_reg, 36, 6, ENFORCE_ENCODING_NONE); /* -> buf[9] */ - - /* ---- normal path: return 0 ---- */ - int dest_reg = mach_get_dest_reg(&ctx, &dest, 0); - ot_check(th_mov_imm(dest_reg, 0, flags_safe(), ENFORCE_ENCODING_32BIT)); /* dest = 0 */ - ot_check(th_b_t4(4)); /* B.W +4 (skip resume) */ - codegen_internal_merge_point(); /* longjmp lands below */ - - /* ---- resume_label: longjmp lands here ---- */ - ot_check(th_mov_imm(dest_reg, 1, flags_safe(), ENFORCE_ENCODING_32BIT)); /* dest = 1 */ - /* ---- end_label ---- */ - - mach_writeback_dest(&dest, dest_reg); - mach_release_all(&ctx); -} - -/* __builtin_longjmp implementation for ARM Thumb-2. - * - * Restores the callee-saved register file (r4-r11, from the hidden save - * area whose address setjmp left in buf[3]) and SP, then jumps to the - * resume address. This function does not return, so every caller-saved - * register is fair game as a temporary. - * - * Buffer layout (must match tcc_gen_machine_setjmp_mop): - * buf[0] = FP, buf[1] = resume_addr, buf[2] = SP, buf[3] = &save_area - */ -ST_FUNC void tcc_gen_machine_longjmp_mop(MachineOperand buf) -{ - MachineCodegenContext ctx = {0}; - int buf_reg; - - if (buf.kind == MACH_OP_NONE) - { - tcc_error("__builtin_longjmp: invalid buffer operand"); - return; - } - - buf_reg = mach_ensure_in_reg(&ctx, &buf, 0); - - /* Copy buf pointer to IP so it survives the register restores */ - ot_check_mov_reg(R_IP, buf_reg, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE, false); - - /* Read resume address, saved SP and save-area pointer into caller-saved - * regs before clobbering anything callee-saved. */ - ot_check_ldr_imm(0, R_IP, 4, 6, ENFORCE_ENCODING_NONE); /* r0 = resume addr */ - ot_check_ldr_imm(1, R_IP, 8, 6, ENFORCE_ENCODING_NONE); /* r1 = saved SP */ - ot_check_ldr_imm(2, R_IP, 12, 6, ENFORCE_ENCODING_NONE); /* r2 = &save_area */ - - /* Restore callee-saved r4-r11 (r7/FP comes from the area too; the copy - * in buf[0] is identical). */ - ot_check_ldr_imm(4, 2, 0, 6, ENFORCE_ENCODING_NONE); /* r4 */ - ot_check_ldr_imm(5, 2, 4, 6, ENFORCE_ENCODING_NONE); /* r5 */ - ot_check_ldr_imm(6, 2, 8, 6, ENFORCE_ENCODING_NONE); /* r6 */ - ot_check_ldr_imm(R_FP, 2, 12, 6, ENFORCE_ENCODING_NONE); /* r7 */ - ot_check_ldr_imm(8, 2, 16, 6, ENFORCE_ENCODING_NONE); /* r8 */ - allow_r9_write = 1; /* restoring the setjmp-time GOT base is the point */ - ot_check_ldr_imm(9, 2, 20, 6, ENFORCE_ENCODING_NONE); /* r9 */ - allow_r9_write = 0; - ot_check_ldr_imm(10, 2, 24, 6, ENFORCE_ENCODING_NONE); /* r10 */ - ot_check_ldr_imm(11, 2, 28, 6, ENFORCE_ENCODING_NONE); /* r11 */ - - /* Restore SP */ - ot_check_mov_reg(R_SP, 1, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE, false); - - /* Jump to resume address (Thumb bit already set by setjmp code) */ - ot_check(th_bx_reg(0)); - - mach_release_all(&ctx); -} - -/* Non-local goto longjmp: restores ALL callee-saved registers (r4-r11), SP, - * then jumps to the resume address. Used for __label__ + nested function goto. - * - * Buffer layout (must match nl_setjmp): - * buf[0-7] = r4-r11, buf[8] = SP, buf[9] = resume_addr - */ -ST_FUNC void tcc_gen_machine_nl_longjmp_mop(MachineOperand buf) -{ - MachineCodegenContext ctx = {0}; - int buf_reg; - - if (buf.kind == MACH_OP_NONE) - { - tcc_error("nl_longjmp: invalid buffer operand"); - return; - } - - if (buf.kind == MACH_OP_CHAIN_REL) - { - /* For chain-relative buffers (non-local goto from nested function), - * we need the ADDRESS of the buffer in the parent frame, not the value. - * mach_ensure_in_reg would load the value; use LEA logic instead. */ - ScratchRegAlloc chain_scratch = {0}; - int chain_used = 0; - uint32_t excl = 0; - int base = resolve_chain_base(tcc_state->ir, buf.u.chain.chain_index, excl, &chain_scratch, &chain_used); - buf_reg = mach_alloc_scratch(&ctx, excl | (1u << (uint32_t)base)); - int32_t off = buf.u.chain.offset; - int sign = (off < 0); - int abs_off = sign ? (int)(-off) : (int)off; - if (abs_off == 0) - { - if (buf_reg != base) - ot_check_mov_reg((uint32_t)buf_reg, (uint32_t)base, flags_safe(), THUMB_SHIFT_DEFAULT, - ENFORCE_ENCODING_NONE, false); - } - else - { - thumb_opcode ins = sign - ? th_sub_imm(buf_reg, base, abs_off, flags_safe(), ENFORCE_ENCODING_NONE) - : th_add_imm(buf_reg, base, abs_off, flags_safe(), ENFORCE_ENCODING_NONE); - if (ins.size != 0) - { - ot_check(ins); - } - else - { - ScratchRegAlloc off_sc = get_scratch_reg_with_save(excl | (1u << (uint32_t)buf_reg) | (1u << (uint32_t)base)); - load_full_const(off_sc.reg, PREG_NONE, LFC_SPLIT(abs_off)); - ot_check(sign ? th_sub_reg(buf_reg, base, off_sc.reg, flags_safe(), THUMB_SHIFT_DEFAULT, - ENFORCE_ENCODING_NONE) - : th_add_reg(buf_reg, base, off_sc.reg, flags_safe(), THUMB_SHIFT_DEFAULT, - ENFORCE_ENCODING_NONE)); - restore_scratch_reg(&off_sc); - } - } - if (chain_used) - restore_scratch_reg(&chain_scratch); - } - else - { - buf_reg = mach_ensure_in_reg(&ctx, &buf, 0); - } - - /* Copy buf pointer to IP so it survives register restores */ - ot_check_mov_reg(R_IP, buf_reg, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE, false); - - /* Load resume address and saved SP into caller-saved regs first - * (before we clobber r4+ with the restore) */ - ot_check_ldr_imm(0, R_IP, 36, 6, ENFORCE_ENCODING_NONE); /* r0 = resume addr */ - ot_check_ldr_imm(1, R_IP, 32, 6, ENFORCE_ENCODING_NONE); /* r1 = saved SP */ - - /* Restore callee-saved registers r4-r11 */ - ot_check_ldr_imm(4, R_IP, 0, 6, ENFORCE_ENCODING_NONE); /* r4 = buf[0] */ - ot_check_ldr_imm(5, R_IP, 4, 6, ENFORCE_ENCODING_NONE); /* r5 = buf[1] */ - ot_check_ldr_imm(6, R_IP, 8, 6, ENFORCE_ENCODING_NONE); /* r6 = buf[2] */ - ot_check_ldr_imm(R_FP, R_IP, 12, 6, ENFORCE_ENCODING_NONE); /* r7 = buf[3] (FP) */ - ot_check_ldr_imm(8, R_IP, 16, 6, ENFORCE_ENCODING_NONE); /* r8 = buf[4] */ - allow_r9_write = 1; - ot_check_ldr_imm(9, R_IP, 20, 6, ENFORCE_ENCODING_NONE); /* r9 = buf[5] */ - allow_r9_write = 0; - ot_check_ldr_imm(10, R_IP, 24, 6, ENFORCE_ENCODING_NONE); /* r10 = buf[6] */ - ot_check_ldr_imm(11, R_IP, 28, 6, ENFORCE_ENCODING_NONE); /* r11 = buf[7] */ - - /* Restore SP */ - ot_check_mov_reg(R_SP, 1, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE, false); - - /* Jump to resume address (Thumb bit already set by setjmp code) */ - ot_check(th_bx_reg(0)); - - mach_release_all(&ctx); -} - -/* ============================================================================ - * __builtin_apply_args / __builtin_apply implementation for ARM Thumb-2 - * ============================================================================ - * - * __builtin_apply_args() returns a pointer to a saved argument block: - * [0] pointer to incoming stack arguments (above saved register area) - * [4] saved r0 - * [8] saved r1 - * [12] saved r2 - * [16] saved r3 - * - * The prologue stores r0-r3 and the stack args pointer when - * func_save_apply_args is set. This handler just computes the address. - * - * __builtin_apply(fn, args, size) restores r0-r3 from the args block, - * calls fn via BLX, and returns the result in dest (r0). - */ - -ST_FUNC void tcc_gen_machine_builtin_apply_args_mop(MachineOperand dest) -{ - MachineCodegenContext ctx = {0}; - - /* The apply_args block lives at tcc_state->apply_args_offset relative to FP. - * Compute FP + adjusted_offset into the dest register. */ - int dest_reg = mach_get_dest_reg(&ctx, &dest, 0); - int offset = tcc_state->apply_args_offset; - tcc_machine_addr_of_stack_slot(dest_reg, offset, 0 /* not param */); - - mach_writeback_dest(&dest, dest_reg); - mach_release_all(&ctx); -} - -ST_FUNC void tcc_gen_machine_builtin_apply_mop(MachineOperand fn, MachineOperand args, MachineOperand dest) -{ - MachineCodegenContext ctx = {0}; - - /* Registers destroyed by the restore-and-call sequence below: r0-r3 are - * reloaded with the saved argument values, and ip(r12)+lr are clobbered by - * the BLX. The args-block base pointer (used by all four restore loads) and - * the callee address must therefore live OUTSIDE this set until used. */ - const uint32_t clobbered = - (1u << R0) | (1u << R1) | (1u << R2) | (1u << R3) | (1u << (uint32_t)R_IP); - - /* Step 1: Materialize the args block pointer, then guarantee it is in a - * register the restore loads won't overwrite. mach_ensure_in_reg returns an - * already-allocated operand register verbatim (ignoring the exclusion mask), - * so when the value already lives in r0-r3 / ip we must relocate it to a - * safe scratch — otherwise the very first load (r0 <- [base+4]) destroys the - * base pointer and the remaining loads read from garbage addresses. */ - int args_reg = mach_ensure_in_reg(&ctx, &args, clobbered); - if (clobbered & (1u << (uint32_t)args_reg)) - { - int safe = mach_alloc_scratch(&ctx, clobbered); - ot_check_mov_reg(safe, args_reg, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE, false); - args_reg = safe; - } - - /* Step 2: Load the function pointer into R12 (IP), which survives the - * register loads below because IP is not one of r0-r3. */ - int fn_reg = mach_ensure_in_reg(&ctx, &fn, (1u << (uint32_t)args_reg)); - if (fn_reg != R_IP) - { - ot_check_mov_reg(R_IP, fn_reg, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE, false); - } - - /* Step 3: Restore r0-r3 from the args block. - * Layout: [+0]=stack_args_ptr, [+4]=r0, [+8]=r1, [+12]=r2, [+16]=r3. */ - ot_check_ldr_imm(R0, args_reg, 4, 6, ENFORCE_ENCODING_NONE); - ot_check_ldr_imm(R1, args_reg, 8, 6, ENFORCE_ENCODING_NONE); - ot_check_ldr_imm(R2, args_reg, 12, 6, ENFORCE_ENCODING_NONE); - ot_check_ldr_imm(R3, args_reg, 16, 6, ENFORCE_ENCODING_NONE); - - /* Step 4: Call the function via BLX R12. - * This clobbers LR and r0-r3 (caller-saved). */ - ot_check(th_blx_reg(R_IP)); - - /* Step 5: Move return value (r0) to dest register. */ - int dest_reg = mach_get_dest_reg(&ctx, &dest, 0); - if (dest_reg != R0) - { - ot_check_mov_reg(dest_reg, R0, flags_safe(), THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE, false); - } - - mach_writeback_dest(&dest, dest_reg); - mach_release_all(&ctx); -} - -ST_FUNC void tcc_gen_machine_backpatch_jump(int address, int offset) -{ - th_patch_call(address, offset); -} - -static int tcc_get_type_size(CType *type) -{ - switch (type->t & VT_BTYPE) - { - case VT_BYTE: - return 1; - case VT_SHORT: - return 2; - case VT_INT: - case VT_LONG: - return 4; - case VT_LLONG: - return 8; - case VT_FLOAT: - return 4; - case VT_DOUBLE: - return 8; - case VT_LDOUBLE: - return 8; // treat long double as double for ARM EABI softcalls - default: - return 0; - } -} - -ST_FUNC const char *tcc_get_abi_softcall_name(SValue *src1, SValue *src2, SValue *dest, TccIrOp op) -{ - const int src1_64bit = tcc_is_64bit_operand(src1); - const int src2_64bit = src2 ? tcc_is_64bit_operand(src2) : 0; - const int dest_64bit = dest ? tcc_is_64bit_type(dest->type.t) : 0; - const int src1_size = tcc_get_type_size(&src1->type); - const int dest_size = dest ? tcc_get_type_size(&dest->type) : 0; - - if (src1_64bit || src2_64bit || dest_64bit) - { - switch (op) - { - case TCCIR_OP_FADD: - return "__aeabi_dadd"; - case TCCIR_OP_FSUB: - return "__aeabi_dsub"; - case TCCIR_OP_FMUL: - return "__aeabi_dmul"; - case TCCIR_OP_FDIV: - return "__aeabi_ddiv"; - case TCCIR_OP_FNEG: - return "__aeabi_dneg"; - default: - break; - } - } - else - { - switch (op) - { - case TCCIR_OP_FADD: - return "__aeabi_fadd"; - case TCCIR_OP_FSUB: - return "__aeabi_fsub"; - case TCCIR_OP_FMUL: - return "__aeabi_fmul"; - case TCCIR_OP_FDIV: - return "__aeabi_fdiv"; - case TCCIR_OP_FNEG: - return "__aeabi_fneg"; - default: - break; - } - } - - switch (op) - { - case TCCIR_OP_CVT_FTOF: - { - if (src1_size == 4 && dest_size == 8) - { - return "__aeabi_f2d"; - } - else if (src1_size == 8 && dest_size == 4) - { - return "__aeabi_d2f"; - } - /* Same size conversion is a no-op, no function needed */ - return NULL; - } - break; - case TCCIR_OP_CVT_FTOI: - { - /* Float/double to integer conversion. - * Map based on destination width, not just VT_BTYPE (since VT_LONG is 32-bit on ARM). - * Use the standard ARM EABI helpers: - * - 32-bit: __aeabi_{f,d}2iz / __aeabi_{f,d}2uiz - * - 64-bit: __aeabi_{f,d}2lz / __aeabi_{f,d}2ulz - */ - const int is_float = (src1_size == 4); - const int is_unsigned = (dest && (dest->type.t & VT_UNSIGNED)) ? 1 : 0; - - if (dest_size == 8) - { - return is_unsigned ? (is_float ? "__aeabi_f2ulz" : "__aeabi_d2ulz") - : (is_float ? "__aeabi_f2lz" : "__aeabi_d2lz"); - } - - return is_unsigned ? (is_float ? "__aeabi_f2uiz" : "__aeabi_d2uiz") : (is_float ? "__aeabi_f2iz" : "__aeabi_d2iz"); - } - break; - case TCCIR_OP_FCMP: - { - /* Get comparison operation from src2.c.i (stored during IR generation) */ - int cmp_op = src2->c.i; - int is_float = (src1_size == 4); - - switch (cmp_op) - { - case TOK_EQ: - return is_float ? "__aeabi_fcmpeq" : "__aeabi_dcmpeq"; - case TOK_NE: - /* NE uses cmpeq and inverts the result */ - return is_float ? "__aeabi_fcmpeq" : "__aeabi_dcmpeq"; - case TOK_LT: - case TOK_ULT: - return is_float ? "__aeabi_fcmplt" : "__aeabi_dcmplt"; - case TOK_LE: - case TOK_ULE: - return is_float ? "__aeabi_fcmple" : "__aeabi_dcmple"; - case TOK_GT: - case TOK_UGT: - return is_float ? "__aeabi_fcmpgt" : "__aeabi_dcmpgt"; - case TOK_GE: - case TOK_UGE: - return is_float ? "__aeabi_fcmpge" : "__aeabi_dcmpge"; - default: - /* Fallback to cfcmple/cdcmple which sets flags */ - return is_float ? "__aeabi_cfcmple" : "__aeabi_cdcmple"; - } - } - break; - case TCCIR_OP_CVT_ITOF: - { - /* Integer to float/double conversion. - * Need to distinguish 32-bit int vs 64-bit long long sources: - * - 32-bit: __aeabi_{ui,i}2{d,f} - * - 64-bit: __aeabi_{ul,l}2{d,f} - */ - int is_unsigned = (src1->type.t & VT_UNSIGNED) ? 1 : 0; - if (src1_size == 8) - { - /* 64-bit integer source (long long / unsigned long long) */ - if (is_unsigned) - return dest_64bit ? "__aeabi_ul2d" : "__aeabi_ul2f"; - return dest_64bit ? "__aeabi_l2d" : "__aeabi_l2f"; - } - /* 32-bit integer source (int / unsigned int) */ - if (is_unsigned) - return dest_64bit ? "__aeabi_ui2d" : "__aeabi_ui2f"; - return dest_64bit ? "__aeabi_i2d" : "__aeabi_i2f"; - } - break; - default: - break; - } - - return NULL; -} - -/* tcc_gen_machine_func_parameter_mop: MachineOperand-based entry point for - * FUNCPARAMVAL / FUNCPARAMVOID. src2_enc must be MACH_OP_IMM holding the - * packed call_id / param_idx value (same encoding as irop_get_imm64_ex). - * src1 is the value being passed (unused here — handled by the call-site ABI). - */ -ST_FUNC void tcc_gen_machine_func_parameter_mop(MachineOperand src1, MachineOperand src2_enc, TccIrOp op) -{ - (void)src1; - - const uint32_t encoded = (uint32_t)src2_enc.u.imm.val; - int call_id = TCCIR_DECODE_CALL_ID(encoded); - int param_index = TCCIR_DECODE_PARAM_IDX(encoded); - - /* Find or create call site for this call_id */ - ThumbGenCallSite *call_site = thumb_get_or_create_call_site(call_id); - if (call_site == NULL) - { - tcc_error("compiler_error: failed to allocate call site for call_id=%d", call_id); - return; - } - - /* FUNCPARAMVOID is a marker for a 0-argument call. - * Ensure the call site exists, but do not create a fake argument entry. */ - if (op == TCCIR_OP_FUNCPARAMVOID) - return; - - /* During dry-run, don't modify the argument list - it causes memory leaks - * when we restore the call sites after dry-run. The argument list is not - * needed for scratch register tracking anyway. */ - if (dry_run_state.active) - return; - - /* Expand argument list if needed */ - if (param_index >= call_site->function_argument_count) - { - int new_count = param_index + 1; - call_site->function_argument_list = (int *)tcc_realloc(call_site->function_argument_list, new_count * sizeof(int)); - /* Initialize new slots */ - for (int i = call_site->function_argument_count; i < new_count; i++) - { - call_site->function_argument_list[i] = -1; - } - call_site->function_argument_count = new_count; - } - - /* Store parameter information - for now just mark as present */ - call_site->function_argument_list[param_index] = 1; /* Mark parameter as present */ -} -/* Emit a nested-function trampoline into the current text section. - * chain_slot_sym: TCC symbol for the chain slot in .data - * func_sym: TCC symbol for the nested function in .text - * - * The trampoline loads the parent frame pointer from the chain slot - * into R10 (the static-chain register) and tail-calls the nested function. - * - * Two variants: - * - GOT-indirect (text_and_data_separation): uses R9-relative GOT loads, - * relocations are R_ARM_GOT32 (linker-resolved, no absolute addresses - * in the code section). - * - Direct: inline literal pool with R_ARM_ABS32 relocations. - */ -ST_FUNC addr_t gen_nested_func_trampoline(Sym *chain_slot_sym, Sym *func_sym) -{ - Section *text_sec = cur_text_section; - int use_got = tcc_state->text_and_data_separation; - - section_prealloc(text_sec, use_got ? 36 : 24); - - /* Align ind to 4-byte boundary */ - while (ind & 3) - text_sec->data[ind++] = 0x00; - - addr_t tramp_start = ind; - - if (use_got) - { - /* GOT-indirect trampoline (32 bytes): - * +0: LDR r12, [pc, #20] ; GOT offset of chain_slot (from +24) - * +4: LDR r10, [r9, r12] ; chain_slot address via GOT - * +8: LDR r10, [r10, #0] ; *chain_slot = parent FP - * +12: LDR r12, [pc, #12] ; GOT offset of function (from +28) - * +16: LDR r12, [r9, r12] ; function address via GOT - * +20: BX r12 ; tail-call - * +22: NOP - * +24: .word 0 ; R_ARM_GOT32 chain_slot - * +28: .word 0 ; R_ARM_GOT32 function - */ - - /* +0: LDR R12, [PC, #20] - F8DF C014 */ - text_sec->data[ind++] = 0xDF; - text_sec->data[ind++] = 0xF8; - text_sec->data[ind++] = 0x14; - text_sec->data[ind++] = 0xC0; - - /* +4: LDR R10, [R9, R12] - F859 A00C */ - text_sec->data[ind++] = 0x59; - text_sec->data[ind++] = 0xF8; - text_sec->data[ind++] = 0x0C; - text_sec->data[ind++] = 0xA0; - - /* +8: LDR R10, [R10, #0] - F8DA A000 */ - text_sec->data[ind++] = 0xDA; - text_sec->data[ind++] = 0xF8; - text_sec->data[ind++] = 0x00; - text_sec->data[ind++] = 0xA0; - - /* +12: LDR R12, [PC, #12] - F8DF C00C */ - text_sec->data[ind++] = 0xDF; - text_sec->data[ind++] = 0xF8; - text_sec->data[ind++] = 0x0C; - text_sec->data[ind++] = 0xC0; - - /* +16: LDR R12, [R9, R12] - F859 C00C */ - text_sec->data[ind++] = 0x59; - text_sec->data[ind++] = 0xF8; - text_sec->data[ind++] = 0x0C; - text_sec->data[ind++] = 0xC0; - - /* +20: BX R12 - 4760 */ - text_sec->data[ind++] = 0x60; - text_sec->data[ind++] = 0x47; - - /* +22: NOP - BF00 */ - text_sec->data[ind++] = 0x00; - text_sec->data[ind++] = 0xBF; - - /* +24: chain slot GOT offset */ - greloc(text_sec, chain_slot_sym, ind, R_ARM_GOT32); - text_sec->data[ind++] = 0x00; - text_sec->data[ind++] = 0x00; - text_sec->data[ind++] = 0x00; - text_sec->data[ind++] = 0x00; - - /* +28: function GOT offset */ - greloc(text_sec, func_sym, ind, R_ARM_GOT32); - text_sec->data[ind++] = 0x00; - text_sec->data[ind++] = 0x00; - text_sec->data[ind++] = 0x00; - text_sec->data[ind++] = 0x00; - } - else - { - /* Direct trampoline (20 bytes): - * +0: LDR r10, [pc, #8] ; chain_slot address (from +12) - * +4: LDR r10, [r10, #0] ; *chain_slot = parent FP - * +8: LDR pc, [pc, #4] ; function address (from +16), tail call - * +12: .word chain_slot ; R_ARM_ABS32 - * +16: .word function ; R_ARM_ABS32 - */ - - /* LDR R10, [PC, #8] - F8DF A008 */ - text_sec->data[ind++] = 0xDF; - text_sec->data[ind++] = 0xF8; - text_sec->data[ind++] = 0x08; - text_sec->data[ind++] = 0xA0; - - /* LDR R10, [R10, #0] - F8DA A000 */ - text_sec->data[ind++] = 0xDA; - text_sec->data[ind++] = 0xF8; - text_sec->data[ind++] = 0x00; - text_sec->data[ind++] = 0xA0; - - /* LDR PC, [PC, #4] - F8DF F004 */ - text_sec->data[ind++] = 0xDF; - text_sec->data[ind++] = 0xF8; - text_sec->data[ind++] = 0x04; - text_sec->data[ind++] = 0xF0; - - /* chain slot address */ - greloc(text_sec, chain_slot_sym, ind, R_ARM_ABS32); - text_sec->data[ind++] = 0x00; - text_sec->data[ind++] = 0x00; - text_sec->data[ind++] = 0x00; - text_sec->data[ind++] = 0x00; - - /* function address */ - greloc(text_sec, func_sym, ind, R_ARM_ABS32); - text_sec->data[ind++] = 0x00; - text_sec->data[ind++] = 0x00; - text_sec->data[ind++] = 0x00; - text_sec->data[ind++] = 0x00; - } - - text_sec->data_offset = ind; - return tramp_start + 1; /* +1 for Thumb interworking bit */ -} diff --git a/source/backend/arch/arm/thumb/arm-thumb-gen.h b/source/backend/arch/arm/thumb/arm-thumb-gen.h new file mode 100644 index 000000000..295fdc564 --- /dev/null +++ b/source/backend/arch/arm/thumb/arm-thumb-gen.h @@ -0,0 +1,621 @@ +/* + * ARMvX-m code generator for TCC + * Uses thumb instruction set + * + * Based on: + * ARM Thumb 2 instruction functions for TCC + * Copyright (c) 2020 Erlend J. Sveen + * from: + * https://git.erlendjs.no/erlendjs/tinycc/-/blob/arm-thumb/arm-thumb-gen.c + * https://git.erlendjs.no/erlendjs/tinycc/-/blob/arm-thumb/arm-thumb-instructions.c + * + * And + * + * ARMv4 code generator for TCC + * + * Copyright (c) 2003 Daniel Glöckner + * Copyright (c) 2012 Thomas Preud'homme + * + * Based on i386-gen.c by Fabrice Bellard + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +/* Declarations shared by the Thumb-2 code generator's source files + * (arm-thumb-gen.c and arm-thumb-{emit,alu,mem,fp,frame,call}.c), which + * used to be one 18k-line TU. Each function or variable declared at the + * bottom is defined in the file its comment names. */ + +#ifndef ARM_THUMB_GEN_H +#define ARM_THUMB_GEN_H + +#if defined(TCC_ARM_EABI) && !defined(TCC_ARM_VFP) +#error "Currently TinyCC only supports float computation with VFP instructions" +#endif + +#ifndef CONFIG_TCC_CPUVER +#define CONFIG_TCC_CPUVER 5 +#endif + +#include "source/backend/arch/arm/arm.h" +#include "source/backend/arch/arm/ssa_opt_arm.h" +#include "arm-thumb-defs.h" +#include "source/opt/include/opt.h" +#include "tcc-chained-hash.h" +#include "tcc.h" +#include "tccir.h" +#include "tccls.h" +#include "tcctype.h" +#include "arm-thumb-callsite.h" +#include "memory/small_sequence.h" + +/* Inline-first literal-pool position scratch (heap only past the inline cap). */ +TCC_SMALL_SEQUENCE_DEFINE(ThumbLitPosSeq, int, 128) + +/* Helper macro: split a 64-bit value into (lo, hi) uint32_t pair for load_full_const. + * Avoids int64_t in function signatures — TCC ARM codegen miscounts int64_t register pairs. */ +#define LFC_SPLIT(v) (uint32_t)((uint64_t)(v)), (uint32_t)((uint64_t)(v) >> 32) + +enum Armv8mRegisters +{ + ARM_R0 = 0, + ARM_R1 = 1, + ARM_R2 = 2, + ARM_R3 = 3, + ARM_R4 = 4, + ARM_R5 = 5, + ARM_R6 = 6, + ARM_R7 = 7, + ARM_R8 = 8, + ARM_R9 = 9, + ARM_R10 = 10, + ARM_R11 = 11, + ARM_R12 = 12, + ARM_SP = 13, + ARM_LR = 14, + ARM_PC = 15 +}; + +#define USING_GLOBALS +#include "tcc.h" + +#include +#include + +#include "source/backend/arch/fpu/arm/fpv5-sp-d16.h" +#include "source/backend/arch/fpu/arm/rp2350-dcp.h" +#include "source/backend/arch/fpu/arm/fpv5-d16.h" +#include "source/backend/arch/arm/thumb/thumb.h" +#include "source/backend/arch/arm/thumb/thop_adr.h" +#include "source/backend/arch/arm/thumb/thop_alu_imm.h" +#include "source/backend/arch/arm/thumb/thop_alu_reg.h" +#include "source/backend/arch/arm/thumb/thop_block.h" +#include "source/backend/arch/arm/thumb/thop_branch.h" +#include "source/backend/arch/arm/thumb/thop_cmp.h" +#include "source/backend/arch/arm/thumb/thop_extend.h" +#include "source/backend/arch/arm/thumb/thop_coproc.h" +#include "source/backend/arch/arm/thumb/thop_vfp.h" +#include "source/backend/arch/arm/thumb/thop_ldr_literal.h" +#include "source/backend/arch/arm/thumb/thop_ldrd.h" +#include "source/backend/arch/arm/thumb/thop_mem_imm.h" +#include "source/backend/arch/arm/thumb/thop_mem_reg.h" +#include "source/backend/arch/arm/thumb/thop_mov.h" +#include "source/backend/arch/arm/thumb/thop_mul.h" +#include "source/backend/arch/arm/thumb/thop_mvn.h" +#include "source/backend/arch/arm/thumb/thop_pld.h" +#include "source/backend/arch/arm/thumb/thop_shift_imm.h" +#include "source/backend/arch/arm/thumb/thop_rev.h" +#include "source/backend/arch/arm/thumb/thop_shift_reg.h" +#include "source/backend/arch/arm/thumb/thop_system.h" + +#include +ST_FUNC void tcc_gen_machine_strldr_cache_set_enabled(int enabled); +ST_FUNC void tcc_gen_machine_imm_cache_reset(void); +/* Structure to track scratch register allocation with potential save/restore */ +typedef struct ScratchRegAlloc +{ + int reg : 29; /* The allocated scratch register (range 0-15 for ARM) */ + uint32_t saved : 2; /* 0=not saved, 1=PUSH to stack, 2=STR to scratch area */ + uint32_t would_save : 1; /* Whether a push was needed (set in both dry-run and real emit) */ +} ScratchRegAlloc; + +typedef struct ScratchRegAllocs +{ + int regs[8]; /* The allocated scratch registers */ + int count; /* Number of registers allocated */ + uint32_t saved_mask; /* Bitmask of registers that were saved (pushed) */ +} ScratchRegAllocs; + +/* ============================================================ + * MachineCodegenContext — per-instruction scratch-register tracker + * ============================================================ + * Used by the MachineOperand-based (_mop) code-generation path. + * Callers allocate scratches via mach_alloc_scratch(), then call + * mach_release_all() at the end of the instruction to pop them in LIFO order. + */ + +/* Forward declarations needed by the mach_* helpers (defined later in this file). */ +typedef thumb_opcode (*thumb_imm_handler_t)(uint32_t rd, uint32_t rn, uint32_t imm, + thumb_flags_behaviour flags_behaviour, + thumb_enforce_encoding enforce_encoding); + +#define MACH_CTX_MAX_SCRATCH 12 + +typedef struct MachineCodegenContext +{ + ScratchRegAlloc scratches[MACH_CTX_MAX_SCRATCH]; + int n_scratch; +} MachineCodegenContext; + +/* --- Hard-float single-precision VFP support --- + * Float operands reach codegen as MACH_OP_VFP_REG (u.reg.r0 = s-register 0-31), + * a distinct kind from GPR MACH_OP_REG. s14/s15 are held out of the allocator + * (arm_init) as fixed VFP scratch so the FP emitters can shuttle GPR/imm/spill + * operands through the FPU without clobbering a live float. is_vfp_reg tests a + * raw allocation register number (LS_VFP_REG_BASE + n) as written by the RA. */ +#define VFP_SCRATCH0 14 +#define VFP_SCRATCH1 15 + +/* ============================================================ + * Dry-Run Code Generation State + * ============================================================ + * Two-pass code generation system for optimal register allocation. + * Pass 1 (Dry Run): Analyze register needs without emitting code + * Pass 2 (Real Emit): Generate code with optimal prologue based on Pass 1 + */ + +typedef struct CodeGenDryRunState +{ + int active; /* 1 = dry run, 0 = real emit */ + uint32_t scratch_regs_pushed; /* Bitmap of regs pushed as scratch */ + int scratch_push_count; /* Total scratch push operations */ + int lr_push_count; /* Times LR specifically was pushed */ + int instruction_count; /* IR instructions processed */ + int max_nested_saves; /* Max nested-call save slots used at any call site */ + int rodata_anchor_sites; /* Shared-.rodata addresses materialised in the body */ +} CodeGenDryRunState; +/* True only for the discovery dry run: "model what would happen" rather than + * "emit what the real pass emits". */ +#define DRY_RUN_MODELLING (dry_run_state.active && !dry_run_rehearsal) + +/* Literal pool dedup uses the same bucket+chain scheme as TinyCC's ELF hashes. + * We only hash entries created through th_literal_pool_find_or_allocate(), so + * plain th_literal_pool_allocate() users stay distinct. */ +#define LITERAL_POOL_HASH_BUCKET_COUNT 512 +#define LITERAL_POOL_LOOKUP_CACHE_SIZE 16 + +typedef struct LiteralPoolLookupCacheEntry +{ + Sym *sym; + int64_t imm; + int pool_index; + uint32_t hash; + int valid; +} LiteralPoolLookupCacheEntry; + +typedef struct LiteralPoolLookupCache +{ + LiteralPoolLookupCacheEntry entries[LITERAL_POOL_LOOKUP_CACHE_SIZE]; +} LiteralPoolLookupCache; + +/* Structure to save/restore thumb_gen_state for dry-run isolation */ +typedef struct ThumbGenStateSnapshot +{ + int code_size; + int literal_pool_count; + int literal_pool_size; + int pool_window_first; + int pool_bytes; + ThumbLiteralPoolEntry *literal_pool; + Sym *cached_global_sym; + int cached_global_reg; + int function_argument_count; + int call_sites_by_id_size; + ThumbGenCallSite *call_sites_by_id; +} ThumbGenStateSnapshot; + +/* ============================================================ + * Branch Instruction Optimization State + * ============================================================ + * Tracks branch instructions during dry-run to select optimal + * 16-bit vs 32-bit encodings based on actual jump distances. + */ + +typedef enum +{ + BRANCH_ENC_UNKNOWN = 0, + BRANCH_ENC_16BIT = 16, + BRANCH_ENC_32BIT = 32 +} BranchEncoding; + +typedef struct BranchInfo +{ + int ir_index; /* IR instruction index of the branch */ + int source_addr; /* Code address where branch is emitted */ + int target_ir; /* Target IR instruction index */ + int target_addr; /* Target code address (computed after dry-run) */ + int offset; /* Computed offset = target - source - 4 */ + int is_conditional; /* 1 = conditional (JUMPIF), 0 = unconditional (JUMP) */ + BranchEncoding encoding; /* Selected encoding after analysis */ +} BranchInfo; + +typedef struct BranchOptState +{ + BranchInfo *branches; /* Array of branch info */ + int branch_count; /* Number of branches */ + int branch_capacity; /* Allocated capacity */ + int optimization_enabled; /* Flag to enable/disable */ + int code_size_reduction; /* Total bytes saved */ +} BranchOptState; + +/* Forward declarations for helpers used by spill preloading. */ +int load_short_from_base(int ir, int base, int fc, int sign); +int load_ushort_from_base(int ir, int base, int fc, int sign); +int load_byte_from_base(int ir, int base, int fc, int sign); +int load_ubyte_from_base(int ir, int base, int fc, int sign); + +/* Immediate-value cache: tracks the last pure-integer constant loaded into + * each register by tcc_machine_load_constant (no symbol involved). Persists + * across IR instruction boundaries so consecutive STORE instructions that + * materialise the same constant can skip the redundant MOV. Reset at jump + * targets and function calls. */ +/* Per-register materialisation cache. `sym == NULL` means the register holds + * the plain constant `value`; `sym != NULL` means it holds the address of that + * symbol plus addend `value` (so a later reference to the same global address + * can skip the redundant literal-pool load). Invalidated per-register on every + * clobbering emit and at IR boundaries, just like the constant cache. */ +typedef struct ImmCacheEntry { int64_t value; Sym *sym; uint8_t valid; } ImmCacheEntry; + +/* Bit layout of tcc_ir_zero_half64_at's byte, as it arrives in bits 18-23 of + * `barrel_shift`. Mirrors source/opt/flat/fusion/zero_half64.c. */ +#define ZH64_S1_LO 0x01u +#define ZH64_S1_HI 0x02u +#define ZH64_S2_LO 0x04u +#define ZH64_S2_HI 0x08u +#define ZH64_D_LO 0x10u +#define ZH64_D_HI 0x20u +/* Bit 6 does not come from zero_half64 -- it is source/opt/flat/fusion/ + * cmp_hi_only.c's verdict, sharing the same side table and the same journey + * through `barrel_shift`: this 64-bit CMP is against a constant whose low half + * is zero and every reader of its flags is a strict order test, so the low + * words cannot change the answer and only the high words need comparing. */ +#define ZH64_CMP_HI 0x40u + +/* --------------------------------------------------------------------------- + * STR -> LDR redundant-reload peephole + * + * Tracks recent immediate-offset STR Rt, [Rn, #imm] emissions and skips the + * subsequent LDR Rt, [Rn, #imm] at the call site when Rt is still known to + * hold the stored value. The cache is reset at every IR instruction + * boundary (via tcc_gen_machine_mov_coalesce_reset, same hook as plan C) + * so that cross-IR equivalences cannot be exploited — any IR op may be a + * branch target, and the runtime register/memory state on an entry-by-jump + * path is not what the emission-order state predicts. + * + * Only puw == 6 (P=1, U=1, W=0 — no writeback) STR/LDR forms are tracked. + * The classifier below recognises the T1 16-bit, T2 16-bit SP-relative, and + * T3 32-bit encodings (those that cover the common stack-spill path). + * --------------------------------------------------------------------------- */ + +typedef struct StrLdrCacheEntry +{ + uint8_t valid; + uint8_t rt; + uint8_t rn; + uint8_t size; /* encoding size, 2 or 4 */ + uint8_t width; /* bytes the access moves; only 4 is ever recorded */ + int imm; + uint32_t puw; +} StrLdrCacheEntry; + +#define STRLDR_CACHE_CAPACITY 8 + +/* The last word access to the frame, if it was the last instruction emitted: + * what frame_word_pair_rewind pairs the next one with. */ +typedef struct FrameWordAccess +{ + int kind; /* 0 none, 1 STR, 2 LDR */ + int start, end; + int reg, base, off; /* off: signed byte offset from base */ +} FrameWordAccess; + +/* Architectural ceiling for a pool window: the patch in + * th_literal_pool_generate() encodes (aligned_position - 4), and both the T1 + * LDR-literal and the LDRD-literal forms carry an 8-bit word-scaled offset, + * i.e. at most 1020 bytes from the load to its literal. */ +#define THUMB_POOL_RANGE_LIMIT 1020 +/* Slack under the ceiling for what the span measurement cannot see coming: a + * flush suppressed by op_in_it_block is deferred to the end of the IT block + * (up to 4 conditioned insns, each able to add a pool entry), and multi-op + * sequences reserve only an estimate of their size. */ +#define THUMB_POOL_FLUSH_SLACK 64 + +typedef thumb_opcode (*thumb_reg_handler_t)(uint32_t rd, uint32_t rn, uint32_t rm, + thumb_flags_behaviour flags_behaviour, thumb_shift shift_type, + thumb_enforce_encoding enforce_encoding); +typedef struct ThumbDataProcessingHandler +{ + thumb_imm_handler_t imm_handler; + thumb_reg_handler_t reg_handler; +} ThumbDataProcessingHandler; + +/* ============================================================ + * thumb_and_imm_form / thumb_emit_and_imm_special + * ============================================================ + * One-instruction lowerings of `AND Rd, Rn, #mask` that never materialize the + * mask. Thumb-2's AND-immediate takes only a modified immediate (8 bits + * rotated, or a byte replicated across the word), so an ordinary bitfield mask + * -- 0x7FF, 0x000FFFFF, 0x007FFFFF -- costs a movw/movt pair or a literal-pool + * load on top of the AND. In the soft-float library that pair is the single + * most common shape there is: every classifier opens with + * `bits & DOUBLE_MANT_MASK`. + * + * The classification is a pure function of the mask so that a caller can + * decide BEFORE resolving operands into registers -- resolving can itself emit + * (a spilled source becomes a load), so a caller that resolved first and then + * fell through to the general path would emit that load twice. + * + * None of these forms writes the flags, so the caller must already have + * established that no flag result is wanted. `flags` is passed through for + * BIC, which does have a flag-setting encoding. + */ +typedef enum ThumbAndImmForm +{ + AND_IMM_NONE = 0, /* nothing better than the general path */ + AND_IMM_UXTB, /* mask 0xFF */ + AND_IMM_UXTH, /* mask 0xFFFF */ + AND_IMM_UBFX, /* low-contiguous run of ones */ + AND_IMM_BIC, /* complement encodes as a modified immediate */ +} ThumbAndImmForm; + +typedef thumb_opcode (*thumb_regonly3_handler_t)(uint32_t rd, uint32_t rn, uint32_t rm); + +typedef thumb_opcode (*thumb_longmul_handler_t)(uint32_t rdlo, uint32_t rdhi, uint32_t rn, uint32_t rm); + +typedef enum ThumbArgMoveKind +{ + THUMB_ARG_MOVE_REG, + THUMB_ARG_MOVE_IMM, + THUMB_ARG_MOVE_IMM64, /* load 64-bit immediate into register pair */ + THUMB_ARG_MOVE_LOCAL_ADDR, /* compute address of local: fp + offset */ + THUMB_ARG_MOVE_STRUCT, /* load struct words into consecutive registers */ + THUMB_ARG_MOVE_MOP, /* generic: load MachineOperand into dst_reg (+ dst_reg_hi for 64-bit) */ +} ThumbArgMoveKind; + +typedef struct ThumbArgMove +{ + ThumbArgMoveKind kind; + int dst_reg; + int dst_reg_hi; /* valid when kind==THUMB_ARG_MOVE_IMM64 */ + int src_reg; /* valid when kind==THUMB_ARG_MOVE_REG */ + uint32_t imm; /* valid when kind==THUMB_ARG_MOVE_IMM */ + uint64_t imm64; /* valid when kind==THUMB_ARG_MOVE_IMM64 */ + Sym *sym; /* valid when kind==THUMB_ARG_MOVE_IMM */ + int local_offset; /* valid when kind==THUMB_ARG_MOVE_LOCAL_ADDR */ + int local_is_param; /* valid when kind==THUMB_ARG_MOVE_LOCAL_ADDR - if true, add offset_to_args */ + int struct_word_count; /* valid when kind==THUMB_ARG_MOVE_STRUCT */ + int struct_src_align; /* struct natural alignment (bytes); gates source LDRD */ + MachineOperand mop; /* valid when kind==THUMB_ARG_MOVE_MOP */ +} ThumbArgMove; + +/* Context for function call generation - reduces parameter passing */ +typedef struct CallGenContext +{ + ThumbGenCallSite *call_site; + TCCAbiCallLayout *layout; + IROperand *args; + MachineOperand *mops; + int argc; + int stack_size; + uint32_t arg_move_dst_mask; /* Registers that will be explicitly written by register arg moves. + * These are safe to clobber as scratch during stack arg placement + * because the subsequent register moves will overwrite them. */ +} CallGenContext; + +/* Copy a (possibly split) struct argument's stack portion into the outgoing + * argument area. `src_align` is the struct's natural alignment in bytes. + * + * Adjacent word pairs are copied with LDRD/STRD instead of two LDR/STR. The + * destination is the outgoing arg area — SP-relative with a word-multiple + * offset and SP 8-byte aligned at the call boundary — so STRD is always + * alignment-safe. LDRD additionally requires the *source* address to be + * 4-byte aligned, which holds exactly when the struct's natural alignment is + * >= 4 (the stack portion starts at base + words_in_regs*4, a word multiple). */ +/* Stack-passed struct arguments at least this many words are copied through + * r0-r3/IP/LR saved around the copy -- LDM/STM, or a memcpy call for an + * unaligned source -- rather than with scratch registers found free one pair + * at a time (~4 bytes a word). Measured with the memcpy form on zig.c .text: + * 32 words 5,822,056; 8 words 5,754,812; 6 words 5,751,108; 4 words + * 5,772,344 -- 8 keeps 6- and 7-word structs on the faster inline copy. */ +#define STACK_ARG_MEMCPY_MIN_WORDS 8 +#define STACK_ARG_UNROLL_MAX_WORDS 16 + +/* One collected immediate stack store, for the grouped/windowed emission path. */ +typedef struct StackImmArg +{ + int off; + uint32_t val; +} StackImmArg; + +/* Defined in arm-thumb-gen.c. */ +extern struct Sym *_lfc_sym; +extern ThumbGeneratorState thumb_gen_state; +extern enum float_abi float_abi; +extern unsigned char text_and_data_separation; +extern unsigned char allow_r9_write; +extern unsigned char pic; +extern unsigned char sb_relative_got; +extern int offset_to_args; +extern uint32_t pushed_registers; +extern int allocated_stack_size; +extern int epilogue_stack_dealloc; +extern int vararg_push_size; +extern uint32_t scratch_global_exclude; +extern int rodata_anchor_reg; +extern int scratch_push_type[128]; +extern int scratch_push_count; +extern int tail_call_pending; +extern int g_debug_current_op; +extern CodeGenDryRunState dry_run_state; +extern int dry_run_rehearsal; +extern int helper_call_sp_bias; +extern int call_args_sp_bias; +extern ThumbLiteralPoolEntry *dry_run_literal_pool; +extern int dry_run_literal_pool_count; +extern int pool_entries_total; +extern int align_pad_max; +extern int dry_run_literal_pool_size; +extern TCCChainedHash literal_pool_hash; +extern LiteralPoolLookupCache literal_pool_last_lookup; +extern int mov_equiv_it_pending; +extern ImmCacheEntry imm_cache[16]; +extern int pool_flush_it_pending; +extern FrameWordAccess frame_word_last; +extern int slot_self_copy_off; +thumb_flags_behaviour flags_safe(void); +Sym *validate_sym_for_reloc(Sym *sym); +int fp_adjust_local_offset(int frame_offset, int is_param); +int param_frame_offset(int param_off); +uint32_t scratch_exclude_baseline(void); +int resolve_chain_base(TCCIRState *ir, int ci, uint32_t exclude_regs, ScratchRegAlloc *out_scratch, + int *used_scratch); +thumb_opcode thumb_call_imm_handler(thumb_imm_handler_t fn, uint32_t rd, uint32_t rn, uint32_t imm, + thumb_flags_behaviour flags, thumb_enforce_encoding encoding); +int is_vfp_reg(int r); +int vfp_num(int r); +int mach_alloc_scratch(MachineCodegenContext *ctx, uint32_t excl); +int mach_alloc_scratch_for_sym(MachineCodegenContext *ctx, uint32_t excl, Sym *sym, int64_t imm); +void mach_release_all(MachineCodegenContext *ctx); +int mach_var_owns_spill_slot(int vreg); +void mach_load_slot(int dest_reg, const MachineOperand *op); +int mach_slot_load_width(const MachineOperand *op); +int mach_slot_store_width(const MachineOperand *op); +int mach_ensure_in_reg(MachineCodegenContext *ctx, const MachineOperand *op, uint32_t excl); +uint32_t mach_reg_excl(const MachineOperand *op); +int mach_ensure_imm_or_reg(MachineCodegenContext *ctx, const MachineOperand *op, uint32_t excl, + thumb_imm_handler_t imm_handler, int dest_reg, int src1_reg, + thumb_flags_behaviour flags, bool *imm_emitted); +int mach_get_dest_reg(MachineCodegenContext *ctx, const MachineOperand *op, uint32_t excl); +void mach_writeback_dest(const MachineOperand *op, int reg); +void tcc_gen_mach_load_to_reg(int dest_reg, const MachineOperand *op); +void map_sym_d(void); +void map_sym_t(void); +int scratch_push_sp_bias(void); +uint32_t literal_pool_hash_func(Sym *sym, int64_t imm); +int literal_pool_lookup_cache_find(LiteralPoolLookupCache *cache, uint32_t full_hash, Sym *sym, + int64_t imm); +void literal_pool_lookup_cache_insert(LiteralPoolLookupCache *cache, uint32_t full_hash, Sym *sym, + int64_t imm, int pool_index); +int literal_pool_hash_find(TCCChainedHash *hash, ThumbLiteralPoolEntry *pool, uint32_t full_hash, + Sym *sym, int64_t imm); +void literal_pool_hash_insert(TCCChainedHash *hash, uint32_t full_hash, int pool_index); +int branch_fits_t1(int offset); +int branch_fits_t2(int offset); +ScratchRegAlloc get_scratch_reg_with_save(uint32_t exclude_regs); +void restore_scratch_reg(ScratchRegAlloc *alloc); +void restore_all_pushed_scratch_regs(void); +int ot_check(thumb_opcode op); +void imm_cache_reset_all(void); +void imm_cache_invalidate_reg(int reg); +int try_scratch_reg_for_sym_addr(Sym *raw_sym, int64_t imm, uint32_t exclude_regs, ScratchRegAlloc *out); +void mov_equiv_reset_all(void); +int mov_equiv_it_block_length(thumb_opcode op); +void mov_equiv_invalidate_reg(int reg); +void mov_equiv_record_mov(int rd, int rm); +int ot_check_mov_reg(uint32_t rd, uint32_t rm, thumb_flags_behaviour flags, thumb_shift shift, + thumb_enforce_encoding enc, bool in_it); +int decode_mov_reg_plain(thumb_opcode op, int *rd_out, int *rm_out); +ST_FUNC void tcc_gen_machine_strldr_cache_reset(void); +void codegen_internal_merge_point(void); +void strldr_cache_invalidate_reg(int reg); +void strldr_cache_invalidate_mem(int rn, int imm, int width); +void strldr_cache_record_access(int rt, int rn, int imm, uint32_t puw, int size, int width, int is_store); +int strldr_cache_try_match_ldr(int rt, int rn, int imm, uint32_t puw, int size, int width); +int strldr_cache_str_is_redundant(int rt, int rn, int imm, uint32_t puw, int size); +int strldr_cache_ldrd_is_redundant(int rt, int rt2, int rn, int imm, uint32_t puw); +int decode_str_ldr_imm(thumb_opcode op, int *is_str_out, int *rt_out, int *rn_out, int *imm_out, + uint32_t *puw_out, int *width_out); +int decode_strd_ldrd_imm(thumb_opcode op, int *is_str_out, int *rn_out, int *imm_out); +int ot_check_ldr_imm(uint32_t rt, uint32_t rn, int imm, uint32_t puw, thumb_enforce_encoding enc); +int ot_check_str_imm(uint32_t rt, uint32_t rn, int imm, uint32_t puw, thumb_enforce_encoding enc); +uint32_t mapcc(int cc); +void o(unsigned int i); +void th_literal_pool_generate(void); +int th_pool_span_after(int upcoming_bytes); +void th_literal_pool_note_entry(const ThumbLiteralPoolEntry *entry); +void th_literal_pool_reserve_upcoming_bytes(int upcoming_bytes); +int th_literal_pool_would_flush_for(int upcoming_bytes); +int is_valid_opcode(thumb_opcode op); +int thumb_op_is_plain_branch(thumb_opcode op); +int thumb_op_is_pure_flag_setter(thumb_opcode op); +int thumb_decode_dest_reg(thumb_opcode op); + +/* Defined in arm-thumb-emit.c. */ +int ot(thumb_opcode op); +thumb_opcode th_generic_mov_imm(uint32_t r, int imm); +ScratchRegAlloc th_offset_to_reg_ex(int off, int sign, uint32_t exclude_regs); +int th_patch_call(int t, int a); +void gadd_sp_ex(int val, int scratch_reg); +void gadd_sp(int val); +int load_word_from_base(int ir, int base, int fc, int sign); +int store_word_to_base(int ir, int base, int fc, int sign); +int sym_is_4_byte_aligned_for_64bit(Sym *sym, int32_t addend); +int try_strd_pair(int lo_reg, int hi_reg, int base, int abs_off, int sign); +int try_ldrd_pair(int lo_reg, int hi_reg, int base, int abs_off, int sign); +ST_FUNC int tcc_gen_machine_try_ldrd_spill(int reg1, int32_t off1, int reg2, int32_t off2); +ST_FUNC int tcc_gen_machine_try_ldrd_base(int reg1, int reg2, int base_reg, int32_t off); +ST_FUNC int tcc_gen_machine_try_strd_base(int reg1, int reg2, int base_reg, int32_t off); +ST_FUNC void tcc_machine_load_spill_slot(int dest_reg, int frame_offset); +ST_FUNC void tcc_machine_store_spill_slot(int src_reg, int frame_offset); +ST_FUNC void tcc_machine_store_param_slot(int src_reg, int frame_offset); +void th_store32_imm_or_reg_ex(int src_reg, uint32_t base_reg, int abs_off, int sign, uint32_t extra_exclude); +void th_store16_imm_or_reg(int src_reg, uint32_t base_reg, int abs_off, int sign); +void th_store8_imm_or_reg(int src_reg, uint32_t base_reg, int abs_off, int sign); +void load_full_const(int r, int r1, uint32_t imm_lo, uint32_t imm_hi); +ST_FUNC void tcc_machine_addr_of_stack_slot(int dest_reg, int frame_offset, int is_param); +ST_FUNC void tcc_machine_load_constant(int dest_reg, int dest_reg_high, int64_t value, int is_64bit, Sym *sym); +void load_from_base(int r, int r1, int irop_btype, int is_unsigned, int fc, int sign, uint32_t base); +thumb_opcode thumb_call_reg_handler(thumb_reg_handler_t fn, uint32_t rd, uint32_t rn, uint32_t rm, + thumb_flags_behaviour flags, thumb_shift shift, + thumb_enforce_encoding encoding); +void thumb_require_materialized_reg(const char *ctx, const char *operand, int reg); +uint32_t thumb_exclude_mask_for_regs(int count, const int *regs); +bool thumb_is_hw_reg(int reg); + +/* Defined in arm-thumb-alu.c. */ +bool mach_op_64_names_memory(const MachineOperand *op); +MachineOperand mach_resolve_deref_64(MachineCodegenContext *mctx, const MachineOperand *op, uint32_t *excl); +MachineOperand mach_make_lo_half(const MachineOperand *op); +MachineOperand mach_make_hi_half(const MachineOperand *op); +void mach_ensure_pair_in_regs(MachineCodegenContext *ctx, const MachineOperand *op64, + uint32_t *excl, int *out_lo, int *out_hi); +ST_FUNC void tcc_gen_machine_assign_mop(MachineOperand src, MachineOperand dest, TccIrOp op); + +/* Defined in arm-thumb-fp.c. */ +MachineOperand mach_make_complex_real(const MachineOperand *op); +MachineOperand mach_make_complex_imag(const MachineOperand *op); + +/* Defined in arm-thumb-frame.c. */ +ST_FUNC void tcc_gen_machine_store_to_stack_ex(int reg, int offset, uint32_t extra_exclude); +void gcall_or_jump_mop(int is_jmp, MachineOperand target); +void load_immediate(int reg, uint32_t imm, Sym *sym, int update_flags); + +/* Defined in arm-thumb-call.c. */ +int thumb_callee_needs_local_call_marker(Sym *sym); +Sym *thumb_local_libc_helper(Sym *sym); +int thumb_callee_in_this_module(const MachineOperand *func_mop); +int thumb_callee_noreturn(const MachineOperand *func_mop); + +#endif /* ARM_THUMB_GEN_H */ diff --git a/source/backend/arch/arm/thumb/arm-thumb-mem.c b/source/backend/arch/arm/thumb/arm-thumb-mem.c new file mode 100644 index 000000000..358371a97 --- /dev/null +++ b/source/backend/arch/arm/thumb/arm-thumb-mem.c @@ -0,0 +1,1223 @@ +/* + * ARMvX-m code generator for TCC + * Uses thumb instruction set + * + * Based on: + * ARM Thumb 2 instruction functions for TCC + * Copyright (c) 2020 Erlend J. Sveen + * from: + * https://git.erlendjs.no/erlendjs/tinycc/-/blob/arm-thumb/arm-thumb-gen.c + * https://git.erlendjs.no/erlendjs/tinycc/-/blob/arm-thumb/arm-thumb-instructions.c + * + * And + * + * ARMv4 code generator for TCC + * + * Copyright (c) 2003 Daniel Glöckner + * Copyright (c) 2012 Thomas Preud'homme + * + * Based on i386-gen.c by Fabrice Bellard + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +/* Thumb-2 code generator: load and store lowering (plain, indexed and + * post-increment) of MachineOperands. */ + +#include "arm-thumb-gen.h" + +/* tcc_gen_machine_load_mop: MachineOperand-based entry point for TCCIR_OP_LOAD. + * + * dest can be MACH_OP_REG, MACH_OP_SPILL, or MACH_OP_PARAM_STACK. + * For spilled destinations, a scratch register is allocated and the result + * is written back to the spill slot after the load completes. + * 64-bit dest is supported: for MACH_OP_REG dest.u.reg.r1 holds the hi + * register; for spilled dests, a second scratch is allocated for hi-half. + * + * src encodes the memory address: + * MACH_OP_REG + needs_deref=true → LDR dest, [src_reg] + * MACH_OP_SPILL → LDR dest, [FP + fp_adjust(offset)] + * MACH_OP_SPILL + needs_deref=true → LLOCAL: LDR ptr,[FP+off]; LDR dest,[ptr] + * MACH_OP_PARAM_STACK → LDR dest, [FP + param_off + offset_to_args] + * MACH_OP_SYMBOL → LDR_literal addr; LDR dest, [addr] + * MACH_OP_IMM → tcc_machine_load_constant (constant load) + */ +ST_FUNC void tcc_gen_machine_load_mop(MachineOperand src, MachineOperand dest, TccIrOp op) +{ + MachineCodegenContext ctx = {0}; + (void)op; + + /* A load whose source is a VFP register is a read out of that s-register (a + * register-promoted float), not a memory load — delegate to the VFP-aware + * assign path, which handles every destination kind. */ + if (src.kind == MACH_OP_VFP_REG) + { + tcc_gen_machine_assign_mop(src, dest, TCCIR_OP_ASSIGN); + return; + } + + /* Determine dest register — allocates scratch if dest is SPILL/PARAM_STACK. */ + const bool dest_is_simple_reg = + (dest.kind == MACH_OP_REG && !dest.needs_deref && dest.u.reg.r0 != (int)PREG_REG_NONE); + const int dest_reg = mach_get_dest_reg(&ctx, &dest, 0); + + /* For 64-bit pairs: get hi-half dest register. */ + int dest_r1 = PREG_REG_NONE; + MachineOperand dest_hi_mop = {0}; + if (dest.is_64bit) + { + if (dest_is_simple_reg) + { + dest_r1 = dest.u.reg.r1; + } + else + { + dest_hi_mop = mach_make_hi_half(&dest); + dest_r1 = mach_get_dest_reg(&ctx, &dest_hi_mop, (1u << (uint32_t)dest_reg)); + } + } + + const int btype = src.btype; + const int is_unsigned = (int)src.is_unsigned; + + switch (src.kind) + { + case MACH_OP_REG: + if (src.needs_deref) + { + /* Register-indirect: LDR dest, [src_reg]. + * 64-bit + proven >= 4-byte alignment (src.align4, from the frontend's + * packed-access tracking): use LDRD directly. LDRD Rt==Rn is fine + * without writeback, so no base-preservation dance is needed. Without + * the proof, load_from_base emits the unaligned-safe LDR pair. */ + if (dest.is_64bit && !dest.is_complex && src.align4 && dest_r1 != (int)PREG_REG_NONE && + try_ldrd_pair(dest_reg, dest_r1, src.u.reg.r0, 0, 0)) + break; + load_from_base(dest_reg, dest_r1, btype, is_unsigned, 0, 0, (uint32_t)src.u.reg.r0); + } + else + { + /* Direct register-to-register (treat as MOV — should be ASSIGN, not LOAD) */ + if (dest_reg != src.u.reg.r0) + ot_check_mov_reg((uint32_t)dest_reg, (uint32_t)src.u.reg.r0, flags_safe(), THUMB_SHIFT_DEFAULT, + ENFORCE_ENCODING_NONE, false); + /* Narrow sub-word parameter values: when a parameter is declared as + * char/short but arrives in a full 32-bit register (AAPCS default + * argument promotion), the upper bits may contain garbage. Emit + * UXTB/SXTB/UXTH/SXTH to truncate to the declared type width. */ + if (btype == IROP_BTYPE_INT8) + { + if (is_unsigned) + ot_check(th_uxtb((uint32_t)dest_reg, (uint32_t)dest_reg, THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); + else + ot_check(th_sxtb((uint32_t)dest_reg, (uint32_t)dest_reg, THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); + } + else if (btype == IROP_BTYPE_INT16) + { + if (is_unsigned) + ot_check(th_uxth((uint32_t)dest_reg, (uint32_t)dest_reg, THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); + else + ot_check(th_sxth((uint32_t)dest_reg, (uint32_t)dest_reg, THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); + } + /* 64-bit pair: also copy the hi-half register */ + if (dest_r1 != PREG_REG_NONE && src.u.reg.r1 >= 0 && dest_r1 != src.u.reg.r1) + ot_check_mov_reg((uint32_t)dest_r1, (uint32_t)src.u.reg.r1, flags_safe(), THUMB_SHIFT_DEFAULT, + ENFORCE_ENCODING_NONE, false); + } + break; + + case MACH_OP_SPILL: + { + const int adj = fp_adjust_local_offset(src.u.spill.offset, 0); + const int sign = (adj < 0), abs_off = sign ? -adj : adj; + const uint32_t base = (uint32_t)(tcc_state->need_frame_pointer ? R_FP : R_SP); + if (!src.needs_deref) + { + /* Load value directly from spill/local slot */ + load_from_base(dest_reg, dest_r1, btype, is_unsigned, abs_off, sign, base); + } + else + { + /* LLOCAL: spill slot holds a pointer; load ptr, then dereference */ + int ptr_r = mach_alloc_scratch(&ctx, ((uint32_t)1u << (uint32_t)dest_reg) | ((uint32_t)1u << base)); + if (!load_word_from_base(ptr_r, (int)base, abs_off, sign)) + { + ScratchRegAlloc rr = th_offset_to_reg_ex(abs_off, sign, ((uint32_t)1u << ptr_r) | ((uint32_t)1u << base)); + ot_check(th_ldr_reg((uint32_t)ptr_r, base, (uint32_t)rr.reg, THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); + restore_scratch_reg(&rr); + } + /* 64-bit + proven alignment: LDRD through the loaded pointer. */ + if (dest.is_64bit && !dest.is_complex && src.align4 && dest_r1 != (int)PREG_REG_NONE && + try_ldrd_pair(dest_reg, dest_r1, ptr_r, 0, 0)) + break; + load_from_base(dest_reg, dest_r1, btype, is_unsigned, 0, 0, (uint32_t)ptr_r); + } + break; + } + + case MACH_OP_PARAM_STACK: + { + const int adj = param_frame_offset(src.u.param.offset); + const int sign = (adj < 0), abs_off = sign ? -adj : adj; + const uint32_t base = (uint32_t)(tcc_state->need_frame_pointer ? R_FP : R_SP); + load_from_base(dest_reg, dest_r1, btype, is_unsigned, abs_off, sign, base); + break; + } + + case MACH_OP_SYMBOL: + { + Sym *sym = src.u.sym.sym ? validate_sym_for_reloc(src.u.sym.sym) : NULL; + const int32_t addend = src.u.sym.addend; + if (!src.needs_deref) + { + /* Load symbol address (+ addend) into dest — no dereference. + * Load symbol address (+ addend) — no dereference. */ + tcc_machine_load_constant(dest_reg, dest_r1, (int64_t)addend, (int)dest.is_64bit, sym); + break; + } + /* needs_deref: load symbol address into scratch, then dereference. */ + int addr_r = mach_alloc_scratch_for_sym(&ctx, (uint32_t)1u << (uint32_t)dest_reg, sym, 0); + tcc_machine_load_constant(addr_r, PREG_REG_NONE, 0, 0, sym); + /* For a 64-bit deref, try LDRD when we can prove the symbol's address + * at `addend` is 4-byte aligned. Otherwise fall back to the pair of + * 32-bit loads via load_from_base. */ + const int sym_sign = (addend < 0), sym_abs = sym_sign ? (int)(-addend) : (int)addend; + if (dest.is_64bit && dest_r1 != PREG_REG_NONE && sym_is_4_byte_aligned_for_64bit(sym, addend) && + try_ldrd_pair(dest_reg, dest_r1, addr_r, sym_abs, sym_sign)) + { + break; + } + load_from_base(dest_reg, dest_r1, btype, is_unsigned, sym_abs, sym_sign, (uint32_t)addr_r); + break; + } + + case MACH_OP_IMM: + if (src.needs_deref) + { + /* Read through an absolute address: `*(volatile T *)0x40000000`, the + * MMIO register idiom. Materialize the address, then load from it — + * the same shape as the MACH_OP_SYMBOL deref path above, and the mirror + * of the store side, which has always done this. Alignment of an + * arbitrary constant address is unknown, so no LDRD. */ + int addr_r = mach_alloc_scratch(&ctx, (uint32_t)1u << (uint32_t)dest_reg); + tcc_machine_load_constant(addr_r, PREG_REG_NONE, src.u.imm.val, 0, NULL); + load_from_base(dest_reg, dest_r1, btype, is_unsigned, 0, 0, (uint32_t)addr_r); + break; + } + /* Not an lvalue: the immediate IS the value. */ + tcc_machine_load_constant(dest_reg, dest_r1, src.u.imm.val, (int)dest.is_64bit, NULL); + break; + + case MACH_OP_FRAME_ADDR: + { + if (!src.needs_deref) + { + /* Load the frame-slot address itself (LEA semantics). */ + tcc_machine_addr_of_stack_slot(dest_reg, src.u.frame.offset, 0); + } + else + { + /* Frame address is a pointer to data — compute addr, then dereference. */ + int addr_r = mach_alloc_scratch(&ctx, (uint32_t)1u << (uint32_t)dest_reg); + tcc_machine_addr_of_stack_slot(addr_r, src.u.frame.offset, 0); + load_from_base(dest_reg, dest_r1, btype, is_unsigned, 0, 0, (uint32_t)addr_r); + } + break; + } + + case MACH_OP_CHAIN_REL: + { + /* Captured variable: load from parent frame via static chain. */ + ScratchRegAlloc chain_scratch = {0}; + int chain_used = 0; + int base = resolve_chain_base(tcc_state->ir, src.u.chain.chain_index, (1u << (uint32_t)dest_reg), &chain_scratch, + &chain_used); + int32_t off = src.u.chain.offset; + int sign = (off < 0), abs_off = sign ? (int)(-off) : (int)off; + load_from_base(dest_reg, dest_r1, btype, is_unsigned, abs_off, sign, (uint32_t)base); + if (chain_used) + restore_scratch_reg(&chain_scratch); + break; + } + + default: + tcc_error("compiler_error: load_mop: unhandled src kind %d", (int)src.kind); + } + + /* Write back result to spill/param slot if dest was not a plain register. */ + if (!dest_is_simple_reg) + { + if (dest.is_64bit) + { + MachineOperand dest_lo_mop = mach_make_lo_half(&dest); + mach_writeback_dest(&dest_lo_mop, dest_reg); + mach_writeback_dest(&dest_hi_mop, dest_r1); + } + else + { + mach_writeback_dest(&dest, dest_reg); + } + } + + mach_release_all(&ctx); +} + +/* tcc_gen_machine_store_mop: MachineOperand-based entry point for TCCIR_OP_STORE. + * + * dest encodes the destination address (memory location to write to). + * src encodes the value to store. + * Store width is determined by dest.btype. + * + * dest kinds handled: + * MACH_OP_REG + needs_deref=true → STR src, [dest_reg] + * MACH_OP_REG (no deref) → MOV dest_reg, src (reg-to-reg) + * MACH_OP_SPILL → STR src, [FP + fp_adjust(offset)] + * MACH_OP_PARAM_STACK → STR src, [FP + param_off + offset_to_args] + * MACH_OP_SYMBOL → load addr, STR src, [addr + addend] + * + * 64-bit src: emits two 32-bit stores at [dest+0] (lo) and [dest+4] (hi) + * for all dest kinds above, plus MACH_OP_IMM and MACH_OP_FRAME_ADDR. + */ +ST_FUNC void tcc_gen_machine_store_mop(MachineOperand dest, MachineOperand src, TccIrOp op) +{ + MachineCodegenContext ctx = {0}; + (void)op; + + /* A store whose destination is a VFP register is a write into that s-register + * (a register-promoted float), not a memory store — delegate to the VFP-aware + * assign path. */ + if (dest.kind == MACH_OP_VFP_REG) + { + tcc_gen_machine_assign_mop(src, dest, TCCIR_OP_ASSIGN); + return; + } + + /* 128-bit complex double store: emit four 32-bit stores for real lo/hi + imag lo/hi. + * Complex double values are 16 bytes: real (8 bytes) at base, imag (8 bytes) at base+8. + * The source is always spilled (force-spilled by the register allocator). */ + if (src.is_64bit && src.is_complex && src.btype == IROP_BTYPE_FLOAT64) + { + /* Split into four 32-bit words using complex then lo/hi splitting. */ + MachineOperand real_part = mach_make_complex_real(&src); + MachineOperand imag_part = mach_make_complex_imag(&src); + MachineOperand w0 = mach_make_lo_half(&real_part); + w0.btype = IROP_BTYPE_INT32; + MachineOperand w1 = mach_make_hi_half(&real_part); + w1.btype = IROP_BTYPE_INT32; + MachineOperand w2 = mach_make_lo_half(&imag_part); + w2.btype = IROP_BTYPE_INT32; + MachineOperand w3 = mach_make_hi_half(&imag_part); + w3.btype = IROP_BTYPE_INT32; + + /* Load all 4 words into registers. */ + const int r0 = mach_ensure_in_reg(&ctx, &w0, 0); + uint32_t excl = (1u << (uint32_t)r0); + const int r1 = mach_ensure_in_reg(&ctx, &w1, excl); + excl |= (1u << (uint32_t)r1); + const int r2 = mach_ensure_in_reg(&ctx, &w2, excl); + excl |= (1u << (uint32_t)r2); + const int r3 = mach_ensure_in_reg(&ctx, &w3, excl); + excl |= (1u << (uint32_t)r3); + + /* Store through dest: 4 × 32-bit stores at [dest+0], [dest+4], [dest+8], [dest+12]. */ + if (dest.kind == MACH_OP_REG && dest.needs_deref) + { + const uint32_t base = (uint32_t)dest.u.reg.r0; + th_store32_imm_or_reg_ex(r0, base, 0, 0, excl | (1u << base)); + th_store32_imm_or_reg_ex(r1, base, 4, 0, excl | (1u << base)); + th_store32_imm_or_reg_ex(r2, base, 8, 0, excl | (1u << base)); + th_store32_imm_or_reg_ex(r3, base, 12, 0, excl | (1u << base)); + } + else if (dest.kind == MACH_OP_SPILL) + { + const int adj = fp_adjust_local_offset(dest.u.spill.offset, 0); + const uint32_t base = (uint32_t)(tcc_state->need_frame_pointer ? R_FP : R_SP); + th_store32_imm_or_reg_ex(r0, base, adj < 0 ? -adj : adj, adj < 0 ? 1 : 0, excl | (1u << base)); + int a1 = adj + 4; + th_store32_imm_or_reg_ex(r1, base, a1 < 0 ? -a1 : a1, a1 < 0 ? 1 : 0, excl | (1u << base)); + int a2 = adj + 8; + th_store32_imm_or_reg_ex(r2, base, a2 < 0 ? -a2 : a2, a2 < 0 ? 1 : 0, excl | (1u << base)); + int a3 = adj + 12; + th_store32_imm_or_reg_ex(r3, base, a3 < 0 ? -a3 : a3, a3 < 0 ? 1 : 0, excl | (1u << base)); + } + else + { + tcc_error("compiler_error: store_mop: unhandled dest kind %d for complex double store", (int)dest.kind); + } + mach_release_all(&ctx); + return; + } + + /* 64-bit store: emit two 32-bit stores for lo and hi halves */ + /* A store is as wide as its destination. A 64-bit integer into a word or + * narrower is a truncation: only the low half goes anywhere. Taking the + * pair path below instead wrote the high half too -- into memory past the + * word, or, for a plain register destination, into its absent second + * register: r1 == -1 encodes as pc (`mov pc, r7`, Zig's InternPool.Key.eql + * truncating aggregateTypeLen's u64 once var_to_param_forward had put the + * u64 in a register pair). */ + if (src.is_64bit && !src.is_complex && src.btype == IROP_BTYPE_INT64 && !dest.is_64bit && + (dest.btype == IROP_BTYPE_INT32 || dest.btype == IROP_BTYPE_INT16 || dest.btype == IROP_BTYPE_INT8)) + { + src = mach_make_lo_half(&src); + src.btype = IROP_BTYPE_INT32; + } + if (src.is_64bit) + { + MachineOperand src_lo = mach_make_lo_half(&src); + src_lo.btype = IROP_BTYPE_INT32; + MachineOperand src_hi = mach_make_hi_half(&src); + src_hi.btype = IROP_BTYPE_INT32; + + uint32_t dest_excl = 0; + if (dest.kind == MACH_OP_REG && dest.needs_deref && + dest.u.reg.r0 >= 0 && dest.u.reg.r0 < 16) + dest_excl = (1u << (uint32_t)dest.u.reg.r0); + + const int lo_reg = mach_ensure_in_reg(&ctx, &src_lo, dest_excl); + uint32_t excl = dest_excl | (thumb_is_hw_reg(lo_reg) ? (1u << (uint32_t)lo_reg) : 0u); + const int hi_reg = mach_ensure_in_reg(&ctx, &src_hi, excl); + excl |= thumb_is_hw_reg(hi_reg) ? (1u << (uint32_t)hi_reg) : 0u; + + switch (dest.kind) + { + case MACH_OP_REG: + if (dest.needs_deref) + { + /* 64-bit pointer-store through a register-held address. STRD needs + * a 4-byte-aligned address on ARMv7-M/v8-M (faults otherwise, + * regardless of UNALIGN_TRP); use it only when the frontend proved + * alignment (dest.align4 — no packed member in the access chain). + * Otherwise the pointer may target packed-struct memory that is only + * 1- or 2-byte aligned, so two plain STRs stay the safe fallback. */ + const uint32_t base = (uint32_t)dest.u.reg.r0; + if (dest.align4 && try_strd_pair(lo_reg, hi_reg, (int)base, 0, 0)) + break; + th_store32_imm_or_reg_ex(lo_reg, base, 0, 0, excl | (1u << base)); + th_store32_imm_or_reg_ex(hi_reg, base, 4, 0, excl | (1u << base)); + } + else + { + /* Reg-pair dst: emit hi first unless lo_reg == dest.r1 (safe-ordering) */ + const int dreg_lo = dest.u.reg.r0; + const int dreg_hi = dest.u.reg.r1; + if (lo_reg == dreg_hi) + { + if (dreg_lo != lo_reg && dreg_lo != (int)PREG_REG_NONE) + ot_check_mov_reg((uint32_t)dreg_lo, (uint32_t)lo_reg, flags_safe(), THUMB_SHIFT_DEFAULT, + ENFORCE_ENCODING_NONE, false); + if (dreg_hi != hi_reg && dreg_hi != (int)PREG_REG_NONE) + ot_check_mov_reg((uint32_t)dreg_hi, (uint32_t)hi_reg, flags_safe(), THUMB_SHIFT_DEFAULT, + ENFORCE_ENCODING_NONE, false); + } + else + { + if (dreg_hi != hi_reg && dreg_hi != (int)PREG_REG_NONE) + ot_check_mov_reg((uint32_t)dreg_hi, (uint32_t)hi_reg, flags_safe(), THUMB_SHIFT_DEFAULT, + ENFORCE_ENCODING_NONE, false); + if (dreg_lo != lo_reg && dreg_lo != (int)PREG_REG_NONE) + ot_check_mov_reg((uint32_t)dreg_lo, (uint32_t)lo_reg, flags_safe(), THUMB_SHIFT_DEFAULT, + ENFORCE_ENCODING_NONE, false); + } + } + break; + + case MACH_OP_SPILL: + { + const int adj = fp_adjust_local_offset(dest.u.spill.offset, 0); + const uint32_t base = (uint32_t)(tcc_state->need_frame_pointer ? R_FP : R_SP); + if (dest.needs_deref) + { + /* LLOCAL: spill slot holds a pointer; load ptr, then store through it */ + int ptr_r = mach_alloc_scratch(&ctx, excl | (1u << base)); + if (!load_word_from_base(ptr_r, (int)base, adj < 0 ? -adj : adj, adj < 0 ? 1 : 0)) + { + ScratchRegAlloc rr = + th_offset_to_reg_ex(adj < 0 ? -adj : adj, adj < 0 ? 1 : 0, (uint32_t)(1u << ptr_r) | (1u << base)); + ot_check(th_ldr_reg((uint32_t)ptr_r, base, (uint32_t)rr.reg, THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); + restore_scratch_reg(&rr); + } + /* Pointer-through store from an LLOCAL spill slot: STRD only when + * the frontend proved >= 4-byte alignment of the target (dest.align4); + * an arbitrary pointer may reference unaligned packed-struct memory. */ + if (dest.align4 && try_strd_pair(lo_reg, hi_reg, ptr_r, 0, 0)) + break; + th_store32_imm_or_reg_ex(lo_reg, (uint32_t)ptr_r, 0, 0, excl | (1u << (uint32_t)ptr_r)); + th_store32_imm_or_reg_ex(hi_reg, (uint32_t)ptr_r, 4, 0, excl | (1u << (uint32_t)ptr_r)); + } + else + { + const int adj_hi = adj + 4; + const int sign = (adj < 0), abs_off = sign ? -adj : adj; + if (!try_strd_pair(lo_reg, hi_reg, (int)base, abs_off, sign)) + { + th_store32_imm_or_reg_ex(lo_reg, base, abs_off, sign, excl | (1u << base)); + th_store32_imm_or_reg_ex(hi_reg, base, adj_hi < 0 ? -adj_hi : adj_hi, adj_hi < 0 ? 1 : 0, + excl | (1u << base)); + } + } + break; + } + + case MACH_OP_PARAM_STACK: + { + const int adj = param_frame_offset(dest.u.param.offset); + const int adj_hi = adj + 4; + const uint32_t base = (uint32_t)(tcc_state->need_frame_pointer ? R_FP : R_SP); + const int sign = (adj < 0), abs_off = sign ? -adj : adj; + if (!try_strd_pair(lo_reg, hi_reg, (int)base, abs_off, sign)) + { + th_store32_imm_or_reg_ex(lo_reg, base, abs_off, sign, excl | (1u << base)); + th_store32_imm_or_reg_ex(hi_reg, base, adj_hi < 0 ? -adj_hi : adj_hi, adj_hi < 0 ? 1 : 0, excl | (1u << base)); + } + break; + } + + case MACH_OP_SYMBOL: + { + /* Global symbol store. STRD needs 4-byte alignment; allow it only when + * the symbol's declared type guarantees natural alignment >= 4 (regular + * scalar globals) or the symbol was explicitly aligned. Packed structs + * and struct-typed globals stay on the STR-pair path. */ + Sym *sym = dest.u.sym.sym ? validate_sym_for_reloc(dest.u.sym.sym) : NULL; + int addr_r = mach_alloc_scratch_for_sym(&ctx, excl, sym, 0); + tcc_machine_load_constant(addr_r, PREG_REG_NONE, 0, 0, sym); + const int32_t addend = dest.u.sym.addend; + const int32_t addend_hi = addend + 4; + const int sign = (addend < 0), abs_off = sign ? (int)(-addend) : (int)addend; + if (sym_is_4_byte_aligned_for_64bit(sym, addend) && try_strd_pair(lo_reg, hi_reg, addr_r, abs_off, sign)) + { + break; + } + th_store32_imm_or_reg_ex(lo_reg, (uint32_t)addr_r, abs_off, sign, excl | (1u << addr_r)); + th_store32_imm_or_reg_ex(hi_reg, (uint32_t)addr_r, addend_hi < 0 ? (int)(-addend_hi) : (int)addend_hi, + addend_hi < 0 ? 1 : 0, excl | (1u << addr_r)); + break; + } + + case MACH_OP_IMM: + { + /* Store to a constant address — alignment unknown, skip STRD. */ + int addr_r = mach_alloc_scratch(&ctx, excl); + tcc_machine_load_constant(addr_r, PREG_REG_NONE, dest.u.imm.val, 0, NULL); + th_store32_imm_or_reg_ex(lo_reg, (uint32_t)addr_r, 0, 0, excl | (1u << addr_r)); + th_store32_imm_or_reg_ex(hi_reg, (uint32_t)addr_r, 4, 0, excl | (1u << addr_r)); + break; + } + + case MACH_OP_FRAME_ADDR: + { + int addr_r = mach_alloc_scratch(&ctx, excl); + tcc_machine_addr_of_stack_slot(addr_r, dest.u.frame.offset, 0 /* not param */); + if (!try_strd_pair(lo_reg, hi_reg, addr_r, 0, 0)) + { + th_store32_imm_or_reg_ex(lo_reg, (uint32_t)addr_r, 0, 0, excl | (1u << addr_r)); + th_store32_imm_or_reg_ex(hi_reg, (uint32_t)addr_r, 4, 0, excl | (1u << addr_r)); + } + break; + } + + case MACH_OP_CHAIN_REL: + { + /* 64-bit captured variable: store lo+hi words to parent frame. */ + ScratchRegAlloc chain_scratch = {0}; + int chain_used = 0; + int base = resolve_chain_base(tcc_state->ir, dest.u.chain.chain_index, excl, &chain_scratch, &chain_used); + int32_t off = dest.u.chain.offset; + int sign = (off < 0), abs_off = sign ? (int)(-off) : (int)off; + int32_t off_hi = off + 4; + int sign_hi = (off_hi < 0), abs_off_hi = sign_hi ? (int)(-off_hi) : (int)off_hi; + if (!try_strd_pair(lo_reg, hi_reg, base, abs_off, sign)) + { + th_store32_imm_or_reg_ex(lo_reg, (uint32_t)base, abs_off, sign, excl | (1u << (uint32_t)base)); + th_store32_imm_or_reg_ex(hi_reg, (uint32_t)base, abs_off_hi, sign_hi, excl | (1u << (uint32_t)base)); + } + if (chain_used) + restore_scratch_reg(&chain_scratch); + break; + } + + default: + tcc_error("compiler_error: store_mop: unhandled dest kind %d for 64-bit src", (int)dest.kind); + } + mach_release_all(&ctx); + return; + } + + const int btype = dest.btype; /* Store width from destination type */ + + /* Fast path: plain-register dest (no deref) — load src directly into dest, + * skipping the intermediate scratch + MOV that the generic path emits. + * Covers IMM, SYMBOL, SPILL, FRAME_ADDR, PARAM_STACK, CHAIN_REL, and + * REG-with-or-without-deref src kinds. */ + if (dest.kind == MACH_OP_REG && !dest.needs_deref && dest.u.reg.r0 != (int)PREG_REG_NONE) + { + tcc_gen_mach_load_to_reg(dest.u.reg.r0, &src); + mach_release_all(&ctx); + return; + } + + /* Get source value register — may allocate a scratch if spilled/const. + * When storing through a register-held pointer, protect the base register + * before materializing immediates or spilled values. */ + uint32_t src_excl = 0; + if (dest.kind == MACH_OP_REG && dest.needs_deref && + dest.u.reg.r0 >= 0 && dest.u.reg.r0 < 16) + src_excl |= (1u << (uint32_t)dest.u.reg.r0); + const int src_reg = mach_ensure_in_reg(&ctx, &src, src_excl); + + switch (dest.kind) + { + case MACH_OP_REG: + if (dest.needs_deref) + { + /* Store through pointer: STR src, [dest_reg] */ + const uint32_t base = (uint32_t)dest.u.reg.r0; + if (btype == IROP_BTYPE_INT8) + th_store8_imm_or_reg(src_reg, base, 0, 0); + else if (btype == IROP_BTYPE_INT16) + th_store16_imm_or_reg(src_reg, base, 0, 0); + else + th_store32_imm_or_reg_ex(src_reg, base, 0, 0, (uint32_t)1u << (uint32_t)src_reg); + } + else + { + /* Register-to-register store (MOV) */ + const int dreg = dest.u.reg.r0; + if (dreg != src_reg && dreg != (int)PREG_REG_NONE) + ot_check_mov_reg((uint32_t)dreg, (uint32_t)src_reg, flags_safe(), THUMB_SHIFT_DEFAULT, + ENFORCE_ENCODING_NONE, false); + } + break; + + case MACH_OP_SPILL: + { + const int adj = fp_adjust_local_offset(dest.u.spill.offset, 0); + const int sign = (adj < 0), abs_off = sign ? -adj : adj; + const uint32_t base = (uint32_t)(tcc_state->need_frame_pointer ? R_FP : R_SP); + if (dest.needs_deref) + { + /* LLOCAL: spill slot holds a pointer; load ptr, then store through it */ + int ptr_r = mach_alloc_scratch(&ctx, (uint32_t)1u << (uint32_t)src_reg | (1u << base)); + if (!load_word_from_base(ptr_r, (int)base, abs_off, sign)) + { + ScratchRegAlloc rr = + th_offset_to_reg_ex(abs_off, sign, (uint32_t)(1u << ptr_r) | (1u << base) | (1u << (uint32_t)src_reg)); + ot_check(th_ldr_reg((uint32_t)ptr_r, base, (uint32_t)rr.reg, THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); + restore_scratch_reg(&rr); + } + if (btype == IROP_BTYPE_INT8) + th_store8_imm_or_reg(src_reg, (uint32_t)ptr_r, 0, 0); + else if (btype == IROP_BTYPE_INT16) + th_store16_imm_or_reg(src_reg, (uint32_t)ptr_r, 0, 0); + else + th_store32_imm_or_reg_ex(src_reg, (uint32_t)ptr_r, 0, 0, + (uint32_t)1u << (uint32_t)src_reg | (1u << (uint32_t)ptr_r)); + } + else if ((btype == IROP_BTYPE_INT8 || btype == IROP_BTYPE_INT16) && dest.vreg >= 0 && + (TCCIR_DECODE_VREG_TYPE(dest.vreg) == TCCIR_VREG_TYPE_TEMP || mach_var_owns_spill_slot(dest.vreg))) + { + /* A temporary's spill slot holds a register image, and every reload of + * it is a word load (mach_load_slot reads narrow only for a narrow + * operand, and a use such as `T ADD #1` is not one). A byte store left + * three stale bytes for that load to pick up: pr82524's + * `foo(y->c.b, w)` spilled both u8 operands and multiplied garbage. + * Store the value extended to a word, the truncation the narrow store + * gave included. The same for a VAR in a spill slot of its own + * (mach_var_owns_spill_slot); an address-taken VAR's slot is its + * narrow home, whose neighbours a word store would clobber -- that + * stays narrow. */ + const int ext = mach_alloc_scratch(&ctx, (uint32_t)1u << (uint32_t)src_reg); + if (btype == IROP_BTYPE_INT8) + ot_check(dest.is_unsigned + ? th_uxtb((uint32_t)ext, (uint32_t)src_reg, THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE) + : th_sxtb((uint32_t)ext, (uint32_t)src_reg, THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); + else + ot_check(dest.is_unsigned + ? th_uxth((uint32_t)ext, (uint32_t)src_reg, THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE) + : th_sxth((uint32_t)ext, (uint32_t)src_reg, THUMB_SHIFT_DEFAULT, ENFORCE_ENCODING_NONE)); + th_store32_imm_or_reg_ex(ext, base, abs_off, sign, (uint32_t)1u << (uint32_t)ext); + } + else + { + if (btype == IROP_BTYPE_INT8) + th_store8_imm_or_reg(src_reg, base, abs_off, sign); + else if (btype == IROP_BTYPE_INT16) + th_store16_imm_or_reg(src_reg, base, abs_off, sign); + else + th_store32_imm_or_reg_ex(src_reg, base, abs_off, sign, (uint32_t)1u << (uint32_t)src_reg); + } + break; + } + + case MACH_OP_PARAM_STACK: + { + const int adj = param_frame_offset(dest.u.param.offset); + const int sign = (adj < 0), abs_off = sign ? -adj : adj; + const uint32_t base = (uint32_t)(tcc_state->need_frame_pointer ? R_FP : R_SP); + if (btype == IROP_BTYPE_INT8) + th_store8_imm_or_reg(src_reg, base, abs_off, sign); + else if (btype == IROP_BTYPE_INT16) + th_store16_imm_or_reg(src_reg, base, abs_off, sign); + else + th_store32_imm_or_reg_ex(src_reg, base, abs_off, sign, (uint32_t)1u << (uint32_t)src_reg); + break; + } + + case MACH_OP_SYMBOL: + { + Sym *sym = dest.u.sym.sym ? validate_sym_for_reloc(dest.u.sym.sym) : NULL; + int addr_r = mach_alloc_scratch_for_sym(&ctx, (uint32_t)1u << (uint32_t)src_reg, sym, 0); + tcc_machine_load_constant(addr_r, PREG_REG_NONE, 0, 0, sym); + const int32_t addend = dest.u.sym.addend; + const int abs_off = addend < 0 ? (int)(-addend) : (int)addend; + const int sign = addend < 0 ? 1 : 0; + if (btype == IROP_BTYPE_INT8) + th_store8_imm_or_reg(src_reg, (uint32_t)addr_r, abs_off, sign); + else if (btype == IROP_BTYPE_INT16) + th_store16_imm_or_reg(src_reg, (uint32_t)addr_r, abs_off, sign); + else + th_store32_imm_or_reg_ex(src_reg, (uint32_t)addr_r, abs_off, sign, (uint32_t)1u << (uint32_t)src_reg); + break; + } + + case MACH_OP_IMM: + { + /* Store to an absolute address — e.g. *(volatile uint32_t*)0xABCD = val */ + int addr_r = mach_alloc_scratch(&ctx, (uint32_t)1u << (uint32_t)src_reg); + tcc_machine_load_constant(addr_r, PREG_REG_NONE, dest.u.imm.val, 0, NULL); + if (btype == IROP_BTYPE_INT8) + th_store8_imm_or_reg(src_reg, (uint32_t)addr_r, 0, 0); + else if (btype == IROP_BTYPE_INT16) + th_store16_imm_or_reg(src_reg, (uint32_t)addr_r, 0, 0); + else + th_store32_imm_or_reg_ex(src_reg, (uint32_t)addr_r, 0, 0, (uint32_t)1u << (uint32_t)src_reg); + break; + } + + case MACH_OP_FRAME_ADDR: + { + /* Store to a frame-relative address; equivalent to MACH_OP_SPILL but via addr computation */ + int addr_r = mach_alloc_scratch(&ctx, (uint32_t)1u << (uint32_t)src_reg); + tcc_machine_addr_of_stack_slot(addr_r, dest.u.frame.offset, 0 /* not param */); + if (btype == IROP_BTYPE_INT8) + th_store8_imm_or_reg(src_reg, (uint32_t)addr_r, 0, 0); + else if (btype == IROP_BTYPE_INT16) + th_store16_imm_or_reg(src_reg, (uint32_t)addr_r, 0, 0); + else + th_store32_imm_or_reg_ex(src_reg, (uint32_t)addr_r, 0, 0, (uint32_t)1u << (uint32_t)src_reg); + break; + } + + case MACH_OP_CHAIN_REL: + { + /* Captured variable: store to parent frame via static chain. */ + ScratchRegAlloc chain_scratch = {0}; + int chain_used = 0; + int base = resolve_chain_base(tcc_state->ir, dest.u.chain.chain_index, (1u << (uint32_t)src_reg), &chain_scratch, + &chain_used); + int32_t off = dest.u.chain.offset; + int sign = (off < 0), abs_off = sign ? (int)(-off) : (int)off; + if (btype == IROP_BTYPE_INT8) + th_store8_imm_or_reg(src_reg, (uint32_t)base, abs_off, sign); + else if (btype == IROP_BTYPE_INT16) + th_store16_imm_or_reg(src_reg, (uint32_t)base, abs_off, sign); + else + th_store32_imm_or_reg_ex(src_reg, (uint32_t)base, abs_off, sign, + (1u << (uint32_t)src_reg) | (1u << (uint32_t)base)); + if (chain_used) + restore_scratch_reg(&chain_scratch); + break; + } + + default: + tcc_error("compiler_error: store_mop: unhandled dest kind %d", (int)dest.kind); + } + + mach_release_all(&ctx); +} + +/* Indexed load: dest = *(base + (index << scale)) + * Generates: LDR dest, [base, index, LSL #scale] + */ +ST_FUNC void tcc_gen_machine_load_indexed_mop(MachineOperand dest, MachineOperand base, MachineOperand index, + MachineOperand scale, TccIrOp op) +{ + MachineCodegenContext ctx = {0}; + (void)op; + + int shift_amount = (scale.kind == MACH_OP_IMM) ? (int)scale.u.imm.val : 2; + if (shift_amount < 0 || shift_amount > 31) + shift_amount = 2; + + /* Fast path: base is &local + constant index — fold into SP/FP-relative load. + * Mirrors the store_indexed FRAME_ADDR fast path. */ + if (!dest.is_64bit && shift_amount == 0 && index.kind == MACH_OP_IMM && + base.kind == MACH_OP_FRAME_ADDR && !base.needs_deref) + { + int combined = base.u.frame.offset + (int)index.u.imm.val; + int adjusted = fp_adjust_local_offset(combined, 0); + int sign = (adjusted < 0); + int abs_off = sign ? -adjusted : adjusted; + if (abs_off <= 4095) + { + const int dest_reg = mach_get_dest_reg(&ctx, &dest, 0); + const int base_reg = tcc_state->need_frame_pointer ? R_FP : R_SP; + load_from_base(dest_reg, PREG_REG_NONE, dest.btype, (int)dest.is_unsigned, abs_off, sign, (uint32_t)base_reg); + mach_writeback_dest(&dest, dest_reg); + mach_release_all(&ctx); + return; + } + } + + /* Fast path: constant-displacement load (scale == 0 and index is an immediate). + * Generated by the displacement-fusion pass when folding `ADD base,#imm; LOAD *` + * into a single `LDR dest,[base,#imm]`, matching GCC's addressing-mode output. */ + if (!dest.is_64bit && shift_amount == 0 && index.kind == MACH_OP_IMM) + { + int imm = (int)index.u.imm.val; + int sign = (imm < 0); + int abs_off = sign ? -imm : imm; + if (abs_off <= 4095) + { + const int dest_reg = mach_get_dest_reg(&ctx, &dest, 0); + uint32_t excl = (1u << (uint32_t)dest_reg); + int base_reg = mach_ensure_in_reg(&ctx, &base, excl); + load_from_base(dest_reg, PREG_REG_NONE, dest.btype, (int)dest.is_unsigned, abs_off, sign, (uint32_t)base_reg); + mach_writeback_dest(&dest, dest_reg); + mach_release_all(&ctx); + return; + } + } + + /* scale 0 → no shift: use THUMB_SHIFT_NONE so the 16-bit T1 register-offset + * encoding (all-low regs) can be selected instead of the wide T32 form. */ + thumb_shift shift = (shift_amount == 0) + ? (thumb_shift){.type = THUMB_SHIFT_NONE, .value = 0, .mode = THUMB_SHIFT_IMMEDIATE} + : (thumb_shift){.type = THUMB_SHIFT_LSL, .value = (uint32_t)shift_amount, .mode = THUMB_SHIFT_IMMEDIATE}; + + /* Fast path: 64-bit constant-displacement load using LDRD [base, #imm]. + * LDRD supports word-aligned offsets in range [-1020, 1020]. + * base.underalign_hint (packed-derived address): skip — LDRD faults on + * unaligned addresses; the generic path below emits an LDR pair instead. */ + if (dest.is_64bit && shift_amount == 0 && index.kind == MACH_OP_IMM && !base.underalign_hint) + { + int imm = (int)index.u.imm.val; + int sign = (imm < 0); + int abs_off = sign ? -imm : imm; + if (abs_off <= 1020 && (abs_off & 3) == 0) + { + const bool dest_is_reg = (dest.kind == MACH_OP_REG && !dest.needs_deref); + int dest_lo, dest_hi; + MachineOperand dest_hi_mop = {0}; + uint32_t excl = 0; + + if (dest_is_reg) + { + dest_lo = dest.u.reg.r0; + if (!thumb_is_hw_reg(dest.u.reg.r1)) + tcc_error("load_indexed_mop: 64-bit dest has invalid r1=%d (r0=%d) — " + "register allocator must produce a valid pair", + dest.u.reg.r1, dest.u.reg.r0); + dest_hi = dest.u.reg.r1; + excl = (1u << (uint32_t)dest_lo) | (1u << (uint32_t)dest_hi); + } + else + { + dest_lo = mach_get_dest_reg(&ctx, &dest, 0); + excl = (1u << (uint32_t)dest_lo); + dest_hi_mop = mach_make_hi_half(&dest); + dest_hi = mach_get_dest_reg(&ctx, &dest_hi_mop, excl); + excl |= (1u << (uint32_t)dest_hi); + } + + int base_reg = mach_ensure_in_reg(&ctx, &base, excl); + uint32_t puw = sign ? 4u : 6u; + ot_check(th_ldrd_imm((uint32_t)dest_lo, (uint32_t)dest_hi, (uint32_t)base_reg, abs_off, puw)); + if (!dest_is_reg) + { + MachineOperand dest_lo_mop = mach_make_lo_half(&dest); + mach_writeback_dest(&dest_lo_mop, dest_lo); + mach_writeback_dest(&dest_hi_mop, dest_hi); + } + mach_release_all(&ctx); + return; + } + } + + /* 64-bit indexed load: compute EA = base + index< 31) + shift_amount = 2; + + /* Fast path: base is &local + constant index — fold into SP/FP-relative store. + * Avoids emitting a separate `ADD base, sp, #frame_off` LEA before the STR, + * cutting one instruction per access in dense local-array initialization. */ + if (!value.is_64bit && shift_amount == 0 && index.kind == MACH_OP_IMM && + base.kind == MACH_OP_FRAME_ADDR && !base.needs_deref) + { + int combined = base.u.frame.offset + (int)index.u.imm.val; + int adjusted = fp_adjust_local_offset(combined, 0); + int sign = (adjusted < 0); + int abs_off = sign ? -adjusted : adjusted; + if (abs_off <= 4095) + { + const int btype = value.btype; + const int base_reg = tcc_state->need_frame_pointer ? R_FP : R_SP; + int value_reg = mach_ensure_in_reg(&ctx, &value, 0); + if (btype == IROP_BTYPE_INT8) + th_store8_imm_or_reg(value_reg, (uint32_t)base_reg, abs_off, sign); + else if (btype == IROP_BTYPE_INT16) + th_store16_imm_or_reg(value_reg, (uint32_t)base_reg, abs_off, sign); + else + th_store32_imm_or_reg_ex(value_reg, (uint32_t)base_reg, abs_off, sign, + (1u << (uint32_t)value_reg) | (1u << (uint32_t)base_reg)); + mach_release_all(&ctx); + return; + } + } + + /* Fast path: constant-displacement store (scale == 0 and index is an immediate). + * Mirrors the load_indexed fast path; emits `STR value,[base,#imm]`. */ + if (!value.is_64bit && shift_amount == 0 && index.kind == MACH_OP_IMM) + { + int imm = (int)index.u.imm.val; + int sign = (imm < 0); + int abs_off = sign ? -imm : imm; + if (abs_off <= 4095) + { + const int btype = value.btype; + int value_reg = mach_ensure_in_reg(&ctx, &value, mach_reg_excl(&base)); + uint32_t excl = (1u << (uint32_t)value_reg); + int base_reg = mach_ensure_in_reg(&ctx, &base, excl); + if (btype == IROP_BTYPE_INT8) + th_store8_imm_or_reg(value_reg, (uint32_t)base_reg, abs_off, sign); + else if (btype == IROP_BTYPE_INT16) + th_store16_imm_or_reg(value_reg, (uint32_t)base_reg, abs_off, sign); + else + th_store32_imm_or_reg_ex(value_reg, (uint32_t)base_reg, abs_off, sign, + (1u << (uint32_t)value_reg) | (1u << (uint32_t)base_reg)); + mach_release_all(&ctx); + return; + } + } + + /* scale 0 → no shift: use THUMB_SHIFT_NONE so the 16-bit T1 register-offset + * encoding (all-low regs) can be selected instead of the wide T32 form. */ + thumb_shift shift = (shift_amount == 0) + ? (thumb_shift){.type = THUMB_SHIFT_NONE, .value = 0, .mode = THUMB_SHIFT_IMMEDIATE} + : (thumb_shift){.type = THUMB_SHIFT_LSL, .value = (uint32_t)shift_amount, .mode = THUMB_SHIFT_IMMEDIATE}; + + /* Fast path: 64-bit constant-displacement store using STRD [base, #imm]. + * base.underalign_hint (packed-derived address): skip — STRD faults on + * unaligned addresses; the generic path below emits an STR pair instead. */ + if (value.is_64bit && shift_amount == 0 && index.kind == MACH_OP_IMM && !base.underalign_hint) + { + int imm = (int)index.u.imm.val; + int sign = (imm < 0); + int abs_off = sign ? -imm : imm; + if (abs_off <= 1020 && (abs_off & 3) == 0) + { + MachineOperand val_lo = mach_make_lo_half(&value); + val_lo.btype = IROP_BTYPE_INT32; + MachineOperand val_hi = mach_make_hi_half(&value); + val_hi.btype = IROP_BTYPE_INT32; + const uint32_t base_excl = mach_reg_excl(&base); + const int lo_reg = mach_ensure_in_reg(&ctx, &val_lo, base_excl); + uint32_t excl = (1u << (uint32_t)lo_reg); + const int hi_reg = mach_ensure_in_reg(&ctx, &val_hi, excl | base_excl); + excl |= (1u << (uint32_t)hi_reg); + int base_reg = mach_ensure_in_reg(&ctx, &base, excl); + uint32_t puw = sign ? 4u : 6u; + ot_check(th_strd_imm((uint32_t)lo_reg, (uint32_t)hi_reg, (uint32_t)base_reg, abs_off, puw)); + mach_release_all(&ctx); + return; + } + } + + /* 64-bit indexed store: compute EA = base + index< 255) + { + mach_release_all(&ctx); + tcc_error("compiler_error: post-increment offset %d out of range (0-255)", offset_imm); + return; + } + const uint32_t puw = 3; /* post-index (p=0), add (u=1), writeback (w=1) */ + + /* 64-bit post-increment load: LDRD dest_lo, dest_hi, [ptr], #offset */ + if (dest.is_64bit) + { + const int dest_lo = mach_get_dest_reg(&ctx, &dest, 0); + MachineOperand dest_hi_mop = mach_make_hi_half(&dest); + const int dest_hi = mach_get_dest_reg(&ctx, &dest_hi_mop, (1u << (uint32_t)dest_lo)); + uint32_t excl = (1u << (uint32_t)dest_lo) | (1u << (uint32_t)dest_hi); + int ptr_reg = mach_ensure_in_reg(&ctx, &ptr, excl); + ot_check( + th_ldrd_imm((uint32_t)dest_lo, (uint32_t)dest_hi, (uint32_t)ptr_reg, offset_imm, puw)); + mach_writeback_dest(&dest_hi_mop, dest_hi); + mach_writeback_dest(&dest, dest_lo); + mach_release_all(&ctx); + return; + } + + const int dest_reg = mach_get_dest_reg(&ctx, &dest, 0); + const int btype = dest.btype; + const int is_unsigned = (int)dest.is_unsigned; + + uint32_t excl = (1u << (uint32_t)dest_reg); + int ptr_reg = mach_ensure_in_reg(&ctx, &ptr, excl); + + if (btype == IROP_BTYPE_INT8) + { + if (is_unsigned) + ot_check(th_ldrb_imm(dest_reg, ptr_reg, offset_imm, puw, ENFORCE_ENCODING_NONE)); + else + ot_check(th_ldrsb_imm(dest_reg, ptr_reg, offset_imm, puw, ENFORCE_ENCODING_NONE)); + } + else if (btype == IROP_BTYPE_INT16) + { + if (is_unsigned) + ot_check(th_ldrh_imm(dest_reg, ptr_reg, offset_imm, puw, ENFORCE_ENCODING_NONE)); + else + ot_check(th_ldrsh_imm(dest_reg, ptr_reg, offset_imm, puw, ENFORCE_ENCODING_NONE)); + } + else + { + ot_check_ldr_imm(dest_reg, ptr_reg, offset_imm, puw, ENFORCE_ENCODING_NONE); + } + mach_writeback_dest(&dest, dest_reg); + mach_release_all(&ctx); +} + +/* Post-increment store: *ptr = value; ptr += offset + * Generates: STR value, [ptr], #offset (puw=3: post-index, add, writeback) + */ +ST_FUNC void tcc_gen_machine_store_postinc_mop(MachineOperand ptr, MachineOperand value, MachineOperand offset, + TccIrOp op) +{ + MachineCodegenContext ctx = {0}; + (void)op; + + int offset_imm = (offset.kind == MACH_OP_IMM) ? (int)offset.u.imm.val : 4; + if (offset_imm < 0 || offset_imm > 255) + { + mach_release_all(&ctx); + tcc_error("compiler_error: post-increment offset %d out of range (0-255)", offset_imm); + return; + } + const uint32_t puw = 3; /* post-index (p=0), add (u=1), writeback (w=1) */ + + /* 64-bit post-increment store: STRD lo, hi, [ptr], #offset */ + if (value.is_64bit) + { + MachineOperand val_lo = mach_make_lo_half(&value); + val_lo.btype = IROP_BTYPE_INT32; + MachineOperand val_hi = mach_make_hi_half(&value); + val_hi.btype = IROP_BTYPE_INT32; + const uint32_t ptr_excl = mach_reg_excl(&ptr); + const int lo_reg = mach_ensure_in_reg(&ctx, &val_lo, ptr_excl); + uint32_t excl = (1u << (uint32_t)lo_reg); + const int hi_reg = mach_ensure_in_reg(&ctx, &val_hi, excl | ptr_excl); + excl |= (1u << (uint32_t)hi_reg); + int ptr_reg = mach_ensure_in_reg(&ctx, &ptr, excl); + ot_check( + th_strd_imm((uint32_t)lo_reg, (uint32_t)hi_reg, (uint32_t)ptr_reg, offset_imm, puw)); + mach_release_all(&ctx); + return; + } + + const int btype = value.btype; + + int value_reg = mach_ensure_in_reg(&ctx, &value, mach_reg_excl(&ptr)); + uint32_t excl = (1u << (uint32_t)value_reg); + int ptr_reg = mach_ensure_in_reg(&ctx, &ptr, excl); + + if (btype == IROP_BTYPE_INT8) + ot_check(th_strb_imm(value_reg, ptr_reg, offset_imm, puw, ENFORCE_ENCODING_NONE)); + else if (btype == IROP_BTYPE_INT16) + ot_check(th_strh_imm(value_reg, ptr_reg, offset_imm, puw, ENFORCE_ENCODING_NONE)); + else + ot_check_str_imm(value_reg, ptr_reg, offset_imm, puw, ENFORCE_ENCODING_NONE); + + mach_release_all(&ctx); +} diff --git a/source/backend/arch/arm/thumb/thop_mul.c b/source/backend/arch/arm/thumb/thop_mul.c index edf4aa474..81029de91 100644 --- a/source/backend/arch/arm/thumb/thop_mul.c +++ b/source/backend/arch/arm/thumb/thop_mul.c @@ -88,6 +88,19 @@ TH_TABLE(TH_UMLAL, "umlal", {&SHAPE_LONG_MUL, 0xfbe00000, long_mul_emit}); TH_TABLE(TH_SMULL, "smull", {&SHAPE_LONG_MUL, 0xfb800000, long_mul_emit}); TH_TABLE(TH_SMLAL, "smlal", {&SHAPE_LONG_MUL, 0xfbc00000, long_mul_emit}); +/* UMAAL: {RdHi:RdLo} = Rn * Rm + RdLo + RdHi. Part of the DSP extension + * (ARMv7E-M, ARMv8-M Mainline with DSP), unlike the four above. */ +static const thop_variant_shape SHAPE_LONG_MUL_DSP = { + .size = THOP_VARIANT_T32, + .rd_place = {8, 4}, + .rn_place = {0, 4}, + .rm_place = {16, 4}, + .ra_place = {12, 4}, + .feat = {.t32 = 1, .dsp = 1}, +}; + +TH_TABLE(TH_UMAAL, "umaal", {&SHAPE_LONG_MUL_DSP, 0xfbe00060, long_mul_emit}); + /* ───── SDIV / UDIV (T32 only) ───── */ static const thop_variant_shape SHAPE_DIV = { @@ -149,6 +162,12 @@ thumb_opcode th_umlal(uint32_t rdlo, uint32_t rdhi, uint32_t rn, uint32_t rm) (thop_args){.rd = rdhi, .rn = rn, .rm = rm, .ra = rdlo}); } +thumb_opcode th_umaal(uint32_t rdlo, uint32_t rdhi, uint32_t rn, uint32_t rm) +{ + return thop_emit(TH_UMAAL.name, TH_UMAAL.variants, TH_UMAAL.variant_count, + (thop_args){.rd = rdhi, .rn = rn, .rm = rm, .ra = rdlo}); +} + thumb_opcode th_smull(uint32_t rdlo, uint32_t rdhi, uint32_t rn, uint32_t rm) { return thop_emit(TH_SMULL.name, TH_SMULL.variants, TH_SMULL.variant_count, diff --git a/source/backend/arch/arm/thumb/thop_mul.h b/source/backend/arch/arm/thumb/thop_mul.h index 65e81fcab..3f39afa2a 100644 --- a/source/backend/arch/arm/thumb/thop_mul.h +++ b/source/backend/arch/arm/thumb/thop_mul.h @@ -10,6 +10,7 @@ thumb_opcode th_mla(uint32_t rd, uint32_t rn, uint32_t rm, uint32_t ra); thumb_opcode th_mls(uint32_t rd, uint32_t rn, uint32_t rm, uint32_t ra); thumb_opcode th_umull(uint32_t rdlo, uint32_t rdhi, uint32_t rn, uint32_t rm); thumb_opcode th_umlal(uint32_t rdlo, uint32_t rdhi, uint32_t rn, uint32_t rm); +thumb_opcode th_umaal(uint32_t rdlo, uint32_t rdhi, uint32_t rn, uint32_t rm); thumb_opcode th_smull(uint32_t rdlo, uint32_t rdhi, uint32_t rn, uint32_t rm); thumb_opcode th_smlal(uint32_t rdlo, uint32_t rdhi, uint32_t rn, uint32_t rm); thumb_opcode th_udiv(uint16_t rd, uint16_t rn, uint16_t rm); diff --git a/source/backend/arch/arm/thumb/thumb-outline.c b/source/backend/arch/arm/thumb/thumb-outline.c new file mode 100644 index 000000000..d0f43f8b1 --- /dev/null +++ b/source/backend/arch/arm/thumb/thumb-outline.c @@ -0,0 +1,1060 @@ +/* + * TCC - Thumb-2 machine outliner (-Os) + * + * Copyright (c) 2026 Mateusz Stadnik + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation. + */ + +/* Runs of IR ops that compile to the same machine code in many functions -- + * Zig's error returns, its slice and hash-map accessors -- are emitted once as + * a shared body and called with BL everywhere else. The work is done per IR + * op, never inside one: the rehearsal pass (ir/codegen.c) records the bytes + * each op produced, a window of consecutive ops whose code is position + * independent and leaves LR and SP alone is looked up among the windows of the + * functions compiled before, and a match is emitted as a call. + * + * Legality is decided on the machine code itself, by a decoder that + * recognises the Thumb-2 forms the backend emits and rejects everything else: + * a branch, a PC-relative access, a write to SP, any use of LR (the BL + * clobbers it) or of PC. An op any of whose bytes did not come through ot() + * -- a literal-pool dump, a branch placeholder -- is rejected as well. + * + * The table is filled as the TU compiles, so a window's first occurrences stay + * inline: the body is created from the rehearsal code of the function whose + * occurrence makes it pay (outline_body_pays), and that function and every + * later one call it. The real pass (pass 2) still generates every op of a + * window it calls instead: ot() records the instructions rather than writing + * them, so every backend cache ot() maintains leaves the window exactly as if + * the code had been inline, and at the window's end they are compared with + * the body and the BL is written in their place. Should they differ from + * what the rehearsal produced -- a cache the pool dumps reset at other points + * -- they are written instead, where they were generated + * (TCC_OUTLINE_STATS=1 counts those, =2 prints them). TCC_OUTLINE_LIMIT=N + * outlines only the first N windows of the compile, and names the function + * of the Nth: a miscompile bisects to one window. + * + * The real pass can only get shorter than the rehearsal by this, as it does by + * branch narrowing; tcc_gen_machine_outline_shrink_between() gives CBZ's lower + * bound the bytes it can lose. A CMP #0 inside a window is not fused with the + * branch after it into CBZ (the rehearsal never fuses). + * + * Only functions whose prologue saves LR and that run the rehearsal: making a + * straight-line function run it too found 126 bytes in the Zig compiler. */ + +#define USING_GLOBALS +#include "tcc.h" +#include "ir.h" + +/* ------------------------------------------------------------ capture */ + +typedef struct +{ + int active; + int base; /* ind at the start of the captured pass */ + int cap; /* halfwords allocated */ + uint16_t *hw; /* captured halfwords, by (ind - base) / 2 */ + uint8_t *set; /* 1 where hw[] came through ot() */ + int *own; /* the IR op that emitted it */ + int nrel, relcap; /* GOT relocations the pass asked for */ + int32_t *rel_off; + int *rel_esym; /* ELF symbol index, -1 if the symbol has none */ +} OutlineCapture; + +static OutlineCapture cap; + +/* The real pass inside a planned window: its instructions go on advancing + * `ind` from the window's address, as if emitted, but land in hw[] -- a pair + * peephole may still rewind over them -- and are compared with the body when + * the window ends. Only then is the BL written there, or, should the code + * differ, the code itself, at the addresses it was generated for. */ +static struct +{ + int active; + int body; + int end_op; /* first op after the window */ + int vbase; /* the window's address */ + int cap; + uint16_t *hw; + uint8_t *set; + int nrel, relcap; + int32_t *rel_off; + int *rel_esym; +} sup; + +static int outline_stats_on(void) +{ + static int v = -1; + if (v < 0) + { + const char *e = getenv("TCC_OUTLINE_STATS"); + v = e ? atoi(e) : 0; /* 2: also print each window that went back inline */ + } + return v; +} + +/* Whether this compile outlines at all: -Os, not disabled, and -- like + * identical code folding -- not when debugging or instrumenting, which would + * see code of one function run from another's. TCC_NO_OUTLINE=1 turns it off. */ +ST_FUNC int tcc_gen_machine_outline_enabled(void) +{ + static int off = -1; + if (off < 0) + off = getenv("TCC_NO_OUTLINE") != NULL; + const TCCState *s1 = tcc_state; + return s1->optimize_size && s1->optimize > 0 && !off && !s1->check_only && !s1->do_debug && !s1->test_coverage && + !s1->do_backtrace; +} + +ST_FUNC void tcc_gen_machine_outline_capture_begin(void) +{ + cap.active = 1; + cap.base = ind; + cap.nrel = 0; + if (cap.hw) + memset(cap.set, 0, (size_t)cap.cap); +} + +ST_FUNC void tcc_gen_machine_outline_capture_end(void) +{ + cap.active = 0; +} + +static void cap_reserve(int n) +{ + if (n <= cap.cap) + return; + int nc = cap.cap ? cap.cap : 4096; + while (nc < n) + nc *= 2; + cap.hw = tcc_realloc(cap.hw, sizeof(uint16_t) * (size_t)nc); + cap.set = tcc_realloc(cap.set, (size_t)nc); + cap.own = tcc_realloc(cap.own, sizeof(int) * (size_t)nc); + memset(cap.set + cap.cap, 0, (size_t)(nc - cap.cap)); + cap.cap = nc; +} + +/* Called by ot() for every instruction, before it is written (or, in a dry + * pass, counted). */ +ST_FUNC void tcc_gen_machine_outline_capture(uint32_t opcode, int size, int at) +{ + if (!cap.active || at < cap.base || (size != 2 && size != 4)) + return; + int k = (at - cap.base) / 2; + cap_reserve(k + 2); + int op = tcc_state->ir ? tcc_state->ir->codegen_instruction_idx : -1; + if (size == 4) + { + cap.hw[k] = (uint16_t)(opcode >> 16); + cap.hw[k + 1] = (uint16_t)opcode; + cap.set[k] = cap.set[k + 1] = 1; + cap.own[k] = cap.own[k + 1] = op; + } + else + { + cap.hw[k] = (uint16_t)opcode; + cap.set[k] = 1; + cap.own[k] = op; + } +} + +/* The GOT_SBREL12 load of `sym` at `at`. The key is the ELF symbol, which + * the real pass would create here anyway: a Sym can be freed with its scope + * and its address reused by another. */ +static void sup_reloc(Sym *sym, int at); + +ST_FUNC void tcc_gen_machine_outline_capture_reloc(Sym *sym, int at) +{ + if (sup.active) + sup_reloc(sym, at); + if (!cap.active) + return; + if (cap.nrel == cap.relcap) + { + cap.relcap = cap.relcap ? cap.relcap * 2 : 64; + cap.rel_off = tcc_realloc(cap.rel_off, sizeof(int32_t) * (size_t)cap.relcap); + cap.rel_esym = tcc_realloc(cap.rel_esym, sizeof(int) * (size_t)cap.relcap); + } + if (sym && sym->c <= 0) + put_extern_sym(sym, NULL, 0, 0); + cap.rel_off[cap.nrel] = at - cap.base; + cap.rel_esym[cap.nrel] = sym && sym->c > 0 ? sym->c : -1; + cap.nrel++; +} + +/* ------------------------------------------------------------ decoder */ + +#define R_IS_SP_LR_PC(r) ((r) >= 13) + +/* Whether the instruction starting with halfword `h1` (and `h2` for a 32-bit + * one) may run from a shared body reached by BL: no branch, no PC-relative + * access, no write to SP, no reference to LR or PC. Anything not positively + * recognised is rejected. */ +static int outline_insn_ok(uint16_t h1, uint16_t h2, int size) +{ + if (size == 2) + { + unsigned top5 = h1 >> 11; + if (top5 <= 0x07) /* shift imm, add/sub reg/imm3, mov/cmp/add/sub imm8: low regs */ + return 1; + if ((h1 & 0xFC00) == 0x4000) /* data processing, low regs */ + return 1; + if ((h1 & 0xFC00) == 0x4400) /* ADD/CMP/MOV high, BX/BLX */ + { + unsigned op = (h1 >> 8) & 3; + if (op == 3) + return 0; + unsigned rdn = ((h1 >> 4) & 8) | (h1 & 7), rm = (h1 >> 3) & 15; + return !R_IS_SP_LR_PC(rdn) && !R_IS_SP_LR_PC(rm); + } + if ((h1 & 0xF800) == 0x4800) /* LDR literal */ + return 0; + if ((h1 & 0xF000) == 0x5000) /* load/store register offset, low regs */ + return 1; + if (top5 >= 0x0C && top5 <= 0x11) /* LDR/STR(B/H) imm, low regs */ + return 1; + if ((h1 & 0xF000) == 0x9000) /* LDR/STR Rt, [SP, #imm]: reads SP */ + return 1; + if ((h1 & 0xF800) == 0xA000) /* ADR */ + return 0; + if ((h1 & 0xF800) == 0xA800) /* ADD Rd, SP, #imm: reads SP */ + return 1; + if ((h1 & 0xFF00) == 0xB200) /* SXTH/SXTB/UXTH/UXTB */ + return 1; + if ((h1 & 0xFF00) == 0xBA00 && ((h1 >> 6) & 3) != 2) /* REV/REV16/REVSH */ + return 1; + if ((h1 & 0xFF00) == 0xBF00) /* IT / hints: IT and NOP only */ + return (h1 & 0x000F) != 0 || h1 == 0xBF00; + if ((h1 & 0xF000) == 0xC000) /* STM/LDM low base, low list */ + return 1; + return 0; /* SP adjust, CBZ, PUSH/POP, B, SVC, UDF, ... */ + } + + /* 32-bit */ + unsigned rn = h1 & 15; + if ((h1 & 0xFE00) == 0xE800) /* load/store multiple, dual, exclusive */ + { + if (h1 & 0x0040) /* dual / exclusive / table branch */ + { + unsigned p = (h1 >> 8) & 1, w = (h1 >> 5) & 1; + if (!p && !w) + return 0; /* LDREX/STREX/TBB */ + unsigned rt = h2 >> 12, rt2 = (h2 >> 8) & 15; + if (rn == 15 || R_IS_SP_LR_PC(rt) || R_IS_SP_LR_PC(rt2)) + return 0; + return !(rn == 13 && w); + } + unsigned op = (h1 >> 7) & 3; + if (op != 1 && op != 2) + return 0; + if (rn >= 13 || (h2 & 0xE000)) + return 0; /* PUSH/POP-like, or SP/LR/PC in the list */ + return 1; + } + if ((h1 & 0xFE00) == 0xEA00) /* data processing, shifted register */ + { + unsigned rd = (h2 >> 8) & 15, rm = h2 & 15, s = (h1 >> 4) & 1; + if (R_IS_SP_LR_PC(rm) || rn == 13 || rn == 14) + return 0; + if (rd == 15) + return s; /* TST/TEQ/CMP/CMN */ + return !R_IS_SP_LR_PC(rd); + } + if ((h1 & 0xF800) == 0xF000) /* data processing immediate / branches */ + { + if (h2 & 0x8000) + return 0; /* branches, misc control */ + unsigned rd = (h2 >> 8) & 15; + if (!(h1 & 0x0200)) /* modified immediate */ + { + unsigned s = (h1 >> 4) & 1; + if (rn == 14) + return 0; + if (rd == 15) + return s; + return !R_IS_SP_LR_PC(rd); /* Rn = SP only read (ADD Rd, SP, #imm) */ + } + unsigned op = (h1 >> 4) & 0x1F; + if (op == 0x04 || op == 0x0C) /* MOVW / MOVT: no Rn */ + return !R_IS_SP_LR_PC(rd); + if (rn == 15 && (op == 0x00 || op == 0x0A)) + return 0; /* ADR */ + if (rn == 14 || (rn == 15 && op != 0x16)) + return 0; /* BFC is BFI with Rn = PC */ + return !R_IS_SP_LR_PC(rd); + } + if ((h1 & 0xFF00) == 0xF800 || (h1 & 0xFF00) == 0xF900) /* load/store single */ + { + unsigned rt = h2 >> 12; + if (rn == 15 || R_IS_SP_LR_PC(rt)) + return 0; + if (rn == 13 && !(h1 & 0x0080) && (h2 & 0x0800) && (h2 & 0x0100)) + return 0; /* pre/post-indexed writeback of SP */ + return 1; + } + if ((h1 & 0xFF00) == 0xFA00) /* data processing register, extends, misc */ + { + unsigned rd = (h2 >> 8) & 15, rm = h2 & 15; + if ((h2 & 0xF000) != 0xF000) + return 0; + return !R_IS_SP_LR_PC(rd) && !R_IS_SP_LR_PC(rm) && rn != 13 && rn != 14; + } + if ((h1 & 0xFF00) == 0xFB00) /* multiply, divide */ + { + unsigned ra = h2 >> 12, rd = (h2 >> 8) & 15, rm = h2 & 15; + return !R_IS_SP_LR_PC(rn) && !R_IS_SP_LR_PC(rd) && !R_IS_SP_LR_PC(rm) && (ra == 15 || !R_IS_SP_LR_PC(ra)); + } + return 0; /* coprocessor / FP / anything else */ +} + +/* ------------------------------------------------------------ windows */ + +#define OUTLINE_MAX_OPS 12 +#define OUTLINE_MIN_BYTES 6 +/* A window's body is created at the occurrence where the ones seen so far, + * all called, would pay for it one and a half times over: n calls save n * + * (s - 4) bytes, the body costs s + 2, and the earlier occurrences are already + * emitted inline. Swept on the Zig compiler built as C at -Os against a fixed + * third occurrence (-54 KB): 1x -52, 1.2x -58, 1.5x -61, 1.7x -59, 2x -60 KB. + * Window lengths of 8 / 20 / 32 ops were within 2 KB of 12. */ +static int outline_body_pays(int seen, int s) +{ + return seen >= 2 && 2 * seen * (s - 4) >= 3 * (s + 2); +} + +static int outline_op_kind_ok(int op) +{ + switch (op) + { + case TCCIR_OP_ADD: case TCCIR_OP_SUB: case TCCIR_OP_MUL: case TCCIR_OP_MLA: case TCCIR_OP_UMULL: + case TCCIR_OP_AND: case TCCIR_OP_OR: case TCCIR_OP_XOR: case TCCIR_OP_SHL: case TCCIR_OP_SAR: + case TCCIR_OP_SHR: case TCCIR_OP_CMP: case TCCIR_OP_SETIF: case TCCIR_OP_TEST_ZERO: + case TCCIR_OP_LOAD: case TCCIR_OP_STORE: case TCCIR_OP_ASSIGN: case TCCIR_OP_LEA: + case TCCIR_OP_LOAD_INDEXED: case TCCIR_OP_STORE_INDEXED: case TCCIR_OP_UBFX: case TCCIR_OP_SBFX: + case TCCIR_OP_BFI: case TCCIR_OP_ZEXT: case TCCIR_OP_PACK64: case TCCIR_OP_BOOL_OR: case TCCIR_OP_BOOL_AND: + return 1; + default: + return 0; + } +} + +static uint64_t fnv_mix(uint64_t h, uint64_t v) +{ + for (int i = 0; i < 8; i++) + { + h ^= (v >> (8 * i)) & 0xFF; + h *= 1099511628211ull; + } + return h; +} + +typedef struct +{ + uint64_t key; + uint32_t count; + int32_t body; /* index into bodies[] + 1, 0 while it has none */ +} OutlineEntry; + +/* A shared body: its code is the window's halfwords plus BX LR. */ +typedef struct +{ + int esym; /* local STT_FUNC symbol */ + uint32_t hw0; /* first halfword in body_hw[] */ + uint32_t rel0; + uint16_t nhw; + uint16_t nrel; +} OutlineBody; + +static OutlineEntry *tab; +static uint32_t tab_cap, tab_n; +static OutlineBody *bodies; +static int nbodies, bodies_cap; +static uint16_t *body_hw; +static uint32_t body_hw_n, body_hw_cap; +static int32_t *body_rel_off; /* bytes from the body start */ +static int *body_rel_esym; +static uint32_t body_rel_n, body_rel_cap; +static Section *outline_sec; +static int64_t st_saved; +static uint64_t st_bodies, st_calls, st_fallbacks; + +static OutlineEntry *tab_find(uint64_t key, int insert) +{ + if (!key) + key = 1; + if (insert && (tab_n + 1) * 4 >= tab_cap * 3) + { + uint32_t oc = tab_cap; + OutlineEntry *ot_ = tab; + tab_cap = tab_cap ? tab_cap * 2 : (1u << 12); + tab = tcc_mallocz(sizeof(OutlineEntry) * tab_cap); + tab_n = 0; + for (uint32_t i = 0; i < oc; i++) + if (ot_[i].key) + { + OutlineEntry *e = tab_find(ot_[i].key, 1); + *e = ot_[i]; + } + tcc_free(ot_); + } + if (!tab_cap) + return NULL; + uint32_t m = tab_cap - 1, i = (uint32_t)(key ^ (key >> 29)) & m; + while (tab[i].key && tab[i].key != key) + i = (i + 1) & m; + if (!tab[i].key) + { + if (!insert) + return NULL; + tab[i].key = key; + tab_n++; + } + return &tab[i]; +} + +static void outline_stats_report(void) +{ + fprintf(stderr, "[OUTLINE] bodies %llu, calls %llu, fallbacks %llu, saved %lld bytes\n", + (unsigned long long)st_bodies, (unsigned long long)st_calls, (unsigned long long)st_fallbacks, + (long long)st_saved); +} + +/* The plan of the function being compiled: plan_body[k] >= 0 where a window + * starts, the ops [k, plan_end[k]) it covers. Built after the rehearsal, + * used by the real pass. */ +static int *plan_body, *plan_end, *plan_save; +static int plan_n; + + +/* Identical code folding merges a static function with an identical earlier + * one, and Zig instantiates many such twins. Outlining must not tell them + * apart -- the first copy inlines a window its twin would call -- so a + * rehearsal identical to an earlier one reuses that function's plan, and its + * windows are not counted again. Keyed by a hash of the rehearsal; every + * window is checked again before it is reused. */ +typedef struct +{ + uint64_t key; + uint32_t first, count; /* triples (op, end op, body) in memo_plan[] */ +} OutlineMemo; +static OutlineMemo *memo; +static uint32_t memo_cap, memo_n; +static int *memo_plan; +static uint32_t memo_plan_n, memo_plan_cap; + +static OutlineMemo *memo_find(uint64_t key, int insert) +{ + if (!key) + key = 1; + if (insert && (memo_n + 1) * 4 >= memo_cap * 3) + { + uint32_t oc = memo_cap; + OutlineMemo *om = memo; + memo_cap = memo_cap ? memo_cap * 2 : 1024; + memo = tcc_mallocz(sizeof(OutlineMemo) * memo_cap); + memo_n = 0; + for (uint32_t i = 0; i < oc; i++) + if (om[i].key) + *memo_find(om[i].key, 1) = om[i]; + tcc_free(om); + } + if (!memo_cap) + return NULL; + uint32_t m = memo_cap - 1, i = (uint32_t)(key ^ (key >> 31)) & m; + while (memo[i].key && memo[i].key != key) + i = (i + 1) & m; + if (!memo[i].key) + { + if (!insert) + return NULL; + memo[i].key = key; + memo[i].first = memo[i].count = 0; + memo_n++; + } + return &memo[i]; +} + +static void memo_push(int v) +{ + if (memo_plan_n == memo_plan_cap) + { + memo_plan_cap = memo_plan_cap ? memo_plan_cap * 2 : 1024; + memo_plan = tcc_realloc(memo_plan, sizeof(int) * memo_plan_cap); + } + memo_plan[memo_plan_n++] = v; +} + +static void plan_clear(void) +{ + tcc_free(plan_body); + tcc_free(plan_end); + tcc_free(plan_save); + plan_body = plan_end = plan_save = NULL; + plan_n = 0; +} + +/* End of a TU: bodies are local to the object file. */ +ST_FUNC void tcc_gen_machine_outline_reset(void) +{ + tcc_free(tab); + tab = NULL; + tab_cap = tab_n = 0; + tcc_free(bodies); + bodies = NULL; + nbodies = bodies_cap = 0; + tcc_free(body_hw); + body_hw = NULL; + body_hw_n = body_hw_cap = 0; + tcc_free(body_rel_off); + tcc_free(body_rel_esym); + body_rel_off = NULL; + body_rel_esym = NULL; + body_rel_n = body_rel_cap = 0; + outline_sec = NULL; + tcc_free(memo); + memo = NULL; + memo_cap = memo_n = 0; + tcc_free(memo_plan); + memo_plan = NULL; + memo_plan_n = memo_plan_cap = 0; + plan_clear(); + tcc_free(sup.hw); + tcc_free(sup.set); + tcc_free(sup.rel_off); + tcc_free(sup.rel_esym); + memset(&sup, 0, sizeof sup); + tcc_free(cap.hw); + tcc_free(cap.set); + tcc_free(cap.own); + tcc_free(cap.rel_off); + tcc_free(cap.rel_esym); + memset(&cap, 0, sizeof cap); +} + +/* Emit the captured bytes [a, b) of the current function as a new body. */ +static int body_create(int a, int b) +{ + TCCState *s1 = tcc_state; + int nhw = (b - a) / 2, k0 = (a - cap.base) / 2; + if (!outline_sec) + { + outline_sec = find_section(s1, ".text.tcc_outlined"); + outline_sec->sh_flags = text_section->sh_flags; + if (outline_sec->sh_addralign < 4) + outline_sec->sh_addralign = 4; + } + if (nbodies == bodies_cap) + { + bodies_cap = bodies_cap ? bodies_cap * 2 : 256; + bodies = tcc_realloc(bodies, sizeof(OutlineBody) * (size_t)bodies_cap); + } + while (body_hw_n + (uint32_t)nhw > body_hw_cap) + { + body_hw_cap = body_hw_cap ? body_hw_cap * 2 : 4096; + body_hw = tcc_realloc(body_hw, sizeof(uint16_t) * body_hw_cap); + } + OutlineBody *bd = &bodies[nbodies]; + bd->hw0 = body_hw_n; + bd->nhw = (uint16_t)nhw; + bd->rel0 = body_rel_n; + bd->nrel = 0; + memcpy(body_hw + body_hw_n, cap.hw + k0, sizeof(uint16_t) * (size_t)nhw); + body_hw_n += (uint32_t)nhw; + + unsigned long off = outline_sec->data_offset; + uint8_t *p = section_ptr_add(outline_sec, (addr_t)(2 * nhw + 2)); + for (int x = 0; x < nhw; x++) + { + p[2 * x] = (uint8_t)cap.hw[k0 + x]; + p[2 * x + 1] = (uint8_t)(cap.hw[k0 + x] >> 8); + } + p[2 * nhw] = 0x70; /* BX LR */ + p[2 * nhw + 1] = 0x47; + for (int r = 0; r < cap.nrel; r++) + { + int ro = cap.rel_off[r] - (a - cap.base); + if (ro < 0 || ro >= b - a) + continue; + if (body_rel_n == body_rel_cap) + { + body_rel_cap = body_rel_cap ? body_rel_cap * 2 : 256; + body_rel_off = tcc_realloc(body_rel_off, sizeof(int32_t) * body_rel_cap); + body_rel_esym = tcc_realloc(body_rel_esym, sizeof(int) * body_rel_cap); + } + body_rel_off[body_rel_n] = ro; + body_rel_esym[body_rel_n] = cap.rel_esym[r]; + body_rel_n++; + bd->nrel++; + put_elf_reloc(symtab_section, outline_sec, off + (unsigned long)ro, R_ARM_GOT_SBREL12, cap.rel_esym[r]); + } + char name[40]; + snprintf(name, sizeof name, "__tcc_outlined.%d", nbodies); + /* A Thumb function's value carries bit 0. */ + bd->esym = put_elf_sym(symtab_section, off + 1, (unsigned long)(2 * nhw + 2), ELFW(ST_INFO)(STB_LOCAL, STT_FUNC), 0, + outline_sec->sh_num, name); + return nbodies++; +} + +/* Whether the captured bytes [a, b) and their relocations are body `bi`: + * the table is keyed by a hash. */ +static int body_matches(int bi, int a, int b) +{ + OutlineBody *bd = &bodies[bi]; + int k0 = (a - cap.base) / 2; + if ((b - a) / 2 != bd->nhw || memcmp(body_hw + bd->hw0, cap.hw + k0, sizeof(uint16_t) * bd->nhw)) + return 0; + int nrel = 0; + for (int r = 0; r < cap.nrel; r++) + { + int ro = cap.rel_off[r] - (a - cap.base); + if (ro < 0 || ro >= b - a) + continue; + if (nrel >= bd->nrel || body_rel_off[bd->rel0 + nrel] != ro || body_rel_esym[bd->rel0 + nrel] != cap.rel_esym[r]) + return 0; + nrel++; + } + return nrel == bd->nrel; +} + +/* Instructions an IT at halfword `h` makes conditional, 0 if `h` is no IT. */ +static int it_length(uint16_t h) +{ + if ((h & 0xFF00) != 0xBF00 || !(h & 0xF)) + return 0; + int n = 4; + for (unsigned m = h & 0xF; !(m & 1); m >>= 1) + n--; + return n; +} + +/* TCC_OUTLINE_LIMIT=N: only the first N windows of the compile are called. */ +static int bisect_ok(void) +{ + static long lim = -2; + static long cnt; + if (lim == -2) + lim = getenv("TCC_OUTLINE_LIMIT") ? atol(getenv("TCC_OUTLINE_LIMIT")) : -1; + if (lim < 0) + return 1; + if (cnt++ < lim) + { + if (cnt == lim) + fprintf(stderr, "[OUTLINE-LAST] %s\n", funcname); + return 1; + } + return 0; +} + +static void plan_add(int n, int k, int end, int bi, int save) +{ + if (!bisect_ok()) + return; + if (!plan_body) + { + plan_n = n; + plan_body = tcc_malloc(sizeof(int) * (size_t)n); + plan_end = tcc_mallocz(sizeof(int) * (size_t)n); + plan_save = tcc_mallocz(sizeof(int) * (size_t)n); + for (int x = 0; x < n; x++) + plan_body[x] = -1; + } + plan_body[k] = bi; + plan_end[k] = end; + plan_save[k] = save; + st_saved += save; + st_calls++; +} + +/* After the rehearsal of a function whose prologue saves LR: pick the windows + * of its code that match a body -- or earn one now -- plan them for the real + * pass, and enter all its windows into the table. `map[i]` is where op i + * starts, `end` where the body ends (the epilogue). Returns how many windows + * were planned. */ +ST_FUNC int tcc_gen_machine_outline_analyze(TCCIRState *ir, const uint32_t *map, int end) +{ + static int reg; + if (!reg && outline_stats_on()) + { + reg = 1; + atexit(outline_stats_report); + } + plan_clear(); + const int n = ir->next_instruction_index; + const uint8_t *btr = ir->codegen_branch_target_reset; + int *ok = tcc_mallocz(sizeof(int) * (size_t)(n + 1)); + uint64_t *h = tcc_mallocz(sizeof(uint64_t) * (size_t)(n + 1)); + int *sz = tcc_mallocz(sizeof(int) * (size_t)(n + 1)); + int it_left = 0; /* conditional instructions still owed to an IT, across ops */ + int walk = 0; /* where the instruction after the last op's ends */ + uint64_t fkey = fnv_mix(0xCBF29CE484222325ull, (uint64_t)n); + for (int i = 0; i < n; i++) + { + IRQuadCompact *q = &ir->compact_instructions[i]; + int a = (int)map[i], b = (int)(i + 1 < n ? map[i + 1] : (uint32_t)end); + sz[i] = b > a ? b - a : 0; + if (b <= a) + continue; + /* Code of an op still inside an IT block opened by an earlier one: that + * op cannot start a window (the BL would be conditional). */ + int good = it_left == 0 && q->op != TCCIR_OP_NOP && outline_op_kind_ok(q->op); + uint64_t hh = 1469598103934665603ull; + int x = a; + /* An instruction may straddle two ops: the frame-word pair peephole + * rewinds over the previous op's LDR/STR to make an LDRD/STRD. */ + if (walk > a) + { + good = 0; + x = walk; + } + for (; x < b;) + { + int k = (x - cap.base) / 2; + if (k < 0 || k >= cap.cap || !cap.set[k]) + { + good = 0; + it_left = 0; + x += 2; + continue; + } + uint16_t h1 = cap.hw[k]; + int isz = ((h1 >> 11) >= 0x1D) ? 4 : 2; + if (isz == 4 && (k + 1 >= cap.cap || !cap.set[k + 1])) + { + good = 0; + it_left = 0; + x += 2; + continue; + } + uint16_t h2 = isz == 4 ? cap.hw[k + 1] : 0; + /* A later op may rewrite this one's code: the frame-word pair peephole + * turns its LDR into the LDRD that also does the next op's load. */ + if (!outline_insn_ok(h1, h2, isz) || cap.own[k] != i) + good = 0; + if (it_left > 0) + it_left--; + else + it_left = it_length(h1); + hh = fnv_mix(hh, ((uint64_t)h1 << 16) | h2); + x += isz; + } + walk = x; + /* An instruction or an IT block running past the op's end. */ + if (x > b || it_left > 0) + good = 0; + for (int r = 0; r < cap.nrel && good; r++) + if (cap.rel_off[r] >= a - cap.base && cap.rel_off[r] < b - cap.base) + { + if (cap.rel_esym[r] <= 0) + good = 0; + hh = fnv_mix(hh, (uint64_t)(uint32_t)cap.rel_esym[r] ^ ((uint64_t)(cap.rel_off[r] - (a - cap.base)) << 40)); + } + ok[i] = good; + h[i] = hh; + fkey = fnv_mix(fkey, hh ^ ((uint64_t)(uint32_t)(a - (int)map[0]) << 32) ^ ((uint64_t)good << 63) ^ (uint64_t)i); + } + /* Ops [k, end) as a window of this function: its size, or 0. */ +#define WINDOW_SIZE(k, end, out) \ + do \ + { \ + int s_ = 0; \ + for (int j_ = (k); j_ < (end) && s_ >= 0; j_++) \ + { \ + IRQuadCompact *q_ = &ir->compact_instructions[j_]; \ + if (j_ > (k) && (q_->is_jump_target || (btr && btr[j_]))) \ + s_ = -1; \ + else if (q_->op == TCCIR_OP_NOP && !sz[j_]) \ + continue; \ + else if (!ok[j_]) \ + s_ = -1; \ + else \ + s_ += sz[j_]; \ + } \ + (out) = s_ > 0 ? s_ : 0; \ + } while (0) + /* Only a function identical code folding may merge: static, not weak. */ + const Sym *fsym = tcc_state->cur_func_sym; + const int foldable = fsym && (fsym->type.t & VT_STATIC) && !fsym->a.weak && !fsym->a.nested_func; + OutlineMemo *fm = foldable ? memo_find(fkey, 0) : NULL; + if (fm) + { + int all = 1; + for (uint32_t t = 0; t < fm->count && all; t++) + { + int k = memo_plan[fm->first + 3 * t], end = memo_plan[fm->first + 3 * t + 1], bi = memo_plan[fm->first + 3 * t + 2]; + int s = 0; + if (end <= n) + WINDOW_SIZE(k, end, s); + all = s > 0 && body_matches(bi, (int)map[k], (int)map[k] + s); + } + if (all) + { + int planned = 0; + for (uint32_t t = 0; t < fm->count; t++) + { + int k = memo_plan[fm->first + 3 * t], end = memo_plan[fm->first + 3 * t + 1], bi = memo_plan[fm->first + 3 * t + 2]; + plan_add(n, k, end, bi, 2 * bodies[bi].nhw - 4); + planned++; + } + tcc_free(ok); + tcc_free(h); + tcc_free(sz); + return planned; + } + } + /* Ops [k, j) of a window, as long as `pass` non-NOP ones: no jump lands + * after its first op. */ +#define WINDOW_EXTEND(k, j, L, s, key, limit, taken) \ + for (; j < n && L < (limit); j++) \ + { \ + IRQuadCompact *q_ = &ir->compact_instructions[j]; \ + if (L > 0 && (q_->is_jump_target || (btr && btr[j]))) \ + break; \ + if (q_->op == TCCIR_OP_NOP && !sz[j]) \ + continue; \ + if (!ok[j] || (taken && taken[j])) \ + break; \ + key = fnv_mix(key, h[j]); \ + s += sz[j]; \ + L++; \ + if (L == (limit)) \ + { \ + j++; \ + break; \ + } \ + } + /* windows starting at each op: greedy, largest first, non-overlapping */ + int planned = 0; + uint8_t *taken = tcc_mallocz((size_t)(n + 1)); + for (int pass = OUTLINE_MAX_OPS; pass >= 1; pass--) + for (int k = 0; k < n; k++) + { + if (!ok[k] || taken[k]) + continue; + uint64_t key = 0x9E3779B97F4A7C15ull; + int s = 0, L = 0, j = k; + WINDOW_EXTEND(k, j, L, s, key, pass, taken); + if (L != pass || s < OUTLINE_MIN_BYTES) + continue; + OutlineEntry *e = tab_find(fnv_mix(key, (uint64_t)s), 0); + if (!e || (!e->body && !outline_body_pays((int)e->count + 1, s))) + continue; + int a = (int)map[k]; + int bi; + if (e->body) + { + bi = e->body - 1; + if (!body_matches(bi, a, a + s)) + continue; /* a hash collision */ + } + else + { + /* This occurrence earns the body; the ones before stay inline. */ + bi = body_create(a, a + s); + e->body = bi + 1; + st_bodies++; + st_saved -= s + 2; + } + plan_add(n, k, j, bi, s - 4); + planned++; + for (int x = k; x < j; x++) + taken[x] = 1; + } + /* enter this function's windows */ + for (int k = 0; k < n; k++) + { + if (!ok[k]) + continue; + uint64_t key = 0x9E3779B97F4A7C15ull; + int s = 0, L = 0, j = k; + uint8_t *no_taken = NULL; + while (j < n && L < OUTLINE_MAX_OPS) + { + int L0 = L; + WINDOW_EXTEND(k, j, L, s, key, L0 + 1, no_taken); + if (L == L0) + break; + if (s >= OUTLINE_MIN_BYTES) + { + OutlineEntry *e = tab_find(fnv_mix(key, (uint64_t)s), 1); + e->count++; + } + } + } +#undef WINDOW_EXTEND +#undef WINDOW_SIZE + /* remember the plan for this function's twins */ + if (foldable) + { + fm = memo_find(fkey, 1); + fm->first = memo_plan_n; + fm->count = 0; + for (int k = 0; k < n && plan_body; k++) + if (plan_body[k] >= 0) + { + memo_push(k); + memo_push(plan_end[k]); + memo_push(plan_body[k]); + fm->count++; + } + } + tcc_free(taken); + tcc_free(ok); + tcc_free(h); + tcc_free(sz); + return planned; +} + +/* ------------------------------------------------------------ real pass */ + +ST_FUNC int tcc_gen_machine_outline_suppressing(void) +{ + return sup.active; +} + +static void sup_reloc(Sym *sym, int at) +{ + if (sup.nrel == sup.relcap) + { + sup.relcap = sup.relcap ? sup.relcap * 2 : 16; + sup.rel_off = tcc_realloc(sup.rel_off, sizeof(int32_t) * (size_t)sup.relcap); + sup.rel_esym = tcc_realloc(sup.rel_esym, sizeof(int) * (size_t)sup.relcap); + } + sup.rel_off[sup.nrel] = at - sup.vbase; + sup.rel_esym[sup.nrel] = sym && sym->c > 0 ? sym->c : -1; + sup.nrel++; +} + +/* Called by ot() for every instruction of the real pass: inside a window the + * instruction is recorded instead of written (returns 1; ot() still advances + * `ind`). */ +ST_FUNC int tcc_gen_machine_outline_suppress(uint32_t opcode, int size) +{ + if (!sup.active) + return 0; + int k = (ind - sup.vbase) / 2; + if (k < 0 || (size != 2 && size != 4)) + tcc_error("internal error: outlined window instruction at 0x%x", ind); + if (k + 2 > sup.cap) + { + int nc = sup.cap ? sup.cap * 2 : 64; + while (nc < k + 2) + nc *= 2; + sup.hw = tcc_realloc(sup.hw, sizeof(uint16_t) * (size_t)nc); + sup.set = tcc_realloc(sup.set, (size_t)nc); + memset(sup.set + sup.cap, 0, (size_t)(nc - sup.cap)); + sup.cap = nc; + } + if (size == 4) + { + sup.hw[k] = (uint16_t)(opcode >> 16); + sup.hw[k + 1] = (uint16_t)opcode; + sup.set[k] = sup.set[k + 1] = 1; + } + else + { + sup.hw[k] = (uint16_t)opcode; + sup.set[k] = 1; + } + return 1; +} + +/* The window's ops are done. If they produced the body, `ind` goes back to + * the window's address and the BL is written there. If not -- a cache the + * pool dumps reset at other points than in the rehearsal -- the code they + * produced is written where it was generated, exactly as without the + * outliner. */ +static void outline_window_close(void) +{ + if (!sup.active) + return; + sup.active = 0; + OutlineBody *bd = &bodies[sup.body]; + const int bytes = ind - sup.vbase, nh = bytes / 2; + int all_set = bytes >= 0 && !(bytes & 1); + for (int x = 0; x < nh && all_set; x++) + all_set = sup.set[x]; + int same = all_set && nh == bd->nhw && !memcmp(sup.hw, body_hw + bd->hw0, sizeof(uint16_t) * (size_t)nh); + int nrel = 0; + for (int r = 0; r < sup.nrel && same; r++) + { + if (sup.rel_off[r] < 0 || sup.rel_off[r] >= bytes) + continue; + if (nrel >= bd->nrel || body_rel_off[bd->rel0 + nrel] != sup.rel_off[r] || + body_rel_esym[bd->rel0 + nrel] != sup.rel_esym[r]) + same = 0; + nrel++; + } + if (same && nrel == bd->nrel) + { + tcc_gen_machine_outline_window_closed(bytes); + ind = sup.vbase; + tcc_gen_machine_outline_emit_bl(bd->esym); + return; + } + + st_fallbacks++; + if (outline_stats_on() > 1) + { + fprintf(stderr, "[OUTLINE-FB] %s at 0x%x: %d/%d halfwords, same %d:", funcname, sup.vbase, nh, bd->nhw, same); + for (int x = 0; x < nh && all_set; x++) + fprintf(stderr, " %04x", sup.hw[x]); + fprintf(stderr, " | body"); + for (int x = 0; x < bd->nhw; x++) + fprintf(stderr, " %04x", body_hw[bd->hw0 + x]); + fprintf(stderr, " | rel %d/%d\n", sup.nrel, bd->nrel); + } + if (!all_set) + tcc_error("internal error: outlined window at 0x%x was not all emitted through ot()", sup.vbase); + ind = sup.vbase; + for (int x = 0; x < nh; x++) + tcc_gen_machine_outline_emit_raw(sup.hw[x]); + for (int r = 0; r < sup.nrel; r++) + if (sup.rel_off[r] >= 0 && sup.rel_off[r] < bytes && sup.rel_esym[r] > 0) + put_elf_reloc(symtab_section, cur_text_section, (unsigned long)(sup.vbase + sup.rel_off[r]), R_ARM_GOT_SBREL12, + sup.rel_esym[r]); +} + +/* The real pass reaches op i, before recording its address. */ +ST_FUNC void tcc_gen_machine_outline_op_end(int i) +{ + if (sup.active && i >= sup.end_op) + outline_window_close(); +} + +/* The real pass recorded op i's address: a planned window starts there (and + * the BL will be there). */ +ST_FUNC void tcc_gen_machine_outline_op_start(int i) +{ + if (!plan_body || i >= plan_n || plan_body[i] < 0 || sup.active) + return; + sup.body = plan_body[i]; + sup.end_op = plan_end[i]; + sup.nrel = 0; + if (sup.cap) + memset(sup.set, 0, (size_t)sup.cap); + tcc_gen_machine_outline_window_open(); + sup.vbase = ind; + sup.active = 1; +} + +/* The real pass is done with the function. */ +ST_FUNC void tcc_gen_machine_outline_function_end(void) +{ + outline_window_close(); + plan_clear(); +} + +/* Bytes the real pass can shed, against the rehearsal, by windows starting + * strictly between two ops. */ +ST_FUNC int tcc_gen_machine_outline_shrink_between(int from_ir, int to_ir) +{ + if (!plan_body) + return 0; + int s = 0; + for (int k = from_ir + 1; k < to_ir && k < plan_n; k++) + if (k >= 0 && plan_body[k] >= 0) + s += plan_save[k]; + return s; +} diff --git a/source/backend/arch/arm/thumb/thumb-tok.h b/source/backend/arch/arm/thumb/thumb-tok.h index a7c2954d3..f565b4589 100644 --- a/source/backend/arch/arm/thumb/thumb-tok.h +++ b/source/backend/arch/arm/thumb/thumb-tok.h @@ -388,6 +388,7 @@ DEF_ASM_BASE(smlal) DEF_ASM_BASE(smull) DEF_ASM_BASE(ssat) DEF_ASM_BASE(udiv) +DEF_ASM_BASE(umaal) DEF_ASM_BASE(umlal) DEF_ASM_BASE(umull) DEF_ASM_BASE(usat) diff --git a/source/backend/arch/unity.mk b/source/backend/arch/unity.mk new file mode 100644 index 000000000..b80abd362 --- /dev/null +++ b/source/backend/arch/unity.mk @@ -0,0 +1,30 @@ +# Unity groups for an arch sub-library: see "Unity build" in the top-level +# Makefile. Include after OBJS is complete and before the rule that archives +# it, with UNITY_LIB set to the library's name in source/unity/unity.mk and +# UNITY_SRC_DIR to this directory relative to $(TOP). UNITY comes from the +# top-level make; run on its own this Makefile builds one TU per file. + +UNITY ?= no +-include $(TOP)/source/unity/unity.mk +# The rules below must not become the including Makefile's default goal. +UNITY_SAVED_GOAL := $(.DEFAULT_GOAL) + +UNITY_HERE = $(foreach g,$(UNITY_GROUPS_$(UNITY_LIB)),$(UNITY_MEMBERS_$g)) + +ifeq ($(UNITY),yes) +UNITY_STALE = $(filter-out $(addprefix $(UNITY_SRC_DIR)/,$(SRCS)),$(UNITY_HERE)) +$(if $(UNITY_STALE),$(error source/unity/unity.mk lists $(UNITY_STALE) under '$(UNITY_LIB)', which no longer builds it; run scripts/gen_unity.py)) +OBJS := $(filter-out $(addprefix $(BUILD_DIR)/,$(notdir $(UNITY_HERE:.c=.o))),$(OBJS)) \ + $(foreach g,$(UNITY_GROUPS_$(UNITY_LIB)),$(BUILD_DIR)/unity_$g.o) +endif + +$(BUILD_DIR)/unity_%.o: $(TOP)/source/unity/%.c $(CORE_HDRS) + @mkdir -p $(dir $@) + $(CC) -o $@ -c $< $(CFLAGS) $(DEFINES) -I$(TOP) -I$(TOP)/source/ir +$(foreach g,$(UNITY_GROUPS_$(UNITY_LIB)),$(eval $(BUILD_DIR)/unity_$g.o : $(addprefix $(TOP)/,$(UNITY_MEMBERS_$g)))) + +# For scripts/gen_unity.py. +unity-info: + @echo 'LIB $(UNITY_LIB) $(addprefix $(UNITY_SRC_DIR)/,$(SRCS))' +.PHONY: unity-info +.DEFAULT_GOAL := $(UNITY_SAVED_GOAL) diff --git a/source/backend/generators/Makefile b/source/backend/generators/Makefile index cdd6be963..90c7a799f 100644 --- a/source/backend/generators/Makefile +++ b/source/backend/generators/Makefile @@ -8,9 +8,10 @@ BACKEND_GENERATORS_CC = $(CC) $(CFLAGS) $(DEFINES) $(BACKEND_GENERATORS_INC) # Object files for the backend generator TUs. BACKEND_GENERATORS_SRC = source/backend/generators/function.c \ - source/backend/generators/regalloc.c + source/backend/generators/regalloc.c \ + source/backend/generators/icf.c -BACKEND_GENERATORS_OBJ = $(patsubst %.c,$(X)%.o,$(BACKEND_GENERATORS_SRC)) +BACKEND_GENERATORS_OBJ = $(call lib-objs,generators,$(BACKEND_GENERATORS_SRC)) # The function generator depends on the full tcc header set and IR headers. $(X)source/backend/generators/function.o: \ @@ -38,6 +39,13 @@ $(X)source/backend/generators/regalloc.o: \ @mkdir -p $(dir $@) $(BACKEND_GENERATORS_CC) -c $< -o $@ +# Identical code folding: drops a generated body equal to one already emitted. +$(X)source/backend/generators/icf.o: \ + source/backend/generators/icf.c \ + source/include/tcc.h $(TCC_CORE_DEPS) + @mkdir -p $(dir $@) + $(BACKEND_GENERATORS_CC) -c $< -o $@ + BACKEND_GENERATORS_LIB = $(X)source/backend/generators/libgenerators.a $(BACKEND_GENERATORS_LIB): $(BACKEND_GENERATORS_OBJ) diff --git a/source/backend/generators/function.c b/source/backend/generators/function.c index 394850c75..80d1f8164 100644 --- a/source/backend/generators/function.c +++ b/source/backend/generators/function.c @@ -52,8 +52,6 @@ extern int func_has_label_addr; extern int local_scope; extern int loc; extern int nb_temp_local_vars; -extern int nb_arg_struct_temps; -extern uint64_t arg_struct_temp_busy; extern int rsym; extern void vpush_type_size(CType *type, int *a); @@ -66,8 +64,6 @@ extern void vpush_type_size(CType *type, int *a); static void gen_instrument_call(Sym *cur_func_sym, const char *hook_name); static void func_vla_arg_code(Sym *arg); static void func_vla_arg(Sym *sym); -static int ir_inline_stash_eligible(Sym *sym, TCCIRState *ir); -static void ir_inline_stash_add(TCCState *s1, Sym *sym, TCCIRState *ir); /* A `V` operand (VAR vreg, is_local + is_lval) names a register-resident scalar; * every other lval — a stack slot, a symbol, a pointer deref — is real memory, @@ -107,6 +103,37 @@ static int ir_body_is_register_only(TCCIRState *ir) return 1; } +/* A static function whose optimized body is at most this many real ops of + * straight-line code (see ir_body_is_straight_leaf) is inlined at every call + * site, whatever its token length. The token gate at registration counts + * source, and Zig's C backend spells one rotate as ten zig.h helper calls: + * math.rotr is 122 tokens, compiles to 20 ops standalone and to two shifts + * and an OR once its constant shift amount arrives, and the Zig compiler + * called it 10.6M times from BLAKE3. Swept on that compiler: 16 ops -3.1% + * instructions, 24 -9.2%, 32 -9.4%. */ +#define SMALL_LEAF_INLINE_OPS 24 + +/* Straight-line leaf: no call, no inline asm, no backward jump. */ +static int ir_body_is_straight_leaf(TCCIRState *ir) +{ + for (int i = 0; i < ir->next_instruction_index; i++) + { + IRQuadCompact *q = &ir->compact_instructions[i]; + if (q->op == TCCIR_OP_FUNCCALLVAL || q->op == TCCIR_OP_FUNCCALLVOID || q->op == TCCIR_OP_INLINE_ASM || + q->op == TCCIR_OP_BUILTIN_APPLY || q->op == TCCIR_OP_IJUMP || q->op == TCCIR_OP_VLA_ALLOC) + return 0; + if (q->op == TCCIR_OP_JUMP || q->op == TCCIR_OP_JUMPIF) + { + IROperand jd = tcc_ir_op_get_dest(ir, q); + if ((int)irop_get_imm64_ex(ir, jd) <= i) + return 0; + } + } + return 1; +} + +TCC_DBG_ENV_STR(dbg_dump_func, "TCC_DUMP_FUNC") + void gen_function(Sym *sym) { struct scope f = {0}; @@ -133,14 +160,31 @@ void gen_function(Sym *sym) gen_fill_nops(newoff - ind); } + func_align_pad = 0; funcname = get_tok_str(sym->v, NULL); func_ind = ind; +#ifdef CONFIG_TCC_DEBUG + /* TCC_DUMP_FUNC=: -dump-ir / -dump-ir-passes print this function only, + * so a whole-TU compile (whose inlining and summaries a cut-down TU does not + * reproduce) can still be dumped for the one function that matters. */ + const int saved_dump_ir = tcc_state->dump_ir; + char *const saved_dump_ir_passes = tcc_state->dump_ir_passes; + if (dbg_dump_func() && strcmp(dbg_dump_func(), funcname)) + { + tcc_state->dump_ir = 0; + tcc_state->dump_ir_passes = NULL; + } +#endif + /* Where this body's relocations begin, for tcc_icf_try_fold. */ + const size_t func_reloc_mark = + cur_text_section->reloc ? cur_text_section->reloc->data_offset : 0; func_vt = sym->type.ref->type; func_var = sym->type.ref->f.func_type == FUNC_ELLIPSIS; func_has_label_addr = 0; tcc_state->cur_func_sym = sym; /* Per-caller budget: see CONST_LOOP_INLINE_BUDGET in gen/builtin/call.c. */ tcc_state->const_loop_inline_used = 0; + tcc_state->nested_inline_budget = NESTED_INLINE_BUDGET; /* NOTE: we patch the symbol size later */ put_extern_sym(sym, cur_text_section, ind + 1, 0); @@ -184,8 +228,6 @@ void gen_function(Sym *sym) if (ir->has_static_chain) loc -= 4; nb_temp_local_vars = 0; - nb_arg_struct_temps = 0; - arg_struct_temp_busy = 0; local_scope = 0; rsym = -1; @@ -193,8 +235,6 @@ void gen_function(Sym *sym) /* -finstrument-functions: emit entry hook call before function body */ if (tcc_state->instrument_functions && !sym->type.ref->f.func_no_instrument) { - tcc_state->force_frame_pointer = 1; - tcc_state->force_lr_save = 1; gen_instrument_call(sym, "__cyg_profile_func_enter"); } @@ -237,6 +277,11 @@ void gen_function(Sym *sym) continue; if (nf->trampoline_needed || nf->nb_captured == 0) continue; + /* Marked auto_inline is a hope, not a fact: a call site can decline for + * reasons of its own, and the surviving call still reads the captures + * through the static chain. nb_real_calls is the fact. */ + if (nf->nb_real_calls > 0) + continue; int called_by_sibling = 0; int func_tok = nf->sym->v & ~SYM_FIELD; for (int si = 0; si < tcc_state->nb_nested_funcs && !called_by_sibling; si++) { @@ -335,6 +380,7 @@ void gen_function(Sym *sym) int nonstatic_global_copier = 0; tcc_ir_opt_run_function_pipeline(ir, sym, func_var, &nonstatic_global_copier); + tcc_ir_backend_fold_pure_forward(ir, sym); tcc_ir_backend_analyze_leaf_and_tail_calls(ir, func_var); if (tcc_state->do_bench) @@ -348,6 +394,19 @@ void gen_function(Sym *sym) tcc_ir_backend_regalloc_pipeline(ir, sym, func_var, &phase_start, funcname, global_label_stack_start); + /* Stores writing back what was just loaded from the same frame bytes: + * copies between objects frame relayout placed on the same slot. */ + if (tcc_state->optimize > 0 && !tcc_ir_opt_pass_disabled("self_store") && tcc_ir_self_store(ir)) + tcc_ir_dump_after_pass(ir, "self_store"); + + /* Repeated single-entry regions (switch case bodies) share one copy. */ + if (tcc_state->optimize > 0 && !tcc_ir_opt_pass_disabled("region_merge") && tcc_ir_region_merge(ir)) + tcc_ir_dump_after_pass(ir, "region_merge"); + + /* Identical block tails jumping to the same place share one copy. */ + if (tcc_state->optimize > 0 && !tcc_ir_opt_pass_disabled("cross_jump") && tcc_ir_cross_jump(ir)) + tcc_ir_dump_after_pass(ir, "cross_jump"); + tcc_ir_codegen_generate(ir); if (ir->barrel_shifts) { @@ -446,6 +505,11 @@ void gen_function(Sym *sym) real_ops++; if (ir->next_instruction_index <= 8 || real_ops == 0) sym->type.ref->f.func_auto_inline = 1; + else if ((sym->type.t & VT_STATIC) && real_ops <= SMALL_LEAF_INLINE_OPS && ir_body_is_straight_leaf(ir)) + { + sym->type.ref->f.func_auto_inline = 1; + sym->type.ref->f.func_small_leaf = 1; + } } /* Keep a non-static global-aggregate-copier with a parameter (the fn1/fn2 @@ -535,8 +599,8 @@ void gen_function(Sym *sym) * The ≤12 floor keeps tiny loop helpers inlinable for the constant-arg * path, where loop_const_sim can still fold the whole thing. */ int loop_body = has_loop && real_ops > 12; - if (call_ops >= 3 || (!naturally_small && ir->next_instruction_index > 24) || - nonstatic_bloat || loop_body) + if (!sym->type.ref->f.func_small_leaf && + (call_ops >= 3 || (!naturally_small && ir->next_instruction_index > 24) || nonstatic_bloat || loop_body)) { sym->type.ref->f.func_auto_inline = 0; } @@ -569,11 +633,40 @@ void gen_function(Sym *sym) } } + /* TCC_INL_DBG: the post-optimization inline classification of every function (with [INLSITE] lines from + * the call sites that expand one). */ + { + static int dbg = -1; + if (dbg < 0) + dbg = getenv("TCC_INL_DBG") != NULL; + if (dbg && ir && sym) + { + int ro = 0, calls = 0; + for (int ii = 0; ii < ir->next_instruction_index; ii++) + { + int op = ir->compact_instructions[ii].op; + ro += op != TCCIR_OP_NOP; + calls += op == TCCIR_OP_FUNCCALLVAL || op == TCCIR_OP_FUNCCALLVOID; + } + fprintf(stderr, "[INL] %s slots=%d ops=%d calls=%d auto=%d leaf=%d evalonly=%d heavy=%d bytes=%d\n", + get_tok_str(sym->v, NULL), ir->next_instruction_index, ro, calls, sym->type.ref->f.func_auto_inline, + sym->type.ref->f.func_small_leaf, sym->type.ref->f.func_eval_only_inline, + sym->type.ref->f.func_inline_call_heavy, ind - func_ind); + } + } tcc_debug_funcend(tcc_state, ind - func_ind); elfsym(sym)->st_size = ind - func_ind; cur_text_section->data_offset = ind; + /* An identical body was generated already: drop this one and let the + * symbol point at it (identical code folding). */ + if (tcc_icf_try_fold(tcc_state, sym, cur_text_section, func_ind, ind - func_ind, func_reloc_mark)) + { + /* The word-alignment pad codegen put in front of the body goes with it. */ + ind = func_ind - func_align_pad; + cur_text_section->data_offset = ind; + } local_scope = 0; label_pop(&global_label_stack, global_label_stack_start, 0); @@ -605,6 +698,10 @@ void gen_function(Sym *sym) /* It's better to crash than to generate wrong code */ cur_text_section = NULL; +#ifdef CONFIG_TCC_DEBUG + tcc_state->dump_ir = saved_dump_ir; + tcc_state->dump_ir_passes = saved_dump_ir_passes; +#endif funcname = ""; /* for safety */ func_vt.t = VT_VOID; /* for safety */ func_var = 0; /* for safety */ @@ -616,14 +713,7 @@ void gen_function(Sym *sym) /* do this after funcend debug info */ next(); - if (ir_inline_stash_eligible(sym, ir)) - { - ir_inline_stash_add(tcc_state, sym, ir); - } - else - { - tcc_ir_free(ir); - } + tcc_ir_free(ir); tcc_state->ir = NULL; /* Publish the fact that this function's body has been compiled in this @@ -633,6 +723,79 @@ void gen_function(Sym *sym) sym->type.ref->f.func_compiled = 1; } +/* Parse SYM's body, the current token stream from its '{', for the + * diagnostics it raises and nothing else: prune_unused_statics dropped the + * function, but an error in it is still an error (gcc reports it too). + * The frontend runs as for gen_function -- parameters in scope, code on -- + * into an IR that is freed afterwards, with tcc_state->check_only keeping + * every other product out of the object (see its comment in tcc.h). No + * optimization, no codegen. Nested function bodies are saved by the parse + * and generated only by codegen, so they are not checked. */ +void gen_function_check_only(Sym *sym) +{ + struct scope f = {0}; + Sym *global_label_stack_start = global_label_stack; + LabelDiffFixup *fixups_start = tcc_state->label_diff_fixups; + const int nested_start = tcc_state->nb_nested_funcs; + Section *saved_text = cur_text_section; + + tcc_state->check_only++; + cur_scope = root_scope = &f; + nocode_wanted = 0; + cur_text_section = tcc_state->check_scratch; + ind = 0; + funcname = get_tok_str(sym->v, NULL); + func_ind = 0; + func_vt = sym->type.ref->type; + func_var = sym->type.ref->f.func_type == FUNC_ELLIPSIS; + func_has_label_addr = 0; + tcc_state->cur_func_sym = sym; + tcc_state->const_loop_inline_used = 0; + + sym_push2(&local_stack, SYM_FIELD, 0, 0); + TCCIRState *ir = tcc_ir_alloc(); + tcc_state->ir = ir; + gen_op_vector_reset(); + ir->is_variadic = func_var; + local_scope = 1; + tcc_ir_params_add(ir, &sym->type); + nb_temp_local_vars = 0; + local_scope = 0; + rsym = -1; + block(0); + + sym_pop(&local_stack, NULL, 0); + sym_pop(&all_cleanups, NULL, 0); + label_pop(&global_label_stack, global_label_stack_start, 0); + while (tcc_state->label_diff_fixups && tcc_state->label_diff_fixups != fixups_start) + { + LabelDiffFixup *next = tcc_state->label_diff_fixups->next; + tcc_free(tcc_state->label_diff_fixups); + tcc_state->label_diff_fixups = next; + } + for (int i = nested_start; i < tcc_state->nb_nested_funcs; i++) + if (tcc_state->nested_funcs[i].func_str) + { + tok_str_free(tcc_state->nested_funcs[i].func_str); + tcc_state->nested_funcs[i].func_str = NULL; + } + tcc_state->nb_nested_funcs = nested_start; + + cur_text_section = saved_text; + funcname = ""; + func_vt.t = VT_VOID; + func_var = 0; + ind = 0; + func_ind = -1; + tcc_state->cur_func_sym = NULL; + nocode_wanted = DATA_ONLY_WANTED; + check_vstack(); + next(); + tcc_ir_free(ir); + tcc_state->ir = NULL; + tcc_state->check_only--; +} + /* ------------------------------------------------------------------ */ /* Helpers that were extracted from tccgen.c because they are only * called from gen_function(). Keeping them here avoids orphaned-static @@ -651,17 +814,8 @@ static void gen_instrument_call(Sym *cur_func_sym, const char *hook_name) /* arg0: address of current function */ vpushsym(&void_ptr_type, cur_func_sym); - /* arg1: return address (call site) = __builtin_return_address(0) - * LR is saved at [FP + PTR_SIZE] in the standard frame record */ - CType ptr_type; - ptr_type.t = VT_VOID; - ptr_type.ref = NULL; - mk_pointer(&ptr_type); - vset(&ptr_type, VT_LOCAL, 0); /* FP value */ - vpushi(PTR_SIZE); - gen_op('+'); - mk_pointer(&vtop->type); - indir(); + /* arg1: return address (call site) = __builtin_return_address(0) */ + vpush_return_address(); /* Push the hook function */ vpush_helper_func(tok_alloc_const(hook_name)); @@ -761,49 +915,3 @@ static void func_vla_arg(Sym *sym) tcc_state->nb_vla_param_exprs = 0; } } - -/* Phase 0 inliner stash: keep optimized IR of eligible `static` functions - * alive past gen_function() so a future inliner pass can splice it into - * callers. Today there are no consumers — this exists to validate the - * lifecycle change (no leaks, no regressions) before the splice logic lands. */ -#ifndef IR_INLINE_STASH_SIZE_BUDGET -#define IR_INLINE_STASH_SIZE_BUDGET 200 -#endif - -static int ir_inline_stash_eligible(Sym *sym, TCCIRState *ir) -{ - if (!sym || !ir) - return 0; - if (!(sym->type.t & VT_STATIC)) - return 0; - if (sym->a.addrtaken) - return 0; - if (!sym->type.ref || sym->type.ref->f.func_type == FUNC_ELLIPSIS) - return 0; - if (ir->has_static_chain) - return 0; - if (ir->nb_nested_funcs > 0) - return 0; - if (ir->naked) - return 0; -#ifdef CONFIG_TCC_ASM - if (ir->inline_asm_count > 0) - return 0; -#endif - if (ir->next_instruction_index > IR_INLINE_STASH_SIZE_BUDGET) - return 0; - return 1; -} - -static void ir_inline_stash_add(TCCState *s1, Sym *sym, TCCIRState *ir) -{ - if (s1->nb_stashed_func_irs >= s1->stashed_func_irs_capacity) - { - s1->stashed_func_irs_capacity = s1->stashed_func_irs_capacity ? s1->stashed_func_irs_capacity * 2 : 4; - s1->stashed_func_irs = - tcc_realloc(s1->stashed_func_irs, s1->stashed_func_irs_capacity * sizeof(StashedFuncIR)); - } - s1->stashed_func_irs[s1->nb_stashed_func_irs].sym = sym; - s1->stashed_func_irs[s1->nb_stashed_func_irs].ir = ir; - s1->nb_stashed_func_irs++; -} diff --git a/source/backend/generators/icf.c b/source/backend/generators/icf.c new file mode 100644 index 000000000..9043a3858 --- /dev/null +++ b/source/backend/generators/icf.c @@ -0,0 +1,426 @@ +/* + * TCC - Identical code folding + * + * Copyright (c) 2026 Mateusz Stadnik + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation. + */ + +/* A generic function instantiated over types that lower the same way compiles + * to the same machine code, once per instantiation: the Zig compiler's C has + * 3,254 such bodies, 10.7% of its .text -- ten copies of one hash-map routine, + * twenty-seven of one error path. A body identical to one already emitted is + * dropped here and its symbol points at the survivor, which is what gcc's + * -fipa-icf does. + * + * The fold happens as the body is generated, while it is still the last thing + * in the section, so nothing that exists already moves: the section and its + * relocation table are rolled back to where this body began, exactly as + * str_lit_pool_merge rolls back a duplicate string literal. A call emitted + * earlier reaches the function through a relocation on its symbol, which now + * resolves to the survivor. + * + * Only a static function whose address nothing takes is folded, so no program + * can observe that two functions became one address. A function designator + * decays to a pointer without `&`, so three things are asked: the `&` flag, + * whether any relocation so far refers to the symbol other than by calling it, + * and whether a body still to be generated names it anywhere but in a direct + * call. Only after the whole TU is parsed, where those answers are complete. + */ + +#include "tcc.h" + +/* A body generated in this TU, by fingerprint. */ +typedef struct IcfEntry +{ + uint64_t hash; + Section *sec; + addr_t start, size; + int nrel; /* relocations the body carries */ + size_t rel_off; /* where they begin in sec->reloc */ + addr_t value; /* the symbol's value (thumb tag included) */ + int self_sym; /* its own symbol, so a self-call matches a self-call */ + struct IcfEntry *next; +} IcfEntry; + +/* Symbols some relocation refers to other than by calling them, and the + * relocation sections scanned so far (relocations arrive as bodies are + * generated, so each is looked at once). */ +static uint8_t *icf_value_sym; +static int icf_value_sym_size; +static size_t *icf_scanned; +static int icf_scanned_size; +/* Identifiers a body still to be generated uses other than as a direct call. */ +static uint8_t *icf_value_tok; +static int icf_value_tok_size; +static int icf_value_tok_done; + +#define ICF_BUCKETS 4096 +static IcfEntry *icf_table[ICF_BUCKETS]; +static IcfEntry *icf_entries; +static int icf_nb_entries, icf_entries_size; + +/* Code in `sec` moved (gen_late_reopt_functions erases a body before + * re-emitting it), so every body recorded there is at an address that no + * longer holds it: forget them. Without this, the re-emitted body matches + * the record of its own erased self. */ +void tcc_icf_section_changed(Section *sec) +{ + for (int i = 0; i < icf_nb_entries; i++) + if (icf_entries[i].sec == sec) + icf_entries[i].sec = NULL; +} + +void tcc_icf_reset(void) +{ + memset(icf_table, 0, sizeof(icf_table)); + tcc_free(icf_entries); + icf_entries = NULL; + icf_nb_entries = icf_entries_size = 0; + tcc_free(icf_value_sym); + icf_value_sym = NULL; + icf_value_sym_size = 0; + tcc_free(icf_scanned); + icf_scanned = NULL; + icf_scanned_size = 0; + tcc_free(icf_value_tok); + icf_value_tok = NULL; + icf_value_tok_size = 0; + icf_value_tok_done = 0; +} + +/* A relocation that calls a function does not take its address; any other one + * does (a pointer in a table, a pointer loaded into a register). */ +static int icf_reloc_is_call(int type) +{ + switch (type) + { + case R_ARM_CALL: + case R_ARM_JUMP24: + case R_ARM_THM_PC22: /* the Thumb BL/BLX tcc emits for a call */ + case R_ARM_THM_JUMP24: + case R_ARM_THM_JUMP19: + case R_ARM_PLT32: + return 1; + default: + return 0; + } +} + +/* Relocations added since the last look, by symbol. */ +static void icf_scan_new_relocs(TCCState *s1) +{ + if (icf_scanned_size < s1->nb_sections) + { + icf_scanned = tcc_realloc(icf_scanned, sizeof(*icf_scanned) * s1->nb_sections); + memset(icf_scanned + icf_scanned_size, 0, sizeof(*icf_scanned) * (s1->nb_sections - icf_scanned_size)); + icf_scanned_size = s1->nb_sections; + } + const int nsyms = s1->symtab ? (int)(s1->symtab->data_offset / sizeof(ElfW(Sym))) : 0; + if (icf_value_sym_size < nsyms) + { + icf_value_sym = tcc_realloc(icf_value_sym, (size_t)nsyms); + memset(icf_value_sym + icf_value_sym_size, 0, (size_t)(nsyms - icf_value_sym_size)); + icf_value_sym_size = nsyms; + } + for (int sn = 1; sn < s1->nb_sections; sn++) + { + Section *sr = s1->sections[sn]; + if (!sr || sr->sh_type != SHT_RELX || !sr->data) + continue; + for (size_t off = icf_scanned[sn]; off + sizeof(ElfW_Rel) <= sr->data_offset; off += sizeof(ElfW_Rel)) + { + const ElfW_Rel *r = (const ElfW_Rel *)(sr->data + off); + const int si = ELFW(R_SYM)(r->r_info); + if (si > 0 && si < icf_value_sym_size && !icf_reloc_is_call(ELFW(R_TYPE)(r->r_info))) + icf_value_sym[si] = 1; + } + icf_scanned[sn] = sr->data_offset; + } +} + +/* Identifiers the bodies still to be generated use other than as `name (`. */ +static void icf_scan_pending_bodies(TCCState *s1) +{ + if (icf_value_tok_done) + return; + icf_value_tok_done = 1; + icf_value_tok_size = tok_ident - TOK_IDENT + 1; + if (icf_value_tok_size < 1) + icf_value_tok_size = 1; + icf_value_tok = tcc_mallocz((size_t)icf_value_tok_size); + for (int pass = 0; pass < 2; pass++) + { + const int n = pass == 0 ? s1->nb_deferred_fns : s1->nb_inline_fns; + for (int i = 0; i < n; i++) + { + TokenString *ts = pass == 0 ? s1->deferred_fns[i]->body : s1->inline_fns[i]->func_str; + if (!ts) + continue; + const int *p = tok_str_buf(ts), *end = p + ts->len; + int prev = 0, prev_is_ident = 0; + while (p < end) + { + int t; + CValue cv; + tok_get(&t, &p, &cv); + if (t == TOK_LINENUM) + continue; + if (prev_is_ident && t != '(') + { + const int idx = prev - TOK_IDENT; + if (idx >= 0 && idx < icf_value_tok_size) + icf_value_tok[idx] = 1; + } + prev_is_ident = t >= TOK_IDENT; + prev = t; + } + if (prev_is_ident) + { + const int idx = prev - TOK_IDENT; + if (idx >= 0 && idx < icf_value_tok_size) + icf_value_tok[idx] = 1; + } + } + } +} + +static int icf_address_may_be_taken(TCCState *s1, Sym *sym) +{ + if (sym->a.addrtaken) + return 1; + icf_scan_new_relocs(s1); + if (sym->c > 0 && sym->c < icf_value_sym_size && icf_value_sym[sym->c]) + return 1; + icf_scan_pending_bodies(s1); + const int idx = (sym->v & ~SYM_FIELD) - TOK_IDENT; + if (idx >= 0 && idx < icf_value_tok_size && icf_value_tok[idx]) + return 1; + return 0; +} + +/* A branch's bytes hold the displacement to its target as it stood when the + * instruction was written, so they differ between two identical bodies sitting + * at different addresses and say nothing about sameness -- the relocation + * entry names the target instead. Every other relocation keeps its addend + * there (ELF REL), which is part of what the body does: `return p;` and + * `return g;` differ only in the addend on their section's symbol. */ +static int icf_reloc_skips_bytes(int type) +{ + return icf_reloc_is_call(type); +} + +static int icf_reloc_width(int type) +{ + (void)type; + return 4; /* every ARM relocation tcc emits covers a word or a pair of halfwords */ +} + +static uint64_t icf_hash_body(Section *sec, addr_t start, addr_t size, const ElfW_Rel *rel, int nrel, int self_sym) +{ + uint64_t h = 1469598103934665603ull; /* FNV-1a */ +#define ICF_MIX(b) \ + do \ + { \ + h = (h ^ (unsigned char)(b)) * 1099511628211ull; \ + } while (0) + unsigned char *masked = tcc_malloc(size ? size : 1); + memcpy(masked, sec->data + start, size); + for (int i = 0; i < nrel; i++) + { + const int type = ELFW(R_TYPE)(rel[i].r_info); + addr_t off = rel[i].r_offset - start; + int w = icf_reloc_width(type); + if (icf_reloc_skips_bytes(type) && off + w <= size) + memset(masked + off, 0, w); + } + for (addr_t i = 0; i < size; i++) + ICF_MIX(masked[i]); + tcc_free(masked); + ICF_MIX(nrel); + for (int i = 0; i < nrel; i++) + { + const addr_t off = rel[i].r_offset - start; + const int type = ELFW(R_TYPE)(rel[i].r_info); + const int sym = ELFW(R_SYM)(rel[i].r_info); + /* A body calling itself matches one calling itself, not one calling the + * other: the reference is to whichever body it belongs to. */ + const int s = (sym == self_sym) ? -1 : sym; + for (unsigned k = 0; k < sizeof(off); k++) + ICF_MIX(off >> (8 * k)); + ICF_MIX(type); + for (unsigned k = 0; k < sizeof(s); k++) + ICF_MIX((unsigned)s >> (8 * k)); + } +#undef ICF_MIX + return h; +} + +/* Byte-for-byte, relocation-for-relocation, so a hash collision cannot fold + * two different bodies. */ +static int icf_same_body(Section *sec, addr_t a_start, addr_t size, const ElfW_Rel *a_rel, int a_nrel, int a_self, + const IcfEntry *b, int b_self) +{ + if (b->size != size || b->nrel != a_nrel) + return 0; + if (b->nrel && !(b->sec->reloc && b->sec->reloc->data)) + return 0; + const ElfW_Rel *b_rel = (const ElfW_Rel *)(b->sec->reloc ? b->sec->reloc->data + b->rel_off : NULL); + for (int i = 0; i < a_nrel; i++) + { + if (a_rel[i].r_offset - a_start != b_rel[i].r_offset - b->start) + return 0; + if (ELFW(R_TYPE)(a_rel[i].r_info) != ELFW(R_TYPE)(b_rel[i].r_info)) + return 0; + int sa = ELFW(R_SYM)(a_rel[i].r_info), sb = ELFW(R_SYM)(b_rel[i].r_info); + if (sa == a_self) + sa = -1; + if (sb == b_self) + sb = -1; + if (sa != sb) + return 0; + } + /* Compare every byte but a branch's displacement. */ + addr_t pos = 0; + for (int i = 0; i <= a_nrel; i++) + { + addr_t next = (i < a_nrel) ? a_rel[i].r_offset - a_start : size; + if (next > size) + next = size; + if (next > pos && memcmp(sec->data + a_start + pos, b->sec->data + b->start + pos, next - pos) != 0) + return 0; + if (i < a_nrel) + { + const int type = ELFW(R_TYPE)(a_rel[i].r_info); + addr_t end = next + icf_reloc_width(type); + if (end > size) + end = size; + if (!icf_reloc_skips_bytes(type) && end > next && + memcmp(sec->data + a_start + next, b->sec->data + b->start + next, end - next) != 0) + return 0; + pos = end; + } + } + return 1; +} + +static void icf_record(uint64_t hash, Section *sec, addr_t start, addr_t size, size_t rel_off, int nrel, addr_t value, + int self_sym) +{ + if (icf_nb_entries >= icf_entries_size) + { + icf_entries_size = icf_entries_size ? icf_entries_size * 2 : 256; + IcfEntry *grown = tcc_realloc(icf_entries, sizeof(*grown) * icf_entries_size); + /* The table holds indices, not pointers, so a move is free. */ + icf_entries = grown; + memset(icf_table, 0, sizeof(icf_table)); + for (int i = 0; i < icf_nb_entries; i++) + { + IcfEntry *e = &icf_entries[i]; + unsigned b = (unsigned)(e->hash % ICF_BUCKETS); + e->next = icf_table[b]; + icf_table[b] = e; + } + } + IcfEntry *e = &icf_entries[icf_nb_entries++]; + e->hash = hash; + e->sec = sec; + e->start = start; + e->size = size; + e->rel_off = rel_off; + e->nrel = nrel; + e->value = value; + e->self_sym = self_sym; + unsigned b = (unsigned)(hash % ICF_BUCKETS); + e->next = icf_table[b]; + icf_table[b] = e; +} + +/* A local symbol inside the dropped range ($t/$d mapping markers) would + * otherwise mark the next function's code as data: objdump applies them by + * address. Nothing relocates against them, so retarget them to SHN_ABS + * rather than delete, which would renumber the symtab. */ +static void icf_drop_range_symbols(TCCState *s1, Section *sec, addr_t start, int keep_sym) +{ + Section *symtab = s1->symtab; + if (!symtab || !symtab->data) + return; + int n = symtab->data_offset / sizeof(ElfW(Sym)); + ElfW(Sym) *syms = (ElfW(Sym) *)symtab->data; + for (int i = 1; i < n; i++) + { + if (i == keep_sym || syms[i].st_shndx != sec->sh_num) + continue; + if ((syms[i].st_value & ~(addr_t)1) >= start) + syms[i].st_shndx = SHN_ABS; + } +} + +/* Called with the body of `sym` occupying [start, start+size) at the end of + * `sec`. Returns 1 when it was dropped for an identical one, having pointed + * the symbol at it; the caller then rewinds `ind`. */ +int tcc_icf_try_fold(TCCState *s1, Sym *sym, Section *sec, addr_t start, addr_t size, size_t reloc_mark) +{ + if (!s1->tu_parsed || s1->optimize == 0 || s1->do_debug || s1->test_coverage || s1->do_backtrace) + return 0; + if (tcc_ir_opt_pass_disabled("icf")) + return 0; + if (!sym || !sec || !sec->data || size == 0 || start + size != sec->data_offset) + return 0; + /* Internal linkage and an address nothing takes: folding is unobservable. */ + if (!(sym->type.t & VT_STATIC) || sym->a.weak || sym->a.nested_func) + return 0; + if (icf_address_may_be_taken(s1, sym)) + return 0; + if (sym->asm_label || !sym->c) + return 0; + ElfSym *esym = elfsym(sym); + if (!esym || esym->st_shndx != sec->sh_num) + return 0; + + const ElfW_Rel *rel = NULL; + int nrel = 0; + if (sec->reloc && sec->reloc->data) + { + if (reloc_mark > sec->reloc->data_offset) + return 0; + rel = (const ElfW_Rel *)(sec->reloc->data + reloc_mark); + nrel = (int)((sec->reloc->data_offset - reloc_mark) / sizeof(ElfW_Rel)); + /* Relocations are appended as the body is generated; a body whose + * relocations are not in address order is rare and not worth sorting. */ + for (int i = 0; i < nrel; i++) + if (rel[i].r_offset < start || rel[i].r_offset >= start + size || + (i > 0 && rel[i].r_offset < rel[i - 1].r_offset)) + return 0; + } + + const int self_sym = sym->c; + const uint64_t hash = icf_hash_body(sec, start, size, rel, nrel, self_sym); + for (IcfEntry *e = icf_table[hash % ICF_BUCKETS]; e; e = e->next) + { + if (e->hash != hash || !e->sec) + continue; + /* The survivor's own symbol index: its self-references were recorded + * against it, so ask which symbol sits at its address. */ + int b_self = e->self_sym; + if (!icf_same_body(sec, start, size, rel, nrel, self_sym, e, b_self)) + continue; + + /* Drop this body: the section and its relocations end where it began. */ + icf_drop_range_symbols(s1, sec, start, sym->c); + if (sec->reloc) + sec->reloc->data_offset = reloc_mark; + sec->data_offset = start; + esym->st_shndx = e->sec->sh_num; + esym->st_value = e->value; + esym->st_size = e->size; + if (sym->type.ref) + sym->type.ref->f.func_icf_folded = 1; + return 1; + } + + icf_record(hash, sec, start, size, reloc_mark, nrel, esym->st_value, self_sym); + return 0; +} diff --git a/source/backend/generators/regalloc.c b/source/backend/generators/regalloc.c index a8db2088a..df2ed8c98 100644 --- a/source/backend/generators/regalloc.c +++ b/source/backend/generators/regalloc.c @@ -112,6 +112,221 @@ static int ir_call_is_tail_positioned(TCCIRState *ir, int call_idx) return is_tail; } +/* ABI-identical parameter or return types: an argument then lands where the + * callee expects it exactly when it lands where this function received it. */ +static int pf_abi_same(CType *a, CType *b) +{ + const int ba = a->t & VT_BTYPE, bb = b->t & VT_BTYPE; + if ((a->t & VT_COMPLEX) != (b->t & VT_COMPLEX)) + return 0; + if (ba == VT_STRUCT || bb == VT_STRUCT) + return ba == bb && a->ref == b->ref; + if (ba == VT_VOID || bb == VT_VOID) + return ba == bb; + if (is_float(a->t) || is_float(b->t)) + return ba == bb; + int aa, ab; + return type_size(a, &aa) == type_size(b, &ab); +} + +/* Does FUNCPARAMVAL source `op` hand on parameter `f` exactly as received? */ +static int pf_arg_is_param(IROperand op, const IRParamForm *f) +{ + switch (f->kind) + { + case IR_PF_REG: + return irop_get_tag(op) == IROP_TAG_VREG && irop_get_vreg(op) == f->vreg && !op.is_lval; + case IR_PF_MEM: + return irop_get_tag(op) == IROP_TAG_STACKOFF && irop_get_vreg(op) == f->vreg && op.is_lval && op.is_param && + irop_get_stack_offset(op) == f->off; + case IR_PF_HOME: + return irop_get_tag(op) == IROP_TAG_STACKOFF && irop_get_vreg(op) < 0 && op.is_lval && !op.is_param && + op.is_local && op.btype == IROP_BTYPE_STRUCT && irop_get_stack_offset(op) == f->off; + default: + return 0; + } +} + +/* A function whose body only hands its parameters, untouched, to one direct + * call of an ABI-identical signature and returns what it returns -- the Zig C + * backend's forwarding wrappers, `f(a0) { g(a0); }` -- finds every argument + * already where the callee expects it: r0-r3 as received, the stack arguments + * in the caller's outgoing area. Its parameters are dropped from the call and + * the call becomes the whole function: a branch (tail_call_only). The one + * body instruction besides the parameters, the call and the return that may + * appear is the prologue's store of a register-passed struct into its home + * slot, from the very registers the callee expects it in. */ +void tcc_ir_backend_fold_pure_forward(TCCIRState *ir, Sym *sym) +{ + if (!ir || !sym || !sym->type.ref || tcc_state->optimize <= 0 || tcc_ir_opt_pass_disabled("pure_forward")) + return; + Sym *fref = sym->type.ref; + if (fref->f.func_type != FUNC_NEW || ir->has_static_chain || ir->captured_count > 0 || ir->naked || + tcc_state->do_debug || tcc_bounds_checking(tcc_state) || tcc_state->instrument_functions) + return; + if ((fref->type.t & VT_BTYPE) == VT_STRUCT || (fref->type.t & VT_COMPLEX)) + return; /* an sret pointer the call would get a fresh temporary for */ + const int np = ir->parameters_count; + if (np < 0 || (np > 0 && !ir->param_forms)) + return; + + const int n = ir->next_instruction_index; + int call = -1, ret = -1, nparams = 0, nstores = 0; + for (int i = 0; i < n; i++) + { + IRQuadCompact *q = &ir->compact_instructions[i]; + switch (q->op) + { + case TCCIR_OP_NOP: + break; + case TCCIR_OP_FUNCPARAMVAL: + nparams++; + break; + case TCCIR_OP_FUNCPARAMVOID: + break; + case TCCIR_OP_STORE: + nstores++; + break; + case TCCIR_OP_FUNCCALLVAL: + case TCCIR_OP_FUNCCALLVOID: + if (call >= 0) + return; + call = i; + break; + case TCCIR_OP_RETURNVALUE: + case TCCIR_OP_RETURNVOID: + if (ret >= 0) + return; + ret = i; + break; + default: + return; + } + } + if (call < 0 || (ret >= 0 && ret < call) || nparams != np) + return; + IRQuadCompact *cq = &ir->compact_instructions[call]; + if (ret >= 0 && ir->compact_instructions[ret].op == TCCIR_OP_RETURNVALUE && + (cq->op != TCCIR_OP_FUNCCALLVAL || + irop_get_vreg(tcc_ir_op_get_src1(ir, &ir->compact_instructions[ret])) != + irop_get_vreg(tcc_ir_op_get_dest(ir, cq)))) + return; + + /* The callee: a direct function of an ABI-identical signature. */ + IROperand fn = tcc_ir_op_get_src1(ir, cq); + if (fn.is_lval || irop_get_vreg(fn) >= 0) + return; + Sym *callee = irop_get_sym_ex(ir, fn); + if (!callee || (callee->type.t & VT_BTYPE) != VT_FUNC || !callee->type.ref) + return; + Sym *cref = callee->type.ref; + if (cref->f.func_type != FUNC_NEW || cref->f.func_call != fref->f.func_call || !pf_abi_same(&fref->type, &cref->type)) + return; + Sym *pa = fref->next, *pb = cref->next; + for (; pa && pb; pa = pa->next, pb = pb->next) + if (!pf_abi_same(&pa->type, &pb->type)) + return; + if (pa || pb) + return; + /* A callee in another module is reached through a thunk that swaps in its + * own R9 and does not restore ours: it must return here. */ + if (tcc_state->text_and_data_separation && !tcc_gen_machine_callee_in_this_module(callee)) + return; + + /* Every argument is the parameter of its position, as received. */ + const int call_id = TCCIR_DECODE_CALL_ID((uint32_t)irop_get_imm64_ex(ir, tcc_ir_op_get_src2(ir, cq))); + uint32_t seen = 0; + if (np > 32) + return; + for (int i = 0; i < call; i++) + { + IRQuadCompact *q = &ir->compact_instructions[i]; + if (q->op != TCCIR_OP_FUNCPARAMVAL) + continue; + uint32_t enc = (uint32_t)irop_get_imm64_ex(ir, tcc_ir_op_get_src2(ir, q)); + int k = TCCIR_DECODE_PARAM_IDX(enc); + if (TCCIR_DECODE_CALL_ID(enc) != call_id || k >= np || (seen & (1u << k)) || + !pf_arg_is_param(tcc_ir_op_get_src1(ir, q), &ir->param_forms[k])) + return; + seen |= 1u << k; + } + /* Stores: a register-passed struct's words into its home, each from the + * PARAM vreg that arrived in the register the callee reads it from. */ + int home_words = 0; + for (int k = 0; k < np; k++) + if (ir->param_forms[k].kind == IR_PF_HOME) + home_words += ir->param_forms[k].words; + if (nstores != home_words) + return; + for (int i = 0; i < call; i++) + { + IRQuadCompact *q = &ir->compact_instructions[i]; + if (q->op != TCCIR_OP_STORE) + continue; + IROperand d = tcc_ir_op_get_dest(ir, q), v = tcc_ir_op_get_src1(ir, q); + int32_t vr = irop_get_vreg(v); + if (irop_get_tag(d) != IROP_TAG_STACKOFF || irop_get_vreg(d) >= 0 || !d.is_lval || d.is_param || + d.btype != IROP_BTYPE_INT32 || irop_get_tag(v) != IROP_TAG_VREG || v.is_lval || vr < 0 || + TCCIR_DECODE_VREG_TYPE(vr) != TCCIR_VREG_TYPE_PARAM) + return; + IRLiveInterval *iv = tcc_ir_vreg_live_interval(ir, vr); + int32_t off = irop_get_stack_offset(d), ok = 0; + for (int k = 0; k < np && !ok; k++) + { + const IRParamForm *f = &ir->param_forms[k]; + int w = (off - f->off) / 4; + ok = f->kind == IR_PF_HOME && off >= f->off && !((off - f->off) & 3) && w < f->words && iv && + iv->incoming_reg0 == f->reg_base + w; + } + if (!ok) + return; + } + + IROperand enc = tcc_ir_op_get_src2(ir, cq); + if (irop_get_tag(enc) != IROP_TAG_IMM32) + return; + enc.u.imm32 = (int32_t)TCCIR_ENCODE_CALL(call_id, 0); + tcc_ir_op_set_src2(ir, cq, enc); + /* The call keeps one FUNCPARAMVOID, as a call without arguments has: the + * backend builds its call sites from them. */ + int marker = 0; + for (int i = 0; i < call; i++) + { + IRQuadCompact *q = &ir->compact_instructions[i]; + if (q->op == TCCIR_OP_FUNCPARAMVAL && !marker) + { + q->op = TCCIR_OP_FUNCPARAMVOID; + tcc_ir_op_set_src2(ir, q, enc); + marker = 1; + } + else if (q->op == TCCIR_OP_FUNCPARAMVAL || q->op == TCCIR_OP_STORE) + q->op = TCCIR_OP_NOP; + } + if (!marker) + { + /* No argument to turn into the marker: a call without parameters is + * already one FUNCPARAMVOID and a call; nothing to rewrite. */ + ir->pure_forward = 0; + return; + } + ir->push_arg_regs = 0; + ir->pure_forward = 1; +} + +/* Does an asm statement in this function clobber LR (a `bl` in its body)? + * Then LR needs saving like a call does: asm_gen_code saves only r4-r11 + * around the statement, so a leaf function returned through the LR the asm + * left -- into itself. Counted as a call that is never a tail call. */ +static int ir_asm_clobbers_lr(TCCIRState *ir) +{ +#ifdef CONFIG_TCC_ASM + for (int a = 0; a < ir->inline_asm_count; ++a) + if (ir->inline_asms[a].asm_str && ir->inline_asms[a].clobber_regs[14]) + return 1; +#endif + return 0; +} + void tcc_ir_backend_analyze_leaf_and_tail_calls(TCCIRState *ir, int func_var) { ir->leaffunc = 1; @@ -144,7 +359,20 @@ void tcc_ir_backend_analyze_leaf_and_tail_calls(TCCIRState *ir, int func_var) } } - if (call_count == 1 && !has_complex_fp && !func_var && !ir->has_static_chain && call_idx >= 0) + if (ir_asm_clobbers_lr(ir)) + { + ir->leaffunc = 0; + call_count = 99; + } + + if (ir->pure_forward && call_count == 1) + { + ir->tail_call_only = 1; + ir->leaffunc = 1; + return; + } + + if (call_count == 1 && !has_complex_fp && !func_var && !ir->push_arg_regs && !ir->has_static_chain && call_idx >= 0) { if (ir_call_is_tail_positioned(ir, call_idx) && !ir_tail_call_returns_hard_float(ir, call_idx)) { @@ -400,6 +628,14 @@ static void run_register_coalescing(TCCIRState *ir) hint_li->r0 = wanted_reg; blocker->r0 = have_reg; + /* The post-RA passes still to run before compute_stack_layout copies every + * interval into its IRLiveInterval read the registers from there, not from + * ls.intervals. Left stale, ra:reload_elim saw `StackLoc <- R0(P0)` then + * `R0(T) <- StackLoc` with T still in the R0 this swap had just taken from + * it, deleted the reload as a no-op, and T's new register was never + * written. */ + tcc_ir_stack_reg_assign(ir, hint_li->vreg, hint_li->stack_location, hint_li->r0, hint_li->r1); + tcc_ir_stack_reg_assign(ir, blocker->vreg, blocker->stack_location, blocker->r0, blocker->r1); ir->ls.dirty_registers |= (1ull << wanted_reg) | (1ull << have_reg); if (ir->ls.live_regs_by_instruction) @@ -422,17 +658,78 @@ static void run_register_coalescing(TCCIRState *ir) } } +/* A vreg's index in one array covering VARs, TEMPs and PARAMs, or -1. */ +static int spill_slot_ref_index(const TCCIRState *ir, int32_t vr) +{ + if (vr < 0) + return -1; + const int pos = TCCIR_DECODE_VREG_POSITION(vr); + switch (TCCIR_DECODE_VREG_TYPE(vr)) + { + case TCCIR_VREG_TYPE_VAR: + return pos < ir->next_local_variable ? pos : -1; + case TCCIR_VREG_TYPE_TEMP: + return pos < ir->next_temporary_variable ? ir->next_local_variable + pos : -1; + case TCCIR_VREG_TYPE_PARAM: + return pos < ir->next_parameter ? ir->next_local_variable + ir->next_temporary_variable + pos : -1; + default: + return -1; + } +} + +/* Static references to each linear-scan interval's vreg: every one of a + * spilled vreg is a load or store of its slot. */ +static uint32_t *spill_slot_weights(TCCIRState *ir) +{ + const int nv = ir->next_local_variable, nt = ir->next_temporary_variable, np = ir->next_parameter; + uint32_t *refs = tcc_mallocz(sizeof(uint32_t) * (size_t)(nv + nt + np + 1)); + for (int i = 0; i < ir->next_instruction_index; i++) + { + const IRQuadCompact *q = &ir->compact_instructions[i]; + if (q->op == TCCIR_OP_NOP) + continue; + IROperand ops[4]; + int n = 0; + if (irop_config[q->op].has_dest) + ops[n++] = tcc_ir_op_get_dest(ir, q); + if (irop_config[q->op].has_src1) + ops[n++] = tcc_ir_op_get_src1(ir, q); + if (irop_config[q->op].has_src2) + ops[n++] = tcc_ir_op_get_src2(ir, q); + if (tcc_ir_op_is_mac(q->op)) + ops[n++] = tcc_ir_op_get_accum(ir, q); + for (int k = 0; k < n; k++) + { + const int idx = spill_slot_ref_index(ir, irop_get_vreg(ops[k])); + if (idx >= 0) + refs[idx]++; + } + } + uint32_t *weights = tcc_malloc(sizeof(uint32_t) * (size_t)(ir->ls.next_interval_index + 1)); + for (int i = 0; i < ir->ls.next_interval_index; i++) + { + const int idx = spill_slot_ref_index(ir, (int)ir->ls.intervals[i].vreg); + weights[i] = idx >= 0 ? refs[idx] : 0; + } + tcc_free(refs); + return weights; +} + /* ================================================================== */ /* Compute final stack layout: min offsets, compact, move coalescing */ /* ================================================================== */ static void compute_stack_layout(TCCIRState *ir, int func_var) { tcc_ls_reset_scratch_cache(&ir->ls); + tcc_ir_mark_nested_captured_params(ir); tcc_ir_avoid_spilling_stack_passed_params(ir); /* Initial min-local-offset scan */ { int min_local_offset = 0; + /* Nested functions reach a parent's captured locals by their frontend + * offsets, so there every offset counts; relayout skips those frames. */ + const int nested = ir->has_static_chain || tcc_state->nb_nested_funcs > 0; (void)0; /* stackoff_count removed — was diagnostic only */ for (int i = 0; i < ir->next_instruction_index; i++) { @@ -447,7 +744,10 @@ static void compute_stack_layout(TCCIRState *ir, int func_var) { if (irop_is_none(ops[j])) continue; - if (irop_get_tag(ops[j]) == IROP_TAG_STACKOFF) + /* A vreg-backed operand's offset is the frontend's watermark, not a + * slot the code addresses; counting it would keep the spills below + * frame objects that relayout has moved up (frame.c). */ + if (irop_get_tag(ops[j]) == IROP_TAG_STACKOFF && (nested || irop_get_vreg(ops[j]) < 0)) { int32_t off = irop_get_stack_offset(ops[j]); if (off < min_local_offset) @@ -459,11 +759,23 @@ static void compute_stack_layout(TCCIRState *ir, int func_var) { loc = min_local_offset; } + /* The incoming static chain is saved at frame offset -4, and that slot is + * not an IR operand — nothing in the scan above can see it. A nested + * function that only forwards the chain to a child of its own (no locals, + * no spills) would otherwise have loc raised to 0 here, and its prologue + * would store the chain below its own SP, where the child's push lands on + * it. gen_function already reserved these four bytes; keep them. */ + if (ir->has_static_chain && loc > -4) + loc = -4; if (func_var && loc > -28) loc = -28; } - tcc_ls_compact_stack_locations(&ir->ls, loc); + { + uint32_t *weights = spill_slot_weights(ir); + tcc_ls_compact_stack_locations_weighted(&ir->ls, loc, weights); + tcc_free(weights); + } /* Nested-chain detection + live vreg bitmap + min_stack_loc */ { @@ -505,7 +817,7 @@ static void compute_stack_layout(TCCIRState *ir, int func_var) if (p <= max_vreg_pos) live_vregs[p / 8] |= (1 << (p % 8)); } - if (q->op == TCCIR_OP_MLA || q->op == TCCIR_OP_LOAD_INDEXED || + if (tcc_ir_op_is_mac(q->op) || q->op == TCCIR_OP_LOAD_INDEXED || q->op == TCCIR_OP_STORE_INDEXED) { int32_t av = irop_get_vreg(tcc_ir_op_get_accum(ir, q)); if (av >= 0 && TCCIR_DECODE_VREG_TYPE(av) != 0) { @@ -562,12 +874,16 @@ static void compute_stack_layout(TCCIRState *ir, int func_var) li->allocation.offset == 0) continue; /* vreg is register-only; stack slot unused */ if (li->allocation.offset != 0) { - int off = li->allocation.offset + ((int)o->u.imm32 - li->original_offset); + int off = li->allocation.offset + (irop_get_stack_offset(*o) - li->original_offset); if (off < min_op_offset) min_op_offset = off; continue; } } + /* Without a slot the operand's offset is only the frontend's + * creation-time watermark: machine_op_from_ir never addresses it + * (it uses the allocation), so it must not size the frame. */ + continue; } int off = (int)irop_get_stack_offset(*o); if (off < min_op_offset) @@ -632,12 +948,13 @@ static void compute_stack_layout(TCCIRState *ir, int func_var) li->allocation.offset == 0) continue; /* register-only vreg; stack slot unused */ if (li->allocation.offset != 0) { - int off = li->allocation.offset + ((int)o->u.imm32 - li->original_offset); + int off = li->allocation.offset + (irop_get_stack_offset(*o) - li->original_offset); if (off < post_min_op_offset) post_min_op_offset = off; continue; } } + continue; /* a frontend watermark, never addressed (see above) */ } int off = (int)irop_get_stack_offset(*o); if (off < post_min_op_offset) @@ -670,6 +987,7 @@ static void compute_stack_layout(TCCIRState *ir, int func_var) tcc_ir_build_stack_layout(ir); } + /* ================================================================== */ /* Compile nested functions and update symbol info */ /* ================================================================== */ @@ -807,7 +1125,7 @@ static void run_post_alloc_passes(TCCIRState *ir, Sym *sym, /* Re-check leaf status after all optimizations */ if (!ir->leaffunc) { - int still_has_call = 0; + int still_has_call = ir_asm_clobbers_lr(ir); for (int i = 0; i < ir->next_instruction_index; ++i) { int op = ir->compact_instructions[i].op; @@ -828,7 +1146,8 @@ static void run_post_alloc_passes(TCCIRState *ir, Sym *sym, * f2d/d2f conversion calls around a math libcall — which the early * analysis could not yet see. The frame/stack guards in codegen still * veto ineligible functions afterwards. */ - if (!ir->tail_call_only && !ir->has_static_chain && sym && sym->type.ref && + if (!ir->tail_call_only && !ir->has_static_chain && !ir->push_arg_regs && !ir_asm_clobbers_lr(ir) && + sym && sym->type.ref && sym->type.ref->f.func_type != FUNC_ELLIPSIS) { int call_count = 0, call_idx = -1, has_complex_fp = 0; @@ -921,6 +1240,13 @@ void tcc_ir_backend_regalloc_pipeline(TCCIRState *ir, Sym *sym, int func_var, tcc_ir_opt_zero_half64(ir); tcc_pass_timing_end(&ra_pt, -1); + /* After zero_half64, which clears the side table both passes write to, and + * under the same "nothing below rewrites operands" requirement. */ + tcc_pass_timing_begin(&ra_pt, "ra:cmp_hi_only"); + if (tcc_state->optimize > 0 && !tcc_ir_opt_pass_disabled("cmp_hi_only")) + tcc_ir_opt_cmp_hi_only(ir); + tcc_pass_timing_end(&ra_pt, -1); + tcc_pass_timing_begin(&ra_pt, "ra:stack_layout"); compute_stack_layout(ir, func_var); tcc_pass_timing_end(&ra_pt, -1); diff --git a/source/backend/generators/regalloc.h b/source/backend/generators/regalloc.h index 805771e28..f411b5293 100644 --- a/source/backend/generators/regalloc.h +++ b/source/backend/generators/regalloc.h @@ -26,6 +26,7 @@ * ir->leaffunc and ir->tail_call_only. Backend property consumed by * prologue/epilogue generation. */ void tcc_ir_backend_analyze_leaf_and_tail_calls(TCCIRState *ir, int func_var); +void tcc_ir_backend_fold_pure_forward(TCCIRState *ir, Sym *sym); /* Run the full register-allocation + codegen-prep tail: RA setup, SSA * regalloc, post-RA micro-opts, jump threading, register coalescing, final diff --git a/source/driver/Makefile b/source/driver/Makefile index 5f389420a..d50842eb9 100644 --- a/source/driver/Makefile +++ b/source/driver/Makefile @@ -19,7 +19,7 @@ DRIVER_SRC = \ DRIVER_HDRS = \ source/driver/libtcc.h -DRIVER_OBJ = $(patsubst %.c,$(X)%.o,$(DRIVER_SRC)) +DRIVER_OBJ = $(call lib-objs,driver,$(DRIVER_SRC)) DRIVER_MAIN_OBJ = $(patsubst %.c,$(X)%.o,$(DRIVER_MAIN_SRC)) DRIVER_LIB = $(X)source/driver/libtccdriver.a diff --git a/source/driver/libtcc.c b/source/driver/libtcc.c index d27a3683e..99ba31dc6 100644 --- a/source/driver/libtcc.c +++ b/source/driver/libtcc.c @@ -961,6 +961,10 @@ LIBTCCAPI TCCState *tcc_new(void) /* SB-relative GOT addressing is on by default for YASOS; disable per build * with -mno-sb-relative-got when a module's GOT exceeds 4096 bytes. */ s->sb_relative_got = 1; + /* Calls to undefined functions that no library in the library paths exports + * keep R9 (no reload after the call); -fno-module-local-calls restores the + * reload after every call out of the translation unit. */ + s->module_local_calls = 1; #else s->text_and_data_separation = 0; #endif @@ -989,6 +993,10 @@ LIBTCCAPI void tcc_delete(TCCState *s1) /* free lazy object files (Phase 2 GC) */ tcc_free_lazy_objfiles(s1); + /* free the -fmodule-local-calls import set (its libtcc1 index lives in the + archive cache freed next) */ + tcc_yaff_import_set_free(s1); + /* free cached archive symbol tables */ tcc_archive_cache_free(s1); @@ -1080,6 +1088,10 @@ LIBTCCAPI int tcc_set_output_type(TCCState *s, int output_type) /* add sections */ tccelf_new(s); + /* Objects need the library paths too: -fmodule-local-calls reads the + * export tables of the shared libraries found there while compiling. */ + tcc_add_library_path(s, CONFIG_TCC_LIBPATHS); + if (output_type == TCC_OUTPUT_OBJ) { /* always elf for objects */ @@ -1087,8 +1099,6 @@ LIBTCCAPI int tcc_set_output_type(TCCState *s, int output_type) return 0; } - tcc_add_library_path(s, CONFIG_TCC_LIBPATHS); - /* paths for crt objects */ tcc_split_path(s, &s->crt_paths, &s->nb_crt_paths, CONFIG_TCC_CRTPREFIX); if (output_type != TCC_OUTPUT_MEMORY && !s->nodefaultlibs && !s->nostdlib) @@ -1725,6 +1735,7 @@ enum TCC_OPTION_sb_relative_got, TCC_OPTION_no_sb_relative_got, TCC_OPTION_fpic, + TCC_OPTION_fno_pic, TCC_OPTION_fpie, TCC_OPTION_no_pie, TCC_OPTION_T, @@ -1777,6 +1788,7 @@ static const TCCOption tcc_options[] = { {"O", TCC_OPTION_O, TCC_OPTION_HAS_ARG | TCC_OPTION_NOSEP}, {"fpie", TCC_OPTION_fpie, 0}, {"fpic", TCC_OPTION_fpic, 0}, + {"fno-pic", TCC_OPTION_fno_pic, 0}, {"no-pie", TCC_OPTION_no_pie, 0}, #if defined(TCC_TARGET_ARM) || defined(TCC_TARGET_ARM_THUMB) {"mfloat-abi=", TCC_OPTION_mfloat_abi, TCC_OPTION_HAS_ARG | TCC_OPTION_NOSEP}, @@ -1886,7 +1898,10 @@ static const FlagDef options_f[] = {{offsetof(TCCState, char_is_unsigned), 0, "u {offsetof(TCCState, opt_float_narrow), 0, "float-narrow"}, {offsetof(TCCState, opt_inline_functions), 0, "inline-functions"}, {offsetof(TCCState, opt_inline_small), 0, "inline-small-functions"}, + {offsetof(TCCState, opt_inline_called_once), 0, "inline-functions-called-once"}, + {offsetof(TCCState, opt_drop_unused_statics), 0, "drop-unused-statics"}, {offsetof(TCCState, instrument_functions), 0, "instrument-functions"}, + {offsetof(TCCState, module_local_calls), 0, "module-local-calls"}, {0, 0, NULL}}; static const FlagDef options_m[] = {{offsetof(TCCState, ms_bitfields), 0, "ms-bitfields"}, {0, 0, NULL}}; @@ -2252,6 +2267,15 @@ PUB_FUNC int tcc_parse_args(TCCState *s, int *pargc, char ***pargv, int optind) case TCC_OPTION_no_pie: s->no_pie = 1; break; + case TCC_OPTION_fno_pic: + /* Position-dependent code with absolute addressing, for code that is + * linked at a fixed address (a kernel): no GOT, no r9 data base, and r9 + * becomes an ordinary allocatable register. */ + s->pic = 0; + s->text_and_data_separation = 0; + s->sb_relative_got = 0; + s->share_rodata = 0; + break; #if defined(TCC_TARGET_ARM) || defined(TCC_TARGET_ARM_THUMB) case TCC_OPTION_mfloat_abi: if (!strcmp(optarg, "soft")) @@ -2454,18 +2478,23 @@ PUB_FUNC int tcc_parse_args(TCCState *s, int *pargc, char ***pargv, int optind) * -O0 -- and because -O comes from a plain option scan, a command line * of `-O2 ... -Os` (toybox's) reset an explicit -O2 back to nothing. * - * They alias -O2. There is no size tier yet: nothing in the pipeline - * trades speed for size, so the honest meaning of "optimize for size" is - * "everything we have", and the alias is where the size-preferring - * decisions get hung when they arrive. Note that on today's numbers -O2 - * is not the smallest output -- over tests/ir_tests -O1 is 511,724 bytes - * of .text against -O2's 540,146, because -O2 spends the difference on - * unrolling and full inlining -- so the first real -Os work is teaching - * this tier to skip those two. */ + * They run the -O2 pipeline and set optimize_size, which the choices + * that trade code size for speed read: no word-alignment pads on branch + * targets, and a block copy past the largest copy stub stays a call. + * -Oz is -Os. Inlining and unrolling are NOT cut back: on Zig's C both + * shrink the code (a lower -finline-limit, -fno-loop-unroll and -O1 are + * all larger), and gcc's own -O2 -> -Os gap there is mostly inlining + * tcc already does not do. */ if (optarg[0] == 's' || optarg[0] == 'z') + { s->optimize = 2; + s->optimize_size = optarg[0] == 'z' ? 2 : 1; + } else + { s->optimize = atoi(optarg); + s->optimize_size = 0; + } /* -O1: scalar cleanup, cheap ARM addressing-mode fusion, constant/range * folding, loop rotation, and light inlining of explicitly-inline / tiny * functions — the same class of work GCC does at -O1. The heavy tier @@ -2492,6 +2521,8 @@ PUB_FUNC int tcc_parse_args(TCCState *s, int *pargc, char ***pargv, int optind) s->opt_float_narrow = 1; /* Narrow double math to float when safe */ s->opt_jump_threading = 1; /* Jump threading optimization */ s->opt_inline_small = 1; /* Inline tiny static/inline functions (≤30 words) */ + s->opt_inline_called_once = 1; /* Inline static functions called from one place, bodies deferred to TU end */ + s->opt_drop_unused_statics = 1; /* Emit only the statics something live refers to */ if (!s->opt_inline_limit_user) s->opt_inline_limit = 30; } diff --git a/source/driver/tcc.c b/source/driver/tcc.c index 57c071c55..caa7c1dbf 100644 --- a/source/driver/tcc.c +++ b/source/driver/tcc.c @@ -89,6 +89,8 @@ static const char help[] = "Tiny C Compiler " TCC_VERSION " - Copyright (C) 2001 " -mfpu=type specify FPU type for ARM hard float\n" " -mno-pic-data-is-text-relative separate .text and .data with base " "register addressing instead of PC-relative\n" + " -fno-pic position-dependent code at a fixed address " + "(kernels): absolute data, no GOT, r9 allocatable\n" #endif "Tools:\n" " create library : tcc -ar [crstvx] lib [files]\n" diff --git a/source/frontend/Makefile b/source/frontend/Makefile index 1ec040da1..a0ad2edef 100644 --- a/source/frontend/Makefile +++ b/source/frontend/Makefile @@ -21,7 +21,7 @@ FRONTEND_HDRS = \ # $(GEN_SRC) comes from source/frontend/gen/Makefile, included first by the # top-level Makefile. -FRONTEND_OBJ = $(patsubst %.c,$(X)%.o,$(FRONTEND_SRC) $(GEN_SRC)) +FRONTEND_OBJ = $(call lib-objs,frontend,$(FRONTEND_SRC) $(GEN_SRC)) FRONTEND_LIB = $(X)source/frontend/libfrontend.a $(X)source/frontend/%.o: source/frontend/%.c $(FRONTEND_HDRS) $(TCC_CORE_DEPS) diff --git a/source/frontend/gen/builtin/call.c b/source/frontend/gen/builtin/call.c index 65f969d4c..83da2f91f 100644 --- a/source/frontend/gen/builtin/call.c +++ b/source/frontend/gen/builtin/call.c @@ -94,6 +94,26 @@ int redirect_call_to_tcc_helper(SValue *saved_args, int nargs, const char *helpe * function, those locals only exist on the stack when actually processing * a call expression, not during every recursive unary() invocation. * This saves ~3000+ bytes per unary() stack frame. */ +/* Whether SYM's body is already being expanded further out: expanding it + * again is recursion, which token replay would unroll without end. */ +static int inline_expansion_in_progress(Sym *sym) +{ + int depth = tcc_state->inline_expansion_depth < INLINE_NEST_MAX ? tcc_state->inline_expansion_depth : INLINE_NEST_MAX; + for (int i = 0; i < depth; i++) + if (tcc_state->inline_expansion_syms[i] == sym) + return 1; + return 0; +} + +/* Whether the call whose callee sits at vtop is a whole expression statement, + * so nothing reads its value. `;` after the call is not enough: `x = f();` and + * `int r = f();` end the same way. The callee must sit directly on the + * statement's base; an assignment's lvalue would be between them. */ +static int call_result_discarded(void) +{ + return tok == ';' && discarded_call_vtop != NULL && vtop - 1 == discarded_call_vtop; +} + void unary_funcall(void) { int n, t, r, size, align; @@ -137,24 +157,32 @@ void unary_funcall(void) * correct chain pointer from our own incoming chain — emitting SET_CHAIN * would clobber it with R7 which may be an unrelated frame pointer. */ int set_chain_ir_idx = -1; + /* The nested function this call targets, if any. Its nb_real_calls counts + * the call sites that actually emitted a call: incremented here, decremented + * again below wherever the body is inlined instead. gen_function needs the + * exact count — "the callee is marked auto_inline" is not the same thing as + * "every call site inlined it", and clearing a capture's addrtaken on that + * guess leaves the surviving call reading chain offset 0. */ + NestedFunc *call_nf = NULL; if (tcc_state->ir && call_func_sym && call_func_sym->a.nested_func) { int emit_set_chain = 1; + for (int ni = 0; ni < tcc_state->nb_nested_funcs; ni++) + { + if (tcc_state->nested_funcs[ni].sym == call_func_sym) + { + call_nf = &tcc_state->nested_funcs[ni]; + break; + } + } + if (call_nf) + call_nf->nb_real_calls++; if (tcc_state->current_nested_func) { /* Caller is a nested function. Determine if callee is our child * (defined inside our body) or a sibling (defined in the same parent * scope). Only emit SET_CHAIN for child calls. */ - NestedFunc *callee_nf = NULL; - for (int ni = 0; ni < tcc_state->nb_nested_funcs; ni++) - { - if (tcc_state->nested_funcs[ni].sym == call_func_sym) - { - callee_nf = &tcc_state->nested_funcs[ni]; - break; - } - } - if (callee_nf && callee_nf->parent_nf != tcc_state->current_nested_func) + if (call_nf && call_nf->parent_nf != tcc_state->current_nested_func) { /* Sibling call: R10 already has the correct parent FP */ emit_set_chain = 0; @@ -260,7 +288,8 @@ void unary_funcall(void) } else { - loc = (loc - size) & -align; + loc = tcc_ir_frame_alloc_ret_temp(loc, size, -align); + tcc_ir_frame_note_type(loc, &s->type); sret_loc = loc; } ret.type = s->type; @@ -321,6 +350,7 @@ void unary_funcall(void) LOG_CODEGEN("FUNCPARAMVAL push: site=sret_param0 call_id=%d param_idx=%d vtop_r=0x%x vtop_vr=%d", call_id, TCCIR_DECODE_PARAM_IDX((uint32_t)num.c.i), vtop->r, vtop->vr); tcc_ir_put(tcc_state->ir, TCCIR_OP_FUNCPARAMVAL, vtop, &num, NULL); + tcc_ir_frame_note_sret_call(call_id, size); } vtop--; nb_args++; @@ -365,7 +395,9 @@ void unary_funcall(void) int can_inline_builtin = 0; int can_inline_eval = 0; int can_optimize_string_builtin = 0; + int can_optimize_fputs_family = 0; const char *func_name = NULL; + int inl_first_obj = 0; /* frame objects of an inline expansion start here (tcc_ir_frame_scope_end) */ /* Check if we have a named function that might be foldable */ if (call_func_sym && call_func_sym->v >= TOK_IDENT) @@ -392,6 +424,15 @@ void unary_funcall(void) } can_optimize_string_builtin = resolve_str_builtin_id(call_func_sym->v, func_name) != STRBI_UNKNOWN; + + /* The fputs lowering below needs the arguments saved. can_try_fold's + * first-letter screen passes fputs and fputs_unlocked but not + * __builtin_fputs_unlocked, which is a lazily declared prototype that + * keeps its own name -- so that one was never lowered, and with a + * user-supplied fputs that aborts inside main (gcc_execute + * builtins/fputs) the call reached it. */ + can_optimize_fputs_family = func_name && (strcmp(func_name, "fputs") == 0 || strcmp(func_name, "fputs_unlocked") == 0 || + strcmp(func_name, "__builtin_fputs_unlocked") == 0); } /* Check if the callee is a small inline function we might evaluate. @@ -537,7 +578,7 @@ void unary_funcall(void) * 7. Replay only the fixed arg tokens for normal call parsing */ if (call_func_sym && call_func_sym->type.ref && call_func_sym->type.ref->f.func_va_arg_pack && - (call_func_sym->type.t & VT_INLINE)) + (call_func_sym->type.t & VT_INLINE) && !tcc_state->check_only) { /* Find the InlineFunc for this symbol */ struct InlineFunc *orig_fn = NULL; @@ -782,10 +823,17 @@ void unary_funcall(void) /* Register clone as inline function */ struct InlineFunc *clone_fn; - clone_fn = tcc_malloc(sizeof *clone_fn + strlen(orig_fn->filename)); + clone_fn = tcc_mallocz(sizeof *clone_fn + strlen(orig_fn->filename)); strcpy(clone_fn->filename, orig_fn->filename); clone_fn->sym = clone_sym; clone_fn->func_str = clone_body; + /* the clone replays the original's body: same #pragma pack state */ + if (orig_fn->pack) + { + size_t pack_size = sizeof(int) * (orig_fn->pack[0] + 2); + clone_fn->pack = tcc_malloc(pack_size); + memcpy(clone_fn->pack, orig_fn->pack, pack_size); + } dynarray_add(&tcc_state->inline_fns, &tcc_state->nb_inline_fns, clone_fn); /* Mark the clone as used so gen_inline_functions compiles it */ @@ -844,24 +892,13 @@ void unary_funcall(void) * This must happen BEFORE the double-complex materialization below, * which converts VT_CONST to VT_LOCAL. */ if ((can_try_fold || can_inline_builtin || can_inline_eval || can_optimize_printf_family || - can_optimize_string_builtin) && + can_optimize_string_builtin || can_optimize_fputs_family) && saved_arg_count < saved_args_cap && !NOEVAL_WANTED) { saved_args[saved_arg_count] = *vtop; - if (aapcs_last_const_init) - { - saved_args_cid[saved_arg_count] = tcc_malloc(aapcs_last_const_init_size); - memcpy(saved_args_cid[saved_arg_count], aapcs_last_const_init, aapcs_last_const_init_size); - saved_args_cid_size[saved_arg_count] = aapcs_last_const_init_size; - aapcs_last_const_init = NULL; - } saved_arg_count++; saved_scratch->saved_arg_count = saved_arg_count; } - else - { - aapcs_last_const_init = NULL; - } /* Materialize constant complex double/ldouble to a temp local. * These are 128-bit values that cannot be represented as a single @@ -1062,7 +1099,8 @@ void unary_funcall(void) * int g(void) { return f(1); } // returns 1 at -O1 */ int inline_evaled = 0; - if (!folded && !inlined && call_func_sym && saved_arg_count == nb_real_args && !NOEVAL_WANTED) + if (!folded && !inlined && call_func_sym && saved_arg_count == nb_real_args && !NOEVAL_WANTED && + !tcc_state->check_only) { if (try_inline_const_eval(call_func_sym, saved_args, saved_arg_count)) { @@ -1072,6 +1110,8 @@ void unary_funcall(void) tcc_state->ir->next_instruction_index = ir_idx_before_args; if (set_chain_ir_idx >= 0 && tcc_state->ir) tcc_state->ir->compact_instructions[set_chain_ir_idx].op = TCCIR_OP_NOP; + if (call_nf && call_nf->nb_real_calls > 0) + call_nf->nb_real_calls--; inline_evaled = 1; } } @@ -1143,10 +1183,10 @@ void unary_funcall(void) * printf/fprintf("%c", ch) → putchar/fputc * printf("%s\n", str) → puts(str) * Also handles __printf_chk/__fprintf_chk (extra flag argument). - * Only optimize in void context (next token is ';'). */ + * Only optimize when the call is a whole expression statement. */ int printf_family_optimized = 0; if (!folded && !inlined && !inline_evaled && !sprintf_family_optimized && can_optimize_printf_family && - saved_arg_count == nb_real_args && nb_args >= pf_min_args && !nocode_wanted && tok == ';') + saved_arg_count == nb_real_args && nb_args >= pf_min_args && !nocode_wanted && call_result_discarded()) { int fmt_len = 0; const char *fmt = try_get_constant_string(&saved_args[pf_fmt_idx], &fmt_len); @@ -1410,9 +1450,8 @@ void unary_funcall(void) * or fputc, but the generic lowering is sufficient and correct here. */ int fputs_family_optimized = 0; if (!folded && !inlined && !inline_evaled && !sprintf_family_optimized && !printf_family_optimized && func_name && - saved_arg_count == nb_real_args && nb_args >= 2 && !nocode_wanted && tok == ';' && - (strcmp(func_name, "fputs") == 0 || strcmp(func_name, "fputs_unlocked") == 0 || - strcmp(func_name, "__builtin_fputs_unlocked") == 0)) + saved_arg_count == nb_real_args && nb_args >= 2 && !nocode_wanted && call_result_discarded() && + can_optimize_fputs_family) { if (ir_idx_before_first_param >= 0) { @@ -1530,16 +1569,25 @@ void unary_funcall(void) * every depth but 0, so the nested attempt never starts. */ (call_func_sym != tcc_state->cur_func_sym || self_inline_ok) && /* Allow nested inlining under controlled conditions: - * - Void/struct/integer return: safe at depth < 3. - * - Pointer return: still gated to depth<1 to avoid the - * store-then-load-through-return-slot phi pattern (930725-1). - * - Depth cap prevents mutual-recursion expansion (pr22379). */ + * - Void/struct/integer return, never a function already being + * expanded (mutual recursion, pr22379), up to INLINE_NEST_MAX + * deep: zig.h's integer helpers nest five deep (saturating sub -> + * overflow sub -> truncate -> shift), under called-once bodies + * that nest as deep as zig's call chains, and a cap of 3 left the + * innermost helpers as calls in every caller. + * - Pointer returns too: the depth<1 gate once kept out the + * store-then-load-through-return-slot phi pattern of 930725-1, + * which no longer reproduces nested (test 512), and it left + * every Zig CBE accessor (`header()`, `acquire()`) called from + * `view()`/`itemPtr()` expansions as a real call. */ (!tcc_state->in_inline_expansion || call_func_sym->a.nested_func || - (tcc_state->inline_expansion_depth < 3 && + (tcc_state->inline_expansion_depth < INLINE_NEST_MAX && + !inline_expansion_in_progress(call_func_sym) && call_func_sym->type.ref && ((call_func_sym->type.ref->type.t & VT_BTYPE) == VT_VOID || (call_func_sym->type.ref->type.t & VT_BTYPE) == VT_STRUCT || + (call_func_sym->type.ref->type.t & VT_BTYPE) == VT_PTR || ((call_func_sym->type.ref->type.t & VT_BTYPE) <= VT_LLONG))))) { /* ---- Token-level inline expansion ---- @@ -1551,14 +1599,10 @@ void unary_funcall(void) int force_always_inline = 0; int has_addr_of_label = 0; int has_inline_asm = 0; - for (int fi = 0; fi < tcc_state->nb_inline_fns; fi++) - { - if (tcc_state->inline_fns[fi]->sym == call_func_sym) - { - inline_fn = tcc_state->inline_fns[fi]; - break; - } - } + /* A body parsed for its diagnostics only expands nothing: the call is + * enough to check, and an expansion could consume a body a live call + * site still owes. */ + inline_fn = tcc_state->check_only ? NULL : inline_fn_lookup(tcc_state, call_func_sym); if (inline_fn && inline_fn->func_str) inline_scan_body_features(inline_fn->func_str, &has_addr_of_label, &has_inline_asm); if (call_func_sym->type.ref && call_func_sym->type.ref->f.func_alwinl && @@ -1668,6 +1712,23 @@ void unary_funcall(void) skip_ipc_cached = 1; } + /* TCC_CALLED_ONCE_DBG: a called-once callee the site declines (budget, depth, asm...). */ + { + static int dbg = -1; + if (dbg < 0) + dbg = getenv("TCC_CALLED_ONCE_DBG") != NULL; + if (dbg && inline_fn && inline_fn->func_str && call_func_sym->type.ref && call_func_sym->type.ref->f.func_called_once && + !(tcc_state->inline_expansion_depth < 32 && inline_fn->func_str->len <= tcc_state->called_once_budget && + !has_addr_of_label && !has_inline_asm && !func_has_label_addr && !skip_ipc_cached)) + { + extern const char *funcname; + fprintf(stderr, "[ONCE] %s len=%d decline=%s budget=%d depth=%d caller=%s\n", func_name, inline_fn->func_str->len, + inline_fn->func_str->len > tcc_state->called_once_budget ? "budget" + : tcc_state->inline_expansion_depth >= 32 ? "depth" + : "site", + tcc_state->called_once_budget, tcc_state->inline_expansion_depth, funcname ? funcname : "?"); + } + } /* Auto-inline: expand small static/inline functions at -O1/-O2 when safe. * Don't auto-inline into functions that use computed gotos (&&label): * inlining increases register pressure which can force the IJMP codegen @@ -1693,8 +1754,17 @@ void unary_funcall(void) tcc_state->current_nested_func)) && /* Only inline functions whose signature is safe: scalar/pointer params * that fit in 32-bit registers, and scalar or struct return types. - * 64-bit types and struct *parameters* are not handled. */ - auto_inline_sig_ok(call_func_sym) && + * 64-bit types and struct *parameters* are not handled. A called-once + * function was already vetted by called_once_body_ok and binds any + * parameter type the expansion supports; nesting is bounded so a deep + * chain of them cannot exhaust the parser's stack, and the caller's + * growth by the called_once_budget. */ + (call_func_sym->type.ref->f.func_called_once + ? tcc_state->inline_expansion_depth < 32 && inline_fn->func_str->len <= tcc_state->called_once_budget + : auto_inline_sig_ok(call_func_sym) && + /* below the first level, within the caller's budget */ + (!tcc_state->in_inline_expansion || inline_fn->func_str->len <= TINY_INLINE_TOKENS || + inline_fn->func_str->len <= tcc_state->nested_inline_budget)) && /* Don't inline if call-site argument count doesn't match the function's * actual parameter count. This can happen when calling through a cast * to an incompatible function pointer type (e.g. ((int(*)(int))bar)(x) @@ -1725,6 +1795,17 @@ void unary_funcall(void) !inline_body_has_apply_args(inline_fn->func_str) && auto_inline_nonstatic_struct_body_ok(call_func_sym, inline_fn->func_str)) { + { + static int dbg = -1; + if (dbg < 0) + dbg = getenv("TCC_INL_DBG") != NULL; + if (dbg) + { + extern const char *funcname; + fprintf(stderr, "[INLSITE] %s into %s depth=%d len=%d\n", get_tok_str(call_func_sym->v & ~SYM_FIELD, NULL), + funcname ? funcname : "?", tcc_state->inline_expansion_depth, inline_fn->func_str->len); + } + } if (TCC_LOG_INLINE_STRUCT) fprintf(stderr, "[auto-inline] callsite: inlining %s\n", get_tok_str(call_func_sym->v & ~SYM_FIELD, NULL)); LOG_INLINE_STRUCT("[auto-inline] callsite: INLINING %s (ret_btype=%d nb_args=%d nb_implicit=%d)", @@ -1734,6 +1815,10 @@ void unary_funcall(void) force_always_inline = 1; if (call_func_sym->type.ref->f.func_inline_call_heavy) inline_fn->inline_count++; + if (call_func_sym->type.ref->f.func_called_once) + tcc_state->called_once_budget -= inline_fn->func_str->len; + else if (tcc_state->in_inline_expansion && inline_fn->func_str->len > TINY_INLINE_TOKENS) + tcc_state->nested_inline_budget -= inline_fn->func_str->len; } else if (TCC_LOG_INLINE_STRUCT) { @@ -1776,6 +1861,14 @@ void unary_funcall(void) } if (inline_fn && inline_fn->func_str && (has_addr_of_label || force_always_inline)) { + /* The frame objects this expansion allocates, its parameter copies + * first, die with it (tcc_ir_frame_scope_end). Taken here, where every + * route into the expansion meets: set only on the auto-inline route it + * stayed 0 for an always_inline or `inline` callee, and the scope end + * then cut the lifetime of every object of the caller -- GNU make's + * target_environment laid its locals over its live hash table and + * corrupted the heap. */ + inl_first_obj = tcc_state->ir ? tcc_state->ir->frame_obj_count : 0; /* --- 1. NOP out FUNCPARAMVALs for this call --- */ if (ir_idx_before_first_param >= 0) { @@ -1800,6 +1893,8 @@ void unary_funcall(void) * the inlined body accesses parent variables directly. */ if (set_chain_ir_idx >= 0 && tcc_state->ir) tcc_state->ir->compact_instructions[set_chain_ir_idx].op = TCCIR_OP_NOP; + if (call_nf && call_nf->nb_real_calls > 0) + call_nf->nb_real_calls--; /* --- 2. Create parameter locals and store arguments --- */ Sym *saved_local = local_stack; @@ -1816,7 +1911,7 @@ void unary_funcall(void) psize = 4; if (palign < 4) palign = 4; - loc = (loc - psize) & -palign; + loc = tcc_ir_frame_alloc(loc, psize, -palign); /* Push parameter symbol FIRST so it gets a vreg assigned. * Unnamed parameters (v == 0) would crash sym_push because @@ -1863,20 +1958,13 @@ void unary_funcall(void) store_dst.c.i = loc; if ((param_sym->type.t & VT_BTYPE) == VT_STRUCT && !(param_sym->type.t & VT_VECTOR)) { - int psz, pal; - psz = type_size(¶m_sym->type, &pal); - if (psz <= 16) - { - vset(&store_dst.type, store_dst.r, store_dst.c.i); - vtop->vr = store_dst.vr; - vpushv(&arg_val); - vstore(); - vtop--; - } - else - { - tcc_ir_put(tcc_state->ir, TCCIR_OP_STORE, &arg_val, NULL, &store_dst); - } + /* A struct argument of any size is the struct itself (AAPCS32 by + * value): copy it into the parameter's slot. */ + vset(&store_dst.type, store_dst.r, store_dst.c.i); + vtop->vr = store_dst.vr; + vpushv(&arg_val); + vstore(); + vtop--; } else { @@ -1893,10 +1981,11 @@ void unary_funcall(void) arg_data = saved_args_cid[pi]; if (arg_data) { - psym->const_init_data = tcc_malloc(psize); - memcpy(psym->const_init_data, arg_data, psize); - psym->const_init_size = psize; - psym->const_init_valid = 1; + SymLocalFacts *f = sym_facts(psym); + f->const_init_data = tcc_malloc(psize); + memcpy(f->const_init_data, arg_data, psize); + f->const_init_size = psize; + f->const_init_valid = 1; } } } @@ -1956,7 +2045,7 @@ void unary_funcall(void) rsize = 4; if (ralign < 4) ralign = 4; - loc = (loc - rsize) & -ralign; + loc = tcc_ir_frame_alloc(loc, rsize, -ralign); inline_ret_loc = loc; } @@ -2000,6 +2089,8 @@ void unary_funcall(void) tcc_state->inline_return_loc = inline_ret_loc; tcc_state->inline_return_vr = inline_ret_vr; tcc_state->inline_return_redirected = 0; + if (tcc_state->inline_expansion_depth < INLINE_NEST_MAX) + tcc_state->inline_expansion_syms[tcc_state->inline_expansion_depth] = call_func_sym; tcc_state->inline_expansion_depth++; root_scope = cur_scope; @@ -2015,10 +2106,13 @@ void unary_funcall(void) inline_ts->data.str = tok_str_buf(inline_fn->func_str); inline_ts->allocated_len = 1; inline_ts->len = inline_fn->func_str->len; + int saved_pack[PACK_STACK_SIZE + 1]; + pp_pack_enter(tcc_state, inline_fn->pack, saved_pack); begin_macro(inline_ts, 2); next(); block(0); end_macro(); + pp_pack_leave(tcc_state, saved_pack); inline_restore_label_bindings(inline_label_tokens, saved_inline_labels, nb_inline_label_tokens); tok = saved_tok; @@ -2031,6 +2125,11 @@ void unary_funcall(void) /* Read back inline_return_loc: the struct return handler may have * redirected it to point at the source local (skipping a memmove). */ inline_ret_loc = tcc_state->inline_return_loc; + /* The callee's objects die here; the local its return was redirected + * into (inline_return_redirected) is the caller's value and lives on. */ + if (tcc_state->ir) + tcc_ir_frame_scope_end(tcc_state->ir, inl_first_obj, tcc_state->ir->next_instruction_index, + tcc_state->inline_return_redirected ? inline_ret_loc : 1); tcc_state->in_inline_expansion = saved_in_inline_expansion; tcc_state->inline_return_loc = saved_inline_return_loc; tcc_state->inline_return_vr = saved_inline_return_vr; @@ -2214,7 +2313,8 @@ void unary_funcall(void) size = (size + regsize - 1) & -regsize; if (ret_align > align) align = ret_align; - loc = (loc - size) & -align; + loc = tcc_ir_frame_alloc_ret_temp(loc, size, -align); + tcc_ir_frame_note_type(loc, &s->type); addr = loc; offset = 0; for (;;) diff --git a/source/frontend/gen/builtin/misc.c b/source/frontend/gen/builtin/misc.c index c1b74f588..11e9d6ca1 100644 --- a/source/frontend/gen/builtin/misc.c +++ b/source/frontend/gen/builtin/misc.c @@ -96,6 +96,28 @@ void gen_bitop1(TccIrOp op) vtop->c.i = 0; /* Clear c.i to avoid corrupting later operations */ } +/* Push this function's return address, __builtin_return_address(0), as a + * void *. The prologue must then save LR even in a leaf; the value is read + * back from that slot (TCCIR_OP_RETURN_ADDRESS), with no frame pointer. */ +ST_FUNC void vpush_return_address(void) +{ + CType type; + SValue dest; + type.t = VT_VOID; + type.ref = NULL; + mk_pointer(&type); + tcc_state->force_lr_save = 1; + svalue_init(&dest); + dest.type = type; + dest.r = 0; + dest.vr = tcc_ir_get_vreg_temp(tcc_state->ir); + tcc_ir_put(tcc_state->ir, TCCIR_OP_RETURN_ADDRESS, NULL, NULL, &dest); + vpush(&type); + vtop->vr = dest.vr; + vtop->r = 0; + vtop->c.i = 0; +} + /* Emit an IR function call to a library helper for a builtin. * Arguments are already on the vstack (1 or 2 args). * func_tok: TOK_xxx or tok_alloc_const("name") for the target function diff --git a/source/frontend/gen/builtin/overflow.c b/source/frontend/gen/builtin/overflow.c index 846239b00..a0f07795d 100644 --- a/source/frontend/gen/builtin/overflow.c +++ b/source/frontend/gen/builtin/overflow.c @@ -102,10 +102,33 @@ void __attribute__((noinline)) unary_builtin_overflow(void) break; } - if (res_bt == VT_LLONG) + /* A 32-bit add or subtract whose operands both convert to the result type + * unchanged takes the same arithmetic checks as the 64-bit case, in 32 + * bits: `r < a` and the sign of `(a ^ r) & (b ^ r)` instead of a 64-bit + * sum truncated and compared back (36-46 bytes per check; Zig's + * @addWithOverflow is one of these on every u32/i32/usize). */ + int fits32 = 0; + if (res_bt == VT_INT && arith_tok != '*') + { + fits32 = 1; + for (int k = -2; k <= -1; k++) + { + CType *ot = &vtop[k].type; + int obt = ot->t & VT_BTYPE, ouns = (ot->t & VT_UNSIGNED) != 0; + if (obt == VT_INT) + fits32 &= ouns == is_unsigned; + else if (obt == VT_SHORT || obt == VT_BYTE || obt == VT_BOOL) + fits32 &= ouns || !is_unsigned; + else + fits32 = 0; + } + } + + if (res_bt == VT_LLONG || fits32) { /* 64-bit result: can't widen further on 32-bit target. - * Use arithmetic overflow checks instead. */ + * Use arithmetic overflow checks instead (and a 32-bit add or + * subtract, fits32, the same way at its own width). */ /* Stack: a b res_ptr → res_ptr a b */ vrott(3); @@ -124,7 +147,9 @@ void __attribute__((noinline)) unary_builtin_overflow(void) CType ll_type; ll_type.ref = NULL; - if (signed_to_unsigned_mul) + if (fits32) + ll_type.t = is_unsigned ? (VT_INT | VT_UNSIGNED) : VT_INT; + else if (signed_to_unsigned_mul) ll_type.t = VT_LLONG; /* signed — preserve sign for overflow check */ else ll_type.t = is_unsigned ? (VT_LLONG | VT_UNSIGNED) : VT_LLONG; @@ -133,6 +158,17 @@ void __attribute__((noinline)) unary_builtin_overflow(void) gen_cast(&ll_type); /* cast a */ vswap(); /* Stack: res_ptr a b */ + /* The checks below read a and b again AFTER the store through res_ptr, + * which may point at either (`__builtin_add_overflow(a, b, &a)`, gcc + * PR85095/PR108789). Widening to long long loaded them; a cast to the + * same 32-bit type does not, so load them here. */ + if (fits32) + { + gv(RC_INT); + vswap(); + gv(RC_INT); + vswap(); + } /* Save copies of a and b for the overflow check. */ vpushv(vtop); /* Stack: res_ptr a b b2 */ @@ -433,19 +469,41 @@ void __attribute__((noinline)) unary_builtin_overflow(void) break; } - /* 32-bit or smaller result: widen to long long, compute, truncate, compare */ + /* 32-bit or smaller result: widen, compute, truncate, compare. When + * the result and both operands are 16 bits or narrower the exact result + * fits a 32-bit int -- an unsigned one for u16 * u16 -- so there is no + * need for long long. */ + int narrow_all = res_bt == VT_SHORT || res_bt == VT_BYTE || res_bt == VT_BOOL; + int ops_unsigned = 1; + for (int k = -2; k <= -1; k++) + { + int obt = vtop[k].type.t & VT_BTYPE; + narrow_all &= obt == VT_SHORT || obt == VT_BYTE || obt == VT_BOOL; + ops_unsigned &= (vtop[k].type.t & VT_UNSIGNED) != 0; + } vrott(3); /* → res_ptr a b */ - /* Widen both operands to (unsigned) long long */ + /* Widen both operands to (unsigned) long long, or to int */ CType wide_type; wide_type.ref = NULL; - wide_type.t = is_unsigned ? (VT_LLONG | VT_UNSIGNED) : VT_LLONG; + if (narrow_all) + wide_type.t = (arith_tok == '*' && ops_unsigned) ? (VT_INT | VT_UNSIGNED) : VT_INT; + else + wide_type.t = is_unsigned ? (VT_LLONG | VT_UNSIGNED) : VT_LLONG; gen_cast(&wide_type); /* cast b */ vswap(); gen_cast(&wide_type); /* cast a */ vswap(); /* Stack: res_ptr a_wide b_wide */ + if (narrow_all) + { + /* As in the fits32 path: loaded before the store through res_ptr. */ + gv(RC_INT); + vswap(); + gv(RC_INT); + vswap(); + } gen_op(arith_tok); /* Stack: res_ptr wide_result */ diff --git a/source/frontend/gen/builtin/simd.c b/source/frontend/gen/builtin/simd.c index 5cd36469f..db50d2f07 100644 --- a/source/frontend/gen/builtin/simd.c +++ b/source/frontend/gen/builtin/simd.c @@ -363,10 +363,11 @@ void __attribute__((noinline)) unary_builtin_shuffle(void) Sym *s; for (s = local_stack; s; s = s->prev) { - if (s->const_init_data && s->const_init_valid && (int)s->c == mask_addr && - s->const_init_size >= elem_count * mask_elem_size) + const SymLocalFacts *f = SYM_FACTS(s); + if (f->const_init_data && f->const_init_valid && (int)s->c == mask_addr && + f->const_init_size >= elem_count * mask_elem_size) { - mask_const = s->const_init_data; + mask_const = f->const_init_data; break; } } diff --git a/source/frontend/gen/builtin/string.c b/source/frontend/gen/builtin/string.c index c736c8e5e..daec58123 100644 --- a/source/frontend/gen/builtin/string.c +++ b/source/frontend/gen/builtin/string.c @@ -41,8 +41,8 @@ const char *try_get_constant_string(SValue *sv, int *out_len) if ((sv->r & (VT_VALMASK | VT_SYM | VT_LVAL)) != (VT_CONST | VT_SYM) && (sv->r & (VT_VALMASK | VT_SYM | VT_LVAL)) != (VT_CONST | VT_SYM | VT_LVAL)) return NULL; - if (!sv->sym) - return NULL; + if (!sv->sym || sv->sym->a.tentative) + return NULL; /* a tentative definition's bytes are not its value yet */ esym = elfsym(sv->sym); if (!esym) diff --git a/source/frontend/gen/core/state.c b/source/frontend/gen/core/state.c index 6e990fbb4..2e90745d4 100644 --- a/source/frontend/gen/core/state.c +++ b/source/frontend/gen/core/state.c @@ -42,8 +42,6 @@ ST_DATA Sym *global_stack; ST_DATA Sym *local_stack; ST_DATA Sym *define_stack; -unsigned char *aapcs_last_const_init; -int aapcs_last_const_init_size; ST_DATA Sym *global_label_stack; ST_DATA Sym *local_label_stack; @@ -109,6 +107,7 @@ ST_DATA SValue *vtop; ST_DATA SValue _vstack[1 + VSTACK_SIZE]; ST_DATA int nocode_wanted; /* no code generation wanted */ +ST_DATA SValue *discarded_call_vtop; ST_DATA int global_expr; /* true if compound literals must be allocated globally (used during initializers parsing */ @@ -117,6 +116,7 @@ ST_DATA int func_var; /* true if current function is variadic (used by return instruction) */ ST_DATA int func_vc; ST_DATA int func_ind; +ST_DATA int func_align_pad; ST_DATA int func_has_label_addr; ST_DATA const char *funcname; ST_DATA CType int_type, func_old_type, func_old_void_type, func_old_char_pointer_type, func_old_void_pointer_type, @@ -130,53 +130,6 @@ struct switch_t *cur_switch; /* current switch */ struct temp_local_variable arr_temp_local_vars[MAX_TEMP_LOCAL_VARIABLE_NUMBER]; int nb_temp_local_vars; -/* Reusable stack slots for by-value struct arguments passed to variadic - * functions (the invisible-copy the AAPCS requires for structs > 16 bytes). - * Unlike get_temp_local_var()'s vstack-based tracking, these slots must stay - * reserved until the whole call is emitted — each argument is lowered to a - * FUNCPARAMVAL and popped off the vstack before the next argument is built, - * so the vstack can no longer witness that the slot is in use. - * - * Instead a busy bitmask tracks live slots, and block() saves/restores it - * around every statement. Two struct-arg copies that are live at the same - * time (e.g. f(a, b, a) — three copies, all read by the one call) therefore - * get distinct slots, while copies from statements that have fully completed - * are reclaimed. GNU statement-expressions used as arguments enter a nested - * block() whose save/restore leaves the enclosing call's reserved slots - * untouched, so they cannot be aliased. */ -#define MAX_ARG_STRUCT_TEMPS 64 -static struct arg_struct_temp -{ - int location; - int size; - int align; -} arg_struct_temps[MAX_ARG_STRUCT_TEMPS]; -int nb_arg_struct_temps; -uint64_t arg_struct_temp_busy; - -int get_arg_struct_temp(int size, int align) -{ - for (int i = 0; i < nb_arg_struct_temps; i++) - { - if (!(arg_struct_temp_busy & ((uint64_t)1 << i)) && - arg_struct_temps[i].size >= size && arg_struct_temps[i].align >= align) - { - arg_struct_temp_busy |= (uint64_t)1 << i; - return arg_struct_temps[i].location; - } - } - loc = (loc - size) & -align; - if (nb_arg_struct_temps < MAX_ARG_STRUCT_TEMPS) - { - int i = nb_arg_struct_temps++; - arg_struct_temps[i].location = loc; - arg_struct_temps[i].size = size; - arg_struct_temps[i].align = align; - arg_struct_temp_busy |= (uint64_t)1 << i; - } - /* Pool exhausted: a fresh never-reused slot is always correct. */ - return loc; -} struct scope *cur_scope, *loop_scope, *root_scope; @@ -247,6 +200,7 @@ ST_FUNC int tccgen_compile(TCCState *s1) pending_aliases = NULL; nb_pending_aliases = 0; nocode_wanted = DATA_ONLY_WANTED; /* no code outside of functions */ + discarded_call_vtop = NULL; debug_modes = (s1->do_debug ? 1 : 0) | s1->test_coverage << 1; tcc_debug_start(s1); @@ -274,6 +228,17 @@ ST_FUNC int tccgen_compile(TCCState *s1) parse_flags = PARSE_FLAG_PREPROCESS | PARSE_FLAG_TOK_NUM | PARSE_FLAG_TOK_STR; next(); decl(VT_CONST); + /* Drop the statics nothing live refers to, and define the objects whose + * initializers were saved (-fdrop-unused-statics). */ + prune_unused_statics(s1); + /* Every definition has been seen: settle the tentative ones before any + * deferred body addresses them. */ + finalize_tentative_definitions(s1); + s1->tu_parsed = 1; + /* Bodies saved for -finline-functions-called-once: generate them now that + * every call site in the TU is known. */ + gen_deferred_function_bodies(s1); + gen_owed_inline_functions(s1); /* End-of-TU analysis: compute call-graph reachability and the set of * static globals with no reachable readers. Must run before * gen_late_reopt_functions so newly-flagged writer functions get picked @@ -345,6 +310,10 @@ ST_FUNC int tccgen_compile(TCCState *s1) ifn->sym = NULL; } } + /* The statics dropped as unused are still checked for errors. */ + check_dropped_statics(s1); + /* What the optimizer left unreferenced goes too. */ + gc_unreferenced_statics(s1); resolve_pending_aliases(); check_vstack(); /* end of translation unit info */ @@ -374,6 +343,11 @@ ST_FUNC void tccgen_finish(TCCState *s1) past inline_restore_label_bindings(), stranding the token/label arrays it would have freed. */ inline_release_hidden_label_bindings(); + free_deferred_functions(s1); + free_dropped_statics(s1); + free_tentative_definitions(s1); + tcc_icf_reset(); + tcc_gen_machine_outline_reset(); str_lit_pool_free(); @@ -431,8 +405,6 @@ ST_FUNC void tccgen_finish(TCCState *s1) } free_inline_functions(s1); - /* Flush stashed static-function IR before sym_pop drops the Sym* keys. */ - ir_inline_stash_flush(s1); sym_pop(&global_stack, NULL, 0); sym_pop(&local_stack, NULL, 0); /* free nested functions array */ @@ -453,8 +425,6 @@ ST_FUNC void tccgen_finish(TCCState *s1) all_cleanups = NULL; pending_gotos = NULL; nb_temp_local_vars = 0; - nb_arg_struct_temps = 0; - arg_struct_temp_busy = 0; global_label_stack = NULL; local_label_stack = NULL; cur_text_section = NULL; diff --git a/source/frontend/gen/decl/attribute.c b/source/frontend/gen/decl/attribute.c index 450409fde..853c5add1 100644 --- a/source/frontend/gen/decl/attribute.c +++ b/source/frontend/gen/decl/attribute.c @@ -170,6 +170,73 @@ int parse_c23_attribute(AttributeDef *ad) return found; } +/* __has_attribute(name): 1 when parse_attribute() gives `name` its meaning, + * 0 when it would ignore it with a warning. Headers use this to choose between + * an attribute and a fallback (zig.h poisons `naked` and `noinline` outright + * when told they are missing), so the answer has to match the parser below -- + * keep the two lists in step. Calling conventions and the PE-only + * dllexport/dllimport/nodecorate are parsed but mean nothing on this target, + * so they report 0, as gcc does on ARM. */ +ST_FUNC int tcc_attribute_supported(int t) +{ + switch (t) + { + case TOK_CLEANUP1: + case TOK_CLEANUP2: + case TOK_CONSTRUCTOR1: + case TOK_CONSTRUCTOR2: + case TOK_DESTRUCTOR1: + case TOK_DESTRUCTOR2: + case TOK_ALWAYS_INLINE1: + case TOK_ALWAYS_INLINE2: + case TOK_NOINLINE1: + case TOK_NOINLINE2: + case TOK_NOIPA1: + case TOK_NOIPA2: + case TOK_SECTION1: + case TOK_SECTION2: + case TOK_ALIAS1: + case TOK_ALIAS2: + case TOK_VISIBILITY1: + case TOK_VISIBILITY2: + case TOK_ALIGNED1: + case TOK_ALIGNED2: + case TOK_PACKED1: + case TOK_PACKED2: + case TOK_WEAK1: + case TOK_WEAK2: + case TOK_NAKED1: + case TOK_NODEBUG1: + case TOK_NODEBUG2: + case TOK_USED1: + case TOK_USED2: + case TOK_UNUSED1: + case TOK_UNUSED2: + case TOK_NORETURN1: + case TOK_NORETURN2: + case TOK_NOINSTRUMENT1: + case TOK_NOINSTRUMENT2: + case TOK_PURE1: + case TOK_PURE2: + case TOK_CONST2: + case TOK_CONST3: + case TOK_VECTOR_SIZE1: + case TOK_VECTOR_SIZE2: + case TOK_MODE1: + case TOK_MODE2: + case TOK_SCALAR_STORAGE_ORDER1: + case TOK_SCALAR_STORAGE_ORDER2: + return 1; + default: + if (t >= TOK_IDENT) + { + const char *attr = get_tok_str(t, NULL); + return !strcmp(attr, "transparent_union") || !strcmp(attr, "__transparent_union__"); + } + return 0; + } +} + /* Parse __attribute__((...)) GNUC extension. */ void parse_attribute(AttributeDef *ad) { @@ -292,10 +359,12 @@ void parse_attribute(AttributeDef *ad) case TOK_NODEBUG2: ad->a.nodebug = 1; break; + case TOK_USED1: + case TOK_USED2: + ad->a.used = 1; + break; case TOK_UNUSED1: case TOK_UNUSED2: - /* currently, no need to handle it because tcc does not - track unused objects */ break; case TOK_NORETURN1: case TOK_NORETURN2: diff --git a/source/frontend/gen/decl/decl.c b/source/frontend/gen/decl/decl.c index 715f9d8c0..37eaa8474 100644 --- a/source/frontend/gen/decl/decl.c +++ b/source/frontend/gen/decl/decl.c @@ -43,6 +43,848 @@ void do_Static_assert(void) skip(';'); } +/* Save a function body for replay away from its definition, together with + * the #pragma pack state it is defined under (see pp_pack_snapshot). When + * nothing compiles the body here (!COMPILED_NOW), its own pack directives are + * applied to the live state now too: they sit at this point of the file, and + * the declarations after the body must see them. */ +static void save_function_body(TokenString **str, int **pack, int compiled_now) +{ + int captures = pp_pack_captures; + *pack = pp_pack_snapshot(tcc_state); + skip_or_save_block(str); + if (!compiled_now && pp_pack_captures != captures) + { + const int *buf = tok_str_buf(*str); + pp_apply_pack_replays(tcc_state, buf, buf + (*str)->len); + } +} + +/* A file-scope function definition's body, from its opening '{' in the + * current token stream: record a static inline as a deferred body, register an + * auto-inline candidate, or generate the code. Called at the definition + * itself, or at the end of the TU for a body gen_deferred_function_bodies + * replays. */ +static void define_global_function(Sym *sym, Section *ad_section) +{ + /* static inline functions are just recorded as a kind + of macro. Their code will be emitted at the end of + the compilation unit only if they are used */ + if (sym->type.t & VT_INLINE) + { + struct InlineFunc *fn; + fn = tcc_mallocz(sizeof *fn + strlen(file->filename)); + strcpy(fn->filename, file->filename); + fn->sym = sym; + dynarray_add(&tcc_state->inline_fns, &tcc_state->nb_inline_fns, fn); + save_function_body(&fn->func_str, &fn->pack, 0); + + /* An explicit `inline` used to *disable* the inliner: this branch + * saved the body and deferred emission, but never set + * func_auto_inline, so every call site emitted a plain call. Only + * functions the user did NOT mark inline reached the auto-inline + * path below. That is backwards, and it costs the most in exactly + * the code that relies on the idiom -- header-defined `static + * inline` helpers. lib/fp/soft is the extreme case: every helper in + * soft_common.h is `static inline`, so __aeabi_dadd made 18 calls to + * one-shift accessors that gcc folds into the body (3.76x on the + * double benchmarks). + * + * Apply the same eligibility test the auto path uses and set the + * flag so call sites replay the body. Deferred emission is a + * strict win over the auto path here: if every call site inlines + * and the address is never taken, gen_inline_functions emits no + * standalone copy at all. + * + * The gate must be token-length based. gen_function's post-opt + * revoke (IR > 8 / call-heavy) cannot help a deferred body -- it + * compiles after the call sites have already been decided. + * + * STATIC ONLY. A non-static `inline` is a C99 inline *definition* + * (and gnu89 `extern inline` another rule again): the standalone + * body is emitted on different terms, and marking it auto_inline + * makes gen_inline_functions skip emission while some call site + * still needs the symbol -- "undefined symbol 'add1_inline'". + * Static inline has internal linkage, so dropping the standalone + * copy once every site inlined is safe, and it is the idiom that + * matters here (all of soft_common.h). */ + if (sym->type.ref && (sym->type.t & VT_STATIC) && + sym->type.ref->f.func_type != FUNC_ELLIPSIS && + !sym->type.ref->f.func_alwinl && !sym->type.ref->f.func_noinline && + (tcc_state->opt_inline_functions || tcc_state->opt_inline_small) && + tcc_state->nb_vla_param_exprs == 0 && fn->func_str) + { + int sig = auto_inline_sig_ok(sym); + int thr = tcc_state->opt_inline_limit > 0 ? tcc_state->opt_inline_limit + : (tcc_state->opt_inline_functions ? 60 : 30); + /* Void-returning with 64-bit params: same narrow cap as the auto + * path (longer bodies trip an IR coalescing bug on narrowed locals). */ + if (sig == 2) + thr = 15; + if (sig && fn->func_str->len <= thr && !inline_body_has_apply_args(fn->func_str) && + !inline_body_has_unsafe_loops(fn->func_str) && !inline_body_has_static_local(fn->func_str)) + { + sym->type.ref->f.func_auto_inline = 1; + /* Body is still owed to gen_inline_functions, unlike the auto + * path below which compiles the standalone copy right here. */ + sym->type.ref->f.func_deferred_inline = 1; + } + } + + /* Scan saved token stream for __builtin_va_arg_pack() usage. + * If found, mark the function so call sites can expand it. */ + if (fn->func_str && sym->type.ref) + { + const int *p = tok_str_buf(fn->func_str); + while (*p != TOK_EOF && *p != 0) + { + if (*p == TOK_builtin_va_arg_pack) + { + sym->type.ref->f.func_va_arg_pack = 1; + break; + } + /* Skip token payload */ + int t = *p++; + switch (t) + { + case TOK_CINT: + case TOK_CUINT: + case TOK_CCHAR: + case TOK_LCHAR: + case TOK_CFLOAT: + case TOK_CFLOAT_I: + case TOK_CINT_I: + case TOK_LINENUM: + case TOK_PACK_REPLAY: +#if LONG_SIZE == 4 + case TOK_CLONG: + case TOK_CULONG: +#endif + p++; + break; + case TOK_CDOUBLE: + case TOK_CDOUBLE_I: + case TOK_CLLONG: + case TOK_CULLONG: +#if LONG_SIZE == 8 + case TOK_CLONG: + case TOK_CULONG: +#endif + p += 2; + break; + case TOK_CLDOUBLE: + case TOK_CLDOUBLE_I: +#if LDOUBLE_SIZE == 8 || defined TCC_USING_DOUBLE_FOR_LDOUBLE + p += 2; +#elif LDOUBLE_SIZE == 12 + p += 3; +#elif LDOUBLE_SIZE == 16 + p += 4; +#endif + break; + case TOK_STR: + case TOK_LSTR: + case TOK_PPNUM: + case TOK_PPSTR: + { + int sz = *p++; + p += (sz + sizeof(int) - 1) / sizeof(int); + break; + } + default: + break; + } + } + } + } + else if (sym->type.ref && sym->type.ref->f.func_type != FUNC_ELLIPSIS && !sym->type.ref->f.func_alwinl && + !sym->type.ref->f.func_noinline && + /* Only auto-inline functions whose signature is safe: scalar/pointer + * params that fit in 32-bit registers, and scalar or struct return + * types. 64-bit types and struct *parameters* are not handled. + * Returns 2 for void+llong signatures (body-length gated below). */ + auto_inline_sig_ok(sym) && (tcc_state->opt_inline_functions || tcc_state->opt_inline_small) && + /* Don't auto-inline functions with VLA parameters: the VLA size + * expressions (which may have side effects like i++) are evaluated + * during function prolog, outside the saved body token stream. + * Inlining would replay only the body, losing those side effects. */ + tcc_state->nb_vla_param_exprs == 0) + { + /* Auto-inline candidate: save the body as a token stream so call + * sites within this TU can replay it. + * + * Static functions: defer standalone compilation; suppress it entirely + * if all call sites are inlined and address not taken. We set + * VT_INLINE so gen_inline_functions handles deferred emission. + * + * Non-static functions: MUST always have a globally-visible symbol for + * other TUs. We compile the standalone definition immediately via + * token-stream replay (same mechanism as gen_inline_functions), then + * keep the token stream in inline_fns for call-site inlining within + * this TU. VT_INLINE is NOT set so ELF linkage stays global. */ + struct InlineFunc *fn; + fn = tcc_mallocz(sizeof *fn + strlen(file->filename)); + strcpy(fn->filename, file->filename); + fn->sym = sym; + fn->func_str = NULL; + save_function_body(&fn->func_str, &fn->pack, 1); + + int threshold = tcc_state->opt_inline_limit > 0 ? tcc_state->opt_inline_limit + : (tcc_state->opt_inline_functions ? 60 : 30); + int is_static = !!(sym->type.t & VT_STATIC); + int body_len = fn->func_str ? fn->func_str->len : 0; + + if (TCC_LOG_INLINE_STRUCT) + fprintf(stderr, "[auto-inline] candidate: %s static=%d len=%d threshold=%d\n", + get_tok_str(sym->v & ~SYM_FIELD, NULL), is_static, body_len, threshold); + LOG_INLINE_STRUCT("[auto-inline] candidate: %s static=%d len=%d threshold=%d ret_btype=%d", + get_tok_str(sym->v & ~SYM_FIELD, NULL), is_static, body_len, threshold, + sym->type.ref ? (sym->type.ref->type.t & VT_BTYPE) : -1); + + Section *saved_text = cur_text_section; + cur_text_section = ad_section ? ad_section : function_text_section(tcc_state, sym); + if (cur_text_section->sh_num > bss_section->sh_num) + cur_text_section->sh_flags = text_section->sh_flags; + + /* Void-returning functions with 64-bit params: only inline very + * short bodies (≤ 15 tokens) — longer bodies may trigger an IR + * coalescing bug with narrowed locals. */ + int void_llong_limit = (auto_inline_sig_ok(sym) == 2) ? 15 : threshold; + if (fn->func_str && body_len <= void_llong_limit && !inline_body_has_apply_args(fn->func_str) && + !inline_body_has_unsafe_loops(fn->func_str)) + { + if (TCC_LOG_INLINE_STRUCT) + fprintf(stderr, "[auto-inline] SMALL: registering %s as inline candidate\n", + get_tok_str(sym->v & ~SYM_FIELD, NULL)); + + /* Small enough: register as inline candidate for call-site replay. + * + * We compile the standalone definition immediately for BOTH static + * and non-static functions. We deliberately do NOT set VT_INLINE: + * - Setting VT_INLINE would defer compilation to gen_inline_functions, + * but alias attributes and other code may reference sym->c before + * gen_inline_functions runs, causing "aliased to undefined symbol" + * errors and similar failures. + * - The standalone definition is always emitted. Under + * -ffunction-sections it lands in its own .text. + * section, and the linker's gc_sections() drops it when + * every call site was inlined and the address never taken. + * (Before function-sections was real, this sentence claimed + * --gc-sections would collect it out of the monolithic + * .text, which was never true -- section-granularity GC + * cannot drop a function from a section it shares.) + * + * fn->func_str is preserved (not consumed) so call-site replay + * can still inline the body later. The compilation uses an owning + * COPY of the token stream. + * + * Save/restore tok+tokc: the replay leaves tok=TOK_EOF which would + * cause the outer decl() loop to stop parsing prematurely. */ + sym->type.ref->f.func_auto_inline = 1; + /* Set VT_INLINE for static functions so can_inline_eval=1 at call sites. + * Non-static functions must NOT get VT_INLINE — their standalone definition + * must remain globally visible for other translation units. */ + if (is_static) + sym->type.t |= VT_INLINE; + dynarray_add(&tcc_state->inline_fns, &tcc_state->nb_inline_fns, fn); + + TokenString *compile_ts = tok_str_alloc(); + if (body_len > 0) + { + int *buf = tcc_malloc(body_len * sizeof(int)); + memcpy(buf, tok_str_buf(fn->func_str), body_len * sizeof(int)); + compile_ts->data.str = buf; + compile_ts->allocated_len = body_len; + compile_ts->len = body_len; + } + int saved_outer_tok = tok; + CValue saved_outer_tokc = tokc; + if (TCC_LOG_INLINE_STRUCT) + fprintf(stderr, "[auto-inline] SMALL: compiling standalone for %s\n", + get_tok_str(sym->v & ~SYM_FIELD, NULL)); + tcc_state->had_nested_funcs = 0; + begin_macro(compile_ts, 1); /* owning: compile_ts freed on end_macro */ + next(); + gen_function(sym); + end_macro(); + tok = saved_outer_tok; + tokc = saved_outer_tokc; + /* Revoke auto-inline for functions that contain nested function + * definitions — their closure/trampoline semantics cannot be + * replicated by token-replay inline expansion. */ + if (tcc_state->had_nested_funcs) + { + sym->type.ref->f.func_auto_inline = 0; + if (is_static) + sym->type.t &= ~VT_INLINE; + } + /* gen_function's post-opt check will revoke auto_inline + * if the compiled IR exceeds 8 instructions. */ + /* If auto_inline was revoked (either by had_nested_funcs above or + * by gen_function's post-opt size/call-count check), the function + * is already compiled standalone and won't be inlined at any + * callsite. If the body is pure and within the eval-only cap, + * promote to func_eval_only_inline so try_inline_const_eval can + * still fold all-constant calls (mirrors the TOO-LARGE branch's + * retroactive promotion below). Otherwise free the token stream + * so gen_inline_functions doesn't re-emit a duplicate body. + * Skip the promotion path when nested funcs revoked: token-replay + * cannot reproduce closure/trampoline semantics. */ + if (!sym->type.ref->f.func_auto_inline) + { + /* A register-only loop body (func_const_arg_loop, set from the + * optimized IR) is exempt from the no-side-effects rule: the + * counter and accumulator updates that trip it (`i++`, `n += x`) + * are exactly what a loop is made of, and the loop is why the + * body was demoted. CTFE re-checks side effects before it + * evaluates, so only the all-constant-argument replay path gains + * these — where ssa:loop_const_sim collapses the expanded loop to + * its final value. */ + int loop_helper = sym->type.ref->f.func_const_arg_loop; + int promote_eval_only = !tcc_state->had_nested_funcs && fn->func_str && body_len <= 160 && + !inline_body_has_apply_args(fn->func_str) && + (loop_helper || !inline_body_has_side_effects(fn->func_str)); + if (promote_eval_only) + { + sym->type.ref->f.func_eval_only_inline = 1; + /* VT_INLINE already set above for static; keep it set so + * gen_inline_functions' eval-only skip branch catches us. */ + } + else if (sym->type.ref->f.func_late_reopt || + sym->type.ref->f.func_keep_tokens_for_noreturn) + { + /* Keep tokens — end-of-TU late_reopt may re-compile (either + * already flagged, or pending noreturn-propagation decision). */ + } + else if (sym->type.ref->f.tu_static_writer) + { + /* Keep tokens — function writes >=1 non-const static global; + * end-of-TU TU-wide DSE analysis may decide to re-compile via + * late_reopt to eliminate dead static stores. */ + } + else + { + if (is_static) + sym->type.t &= ~VT_INLINE; + if (fn->func_str) + { + tok_str_free(fn->func_str); + fn->func_str = NULL; + } + fn->sym = NULL; + } + } + if (TCC_LOG_INLINE_STRUCT) + fprintf(stderr, "[auto-inline] SMALL: done compiling %s sym->c=%d\n", + get_tok_str(sym->v & ~SYM_FIELD, NULL), sym->c); + } + else + { + /* Too large to inline-expand, but if the body is pure and under + * a higher eval-only cap, keep the token stream so + * try_inline_const_eval can still fold all-constant calls. + * Regular inline-expansion paths skip functions tagged + * func_eval_only_inline. */ + const int eval_only_cap = 160; + int has_apply = fn->func_str ? inline_body_has_apply_args(fn->func_str) : 0; + int has_side = fn->func_str ? inline_body_has_side_effects(fn->func_str) : 1; + int eval_only_candidate = fn->func_str && body_len <= eval_only_cap && !has_apply && !has_side; + + if (TCC_LOG_INLINE_STRUCT) + fprintf( + stderr, + "[auto-inline] TOO LARGE: compiling %s normally (len=%d > threshold=%d) cap=%d apply=%d side=%d%s\n", + get_tok_str(sym->v & ~SYM_FIELD, NULL), body_len, threshold, eval_only_cap, has_apply, has_side, + eval_only_candidate ? " (eval-only retained)" : ""); + + if (eval_only_candidate) + { + sym->type.ref->f.func_eval_only_inline = 1; + if (is_static) + sym->type.t |= VT_INLINE; + dynarray_add(&tcc_state->inline_fns, &tcc_state->nb_inline_fns, fn); + + TokenString *compile_ts = tok_str_alloc(); + int *buf = tcc_malloc(body_len * sizeof(int)); + memcpy(buf, tok_str_buf(fn->func_str), body_len * sizeof(int)); + compile_ts->data.str = buf; + compile_ts->allocated_len = body_len; + compile_ts->len = body_len; + + int saved_outer_tok = tok; + CValue saved_outer_tokc = tokc; + begin_macro(compile_ts, 1); + next(); + gen_function(sym); + end_macro(); + tok = saved_outer_tok; + tokc = saved_outer_tokc; + /* Promoted as a small straight-line leaf: inline it at every call + * site, not only the all-constant ones. */ + if (sym->type.ref->f.func_small_leaf) + sym->type.ref->f.func_eval_only_inline = 0; + } + else if (fn->func_str) + { + int saved_outer_tok = tok; + CValue saved_outer_tokc = tokc; + const int post_opt_inline_cap = 512; + if (body_len <= post_opt_inline_cap) + { + /* Preserve token stream for post-optimization re-inlining. + * Compile from a copy (same pattern as the small-function path). */ + dynarray_add(&tcc_state->inline_fns, &tcc_state->nb_inline_fns, fn); + TokenString *compile_ts = tok_str_alloc(); + int *buf = tcc_malloc(body_len * sizeof(int)); + memcpy(buf, tok_str_buf(fn->func_str), body_len * sizeof(int)); + compile_ts->data.str = buf; + compile_ts->allocated_len = body_len; + compile_ts->len = body_len; + begin_macro(compile_ts, 1); + next(); + gen_function(sym); + end_macro(); + if (sym->type.ref->f.func_small_leaf) { + /* Promoted as a small straight-line leaf (see gen_function): + * its optimized body is what gets duplicated, not its tokens, + * so it inlines at every call site. */ + } else if (sym->type.ref->f.func_auto_inline) { + /* Retroactively promoted: convert to eval-only so callsite + * inlining only fires when ALL args are compile-time constants. + * The original body is too large for unconditional inlining. */ + sym->type.ref->f.func_auto_inline = 0; + sym->type.ref->f.func_eval_only_inline = 1; + } else if (sym->type.ref->f.func_late_reopt || + sym->type.ref->f.func_keep_tokens_for_noreturn) { + /* Keep tokens — end-of-TU late_reopt may re-compile. */ + } else if (sym->type.ref->f.tu_static_writer) { + /* Keep tokens — TU-wide DSE may flag this for re-compile. */ + } else { + /* Not promoted: prevent gen_inline_functions re-compilation. */ + tok_str_free(fn->func_str); + fn->func_str = NULL; + fn->sym = NULL; + } + } + else + { + /* Body exceeds post_opt_inline_cap. If the function writes a + * non-const static, we still need its tokens for end-of-TU + * re-compilation; preserve them by going through the same + * inline_fns path used for the smaller-body case. */ + if (tcc_state->opt_dead_store) { + TokenString *compile_ts = tok_str_alloc(); + int *buf = tcc_malloc(body_len * sizeof(int)); + memcpy(buf, tok_str_buf(fn->func_str), body_len * sizeof(int)); + compile_ts->data.str = buf; + compile_ts->allocated_len = body_len; + compile_ts->len = body_len; + dynarray_add(&tcc_state->inline_fns, &tcc_state->nb_inline_fns, fn); + begin_macro(compile_ts, 1); + next(); + gen_function(sym); + end_macro(); + if (!sym->type.ref->f.tu_static_writer && + !sym->type.ref->f.func_late_reopt && + !sym->type.ref->f.func_keep_tokens_for_noreturn && + /* A small straight-line leaf inlines at every call site + * whatever its source length: its optimized size (a few + * ops) is what gets duplicated. Zig's MultiArrayList.slice + * is 536 tokens and 8 ops once its loop unrolls. */ + !sym->type.ref->f.func_small_leaf) { + /* Not a static writer — discard tokens to save memory. */ + tok_str_free(fn->func_str); + fn->func_str = NULL; + fn->sym = NULL; + } + } else { + TokenString *ts = fn->func_str; + fn->func_str = NULL; + begin_macro(ts, 1); + next(); + gen_function(sym); + end_macro(); + tcc_free(fn->pack); + tcc_free(fn); + } + } + tok = saved_outer_tok; + tokc = saved_outer_tokc; + } + else + { + tcc_free(fn->pack); + tcc_free(fn); + } + } + + cur_text_section = saved_text; + } + else + { + /* compute text section */ + cur_text_section = ad_section; + if (!cur_text_section) + cur_text_section = function_text_section(tcc_state, sym); + else if (cur_text_section->sh_num > bss_section->sh_num) + cur_text_section->sh_flags = text_section->sh_flags; + /* When -fdead-store-elimination is enabled, save the body as a + * token stream so the end-of-TU late_reopt pass can re-compile + * this function if TU-wide analysis flags it as a writer of a + * static global with no reachable readers. This path covers + * functions that auto_inline_sig_ok rejects (e.g., double/long + * double params), which would otherwise never reach gen_late_reopt + * because they are not in inline_fns. Bound the saved body + * length so very large functions don't pin extra memory. */ + const int late_reopt_cap = 512; + if (tcc_state->opt_dead_store) + { + struct InlineFunc *fn = tcc_mallocz(sizeof *fn + strlen(file->filename)); + strcpy(fn->filename, file->filename); + fn->sym = sym; + fn->func_str = NULL; + save_function_body(&fn->func_str, &fn->pack, 1); + int body_len = fn->func_str ? fn->func_str->len : 0; + if (fn->func_str && body_len > 0 && body_len <= late_reopt_cap) + { + dynarray_add(&tcc_state->inline_fns, &tcc_state->nb_inline_fns, fn); + TokenString *compile_ts = tok_str_alloc(); + int *buf = tcc_malloc(body_len * sizeof(int)); + memcpy(buf, tok_str_buf(fn->func_str), body_len * sizeof(int)); + compile_ts->data.str = buf; + compile_ts->allocated_len = body_len; + compile_ts->len = body_len; + int saved_outer_tok = tok; + CValue saved_outer_tokc = tokc; + tcc_state->had_nested_funcs = 0; + begin_macro(compile_ts, 1); + next(); + gen_function(sym); + end_macro(); + tok = saved_outer_tok; + tokc = saved_outer_tokc; + /* Token-replay cannot reproduce nested function closure + * semantics during re-compile; drop tokens in that case. + * Otherwise keep only bodies that either write statics for + * TU-wide DSE or were speculatively preserved for call-fact + * late reopt. */ + if (tcc_state->had_nested_funcs || + (!sym->type.ref->f.tu_static_writer && + !sym->type.ref->f.func_keep_tokens_for_noreturn && + !sym->type.ref->f.func_late_reopt)) + { + tok_str_free(fn->func_str); + fn->func_str = NULL; + fn->sym = NULL; + } + } + else + { + /* Body empty or exceeds the cap: compile via replay (if we + * have a saved stream) without inline_fns preservation. */ + if (fn->func_str) + { + int saved_outer_tok2 = tok; + CValue saved_outer_tokc2 = tokc; + TokenString *ts = fn->func_str; + fn->func_str = NULL; + begin_macro(ts, 1); + next(); + gen_function(sym); + end_macro(); + tok = saved_outer_tok2; + tokc = saved_outer_tokc2; + } + else + { + gen_function(sym); + } + tcc_free(fn->pack); + tcc_free(fn); + } + } + else + { + gen_function(sym); + } + /* Nested functions are now compiled inside gen_function, + * before pop_local_syms, so parent locals are still accessible. */ + } +} + +/* -finline-functions-called-once: save a function's body instead of + * generating it, so the end of the TU (gen_deferred_function_bodies) knows + * every body -- and every call site -- before it generates any of them. A + * non-static function is deferred too: it is always emitted, but a caller + * compiled at its definition could inline nothing defined further down. A + * static inline, an old-style or variadic definition, and one whose VLA + * parameters evaluate expressions at the definition are left alone. */ +/* Whether T, the token before a name followed by '(', ends the declaration + * specifiers of a block-scope declaration (`zig_extern zig_i128 __divti3(...);` + * in zig.h): a type keyword or a typedef name. The name is being declared, + * with its prototype, not called. */ +static int tok_ends_decl_specifiers(int t) +{ + switch (t) + { + case TOK_VOID: case TOK_CHAR: case TOK_INT: case TOK_FLOAT: case TOK_DOUBLE: case TOK_BOOL: + case TOK_SHORT: case TOK_LONG: case TOK_UNSIGNED: case TOK_SIGNED1: case TOK_SIGNED2: case TOK_SIGNED3: + case TOK_CONST1: case TOK_CONST2: case TOK_CONST3: + case TOK_VOLATILE1: case TOK_VOLATILE2: case TOK_VOLATILE3: + return 1; + } + if (t < TOK_UIDENT) + return 0; + Sym *s = sym_find(t); + return s && (s->type.t & VT_TYPEDEF); +} + +/* Whether BODY calls a function that has no prototype here: an identifier + * with no declaration in scope (an implicit declaration), or one declared + * without a prototype. Replayed at the end of the TU, the call would be + * checked against a prototype that only appears further down -- pr47428's + * `fn (0)` became "too few arguments". Token replay cannot restore the scope + * as of the definition, so such a body is generated here instead. An + * identifier that is a parameter, or that the body mentioned before (a local + * function pointer), is not a file-scope reference; nor is a predefined token + * (a builtin unary() expands itself, or a library name like memcpy). */ +static int body_calls_unprototyped(Sym *sym, TokenString *body) +{ + const int *start = tok_str_buf(body), *p = start, *end = start + body->len; + int prev = 0; + CValue cv; + while (p < end) + { + int t; + const int *at = p, *next_tok; + tok_get(&t, &p, &cv); + for (next_tok = p; next_tok < end && *next_tok == TOK_LINENUM;) + next_tok += 2; + if (t >= TOK_UIDENT && prev != '.' && prev != TOK_ARROW && next_tok < end && *next_tok == '(') + { + Sym *s = sym_find(t); + if (s && !IS_ASM_SYM(s)) + { + if ((s->type.t & VT_BTYPE) == VT_FUNC && s->type.ref->f.func_type == FUNC_OLD) + return 1; + } + else if (t >= TOK_BUILTIN_END) + { + int local = tok_ends_decl_specifiers(prev); + for (Sym *param = sym->type.ref->next; param && !local; param = param->next) + local = (param->v & ~SYM_FIELD) == t; + for (const int *q = start; q < at && !local;) + { + int u; + CValue ucv; + tok_get(&u, &q, &ucv); + local = u == t; + } + if (!local) + return 1; + } + } + if (t != TOK_LINENUM) + prev = t; + } + return 0; +} + +static int defer_function_body(Sym *sym, Section *section) +{ + if (!(tcc_state->opt_inline_called_once || tcc_state->opt_drop_unused_statics) || tcc_state->nb_vla_param_exprs || + (sym->type.t & VT_INLINE) || !sym->type.ref || sym->type.ref->f.func_type != FUNC_NEW || sym->a.nested_func) + return 0; + DeferredFunc *d = tcc_mallocz(sizeof *d + strlen(file->filename)); + strcpy(d->filename, file->filename); + d->sym = sym; + d->section = section; + save_function_body(&d->body, &d->pack, 0); + if (body_calls_unprototyped(sym, d->body)) + { + /* Generate it now, from the saved tokens, as if never deferred. */ + int saved_tok = tok, saved_line = file->line_num; + CValue saved_tokc = tokc; + define_deferred_function(d); + tok = saved_tok; + tokc = saved_tokc; + file->line_num = saved_line; + tcc_free(d->pack); + tcc_free(d); + return 1; + } + dynarray_add(&tcc_state->deferred_fns, &tcc_state->nb_deferred_fns, d); + return 1; +} + +/* Generate a body defer_function_body saved: replay its tokens through the + * definition path it skipped. */ +void define_deferred_function(DeferredFunc *d) +{ + TokenString *body = d->body; + int saved_pack[PACK_STACK_SIZE + 1]; + int saved_budget = called_once_budget_begin(body->len); + d->body = NULL; + tccpp_putfile(d->filename); + pp_pack_enter(tcc_state, d->pack, saved_pack); + begin_macro(body, 1); + next(); + define_global_function(d->sym, d->section); + end_macro(); + pp_pack_leave(tcc_state, saved_pack); + called_once_budget_end(saved_budget); +} + +/* A copy of STR, to replay without consuming it. */ +TokenString *tok_str_clone(TokenString *str) +{ + TokenString *copy = tok_str_alloc(); + for (const int *p = tok_str_buf(str), *end = p + str->len; p < end; p++) + tok_str_add(copy, *p); + return copy; +} + +/* Complete TYPE, an array whose size its initializer decides, from the saved + * tokens INIT: the size-only pass decl_initializer_alloc runs first, over a + * copy, so INIT stays whole for the replay. */ +static void size_array_from_saved_init(CType *type, TokenString *init) +{ + TokenString *copy = tok_str_clone(init); + int saved_tok = tok, saved_line = file->line_num, saved_nocode_wanted = nocode_wanted; + CValue saved_tokc = tokc; + init_params ip = {0}; + /* Unshare the size-holding ref, as decl_initializer_alloc does: a typedef + * of an unsized array keeps its own. */ + type->ref = sym_push(SYM_FIELD, &type->ref->type, 0, type->ref->c); + ip.flex_array_ref = type->ref; + nocode_wanted |= DATA_ONLY_WANTED; + begin_macro(copy, 1); + next(); + decl_initializer(&ip, type, 0, DIF_FIRST | DIF_SIZE_ONLY, -1); + end_macro(); + nocode_wanted = saved_nocode_wanted; + tok = saved_tok; + tokc = saved_tokc; + file->line_num = saved_line; +} + +/* -fdrop-unused-statics: save the initializer of a file-scope static object + * instead of parsing it, so the end of the TU (prune_unused_statics) emits it + * only if something live refers to it. The symbol is declared now, with its + * complete type, so every later mention resolves; until it is defined it is + * undefined (or still tentative), which no initializer-reading fold trusts. + * An object whose size its initializer decides, or that carries attributes + * about where or whether it is emitted, is defined here as before. */ +static int defer_static_data(CType *type, AttributeDef *ad, int r, int v) +{ + int align; + if (!tcc_state->opt_drop_unused_statics || !(type->t & VT_STATIC) || (type->t & VT_EXTERN) || ad->section || + ad->a.used || ad->alias_target || ad->asm_label || ad->cleanup_func) + return 0; + const int unsized = type_size(type, &align) < 0; + if (unsized && (!(type->t & VT_ARRAY) || type_size(&type->ref->type, &align) < 0)) + return 0; +#ifdef CONFIG_TCC_BCHECK + if (tcc_state->do_bounds_check) + return 0; +#endif + if ((type->t & VT_BTYPE) == VT_STRUCT) + { + Sym *field = type->ref->next; + while (field && field->next) + field = field->next; + if (field && (field->type.t & VT_ARRAY) && field->type.ref->c < 0) + return 0; /* a flexible array member: the initializer sizes it */ + } + Sym *sym = sym_find(v); + if (sym) + { + ElfSym *esym = sym->c ? elfsym(sym) : NULL; + if (!(sym->type.t & VT_EXTERN) || (esym && esym->st_shndx != SHN_UNDEF && esym->st_shndx != SHN_COMMON)) + return 0; /* already defined: let the definition path report it */ + } + TokenString *init; + skip_or_save_block(&init); + if (unsized) + { + size_array_from_saved_init(type, init); + if (type_size(type, &align) < 0) + tcc_error("unknown type size"); + } + /* Declared, not defined, until the replay: the way a tentative static + * is (decl() marks an uninitialized one VT_EXTERN). */ + CType declared = *type; + declared.t |= VT_EXTERN; + if (sym) + patch_storage(sym, ad, &declared); + else + { + sym = sym_push(v, &declared, r | VT_SYM, 0); + patch_storage(sym, ad, NULL); + } + DeferredData *d = tcc_mallocz(sizeof *d + strlen(file->filename)); + strcpy(d->filename, file->filename); + d->sym = sym; + d->type = *type; + d->ad = *ad; + d->r = r; + d->init = init; + dynarray_add(&tcc_state->deferred_data, &tcc_state->nb_deferred_data, d); + return 1; +} + +/* Define an object defer_static_data saved: replay its initializer through + * the allocation it skipped. */ +void define_deferred_data(DeferredData *d) +{ + TokenString *init = d->init; + d->init = NULL; + tccpp_putfile(d->filename); + begin_macro(init, 1); + next(); + decl_initializer_alloc(&d->type, &d->ad, d->r, 1, d->sym->v, 1); + if (tok != TOK_EOF) + expect("';'"); + end_macro(); +} + +/* Parse the initializer of an object prune_unused_statics dropped, for its + * diagnostics only: an invalid initializer is an error whether or not the + * object is used (96_nodata_wanted). Under check_only nothing reaches the + * object file; the initializer is laid out in check_scratch. */ +void check_dropped_deferred_data(DeferredData *d, TokenString *init) +{ + int align, size = type_size(&d->type, &align); + init_params p = {0}; + p.sec = tcc_state->check_scratch; + const unsigned long addr = section_add(p.sec, size > 0 ? size : 0, align > 0 ? align : 1); + const int saved_nocode_wanted = nocode_wanted; + nocode_wanted = DATA_ONLY_WANTED; + tcc_state->check_only++; + tccpp_putfile(d->filename); + begin_macro(init, 1); + next(); + decl_initializer(&p, &d->type, addr, DIF_FIRST, -1); + if (tok != TOK_EOF) + expect("';'"); + end_macro(); + tcc_state->check_only--; + nocode_wanted = saved_nocode_wanted; +} + +/* The same for a function body (gen_function_check_only). */ +void check_dropped_function(DeferredFunc *d, TokenString *body) +{ + int saved_pack[PACK_STACK_SIZE + 1]; + tccpp_putfile(d->filename); + pp_pack_enter(tcc_state, d->pack, saved_pack); + begin_macro(body, 1); + next(); + gen_function_check_only(d->sym); + end_macro(); + pp_pack_leave(tcc_state, saved_pack); +} + /* 'l' is VT_LOCAL or VT_CONST to define default storage type or VT_CMP if parsing old style parameter list or VT_JMP if parsing c99 for decl: for (int i = 0, ...) */ @@ -306,6 +1148,7 @@ int decl(int l) * address-taken). Write captures produce VAR-to-VAR IR patterns * the optimizer can mishandle after inlining. */ if (tcc_state->ir && nf->func_str && + !nf->sym->type.ref->f.func_noinline && (tcc_state->opt_inline_functions || tcc_state->opt_inline_small) && auto_inline_sig_ok(nf->sym) && nf->nb_nlgotos == 0 && nf->nb_addr_labels == 0 && @@ -501,534 +1344,8 @@ int decl(int l) /* put function symbol */ type.t &= ~VT_EXTERN; sym = external_sym(v, &type, 0, &ad); - - /* static inline functions are just recorded as a kind - of macro. Their code will be emitted at the end of - the compilation unit only if they are used */ - if (sym->type.t & VT_INLINE) - { - struct InlineFunc *fn; - fn = tcc_mallocz(sizeof *fn + strlen(file->filename)); - strcpy(fn->filename, file->filename); - fn->sym = sym; - dynarray_add(&tcc_state->inline_fns, &tcc_state->nb_inline_fns, fn); - skip_or_save_block(&fn->func_str); - - /* An explicit `inline` used to *disable* the inliner: this branch - * saved the body and deferred emission, but never set - * func_auto_inline, so every call site emitted a plain call. Only - * functions the user did NOT mark inline reached the auto-inline - * path below. That is backwards, and it costs the most in exactly - * the code that relies on the idiom -- header-defined `static - * inline` helpers. lib/fp/soft is the extreme case: every helper in - * soft_common.h is `static inline`, so __aeabi_dadd made 18 calls to - * one-shift accessors that gcc folds into the body (3.76x on the - * double benchmarks). - * - * Apply the same eligibility test the auto path uses and set the - * flag so call sites replay the body. Deferred emission is a - * strict win over the auto path here: if every call site inlines - * and the address is never taken, gen_inline_functions emits no - * standalone copy at all. - * - * The gate must be token-length based. gen_function's post-opt - * revoke (IR > 8 / call-heavy) cannot help a deferred body -- it - * compiles after the call sites have already been decided. - * - * STATIC ONLY. A non-static `inline` is a C99 inline *definition* - * (and gnu89 `extern inline` another rule again): the standalone - * body is emitted on different terms, and marking it auto_inline - * makes gen_inline_functions skip emission while some call site - * still needs the symbol -- "undefined symbol 'add1_inline'". - * Static inline has internal linkage, so dropping the standalone - * copy once every site inlined is safe, and it is the idiom that - * matters here (all of soft_common.h). */ - if (sym->type.ref && (sym->type.t & VT_STATIC) && - sym->type.ref->f.func_type != FUNC_ELLIPSIS && - !sym->type.ref->f.func_alwinl && !sym->type.ref->f.func_noinline && - (tcc_state->opt_inline_functions || tcc_state->opt_inline_small) && - tcc_state->nb_vla_param_exprs == 0 && fn->func_str) - { - int sig = auto_inline_sig_ok(sym); - int thr = tcc_state->opt_inline_limit > 0 ? tcc_state->opt_inline_limit - : (tcc_state->opt_inline_functions ? 60 : 30); - /* Void-returning with 64-bit params: same narrow cap as the auto - * path (longer bodies trip an IR coalescing bug on narrowed locals). */ - if (sig == 2) - thr = 15; - if (sig && fn->func_str->len <= thr && !inline_body_has_apply_args(fn->func_str) && - !inline_body_has_unsafe_loops(fn->func_str) && !inline_body_has_static_local(fn->func_str)) - { - sym->type.ref->f.func_auto_inline = 1; - /* Body is still owed to gen_inline_functions, unlike the auto - * path below which compiles the standalone copy right here. */ - sym->type.ref->f.func_deferred_inline = 1; - } - } - - /* Scan saved token stream for __builtin_va_arg_pack() usage. - * If found, mark the function so call sites can expand it. */ - if (fn->func_str && sym->type.ref) - { - const int *p = tok_str_buf(fn->func_str); - while (*p != TOK_EOF && *p != 0) - { - if (*p == TOK_builtin_va_arg_pack) - { - sym->type.ref->f.func_va_arg_pack = 1; - break; - } - /* Skip token payload */ - int t = *p++; - switch (t) - { - case TOK_CINT: - case TOK_CUINT: - case TOK_CCHAR: - case TOK_LCHAR: - case TOK_CFLOAT: - case TOK_CFLOAT_I: - case TOK_CINT_I: - case TOK_LINENUM: - case TOK_PACK_REPLAY: -#if LONG_SIZE == 4 - case TOK_CLONG: - case TOK_CULONG: -#endif - p++; - break; - case TOK_CDOUBLE: - case TOK_CDOUBLE_I: - case TOK_CLLONG: - case TOK_CULLONG: -#if LONG_SIZE == 8 - case TOK_CLONG: - case TOK_CULONG: -#endif - p += 2; - break; - case TOK_CLDOUBLE: - case TOK_CLDOUBLE_I: -#if LDOUBLE_SIZE == 8 || defined TCC_USING_DOUBLE_FOR_LDOUBLE - p += 2; -#elif LDOUBLE_SIZE == 12 - p += 3; -#elif LDOUBLE_SIZE == 16 - p += 4; -#endif - break; - case TOK_STR: - case TOK_LSTR: - case TOK_PPNUM: - case TOK_PPSTR: - { - int sz = *p++; - p += (sz + sizeof(int) - 1) / sizeof(int); - break; - } - default: - break; - } - } - } - } - else if (sym->type.ref && sym->type.ref->f.func_type != FUNC_ELLIPSIS && !sym->type.ref->f.func_alwinl && - !sym->type.ref->f.func_noinline && - /* Only auto-inline functions whose signature is safe: scalar/pointer - * params that fit in 32-bit registers, and scalar or struct return - * types. 64-bit types and struct *parameters* are not handled. - * Returns 2 for void+llong signatures (body-length gated below). */ - auto_inline_sig_ok(sym) && (tcc_state->opt_inline_functions || tcc_state->opt_inline_small) && - /* Don't auto-inline functions with VLA parameters: the VLA size - * expressions (which may have side effects like i++) are evaluated - * during function prolog, outside the saved body token stream. - * Inlining would replay only the body, losing those side effects. */ - tcc_state->nb_vla_param_exprs == 0) - { - /* Auto-inline candidate: save the body as a token stream so call - * sites within this TU can replay it. - * - * Static functions: defer standalone compilation; suppress it entirely - * if all call sites are inlined and address not taken. We set - * VT_INLINE so gen_inline_functions handles deferred emission. - * - * Non-static functions: MUST always have a globally-visible symbol for - * other TUs. We compile the standalone definition immediately via - * token-stream replay (same mechanism as gen_inline_functions), then - * keep the token stream in inline_fns for call-site inlining within - * this TU. VT_INLINE is NOT set so ELF linkage stays global. */ - struct InlineFunc *fn; - fn = tcc_mallocz(sizeof *fn + strlen(file->filename)); - strcpy(fn->filename, file->filename); - fn->sym = sym; - fn->func_str = NULL; - skip_or_save_block(&fn->func_str); - - int threshold = tcc_state->opt_inline_limit > 0 ? tcc_state->opt_inline_limit - : (tcc_state->opt_inline_functions ? 60 : 30); - int is_static = !!(sym->type.t & VT_STATIC); - int body_len = fn->func_str ? fn->func_str->len : 0; - - if (TCC_LOG_INLINE_STRUCT) - fprintf(stderr, "[auto-inline] candidate: %s static=%d len=%d threshold=%d\n", - get_tok_str(sym->v & ~SYM_FIELD, NULL), is_static, body_len, threshold); - LOG_INLINE_STRUCT("[auto-inline] candidate: %s static=%d len=%d threshold=%d ret_btype=%d", - get_tok_str(sym->v & ~SYM_FIELD, NULL), is_static, body_len, threshold, - sym->type.ref ? (sym->type.ref->type.t & VT_BTYPE) : -1); - - Section *saved_text = cur_text_section; - cur_text_section = ad.section ? ad.section : function_text_section(tcc_state, sym); - if (cur_text_section->sh_num > bss_section->sh_num) - cur_text_section->sh_flags = text_section->sh_flags; - - /* Void-returning functions with 64-bit params: only inline very - * short bodies (≤ 15 tokens) — longer bodies may trigger an IR - * coalescing bug with narrowed locals. */ - int void_llong_limit = (auto_inline_sig_ok(sym) == 2) ? 15 : threshold; - if (fn->func_str && body_len <= void_llong_limit && !inline_body_has_apply_args(fn->func_str) && - !inline_body_has_unsafe_loops(fn->func_str)) - { - if (TCC_LOG_INLINE_STRUCT) - fprintf(stderr, "[auto-inline] SMALL: registering %s as inline candidate\n", - get_tok_str(sym->v & ~SYM_FIELD, NULL)); - - /* Small enough: register as inline candidate for call-site replay. - * - * We compile the standalone definition immediately for BOTH static - * and non-static functions. We deliberately do NOT set VT_INLINE: - * - Setting VT_INLINE would defer compilation to gen_inline_functions, - * but alias attributes and other code may reference sym->c before - * gen_inline_functions runs, causing "aliased to undefined symbol" - * errors and similar failures. - * - The standalone definition is always emitted. Under - * -ffunction-sections it lands in its own .text. - * section, and the linker's gc_sections() drops it when - * every call site was inlined and the address never taken. - * (Before function-sections was real, this sentence claimed - * --gc-sections would collect it out of the monolithic - * .text, which was never true -- section-granularity GC - * cannot drop a function from a section it shares.) - * - * fn->func_str is preserved (not consumed) so call-site replay - * can still inline the body later. The compilation uses an owning - * COPY of the token stream. - * - * Save/restore tok+tokc: the replay leaves tok=TOK_EOF which would - * cause the outer decl() loop to stop parsing prematurely. */ - sym->type.ref->f.func_auto_inline = 1; - /* Set VT_INLINE for static functions so can_inline_eval=1 at call sites. - * Non-static functions must NOT get VT_INLINE — their standalone definition - * must remain globally visible for other translation units. */ - if (is_static) - sym->type.t |= VT_INLINE; - dynarray_add(&tcc_state->inline_fns, &tcc_state->nb_inline_fns, fn); - - TokenString *compile_ts = tok_str_alloc(); - if (body_len > 0) - { - int *buf = tcc_malloc(body_len * sizeof(int)); - memcpy(buf, tok_str_buf(fn->func_str), body_len * sizeof(int)); - compile_ts->data.str = buf; - compile_ts->allocated_len = body_len; - compile_ts->len = body_len; - } - int saved_outer_tok = tok; - CValue saved_outer_tokc = tokc; - if (TCC_LOG_INLINE_STRUCT) - fprintf(stderr, "[auto-inline] SMALL: compiling standalone for %s\n", - get_tok_str(sym->v & ~SYM_FIELD, NULL)); - tcc_state->had_nested_funcs = 0; - begin_macro(compile_ts, 1); /* owning: compile_ts freed on end_macro */ - next(); - gen_function(sym); - end_macro(); - tok = saved_outer_tok; - tokc = saved_outer_tokc; - /* Revoke auto-inline for functions that contain nested function - * definitions — their closure/trampoline semantics cannot be - * replicated by token-replay inline expansion. */ - if (tcc_state->had_nested_funcs) - { - sym->type.ref->f.func_auto_inline = 0; - if (is_static) - sym->type.t &= ~VT_INLINE; - } - /* gen_function's post-opt check will revoke auto_inline - * if the compiled IR exceeds 8 instructions. */ - /* If auto_inline was revoked (either by had_nested_funcs above or - * by gen_function's post-opt size/call-count check), the function - * is already compiled standalone and won't be inlined at any - * callsite. If the body is pure and within the eval-only cap, - * promote to func_eval_only_inline so try_inline_const_eval can - * still fold all-constant calls (mirrors the TOO-LARGE branch's - * retroactive promotion below). Otherwise free the token stream - * so gen_inline_functions doesn't re-emit a duplicate body. - * Skip the promotion path when nested funcs revoked: token-replay - * cannot reproduce closure/trampoline semantics. */ - if (!sym->type.ref->f.func_auto_inline) - { - /* A register-only loop body (func_const_arg_loop, set from the - * optimized IR) is exempt from the no-side-effects rule: the - * counter and accumulator updates that trip it (`i++`, `n += x`) - * are exactly what a loop is made of, and the loop is why the - * body was demoted. CTFE re-checks side effects before it - * evaluates, so only the all-constant-argument replay path gains - * these — where ssa:loop_const_sim collapses the expanded loop to - * its final value. */ - int loop_helper = sym->type.ref->f.func_const_arg_loop; - int promote_eval_only = !tcc_state->had_nested_funcs && fn->func_str && body_len <= 160 && - !inline_body_has_apply_args(fn->func_str) && - (loop_helper || !inline_body_has_side_effects(fn->func_str)); - if (promote_eval_only) - { - sym->type.ref->f.func_eval_only_inline = 1; - /* VT_INLINE already set above for static; keep it set so - * gen_inline_functions' eval-only skip branch catches us. */ - } - else if (sym->type.ref->f.func_late_reopt || - sym->type.ref->f.func_keep_tokens_for_noreturn) - { - /* Keep tokens — end-of-TU late_reopt may re-compile (either - * already flagged, or pending noreturn-propagation decision). */ - } - else if (sym->type.ref->f.tu_static_writer) - { - /* Keep tokens — function writes >=1 non-const static global; - * end-of-TU TU-wide DSE analysis may decide to re-compile via - * late_reopt to eliminate dead static stores. */ - } - else - { - if (is_static) - sym->type.t &= ~VT_INLINE; - if (fn->func_str) - { - tok_str_free(fn->func_str); - fn->func_str = NULL; - } - fn->sym = NULL; - } - } - if (TCC_LOG_INLINE_STRUCT) - fprintf(stderr, "[auto-inline] SMALL: done compiling %s sym->c=%d\n", - get_tok_str(sym->v & ~SYM_FIELD, NULL), sym->c); - } - else - { - /* Too large to inline-expand, but if the body is pure and under - * a higher eval-only cap, keep the token stream so - * try_inline_const_eval can still fold all-constant calls. - * Regular inline-expansion paths skip functions tagged - * func_eval_only_inline. */ - const int eval_only_cap = 160; - int has_apply = fn->func_str ? inline_body_has_apply_args(fn->func_str) : 0; - int has_side = fn->func_str ? inline_body_has_side_effects(fn->func_str) : 1; - int eval_only_candidate = fn->func_str && body_len <= eval_only_cap && !has_apply && !has_side; - - if (TCC_LOG_INLINE_STRUCT) - fprintf( - stderr, - "[auto-inline] TOO LARGE: compiling %s normally (len=%d > threshold=%d) cap=%d apply=%d side=%d%s\n", - get_tok_str(sym->v & ~SYM_FIELD, NULL), body_len, threshold, eval_only_cap, has_apply, has_side, - eval_only_candidate ? " (eval-only retained)" : ""); - - if (eval_only_candidate) - { - sym->type.ref->f.func_eval_only_inline = 1; - if (is_static) - sym->type.t |= VT_INLINE; - dynarray_add(&tcc_state->inline_fns, &tcc_state->nb_inline_fns, fn); - - TokenString *compile_ts = tok_str_alloc(); - int *buf = tcc_malloc(body_len * sizeof(int)); - memcpy(buf, tok_str_buf(fn->func_str), body_len * sizeof(int)); - compile_ts->data.str = buf; - compile_ts->allocated_len = body_len; - compile_ts->len = body_len; - - int saved_outer_tok = tok; - CValue saved_outer_tokc = tokc; - begin_macro(compile_ts, 1); - next(); - gen_function(sym); - end_macro(); - tok = saved_outer_tok; - tokc = saved_outer_tokc; - } - else if (fn->func_str) - { - int saved_outer_tok = tok; - CValue saved_outer_tokc = tokc; - const int post_opt_inline_cap = 512; - if (body_len <= post_opt_inline_cap) - { - /* Preserve token stream for post-optimization re-inlining. - * Compile from a copy (same pattern as the small-function path). */ - dynarray_add(&tcc_state->inline_fns, &tcc_state->nb_inline_fns, fn); - TokenString *compile_ts = tok_str_alloc(); - int *buf = tcc_malloc(body_len * sizeof(int)); - memcpy(buf, tok_str_buf(fn->func_str), body_len * sizeof(int)); - compile_ts->data.str = buf; - compile_ts->allocated_len = body_len; - compile_ts->len = body_len; - begin_macro(compile_ts, 1); - next(); - gen_function(sym); - end_macro(); - if (sym->type.ref->f.func_auto_inline) { - /* Retroactively promoted: convert to eval-only so callsite - * inlining only fires when ALL args are compile-time constants. - * The original body is too large for unconditional inlining. */ - sym->type.ref->f.func_auto_inline = 0; - sym->type.ref->f.func_eval_only_inline = 1; - } else if (sym->type.ref->f.func_late_reopt || - sym->type.ref->f.func_keep_tokens_for_noreturn) { - /* Keep tokens — end-of-TU late_reopt may re-compile. */ - } else if (sym->type.ref->f.tu_static_writer) { - /* Keep tokens — TU-wide DSE may flag this for re-compile. */ - } else { - /* Not promoted: prevent gen_inline_functions re-compilation. */ - tok_str_free(fn->func_str); - fn->func_str = NULL; - fn->sym = NULL; - } - } - else - { - /* Body exceeds post_opt_inline_cap. If the function writes a - * non-const static, we still need its tokens for end-of-TU - * re-compilation; preserve them by going through the same - * inline_fns path used for the smaller-body case. */ - if (tcc_state->opt_dead_store) { - TokenString *compile_ts = tok_str_alloc(); - int *buf = tcc_malloc(body_len * sizeof(int)); - memcpy(buf, tok_str_buf(fn->func_str), body_len * sizeof(int)); - compile_ts->data.str = buf; - compile_ts->allocated_len = body_len; - compile_ts->len = body_len; - dynarray_add(&tcc_state->inline_fns, &tcc_state->nb_inline_fns, fn); - begin_macro(compile_ts, 1); - next(); - gen_function(sym); - end_macro(); - if (!sym->type.ref->f.tu_static_writer && - !sym->type.ref->f.func_late_reopt && - !sym->type.ref->f.func_keep_tokens_for_noreturn) { - /* Not a static writer — discard tokens to save memory. */ - tok_str_free(fn->func_str); - fn->func_str = NULL; - fn->sym = NULL; - } - } else { - TokenString *ts = fn->func_str; - fn->func_str = NULL; - begin_macro(ts, 1); - next(); - gen_function(sym); - end_macro(); - tcc_free(fn); - } - } - tok = saved_outer_tok; - tokc = saved_outer_tokc; - } - else - { - tcc_free(fn); - } - } - - cur_text_section = saved_text; - } - else - { - /* compute text section */ - cur_text_section = ad.section; - if (!cur_text_section) - cur_text_section = function_text_section(tcc_state, sym); - else if (cur_text_section->sh_num > bss_section->sh_num) - cur_text_section->sh_flags = text_section->sh_flags; - /* When -fdead-store-elimination is enabled, save the body as a - * token stream so the end-of-TU late_reopt pass can re-compile - * this function if TU-wide analysis flags it as a writer of a - * static global with no reachable readers. This path covers - * functions that auto_inline_sig_ok rejects (e.g., double/long - * double params), which would otherwise never reach gen_late_reopt - * because they are not in inline_fns. Bound the saved body - * length so very large functions don't pin extra memory. */ - const int late_reopt_cap = 512; - if (tcc_state->opt_dead_store) - { - struct InlineFunc *fn = tcc_mallocz(sizeof *fn + strlen(file->filename)); - strcpy(fn->filename, file->filename); - fn->sym = sym; - fn->func_str = NULL; - skip_or_save_block(&fn->func_str); - int body_len = fn->func_str ? fn->func_str->len : 0; - if (fn->func_str && body_len > 0 && body_len <= late_reopt_cap) - { - dynarray_add(&tcc_state->inline_fns, &tcc_state->nb_inline_fns, fn); - TokenString *compile_ts = tok_str_alloc(); - int *buf = tcc_malloc(body_len * sizeof(int)); - memcpy(buf, tok_str_buf(fn->func_str), body_len * sizeof(int)); - compile_ts->data.str = buf; - compile_ts->allocated_len = body_len; - compile_ts->len = body_len; - int saved_outer_tok = tok; - CValue saved_outer_tokc = tokc; - tcc_state->had_nested_funcs = 0; - begin_macro(compile_ts, 1); - next(); - gen_function(sym); - end_macro(); - tok = saved_outer_tok; - tokc = saved_outer_tokc; - /* Token-replay cannot reproduce nested function closure - * semantics during re-compile; drop tokens in that case. - * Otherwise keep only bodies that either write statics for - * TU-wide DSE or were speculatively preserved for call-fact - * late reopt. */ - if (tcc_state->had_nested_funcs || - (!sym->type.ref->f.tu_static_writer && - !sym->type.ref->f.func_keep_tokens_for_noreturn && - !sym->type.ref->f.func_late_reopt)) - { - tok_str_free(fn->func_str); - fn->func_str = NULL; - fn->sym = NULL; - } - } - else - { - /* Body empty or exceeds the cap: compile via replay (if we - * have a saved stream) without inline_fns preservation. */ - if (fn->func_str) - { - int saved_outer_tok2 = tok; - CValue saved_outer_tokc2 = tokc; - TokenString *ts = fn->func_str; - fn->func_str = NULL; - begin_macro(ts, 1); - next(); - gen_function(sym); - end_macro(); - tok = saved_outer_tok2; - tokc = saved_outer_tokc2; - } - else - { - gen_function(sym); - } - tcc_free(fn); - } - } - else - { - gen_function(sym); - } - /* Nested functions are now compiled inside gen_function, - * before pop_local_syms, so parent locals are still accessible. */ - } + if (!defer_function_body(sym, ad.section)) + define_global_function(sym, ad.section); break; } else @@ -1135,7 +1452,8 @@ int decl(int l) else if (l == VT_CONST) /* uninitialized global variables may be overridden */ type.t |= VT_EXTERN; - decl_initializer_alloc(&type, &ad, r, has_init, v, l == VT_CONST); + if (!(has_init && l == VT_CONST && defer_static_data(&type, &ad, r, v))) + decl_initializer_alloc(&type, &ad, r, has_init, v, l == VT_CONST); } if (ad.alias_target && l == VT_CONST) diff --git a/source/frontend/gen/decl/declarator.c b/source/frontend/gen/decl/declarator.c index c8544fa0c..dc6384e64 100644 --- a/source/frontend/gen/decl/declarator.c +++ b/source/frontend/gen/decl/declarator.c @@ -283,6 +283,11 @@ int post_type(CType *type, AttributeDef *ad, int storage, int td) tcc_error("declaration of an array of incomplete type elements"); t1 |= type->t & VT_VLA; + /* An element whose size is only known at run time (a struct with a VLA + * member, GNU C) makes the array variable-length too: its fixed-size + * frame slot would be overrun by the second element. */ + if (local_stack && !(storage & VT_STATIC) && struct_has_vla_member(type)) + t1 |= VT_VLA; if (t1 & VT_VLA) { diff --git a/source/frontend/gen/expr/atomic.c b/source/frontend/gen/expr/atomic.c index c59e560e4..a128e13d3 100644 --- a/source/frontend/gen/expr/atomic.c +++ b/source/frontend/gen/expr/atomic.c @@ -23,6 +23,85 @@ #include "gen_priv.h" +#define __ATOMIC_SEQ_CST_VALUE 5 /* the memory orders as numbers them */ + +/* A memory barrier: a call to __tcc_dmb, which the ARM backend emits as one + * DMB instead (tcc_gen_machine_func_call_mop). Every pass already treats a + * call as reading and writing any memory, so nothing moves across it. */ +static void gen_atomic_barrier(void) +{ + const int call_id = tcc_state->ir->next_call_id++; + SValue call_id_sv = tcc_ir_svalue_call_id_argc(call_id, 0); + vpush_helper_func(tok_alloc_const("__tcc_dmb")); + tcc_ir_put(tcc_state->ir, TCCIR_OP_FUNCCALLVOID, vtop, &call_id_sv, NULL); + vpop(); +} + +/* __atomic_load/__atomic_store of a 1-, 2- or 4-byte object, inline: a plain + * LDR/STR of those sizes is single-copy atomic on ARMv7-M/ARMv8-M, and the + * ordering is the standard C11 mapping for the architecture -- + * load: relaxed: ldr; acquire/consume: ldr; dmb; seq_cst: ldr; dmb + * store: relaxed: str; release: dmb; str; seq_cst: dmb; str; dmb + * with an order that is not a constant, or not valid for the access, taken + * as seq_cst. A store, and a relaxed load, is a volatile access, so it is + * neither dropped, merged nor moved out of a loop. An ordered load needs no + * such mark: the barrier call after it already reads and writes all memory, + * so no pass merges it with another load or hoists it, and a volatile access + * would make every pass treat each unmarked access of the whole function (of + * any caller it is inlined into) as possibly volatile -- Zig's InternPool + * accessors do an acquire load on every lookup. + * Read-modify-write stays a call: an LDREX/STREX loop never completes on + * the RP2350's PSRAM, so how to make it atomic is the runtime's to decide. + * vtop holds the arguments the template left: [ptr, order] for a load, + * [ptr, value, order] for a store. Returns 0 to fall back to the call. */ +static int inline_atomic_access(int atok, int size, CType *atom) +{ +#if defined TCC_TARGET_ARM + if (!tcc_state->ir || NOEVAL_WANTED || (size != 1 && size != 2 && size != 4) || + (atok != TOK___atomic_load && atok != TOK___atomic_load_n && atok != TOK___atomic_store)) + return 0; + int order = __ATOMIC_SEQ_CST_VALUE; + if ((vtop->r & (VT_VALMASK | VT_LVAL | VT_SYM)) == VT_CONST) + order = (int)vtop->c.i; + vpop(); + CType vol = *atom; + vol.t |= VT_VOLATILE; + if (atok == TOK___atomic_load || atok == TOK___atomic_load_n) + { + if (order != 0 && order != 1 && order != 2) + order = __ATOMIC_SEQ_CST_VALUE; + if (order != 0 && !(atom->t & VT_VOLATILE)) + vol.t &= ~VT_VOLATILE; + mk_pointer(&vol); + vtop->type = vol; + indir(); + gv(RC_INT); + if (order != 0) + gen_atomic_barrier(); + return 1; + } + if (order != 0 && order != 3) + order = __ATOMIC_SEQ_CST_VALUE; + if (order != 0) + gen_atomic_barrier(); + vswap(); + mk_pointer(&vol); + vtop->type = vol; + indir(); + vswap(); + vstore(); + vpop(); + if (order == __ATOMIC_SEQ_CST_VALUE) + gen_atomic_barrier(); + return 1; +#else + (void)atok; + (void)size; + (void)atom; + return 0; +#endif +} + void parse_atomic(int atok) { int size, align, arg, t, save = 0; @@ -47,6 +126,7 @@ void parse_atomic(int atok) /* keep in order of appearance in tcctok.h: */ /* __atomic_store */ "alm.?", /* __atomic_load */ "Asm.v", + /* __atomic_load_n */ "Am.v", /* __atomic_exchange */ "alsm.v", /* __atomic_compare_exchange */ "aplbmm.b", /* __atomic_fetch_add */ "avm.v", @@ -131,7 +211,19 @@ void parse_atomic(int atok) break; } - snprintf(buf, sizeof(buf), "%s_%d", get_tok_str(atok, 0), size); + if (inline_atomic_access(atok, size, atom)) + { + if ((ct.t & VT_BTYPE) == VT_VOID) + { + vpushi(0); + vtop->type = ct; + return; + } + goto result; + } + + /* __atomic_load_n is __atomic_load returning the value: same helper. */ + snprintf(buf, sizeof(buf), "%s_%d", get_tok_str(atok == TOK___atomic_load_n ? TOK___atomic_load : atok, 0), size); vpush_helper_func(tok_alloc_const(buf)); { int call_argc = arg - save; @@ -185,6 +277,7 @@ void parse_atomic(int atok) vtop->type.t = VT_INT; #endif } +result: gen_cast(&ct); if (save) { diff --git a/source/frontend/gen/expr/cond.c b/source/frontend/gen/expr/cond.c index 70a2103d8..b98382451 100644 --- a/source/frontend/gen/expr/cond.c +++ b/source/frontend/gen/expr/cond.c @@ -90,34 +90,37 @@ void expr_landor(int op) } nocode_wanted -= f; } - else if (tcc_state->ir != NULL && i == 0 && (vtop->r & (VT_VALMASK | VT_LVAL | VT_SYM)) == VT_CONST) + else if (tcc_state->ir != NULL && (vtop->r & (VT_VALMASK | VT_LVAL | VT_SYM)) == VT_CONST) { - /* IR mode, || only: the last operand is a compile-time constant false - * but earlier operands were runtime values (cc=0, f=0). + /* IR mode: the last operand is a compile-time constant but earlier + * operands were runtime values (cc=0, f=0). * - * gvtst_set() would create a synthetic VT_CMP(TOK_EQ) with no real - * CMP instruction, causing the next JUMPIF to reuse stale condition - * flags from a prior comparison — producing the wrong branch direction. + * gvtst_set() would create a synthetic VT_CMP with no real CMP + * instruction, so whatever reads it -- a SETIF materialising the value + * or the next JUMPIF -- takes stale flags from a prior comparison. * - * For ||, the chain 't' holds "jump-when-true" entries. A constant- - * false last operand means the overall result depends solely on - * whether a prior operand was true → encode as VT_JMP with chain t. + * The constant is the operator's neutral element here (a false one + * sets f above), so the result is decided by the chain alone: + * || 't' holds "jump-when-true" entries -> VT_JMP + * && 't' holds "jump-when-false" entries -> VT_JMPI * - * (For &&, i==1, the synthetic VT_CMP(TOK_NE) from gvtst_set happens - * to match the stale flags correctly on the fallthrough path, so the - * original codepath is valid and must not be replaced.) + * && used to go through gvtst_set on the theory that the stale flags + * happen to say "true" on the fallthrough path. They do only when the + * previous operand's jump was on EQ: after `!x`, whose jump is on NE, + * the fallthrough flags say EQ and `a && !x && 1` came out 0. The + * self-hosted tcc hit it in frame_record's `guard` (tcc_bounds_checking + * is the constant 0), dropping the guard byte from every frame object. */ int const_val = (vtop->c.i != 0); vpop(); - if (const_val == 0) + if (const_val == i) { - /* false || … — outcome depends on chain only. */ - vseti(VT_JMP, t); + vseti(i ? VT_JMPI : VT_JMP, t); } else { - /* true || … — always true. */ - vpushi(1); + /* Not reached: a constant that decides the result sets f. */ + vpushi(const_val); tcc_ir_backpatch_to_here(tcc_state->ir, t); } } diff --git a/source/frontend/gen/expr/indir.c b/source/frontend/gen/expr/indir.c index e774af316..b9f95f840 100644 --- a/source/frontend/gen/expr/indir.c +++ b/source/frontend/gen/expr/indir.c @@ -72,7 +72,7 @@ ST_FUNC void indir(void) * only under nocode_wanted (speculative try_inline_const_eval) so regular * code generation is unaffected. */ if (nocode_wanted && (vtop->r & (VT_VALMASK | VT_SYM | VT_LVAL)) == (VT_CONST | VT_SYM | VT_LVAL) && vtop->sym && - !vtop->sym->a.possibly_written && !(vtop->type.t & (VT_ARRAY | VT_VLA))) + !vtop->sym->a.possibly_written && !vtop->sym->a.tentative && !(vtop->type.t & (VT_ARRAY | VT_VLA))) { int btype = vtop->type.t & VT_BTYPE; if (btype == VT_BYTE || btype == VT_SHORT || btype == VT_INT || btype == VT_LLONG || btype == VT_BOOL || @@ -132,8 +132,6 @@ void gfunc_param_typed(Sym *func, Sym *arg) /* Handle struct/union arguments for unprototyped/variadic calls. */ if ((vtop->type.t & VT_BTYPE) == VT_STRUCT) { - int align, size = type_size(&vtop->type, &align); - /* VLA structs have runtime-determined size (type_size returns 0). * Pass by invisible reference: the VLA struct's stack slot already * contains a pointer to the VLA-allocated data. Load that pointer @@ -150,71 +148,9 @@ void gfunc_param_typed(Sym *func, Sym *arg) return; } - if (size > 16) - { - if (nocode_wanted) - return; - - if (!(vtop->r & VT_LVAL)) - { - tcc_error("cannot pass large struct by value"); - } - - /* Allocate a stack slot for the struct copy. - * - * For a non-variadic argument the slot is converted to a pointer by - * the gaddrof() below, which drops the VR_TEMP_LOCAL marker from the - * vstack — so get_temp_local_var() could reuse the same slot for a - * sibling struct argument in the same call, aliasing the two copies - * (GCC PR 67226). Those keep a fresh, never-reused slot. - * - * A variadic anonymous argument instead stays a struct lvalue (no - * gaddrof — see the FUNC_ELLIPSIS return below), so it is safe to draw - * its copy from the call-scoped arg-struct temp pool: that pool keeps - * concurrently-live copies in distinct slots while reclaiming slots - * from completed statements, collapsing the one-copy-per-call-site - * stack growth seen when marshaling many large by-value variadic - * structs. */ - int tmp_loc; - if (func_type == FUNC_ELLIPSIS) - { - tmp_loc = get_arg_struct_temp(size, align); - } - else - { - loc = (loc - size) & -align; - tmp_loc = loc; - } - - /* Store the source struct into the temporary destination. - * vstore() will emit a memmove() for struct types. */ - { - SValue dst; - memset(&dst, 0, sizeof(dst)); - dst.type = vtop->type; - dst.r = VT_LOCAL | VT_LVAL; - dst.vr = -1; - dst.c.i = tmp_loc; - vpushv(&dst); - vswap(); - vstore(); - } - - if (func_type == FUNC_ELLIPSIS) - { - /* Variadic anonymous argument: keep as struct lvalue so the - * backend decomposes it into words for register/stack placement. - * va_arg reads the raw data from the va area, not a pointer. */ - return; - } - - /* Unprototyped (FUNC_OLD) call: the callee may have been compiled - * with a prototype and expect invisible reference (pointer) for - * structs > 16 bytes. Convert the temp copy to a pointer arg. */ - mk_pointer(&vtop->type); - gaddrof(); - return; - } + /* Any other struct goes by value whatever its size: AAPCS32 places it + * in the free core registers and then on the stack, like a prototyped + * argument (a variadic one is read back by value by va_arg). */ } /* default casting : only need to convert float to double */ @@ -255,65 +191,10 @@ void gfunc_param_typed(Sym *func, Sym *arg) } } - /* ARM EABI AAPCS: Composite types (struct/union) larger than 4 words (16 bytes) - * must be passed by invisible reference - the caller passes a pointer. - * Check if this is a large struct that should be passed by reference. */ - if ((type.t & VT_BTYPE) == VT_STRUCT) - { - int align, size = type_size(&type, &align); - if (size > 16) - { - /* Pass by invisible reference: caller must allocate a temporary copy - * and pass a pointer to that copy (AAPCS). Passing the original object's - * address would break C's by-value semantics. - */ - if (nocode_wanted) - return; - - if (!(vtop->r & VT_LVAL)) - { - /* For now we require an lvalue source; most struct expressions in TCC - * are materialized as lvalues already. - */ - tcc_error("cannot pass large struct by value"); - } - - /* Always allocate a fresh stack slot for the struct copy. - * Do NOT use get_temp_local_var() here: after gaddrof() converts - * the lvalue to a pointer, the VR_TEMP_LOCAL marker is lost from - * vstack, causing get_temp_local_var() to reuse the same slot for - * a subsequent struct argument in the same call. This would make - * both struct copies alias the same memory. (See GCC PR 67226.) */ - loc = (loc - size) & -align; - int tmp_loc = loc; - - /* Store the source struct into the temporary destination. - * vstore() will emit a memmove() for struct types. - */ - { - SValue dst; - memset(&dst, 0, sizeof(dst)); - dst.type = type; - dst.r = VT_LOCAL | VT_LVAL; - dst.vr = -1; - dst.c.i = tmp_loc; - vpushv(&dst); - vswap(); - vstore(); - } - - /* Save const_init_data before gaddrof invalidates it — the - * inline expansion needs it for compile-time vector folding. */ - aapcs_last_const_init = find_sv_const_init(vtop, size); - aapcs_last_const_init_size = aapcs_last_const_init ? size : 0; - - /* Convert the temp lvalue to a pointer argument. */ - mk_pointer(&vtop->type); - gaddrof(); - return; - } - } - + /* A struct or union of any size is passed by value (AAPCS32): the call + * lowering copies it into the argument registers and stack, so no + * temporary copy is made here. (A pointer to a copy for composites over + * 16 bytes is the AArch64 rule.) */ gen_assign_cast(&type); } } diff --git a/source/frontend/gen/expr/primary.c b/source/frontend/gen/expr/primary.c index 7761e01e5..90478d81e 100644 --- a/source/frontend/gen/expr/primary.c +++ b/source/frontend/gen/expr/primary.c @@ -534,7 +534,8 @@ __attribute__((noinline)) int unary_primary(void) /* Recognize compile-time-constant lvalue accesses to read-only data. * For example, string literal subscript "hi"[0] is a compile-time * constant even though it presents as an lvalue (VT_LVAL set). */ - if (n == 0 && (vtop->r & (VT_VALMASK | VT_LVAL | VT_SYM)) == (VT_CONST | VT_LVAL | VT_SYM) && vtop->sym) + if (n == 0 && (vtop->r & (VT_VALMASK | VT_LVAL | VT_SYM)) == (VT_CONST | VT_LVAL | VT_SYM) && vtop->sym && + !vtop->sym->a.tentative) { ElfSym *esym = elfsym(vtop->sym); if (esym && esym->st_shndx > 0 && esym->st_shndx < tcc_state->nb_sections) @@ -591,6 +592,16 @@ __attribute__((noinline)) int unary_primary(void) break; case TOK_builtin_unreachable: parse_builtin_params(0, ""); /* just skip '()' */ + /* A marked TRAP: control never gets here, and saying so in the IR lets + * the branches that lead here go (tcc_ir_opt_unreachable_fold). */ + if (tcc_state->ir && !nocode_wanted) + { + const int before = tcc_state->ir->next_instruction_index; + tcc_ir_put(tcc_state->ir, TCCIR_OP_TRAP, NULL, NULL, NULL); + if (tcc_state->ir->next_instruction_index > before && + tcc_state->ir->compact_instructions[before].op == TCCIR_OP_TRAP) + tcc_state->ir->compact_instructions[before].unreachable = 1; + } type.t = VT_VOID; vpush(&type); CODE_OFF(); @@ -1103,21 +1114,16 @@ __attribute__((noinline)) int unary_primary(void) vpushi(0); vtop->type = type; } + else if (tok1 == TOK_builtin_return_address) + { + /* level 0: the LR the prologue saved, read off SP */ + vpush_return_address(); + } else { - /* level == 0: force standard frame record {FP, LR} */ + /* __builtin_frame_address(0) is the frame pointer itself */ tcc_state->force_frame_pointer = 1; - if (tok1 == TOK_builtin_return_address) - tcc_state->force_lr_save = 1; vset(&type, VT_LOCAL, 0); /* FP value */ - if (tok1 == TOK_builtin_return_address) - { - /* LR is at [FP + PTR_SIZE] in the standard frame record */ - vpushi(PTR_SIZE); - gen_op('+'); - mk_pointer(&vtop->type); - indir(); - } } #else /* Non-ARM targets: original chain-walking implementation */ @@ -1612,6 +1618,7 @@ __attribute__((noinline)) int unary_primary(void) /* atomic operations */ case TOK___atomic_store: case TOK___atomic_load: + case TOK___atomic_load_n: case TOK___atomic_exchange: case TOK___atomic_compare_exchange: case TOK___atomic_fetch_add: @@ -1810,7 +1817,7 @@ __attribute__((noinline)) int unary_primary(void) * initializers. */ if ((s->type.t & VT_CONSTANT) && !(s->type.t & VT_VOLATILE) && !(s->type.t & VT_ARRAY) && !(s->type.t & VT_VLA) && (s->type.t & VT_BTYPE) != VT_FUNC && (s->type.t & VT_BTYPE) != VT_STRUCT && - (s->type.t & VT_BTYPE) != VT_PTR && s->c > 0) + (s->type.t & VT_BTYPE) != VT_PTR && s->c > 0 && !s->a.tentative) { ElfSym *esym = elfsym(s); if (esym && esym->st_shndx != SHN_UNDEF && esym->st_shndx != SHN_COMMON && diff --git a/source/frontend/gen/gen_priv.h b/source/frontend/gen/gen_priv.h index 1b5e277c3..35312ad8d 100644 --- a/source/frontend/gen/gen_priv.h +++ b/source/frontend/gen/gen_priv.h @@ -78,6 +78,13 @@ int gjmp_addr_acs(int a); #define vstack (_vstack + 1) #define NODATA_WANTED (nocode_wanted > 0) /* no static data output wanted either */ +/* No static-storage bytes wanted: NODATA_WANTED, or a dropped static is being + * parsed for its diagnostics only (check_only). That cannot be a nocode_wanted + * bit: its initializers need DATA_ONLY_WANTED (the sign bit) to fold as static + * initializers do, and its function bodies generate code (into an IR that is + * freed) -- so locals are laid out as ever, and only what would land in a + * section is held back. */ +#define NOSTATIC_WANTED (NODATA_WANTED || tcc_state->check_only) #define DATA_ONLY_WANTED 0x80000000 /* ON outside of functions and for static initializers */ /* no code output after unconditional jumps such as with if (tcc_state->optimize > 0) ... */ @@ -258,7 +265,6 @@ extern int nb_temp_local_vars; * block() saves/restores it around every statement so two call expressions * that are lexically nested (e.g. inside a GNU statement expression used as * an argument) do not alias each other's reserved slots. */ -extern uint64_t arg_struct_temp_busy; /* sym_push()'s bump allocator. Producer/owner: sym/symtab.c. */ extern Sym *sym_free_first; @@ -273,8 +279,6 @@ extern FuncallScratch *funcall_scratch_stack; /* AAPCS invisible-copy handshake: set while typing a call's formal * parameters (expr/indir.c's gfunc_param_typed()), consumed while lowering * the actual arguments (builtin/call.c's unary_funcall()). */ -extern unsigned char *aapcs_last_const_init; -extern int aapcs_last_const_init_size; /* Producer/consumer: decl/declarator.c's post_type() alone; kept here (not * file-local) only because promoting it cost nothing once str_lit_pool @@ -389,7 +393,6 @@ int is_integer_btype(int bt); /* --- core/state.c --- */ Sym *find_local_scalar_sym_by_offset(int offset); void funcall_scratch_pop_free(FuncallScratch *fs); -int get_arg_struct_temp(int size, int align); /* --- core/suppress.c --- */ int gind(); @@ -447,6 +450,8 @@ int unary_paren(void); /* --- init/alloc.c --- */ void decl_initializer(init_params *p, CType *type, unsigned long c, int flags, int vreg); void decl_initializer_alloc(CType *type, AttributeDef *ad, int r, int has_init, int v, int global); +ST_FUNC void finalize_tentative_definitions(TCCState *s1); +ST_FUNC void free_tentative_definitions(TCCState *s1); /* --- init/initializer.c --- */ void decl_design_flex(init_params *p, Sym *ref, int index); @@ -488,8 +493,23 @@ int try_inline_const_eval(Sym *func_sym, SValue *args, int nb_args); void free_inline_functions(TCCState *s); Section *function_text_section(TCCState *s1, Sym *sym); void gen_inline_functions(TCCState *s); +void gen_owed_inline_functions(TCCState *s); +InlineFunc *inline_fn_lookup(TCCState *s, Sym *sym); +void gen_deferred_function_bodies(TCCState *s); +void free_deferred_functions(TCCState *s); +void define_deferred_function(DeferredFunc *d); +void define_deferred_data(DeferredData *d); +void check_dropped_deferred_data(DeferredData *d, TokenString *init); +void check_dropped_function(DeferredFunc *d, TokenString *body); +void check_dropped_statics(TCCState *s); +void gc_unreferenced_statics(TCCState *s); +void free_dropped_statics(TCCState *s); +void gen_function_check_only(Sym *sym); +TokenString *tok_str_clone(TokenString *str); +void prune_unused_statics(TCCState *s); +int called_once_budget_begin(int own_len); +void called_once_budget_end(int saved); void gen_late_reopt_functions(TCCState *s); -void ir_inline_stash_flush(TCCState *s1); /* --- nested/nested.c --- */ void prescan_captured_vars(NestedFunc *nf, Sym *parent_local_stack, NestedFunc *explicit_parent_nf); @@ -562,6 +582,11 @@ int struct_is_single_1byte_scalar_member(const CType *type); int struct_is_single_2byte_scalar_member(const CType *type); int struct_is_small_bitfield_word(const CType *type); int struct_member_copy_safe(const CType *type); +#define SMALL_AGGREGATE_COPY_MAX 16 +int small_aggregate_copy_plan(const CType *stype, int size, int align, int src_off, int dst_off, int *w_out, + unsigned char *covered); +void ir_emit_small_aggregate_copy(const SValue *src, int src_deref, const SValue *dst, int dst_deref, int size, + int w, const unsigned char *covered); /* --- sym/attr_merge.c --- */ void apply_alias_attribute(Sym *alias_sym, int target_tok); @@ -571,6 +596,7 @@ void merge_funcattr(struct FuncAttr *fa, struct FuncAttr *fa1); void merge_symattr(struct SymAttr *sa, struct SymAttr *sa1); void patch_storage(Sym *sym, AttributeDef *ad, CType *type); void resolve_pending_aliases(void); +void mark_pending_alias_targets_used(void); void sym_copy_ref(Sym *s, Sym **ps); /* --- sym/symtab.c --- */ diff --git a/source/frontend/gen/init/alloc.c b/source/frontend/gen/init/alloc.c index 0be7c7378..4759c3364 100644 --- a/source/frontend/gen/init/alloc.c +++ b/source/frontend/gen/init/alloc.c @@ -154,10 +154,10 @@ void decl_initializer(init_params *p, CType *type, unsigned long c, int flags, i if (p->sec && size1 == 1) { init_assert(p, c + nb); - if (!NODATA_WANTED) + if (!NOSTATIC_WANTED) memcpy(p->sec->data + c, initstr.data, nb); } - else if (tcc_state->ir && size1 == 1 && nb >= 8 && !NODATA_WANTED) + else if (tcc_state->ir && size1 == 1 && nb >= 8 && !NOSTATIC_WANTED) { /* Bulk copy string literal from .rodata instead of byte-by-byte stores. * Matches GCC: memcpy(dest, .rodata, str_len) + memset(trailing, 0, rem) */ @@ -427,6 +427,56 @@ static int type_contains_pointer(CType *type) are parsed. If 'v' is zero, then a reference to the new object is put in the value stack. If 'has_init' is 2, a special parsing is done to handle string constants. */ +/* A definition's storage: the bytes its tentative declaration already took, + * when they fit -- same section, same size, suitably aligned. Zig's C + * declares every static first and defines it further down, so without this + * each object is laid out twice. */ +static int tentative_storage_reuse(Sym *sym, int has_init, Section *sec, int size, int align, int *addr) +{ + if (!sym || !has_init || !sym->a.tentative || !sym->c) + return 0; + ElfSym *esym = elfsym(sym); + if (!esym || esym->st_shndx != sec->sh_num || esym->st_size != (addr_t)size || (esym->st_value & (align - 1))) + return 0; + *addr = (int)esym->st_value; + return 1; +} + +/* The end of the TU: every definition has been seen. A tentative definition + * none of them initialized is a zero-filled one (C11 6.9.2p2); if it waits in + * COMMON it is placed in .bss, unless a non-static object may stay common + * (-fcommon). Runs before the deferred bodies are generated, so they see + * every object where it stays. */ +ST_FUNC void finalize_tentative_definitions(TCCState *s1) +{ + /* File scope, whatever the last body generated here left behind (a body + * ending in unreachable code leaves code, and so data, switched off). */ + const int saved_nocode_wanted = nocode_wanted; + nocode_wanted = DATA_ONLY_WANTED; + for (int i = 0; i < s1->nb_tentative_syms; i++) + { + Sym *sym = s1->tentative_syms[i]; + sym->a.tentative = 0; + ElfSym *esym = sym->c ? elfsym(sym) : NULL; + if (!esym || esym->st_shndx != SHN_COMMON || sym->a.tu_unused) + continue; + if (!(sym->type.t & VT_STATIC) && !s1->nocommon) + continue; + const unsigned long size = esym->st_size; + const int align = (int)esym->st_value; + put_extern_sym(sym, bss_section, section_add(bss_section, size, align), size); + } + nocode_wanted = saved_nocode_wanted; + free_tentative_definitions(s1); +} + +ST_FUNC void free_tentative_definitions(TCCState *s1) +{ + tcc_free(s1->tentative_syms); + s1->tentative_syms = NULL; + s1->nb_tentative_syms = 0; +} + void decl_initializer_alloc(CType *type, AttributeDef *ad, int r, int has_init, int v, int global) { int size, align, addr; @@ -557,7 +607,7 @@ void decl_initializer_alloc(CType *type, AttributeDef *ad, int r, int has_init, align = 1; } - if (!v && NODATA_WANTED) + if (!v && (NODATA_WANTED || (tcc_state->check_only && (r & VT_VALMASK) == VT_CONST))) { size = 0, align = 1; } @@ -607,7 +657,8 @@ void decl_initializer_alloc(CType *type, AttributeDef *ad, int r, int has_init, int slot_align = align; if (size >= 4 && slot_align < 4 && (type->t & VT_BTYPE) == VT_STRUCT) slot_align = 4; - loc = (loc - size) & -slot_align; + loc = tcc_ir_frame_alloc(loc, size, -slot_align); + tcc_ir_frame_note_type(loc, type); } } addr = loc; @@ -622,16 +673,22 @@ void decl_initializer_alloc(CType *type, AttributeDef *ad, int r, int has_init, if (v) { /* local variable */ + sym = sym_push(v, type, r, addr); + vreg = sym->vreg; #ifdef CONFIG_TCC_ASM + /* `register T x __asm("rN")`: x is an ordinary local whose value is + * placed in rN only where it is an asm operand. Record rN in sym->r + * after sym_push, which gives register-candidate locals their vreg only + * while the value mask still says VT_LOCAL -- recording it before left x + * without a vreg, so its uses went to a stack slot that was never + * allocated and aliased the previous local. */ if (ad->asm_label) { int reg = asm_parse_regvar(ad->asm_label); if (reg >= 0) - r = (r & ~VT_VALMASK) | reg; + sym->r = (sym->r & ~VT_VALMASK) | reg; } #endif - sym = sym_push(v, type, r, addr); - vreg = sym->vreg; if (ad->cleanup_func) { Sym *cls = sym_push2(&all_cleanups, SYM_FIELD | ++cur_scope->cl.n, 0, 0); @@ -650,10 +707,11 @@ void decl_initializer_alloc(CType *type, AttributeDef *ad, int r, int has_init, if (has_init && size > 0 && size <= 256 && ((type->t & VT_ARRAY) || (type->t & VT_VECTOR)) && !(type->t & VT_VLA)) { - sym->const_init_data = tcc_mallocz(size); - sym->const_init_size = size; - sym->const_init_valid = 1; - sym->const_init_in_progress = 1; + SymLocalFacts *f = sym_facts(sym); + f->const_init_data = tcc_mallocz(size); + f->const_init_size = size; + f->const_init_valid = 1; + f->const_init_in_progress = 1; p.const_init_sym = sym; p.const_init_base = addr; } @@ -669,10 +727,11 @@ void decl_initializer_alloc(CType *type, AttributeDef *ad, int r, int has_init, { Sym *anon = sym_push2(&local_stack, SYM_FIRST_ANOM, type->t, addr); anon->type.ref = type->ref; - anon->const_init_data = tcc_mallocz(size); - anon->const_init_size = size; - anon->const_init_valid = 1; - anon->const_init_in_progress = 1; + SymLocalFacts *f = sym_facts(anon); + f->const_init_data = tcc_mallocz(size); + f->const_init_size = size; + f->const_init_valid = 1; + f->const_init_in_progress = 1; p.const_init_sym = anon; p.const_init_base = addr; } @@ -681,6 +740,14 @@ void decl_initializer_alloc(CType *type, AttributeDef *ad, int r, int has_init, else { sym = NULL; + /* A file-scope object without an initializer is a tentative definition + * (C11 6.9.2): the definition may still follow, or it is zero-filled at + * the end of the TU (finalize_tentative_definitions). */ + const int tentative = v && global && !has_init +#ifdef CONFIG_TCC_BCHECK + && !bcheck +#endif + ; if (v && global) { /* see if the symbol was already defined */ @@ -709,7 +776,10 @@ void decl_initializer_alloc(CType *type, AttributeDef *ad, int r, int has_init, CType *tp = type; while ((tp->t & (VT_BTYPE | VT_ARRAY)) == (VT_PTR | VT_ARRAY)) tp = &tp->ref->type; - if (tp->t & VT_CONSTANT) + /* A tentative one waits in COMMON, const or not (below): the + * definition that may follow is placed where it belongs, and one + * nothing refers to is dropped (prune_unused_statics). */ + if ((tp->t & VT_CONSTANT) && !tentative) { /* RELRO: with -share-rodata, a const object whose type contains a pointer can hold a relocation, so it must live in the writable @@ -730,13 +800,19 @@ void decl_initializer_alloc(CType *type, AttributeDef *ad, int r, int has_init, /*if (tcc_state->g_debug & 4) tcc_warning("rw data: %s", get_tok_str(v, 0));*/ } - else if (tcc_state->nocommon) + /* A tentative one waits in COMMON: its definition would move it to + * .data, leaving the .bss bytes it took unused. */ + else if (tcc_state->nocommon && !tentative) sec = bss_section; } + /* A static object of a function parsed for its diagnostics only. */ + if (sec && tcc_state->check_only) + sec = tcc_state->check_scratch; if (sec) { - addr = section_add(sec, size, align); + if (!tentative_storage_reuse(sym, has_init, sec, size, align, &addr)) + addr = section_add(sec, size, align); #ifdef CONFIG_TCC_BCHECK /* add padding if bound check */ if (bcheck) @@ -759,6 +835,13 @@ void decl_initializer_alloc(CType *type, AttributeDef *ad, int r, int has_init, } /* update symbol definition */ put_extern_sym(sym, sec, addr, size); + if (tentative && !sym->a.tentative) + { + sym->a.tentative = 1; + dynarray_add(&tcc_state->tentative_syms, &tcc_state->nb_tentative_syms, sym); + } + else if (!tentative) + sym->a.tentative = 0; } else { @@ -945,7 +1028,7 @@ void decl_initializer_alloc(CType *type, AttributeDef *ad, int r, int has_init, * is entirely load-time-constant do we emit the memcpy and skip the * per-element path. Otherwise rewind and fall through to normal init. */ int templated = 0; - if (!sec && tcc_state->ir && has_init && !NODATA_WANTED && (type->t & VT_ARRAY) && !(type->t & VT_VLA) && + if (!sec && tcc_state->ir && has_init && !NOSTATIC_WANTED && (type->t & VT_ARRAY) && !(type->t & VT_VLA) && !(type->t & VT_COMPLEX) && size > 256) { /* Both paths leave the live token at the start of the initializer: @@ -1018,7 +1101,14 @@ void decl_initializer_alloc(CType *type, AttributeDef *ad, int r, int has_init, cargs[2].c.i = size; cargs[2].vr = -1; - gen_ir_void_call_args(cargs, 3, TOK_memcpy); + int copy_tok = TOK_memcpy; +#ifdef TCC_ARM_EABI + /* Local and template share the type's alignment: a word-aligned + * copy the backend expands inline (thumb_inline_aligned_copy_call). */ + if (align >= 4 && !(size & 3)) + copy_tok = TOK_memmove4; +#endif + gen_ir_void_call_args(cargs, 3, copy_tok); templated = 1; } else @@ -1035,7 +1125,7 @@ void decl_initializer_alloc(CType *type, AttributeDef *ad, int r, int has_init, decl_initializer(&p, type, addr, DIF_FIRST, vreg); if (p.const_init_sym) - p.const_init_sym->const_init_in_progress = 0; + p.const_init_sym->facts->const_init_in_progress = 0; tcc_state->nrvo_target_ptr_vreg = saved_nrvo_ptr_vreg; tcc_state->nrvo_target_active = saved_nrvo_active; diff --git a/source/frontend/gen/init/initializer.c b/source/frontend/gen/init/initializer.c index faf66ea05..80440f3bd 100644 --- a/source/frontend/gen/init/initializer.c +++ b/source/frontend/gen/init/initializer.c @@ -41,6 +41,8 @@ void skip_or_save_block(TokenString **str) *str = tok_str_alloc(); pp_pragma_capture = str ? *str : saved_capture; + int recorded_len = 0; + while (1) { int t = tok; @@ -57,7 +59,10 @@ void skip_or_save_block(TokenString **str) break; } if (str) + { tok_str_add_tok(*str); + recorded_len = (*str)->len; + } next(); if (t == '{' || t == '(' || t == '[') { @@ -72,7 +77,21 @@ void skip_or_save_block(TokenString **str) } pp_pragma_capture = saved_capture; if (str) + { + /* The lookahead read after the last recorded token (the token after a + body's closing brace, or the terminator) ran any #pragma pack in + between with capture still on. Those directives are outside the saved + tokens: apply them now, in file order, and drop them from the stream. + Left in, `f(){...} #pragma pack(1) struct S {...};` laid S out unpacked + and replayed pack(1) whenever f's body was. */ + if ((*str)->len > recorded_len) + { + const int *buf = tok_str_buf(*str); + pp_apply_pack_replays(tcc_state, buf + recorded_len, buf + (*str)->len); + (*str)->len = recorded_len; + } tok_str_add(*str, TOK_EOF); + } } #define EXPR_CONST 1 @@ -108,7 +127,7 @@ void parse_init_elem(int expr_type) #if 1 void init_assert(init_params *p, int offset) { - if (p->sec ? !NODATA_WANTED && offset > p->sec->data_offset : !nocode_wanted && offset > p->local_offset) + if (p->sec ? !NOSTATIC_WANTED && offset > p->sec->data_offset : !nocode_wanted && offset > p->local_offset) tcc_internal_error("initializer overflow"); } #else @@ -512,7 +531,7 @@ void init_putv(init_params *p, CType *type, unsigned long c, int vreg) !((vtop->r & VT_CONST) && vtop->sym->v >= SYM_FIRST_ANOM)) tcc_error("initializer element is not computable at load time"); - if (NODATA_WANTED) + if (NOSTATIC_WANTED) { vtop--; print_vstack("init_putv"); @@ -711,17 +730,17 @@ void init_putv(init_params *p, CType *type, unsigned long c, int vreg) /* Capture scalar constant into the tracked sym's const_init_data * before vstore (which pops the value). Buffer was zeroed at * allocation time, so zero values can be silently dropped. */ - if (p->const_init_sym && p->const_init_sym->const_init_valid) + if (p->const_init_sym && p->const_init_sym->facts->const_init_valid) { int rel_off = (int)c - p->const_init_base; int bt = type->t & VT_BTYPE; - if (rel_off >= 0 && rel_off + size <= p->const_init_sym->const_init_size && !(type->t & VT_BITFIELD) && + if (rel_off >= 0 && rel_off + size <= p->const_init_sym->facts->const_init_size && !(type->t & VT_BITFIELD) && bt != VT_STRUCT) { if ((vtop->r & (VT_VALMASK | VT_LVAL | VT_SYM)) == VT_CONST) { uint64_t cval = (uint64_t)vtop->c.i; - unsigned char *dst = p->const_init_sym->const_init_data + rel_off; + unsigned char *dst = p->const_init_sym->facts->const_init_data + rel_off; switch (size) { case 1: @@ -737,13 +756,13 @@ void init_putv(init_params *p, CType *type, unsigned long c, int vreg) write64le(dst, cval); break; default: - p->const_init_sym->const_init_valid = 0; + p->const_init_sym->facts->const_init_valid = 0; break; } } else { - p->const_init_sym->const_init_valid = 0; + p->const_init_sym->facts->const_init_valid = 0; } } } diff --git a/source/frontend/gen/inline/analysis.c b/source/frontend/gen/inline/analysis.c index 75462450f..6b5ca51b3 100644 --- a/source/frontend/gen/inline/analysis.c +++ b/source/frontend/gen/inline/analysis.c @@ -98,19 +98,25 @@ int auto_inline_sig_ok(Sym *func_sym) return 0; if (p->type.t & VT_COMPLEX) return 0; - /* Only inline small plain structs (≤16 bytes). Non-static functions with - * struct params carrying pointer members can have complex aliasing (a - * pointer member aliasing another param) that the optimizer mishandles - * after inlining — so for non-static functions require the struct to be - * pure scalar/bitfield data. Vector types and large structs are always - * rejected. */ + /* Struct parameters: a non-static function takes only small plain + * structs (<=16 bytes, pure scalar/bitfield data -- a pointer member + * aliasing another param is mishandled after inlining). A static one + * takes any size: the expansion binds the argument by the same copy the + * call would make, and the copy is what inlining removes -- Zig passes + * whole structs by value to accessors that read one field of them + * (Sema.Block.src reads src_base_inst of a 100+ byte Block; the slice + * accessors take a 20-byte MultiArrayList.Slice). zig.c -O2 .text -47 KB; + * 64 bytes took -38 KB of it. Vector types are always rejected. */ if ((p->type.t & VT_BTYPE) == VT_STRUCT) { if (p->type.t & VT_VECTOR) return 0; int sz, al; sz = type_size(&p->type, &al); - if (sz > 16) + static int small_only = -1; /* TCC_INLINE_STRUCT_SMALL: the old 16-byte limit for static ones too */ + if (small_only < 0) + small_only = getenv("TCC_INLINE_STRUCT_SMALL") != NULL; + if (sz > 16 && (small_only || !(func_sym->type.t & VT_STATIC))) return 0; if (!(func_sym->type.t & VT_STATIC) && struct_has_pointer_member(&p->type)) return 0; diff --git a/source/frontend/gen/inline/const_eval.c b/source/frontend/gen/inline/const_eval.c index 21aecf942..9e0c649a9 100644 --- a/source/frontend/gen/inline/const_eval.c +++ b/source/frontend/gen/inline/const_eval.c @@ -87,6 +87,7 @@ int try_inline_const_eval(Sym *func_sym, SValue *args, int nb_args) int success = 0; jmp_buf saved_jmp_buf; int saved_nb_errors; + int saved_pack[PACK_STACK_SIZE + 1]; void (*saved_error_func)(void *opaque, const char *msg); void *saved_error_opaque; int saved_overlay_n; @@ -123,7 +124,7 @@ int try_inline_const_eval(Sym *func_sym, SValue *args, int nb_args) * are typically zero with a pending relocation (e.g. static T *p = &x), * so reading raw bytes would yield a bogus null value. */ if ((args[i].r & (VT_VALMASK | VT_SYM | VT_LVAL)) == (VT_CONST | VT_SYM | VT_LVAL) && args[i].sym && - !args[i].sym->a.possibly_written && !(args[i].type.t & (VT_ARRAY | VT_VLA))) + !args[i].sym->a.possibly_written && !args[i].sym->a.tentative && !(args[i].type.t & (VT_ARRAY | VT_VLA))) { int btype = args[i].type.t & VT_BTYPE; if (btype == VT_BYTE || btype == VT_SHORT || btype == VT_INT || btype == VT_LLONG || btype == VT_BOOL) @@ -372,6 +373,7 @@ int try_inline_const_eval(Sym *func_sym, SValue *args, int nb_args) ts->data.str = tok_str_buf(fn->func_str); ts->allocated_len = 1; /* pretend heap so tok_str_buf returns data.str */ ts->len = fn->func_str->len; + pp_pack_enter(tcc_state, fn->pack, saved_pack); begin_macro(ts, 2); /* Set up error recovery: expressions like x++ on a constant parameter @@ -600,6 +602,7 @@ int try_inline_const_eval(Sym *func_sym, SValue *args, int nb_args) * (e.g. string literals via decl_initializer_alloc) may push extra macro * stack entries (unget_tok) that aren't popped before we reach cleanup. */ end_macro_to(ts); + pp_pack_leave(tcc_state, saved_pack); /* Restore state */ nocode_wanted = saved_nocode_wanted; diff --git a/source/frontend/gen/inline/emit.c b/source/frontend/gen/inline/emit.c index 542c75249..1edf5326c 100644 --- a/source/frontend/gen/inline/emit.c +++ b/source/frontend/gen/inline/emit.c @@ -22,35 +22,25 @@ * Split out of tccgen.c; see docs/plan_tccgen_split.md. */ #include "gen_priv.h" +#include "tccdbgenv.h" /* parse a function defined by symbol 'sym' and generate its code in 'cur_text_section' */ -void ir_inline_stash_flush(TCCState *s1) -{ - for (int i = 0; i < s1->nb_stashed_func_irs; i++) - { - if (s1->stashed_func_irs[i].ir) - tcc_ir_free(s1->stashed_func_irs[i].ir); - } - tcc_free(s1->stashed_func_irs); - s1->stashed_func_irs = NULL; - s1->nb_stashed_func_irs = 0; - s1->stashed_func_irs_capacity = 0; -} - /* Remove [start, start+size) bytes from `sec`, shifting trailing data - * down. Updates symbol values and relocation offsets accordingly so + * down by a multiple of `unit` (a power of two: the alignment the tail must + * keep). Updates symbol values and relocation offsets accordingly so * the section stays self-consistent. Used by gen_late_reopt_functions * to reclaim the original code range of a function that is about to be - * re-emitted. Cross-section relocations resolve via symbol indices, so + * re-emitted, and by gc_unreferenced_statics. Cross-section relocations resolve via symbol indices, so * only this section's own reloc table needs r_offset adjustment — except * debug info (DWARF), which records text PCs as section symbol + * IN-PLACE addend: those addends are rewritten here (shifted past the * erased range, tombstoned inside it), so the erase is safe under -g. */ -static void erase_text_range(TCCState *s, Section *sec, addr_t start, addr_t size) +static void erase_section_range(TCCState *s, Section *sec, addr_t start, addr_t size, addr_t unit) { - if (size == 0 || !sec || !sec->data) + const int nobits = sec && sec->sh_type == SHT_NOBITS; + if (size == 0 || !sec || (!sec->data && !nobits)) return; if (start + size > sec->data_offset) return; @@ -64,16 +54,27 @@ static void erase_text_range(TCCState *s, Section *sec, addr_t start, addr_t siz * misaligned its `&al` pool entry). Function code is halfword-granular, so * `size` is even but may be 2 (mod 4); reclaim only a multiple-of-4 span and * leave up to 2 dead filler bytes so the tail's alignment is preserved. */ - addr_t shift = size & ~(addr_t)3; + addr_t shift = size & ~(unit - 1); if (shift == 0) return; + /* What identical code folding recorded about this section describes + * addresses that are about to move. */ + tcc_icf_section_changed(sec); + /* Shift the section's tail data down (by `shift`, not `size`). */ size_t tail_offset = (size_t)(start + size); size_t tail_len = sec->data_offset - tail_offset; - if (tail_len > 0) + if (tail_len > 0 && !nobits) memmove(sec->data + (tail_offset - shift), sec->data + tail_offset, tail_len); sec->data_offset -= shift; + /* Zero the vacated tail. The next thing appended here takes it as grown + * section memory, and a string literal relies on that being zero for its + * terminator: with the old bytes left in place, the second file of a + * two-file compile (120_alias at -O1) got "in unit2:\n" ending in the tail + * of the first file's format string instead of a NUL. */ + if (!nobits) + memset(sec->data + sec->data_offset, 0, shift); /* Adjust symbols that point into this section. */ Section *symtab = s->symtab; /* union alias of symtab_section */ @@ -219,6 +220,12 @@ static void erase_text_range(TCCState *s, Section *sec, addr_t start, addr_t siz } } +/* Code: the shift keeps the tail's 4-byte alignment (see above). */ +static void erase_text_range(TCCState *s, Section *sec, addr_t start, addr_t size) +{ + erase_section_range(s, sec, start, size, 4); +} + /* End-of-TU re-optimization pass. Functions whose IR contained a * would-be fold of a non-const `static` global blocked by the * VT_CONSTANT gate at first compile are marked func_late_reopt. Now @@ -247,6 +254,7 @@ void gen_late_reopt_functions(TCCState *s) * counters still record first-compile offsets nothing rewrites, so * that gate stays. */ int do_compact = !s->test_coverage; + int *aliases = NULL, nalias = 0, alias_cap = 0; for (i = 0; i < s->nb_inline_fns; ++i) { @@ -256,6 +264,10 @@ void gen_late_reopt_functions(TCCState *s) continue; if (!sym->type.ref->f.func_late_reopt) continue; + /* Its body is another function's now (tcc_icf_try_fold); re-emitting + * would erase that one's range and orphan every other caller. */ + if (sym->type.ref->f.func_icf_folded) + continue; /* Must still have saved tokens (the auto-inline post-emit path * preserves them when func_late_reopt is set). */ if (!fn->func_str) @@ -284,6 +296,26 @@ void gen_late_reopt_functions(TCCState *s) { addr_t old_start = esym->st_value & ~(addr_t)1; /* drop thumb LSB tag */ addr_t old_size = esym->st_size; + /* Identical code folding may have pointed other statics at this body + * (tcc_icf_try_fold): they alias it and follow it to where it is + * re-emitted. erase_text_range leaves a symbol inside the erased range + * where it was -- on the range's filler and the code shifted down after + * it -- and every call to the alias ran into the next function (Zig's + * link.File.startProgress, folded onto an identical function that late + * reopt re-emitted). */ + ElfW(Sym) *syms = (ElfW(Sym) *)symtab_section->data; + const int nsyms = symtab_section->data_offset / sizeof(ElfW(Sym)); + for (int k = 1; k < nsyms; k++) + if (k != sym->c && syms[k].st_shndx == late_reopt_sec->sh_num && + ELFW(ST_TYPE)(syms[k].st_info) == STT_FUNC && (syms[k].st_value & ~(addr_t)1) == old_start) + { + if (nalias == alias_cap) + { + alias_cap = alias_cap ? alias_cap * 2 : 4; + aliases = tcc_realloc(aliases, sizeof(int) * alias_cap); + } + aliases[nalias++] = k; + } erase_text_range(s, late_reopt_sec, old_start, old_size); } } @@ -304,11 +336,28 @@ void gen_late_reopt_functions(TCCState *s) Section *saved_text = cur_text_section; cur_text_section = late_reopt_sec; + int saved_pack[PACK_STACK_SIZE + 1]; + int saved_budget = called_once_budget_begin(body_len); tccpp_putfile(fn->filename); + pp_pack_enter(s, fn->pack, saved_pack); begin_macro(compile_ts, 1); next(); gen_function(sym); end_macro(); + pp_pack_leave(s, saved_pack); + called_once_budget_end(saved_budget); + if (nalias) + { + ElfW(Sym) *syms = (ElfW(Sym) *)symtab_section->data; + const ElfW(Sym) *body = &syms[sym->c]; + for (int k = 0; k < nalias; k++) + { + syms[aliases[k]].st_shndx = body->st_shndx; + syms[aliases[k]].st_value = body->st_value; + syms[aliases[k]].st_size = body->st_size; + } + nalias = 0; + } tok = saved_outer_tok; tokc = saved_outer_tokc; @@ -328,6 +377,7 @@ void gen_late_reopt_functions(TCCState *s) } } + tcc_free(aliases); tcc_close(); s->ir_late_reopt_phase = 0; } @@ -355,7 +405,10 @@ Section *function_text_section(TCCState *s1, Sym *sym) return sec; } -void gen_inline_functions(TCCState *s) +/* Generate the saved bodies the TU still owes: those referenced (sym->c) or + * not internal. With OWED_ONLY, just the `static inline` bodies referenced so + * far that were never compiled at all -- see gen_owed_inline_functions. */ +static void emit_inline_functions(TCCState *s, int owed_only) { Sym *sym; int inline_generated, i; @@ -398,6 +451,9 @@ void gen_inline_functions(TCCState *s) sym ? get_tok_str(sym->v & ~SYM_FIELD, NULL) : "", sym ? sym->c : -1, sym ? !!(sym->type.t & VT_INLINE) : -1, (sym && sym->a.addrtaken) ? 1 : 0, (sym && sym->type.ref && sym->type.ref->f.func_auto_inline) ? 1 : 0); + if (owed_only && (!sym || !(sym->type.t & VT_INLINE) || !sym->c || !elfsym(sym) || + elfsym(sym)->st_shndx != SHN_UNDEF)) + continue; if (sym && (sym->c || !(sym->type.t & VT_INLINE))) { /* Skip original va_arg_pack functions - only their clones get compiled */ @@ -406,12 +462,17 @@ void gen_inline_functions(TCCState *s) /* the function was used or forced (and then not internal): generate its code and convert it to a normal function */ fn->sym = NULL; + int saved_pack[PACK_STACK_SIZE + 1]; + int saved_budget = called_once_budget_begin(fn->func_str->len); tccpp_putfile(fn->filename); + pp_pack_enter(s, fn->pack, saved_pack); begin_macro(fn->func_str, 1); next(); cur_text_section = function_text_section(s, sym); gen_function(sym); end_macro(); + pp_pack_leave(s, saved_pack); + called_once_budget_end(saved_budget); inline_generated = 1; } @@ -420,6 +481,25 @@ void gen_inline_functions(TCCState *s) tcc_close(); } +void gen_inline_functions(TCCState *s) +{ + emit_inline_functions(s, 0); +} + +/* The end-of-TU analyses read facts that parsing a body records: a static's + * possibly_written/addrtaken (the late_reopt global_init fold) and its readers + * and writers (tcc_ir_tu_analyze_dead_statics). A `static inline` body a call + * site did not expand -- address taken, or a call the inliner declined -- is + * parsed only when gen_inline_functions emits it, which used to be after both: + * a static written only there folded to its initializer, and a store read only + * there was dropped as dead. Emit those bodies first. Whether one is owed is + * already settled by sym->c; a body still undefined in the symbol table was + * never compiled, unlike the kept-for-reopt entries sharing inline_fns. */ +void gen_owed_inline_functions(TCCState *s) +{ + emit_inline_functions(s, 1); +} + void free_inline_functions(TCCState *s) { int i; @@ -429,6 +509,869 @@ void free_inline_functions(TCCState *s) struct InlineFunc *fn = s->inline_fns[i]; if (fn->sym) tok_str_free(fn->func_str); + tcc_free(fn->pack); } dynarray_reset(&s->inline_fns, &s->nb_inline_fns); + tcc_free(s->inline_fn_by_tok); + s->inline_fn_by_tok = NULL; + s->inline_fn_by_tok_size = 0; + s->inline_fn_indexed = 0; +} + +/* The inline_fns entry holding `sym`'s body, or NULL. Entries are indexed by + * symbol token as they are appended; a call site used to scan the whole list, + * which with every function body deferred to the end of the TU made each call + * O(functions). */ +InlineFunc *inline_fn_lookup(TCCState *s, Sym *sym) +{ + for (; s->inline_fn_indexed < s->nb_inline_fns; s->inline_fn_indexed++) + { + InlineFunc *fn = s->inline_fns[s->inline_fn_indexed]; + if (!fn || !fn->sym) + continue; + int k = (fn->sym->v & ~SYM_FIELD) - TOK_IDENT; + if (k < 0) + continue; + if (k >= s->inline_fn_by_tok_size) + { + int n = s->inline_fn_by_tok_size ? s->inline_fn_by_tok_size : 256; + while (n <= k) + n *= 2; + s->inline_fn_by_tok = tcc_realloc(s->inline_fn_by_tok, sizeof(InlineFunc *) * n); + memset(s->inline_fn_by_tok + s->inline_fn_by_tok_size, 0, + sizeof(InlineFunc *) * (n - s->inline_fn_by_tok_size)); + s->inline_fn_by_tok_size = n; + } + /* First entry for a symbol wins, as the linear scan it replaces did. */ + InlineFunc *old = s->inline_fn_by_tok[k]; + if (!old || old->sym != fn->sym) + s->inline_fn_by_tok[k] = fn; + } + int k = (sym->v & ~SYM_FIELD) - TOK_IDENT; + if (k < 0 || k >= s->inline_fn_by_tok_size || !s->inline_fn_by_tok[k]) + return NULL; + InlineFunc *fn = s->inline_fn_by_tok[k]; + if (fn->sym == sym) + return fn; + /* The indexed entry was consumed, or belongs to another symbol of the same + * name (a nested function): fall back to the scan. */ + for (int i = 0; i < s->nb_inline_fns; i++) + if (s->inline_fns[i]->sym == sym) + return s->inline_fns[i]; + return NULL; +} + +/* ---- -finline-functions-called-once ------------------------------------ + * + * Every static function body was saved at its definition (decl.c + * defer_function_body) instead of generated. With all of them known, each + * function's references are counted over the saved bodies: a static function + * with exactly one call, no other mention and nothing already referencing it + * (sym->c: code or data emitted before the end of the TU) is inlined at that + * one call site whatever its own size, as long as the caller stays within its + * growth budget (called_once_budget_begin). It becomes a deferred inline body, which + * gen_inline_functions emits only if some reference survives -- a call site + * that declined to inline it -- so declining is always safe. The rest are + * generated here, callees before callers, so the size-gated auto-inliner also + * sees every small callee's body before its callers are compiled. */ + +/* Whether BODY uses alloca or __builtin_frame_address. alloca memory lives + * until the function returns: expanded into a caller, a block freed at each + * return of the callee would pile up in the caller's frame -- every + * iteration of a loop around the call -- which is why gcc does not inline + * such functions either. The frame address would be the caller's. Both + * also force a frame pointer on the caller, which loses r7 to allocation. */ +static int body_uses_frame_builtins(TokenString *body) +{ + const int *p = tok_str_buf(body), *end = p + body->len; + while (p < end) + { + int t; + CValue cv; + tok_get(&t, &p, &cv); + if (t == TOK_builtin_frame_address || t == TOK_alloca || t == TOK_builtin_alloca) + return 1; + } + return 0; +} + +/* Can the token-replay inliner express a called-once body? (Constructs it + * cannot reproduce, or whose meaning would change in the caller -- not a + * size or benefit judgement.) */ +static int called_once_body_ok(Sym *sym, TokenString *body) +{ + Sym *ref = sym->type.ref; + if (!ref || ref->f.func_noinline || ref->f.func_ctor || ref->f.func_dtor || sym->a.weak || + (ref->type.t & (VT_COMPLEX | VT_VECTOR))) + return 0; + for (Sym *p = ref->next; p; p = p->next) + if (p->type.t & (VT_COMPLEX | VT_VECTOR | VT_VLA)) + return 0; + int addr_of_label = 0, inline_asm = 0; + inline_scan_body_features(body, &addr_of_label, &inline_asm); + if (addr_of_label || inline_asm || inline_body_has_static_local(body) || inline_body_has_apply_args(body)) + return 0; + if ((ref->type.t & VT_BTYPE) != VT_VOID && !inline_body_has_return_stmt(body)) + return 0; + if (body_uses_frame_builtins(body)) + return 0; + return 1; +} + +/* How much a function may grow by expanding called-once functions, in saved + * body ints: up to CALLED_ONCE_LARGE_BODY in all, or to twice its own length + * when it is already larger -- gcc's large-function-insns/-growth rule for + * the same transform. Every called-once callee is a size win on its own, but + * a dispatcher calling hundreds of them (zig's Sema.analyzeBodyInner) became + * one 200,000-instruction function: register allocation there costs more + * than all the calls it saves, and no device has the memory to compile it. + * A callee that does not fit stays a call; its body is emitted standalone. + * Returns the previous budget, for called_once_budget_end. */ +/* 16000: zig.c -O2 .text 4,295,178 at 8000 -> 4,283,054; 12000 -4.5 KB, 20000+ back + * to +4 KB, and a growth factor of 3 or 4 +35..+59 KB -- the losses sit in + * callers that were already large, where every value the merged body keeps + * costs a spill and a frame past the 1020-byte short-encoding reach. */ +#define CALLED_ONCE_LARGE_BODY 16000 + +int called_once_budget_begin(int own_len) +{ + int saved = tcc_state->called_once_budget; + /* TCC_CALLED_ONCE_LARGE / TCC_CALLED_ONCE_GROWTH: the two limits, for experiments. */ + static int large = -1, growth = -1; + if (large < 0) + { + const char *e = getenv("TCC_CALLED_ONCE_LARGE"); + large = e ? atoi(e) : CALLED_ONCE_LARGE_BODY; + e = getenv("TCC_CALLED_ONCE_GROWTH"); + growth = e ? atoi(e) : 2; + } + int limit = own_len * growth > large ? own_len * growth : large; + tcc_state->called_once_budget = limit - own_len; + return saved; +} + +void called_once_budget_end(int saved) +{ + tcc_state->called_once_budget = saved; +} + +/* Scratch arrays of gen_deferred_function_bodies, kept where + * free_deferred_functions can release them: a compile error inside a + * deferred body longjmps out past the normal frees. */ +static void **deferred_scratch; +static int nb_deferred_scratch; + +static void *deferred_alloc(size_t size) +{ + void *p = tcc_mallocz(size ? size : 1); + dynarray_add(&deferred_scratch, &nb_deferred_scratch, p); + return p; +} + +/* Release the deferred bodies and the scratch; tccgen_finish calls it on + * every exit, including an error's. */ +void free_deferred_functions(TCCState *s) +{ + for (int i = 0; i < s->nb_deferred_fns; i++) + { + if (s->deferred_fns[i]->body) + tok_str_free(s->deferred_fns[i]->body); + tcc_free(s->deferred_fns[i]->pack); + } + dynarray_reset(&s->deferred_fns, &s->nb_deferred_fns); /* frees the entries too */ + for (int i = 0; i < s->nb_deferred_data; i++) + if (s->deferred_data[i]->init) + tok_str_free(s->deferred_data[i]->init); + dynarray_reset(&s->deferred_data, &s->nb_deferred_data); + for (int i = 0; i < nb_deferred_scratch; i++) + tcc_free(deferred_scratch[i]); + tcc_free(deferred_scratch); + deferred_scratch = NULL; + nb_deferred_scratch = 0; +} + +/* ---- -fdrop-unused-statics ------------------------------------------- + * + * At the end of parsing, a static function or initialized static object + * nothing live refers to is dropped, as gcc does at -O1: generated C + * (zig's) defines every helper and table it might need, and a program keeps + * few of them. The candidates are the saved bodies (defer_function_body), + * the saved initializers (defer_static_data), the static inline bodies + * waiting to be emitted, and the static tentative definitions. Roots are + * what must be emitted anyway: a non-static function, one marked used, + * weak, constructor/destructor or placed in a section, an alias target, and + * any candidate a relocation of already emitted code or data refers to. + * Liveness then flows along identifier mentions in the saved tokens -- a + * name is a reference however it is used (called, address taken, sizeof), + * which can only keep too much. The live objects are defined here, in + * declaration order, before the tentative definitions are settled and before + * any body is generated, so every fold of an initializer sees its bytes. */ + +static void tombstone_unused_static(Sym *sym) +{ + sym->a.tu_unused = 1; + ElfSym *esym = sym->c ? elfsym(sym) : NULL; + if (esym && (esym->st_shndx == SHN_UNDEF || esym->st_shndx == SHN_COMMON)) + { + esym->st_shndx = SHN_ABS; + esym->st_value = 0; + esym->st_size = 0; + } +} + +/* A scratch section for what a check-only parse lays out (check_only in + * tcc.h): never registered with the TU, so nothing in it is output. */ +static void check_scratch_begin(TCCState *s) +{ + if (s->check_scratch) + return; + s->check_scratch = tcc_mallocz(sizeof(Section) + 16); + s->check_scratch->s1 = s; + s->check_scratch->sh_type = SHT_PROGBITS; + s->check_scratch->sh_addralign = 1; +} + +/* TCC_CHECK_ALL_STATICS: also run the check-only parse over every live + * deferred body and initializer, just before it is generated -- a stress + * mode that puts check_only through the whole test corpus. */ +TCC_DBG_ENV_FLAG(check_every_static, "TCC_CHECK_ALL_STATICS") + +void prune_unused_statics(TCCState *s) +{ + const int nf = s->nb_deferred_fns, nd = s->nb_deferred_data; + if (!s->opt_drop_unused_statics || (!nf && !nd)) + goto define_all; + + /* Nodes: deferred functions, deferred objects, static inline bodies, + * static tentative definitions -- in that order. */ + const int ni = s->nb_inline_fns, nt = s->nb_tentative_syms; + const int cap = nf + nd + ni + nt; + Sym **node_sym = deferred_alloc(sizeof(Sym *) * cap); + TokenString **node_toks = deferred_alloc(sizeof(TokenString *) * cap); + uint8_t *root = deferred_alloc(cap), *live = deferred_alloc(cap); + int n = 0; + for (int i = 0; i < nf; i++) + { + DeferredFunc *d = s->deferred_fns[i]; + Sym *sym = d->sym; + node_sym[n] = sym; + node_toks[n] = d->body; + root[n] = !(sym->type.t & VT_STATIC) || sym->a.used || sym->a.weak || d->section || + (sym->type.ref && (sym->type.ref->f.func_ctor || sym->type.ref->f.func_dtor)); + n++; + } + for (int i = 0; i < nd; i++) + { + node_sym[n] = s->deferred_data[i]->sym; + node_toks[n] = s->deferred_data[i]->init; + root[n] = node_sym[n]->a.used; + n++; + } + for (int i = 0; i < ni; i++) + { + InlineFunc *fn = s->inline_fns[i]; + if (!fn || !fn->sym || !fn->func_str) + continue; + Sym *sym = fn->sym; + ElfSym *esym = sym->c ? elfsym(sym) : NULL; + node_sym[n] = sym; + node_toks[n] = fn->func_str; + /* Already compiled (a body kept for re-optimization): what it refers + * to stays referred to. */ + root[n] = !(sym->type.t & VT_STATIC) || sym->a.used || (esym && esym->st_shndx != SHN_UNDEF); + n++; + } + const int tent_base = n; + for (int i = 0; i < nt; i++) + { + Sym *sym = s->tentative_syms[i]; + if (!(sym->type.t & VT_STATIC) || !sym->a.tentative) + continue; + node_sym[n] = sym; + node_toks[n] = NULL; + root[n] = sym->a.used; + n++; + } + + /* Names -> nodes (a chain per name; an object with a deferred definition + * is also a tentative node, harmlessly). */ + const int ntok = tok_ident - TOK_IDENT; + int *head = deferred_alloc(sizeof(int) * (ntok > 0 ? ntok : 1)); + int *next_same = deferred_alloc(sizeof(int) * (n ? n : 1)); + for (int k = 0; k < ntok; k++) + head[k] = -1; + for (int i = 0; i < n; i++) + { + int k = (node_sym[i]->v & ~SYM_FIELD) - TOK_IDENT; + next_same[i] = -1; + if (k >= 0 && k < ntok) + { + next_same[i] = head[k]; + head[k] = i; + } + } + + /* Relocations of what is already emitted, and alias targets. */ + const int nelf = symtab_section->data_offset / sizeof(ElfW(Sym)); + int *by_elf = deferred_alloc(sizeof(int) * (nelf ? nelf : 1)); + for (int e = 0; e < nelf; e++) + by_elf[e] = -1; + for (int i = 0; i < n; i++) + if (node_sym[i]->c > 0 && node_sym[i]->c < nelf) + by_elf[node_sym[i]->c] = i; + for (int sh = 1; sh < s->nb_sections; sh++) + { + Section *sec = s->sections[sh]; + if (!sec || !(sec->sh_flags & SHF_ALLOC) || !sec->reloc) + continue; + ElfW_Rel *rel = (ElfW_Rel *)sec->reloc->data, *rel_end = (ElfW_Rel *)(sec->reloc->data + sec->reloc->data_offset); + for (; rel < rel_end; rel++) + { + int e = ELFW(R_SYM)(rel->r_info); + if (e > 0 && e < nelf && by_elf[e] >= 0) + root[by_elf[e]] = 1; + } + } + mark_pending_alias_targets_used(); + for (int i = 0; i < n; i++) + if (node_sym[i]->a.used) + root[i] = 1; + + /* Mentions, breadth first from the roots. A name after '.' or '->' is a + * member, not a reference. */ + int *work = deferred_alloc(sizeof(int) * (n ? n : 1)), nw = 0; + for (int i = 0; i < n; i++) + if (root[i]) + live[i] = 1, work[nw++] = i; + while (nw) + { + TokenString *str = node_toks[work[--nw]]; + if (!str) + continue; + const int *p = tok_str_buf(str), *end = p + str->len; + int prev = 0; + while (p < end) + { + int t; + CValue cv; + tok_get(&t, &p, &cv); + if (t >= TOK_IDENT && t - TOK_IDENT < ntok && prev != '.' && prev != TOK_ARROW) + for (int j = head[t - TOK_IDENT]; j >= 0; j = next_same[j]) + if (!live[j]) + live[j] = 1, work[nw++] = j; + if (t != TOK_LINENUM) + prev = t; + } + } + + /* A symbol reached under any of its nodes is live under all of them. */ + for (int i = 0; i < n; i++) + { + int k = (node_sym[i]->v & ~SYM_FIELD) - TOK_IDENT; + if (live[i] && k >= 0 && k < ntok) + for (int j = head[k]; j >= 0; j = next_same[j]) + if (node_sym[j] == node_sym[i]) + live[j] = 1; + } + + /* The dropped ones move to their own lists: check_dropped_statics parses + * them for their diagnostics once everything live is generated. */ + int kept = 0; + for (int i = 0; i < nf; i++) + { + DeferredFunc *d = s->deferred_fns[i]; + if (live[i]) + s->deferred_fns[kept++] = d; + else + { + dynarray_add(&s->dropped_fns, &s->nb_dropped_fns, d); + tombstone_unused_static(d->sym); + } + } + s->nb_deferred_fns = kept; + kept = 0; + for (int i = 0; i < nd; i++) + { + DeferredData *d = s->deferred_data[i]; + if (live[nf + i]) + s->deferred_data[kept++] = d; + else + { + dynarray_add(&s->dropped_data, &s->nb_dropped_data, d); + tombstone_unused_static(d->sym); + } + } + s->nb_deferred_data = kept; + for (int i = tent_base; i < n; i++) + if (!live[i]) + tombstone_unused_static(node_sym[i]); + +define_all: + if (s->nb_deferred_data) + { + const int saved_nocode_wanted = nocode_wanted; + nocode_wanted = DATA_ONLY_WANTED; + tcc_open_bf(s, ":deferred:", 0); + for (int i = 0; i < s->nb_deferred_data; i++) + { + DeferredData *d = s->deferred_data[i]; + if (check_every_static()) + { + check_scratch_begin(s); + check_dropped_deferred_data(d, tok_str_clone(d->init)); + } + define_deferred_data(d); + } + tcc_close(); + nocode_wanted = saved_nocode_wanted; + } +} + +/* ---- end-of-TU garbage collection ------------------------------------ + * + * prune_unused_statics decides from the source; what the optimizer does + * afterwards can leave more behind: a static function whose every call was + * inlined, an object whose stores dead_static_store removed, and whatever + * only those referred to. Here the generated object itself is collected, + * the way --gc-sections would at link time but per static: each local + * function or object symbol is a node (symbols whose ranges overlap -- ICF + * aliases -- share one), relocations are the edges, and the roots are + * everything else: code or data outside any node, global and weak symbols, + * statics marked used or aliased. A relocation against a section symbol + * names no node, so it pins its whole target section. The unreachable + * nodes are erased with erase_section_range, which keeps symbols, + * relocations and debug addends consistent, their symbols tombstoned. An + * undefined global nothing refers to any more -- its callers erased here, + * or rewritten by dead_static_store or late reopt -- becomes weak, so the + * link does not go looking for what nothing uses. */ + +typedef struct GcNode +{ + int sec; + addr_t start, end; +} GcNode; + +static int gc_node_cmp(const void *a, const void *b) +{ + const GcNode *x = a, *y = b; + if (x->sec != y->sec) + return x->sec < y->sec ? -1 : 1; + if (x->start != y->start) + return x->start < y->start ? -1 : 1; + return (x->end > y->end) - (x->end < y->end); +} + +/* The node of section `sec` containing `addr`, or -1. Nodes are sorted and + * disjoint; sec_first/sec_count index each section's run. */ +static int gc_node_at(const GcNode *nodes, const int *sec_first, const int *sec_count, int sec, addr_t addr) +{ + int lo = sec_first[sec], hi = lo + sec_count[sec] - 1; + while (lo <= hi) + { + int mid = (lo + hi) / 2; + if (addr < nodes[mid].start) + hi = mid - 1; + else if (addr >= nodes[mid].end) + lo = mid + 1; + else + return mid; + } + return -1; +} + +void gc_unreferenced_statics(TCCState *s) +{ + if (!s->opt_drop_unused_statics || s->test_coverage || !symtab_section || !symtab_section->data) + return; + const int nsyms = symtab_section->data_offset / sizeof(ElfW(Sym)); + const int nsec = s->nb_sections; + ElfW(Sym) *syms = (ElfW(Sym) *)symtab_section->data; + + /* Candidate ranges: local functions and objects with a size, in an + * allocated section. */ + GcNode *nodes = tcc_malloc(sizeof(GcNode) * (nsyms ? nsyms : 1)); + int nn = 0; + for (int i = 1; i < nsyms; i++) + { + ElfW(Sym) *e = &syms[i]; + int type = ELFW(ST_TYPE)(e->st_info); + if (ELFW(ST_BIND)(e->st_info) != STB_LOCAL || (type != STT_FUNC && type != STT_OBJECT) || !e->st_size) + continue; + if (e->st_shndx == SHN_UNDEF || e->st_shndx >= nsec || !s->sections[e->st_shndx] || + !(s->sections[e->st_shndx]->sh_flags & SHF_ALLOC)) + continue; + addr_t start = e->st_value & ~(addr_t)1; /* Thumb bit */ + nodes[nn].sec = e->st_shndx; + nodes[nn].start = start; + nodes[nn].end = start + e->st_size; + nn++; + } + if (!nn) + { + tcc_free(nodes); + return; + } + qsort(nodes, nn, sizeof *nodes, gc_node_cmp); + /* Merge overlapping ranges (ICF aliases, a symbol inside another). */ + int m = 0; + for (int i = 0; i < nn; i++) + { + if (m && nodes[m - 1].sec == nodes[i].sec && nodes[i].start < nodes[m - 1].end) + { + if (nodes[i].end > nodes[m - 1].end) + nodes[m - 1].end = nodes[i].end; + } + else + nodes[m++] = nodes[i]; + } + nn = m; + int *sec_first = tcc_mallocz(sizeof(int) * nsec), *sec_count = tcc_mallocz(sizeof(int) * nsec); + for (int i = nn - 1; i >= 0; i--) + sec_first[nodes[i].sec] = i, sec_count[nodes[i].sec]++; + + uint8_t *live = tcc_mallocz(nn), *pinned = tcc_mallocz(nsec); + int *edge_count = tcc_mallocz(sizeof(int) * (nn + 1)); + /* A relocation: from the node containing its offset (or a root, -1) to + * the node containing its target (or nothing, -1). */ + int nrel_total = 0; + for (int sh = 1; sh < nsec; sh++) + { + Section *sec = s->sections[sh]; + if (sec && (sec->sh_flags & SHF_ALLOC) && sec->reloc) + nrel_total += sec->reloc->data_offset / sizeof(ElfW_Rel); + } + int *rel_from = tcc_malloc(sizeof(int) * (nrel_total ? nrel_total : 1)); + int *rel_to = tcc_malloc(sizeof(int) * (nrel_total ? nrel_total : 1)); + int nr = 0; + for (int sh = 1; sh < nsec; sh++) + { + Section *sec = s->sections[sh]; + if (!sec || !(sec->sh_flags & SHF_ALLOC) || !sec->reloc) + continue; + ElfW_Rel *rel = (ElfW_Rel *)sec->reloc->data, *rel_end = (ElfW_Rel *)(sec->reloc->data + sec->reloc->data_offset); + for (; rel < rel_end; rel++) + { + int t = ELFW(R_SYM)(rel->r_info); + if (t <= 0 || t >= nsyms) + continue; + ElfW(Sym) *e = &syms[t]; + if (e->st_shndx == SHN_UNDEF || e->st_shndx >= nsec) + continue; + if (ELFW(ST_TYPE)(e->st_info) == STT_SECTION) + { + pinned[e->st_shndx] = 1; + continue; + } + int to = gc_node_at(nodes, sec_first, sec_count, e->st_shndx, e->st_value & ~(addr_t)1); + if (to < 0) + continue; + int from = gc_node_at(nodes, sec_first, sec_count, sh, rel->r_offset); + rel_from[nr] = from; + rel_to[nr] = to; + nr++; + if (from >= 0) + edge_count[from]++; + } + } + /* Edges by source node (CSR). */ + int *edge_start = tcc_mallocz(sizeof(int) * (nn + 1)); + for (int i = 0; i < nn; i++) + edge_start[i + 1] = edge_start[i] + edge_count[i]; + int *edges = tcc_malloc(sizeof(int) * (edge_start[nn] ? edge_start[nn] : 1)); + memset(edge_count, 0, sizeof(int) * (nn + 1)); + int *work = tcc_malloc(sizeof(int) * nn), nw = 0; +#define GC_MARK(n) \ + do \ + { \ + int gc_n_ = (n); \ + if (gc_n_ >= 0 && !live[gc_n_]) \ + live[gc_n_] = 1, work[nw++] = gc_n_; \ + } while (0) + for (int r = 0; r < nr; r++) + { + if (rel_from[r] < 0) + GC_MARK(rel_to[r]); + else + edges[edge_start[rel_from[r]] + edge_count[rel_from[r]]++] = rel_to[r]; + } + /* Roots: pinned sections, global and weak symbols, used statics. */ + for (int i = 0; i < nn; i++) + if (pinned[nodes[i].sec]) + GC_MARK(i); + for (int i = 1; i < nsyms; i++) + { + ElfW(Sym) *e = &syms[i]; + if (ELFW(ST_BIND)(e->st_info) != STB_LOCAL && e->st_shndx != SHN_UNDEF && e->st_shndx < nsec) + GC_MARK(gc_node_at(nodes, sec_first, sec_count, e->st_shndx, e->st_value & ~(addr_t)1)); + } + for (Sym *g = global_stack; g; g = g->prev) + if (g->a.used && g->c > 0 && g->c < nsyms && syms[g->c].st_shndx < nsec) + GC_MARK(gc_node_at(nodes, sec_first, sec_count, syms[g->c].st_shndx, syms[g->c].st_value & ~(addr_t)1)); + while (nw) + { + int n = work[--nw]; + for (int k = edge_start[n]; k < edge_start[n + 1]; k++) + GC_MARK(edges[k]); + } +#undef GC_MARK + + /* Erase the dead nodes, highest address first so the lower ones stay put. + * Their symbols go first: erase_section_range leaves symbols inside the + * range where they were. */ + for (int i = nn - 1; i >= 0; i--) + { + if (live[i]) + continue; + Section *sec = s->sections[nodes[i].sec]; + for (int k = 1; k < nsyms; k++) + { + ElfW(Sym) *e = &syms[k]; + int type = ELFW(ST_TYPE)(e->st_info); + if (e->st_shndx != nodes[i].sec || type == STT_SECTION || type == STT_FILE) + continue; + addr_t v = e->st_value & ~(addr_t)1; + if (v < nodes[i].start || v >= nodes[i].end) + continue; + const char *nm = symtab_section->link ? (const char *)symtab_section->link->data + e->st_name : ""; + if (nm[0] == '$' && (nm[1] == 't' || nm[1] == 'd') && nm[2] == 0) + continue; /* mapping markers: erase_section_range retargets them */ + e->st_shndx = SHN_ABS; + e->st_value = 0; + e->st_size = 0; + } + addr_t unit = sec->sh_addralign > 0 ? sec->sh_addralign : 1; + if ((sec->sh_flags & SHF_EXECINSTR) && unit < 4) + unit = 4; + erase_section_range(s, sec, nodes[i].start, nodes[i].end - nodes[i].start, unit); + } + + /* Undefined globals nothing refers to any more. */ + int *refs_after = tcc_mallocz(sizeof(int) * nsyms); + for (int sh = 1; sh < nsec; sh++) + { + Section *sr = s->sections[sh]; + if (!sr || sr->sh_type != SHT_RELX || sr->link != symtab_section || !sr->data) + continue; + for (ElfW_Rel *rel = (ElfW_Rel *)sr->data; (unsigned char *)rel < sr->data + sr->data_offset; rel++) + if (ELFW(R_SYM)(rel->r_info) < (unsigned)nsyms) + refs_after[ELFW(R_SYM)(rel->r_info)]++; + } + for (int i = 1; i < nsyms; i++) + if (syms[i].st_shndx == SHN_UNDEF && ELFW(ST_BIND)(syms[i].st_info) == STB_GLOBAL && !refs_after[i]) + syms[i].st_info = ELFW(ST_INFO)(STB_WEAK, ELFW(ST_TYPE)(syms[i].st_info)); + tcc_free(refs_after); + + tcc_free(work); + tcc_free(edges); + tcc_free(edge_start); + tcc_free(rel_to); + tcc_free(rel_from); + tcc_free(edge_count); + tcc_free(pinned); + tcc_free(live); + tcc_free(sec_count); + tcc_free(sec_first); + tcc_free(nodes); +} + +/* Parse the statics prune_unused_statics dropped, for their diagnostics + * only, after everything live has been generated -- so what the parse + * records on symbols (possibly_written, addrtaken) cannot pessimize code. + * Debug output is off: no line table or DIE may describe them. */ +void check_dropped_statics(TCCState *s) +{ + if (!s->nb_dropped_fns && !s->nb_dropped_data) + return; + check_scratch_begin(s); + const char saved_debug_modes = debug_modes; + debug_modes = 0; + tcc_open_bf(s, ":dropped:", 0); + for (int i = 0; i < s->nb_dropped_data; i++) + { + DeferredData *d = s->dropped_data[i]; + TokenString *init = d->init; + d->init = NULL; + check_dropped_deferred_data(d, init); + } + for (int i = 0; i < s->nb_dropped_fns; i++) + { + DeferredFunc *d = s->dropped_fns[i]; + TokenString *body = d->body; + d->body = NULL; + check_dropped_function(d, body); + } + tcc_close(); + debug_modes = saved_debug_modes; + free_dropped_statics(s); +} + +/* Also on every exit, an error's included (tccgen_finish). */ +void free_dropped_statics(TCCState *s) +{ + for (int i = 0; i < s->nb_dropped_fns; i++) + { + if (s->dropped_fns[i]->body) + tok_str_free(s->dropped_fns[i]->body); + tcc_free(s->dropped_fns[i]->pack); + } + dynarray_reset(&s->dropped_fns, &s->nb_dropped_fns); + for (int i = 0; i < s->nb_dropped_data; i++) + if (s->dropped_data[i]->init) + tok_str_free(s->dropped_data[i]->init); + dynarray_reset(&s->dropped_data, &s->nb_dropped_data); + if (s->check_scratch) + { + tcc_free(s->check_scratch->data); + tcc_free(s->check_scratch); + s->check_scratch = NULL; + } + s->check_only = 0; +} + +void gen_deferred_function_bodies(TCCState *s) +{ + const int n = s->nb_deferred_fns; + if (!n) + return; + const int ntok = tok_ident - TOK_IDENT; + int *by_tok = deferred_alloc(sizeof(int) * (ntok > 0 ? ntok : 1)); + for (int k = 0; k < ntok; k++) + by_tok[k] = -1; + for (int i = 0; i < n; i++) + { + int k = (s->deferred_fns[i]->sym->v & ~SYM_FIELD) - TOK_IDENT; + if (k >= 0 && k < ntok) + by_tok[k] = i; + } + + /* References to deferred functions from the saved bodies: a name followed + * by '(' is a call; anything else (address taken, sizeof, a same-named + * local) is another mention, which rules the function out. */ + int *calls = deferred_alloc(sizeof(int) * n), *mentions = deferred_alloc(sizeof(int) * n); + int *caller = deferred_alloc(sizeof(int) * n); + int *edge_start = deferred_alloc(sizeof(int) * (n + 1)), nedges = 0, edge_cap = 0; + int *edges = NULL; + for (int i = 0; i < n; i++) + { + edge_start[i] = nedges; + const int *p = tok_str_buf(s->deferred_fns[i]->body); + int t; + CValue cv; + tok_get(&t, &p, &cv); + while (t != TOK_EOF && t != 0) + { + int next_t; + CValue ncv; + tok_get(&next_t, &p, &ncv); + if (t >= TOK_IDENT && t - TOK_IDENT < ntok && by_tok[t - TOK_IDENT] >= 0) + { + int j = by_tok[t - TOK_IDENT]; + if (next_t == '(') + { + calls[j]++; + caller[j] = i; + if (nedges == edge_cap) + { + /* Grown by copying into a fresh tracked array: a realloc'd block + * would go stale in the scratch list. */ + int *grown = deferred_alloc(sizeof(int) * (edge_cap ? edge_cap * 2 : 1024)); + if (nedges) + memcpy(grown, edges, sizeof(int) * nedges); + edges = grown; + edge_cap = edge_cap ? edge_cap * 2 : 1024; + } + edges[nedges++] = j; + } + else + mentions[j]++; + } + t = next_t; + } + } + edge_start[n] = nedges; + + uint8_t *once = deferred_alloc(n); + for (int j = 0; j < n; j++) + { + DeferredFunc *d = s->deferred_fns[j]; + Sym *sym = d->sym; + if (!s->opt_inline_called_once || !(sym->type.t & VT_STATIC) || calls[j] != 1 || mentions[j] || + caller[j] == j || sym->c || sym->a.addrtaken || !d->body || !called_once_body_ok(sym, d->body)) + { + /* TCC_CALLED_ONCE_DBG: why a function with one call stays a call. */ + static int dbg = -1; + if (dbg < 0) + dbg = getenv("TCC_CALLED_ONCE_DBG") != NULL; + if (dbg && calls[j] == 1 && s->opt_inline_called_once) + fprintf(stderr, "[ONCE] %s len=%d decline=%s caller=%s\n", get_tok_str(sym->v, NULL), d->body ? d->body->len : -1, + !(sym->type.t & VT_STATIC) ? "not-static" + : mentions[j] ? "mentioned" + : caller[j] == j ? "self" + : sym->c ? "emitted" + : sym->a.addrtaken ? "addrtaken" + : !d->body ? "no-body" + : "body-rules", + caller[j] >= 0 && caller[j] < n ? get_tok_str(s->deferred_fns[caller[j]]->sym->v, NULL) : "?"); + continue; + } + once[j] = 1; + /* A deferred inline body: the call site replays it, gen_inline_functions + * emits it only if a reference survives. */ + InlineFunc *fn = tcc_mallocz(sizeof *fn + strlen(d->filename)); + strcpy(fn->filename, d->filename); + fn->sym = sym; + fn->func_str = d->body; + fn->pack = d->pack; + d->body = NULL; + d->pack = NULL; + sym->type.t |= VT_INLINE; + sym->type.ref->f.func_auto_inline = 1; + sym->type.ref->f.func_deferred_inline = 1; + sym->type.ref->f.func_called_once = 1; + dynarray_add(&s->inline_fns, &s->nb_inline_fns, fn); + } + + /* Generate the others callees first (post-order over the call edges, + * iteratively: call chains in generated C run deep). */ + uint8_t *state = deferred_alloc(n); /* 0 new, 1 on the stack, 2 done */ + int *stack = deferred_alloc(sizeof(int) * 2 * (n + 1)), sp = 0; + tcc_open_bf(s, ":deferred:", 0); + for (int root = 0; root < n; root++) + { + if (state[root]) + continue; + stack[sp++] = root; + stack[sp++] = edge_start[root]; + state[root] = 1; + while (sp) + { + int i = stack[sp - 2], e = stack[sp - 1]; + if (e < edge_start[i + 1]) + { + stack[sp - 1] = e + 1; + int j = edges[e]; + if (!state[j]) + { + state[j] = 1; + stack[sp++] = j; + stack[sp++] = edge_start[j]; + } + continue; + } + sp -= 2; + state[i] = 2; + if (!once[i] && s->deferred_fns[i]->body) + { + if (check_every_static()) + { + check_scratch_begin(s); + check_dropped_function(s->deferred_fns[i], tok_str_clone(s->deferred_fns[i]->body)); + } + define_deferred_function(s->deferred_fns[i]); + } + } + } + tcc_close(); + + free_deferred_functions(s); } diff --git a/source/frontend/gen/nested/nested.c b/source/frontend/gen/nested/nested.c index 6971f463c..097e9d9b5 100644 --- a/source/frontend/gen/nested/nested.c +++ b/source/frontend/gen/nested/nested.c @@ -43,7 +43,9 @@ static NestedFunc *find_nested_func_by_sym(Sym *sym) void setup_nested_func_trampoline(Sym *s) { NestedFunc *nf = find_nested_func_by_sym(s); - if (!nf) + /* A body parsed for its diagnostics only (check_only) has no code to need + * a trampoline, and the chain slot and trampoline symbols would outlive it. */ + if (!nf || tcc_state->check_only) return; nf->trampoline_needed = 1; @@ -157,6 +159,7 @@ typedef struct int ind; int rsym; int func_ind; + int func_align_pad; const char *funcname; CType func_vt; int func_var; @@ -199,6 +202,7 @@ void compile_nested_functions(Sym *parent_sym) saved.ind = ind; saved.rsym = rsym; saved.func_ind = func_ind; + saved.func_align_pad = func_align_pad; saved.funcname = funcname; saved.func_vt = func_vt; saved.func_var = func_var; @@ -383,6 +387,7 @@ void compile_nested_functions(Sym *parent_sym) the parent's codegen will emit at the CURRENT ind (after nested funcs) */ rsym = saved.rsym; func_ind = saved.func_ind; + func_align_pad = saved.func_align_pad; funcname = saved.funcname; func_vt = saved.func_vt; func_var = saved.func_var; diff --git a/source/frontend/gen/op/fold_math.c b/source/frontend/gen/op/fold_math.c index e59d9f2ca..d848a953b 100644 --- a/source/frontend/gen/op/fold_math.c +++ b/source/frontend/gen/op/fold_math.c @@ -163,9 +163,9 @@ int svalue_get_conservative_max_u64(SValue *sv, unsigned long long *out_max) { Sym *sym = find_local_scalar_sym_for_svalue(sv); - if (sym && sym->objsize_max_valid) + if (sym && SYM_FACTS(sym)->objsize_max_valid) { - *out_max = sym->objsize_max_value; + *out_max = SYM_FACTS(sym)->objsize_max_value; return 1; } } @@ -190,9 +190,9 @@ int svalue_get_conservative_string_bytes_u64(SValue *sv, unsigned long long *out { Sym *sym = find_local_scalar_sym_for_svalue(sv); - if (sym && sym->objsize_strlen_valid) + if (sym && SYM_FACTS(sym)->objsize_strlen_valid) { - *out_max = sym->objsize_strlen_value; + *out_max = SYM_FACTS(sym)->objsize_strlen_value; return 1; } } @@ -233,8 +233,10 @@ int chk_get_conservative_sprintf_bytes(int tok, int fmt_idx, SValue *all_args, i void update_local_scalar_max_bound(SValue *dst, SValue *src) { Sym *sym; + SymLocalFacts *f; unsigned long long max_value; unsigned long long max_strlen; + int have_max, have_strlen; if ((dst->r & VT_VALMASK) != VT_LOCAL) return; @@ -242,29 +244,35 @@ void update_local_scalar_max_bound(SValue *dst, SValue *src) if (!sym) return; - if (!svalue_get_conservative_max_u64(src, &max_value)) + have_max = svalue_get_conservative_max_u64(src, &max_value); + have_strlen = svalue_get_conservative_string_bytes_u64(src, &max_strlen); + /* Nothing known and nothing recorded: clearing would only allocate zeroes. */ + if (!have_max && !have_strlen && !sym->facts) + return; + f = sym_facts(sym); + + if (!have_max) { - sym->objsize_max_valid = 0; - sym->objsize_max_value = 0; + f->objsize_max_valid = 0; + f->objsize_max_value = 0; } - - if (svalue_get_conservative_max_u64(src, &max_value)) + else { - if (!sym->objsize_max_valid || max_value > sym->objsize_max_value) - sym->objsize_max_value = max_value; - sym->objsize_max_valid = 1; + if (!f->objsize_max_valid || max_value > f->objsize_max_value) + f->objsize_max_value = max_value; + f->objsize_max_valid = 1; } - if (svalue_get_conservative_string_bytes_u64(src, &max_strlen)) + if (have_strlen) { - if (!sym->objsize_strlen_valid || max_strlen > sym->objsize_strlen_value) - sym->objsize_strlen_value = max_strlen; - sym->objsize_strlen_valid = 1; + if (!f->objsize_strlen_valid || max_strlen > f->objsize_strlen_value) + f->objsize_strlen_value = max_strlen; + f->objsize_strlen_valid = 1; } else { - sym->objsize_strlen_valid = 0; - sym->objsize_strlen_value = 0; + f->objsize_strlen_valid = 0; + f->objsize_strlen_value = 0; } } /* ============================================================================ diff --git a/source/frontend/gen/op/vector.c b/source/frontend/gen/op/vector.c index 4c68a0709..f1652078d 100644 --- a/source/frontend/gen/op/vector.c +++ b/source/frontend/gen/op/vector.c @@ -71,8 +71,9 @@ unsigned char *find_sv_const_init(const SValue *sv, int min_size) Sym *s; for (s = local_stack; s; s = s->prev) { - if (s->const_init_data && s->const_init_valid && (int)s->c == addr && s->const_init_size >= min_size) - return s->const_init_data; + const SymLocalFacts *f = SYM_FACTS(s); + if (f->const_init_data && f->const_init_valid && (int)s->c == addr && f->const_init_size >= min_size) + return f->const_init_data; } return NULL; } @@ -98,9 +99,10 @@ unsigned char *find_sv_vec_literal_init(const SValue *sv, int min_size) Sym *s; for (s = local_stack; s; s = s->prev) { - if (s->const_init_data && s->const_init_valid && s->v >= SYM_FIRST_ANOM && (int)s->c == addr && - s->const_init_size >= min_size) - return s->const_init_data; + const SymLocalFacts *f = SYM_FACTS(s); + if (f->const_init_data && f->const_init_valid && s->v >= SYM_FIRST_ANOM && (int)s->c == addr && + f->const_init_size >= min_size) + return f->const_init_data; } return NULL; } @@ -197,10 +199,11 @@ static int64_t eval_vec_const_op(int op, int64_t a, int64_t b, int is_unsigned) void attach_const_init_to_temp(int frame_offset, int size, const unsigned char *data) { Sym *s = sym_push2(&local_stack, SYM_FIRST_ANOM, VT_INT, frame_offset); - s->const_init_data = tcc_malloc(size); - memcpy(s->const_init_data, data, size); - s->const_init_size = size; - s->const_init_valid = 1; + SymLocalFacts *f = sym_facts(s); + f->const_init_data = tcc_malloc(size); + memcpy(f->const_init_data, data, size); + f->const_init_size = size; + f->const_init_valid = 1; } /* ---- element-major fusion of chained vector expressions ------------------ diff --git a/source/frontend/gen/stmt/block.c b/source/frontend/gen/stmt/block.c index ff61cb16a..951e64568 100644 --- a/source/frontend/gen/stmt/block.c +++ b/source/frontend/gen/stmt/block.c @@ -143,22 +143,7 @@ static void lblock(int *bsym, int *csym) } } -static void block_1(int flags); - -/* Wrapper that scopes the variadic struct-argument temp pool to one - * statement. Slots reserved while parsing this statement (and its - * sub-expressions) are released on exit so sibling statements reuse them, - * but a nested block() — e.g. a GNU statement-expression used as a call - * argument — saves/restores the mask and so cannot recycle a slot the - * enclosing call still has in flight. */ void block(int flags) -{ - uint64_t saved_arg_struct_busy = arg_struct_temp_busy; - block_1(flags); - arg_struct_temp_busy = saved_arg_struct_busy; -} - -static void block_1(int flags) { int a, b, c, d, e, t; struct scope o; @@ -172,6 +157,10 @@ static void block_1(int flags) } again: + /* Only the expression statement below discards a value; every other + * statement (a declaration's initializer, a nested one inside a + * statement expression) uses it. */ + discarded_call_vtop = NULL; t = tok; /* If the token carries a value, next() might destroy it. Only with invalid code such as f(){"123"4;} */ @@ -907,7 +896,9 @@ static void block_1(int flags) } else { + discarded_call_vtop = vtop; gexpr(); + discarded_call_vtop = NULL; tcc_ir_codegen_drop_return(tcc_state->ir); gv_discarded_volatile(); vpop(); diff --git a/source/frontend/gen/stmt/cleanup.c b/source/frontend/gen/stmt/cleanup.c index 2472790ca..3ebaedc9f 100644 --- a/source/frontend/gen/stmt/cleanup.c +++ b/source/frontend/gen/stmt/cleanup.c @@ -151,7 +151,7 @@ static int try_inline_cleanup_call(Sym *fs, Sym *vs) psize = 4; if (palign < 4) palign = 4; - loc = (loc - psize) & -palign; + loc = tcc_ir_frame_alloc(loc, psize, -palign); int pv = param_sym->v & ~SYM_FIELD; if (pv == 0) @@ -210,10 +210,13 @@ static int try_inline_cleanup_call(Sym *fs, Sym *vs) inline_ts->data.str = tok_str_buf(inline_fn->func_str); inline_ts->allocated_len = 1; inline_ts->len = inline_fn->func_str->len; + int saved_pack[PACK_STACK_SIZE + 1]; + pp_pack_enter(tcc_state, inline_fn->pack, saved_pack); begin_macro(inline_ts, 2); next(); block(0); end_macro(); + pp_pack_leave(tcc_state, saved_pack); inline_restore_label_bindings(inline_label_tokens, saved_inline_labels, nb_inline_label_tokens); tok = saved_tok; diff --git a/source/frontend/gen/stmt/ret.c b/source/frontend/gen/stmt/ret.c index cedcfefe8..4730b1068 100644 --- a/source/frontend/gen/stmt/ret.c +++ b/source/frontend/gen/stmt/ret.c @@ -90,7 +90,7 @@ void gfunc_return(CType *func_type) /* Source is an rvalue (register pair) — spill to temp local. * This path handles _Complex float/int (8 bytes) which can fit * in a register pair and be stored via a single 64-bit STORE. */ - loc = (loc - complex_size) & -complex_align; + loc = tcc_ir_frame_alloc(loc, complex_size, -complex_align); int tmp_loc = loc; SValue tmp_dst; @@ -290,14 +290,14 @@ void gfunc_return(CType *func_type) int s_size, s_align; s_size = type_size(func_type, &s_align); int has_struct_vla = struct_has_vla_member(func_type); - /* Cap at 8 bytes to avoid a store-forwarding width-mismatch issue - * in the IR optimizer: a 4-byte LOAD that should see an earlier - * wider STORE can incorrectly forward from a stale narrow STORE at - * the same address. Triggers on 16-byte vector literals built from - * scalar components (e.g. (__m128i){a, b} — zero-init + 8-byte - * stores), pr92618. */ + /* Up to 16 bytes. This stopped at 8 while store forwarding could + * feed a 4-byte LOAD from a stale narrow STORE where an earlier wider + * one covered the address (pr92618's 16-byte vector literal built + * from scalar components: zero-init + 8-byte stores); pr92618 passes + * at 16 now, as vstore's same cap does -- and like it, only when + * optimizing, where the word copies fold. */ if (tcc_state->ir && !has_struct_vla && (vtop->r & VT_LVAL) && s_size > 0 && - s_size <= 8 && !(s_size & 3) && !(s_align & 3) && !NOEVAL_WANTED) + s_size <= (tcc_state->optimize > 0 ? 16 : 8) && !(s_size & 3) && !(s_align & 3) && !NOEVAL_WANTED) { SValue src_mem = *vtop; @@ -450,7 +450,7 @@ void gfunc_return(CType *func_type) (align & (ret_align - 1)) && ((vtop->r & VT_VALMASK) < VT_CONST /* pointer to struct */ || (vtop->c.i & (ret_align - 1)))) { - loc = (loc - size) & -ret_align; + loc = tcc_ir_frame_alloc(loc, size, -ret_align); addr = loc; type = *func_type; vset(&type, VT_LOCAL | VT_LVAL, addr); diff --git a/source/frontend/gen/store/struct_copy.c b/source/frontend/gen/store/struct_copy.c index d9bd8f1a8..d23f5beda 100644 --- a/source/frontend/gen/store/struct_copy.c +++ b/source/frontend/gen/store/struct_copy.c @@ -253,3 +253,165 @@ int struct_is_single_1byte_scalar_member(const CType *type) return 0; return type_size(&f->type, &align) == 1; } + +/* Mark the bytes of `t` placed at `base` as covered, rejecting anything the + * chunked copy below cannot move without splitting an access: a scalar or + * bitfield unit wider than the chunk width `w` (a later read of it would be + * wider than the stores that wrote it, which store-load forwarding must not + * partial-forward), or one not naturally aligned inside the aggregate. */ +static int small_copy_cover(const CType *t, int base, int w, int size, unsigned char *covered) +{ + int align, n; + + if (t->t & VT_ARRAY) + { + const CType *et = pointed_type((CType *)t); + int esize = type_size((CType *)et, &align); + if (!t->ref || t->ref->c < 0 || esize <= 0) + return 0; /* flexible/incomplete array member */ + for (int i = 0; i < t->ref->c; i++) + if (!small_copy_cover(et, base + i * esize, w, size, covered)) + return 0; + return 1; + } + if ((t->t & VT_BTYPE) == VT_STRUCT) /* struct or union */ + { + if (!t->ref) + return 0; + for (Sym *f = t->ref->next; f; f = f->next) + { + if (f->type.t & VT_BITFIELD) + { + int bw = bitfield_unit_width(f); + int off = base + (f->c & ~(bw - 1)); + if (bw != 1 && bw != 2 && bw != 4) + return 0; + if (bw > w || off < 0 || off + bw > size || (off % bw)) + return 0; + for (int k = off; k < off + bw; k++) + covered[k] = 1; + } + else if (!small_copy_cover(&f->type, base + f->c, w, size, covered)) + return 0; + } + return 1; + } + if (t->t & (VT_VECTOR | VT_COMPLEX | VT_VLA)) + return 0; + n = type_size((CType *)t, &align); + if (n == 0) + return 1; /* zero-sized member */ + if ((n != 1 && n != 2 && n != 4) || n > w || base < 0 || base + n > size || (base % n)) + return 0; /* 8-byte scalars keep memmove: a split one would be read wider */ + for (int k = base; k < base + n; k++) + covered[k] = 1; + return 1; +} + +/* Plan the copy of a small aggregate for ir_emit_small_aggregate_copy: every + * chunk has one width w = min(4, the type's alignment, the alignment of each + * frame/global base offset), so each field sits inside a single chunk and no + * chunk is narrower than a field it holds. Returns the number of chunks that + * carry data (pure-padding chunks are not copied), or 0 if the type cannot be + * tiled that way. `src_off`/`dst_off` are the c.i of a LOCAL/GLOBAL side, or + * -1 for a register-deref side, which is trusted to the type's alignment. */ +int small_aggregate_copy_plan(const CType *stype, int size, int align, int src_off, int dst_off, int *w_out, + unsigned char *covered) +{ + int w = align < 4 ? align : 4, chunks = 0; + if (size <= 0 || size > SMALL_AGGREGATE_COPY_MAX || w <= 0) + return 0; + while (w > 1 && ((src_off >= 0 && (src_off % w)) || (dst_off >= 0 && (dst_off % w)))) + w >>= 1; + if (size % w) + return 0; + memset(covered, 0, SMALL_AGGREGATE_COPY_MAX); + if (!small_copy_cover(stype, 0, w, size, covered)) + return 0; + for (int p = 0; p < size; p += w) + { + int any = 0; + for (int k = p; k < p + w; k++) + any |= covered[k]; + chunks += any; + } + *w_out = w; + return chunks; +} + +/* Address `side` + off as an lvalue of type `ct`. A LOCAL/GLOBAL side takes the + * offset in c.i; a register-deref side (whose c.i the codegen does not honour) + * gets an explicit ADD, which displacement fusion later folds into the access. */ +static SValue small_copy_lval(const SValue *side, int is_deref, int off, CType ct) +{ + SValue v; + svalue_init(&v); + v.type = ct; + if (is_deref && off != 0) + { + SValue base, imm, ptr; + svalue_init(&base); + base.type.t = VT_PTR; + base.vr = side->vr; + base.r = 0; + svalue_init(&imm); + imm.type.t = VT_INT; + imm.r = VT_CONST; + imm.vr = -1; + imm.c.i = off; + svalue_init(&ptr); + ptr.type.t = VT_PTR; + ptr.vr = tcc_ir_get_vreg_temp(tcc_state->ir); + ptr.r = 0; + tcc_ir_put(tcc_state->ir, TCCIR_OP_ADD, &base, &imm, &ptr); + v.r = VT_LVAL; + v.vr = ptr.vr; + return v; + } + v.r = side->r; + v.vr = side->vr; + v.sym = side->sym; + v.c.i = side->c.i + off; + return v; +} + +/* Copy a small aggregate planned by small_aggregate_copy_plan as w-byte LOAD / + * STORE pairs visible to the optimizer, instead of an opaque __aeabi_memmove + * call: every LOAD first, then every STORE, so an overlapping copy still has + * memmove semantics. */ +void ir_emit_small_aggregate_copy(const SValue *src, int src_deref, const SValue *dst, int dst_deref, int size, + int w, const unsigned char *covered) +{ + int tmp_vr[SMALL_AGGREGATE_COPY_MAX]; + CType ct; + ct.ref = NULL; + ct.t = (w == 1 ? (VT_BYTE | VT_UNSIGNED) : w == 2 ? (VT_SHORT | VT_UNSIGNED) : VT_INT); + + for (int p = 0; p < size; p += w) + { + int any = 0; + for (int k = p; k < p + w; k++) + any |= covered[k]; + tmp_vr[p / w] = -1; + if (!any) + continue; /* padding only */ + SValue s = small_copy_lval(src, src_deref, p, ct), tmp; + svalue_init(&tmp); + tmp.type = ct; + tmp.r = 0; + tmp.vr = tcc_ir_get_vreg_temp(tcc_state->ir); + tcc_ir_put(tcc_state->ir, TCCIR_OP_LOAD, &s, NULL, &tmp); + tmp_vr[p / w] = tmp.vr; + } + for (int p = 0; p < size; p += w) + { + if (tmp_vr[p / w] < 0) + continue; + SValue tmp, d = small_copy_lval(dst, dst_deref, p, ct); + svalue_init(&tmp); + tmp.type = ct; + tmp.r = 0; + tmp.vr = tmp_vr[p / w]; + tcc_ir_put(tcc_state->ir, TCCIR_OP_STORE, &tmp, NULL, &d); + } +} diff --git a/source/frontend/gen/store/vstore.c b/source/frontend/gen/store/vstore.c index 689cbe21f..bbafa3d4a 100644 --- a/source/frontend/gen/store/vstore.c +++ b/source/frontend/gen/store/vstore.c @@ -23,12 +23,32 @@ #include "gen_priv.h" +/* Most chunks a small aggregate copy may take inline before it stays a + * __aeabi_memmove call (see small_aggregate_copy_plan). Re-measured on the + * Zig compiler's C (70 MB) once codegen began fusing frame-slot word copies + * into LDM/STM (94b95514), which the 2 this was first measured at predates: + * -O2 .text 5,194,592 at 2, 5,171,632 at 3, 5,169,236 at 4 and 5,169,572 at + * 6; -O1 5,157,884 -> 5,133,380 at 4. Over tinycc's own 360 translation + * units it is a wash (1,329,510 -> 1,329,470), so 4 costs nothing where the + * copies are few and pays where they are not. Swept again once the codegen + * began fusing these batched runs into LDM/STM too (2e7508a0), in case that + * was what held the ceiling down: it is not. zig.c -O2 .text 5,015,252 at 4, + * 5,015,268 at 5, 5,015,576 at 6, 5,018,536 at 8 -- past four words the run + * rarely finds that many free registers, and the loads and stores it falls + * back to are bigger than the call they replaced. */ +TCC_DBG_ENV_INT(small_aggregate_copy_max_chunks, "TCC_SMALL_COPY_CHUNKS", 4) + /* store vtop in lvalue pushed on stack */ ST_FUNC void vstore(void) { int sbt, dbt, ft, r, size, align, bit_size, bit_pos, delayed_cast; SValue orig_src = *vtop; SValue orig_dst = vtop[-1]; + /* Either side reached through a packed member (or a misaligned offset) may + * sit at any address, whatever its type's alignment says: such a copy must + * not name the word-aligned __aeabi_memmove4/8, whose contract -- and the + * backend's inline LDM/STM expansion of it -- needs aligned pointers. */ + const int copy_underaligned = orig_src.underaligned || orig_dst.underaligned; /* Eagerly snapshot source const_init_data before gaddrof in the memmove * path invalidates it. Used at vstore_done to propagate through struct @@ -70,14 +90,14 @@ ST_FUNC void vstore(void) Sym *s; for (s = local_stack; s; s = s->prev) { - if (!s->const_init_data || !s->const_init_valid) + if (!s->facts || !s->facts->const_init_data || !s->facts->const_init_valid) continue; - if (s->const_init_in_progress) + if (s->facts->const_init_in_progress) continue; int base = (int)s->c; - if (dst_off + dst_size > base && dst_off < base + s->const_init_size) + if (dst_off + dst_size > base && dst_off < base + s->facts->const_init_size) { - s->const_init_valid = 0; + s->facts->const_init_valid = 0; } } } @@ -524,7 +544,7 @@ ST_FUNC void vstore(void) /* size */ vpushi(complex_size); #ifdef TCC_ARM_EABI - if (!(complex_align & 3)) + if (!(complex_align & 3) && !copy_underaligned) vpush_helper_func(TOK_memmove4); else #endif @@ -781,33 +801,29 @@ ST_FUNC void vstore(void) * assignments. */ #define IS_REG_DEREF_LVAL(r) \ (((r) & VT_LVAL) && ((r) & VT_VALMASK) < VT_CONST) - /* Cap at 8 bytes: see same-named cap in gfunc_return's struct path - * — store-forwarding width-mismatch in the optimizer would feed stale - * zero-init bytes to the LOADs we emit otherwise. - * - * That hazard needs a LOCAL/GLOBAL source: it is an earlier store to the - * source's own storage, at a different width than our LOAD, that the - * forwarder mis-narrows (pr92618's 16-byte vector literal built from - * scalar components). When BOTH sides are register-deref pointers the - * source is opaque memory with no such store in view, so the plain - * `*d = *s` shape — the common one, and the one that otherwise costs a - * __aeabi_memmove4 call — is allowed up to 16 bytes. Sixteen is also the - * point where an LDM/STM pair stops fitting the scratch budget. */ - /* The same reasoning admits the MIRROR shape, `local = *p` / `local = - * arr[i]` — a register-deref SOURCE read into a frame slot. It is the - * commonest struct-copy idiom in C and the one the size-tuned mixed-copy - * cap above (4 bytes) sends to __aeabi_memmove4: mibench_dijkstra's - * `item_t current = queue[head++]` is 12 bytes, so it paid a ~110-cycle - * call 20,042 times per benchmark run where gcc emits two loads. The - * width-mismatch hazard is a property of the SOURCE, and an opaque - * pointer deref has no store to mis-narrow, so this direction gets the - * same 16-byte ceiling as the both-reg-deref case. */ +/* A register deref's address is its vreg alone: svalue_to_iroperand drops + * c.i for a vreg lvalue, as the memmove path below does. c.i may be nonzero + * all the same -- `*p` for a pointer loaded from a local keeps that local's + * frame offset -- which is no displacement, so it does not keep such a copy + * off the chunked path. Zig's C backend reads every array through a pointer + * local (`t20 = (T *)p; t21 = *t20;`, a [4]u8 per std.mem.readInt). */ + /* Up to 16 bytes, the point where an LDM/STM pair stops fitting the + * scratch budget, for every shape. A frame-slot or global SOURCE into a + * register-deref destination used to stop at 8: an earlier store to the + * source's own storage, at a different width than these LOADs, was + * mis-narrowed by store forwarding (pr92618's 16-byte vector literal + * built from scalar components) and fed stale zero-init bytes to the + * copy. pr92618 passes at 16 now, and the shape is zig's commonest copy: + * `return t;` of a local 12-byte error union into the caller's sret + * buffer, 146 __aeabi_memmove4 calls in a std.debug.print hello. + * Only when optimizing: the word LOAD/STOREs pay off because forwarding + * folds them; at -O0 they stay, and three of them plus their address + * arithmetic outweigh the call (zig.c at -O0 grew 118 KB). */ int src_reg_deref_lval = IS_REG_DEREF_LVAL(vtop[0].r); int dst_reg_deref_lval = IS_REG_DEREF_LVAL(vtop[-1].r); int dst_slot_lval = IS_LOCAL_LVAL(vtop[-1].r) || IS_GLOBAL_LVAL(vtop[-1].r); - int both_reg_deref = dst_reg_deref_lval && src_reg_deref_lval; int slot_from_deref = dst_slot_lval && src_reg_deref_lval; - int inline_copy_max = (both_reg_deref || slot_from_deref) ? 16 : 8; + int inline_copy_max = (tcc_state->optimize > 0 || (dst_reg_deref_lval && src_reg_deref_lval) || slot_from_deref) ? 16 : 8; if (tcc_state->ir && !has_vla && size > 0 && size <= inline_copy_max && !(size & 3) && !(align & 3) && !NOEVAL_WANTED && ((dst_reg_deref_lval && @@ -990,6 +1006,44 @@ ST_FUNC void vstore(void) vtop->type = saved_struct_type; goto vstore_done; } + /* Any other small aggregate between frame slots, globals and register + * derefs: copy it as width-uniform LOAD/STORE chunks rather than a + * __aeabi_memmove call (see small_aggregate_copy_plan). Zig's C backend is + * the heavy user -- every `!void` result is a 2-byte struct copied through + * a local, and error unions and slices are written through out-pointers. */ + if (tcc_state->ir && !has_vla && !NOEVAL_WANTED && size > 0 && size <= SMALL_AGGREGATE_COPY_MAX) + { + int src_deref = IS_REG_DEREF_LVAL(vtop[0].r) && vtop[0].vr >= 0; + int dst_deref = IS_REG_DEREF_LVAL(vtop[-1].r) && vtop[-1].vr >= 0; + int src_slot = IS_LOCAL_LVAL(vtop[0].r) || IS_GLOBAL_LVAL(vtop[0].r); + int dst_slot = IS_LOCAL_LVAL(vtop[-1].r) || IS_GLOBAL_LVAL(vtop[-1].r); + unsigned char covered[SMALL_AGGREGATE_COPY_MAX]; + int w = 0, chunks = 0; + if ((src_deref || src_slot) && (dst_deref || dst_slot)) + chunks = small_aggregate_copy_plan(&saved_struct_type, size, align, src_slot ? (int)vtop[0].c.i : -1, + dst_slot ? (int)vtop[-1].c.i : -1, &w, covered); + if (chunks > 0 && chunks <= small_aggregate_copy_max_chunks()) + { + SValue src = vtop[0]; + SValue dst = vtop[-1]; + vtop--; /* pop src; vtop = dst (kept as result lvalue) */ + if (!(IS_LOCAL_LVAL(src.r) && IS_LOCAL_LVAL(dst.r) && src.c.i == dst.c.i)) + ir_emit_small_aggregate_copy(&src, src_deref, &dst, dst_deref, size, w, covered); + vtop->type = saved_struct_type; + goto vstore_done; + } + } + /* A copy of a frame slot onto itself: `t = f();` returns f's struct through + * the hidden result pointer straight into t's slot, then the store copies + * t onto t -- a 96-byte __aeabi_memmove per Wyhash.hash call in Zig's + * compiler. The small-aggregate path above already skips this. */ + if (tcc_state->ir && !has_vla && IS_LOCAL_LVAL(vtop[0].r) && IS_LOCAL_LVAL(vtop[-1].r) && + vtop[0].c.i == vtop[-1].c.i && vtop[0].vr == vtop[-1].vr) + { + vtop--; /* pop src; vtop = dst (kept as result lvalue) */ + vtop->type = saved_struct_type; + goto vstore_done; + } #undef IS_REG_DEREF_LVAL #undef IS_LOCAL_LVAL #undef IS_GLOBAL_LVAL @@ -1051,9 +1105,9 @@ ST_FUNC void vstore(void) vpushi(size); /* Use memmove, rather than memcpy, as dest and src may be same: */ #ifdef TCC_ARM_EABI - if (!(align & 7)) + if (!(align & 7) && !copy_underaligned) vpush_helper_func(TOK_memmove8); - else if (!(align & 3)) + else if (!(align & 3) && !copy_underaligned) vpush_helper_func(TOK_memmove4); else #endif @@ -1094,7 +1148,7 @@ ST_FUNC void vstore(void) Sym *dst_sym = NULL; for (Sym *s = local_stack; s; s = s->prev) { - if ((int)s->c == dst_addr && s->const_init_size >= size) + if ((int)s->c == dst_addr && SYM_FACTS(s)->const_init_size >= size) { dst_sym = s; break; @@ -1102,11 +1156,12 @@ ST_FUNC void vstore(void) } if (dst_sym) { - if (!dst_sym->const_init_data) - dst_sym->const_init_data = tcc_malloc(vstore_src_cid_size); - memcpy(dst_sym->const_init_data, vstore_src_cid, vstore_src_cid_size); - dst_sym->const_init_size = vstore_src_cid_size; - dst_sym->const_init_valid = 1; + SymLocalFacts *f = sym_facts(dst_sym); + if (!f->const_init_data) + f->const_init_data = tcc_malloc(vstore_src_cid_size); + memcpy(f->const_init_data, vstore_src_cid, vstore_src_cid_size); + f->const_init_size = vstore_src_cid_size; + f->const_init_valid = 1; } else { diff --git a/source/frontend/gen/sym/attr_merge.c b/source/frontend/gen/sym/attr_merge.c index f13c38955..72c8f59ee 100644 --- a/source/frontend/gen/sym/attr_merge.c +++ b/source/frontend/gen/sym/attr_merge.c @@ -43,6 +43,7 @@ void merge_symattr(struct SymAttr *sa, struct SymAttr *sa1) sa->dllimport |= sa1->dllimport; sa->naked |= sa1->naked; sa->transparent_union |= sa1->transparent_union; + sa->used |= sa1->used; } /* Merge function attributes. */ @@ -209,6 +210,7 @@ static Sym *sym_copy(Sym *s0, Sym **ps) { Sym *s; s = sym_malloc(), *s = *s0; + s->facts = NULL; /* s0 keeps them: a copy on another stack must not share them */ s->prev = *ps, *ps = s; if (s->v < SYM_FIRST_ANOM) { @@ -274,12 +276,11 @@ Sym *external_sym(int v, CType *type, int r, AttributeDef *ad) s = global_identifier_push(v, type->t, 0); s->r |= r; s->a = ad->a; - /* Merge function attributes (pure, const, etc.) without overwriting - * func_type and func_args which are set from type.ref->f */ - if (ad->f.func_pure) - s->f.func_pure = 1; - if (ad->f.func_const) - s->f.func_const = 1; + /* pure/const are NOT set on s->f: a global symbol's f shares its word + * with sym_scope, and the lookup above takes a non-zero sym_scope for a + * local -- so the next reference pushed a second, undefined global (a + * static pure function called through a separate `U f`). The declaration + * merges them into type.ref->f, which is where the readers look. */ s->asm_label = ad->asm_label; s->type.ref = type->ref; /* copy type to the global stack */ @@ -355,6 +356,18 @@ void apply_alias_attribute(Sym *alias_sym, int target_tok) queue_alias_symbol(alias_sym, target_tok); } +/* An alias target is emitted whether or not anything else refers to it + * (prune_unused_statics runs before the pending aliases are resolved). */ +void mark_pending_alias_targets_used(void) +{ + for (int i = 0; i < nb_pending_aliases; i++) + { + Sym *target = find_global_alias_target_sym(pending_aliases[i].target_tok); + if (target) + target->a.used = 1; + } +} + void resolve_pending_aliases(void) { int i, write_idx, progress; diff --git a/source/frontend/gen/sym/elfsym.c b/source/frontend/gen/sym/elfsym.c index e8bab20e1..1630ec8b4 100644 --- a/source/frontend/gen/sym/elfsym.c +++ b/source/frontend/gen/sym/elfsym.c @@ -181,6 +181,8 @@ ST_FUNC void put_extern_sym2(Sym *sym, int sh_num, addr_t value, unsigned long s ST_FUNC void put_extern_sym(Sym *sym, Section *s, addr_t value, unsigned long size) { + if (tcc_state->check_only) + return; if (nocode_wanted && (NODATA_WANTED || (s && s == cur_text_section))) return; put_extern_sym2(sym, s ? s->sh_num : SHN_UNDEF, value, size, 1); @@ -191,7 +193,7 @@ ST_FUNC void greloca(Section *s, Sym *sym, unsigned long offset, int type, addr_ { int c = 0; - if (nocode_wanted && s == cur_text_section) + if (tcc_state->check_only || (nocode_wanted && s == cur_text_section)) return; if (sym) diff --git a/source/frontend/gen/sym/symtab.c b/source/frontend/gen/sym/symtab.c index 4adacad06..e7d3b3058 100644 --- a/source/frontend/gen/sym/symtab.c +++ b/source/frontend/gen/sym/symtab.c @@ -60,13 +60,28 @@ Sym *sym_malloc(void) #endif } -ST_INLN void sym_free(Sym *sym) +const SymLocalFacts sym_no_facts; + +SymLocalFacts *sym_facts(Sym *s) +{ + if (!s->facts) + s->facts = tcc_mallocz(sizeof(SymLocalFacts)); + return s->facts; +} + +static void sym_free_facts(Sym *s) { - if (sym->const_init_data) + if (s->facts) { - tcc_free(sym->const_init_data); - sym->const_init_data = NULL; + tcc_free(s->facts->const_init_data); + tcc_free(s->facts); + s->facts = NULL; } +} + +ST_INLN void sym_free(Sym *sym) +{ + sym_free_facts(sym); #ifndef SYM_DEBUG /* Poison freed symbols to detect use-after-free */ sym->v = 0xDEADBEEF; @@ -184,7 +199,10 @@ ST_FUNC Sym *sym_push(int v, CType *type, int r, int c) { IRLiveInterval *iv = tcc_ir_vreg_live_interval(tcc_state->ir, vreg); if (iv) + { iv->is_volatile = (type->t & VT_VOLATILE) != 0; + iv->is_struct = (type->t & VT_BTYPE) == VT_STRUCT; + } } /* For stack-passed params (VT_LOCAL), c is the stack offset; * for register params, c is the parameter index */ @@ -314,10 +332,10 @@ ST_FUNC void sym_pop(Sym **ptop, Sym *b, int keep) ps = &ts->sym_identifier; *ps = s->prev_tok; } - if (!keep && s->const_init_data) + if (!keep && s->facts && s->facts->const_init_data) { - tcc_free(s->const_init_data); - s->const_init_data = NULL; + tcc_free(s->facts->const_init_data); + s->facts->const_init_data = NULL; } /* Don't free symbols that have been exported to ELF (sym->c != 0) as they may still be referenced by IR instructions */ @@ -394,7 +412,8 @@ ST_FUNC void label_pop(Sym **ptop, Sym *slast, int keep) codegen, so orig_ir_to_code_mapping is NULL. Defer resolution of addr-taken labels by moving them to global_label_stack, which is popped AFTER codegen when the mapping is available. */ - if (addr_taken && tcc_state->ir && !tcc_state->ir->orig_ir_to_code_mapping && ptop != &global_label_stack) + if (addr_taken && tcc_state->ir && !tcc_state->ir->orig_ir_to_code_mapping && ptop != &global_label_stack && + !tcc_state->check_only) /* a checked-only body has no codegen to wait for */ { /* Unlink from table_ident now (function scope is ending) */ if (s->r != LABEL_GONE) diff --git a/source/frontend/gen/type/cast.c b/source/frontend/gen/type/cast.c index 034ae53d6..0fb07bf55 100644 --- a/source/frontend/gen/type/cast.c +++ b/source/frontend/gen/type/cast.c @@ -690,6 +690,12 @@ void gen_cast(CType *type) if (sf || df) { + /* A comparison still in the flags (`(double)!f()`, `d = a < b`) is + * only a value once SETIF materialises it: vtop->vr names the + * compared operand, so converting it as it stands converted the + * operand -- toybox awk's "num = ! get_set_logical()" stored the + * un-negated result and every awk loop body was skipped. */ + tcc_ir_codegen_cmp_jmp_set(tcc_state->ir); if (sf && df) { /* convert from fp to fp - emit IR operation */ diff --git a/source/frontend/gen/type/size.c b/source/frontend/gen/type/size.c index 83131ce7d..a732bcf0a 100644 --- a/source/frontend/gen/type/size.c +++ b/source/frontend/gen/type/size.c @@ -58,6 +58,13 @@ int compute_aapcs_natural_alignment(const CType *type) type_size(&f->type, &member_align); if (f->a.packed || s->a.packed) member_align = 1; + /* An alignment given on the member itself (_Alignas, aligned) counts, + * packed or not: gcc's doubleword test reads each field's DECL_ALIGN, + * which includes it. Zig's C backend emits exactly that shape -- + * `zig_align(8) zig_packed(union {...}) payload` -- and gcc passed such + * a struct from an even register while tcc used the next free one. */ + if (f->a.aligned && (1 << (f->a.aligned - 1)) > member_align) + member_align = 1 << (f->a.aligned - 1); if (member_align > max_align) max_align = member_align; } diff --git a/source/frontend/gen/value/load.c b/source/frontend/gen/value/load.c index 38f73ce12..d6c3ad482 100644 --- a/source/frontend/gen/value/load.c +++ b/source/frontend/gen/value/load.c @@ -60,7 +60,7 @@ int get_temp_local_var(int size, int align, int *vr_out) return temp_var->location; } } - loc = (loc - size) & -align; + loc = tcc_ir_frame_alloc(loc, size, -align); if (nb_temp_local_vars < MAX_TEMP_LOCAL_VARIABLE_NUMBER) { temp_var = &arr_temp_local_vars[nb_temp_local_vars]; @@ -98,13 +98,13 @@ ST_FUNC void gaddrof(void) Sym *s; for (s = local_stack; s; s = s->prev) { - if (!s->const_init_data || !s->const_init_valid) + if (!s->facts || !s->facts->const_init_data || !s->facts->const_init_valid) continue; - if (s->const_init_in_progress) + if (s->facts->const_init_in_progress) continue; if ((int)s->c == off) { - s->const_init_valid = 0; + s->facts->const_init_valid = 0; break; } } @@ -513,7 +513,7 @@ ST_FUNC int gv(int rc) init_params p = {rodata_section}; unsigned long offset; size = type_size(&vtop->type, &align); - if (NODATA_WANTED) + if (NOSTATIC_WANTED) size = 0, align = 1; offset = section_add(p.sec, size, align); vpush_ref(&vtop->type, p.sec, offset, size); diff --git a/source/frontend/gen/value/longlong.c b/source/frontend/gen/value/longlong.c index f908ed124..f26ec7dd5 100644 --- a/source/frontend/gen/value/longlong.c +++ b/source/frontend/gen/value/longlong.c @@ -705,7 +705,11 @@ void gen_opl(int op) case TOK_SAR: case TOK_SHR: case TOK_SHL: - if (tcc_state->ir) + /* -Os keeps the helper call for a variable count: the call is a BL and + * the argument moves, the inline sequence eight instructions (~22 bytes) + * at every site -- 440 of them in zig.c. */ + if (tcc_state->ir && + ((vtop->r & (VT_VALMASK | VT_LVAL | VT_SYM)) == VT_CONST || !tcc_state->optimize_size)) { /* IR mode: generate a single 64-bit shift instruction directly. * The lexpand/lbuild decomposition produces intermediate 32-bit values diff --git a/source/frontend/gen/value/strlit_pool.c b/source/frontend/gen/value/strlit_pool.c index b92bbd7b4..987fa2532 100644 --- a/source/frontend/gen/value/strlit_pool.c +++ b/source/frontend/gen/value/strlit_pool.c @@ -91,7 +91,7 @@ void str_lit_pool_merge(addr_t pre_off) if (tcc_state->do_bounds_check) return; /* bound-check padding breaks the tail-append invariant */ #endif - if (NODATA_WANTED) + if (NOSTATIC_WANTED) return; if (!vtop->sym || vtop->c.i != 0) return; diff --git a/source/frontend/gen/value/vstack.c b/source/frontend/gen/value/vstack.c index bdd6245cd..77d904887 100644 --- a/source/frontend/gen/value/vstack.c +++ b/source/frontend/gen/value/vstack.c @@ -347,7 +347,8 @@ ST_FUNC Sym *get_sym_ref(CType *type, Section *sec, unsigned long offset, unsign * them. Those IR instructions are later removed by DCE, but the * symbol must exist during compilation. Under NODATA_WANTED the * caller passes size=0, so no section data is wasted. */ - put_extern_sym2(sym, sec ? sec->sh_num : SHN_UNDEF, offset, size, 1); + if (!tcc_state->check_only) /* its IR is freed: nothing may name it */ + put_extern_sym2(sym, sec ? sec->sh_num : SHN_UNDEF, offset, size, 1); return sym; } diff --git a/source/frontend/tccasm.c b/source/frontend/tccasm.c index 258caec31..34320ff48 100644 --- a/source/frontend/tccasm.c +++ b/source/frontend/tccasm.c @@ -1721,6 +1721,13 @@ static void parse_asm_operands(ASMOperand *operands, int *nb_operands_ptr, int i skip('('); gexpr(); maybe_substitute_inline_const_arg(vtop); + /* Record a local register variable's register now, while its Sym is in + * scope: by the time the IR lowers the asm the value may live in any + * register (or be spilled) and the function's locals have been popped. */ + op->regvar = 0; + if (vtop->sym && (vtop->r & (VT_VALMASK | VT_LVAL)) == (VT_LOCAL | VT_LVAL) && !(vtop->sym->r & VT_PARAM) && + (vtop->sym->r & VT_VALMASK) < VT_CONST) + op->regvar = 1 + (vtop->sym->r & VT_VALMASK); if (is_output) { if (!(vtop->type.t & VT_ARRAY)) diff --git a/source/frontend/tccpp.c b/source/frontend/tccpp.c index 81d440b73..8ac04d4cd 100644 --- a/source/frontend/tccpp.c +++ b/source/frontend/tccpp.c @@ -42,6 +42,7 @@ ST_DATA CValue tokc; ST_DATA const int *macro_ptr; ST_DATA CString tokcstr; /* current parsed string, if any */ ST_DATA TokenString *pp_pragma_capture; /* see tcc.h */ +ST_DATA int pp_pack_captures; /* see tcc.h */ /* display benchmark infos */ ST_DATA int tok_ident; @@ -1912,6 +1913,64 @@ ST_FUNC void pp_apply_pack_replay(TCCState *s1, int code) } } +/* Apply every deferred #pragma pack action in the saved tokens [P, END) to the + live pack state. */ +ST_FUNC void pp_apply_pack_replays(TCCState *s1, const int *p, const int *end) +{ + while (p < end) + { + int t; + CValue cv; + tok_get(&t, &p, &cv); + if (t == TOK_PACK_REPLAY) + pp_apply_pack_replay(s1, cv.i); + } +} + +/* The #pragma pack state as {depth, stack[0..depth]}. A saved function body + replayed away from its definition (at a call site, or at the end of the TU) + lays out its structs under the state it was defined in, not the state around + the replay. pp_pack_snapshot returns a heap copy, or NULL for the default + state (the case for nearly every function); pp_pack_enter saves the live + state into SAVED (PACK_STACK_SIZE + 1 ints) and installs SNAP, and + pp_pack_leave puts SAVED back, so directives inside the body do not leak + past the replay. */ +static void pp_pack_install(TCCState *s1, const int *snap) +{ + if (!snap) + { + s1->pack_stack[0] = 0; + s1->pack_stack_ptr = s1->pack_stack; + return; + } + memcpy(s1->pack_stack, snap + 1, sizeof(int) * (snap[0] + 1)); + s1->pack_stack_ptr = s1->pack_stack + snap[0]; +} + +ST_FUNC int *pp_pack_snapshot(TCCState *s1) +{ + int depth = s1->pack_stack_ptr - s1->pack_stack; + if (depth == 0 && s1->pack_stack[0] == 0) + return NULL; + int *snap = tcc_malloc(sizeof(int) * (depth + 2)); + snap[0] = depth; + memcpy(snap + 1, s1->pack_stack, sizeof(int) * (depth + 1)); + return snap; +} + +ST_FUNC void pp_pack_enter(TCCState *s1, const int *snap, int *saved) +{ + int depth = s1->pack_stack_ptr - s1->pack_stack; + saved[0] = depth; + memcpy(saved + 1, s1->pack_stack, sizeof(int) * (depth + 1)); + pp_pack_install(s1, snap); +} + +ST_FUNC void pp_pack_leave(TCCState *s1, const int *saved) +{ + pp_pack_install(s1, saved); +} + #if 0 #define TOK_GET(t, p, c) tok_get(t, p, c) #else @@ -2234,6 +2293,16 @@ static void pp_parse_assertion(void) tokc.i = pp_assertion_value(kind_tok, value_tok); } +/* The next preprocessor-expression token, never macro-expanded: from the + * current substitution result when there is one, else from the file. */ +static void pp_next_nosubst(void) +{ + if (macro_ptr) + next(); + else + next_nomacro(); +} + /* eval an expression for #if/#elif */ static int expr_preprocess(TCCState *s1) { @@ -2270,7 +2339,8 @@ static int expr_preprocess(TCCState *s1) if (s1->run_test) maybe_run_test(s1); c = 0; - if (define_find(tok) || tok == TOK___HAS_INCLUDE || tok == TOK___HAS_INCLUDE_NEXT) + if (define_find(tok) || tok == TOK___HAS_INCLUDE || tok == TOK___HAS_INCLUDE_NEXT || + tok == TOK___HAS_ATTRIBUTE) c = 1; if (t == '(') { @@ -2281,6 +2351,25 @@ static int expr_preprocess(TCCState *s1) tok = TOK_CINT; tokc.i = c; } + else if (tok == TOK___HAS_ATTRIBUTE) + { + /* Usually reached through a macro (zig.h's zig_has_attribute), so the + * operand is still in the substituted stream, which only next() reads; + * the operand was kept unexpanded there (see macro_subst). Read straight + * from the file, next() would expand it, so use the raw lexer. */ + pp_next_nosubst(); + if (tok != '(') + expect("'(' after __has_attribute"); + pp_next_nosubst(); + if (tok < TOK_IDENT) + expect("attribute name in __has_attribute"); + c = tcc_attribute_supported(tok); + pp_next_nosubst(); + if (tok != ')') + expect("')'"); + tok = TOK_CINT; + tokc.i = c; + } else if (tok == TOK___HAS_INCLUDE || tok == TOK___HAS_INCLUDE_NEXT) { t = tok; @@ -2318,6 +2407,36 @@ static int expr_preprocess(TCCState *s1) return c != 0; } +/* __has_builtin: builtins parse_builtin and friends implement. A "no" sends a + * header to its portable C fallback: zig.h built its byte swaps from shifts + * and ORs (five instructions a u16) while REV16 was there. + * + * The __builtin_*_overflow family is implemented but answered "no": zig.h's + * builtin branch takes the result's address (`&full_res`, which then lives + * in memory) and ORs the flag with range checks, and that costs more than its + * fallback's `*res < lhs` -- zig.c -Os +7 KB. Answer "yes" once both of + * those compile as well as the fallback does. */ +static const char *const tcc_builtins_supported[] = { + "__builtin_bswap16", "__builtin_bswap32", "__builtin_bswap64", + "__builtin_clz", "__builtin_clzl", "__builtin_clzll", "__builtin_ctz", "__builtin_ctzl", "__builtin_ctzll", + "__builtin_popcount", "__builtin_popcountl", "__builtin_popcountll", + "__builtin_unreachable", "__builtin_trap", "__builtin_expect", "__builtin_frame_address", + "__builtin_return_address", "__builtin_prefetch", "__builtin_nan", "__builtin_inf", + "__builtin_constant_p", "__builtin_types_compatible_p", "__builtin_choose_expr", + "__builtin_memcpy", "__builtin_memmove", "__builtin_memset", "__builtin_memcmp", "__builtin_strlen", + "__builtin_abs", "__builtin_labs", "__builtin_llabs", "__builtin_alloca", "__builtin_offsetof", + "__builtin_va_start", "__builtin_va_end", "__builtin_va_arg", "__builtin_va_copy", + NULL}; + +ST_FUNC int tcc_builtin_supported(int t) +{ + const char *name = get_tok_str(t, NULL); + for (int i = 0; tcc_builtins_supported[i]; i++) + if (!strcmp(name, tcc_builtins_supported[i])) + return 1; + return 0; +} + ST_FUNC void pp_error(CString *cs) { cstr_printf(cs, "bad preprocessor expression: #%s", get_tok_str(pp_expr, 0)); @@ -2599,6 +2718,7 @@ static int pragma_parse(TCCState *s1) CValue cv; cv.i = ((unsigned)rec_kind << 16) | (rec_value & 0xffff); tok_str_add2(pp_pragma_capture, TOK_PACK_REPLAY, &cv); + pp_pack_captures++; } } else if (tok == TOK_comment) @@ -2722,7 +2842,8 @@ ST_FUNC void preprocess(int is_bof) file->ifndef_macro = tok; } } - if (define_find(tok) || tok == TOK___HAS_INCLUDE || tok == TOK___HAS_INCLUDE_NEXT) + if (define_find(tok) || tok == TOK___HAS_INCLUDE || tok == TOK___HAS_INCLUDE_NEXT || + tok == TOK___HAS_ATTRIBUTE) c ^= 1; next_nomacro(); do_if: @@ -4701,7 +4822,32 @@ static int macro_subst_tok(TokenString *tok_str, Sym **nested_list, Sym *s) char buf[32], *cstrval = buf; /* special macros */ - if (v == TOK___LINE__ || v == TOK___COUNTER__) + if (v == TOK___HAS_BUILTIN) + { + /* Anywhere, not only in #if: zig.h pastes the 0/1 answer into a name + * (zig_expand_import_##has) to choose a libm import. */ + int saved_parse_flags = parse_flags, name; + parse_flags |= PARSE_FLAG_SPACES | PARSE_FLAG_LINEFEED | PARSE_FLAG_ACCEPT_STRAYS; + do + t = next_argstream(nested_list, NULL); + while (t == ' ' || t == TOK_LINEFEED); + if (t != '(') + expect("'(' after __has_builtin"); + do + t = next_argstream(nested_list, NULL); + while (t == ' ' || t == TOK_LINEFEED); + name = t; + do + t = next_argstream(nested_list, NULL); + while (t == ' ' || t == TOK_LINEFEED); + parse_flags = saved_parse_flags; + if (t != ')') + expect("')' after the __has_builtin operand"); + snprintf(buf, sizeof(buf), "%d", name >= TOK_IDENT && tcc_builtin_supported(name)); + t = TOK_PPNUM; + goto add_cstr1; + } + else if (v == TOK___LINE__ || v == TOK___COUNTER__) { t = v == TOK___LINE__ ? file->line_num : pp_counter++; snprintf(buf, sizeof(buf), "%d", t); @@ -4799,8 +4945,10 @@ static int macro_subst(TokenString *tok_str, Sym **nested_list, const int *macro tok_str_add2_spc(tok_str, t, &cval); if (nosubst && t != '(') nosubst = 0; - /* GCC supports 'defined' as result of a macro substitution */ - if (t == TOK_DEFINED && pp_expr) + /* GCC supports 'defined' as result of a macro substitution; the + operand of __has_attribute is likewise an attribute name, not a + macro to expand. */ + if ((t == TOK_DEFINED || t == TOK___HAS_ATTRIBUTE) && pp_expr) nosubst = 1; } } @@ -4948,8 +5096,16 @@ ST_FUNC HOT void next(void) if (t == TOK_PACK_REPLAY) { /* deferred #pragma pack action: apply it and stay invisible to the - parser by fetching the next real token. */ - pp_apply_pack_replay(s1, tokc.i); + parser by fetching the next real token. While a replayed body is + itself being saved again (skip_or_save_block), keep the action in + the new stream instead, as for a directive read from the file. */ + if (pp_pragma_capture) + { + tok_str_add2(pp_pragma_capture, TOK_PACK_REPLAY, &tokc); + pp_pack_captures++; + } + else + pp_apply_pack_replay(s1, tokc.i); goto redo; } goto convert; @@ -5404,6 +5560,8 @@ static void tcc_predefs_base(TCCState *s1, CString *cs, int is_asm, int include_ putdef(cs, "__CHAR_UNSIGNED__"); if (s1->optimize > 0) putdef(cs, "__OPTIMIZE__"); + if (s1->optimize_size) + putdef(cs, "__OPTIMIZE_SIZE__"); if (s1->option_pthread) putdef(cs, "_REENTRANT"); if (s1->leading_underscore) @@ -5622,6 +5780,7 @@ ST_FUNC void tccpp_new(TCCState *s) define_push(TOK___DATE__, MACRO_OBJ, NULL, NULL); define_push(TOK___TIME__, MACRO_OBJ, NULL, NULL); define_push(TOK___COUNTER__, MACRO_OBJ, NULL, NULL); + define_push(TOK___HAS_BUILTIN, MACRO_OBJ, NULL, NULL); } ST_FUNC void tccpp_delete(TCCState *s) diff --git a/source/frontend/tcctok.h b/source/frontend/tcctok.h index 5e47a3b3b..4c04d6d24 100644 --- a/source/frontend/tcctok.h +++ b/source/frontend/tcctok.h @@ -107,6 +107,8 @@ DEF(TOK___VA_ARGS__, "__VA_ARGS__") DEF(TOK___COUNTER__, "__COUNTER__") DEF(TOK___HAS_INCLUDE, "__has_include") DEF(TOK___HAS_INCLUDE_NEXT, "__has_include_next") +DEF(TOK___HAS_ATTRIBUTE, "__has_attribute") +DEF(TOK___HAS_BUILTIN, "__has_builtin") /* special identifiers */ DEF(TOK___FUNC__, "__func__") @@ -132,6 +134,8 @@ DEF(TOK_WEAK1, "weak") DEF(TOK_WEAK2, "__weak__") DEF(TOK_ALIAS1, "alias") DEF(TOK_ALIAS2, "__alias__") +DEF(TOK_USED1, "used") +DEF(TOK_USED2, "__used__") DEF(TOK_UNUSED1, "unused") DEF(TOK_UNUSED2, "__unused__") DEF(TOK_NODEBUG1, "nodebug") @@ -396,6 +400,7 @@ DEF(TOK_builtin_va_start, "__builtin_va_start") #define DEF_ATOMIC(ID) DEF(TOK_##__##ID, "__" #ID) DEF_ATOMIC(atomic_store) DEF_ATOMIC(atomic_load) +DEF_ATOMIC(atomic_load_n) DEF_ATOMIC(atomic_exchange) DEF_ATOMIC(atomic_compare_exchange) DEF_ATOMIC(atomic_fetch_add) diff --git a/source/include/tcc.h b/source/include/tcc.h index 69c220001..2719f1eb8 100644 --- a/source/include/tcc.h +++ b/source/include/tcc.h @@ -21,8 +21,15 @@ #ifndef _TCC_H #define _TCC_H +/* The Makefile also passes these on the command line, so a file that includes a + * system header before tcc.h gets the same declarations (and a unity group the + * same headers whichever member includes them first); see gen_unity.py. */ +#ifndef _GNU_SOURCE #define _GNU_SOURCE +#endif +#ifndef _DARWIN_C_SOURCE #define _DARWIN_C_SOURCE +#endif #include "config.h" #include @@ -435,6 +442,20 @@ typedef struct Sym Sym; ~10 KiB .bss (513->257 * 40 B SValue). Clean tcc_error on overflow. */ #define STRING_MAX_SIZE 1024 #define TOKSTR_MAX_SIZE 256 +/* Nested auto-inline expansion: how deep, and how many body tokens one + * function may take in through expansions below the first level. A body of + * at most TINY_INLINE_TOKENS is smaller than the call it replaces and costs + * nothing from the budget. 24 was too few for that: zig.h's 128-bit helpers + * (zig_shr_u128, 92 tokens; the truncates, 20-48) take their zig_u128 by value + * -- 16 bytes each way through registers and the stack -- and fold to a few + * instructions once the constant shift or width arrives, yet a large caller + * spent the budget before reaching them (Wyhash.hash kept 8 calls to + * zig_u64_truncate_u64 alone). At 96 the Zig compiler's .text shrinks 55 KB + * and it runs 3.3% fewer instructions; 128-256 change nothing more, and the + * 2441-file corpus is byte-identical at -O1/-O2 (no smaller caller runs out). */ +#define INLINE_NEST_MAX 32 +#define NESTED_INLINE_BUDGET 4096 +#define TINY_INLINE_TOKENS 96 #define PACK_STACK_SIZE 8 #define TOK_HASH_SIZE 2048 /* must be a power of two. YASOS: 4096 -> 2048 saves 8 KiB @@ -562,9 +583,18 @@ struct SymAttr by tcc_ir_tu_analyze_dead_statics; read by dead-static-store-elim during the end-of-TU late_reopt phase. */ - param_volatile : 1; /* original parameter declaration was volatile + param_volatile : 1, /* original parameter declaration was volatile before function-type normalization stripped top-level qualifiers. */ + tentative : 1, /* file-scope object declared without an + initializer, whose definition may still + follow: its bytes are not its value yet. + Cleared by the definition or at the end + of the TU (finalize_tentative_definitions). */ + used : 1, /* __attribute__((used)): emitted even when + nothing in the TU refers to it. */ + tu_unused : 1; /* a static nothing live in the TU refers to + (prune_unused_statics): never emitted. */ }; /* function attributes or temporary attributes for parsing */ @@ -585,6 +615,7 @@ struct FuncAttr func_rewritten_extern_inline : 1, /* extern inline rewritten to non-extern inline-only def */ func_outofline_needed : 1, /* always_inline call could not stay call-site-only */ func_auto_inline : 1, /* compiler-selected auto-inline candidate (small func) */ + func_icf_folded : 1, /* body dropped: the symbol points at an identical one (tcc_icf_try_fold) */ func_deferred_inline : 1, /* auto-inline candidate that was NOT compiled at definition time: a `static inline` whose body gen_inline_functions still owes the TU. Emit it there iff some call site fell back to a real call @@ -602,9 +633,13 @@ struct FuncAttr func_compiled : 1, /* gen_function has completed for this sym at least once — distinguishes forward-declared-not-yet-defined functions from already-emitted ones, used by late_reopt triggering for inter-procedural noreturn propagation */ - func_keep_tokens_for_noreturn : 1; /* tokens preserved so end-of-TU noreturn propagation can decide whether to + func_keep_tokens_for_noreturn : 1, /* tokens preserved so end-of-TU noreturn propagation can decide whether to re-emit — separate from func_late_reopt so we don't trigger unnecessary re-emit before we know if any callee turned out to be noreturn */ + func_called_once : 1, /* static, one call site in the TU, address never taken: that site inlines it + whatever its size (-finline-functions-called-once) */ + func_small_leaf : 1; /* static leaf whose optimized body is a few straight-line ops: inlined + whatever its token length (see gen_function's post-opt promotion) */ }; /* symbol management */ @@ -642,14 +677,21 @@ struct Sym struct Sym *cleanupstate; /* in defined labels */ int *vla_array_str; /* vla array code */ }; - struct Sym *prev; /* prev symbol in stack */ - struct Sym *prev_tok; /* previous symbol for this token */ - int vla_size_loc; /* for structs with VLA members: stack offset holding - runtime total struct size (0 = not a VLA struct) */ + struct Sym *prev; /* prev symbol in stack */ + struct Sym *prev_tok; /* previous symbol for this token */ + struct SymLocalFacts *facts; /* NULL unless a local variable recorded some (SYM_FACTS / sym_facts) */ +}; + +/* Facts a few local variables carry, kept out of Sym so the tens of thousands + * of prototypes, fields and type nodes a TU creates do not pay for them (Sym + * was 88 bytes on ARM, 48 without them; the symbol table is the largest part + * of a compile's memory that lives for the whole TU). Read through + * SYM_FACTS(s), which yields an all-zero record when s has none; write through + * sym_facts(s), which allocates it. Freed with the Sym (sym_free). */ +typedef struct SymLocalFacts +{ unsigned long long objsize_max_value; /* conservative max scalar value assigned locally */ unsigned long long objsize_strlen_value; /* conservative max NUL-terminated string bytes */ - unsigned char objsize_max_valid; - unsigned char objsize_strlen_valid; /* Captured constant initializer bytes for small local arrays/vectors. * Set by decl_initializer_alloc when all init values are compile-time * constants; invalidated by vstore when the variable is later written. @@ -657,9 +699,15 @@ struct Sym * mask loads into constant indices. */ unsigned char *const_init_data; int const_init_size; + unsigned char objsize_max_valid; + unsigned char objsize_strlen_valid; unsigned char const_init_valid; unsigned char const_init_in_progress; -}; +} SymLocalFacts; + +extern const SymLocalFacts sym_no_facts; +#define SYM_FACTS(s) ((const SymLocalFacts *)((s)->facts ? (s)->facts : &sym_no_facts)) +SymLocalFacts *sym_facts(struct Sym *s); #include "source/ir/machine_op.h" #include "source/ir/tccir.h" @@ -895,10 +943,36 @@ typedef struct InlineFunc { TokenString *func_str; Sym *sym; + int *pack; /* #pragma pack state at the body (pp_pack_snapshot), NULL = default */ int inline_count; /* number of auto-inline expansions performed so far (call-heavy budget) */ char filename[1]; } InlineFunc; +/* A file-scope function definition whose body is generated at the end of the + * TU (gen_deferred_function_bodies): its tokens from the opening '{', and the + * section attribute the definition carried. */ +typedef struct DeferredFunc +{ + Sym *sym; + Section *section; + TokenString *body; + int *pack; /* #pragma pack state at the body (pp_pack_snapshot), NULL = default */ + char filename[1]; +} DeferredFunc; + +/* A file-scope static object whose initializer is parsed at the end of the TU + * (prune_unused_statics), and only if something live refers to it: its tokens + * from the first after '=', and the declaration they initialize. */ +typedef struct DeferredData +{ + Sym *sym; + CType type; + AttributeDef ad; + int r; + TokenString *init; + char filename[1]; +} DeferredData; + /* nested functions */ #define MAX_CAPTURED_VARS 32 #define MAX_NONLOCAL_GOTOS 8 @@ -921,6 +995,7 @@ typedef struct NestedFunc int needs_chain_save; /* 1 if a child func needs multi-hop chain (depth>1) */ int compiled; /* number of captured parent variables */ int trampoline_needed; /* address of this nested function was taken */ + int nb_real_calls; /* call sites that emitted a real call (were not inlined) */ Sym *trampoline_tcc_sym; /* TCC symbol for trampoline code (.text) */ Sym *chain_slot_tcc_sym; /* TCC symbol for chain slot (.data) */ /* Non-local goto support: nested function does 'goto label' targeting parent __label__ */ @@ -992,6 +1067,8 @@ typedef struct ASMOperand int input_index; /* if >= 0, gives reference to an input constraint */ int priority; /* priority, used to assign registers */ int reg; /* if >= 0, register number used for this operand */ + int regvar; /* 1 + register a `register T x __asm("rN")` operand names; + 0 (what zero-initialisation gives) means none */ int is_llong; /* true if double register value */ int is_memory; /* true if memory operand */ int is_rw; /* for '+' modifier */ @@ -1030,15 +1107,6 @@ typedef struct ArchiveSymbolCache unsigned int loaded_member_mask; /* dedup set mask */ } ArchiveSymbolCache; -/* Per-TU stash of optimized IR for `static` functions that look eligible - * for later inlining. Populated at the end of gen_function(); flushed at - * tccgen_finish(). Phase 0: stash only, no consumers — exists to validate - * the lifecycle change before the inliner pass lands. */ -typedef struct StashedFuncIR -{ - Sym *sym; - TCCIRState *ir; -} StashedFuncIR; struct TCCState { @@ -1052,6 +1120,9 @@ struct TCCState unsigned char symbolic; /* if true, resolve symbols in the current module first */ unsigned char filetype; /* file type for compilation (NONE,C,ASM) */ unsigned char optimize; /* only to #define __OPTIMIZE__ */ + unsigned char optimize_size; /* -Os (1) / -Oz (2): the -O2 pipeline, but + where a choice trades code size for + speed, take the smaller code */ unsigned char option_pthread; /* -pthread option */ unsigned char enable_new_dtags; /* -Wl,--enable-new-dtags */ unsigned char gc_sections; /* -Wl,--gc-sections: garbage collect unused sections */ @@ -1145,6 +1216,10 @@ struct TCCState int const_loop_inline_used; unsigned char opt_inline_functions; /* -finline-functions: auto-inline small functions at -O2 */ unsigned char opt_inline_small; /* -finline-small-functions: auto-inline tiny functions at -O1 */ + unsigned char opt_inline_called_once; /* -finline-functions-called-once: defer function bodies to the end of the + TU and inline each static function called from one place there */ + unsigned char opt_drop_unused_statics; /* -fdrop-unused-statics: defer static functions and initialized static + objects to the end of the TU and emit only those something live refers to */ unsigned char opt_ipc; /* interprocedural constant propagation */ int opt_inline_limit; /* -finline-limit=N: token-stream word threshold (default 0=use level default) */ unsigned char opt_inline_limit_user; /* -finline-limit=N given explicitly: -O levels must not override it */ @@ -1257,6 +1332,21 @@ struct TCCState YaffLib *yaff_libs; int nb_yaff_libs; + /* -fmodule-local-calls (default on under text_and_data_separation): a call + to an undefined global function that no shared library in the library + paths exports is compiled as module-local, so its R9 reload is dropped + (arm-thumb-gen.c thumb_callee_in_this_module). Whether the assumption held + is checked at link time through R_ARM_YASOS_LOCAL_CALL. The import set is + built lazily by tcc_yaff_import_set_has: the export tables of every + lib*.so in the library paths, minus what libtcc1.a defines (the archive is + pulled before the shared libraries resolve, so its members win). */ + unsigned char module_local_calls; + unsigned char import_set_state; /* 0 = not built, 1 = built, 2 = unavailable */ + YaffLib *import_libs; + int nb_import_libs; + char *import_libtcc1; /* path of the libtcc1.a whose index was read, or NULL */ + char *import_fp_archive; /* fp/lib.a when the FP runtime links statically, or NULL */ + /* include paths */ char **include_paths; int nb_include_paths; @@ -1313,6 +1403,40 @@ struct TCCState only if referenced */ struct InlineFunc **inline_fns; int nb_inline_fns; + /* inline_fns by symbol token, so a call site finds its callee's body without + * scanning every entry (inline_fn_lookup). */ + struct InlineFunc **inline_fn_by_tok; + int inline_fn_by_tok_size; + int inline_fn_indexed; /* inline_fns entries already in inline_fn_by_tok */ + /* Function bodies saved at their definition for end-of-TU generation + * (-finline-functions-called-once, gen_deferred_function_bodies). */ + struct DeferredFunc **deferred_fns; + int nb_deferred_fns; + /* Initialized static objects saved for -fdrop-unused-statics. */ + struct DeferredData **deferred_data; + int nb_deferred_data; + /* The statics prune_unused_statics dropped: parsed at the end of the TU for + * their diagnostics only (check_dropped_statics), never emitted. */ + struct DeferredFunc **dropped_fns; + int nb_dropped_fns; + struct DeferredData **dropped_data; + int nb_dropped_data; + /* > 0 while a dropped static is parsed for its diagnostics: nothing it + * produces may reach the object -- no data (NODATA_WANTED), no symbol + * (put_extern_sym, get_sym_ref), no relocation (greloca). Its code goes to + * an IR that is freed, its static objects to check_scratch. */ + int check_only; + struct Section *check_scratch; + /* Objects declared without an initializer at file scope, settled at the end + * of the TU (finalize_tentative_definitions). */ + struct Sym **tentative_syms; + int nb_tentative_syms; + /* The whole TU has been parsed: what is generated from here on (deferred + * bodies, owed inline bodies) sees every declaration and every use. */ + int tu_parsed; + /* Body length (saved-token ints) the function being generated may still + * absorb by expanding called-once functions (called_once_budget_begin). */ + int called_once_budget; /* Current function symbol being compiled. Set by gen_function so IR opt * passes can mark the function for end-of-TU re-optimization. */ @@ -1504,12 +1628,6 @@ struct TCCState CString linker_arg; /* collect -Wl options */ int thumb_func; TCCIRState *ir; - /* Inliner stash: optimized IR for eligible `static` functions, kept alive - * past the per-function gen_function() free so a future inliner pass can - * splice it into callers. Phase 0: no consumers yet. */ - StashedFuncIR *stashed_func_irs; - int nb_stashed_func_irs; - int stashed_func_irs_capacity; /* Nested functions - saved token streams for functions defined inside other functions */ NestedFunc *nested_funcs; int nb_nested_funcs; @@ -1537,6 +1655,12 @@ struct TCCState stream at a call site, these track the return value destination. */ uint8_t in_inline_expansion; /* nonzero while expanding inline body */ uint8_t inline_expansion_depth; /* nested expansion depth, capped to bound work */ + /* The functions whose bodies are being expanded, outermost first: a callee + * already here is recursion, never expanded again. */ + struct Sym *inline_expansion_syms[INLINE_NEST_MAX]; + /* Tokens nested auto-inline expansions may still add to the function being + * compiled (reset by gen_function): helpers calling helpers can fan out. */ + int nested_inline_budget; int inline_return_loc; /* stack offset for storing return value */ int inline_return_vr; /* vreg bound to that slot for scalar returns, * -1 when the return goes through memory only. @@ -1601,6 +1725,16 @@ struct TCCState struct LabelDiffFixup *label_diff_fixups; }; + +/* Whether built-in bound checking is on; constant 0 in a compiler built + * without it (CONFIG_TCC_BCHECK=0, as the YasOS cross is), where the + * do_bounds_check member does not exist. */ +#ifdef CONFIG_TCC_BCHECK +#define tcc_bounds_checking(s) ((s)->do_bounds_check) +#else +#define tcc_bounds_checking(s) 0 +#endif + /* String/memory builtin IDs for table-driven dispatch. * Used by tccgen.c and ir/opt.c to avoid repeated strcmp. */ enum StrBuiltinId @@ -2150,6 +2284,9 @@ ST_DATA CString tokcstr; /* current parsed string, if any */ pack state is applied at the struct's position during replay, not eagerly during the recording scan. */ ST_DATA TokenString *pp_pragma_capture; +/* Count of #pragma pack actions ever added to a pp_pragma_capture stream: lets + a caller of skip_or_save_block tell whether the saved tokens hold any. */ +ST_DATA int pp_pack_captures; /* display benchmark infos */ ST_DATA int tok_ident; @@ -2204,6 +2341,10 @@ ST_FUNC void tok_str_add2(TokenString *s, int t, CValue *cv); ST_FUNC void tok_str_add_tok(TokenString *s); ST_FUNC void tok_get(int *t, const int **pp, CValue *cv); ST_FUNC void pp_apply_pack_replay(TCCState *s1, int code); +ST_FUNC void pp_apply_pack_replays(TCCState *s1, const int *p, const int *end); +ST_FUNC int *pp_pack_snapshot(TCCState *s1); +ST_FUNC void pp_pack_enter(TCCState *s1, const int *snap, int *saved); +ST_FUNC void pp_pack_leave(TCCState *s1, const int *saved); ST_INLN void define_push(int v, int macro_type, int *str, Sym *first_arg); ST_FUNC void define_undef(Sym *s); ST_INLN Sym *define_find(int v); @@ -2245,6 +2386,31 @@ static inline int toup(int c) return (c >= 'a' && c <= 'z') ? c - 'a' + 'A' : c; } +/* Whether parse_attribute() acts on the attribute spelled by token `t` + * (backs the __has_attribute preprocessor operator). */ +ST_FUNC int tcc_attribute_supported(int t); +ST_FUNC int tcc_builtin_supported(int t); +/* thumb-outline.c: -Os machine outliner */ +ST_FUNC int tcc_gen_machine_outline_enabled(void); +ST_FUNC void tcc_gen_machine_outline_capture_begin(void); +ST_FUNC void tcc_gen_machine_outline_capture_end(void); +ST_FUNC void tcc_gen_machine_outline_capture(uint32_t opcode, int size, int at); +ST_FUNC void tcc_gen_machine_outline_capture_reloc(struct Sym *sym, int at); +struct TCCIRState; +ST_FUNC int tcc_gen_machine_outline_analyze(struct TCCIRState *ir, const uint32_t *map, int end); +ST_FUNC void tcc_gen_machine_outline_op_end(int i); +ST_FUNC void tcc_gen_machine_outline_op_start(int i); +ST_FUNC void tcc_gen_machine_outline_function_end(void); +ST_FUNC int tcc_gen_machine_outline_suppress(uint32_t opcode, int size); +ST_FUNC int tcc_gen_machine_outline_suppressing(void); +ST_FUNC int tcc_gen_machine_outline_shrink_between(int from_ir, int to_ir); +ST_FUNC void tcc_gen_machine_outline_reset(void); +ST_FUNC void tcc_gen_machine_outline_window_open(void); +ST_FUNC void tcc_gen_machine_outline_window_closed(int bytes); +ST_FUNC void tcc_gen_machine_outline_emit_bl(int esym); +ST_FUNC void tcc_gen_machine_outline_emit_raw(uint16_t hw); +ST_FUNC int tcc_gen_machine_prolog_saved_lr(void); + /* ------------ tccgen.c ------------ */ #define SYM_POOL_NB (8192 / sizeof(Sym)) @@ -2260,12 +2426,15 @@ ST_DATA int rsym, anon_sym, ind, loc; ST_DATA char debug_modes; ST_DATA int nocode_wanted; /* true if no code generation wanted for an expression */ +ST_DATA SValue *discarded_call_vtop; /* vtop where an expression statement began, while its + value is being thrown away; NULL anywhere else */ ST_DATA int global_expr; /* true if compound literals must be allocated globally (used during initializers parsing */ ST_DATA CType func_vt; /* current function return type (used by return instruction) */ ST_DATA int func_var; /* true if current function is variadic */ ST_DATA int func_vc; ST_DATA int func_ind; +ST_DATA int func_align_pad; /* bytes codegen padded in front of the body (ir/codegen.c) */ ST_DATA int func_has_label_addr; /* true if current function uses &&label (computed goto) */ ST_DATA const char *funcname; @@ -2284,6 +2453,12 @@ ST_FUNC ElfSym *elfsym(Sym *); ST_FUNC void update_storage(Sym *sym); ST_FUNC void put_extern_sym2(Sym *sym, int sh_num, addr_t value, unsigned long size, int can_add_underscore); ST_FUNC void put_extern_sym(Sym *sym, Section *section, addr_t value, unsigned long size); + +/* Identical code folding: a generated body equal to one already emitted is + * dropped and its symbol points at the survivor (backend/generators/icf.c). */ +int tcc_icf_try_fold(TCCState *s1, Sym *sym, Section *sec, addr_t start, addr_t size, size_t reloc_mark); +void tcc_icf_reset(void); +void tcc_icf_section_changed(Section *sec); #if PTR_SIZE == 4 ST_FUNC void greloc(Section *s, Sym *sym, unsigned long offset, int type); #endif @@ -2304,6 +2479,7 @@ ST_FUNC Sym *global_identifier_push(int v, int t, int c); ST_FUNC Sym *external_global_sym(int v, CType *type); ST_FUNC Sym *external_helper_sym(int v); ST_FUNC void vpush_helper_func(int v); +ST_FUNC void vpush_return_address(void); ST_FUNC void vpush_typed_helper_func(int v, CType *type); ST_FUNC void vset(CType *type, int r, int v); ST_FUNC void vset_VT_CMP(int op); @@ -2389,6 +2565,12 @@ ST_FUNC int find_elf_sym(Section *s, const char *name); ST_FUNC int tcc_dynsym_find(TCCState *s1, const char *name); /* find_elf_sym(dynsymtab) + lazy YAFF resolve */ ST_FUNC int tcc_yaff_resolve(TCCState *s1, const char *name); /* on-disk-hash lookup + intern; 0 if absent */ ST_FUNC void tcc_yaff_libs_free(TCCState *s1); +/* 1 if a shared library in the library paths exports `name` (and libtcc1.a does + not define it), 0 if not, -1 when no import set could be built. */ +ST_FUNC int tcc_yaff_import_set_has(TCCState *s1, const char *name); +/* 1 if the libtcc1.a found next to the import set defines `name`. */ +ST_FUNC int tcc_yaff_libtcc1_has(TCCState *s1, const char *name); +ST_FUNC void tcc_yaff_import_set_free(TCCState *s1); ST_FUNC void put_elf_reloc(Section *symtab, Section *s, unsigned long offset, int type, int symbol); ST_FUNC void put_elf_reloca(Section *symtab, Section *s, unsigned long offset, int type, int symbol, addr_t addend); @@ -2406,6 +2588,11 @@ ST_FUNC void tcc_load_referenced_sections(TCCState *s1); ST_FUNC void tcc_free_lazy_objfiles(TCCState *s1); ST_FUNC int tcc_load_archive(TCCState *s1, int fd, int alacarte); ST_FUNC void tcc_archive_cache_free(TCCState *s1); +/* Read an archive's symbol index into the archive symbol cache without loading + a member; 1 on success, 0 if `path` is not an archive with an index. */ +ST_FUNC int tcc_archive_index_load(TCCState *s1, const char *path); +/* Does the index read by tcc_archive_index_load(path) name `name`? */ +ST_FUNC int tcc_archive_index_has(TCCState *s1, const char *path, const char *name); ST_FUNC int tcc_group_has_satisfiable_undefs(TCCState *s1); ST_FUNC void add_array(TCCState *s1, const char *sec, int c); @@ -2428,6 +2615,8 @@ ST_FUNC void tccelf_add_crtend(TCCState *s1); #if defined TCC_TARGET_ARM ST_FUNC void tccelf_add_arm_fp_lib(TCCState *s1); ST_FUNC int tccelf_arm_fp_lib_is_shared(TCCState *s1); +/* The __aeabi_ runtime this link binds ("softfp", "rp2350fp", ...), NULL for hard float. */ +ST_FUNC const char *tccelf_get_fp_lib_name(TCCState *s1); #endif #endif #ifndef TCC_TARGET_PE @@ -2775,6 +2964,10 @@ ST_FUNC void tcc_gen_machine_bitop1_mop(MachineOperand src1, MachineOperand dest /* Does the active target encode clz/rbit/rev/rev16? Front ends use this to * decide between the native opcode and the libgcc helper call. */ ST_FUNC int tcc_machine_has_bit_ops(void); +/* UMAAL (DSP extension): gates the umaal fusion; see tcc_gen_machine_umaal_mop. */ +ST_FUNC int tcc_machine_has_umaal(void); +ST_FUNC void tcc_gen_machine_umaal_mop(MachineOperand src1, MachineOperand src2, MachineOperand accum, + MachineOperand dest); ST_FUNC void tcc_gen_machine_assign_mop(MachineOperand src, MachineOperand dest, TccIrOp op); /* Same, plus zero_half64's verdict in `zh` (bits 4-5: which halves of a 64-bit * destination no consumer reads and so need not be written). */ @@ -2837,6 +3030,11 @@ ST_FUNC void tcc_gen_machine_dry_run_set_rehearsal(int on); ST_FUNC int tcc_gen_machine_cbz_forward_ok(int32_t target_ir, int current_ir_idx); ST_FUNC int tcc_gen_machine_pool_flushes_total(void); ST_FUNC int tcc_gen_machine_pool_entries_total(void); + +/* Pad the instruction stream with a 2-byte NOP when the next instruction would + * land halfword-odd. Called for an IRQuadCompact carrying `align_target`; see + * that field in tccir.h for the RP2350 measurement that motivates it. */ +ST_FUNC void tcc_gen_machine_align_branch_target(int align); ST_FUNC int tcc_gen_machine_const_needs_pool(int32_t value); ST_FUNC int tcc_gen_machine_pending_pool_size(void); ST_FUNC int tcc_gen_machine_cbz_jump_mop(int rn, int nonzero, int32_t target_ir, int ir_idx); @@ -2847,6 +3045,7 @@ ST_FUNC int tcc_gen_machine_switch_load_dry_run_size(int num_entries); ST_FUNC void tcc_gen_machine_switch_load_mop(MachineOperand src, MachineOperand dest, struct TCCIRSwitchValueTable *vtab, struct TCCIRState *ir, int ir_idx); ST_FUNC void tcc_gen_machine_set_chain(void); +ST_FUNC int tcc_gen_machine_frame_top_offset(void); ST_FUNC void tcc_gen_machine_restore_chain(void); ST_FUNC void tcc_gen_machine_init_chain_slot(IROperand src1); ST_FUNC void tcc_gen_machine_backpatch_jump(int address, int offset); @@ -2890,6 +3089,9 @@ ST_FUNC void tcc_gen_machine_mov_equiv_reset(void); ST_FUNC void tcc_gen_machine_reserve_pool_bytes(int upcoming_bytes); ST_FUNC void tcc_gen_machine_strldr_cache_reset(void); ST_FUNC void tcc_gen_machine_strldr_cache_set_enabled(int enabled); +ST_FUNC void tcc_gen_machine_str_elide_set_enabled(int enabled); +ST_FUNC void tcc_gen_machine_slot_self_copy_set_enabled(int enabled); +ST_FUNC void tcc_gen_machine_frame_word_pair_set_enabled(int enabled); ST_FUNC void tcc_gen_machine_imm_cache_reset(void); ST_FUNC void tcc_gen_machine_imm_cache_invalidate_live(uint32_t live_mask); @@ -2907,6 +3109,7 @@ ST_FUNC void tcc_gen_machine_nl_longjmp_mop(MachineOperand buf); /* __builtin_apply_args / __builtin_apply instruction generation */ ST_FUNC void tcc_gen_machine_builtin_apply_args_mop(MachineOperand dest); +ST_FUNC void tcc_gen_machine_return_address_mop(MachineOperand dest); ST_FUNC void tcc_gen_machine_builtin_apply_mop(MachineOperand fn, MachineOperand args, MachineOperand dest); /* Block copy from const data to stack (LDM/STM on ARM) */ @@ -2914,6 +3117,22 @@ ST_FUNC void tcc_gen_machine_block_copy_mop(TCCIRState *ir, IROperand dest, IROp /* Block copy between spill slots using LDM/STM (peephole for consecutive LOAD+STORE pairs) */ ST_FUNC void tcc_gen_machine_spill_block_copy(int32_t src_spill_off, int32_t dst_spill_off, int nwords); +ST_FUNC int tcc_gen_machine_calls_reload_r9(struct TCCIRState *ir); +/* Is a direct call to `sym` known (or, under -fmodule-local-calls, assumed) to + bind inside this module, so that R9 survives it? */ +ST_FUNC int tcc_gen_machine_callee_in_this_module(Sym *sym); +ST_FUNC int tcc_gen_machine_spill_block_copy_free(int32_t src_spill_off, int32_t dst_spill_off, int nwords, + uint32_t also_free); + +/* A side of tcc_gen_machine_reg_block_copy that is a frame slot rather than a + * base register: its offset is a spill-slot offset, to be taken from whichever + * register the frame is addressed through. */ +#define MACH_BLOCK_COPY_FRAME (-1) + +/* Block copy of `nwords` words between two addresses, each a base register (a + * pointer deref) or MACH_BLOCK_COPY_FRAME, through registers the caller owns */ +ST_FUNC int tcc_gen_machine_reg_block_copy(int src_base, int32_t src_off, int dst_base, int32_t dst_off, + const int *regs, int nwords, int end_idx); /* Conditional select: dest = (cond) ? then_val : else_val (ITE on ARM) */ ST_FUNC void tcc_gen_machine_select_mop(MachineOperand then_val, MachineOperand else_val, MachineOperand dest, diff --git a/source/ir/Makefile b/source/ir/Makefile index c42573cb6..7a773fd62 100644 --- a/source/ir/Makefile +++ b/source/ir/Makefile @@ -13,11 +13,19 @@ IR_SRC = \ source/ir/pool.c \ source/ir/vreg.c \ source/ir/stack.c \ + source/ir/frame.c \ + source/ir/cross_jump.c \ + source/ir/self_store.c \ + source/ir/dead_def.c \ source/ir/dump.c \ source/ir/codegen.c \ source/ir/cfg.c \ source/ir/ssa.c \ source/ir/regalloc.c \ + source/ir/regalloc_scan.c \ + source/ir/regalloc_phi.c \ + source/ir/regalloc_entry.c \ + source/ir/regalloc_post.c \ source/ir/machine_op.c \ source/ir/tccir_operand.c @@ -25,7 +33,7 @@ IR_HDRS = $(wildcard source/ir/*.h) # $(IR_GEN_FILES) comes from source/ir/gen/Makefile, included first by the # top-level Makefile. -IR_OBJ = $(patsubst %.c,$(X)%.o,$(IR_SRC) $(IR_GEN_FILES)) +IR_OBJ = $(call lib-objs,ir,$(IR_SRC) $(IR_GEN_FILES)) IR_LIB = $(X)source/ir/libir.a $(X)source/ir/%.o: source/ir/%.c $(IR_HDRS) $(TCC_CORE_DEPS) diff --git a/source/ir/codegen.c b/source/ir/codegen.c index 5a831bfd5..60cba6271 100644 --- a/source/ir/codegen.c +++ b/source/ir/codegen.c @@ -125,7 +125,7 @@ void tcc_ir_fill_registers(TCCIRState *ir, SValue *sv) * (negative FP offset), not the caller's argument area, so VT_PARAM * would incorrectly add offset_to_args. */ int spilled_param_flag = 0; - if ((old_r & VT_PARAM) && interval->incoming_reg0 < 0) + if ((old_r & VT_PARAM) && interval->incoming_reg0 < 0 && !interval->nested_home) { spilled_param_flag = VT_PARAM; } @@ -163,7 +163,7 @@ static int ir_codegen_op_touches_volatile(TCCIRState *ir, IRQuadCompact *q) if (irop_config[q->op].has_dest) ops[no++] = tcc_ir_op_get_dest(ir, q); if (irop_config[q->op].has_src1) ops[no++] = tcc_ir_op_get_src1(ir, q); if (irop_config[q->op].has_src2) ops[no++] = tcc_ir_op_get_src2(ir, q); - if (q->op == TCCIR_OP_MLA) ops[no++] = tcc_ir_op_get_accum(ir, q); + if (tcc_ir_op_is_mac(q->op)) ops[no++] = tcc_ir_op_get_accum(ir, q); for (int k = 0; k < no; k++) { int32_t vr = irop_get_vreg(ops[k]); @@ -185,6 +185,22 @@ static int ir_codegen_op_touches_volatile(TCCIRState *ir, IRQuadCompact *q) * Parameter Register Allocation * ============================================================================ */ +/* A stack-passed parameter is read from the caller's argument area, unless + * tcc_ir_avoid_spilling_stack_passed_params left it the core registers linear + * scan gave it; the prologue loads those. */ +static void stack_param_home(IRLiveInterval *interval) +{ + /* A parameter a nested function captures keeps its frame slot: that slot is + * the capture's target, and the prologue homes the incoming value into it. */ + if (interval->nested_home && interval->allocation.offset != 0) + return; + if (interval->allocation.r0 <= 12 && interval->allocation.offset == 0) + return; + interval->allocation.r0 = PREG_NONE; + interval->allocation.r1 = PREG_NONE; + interval->allocation.offset = 0; +} + void tcc_ir_register_allocation_params(TCCIRState *ir) { /* For leaf functions: parameters can stay in registers r0-r3, UNLESS @@ -230,6 +246,16 @@ void tcc_ir_register_allocation_params(TCCIRState *ir) continue; } } + /* Placed on the caller's stack by the ABI (see params_mark_stack): every + * core register is used or skipped by now. */ + if (interval && interval->incoming_stack) + { + interval->incoming_reg0 = -1; + interval->incoming_reg1 = -1; + stack_param_home(interval); + argno = 4; + continue; + } /* is_double for soft-float (LS_REG_TYPE_DOUBLE_SOFT) or is_llong for 64-bit */ int is_64bit = interval && (interval->is_double || interval->is_llong || interval->is_complex); @@ -283,11 +309,7 @@ void tcc_ir_register_allocation_params(TCCIRState *ir) */ if (interval->original_offset == 0) interval->original_offset = (argno - 4) * 4; - /* Don't overwrite allocator spill slots with caller-stack offsets. - */ - interval->allocation.r0 = PREG_NONE; - interval->allocation.r1 = PREG_NONE; - interval->allocation.offset = 0; + stack_param_home(interval); } argno += 2; } @@ -309,11 +331,7 @@ void tcc_ir_register_allocation_params(TCCIRState *ir) */ if (interval->original_offset == 0) interval->original_offset = (argno - 4) * 4; - /* Don't overwrite allocator spill slots with caller-stack offsets. - */ - interval->allocation.r0 = PREG_NONE; - interval->allocation.r1 = PREG_NONE; - interval->allocation.offset = 0; + stack_param_home(interval); } argno++; } @@ -420,6 +438,109 @@ void tcc_ir_mark_return_value_incoming_regs(TCCIRState *ir) } } +static int ls_gpr(int r) +{ + return r >= 0 && r <= 12; +} + +/* Whether stack-passed parameter `ls` may stay in the core register(s) linear + * scan assigned it. Its value exists before instruction 0, so no other value + * live at entry may share them: the boundary rules that let an interval take + * a register at the instruction where another one's last use sits do not hold + * for the prologue's load, which happens before either. */ +static int stack_param_keeps_register(TCCIRState *ir, const LSLiveInterval *ls) +{ + if (ls->addrtaken || ls->stack_location != 0 || !ls_gpr(ls->r0)) + return 0; + if (ls->reg_type == LS_REG_TYPE_LLONG || ls->reg_type == LS_REG_TYPE_DOUBLE_SOFT) + { + if (!ls_gpr(ls->r1) || ls->r1 == ls->r0) + return 0; + } + else if (ls->reg_type != LS_REG_TYPE_INT || ls->r1 >= 0) + return 0; + IRLiveInterval *iv = tcc_ir_vreg_live_interval(ir, (int)ls->vreg); + if (!iv || iv->addrtaken || iv->is_struct || iv->is_complex || iv->use_vfp || iv->is_volatile) + return 0; + if ((iv->is_double || iv->is_llong) != (ls->r1 >= 0)) + return 0; + const uint32_t mine = (1u << ls->r0) | (ls->r1 >= 0 ? 1u << ls->r1 : 0); + for (int k = 0; k < ir->ls.next_interval_index; ++k) + { + const LSLiveInterval *o = &ir->ls.intervals[k]; + if (o == ls) + continue; + const int at_entry = TCCIR_DECODE_VREG_TYPE((int)o->vreg) == TCCIR_VREG_TYPE_PARAM || + (ir->has_static_chain && (int)o->vreg == ir->static_chain_vreg); + if (!at_entry || o->stack_location != 0) + continue; + uint32_t theirs = 0; + if (ls_gpr(o->r0)) + theirs |= 1u << o->r0; + if (ls_gpr(o->r1)) + theirs |= 1u << o->r1; + if (theirs & mine) + return 0; + } + return 1; +} + +/* GNU nested functions capture the enclosing function's variables BY REFERENCE, + * through a static chain pointing at the parent's frame. A captured variable + * must therefore have a stable home at a fixed offset in that frame, and every + * parent access must go through it. Locals get one for free: the capture marks + * them address-taken and linear scan spills them. Parameters do not — a + * register-passed one lives in a register, and a stack-passed one lives in the + * CALLER's argument area, at an offset_to_args-relative position the child + * cannot name (the children are compiled from finalize_nested_functions, before + * the parent's prologue fixes offset_to_args). + * + * True when `vreg` is a parameter of the function currently being compiled and + * some nested function of THAT function captures it. The parent_nf filter is + * what keeps a child's own PARAM 0 from matching its parent's captured PARAM 0: + * when the parent is being compiled current_nested_func is NULL, when child C + * is being compiled it is C, and a capture belongs to whichever function + * declared it. Stateless — derived from the live nested_funcs table, which is + * cleared only at the end of compile_nested_functions — so it is immune to the + * addrtaken resets that run between parsing and register allocation. */ +int tcc_ir_param_is_captured_by_nested(int vreg) +{ + if (!tcc_state || tcc_state->nb_nested_funcs == 0) + return 0; + if (TCCIR_DECODE_VREG_TYPE(vreg) != TCCIR_VREG_TYPE_PARAM) + return 0; + for (int i = 0; i < tcc_state->nb_nested_funcs; i++) + { + const NestedFunc *nf = &tcc_state->nested_funcs[i]; + if (nf->parent_nf != tcc_state->current_nested_func) + continue; + for (int j = 0; j < nf->nb_captured; j++) + if (nf->captured_vregs[j] == vreg) + return 1; + } + return 0; +} + +/* Decide, once and for the whole function, which parameters need a parent-frame + * home because a nested function captures them. `addrtaken` carries the rest + * of the decision: prescan_captured_vars sets it for every capture, and + * gen_function clears it again for the nested functions that were inlined at + * every call site — those need no chain, so their captures need no home. Run + * before compute_stack_layout so the three places that would otherwise reset a + * parameter to its caller-side home can see the flag. */ +void tcc_ir_mark_nested_captured_params(TCCIRState *ir) +{ + if (!ir || !tcc_state || tcc_state->nb_nested_funcs == 0) + return; + for (int vreg = 0; vreg < ir->next_parameter; ++vreg) + { + const int encoded_vreg = (TCCIR_VREG_TYPE_PARAM << 28) | vreg; + IRLiveInterval *interval = tcc_ir_vreg_live_interval(ir, encoded_vreg); + if (interval && interval->addrtaken && tcc_ir_param_is_captured_by_nested(encoded_vreg)) + interval->nested_home = 1; + } +} + void tcc_ir_avoid_spilling_stack_passed_params(TCCIRState *ir) { if (!ir) @@ -468,6 +589,12 @@ void tcc_ir_avoid_spilling_stack_passed_params(TCCIRState *ir) continue; } + if (interval->incoming_stack) + { + is_stack_passed[vreg] = 1; + argno = 4; + continue; + } const int is_64bit = interval->is_double || interval->is_llong; if (is_64bit && (argno & 1)) argno++; /* align 64-bit to even reg pair */ @@ -479,6 +606,34 @@ void tcc_ir_avoid_spilling_stack_passed_params(TCCIRState *ir) argno += is_64bit ? 2 : 1; } + /* Static references to each parameter, reads and writes alike. */ + uint16_t *refs = tcc_mallocz(sizeof(uint16_t) * (size_t)param_count); + for (int i = 0; i < ir->next_instruction_index; ++i) + { + const IRQuadCompact *q = &ir->compact_instructions[i]; + if (q->op == TCCIR_OP_NOP) + continue; + IROperand ops[4]; + int n = 0; + if (irop_config[q->op].has_dest) + ops[n++] = tcc_ir_op_get_dest(ir, q); + if (irop_config[q->op].has_src1) + ops[n++] = tcc_ir_op_get_src1(ir, q); + if (irop_config[q->op].has_src2) + ops[n++] = tcc_ir_op_get_src2(ir, q); + if (tcc_ir_op_is_mac(q->op)) + ops[n++] = tcc_ir_op_get_accum(ir, q); + for (int k = 0; k < n; ++k) + { + const int32_t vr = irop_get_vreg(ops[k]); + if (vr < 0 || TCCIR_DECODE_VREG_TYPE(vr) != TCCIR_VREG_TYPE_PARAM) + continue; + const int p = TCCIR_DECODE_VREG_POSITION(vr); + if (p < param_count && refs[p] < UINT16_MAX) + refs[p]++; + } + } + /* Rewrite linear-scan results: stack-passed params already have a memory * home (caller arg area), so drop any local spill slot. Also force * address-taken stack params to remain in memory. */ @@ -493,14 +648,28 @@ void tcc_ir_avoid_spilling_stack_passed_params(TCCIRState *ir) if (!is_stack_passed[pidx]) continue; - /* Stack-passed params live in the caller's argument area. If linear-scan - * assigned them a register (without spilling), the prolog won't load - * them — always reset r0/r1 to force use of the incoming stack location. */ + /* A parameter the allocator gave core registers keeps them: the prologue + * loads it once (tcc_gen_machine_prolog), where every use would otherwise + * load it from the caller's area, a far access in a big frame. Read + * once, it is better left there: the one use loads it straight where it + * is needed, while the prologue's copy would still have to be moved. */ + /* Captured by a nested function: the linear-scan slot IS its home, and + * the prologue copies the incoming value into it. Resetting it here + * would send the parent back to the caller's argument area and leave the + * capture resolving to chain offset 0. */ + { + IRLiveInterval *li = tcc_ir_vreg_live_interval(ir, (int)ls->vreg); + if (li && li->nested_home && ls->stack_location != 0) + continue; + } + if (refs[pidx] >= 2 && stack_param_keeps_register(ir, ls)) + continue; ls->r0 = PREG_NONE; ls->r1 = PREG_NONE; ls->stack_location = 0; } + tcc_free(refs); tcc_free(is_stack_passed); } @@ -1004,7 +1173,8 @@ static void tcc_ir_codegen_inline_asm_by_id(TCCIRState *ir, int id) * not the value. That pointer is read by both prolog load and epilog * store, so it must survive across the asm body. Keep lvalue-operand * registers reserved so the value gets a distinct register. */ - if (vals[i].r & VT_LVAL) + /* A register variable's operand is a local's value, not an address. */ + if ((vals[i].r & VT_LVAL) && !ops[i].regvar) continue; if (!vals[i].pr0_spilled && vals[i].pr0_reg != PREG_REG_NONE && vals[i].pr0_reg < NB_ASM_REGS) reserved_regs[vals[i].pr0_reg] = 0; @@ -1270,12 +1440,101 @@ static int32_t *demote_ref_cache_build(TCCIRState *ir, int n_insns) } /* Prices the O(interval length) rescan this pass does per candidate. */ +/* TCC_DYN_ARGS_MIN: calls passing more stack-argument bytes than this get + * their own `sub sp` window instead of the frame's reserved area; 0 = never. */ +TCC_DBG_ENV_INT(dyn_args_min, "TCC_DYN_ARGS_MIN", 256) +/* Largest outgoing area the windows are used for: past it the scratch save + * slots above the reserved area could leave STR's 4 KB reach. */ +#define DYN_ARGS_MAX 3072 + TCC_DBG_ENV_FLAG(cg_no_scratch_demote, "TCC_NO_SCRATCH_DEMOTE") /* Dumps the post-opt IR that codegen is about to consume. */ TCC_DBG_ENV_FLAG(cg_dump_ir_cg, "DUMP_IR_CG") /* A/B levers for the two codegen walks priced in docs/remote_smoke_speedup_plan.md. */ TCC_DBG_ENV_FLAG(cg_keep_fwd_dry, "TCC_KEEP_FWD_DRY") TCC_DBG_ENV_FLAG(cg_no_rehearsal, "TCC_NO_REHEARSAL") +TCC_DBG_ENV_INT(cg_align_loops, "TCC_ALIGN_LOOPS", 4) +/* TCC_NO_FUNC_ALIGN=1: leave a literal-reading body at the 2 mod 4 start it got. */ +TCC_DBG_ENV_FLAG(cg_no_func_align, "TCC_NO_FUNC_ALIGN") + +/* Word-align the start of a body that reads its literal pool. A literal + * load's offset and the pad before each pool dump depend on where the code + * sits mod 4, so the same function compiled at a start that is 2 mod 4 is + * different machine code -- and identical code folding (icf.c) compares + * machine code: Zig's C had generic instantiations folding or not by the + * parity the functions before them happened to leave. Only before anything + * of the body is emitted; the pad goes with the body if it is folded + * (function.c). -O0 keeps its layout. */ +static void cg_word_align_start(void) +{ + if (!(ind & 2) || ind != func_ind || tcc_state->optimize == 0 || tcc_state->do_debug || cg_no_func_align()) + return; + gen_fill_nops(2); + func_ind = ind; + func_align_pad = 2; + if (tcc_state->cur_func_sym && tcc_state->cur_func_sym->c) + elfsym(tcc_state->cur_func_sym)->st_value = ind + 1; +} + +/* Whether a body may read a literal pool: a symbol's address, a float or + * 64-bit constant, or a 32-bit one outside what MOVS/MOVW/MVN build (a call's + * target is a BL, not a load). Over-answering costs a 2-byte pad, + * under-answering a missed fold -- never correctness. */ +static int cg_may_read_pool(TCCIRState *ir) +{ + for (int i = 0; i < ir->next_instruction_index; i++) + { + IRQuadCompact *q = &ir->compact_instructions[i]; + if (q->op == TCCIR_OP_NOP) + continue; + const int is_call = q->op == TCCIR_OP_FUNCCALLVAL || q->op == TCCIR_OP_FUNCCALLVOID; + for (int s = 0; s < 3; s++) + { + if ((s == 0 && !irop_config[q->op].has_dest) || (s == 1 && !irop_config[q->op].has_src1) || + (s == 2 && !irop_config[q->op].has_src2) || (s == 1 && is_call)) + continue; + IROperand o = s == 0 ? tcc_ir_op_get_dest(ir, q) : s == 1 ? tcc_ir_op_get_src1(ir, q) : tcc_ir_op_get_src2(ir, q); + const int tag = irop_get_tag(o); + if (tag == IROP_TAG_SYMREF || tag == IROP_TAG_I64 || tag == IROP_TAG_F64 || tag == IROP_TAG_F32) + return 1; + if (tag == IROP_TAG_IMM32) + { + const int32_t v = (int32_t)irop_get_imm32(o); + if (v < -256 || v > 65535) + return 1; + } + } + } + return 0; +} + + +/* CBZ/CBNZ set no flags, so the compare it replaces must have no OTHER reader. + * ra:redundant_cmp deletes a duplicate CMP on either of a JUMPIF's edges + * precisely because the earlier compare's flags survive there, which leaves a + * conditional on that edge depending on the flags this fusion would stop + * producing. gcc.c-torture 20010221-1's `if (i != 0) x = (i > 0) ? i : 0;` + * lowers to CMP i,#0 / JUMPIF EQ / JUMPIF GT; fusing the first pair into + * `cbz r1` left the `bgt` reading whatever was in the flags, and x came out + * 45 instead of 1. Latent since the peephole landed -- any layout change that + * brings the branch into CBZ range exposes it. */ +static int ir_codegen_flags_read_at(TCCIRState *ir, int from) +{ + int n = ir->next_instruction_index; + if (from < 0) + return 1; /* unknown edge: assume a reader */ + while (from < n && ir->compact_instructions[from].op == TCCIR_OP_NOP) + from++; + if (from >= n) + return 0; + int op = ir->compact_instructions[from].op; + return op == TCCIR_OP_JUMPIF || op == TCCIR_OP_SETIF || op == TCCIR_OP_SELECT; +} + +static int demote_to_param_home(const IRLiveInterval *ir_iv, int32_t vreg) +{ + return TCCIR_DECODE_VREG_TYPE(vreg) == TCCIR_VREG_TYPE_PARAM && ir_iv->incoming_reg0 < 0; +} static int try_demote_scratch_conflict(TCCIRState *ir, int r, int insn_i, const uint16_t *dry_insn_saves, const int *dry_insn_scratch, int n_insns, int32_t **ref_cache) @@ -1389,12 +1648,23 @@ static int try_demote_scratch_conflict(TCCIRState *ir, int r, int insn_i, const fprintf(stderr, "[phase3-demote] insn=%d vreg=%d R%d -> memory (blocked=%d refs=%d range=%d..%d)\n", insn_i, (int)ls_iv->vreg, r, blocked, refs, start, end); - /* --- Apply the demotion. The real slot is assigned by the caller. --- */ + /* --- Apply the demotion. The real slot is assigned by the caller, except + * for a stack-passed parameter: it goes back to the caller's argument area, + * which already holds it (the prologue loads only register-resident ones). */ ls_iv->r0 = -1; - ls_iv->stack_location = DEMOTE_PENDING_SLOT; - ir_iv->allocation.r0 = PREG_SPILLED | PREG_REG_NONE; - ir_iv->allocation.r1 = PREG_NONE; - ir_iv->allocation.offset = DEMOTE_PENDING_SLOT; + if (demote_to_param_home(ir_iv, vreg)) + { + ir_iv->allocation.r0 = PREG_NONE; + ir_iv->allocation.r1 = PREG_NONE; + ir_iv->allocation.offset = 0; + } + else + { + ls_iv->stack_location = DEMOTE_PENDING_SLOT; + ir_iv->allocation.r0 = PREG_SPILLED | PREG_REG_NONE; + ir_iv->allocation.r1 = PREG_NONE; + ir_iv->allocation.offset = DEMOTE_PENDING_SLOT; + } /* No other interval claims r over [start, end] (checked above), so the whole * window can be released unconditionally. */ @@ -1935,7 +2205,7 @@ static int ir_codegen_count_vreg_uses(TCCIRState *ir, int32_t vreg) uses++; if (irop_config[q->op].has_src2 && irop_get_vreg(tcc_ir_op_get_src2(ir, q)) == vreg) uses++; - if (q->op == TCCIR_OP_MLA && q->operand_base + 3 < ir->iroperand_pool_count && + if (tcc_ir_op_is_mac(q->op) && q->operand_base + 3 < ir->iroperand_pool_count && irop_get_vreg(ir->iroperand_pool[q->operand_base + 3]) == vreg) uses++; } @@ -1971,7 +2241,7 @@ static int ir_codegen_vreg_used_elsewhere(TCCIRState *ir, int32_t vreg, int def_ return 1; if (irop_config[q->op].has_src2 && irop_get_vreg(tcc_ir_op_get_src2(ir, q)) == vreg) return 1; - if (q->op == TCCIR_OP_MLA && q->operand_base + 3 < ir->iroperand_pool_count && + if (tcc_ir_op_is_mac(q->op) && q->operand_base + 3 < ir->iroperand_pool_count && irop_get_vreg(ir->iroperand_pool[q->operand_base + 3]) == vreg) return 1; } @@ -2060,14 +2330,106 @@ static MopArgs decode_mop_args(TCCIRState *ir, IRQuadCompact *cq, const IROperan /* ============================================================================ * OPTION B: MopArgs cache helper * ============================================================================ - * During the dry-run, decoded dest/src1/src2 operands are stored in mop_cache. + * During the dry-run, decoded operands are stored in mop_cache. * During the real-run (when the cache is valid), they are read back directly, * skipping the interval-table lookups in decode_mop_args. * - * Instructions that use scale or accum (indexed loads/stores, MLA) are rare; - * those slots are not cached — they fall through to a full decode in both passes. + * Storage: per instruction only the slots its MopSpec decodes (the others are + * never written by decode_mop_args, so nothing reads them), as full + * MachineOperands behind a slot-mask header, packed into fixed-size chunks. + * A flat MopArgs per instruction was 80 bytes on the device and the largest + * allocation of a big function's codegen. at[i] = 1 + the record's offset + * (chunk * MOP_CACHE_CHUNK + offset in chunk); 0 = never recorded. * ============================================================================ */ -static inline MopArgs ir_decode_cached(int is_dry_run, int use_mop_cache, MopArgs *mop_cache, int i, TCCIRState *ir, +#define MOP_CACHE_CHUNK 16384 +#define MOP_CACHE_HDR 8 /* slot mask byte, padded to MachineOperand's alignment */ + +typedef struct MopCache +{ + uint32_t *at; + unsigned char **chunks; + int nchunks; + uint32_t fill; /* bytes used in the last chunk */ +} MopCache; + +static MopCache *mop_cache_new(int n) +{ + MopCache *mc = tcc_mallocz(sizeof(*mc)); + mc->at = tcc_mallocz(sizeof(uint32_t) * n); + return mc; +} + +void tcc_ir_mop_cache_free(void *cache) +{ + MopCache *mc = cache; + if (!mc) + return; + for (int c = 0; c < mc->nchunks; c++) + tcc_free(mc->chunks[c]); + tcc_free(mc->chunks); + tcc_free(mc->at); + tcc_free(mc); +} + +static unsigned mop_spec_mask(MopSpec spec) +{ + return (spec.dest ? 1u : 0) | (spec.src1 ? 2u : 0) | (spec.src2 ? 4u : 0) | (spec.scale ? 8u : 0) | + (spec.accum ? 16u : 0); +} + +static unsigned char *mop_cache_record(MopCache *mc, int i) +{ + uint32_t at = mc->at[i]; + return at ? mc->chunks[(at - 1) / MOP_CACHE_CHUNK] + (at - 1) % MOP_CACHE_CHUNK : NULL; +} + +/* Record instruction i's decode. A later dry-run pass re-decoding i with the + * same slots overwrites the record in place, as the flat array did. */ +static void mop_cache_put(MopCache *mc, int i, const MopArgs *a, MopSpec spec) +{ + unsigned mask = mop_spec_mask(spec); + unsigned char *rec = mop_cache_record(mc, i); + if (!rec || rec[0] != mask) + { + uint32_t need = MOP_CACHE_HDR + (uint32_t)__builtin_popcount(mask) * sizeof(MachineOperand); + if (mc->nchunks == 0 || mc->fill + need > MOP_CACHE_CHUNK) + { + mc->chunks = tcc_realloc(mc->chunks, sizeof(*mc->chunks) * (mc->nchunks + 1)); + mc->chunks[mc->nchunks++] = tcc_malloc(MOP_CACHE_CHUNK); + mc->fill = 0; + } + rec = mc->chunks[mc->nchunks - 1] + mc->fill; + mc->at[i] = 1 + (uint32_t)(mc->nchunks - 1) * MOP_CACHE_CHUNK + mc->fill; + mc->fill += need; + rec[0] = (unsigned char)mask; + } + MachineOperand *o = (MachineOperand *)(rec + MOP_CACHE_HDR); + if (mask & 1) *o++ = a->dest; + if (mask & 2) *o++ = a->src1; + if (mask & 4) *o++ = a->src2; + if (mask & 8) *o++ = a->scale; + if (mask & 16) *o++ = a->accum; +} + +/* Replay instruction i's recorded decode into *a; 0 when i was never recorded + * or its record lacks a slot `spec` needs. */ +static int mop_cache_get(MopCache *mc, int i, MopArgs *a, MopSpec spec) +{ + unsigned char *rec = mop_cache_record(mc, i); + if (!rec || (mop_spec_mask(spec) & ~(unsigned)rec[0])) + return 0; + unsigned mask = rec[0]; + const MachineOperand *o = (const MachineOperand *)(rec + MOP_CACHE_HDR); + memset(a, 0, sizeof(*a)); + if (mask & 1) a->dest = *o++; + if (mask & 2) a->src1 = *o++; + if (mask & 4) a->src2 = *o++; + if (mask & 8) a->scale = *o++; + if (mask & 16) a->accum = *o++; + return 1; +} + +static inline MopArgs ir_decode_cached(int is_dry_run, int use_mop_cache, MopCache *mop_cache, int i, TCCIRState *ir, IRQuadCompact *cq, const IROperand *src1_ir, const IROperand *src2_ir, const IROperand *dest_ir, MopSpec spec) { @@ -2083,7 +2445,7 @@ static inline MopArgs ir_decode_cached(int is_dry_run, int use_mop_cache, MopArg * (ptr fuzz seed 30436: `ldr r8, [...]` clobbered by stale-cache `mov r8, ip`). */ if (!is_dry_run && use_mop_cache) { - MopArgs cached = mop_cache[i]; + MopArgs cached; /* A peephole that skips an instruction (i = next_i; break) can fire in the * dry-run but not the real-run when its decision depends on pass-varying * state. The STRD-spill fusion is one such case: it keys on the @@ -2091,11 +2453,12 @@ static inline MopArgs ir_decode_cached(int is_dry_run, int use_mop_cache, MopArg * term is 0 during the dry-run (the prologue that sets it runs only before * the real pass) but final during the real-run. A large frame can therefore * make the dry-run fuse-and-skip instruction i while the real-run does not, - * leaving mop_cache[i] never written (zero-initialised → all MACH_OP_NONE). - * A genuinely decoded store/load always materialises dest or src1, so an - * all-NONE pair marks an unpopulated slot: re-decode instead of returning - * the stale sentinel (which would trip the MACH_OP_NONE codegen assert). */ - if (cached.dest.kind != MACH_OP_NONE || cached.src1.kind != MACH_OP_NONE) + * leaving instruction i never recorded. A genuinely decoded store/load + * always materialises dest or src1, so a record with neither is treated + * like a missing one: re-decode instead of returning the stale sentinel + * (which would trip the MACH_OP_NONE codegen assert). */ + if (mop_cache_get(mop_cache, i, &cached, spec) && + (cached.dest.kind != MACH_OP_NONE || cached.src1.kind != MACH_OP_NONE)) return cached; } @@ -2103,7 +2466,7 @@ static inline MopArgs ir_decode_cached(int is_dry_run, int use_mop_cache, MopArg /* Dry-run: store the decoded operands for real-run replay. */ if (is_dry_run && mop_cache) - mop_cache[i] = a; + mop_cache_put(mop_cache, i, &a, spec); return a; } @@ -2115,6 +2478,189 @@ static inline MopArgs ir_decode_cached(int is_dry_run, int use_mop_cache, MopArg * UT_COVERAGE_ONLY_SRCS), so a definition over there fails to link `ut`. */ unsigned long tcc_dryreason[8]; +/* Operand slots naming each TEMP vreg, by position: a copy peephole may drop a + * loaded temporary only when its STORE is its one use. */ +static int *ir_codegen_temp_uses(TCCIRState *ir, int *count) +{ + int n = ir->next_temporary_variable > 0 ? ir->next_temporary_variable : 1; + int *uses = tcc_mallocz(sizeof(int) * n); + for (int i = 0; i < ir->next_instruction_index; i++) + { + IRQuadCompact *q = &ir->compact_instructions[i]; + if (q->op == TCCIR_OP_NOP) + continue; + int nops = irop_config[q->op].has_dest + irop_config[q->op].has_src1 + irop_config[q->op].has_src2; + if (tcc_ir_op_is_mac(q->op) || q->op == TCCIR_OP_LOAD_INDEXED || q->op == TCCIR_OP_STORE_INDEXED || + q->op == TCCIR_OP_SELECT) + nops++; + for (int k = 0; k < nops; k++) + { + int32_t vr = irop_get_vreg(ir->iroperand_pool[q->operand_base + k]); + if (vr >= 0 && TCCIR_DECODE_VREG_TYPE(vr) == TCCIR_VREG_TYPE_TEMP && TCCIR_DECODE_VREG_POSITION(vr) < n) + uses[TCCIR_DECODE_VREG_POSITION(vr)]++; + } + } + *count = n; + return uses; +} + +/* A loaded TEMP whose only other mention is the STORE that takes it. */ +static int ir_codegen_temp_dies_at_store(const int *uses, int nuses, int32_t vr) +{ + return uses && vr >= 0 && TCCIR_DECODE_VREG_TYPE(vr) == TCCIR_VREG_TYPE_TEMP && + TCCIR_DECODE_VREG_POSITION(vr) < nuses && uses[TCCIR_DECODE_VREG_POSITION(vr)] == 2; +} + +/* Most words one LDM/STM pair of a batched aggregate copy may carry. */ +#define MACH_BLOCK_COPY_MAX_WORDS 8 + +typedef struct +{ + int src_base, dst_base; /* a register, or MACH_BLOCK_COPY_FRAME */ + int32_t src_off, dst_off; + int regs[MACH_BLOCK_COPY_MAX_WORDS]; + int nwords; + int last_i; /* the run's last store */ +} BatchedWordCopy; + +/* One word of a batched aggregate copy: the address it touches and the register + * the word passes through. `base` is a physical register, or + * MACH_BLOCK_COPY_FRAME for a frame slot whose offset the backend resolves. */ +typedef struct +{ + int base; + int32_t off; + int reg; + int32_t vr; +} CopyWordAccess; + +/* Read instruction `idx` as one 32-bit word load (or store, `is_load` 0) of + * such a copy: a frame slot, a plain pointer deref, or a pointer deref with a + * constant displacement. Anything wider, narrower, volatile, scaled or landed + * on by a jump is not one. */ +static int ir_codegen_copy_word_access(TCCIRState *ir, const uint8_t *branch_target_reset, int idx, int is_load, + int is_dry_run, CopyWordAccess *w) +{ + if (idx < 0 || idx >= ir->next_instruction_index) + return 0; + IRQuadCompact *q = &ir->compact_instructions[idx]; + if (q->is_jump_target || (branch_target_reset && branch_target_reset[idx])) + return 0; + const int op = q->op; + const int indexed = op == (is_load ? TCCIR_OP_LOAD_INDEXED : TCCIR_OP_STORE_INDEXED); + if (!indexed && op != (is_load ? TCCIR_OP_LOAD : TCCIR_OP_STORE)) + return 0; + + IROperand src1_ir = tcc_ir_op_get_src1(ir, q); + IROperand src2_ir = tcc_ir_op_get_src2(ir, q); + IROperand dest_ir = tcc_ir_op_get_dest(ir, q); + const MopSpec spec = indexed ? (is_load ? (MopSpec){.dest = 2, .src1 = 1, .src2 = 1, .scale = 1} + : (MopSpec){.dest = 1, .src1 = 1, .src2 = 1, .scale = 1}) + : (MopSpec){.dest = 1, .src1 = 2}; + MopArgs a = ir_decode_cached(is_dry_run, 0, NULL, idx, ir, q, &src1_ir, &src2_ir, &dest_ir, spec); + + /* The word itself: the load's destination, the store's source. */ + const MachineOperand *val = is_load ? &a.dest : &a.src1; + const IROperand *val_ir = is_load ? &dest_ir : &src1_ir; + if (val->kind != MACH_OP_REG || val->needs_deref || val->is_64bit || + (val->btype != IROP_BTYPE_INT32 && val->btype != IROP_BTYPE_FLOAT32)) + return 0; + w->reg = val->u.reg.r0; + w->vr = irop_get_vreg(*val_ir); + + /* The address: the load's source, the store's destination. */ + const MachineOperand *adr = is_load ? &a.src1 : &a.dest; + const IROperand *adr_ir = is_load ? &src1_ir : &dest_ir; + if (tcc_ir_access_is_volatile(ir, *adr_ir) || adr->underalign_hint || val->underalign_hint) + return 0; + if (indexed) + { + if (adr->kind != MACH_OP_REG || adr->needs_deref || a.scale.kind != MACH_OP_IMM || a.scale.u.imm.val != 0 || + a.src2.kind != MACH_OP_IMM) + return 0; + w->base = adr->u.reg.r0; + w->off = (int32_t)a.src2.u.imm.val; + return 1; + } + if (adr->kind == MACH_OP_SPILL && !adr->needs_deref) + { + w->base = MACH_BLOCK_COPY_FRAME; + w->off = adr->u.spill.offset; + return 1; + } + if (adr->kind == MACH_OP_REG && adr->needs_deref) + { + w->base = adr->u.reg.r0; + w->off = 0; + return 1; + } + return 0; +} + +/* A small aggregate copied through a register deref comes out of the frontend + * batched -- every word loaded, then every word stored, so an overlapping copy + * keeps memmove semantics (ir_emit_small_aggregate_copy). Match the whole run + * here: the loads walk one base upwards, the stores walk another in the same + * order, and each word dies at its store, so the registers regalloc gave the + * loads are free for one LDM and one STM. Fills `out` and returns the index of + * the last store, or 0 when the run is not one. */ +static int ir_codegen_scan_batched_word_copy(TCCIRState *ir, const uint8_t *branch_target_reset, int start_i, + const int *temp_uses, int temp_uses_n, int is_dry_run, + BatchedWordCopy *out) +{ + CopyWordAccess w; + if (!ir_codegen_copy_word_access(ir, branch_target_reset, start_i, 1, is_dry_run, &w)) + return 0; + + int32_t vrs[MACH_BLOCK_COPY_MAX_WORDS]; + int n = 0, last = start_i; + out->src_base = w.base; + out->src_off = w.off; + for (;;) + { + if (!ir_codegen_temp_dies_at_store(temp_uses, temp_uses_n, w.vr)) + return 0; + for (int k = 0; k < n; k++) + if (out->regs[k] == w.reg) + return 0; /* two live words in one register: not this shape */ + out->regs[n] = w.reg; + vrs[n] = w.vr; + n++; + if (n == MACH_BLOCK_COPY_MAX_WORDS) + break; + const int next = ir_codegen_next_nonnop_no_label(ir, branch_target_reset, last); + CopyWordAccess nw; + if (!ir_codegen_copy_word_access(ir, branch_target_reset, next, 1, is_dry_run, &nw) || nw.base != out->src_base || + nw.off != out->src_off + 4 * n) + break; + last = next; + w = nw; + } + if (n < 3) + return 0; + + /* The stores, in the order the words were loaded. */ + for (int k = 0; k < n; k++) + { + const int next = ir_codegen_next_nonnop_no_label(ir, branch_target_reset, last); + CopyWordAccess sw; + if (!ir_codegen_copy_word_access(ir, branch_target_reset, next, 0, is_dry_run, &sw) || sw.vr != vrs[k] || + sw.reg != out->regs[k]) + return 0; + if (k == 0) + { + out->dst_base = sw.base; + out->dst_off = sw.off; + } + else if (sw.base != out->dst_base || sw.off != out->dst_off + 4 * k) + return 0; + last = next; + } + out->nwords = n; + out->last_i = last; + return 1; +} + void tcc_ir_codegen_generate(TCCIRState *ir) { IRQuadCompact *cq; @@ -2177,6 +2723,9 @@ void tcc_ir_codegen_generate(TCCIRState *ir) int call_count = 0; int has_softfloat_ops = 0; int max_nested_save_regs = 0; + /* Each call's stack-argument bytes by call id, for the window policy below. */ + int32_t *call_stack = NULL; + int call_stack_size = 0; for (int i = 0; i < ir->next_instruction_index; i++) { const IRQuadCompact *q = &ir->compact_instructions[i]; @@ -2225,6 +2774,20 @@ void tcc_ir_codegen_generate(TCCIRState *ir) stack = (stack + 7) & ~7; /* 8-byte align (AAPCS) */ if (stack > max_outgoing) max_outgoing = stack; + if (stack > 0 && call_id >= 0) + { + if (call_id >= call_stack_size) + { + int n = call_stack_size ? call_stack_size : 64; + while (n <= call_id) + n *= 2; + call_stack = tcc_realloc(call_stack, sizeof(int32_t) * n); + memset(call_stack + call_stack_size, 0, sizeof(int32_t) * (n - call_stack_size)); + call_stack_size = n; + } + if (stack > call_stack[call_id]) + call_stack[call_id] = stack; + } /* Compute actual nested-call save needs using liveness data. * Only R0-R3 that hold values live BEFORE argument setup AND those @@ -2326,7 +2889,7 @@ void tcc_ir_codegen_generate(TCCIRState *ir) * text_and_data_separation requires R9 save/restore around the call. * With stack args, the pre-reserved outgoing area would need to be set up * before the branch, complicating frame teardown. */ - if (ir->tail_call_only && (max_outgoing > 0 || tcc_state->text_and_data_separation)) + if (ir->tail_call_only && (max_outgoing > 0 || (tcc_state->text_and_data_separation && !ir->pure_forward))) { ir->tail_call_only = 0; ir->leaffunc = 0; @@ -2341,9 +2904,9 @@ void tcc_ir_codegen_generate(TCCIRState *ir) save_regs++; /* R9 */ ir->call_nested_save_size = save_regs * 4; } - else if (call_count >= 1 && tcc_state->text_and_data_separation) + else if (call_count >= 1 && tcc_state->text_and_data_separation && tcc_gen_machine_calls_reload_r9(ir)) { - ir->call_nested_save_size = 4; /* R9 only */ + ir->call_nested_save_size = 4; /* R9 only: its one reader is a call's reload */ } else { @@ -2359,6 +2922,37 @@ void tcc_ir_codegen_generate(TCCIRState *ir) if (ir->has_static_chain) tcc_state->need_frame_pointer = 1; + /* The outgoing area sits at the bottom of the frame, so every local is + * addressed past it: one call passing 2 KB of stack arguments puts every + * [sp, #k] of the function out of the 16-bit form's 1020-byte reach (a + * Zig AstGen function went from a 1 KB to a 5 KB frame when a callee with + * such a call was inlined into it). The area is reserved only for the + * calls that pass at most TCC_DYN_ARGS_MIN bytes; one that passes more + * drops SP by the difference around itself alone (tcc_gen_machine_func_ + * call_mop), a `sub sp` and an `add sp`. Frame-relative SP offsets take + * the window into account through the same bias as a scratch push. Not + * with VLAs or alloca (SP already moves, and the area is re-reserved below + * each block), and the reserved area stays within reach of the scratch + * save slots above it, whose PUSH fallback cannot live inside a window. */ + tcc_free(ir->call_dyn_extra); + ir->call_dyn_extra = NULL; + ir->call_dyn_extra_size = 0; + if (call_stack && dyn_args_min() > 0 && max_outgoing > dyn_args_min() && max_outgoing <= DYN_ARGS_MAX && + !tcc_state->func_dynamic_sp && !tcc_state->force_frame_pointer && !tcc_state->do_debug) + { + int keep = 0; + for (int c = 0; c < call_stack_size; c++) + if (call_stack[c] <= dyn_args_min() && call_stack[c] > keep) + keep = call_stack[c]; + ir->call_dyn_extra = tcc_mallocz(sizeof(int32_t) * call_stack_size); + ir->call_dyn_extra_size = call_stack_size; + for (int c = 0; c < call_stack_size; c++) + if (call_stack[c] > keep) + ir->call_dyn_extra[c] = call_stack[c] - keep; + ir->call_outgoing_size = keep; + } + tcc_free(call_stack); + /* A stack-passed parameter used to force FP here whenever the function * also made a call. It does not need to: the MACH_OP_PARAM_STACK path * adds offset_to_args, and without FP the SP-relative form folds in the @@ -2515,6 +3109,98 @@ void tcc_ir_codegen_generate(TCCIRState *ir) has_forward_branch = 1; } + /* Word-align EVERY backward-branch target (see IRQuadCompact.align_target). + * ssa:loop_bottom_test marks the heads it creates itself; this covers the + * loops that were already bottom-tested — ssa:loop_rotate's output, the + * frontend's `do/while`, and every counted loop the pipeline leaves with a + * conditional back-edge. At most 2 bytes per loop, paid once on ENTRY, + * against a refill fetch cycle per ITERATION. + * + * The width of the head instruction is deliberately NOT consulted, and that + * was measured, not assumed: filtering to 32-bit heads (the shape the + * bench_memcpy discovery happened to have) took the rig set win from -0.943% + * to -0.059%. All six of mibench_sha's loop heads are 16-bit and aligning + * them is worth -4.98% on its own; the M33 fetches a word at a time, so a + * target at `addr%4==2` wastes half of the first fetch after the branch + * whatever sits there. Knob TCC_ALIGN_LOOPS. */ + /* Word-align branch targets (see IRQuadCompact.align_target). Levels, with + * the rig set totals they were priced at, each against a size-matched arm: + * + * 1 backward targets only -0.944% vs no alignment, .text +0.34% + * 4 + forward targets INSIDE a loop -3.447% on top of 1, .text +0.59% + * 3 + every forward target -4.047% on top of 1, .text +2.34% + * 2 backward only, to 8 bytes +0.647% -- REJECTED, a loss + * + * 4 is the default: it takes 85% of what aligning every forward target is + * worth for an eighth of the extra code, because a forward target outside + * every loop is an `if` join reached ONCE and only pays the pad. Inside a + * loop it is reached once per iteration, which is the same trade the + * backward targets already won (mibench_dijkstra's hot `run_dijkstra_path` + * had four such joins at `mod4 == 2`; aligning them is -7.2% on its own). + * 8-byte alignment loses: up to three wasted NOPs per loop ENTRY buy no + * further fetch (matrix_mul +2.4%, dijkstra +1.3%). */ + /* -Os pads nothing: every pad is up to 2 bytes for a fetch cycle, the + * trade that tier declines (zig.c: 72 KB of NOPs). */ + const int align_level = tcc_state->optimize_size ? 0 : cg_align_loops(); + const int align_bytes = (align_level == 2) ? 8 : 4; + if (tcc_state->optimize >= 1 && align_level >= 1) + { + /* Loop extents, as [target, back-branch] spans. A forward branch target + * inside one is an `if` join reached once per ITERATION, which is what + * makes it worth the 2 bytes; a forward target outside every loop is + * reached once and only pays. */ + int nspans = 0; + int *spans = NULL; + if (align_level == 4 && ir->next_instruction_index > 0) + spans = tcc_malloc(sizeof(int) * 2 * (size_t)ir->next_instruction_index); + for (int bi = 0; bi < ir->next_instruction_index; bi++) + { + IRQuadCompact *bq = &ir->compact_instructions[bi]; + if (bq->op != TCCIR_OP_JUMP && bq->op != TCCIR_OP_JUMPIF) + continue; + IROperand bdest = tcc_ir_op_get_dest(ir, bq); + if (irop_is_none(bdest)) + continue; + int bt = (int)bdest.u.imm32; + if (bt < 0 || bt >= ir->next_instruction_index) + continue; + if (bt < bi) + { + ir->compact_instructions[bt].align_target = 1; + if (spans) + { + spans[2 * nspans] = bt; + spans[2 * nspans + 1] = bi; + nspans++; + } + } + else if (align_level == 3) + ir->compact_instructions[bt].align_target = 1; + } + if (spans) + { + for (int bi = 0; bi < ir->next_instruction_index; bi++) + { + IRQuadCompact *bq = &ir->compact_instructions[bi]; + if (bq->op != TCCIR_OP_JUMP && bq->op != TCCIR_OP_JUMPIF) + continue; + IROperand bdest = tcc_ir_op_get_dest(ir, bq); + if (irop_is_none(bdest)) + continue; + int bt = (int)bdest.u.imm32; + if (bt <= bi || bt >= ir->next_instruction_index) + continue; + for (int sp = 0; sp < nspans; sp++) + if (bt > spans[2 * sp] && bt <= spans[2 * sp + 1]) + { + ir->compact_instructions[bt].align_target = 1; + break; + } + } + tcc_free(spans); + } + } + /* TEMPORARY INSTRUMENT: why does the dry walk run? cg:dry is 68.5 s of the corpus (48% of codegen, the single largest item in the compiler) and the answer decides between a conservative analytic scratch estimate and a @@ -2588,6 +3274,7 @@ void tcc_ir_codegen_generate(TCCIRState *ir) case TCCIR_OP_IMOD: case TCCIR_OP_UMULL: case TCCIR_OP_SMULL: + case TCCIR_OP_UMAAL: case TCCIR_OP_BLOCK_COPY: case TCCIR_OP_VLA_ALLOC: case TCCIR_OP_VLA_SP_SAVE: @@ -2622,7 +3309,7 @@ void tcc_ir_codegen_generate(TCCIRState *ir) } /* Calls with stack-passed args: the call-site setup may need scratch * to materialise the argument values into SP-relative slots. */ - if (!might_need_scratch && ir->call_outgoing_size > 0) + if (!might_need_scratch && (ir->call_outgoing_size > 0 || ir->call_dyn_extra)) might_need_scratch = 1, tcc_dryreason[4]++; /* Incoming stack params: reads from [sp + offset_to_args] may collide * with live argument registers, forcing get_scratch_reg_with_save to @@ -2654,6 +3341,16 @@ void tcc_ir_codegen_generate(TCCIRState *ir) * callee-saved registers from dirty_registers before prologue. */ ir_codegen_pre_patch_funcparam_allocations(ir); ir_codegen_recompute_dirty_from_allocations(ir); + /* Every vreg is where it will stay: merge the tails that differ only in + * which vreg brings a value in through the same location + * (cross_jump.c). Before the rehearsal, so its address map is of the + * merged code; the removed instructions' cached operands go too. */ + if (tcc_state->optimize > 0 && !tcc_ir_opt_pass_disabled("cross_jump_alloc")) + tcc_ir_cross_jump_alloc(ir); /* no operand cache built yet on this path */ + /* The real pass alone lays this body out: nothing measured its pool, + * so ask its IR. */ + if (cg_may_read_pool(ir)) + cg_word_align_start(); if (!ir->naked) tcc_gen_machine_prolog(ir->leaffunc, ir->ls.dirty_registers, stack_size, extra_prologue_regs); if (!ir->naked) @@ -2675,12 +3372,11 @@ void tcc_ir_codegen_generate(TCCIRState *ir) * ============================================================================ */ /* Option B: allocate per-instruction MopArgs cache for the dry-run. * Not used when the dry-run is skipped (can_skip_dry_run). */ - /* Zero-initialised: an unwritten slot reads back as all-MACH_OP_NONE, which - * ir_decode_cached() treats as "not populated in the dry-run" and re-decodes - * (see the cache-hit path there). */ - MopArgs *mop_cache = (!can_skip_dry_run && ir->next_instruction_index > 0) - ? tcc_mallocz(ir->next_instruction_index * sizeof(MopArgs)) - : NULL; + /* An instruction the dry-run never recorded is re-decoded by + * ir_decode_cached() (see the cache-hit path there). */ + MopCache *mop_cache = (!can_skip_dry_run && ir->next_instruction_index > 0) + ? mop_cache_new(ir->next_instruction_index) + : NULL; ir->codegen_mop_cache = mop_cache; int use_mop_cache = 0; @@ -2766,6 +3462,11 @@ void tcc_ir_codegen_generate(TCCIRState *ir) this decision was priced with). */ int cg_skip_rehearsal = tcc_state->optimize == 0 || cg_no_rehearsal(); TCCPassTimer cg_pt = {0}; + int temp_uses_n = 0; + int *temp_uses = tcc_state->optimize > 0 ? ir_codegen_temp_uses(ir, &temp_uses_n) : NULL; + int p0_pool_entries = 0; /* literal-pool entries the discovery pass allocated */ + int outline_capturing = 0; + int outline_planned = 0; /* windows the real pass calls instead */ for (int pass = pass_start; pass < 3; pass++) { if (pass == 1 && cg_skip_rehearsal) @@ -2816,6 +3517,13 @@ void tcc_ir_codegen_generate(TCCIRState *ir) tcc_gen_machine_branch_opt_init(); tcc_gen_machine_dry_run_start(); tcc_gen_machine_dry_run_set_rehearsal(is_rehearsal); + /* -Os outliner: record the rehearsal's code of a body whose prologue + * saved LR (a BL from it clobbers LR). */ + if (is_rehearsal && !ir->naked && tcc_gen_machine_outline_enabled() && tcc_gen_machine_prolog_saved_lr()) + { + outline_capturing = 1; + tcc_gen_machine_outline_capture_begin(); + } tcc_gen_machine_reset_scratch_state(); tcc_ir_spill_cache_clear(&ir->spill_cache); } @@ -2836,6 +3544,11 @@ void tcc_ir_codegen_generate(TCCIRState *ir) const int strldr_reset_every_op = ir->func_has_volatile_access || tcc_ir_opt_pass_disabled("codegen:strldr_xir"); tcc_gen_machine_strldr_cache_set_enabled(!strldr_reset_every_op); + tcc_gen_machine_str_elide_set_enabled(!tcc_ir_opt_pass_disabled("codegen:str_elide")); + tcc_gen_machine_slot_self_copy_set_enabled(!ir->func_has_volatile_access && + !tcc_ir_opt_pass_disabled("codegen:slot_self_copy")); + tcc_gen_machine_frame_word_pair_set_enabled(!ir->func_has_volatile_access && + !tcc_ir_opt_pass_disabled("codegen:frame_word_pair")); /* ---- Instruction loop ---- */ for (int i = 0; i < ir->next_instruction_index; i++) @@ -2860,6 +3573,18 @@ void tcc_ir_codegen_generate(TCCIRState *ir) /* Debug tracking: update current op for ot_check failure reporting */ g_debug_current_op = (int)cq->op; + /* Word-align a bottom-tested loop head before its address is recorded, so + * the back-edge branch lands on a 4-aligned instruction (see + * IRQuadCompact.align_target). The pad sits BEFORE the target, so it is + * executed only on fall-through entry, never on the back edge. */ + if (cq->align_target && !tcc_state->optimize_size) + tcc_gen_machine_align_branch_target(align_bytes); + + /* -Os outliner: a window the real pass calls instead ends before this + * op's address is taken... */ + if (outline_planned && !is_dry_run) + tcc_gen_machine_outline_op_end(i); + ir_to_code_mapping[i] = ind; /* Snapshot pool pressure for can_narrow_forward_branch. */ @@ -2927,6 +3652,9 @@ void tcc_ir_codegen_generate(TCCIRState *ir) orig_ir_to_code_mapping[cq->orig_index] = ind; tcc_debug_line_num(tcc_state, cq->line_num); } + /* ... and one starts after it: a jump to this op lands on the BL. */ + if (outline_planned && !is_dry_run) + tcc_gen_machine_outline_op_start(i); /* Get operand copies from iroperand_pool (compact representation) */ IROperand src1_ir = tcc_ir_op_get_src1(ir, cq); @@ -3044,8 +3772,10 @@ void tcc_ir_codegen_generate(TCCIRState *ir) } case TCCIR_OP_TEST_ZERO: { - /* CBZ/CBNZ peephole for TEST_ZERO: same as CMP #0 pattern */ - if (cbz_enabled) + /* CBZ/CBNZ peephole for TEST_ZERO: same as CMP #0 pattern. Not + * inside a window the -Os outliner calls: the rehearsal emitted the + * compare into the shared body. */ + if (cbz_enabled && !tcc_gen_machine_outline_suppressing()) { MopArgs cbz_a = DECODE(.dest = 1, .src1 = 1, .src2 = 1); if (!cbz_a.src1.is_64bit && cbz_a.src1.kind == MACH_OP_REG && !cbz_a.src1.needs_deref && @@ -3062,7 +3792,8 @@ void tcc_ir_codegen_generate(TCCIRState *ir) { IROperand jdest = tcc_ir_op_get_dest(ir, &ir->compact_instructions[next_j]); int target_ir = irop_is_none(jdest) ? -1 : (int)jdest.u.imm32; - if (target_ir > i && target_ir < (int)ir->ir_to_code_mapping_size) + if (target_ir > i && target_ir < (int)ir->ir_to_code_mapping_size && + !ir_codegen_flags_read_at(ir, next_j + 1) && !ir_codegen_flags_read_at(ir, target_ir)) { if (tcc_gen_machine_cbz_forward_ok(target_ir, i)) { @@ -3107,6 +3838,12 @@ void tcc_ir_codegen_generate(TCCIRState *ir) } break; } + case TCCIR_OP_UMAAL: + { + MopArgs a = DECODE(.dest = 1, .src1 = 1, .src2 = 1, .accum = 1); + SCRATCH_WRAP(tcc_gen_machine_umaal_mop(a.src1, a.src2, a.accum, a.dest)); + break; + } case TCCIR_OP_UMULL: case TCCIR_OP_SMULL: { @@ -3275,8 +4012,9 @@ void tcc_ir_codegen_generate(TCCIRState *ir) break; } /* CBZ/CBNZ peephole: CMP rN, #0 followed by JUMPIF EQ/NE. - * Only in real pass with valid dry-run distance estimates. */ - if (cbz_enabled) + * Only in real pass with valid dry-run distance estimates, and not + * inside a window the -Os outliner calls (see TEST_ZERO). */ + if (cbz_enabled && !tcc_gen_machine_outline_suppressing()) { MopArgs cbz_a = DECODE(.dest = 1, .src1 = 1, .src2 = 1); if (cbz_a.src2.kind == MACH_OP_IMM && cbz_a.src2.u.imm.val == 0 && !cbz_a.src1.is_64bit && @@ -3294,7 +4032,8 @@ void tcc_ir_codegen_generate(TCCIRState *ir) { IROperand jdest = tcc_ir_op_get_dest(ir, &ir->compact_instructions[next_j]); int target_ir = irop_is_none(jdest) ? -1 : (int)jdest.u.imm32; - if (target_ir > i && target_ir < (int)ir->ir_to_code_mapping_size) + if (target_ir > i && target_ir < (int)ir->ir_to_code_mapping_size && + !ir_codegen_flags_read_at(ir, next_j + 1) && !ir_codegen_flags_read_at(ir, target_ir)) { if (tcc_gen_machine_cbz_forward_ok(target_ir, i)) { @@ -3470,6 +4209,26 @@ void tcc_ir_codegen_generate(TCCIRState *ir) tcc_error("compiler_error: LOAD operand produced MACH_OP_NONE (i=%d dest_kind=%d src_kind=%d)", i, a.dest.kind, a.src1.kind); + /* A small aggregate copied through a pointer arrives batched -- every + * word loaded, then every word stored -- so it fuses into one LDM and + * one STM whatever either side is addressed through. */ + { + BatchedWordCopy bc; + if (ir_codegen_scan_batched_word_copy(ir, branch_target_reset, i, temp_uses, temp_uses_n, is_dry_run, &bc)) + { + int done = 0; + SCRATCH_WRAP(done = tcc_gen_machine_reg_block_copy(bc.src_base, bc.src_off, bc.dst_base, bc.dst_off, + bc.regs, bc.nwords, bc.last_i)); + if (done) + { + tcc_ir_spill_cache_clear(&ir->spill_cache); + tcc_gen_machine_imm_cache_reset(); + i = bc.last_i; + break; + } + } + } + /* Block copy peephole: consecutive LOAD-from-spill + STORE-to-spill pairs * with sequential offsets → single LDM/STM block copy. * Safety: all loads must use the same destination register, proving each @@ -3478,9 +4237,10 @@ void tcc_ir_codegen_generate(TCCIRState *ir) if (a.dest.kind == MACH_OP_REG && !a.dest.needs_deref && a.src1.kind == MACH_OP_SPILL && !a.src1.needs_deref && !a.src1.is_64bit && (a.src1.btype == IROP_BTYPE_INT32 || a.src1.btype == IROP_BTYPE_FLOAT32) && - (a.src1.u.spill.offset & 3) == 0) + (a.src1.u.spill.offset & 3) == 0 && !tcc_ir_access_is_volatile(ir, src1_ir)) { int first_load_reg = a.dest.u.reg.r0; + int32_t first_load_vr = irop_get_vreg(dest_ir); int store_i = ir_codegen_next_nonnop_no_label(ir, branch_target_reset, i); if (store_i >= 0 && ir->compact_instructions[store_i].op == TCCIR_OP_STORE && !ir->compact_instructions[store_i].is_jump_target) @@ -3493,15 +4253,23 @@ void tcc_ir_codegen_generate(TCCIRState *ir) &s_src1, &s_src2, &s_dest, (MopSpec){.dest = 1, .src1 = 2}); + /* Each loaded value must die at its store: proven when it is a + * TEMP the store is the one use of, or when every load of the run + * reuses one register (each overwrites the last). */ + int all_one_reg = 1; if (sa.dest.kind == MACH_OP_SPILL && !sa.dest.needs_deref && !sa.src1.is_64bit && sa.src1.kind == MACH_OP_REG && sa.src1.u.reg.r0 == first_load_reg && + irop_get_vreg(s_src1) == first_load_vr && (sa.dest.btype == IROP_BTYPE_INT32 || sa.dest.btype == IROP_BTYPE_FLOAT32) && - (sa.dest.u.spill.offset & 3) == 0) + (sa.dest.u.spill.offset & 3) == 0 && !tcc_ir_access_is_volatile(ir, s_dest)) { + int dies = ir_codegen_temp_dies_at_store(temp_uses, temp_uses_n, first_load_vr); int32_t src_base = a.src1.u.spill.offset; int32_t dst_base = sa.dest.u.spill.offset; int count = 1; int last_i = store_i; + int prev_last_i = -1; + int last_dies = dies; while (count < 32) { @@ -3520,10 +4288,16 @@ void tcc_ir_codegen_generate(TCCIRState *ir) if (la.src1.kind != MACH_OP_SPILL || la.src1.needs_deref || la.src1.is_64bit || la.src1.u.spill.offset != src_base + count * 4 || - (la.src1.btype != IROP_BTYPE_INT32 && la.src1.btype != IROP_BTYPE_FLOAT32)) + (la.src1.btype != IROP_BTYPE_INT32 && la.src1.btype != IROP_BTYPE_FLOAT32) || + tcc_ir_access_is_volatile(ir, l_src1)) break; - if (la.dest.kind != MACH_OP_REG || la.dest.u.reg.r0 != first_load_reg) + int32_t load_vr = irop_get_vreg(l_dest); + if (la.dest.kind != MACH_OP_REG || la.dest.needs_deref) + break; + int same_reg = la.dest.u.reg.r0 == first_load_reg; + int this_dies = ir_codegen_temp_dies_at_store(temp_uses, temp_uses_n, load_vr); + if (!same_reg && !this_dies) break; int next_store_i = ir_codegen_next_nonnop_no_label(ir, branch_target_reset, next_load_i); @@ -3541,15 +4315,50 @@ void tcc_ir_codegen_generate(TCCIRState *ir) if (sa2.dest.kind != MACH_OP_SPILL || sa2.dest.needs_deref || sa2.src1.is_64bit || sa2.dest.u.spill.offset != dst_base + count * 4 || - sa2.src1.kind != MACH_OP_REG || sa2.src1.u.reg.r0 != first_load_reg || - (sa2.dest.btype != IROP_BTYPE_INT32 && sa2.dest.btype != IROP_BTYPE_FLOAT32)) + sa2.src1.kind != MACH_OP_REG || sa2.src1.u.reg.r0 != la.dest.u.reg.r0 || + irop_get_vreg(s2_src1) != load_vr || + (sa2.dest.btype != IROP_BTYPE_INT32 && sa2.dest.btype != IROP_BTYPE_FLOAT32) || + tcc_ir_access_is_volatile(ir, s2_dest)) break; + all_one_reg &= same_reg; + dies &= this_dies; count++; + prev_last_i = last_i; last_i = next_store_i; + last_dies = this_dies; + } + + /* One register for every load proves each word dead only where + * the next load overwrites it; nothing overwrites the last, and + * the copy leaves that register holding something else. Zig's + * analyzeInstSwitchBr read the last word of such a run again + * (load_cse had forwarded it) and got memmove's return value. + * Leave that pair to be emitted on its own. */ + if (!dies && all_one_reg && !last_dies && count > 1) + { + count--; + last_i = prev_last_i; } - if (count >= 8) + /* Loading a whole chunk before storing it matches the pairwise + * order only when the ranges do not overlap. */ + const int apart = dst_base + count * 4 <= src_base || src_base + count * 4 <= dst_base; + if (apart && (dies || all_one_reg) && count >= 3) + { + int done = 0; + /* The first load's register holds only that temporary. */ + uint32_t also_free = dies && first_load_reg >= 0 && first_load_reg < 16 ? 1u << first_load_reg : 0; + SCRATCH_WRAP(done = tcc_gen_machine_spill_block_copy_free(src_base, dst_base, count, also_free)); + if (done) + { + tcc_ir_spill_cache_clear(&ir->spill_cache); + tcc_gen_machine_imm_cache_reset(); + i = last_i; + break; + } + } + if (apart && all_one_reg && count >= 8) { SCRATCH_WRAP(tcc_gen_machine_spill_block_copy(src_base, dst_base, count)); i = last_i; @@ -3821,6 +4630,26 @@ void tcc_ir_codegen_generate(TCCIRState *ir) { MopArgs a = DECODE(.dest = 2, .src1 = 1, .src2 = 1, .scale = 1); + /* A small aggregate copied through a pointer arrives batched -- every + * word loaded, then every word stored -- so it fuses into one LDM and + * one STM whatever either side is addressed through. */ + { + BatchedWordCopy bc; + if (ir_codegen_scan_batched_word_copy(ir, branch_target_reset, i, temp_uses, temp_uses_n, is_dry_run, &bc)) + { + int done = 0; + SCRATCH_WRAP(done = tcc_gen_machine_reg_block_copy(bc.src_base, bc.src_off, bc.dst_base, bc.dst_off, + bc.regs, bc.nwords, bc.last_i)); + if (done) + { + tcc_ir_spill_cache_clear(&ir->spill_cache); + tcc_gen_machine_imm_cache_reset(); + i = bc.last_i; + break; + } + } + } + /* LDRD pairing: two adjacent 32-bit LOAD_INDEXED ops with the same * base register, scale=0, and constant offsets differing by 4 can * fold to a single LDRD. Mirrors the SPILL-slot LDRD peephole @@ -4206,8 +5035,12 @@ void tcc_ir_codegen_generate(TCCIRState *ir) } /* fall through to RETURNVOID */ case TCCIR_OP_RETURNVOID: - /* Real-run: emit jump to epilogue (backpatched later). - * Dry-run: no-op (we don't track return_jump_addrs). + /* Emit the jump to the epilogue (backpatched later) in every pass. + * The dry passes only size it -- tcc_gen_machine_jump_mop emits the + * same 4-byte placeholder as for any branch -- but they must: the + * rehearsal's layout bounds the real offsets that CBZ fusion and + * branch narrowing commit to, and a return missing from it made a + * range 2-4 bytes longer than modelled (a fused CBZ landed at 128). * Skip the jump if all remaining instructions are NOPs — * the epilogue immediately follows, so the branch is a no-op. */ { @@ -4220,12 +5053,13 @@ void tcc_ir_codegen_generate(TCCIRState *ir) break; } } - if (!is_dry_run && has_trailing_code) + if (has_trailing_code) { /* Target -1 means "the epilogue", which is not an IR index. The * emitter sizes it from the rehearsal's end-of-body address. */ int ret_branch_size = tcc_gen_machine_jump_mop(cq->op, -1, i); - return_jump_addrs[num_return_jumps++] = ind - ret_branch_size; + if (!is_dry_run) + return_jump_addrs[num_return_jumps++] = ind - ret_branch_size; } } break; @@ -4502,7 +5336,8 @@ void tcc_ir_codegen_generate(TCCIRState *ir) break; } case TCCIR_OP_TRAP: - tcc_gen_machine_trap_mop(); + if (!cq->unreachable) /* __builtin_unreachable: no code */ + tcc_gen_machine_trap_mop(); break; case TCCIR_OP_SETJMP: { @@ -4534,6 +5369,12 @@ void tcc_ir_codegen_generate(TCCIRState *ir) SCRATCH_WRAP(tcc_gen_machine_builtin_apply_args_mop(a.dest)); break; } + case TCCIR_OP_RETURN_ADDRESS: + { + MopArgs a = DECODE(.dest = 1); + SCRATCH_WRAP(tcc_gen_machine_return_address_mop(a.dest)); + break; + } case TCCIR_OP_BUILTIN_APPLY: { MopArgs a = DECODE(.dest = 1, .src1 = 1, .src2 = 1); @@ -4563,6 +5404,14 @@ void tcc_ir_codegen_generate(TCCIRState *ir) tcc_error("inline asm not supported"); #endif } + /* The asm body writes registers without passing through ot(), so no + * register-content cache saw those writes: an imm_cache entry naming + * an operand register (e.g. "r0 = sp" from materializing &local) would + * be reused after the asm overwrote it. Drop them all, in the dry run + * too, so both passes leave the statement with the same cache state. */ + tcc_gen_machine_imm_cache_reset(); + tcc_gen_machine_mov_equiv_reset(); + tcc_gen_machine_strldr_cache_reset(); break; case TCCIR_OP_BLOCK_COPY: { @@ -4622,12 +5471,23 @@ void tcc_ir_codegen_generate(TCCIRState *ir) * This restores any pushed scratch registers and resets the global exclude mask. */ tcc_gen_machine_end_instruction(); } + if (outline_planned && !is_dry_run) + { + tcc_gen_machine_outline_function_end(); + outline_planned = 0; + } /* ---- Pass-specific finalisation ---- */ if (is_rehearsal) { /* The rehearsal exists only to produce an address map the real pass will * match. Capture it, then rewind everything the real pass redoes. */ + if (outline_capturing) + { + tcc_gen_machine_outline_capture_end(); + outline_planned = tcc_gen_machine_outline_analyze(ir, ir_to_code_mapping, ind); + outline_capturing = 0; + } tcc_gen_machine_dry_run_end(); tcc_gen_machine_dry_run_set_rehearsal(0); @@ -4658,6 +5518,7 @@ void tcc_ir_codegen_generate(TCCIRState *ir) else if (is_dry_run) { /* End dry-run and analyze results */ + p0_pool_entries = tcc_gen_machine_pool_entries_total(); tcc_gen_machine_dry_run_end(); /* Analyze branch offsets and select optimal encodings */ @@ -4689,6 +5550,11 @@ void tcc_ir_codegen_generate(TCCIRState *ir) /* Restore state for real code generation */ ind = saved_ind; loc = saved_loc; + + /* After the discovery pass, before the rehearsal: both remaining + * passes lay out from the aligned start. */ + if (p0_pool_entries > 0) + cg_word_align_start(); ir->call_outgoing_base = saved_call_outgoing_base; ir->call_nested_save_base = saved_call_nested_save_base; ir->codegen_instruction_idx = saved_codegen_idx; @@ -4784,7 +5650,9 @@ void tcc_ir_codegen_generate(TCCIRState *ir) ir->next_instruction_index, &demote_ref_cache); if (dv >= 0) { - demoted_vregs[demoted_count++] = dv; + IRLiveInterval *dli = tcc_ir_get_live_interval(ir, dv); + if (!dli || !demote_to_param_home(dli, dv)) + demoted_vregs[demoted_count++] = dv; any_fixup = 1; alt_fixed = 1; } @@ -4804,7 +5672,7 @@ void tcc_ir_codegen_generate(TCCIRState *ir) { tcc_ls_reset_scratch_cache(&ir->ls); /* Interval table was mutated: cached MopArgs are stale, discard. */ - tcc_free(mop_cache); + tcc_ir_mop_cache_free(mop_cache); mop_cache = NULL; ir->codegen_mop_cache = NULL; } @@ -4939,6 +5807,18 @@ void tcc_ir_codegen_generate(TCCIRState *ir) (void)original_leaffunc; ir_codegen_pre_patch_funcparam_allocations(ir); ir_codegen_recompute_dirty_from_allocations(ir); + /* Every vreg is where it will stay: merge the tails that differ only in + * which vreg brings a value in through the same location + * (cross_jump.c). Before the rehearsal, so its address map is of the + * merged code; the removed instructions' cached operands go too. */ + if (tcc_state->optimize > 0 && !tcc_ir_opt_pass_disabled("cross_jump_alloc") && + tcc_ir_cross_jump_alloc(ir) > 0) + { + tcc_ir_mop_cache_free(mop_cache); + mop_cache = NULL; + ir->codegen_mop_cache = NULL; + use_mop_cache = 0; + } if (!ir->naked) tcc_gen_machine_prolog(ir->leaffunc, ir->ls.dirty_registers, stack_size, extra_prologue_regs); if (!ir->naked) @@ -4949,7 +5829,7 @@ void tcc_ir_codegen_generate(TCCIRState *ir) if (cg_pt.active) tcc_pass_timing_end(&cg_pt, -1); - tcc_free(mop_cache); + tcc_ir_mop_cache_free(mop_cache); ir->codegen_mop_cache = NULL; if (cbz_dry_mapping) tcc_free(cbz_dry_mapping); @@ -4980,7 +5860,10 @@ void tcc_ir_codegen_generate(TCCIRState *ir) if (!ir->naked) { - if (!ir->noreturn && !ir->tail_call_only) + /* A tail-call-only function ends in the branch to its callee, but an + * earlier `return` still jumps to the epilogue: without one it ran into + * the next function (`if (a == 63) return b; return f(a, b);`). */ + if (!ir->noreturn && (!ir->tail_call_only || num_return_jumps > 0)) tcc_gen_machine_epilog(ir->leaffunc); else tcc_gen_machine_finish_noreturn(); @@ -5000,6 +5883,7 @@ void tcc_ir_codegen_generate(TCCIRState *ir) ir->codegen_dry_insn_saves = NULL; tcc_free(dry_insn_scratch); ir->codegen_dry_insn_scratch = NULL; + tcc_free(temp_uses); } /* ============================================================================ diff --git a/source/ir/core.h b/source/ir/core.h index 930669b92..111eab4f4 100644 --- a/source/ir/core.h +++ b/source/ir/core.h @@ -26,6 +26,8 @@ struct TCCIRState *tcc_ir_alloc(void); /* Free IR block and all associated memory */ void tcc_ir_free(struct TCCIRState *ir); +/* Free codegen's dry-run operand cache (TCCIRState.codegen_mop_cache). */ +void tcc_ir_mop_cache_free(void *cache); /* Reset IR block for reuse (keeps allocations) */ void tcc_ir_reset(struct TCCIRState *ir); diff --git a/source/ir/cross_jump.c b/source/ir/cross_jump.c new file mode 100644 index 000000000..f0c571a66 --- /dev/null +++ b/source/ir/cross_jump.c @@ -0,0 +1,1571 @@ +/* + * TinyCC - Tiny C Compiler + * + * Cross-jumping: merge identical block tails that jump to the same place. + * + * The Zig C backend spells out an error return at every `try`: build the error + * union, store it through the result pointer, jump to the epilogue. After + * register allocation many of those tails are the same instructions on the + * same registers and slots, differing only in which block they end. A tail + * that repeats another tail ending in a jump to the same target is replaced by + * a jump into that other tail. + * + * Runs on the final IR, between register allocation and code generation. The + * code generator still moves some values while it emits (a value whose + * register an instruction needs as scratch is moved or spilled), so equal + * allocations do not make equal code. Two tails match when their ops, + * immediates, frame slots, symbols and annotations are equal and every vreg + * operand is either the same vreg in both -- one value, wherever it ends up -- + * or defined inside its tail and used nowhere else, consistently paired with + * its counterpart: only the kept copy runs, and it computes its own values. + * Registers live inside the shared tail on the other path are ones the tail + * reads or ones live at the common target, so the kept code, which never + * takes a live register as scratch, leaves them alone. The kept tail's first + * instruction becomes a jump target, which resets the code generator's + * register, flag and memory caches there. + * + * Not merged: calls and their arguments (per-call bookkeeping), control flow, + * inline asm, VLA and setjmp ops, a tail that would begin by consuming flags + * set before it, and a tail with a jump target inside the part removed. + */ + +#include "ir.h" + +#define CJ_MAX_TAIL 64 + +static int cj_mergeable_op(int op) +{ + switch (op) + { + case TCCIR_OP_NOP: + case TCCIR_OP_JUMP: + case TCCIR_OP_JUMPIF: + case TCCIR_OP_IJUMP: + case TCCIR_OP_SWITCH_TABLE: + case TCCIR_OP_RETURNVALUE: + case TCCIR_OP_RETURNVOID: + case TCCIR_OP_FUNCPARAMVAL: + case TCCIR_OP_FUNCPARAMVOID: + case TCCIR_OP_FUNCCALLVAL: + case TCCIR_OP_FUNCCALLVOID: + case TCCIR_OP_SET_CHAIN: + case TCCIR_OP_SETJMP: + case TCCIR_OP_NL_SETJMP: + case TCCIR_OP_INLINE_ASM: + case TCCIR_OP_ASM_INPUT: + case TCCIR_OP_ASM_OUTPUT: + case TCCIR_OP_VLA_ALLOC: + case TCCIR_OP_VLA_SP_SAVE: + case TCCIR_OP_VLA_SP_RESTORE: + return 0; + default: + return 1; + } +} + +static int cj_reads_flags(int op) +{ + return op == TCCIR_OP_SETIF || op == TCCIR_OP_SELECT || op == TCCIR_OP_JUMPIF; +} + +static int cj_sets_flags(int op) +{ + return op == TCCIR_OP_CMP || op == TCCIR_OP_TEST_ZERO || op == TCCIR_OP_FCMP; +} + +static int cj_has_slot3(int op) +{ + return tcc_ir_op_is_mac(op) || op == TCCIR_OP_LOAD_INDEXED || op == TCCIR_OP_STORE_INDEXED || op == TCCIR_OP_SELECT; +} + +/* Struct operands name their type by an index into a per-operand pool, so the + * same type has many indices: compare the types. */ +static int cj_struct_same(IROperand a, IROperand b) +{ + if (a.u.s.aux_data != b.u.s.aux_data) + return 0; + CType *ta = irop_get_ctype(a), *tb = irop_get_ctype(b); + return ta == tb || (ta && tb && ta->t == tb->t && ta->ref == tb->ref); +} + +static int cj_operand_same(TCCIRState *ir, IROperand a, IROperand b) +{ + if (irop_is_none(a) || irop_is_none(b)) + return irop_is_none(a) && irop_is_none(b); + if (a.tag != b.tag || a.btype != b.btype || a.is_lval != b.is_lval || a.is_llocal != b.is_llocal || + a.is_local != b.is_local || a.is_const != b.is_const || a.is_complex != b.is_complex || + a.is_unsigned != b.is_unsigned || a.is_static != b.is_static || a.is_sym != b.is_sym || + a.is_param != b.is_param || a.aux != b.aux) + return 0; + int32_t va = irop_get_vreg(a), vb = irop_get_vreg(b); + if ((va < 0) != (vb < 0) || (va < 0 && va != vb)) + return 0; + if (va >= 0 && TCCIR_DECODE_VREG_TYPE(va) != TCCIR_DECODE_VREG_TYPE(vb)) + return 0; /* which vreg is checked by cj_verify */ + switch (a.tag) + { + case IROP_TAG_VREG: + return a.btype != IROP_BTYPE_STRUCT || cj_struct_same(a, b); + case IROP_TAG_IMM32: + case IROP_TAG_STACKOFF: + case IROP_TAG_F32: + if (a.btype == IROP_BTYPE_STRUCT) + return cj_struct_same(a, b); + return a.u.imm32 == b.u.imm32; + case IROP_TAG_I64: + case IROP_TAG_F64: + return irop_get_imm64_ex(ir, a) == irop_get_imm64_ex(ir, b); + case IROP_TAG_SYMREF: + { + IRPoolSymref *sa = irop_get_symref_ex(ir, a), *sb = irop_get_symref_ex(ir, b); + return sa && sb && sa->sym == sb->sym && sa->addend == sb->addend; + } + default: + return 0; + } +} + +static int cj_insn_same(TCCIRState *ir, IRQuadCompact *a, IRQuadCompact *b) +{ + if (a->op != b->op || !cj_mergeable_op(a->op)) + return 0; + if ((irop_config[a->op].has_dest && !cj_operand_same(ir, tcc_ir_op_get_dest(ir, a), tcc_ir_op_get_dest(ir, b))) || + (irop_config[a->op].has_src1 && !cj_operand_same(ir, tcc_ir_op_get_src1(ir, a), tcc_ir_op_get_src1(ir, b))) || + (irop_config[a->op].has_src2 && !cj_operand_same(ir, tcc_ir_op_get_src2(ir, a), tcc_ir_op_get_src2(ir, b)))) + return 0; + if (cj_has_slot3(a->op) && + !cj_operand_same(ir, ir->iroperand_pool[a->operand_base + 3], ir->iroperand_pool[b->operand_base + 3])) + return 0; + return tcc_ir_barrel_shift_at(ir, a) == tcc_ir_barrel_shift_at(ir, b) && + tcc_ir_shift64_dead_half_at(ir, a) == tcc_ir_shift64_dead_half_at(ir, b) && + tcc_ir_zero_half64_at(ir, a) == tcc_ir_zero_half64_at(ir, b) && + tcc_ir_bfi_params_at(ir, a) == tcc_ir_bfi_params_at(ir, b); +} + +/* A dense index for a vreg (TEMP, VAR, PARAM), or -1. */ +static int cj_vkey(TCCIRState *ir, int32_t vr) +{ + if (vr < 0) + return -1; + int pos = TCCIR_DECODE_VREG_POSITION(vr), nt = ir->next_temporary_variable, nv = ir->next_local_variable; + switch (TCCIR_DECODE_VREG_TYPE(vr)) + { + case TCCIR_VREG_TYPE_TEMP: + return pos < nt ? pos : -1; + case TCCIR_VREG_TYPE_VAR: + return pos < nv ? nt + pos : -1; + case TCCIR_VREG_TYPE_PARAM: + return pos < ir->next_parameter ? nt + nv + pos : -1; + default: + return -1; + } +} + +static IROperand cj_slot(TCCIRState *ir, IRQuadCompact *q, int s) +{ + if (s == 0 && irop_config[q->op].has_dest) + return tcc_ir_op_get_dest(ir, q); + if (s == 1 && irop_config[q->op].has_src1) + return tcc_ir_op_get_src1(ir, q); + if (s == 2 && irop_config[q->op].has_src2) + return tcc_ir_op_get_src2(ir, q); + if (s == 3 && cj_has_slot3(q->op)) + return ir->iroperand_pool[q->operand_base + 3]; + return IROP_NONE; +} + +/* Does slot s of q define the vreg it names (rather than read it)? A store's + * dest through a TEMP is the address it writes through, a use. */ +static int cj_defines(IRQuadCompact *q, int s, IROperand o) +{ + if (s != 0 || q->op == TCCIR_OP_STORE_INDEXED || q->op == TCCIR_OP_STORE_POSTINC) + return 0; + if (TCCIR_DECODE_VREG_TYPE(irop_get_vreg(o)) == TCCIR_VREG_TYPE_TEMP) + return !o.is_lval; + return !o.is_llocal; /* a VAR is written through its slot operand */ +} + +/* How many of the tail instructions, counted from the end, can be merged: + * every vreg operand is the same vreg in both copies, or a vreg first defined + * in its copy, paired one to one with its counterpart and occurring nowhere + * outside the tail. Returns the verified length (0 if none). */ +static int cj_verify(TCCIRState *ir, const int *ta, const int *tb, int k, const int *occ, int *pa, int *pb, int *ca, + int *cb) +{ + while (k >= 2) + { + int np = 0, fail = -1; + for (int m = k - 1; m >= 0 && fail < 0; m--) + { + IRQuadCompact *qa = &ir->compact_instructions[ta[m]], *qb = &ir->compact_instructions[tb[m]]; + for (int s = 0; s < 4 && fail < 0; s++) + { + IROperand oa = cj_slot(ir, qa, s), ob = cj_slot(ir, qb, s); + int va = cj_vkey(ir, irop_get_vreg(oa)), vb = cj_vkey(ir, irop_get_vreg(ob)); + if (va < 0 && vb < 0) + continue; + int p = -1; + for (int x = 0; x < np; x++) + if (pa[x] == va || pb[x] == vb) + p = x; + if (p >= 0) + { + if (pa[p] != va || pb[p] != vb) + fail = m; + else + ca[p]++, cb[p]++; + continue; + } + if (va == vb) + continue; /* the same value in both */ + if (va < 0 || vb < 0 || !cj_defines(qa, s, oa) || !cj_defines(qb, s, ob)) + { + fail = m; + continue; + } + pa[np] = va, pb[np] = vb, ca[np] = 1, cb[np] = 1; + np++; + } + } + if (fail < 0) + { + for (int x = 0; x < np; x++) + if (ca[x] != occ[pa[x]] || cb[x] != occ[pb[x]]) + return 0; /* used outside its tail */ + return k; + } + k = fail; /* keep only what follows the failing instruction */ + } + return 0; +} + +/* The previous non-NOP instruction before `i`, or -1. */ +static int cj_prev(TCCIRState *ir, int i) +{ + for (i--; i >= 0; i--) + if (ir->compact_instructions[i].op != TCCIR_OP_NOP) + return i; + return -1; +} + +int tcc_ir_cross_jump(TCCIRState *ir) +{ + const int n = ir->next_instruction_index; + if (n < 4 || ir->func_has_label_addr) + return 0; + int *jumps = tcc_malloc(sizeof(int) * n); + int nj = 0; + for (int i = 0; i < n; i++) + if (ir->compact_instructions[i].op == TCCIR_OP_JUMP) + jumps[nj++] = i; + + int *tail_a = tcc_malloc(sizeof(int) * n), *tail_b = tcc_malloc(sizeof(int) * n); + /* Every vreg's operand occurrences, to tell a tail-local value. */ + const int nkeys = ir->next_temporary_variable + ir->next_local_variable + ir->next_parameter + 1; + int *occ = tcc_mallocz(sizeof(int) * nkeys); + for (int i = 0; i < n; i++) + { + IRQuadCompact *q = &ir->compact_instructions[i]; + if (q->op == TCCIR_OP_NOP) + continue; + for (int s = 0; s < 4; s++) + { + int key = cj_vkey(ir, irop_get_vreg(cj_slot(ir, q, s))); + if (key >= 0) + occ[key]++; + } + } + /* Scratch for cj_verify: at most 4 pairs per instruction. */ + int *pa = tcc_malloc(sizeof(int) * 4 * (CJ_MAX_TAIL + 1)), *pb = tcc_malloc(sizeof(int) * 4 * (CJ_MAX_TAIL + 1)); + int *ca = tcc_malloc(sizeof(int) * 4 * (CJ_MAX_TAIL + 1)), *cb = tcc_malloc(sizeof(int) * 4 * (CJ_MAX_TAIL + 1)); + int merged = 0; + for (int x = 0; x < nj; x++) + { + int ja = jumps[x]; + IRQuadCompact *qa = &ir->compact_instructions[ja]; + if (qa->op != TCCIR_OP_JUMP) + continue; + int target = irop_get_imm32(tcc_ir_op_get_dest(ir, qa)); + for (int y = x + 1; y < nj; y++) + { + int jb = jumps[y]; + IRQuadCompact *qb = &ir->compact_instructions[jb]; + if (qb->op != TCCIR_OP_JUMP || qb->is_jump_target || irop_get_imm32(tcc_ir_op_get_dest(ir, qb)) != target) + continue; + /* The longest common tail, walked backwards in step. A path entering + * the removed copy anywhere but at its first instruction -- a jump target + * on a skipped NOP included -- would run the whole kept tail instead of + * the part after its entry, so the walk stops before such an entry. The + * copy's first instruction may be a target (its paths then fall through + * NOPs to the redirected jump and run exactly that tail). The kept copy + * is left unchanged. */ + int k = 0; + for (int ia = ja, ib = jb;;) + { + int na = cj_prev(ir, ia), nb = cj_prev(ir, ib); + if (na < 0 || nb < 0 || na == nb || na == jb || nb == ja) + break; + int entry = 0; + for (int m = nb + 1; m < ib && !entry; m++) + entry = ir->compact_instructions[m].is_jump_target; + IRQuadCompact *a = &ir->compact_instructions[na], *b = &ir->compact_instructions[nb]; + if (entry || !cj_insn_same(ir, a, b)) + break; + tail_a[k] = na; + tail_b[k] = nb; + k++; + if (k >= CJ_MAX_TAIL) + break; + ia = na; + ib = nb; + if (b->is_jump_target || a->is_jump_target) + break; + } + /* Do not begin with a flag consumer whose flags were set before the + * tail: trim from the front until every consumer follows a producer. */ + for (;;) + { + int flags_ok = 1, seen_producer = 0, drop = -1; + for (int m = k - 1; m >= 0; m--) + { + int op = ir->compact_instructions[tail_a[m]].op; + if (cj_reads_flags(op) && !seen_producer) + { + flags_ok = 0; + drop = m; + break; + } + if (cj_sets_flags(op)) + seen_producer = 1; + } + if (flags_ok) + break; + k = drop; /* keep only the instructions after the consumer */ + } + k = k >= 2 ? cj_verify(ir, tail_a, tail_b, k, occ, pa, pb, ca, cb) : 0; + if (k < 2) + continue; + /* Removed copy: tail_b[k-1] .. jb. A jump target inside it other than + * its first instruction would lose its code. */ + int ok = 1; + for (int m = 0; m < k - 1 && ok; m++) + ok = !ir->compact_instructions[tail_b[m]].is_jump_target; + if (!ok) + continue; + int start_a = tail_a[k - 1], start_b = tail_b[k - 1]; + for (int m = 0; m < k; m++) + ir->compact_instructions[tail_b[m]].op = TCCIR_OP_NOP; + tcc_ir_set_dest(ir, jb, irop_make_imm32(-1, start_a, IROP_BTYPE_INT32)); + ir->compact_instructions[start_a].is_jump_target = 1; + (void)start_b; + merged++; + } + } + tcc_free(cb); + tcc_free(ca); + tcc_free(pb); + tcc_free(pa); + tcc_free(occ); + tcc_free(tail_b); + tcc_free(tail_a); + tcc_free(jumps); + return merged; +} + +/* + * Tail merging by final location, run inside code generation. + * + * tcc_ir_cross_jump above pairs values by vreg, so two tails that differ only + * in which vreg brings a value in stay apart even when both vregs sit in the + * same register -- Zig's error return after each `try`: + * + * R1(T43) <-- R1(T253) [ASSIGN] R1(T73) <-- R1(T254) [ASSIGN] + * R10(T323)***DEREF*** <-- #0xaaaaaaaa R10(T323)***DEREF*** <-- #0xaaaaaaaa + * R10(T323) <-- R1(T43) STORE_INDEXED #4 R10(T323) <-- R1(T73) STORE_INDEXED #4 + * JMP to 361 JMP to 361 + * + * T253 and T254 are each the error code of a different call; on either path + * the tail reads it from R1. This pass merges such tails once the locations + * are final: after the discovery pass of tcc_ir_codegen_generate, which is the + * last thing that moves a vreg (scratch-conflict reassignment, demotion to a + * frame slot). A value flowing into the tail may then be a different vreg in + * each copy when both have the same allocation -- same registers, or the same + * frame slot -- and neither copy redefines it. Everything else follows + * tcc_ir_cross_jump: the same vreg, or defined inside its copy, used nowhere + * else, and paired one to one. + * + * Tails ending in a return merge too (RETURNVOID with RETURNVOID; RETURNVALUE + * with RETURNVALUE of the same value by those rules): both leave through the + * one epilogue. The removed copy's instructions become NOPs, one of them the + * jump to the kept copy's first instruction. + */ + +/* Do two vregs live in the same place for the whole function? */ +static int cjl_same_location(TCCIRState *ir, int32_t va, int32_t vb) +{ + int ta = TCCIR_DECODE_VREG_TYPE(va), tb = TCCIR_DECODE_VREG_TYPE(vb); + if (ta != tb || (ta != TCCIR_VREG_TYPE_TEMP && ta != TCCIR_VREG_TYPE_VAR)) + return 0; + IRLiveInterval *a = tcc_ir_get_live_interval(ir, va), *b = tcc_ir_get_live_interval(ir, vb); + if (!a || !b || a->addrtaken || b->addrtaken || a->is_lvalue || b->is_lvalue) + return 0; + if (a->is_float != b->is_float || a->is_double != b->is_double || a->is_llong != b->is_llong || + a->is_complex != b->is_complex || a->use_vfp != b->use_vfp || a->is_struct != b->is_struct) + return 0; + if (a->allocation.r0 != b->allocation.r0 || a->allocation.r1 != b->allocation.r1) + return 0; + if (a->allocation.r0 == (PREG_SPILLED | PREG_REG_NONE) || (a->allocation.r0 & PREG_SPILLED)) + { + if (a->allocation.offset != b->allocation.offset) + return 0; + } + else if ((a->allocation.r0 & PREG_REG_NONE) == PREG_REG_NONE) + return 0; /* no location at all */ + if (a->remat_kind != b->remat_kind || (a->remat_kind && a->remat_imm != b->remat_imm)) + return 0; + return 1; +} + +/* Does any instruction of the tail define vreg vr? */ +static int cjl_defined_in(TCCIRState *ir, const int *tail, int k, int32_t key) +{ + for (int m = 0; m < k; m++) + { + IRQuadCompact *q = &ir->compact_instructions[tail[m]]; + for (int s = 0; s < 4; s++) + { + IROperand o = cj_slot(ir, q, s); + if (cj_vkey(ir, irop_get_vreg(o)) == key && cj_defines(q, s, o)) + return 1; + } + } + return 0; +} + +static int cjl_is_call(int op) +{ + return op == TCCIR_OP_FUNCPARAMVAL || op == TCCIR_OP_FUNCPARAMVOID || op == TCCIR_OP_FUNCCALLVAL || + op == TCCIR_OP_FUNCCALLVOID; +} + +static int cjl_call_id(TCCIRState *ir, IRQuadCompact *q) +{ + return TCCIR_DECODE_CALL_ID(irop_get_imm64_ex(ir, tcc_ir_op_get_src2(ir, q))); +} + +/* cj_insn_same, and calls: a call's parameters and the call carry the call id + * in src2's high half, which cjl_verify pairs; the low half (parameter index + * or argument count) must match. */ +static int cjl_insn_same(TCCIRState *ir, IRQuadCompact *a, IRQuadCompact *b) +{ + if (!cjl_is_call(a->op) || a->op != b->op) + return cj_insn_same(ir, a, b); + for (int s = 0; s < 4; s++) + { + IROperand oa = cj_slot(ir, a, s), ob = cj_slot(ir, b, s); + if (s == 2) + { + if (oa.tag != ob.tag || (irop_get_imm64_ex(ir, oa) & 0xFFFF) != (irop_get_imm64_ex(ir, ob) & 0xFFFF)) + return 0; + continue; + } + if (!cj_operand_same(ir, oa, ob)) + return 0; + } + return tcc_ir_barrel_shift_at(ir, a) == tcc_ir_barrel_shift_at(ir, b) && + tcc_ir_shift64_dead_half_at(ir, a) == tcc_ir_shift64_dead_half_at(ir, b) && + tcc_ir_zero_half64_at(ir, a) == tcc_ir_zero_half64_at(ir, b) && + tcc_ir_bfi_params_at(ir, a) == tcc_ir_bfi_params_at(ir, b); +} + +/* Call ids of the tail paired one to one, every instruction of a paired call + * inside its tail. Returns 1, or 0 with *cut set to the number of tail + * instructions after the first call found cut in two (the tail starts + * between its parameters). */ +static int cjl_calls_whole(TCCIRState *ir, const int *ta, const int *tb, int k, const int *callocc, int ncalls, + int *ida, int *idb, int *cnt, int *cut) +{ + int np = 0; + for (int m = 0; m < k; m++) + { + IRQuadCompact *qa = &ir->compact_instructions[ta[m]], *qb = &ir->compact_instructions[tb[m]]; + if (!cjl_is_call(qa->op)) + continue; + int a = cjl_call_id(ir, qa), b = cjl_call_id(ir, qb); + if (a < 0 || b < 0 || a >= ncalls || b >= ncalls) + { + *cut = m; + return 0; + } + int p = -1; + for (int x = 0; x < np; x++) + if (ida[x] == a || idb[x] == b) + p = x; + if (p < 0) + { + ida[np] = a, idb[np] = b, cnt[np] = 0; + p = np++; + } + else if (ida[p] != a || idb[p] != b) + { + *cut = m; + return 0; + } + cnt[p]++; + } + for (int x = 0; x < np; x++) + if (cnt[x] != callocc[ida[x]] || cnt[x] != callocc[idb[x]]) + { + /* keep what follows this call's last instruction in the tail */ + int last = k; + for (int m = 0; m < k; m++) + { + IRQuadCompact *qa = &ir->compact_instructions[ta[m]]; + if (cjl_is_call(qa->op) && cjl_call_id(ir, qa) == ida[x]) + { + last = m; + break; + } + } + *cut = last; + return 0; + } + return 1; +} + +/* cj_verify with location-matched values flowing in. The terminators ra/rb, + * when >= 0, are RETURNVALUEs whose operand must match like any other read. */ +static int cjl_verify(TCCIRState *ir, int *ta, int *tb, int k, int ra, int rb, const int *occ, int *pa, int *pb, + int *ca, int *cb, char *live_in) +{ + while (k >= 1) + { + int np = 0, fail = -1; + for (int m = k - 1; m >= 0 && fail < 0; m--) + { + IRQuadCompact *qa = &ir->compact_instructions[ta[m]], *qb = &ir->compact_instructions[tb[m]]; + for (int s = 0; s < 4 && fail < 0; s++) + { + IROperand oa = cj_slot(ir, qa, s), ob = cj_slot(ir, qb, s); + int va = cj_vkey(ir, irop_get_vreg(oa)), vb = cj_vkey(ir, irop_get_vreg(ob)); + if (va < 0 && vb < 0) + continue; + int p = -1; + for (int x = 0; x < np; x++) + if (pa[x] == va || pb[x] == vb) + p = x; + if (p >= 0) + { + if (pa[p] != va || pb[p] != vb) + fail = m; + else + ca[p]++, cb[p]++; + continue; + } + if (va == vb) + continue; + if (va < 0 || vb < 0) + { + fail = m; + continue; + } + int da = cj_defines(qa, s, oa), db = cj_defines(qb, s, ob); + if (da && db) + { + pa[np] = va, pb[np] = vb, ca[np] = 1, cb[np] = 1, live_in[np] = 0; + np++; + continue; + } + if (!da && !db && cjl_same_location(ir, irop_get_vreg(oa), irop_get_vreg(ob)) && + !cjl_defined_in(ir, ta, k, va) && !cjl_defined_in(ir, tb, k, vb)) + { + pa[np] = va, pb[np] = vb, ca[np] = 1, cb[np] = 1, live_in[np] = 1; + np++; + continue; + } + fail = m; + } + } + if (fail < 0 && ra >= 0) + { + /* The returned value: same vreg, a pair from the tail, or a value + * flowing in through the same location. */ + IROperand oa = tcc_ir_op_get_src1(ir, &ir->compact_instructions[ra]); + IROperand ob = tcc_ir_op_get_src1(ir, &ir->compact_instructions[rb]); + int va = cj_vkey(ir, irop_get_vreg(oa)), vb = cj_vkey(ir, irop_get_vreg(ob)); + if (va >= 0 || vb >= 0) + { + int p = -1; + for (int x = 0; x < np; x++) + if (pa[x] == va || pb[x] == vb) + p = x; + if (p >= 0) + { + if (pa[p] != va || pb[p] != vb) + return 0; + else + ca[p]++, cb[p]++; + } + else if (va != vb) + { + if (va >= 0 && vb >= 0 && cjl_same_location(ir, irop_get_vreg(oa), irop_get_vreg(ob)) && + !cjl_defined_in(ir, ta, k, va) && !cjl_defined_in(ir, tb, k, vb)) + { + pa[np] = va, pb[np] = vb, ca[np] = 1, cb[np] = 1, live_in[np] = 1; + np++; + } + else + return 0; /* the returned values differ whatever the tail */ + } + } + } + if (fail < 0) + { + for (int x = 0; x < np; x++) + if (!live_in[x] && (ca[x] != occ[pa[x]] || cb[x] != occ[pb[x]])) + return 0; /* used outside its tail */ + return k; + } + k = fail; /* keep only what follows the failing instruction */ + } + return 0; +} + +static int cjl_is_exit(int op) +{ + return op == TCCIR_OP_JUMP || op == TCCIR_OP_RETURNVOID || op == TCCIR_OP_RETURNVALUE; +} + +/* Do two exits leave to the same place? */ +static int cjl_same_exit(TCCIRState *ir, IRQuadCompact *a, IRQuadCompact *b) +{ + if (a->op != b->op) + return 0; + if (a->op == TCCIR_OP_JUMP) + return irop_get_imm32(tcc_ir_op_get_dest(ir, a)) == irop_get_imm32(tcc_ir_op_get_dest(ir, b)); + if (a->op == TCCIR_OP_RETURNVALUE) + { + IROperand oa = tcc_ir_op_get_src1(ir, a), ob = tcc_ir_op_get_src1(ir, b); + return cj_operand_same(ir, oa, ob) && !tcc_ir_barrel_shift_at(ir, a) && !tcc_ir_barrel_shift_at(ir, b); + } + return 1; +} + +int tcc_ir_cross_jump_alloc(TCCIRState *ir) +{ + const int n = ir->next_instruction_index; + if (n < 4 || ir->func_has_label_addr || tcc_ir_calls_returns_twice(ir)) + return 0; + int *exits = tcc_malloc(sizeof(int) * n); + int ne = 0; + for (int i = 0; i < n; i++) + if (cjl_is_exit(ir->compact_instructions[i].op)) + exits[ne++] = i; + + int *tail_a = tcc_malloc(sizeof(int) * (CJ_MAX_TAIL + 1)), *tail_b = tcc_malloc(sizeof(int) * (CJ_MAX_TAIL + 1)); + const int nkeys = ir->next_temporary_variable + ir->next_local_variable + ir->next_parameter + 1; + int *occ = tcc_mallocz(sizeof(int) * nkeys); + for (int i = 0; i < n; i++) + { + IRQuadCompact *q = &ir->compact_instructions[i]; + if (q->op == TCCIR_OP_NOP) + continue; + for (int s = 0; s < 4; s++) + { + int key = cj_vkey(ir, irop_get_vreg(cj_slot(ir, q, s))); + if (key >= 0) + occ[key]++; + } + } + const int ncalls = ir->next_call_id + 1; + int *callocc = tcc_mallocz(sizeof(int) * ncalls); + for (int i = 0; i < n; i++) + { + IRQuadCompact *q = &ir->compact_instructions[i]; + if (cjl_is_call(q->op)) + { + int id = cjl_call_id(ir, q); + if (id >= 0 && id < ncalls) + callocc[id]++; + } + } + const int np_max = 4 * (CJ_MAX_TAIL + 1) + 1; + int *pa = tcc_malloc(sizeof(int) * np_max), *pb = tcc_malloc(sizeof(int) * np_max); + int *ca = tcc_malloc(sizeof(int) * np_max), *cb = tcc_malloc(sizeof(int) * np_max); + char *live_in = tcc_malloc(np_max); + int merged = 0; + for (int x = 0; x < ne; x++) + { + int ja = exits[x]; + IRQuadCompact *qa = &ir->compact_instructions[ja]; + if (!cjl_is_exit(qa->op)) + continue; + for (int y = x + 1; y < ne; y++) + { + int jb = exits[y]; + IRQuadCompact *qb = &ir->compact_instructions[jb]; + if (!cjl_is_exit(qb->op) || qb->is_jump_target || !cjl_same_exit(ir, qa, qb)) + continue; + /* The longest common tail, walked backwards in step (see + * tcc_ir_cross_jump). */ + int k = 0; + for (int ia = ja, ib = jb;;) + { + int na = cj_prev(ir, ia), nb = cj_prev(ir, ib); + if (na < 0 || nb < 0 || na == nb || na == jb || nb == ja) + break; + int entry = 0; + for (int m = nb + 1; m < ib && !entry; m++) + entry = ir->compact_instructions[m].is_jump_target; + IRQuadCompact *a = &ir->compact_instructions[na], *b = &ir->compact_instructions[nb]; + if (entry || cjl_is_exit(a->op) || cjl_is_exit(b->op) || !cjl_insn_same(ir, a, b)) + break; + tail_a[k] = na; + tail_b[k] = nb; + k++; + if (k >= CJ_MAX_TAIL) + break; + ia = na; + ib = nb; + if (b->is_jump_target || a->is_jump_target) + break; + } + for (;;) + { + int flags_ok = 1, seen_producer = 0, drop = -1; + for (int m = k - 1; m >= 0; m--) + { + int op = ir->compact_instructions[tail_a[m]].op; + if (cj_reads_flags(op) && !seen_producer) + { + flags_ok = 0; + drop = m; + break; + } + if (cj_sets_flags(op)) + seen_producer = 1; + } + if (flags_ok) + break; + k = drop; + } + int ra = qa->op == TCCIR_OP_RETURNVALUE ? ja : -1, rb = qa->op == TCCIR_OP_RETURNVALUE ? jb : -1; + for (;;) + { + int cut; + k = k >= 1 ? cjl_verify(ir, tail_a, tail_b, k, ra, rb, occ, pa, pb, ca, cb, live_in) : 0; + if (k < 1 || cjl_calls_whole(ir, tail_a, tail_b, k, callocc, ncalls, pa, pb, ca, &cut)) + break; + k = cut; + } + /* A plain jump pays for itself only with two instructions merged; a + * return is replaced by a jump, so one instruction plus the return. */ + if (k < (qa->op == TCCIR_OP_JUMP ? 2 : 1)) + continue; + int ok = 1; + for (int m = 0; m < k - 1 && ok; m++) + ok = !ir->compact_instructions[tail_b[m]].is_jump_target; + if (!ok) + continue; + /* The jump goes into a removed instruction with a destination slot: + * the removed exit itself when it has one (JUMP, RETURNVALUE). */ + int jslot = -1; + if (qb->op != TCCIR_OP_RETURNVOID) + jslot = jb; + else + for (int m = 0; m < k && jslot < 0; m++) + if (irop_config[ir->compact_instructions[tail_b[m]].op].has_dest || + irop_config[ir->compact_instructions[tail_b[m]].op].has_src1) + jslot = tail_b[m]; + if (jslot < 0) + continue; + int start_a = tail_a[k - 1]; + for (int m = 0; m < k; m++) + ir->compact_instructions[tail_b[m]].op = TCCIR_OP_NOP; + if (jslot != jb) + ir->compact_instructions[jb].op = TCCIR_OP_NOP; + IRQuadCompact *jq = &ir->compact_instructions[jslot]; + /* pool[operand_base] is the first operand slot of whatever op was here */ + ir->iroperand_pool[jq->operand_base] = irop_make_imm32(-1, start_a, IROP_BTYPE_INT32); + jq->op = TCCIR_OP_JUMP; + ir->compact_instructions[start_a].is_jump_target = 1; + merged++; + } + } + tcc_free(callocc); + tcc_free(live_in); + tcc_free(cb); + tcc_free(ca); + tcc_free(pb); + tcc_free(pa); + tcc_free(occ); + tcc_free(tail_b); + tcc_free(tail_a); + tcc_free(exits); + return merged; +} + +/* + * Region merging: whole single-entry regions that repeat. + * + * A switch over a Zig tagged union spells out one case body per tag, and after + * SRA and frame colouring many of those bodies are the same code: loads from + * the same slots, the same call, the same branches to the same places. Each is + * a region -- a straight run of instructions entered only at its first one, + * ending in an unconditional transfer, with every branch inside it going either + * to the same place in both copies or to the same position within its own copy. + * A region that repeats an earlier one becomes a jump to it. + * + * Same rules as the tail merge above for values: a vreg is the same vreg in + * both copies, or first defined in its copy and occurring nowhere else, paired + * one to one. Calls may be merged here: a call's parameters and the call carry + * a call id, and ids are paired the same way (every instruction with the id + * inside the region). Not merged: indirect jumps, switch dispatch, inline asm, + * VLA and setjmp ops, a region that begins by consuming flags set before it, and + * a region whose inner jump targets are reached from outside it. + */ + +#define RM_MAX_LEN 4096 +#define RM_MIN_LEN 3 +#define RM_BUCKET_TRIES 32 + +static int rm_op_ok(int op) +{ + switch (op) + { + case TCCIR_OP_JUMP: + case TCCIR_OP_JUMPIF: + case TCCIR_OP_RETURNVALUE: + case TCCIR_OP_RETURNVOID: + case TCCIR_OP_FUNCPARAMVAL: + case TCCIR_OP_FUNCPARAMVOID: + case TCCIR_OP_FUNCCALLVAL: + case TCCIR_OP_FUNCCALLVOID: + return 1; + case TCCIR_OP_NOP: + return 0; + default: + return cj_mergeable_op(op); + } +} + +static int rm_is_call_op(int op) +{ + return op == TCCIR_OP_FUNCPARAMVAL || op == TCCIR_OP_FUNCPARAMVOID || op == TCCIR_OP_FUNCCALLVAL || + op == TCCIR_OP_FUNCCALLVOID; +} + +static int rm_is_jump(int op) +{ + return op == TCCIR_OP_JUMP || op == TCCIR_OP_JUMPIF; +} + +static int rm_ends_region(int op) +{ + return op == TCCIR_OP_JUMP || op == TCCIR_OP_RETURNVALUE || op == TCCIR_OP_RETURNVOID; +} + +/* Everything but jump targets and call ids, which rm_match pairs itself. */ +static int rm_insn_same(TCCIRState *ir, IRQuadCompact *a, IRQuadCompact *b) +{ + if (a->op != b->op || !rm_op_ok(a->op)) + return 0; + for (int s = 0; s < 4; s++) + { + if (s == 0 && rm_is_jump(a->op)) + continue; + IROperand oa = cj_slot(ir, a, s), ob = cj_slot(ir, b, s); + if (s == 2 && rm_is_call_op(a->op)) + { + if (oa.tag != ob.tag || (irop_get_imm64_ex(ir, oa) & 0xFFFF) != (irop_get_imm64_ex(ir, ob) & 0xFFFF)) + return 0; + continue; + } + if (!cj_operand_same(ir, oa, ob)) + return 0; + } + return tcc_ir_barrel_shift_at(ir, a) == tcc_ir_barrel_shift_at(ir, b) && + tcc_ir_shift64_dead_half_at(ir, a) == tcc_ir_shift64_dead_half_at(ir, b) && + tcc_ir_zero_half64_at(ir, a) == tcc_ir_zero_half64_at(ir, b) && + tcc_ir_bfi_params_at(ir, a) == tcc_ir_bfi_params_at(ir, b); +} + +static int rm_next(TCCIRState *ir, int i, int n) +{ + while (i < n && ir->compact_instructions[i].op == TCCIR_OP_NOP) + i++; + return i; +} + +static int rm_call_id(TCCIRState *ir, IRQuadCompact *q) +{ + return TCCIR_DECODE_CALL_ID(irop_get_imm64_ex(ir, tcc_ir_op_get_src2(ir, q))); +} + +#define RM_NOCASE INT64_MIN +#define RM_MULTI (INT64_MIN + 1) +#define RM_MAX_PARAMS 8 + +typedef struct RMState +{ + TCCIRState *ir; + int n; + int *ord; /* non-NOP instructions before each index */ + int *occ; /* operand occurrences per vreg key */ + int *callocc; /* instructions per call id */ + int *jrefs; /* JUMP/JUMPIF references per target */ + int *trefs; /* switch table references per target */ + unsigned char *dead; + int *ia, *ib; /* matched instruction pairs */ + /* pairing maps, valid where stamp == gen */ + int *vmap_a, *vmap_b, *vcnt, *vstamp_a, *vstamp_b; + int *cmap_a, *cmap_b, *ccnt, *cstamp_a, *cstamp_b; + int gen; + /* Case-constant merging (NULL arrays when off). */ + int64_t *caseval; /* case value reaching each entry only from its dispatch, or RM_NOCASE/RM_MULTI */ + int *selsw; /* the SWITCH_TABLE dispatching to each entry */ + IROperand *swsel; /* per SWITCH_TABLE: an operand holding the case value itself, or IROP_NONE */ + signed char *pslot; /* operand slot to rewrite to the selector, or -1 */ + int *pown; /* entry of the region owning a rewrite / a parameterised region's instructions */ + int *npar; /* rewrites per parameterised entry */ + unsigned char *exact_in; /* an exact copy was merged into this entry */ + int nparam, param_at[RM_MAX_PARAMS], param_slot[RM_MAX_PARAMS]; +} RMState; + +/* Pair key ka (copy A) with kb (copy B); returns 0 on a conflict. */ +static int rm_pair(RMState *st, int *map_a, int *map_b, int *cnt, int *stamp_a, int *stamp_b, int ka, int kb, + int may_start) +{ + int seen_a = stamp_a[ka] == st->gen, seen_b = stamp_b[kb] == st->gen; + if (seen_a || seen_b) + { + if (!seen_a || !seen_b || map_a[ka] != kb || map_b[kb] != ka) + return 0; + cnt[ka]++; + return 1; + } + if (!may_start) + return 0; + stamp_a[ka] = stamp_b[kb] = st->gen; + map_a[ka] = kb; + map_b[kb] = ka; + cnt[ka] = 1; + return 1; +} + +/* + * Case-constant merging. A Zig `switch (x.tag) { .k => .{ .tag = k, ... } }` + * spells out bodies that are the same except for the constant k, the value of + * the selector on the way in. Such bodies match when each copy's constant is + * the case value of the single table entry reaching it; the kept copy then + * reads the selector instead of its constant. Rewrites are recorded while + * matching and applied at the end, so later bodies still compare constants. + * + * The selector must still hold the case value where the kept copy reads it: + * its location (register or frame slot, after allocation) must be reserved + * over the whole copy and written by nothing in it. + */ + +/* An operand naming a 32-bit-or-narrower integer value held in a vreg. */ +static int rm_sel_operand_ok(IROperand o) +{ + int32_t vr = irop_get_vreg(o); + if (vr < 0 || o.tag != IROP_TAG_VREG || o.is_lval || o.is_llocal || o.is_complex) + return 0; + if (o.btype != IROP_BTYPE_INT32 && o.btype != IROP_BTYPE_INT8 && o.btype != IROP_BTYPE_INT16) + return 0; + int t = TCCIR_DECODE_VREG_TYPE(vr); + return t == TCCIR_VREG_TYPE_TEMP || t == TCCIR_VREG_TYPE_VAR || t == TCCIR_VREG_TYPE_PARAM; +} + +/* Where a vreg lives after allocation: core registers (mask) or a frame + * offset. Returns 0 when that cannot be told. */ +static int rm_loc(TCCIRState *ir, int32_t vr, uint32_t *regs, int *off, int *is_stack, int *size) +{ + IRLiveInterval *li = tcc_ir_try_get_live_interval(ir, vr); + if (!li) + return 0; + unsigned r0 = li->allocation.r0, r1 = li->allocation.r1; + *regs = 0; + *off = 0; + *is_stack = 0; + *size = li->is_llong || li->is_double || li->is_complex ? 8 : 4; + if (r0 != PREG_NONE && !(r0 & PREG_SPILLED) && li->allocation.offset == 0) + { + if (!LS_IS_VFP_REG(r0)) + *regs |= 1u << (r0 & PREG_REG_NONE); + if (r1 != PREG_NONE && r1 != 0xFFFF && !(r1 & PREG_SPILLED) && !LS_IS_VFP_REG(r1)) + *regs |= 1u << (r1 & PREG_REG_NONE); + return 1; + } + if (li->allocation.offset != 0) + { + *off = li->allocation.offset; + *is_stack = 1; + return 1; + } + return 0; +} + +/* Does anything in [from, to] write the location of `sel`, or name another + * value living there? */ +static int rm_sel_clobbered(RMState *st, int from, int to, int32_t sel, uint32_t regs, int off, int is_stack) +{ + TCCIRState *ir = st->ir; + for (int i = from; i <= to; i++) + { + IRQuadCompact *q = &ir->compact_instructions[i]; + if (q->op == TCCIR_OP_NOP) + continue; + for (int s = 0; s < 4; s++) + { + IROperand o = cj_slot(ir, q, s); + if (irop_is_none(o)) + continue; + int32_t vr = irop_get_vreg(o); + if (vr < 0) + { + if (is_stack && s == 0 && o.tag == IROP_TAG_STACKOFF && !o.is_llocal) + { + /* A direct frame write: a scalar store names its bytes, <= 8. */ + if (q->op != TCCIR_OP_STORE || o.btype == IROP_BTYPE_STRUCT || o.is_param) + return 1; + int32_t w = irop_get_stack_offset(o); + int wsz = o.btype == IROP_BTYPE_INT64 || o.btype == IROP_BTYPE_FLOAT64 ? 8 : 4; + if (w < off + 4 && off < w + wsz) + return 1; + } + continue; + } + if (vr == sel) + { + if (cj_defines(q, s, o)) + return 1; + continue; + } + uint32_t r; + int o2, st2, sz2; + if (!rm_loc(ir, vr, &r, &o2, &st2, &sz2)) + { + if (is_stack) + return 1; + continue; /* not in a register */ + } + if (!is_stack && (r & regs)) + return 1; + if (is_stack && st2 && o2 < off + 4 && off < o2 + sz2) + return 1; + } + } + return 0; +} + +/* The operand holding the case value itself at SWITCH_TABLE `w`, or IROP_NONE. + * The table indexes with `sel - min_val`; for min_val != 0 look through the + * `idx <-- sel SUB #min_val` right before the dispatch. */ +static IROperand rm_case_selector(RMState *st, int w, int64_t min_val) +{ + TCCIRState *ir = st->ir; + IRQuadCompact *qw = &ir->compact_instructions[w]; + IROperand idx = tcc_ir_op_get_src1(ir, qw); + if (!rm_sel_operand_ok(idx)) + return IROP_NONE; + if (min_val == 0) + return idx; + if (min_val != (int32_t)min_val || TCCIR_DECODE_VREG_TYPE(irop_get_vreg(idx)) != TCCIR_VREG_TYPE_TEMP) + return IROP_NONE; + for (int p = w; p-- > 0;) + { + IRQuadCompact *q = &ir->compact_instructions[p]; + if (ir->compact_instructions[p + 1].is_jump_target || st->trefs[p + 1] || st->jrefs[p + 1]) + return IROP_NONE; /* another way in between the index and the dispatch */ + if (q->op == TCCIR_OP_NOP) + continue; + if (q->op == TCCIR_OP_JUMP || q->op == TCCIR_OP_SWITCH_TABLE || + q->op == TCCIR_OP_IJUMP || rm_is_call_op(q->op)) + return IROP_NONE; + IROperand d = irop_config[q->op].has_dest ? tcc_ir_op_get_dest(ir, q) : IROP_NONE; + if (irop_get_vreg(d) != irop_get_vreg(idx)) + continue; + if (q->op != TCCIR_OP_SUB || !cj_defines(q, 0, d) || d.btype != IROP_BTYPE_INT32) + return IROP_NONE; + IROperand x = tcc_ir_op_get_src1(ir, q), k = tcc_ir_op_get_src2(ir, q); + if (k.tag != IROP_TAG_IMM32 || k.btype == IROP_BTYPE_STRUCT || k.u.imm32 != (int32_t)min_val || + !rm_sel_operand_ok(x) || tcc_ir_barrel_shift_at(ir, q)) + return IROP_NONE; + uint32_t regs; + int off, is_stack, size; + if (!rm_loc(ir, irop_get_vreg(x), ®s, &off, &is_stack, &size) || + rm_sel_clobbered(st, p, w - 1, irop_get_vreg(x), regs, off, is_stack)) + return IROP_NONE; + return x; + } + return IROP_NONE; +} + +static int rm_imm_is_case(int32_t imm, int64_t cv) +{ + if (cv == RM_NOCASE || cv == RM_MULTI) + return 0; + return (int64_t)imm == cv || (int64_t)(uint32_t)imm == cv; +} + +/* qa/qb differ only in one immediate that is each copy's own case value, read + * by an op that can take the selector in its place instead. */ +static int rm_insn_same_case(RMState *st, int a, int b, int ia, IRQuadCompact *qa, IRQuadCompact *qb) +{ + TCCIRState *ir = st->ir; + if (!st->caseval || qa->op != qb->op || + (qa->op != TCCIR_OP_LOAD && qa->op != TCCIR_OP_STORE && qa->op != TCCIR_OP_ASSIGN)) + return 0; + if (st->selsw[a] < 0 || st->selsw[a] != st->selsw[b] || irop_is_none(st->swsel[st->selsw[a]])) + return 0; + if (!cj_operand_same(ir, cj_slot(ir, qa, 0), cj_slot(ir, qb, 0))) + return 0; + IROperand oa = cj_slot(ir, qa, 1), ob = cj_slot(ir, qb, 1); + if (oa.tag != IROP_TAG_IMM32 || ob.tag != IROP_TAG_IMM32 || oa.btype == IROP_BTYPE_STRUCT || + irop_get_vreg(oa) >= 0 || irop_get_vreg(ob) >= 0 || oa.btype != ob.btype || oa.is_lval || ob.is_lval || + oa.is_llocal != ob.is_llocal || oa.is_local != ob.is_local || oa.is_const != ob.is_const || + oa.is_complex != ob.is_complex || oa.is_unsigned != ob.is_unsigned || oa.is_static != ob.is_static || + oa.is_sym || ob.is_sym || oa.is_param != ob.is_param || oa.aux != ob.aux) + return 0; + if (!rm_imm_is_case(oa.u.imm32, st->caseval[a]) || !rm_imm_is_case(ob.u.imm32, st->caseval[b])) + return 0; + if (tcc_ir_barrel_shift_at(ir, qa) || tcc_ir_barrel_shift_at(ir, qb) || tcc_ir_shift64_dead_half_at(ir, qa) || + tcc_ir_shift64_dead_half_at(ir, qb) || tcc_ir_zero_half64_at(ir, qa) || tcc_ir_zero_half64_at(ir, qb) || + tcc_ir_bfi_params_at(ir, qa) != tcc_ir_bfi_params_at(ir, qb)) + return 0; + if (st->nparam >= RM_MAX_PARAMS) + return 0; + st->param_at[st->nparam] = ia; + st->param_slot[st->nparam] = 1; + st->nparam++; + return 1; +} + +/* A match with case-constant differences (region A = st->ia[0..k-1]): may A + * read the selector in place of its constants? On success A is recorded as + * parameterised. */ +static int rm_case_accept(RMState *st, int a, int k) +{ + TCCIRState *ir = st->ir; + int end_a = st->ia[k - 1]; + if (st->exact_in[a]) + return 0; /* a copy that is not this case already runs A */ + if (st->npar[a]) + { + /* A already reads the selector: the same places, nothing more. */ + if (st->nparam != st->npar[a]) + return 0; + for (int p = 0; p < st->nparam; p++) + if (st->pslot[st->param_at[p]] != st->param_slot[p] || st->pown[st->param_at[p]] != a) + return 0; + return 1; + } + for (int t = a; t <= end_a; t++) + if (st->pown[t] >= 0 || (t > a && (st->exact_in[t] || st->trefs[t]))) + return 0; + /* Inner jump targets of A reached only from inside A. */ + for (int t = a + 1; t <= end_a; t++) + { + if (!st->jrefs[t]) + continue; + int inside = 0; + for (int m = 0; m < k; m++) + { + IRQuadCompact *q = &ir->compact_instructions[st->ia[m]]; + if (rm_is_jump(q->op) && irop_get_imm32(tcc_ir_op_get_dest(ir, q)) == t) + inside++; + } + if (inside != st->jrefs[t]) + return 0; + } + /* The selector's location is reserved over A and written by nothing in it. */ + IROperand sel = st->swsel[st->selsw[a]]; + int32_t vr = irop_get_vreg(sel); + uint32_t regs; + int off, is_stack, size; + if (!rm_loc(ir, vr, ®s, &off, &is_stack, &size)) + return 0; + IRLiveInterval *li = tcc_ir_try_get_live_interval(ir, vr); + if (!li || li->addrtaken || li->start > (uint32_t)a || li->end < (uint32_t)end_a) + return 0; + if (!is_stack) + { + if (regs & ~0x0FF0u) + return 0; /* only callee-saved r4-r11 survive what A may call */ + if (ir->ls.live_regs_by_instruction) + for (int t = a; t <= end_a; t++) + if (t >= ir->ls.live_regs_by_instruction_size || (ir->ls.live_regs_by_instruction[t] & regs) != regs) + return 0; + } + if (rm_sel_clobbered(st, a, end_a, vr, regs, off, is_stack)) + return 0; + for (int p = 0; p < st->nparam; p++) + st->pslot[st->param_at[p]] = (signed char)st->param_slot[p]; + for (int t = a; t <= end_a; t++) + st->pown[t] = a; + st->npar[a] = st->nparam; + int key = cj_vkey(ir, vr); + if (key >= 0) + st->occ[key] += st->nparam; /* the new reads keep regions from pairing it */ + return 1; +} + +/* The instruction before `i` (NOPs skipped) never falls into it. */ +static int rm_no_fallthrough(TCCIRState *ir, int i) +{ + for (int p = i - 1; p >= 0; p--) + { + int op = ir->compact_instructions[p].op; + if (op == TCCIR_OP_NOP) + continue; + return op == TCCIR_OP_JUMP || op == TCCIR_OP_RETURNVALUE || op == TCCIR_OP_RETURNVOID || + op == TCCIR_OP_SWITCH_TABLE; + } + return 1; +} + +/* Match the region at entry `a` against the one at entry `b` (a < b). + * Returns the number of matched instructions, 0 if they differ. */ +static int rm_match(RMState *st, int a, int b) +{ + TCCIRState *ir = st->ir; + const int n = st->n; + int ia = a, ib = b, k = 0, pend = a, seen_producer = 0, shared_min = -1; + st->nparam = 0; + for (;;) + { + ia = rm_next(ir, ia, n); + ib = rm_next(ir, ib, n); + if (ia >= b || ib >= n || k >= RM_MAX_LEN || st->dead[ia] || st->dead[ib]) + return 0; + IRQuadCompact *qa = &ir->compact_instructions[ia], *qb = &ir->compact_instructions[ib]; + if (!rm_insn_same(ir, qa, qb) && !rm_insn_same_case(st, a, b, ia, qa, qb)) + return 0; + if (cj_reads_flags(qa->op) && !seen_producer) + return 0; + if (cj_sets_flags(qa->op)) + seen_producer = 1; + if (rm_is_jump(qa->op)) + { + int ta = irop_get_imm32(tcc_ir_op_get_dest(ir, qa)), tb = irop_get_imm32(tcc_ir_op_get_dest(ir, qb)); + if (ta != tb) + { + /* Both inside their own copy, at the same position. */ + if (ta < a || ta >= b || tb < b || tb >= n || st->ord[ta] - st->ord[a] != st->ord[tb] - st->ord[b]) + return 0; + if (ta > pend) + pend = ta; + } + else if (ta >= b && (shared_min < 0 || ta < shared_min)) + shared_min = ta; /* must lie past the copy being removed */ + } + st->ia[k] = ia; + st->ib[k] = ib; + k++; + if (rm_ends_region(qa->op) && pend <= ia) + break; + ia++; + ib++; + } + if (k < RM_MIN_LEN) + return 0; + int end_b = st->ib[k - 1]; + if (shared_min >= 0 && shared_min <= end_b) + return 0; /* a shared target inside the copy being removed */ + + /* Inner jump targets of B reached only from inside B. */ + for (int t = b + 1; t <= end_b; t++) + { + if (st->trefs[t]) + return 0; + if (!st->jrefs[t]) + continue; + int inside = 0; + for (int m = 0; m < k; m++) + { + IRQuadCompact *q = &ir->compact_instructions[st->ib[m]]; + if (rm_is_jump(q->op) && irop_get_imm32(tcc_ir_op_get_dest(ir, q)) == t) + inside++; + } + if (inside != st->jrefs[t]) + return 0; + } + + /* Values and calls paired one to one. */ + st->gen++; + for (int m = 0; m < k; m++) + { + IRQuadCompact *qa = &ir->compact_instructions[st->ia[m]], *qb = &ir->compact_instructions[st->ib[m]]; + if (rm_is_call_op(qa->op) && + !rm_pair(st, st->cmap_a, st->cmap_b, st->ccnt, st->cstamp_a, st->cstamp_b, rm_call_id(ir, qa), + rm_call_id(ir, qb), 1)) + return 0; + for (int s = 0; s < 4; s++) + { + IROperand oa = cj_slot(ir, qa, s), ob = cj_slot(ir, qb, s); + int va = cj_vkey(ir, irop_get_vreg(oa)), vb = cj_vkey(ir, irop_get_vreg(ob)); + if (va < 0 && vb < 0) + continue; + if (va < 0 || vb < 0) + return 0; + int paired = st->vstamp_a[va] == st->gen || st->vstamp_b[vb] == st->gen; + if (!paired && va == vb) + continue; /* the same value in both */ + if (!rm_pair(st, st->vmap_a, st->vmap_b, st->vcnt, st->vstamp_a, st->vstamp_b, va, vb, + cj_defines(qa, s, oa) && cj_defines(qb, s, ob))) + return 0; + } + } + for (int m = 0; m < k; m++) + { + IRQuadCompact *qa = &ir->compact_instructions[st->ia[m]], *qb = &ir->compact_instructions[st->ib[m]]; + if (rm_is_call_op(qa->op)) + { + int ca = rm_call_id(ir, qa), cb = rm_call_id(ir, qb); + if (st->ccnt[ca] != st->callocc[ca] || st->ccnt[ca] != st->callocc[cb]) + return 0; + } + for (int s = 0; s < 4; s++) + { + int va = cj_vkey(ir, irop_get_vreg(cj_slot(ir, qa, s))); + if (va < 0 || st->vstamp_a[va] != st->gen) + continue; + int vb = st->vmap_a[va]; + if (st->vcnt[va] != st->occ[va] || st->vcnt[va] != st->occ[vb]) + return 0; /* used outside its region */ + } + } + return k; +} + +static unsigned rm_hash(TCCIRState *ir, int i, int n) +{ + unsigned h = 2166136261u; + for (int c = 0; c < 4; c++) + { + i = rm_next(ir, i, n); + if (i >= n) + break; + int op = ir->compact_instructions[i].op; + h = (h ^ op) * 16777619u; + if (rm_ends_region(op)) + break; /* what follows belongs to another region */ + i++; + } + return h; +} + +int tcc_ir_region_merge(TCCIRState *ir) +{ + const int n = ir->next_instruction_index; + if (n < 8 || ir->func_has_label_addr) + return 0; + for (int i = 0; i < n; i++) + if (ir->compact_instructions[i].op == TCCIR_OP_IJUMP) + return 0; + + RMState st = {0}; + int merged = 0; + st.ir = ir; + st.n = n; + const int nkeys = ir->next_temporary_variable + ir->next_local_variable + ir->next_parameter + 1; + const int ncalls = ir->next_call_id + 1; + st.ord = tcc_malloc(sizeof(int) * (n + 1)); + st.occ = tcc_mallocz(sizeof(int) * nkeys); + st.callocc = tcc_mallocz(sizeof(int) * ncalls); + st.jrefs = tcc_mallocz(sizeof(int) * n); + st.trefs = tcc_mallocz(sizeof(int) * n); + st.dead = tcc_mallocz(n); + st.ia = tcc_malloc(sizeof(int) * RM_MAX_LEN); + st.ib = tcc_malloc(sizeof(int) * RM_MAX_LEN); + st.vmap_a = tcc_malloc(sizeof(int) * nkeys); + st.vmap_b = tcc_malloc(sizeof(int) * nkeys); + st.vcnt = tcc_malloc(sizeof(int) * nkeys); + st.vstamp_a = tcc_mallocz(sizeof(int) * nkeys); + st.vstamp_b = tcc_mallocz(sizeof(int) * nkeys); + st.cmap_a = tcc_malloc(sizeof(int) * ncalls); + st.cmap_b = tcc_malloc(sizeof(int) * ncalls); + st.ccnt = tcc_malloc(sizeof(int) * ncalls); + st.cstamp_a = tcc_mallocz(sizeof(int) * ncalls); + st.cstamp_b = tcc_mallocz(sizeof(int) * ncalls); + if (!tcc_ir_opt_pass_disabled("region_merge_case") && ir->num_switch_tables > 0) + { + st.caseval = tcc_malloc(sizeof(int64_t) * n); + st.selsw = tcc_malloc(sizeof(int) * n); + st.swsel = tcc_malloc(sizeof(IROperand) * n); + st.pslot = tcc_malloc(n); + st.pown = tcc_malloc(sizeof(int) * n); + st.npar = tcc_mallocz(sizeof(int) * n); + st.exact_in = tcc_mallocz(n); + for (int i = 0; i < n; i++) + { + st.caseval[i] = RM_NOCASE; + st.selsw[i] = -1; + st.swsel[i] = IROP_NONE; + st.pslot[i] = -1; + st.pown[i] = -1; + } + } + + int c = 0; + for (int i = 0; i < n; i++) + { + IRQuadCompact *q = &ir->compact_instructions[i]; + st.ord[i] = c; + if (q->op == TCCIR_OP_NOP) + continue; + c++; + for (int s = 0; s < 4; s++) + { + int key = cj_vkey(ir, irop_get_vreg(cj_slot(ir, q, s))); + if (key >= 0) + st.occ[key]++; + } + if (rm_is_call_op(q->op)) + { + int id = rm_call_id(ir, q); + if (id < 0 || id >= ncalls) + goto out; /* unknown encoding: leave the function alone */ + st.callocc[id]++; + } + if (rm_is_jump(q->op)) + { + int t = irop_get_imm32(tcc_ir_op_get_dest(ir, q)); + if (t >= 0 && t < n) + st.jrefs[t]++; + } + if (q->op == TCCIR_OP_SWITCH_TABLE) + { + int id = (int)irop_get_imm64_ex(ir, tcc_ir_op_get_src2(ir, q)); + if (id >= 0 && id < ir->num_switch_tables) + { + TCCIRSwitchTable *tab = &ir->switch_tables[id]; + for (int e = 0; e < tab->num_entries; e++) + if (tab->targets[e] >= 0 && tab->targets[e] < n) + { + int t = tab->targets[e]; + st.trefs[t]++; + if (st.caseval) + { + st.caseval[t] = st.caseval[t] == RM_NOCASE ? tab->min_val + e : RM_MULTI; + st.selsw[t] = i; + } + } + if (tab->default_target >= 0 && tab->default_target < n) + { + st.trefs[tab->default_target]++; + if (st.caseval) + st.caseval[tab->default_target] = RM_MULTI; + } + } + else if (st.caseval) + goto out; /* a table we cannot read: leave the function alone */ + } + } + st.ord[n] = c; + if (st.caseval) + for (int i = 0; i < n; i++) + { + IRQuadCompact *q = &ir->compact_instructions[i]; + if (q->op == TCCIR_OP_SWITCH_TABLE) + { + TCCIRSwitchTable *tab = &ir->switch_tables[irop_get_imm64_ex(ir, tcc_ir_op_get_src2(ir, q))]; + if (tab->num_entries > 0) + st.swsel[i] = rm_case_selector(&st, i, tab->min_val); + } + if (st.jrefs[i] || !rm_no_fallthrough(ir, i)) + st.caseval[i] = RM_MULTI; + } + + /* Entries bucketed by the ops they start with. */ + enum { NB = 1024 }; + int *head = tcc_malloc(sizeof(int) * NB), *next = tcc_malloc(sizeof(int) * n); + for (int b = 0; b < NB; b++) + head[b] = -1; + for (int b = 0; b < n; b++) + { + IRQuadCompact *qb = &ir->compact_instructions[b]; + if (!qb->is_jump_target && !st.trefs[b]) + continue; + if (st.dead[b]) + continue; + unsigned h = rm_hash(ir, b, n) & (NB - 1); + int done = 0, tries = 0; + for (int a = head[h]; a >= 0 && !done && tries < RM_BUCKET_TRIES; a = next[a], tries++) + { + if (st.dead[a]) + continue; + int k = rm_match(&st, a, b); + if (!k) + continue; + int fb = st.ib[0]; + IRQuadCompact *qf = &ir->compact_instructions[fb]; + if (!irop_config[qf->op].has_dest && !irop_config[qf->op].has_src1 && !irop_config[qf->op].has_src2) + continue; /* no operand slot for the jump's target */ + if (st.caseval) + { + if (st.nparam ? !rm_case_accept(&st, a, k) : st.pown[a] >= 0) + continue; /* an exact copy cannot run a region that reads the selector */ + if (!st.nparam) + st.exact_in[a] = 1; + } + for (int m = 1; m < k; m++) + { + ir->compact_instructions[st.ib[m]].op = TCCIR_OP_NOP; + st.dead[st.ib[m]] = 1; + } + for (int t = b + 1; t <= st.ib[k - 1]; t++) + st.dead[t] = 1; + qf->op = TCCIR_OP_JUMP; + tcc_ir_op_set_dest(ir, qf, irop_make_imm32(-1, a, IROP_BTYPE_INT32)); + st.dead[fb] = 1; + ir->compact_instructions[a].is_jump_target = 1; + merged++; + done = 1; + } + if (!done) + { + next[b] = head[h]; + head[h] = b; + } + } + tcc_free(next); + tcc_free(head); + /* Kept copies read the selector where they had their case value. */ + if (st.caseval) + for (int i = 0; i < n; i++) + if (st.pslot[i] >= 0) + { + IRQuadCompact *q = &ir->compact_instructions[i]; + tcc_ir_op_set_src1(ir, q, st.swsel[st.selsw[st.pown[i]]]); + } +out: + tcc_free(st.exact_in); + tcc_free(st.npar); + tcc_free(st.pown); + tcc_free(st.pslot); + tcc_free(st.swsel); + tcc_free(st.selsw); + tcc_free(st.caseval); + tcc_free(st.cstamp_b); + tcc_free(st.cstamp_a); + tcc_free(st.ccnt); + tcc_free(st.cmap_b); + tcc_free(st.cmap_a); + tcc_free(st.vstamp_b); + tcc_free(st.vstamp_a); + tcc_free(st.vcnt); + tcc_free(st.vmap_b); + tcc_free(st.vmap_a); + tcc_free(st.ib); + tcc_free(st.ia); + tcc_free(st.dead); + tcc_free(st.trefs); + tcc_free(st.jrefs); + tcc_free(st.callocc); + tcc_free(st.occ); + tcc_free(st.ord); + return merged; +} diff --git a/source/ir/dead_def.c b/source/ir/dead_def.c new file mode 100644 index 000000000..04332d85e --- /dev/null +++ b/source/ir/dead_def.c @@ -0,0 +1,239 @@ +/* + * TinyCC - Tiny C Compiler + * + * Dead TEMP definitions reaching register allocation. + * + * Two shapes reach the code generator as instructions whose result nothing + * reads: + * + * T8 <-- Addr[StackLoc[-88]] address of an object whose copy frame + * T9 <-- Addr[StackLoc[-88]] colouring merged away: the copy is gone, + * the operands' LEAs are not + * + * T13 <-- #0 [ASSIGN] a compound literal's zero fill, split into + * T13 <-- GlobalSym [STORE] field TEMPs by SRA: the field's real value + * PARAM1 T13 overwrites the zero before anything reads it + * + * In the Zig compiler built as C they were `add rX,sp,#k` pairs and + * `movs rX,#0; ldr rX,[pc,#k]` pairs, 18 KB. + * + * A definition of a TEMP by a side-effect-free op is deleted when + * - no instruction names the TEMP any more (other than as this plain + * destination), or + * - the same TEMP is redefined, as a plain destination and without being + * read, later in the same straight run of code, before any instruction + * names it: every path from the first definition passes the second one. + * A TEMP's value is a whole register (a spilled TEMP's slot is written as a + * word), so any definition of it replaces all of the old value. + * + * Runs inside the allocator, after its SSA passes and before its intervals, + * where a TEMP is still read only under its own name. After it, graph coalescing erases the copies between vregs that + * share a register (a phi's `T21 <- T23`), and T23's value is then read as + * T21 at the join: an unnamed use this pass cannot see. + */ + +#include "ir.h" + +#define DD_MAX_DIST 64 + +static int dd_has_slot3(int op) +{ + return tcc_ir_op_is_mac(op) || op == TCCIR_OP_LOAD_INDEXED || op == TCCIR_OP_STORE_INDEXED || op == TCCIR_OP_SELECT; +} + +static IROperand dd_slot(TCCIRState *ir, IRQuadCompact *q, int s) +{ + if (s == 0 && irop_config[q->op].has_dest) + return tcc_ir_op_get_dest(ir, q); + if (s == 1 && irop_config[q->op].has_src1) + return tcc_ir_op_get_src1(ir, q); + if (s == 2 && irop_config[q->op].has_src2) + return tcc_ir_op_get_src2(ir, q); + if (s == 3 && dd_has_slot3(q->op)) + return ir->iroperand_pool[q->operand_base + 3]; + return IROP_NONE; +} + +/* Ops that compute their destination from their operands and do nothing + * else: no memory write, no control transfer, no flags another op consumes + * (the carry-generating halves of a 64-bit add/subtract are left alone). */ +static int dd_pure_op(int op) +{ + switch (op) + { + case TCCIR_OP_ASSIGN: + case TCCIR_OP_LEA: + case TCCIR_OP_LOAD: + case TCCIR_OP_ADD: + case TCCIR_OP_SUB: + case TCCIR_OP_MUL: + case TCCIR_OP_AND: + case TCCIR_OP_OR: + case TCCIR_OP_XOR: + case TCCIR_OP_SHL: + case TCCIR_OP_SAR: + case TCCIR_OP_SHR: + case TCCIR_OP_ROR: + case TCCIR_OP_ZEXT: + case TCCIR_OP_UBFX: + case TCCIR_OP_SBFX: + case TCCIR_OP_SETIF: + return 1; + default: + return 0; + } +} + +/* Ops that write their whole destination and read it only through their + * explicit operands (BFI reads the destination it inserts into; the + * post-increment forms write their base too). */ +static int dd_full_def_op(int op) +{ + return dd_pure_op(op) || op == TCCIR_OP_STORE || op == TCCIR_OP_LOAD_INDEXED || op == TCCIR_OP_FUNCCALLVAL; +} + +/* The TEMP a plain (register) destination names, or -1. */ +static int32_t dd_plain_temp_dest(TCCIRState *ir, IRQuadCompact *q) +{ + if (!irop_config[q->op].has_dest) + return -1; + IROperand d = tcc_ir_op_get_dest(ir, q); + int32_t vr = irop_get_vreg(d); + if (vr < 0 || d.is_lval || d.is_llocal || irop_get_tag(d) != IROP_TAG_VREG || + TCCIR_DECODE_VREG_TYPE(vr) != TCCIR_VREG_TYPE_TEMP) + return -1; + return vr; +} + +static int dd_width(IROperand op) +{ + int bt = irop_get_btype(op); + return (bt == IROP_BTYPE_INT64 || bt == IROP_BTYPE_FLOAT64) ? 8 : 4; +} + +/* A side annotation that changes what the instruction reads or writes. */ +static int dd_annotated(TCCIRState *ir, IRQuadCompact *q) +{ + return tcc_ir_barrel_shift_at(ir, q) || tcc_ir_shift64_dead_half_at(ir, q) || tcc_ir_zero_half64_at(ir, q) || + tcc_ir_bfi_params_at(ir, q); +} + +/* Instructions that end a straight run: after them the next instruction in + * the array is not the only continuation. */ +static int dd_ends_run(int op) +{ + switch (op) + { + case TCCIR_OP_JUMP: + case TCCIR_OP_JUMPIF: + case TCCIR_OP_IJUMP: + case TCCIR_OP_SWITCH_TABLE: + case TCCIR_OP_SWITCH_LOAD: + case TCCIR_OP_RETURNVALUE: + case TCCIR_OP_RETURNVOID: + case TCCIR_OP_SETJMP: + case TCCIR_OP_NL_SETJMP: + case TCCIR_OP_LONGJMP: + case TCCIR_OP_NL_LONGJMP: + case TCCIR_OP_BUILTIN_RETURN: + case TCCIR_OP_INLINE_ASM: + case TCCIR_OP_TRAP: + return 1; + default: + return 0; + } +} + +static void dd_count(TCCIRState *ir, IRQuadCompact *q, int *occ, int nt, int delta) +{ + for (int s = 0; s < 4; s++) + { + int32_t vr = irop_get_vreg(dd_slot(ir, q, s)); + if (vr < 0 || TCCIR_DECODE_VREG_TYPE(vr) != TCCIR_VREG_TYPE_TEMP) + continue; + int pos = TCCIR_DECODE_VREG_POSITION(vr); + if (pos < nt) + occ[pos] += delta; + } +} + +/* Is the definition at i overwritten before any read, on the straight run + * that follows it? */ +static int dd_overwritten(TCCIRState *ir, int i, int32_t vr, int width) +{ + const int n = ir->next_instruction_index; + IRQuadCompact *def = &ir->compact_instructions[i]; + if (dd_ends_run(def->op)) + return 0; + for (int j = i + 1, dist = 0; j < n && dist < DD_MAX_DIST; j++) + { + IRQuadCompact *q = &ir->compact_instructions[j]; + if (q->op == TCCIR_OP_NOP) + continue; + dist++; + int named = 0; + for (int s = 1; s < 4 && !named; s++) + named = irop_get_vreg(dd_slot(ir, q, s)) == vr; + if (named) + return 0; + if (irop_config[q->op].has_dest && irop_get_vreg(tcc_ir_op_get_dest(ir, q)) == vr) + return dd_plain_temp_dest(ir, q) == vr && dd_full_def_op(q->op) && !dd_annotated(ir, q) && + dd_width(tcc_ir_op_get_dest(ir, q)) >= width; + if (dd_ends_run(q->op)) + return 0; + } + return 0; +} + +int tcc_ir_dead_def(TCCIRState *ir) +{ + const int n = ir->next_instruction_index; + const int nt = ir->next_temporary_variable; + if (n < 2 || nt <= 0 || ir->func_has_label_addr) + return 0; + /* Operand occurrences per TEMP, destinations included. */ + int *occ = tcc_mallocz(sizeof(int) * nt); + for (int i = 0; i < n; i++) + { + IRQuadCompact *q = &ir->compact_instructions[i]; + if (q->op != TCCIR_OP_NOP) + dd_count(ir, q, occ, nt, 1); + } + + int removed = 0, changed = 1; + while (changed) + { + changed = 0; + for (int i = n - 1; i >= 0; i--) + { + IRQuadCompact *q = &ir->compact_instructions[i]; + if (!dd_pure_op(q->op) || dd_annotated(ir, q)) + continue; + int32_t vr = dd_plain_temp_dest(ir, q); + if (vr < 0) + continue; + int pos = TCCIR_DECODE_VREG_POSITION(vr); + if (pos >= nt) + continue; + /* Any memory operand may be volatile (`T <- *p ADD #1` reads *p). */ + int vol = 0; + for (int s = 1; s < 4 && !vol; s++) + { + IROperand o = dd_slot(ir, q, s); + vol = !irop_is_none(o) && tcc_ir_access_is_volatile(ir, o); + } + if (vol) + continue; + /* Named only as this destination, or overwritten before a read. A + * TEMP the op itself also reads (`T <- T + 1`) counts more than once. */ + if (occ[pos] != 1 && !dd_overwritten(ir, i, vr, dd_width(tcc_ir_op_get_dest(ir, q)))) + continue; + dd_count(ir, q, occ, nt, -1); + q->op = TCCIR_OP_NOP; + removed++; + changed = 1; + } + } + tcc_free(occ); + return removed; +} diff --git a/source/ir/dump.c b/source/ir/dump.c index 1e80683ef..fb99ebbfe 100644 --- a/source/ir/dump.c +++ b/source/ir/dump.c @@ -61,6 +61,8 @@ const char *tcc_ir_get_op_name(TccIrOp op) return "REV"; case TCCIR_OP_REV16: return "REV16"; + case TCCIR_OP_RETURN_ADDRESS: + return "RETURN_ADDRESS"; case TCCIR_OP_PDIV: return "PDIV"; case TCCIR_OP_UDIV: @@ -168,6 +170,8 @@ const char *tcc_ir_get_op_name(TccIrOp op) return "INIT_CHAIN_SLOT"; case TCCIR_OP_MLA: return "MLA"; + case TCCIR_OP_UMAAL: + return "UMAAL"; case TCCIR_OP_SWITCH_TABLE: return "SWITCH_TABLE"; case TCCIR_OP_SWITCH_LOAD: @@ -571,6 +575,11 @@ void tcc_ir_dump_compact(TCCIRState *ir, IRQuadCompact *q, int pc, FILE *out) IROperand dest = tcc_ir_op_get_dest(ir, q); fprintf(out, "%04d: ", pc); + if (op == TCCIR_OP_TRAP && q->unreachable) + { + fprintf(out, "UNREACHABLE\n"); + return; + } switch (op) { case TCCIR_OP_NOP: @@ -601,12 +610,13 @@ void tcc_ir_dump_compact(TCCIRState *ir, IRQuadCompact *q, int pc, FILE *out) fprintf(out, "IJMP "); break; case TCCIR_OP_MLA: + case TCCIR_OP_UMAAL: tcc_ir_dump_op(ir, dest, out); fprintf(out, " <-- "); tcc_ir_dump_op(ir, src1, out); - fprintf(out, " MLA "); + fprintf(out, op == TCCIR_OP_MLA ? " MLA " : " UMAAL "); tcc_ir_dump_op(ir, src2, out); - fprintf(out, " + "); + fprintf(out, op == TCCIR_OP_MLA ? " + " : " + lo+hi "); break; default: tcc_ir_dump_op(ir, dest, out); @@ -619,7 +629,7 @@ void tcc_ir_dump_compact(TCCIRState *ir, IRQuadCompact *q, int pc, FILE *out) { fprintf(out, "(cond=0x%lx)", (unsigned long)irop_get_imm64_ex(ir, src1)); } - else if (op != TCCIR_OP_JUMPIF && op != TCCIR_OP_MLA) + else if (op != TCCIR_OP_JUMPIF && !tcc_ir_op_is_mac(op)) { tcc_ir_dump_op(ir, src1, out); } @@ -636,6 +646,7 @@ void tcc_ir_dump_compact(TCCIRState *ir, IRQuadCompact *q, int pc, FILE *out) case TCCIR_OP_FUNCPARAMVAL: case TCCIR_OP_FUNCCALLVAL: case TCCIR_OP_MLA: + case TCCIR_OP_UMAAL: break; default: fprintf(out, " %s ", tcc_ir_get_op_name((TccIrOp)op)); @@ -700,7 +711,7 @@ void tcc_ir_dump_compact(TCCIRState *ir, IRQuadCompact *q, int pc, FILE *out) } fprintf(out, "\""); } - else if (op == TCCIR_OP_MLA) + else if (tcc_ir_op_is_mac(op)) { IROperand accum = tcc_ir_op_get_accum(ir, q); tcc_ir_dump_op(ir, accum, out); diff --git a/source/ir/frame.c b/source/ir/frame.c new file mode 100644 index 000000000..c56960635 --- /dev/null +++ b/source/ir/frame.c @@ -0,0 +1,2253 @@ +/* + * TCC IR - Frame relayout + * + * Copyright (c) 2026 Mateusz Stadnik + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation. + */ + +/* + * The frontend gives every stack object (aggregates, address-taken temporaries, + * call and return slots) its own frame slot at the point it is declared, and + * that layout was final. After optimization many of those objects are no + * longer referenced at all -- a copy forwarded away, a temporary whose only + * reader was folded -- yet they still size the frame, and every live object + * below them pays for the distance: offsets past 1020 no longer fit the 16-bit + * LDR/STR [sp] forms, and past 4095 need an extra instruction. In the Zig + * compiler built as C, dropping them shrinks the summed frames by 16%, and + * letting objects whose lifetimes do not overlap share bytes by 35%. + * + * tcc_ir_frame_relayout lays the frame out again. It runs just before the + * linear scan, once every pass that adds or drops a frame reference is done: + * + * - The frame region [loc, 0) is cut into SEGMENTS: every object the frontend + * allocated through tcc_ir_frame_alloc*, with its padding, plus the gaps + * between them. A gap is an allocation made directly on `loc` -- + * parameter homes the prologue stores to, setjmp and __label__ buffers, + * VLA pointer slots -- and is PINNED: never dropped, never moved. + * - A segment is live if any stack operand points into it. + * - Colouring (frame_colour, below) then gives each live object a lifetime + * and places objects with disjoint lifetimes on the same bytes, the most + * referenced ones nearest sp, where the short [sp, #imm] forms reach. + * - Where colouring does not apply (computed gotos, a call that returns + * twice) or does no better, the dead segments are dropped and the runs of + * consecutive live ones packed towards 0 as blocks, so an address computed + * across segments inside a run stays valid. Each run moves by a multiple + * of the strictest alignment in it, capped at 8; a run holding a pinned + * gap stays where it is. + * - Every stack operand is rewritten by its segment's displacement, and *loc + * is raised to the new bottom, so the register allocator places spills + * directly below the objects. + * + * Only operands WITHOUT a vreg name frontend objects: a vreg-backed StackLoc is + * resolved relative to the vreg's own original offset and allocator slot. + * Functions whose frame is reachable other than through IR operands are left + * alone: -g (debug info records the original offsets), bounds checking, + * variadic functions (register save area), nested functions and captures + * (the static chain reads the parent's frame by offset), VLAs, inline asm, + * setjmp. + */ + +#define USING_GLOBALS +#include "ir.h" +#include "opt_utils.h" +#include + +#define FRAME_OBJ_ARG_COPY 1 /* a by-value argument's copy, never user-visible */ +#define FRAME_OBJ_RET_TEMP 2 /* a call's struct result */ + +static int frame_record(int loc, int size, int mask, int flags) +{ + TCCIRState *ir = tcc_state ? tcc_state->ir : NULL; + /* When the layout may be recoloured, a guard byte above the object keeps its + * one-past-end address inside its own record: otherwise it would equal the + * start of the object allocated before it, and could not be told apart. */ + int guard = ir && tcc_state->optimize > 0 && !tcc_state->do_debug && !tcc_bounds_checking(tcc_state); + /* The guard must not cost an object the word alignment it would otherwise + * have had: inline copies into a char array use LDM/STM, which fault on an + * unaligned address. */ + if (guard && size >= 4 && !(((loc - size) & mask) & 3)) + mask &= -4; + const int off = (loc - size - guard) & mask; + /* Recorded with the padding above it: the padding is freed or moved together + * with the object, instead of forming an unknown (pinned) gap. */ + if (ir && loc > off) + { + if (ir->frame_obj_count == ir->frame_obj_cap) + { + ir->frame_obj_cap = ir->frame_obj_cap ? ir->frame_obj_cap * 2 : 32; + ir->frame_objs = tcc_realloc(ir->frame_objs, sizeof(int32_t) * 4 * ir->frame_obj_cap); + } + int32_t *o = &ir->frame_objs[4 * ir->frame_obj_count++]; + o[0] = off; + o[1] = loc - off; + o[2] = size > 0 ? size : 1; + o[3] = flags; + } + return off; +} + +/* Mark the bytes of a `t` at byte `at` that some member of it occupies. A + * bit-field is charged its whole storage unit and every byte its bits reach; + * anything nested too deep, or of unknown size, all of its bytes. */ +static void frame_cover(CType *t, int at, uint64_t *cov, int depth) +{ + int align, size = type_size(t, &align); + if (size <= 0 || at >= 64) + return; + const int bt = t->t & VT_BTYPE; + if (depth < 8 && bt == VT_STRUCT && t->ref && !(t->t & VT_BITFIELD)) + { + for (Sym *f = t->ref->next; f; f = f->next) + { + frame_cover(&f->type, at + f->c, cov, depth + 1); + if (f->type.t & VT_BITFIELD) + { + int end = (BIT_POS(f->type.t) + BIT_SIZE(f->type.t) + 7) / 8; + for (int b = at + f->c; b < at + f->c + end && b < 64; b++) + *cov |= 1ull << b; + } + } + return; + } + if (depth < 8 && bt == VT_PTR && (t->t & VT_ARRAY) && t->ref) + { + int ea, es = type_size(&t->ref->type, &ea); + if (es > 0) + { + for (int e = 0; e < size / es && at + e * es < 64; e++) + frame_cover(&t->ref->type, at + e * es, cov, depth + 1); + return; + } + } + for (int b = at; b < at + size && b < 64; b++) + *cov |= 1ull << b; +} + +/* C leaves the padding bytes of a struct unspecified whenever a member is + * stored (C11 6.2.6.1p6), so a store may write anything there: SRA uses it to + * store a u16 into the low half of a word field without keeping the other + * half (sra.c). The bytes of the record above the object -- alignment, the + * guard byte -- belong to no member either. */ +void tcc_ir_frame_note_type(int off, CType *type) +{ + TCCIRState *ir = tcc_state ? tcc_state->ir : NULL; + if (!ir || !type || !ir->frame_obj_count || ir->frame_objs[4 * (ir->frame_obj_count - 1)] != off) + return; + const int32_t *o = &ir->frame_objs[4 * (ir->frame_obj_count - 1)]; + uint64_t cov = 0; + frame_cover(type, 0, &cov, 0); + const int ext = o[1] < 64 ? o[1] : 64; + uint64_t pad = ~cov & (ext == 64 ? ~0ull : (1ull << ext) - 1); + if (!pad) + return; + if (ir->frame_pad_count == ir->frame_pad_cap) + { + ir->frame_pad_cap = ir->frame_pad_cap ? ir->frame_pad_cap * 2 : 16; + ir->frame_pads = tcc_realloc(ir->frame_pads, sizeof(int32_t) * 3 * ir->frame_pad_cap); + } + int32_t *p = &ir->frame_pads[3 * ir->frame_pad_count++]; + p[0] = off; + p[1] = (int32_t)(uint32_t)pad; + p[2] = (int32_t)(uint32_t)(pad >> 32); +} + +uint64_t tcc_ir_frame_padding(TCCIRState *ir, int lo, int hi) +{ + if (!ir || ir->frame_relaid || !ir->frame_pad_count || hi - lo > 64 || hi <= lo) + return 0; + uint64_t pad = ~0ull; + int objs = 0; + for (int k = 0; k < ir->frame_obj_count; k++) + { + const int32_t *o = &ir->frame_objs[4 * k]; + if (o[0] >= hi || o[0] + o[1] <= lo) + continue; + if (o[0] != lo) + return 0; + objs++; + } + int noted = 0; + for (int k = 0; k < ir->frame_pad_count; k++) + { + const int32_t *p = &ir->frame_pads[3 * k]; + if (p[0] != lo) + continue; + pad &= (uint64_t)(uint32_t)p[1] | ((uint64_t)(uint32_t)p[2] << 32); + noted++; + } + /* Every object recorded here must have told its type: one with no padding + * noted, or none known, keeps all of its bytes. */ + return objs && noted == objs ? pad : 0; +} + +int tcc_ir_frame_alloc(int loc, int size, int mask) +{ + return frame_record(loc, size, mask, 0); +} + +int tcc_ir_frame_alloc_arg_copy(int loc, int size, int mask) +{ + return frame_record(loc, size, mask, FRAME_OBJ_ARG_COPY); +} + +int tcc_ir_frame_alloc_ret_temp(int loc, int size, int mask) +{ + return frame_record(loc, size, mask, FRAME_OBJ_RET_TEMP); +} + +/* The objects recorded since `first_obj` belong to an inlined callee whose + * body ends at instruction `insn`: their lifetime ends there whatever their + * address did (the callee's locals died at its return). An object already + * scoped by a nested expansion keeps its (earlier) end; the one at `keep` + * (a callee local the return was redirected into) outlives the body. */ +void tcc_ir_frame_scope_end(TCCIRState *ir, int first_obj, int insn, int keep) +{ + if (!ir || insn < 0 || insn >= (1 << 26)) + return; + for (int k = first_obj; k < ir->frame_obj_count; k++) + { + int32_t *o = &ir->frame_objs[4 * k]; + if (!(o[3] >> 4) && o[0] != keep) + o[3] |= (insn + 1) << 4; + } +} + +void tcc_ir_frame_note_sret_call(int call_id, int size) +{ + TCCIRState *ir = tcc_state ? tcc_state->ir : NULL; + if (!ir || call_id < 0) + return; + if (call_id >= ir->sret_calls_size) + { + int n = ir->sret_calls_size ? ir->sret_calls_size : 64; + while (n <= call_id) + n *= 2; + ir->sret_calls = tcc_realloc(ir->sret_calls, sizeof(int32_t) * n); + memset(ir->sret_calls + ir->sret_calls_size, 0, sizeof(int32_t) * (n - ir->sret_calls_size)); + ir->sret_calls_size = n; + } + ir->sret_calls[call_id] = size > 0 ? size : 1; +} + +static int frame_idx_cmp(const void *a, const void *b) +{ + const int32_t *x = a, *y = b; + return x[0] < y[0] ? -1 : x[0] > y[0]; +} + +int tcc_ir_frame_object_size_at(TCCIRState *ir, int start) +{ + if (!ir || ir->frame_relaid) + return -1; + for (int k = 0; k < ir->frame_obj_count; k++) + if (ir->frame_objs[4 * k] == start) + return ir->frame_objs[4 * k + 2]; + return -1; +} + +int tcc_ir_frame_object_at(TCCIRState *ir, int off, int *lo, int *hi) +{ + if (!ir || ir->frame_relaid || !ir->frame_obj_count) + return 0; + if (ir->frame_index_objs != ir->frame_obj_count) + { + /* Rebuild: extents sorted by start, overlapping ones merged. */ + int n = ir->frame_obj_count, m = 0; + int32_t *e = tcc_malloc(sizeof(int32_t) * 2 * n); + for (int k = 0; k < n; k++) + { + e[2 * k] = ir->frame_objs[4 * k]; + e[2 * k + 1] = ir->frame_objs[4 * k] + ir->frame_objs[4 * k + 1]; + } + qsort(e, n, sizeof(int32_t) * 2, frame_idx_cmp); + for (int k = 0; k < n; k++) + { + if (m && e[2 * k] < e[2 * m - 1]) + { + if (e[2 * k + 1] > e[2 * m - 1]) + e[2 * m - 1] = e[2 * k + 1]; + continue; + } + e[2 * m] = e[2 * k]; + e[2 * m + 1] = e[2 * k + 1]; + m++; + } + tcc_free(ir->frame_index); + ir->frame_index = e; + ir->frame_index_n = m; + ir->frame_index_objs = n; + } + int a = 0, b = ir->frame_index_n - 1; + while (a <= b) + { + int mid = (a + b) / 2; + if (off < ir->frame_index[2 * mid]) + b = mid - 1; + else if (off >= ir->frame_index[2 * mid + 1]) + a = mid + 1; + else + { + *lo = ir->frame_index[2 * mid]; + *hi = ir->frame_index[2 * mid + 1]; + return 1; + } + } + return 0; +} + +/* GCC's list (special_function_p), leading underscores ignored. */ +static int frame_name_returns_twice(const char *name) +{ + static const char *const names[] = {"setjmp", "sigsetjmp", "savectx", "vfork", + "qsetjmp", "getcontext", "setjmp_syscall", "secure_setjmp"}; + while (*name == '_') + name++; + for (unsigned k = 0; k < sizeof names / sizeof names[0]; k++) + if (!strcmp(name, names[k])) + return 1; + return 0; +} + +int tcc_ir_calls_returns_twice(TCCIRState *ir) +{ + for (int i = 0; i < ir->next_instruction_index; i++) + { + IRQuadCompact *q = &ir->compact_instructions[i]; + if (q->op == TCCIR_OP_SETJMP || q->op == TCCIR_OP_NL_SETJMP) + return 1; + if (q->op != TCCIR_OP_FUNCCALLVAL && q->op != TCCIR_OP_FUNCCALLVOID) + continue; + IROperand callee = tcc_ir_op_get_src1(ir, q); + if (irop_get_tag(callee) != IROP_TAG_SYMREF) + continue; + Sym *sym = irop_get_sym_ex(ir, callee); + if (sym && frame_name_returns_twice(get_tok_str(sym->v, NULL))) + return 1; + } + return 0; +} + +typedef struct +{ + int start, end; /* [start, end), start < end <= 0: the object's record */ + int size; /* the bytes the object itself occupies from start */ + uint8_t live; + uint8_t pinned; /* a gap: some untracked allocation, never dropped or moved */ + uint8_t kind; /* FRAME_OBJ_* */ + int scope_end; /* first instruction past the inlined body that owns it, 0 = none */ + uint8_t escaped; /* its address may be used anywhere after it is taken */ + uint8_t precise; /* lifetime from liveness (frame_precise_lifetimes) */ + int first[2], last[2]; /* lifetime in source order and in RPO (frame_rpo); + * first > last if none */ + int refs; /* instructions referencing it: how hot it is */ + int unit; + int delta; /* displacement applied to offsets inside the segment */ +} FrameSeg; + +static int frame_obj_cmp(const void *a, const void *b) +{ + const int32_t *x = a, *y = b; + return x[0] < y[0] ? -1 : x[0] > y[0]; +} + +/* Largest power of two (<= 8) dividing a frame offset. */ +static int frame_offset_align(int off) +{ + int a = 8; + while (a > 1 && (off & (a - 1))) + a >>= 1; + return a; +} + +static int frame_seg_find(const FrameSeg *seg, int n, int off) +{ + int lo = 0, hi = n - 1; + while (lo <= hi) + { + int mid = (lo + hi) / 2; + if (off < seg[mid].start) + hi = mid - 1; + else if (off >= seg[mid].end) + lo = mid + 1; + else + return mid; + } + return -1; +} + +/* An operand machine_op_from_ir resolves to a concrete frame slot (its case 3: + * no vreg, stack-relative). Immediates and symbols with a stray is_local are + * resolved before that case and never name the frame. */ +static int frame_operand_concrete(IROperand op, int *off) +{ + if (irop_is_none(op) || irop_get_vreg(op) >= 0 || op.is_param) + return 0; + int tag = irop_get_tag(op); + if (tag != IROP_TAG_STACKOFF && !(tag == IROP_TAG_VREG && (op.is_local || op.is_llocal))) + return 0; + *off = irop_get_stack_offset(op); + return 1; +} + +static int frame_operand_offset(IROperand op, int bottom, int *off) +{ + return frame_operand_concrete(op, off) && *off >= bottom && *off < 0; +} + +static int frame_function_eligible(TCCIRState *ir) +{ + if (tcc_state->do_debug || tcc_bounds_checking(tcc_state)) + return 0; + if (ir->is_variadic || ir->has_static_chain || ir->captured_count > 0 || tcc_state->nb_nested_funcs > 0) + return 0; + for (int i = 0; i < ir->next_instruction_index; i++) + { + switch (ir->compact_instructions[i].op) + { + case TCCIR_OP_VLA_ALLOC: + case TCCIR_OP_VLA_SP_SAVE: + case TCCIR_OP_VLA_SP_RESTORE: + case TCCIR_OP_ASM_INPUT: + case TCCIR_OP_INLINE_ASM: + case TCCIR_OP_ASM_OUTPUT: + case TCCIR_OP_SETJMP: + case TCCIR_OP_NL_SETJMP: + return 0; + default: + break; + } + } + return 1; +} + +/* ---- Step 1 fallback: drop dead segments, pack live runs towards 0 ---- */ + +/* Consecutive live segments form a run that moves as one block; a run holding + * a pinned gap stays where it is. Returns the new bottom. */ +static int frame_pack_runs(FrameSeg *seg, int nseg) +{ + int top = 0; + for (int s = nseg - 1; s >= 0;) + { + if (!seg[s].live) + { + s--; + continue; + } + int r_hi = s, r_lo = s, align = 1, pinned = 0; + for (; r_lo >= 0 && seg[r_lo].live; r_lo--) + { + int a = frame_offset_align(seg[r_lo].start); + if (a > align) + align = a; + pinned |= seg[r_lo].pinned; + } + r_lo++; + int room = top - seg[r_hi].end; + int delta = pinned ? 0 : room - room % align; + for (int k = r_lo; k <= r_hi; k++) + seg[k].delta = delta; + top = seg[r_lo].start + delta; + s = r_lo - 1; + } + return top; +} + +/* ---- Lifetime colouring ---- + * + * An object is live from its first to its last reference, where a reference is + * any operand naming it or holding a pointer into it. The range is kept in + * two orders of the instructions: source order, and reverse post-order of the + * control-flow graph (frame_rpo). + * Pointers are followed through vregs (flow-insensitively: a vreg that may + * point to two objects gives up on both). An object whose address escapes -- + * stored to memory, passed to a call, returned, into a vreg held in memory -- + * stays live to the end of the function. The address of a call temporary + * passed to its call, of any object passed as a struct-return buffer, or of + * any object passed to memcpy, strlen and the like is not an escape: the + * callee cannot keep it. Every range that intersects a loop (a back edge) is + * widened to the whole loop, until no range changes. + * + * Two objects whose ranges are disjoint in either order may then share bytes: + * in that order any execution path from a reference of the later one back to + * the earlier one crosses a backward edge whose span the later one's range + * already covers. Neither order wins everywhere: tinycc emits a switch's + * dispatch after its cases, so in source order its jumps into them look like + * loops; the Zig C backend's `if (c) { ...; goto out; }` inside a loop puts + * the rest of the function inside the loop's span in RPO. */ + +#define TAINT_NONE (-1) +#define TAINT_MULTI (-2) + +enum +{ + FOP_NONE, + FOP_MEM, /* a frame object's bytes */ + FOP_ADDR, /* a frame object's address */ + FOP_VAL, /* a vreg's value */ + FOP_DEREF, /* memory through a vreg's value */ + FOP_VAR_ADDR /* the address of a VAR/PARAM (lives in an allocator slot) */ +}; + +typedef struct +{ + TCCIRState *ir; + FrameSeg *seg; + int nseg, bottom, n; + int32_t *taint[4]; /* by vreg type: VAR, TEMP, PARAM */ + int taint_size[4]; + uint8_t *var_addr_taken[4]; + /* A vreg defined once, to a known address inside a frame object: the object + * (vseg, -1 if not known) and the offset in it (vrel). */ + int32_t *vseg[4], *vrel[4]; + int32_t *vdef_at[4]; /* the instruction defining such a pointer */ + uint8_t *vdefs[4]; + int changed, bad; + /* Whole-object copies `memmove(&D, &S, size)` at which S's lifetime ends + * and D's begins (frame_find_copies): the call instruction and the two + * segments. frame_colour gives them the same bytes, and the copy goes. */ + int *copy_at, *copy_pos, *copy_from, *copy_from_pos, *copy_dst, *copy_src, ncopy; +} FrameLive; + +static int frame_classify(FrameLive *fl, IROperand op, int *id) +{ + if (irop_is_none(op)) + return FOP_NONE; + int32_t vr = irop_get_vreg(op); + if (vr < 0) + { + int off, s; + if (!frame_operand_offset(op, fl->bottom, &off) || (s = frame_seg_find(fl->seg, fl->nseg, off)) < 0) + return FOP_NONE; + *id = s; + return (op.is_lval || op.is_llocal) ? FOP_MEM : FOP_ADDR; + } + int type = TCCIR_DECODE_VREG_TYPE(vr), pos = TCCIR_DECODE_VREG_POSITION(vr); + if (type < 1 || type > 3 || pos >= fl->taint_size[type]) + { + fl->bad = 1; + return FOP_NONE; + } + *id = vr; + int tag = irop_get_tag(op); + if (tag == IROP_TAG_VREG) + return op.is_lval ? FOP_DEREF : FOP_VAL; + if (tag == IROP_TAG_STACKOFF) + { + if (op.is_llocal) + return FOP_DEREF; + return op.is_lval ? FOP_VAL : FOP_VAR_ADDR; + } + return FOP_NONE; +} + +static int32_t *frame_taint_ref(FrameLive *fl, int32_t vr) +{ + return &fl->taint[TCCIR_DECODE_VREG_TYPE(vr)][TCCIR_DECODE_VREG_POSITION(vr)]; +} + +static void frame_escape(FrameLive *fl, int t) +{ + if (t >= 0) + fl->seg[t].escaped = 1; +} + +static int frame_taint_merge(FrameLive *fl, int a, int b) +{ + if (b == TAINT_NONE || a == b) + return a; + if (a == TAINT_NONE) + return b; + frame_escape(fl, a); + frame_escape(fl, b); + return TAINT_MULTI; +} + +static void frame_taint_join(FrameLive *fl, int32_t vr, int t) +{ + int32_t *p = frame_taint_ref(fl, vr); + int m = frame_taint_merge(fl, *p, t); + if (m != *p) + { + *p = m; + fl->changed = 1; + } +} + +/* The object a value operand points into, if any. */ +static int frame_op_taint(FrameLive *fl, IROperand op) +{ + int id, k = frame_classify(fl, op, &id); + if (k == FOP_ADDR) + return id; + if (k == FOP_VAL) + return *frame_taint_ref(fl, id); + return TAINT_NONE; +} + +/* A reference to segment `s` by instruction `i` (at RPO position `p`). */ +static void frame_use(FrameLive *fl, int s, int i, int p) +{ + if (s < 0) + return; + FrameSeg *g = &fl->seg[s]; + if (i < g->first[0]) + g->first[0] = i; + if (i > g->last[0]) + g->last[0] = i; + if (p < g->first[1]) + g->first[1] = p; + if (p > g->last[1]) + g->last[1] = p; +} + +/* Ops whose result is computed from their sources' values: a pointer operand + * may come out as (derived from) the result. */ +static int frame_op_passes_value(int op) +{ + switch (op) + { + case TCCIR_OP_ADD: + case TCCIR_OP_ADC_USE: + case TCCIR_OP_ADC_GEN: + case TCCIR_OP_SUB: + case TCCIR_OP_SUBC_USE: + case TCCIR_OP_SUBC_GEN: + case TCCIR_OP_MUL: + case TCCIR_OP_MLA: + case TCCIR_OP_UMULL: + case TCCIR_OP_UMAAL: + case TCCIR_OP_SMULL: + case TCCIR_OP_DIV: + case TCCIR_OP_UDIV: + case TCCIR_OP_PDIV: + case TCCIR_OP_UMOD: + case TCCIR_OP_IMOD: + case TCCIR_OP_AND: + case TCCIR_OP_OR: + case TCCIR_OP_XOR: + case TCCIR_OP_SHL: + case TCCIR_OP_SAR: + case TCCIR_OP_SHR: + case TCCIR_OP_ROR: + case TCCIR_OP_ASSIGN: + case TCCIR_OP_LEA: + case TCCIR_OP_SELECT: + case TCCIR_OP_UBFX: + case TCCIR_OP_SBFX: + case TCCIR_OP_BFI: + case TCCIR_OP_ZEXT: + case TCCIR_OP_PACK64: + case TCCIR_OP_SETIF: + case TCCIR_OP_BOOL_OR: + case TCCIR_OP_BOOL_AND: + case TCCIR_OP_CLZ: + case TCCIR_OP_RBIT: + case TCCIR_OP_REV: + case TCCIR_OP_REV16: + return 1; + default: + return 0; + } +} + +/* Ops that read their sources without letting a pointer among them escape. */ +static int frame_op_keeps_pointers(int op) +{ + if (frame_op_passes_value(op)) + return 1; + switch (op) + { + case TCCIR_OP_CMP: + case TCCIR_OP_TEST_ZERO: + case TCCIR_OP_JUMP: + case TCCIR_OP_JUMPIF: + case TCCIR_OP_LOAD: + case TCCIR_OP_LOAD_INDEXED: + case TCCIR_OP_LOAD_POSTINC: + case TCCIR_OP_BLOCK_COPY: + case TCCIR_OP_SWITCH_TABLE: + case TCCIR_OP_SWITCH_LOAD: + case TCCIR_OP_PREFETCH: + case TCCIR_OP_NOP: + return 1; + default: + return 0; + } +} + +/* The fourth operand slot some ops keep at operand_base + 3. */ +static int frame_op_has_slot3(int op) +{ + return tcc_ir_op_is_mac(op) || op == TCCIR_OP_LOAD_INDEXED || op == TCCIR_OP_STORE_INDEXED || op == TCCIR_OP_SELECT; +} + +static int frame_call_id(TCCIRState *ir, IROperand enc) +{ + return TCCIR_DECODE_CALL_ID((uint32_t)irop_get_imm64_ex(ir, enc)); +} + +/* C library functions that keep no pointer argument past the call: 1, or 2 + * when the result may point into the first argument (memcpy returns it, + * strchr returns a pointer inside it). */ +static int frame_callee_nocapture(TCCIRState *ir, IRQuadCompact *call) +{ + static const char *const keeps_none[] = {"memcmp", "bcmp", "strcmp", "strncmp", + "strlen", "strnlen", "strspn", "strcspn", + "__aeabi_memcpy", "__aeabi_memcpy4", "__aeabi_memcpy8", + "__aeabi_memmove", "__aeabi_memmove4", "__aeabi_memmove8", + "__aeabi_memset", "__aeabi_memset4", "__aeabi_memset8", + "__aeabi_memclr", "__aeabi_memclr4", "__aeabi_memclr8"}; + static const char *const returns_arg0[] = {"memcpy", "memmove", "memset", "strcpy", "strncpy", "strcat", + "strncat", "strchr", "strrchr", "memchr", "strstr", "strpbrk"}; + IROperand callee = tcc_ir_op_get_src1(ir, call); + if (irop_get_tag(callee) != IROP_TAG_SYMREF) + return 0; + Sym *sym = irop_get_sym_ex(ir, callee); + const char *name = sym ? get_tok_str(sym->v, NULL) : NULL; + if (!name) + return 0; + for (unsigned k = 0; k < sizeof keeps_none / sizeof keeps_none[0]; k++) + if (!strcmp(name, keeps_none[k])) + return 1; + for (unsigned k = 0; k < sizeof returns_arg0 / sizeof returns_arg0[0]; k++) + if (!strcmp(name, returns_arg0[k])) + return 2; + return 0; +} + +/* The basic blocks frame_rpo found, kept for frame_precise_lifetimes. */ +typedef struct +{ + int nb; + int *bstart; /* nb + 1: first instruction of each block, then n */ + int *block_of; /* n: the block of each instruction */ + int *succ_at; /* nb + 1: successors of b are succ[succ_at[b] .. succ_at[b+1]) */ + int *succ; + int *post; /* the reachable blocks in DFS post-order */ + int npost; + uint8_t *seen; /* nb: reachable from the entry */ +} FrameCFG; + +static void frame_cfg_free(FrameCFG *cfg) +{ + tcc_free(cfg->bstart); + tcc_free(cfg->block_of); + tcc_free(cfg->succ_at); + tcc_free(cfg->succ); + tcc_free(cfg->post); + tcc_free(cfg->seen); + memset(cfg, 0, sizeof *cfg); +} + +/* For a C library call that overwrites what its argument 0 points to -- the + * memcpy, memmove and memset families -- the index of the argument holding + * the byte count; else -1. */ +static int frame_callee_arg0_size_index(TCCIRState *ir, IRQuadCompact *call) +{ + static const char *const size2[] = {"memcpy", "memmove", "memset", + "__aeabi_memcpy", "__aeabi_memcpy4", "__aeabi_memcpy8", + "__aeabi_memmove", "__aeabi_memmove4", "__aeabi_memmove8"}; + static const char *const size1[] = {"__aeabi_memset", "__aeabi_memset4", "__aeabi_memset8", + "__aeabi_memclr", "__aeabi_memclr4", "__aeabi_memclr8"}; + IROperand callee = tcc_ir_op_get_src1(ir, call); + if (irop_get_tag(callee) != IROP_TAG_SYMREF) + return -1; + Sym *sym = irop_get_sym_ex(ir, callee); + const char *name = sym ? get_tok_str(sym->v, NULL) : NULL; + if (!name) + return -1; + for (unsigned k = 0; k < sizeof size2 / sizeof size2[0]; k++) + if (!strcmp(name, size2[k])) + return 2; + for (unsigned k = 0; k < sizeof size1 / sizeof size1[0]; k++) + if (!strcmp(name, size1[k])) + return 1; + return -1; +} + +/* Positions of the instructions in reverse post-order of the control-flow + * graph, and its back edges as (target, source) position pairs. In RPO every + * edge that is not a back edge goes forward, so a path can only return to an + * earlier position across a back edge -- which the lifetime widening needs. + * Source order would not do: a switch's dispatch follows its cases, and its + * forward jumps into them would all look like loops. Returns 0 on a switch the + * graph cannot follow. Unreachable blocks go last, in source order. */ +static int frame_rpo(TCCIRState *ir, int n, int *pos, int **be_out, int *nbe_out, int **sbe_out, int *nsbe_out, + FrameCFG *cfg) +{ + uint8_t *leader = tcc_mallocz(n + 1); + int *block_of = tcc_malloc(sizeof(int) * n); + int ok = 0, nb = 0; + int *bstart = NULL, *succ = NULL, *succ_at = NULL, *post = NULL, *stack = NULL, *next_succ = NULL; + uint8_t *seen = NULL; + int *be = NULL, nbe = 0, cap = 0; + int *sbe = NULL, nsbe = 0, scap2 = 0; + + leader[0] = 1; + for (int i = 0; i < n; i++) + { + IRQuadCompact *q = &ir->compact_instructions[i]; + if (q->op == TCCIR_OP_JUMP || q->op == TCCIR_OP_JUMPIF) + { + /* A target past the last instruction is the epilogue: an exit. */ + int t = (int)irop_get_imm64_ex(ir, tcc_ir_op_get_dest(ir, q)); + if (t >= 0 && t < n) + leader[t] = 1; + } + else if (q->op == TCCIR_OP_SWITCH_TABLE) + { + int tid = (int)irop_get_imm64_ex(ir, tcc_ir_op_get_src2(ir, q)); + if (tid < 0 || tid >= ir->num_switch_tables) + goto done; + TCCIRSwitchTable *tab = &ir->switch_tables[tid]; + for (int k = -1; k < tab->num_entries; k++) + { + int t = k < 0 ? tab->default_target : tab->targets[k]; + if (t >= 0 && t < n) + leader[t] = 1; + } + } + if (q->op == TCCIR_OP_JUMP || q->op == TCCIR_OP_JUMPIF || q->op == TCCIR_OP_SWITCH_TABLE || + q->op == TCCIR_OP_RETURNVALUE || q->op == TCCIR_OP_RETURNVOID || q->op == TCCIR_OP_IJUMP) + leader[i + 1] = 1; + } + for (int i = 0; i < n; i++) + { + if (leader[i]) + nb++; + block_of[i] = nb - 1; + } + bstart = tcc_malloc(sizeof(int) * (nb + 1)); + for (int i = 0, b = 0; i < n; i++) + if (leader[i]) + bstart[b++] = i; + bstart[nb] = n; + + /* Successor lists. */ + succ_at = tcc_malloc(sizeof(int) * (nb + 1)); + int nsucc = 0, scap = nb * 2 + 4; + succ = tcc_malloc(sizeof(int) * scap); + for (int b = 0; b < nb; b++) + { + succ_at[b] = nsucc; + IRQuadCompact *q = &ir->compact_instructions[bstart[b + 1] - 1]; + int fall = q->op != TCCIR_OP_JUMP && q->op != TCCIR_OP_SWITCH_TABLE && q->op != TCCIR_OP_RETURNVALUE && + q->op != TCCIR_OP_RETURNVOID && q->op != TCCIR_OP_IJUMP; + int extra = q->op == TCCIR_OP_SWITCH_TABLE ? ir->switch_tables[irop_get_imm64_ex(ir, tcc_ir_op_get_src2(ir, q))].num_entries + 1 : 1; + if (nsucc + extra + 1 > scap) + { + scap = (nsucc + extra + 1) * 2; + succ = tcc_realloc(succ, sizeof(int) * scap); + } + /* Visit order, which decides the layout: jumps forward in the source + * first, then the fallthrough, then jumps back. The first visited comes + * last in RPO, so a loop's body follows its header and precedes its exits + * whether the loop tests at the top or the bottom, and a switch's cases + * follow its dispatch however the dispatch was laid out. */ + for (int pass = 0; pass < 3; pass++) + { + if (pass == 1) + { + if (fall && b + 1 < nb) + succ[nsucc++] = b + 1; + continue; + } + int back = pass == 2; + if (q->op == TCCIR_OP_JUMP || q->op == TCCIR_OP_JUMPIF) + { + int ti = (int)irop_get_imm64_ex(ir, tcc_ir_op_get_dest(ir, q)); + if (ti >= 0 && ti < n && (block_of[ti] <= b) == back) + succ[nsucc++] = block_of[ti]; + } + else if (q->op == TCCIR_OP_SWITCH_TABLE) + { + TCCIRSwitchTable *tab = &ir->switch_tables[irop_get_imm64_ex(ir, tcc_ir_op_get_src2(ir, q))]; + for (int k = -1; k < tab->num_entries; k++) + { + int t = k < 0 ? tab->default_target : tab->targets[k]; + if (t >= 0 && t < n && (block_of[t] <= b) == back) + succ[nsucc++] = block_of[t]; + } + } + } + } + succ_at[nb] = nsucc; + + /* Iterative DFS post-order from the entry. */ + post = tcc_malloc(sizeof(int) * nb); + stack = tcc_malloc(sizeof(int) * nb); + next_succ = tcc_malloc(sizeof(int) * nb); + seen = tcc_mallocz(nb); + int npost = 0, sp = 0; + stack[sp++] = 0; + seen[0] = 1; + next_succ[0] = succ_at[0]; + while (sp) + { + int b = stack[sp - 1]; + if (next_succ[b] < succ_at[b + 1]) + { + int v = succ[next_succ[b]++]; + if (!seen[v]) + { + seen[v] = 1; + next_succ[v] = succ_at[v]; + stack[sp++] = v; + } + } + else + { + post[npost++] = b; + sp--; + } + } + + int p = 0; + for (int k = npost - 1; k >= 0; k--) + for (int i = bstart[post[k]]; i < bstart[post[k] + 1]; i++) + pos[i] = p++; + for (int b = 0; b < nb; b++) + if (!seen[b]) + for (int i = bstart[b]; i < bstart[b + 1]; i++) + pos[i] = p++; + + for (int b = 0; b < nb; b++) + { + if (!seen[b]) + continue; + int src = pos[bstart[b + 1] - 1], src_i = bstart[b + 1] - 1; + for (int k = succ_at[b]; k < succ_at[b + 1]; k++) + { + int dst = pos[bstart[succ[k]]], dst_i = bstart[succ[k]]; + if (dst <= src) + { + if (nbe == cap) + { + cap = cap ? cap * 2 : 16; + be = tcc_realloc(be, sizeof(int) * 2 * cap); + } + be[2 * nbe] = dst; + be[2 * nbe + 1] = src; + nbe++; + } + if (dst_i <= src_i) + { + if (nsbe == scap2) + { + scap2 = scap2 ? scap2 * 2 : 16; + sbe = tcc_realloc(sbe, sizeof(int) * 2 * scap2); + } + sbe[2 * nsbe] = dst_i; + sbe[2 * nsbe + 1] = src_i; + nsbe++; + } + } + } + ok = 1; + cfg->nb = nb; + cfg->bstart = bstart; + cfg->block_of = block_of; + cfg->succ_at = succ_at; + cfg->succ = succ; + cfg->post = post; + cfg->npost = npost; + cfg->seen = seen; + bstart = block_of = succ_at = succ = post = NULL; + seen = NULL; + +done: + tcc_free(leader); + tcc_free(block_of); + tcc_free(bstart); + tcc_free(succ); + tcc_free(succ_at); + tcc_free(post); + tcc_free(stack); + tcc_free(next_succ); + tcc_free(seen); + if (!ok) + { + tcc_free(be); + tcc_free(sbe); + be = sbe = NULL, nbe = nsbe = 0; + } + *be_out = be; + *nbe_out = nbe; + *sbe_out = sbe; + *nsbe_out = nsbe; + return ok; +} + +/* ---- Liveness with kills (frame_precise_lifetimes) ---- + * + * The ranges above take an object to be live from its first to its last + * reference and then widen them over every loop they touch: a value may be + * carried around the back edge. For an object whose uses inside the loop + * are all preceded by a full write, nothing is carried, and the widening + * makes every such object -- the locals of each case of a loop around a + * switch, of each called-once body inlined into one -- live across the + * whole loop, so none of them share bytes. + * + * Each reference is recorded as an event: a USE may read the object, a REF + * only takes its address (the pointer's later uses are events of their own), + * a WRITE stores bytes [off, off + width) of it, and a KILL overwrites all of + * it. Per block, the first USE before the object is fully overwritten makes + * it live on entry; writes that cover it first -- one store, a struct result, + * a block copy, or several stores between them -- kill it. Backward liveness + * over the CFG then gives the blocks it is live through, and its lifetime is + * the hull of those blocks and its references, in each order: two objects + * whose hulls are disjoint in an order are never live at the same point. No + * widening is needed, the liveness already follows the back edges. */ +enum +{ + FEV_USE, + FEV_REF, + FEV_WRITE, + FEV_KILL +}; + +typedef struct +{ + int i, s, kind, off, width; +} FrameEv; + +typedef struct +{ + FrameEv *v; + int n, cap; +} FrameEvents; + +static void frame_ev_add(FrameEvents *ev, int i, int s, int kind, int off, int width) +{ + if (s < 0 || i < 0) + return; + if (ev->n == ev->cap) + { + ev->cap = ev->cap ? ev->cap * 2 : 256; + ev->v = tcc_realloc(ev->v, sizeof(FrameEv) * ev->cap); + } + ev->v[ev->n++] = (FrameEv){i, s, kind, off, width}; +} + +/* Bytes an access of this operand's type moves, or 0 if unknown. */ +static int frame_btype_width(IROperand op) +{ + switch (irop_get_btype(op)) + { + case IROP_BTYPE_INT8: + return 1; + case IROP_BTYPE_INT16: + return 2; + case IROP_BTYPE_INT32: + case IROP_BTYPE_FLOAT32: + return 4; + case IROP_BTYPE_INT64: + case IROP_BTYPE_FLOAT64: + return 8; + default: + return 0; + } +} + +/* The event for operand `opnd` (classified `kind`, naming segment `t`) of + * instruction `i`; role 0 is the destination. `call` is the call reading a + * FUNCPARAMVAL, and `writes` the bytes that call stores through it (a struct + * result, a memcpy/memset destination) or 0. A USE records the bytes it + * reads when known, width 0 when not. */ +static void frame_ev_operand(FrameLive *fl, FrameEvents *ev, int i, int op, int role, int kind, IROperand opnd, int t, + int call, int writes) +{ + /* Where in object t the operand points: a direct frame operand, or a + * pointer vreg with one definition to a known address in it. */ + int rel = -1, off; + if ((kind == FOP_MEM || kind == FOP_ADDR) && frame_operand_offset(opnd, fl->bottom, &off)) + rel = off - fl->seg[t].start; + else if (kind == FOP_VAL || kind == FOP_DEREF) + { + int32_t vr = irop_get_vreg(opnd); + int ty = TCCIR_DECODE_VREG_TYPE(vr), ps = TCCIR_DECODE_VREG_POSITION(vr); + if (vr >= 0 && ty >= 1 && ty <= 3 && ps < fl->taint_size[ty] && fl->vseg[ty][ps] == t) + rel = fl->vrel[ty][ps]; + } + if (kind == FOP_MEM || kind == FOP_DEREF) + { + /* The bytes at rel: directly, or through the pointer. */ + int w = op == TCCIR_OP_VLA_SP_SAVE ? 4 : frame_btype_width(opnd); + if (op == TCCIR_OP_BLOCK_COPY) + { + IROperand size = tcc_ir_op_get_src2(fl->ir, &fl->ir->compact_instructions[i]); + w = irop_is_immediate(size) ? (int)irop_get_imm64_ex(fl->ir, size) : 0; + } + if (rel < 0 || w <= 0) + rel = 0, w = 0; + if (role == 0 && (op == TCCIR_OP_STORE || op == TCCIR_OP_ASSIGN || op == TCCIR_OP_VLA_SP_SAVE || + op == TCCIR_OP_BLOCK_COPY)) + frame_ev_add(ev, i, t, w > 0 ? FEV_WRITE : FEV_USE, rel, w); + else + frame_ev_add(ev, i, t, FEV_USE, rel, w); + return; + } + if (kind == FOP_ADDR || kind == FOP_VAL) + { + if (op == TCCIR_OP_FUNCPARAMVAL) + { + frame_ev_add(ev, i, t, FEV_REF, 0, 0); + if (writes > 0 && rel >= 0) + frame_ev_add(ev, call, t, FEV_WRITE, rel, writes); + else + frame_ev_add(ev, call, t, FEV_USE, 0, 0); + return; + } + if (role == 0 && op == TCCIR_OP_BLOCK_COPY) + { + frame_ev_operand(fl, ev, i, op, role, FOP_MEM, opnd, t, call, writes); + return; + } + if (frame_op_passes_value(op) || op == TCCIR_OP_CMP || op == TCCIR_OP_TEST_ZERO) + { + frame_ev_add(ev, i, t, FEV_REF, 0, 0); + return; + } + } + frame_ev_add(ev, i, t, FEV_USE, 0, 0); +} + +static int frame_ev_cmp(const void *a, const void *b) +{ + const FrameEv *x = a, *y = b; + if (x->i != y->i) + return x->i < y->i ? -1 : 1; + /* Within one instruction the sources are read before the result is + * written: a use first. */ + int ux = x->kind == FEV_USE, uy = y->kind == FEV_USE; + return uy - ux; +} + +/* Cap on each liveness bit array (words): beyond it the widened ranges + * stand, rather than spend the memory. */ +#define FRAME_LIVE_MAX_WORDS (1 << 21) + +static void frame_precise_lifetimes(FrameLive *fl, FrameCFG *cfg, const int *pos, FrameEvents *ev, const int *be, + int nbe, const int *sbe, int nsbe) +{ + const int nseg = fl->nseg, nb = cfg->nb; + if (nb <= 0 || ev->n == 0) + return; + + /* Candidates: objects the widening would grow -- live, never escaping + * (a pointer to an escaped one may read it anywhere). */ + int *cand = tcc_malloc(sizeof(int) * (nseg > 0 ? nseg : 1)); + int K = 0; + for (int s = 0; s < nseg; s++) + { + FrameSeg *g = &fl->seg[s]; + cand[s] = -1; + if (!g->live || g->escaped || g->pinned || g->first[0] > g->last[0]) + continue; + int loopy = 0; + for (int o = 0; o < 2 && !loopy; o++) + { + const int *e = o ? be : sbe; + const int ne = o ? nbe : nsbe; + for (int k = 0; k < ne && !loopy; k++) + loopy = g->first[o] <= e[2 * k + 1] && g->last[o] >= e[2 * k]; + } + if (loopy) + cand[s] = K++; + } + const int W = (K + 31) / 32; + if (K == 0 || (int64_t)nb * W > FRAME_LIVE_MAX_WORDS) + { + tcc_free(cand); + return; + } + + qsort(ev->v, ev->n, sizeof(FrameEv), frame_ev_cmp); + uint32_t *gen = tcc_mallocz(sizeof(uint32_t) * nb * W); + uint32_t *kill = tcc_mallocz(sizeof(uint32_t) * nb * W); + uint32_t *in = tcc_mallocz(sizeof(uint32_t) * nb * W); + uint32_t *out = tcc_mallocz(sizeof(uint32_t) * nb * W); + uint32_t *acc = tcc_mallocz(sizeof(uint32_t) * W); + uint8_t *bad = tcc_mallocz(K), *killed = tcc_mallocz(K), *closed = tcc_mallocz(K); + int *stamp = tcc_malloc(sizeof(int) * K); + uint64_t *cover = tcc_malloc(sizeof(uint64_t) * K); + /* The bytes of each object ever read (up to 64 bytes; beyond, all): only + * those need covering for a kill -- a byte nothing reads carries nothing. */ + uint64_t *rmask = tcc_mallocz(sizeof(uint64_t) * K); + for (int k = 0; k < K; k++) + stamp[k] = -1; + for (int x = 0; x < ev->n; x++) + { + const FrameEv *e = &ev->v[x]; + int k = cand[e->s]; + if (k < 0 || e->kind != FEV_USE) + continue; + const int size = fl->seg[e->s].size; + if (size > 64 || e->width <= 0 || e->off < 0 || e->off >= size) + rmask[k] = ~0ull; + else + { + int hi = e->off + e->width < size ? e->off + e->width : size; + rmask[k] |= (hi - e->off >= 64 ? ~0ull : ((1ull << (hi - e->off)) - 1)) << e->off; + } + } + + for (int x = 0; x < ev->n; x++) + { + const FrameEv *e = &ev->v[x]; + int k = cand[e->s]; + if (k < 0) + continue; + int b = cfg->block_of[e->i]; + if (!cfg->seen[b]) + { + bad[k] = 1; /* in a block the CFG does not reach: leave it widened */ + continue; + } + if (stamp[k] != b) + { + stamp[k] = b; + closed[k] = 0; + cover[k] = 0; + } + if (closed[k]) + continue; + const int size = fl->seg[e->s].size; + uint32_t bit = 1u << (k & 31); + switch (e->kind) + { + case FEV_USE: + gen[b * W + k / 32] |= bit; + closed[k] = 1; + break; + case FEV_WRITE: + if (e->off <= 0 && e->off + e->width >= size) + goto kill_it; + if (size <= 64 && e->off >= 0 && e->off < size) + { + int hi = e->off + e->width < size ? e->off + e->width : size; + uint64_t m = (hi - e->off >= 64 ? ~0ull : ((1ull << (hi - e->off)) - 1)) << e->off; + uint64_t full = (size >= 64 ? ~0ull : (1ull << size) - 1) & rmask[k]; + cover[k] |= m; + if ((cover[k] & full) == full) + goto kill_it; + } + break; + case FEV_KILL: + kill_it: + kill[b * W + k / 32] |= bit; + killed[k] = 1; + closed[k] = 1; + break; + default: + break; + } + } + + /* Backward liveness, blocks in post-order, to a fixpoint. */ + int changed; + do + { + changed = 0; + for (int x = 0; x < cfg->npost; x++) + { + int b = cfg->post[x]; + memset(acc, 0, sizeof(uint32_t) * W); + for (int j = cfg->succ_at[b]; j < cfg->succ_at[b + 1]; j++) + { + const uint32_t *si = &in[cfg->succ[j] * W]; + for (int w = 0; w < W; w++) + acc[w] |= si[w]; + } + uint32_t *bo = &out[b * W], *bi = &in[b * W]; + const uint32_t *bg = &gen[b * W], *bk = &kill[b * W]; + for (int w = 0; w < W; w++) + { + uint32_t ni = bg[w] | (acc[w] & ~bk[w]); + if (ni != bi[w]) + bi[w] = ni, changed = 1; + bo[w] = acc[w]; + } + } + } while (changed); + + /* Lifetime = hull of the references and of the blocks live through. */ + for (int s = 0; s < nseg; s++) + { + int k = cand[s]; + if (k < 0 || bad[k] || !killed[k]) + continue; + FrameSeg *g = &fl->seg[s]; + uint32_t bit = 1u << (k & 31); + for (int b = 0; b < nb; b++) + { + if (!cfg->seen[b] || !((in[b * W + k / 32] | out[b * W + k / 32]) & bit)) + continue; + int lo = cfg->bstart[b], hi = cfg->bstart[b + 1] - 1; + if (lo < g->first[0]) + g->first[0] = lo; + if (hi > g->last[0]) + g->last[0] = hi; + if (pos[lo] < g->first[1]) + g->first[1] = pos[lo]; + if (pos[hi] > g->last[1]) + g->last[1] = pos[hi]; + } + g->precise = 1; + } + + tcc_free(cand); + tcc_free(gen); + tcc_free(kill); + tcc_free(in); + tcc_free(out); + tcc_free(acc); + tcc_free(bad); + tcc_free(killed); + tcc_free(closed); + tcc_free(stamp); + tcc_free(cover); + tcc_free(rmask); +} + +/* Lifetimes of the live segments. Returns 0 if they cannot be computed. */ +/* The block-copy helpers whose arguments are (dst, src, n). */ +static int frame_is_copy_helper(TCCIRState *ir, IRQuadCompact *call) +{ + static const char *const names[] = {"memcpy", "memmove", "__aeabi_memcpy", + "__aeabi_memcpy4", "__aeabi_memcpy8", "__aeabi_memmove", + "__aeabi_memmove4", "__aeabi_memmove8"}; + IROperand callee = tcc_ir_op_get_src1(ir, call); + if (irop_get_tag(callee) != IROP_TAG_SYMREF) + return 0; + Sym *sym = irop_get_sym_ex(ir, callee); + const char *name = sym ? get_tok_str(sym->v, NULL) : NULL; + for (unsigned k = 0; name && k < sizeof names / sizeof names[0]; k++) + if (!strcmp(name, names[k])) + return 1; + return 0; +} + +/* Record the copies that could share their objects' bytes: a whole-object + * block copy between two frame objects, where the call is the source's last + * use and the destination's first. frame_use attributes an argument to its + * call, so a source passed by its address, or by a pointer defined once to + * its start, ends there. The destination's first use is the call too when it + * is passed by its address; when it is passed by such a pointer (the usual + * shape), it is the instruction that defines the pointer, which must then be + * in the call's block with nothing between them touching the destination. + * zig's C copies structs from temporary to temporary this way, one call each + * (`t5 = t4;`). */ +static int frame_ptr_to_start(FrameLive *fl, IROperand a, int kind, int id, int *def) +{ + int off; + *def = -1; + if (kind == FOP_ADDR && frame_operand_offset(a, fl->bottom, &off) && off == fl->seg[id].start) + return id; + if (kind == FOP_VAL && irop_get_tag(a) == IROP_TAG_VREG) + { + int ty = TCCIR_DECODE_VREG_TYPE(id), ps = TCCIR_DECODE_VREG_POSITION(id); + if (fl->vseg[ty][ps] >= 0 && fl->vrel[ty][ps] == 0) + { + *def = fl->vdef_at[ty][ps]; + return fl->vseg[ty][ps]; + } + } + return -1; +} + +/* Whether instruction j touches segment d other than as an argument of call + * i: by its address, its bytes, or a pointer frame_lifetimes traced into it. */ +static int frame_insn_touches(FrameLive *fl, int j, int d, int call, const int *param_call) +{ + TCCIRState *ir = fl->ir; + IRQuadCompact *q = &ir->compact_instructions[j]; + if (q->op == TCCIR_OP_NOP || (q->op == TCCIR_OP_FUNCPARAMVAL && param_call[j] == call)) + return 0; + int nops = irop_config[q->op].has_dest + irop_config[q->op].has_src1 + irop_config[q->op].has_src2 + + frame_op_has_slot3(q->op); + for (int k = 0; k < nops; k++) + { + uint32_t pi = q->operand_base + k; + if (pi >= (uint32_t)ir->iroperand_pool_count) + continue; + int id, kind = frame_classify(fl, ir->iroperand_pool[pi], &id); + if ((kind == FOP_ADDR || kind == FOP_MEM) && id == d) + return 1; + if ((kind == FOP_VAL || kind == FOP_DEREF) && irop_get_vreg(ir->iroperand_pool[pi]) >= 0) + { + int ty = TCCIR_DECODE_VREG_TYPE(id), ps = TCCIR_DECODE_VREG_POSITION(id); + if (fl->vseg[ty][ps] == d || fl->taint[ty][ps] == d) + return 1; + } + } + return 0; +} + +static void frame_find_copies(FrameLive *fl, const int *call_params, const int *param_next, const int *param_call, + const int *pos, const FrameCFG *cfg) +{ + TCCIRState *ir = fl->ir; + for (int i = 0; i < fl->n; i++) + { + IRQuadCompact *q = &ir->compact_instructions[i]; + if (q->op != TCCIR_OP_FUNCCALLVOID || call_params[i] < 0 || !frame_is_copy_helper(ir, q)) + continue; + int d = -1, s = -1, ddef = -1, sdef, id; + int64_t len = -1; + for (int p = call_params[i]; p >= 0; p = param_next[p]) + { + IRQuadCompact *pq = &ir->compact_instructions[p]; + IROperand a = tcc_ir_op_get_src1(ir, pq); + int idx = TCCIR_DECODE_PARAM_IDX((uint32_t)irop_get_imm64_ex(ir, tcc_ir_op_get_src2(ir, pq))); + int kind = frame_classify(fl, a, &id); + if (idx == 0) + d = frame_ptr_to_start(fl, a, kind, id, &ddef); + else if (idx == 1) + s = frame_ptr_to_start(fl, a, kind, id, &sdef); + else if (idx == 2 && irop_is_immediate(a)) + len = irop_get_imm64_ex(ir, a); + } + if (d < 0 || s < 0 || d == s) + continue; + FrameSeg *gd = &fl->seg[d], *gs = &fl->seg[s]; + /* Only [0, len) is copied, and it becomes the same bytes of both: the + * destination may be bigger (a value stored into the head of a larger + * object) or the source (the head of a larger object taken out). */ + if (gd->pinned || gs->pinned || gd->escaped || gs->escaped || len <= 0 || len > gd->size || len > gs->size) + continue; + if (gs->last[0] != i || gs->last[1] != pos[i]) + continue; + int from = i; + if (ddef >= 0) + { + /* The pointer's definition opens the destination's lifetime. */ + if (ddef > i || cfg->block_of[ddef] != cfg->block_of[i]) + continue; + int clear = 1; + for (int j = ddef + 1; j < i && clear; j++) + clear = !frame_insn_touches(fl, j, d, i, param_call); + if (!clear) + continue; + from = ddef; + } + if (gd->first[0] != from || gd->first[1] != pos[from]) + continue; + if (fl->ncopy % 64 == 0) + { + int cap = fl->ncopy + 64; + fl->copy_at = tcc_realloc(fl->copy_at, sizeof(int) * cap); + fl->copy_pos = tcc_realloc(fl->copy_pos, sizeof(int) * cap); + fl->copy_from = tcc_realloc(fl->copy_from, sizeof(int) * cap); + fl->copy_from_pos = tcc_realloc(fl->copy_from_pos, sizeof(int) * cap); + fl->copy_dst = tcc_realloc(fl->copy_dst, sizeof(int) * cap); + fl->copy_src = tcc_realloc(fl->copy_src, sizeof(int) * cap); + } + fl->copy_at[fl->ncopy] = i; + fl->copy_pos[fl->ncopy] = pos[i]; + fl->copy_from[fl->ncopy] = from; + fl->copy_from_pos[fl->ncopy] = pos[from]; + fl->copy_dst[fl->ncopy] = d; + fl->copy_src[fl->ncopy] = s; + fl->ncopy++; + } +} + +static int frame_lifetimes(FrameLive *fl) +{ + TCCIRState *ir = fl->ir; + const int n = fl->n; + const int ncall = ir->next_call_id + 1; + int ok = 0; + + static const int types[3] = {TCCIR_VREG_TYPE_VAR, TCCIR_VREG_TYPE_TEMP, TCCIR_VREG_TYPE_PARAM}; + fl->taint_size[TCCIR_VREG_TYPE_VAR] = ir->variables_live_intervals_size; + fl->taint_size[TCCIR_VREG_TYPE_TEMP] = ir->temporary_variables_live_intervals_size; + fl->taint_size[TCCIR_VREG_TYPE_PARAM] = ir->parameters_live_intervals_size; + for (int k = 0; k < 3; k++) + { + int t = types[k], sz = fl->taint_size[t] > 0 ? fl->taint_size[t] : 1; + fl->taint[t] = tcc_malloc(sizeof(int32_t) * sz); + for (int j = 0; j < sz; j++) + fl->taint[t][j] = TAINT_NONE; + fl->var_addr_taken[t] = tcc_mallocz(sz); + } + + /* Each FUNCPARAM's call (the next call with its id), and its call's params. */ + int *param_call = tcc_malloc(sizeof(int) * (n > 0 ? n : 1)); + int *call_at = tcc_malloc(sizeof(int) * ncall); + int *param_next = tcc_malloc(sizeof(int) * (n > 0 ? n : 1)); + int *call_params = tcc_malloc(sizeof(int) * (n > 0 ? n : 1)); + uint8_t *nocapture = tcc_mallocz(n > 0 ? n : 1); + int *pos = tcc_malloc(sizeof(int) * (n > 0 ? n : 1)); + int *be = NULL, nbe = 0, *sbe = NULL, nsbe = 0; + FrameCFG cfg = {0}; + FrameEvents ev = {0}; + for (int c = 0; c < ncall; c++) + call_at[c] = -1; + if (!frame_rpo(ir, n, pos, &be, &nbe, &sbe, &nsbe, &cfg)) + goto out; + for (int i = n - 1; i >= 0; i--) + { + IRQuadCompact *q = &ir->compact_instructions[i]; + param_call[i] = -1; + call_params[i] = -1; + param_next[i] = -1; + if (q->op == TCCIR_OP_FUNCCALLVAL || q->op == TCCIR_OP_FUNCCALLVOID) + { + int c = frame_call_id(ir, tcc_ir_op_get_src2(ir, q)); + if (c < 0 || c >= ncall) + goto out; + call_at[c] = i; + nocapture[i] = (uint8_t)frame_callee_nocapture(ir, q); + } + else if (q->op == TCCIR_OP_FUNCPARAMVAL) + { + int c = frame_call_id(ir, tcc_ir_op_get_src2(ir, q)); + if (c < 0 || c >= ncall) + goto out; + int ci = call_at[c]; + param_call[i] = ci >= 0 ? ci : i; + if (ci >= 0) + { + param_next[i] = call_params[ci]; + call_params[ci] = i; + } + } + } + + /* Pointer flow through vregs, to a fixpoint. */ + for (int i = 0; i < n; i++) + { + IRQuadCompact *q = &ir->compact_instructions[i]; + if (q->op == TCCIR_OP_NOP) + continue; + int nops = irop_config[q->op].has_dest + irop_config[q->op].has_src1 + irop_config[q->op].has_src2 + + frame_op_has_slot3(q->op); + for (int k = 0; k < nops; k++) + { + int id; + if (frame_classify(fl, ir->iroperand_pool[q->operand_base + k], &id) == FOP_VAR_ADDR) + fl->var_addr_taken[TCCIR_DECODE_VREG_TYPE(id)][TCCIR_DECODE_VREG_POSITION(id)] = 1; + } + } + do + { + fl->changed = 0; + for (int i = 0; i < n && !fl->bad; i++) + { + IRQuadCompact *q = &ir->compact_instructions[i]; + int op = q->op, id; + if (op == TCCIR_OP_NOP || !irop_config[op].has_dest) + continue; + IROperand dest = tcc_ir_op_get_dest(ir, q); + if (frame_classify(fl, dest, &id) != FOP_VAL) + continue; + if (op == TCCIR_OP_FUNCCALLVAL) + { + /* The result may point into a struct result passed on (`f().arr`). + * Any other object reaching a call escapes anyway, and an argument's + * copy is never seen by the callee. */ + for (int p = call_params[i]; p >= 0; p = param_next[p]) + { + IRQuadCompact *pq = &ir->compact_instructions[p]; + int t = frame_op_taint(fl, tcc_ir_op_get_src1(ir, pq)); + int idx = TCCIR_DECODE_PARAM_IDX((uint32_t)irop_get_imm64_ex(ir, tcc_ir_op_get_src2(ir, pq))); + if (t >= 0 && (fl->seg[t].kind == FRAME_OBJ_RET_TEMP || (nocapture[i] == 2 && idx == 0))) + frame_taint_join(fl, id, t); + } + } + else if (op == TCCIR_OP_LOAD) + { + /* `T <-- V [LOAD]` reads a vreg's value -- a move, and V may hold a + * frame address -- where `T <-- P***DEREF***` and `T <-- StackLoc[x]` + * read memory, which frame_op_taint leaves out: a pointer only gets + * into memory through a store, which escapes it. Zig's AstGen.generate + * moved the address of its by-value Ast's copy through such a LOAD + * into `astgen.tree`, so storing it escaped nothing, and the copy was + * coloured over while the tree was still read through the pointer. */ + frame_taint_join(fl, id, frame_op_taint(fl, tcc_ir_op_get_src1(ir, q))); + } + else if (op == TCCIR_OP_STORE || frame_op_passes_value(op)) + { + if (irop_config[op].has_src1) + frame_taint_join(fl, id, frame_op_taint(fl, tcc_ir_op_get_src1(ir, q))); + if (irop_config[op].has_src2) + frame_taint_join(fl, id, frame_op_taint(fl, tcc_ir_op_get_src2(ir, q))); + if (tcc_ir_op_is_mac(op)) + frame_taint_join(fl, id, frame_op_taint(fl, tcc_ir_op_get_accum(ir, q))); + } + } + } while (fl->changed && !fl->bad); + if (fl->bad) + goto out; + + /* Pointers defined once, to a known place in a frame object: the address + * itself, or one such pointer moved or offset by a constant. A store + * through one writes known bytes (frame_ev_operand). */ + for (int k = 0; k < 3; k++) + { + int t = types[k], sz = fl->taint_size[t] > 0 ? fl->taint_size[t] : 1; + fl->vseg[t] = tcc_malloc(sizeof(int32_t) * sz); + fl->vrel[t] = tcc_malloc(sizeof(int32_t) * sz); + fl->vdef_at[t] = tcc_malloc(sizeof(int32_t) * sz); + fl->vdefs[t] = tcc_mallocz(sz); + for (int j = 0; j < sz; j++) + fl->vseg[t][j] = -1; + } + for (int i = 0; i < n; i++) + { + IRQuadCompact *q = &ir->compact_instructions[i]; + if (q->op == TCCIR_OP_NOP || !irop_config[q->op].has_dest) + continue; + IROperand d = tcc_ir_op_get_dest(ir, q); + int32_t vr = irop_get_vreg(d); + int ty = TCCIR_DECODE_VREG_TYPE(vr), ps = TCCIR_DECODE_VREG_POSITION(vr); + if (irop_dest_defines_vreg(d) && ty >= 1 && ty <= 3 && ps < fl->taint_size[ty] && fl->vdefs[ty][ps] < 2) + fl->vdefs[ty][ps]++; + } + for (int round = 0; round < 3; round++) + for (int i = 0; i < n; i++) + { + IRQuadCompact *q = &ir->compact_instructions[i]; + int op = q->op; + if (op != TCCIR_OP_LEA && op != TCCIR_OP_ASSIGN && op != TCCIR_OP_ADD && op != TCCIR_OP_SUB) + continue; + IROperand d = tcc_ir_op_get_dest(ir, q); + int32_t vr = irop_get_vreg(d); + int ty = TCCIR_DECODE_VREG_TYPE(vr), ps = TCCIR_DECODE_VREG_POSITION(vr); + /* A PARAM also arrives defined, and a VAR whose address is taken may + * be written through it: one IR definition says nothing for those. */ + if (vr < 0 || irop_get_tag(d) != IROP_TAG_VREG || d.is_lval || + (ty != TCCIR_VREG_TYPE_TEMP && ty != TCCIR_VREG_TYPE_VAR) || ps >= fl->taint_size[ty] || + fl->vdefs[ty][ps] != 1 || fl->vseg[ty][ps] >= 0 || fl->var_addr_taken[ty][ps]) + continue; + IROperand a = tcc_ir_op_get_src1(ir, q); + int id, kind = frame_classify(fl, a, &id), seg = -1, rel = 0, off; + if (kind == FOP_ADDR && frame_operand_offset(a, fl->bottom, &off)) + seg = id, rel = off - fl->seg[id].start; + else if (kind == FOP_VAL && irop_get_tag(a) == IROP_TAG_VREG) + { + int aty = TCCIR_DECODE_VREG_TYPE(id), aps = TCCIR_DECODE_VREG_POSITION(id); + if (fl->vseg[aty][aps] >= 0) + seg = fl->vseg[aty][aps], rel = fl->vrel[aty][aps]; + } + if (seg < 0) + continue; + if (op == TCCIR_OP_ADD || op == TCCIR_OP_SUB) + { + IROperand b = tcc_ir_op_get_src2(ir, q); + if (!irop_is_immediate(b)) + continue; + int64_t c = irop_get_imm64_ex(ir, b); + rel += op == TCCIR_OP_SUB ? -(int)c : (int)c; + } + else if (irop_config[op].has_src2 && !irop_is_none(tcc_ir_op_get_src2(ir, q))) + continue; + fl->vseg[ty][ps] = seg; + fl->vrel[ty][ps] = rel; + fl->vdef_at[ty][ps] = i; + } + + /* References and escapes. Pool entries no instruction operand covers are + * orphans a pass replaced; they are never executed. */ + for (int i = 0; i < n; i++) + { + IRQuadCompact *q = &ir->compact_instructions[i]; + int op = q->op; + if (op == TCCIR_OP_NOP) + continue; + int nops = irop_config[op].has_dest + irop_config[op].has_src1 + irop_config[op].has_src2 + frame_op_has_slot3(op); + for (int k = 0; k < nops; k++) + { + uint32_t pi = q->operand_base + k; + if (pi >= (uint32_t)ir->iroperand_pool_count) + continue; + int id, kind = frame_classify(fl, ir->iroperand_pool[pi], &id); + int t = TAINT_NONE; + if (kind == FOP_MEM || kind == FOP_ADDR) + t = id; + else if (kind == FOP_VAL || kind == FOP_DEREF) + t = *frame_taint_ref(fl, id); + frame_use(fl, t, i, pos[i]); + if (t >= 0) + { + fl->seg[t].refs++; + int sret = 0; + if (op == TCCIR_OP_FUNCPARAMVAL && k == 0) + { + IROperand enc = tcc_ir_op_get_src2(ir, q); + int c = frame_call_id(ir, enc); + if (TCCIR_DECODE_PARAM_IDX((uint32_t)irop_get_imm64_ex(ir, enc)) == 0) + { + int ci = param_call[i]; + int size_idx = ci != i ? frame_callee_arg0_size_index(ir, &ir->compact_instructions[ci]) : -1; + if (c >= 0 && c < ir->sret_calls_size && ir->sret_calls[c]) + sret = ir->sret_calls[c]; + else if (size_idx >= 0) + for (int pp = call_params[ci]; pp >= 0; pp = param_next[pp]) + { + IRQuadCompact *pq = &ir->compact_instructions[pp]; + IROperand pe = tcc_ir_op_get_src2(ir, pq), pv = tcc_ir_op_get_src1(ir, pq); + if (TCCIR_DECODE_PARAM_IDX((uint32_t)irop_get_imm64_ex(ir, pe)) == size_idx && + irop_is_immediate(pv)) + sret = (int)irop_get_imm64_ex(ir, pv); + } + } + } + int has_dest = irop_config[op].has_dest; + frame_ev_operand(fl, &ev, i, op, has_dest && k == 0 ? 0 : 1, kind, ir->iroperand_pool[pi], t, + op == TCCIR_OP_FUNCPARAMVAL ? param_call[i] : i, sret); + } + /* A parameter is read by its call. */ + if (op == TCCIR_OP_FUNCPARAMVAL) + frame_use(fl, t, param_call[i], pos[param_call[i]]); + } + + int has_dest = irop_config[op].has_dest, id; + switch (op) + { + case TCCIR_OP_STORE: + if (frame_classify(fl, tcc_ir_op_get_dest(ir, q), &id) != FOP_VAL) + frame_escape(fl, frame_op_taint(fl, tcc_ir_op_get_src1(ir, q))); + break; + case TCCIR_OP_STORE_INDEXED: + case TCCIR_OP_STORE_POSTINC: + frame_escape(fl, frame_op_taint(fl, tcc_ir_op_get_src1(ir, q))); + break; + case TCCIR_OP_FUNCPARAMVAL: + { + int t = frame_op_taint(fl, tcc_ir_op_get_src1(ir, q)); + if (t >= 0) + { + IROperand enc = tcc_ir_op_get_src2(ir, q); + int c = frame_call_id(ir, enc); + int idx = TCCIR_DECODE_PARAM_IDX((uint32_t)irop_get_imm64_ex(ir, enc)); + int sret = idx == 0 && c < ir->sret_calls_size && ir->sret_calls[c]; + if (!fl->seg[t].kind && !sret && !(param_call[i] != i && nocapture[param_call[i]])) + frame_escape(fl, t); + } + break; + } + default: + if (!frame_op_keeps_pointers(op)) + { + /* Any other consumer of a pointer value lets it go. */ + if (irop_config[op].has_src1) + frame_escape(fl, frame_op_taint(fl, tcc_ir_op_get_src1(ir, q))); + if (irop_config[op].has_src2) + frame_escape(fl, frame_op_taint(fl, tcc_ir_op_get_src2(ir, q))); + if (has_dest && op != TCCIR_OP_FUNCCALLVAL) + frame_escape(fl, frame_op_taint(fl, tcc_ir_op_get_dest(ir, q))); + } + break; + } + } + for (int k = 0; k < 3; k++) + { + int t = types[k]; + for (int j = 0; j < fl->taint_size[t]; j++) + if (fl->var_addr_taken[t][j]) + frame_escape(fl, fl->taint[t][j]); + } + if (fl->bad) + goto out; + + /* An escaped object may be read through its pointer anywhere after the + * address was taken -- until the end of the function, or of the inlined + * body that declared it (tcc_ir_frame_scope_end): its lifetime ended at + * the callee's return, so nothing past that point may still read it. In + * RPO its extent is unknown; source order alone then decides sharing. */ + for (int s = 0; s < fl->nseg; s++) + if (fl->seg[s].escaped && fl->seg[s].first[0] <= fl->seg[s].last[0]) + { + int end = fl->seg[s].scope_end && fl->seg[s].scope_end - 1 <= n - 1 ? fl->seg[s].scope_end - 1 : n - 1; + if (end < fl->seg[s].first[0]) + end = n - 1; + fl->seg[s].last[0] = end; + fl->seg[s].last[1] = n - 1; + } + + frame_precise_lifetimes(fl, &cfg, pos, &ev, be, nbe, sbe, nsbe); + + /* Widen every other range over the loops it intersects, in each order. */ + for (int o = 0; o < 2; o++) + { + const int *e = o ? be : sbe; + const int ne = o ? nbe : nsbe; + for (int s = 0; s < fl->nseg; s++) + { + FrameSeg *g = &fl->seg[s]; + if (g->first[o] > g->last[o] || g->precise) + continue; + int grew; + do + { + grew = 0; + for (int k = 0; k < ne; k++) + if (g->first[o] <= e[2 * k + 1] && g->last[o] >= e[2 * k]) + { + if (e[2 * k] < g->first[o]) + g->first[o] = e[2 * k], grew = 1; + if (e[2 * k + 1] > g->last[o]) + g->last[o] = e[2 * k + 1], grew = 1; + } + } while (grew); + } + } + frame_find_copies(fl, call_params, param_next, param_call, pos, &cfg); + ok = 1; + +out: + frame_cfg_free(&cfg); + tcc_free(ev.v); + tcc_free(be); + tcc_free(sbe); + tcc_free(pos); + for (int k = 0; k < 3; k++) + { + tcc_free(fl->taint[types[k]]); + tcc_free(fl->var_addr_taken[types[k]]); + tcc_free(fl->vseg[types[k]]); + tcc_free(fl->vrel[types[k]]); + tcc_free(fl->vdef_at[types[k]]); + tcc_free(fl->vdefs[types[k]]); + } + tcc_free(param_call); + tcc_free(nocapture); + tcc_free(call_at); + tcc_free(param_next); + tcc_free(call_params); + return ok; +} + +/* A colouring unit: one live object, placed by the bytes it occupies. */ +typedef struct +{ + int start, end, align; + int first[2], last[2]; + int refs; + uint8_t pinned; + int new_start; +} FrameUnit; + +typedef struct +{ + int start, end; +} FrameSpan; + +static int frame_span_start_asc(const void *a, const void *b) +{ + const FrameSpan *x = a, *y = b; + return x->start < y->start ? -1 : x->start > y->start; +} + +/* Place the `nm` movable units in `order` bottom-up: each at the lowest + * distance above the frame bottom that the units already placed with an + * overlapping lifetime leave free. The code addresses the frame from sp, + * just below it, so the first units placed get the short [sp, #imm] + * encodings. The block goes under the lowest pinned unit. Returns the new + * frame bottom. */ +static int frame_place(FrameUnit *u, int nu, const int *order, int nm, FrameSpan *spans) +{ + int ceiling = 0, height = 0; + for (int k = 0; k < nu; k++) + if (u[k].pinned && u[k].start < ceiling) + ceiling = u[k].start; + for (int oi = 0; oi < nm; oi++) + { + FrameUnit *x = &u[order[oi]]; + int nsp = 0; + for (int pi = 0; pi < oi; pi++) + { + FrameUnit *y = &u[order[pi]]; + /* Disjoint in either order is enough (see frame_lifetimes). */ + if (y->first[0] <= x->last[0] && x->first[0] <= y->last[0] && y->first[1] <= x->last[1] && + x->first[1] <= y->last[1]) + { + spans[nsp].start = y->new_start; + spans[nsp].end = y->new_start + (y->end - y->start); + nsp++; + } + } + qsort(spans, nsp, sizeof *spans, frame_span_start_asc); + int size = x->end - x->start, d = 0; + /* The bottom is 8-aligned, so a distance keeps the object's alignment + * when it is congruent with the object's original offset. */ + d += ((x->start - d) % x->align + x->align) % x->align; + for (int k = 0; k < nsp; k++) + { + if (spans[k].start >= d + size) + break; + if (spans[k].end <= d) + continue; + d = spans[k].end; + d += ((x->start - d) % x->align + x->align) % x->align; + } + x->new_start = d; /* a distance for now */ + if (d + size > height) + height = d + size; + } + int base = (ceiling - height) & ~7; + for (int oi = 0; oi < nm; oi++) + u[order[oi]].new_start += base; + for (int k = 0; k < nu; k++) + if (u[k].pinned) + u[k].new_start = u[k].start; + return base < ceiling ? base : ceiling; +} + +/* Hottest first: references per byte, then more references. */ +static FrameUnit *frame_units_gl; +static int frame_unit_hot_first(const void *a, const void *b) +{ + const FrameUnit *x = &frame_units_gl[*(const int *)a], *y = &frame_units_gl[*(const int *)b]; + int64_t lx = (int64_t)x->refs * (y->end - y->start), ly = (int64_t)y->refs * (x->end - x->start); + if (lx != ly) + return lx > ly ? -1 : 1; + if (x->refs != y->refs) + return x->refs > y->refs ? -1 : 1; + return x->start < y->start ? -1 : x->start > y->start; +} + +static int frame_grp_find(int *grp, int s) +{ + while (grp[s] != s) + s = grp[s] = grp[grp[s]]; + return s; +} + +static void frame_copies_free(FrameLive *fl) +{ + tcc_free(fl->copy_at); + tcc_free(fl->copy_pos); + tcc_free(fl->copy_from); + tcc_free(fl->copy_from_pos); + tcc_free(fl->copy_dst); + tcc_free(fl->copy_src); + fl->copy_at = fl->copy_pos = fl->copy_from = fl->copy_from_pos = fl->copy_dst = fl->copy_src = NULL; + fl->ncopy = 0; +} + +/* Colour the live segments. Returns the new bottom, or 1 when colouring does + * not apply or does worse than packing. */ +TCC_DBG_ENV_FLAG(frame_dbg, "TCC_FRAME_DBG") + +/* The copies the returned layout merged are handed back in *merged_at (their + * instruction indices, *nmerged of them) rather than deleted here: the caller + * may throw this layout away and colour again, and a copy deleted for a layout + * that is not kept leaves its two objects apart with nothing copying between + * them. */ +static int frame_colour(TCCIRState *ir, FrameSeg *seg, int nseg, int bottom, int **merged_at, int *nmerged) +{ + *merged_at = NULL; + *nmerged = 0; + FrameLive fl = {0}; + fl.ir = ir; + fl.seg = seg; + fl.nseg = nseg; + fl.bottom = bottom; + fl.n = ir->next_instruction_index; + for (int s = 0; s < nseg; s++) + { + seg[s].first[0] = seg[s].first[1] = INT_MAX; + seg[s].last[0] = seg[s].last[1] = -1; + } + if (!frame_lifetimes(&fl)) + { + frame_copies_free(&fl); + return 1; + } + + /* Copy coalescing: the two objects of a copy frame_find_copies recorded + * become one colouring unit, so they get the same bytes and the copy + * copies them onto themselves. Chains (`t5 = t4; t6 = t5;`) merge through + * the groups, each merge re-checked against the group's whole lifetime; a + * group is as big as its biggest object, all of them at its start. The + * group keeps the start whose alignment is the stricter; the other object + * must sit at the same offset modulo its own alignment. */ + int *grp = tcc_malloc(sizeof(int) * (nseg > 0 ? nseg : 1)); + uint8_t *merged = tcc_mallocz(fl.ncopy > 0 ? fl.ncopy : 1); + for (int s = 0; s < nseg; s++) + grp[s] = s; + for (int c = 0; c < fl.ncopy; c++) + { + int rd = frame_grp_find(grp, fl.copy_dst[c]), rs = frame_grp_find(grp, fl.copy_src[c]); + if (rd == rs) + continue; + /* The source group ends at the call, the destination group starts no + * earlier than the copy (the call, or its destination pointer). */ + if (seg[rs].last[0] > fl.copy_at[c] || seg[rs].last[1] > fl.copy_pos[c] || seg[rd].first[0] < fl.copy_from[c] || + seg[rd].first[1] < fl.copy_from_pos[c]) + continue; + int keep = frame_offset_align(seg[rd].start) >= frame_offset_align(seg[rs].start) ? rd : rs; + int other = keep == rd ? rs : rd, oa = frame_offset_align(seg[other].start); + if (((seg[other].start - seg[keep].start) % oa + oa) % oa) + continue; + grp[other] = keep; + if (seg[other].size > seg[keep].size) + seg[keep].size = seg[other].size; + for (int o = 0; o < 2; o++) + { + if (seg[other].first[o] < seg[keep].first[o]) + seg[keep].first[o] = seg[other].first[o]; + if (seg[other].last[o] > seg[keep].last[o]) + seg[keep].last[o] = seg[other].last[o]; + } + seg[keep].refs += seg[other].refs; + merged[c] = 1; + } + + FrameUnit *u = tcc_malloc(sizeof(FrameUnit) * (nseg > 0 ? nseg : 1)); + int nu = 0; + for (int s = 0; s < nseg; s++) + { + FrameSeg *g = &seg[s]; + g->unit = -1; + if (!g->live || (!g->pinned && g->first[0] > g->last[0]) || frame_grp_find(grp, s) != s) + continue; + FrameUnit *x = &u[nu]; + g->unit = nu++; + x->start = g->start; + x->end = g->pinned ? g->end : g->start + g->size; + x->align = frame_offset_align(g->start); + x->pinned = g->pinned; + for (int o = 0; o < 2; o++) + { + x->first[o] = g->pinned ? 0 : g->first[o]; + x->last[o] = g->pinned ? fl.n : g->last[o]; + } + x->refs = g->refs; + x->new_start = 1; + } + for (int s = 0; s < nseg; s++) + if (frame_grp_find(grp, s) != s && seg[s].live) + seg[s].unit = seg[frame_grp_find(grp, s)].unit; + + int *order = tcc_malloc(sizeof(int) * (nu > 0 ? nu : 1)); + FrameSpan *spans = tcc_malloc(sizeof(FrameSpan) * (nu > 0 ? nu : 1)); + int nm = 0; + for (int k = 0; k < nu; k++) + if (!u[k].pinned) + order[nm++] = k; + frame_units_gl = u; + qsort(order, nm, sizeof(int), frame_unit_hot_first); + frame_units_gl = NULL; + int placed = frame_place(u, nu, order, nm, spans); + if (frame_dbg()) + for (int s = 0; s < nseg; s++) + fprintf(stderr, "[FRAME] seg %d [%d,+%d) live=%d pin=%d kind=%d esc=%d prec=%d idx=[%d,%d] pos=[%d,%d] refs=%d unit=%d -> %d\n", + s, seg[s].start, seg[s].size, seg[s].live, seg[s].pinned, seg[s].kind, seg[s].escaped, seg[s].precise, + seg[s].first[0], seg[s].last[0], seg[s].first[1], seg[s].last[1], seg[s].refs, seg[s].unit, + seg[s].unit >= 0 ? u[seg[s].unit].new_start : 0); + + /* Keep it only if its frame is no bigger than packing live runs in place. */ + FrameSeg *tmp = tcc_malloc(sizeof(FrameSeg) * (nseg > 0 ? nseg : 1)); + memcpy(tmp, seg, sizeof(FrameSeg) * nseg); + int packed = frame_pack_runs(tmp, nseg); + tcc_free(tmp); + int result = 1; + if (placed >= packed) + { + for (int s = 0; s < nseg; s++) + seg[s].delta = seg[s].unit >= 0 ? u[seg[s].unit].new_start - seg[s].start : 0; + result = placed; + /* The merged copies now copy an object onto itself. */ + *merged_at = tcc_malloc(sizeof(int) * (fl.ncopy > 0 ? fl.ncopy : 1)); + for (int c = 0; c < fl.ncopy; c++) + if (merged[c]) + (*merged_at)[(*nmerged)++] = fl.copy_at[c]; + } + tcc_free(grp); + tcc_free(merged); + frame_copies_free(&fl); + tcc_free(order); + tcc_free(spans); + tcc_free(u); + return result; +} + +int tcc_ir_frame_relayout(TCCIRState *ir, int *ploc) +{ + int bottom = *ploc; + /* Once only: after it, the operands no longer match frame_objs. */ + if (!ir || ir->frame_relaid || bottom >= 0 || ir->frame_obj_count == 0 || !frame_function_eligible(ir)) + return 0; + ir->frame_relaid = 1; + + /* Objects -> disjoint segments tiling [bottom, 0), gaps included. */ + int nobj = ir->frame_obj_count; + qsort(ir->frame_objs, nobj, sizeof(int32_t) * 4, frame_obj_cmp); + /* `bottom` is the lowest offset an operand still names (compute_min_stack_ref), + * so an object whose low words lost their last reference -- a dead store + * removed -- starts below it. Measure the frame from that object instead: + * otherwise it is pinned in place at the bottom and nothing above it can + * pack (a single 8-byte temporary kept a 43 KB frame from colouring down + * to 8.7 KB). Its unreferenced low bytes simply move with it. */ + if (ir->frame_objs[0] < bottom) + bottom = ir->frame_objs[0]; + FrameSeg *seg = tcc_mallocz(sizeof(FrameSeg) * (2 * nobj + 2)); + int nseg = 0, cur = bottom; + for (int k = 0; k < nobj; k++) + { + const int32_t *o = &ir->frame_objs[4 * k]; + int s = o[0], e = s + o[1]; + if (s < bottom || e > 0) + { + /* Not wholly inside the frame the optimizer left: keep it where it is. */ + if (e > 0) + e = 0; + if (s < bottom) + s = bottom; + if (e <= cur) + continue; + if (s < cur) + s = cur; + if (s > cur) + seg[nseg++] = (FrameSeg){.start = cur, .end = s, .size = s - cur, .live = 1, .pinned = 1}; + seg[nseg++] = (FrameSeg){.start = s, .end = e, .size = e - s, .live = 1, .pinned = 1}; + cur = e; + continue; + } + if (e <= cur) + continue; /* inside an earlier object */ + if (s < cur) + { + /* Overlapping objects share bytes, so they live and move as one. */ + FrameSeg *g = &seg[nseg - 1]; + g->end = e; + if (s + o[2] > g->start + g->size) + g->size = s + o[2] - g->start; + g->kind &= o[3] & 0xF; + /* Sharing bytes, the two live as long as the longer one. */ + g->scope_end = g->scope_end && (o[3] >> 4) ? (g->scope_end > (o[3] >> 4) ? g->scope_end : (o[3] >> 4)) : 0; + cur = e; + continue; + } + if (s > cur) + seg[nseg++] = (FrameSeg){.start = cur, .end = s, .size = s - cur, .live = 1, .pinned = 1}; + seg[nseg++] = (FrameSeg){ + .start = s, .end = e, .size = o[2] < e - s ? o[2] : e - s, .kind = (uint8_t)(o[3] & 0xF), .scope_end = o[3] >> 4}; + cur = e; + } + if (cur < 0) + seg[nseg++] = (FrameSeg){.start = cur, .end = 0, .size = -cur, .live = 1, .pinned = 1}; + + /* Liveness over the whole operand pool rather than per instruction: every + * operand slot is covered whatever its position, and an orphaned or NOP'd + * operand only keeps an object alive. + * + * A STRUCT operand keeps its offset in the 16-bit u.s.aux_data, and + * colouring moves the hottest objects to the frame bottom -- below -32768 + * once the frame is that big, where the offset wraps (a 12-byte struct + * copy under a 32 KB buffer read StackLoc[32760] for StackLoc[-32776]). + * In such a frame an object a STRUCT operand names stays where it is: its + * offset already fits, and packing only ever moves objects up. */ + IROperand *pool = ir->iroperand_pool; + const int npool = ir->iroperand_pool_count; + for (int i = 0; i < npool; i++) + { + int off, s; + if (frame_operand_offset(pool[i], bottom, &off) && (s = frame_seg_find(seg, nseg, off)) >= 0) + seg[s].live = 1; + } + + /* Laid out once without regard to the 16-bit offsets, and again with every + * object a STRUCT operand names pinned in place should one of them have + * landed below -32768 (the layout only ever shrinks such a frame: a + * function whose 38 KB of inlined callees' temporaries had 655 struct + * objects pinned across a 33 KB frame comes down to a few KB, and with it + * every access from movw + register-indexed to a 16-bit [sp, #imm]). */ + FrameSeg *seg0 = tcc_malloc(sizeof(FrameSeg) * (nseg > 0 ? nseg : 1)); + memcpy(seg0, seg, sizeof(FrameSeg) * nseg); + int new_top = 1; + int *merged_at = NULL, nmerged = 0; + for (int attempt = 0; attempt < 2; attempt++) + { + tcc_free(merged_at); + merged_at = NULL; + nmerged = 0; + if (attempt) + { + memcpy(seg, seg0, sizeof(FrameSeg) * nseg); + for (int i = 0; i < npool; i++) + { + int off, s; + if (pool[i].btype == IROP_BTYPE_STRUCT && frame_operand_offset(pool[i], bottom, &off) && + (s = frame_seg_find(seg, nseg, off)) >= 0) + seg[s].pinned = 1; + } + } + new_top = 1; + if (tcc_state->optimize > 0 && !tcc_ir_opt_pass_disabled("frame_colour") && !ir->func_has_label_addr && + !tcc_ir_calls_returns_twice(ir)) + { + int ijump = 0; + for (int i = 0; i < ir->next_instruction_index; i++) + ijump |= ir->compact_instructions[i].op == TCCIR_OP_IJUMP; + if (!ijump) + new_top = frame_colour(ir, seg, nseg, bottom, &merged_at, &nmerged); + } + if (new_top > 0) + { + int dead = 0; + for (int s = 0; s < nseg; s++) + dead += !seg[s].live; + if (!dead) + { + tcc_free(merged_at); + tcc_free(seg0); + tcc_free(seg); + return 0; + } + new_top = frame_pack_runs(seg, nseg); + } + if (bottom >= -32768 || attempt) + break; + int fits = 1; + for (int i = 0; i < npool && fits; i++) + { + int off, s; + if (pool[i].btype == IROP_BTYPE_STRUCT && frame_operand_offset(pool[i], bottom, &off) && + (s = frame_seg_find(seg, nseg, off)) >= 0 && off + seg[s].delta < -32768) + fits = 0; + } + if (fits) + break; + } + tcc_free(seg0); + /* Only now, with the layout settled, do the kept attempt's merged copies go. */ + for (int k = 0; k < nmerged; k++) + { + ir_opt_nop_call_params(ir, merged_at[k]); + ir->compact_instructions[merged_at[k]].op = TCCIR_OP_NOP; + } + tcc_free(merged_at); + + for (int i = 0; i < npool; i++) + { + int off, s; + if (!frame_operand_offset(pool[i], bottom, &off) || (s = frame_seg_find(seg, nseg, off)) < 0 || !seg[s].delta) + continue; + if (pool[i].btype == IROP_BTYPE_STRUCT) + { + if (off + seg[s].delta < -32768) + tcc_error("internal: frame relayout moved a struct operand to %d, out of the 16-bit encoding range", + off + seg[s].delta); + pool[i].u.s.aux_data = (int16_t)(off + seg[s].delta); + } + else + pool[i].u.imm32 = off + seg[s].delta; + } + + tcc_free(seg); + int new_bottom = new_top & ~7; + if (new_bottom <= bottom) + return 0; + *ploc = new_bottom; + return new_bottom - bottom; +} diff --git a/source/ir/gen/config.c b/source/ir/gen/config.c index 716f8786d..129ae3391 100644 --- a/source/ir/gen/config.c +++ b/source/ir/gen/config.c @@ -22,6 +22,7 @@ const IRRegistersConfig irop_config[] = { [TCCIR_OP_MUL] = {1, 1, 1}, [TCCIR_OP_MLA] = {1, 1, 1}, /* MLA has accumulator as extra operand at pool[operand_base+3] */ [TCCIR_OP_UMULL] = {1, 1, 1}, + [TCCIR_OP_UMAAL] = {1, 1, 1}, /* accum (both addends) at pool[operand_base+3], as MLA */ [TCCIR_OP_SMULL] = {1, 1, 1}, [TCCIR_OP_DIV] = {1, 1, 1}, [TCCIR_OP_UMOD] = {1, 1, 1}, @@ -126,6 +127,8 @@ const IRRegistersConfig irop_config[] = { [TCCIR_OP_RBIT] = {1, 1, 0}, [TCCIR_OP_REV] = {1, 1, 0}, [TCCIR_OP_REV16] = {1, 1, 0}, + /* __builtin_return_address(0): dest only */ + [TCCIR_OP_RETURN_ADDRESS] = {1, 0, 0}, } ; // clang-format on diff --git a/source/ir/gen/params.c b/source/ir/gen/params.c index 537abb707..ec3b518f5 100644 --- a/source/ir/gen/params.c +++ b/source/ir/gen/params.c @@ -11,6 +11,10 @@ #define USING_GLOBALS #include "ir.h" +/* The form the parameter being processed takes in the body, when it can be + * described (see IRParamForm); NULL when not recording. */ +static IRParamForm *pf_cur; + /* Forward declarations for internal helpers */ static void tcc_ir_params_add_hidden_sret(TCCIRState *ir, CType *func_type); static void tcc_ir_params_process_arguments(TCCIRState *ir, Sym *param_list, TCCAbiCallLayout *call_layout); @@ -67,7 +71,14 @@ static void tcc_ir_params_add_hidden_sret(TCCIRState *ir, CType *func_type) /* Struct returned via hidden pointer in first parameter (r0) */ SValue src, dst; - loc = (loc - PTR_SIZE) & -PTR_SIZE; + /* A frame OBJECT, not a raw slot: SRA then treats the pointer like any + * other word-sized local and promotes it, so the returns read a register + * instead of reloading the slot (zig.c: every struct-returning function + * reloaded it at each `return`). */ + if (getenv("TCC_SRET_HOME_SLOT")) /* the old raw slot, for experiments */ + loc = (loc - PTR_SIZE) & -PTR_SIZE; + else + loc = tcc_ir_frame_alloc(loc, PTR_SIZE, -PTR_SIZE); func_vc = loc; /* Consume a PARAM vreg for the hidden sret pointer */ @@ -105,12 +116,16 @@ static void tcc_ir_params_process_arguments(TCCIRState *ir, Sym *param_list, TCC ir->parameters_count = (int8_t)arg_count; ir->named_arg_reg_bytes = 0; ir->named_arg_stack_bytes = 0; + tcc_free(ir->param_forms); + ir->param_forms = arg_count > 0 ? tcc_mallocz(sizeof(IRParamForm) * arg_count) : NULL; } /* Process each parameter */ for (sym = param_list; sym; sym = sym->next, ++arg_index) { + pf_cur = ir && ir->param_forms && arg_index < 127 ? &ir->param_forms[arg_index] : NULL; tcc_ir_params_process_single(ir, sym, arg_index, call_layout); + pf_cur = NULL; } } @@ -176,6 +191,27 @@ void tcc_ir_params_process_single(TCCIRState *ir, Sym *sym, int arg_index, TCCAb } } +/* A parameter the ABI placed wholly on the caller's stack. Recorded on its + * PARAM vreg because tcc_ir_register_allocation_params re-derives register + * placement by counting, and cannot tell a register an 8-aligned struct + * skipped (r3 before a stack-passed S24L) from one still free. */ +static void params_mark_stack(TCCIRState *ir, Sym *ps) +{ + if (!ir || !ps || ps->vreg < 0 || TCCIR_DECODE_VREG_TYPE(ps->vreg) != TCCIR_VREG_TYPE_PARAM) + return; + IRLiveInterval *iv = tcc_ir_vreg_live_interval(ir, ps->vreg); + if (iv) + iv->incoming_stack = 1; +} + +/* Offset of a stack-passed parameter as the IR names it: from the first stack + * argument, or from the pushed r0 when the prologue pushes r0-r3 for an + * in-place split struct (see tcc_ir_params_process_struct). */ +static int param_stack_off(const TCCIRState *ir, const TCCAbiArgLoc *loc_info) +{ + return loc_info->stack_off + (ir && ir->push_arg_regs ? 16 : 0); +} + void tcc_ir_params_update_tracking(TCCIRState *ir, TCCAbiArgLoc loc_info, TCCAbiCallLayout *layout) { if (!ir) @@ -198,7 +234,8 @@ void tcc_ir_params_update_tracking(TCCIRState *ir, TCCAbiArgLoc loc_info, TCCAbi } else { - int end = loc_info.stack_off + loc_info.size; + /* A stack slot is a whole number of words (a 17-byte struct takes 20). */ + int end = loc_info.stack_off + ((loc_info.size + 3) & ~3); if (end > ir->named_arg_stack_bytes) ir->named_arg_stack_bytes = end; } @@ -219,63 +256,9 @@ void tcc_ir_params_update_tracking(TCCIRState *ir, TCCAbiArgLoc loc_info, TCCAbi void tcc_ir_params_process_struct(TCCIRState *ir, Sym *sym, CType *type, int size, int align, TCCAbiArgLoc *loc_info, TCCAbiCallLayout *call_layout, int arg_index) { - const int invisible_ref = - (call_layout->arg_flags && (call_layout->arg_flags[arg_index] & TCC_ABI_ARG_FLAG_INVISIBLE_REF)); int slot_align = align < 4 ? 4 : align; int flags = 0, addr = 0; - if (invisible_ref) - { - /* Large struct passed as hidden pointer */ - loc = (loc - PTR_SIZE) & -PTR_SIZE; - const int ptr_slot = loc; - const int ptr_param_vr = tcc_ir_get_vreg_param(ir); - - IRLiveInterval *ptr_iv = tcc_ir_vreg_live_interval(ir, ptr_param_vr); - if (ptr_iv) - { - if (loc_info->kind == TCC_ABI_LOC_REG) - { - /* Invisible-ref pointer passed in a register. - * Set incoming register so tcc_ir_mark_param_incoming_regs skips - * this vreg and doesn't re-assign it based on sequential argno. */ - ptr_iv->incoming_reg0 = loc_info->reg_base; - ptr_iv->incoming_reg1 = -1; - } - else - { - /* Invisible-ref pointer passed on the stack (all argument registers - * exhausted). Mark as stack-passed and record the caller-frame - * offset so PARAM_STACK materialisation picks it up correctly. */ - ptr_iv->incoming_reg0 = -1; - ptr_iv->incoming_reg1 = -1; - tcc_ir_set_original_offset(ir, ptr_param_vr, loc_info->stack_off); - } - } - - SValue src, dst; - memset(&src, 0, sizeof(src)); - memset(&dst, 0, sizeof(dst)); - src.type.t = VT_PTR; - src.r = 0; - src.vr = ptr_param_vr; - dst.type.t = VT_PTR; - dst.r = VT_LOCAL | VT_LVAL; - dst.vr = -1; - dst.c.i = ptr_slot; - tcc_ir_put(ir, TCCIR_OP_STORE, &src, NULL, &dst); - - flags = VT_LVAL | VT_LLOCAL; - addr = ptr_slot; - { - int v = sym->v & ~SYM_FIELD; - if (!v) - v = anon_sym++; - sym_push(v, type, flags, addr); - } - return; - } - if (loc_info->kind == TCC_ABI_LOC_REG) { /* Struct passed in registers - spill to local home */ @@ -319,12 +302,37 @@ void tcc_ir_params_process_struct(TCCIRState *ir, Sym *sym, CType *type, int siz v = anon_sym++; sym_push(v, type, flags, addr); } + if (pf_cur) + *pf_cur = (IRParamForm){.kind = IR_PF_HOME, .reg_base = (int8_t)loc_info->reg_base, .off = struct_slot, + .words = word_count}; + return; + } + + if (loc_info->kind == TCC_ABI_LOC_REG_STACK && loc_info->stack_off == 0 && !call_layout->is_variadic) + { + /* Struct straddles r3 and the stack. The prologue pushes r0-r3 right + * below the stack arguments (push_arg_regs), which makes it contiguous in + * memory: use it in place from its first register's slot instead of + * copying it word by word into the frame. Parameter offsets are then + * measured from the pushed r0 (offset_to_args stops there), so they stay + * non-negative -- disjoint from the frame's negative local offsets, which + * passes comparing slot offsets rely on -- and every stack parameter + * after this one sits 16 bytes further up (param_stack_off). */ + ir->push_arg_regs = 1; + int v = sym->v & ~SYM_FIELD; + if (!v) + v = anon_sym++; + Sym *ps = sym_push(v, type, VT_PARAM | VT_LVAL | VT_LOCAL, loc_info->reg_base * 4); + params_mark_stack(ir, ps); + if (pf_cur && ps->vreg >= 0) + *pf_cur = (IRParamForm){.kind = IR_PF_MEM, .vreg = ps->vreg, .off = loc_info->reg_base * 4}; return; } if (loc_info->kind == TCC_ABI_LOC_REG_STACK) { - /* Struct straddles registers and stack */ + /* Struct straddles registers and stack, the stack part not first on the + * stack (only under hard-float, after VFP arguments spilled): copy it. */ int slot_size = tcc_abi_align_up_int(size, 4); loc = (loc - slot_size) & -slot_align; const int struct_slot = loc; @@ -401,12 +409,15 @@ void tcc_ir_params_process_struct(TCCIRState *ir, Sym *sym, CType *type, int siz /* Struct passed on stack */ flags = VT_PARAM | VT_LVAL | VT_LOCAL; - addr = loc_info->stack_off; + addr = param_stack_off(ir, loc_info); { int v = sym->v & ~SYM_FIELD; if (!v) v = anon_sym++; - sym_push(v, type, flags, addr); + Sym *ps = sym_push(v, type, flags, addr); + params_mark_stack(ir, ps); + if (pf_cur && ps->vreg >= 0) + *pf_cur = (IRParamForm){.kind = IR_PF_MEM, .vreg = ps->vreg, .off = addr}; } } @@ -438,7 +449,7 @@ void tcc_ir_params_process_scalar(TCCIRState *ir, Sym *sym, CType *type, TCCAbiA else { flags = VT_PARAM | VT_LVAL | VT_LOCAL; - addr = loc_info->stack_off; + addr = param_stack_off(ir, loc_info); } sym->r |= ~(VT_LVAL | VT_LLOCAL); @@ -446,7 +457,12 @@ void tcc_ir_params_process_scalar(TCCIRState *ir, Sym *sym, CType *type, TCCAbiA int v = sym->v & ~SYM_FIELD; if (!v) v = anon_sym++; - sym_push(v, &pushed_type, flags, addr); + Sym *ps = sym_push(v, &pushed_type, flags, addr); + if (loc_info->kind == TCC_ABI_LOC_STACK) + params_mark_stack(ir, ps); + if (pf_cur && ps->vreg >= 0 && !variadic) + *pf_cur = (IRParamForm){.kind = loc_info->kind == TCC_ABI_LOC_STACK ? IR_PF_MEM : IR_PF_REG, .vreg = ps->vreg, + .off = addr}; } int tcc_ir_local_add(TCCIRState *ir, Sym *sym, int stack_offset) diff --git a/source/ir/gen/put.c b/source/ir/gen/put.c index daf3b0e08..437afc597 100644 --- a/source/ir/gen/put.c +++ b/source/ir/gen/put.c @@ -18,7 +18,7 @@ static void ir_ensure_sym_registered(SValue *sv) { Sym *sym = sv->sym; /* Check if this is an anonymous symbol that hasn't been registered yet */ - if ((sym->v & ~0x0FFFFFFF) == SYM_FIRST_ANOM && sym->c == 0) + if ((sym->v & ~0x0FFFFFFF) == SYM_FIRST_ANOM && sym->c == 0 && !tcc_state->check_only) { /* Use put_extern_sym2 directly to bypass nocode_wanted check. * We need the symbol registered in ELF even if we're in a "nocode" section @@ -39,6 +39,41 @@ static int ir_operand_is_stack_addr(const SValue *sv) return 0; } +/* A STRUCT-typed stack operand keeps its frame offset in the int16_t + * u.s.aux_data half of the split encoding (tccir_operand.c), so a struct more + * than 32 KiB from the frame base -- a local behind a big array, a by-value + * parameter behind a big one -- cannot be named directly: the offset would be + * truncated (pr28982b: a 256 KiB local at FP-0x40004 became FP-4). Name it + * through its address instead. An address-of (no VT_LVAL) only needs a + * scalar type, whose STACKOFF carries a full imm32; a struct value gets that + * address LEA'd into a TEMP and is read through the pointer. */ +static SValue *ir_put_far_struct(TCCIRState *ir, SValue *sv, SValue *tmp) +{ + if (!sv || (sv->type.t & VT_BTYPE) != VT_STRUCT || (sv->r & VT_SYM) || (sv->r & VT_VALMASK) != VT_LOCAL || + ((int32_t)sv->c.i >= -32768 && (int32_t)sv->c.i <= 32767)) + return sv; + *tmp = *sv; + if (!(sv->r & VT_LVAL)) + { + tmp->type.t = VT_PTR; + tmp->type.ref = NULL; + return tmp; + } + SValue src = *sv; + src.r &= ~VT_LVAL; + src.type.t = VT_PTR; + src.type.ref = NULL; + SValue addr; + memset(&addr, 0, sizeof(addr)); + addr.type.t = VT_PTR; + addr.vr = tcc_ir_get_vreg_temp(ir); + tcc_ir_put(ir, TCCIR_OP_LEA, &src, NULL, &addr); + tmp->r = VT_LVAL; + tmp->vr = addr.vr; + tmp->c.i = 0; + return tmp; +} + /* Defined in ir/gen/softfloat.c */ int ir_put_soft_call_fpu_if_needed(TCCIRState *ir, TccIrOp op, SValue *src1, SValue *src2, SValue *dest); @@ -64,6 +99,12 @@ int tcc_ir_put(TCCIRState *ir, TccIrOp op, SValue *src1, SValue *src2, SValue *d ir_ensure_sym_registered(src2); ir_ensure_sym_registered(dest); + SValue far_src1, far_src2; + if (irop_config[op].has_src1 == 1) + src1 = ir_put_far_struct(ir, src1, &far_src1); + if (irop_config[op].has_src2 == 1) + src2 = ir_put_far_struct(ir, src2, &far_src2); + /* Check if we need to use soft-float call instead of native FPU instruction. * Skip this for complex operations - they need special handling in the code generator. */ if (tcc_ir_type_op_needs_fpu(op) && !((dest && (dest->type.t & VT_COMPLEX)) || @@ -245,7 +286,16 @@ int tcc_ir_put(TCCIRState *ir, TccIrOp op, SValue *src1, SValue *src2, SValue *d const int prev_is_64bit = irop_is_64bit(prev_dest_irop); const int new_is_64bit = irop_is_64bit(dest_irop); const int width_match = (prev_is_64bit == new_is_64bit); - const int can_coalesce = (!ir->prevent_coalescing) && width_match && + /* A vreg-backed slot written at a sub-offset (`__imag__ x = ...` on a + * complex parameter or local: StackLoc[orig + 8] backed by x's vreg) keeps + * that offset only in the STACKOFF operand. Renaming the previous + * instruction's TEMP dest to the bare vreg would write the real half. */ + const int dest_vr = irop_get_vreg(dest_irop); + IRLiveInterval *dest_iv = (irop_get_tag(dest_irop) == IROP_TAG_STACKOFF && tcc_ir_vreg_is_valid(ir, dest_vr)) + ? tcc_ir_vreg_live_interval(ir, dest_vr) + : NULL; + const int dest_subslot = dest_iv && irop_get_stack_offset(dest_irop) != dest_iv->original_offset; + const int can_coalesce = (!ir->prevent_coalescing) && width_match && !dest_subslot && (TCCIR_DECODE_VREG_TYPE(irop_get_vreg(src1_irop)) == TCCIR_VREG_TYPE_TEMP) && !src1_irop.is_lval && (irop_get_vreg(src1_irop) == prev_dest_vr); if (can_coalesce) diff --git a/source/ir/gen/state.c b/source/ir/gen/state.c index 178e0d55d..1a6f34dae 100644 --- a/source/ir/gen/state.c +++ b/source/ir/gen/state.c @@ -165,6 +165,32 @@ void tcc_ir_free(TCCIRState *ir) tcc_free(ir->active_set); } + tcc_free(ir->param_forms); + ir->param_forms = NULL; + if (ir->frame_objs) + { + tcc_free(ir->frame_objs); + ir->frame_objs = NULL; + } + tcc_free(ir->frame_pads); + ir->frame_pads = NULL; + if (ir->sret_calls) + { + tcc_free(ir->sret_calls); + ir->sret_calls = NULL; + } + if (ir->call_dyn_extra) + { + tcc_free(ir->call_dyn_extra); + ir->call_dyn_extra = NULL; + } + ir->call_dyn_extra_size = 0; + if (ir->frame_index) + { + tcc_free(ir->frame_index); + ir->frame_index = NULL; + } + if (ir->ir_to_code_mapping) { tcc_free(ir->ir_to_code_mapping); @@ -259,7 +285,7 @@ void tcc_ir_free(TCCIRState *ir) ir->codegen_dry_insn_scratch = NULL; tcc_free(ir->codegen_dry_insn_saves); ir->codegen_dry_insn_saves = NULL; - tcc_free(ir->codegen_mop_cache); + tcc_ir_mop_cache_free(ir->codegen_mop_cache); ir->codegen_mop_cache = NULL; tcc_free(ir->codegen_cbz_dry_mapping); ir->codegen_cbz_dry_mapping = NULL; diff --git a/source/ir/machine_op.c b/source/ir/machine_op.c index ed1594c28..18191503e 100644 --- a/source/ir/machine_op.c +++ b/source/ir/machine_op.c @@ -290,9 +290,14 @@ MachineOperand machine_op_from_ir(TCCIRState *ir, const IROperand *op) use_llocal = 1; } - /* Only preserve is_param for stack-passed parameters (incoming_reg0 < 0). */ + /* Only preserve is_param for stack-passed parameters (incoming_reg0 < 0) + * that still live in the caller's argument area. One a nested function + * captures was given a home in THIS frame (nested_home, see + * tcc_ir_mark_nested_captured_params) and the prologue copied the incoming + * value into it: reading it as a param would add offset_to_args to an + * offset that is already frame-relative. */ int spilled_param = 0; - if (op->is_param && interval->incoming_reg0 < 0) + if (op->is_param && interval->incoming_reg0 < 0 && !interval->nested_home) { spilled_param = 1; } diff --git a/source/ir/machine_op.h b/source/ir/machine_op.h index 81ad2a25f..1dfcab352 100644 --- a/source/ir/machine_op.h +++ b/source/ir/machine_op.h @@ -66,22 +66,26 @@ typedef enum * register file from the GPRs of MACH_OP_REG. */ } MachineOperandKind; +/* Packed to 16 bytes on 32-bit hosts (was 32): codegen keeps five of these per + * IR instruction for the whole function (the dry-run MopArgs cache), which was + * the peak of compiling a big function on the device. The flags are 1-bit, so + * only ever assign them 0/1 values (compare results, (bool) casts). */ typedef struct MachineOperand { - MachineOperandKind kind; /* How to materialize this operand */ - int btype; /* IROP_BTYPE_* — compressed base type */ - int vreg; /* Original vreg (for debug / liveness queries) */ - bool needs_deref; /* Emit a load through this address (VT_LVAL) */ - bool is_64bit; /* Two-register value (INT64 or FLOAT64) */ - bool is_unsigned; /* Unsigned type (VT_UNSIGNED) */ - bool is_complex; /* Complex type (VT_COMPLEX) */ - bool align4; /* 64-bit deref only: the accessed address is proven - * >= 4-byte aligned, so LDRD/STRD may be used through - * a general base register (IROperand.align4_ok). */ - bool underalign_hint; /* Base of an indexed access whose chain crossed a - * packed member: address may be < 4-byte aligned, so - * the 64-bit indexed lowering must avoid LDRD/STRD - * (IROperand.underalign_hint). */ + uint8_t kind; /* MachineOperandKind: how to materialize this operand */ + uint8_t btype; /* IROP_BTYPE_* — compressed base type */ + uint8_t needs_deref : 1; /* Emit a load through this address (VT_LVAL) */ + uint8_t is_64bit : 1; /* Two-register value (INT64 or FLOAT64) */ + uint8_t is_unsigned : 1; /* Unsigned type (VT_UNSIGNED) */ + uint8_t is_complex : 1; /* Complex type (VT_COMPLEX) */ + uint8_t align4 : 1; /* 64-bit deref only: the accessed address is proven + * >= 4-byte aligned, so LDRD/STRD may be used through + * a general base register (IROperand.align4_ok). */ + uint8_t underalign_hint : 1; /* Base of an indexed access whose chain crossed a + * packed member: address may be < 4-byte aligned, so + * the 64-bit indexed lowering must avoid LDRD/STRD + * (IROperand.underalign_hint). */ + int vreg; /* Original vreg (for debug / liveness queries) */ union { struct diff --git a/source/ir/regalloc.c b/source/ir/regalloc.c index 67ceed40f..33fc1959a 100644 --- a/source/ir/regalloc.c +++ b/source/ir/regalloc.c @@ -21,75 +21,38 @@ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ -#define USING_GLOBALS -#include "ir.h" -#include "regalloc.h" -#include "cfg.h" -#include "ssa.h" -#include "mem_ssa.h" -#include "ssa_opt.h" -#include "opt/ssa/branch.h" -#include "const_string_fold.h" -#include "bitop_const_fold.h" -#include "opt/ssa/branch.h" -#include "opt/ssa/fold.h" -#include "opt/ssa/var_imm_prop.h" -#include "opt/ssa/cprop.h" -#include "global_addr_hoist.h" -#include "load_cse.h" -#include "diamond_store_fwd.h" +/* SSA register allocator: live intervals, phi register hints and + * outgoing-parameter affinity. The rest of the allocator is in + * regalloc_{scan,phi,entry,post}.c; see regalloc_priv.h. */ -#include "opt/ssa/strength.h" -#include "opt/ssa/reassoc.h" -#include "opt/ssa/gvn.h" -#include "opt/ssa/vrp.h" -#include "opt/ssa/setif_or_taut.h" -#include "opt/ssa/bool_norm.h" -#include "opt/ssa/cmp_offset_fold.h" -#include "opt_pipeline.h" -#include "opt.h" -#include "licm.h" -#include "opt_loop_utils.h" +#include "regalloc_priv.h" -#include "memory/bitspan.h" -#include "memory/bit_matrix.h" - - -#define RA_DBG(fmt, ...) LOG_LS(fmt, ##__VA_ARGS__) - -/* ============================================================================ - * SSA Live Interval - * ============================================================================ */ - -typedef struct SSAInterval { - int32_t vreg; - uint32_t start; - uint32_t end; - int8_t r0; - int8_t r1; - int32_t stack_location; - uint8_t crosses_call : 1; - uint8_t crosses_real_call : 1; /* crosses a real FUNCCALL (VFP caller-saved clobber) — excludes native FP-op implicit calls */ - uint8_t addrtaken : 1; - uint8_t is_volatile : 1; /* volatile-qualified local: force to a stack slot so every access is a real ldr/str */ - uint8_t is_param : 1; - uint8_t reg_shared : 1; /* cur shares hr with another active interval (return-block tail); skip expire-free and active push */ - uint8_t loop_phi_locked : 1; /* absorbed a loop-phi partner (carries a loop-carried value across the whole loop body); must not be evicted — spilling it mid-loop would not reload the partner's uses and corrupts the IV */ - uint8_t reg_type; - uint16_t use_count; - uint16_t narrow_uses; /* static count of references from ops with 16-bit encodings (want r0-r7) */ - int8_t precolored; - int8_t pref_reg; /* soft hint: prefer this physical reg if available (e.g. r0 for RETURNVALUE feeders) */ - int32_t hint_vreg; - int32_t coalesce_to; /* graph coalescing: vreg of the representative this one merged into (-1 = rep / not merged) */ - uint8_t co_member; /* 1 if part of a graph-coalesced class (rep or member) — the in-scan transfer must leave it alone */ -} SSAInterval; +/* ra_alive_share deliberately leaves a SECOND live holder on a register the + * allocator still shows as owned by the first, so every site that frees a + * register or hands it on has to ask whether anyone else is still living + * there. The expire path asks it with survivor_int; eviction learned to ask + * it in 030e6fa7. True when some active interval OTHER than `skip` holds hr. */ +int ra_reg_has_other_holder(SSAInterval **active, int active_count, + const SSAInterval *skip, int hr) +{ + if (hr < 0) + return 0; + for (int k = 0; k < active_count; k++) + { + const SSAInterval *a = active[k]; + if (a == skip) + continue; + if (a->r0 == hr || a->r1 == hr) + return 1; + } + return 0; +} /* ============================================================================ * Call-site prefix sum (reused from ir/live.c pattern) * ============================================================================ */ -static int ir_op_is_implicit_call_ra(TccIrOp op) +int ir_op_is_implicit_call_ra(TccIrOp op) { const FloatingPointConfig *fpu = architecture_config.fpu; if (!fpu) @@ -107,8 +70,7 @@ static int ir_op_is_implicit_call_ra(TccIrOp op) default: return 0; } } - -static int *ra_build_call_prefix(TCCIRState *ir) +int *ra_build_call_prefix(TCCIRState *ir) { int n = ir->next_instruction_index; if (n <= 0) @@ -121,7 +83,7 @@ static int *ra_build_call_prefix(TCCIRState *ir) int is_call = (op == TCCIR_OP_FUNCCALLVOID || op == TCCIR_OP_FUNCCALLVAL || op == TCCIR_OP_BUILTIN_APPLY || ir_op_is_implicit_call_ra(op)); /* A large BLOCK_COPY lowers to a memcpy() call in the backend, clobbering - * the caller-saved registers. The inline (small) lowering saves/restores + * the caller-saved registers. The (small) lowering saves/restores * everything it touches, so only the memcpy-sized copies count as calls. */ if (!is_call && op == TCCIR_OP_BLOCK_COPY) { int bc_size = (int)irop_get_imm64_ex(ir, tcc_ir_op_get_src2(ir, q)); @@ -207,7 +169,25 @@ static int ra_has_switch_in_range(const int *prefix, int start, int end, int n) return (prefix[end + 1] - prefix[start + 1]) != 0; } -static const char *ra_vreg_type_char(int type) +/* Prefix sum of INLINE_ASM statements. The allocator does not see an asm + * statement's register effects: its clobber list, and the registers the + * constraint solver picks for operands (r0-r8, or the one a `register ... + * __asm("rN")` variable names), are only known when the backend emits it. + * asm_gen_code preserves callee-saved r4-r11 around the statement but not + * r0-r3/r12, so a value live across an asm must not sit in a caller-saved + * register -- the same constraint a call imposes. */ +static int *ra_build_asm_prefix(TCCIRState *ir) +{ + int n = ir->next_instruction_index; + if (n <= 0) + return NULL; + int *prefix = tcc_malloc(sizeof(int) * (n + 1)); + prefix[0] = 0; + for (int i = 0; i < n; i++) + prefix[i + 1] = prefix[i] + (ir->compact_instructions[i].op == TCCIR_OP_INLINE_ASM); + return prefix; +} +const char *ra_vreg_type_char(int type) { switch (type) { case TCCIR_VREG_TYPE_VAR: return "V"; @@ -216,17 +196,7 @@ static const char *ra_vreg_type_char(int type) default: return "?"; } } - -/* ============================================================================ - * SSA Live Interval Building - * ============================================================================ */ - -/* Defined with the graph coalescer below; used by the back-edge extension to - * classify the instruction at a jump target. */ -static void ra_co_ops(TCCIRState *ir, IRQuadCompact *q, - int32_t *out_def, int *has_def, int32_t uses[4], int *nuse); - -static void ra_build_intervals(TCCIRState *ir, IRCFG *cfg, IRSSAState *ssa, +void ra_build_intervals(TCCIRState *ir, IRCFG *cfg, IRSSAState *ssa, SSAInterval **out_intervals, int *out_count, const int *call_prefix, int *out_max_vreg_pos) { @@ -242,6 +212,7 @@ static void ra_build_intervals(TCCIRState *ir, IRCFG *cfg, IRSSAState *ssa, /* SWITCH_TABLE/SWITCH_LOAD dispatch clobbers R_IP (R12); see * ra_has_switch_in_range below. */ int *switch_prefix = ra_build_switch_prefix(ir); + int *asm_prefix = ra_build_asm_prefix(ir); int *real_call_prefix = ra_build_real_call_prefix(ir); /* Allocate per-vreg start/end tracking indexed by encoded vreg. @@ -291,7 +262,7 @@ static void ra_build_intervals(TCCIRState *ir, IRCFG *cfg, IRSSAState *ssa, int nsrcs = 0; if (irop_config[q->op].has_src1) srcs[nsrcs++] = tcc_ir_op_get_src1(ir, q); if (irop_config[q->op].has_src2) srcs[nsrcs++] = tcc_ir_op_get_src2(ir, q); - if (q->op == TCCIR_OP_MLA) srcs[nsrcs++] = tcc_ir_op_get_accum(ir, q); + if (tcc_ir_op_is_mac(q->op)) srcs[nsrcs++] = tcc_ir_op_get_accum(ir, q); for (int k = 0; k < nsrcs; k++) { int32_t svr = irop_get_vreg(srcs[k]); if (svr < 0 || !tcc_ir_vreg_is_valid(ir, svr)) continue; @@ -338,7 +309,7 @@ static void ra_build_intervals(TCCIRState *ir, IRCFG *cfg, IRSSAState *ssa, } } /* MLA accumulator (4th operand) */ - if (q->op == TCCIR_OP_MLA) { + if (tcc_ir_op_is_mac(q->op)) { IROperand acc = tcc_ir_op_get_accum(ir, q); int32_t vr = irop_get_vreg(acc); if (vr >= 0 && tcc_ir_vreg_is_valid(ir, vr)) { @@ -390,12 +361,18 @@ static void ra_build_intervals(TCCIRState *ir, IRCFG *cfg, IRSSAState *ssa, * That is a register write, not a memory write — without a def here * the temp reads as live-from-entry, and a block-local scratch (an * inlined helper's per-case var) balloons across the whole loop and - * detonates pressure. Full-width writes only: a narrow store leaves - * the temp's upper bytes live-before (partial def). */ + * detonates pressure. A narrow write is a whole def too: into a + * register it is a MOV of the source register, and into the temp's + * spill slot a word store of the value extended (9be4d176), so no byte + * of the old value survives either way. Treating it as a partial def + * made every byte temp an inlined Zig helper leaves behind live from + * entry, pinning a callee-saved register across the whole function -- + * six of them starved std.HashMap's probe loop into stack slots. */ else if (q->op == TCCIR_OP_STORE && vr >= 0 && tcc_ir_vreg_is_valid(ir, vr) && TCCIR_DECODE_VREG_TYPE(vr) == TCCIR_VREG_TYPE_TEMP && !d.is_lval && !d.is_local && - (d.btype == IROP_BTYPE_INT32 || d.btype == IROP_BTYPE_FLOAT32 || + (d.btype == IROP_BTYPE_INT8 || d.btype == IROP_BTYPE_INT16 || + d.btype == IROP_BTYPE_INT32 || d.btype == IROP_BTYPE_FLOAT32 || d.btype == IROP_BTYPE_INT64 || d.btype == IROP_BTYPE_FLOAT64)) { int didx = VREG_IDX(vr); int read_as_src = didx < table_size && @@ -728,6 +705,39 @@ static void ra_build_intervals(TCCIRState *ir, IRCFG *cfg, IRSSAState *ssa, } } while (changed); + /* Pass 2: escapes. The address passed to a call (which may return it, + * as a `mem.asBytes(&x)` wrapper does, or keep it) or stored to memory + * comes back through a value the flow above cannot follow, so the root + * stays live to the function end. Without this, `t = as_bytes(&b); + * ...; memcpy(dst, t, 1)` gave b's slot to a later address-taken VAR + * while t still pointed into it (Zig's autoHash of a u8 hashed the low + * byte of a pointer -- `std.lang is corrupt` from the -O0 compiler). */ + for (int i = 0; i < n; i++) { + IRQuadCompact *q = &ir->compact_instructions[i]; + IROperand v; + if (q->op == TCCIR_OP_FUNCPARAMVAL || q->op == TCCIR_OP_STORE || q->op == TCCIR_OP_RETURNVALUE) + v = tcc_ir_op_get_src1(ir, q); + else + continue; + if (v.is_lval || v.is_llocal) + continue; /* the value read through it, not the address */ + int32_t v_vr = irop_get_vreg(v); + if (v_vr < 0 || !tcc_ir_vreg_is_valid(ir, v_vr)) + continue; + int vidx = VREG_IDX(v_vr); + if (vidx >= table_size) + continue; + int root = -1; + if (irop_get_tag(v) == IROP_TAG_STACKOFF && v.is_local) { + IRLiveInterval *li = tcc_ir_vreg_live_interval(ir, v_vr); + if (li && li->addrtaken) + root = vidx; + } else if (taint_root[vidx] >= 0) + root = taint_root[vidx]; + if (root >= 0 && ends[root] < (uint32_t)(n - 1)) + ends[root] = (uint32_t)(n - 1); + } + tcc_free(taint_root); } @@ -966,9 +976,14 @@ static void ra_build_intervals(TCCIRState *ir, IRCFG *cfg, IRSSAState *ssa, * stays 0 (lets the allocator place it in a caller-saved arg reg). * For other vregs whose end lands at a call (e.g., the function * pointer of an indirect call), keep the conservative crosses_call=1. */ - iv->crosses_real_call = ra_has_call_in_range(real_call_prefix, iv->start, iv->end, n) ? 1 : 0; + /* A parameter is defined before instruction 0, so a call AT 0 is inside + * its range too: a leaf whose body opens with a memcpy-sized BLOCK_COPY + * (a local's rodata initialiser) had its first parameter precolored in + * r0 across the copy. */ + const int xstart = type == TCCIR_VREG_TYPE_PARAM ? -1 : (int)iv->start; + iv->crosses_real_call = ra_has_call_in_range(real_call_prefix, xstart, iv->end, n) ? 1 : 0; if (!iv->crosses_call) { - iv->crosses_call = ra_has_call_in_range(call_prefix, iv->start, iv->end, n); + iv->crosses_call = ra_has_call_in_range(call_prefix, xstart, iv->end, n); if (!iv->crosses_call && iv->end < (uint32_t)n) { TccIrOp eop = ir->compact_instructions[iv->end].op; if (eop == TCCIR_OP_FUNCCALLVAL || eop == TCCIR_OP_FUNCCALLVOID) { @@ -989,7 +1004,13 @@ static void ra_build_intervals(TCCIRState *ir, IRCFG *cfg, IRSSAState *ssa, * (fuzz seed 102: at -O2 the loop-carried checksum `cs` was placed in * R12 and clobbered by the switch dispatch, corrupting the result.) */ if (!iv->crosses_call) - iv->crosses_call = ra_has_switch_in_range(switch_prefix, iv->start, iv->end, n); + iv->crosses_call = ra_has_switch_in_range(switch_prefix, xstart, iv->end, n); + + /* Asm crossing: see ra_build_asm_prefix. Same inclusive-end range test + * as the switch case, since a back-edge extension can end an interval + * exactly at the asm statement it is live across. */ + if (!iv->crosses_call) + iv->crosses_call = ra_has_switch_in_range(asm_prefix, xstart, iv->end, n); /* Params: start at 0, precolor if in register. * Do NOT bump end past its last actual use — the pref_reg boundary @@ -1046,6 +1067,7 @@ static void ra_build_intervals(TCCIRState *ir, IRCFG *cfg, IRSSAState *ssa, if (out_max_vreg_pos) *out_max_vreg_pos = max_vreg_pos; if (switch_prefix) tcc_free(switch_prefix); + if (asm_prefix) tcc_free(asm_prefix); if (real_call_prefix) tcc_free(real_call_prefix); #undef VREG_IDX } @@ -1054,7 +1076,7 @@ static void ra_build_intervals(TCCIRState *ir, IRCFG *cfg, IRSSAState *ssa, * Phi Register Hints * ============================================================================ */ -static void ra_build_phi_hints(SSAInterval *intervals, int count, +void ra_build_phi_hints(SSAInterval *intervals, int count, IRSSAState *ssa, IRCFG *cfg, int max_vreg_pos) { if (!ssa || !ssa->block_phis || !cfg || max_vreg_pos <= 0) @@ -1115,7 +1137,7 @@ static void ra_build_phi_hints(SSAInterval *intervals, int count, * carries `dest = src` copies at each former phi edge. Each such copy * is a place where we'd like dest and src to share a register so the * post-RA move-coalescing pass can erase the mov rX, rX. */ -static void ra_build_assign_hints(SSAInterval *intervals, int count, +void ra_build_assign_hints(SSAInterval *intervals, int count, TCCIRState *ir, int max_vreg_pos) { if (max_vreg_pos <= 0) return; @@ -1190,7 +1212,7 @@ static void ra_build_assign_hints(SSAInterval *intervals, int count, * and must be skipped. INT64 needs a register pair and isn't expressible * as a single-reg hint. is_lval sources are real memory dereferences, * not pass-through copies. */ -static void ra_build_load_param_hints(SSAInterval *intervals, int count, +void ra_build_load_param_hints(SSAInterval *intervals, int count, TCCIRState *ir, int max_vreg_pos) { if (max_vreg_pos <= 0) return; @@ -1256,7 +1278,7 @@ static void ra_build_load_param_hints(SSAInterval *intervals, int count, * (the dominant residual cost of BFI lowering). Soft, one-directional hint: if * the register isn't available the emitter still falls back to the mov, so this * can only remove instructions, never add them. */ -static void ra_build_bfi_hints(SSAInterval *intervals, int count, +void ra_build_bfi_hints(SSAInterval *intervals, int count, TCCIRState *ir, int max_vreg_pos) { if (max_vreg_pos <= 0) return; @@ -1317,7 +1339,7 @@ static void ra_build_bfi_hints(SSAInterval *intervals, int count, * loaded value goes into the arg at codegen time, not T itself) and * 64-bit args (need a register pair which single-reg affinity can't * express). */ -static void ra_build_outgoing_param_hints(SSAInterval *intervals, int count, +void ra_build_outgoing_param_hints(SSAInterval *intervals, int count, TCCIRState *ir, int max_vreg_pos) { if (max_vreg_pos <= 0) return; @@ -1384,6694 +1406,3 @@ static void ra_build_outgoing_param_hints(SSAInterval *intervals, int count, tcc_free(vreg_to_iv); #undef OPH_VREG_IDX } - -/* ============================================================================ - * Linear Scan Allocation - * ============================================================================ */ - -static int sort_by_start(const void *a, const void *b) -{ - const SSAInterval *ia = (const SSAInterval *)a; - const SSAInterval *ib = (const SSAInterval *)b; - if (ia->is_param && !ib->is_param) return -1; - if (!ia->is_param && ib->is_param) return 1; - if (ia->start < ib->start) return -1; - if (ia->start > ib->start) return 1; - /* Equal starts: precolored intervals first, so their fixed registers are - * claimed before the scan hands the same register to a non-precolored - * interval (the precolored assignment does not check int_free). Then by - * vreg — qsort is not stable, and leaving ties unspecified makes the - * allocation depend on the libc's qsort (host glibc and the device libc - * order equal elements differently). */ - if (ia->precolored >= 0 && ib->precolored < 0) return -1; - if (ia->precolored < 0 && ib->precolored >= 0) return 1; - if (ia->vreg < ib->vreg) return -1; - if (ia->vreg > ib->vreg) return 1; - return 0; -} - -/* Safety check for loop-carried phi coalescing. - * - * Returns 1 if cur can take partner's physical register even though their - * live intervals overlap. The case: cur is defined inside the partner's - * live range by an instruction that reads partner as a source, and - * partner's only remaining use past that def is a single ASSIGN with - * dest = partner, src1 = cur (i.e. the explicit back-edge phi copy left - * over after SSA destruction). Under that pattern partner's value is - * "killed" at cur's def: after the defining instruction R holds cur's - * value, the back-edge ASSIGN becomes mov R, R (elided), and R remains - * the carrier for the same logical loop variable across iterations. - * - * Note: cur->start may have been pulled back to the loop entry by the - * back-edge extension (so cur's "official" start is before its actual - * def). We locate the def position by scanning for the first instruction - * in [cur->start, partner->end] whose dest is cur. - * - * Reads of partner *before* the def are fine: at those positions the - * register holds partner's value (which equals what cur will be assigned - * from in the previous iteration via the back-edge ASSIGN). Reads - * *after* the def — except for the back-edge ASSIGN itself — are not - * fine: they would see cur's value, not partner's. */ -static int ra_safe_loop_phi_coalesce(TCCIRState *ir, SSAInterval *cur, SSAInterval *partner) -{ - int n = ir->next_instruction_index; - int cur_start = (int)cur->start; - int partner_end = (int)partner->end; - int32_t cur_vreg = cur->vreg; - int32_t partner_vreg = partner->vreg; - - if (cur_start < 0 || cur_start >= n || partner_end < cur_start || partner_end >= n) - return 0; - - /* cur (the loop update) must be consumed by the back-edge copy partner<-cur - * at partner_end, so cur must NOT outlive partner. If cur->end > partner_end - * then cur and partner are two DISTINCT values that both span the loop body - * (they interfere), and sharing one register conflates them. This was a - * self-host miscompile: the cross coalesced a pointer-holding interval with - * an index-holding one in ra_coalesce_graph (cur.end > partner.end), yielding - * a register used as both index and pointer, which corrupted that pass's own - * coalescing decisions on later compiles. Legitimate loop-IV updates have - * cur.end <= partner.end (the update is dead after the back-edge copy). */ - if ((int)cur->end > partner_end) - return 0; - - /* Locate cur's def: first instruction in [cur_start, partner_end] whose - * non-STORE dest is cur AND which reads partner as a source. */ - int def_pos = -1; - for (int j = cur_start; j <= partner_end; j++) { - IRQuadCompact *q = &ir->compact_instructions[j]; - if (q->op == TCCIR_OP_NOP) continue; - if (!irop_config[q->op].has_dest) continue; - if (q->op == TCCIR_OP_STORE || q->op == TCCIR_OP_STORE_INDEXED || - q->op == TCCIR_OP_STORE_POSTINC) continue; - IROperand d = tcc_ir_op_get_dest(ir, q); - if (!irop_has_vreg(d) || irop_get_vreg(d) != cur_vreg) continue; - - int reads_partner = 0; - if (irop_config[q->op].has_src1) { - IROperand s = tcc_ir_op_get_src1(ir, q); - if (irop_has_vreg(s) && irop_get_vreg(s) == partner_vreg) reads_partner = 1; - } - if (!reads_partner && irop_config[q->op].has_src2) { - IROperand s = tcc_ir_op_get_src2(ir, q); - if (irop_has_vreg(s) && irop_get_vreg(s) == partner_vreg) reads_partner = 1; - } - if (!reads_partner && q->op == TCCIR_OP_MLA) { - IROperand s = tcc_ir_op_get_accum(ir, q); - if (irop_has_vreg(s) && irop_get_vreg(s) == partner_vreg) reads_partner = 1; - } - if (!reads_partner) return 0; - def_pos = j; - break; - } - if (def_pos < 0) return 0; - - int found_back_copy = 0; - for (int j = def_pos + 1; j <= partner_end; j++) { - IRQuadCompact *q = &ir->compact_instructions[j]; - if (q->op == TCCIR_OP_NOP) continue; - - /* cur must be defined only at def_pos. The override's correctness rests on - * "after def_pos the register holds cur's value and the back-edge copy is - * mov R,R"; a *second* def of cur before the back-edge breaks that — the - * register then carries an intermediate value while partner is still - * (textually) live, and coalescing conflates two distinct values. This - * happens when def_pos is a copy `cur <- partner` at the top of an OUTER - * loop body and cur is then re-assigned inside a nested (rotated) inner - * loop before the outer back-edge copy `partner <- cur` (longlong seed 218: - * g12-carried hash T160<-T161, re-defined inside the rotated g16 loop). The - * linear scan cannot model the inner back-edge, so reject conservatively. */ - if (irop_config[q->op].has_dest) { - IROperand cd = tcc_ir_op_get_dest(ir, q); - if (irop_has_vreg(cd) && irop_get_vreg(cd) == cur_vreg) - return 0; - } - - int uses_partner_as_src = 0; - if (irop_config[q->op].has_src1) { - IROperand s = tcc_ir_op_get_src1(ir, q); - if (irop_has_vreg(s) && irop_get_vreg(s) == partner_vreg) - uses_partner_as_src = 1; - } - if (!uses_partner_as_src && irop_config[q->op].has_src2) { - IROperand s = tcc_ir_op_get_src2(ir, q); - if (irop_has_vreg(s) && irop_get_vreg(s) == partner_vreg) - uses_partner_as_src = 1; - } - if (!uses_partner_as_src && q->op == TCCIR_OP_MLA) { - IROperand s = tcc_ir_op_get_accum(ir, q); - if (irop_has_vreg(s) && irop_get_vreg(s) == partner_vreg) - uses_partner_as_src = 1; - } - if (uses_partner_as_src) { - if (q->op != TCCIR_OP_ASSIGN) return 0; - IROperand d = tcc_ir_op_get_dest(ir, q); - if (!irop_has_vreg(d) || irop_get_vreg(d) != partner_vreg) return 0; - IROperand s = tcc_ir_op_get_src1(ir, q); - if (!irop_has_vreg(s) || irop_get_vreg(s) != cur_vreg) return 0; - if (found_back_copy) return 0; - found_back_copy = 1; - continue; - } - - if (irop_config[q->op].has_dest) { - int dest_is_use = (q->op == TCCIR_OP_STORE || q->op == TCCIR_OP_STORE_INDEXED || - q->op == TCCIR_OP_STORE_POSTINC); - if (dest_is_use) { - IROperand d = tcc_ir_op_get_dest(ir, q); - if (irop_has_vreg(d) && irop_get_vreg(d) == partner_vreg) - return 0; - } else { - IROperand d = tcc_ir_op_get_dest(ir, q); - if (irop_has_vreg(d) && irop_get_vreg(d) == partner_vreg) { - if (q->op != TCCIR_OP_ASSIGN) return 0; - IROperand s = tcc_ir_op_get_src1(ir, q); - if (!irop_has_vreg(s) || irop_get_vreg(s) != cur_vreg) return 0; - if (found_back_copy) return 0; - found_back_copy = 1; - } - } - } - } - return found_back_copy; -} - -/* Returns 1 if instruction q references `vreg` as any source or destination - * operand (read or write). Mirrors the operand enumeration ra_build_intervals - * uses (src1, src2, MLA accumulator, dest including STORE-class). */ -static int ra_instr_touches_vreg(TCCIRState *ir, IRQuadCompact *q, int32_t vreg) -{ - if (q->op == TCCIR_OP_NOP) - return 0; - if (irop_config[q->op].has_src1) { - IROperand s = tcc_ir_op_get_src1(ir, q); - if (irop_has_vreg(s) && irop_get_vreg(s) == vreg) return 1; - } - if (irop_config[q->op].has_src2) { - IROperand s = tcc_ir_op_get_src2(ir, q); - if (irop_has_vreg(s) && irop_get_vreg(s) == vreg) return 1; - } - if (q->op == TCCIR_OP_MLA) { - IROperand s = tcc_ir_op_get_accum(ir, q); - if (irop_has_vreg(s) && irop_get_vreg(s) == vreg) return 1; - } - if (irop_config[q->op].has_dest) { - IROperand d = tcc_ir_op_get_dest(ir, q); - if (irop_has_vreg(d) && irop_get_vreg(d) == vreg) return 1; - } - return 0; -} - -/* Control-flow-aware safety check for "exit-phi" coalescing. - * - * cur is defined by a pure copy `cur <- partner` (an ASSIGN whose only source - * is partner — the SSA-destruction copy left at a loop-exit / block-merge - * edge). Linear scan models partner with a single [start,end] interval whose - * end is partner's last *textual* use. Under TCC's inverted-header loop - * layout the loop body is emitted *after* the loop's exit edge, so partner's - * body uses sit textually below the copy and the interval spuriously overlaps - * cur — even though, in the actual control flow, reaching the copy means the - * loop has exited and partner is dead. ra_safe_loop_phi_coalesce only covers - * the back-edge phi (`partner <- cur`), not this forward exit phi. - * - * Returns 1 when no instruction reachable from *after* the copy reads or - * writes partner — i.e. partner is genuinely dead past the copy. cur and - * partner are then the same logical value over disjoint control-flow regions - * and may share a register; the residual `mov R,R` is erased by the post-RA - * move-coalescing pass. - * - * Reachability is computed directly on the current instruction stream (jump - * targets define the edges). The shared `cfg` cannot be used here: it is - * built before ra_resolve_phis inserts these very copies and is stale by the - * time the linear scan runs. Any indirect/multi-target transfer (IJUMP, - * SWITCH_*) whose successors we cannot enumerate forces a conservative - * reject. */ -static int ra_safe_exit_phi_coalesce(TCCIRState *ir, SSAInterval *cur, SSAInterval *partner) -{ - int n = ir->next_instruction_index; - int32_t cur_vreg = cur->vreg; - int32_t partner_vreg = partner->vreg; - if (cur->start < 0 || (int)cur->start >= n) - return 0; - - /* cur must be partner's forward continuation, which means it outlives - * partner: cur->end >= partner->end. When partner outlives cur, partner is - * still needed on some path after cur dies (e.g. a parameter live across all - * arms of a switch while cur is a copy in one arm); transferring partner's - * register to cur and dropping partner from the active set would then free - * the register while partner is still live elsewhere, corrupting it. */ - if ((int)partner->end > (int)cur->end) - return 0; - - /* Locate cur's first def and require it to be a pure copy `cur <- partner`. */ - int def_pos = -1; - for (int j = (int)cur->start; j < n; j++) { - IRQuadCompact *q = &ir->compact_instructions[j]; - if (q->op == TCCIR_OP_NOP) continue; - if (!irop_config[q->op].has_dest) continue; - IROperand d = tcc_ir_op_get_dest(ir, q); - if (!irop_has_vreg(d) || irop_get_vreg(d) != cur_vreg) continue; - if (q->op != TCCIR_OP_ASSIGN) return 0; - IROperand s = tcc_ir_op_get_src1(ir, q); - if (!irop_has_vreg(s) || irop_get_vreg(s) != partner_vreg) return 0; - def_pos = j; - break; - } - if (def_pos < 0) - return 0; - - /* DFS over instructions reachable from after the copy. Each instruction is - * marked visited *when pushed*, so it is enqueued at most once and the stack - * never exceeds n entries. If a back-edge re-enters def_pos itself, the - * copy's own read of partner is seen and the case is conservatively rejected. */ - uint8_t *visited = tcc_mallocz(n); - int *stack = tcc_malloc(sizeof(int) * n); - int sp = 0, dead = 1; -#define RA_EXITPHI_PUSH(x) do { int _x = (x); \ - if (_x >= 0 && _x < n && !visited[_x]) { visited[_x] = 1; stack[sp++] = _x; } } while (0) - RA_EXITPHI_PUSH(def_pos + 1); - - while (sp > 0) { - int i = stack[--sp]; - IRQuadCompact *q = &ir->compact_instructions[i]; - if (q->op == TCCIR_OP_NOP) { - RA_EXITPHI_PUSH(i + 1); - continue; - } - if (ra_instr_touches_vreg(ir, q, partner_vreg)) { dead = 0; break; } - - if (q->op == TCCIR_OP_RETURNVOID || q->op == TCCIR_OP_RETURNVALUE || - q->op == TCCIR_OP_TRAP) - continue; /* no successor */ - if (q->op == TCCIR_OP_IJUMP || q->op == TCCIR_OP_SWITCH_TABLE || - q->op == TCCIR_OP_SWITCH_LOAD) { dead = 0; break; } /* unknown targets */ - if (q->op == TCCIR_OP_JUMP) { - RA_EXITPHI_PUSH((int)irop_get_imm64_ex(ir, tcc_ir_op_get_dest(ir, q))); - continue; - } - if (q->op == TCCIR_OP_JUMPIF) { - RA_EXITPHI_PUSH((int)irop_get_imm64_ex(ir, tcc_ir_op_get_dest(ir, q))); - RA_EXITPHI_PUSH(i + 1); - continue; - } - /* Ordinary instruction (including calls — keep the fall-through; a - * noreturn call's dead fall-through only makes the check more - * conservative). */ - RA_EXITPHI_PUSH(i + 1); - } -#undef RA_EXITPHI_PUSH - - /* The reachable set is now complete in `visited` (the loop only breaks early - * when dead is already 0). cur may legitimately take partner's register - * only if every definition of cur is one of: - * (a) the copy itself (def_pos), or - * (b) reachable from the copy — cur is the forward continuation of partner - * (e.g. the loop increment), or - * (c) another pure copy `cur <- partner` from the SAME partner — a parallel - * SSA phi copy on a different incoming edge of the same merge; it also - * collapses to mov R,R once cur and partner share R, so it is harmless. - * A def from a DIFFERENT source on a sibling path (a true merge phi, e.g. a - * return-value phi `ret <- a` / `ret <- b`) makes cur hold a value unrelated - * to partner there; coalescing cur into partner's register would corrupt it. */ - if (dead) { - for (int j = 0; j < n; j++) { - if (j == def_pos) continue; - IRQuadCompact *q = &ir->compact_instructions[j]; - if (q->op == TCCIR_OP_NOP || !irop_config[q->op].has_dest) continue; - if (q->op == TCCIR_OP_STORE || q->op == TCCIR_OP_STORE_INDEXED || - q->op == TCCIR_OP_STORE_POSTINC) continue; - IROperand d = tcc_ir_op_get_dest(ir, q); - if (!irop_has_vreg(d) || irop_get_vreg(d) != cur_vreg) continue; - if (visited[j]) continue; /* (b) forward continuation */ - /* (c) a parallel `cur <- partner` copy is fine; anything else rejects. */ - int parallel_copy = 0; - if (q->op == TCCIR_OP_ASSIGN) { - IROperand s = tcc_ir_op_get_src1(ir, q); - if (irop_has_vreg(s) && irop_get_vreg(s) == partner_vreg) - parallel_copy = 1; - } - if (!parallel_copy) { dead = 0; break; } - } - } - - tcc_free(visited); - tcc_free(stack); - return dead; -} - -/* Pre-allocation frame-pointer prediction. The real decision is made after - * allocation — tcc_ir_codegen_generate forces FP for VLA, soft-float ops and - * static chain; the Thumb prologue for variadic functions and force_lr_save — - * so freeing the FP register for allocation is safe only when none of those - * can fire. This must remain a SUPERSET of every post-allocation forcing - * site: add the condition HERE FIRST when adding one there. Inline asm does - * not force a frame pointer, but its text can name r7 outright, which the - * allocator cannot see — keep the register reserved in such functions. */ -/* ============================================================================ - * Accurate per-vreg liveness, for the allocator's last-resort register share - * - * ra_build_intervals models every value as ONE contiguous [start,end] range. - * That is a strict over-approximation across a diamond: a value defined above - * an if/else and read only in ONE arm looks live in the other arm too, so the - * linear scan sees a false conflict and spills a value that could have had a - * register for free. `__aeabi_dadd`'s alignment block is the shape: `exp_diff` - * is read in the `exp_diff > 0` arm and again in the `exp_diff < 0` arm, so its - * register looks busy in BOTH -- and the 64-bit temp defined there was written - * to the frame and read back at the very next instruction. - * - * This computes real liveness (backward dataflow over the CFG, the same - * def/use model the graph coalescer trusts, deferred PARAM sources included) - * and records, per vreg, every instruction index at which it is live-in or - * live-out. ra_linear_scan consults it ONLY when it is about to spill: if an - * active interval is provably dead across the whole of the new interval's - * range, the two cannot conflict on any path and can share one register. - * - * Unknown answers are reported BUSY, so a failure to build (un-enumerated CFG - * edges, a vreg outside the table, an oversized function) can only cost a - * sharing opportunity, never create a clobber. - * ========================================================================== */ -typedef struct RaAliveInfo -{ - int valid; - int n; /* instruction indices covered: [0, n) */ - int tbl; /* vreg slot table size */ - int maxpos; - int words; /* uint64 words per vreg row */ - int ndense; - int *dense; /* tbl entries: slot -> row index, or -1 */ - uint64_t *rows; /* ndense rows of `words` words */ -} RaAliveInfo; - -#define RA_ALIVE_VIDX(info, vr) \ - ((TCCIR_DECODE_VREG_TYPE(vr) * (info)->maxpos) + TCCIR_DECODE_VREG_POSITION(vr)) - -/* Cap: a row per vreg over the whole instruction range. Beyond this the - * bitmap is not worth the compile time or the memory, and the allocator just - * keeps its previous behaviour. */ -#define RA_ALIVE_MAX_WORDS (1 << 18) /* 2 MiB */ - -/* Row of instruction indices at which `vreg` is live (live-in or live-out), - * or NULL when the answer is not known. */ -static const uint64_t *ra_alive_row(const RaAliveInfo *info, int32_t vreg) -{ - if (!info || !info->valid) - return NULL; - int vi = RA_ALIVE_VIDX(info, vreg); - if (vi < 0 || vi >= info->tbl) - return NULL; - int d = info->dense[vi]; - if (d < 0) - return NULL; - return info->rows + (size_t)d * (size_t)info->words; -} - -/* 1 if `vreg` may be live anywhere in [lo,hi]. Conservative: unknown -> 1. - * - * The range asked about is the NEW interval's [start,end] -- the window over - * which the allocator guarantees it the register -- and NOT its accurate live - * set. Those differ: the interval over-approximates, and a donor live inside - * one of its holes has still had its value destroyed by the def at `start`. - * Testing the two live SETS against each other instead looks stronger and is - * wrong; it miscompiles bench_double.c's Mandelbrot kernel. */ -static int ra_alive_busy(const RaAliveInfo *info, int32_t vreg, uint32_t lo, uint32_t hi) -{ - const uint64_t *row = ra_alive_row(info, vreg); - if (!row) - return 1; - if (hi >= (uint32_t)info->n) - hi = (uint32_t)info->n - 1; - if (lo > hi) - return 1; - for (uint32_t k = lo; k <= hi; k++) - { - uint32_t w = k >> 6; - if ((k & 63) == 0 && row[w] == 0 && k + 63 <= hi) - { - k += 63; - continue; - } - if (row[w] & (1ull << (k & 63))) - return 1; - } - return 0; -} - -static void ra_alive_free(RaAliveInfo *info) -{ - if (!info) - return; - tcc_free(info->dense); - tcc_free(info->rows); - memset(info, 0, sizeof *info); -} - -/* Cheap pre-check so the dataflow is not built for a function that could never - * use it. A graph-coalesced class is disqualifying by construction: the - * representative's register carries the whole class, which its own vreg's - * liveness does not describe. The return-block share is per-interval and is - * handled at the donor (`reg_shared`) plus an interlock in the scan. */ -static int ra_alive_worth_building(TCCIRState *ir, const SSAInterval *intervals, int count) -{ - if (tcc_state->optimize < 1) - return 0; - if (tcc_ir_opt_pass_disabled("ra:alive_share")) - return 0; - for (int i = 0; i < count; i++) - if (intervals[i].coalesce_to >= 0 || intervals[i].co_member) - return 0; - /* A back-edge is disqualifying, and MEASURED so: with loops admitted - * double_add improves a further 1.9% and bench_double.c's Mandelbrot kernel - * MISCOMPILES. A loop puts values on a register that no single vreg's - * liveness describes -- the loop-phi absorb is only the visible half -- so - * the donor filters are not sufficient there. */ - const int n = ir->next_instruction_index; - for (int i = 0; i < n; i++) - { - IRQuadCompact *q = &ir->compact_instructions[i]; - if (q->op != TCCIR_OP_JUMP && q->op != TCCIR_OP_JUMPIF) - continue; - int t = (int)irop_get_imm64_ex(ir, tcc_ir_op_get_dest(ir, q)); - if (t >= 0 && t <= i) - return 0; - } - return 1; -} - -static void ra_alive_build(TCCIRState *ir, RaAliveInfo *info, int max_vreg_pos) -{ - memset(info, 0, sizeof *info); - const int n = ir->next_instruction_index; - if (n <= 0 || max_vreg_pos <= 0) - return; - if (tcc_state->optimize < 1) - return; - /* Un-enumerated edges make the dataflow unsound; same guard the coalescer - * and the scratch-liveness refinement use. */ - for (int i = 0; i < n; i++) - { - int op = ir->compact_instructions[i].op; - if (op == TCCIR_OP_IJUMP || op == TCCIR_OP_SWITCH_TABLE || op == TCCIR_OP_SWITCH_LOAD) - return; - } - - IRCFG *cfg = tcc_ir_cfg_build(ir); - if (!cfg) - return; - tcc_ir_cfg_compute_rpo(cfg); - const int nb = cfg->num_blocks; - if (nb <= 0) - { - tcc_ir_cfg_free(cfg); - return; - } - - const int maxpos = max_vreg_pos; - const int tbl = 4 * maxpos; - #define AVIDX(vr) ((TCCIR_DECODE_VREG_TYPE(vr) * maxpos) + TCCIR_DECODE_VREG_POSITION(vr)) - - int *dense = tcc_malloc(sizeof(int) * tbl); - for (int i = 0; i < tbl; i++) - dense[i] = -1; - int ndense = 0; - for (int i = 0; i < n; i++) - { - int32_t def = -1, hd = 0, uses[4], nu = 0; - ra_co_ops(ir, &ir->compact_instructions[i], &def, &hd, uses, &nu); - for (int k = 0; k < nu; k++) - { - if (!tcc_ir_vreg_is_valid(ir, uses[k])) continue; - int u = AVIDX(uses[k]); - if (u >= 0 && u < tbl && dense[u] < 0) dense[u] = ndense++; - } - if (hd && tcc_ir_vreg_is_valid(ir, def)) - { - int d = AVIDX(def); - if (d >= 0 && d < tbl && dense[d] < 0) dense[d] = ndense++; - } - } - if (ndense == 0) - { - tcc_free(dense); - tcc_ir_cfg_free(cfg); - return; - } - - const int words = (n + 63) / 64; - if ((size_t)ndense * (size_t)words > RA_ALIVE_MAX_WORDS) - { - tcc_free(dense); - tcc_ir_cfg_free(cfg); - return; - } - - /* Deferred PARAM sources are read again at their CALL, after anything - * between the PARAM quad and the call; ra_build_intervals extends ends for - * exactly this, and the liveness has to model it or a share would clobber an - * argument already marshalled. */ - int *call_param_head = tcc_malloc(sizeof(int) * n); - int *param_sibling = tcc_malloc(sizeof(int) * n); - for (int i = 0; i < n; i++) { call_param_head[i] = -1; param_sibling[i] = -1; } - for (int i = 0; i < n; i++) - { - IRQuadCompact *q = &ir->compact_instructions[i]; - if (q->op != TCCIR_OP_FUNCPARAMVAL) continue; - IROperand s1 = tcc_ir_op_get_src1(ir, q); - if (!irop_has_vreg(s1) || irop_is_immediate(s1)) continue; - int cid = TCCIR_DECODE_CALL_ID(irop_get_imm64_ex(ir, tcc_ir_op_get_src2(ir, q))); - if (cid < 0) continue; - for (int j = i + 1; j < n; j++) - { - IRQuadCompact *qq = &ir->compact_instructions[j]; - if (qq->op != TCCIR_OP_FUNCCALLVOID && qq->op != TCCIR_OP_FUNCCALLVAL) continue; - if (TCCIR_DECODE_CALL_ID(irop_get_imm64_ex(ir, tcc_ir_op_get_src2(ir, qq))) == cid) - { - param_sibling[i] = call_param_head[j]; - call_param_head[j] = i; - break; - } - } - } - #define AV_FOR_PARAM_USES(call_idx, vr_, body) \ - do { \ - for (int p_ = call_param_head[(call_idx)]; p_ >= 0; p_ = param_sibling[p_]) { \ - IROperand ps_ = tcc_ir_op_get_src1(ir, &ir->compact_instructions[p_]); \ - int32_t vr_ = irop_get_vreg(ps_); \ - if (!tcc_ir_vreg_is_valid(ir, vr_)) continue; \ - body \ - } \ - } while (0) - - const int bw = (ndense + 63) / 64; - uint64_t *use_b = tcc_mallocz(sizeof(uint64_t) * (size_t)nb * bw); - uint64_t *def_b = tcc_mallocz(sizeof(uint64_t) * (size_t)nb * bw); - uint64_t *in_b = tcc_mallocz(sizeof(uint64_t) * (size_t)nb * bw); - uint64_t *out_b = tcc_mallocz(sizeof(uint64_t) * (size_t)nb * bw); - - for (int b = 0; b < nb; b++) - { - uint64_t *ub = use_b + (size_t)b * bw, *db = def_b + (size_t)b * bw; - int s = cfg->blocks[b].start_idx, e = cfg->blocks[b].end_idx; - for (int i = s; i < e && i < n; i++) - { - int32_t def = -1, hd = 0, uses[4], nu = 0; - ra_co_ops(ir, &ir->compact_instructions[i], &def, &hd, uses, &nu); - for (int k = 0; k < nu; k++) - { - if (!tcc_ir_vreg_is_valid(ir, uses[k])) continue; - int u = AVIDX(uses[k]); if (u < 0 || u >= tbl || dense[u] < 0) continue; - if (!tcc_bitspan_test(db, dense[u])) tcc_bitspan_set(ub, dense[u]); - } - AV_FOR_PARAM_USES(i, vr, { - int u = AVIDX(vr); if (u < 0 || u >= tbl || dense[u] < 0) continue; - if (!tcc_bitspan_test(db, dense[u])) tcc_bitspan_set(ub, dense[u]); - }); - if (hd && tcc_ir_vreg_is_valid(ir, def)) - { - int d = AVIDX(def); if (d >= 0 && d < tbl && dense[d] >= 0) tcc_bitspan_set(db, dense[d]); - } - } - } - - int changed = 1, guard = 0; - while (changed && guard++ < nb + 4) - { - changed = 0; - for (int ri = cfg->rpo_count - 1; ri >= 0; ri--) - { - int b = cfg->rpo_order ? cfg->rpo_order[ri] : ri; - if (b < 0 || b >= nb) continue; - uint64_t *lo = out_b + (size_t)b * bw, *li = in_b + (size_t)b * bw; - uint64_t *ub = use_b + (size_t)b * bw, *db = def_b + (size_t)b * bw; - tcc_bitspan_zero(lo, bw); - for (int si = 0; si < cfg->blocks[b].num_succs; si++) - { - int sb = cfg->blocks[b].succs[si]; - if (sb < 0 || sb >= nb) continue; - tcc_bitspan_or(lo, in_b + (size_t)sb * bw, bw); - } - changed |= tcc_bitspan_or_andnot(li, ub, lo, db, bw); - } - } - - uint64_t *rows = tcc_mallocz(sizeof(uint64_t) * (size_t)ndense * words); - uint64_t *live = tcc_malloc(sizeof(uint64_t) * bw); - for (int b = 0; b < nb; b++) - { - int s = cfg->blocks[b].start_idx, e = cfg->blocks[b].end_idx; - tcc_bitspan_copy(live, out_b + (size_t)b * bw, bw); - for (int i = (e < n ? e : n) - 1; i >= s; i--) - { - /* Mark live-OUT of i, then live-IN of i: a vreg counted at either end - * must keep its register across i. */ - tcc_bitspan_for_each_set(live, bw, ndense, di) - { - rows[(size_t)di * words + (i >> 6)] |= (1ull << (i & 63)); - } - int32_t def = -1, hd = 0, uses[4], nu = 0; - ra_co_ops(ir, &ir->compact_instructions[i], &def, &hd, uses, &nu); - if (hd && tcc_ir_vreg_is_valid(ir, def)) - { - int d = AVIDX(def); - if (d >= 0 && d < tbl && dense[d] >= 0) tcc_bitspan_reset(live, dense[d]); - } - for (int k = 0; k < nu; k++) - { - if (!tcc_ir_vreg_is_valid(ir, uses[k])) continue; - int u = AVIDX(uses[k]); if (u < 0 || u >= tbl || dense[u] < 0) continue; - tcc_bitspan_set(live, dense[u]); - } - AV_FOR_PARAM_USES(i, vr, { - int u = AVIDX(vr); if (u < 0 || u >= tbl || dense[u] < 0) continue; - tcc_bitspan_set(live, dense[u]); - }); - tcc_bitspan_for_each_set(live, bw, ndense, di) - { - rows[(size_t)di * words + (i >> 6)] |= (1ull << (i & 63)); - } - } - } - - tcc_free(live); - tcc_free(use_b); tcc_free(def_b); tcc_free(in_b); tcc_free(out_b); - tcc_free(call_param_head); tcc_free(param_sibling); - tcc_ir_cfg_free(cfg); - #undef AV_FOR_PARAM_USES - #undef AVIDX - - info->valid = 1; - info->n = n; - info->tbl = tbl; - info->maxpos = maxpos; - info->words = words; - info->ndense = ndense; - info->dense = dense; - info->rows = rows; -} - -static int ra_may_need_frame_pointer(const TCCIRState *ir) -{ - if (tcc_state->force_frame_pointer || tcc_state->force_lr_save) - return 1; - if (func_var) - return 1; - if (ir->has_static_chain || tcc_state->nb_nested_funcs > 0) - return 1; - for (int i = 0; i < ir->next_instruction_index; i++) - { - const int op = ir->compact_instructions[i].op; - if (op == TCCIR_OP_VLA_ALLOC) - return 1; - if (op >= TCCIR_OP_FADD && op <= TCCIR_OP_CVT_FTOI) - return 1; - if (op == TCCIR_OP_ASM_INPUT || op == TCCIR_OP_INLINE_ASM || op == TCCIR_OP_ASM_OUTPUT) - return 1; - } - return 0; -} - -/* Can physical register `r` be handed to `cur` even though an active interval - * still owns it? Yes exactly when EVERY active interval holding it is provably - * dead across the whole of cur's range: then no execution reads that register's - * old value from inside the range, and on the paths where the old value IS read - * later, cur was never defined and never wrote it. - * - * Every donor property that would make its vreg's own liveness an incomplete - * description of what the register holds -- a coalesced class, a loop-phi - * absorb, a return-block share, an address-taken home -- disqualifies it. */ -static int ra_alive_reg_shareable(const RaAliveInfo *alive, SSAInterval *cur, - SSAInterval **active, int active_count, int r, - const RegAllocTarget *target) -{ - if (!alive || !alive->valid) - return 0; - if (r < 0 || r >= tcc_state->registers_for_allocator) - return 0; - if (cur->crosses_call) - { - int ok = 0; - for (int ci = 0; ci < target->int_class.num_callee_saved; ci++) - if (target->int_class.callee_saved[ci] == r) { ok = 1; break; } - if (!ok) - return 0; - } - int holders = 0; - for (int j = 0; j < active_count; j++) - { - SSAInterval *a = active[j]; - if (a == cur || a->stack_location != 0) - continue; - if (a->r0 != r && a->r1 != r) - continue; - holders++; - if (a->precolored >= 0 || a->co_member || a->loop_phi_locked || a->reg_shared || a->addrtaken) - return 0; - if (a->reg_type == LS_REG_TYPE_FLOAT || a->reg_type == LS_REG_TYPE_DOUBLE) - return 0; - if (ra_alive_busy(alive, (int32_t)a->vreg, cur->start, cur->end)) - return 0; - } - return holders > 0; -} - -static void ra_linear_scan(TCCIRState *ir, SSAInterval *intervals, int count, - const RegAllocTarget *target, int spill_base, - uint64_t *out_dirty_int, uint64_t *out_dirty_fp, - int max_vreg_pos, int has_call, const RaAliveInfo *alive) -{ - if (count <= 0) return; - - qsort(intervals, count, sizeof(SSAInterval), sort_by_start); - - /* Accurate-liveness register sharing; ra_alive_worth_building did the - * gating, and a failed build leaves `valid` clear. - * - * The return-block share below (`reg_shared`) is the one other mechanism - * that puts a second owner on a register, and it leaves that owner OUT of - * `active` -- so neither mechanism can see the other's extra holder. The - * two are interlocked: whichever fires first in a function locks the other - * out for the rest of it. */ - int share_ok = (alive && alive->valid); - int alive_shared_used = 0; - int reg_shared_used = 0; - - /* Build vreg -> interval lookup for phi hint resolution */ - int hint_tbl_size = (max_vreg_pos > 0) ? 4 * max_vreg_pos : 1; - SSAInterval **vreg_to_iv = tcc_mallocz(sizeof(SSAInterval *) * hint_tbl_size); - #define HINT_IDX(vr) \ - ((TCCIR_DECODE_VREG_TYPE(vr) * max_vreg_pos) + TCCIR_DECODE_VREG_POSITION(vr)) - for (int i = 0; i < count; i++) { - int idx = HINT_IDX(intervals[i].vreg); - if (idx >= 0 && idx < hint_tbl_size) - vreg_to_iv[idx] = &intervals[i]; - } - - /* Register availability bitmaps */ - uint64_t int_avail = 0; - for (int i = 0; i < target->int_class.num_caller_saved; i++) - int_avail |= (1ull << target->int_class.caller_saved[i]); - for (int i = 0; i < target->int_class.num_callee_saved; i++) - int_avail |= (1ull << target->int_class.callee_saved[i]); - - uint64_t fp_avail = 0; - for (int i = 0; i < target->fp_class.num_caller_saved; i++) - fp_avail |= (1ull << target->fp_class.caller_saved[i]); - for (int i = 0; i < target->fp_class.num_callee_saved; i++) - fp_avail |= (1ull << target->fp_class.callee_saved[i]); - - /* Respect tcc_state->registers_for_allocator limit */ - uint64_t int_allowed = int_avail & tcc_state->registers_map_for_allocator; - uint64_t fp_allowed = fp_avail & tcc_state->float_registers_map_for_allocator; - - /* Nested-function trampolines load the static chain into the static-chain - * register (R10) and tail-jump to the nested function, clobbering R10 - * without restoring it — even though R10 is AAPCS callee-saved. A parent - * that defines nested functions calls them (directly, or via a function - * pointer through an ABI-compliant helper) and would see any value the - * allocator parked in R10 corrupted across that call (nestfunc-2: the - * i/j/k loop counters lived in R10 and the inner foo() call clobbered it → - * infinite loop). Reserve R10 in such parents. Nested functions - * themselves (has_static_chain) hold the live chain in R10 and are handled - * separately, so leave their allocation untouched. */ - if (tcc_state->nb_nested_funcs > 0 && !ir->has_static_chain) - int_allowed &= ~(1ull << (uint64_t)architecture_config.static_chain_reg); - - /* The frame-pointer register (R7 on Thumb) is held out of the target's - * allocator map because the backend needs it as the frame base — but only - * in the minority of functions that actually get a frame pointer. The FP - * decision is made AFTER allocation (tcc_ir_codegen_generate and the - * prologue), so free the register here only when no forcing condition can - * fire. ra_may_need_frame_pointer must stay a SUPERSET of those - * conditions: predicting FP when none materializes just leaves the - * register unused, but the reverse would hand out the frame base. The - * prologue hard-fails if they ever disagree. A low register matters - * doubly on Thumb: it takes 16-bit encodings that R8/R10/R11 cannot, and - * every value it holds is one fewer spill. */ - if (target->frame_pointer_reg >= 0 && !ra_may_need_frame_pointer(ir)) - int_allowed |= int_avail & (1ull << (uint64_t)target->frame_pointer_reg); - - /* Withhold one register in functions that address shared .rodata often. - * Each such reference otherwise reloads the segment's runtime base from the - * GOT through a scratch register it has to push and pop; parking the base in - * a callee-saved register for the whole body turns every one of them into a - * bare ADD. The backend claims the register after allocation, from whatever - * this pass left free (tcc_gen_machine_rodata_anchor_claim), so all this has - * to do is leave one unallocated — and only where the references pay for it, - * since a withheld register is one fewer for values. Purely a size trade: - * if the backend declines to claim, the register is merely unused. */ - if (target->rodata_anchor_reg >= 0 && target->rodata_anchor_sites && - target->rodata_anchor_sites(ir, RA_RODATA_ANCHOR_MIN_SITES) >= RA_RODATA_ANCHOR_MIN_SITES) - int_allowed &= ~(1ull << (uint64_t)target->rodata_anchor_reg); - - uint64_t int_free = int_allowed; - uint64_t fp_free = fp_allowed; - uint64_t dirty_int = 0; - uint64_t dirty_fp = 0; - - /* Active set sorted by end point */ - SSAInterval **active = tcc_malloc(sizeof(SSAInterval *) * count); - int active_count = 0; - - /* Active addrtaken intervals — tracked separately because the main `active` - * set is for register-resident intervals; addrtaken intervals always spill - * and were previously dropped on the floor. We track them so their stack - * slots can be returned to a free list once the interval ends. */ - SSAInterval **active_addrtaken = tcc_malloc(sizeof(SSAInterval *) * count); - int active_addrtaken_count = 0; - uint32_t active_addrtaken_min_end = UINT32_MAX; - - /* Free list of expired 4-byte addrtaken stack slots, available for reuse - * by later addrtaken intervals. Only 4-byte slots are tracked here; other - * sizes fall through to fresh allocation, matching the legacy behavior of - * always assigning `spill_loc -= 4` regardless of value width. */ - int *free_slots_4 = tcc_malloc(sizeof(int) * count); - int free_slots_4_count = 0; - - int spill_loc = spill_base; - - for (int i = 0; i < count; i++) { - SSAInterval *cur = &intervals[i]; - - /* Graph coalescing: non-representative members are merged into their - * representative's interval and inherit its register after the scan. Skip - * them so they neither consume a register nor enter the active set. */ - if (cur->coalesce_to >= 0) - continue; - - /* Expire old intervals. - * - * First compute which hard registers the SURVIVING (still-live) intervals - * still occupy. A register may be held by two active intervals at once - * when a phi/merge copy coalesced them onto the same register: e.g. an - * if/else merge temp for a variable inherits the register from one arm - * (via its single hint_vreg) while the other arm's interval is still live - * and also holds that register. If the shorter of the two expires first, - * returning its register to the free pool would let a later allocation - * (a loop-body temp, say) clobber the value the longer, still-live - * interval needs after the loop. Only free a register when no surviving - * interval still holds it. */ - uint64_t survivor_int = 0, survivor_fp = 0; - for (int j = 0; j < active_count; j++) { - SSAInterval *a = active[j]; - if (a->end >= cur->start && a->r0 >= 0 && a->stack_location == 0) { - if (a->reg_type == LS_REG_TYPE_FLOAT || a->reg_type == LS_REG_TYPE_DOUBLE) { - survivor_fp |= (1ull << a->r0); - if (a->r1 >= 0) survivor_fp |= (1ull << a->r1); - } else { - survivor_int |= (1ull << a->r0); - if (a->r1 >= 0) survivor_int |= (1ull << a->r1); - } - } - } - int w = 0; - for (int j = 0; j < active_count; j++) { - SSAInterval *a = active[j]; - if (a->end < cur->start) { - /* Free register — but not if cur shared hr with another active - * interval (reg_shared): the partner still logically owns hr, - * so freeing it here would let a later allocation clobber the - * loop body's view of partner. Likewise skip any register a - * surviving active interval still holds (coalesced sharing). */ - if (a->r0 >= 0 && a->stack_location == 0 && !a->reg_shared) { - if (a->reg_type == LS_REG_TYPE_FLOAT || a->reg_type == LS_REG_TYPE_DOUBLE) { - if (!(survivor_fp & (1ull << a->r0))) fp_free |= (1ull << a->r0); - if (a->r1 >= 0 && !(survivor_fp & (1ull << a->r1))) fp_free |= (1ull << a->r1); - } else { - if (!(survivor_int & (1ull << a->r0))) int_free |= (1ull << a->r0); - if (a->r1 >= 0 && !(survivor_int & (1ull << a->r1))) int_free |= (1ull << a->r1); - } - } - } else { - active[w++] = a; - } - } - active_count = w; - - /* Expire old addrtaken intervals — return their 4-byte slots to the - * free list so later non-overlapping addrtaken intervals can reuse them. - * Skip the scan when no active entry can expire (min_end >= cur->start). */ - if (active_addrtaken_count > 0 && active_addrtaken_min_end < cur->start) { - int wa = 0; - active_addrtaken_min_end = UINT32_MAX; - for (int j = 0; j < active_addrtaken_count; j++) { - SSAInterval *a = active_addrtaken[j]; - if (a->end < cur->start) { - free_slots_4[free_slots_4_count++] = a->stack_location; - } else { - active_addrtaken[wa++] = a; - if (a->end < active_addrtaken_min_end) - active_addrtaken_min_end = a->end; - } - } - active_addrtaken_count = wa; - } - - /* Address-taken: force spill. - * Reuse an expired addrtaken slot when one is available; otherwise grow - * the spill area. Slot reuse is correct here because the addrtaken - * extension pass in ra_build_intervals has already pushed V's end past - * the death of any pointer derived from V — so two intervals with - * non-overlapping (extended) lifetimes truly access disjoint memory - * windows. Track in active_addrtaken so the slot returns to the free - * list when cur expires. - * A volatile local is forced to memory the same way: every access must - * be a real ldr/str, so it must never occupy a register. */ - if (cur->addrtaken || cur->is_volatile) { - if (free_slots_4_count > 0) { - cur->stack_location = free_slots_4[--free_slots_4_count]; - } else { - spill_loc -= 4; - cur->stack_location = spill_loc; - } - active_addrtaken[active_addrtaken_count++] = cur; - if (cur->end < active_addrtaken_min_end) - active_addrtaken_min_end = cur->end; - continue; - } - - /* Precolored: assign fixed register */ - if (cur->precolored >= 0) { - int reg = cur->precolored; - cur->r0 = reg; - if (LS_IS_VFP_REG(reg)) { - /* Hard-float float param precolored to its incoming VFP register. */ - fp_free &= ~(1ull << (uint64_t)LS_VFP_REG_NUM(reg)); - dirty_fp |= (1ull << (uint64_t)LS_VFP_REG_NUM(reg)); - } else { - int_free &= ~(1ull << reg); - dirty_int |= (1ull << reg); - } - /* Insert into active set */ - active[active_count++] = cur; - continue; - } - - /* Float/double: use FP class */ - if (cur->reg_type == LS_REG_TYPE_FLOAT) { - int reg = -1; - /* Every allocatable single-precision VFP register (s0-s13) is caller-saved - * and there are no callee-saved VFP registers in the allocation map, so a - * float that is live across a REAL call cannot stay in a VFP register — - * force it to a stack slot. Native FP-op "implicit calls" (a single- - * precision vadd.f32) clobber nothing, so they do not force a spill. */ - if (!cur->crosses_real_call && fp_free) { - reg = __builtin_ctzll(fp_free); - fp_free &= ~(1ull << reg); - dirty_fp |= (1ull << reg); - } - if (reg >= 0) { - cur->r0 = LS_VFP_REG_BASE + reg; - active[active_count++] = cur; - } else { - spill_loc -= 4; - cur->stack_location = spill_loc; - } - continue; - } - - if (cur->reg_type == LS_REG_TYPE_DOUBLE) { - /* Need even-aligned pair */ - int reg = -1; - for (int r = 0; r < 30; r += 2) { - if ((fp_free & (3ull << r)) == (3ull << r)) { - reg = r; - break; - } - } - if (reg >= 0) { - fp_free &= ~(3ull << reg); - dirty_fp |= (3ull << reg); - cur->r0 = LS_VFP_REG_BASE + reg; - cur->r1 = LS_VFP_REG_BASE + reg + 1; - active[active_count++] = cur; - } else { - spill_loc -= 8; - cur->stack_location = spill_loc; - } - continue; - } - - /* 64-bit integer or soft-float double: need register pair */ - if (cur->reg_type == LS_REG_TYPE_LLONG || cur->reg_type == LS_REG_TYPE_DOUBLE_SOFT || - cur->reg_type == LS_REG_TYPE_COMPLEX_FLOAT) { - int r0 = -1, r1 = -1; - - /* Return-pair preference for 64-bit call results. - * - * A call returns its 64-bit result in r0:r1. When that result does not - * cross another call (it is consumed before the next call — e.g. it - * feeds that call's first argument), keeping it in r0:r1 avoids a move - * from the return pair to a callee-saved/other pair. This is the - * dominant cost in chained soft-float expressions: in pr58574's Horner - * polynomials each __aeabi_dmul / __aeabi_dadd result feeds the next - * helper call, and parking the result anywhere but r0:r1 forces a dead - * mov of the return value to its home pair. - * - * Boundary reuse: the pair may still be held by the just-returned call's - * last argument(s), whose intervals end exactly at this call. Those - * values were copied into their AAPCS argument registers before the call - * and are dead afterwards (the call clobbered r0:r1 with the result), so - * the pair is free to take. Evict any such boundary occupant — the same - * idea as the single-reg pref_reg boundary eviction below. */ - /* The mirror case: a pair the function RETURNS. AAPCS requires it in - * r0:r1 at the return, so computing it there directly avoids - * `mov r0,r4 / mov r1,r5` AND the push/pop of the callee-saved pair it - * would otherwise occupy — `long long add64(a,b){return a+b;}` goes from - * 7 instructions to 3, matching GCC. Same !crosses_call requirement: a - * call in between would clobber r0:r1. */ - int wants_return_pair = 0; - if (!cur->crosses_call && 1 < tcc_state->registers_for_allocator) { - if ((int)cur->start < ir->next_instruction_index && - ir->compact_instructions[cur->start].op == TCCIR_OP_FUNCCALLVAL) { - wants_return_pair = 1; /* incoming: this pair IS a call's result */ - } else if ((int)cur->end < ir->next_instruction_index) { - IRQuadCompact *eq = &ir->compact_instructions[cur->end]; - if (eq->op == TCCIR_OP_RETURNVALUE && - irop_get_vreg(tcc_ir_op_get_src1(ir, eq)) == cur->vreg) - wants_return_pair = 1; /* outgoing: this pair is what we return */ - } - } - if (wants_return_pair) { - int want0 = 0, want1 = 1; /* r0:r1 */ - int avail0 = (int_free & (1ull << want0)) != 0; - int avail1 = (int_free & (1ull << want1)) != 0; - int evict0 = -1, evict1 = -1; - for (int k = 0; k < active_count && (!avail0 || !avail1); k++) { - SSAInterval *a = active[k]; - if (a->stack_location != 0 || a->end != cur->start) - continue; - if (!avail0 && (a->r0 == want0 || a->r1 == want0)) { evict0 = k; avail0 = 1; } - if (!avail1 && (a->r0 == want1 || a->r1 == want1)) { evict1 = k; avail1 = 1; } - } - if (avail0 && avail1) { - int idxs[2], ni = 0; - if (evict0 >= 0) idxs[ni++] = evict0; - if (evict1 >= 0 && evict1 != evict0) idxs[ni++] = evict1; - if (ni == 2 && idxs[0] < idxs[1]) { int t = idxs[0]; idxs[0] = idxs[1]; idxs[1] = t; } - for (int e = 0; e < ni; e++) { - SSAInterval *a = active[idxs[e]]; - int_free |= (1ull << a->r0); - if (a->r1 >= 0) int_free |= (1ull << a->r1); - active[idxs[e]] = active[--active_count]; - } - r0 = want0; r1 = want1; - } - } - - if (r0 < 0 && cur->crosses_call && target->int_class.pair_align) { - /* Prefer callee-saved even-aligned pairs (R4:R5, R6:R7, R8:R9, R10:R11) */ - for (int r = 4; r < 12; r += 2) { - if ((int_free & (3ull << r)) == (3ull << r)) { - r0 = r; r1 = r + 1; - break; - } - } - } - if (r0 < 0 && !cur->crosses_call && target->int_class.pair_align) { - /* Try any even-aligned pair (only for non-call-crossing) */ - for (int r = 0; r < 12; r += 2) { - if ((int_free & (3ull << r)) == (3ull << r)) { - r0 = r; r1 = r + 1; - break; - } - } - } - if (r0 < 0 && !cur->crosses_call) { - /* Fallback: any two registers (only for non-call-crossing) */ - int first = -1; - for (int r = 0; r < 13; r++) { - if (int_free & (1ull << r)) { - if (first < 0) first = r; - else { r0 = first; r1 = r; break; } - } - } - } - if (r0 < 0 && cur->crosses_call && cur->reg_type == LS_REG_TYPE_LLONG) { - /* Fallback: any two callee-saved registers (need not be an aligned - * pair). Thumb-2 LDRD/STRD accept any two distinct r0..r12/r14 - * (try_ldrd_pair / try_strd_pair in arm-thumb-gen.c only enforce - * lo != hi and Rn != SP/PC). Other 64-bit ops (cmp+sbcs, adds+adcs, - * load/store as two 32-bit halves) likewise have no pair-alignment - * requirement. Without this fallback, a call-crossing 64-bit value - * spills whenever the small set of aligned callee-saved pairs is - * busy — e.g. when R7 is excluded as Thumb FP, R4:R5/R6:R7/R8:R9/ - * R10:R11 quickly collide with the loop IV, array base, and another - * live 64-bit value. - * - * If two free callee-saved are not immediately available, evict up to - * two single-INT victims (lowest-use, end > cur->end, currently in a - * callee-saved reg) to free the pair. Mirrors the eviction policy - * used for single-INT spill below, just iterated to fill both slots. - * Bail out if a candidate victim has use_count >= cur->use_count to - * avoid spilling a hotter interval. - * - * Restricted to LS_REG_TYPE_LLONG to avoid evicting INT base pointers - * for soft-double / complex-float pairs, where the cost/benefit shifts - * (softfloat helpers reload args anyway, and complex temporaries are - * often short-lived). Enabling it for those types regressed - * gcc-execute/20021120-1::foo by 200+ insns in benchmarks. */ - for (int round = 0; r0 < 0 && round < 3; round++) { - int first = -1; - for (int ci = 0; ci < target->int_class.num_callee_saved; ci++) { - int r = target->int_class.callee_saved[ci]; - if (int_free & (1ull << r)) { - if (first < 0) first = r; - else { r0 = first; r1 = r; break; } - } - } - if (r0 >= 0) break; - SSAInterval *victim = NULL; - int victim_idx = -1; - uint16_t victim_uses = UINT16_MAX; - for (int j = 0; j < active_count; j++) { - SSAInterval *a = active[j]; - if (a->precolored >= 0) continue; - if (a->reg_type != LS_REG_TYPE_INT) continue; - if (a->end <= cur->end) continue; - if (a->r0 < 0 || a->stack_location != 0) continue; - /* Never evict a loop-phi-locked interval: it holds a loop-carried - * value whose register is SHARED with a coalesce partner that stays - * live (see the loop-phi coalescing above and the identical guard in - * the single-register spill victim scan below). Spilling it here - * frees a register the partner still occupies, so a later pair - * allocation would double-book it and clobber the loop-carried value - * (combo_num seed 84127: the g16 loop counter lost to a 64-bit OR's - * high half). */ - if (a->loop_phi_locked) continue; - int is_callee = 0; - for (int ci = 0; ci < target->int_class.num_callee_saved; ci++) { - if (target->int_class.callee_saved[ci] == a->r0) { is_callee = 1; break; } - } - if (!is_callee) continue; - if (a->use_count < victim_uses) { - victim_uses = a->use_count; - victim = a; - victim_idx = j; - } - } - if (!victim || victim_uses >= cur->use_count) break; - int_free |= (1ull << victim->r0); - spill_loc -= 4; - victim->stack_location = spill_loc; - victim->r0 = -1; - active[victim_idx] = active[--active_count]; - } - } - if (r0 >= 0) { - int_free &= ~((1ull << r0) | (1ull << r1)); - dirty_int |= ((1ull << r0) | (1ull << r1)); - cur->r0 = r0; - cur->r1 = r1; - active[active_count++] = cur; - } else { - /* Boundary reuse for 64-bit pairs. - * - * An interval whose end == cur->start is read (at the latest) by the - * very instruction that defines cur, so its pair is free the moment - * that instruction retires. The strict `<` expiration above still - * counts it as active, which is why the pair paths saw 0-1 free - * registers and spilled temps that live for a single instruction. - * Handing cur exactly that pair makes dest == that source, and the - * dest-equals-source case is safe for the ops below: - * - * ADD/SUB/AND/OR/XOR : thumb_emit_data_processing_mop64 writes the - * low half then reads the high half, so an EXACT pair match is - * fine (partial overlap is not — we never create one, since we - * take both registers of one interval). - * SHL/SHR/SAR : thumb_emit_shift64_mop is already alias - * aware (it computes the cross-shift into a temp first and - * reorders the halves when dst aliases src). - * ASSIGN : a copy onto itself; both movs become - * mov rX, rX and are elided. - * - * MUL/UMULL/MLA are excluded: UMULL requires its destination pair not - * to overlap the sources at all. Calls are excluded because the - * result lands in fixed registers. This is the pair analogue of the - * single-INT phi-hint boundary case further below, whose comment - * blanket-excluded pairs; the shift emitter turned out to be safe. */ - int def_op = (cur->start < ir->next_instruction_index) - ? ir->compact_instructions[cur->start].op : -1; - int op_ok = (def_op == TCCIR_OP_ADD || def_op == TCCIR_OP_SUB || def_op == TCCIR_OP_AND || - def_op == TCCIR_OP_OR || def_op == TCCIR_OP_XOR || def_op == TCCIR_OP_SHL || - def_op == TCCIR_OP_SHR || def_op == TCCIR_OP_SAR || def_op == TCCIR_OP_ASSIGN); - int reused = 0; - if (op_ok && cur->reg_type == LS_REG_TYPE_LLONG) { - for (int k = 0; k < active_count; k++) { - SSAInterval *p = active[k]; - if (p->end != cur->start || p->r0 < 0 || p->r1 < 0 || p->stack_location != 0) - continue; - if (p->co_member || cur->co_member || p->loop_phi_locked) - continue; - if (p->r0 >= tcc_state->registers_for_allocator || - p->r1 >= tcc_state->registers_for_allocator) - continue; - if (cur->crosses_call) { - int c0 = 0, c1 = 0; - for (int ci = 0; ci < target->int_class.num_callee_saved; ci++) { - if (target->int_class.callee_saved[ci] == p->r0) c0 = 1; - if (target->int_class.callee_saved[ci] == p->r1) c1 = 1; - } - if (!c0 || !c1) - continue; - } - cur->r0 = p->r0; - cur->r1 = p->r1; - ir->ls.dirty_registers |= (1ull << p->r0) | (1ull << p->r1); - active[k] = active[--active_count]; - active[active_count++] = cur; - reused = 1; - break; - } - } - if (reused) - continue; - - /* Share two registers whose current owners are dead across cur's whole - * range, rather than writing a 64-bit temp to the frame and reading it - * straight back. */ - if (share_ok && !reg_shared_used) - { - int p0 = -1, p1 = -1; - for (int r = 0; r < 13 && p1 < 0; r++) - { - if (int_free & (1ull << r)) - continue; /* a free register would have been taken above */ - if (!ra_alive_reg_shareable(alive, cur, active, active_count, r, target)) - continue; - if (p0 < 0) p0 = r; else p1 = r; - } - if (p1 >= 0) - { - dirty_int |= ((1ull << p0) | (1ull << p1)); - cur->r0 = p0; - cur->r1 = p1; - alive_shared_used = 1; - active[active_count++] = cur; - RA_DBG(" alive_share pair T%d [%u,%u] -> R%d:R%d", - TCCIR_DECODE_VREG_POSITION(cur->vreg), cur->start, cur->end, p0, p1); - continue; - } - } - - spill_loc -= 8; - cur->stack_location = spill_loc; - } - continue; - } - - /* Complex double: always spill (128-bit) */ - if (cur->reg_type == LS_REG_TYPE_COMPLEX_DOUBLE) { - spill_loc -= 16; - cur->stack_location = spill_loc; - continue; - } - - /* Integer: single register */ - int reg = -1; - RA_DBG(" alloc T%d [%u,%u] xcall=%d narrow=%u int_free=0x%llx active=%d", - TCCIR_DECODE_VREG_POSITION(cur->vreg), cur->start, cur->end, - cur->crosses_call, cur->narrow_uses, (unsigned long long)int_free, active_count); - - /* Phi-coalescing: try the hinted partner's register first. - * ra_build_phi_hints set hint_vreg to a vreg this interval used to - * share a phi edge with (now resolved into an explicit ASSIGN copy). - * If the partner has expired and freed its register, taking that - * same register here turns the explicit copy into mov rX, rX which - * the post-RA move-coalescing pass erases. - * - * Boundary case: when partner->end == cur->start, the standard `<` - * expiration kept partner active, so its register looks busy here. - * But within a single ARM 3-operand instruction (or ASSIGN/`mov`), - * sources are read before the dest is written — so the same register - * can serve both. Allow the hint to take that register and force - * partner out of the active set so subsequent allocations see it as - * free. Restricted to single-register INT to avoid corrupting register - * pairs (umull, ll-shift, etc.) where the architecture forbids dest/ - * source overlap. */ - if (cur->hint_vreg >= 0 && max_vreg_pos > 0 && !cur->co_member) { - int hint_idx = HINT_IDX(cur->hint_vreg); - if (hint_idx >= 0 && hint_idx < hint_tbl_size) { - SSAInterval *partner = vreg_to_iv[hint_idx]; - if (partner && !partner->co_member && partner->r0 >= 0 && partner->r1 < 0 && - partner->stack_location == 0) { - int hr = partner->r0; - int hr_free = (int_free & (1ull << hr)) != 0; - int boundary = !hr_free && partner->end == cur->start && - cur->reg_type == LS_REG_TYPE_INT && - partner->reg_type == LS_REG_TYPE_INT; - if ((hr_free || boundary) && - hr < tcc_state->registers_for_allocator) { - int ok = 1; - if (cur->crosses_call) { - ok = 0; - for (int ci = 0; ci < target->int_class.num_callee_saved; ci++) { - if (target->int_class.callee_saved[ci] == hr) { ok = 1; break; } - } - } - if (ok) { - reg = hr; - if (boundary) { - /* Force partner out of active so its register doesn't - * appear taken to subsequent intervals. */ - for (int k = 0; k < active_count; k++) { - if (active[k] == partner) { - int_free |= (1ull << hr); - active[k] = active[--active_count]; - break; - } - } - } - } - } - } - } - } - - /* Loop-carried phi coalescing. - * - * The standard boundary case above fires when partner expires exactly - * at cur->start. A loop-carried phi (sum_array's `sum + arr[i]`, - * for example) extends the partner's live interval to the back-edge - * JMP — past cur's def — so partner's register looks taken at the - * point we want to coalesce. If cur's defining instruction reads - * partner and the only remaining use of partner before partner->end - * is an ASSIGN `partner <-- cur` (the back-edge copy), the two share - * a value from cur->start onward and can share the register. - * - * Transfer ownership of R from partner to cur in the active set: - * extend cur->end to cover partner->end, remove partner from active, - * and let cur represent R there. Both vregs keep r0 = R so codegen - * resolves either name to the same register, and the back-edge - * ASSIGN becomes mov R, R which post-RA cleanup elides. Without - * this transfer R would be double-counted and the expire phase would - * free it as soon as partner's original end lapses while cur is still - * alive in R. */ - int coalesced_loop_phi = 0; - if (reg < 0 && cur->hint_vreg >= 0 && max_vreg_pos > 0 && !cur->co_member && - cur->reg_type == LS_REG_TYPE_INT && tcc_state->optimize >= 1) { - int hint_idx = HINT_IDX(cur->hint_vreg); - if (hint_idx >= 0 && hint_idx < hint_tbl_size) { - SSAInterval *partner = vreg_to_iv[hint_idx]; - if (partner && !partner->co_member && partner->r0 >= 0 && partner->r1 < 0 && - partner->stack_location == 0 && - partner->reg_type == LS_REG_TYPE_INT && - (int)partner->end > (int)cur->start) { - int hr = partner->r0; - int partner_active_idx = -1; - for (int k = 0; k < active_count; k++) { - if (active[k] == partner) { partner_active_idx = k; break; } - } - if (partner_active_idx >= 0 && !(int_free & (1ull << hr)) && - hr < tcc_state->registers_for_allocator) { - int ok = 1; - if (cur->crosses_call) { - ok = 0; - for (int ci = 0; ci < target->int_class.num_callee_saved; ci++) { - if (target->int_class.callee_saved[ci] == hr) { ok = 1; break; } - } - } - if (ok && (ra_safe_loop_phi_coalesce(ir, cur, partner) || - ra_safe_exit_phi_coalesce(ir, cur, partner))) { - reg = hr; - coalesced_loop_phi = 1; - if (partner->end > cur->end) - cur->end = partner->end; - cur->r0 = reg; - /* cur now carries partner's loop-carried value over the extended - * range; the partner is gone from active, so cur is the sole - * holder of hr that the partner's remaining uses depend on. - * Evicting cur mid-loop would spill it without reloading those - * partner uses → IV corruption. Lock it against eviction. */ - cur->loop_phi_locked = 1; - active[partner_active_idx] = active[--active_count]; - } - } - } - } - } - - /* Soft preference: try iv->pref_reg first (e.g., r0 for vregs that - * feed RETURNVALUE). If free, take it. If a still-active interval - * ends exactly at cur->start and holds that register, evict it and - * take it (boundary case — same logic as the phi-coalescing hint). */ - if (reg < 0 && cur->pref_reg >= 0 && cur->reg_type == LS_REG_TYPE_INT) { - int hr = (int)cur->pref_reg; - if (hr < tcc_state->registers_for_allocator) { - int hr_free = (int_free & (1ull << hr)) != 0; - int ok = 1; - if (cur->crosses_call) { - ok = 0; - for (int ci = 0; ci < target->int_class.num_callee_saved; ci++) { - if (target->int_class.callee_saved[ci] == hr) { ok = 1; break; } - } - } - if (ok) { - if (hr_free) { - reg = hr; - } else { - /* Boundary: an active INT interval ending at cur->start in hr. */ - for (int k = 0; k < active_count; k++) { - SSAInterval *a = active[k]; - if (a->r0 == hr && a->r1 < 0 && a->end == cur->start && - a->stack_location == 0 && a->reg_type == LS_REG_TYPE_INT) { - int_free |= (1ull << hr); - active[k] = active[--active_count]; - reg = hr; - break; - } - } - } - /* Return-block register sharing: cur is a single-use RETURNVALUE - * feeder whose def at cur->start and consumer at cur->end form - * a return tail. Control returns to the caller at cur->end, so - * we can SHARE hr with whatever interval `partner` is holding - * it — cur's def overwrites hr, but we never re-execute past - * the return on this control-flow path. - * - * Safety conditions: - * 1. cur->end is RETURNVALUE/RETURNVOID. - * 2. partner's vreg is not READ at any instruction in - * [cur->start, cur->end] — cur's def would clobber it. - * - * Sharing model: cur->r0 = hr, but we do NOT remove partner - * from active, do NOT mark hr free in int_free, and set - * cur->reg_shared so cur's expire phase does not free hr - * (partner still logically owns it). Other CFG paths through - * partner's live range emit their own reads of hr unaffected — - * those paths never execute cur's def, so partner's value - * remains intact in hr along them. */ - if (reg < 0 && !alive_shared_used && cur->end > cur->start && - (int)cur->end < ir->next_instruction_index) { - IRQuadCompact *eq = &ir->compact_instructions[cur->end]; - if (eq->op == TCCIR_OP_RETURNVALUE || eq->op == TCCIR_OP_RETURNVOID) { - for (int k = 0; k < active_count; k++) { - SSAInterval *a = active[k]; - if (a->r0 != hr || a->r1 >= 0 || a->stack_location != 0 || - a->reg_type != LS_REG_TYPE_INT) - continue; - int conflict = 0; - for (int p = (int)cur->start; p <= (int)cur->end && !conflict; p++) { - IRQuadCompact *pq = &ir->compact_instructions[p]; - /* Any operand reference to the partner clobbers the share: - * cur's def at cur->start overwrites hr, so partner must not be - * needed anywhere in the range. Use ra_instr_touches_vreg so a - * STORE-class op's dest (its base *pointer*, which the store - * READS) and an MLA accumulator count — a naive src1/src2 scan - * missed a partner used as a store base and shared hr anyway, - * emitting `str rX, [rX]` (value written through itself). */ - if (ra_instr_touches_vreg(ir, pq, a->vreg)) { conflict = 1; break; } - } - if (!conflict) { - cur->r0 = hr; - cur->reg_shared = 1; - reg_shared_used = 1; - dirty_int |= (1ull << hr); - reg = hr; - break; - } - } - } - } - } - } - } - - if (reg < 0 && cur->crosses_call) { - /* Prefer callee-saved */ - for (int ci = 0; ci < target->int_class.num_callee_saved; ci++) { - int r = target->int_class.callee_saved[ci]; - if (int_free & (1ull << r)) { reg = r; break; } - } - } - if (reg < 0 && !cur->crosses_call && - (int)cur->start < ir->next_instruction_index) { - IRQuadCompact *dq = &ir->compact_instructions[cur->start]; - if (dq->op == TCCIR_OP_MLA && - irop_get_vreg(tcc_ir_op_get_dest(ir, dq)) == cur->vreg) { - IROperand accum = tcc_ir_op_get_accum(ir, dq); - if (accum.is_sym) { - static const int sym_mla_order[] = {10, 11, 8, 9}; - for (int oi = 0; oi < 4; oi++) { - int r = sym_mla_order[oi]; - if (r < tcc_state->registers_for_allocator && - (int_free & (1ull << r))) { - reg = r; - break; - } - } - } - } - } - if (reg < 0 && !cur->crosses_call) { - /* Narrow-dense intervals prefer r4-r7 over r12 for 16-bit encodings; gates in docs/regalloc_narrow_pref.md */ - static const int alloc_order[] = {0, 1, 2, 3, 12, 4, 5, 6, 7, 8, 9, 10, 11}; - static const int alloc_order_narrow[] = {0, 1, 2, 3, 4, 5, 6, 7, 12, 8, 9, 10, 11}; - int narrow_dense = cur->narrow_uses > 0 && - (uint32_t)cur->narrow_uses * 8 >= (cur->end - cur->start); - const int *order = narrow_dense ? alloc_order_narrow : alloc_order; - for (int pass = 0; pass < 2 && reg < 0; pass++) { - for (int oi = 0; oi < 13; oi++) { - int r = order[oi]; - if (r >= tcc_state->registers_for_allocator) continue; - if (!(int_free & (1ull << r))) continue; - if (pass == 0 && order == alloc_order_narrow) { - if (r >= 4 && r <= 7 && !(dirty_int & (1ull << r)) && !has_call && - cur->narrow_uses < 2) - continue; - if (r >= 8 && r != 12) - continue; - } - reg = r; - break; - } - } - } - - if (cur->reg_shared) { - /* Return-block share: cur->r0 was set in the pref_reg path. - * Don't touch int_free (partner still owns hr) and don't add cur - * to active (cur's expire would otherwise hit the !reg_shared - * guard but adding it is just bookkeeping; the simpler invariant - * is "shared cur never enters active"). */ - } else if (coalesced_loop_phi) { - /* Partner was removed from active above; transfer R's ownership - * to cur. int_free stays unchanged (R was taken via partner, - * now taken via cur). */ - dirty_int |= (1ull << reg); - active[active_count++] = cur; - } else if (reg >= 0) { - int_free &= ~(1ull << reg); - dirty_int |= (1ull << reg); - cur->r0 = reg; - active[active_count++] = cur; - } else { - /* Before spilling, look for a register whose current owner is provably - * dead across cur's whole range: sharing it costs nothing, where a spill - * costs a store plus a reload at every use. */ - if (share_ok && !reg_shared_used) { - int sh = -1; - for (int r = 0; r < 13; r++) { - if (int_free & (1ull << r)) continue; /* a free one was taken above */ - if (!ra_alive_reg_shareable(alive, cur, active, active_count, r, target)) continue; - sh = r; - break; - } - if (sh >= 0) { - dirty_int |= (1ull << sh); - cur->r0 = sh; - alive_shared_used = 1; - active[active_count++] = cur; - RA_DBG(" alive_share T%d [%u,%u] -> R%d", - TCCIR_DECODE_VREG_POSITION(cur->vreg), cur->start, cur->end, sh); - continue; - } - } - /* Spill: among intervals that extend past cur, evict the one - * with the lowest spill cost (fewest loop-weighted uses). - * use_count is already weighted by loop depth (4^depth per use), - * so this prefers evicting intervals with few loop-hot uses. */ - SSAInterval *victim = NULL; - int victim_idx = -1; - uint16_t victim_uses = UINT16_MAX; - for (int j = 0; j < active_count; j++) { - SSAInterval *a = active[j]; - if (a->precolored >= 0) continue; - if (a->reg_type != LS_REG_TYPE_INT) continue; - if (a->end <= cur->end) continue; - /* Never evict a loop-phi-locked interval: it holds a loop-carried value - * (its absorbed partner's uses still read this register across the loop - * body) and spilling it here would not reload those uses. */ - if (a->loop_phi_locked) continue; - if (a->use_count < victim_uses || - (a->use_count == victim_uses && victim && a->end > victim->end)) { - victim_uses = a->use_count; - victim = a; - victim_idx = j; - } - } - if (victim) { - /* Evict victim, give its register to cur */ - reg = victim->r0; - victim->r0 = -1; - spill_loc -= 4; - victim->stack_location = spill_loc; - /* Remove victim from active */ - active[victim_idx] = active[--active_count]; - cur->r0 = reg; - active[active_count++] = cur; - } else { - spill_loc -= 4; - cur->stack_location = spill_loc; - } - } - } - - tcc_free(active); - tcc_free(active_addrtaken); - tcc_free(free_slots_4); - tcc_free(vreg_to_iv); - #undef HINT_IDX - - if (TCC_LOG_LS) { - RA_DBG("SSA ra_linear_scan: allocation results (%d intervals)", count); - for (int i = 0; i < count; i++) { - SSAInterval *iv = &intervals[i]; - int type = TCCIR_DECODE_VREG_TYPE(iv->vreg); - int pos = TCCIR_DECODE_VREG_POSITION(iv->vreg); - if (iv->stack_location != 0) { - RA_DBG(" %s%d [%u,%u] -> spill(%d)", ra_vreg_type_char(type), pos, - iv->start, iv->end, iv->stack_location); - } else if (iv->r1 >= 0) { - RA_DBG(" %s%d [%u,%u] -> R%d:R%d", ra_vreg_type_char(type), pos, - iv->start, iv->end, iv->r0, iv->r1); - } else { - RA_DBG(" %s%d [%u,%u] -> R%d", ra_vreg_type_char(type), pos, - iv->start, iv->end, iv->r0); - } - } - RA_DBG(" dirty_int=0x%llx dirty_fp=0x%llx", (unsigned long long)dirty_int, (unsigned long long)dirty_fp); - } - - *out_dirty_int = dirty_int; - *out_dirty_fp = dirty_fp; -} - -/* ============================================================================ - * Write Results to IR - * ============================================================================ */ - -static void ra_write_results(TCCIRState *ir, SSAInterval *intervals, int count) -{ - /* Clear LS intervals and repopulate from SSA results */ - tcc_ls_clear_live_intervals(&ir->ls); - - for (int i = 0; i < count; i++) { - SSAInterval *iv = &intervals[i]; - tcc_ls_add_live_interval(&ir->ls, iv->vreg, iv->start, iv->end, - iv->crosses_call, iv->addrtaken, iv->reg_type, - 0, iv->precolored); - LSLiveInterval *lsi = &ir->ls.intervals[ir->ls.next_interval_index - 1]; - lsi->r0 = iv->r0; - lsi->r1 = iv->r1; - lsi->stack_location = iv->stack_location; - lsi->co_member = iv->co_member; - - /* Also write to IRLiveInterval for codegen */ - tcc_ir_stack_reg_assign(ir, iv->vreg, iv->stack_location, iv->r0, iv->r1); - IRLiveInterval *li = tcc_ir_vreg_live_interval(ir, iv->vreg); - if (li) { - li->start = iv->start; - li->end = iv->end; - li->crosses_call = iv->crosses_call; - } - } -} - -/* ============================================================================ - * Phi Resolution - * ============================================================================ */ - -typedef struct RAPhiCopy { - int32_t dest_vreg; - int32_t src_vreg; - int btype; - uint8_t emitted; -} RAPhiCopy; - -typedef struct RAPhiCopyRecord { - int32_t src_vreg; - int new_instr_idx; -} RAPhiCopyRecord; - -typedef struct RAPhiStats { - int phi_operands; - int parallel_requested; - int parallel_emitted; - int cycle_temporaries; - int participant_spills; - int32_t *participants; - int participant_count; - int participant_cap; -} RAPhiStats; - -static void ra_phi_stats_add_participant(RAPhiStats *stats, int32_t vreg) -{ - if (!TCC_LOG_LS || !stats || vreg < 0) - return; - - for (int i = 0; i < stats->participant_count; i++) - if (stats->participants[i] == vreg) - return; - - if (stats->participant_count >= stats->participant_cap) { - int cap = stats->participant_cap ? stats->participant_cap * 2 : 16; - stats->participants = tcc_realloc(stats->participants, - cap * sizeof(*stats->participants)); - stats->participant_cap = cap; - } - stats->participants[stats->participant_count++] = vreg; -} - -static int ra_phi_stats_has_participant(RAPhiStats *stats, int32_t vreg) -{ - if (!TCC_LOG_LS || !stats) - return 0; - for (int i = 0; i < stats->participant_count; i++) - if (stats->participants[i] == vreg) - return 1; - return 0; -} - -#define RA_MAX_PHI_COPY_RECORDS 512 - -static int ra_interval_reg_count(IRLiveInterval *li, int regs[2]) -{ - int n = 0; - if (!li || li->allocation.offset != 0) - return 0; - int r0 = li->allocation.r0; - if ((r0 & PREG_SPILLED) == 0 && r0 != PREG_NONE && r0 != 0xffff) - regs[n++] = r0; - int r1 = li->allocation.r1; - if ((r1 & PREG_SPILLED) == 0 && r1 != PREG_NONE && r1 != 0xffff) - regs[n++] = r1; - return n; -} - -static int ra_interval_regs_overlap(IRLiveInterval *a, IRLiveInterval *b) -{ - int ar[2], br[2]; - int an = ra_interval_reg_count(a, ar); - int bn = ra_interval_reg_count(b, br); - for (int i = 0; i < an; i++) - for (int j = 0; j < bn; j++) - if (ar[i] == br[j]) - return 1; - return 0; -} - -static int ra_interval_locations_overlap(IRLiveInterval *a, IRLiveInterval *b) -{ - if (!a || !b) - return 0; - if (a->allocation.offset != 0 || b->allocation.offset != 0) - return a->allocation.offset != 0 && a->allocation.offset == b->allocation.offset; - return ra_interval_regs_overlap(a, b); -} - -/* Set to 1 while running ra_resolve_phis BEFORE register allocation, where - * no allocation info exists. In that mode, copy-elision (identity check) - * and physical-register clobber detection must fall back to vreg-level - * reasoning. */ -static int ra_phi_resolve_pre_ra_mode = 0; - -/* Post-allocation dead register-copy elimination. - * - * Phi resolution (ra_resolve_phis) emits an ASSIGN copy for every phi operand, - * including phi destinations that are never read. Such a copy is dead code — - * but because its destination has no uses (an empty live range), the linear - * scan freely gives it a register that simultaneously holds a DIFFERENT value - * which is live across the merge (the two do not interfere). The dead copy - * then overwrites that register, destroying the live value. - * switch fuzz seed 198468 -O1: at a loop-exit merge the dead exit-phi copy - * `R6(T121) <- T130` clobbered R6, which still held the live u6 read one - * instruction later by `T106 <- R6(T104)`, so u6 read back as 0. - * - * These copies must NOT be suppressed before allocation: removing them changes - * register pressure and perturbs every allocation decision, regressing - * unrelated code by shifting the scan onto latent bugs. They are NOP'd here, - * after allocation AND after every liveness / scratch bitmap has been built, so - * the allocation is byte-identical to before and only the emitted code changes. - * - * A copy is removable when its destination is an SSA TEMP — never - * address-taken, so every read is an explicit operand — that appears as a read - * operand nowhere. Reads are enumerated exactly as ra_build_intervals counts - * uses: src1, src2, the MLA accumulator, and the address of a STORE-class op. - * Iterate to a fixed point so a chain of dead copies (each read only by the - * next) collapses completely. */ -static void ra_eliminate_dead_reg_copies(TCCIRState *ir) -{ - int max_pos = ir->next_local_variable; - if (ir->next_temporary_variable > max_pos) max_pos = ir->next_temporary_variable; - if (ir->next_parameter > max_pos) max_pos = ir->next_parameter; - if (max_pos <= 0) - return; - size_t map_size = (size_t)4 * max_pos; - uint8_t *read_map = tcc_malloc(map_size); - int n = ir->next_instruction_index; - - #define RA_DEADCOPY_IDX(vr) \ - (TCCIR_DECODE_VREG_TYPE(vr) * max_pos + TCCIR_DECODE_VREG_POSITION(vr)) - #define RA_DEADCOPY_MARK(vr) do { \ - int32_t _v = (vr); \ - if (_v >= 0) { \ - long _i = RA_DEADCOPY_IDX(_v); \ - if (_i >= 0 && (size_t)_i < map_size) read_map[_i] = 1; \ - } \ - } while (0) - - for (;;) { - memset(read_map, 0, map_size); - - for (int i = 0; i < n; i++) { - IRQuadCompact *q = &ir->compact_instructions[i]; - if (q->op == TCCIR_OP_NOP) - continue; - if (irop_config[q->op].has_src1) { - IROperand s = tcc_ir_op_get_src1(ir, q); - if (irop_has_vreg(s)) RA_DEADCOPY_MARK(irop_get_vreg(s)); - } - if (irop_config[q->op].has_src2) { - IROperand s = tcc_ir_op_get_src2(ir, q); - if (irop_has_vreg(s)) RA_DEADCOPY_MARK(irop_get_vreg(s)); - } - if (q->op == TCCIR_OP_MLA) { - IROperand s = tcc_ir_op_get_accum(ir, q); - if (irop_has_vreg(s)) RA_DEADCOPY_MARK(irop_get_vreg(s)); - } - /* STORE-class ops read their "dest" operand (the memory address). */ - if (q->op == TCCIR_OP_STORE || q->op == TCCIR_OP_STORE_INDEXED || - q->op == TCCIR_OP_STORE_POSTINC) { - IROperand d = tcc_ir_op_get_dest(ir, q); - if (irop_has_vreg(d)) RA_DEADCOPY_MARK(irop_get_vreg(d)); - } - } - - int changed = 0; - for (int i = 0; i < n; i++) { - IRQuadCompact *q = &ir->compact_instructions[i]; - if (q->op != TCCIR_OP_ASSIGN) - continue; - IROperand d = tcc_ir_op_get_dest(ir, q); - if (!irop_has_vreg(d)) - continue; - int32_t dv = irop_get_vreg(d); - if (TCCIR_DECODE_VREG_TYPE(dv) != TCCIR_VREG_TYPE_TEMP) - continue; - long idx = RA_DEADCOPY_IDX(dv); - if (idx < 0 || (size_t)idx >= map_size || read_map[idx]) - continue; /* destination is read somewhere — live copy, keep */ - /* Dead copy into a never-read temp: NOP it. The slot (and any - * is_jump_target flag on it) is preserved so branch targets still - * resolve to a valid index; codegen skips NOPs. */ - q->op = TCCIR_OP_NOP; - changed = 1; - } - - if (!changed) - break; - } - - #undef RA_DEADCOPY_MARK - #undef RA_DEADCOPY_IDX - tcc_free(read_map); -} - -static int ra_phi_copy_is_identity(IRLiveInterval *dest_li, IRLiveInterval *src_li) -{ - if (!dest_li || !src_li) - return 0; - /* Pre-RA: never collapse; allocation may still place them in the same - * register, in which case the post-RA move-coalescing pass will erase - * the redundant copy. */ - if (ra_phi_resolve_pre_ra_mode) - return 0; - if (dest_li->allocation.offset != 0 || src_li->allocation.offset != 0) - return dest_li->allocation.offset == src_li->allocation.offset; - return dest_li->allocation.r0 == src_li->allocation.r0 && - dest_li->allocation.r1 == src_li->allocation.r1; -} - -static int ra_phi_copy_dest_clobbers_pending_source(TCCIRState *ir, RAPhiCopy *copies, - int copy_count, int copy_idx) -{ - /* Pre-RA: clobber means "copy_idx writes a vreg that another pending - * copy still needs to read". Compare by vreg, not by physical - * register (which doesn't exist yet). */ - if (ra_phi_resolve_pre_ra_mode) { - int32_t dest_vr = copies[copy_idx].dest_vreg; - for (int i = 0; i < copy_count; i++) { - if (i == copy_idx || copies[i].emitted) - continue; - if (copies[i].src_vreg == dest_vr) - return 1; - } - return 0; - } - - IRLiveInterval *dest_li = tcc_ir_vreg_live_interval(ir, copies[copy_idx].dest_vreg); - if (!dest_li) - return 0; - - for (int i = 0; i < copy_count; i++) { - if (i == copy_idx || copies[i].emitted) - continue; - if (!tcc_ir_vreg_is_valid(ir, copies[i].src_vreg)) - continue; - IRLiveInterval *src_li = tcc_ir_vreg_live_interval(ir, copies[i].src_vreg); - if (ra_interval_locations_overlap(dest_li, src_li)) - return 1; - } - return 0; -} - -static int ra_invert_cond(int tok) -{ - switch (tok) { - case TOK_ULT: return TOK_UGE; - case TOK_UGE: return TOK_ULT; - case TOK_EQ: return TOK_NE; - case TOK_NE: return TOK_EQ; - case TOK_ULE: return TOK_UGT; - case TOK_UGT: return TOK_ULE; - case TOK_LT: return TOK_GE; - case TOK_GE: return TOK_LT; - case TOK_LE: return TOK_GT; - case TOK_GT: return TOK_LE; - default: return tok ^ 1; - } -} - -static int ra_phi_copy_needed(TCCIRState *ir, IRPhiNode *phi, int operand_idx) -{ - IRLiveInterval *dest_li = tcc_ir_vreg_live_interval(ir, phi->dest_vreg); - if (!dest_li) - return 0; - /* Pre-RA: emit a copy for every operand. Post-RA coalescing will drop - * any that turn out to land in the same physical register. Copies into a - * phi destination that is never read are NOT suppressed here: doing so - * pre-RA changes register pressure and perturbs the whole allocation. They - * are stripped after allocation instead — see ra_eliminate_dead_reg_copies. */ - if (ra_phi_resolve_pre_ra_mode) { - if (!tcc_ir_vreg_is_valid(ir, phi->operands[operand_idx].vreg)) - return 0; - return 1; - } - if (dest_li->allocation.r0 == PREG_NONE && dest_li->allocation.offset == 0) - return 0; - IRLiveInterval *src_li = NULL; - if (tcc_ir_vreg_is_valid(ir, phi->operands[operand_idx].vreg)) - src_li = tcc_ir_vreg_live_interval(ir, phi->operands[operand_idx].vreg); - if (ra_phi_copy_is_identity(dest_li, src_li)) { - if (src_li) - src_li->phi_pinned = 1; - return 0; - } - return 1; -} - -/* Candidate (phi, operand) pairs bucketed by predecessor block, in exactly the - * order a block-major scan would visit them. - * - * ra_collect_phi_copies_for_pred() with no successor filter wants every phi - * operand naming one predecessor, and scanning all blocks' phis to find them - * is O(blocks x phi operands) -- once per predecessor, so quadratic. pr34093's - * 1000-case switch still spent ~25% of its compile there after the filtered - * scans were fixed. The buckets hold CANDIDATES, not decisions: - * ra_phi_copy_needed() is still asked about each one, in the same order as - * before, so its phi_pinned side effect is unchanged. */ -typedef struct RAPhiPredIndex -{ - int *off; /* num_blocks + 1 bucket bounds */ - IRPhiNode **phi; /* off[nb] entries */ - int *opidx; -} RAPhiPredIndex; - -static int ra_phi_pred_candidate(const IRPhiNode *phi, int pi, int nb) -{ - int pred = phi->operands[pi].pred_block; - if (pred < 0 || pred >= nb || phi->operands[pi].vreg < 0) - return -1; - return pred; -} - -static void ra_phi_pred_index_build(IRCFG *cfg, IRSSAState *ssa, RAPhiPredIndex *idx) -{ - const int nb = cfg->num_blocks; - - idx->off = tcc_mallocz(sizeof(int) * (nb + 1)); - idx->phi = NULL; - idx->opidx = NULL; - - for (int sb = 0; sb < nb; sb++) - for (IRPhiNode *phi = ssa->block_phis[sb]; phi; phi = phi->next) - for (int pi = 0; pi < phi->num_operands; pi++) { - int pred = ra_phi_pred_candidate(phi, pi, nb); - if (pred >= 0) - idx->off[pred + 1]++; - } - for (int b = 0; b < nb; b++) - idx->off[b + 1] += idx->off[b]; - - const int total = idx->off[nb]; - if (!total) - return; - - idx->phi = tcc_malloc(sizeof(IRPhiNode *) * total); - idx->opidx = tcc_malloc(sizeof(int) * total); - - int *cursor = tcc_malloc(sizeof(int) * nb); - memcpy(cursor, idx->off, sizeof(int) * nb); - for (int sb = 0; sb < nb; sb++) - for (IRPhiNode *phi = ssa->block_phis[sb]; phi; phi = phi->next) - for (int pi = 0; pi < phi->num_operands; pi++) { - int pred = ra_phi_pred_candidate(phi, pi, nb); - if (pred < 0) - continue; - idx->phi[cursor[pred]] = phi; - idx->opidx[cursor[pred]] = pi; - cursor[pred]++; - } - tcc_free(cursor); -} - -static void ra_phi_pred_index_free(RAPhiPredIndex *idx) -{ - tcc_free(idx->off); - tcc_free(idx->phi); - tcc_free(idx->opidx); - idx->off = NULL; - idx->phi = NULL; - idx->opidx = NULL; -} - -/* Resolve a successor filter to the block range the scanners below must walk. - * - * Both took the filter as a per-iteration equality test, so a call that wanted - * ONE block still walked all of them -- and ra_resolve_phis calls them once per - * predecessor, which is O(blocks^2). pr34093's 1000-case switch spent 32% of - * its entire compile in ra_collect_phi_copies_for_pred that way. Same fix, and - * same reason, as the copies_per_block aggregation in ra_resolve_phis. */ -static void ra_phi_scan_range(const IRCFG *cfg, int succ_filter, int *first, int *last) -{ - if (succ_filter < 0) { - *first = 0; - *last = cfg->num_blocks - 1; - } else if (succ_filter < cfg->num_blocks) { - *first = *last = succ_filter; - } else { - /* Names no block: scan nothing, as the equality test used to. */ - *first = 0; - *last = -1; - } -} - -static int ra_count_phi_copies_for_pred(TCCIRState *ir, IRCFG *cfg, IRSSAState *ssa, - int pred_block, int succ_filter) -{ - int count = 0; - int first, last; - ra_phi_scan_range(cfg, succ_filter, &first, &last); - for (int sb = first; sb <= last; sb++) { - for (IRPhiNode *phi = ssa->block_phis[sb]; phi; phi = phi->next) { - for (int pi = 0; pi < phi->num_operands; pi++) { - if (phi->operands[pi].pred_block != pred_block || phi->operands[pi].vreg < 0) - continue; - if (ra_phi_copy_needed(ir, phi, pi)) - count++; - } - } - } - return count; -} - -static int ra_collect_phi_copies_for_pred(TCCIRState *ir, IRCFG *cfg, IRSSAState *ssa, - int pred_block, int succ_filter, - const RAPhiPredIndex *idx, RAPhiCopy *copies) -{ - int copy_count = 0; - int first, last; - - if (succ_filter < 0 && idx->phi && pred_block >= 0 && pred_block < cfg->num_blocks) { - for (int e = idx->off[pred_block]; e < idx->off[pred_block + 1]; e++) { - IRPhiNode *phi = idx->phi[e]; - int pi = idx->opidx[e]; - if (!ra_phi_copy_needed(ir, phi, pi)) - continue; - copies[copy_count].dest_vreg = phi->dest_vreg; - copies[copy_count].src_vreg = phi->operands[pi].vreg; - copies[copy_count].btype = phi->btype; - copies[copy_count].emitted = 0; - copy_count++; - } - return copy_count; - } - - ra_phi_scan_range(cfg, succ_filter, &first, &last); - for (int sb = first; sb <= last; sb++) { - for (IRPhiNode *phi = ssa->block_phis[sb]; phi; phi = phi->next) { - for (int pi = 0; pi < phi->num_operands; pi++) { - if (phi->operands[pi].pred_block != pred_block || phi->operands[pi].vreg < 0) - continue; - if (!ra_phi_copy_needed(ir, phi, pi)) - continue; - copies[copy_count].dest_vreg = phi->dest_vreg; - copies[copy_count].src_vreg = phi->operands[pi].vreg; - copies[copy_count].btype = phi->btype; - copies[copy_count].emitted = 0; - copy_count++; - } - } - } - return copy_count; -} - -static void ra_emit_phi_copy(TCCIRState *ir, IRQuadCompact *new_instrs, int *wp, - int *pool_wp, const RAPhiCopy *copy, - RAPhiCopyRecord *records, int *record_count) -{ - IROperand dest_op; - memset(&dest_op, 0, sizeof(dest_op)); - irop_set_vreg(&dest_op, copy->dest_vreg); - dest_op.tag = IROP_TAG_VREG; - dest_op.btype = copy->btype; - - IROperand src_op; - memset(&src_op, 0, sizeof(src_op)); - irop_set_vreg(&src_op, copy->src_vreg); - src_op.tag = IROP_TAG_VREG; - src_op.btype = copy->btype; - - ir->iroperand_pool[*pool_wp] = dest_op; - ir->iroperand_pool[*pool_wp + 1] = src_op; - - if (records && record_count && *record_count < RA_MAX_PHI_COPY_RECORDS) { - records[*record_count].src_vreg = copy->src_vreg; - records[*record_count].new_instr_idx = *wp; - (*record_count)++; - } - - new_instrs[*wp].op = TCCIR_OP_ASSIGN; - new_instrs[*wp].operand_base = *pool_wp; - new_instrs[*wp].line_num = 0; - new_instrs[*wp].is_jump_target = 0; - (*wp)++; - *pool_wp += 2; -} - -static int ra_btype_stack_size(int btype) -{ - switch (btype) { - case IROP_BTYPE_INT64: - case IROP_BTYPE_FLOAT64: - return 8; - default: - return 4; - } -} - -static void ra_note_vreg_type_for_btype(TCCIRState *ir, int32_t vreg, int btype) -{ - if (!tcc_ir_vreg_is_valid(ir, vreg)) - return; - if (btype == IROP_BTYPE_FLOAT32 || btype == IROP_BTYPE_FLOAT64) - tcc_ir_vreg_type_set_fp(ir, vreg, 1, btype == IROP_BTYPE_FLOAT64); - else if (btype == IROP_BTYPE_INT64) - tcc_ir_vreg_type_set_64bit(ir, vreg); -} - -static void ra_record_stack_operand_min(IROperand op, int *min_offset) -{ - if (op.tag != IROP_TAG_STACKOFF) - return; - int off = irop_get_stack_offset(op); - if (off < *min_offset) - *min_offset = off; -} - -static int ra_find_phi_spill_cursor(TCCIRState *ir) -{ - int min_offset = 0; - for (int i = 0; i < ir->ls.next_interval_index; i++) { - LSLiveInterval *lsi = &ir->ls.intervals[i]; - if (lsi->stack_location < min_offset) - min_offset = lsi->stack_location; - } - for (int i = 0; i < ir->next_instruction_index; i++) { - IRQuadCompact *q = &ir->compact_instructions[i]; - if (irop_config[q->op].has_dest) - ra_record_stack_operand_min(tcc_ir_op_get_dest(ir, q), &min_offset); - if (irop_config[q->op].has_src1) - ra_record_stack_operand_min(tcc_ir_op_get_src1(ir, q), &min_offset); - if (irop_config[q->op].has_src2) - ra_record_stack_operand_min(tcc_ir_op_get_src2(ir, q), &min_offset); - if (q->op == TCCIR_OP_MLA) - ra_record_stack_operand_min(tcc_ir_op_get_accum(ir, q), &min_offset); - } - return min_offset; -} - -static int32_t ra_create_phi_temp(TCCIRState *ir, int btype, int start, int end, - int *phi_spill_cursor) -{ - int32_t tmp_vreg = tcc_ir_vreg_alloc_temp(ir); - if (tmp_vreg < 0) - return -1; - - int size = ra_btype_stack_size(btype); - *phi_spill_cursor -= size; - if (size > 4 && (*phi_spill_cursor & 7)) - *phi_spill_cursor &= ~7; - - ra_note_vreg_type_for_btype(ir, tmp_vreg, btype); - tcc_ls_add_live_interval(&ir->ls, tmp_vreg, start, end, 0, 0, - tcc_ir_vreg_type_get(ir, tmp_vreg), 0, -1); - LSLiveInterval *lsi = &ir->ls.intervals[ir->ls.next_interval_index - 1]; - lsi->r0 = -1; - lsi->r1 = -1; - lsi->stack_location = *phi_spill_cursor; - - tcc_ir_stack_reg_assign(ir, tmp_vreg, *phi_spill_cursor, -1, -1); - IRLiveInterval *li = tcc_ir_vreg_live_interval(ir, tmp_vreg); - if (li) { - li->start = (uint32_t)start; - li->end = (uint32_t)end; - } - - return tmp_vreg; -} - -static void ra_emit_scheduled_phi_copies(TCCIRState *ir, IRQuadCompact *new_instrs, - int *wp, int *pool_wp, RAPhiCopy *copies, - int copy_count, int block, - RAPhiCopyRecord *records, int *record_count, - int *phi_spill_cursor, RAPhiStats *stats) -{ - int start_wp = *wp; - int emitted = 0; - while (emitted < copy_count) { - int progress = 0; - for (int ci = 0; ci < copy_count; ci++) { - if (copies[ci].emitted) - continue; - if (ra_phi_copy_dest_clobbers_pending_source(ir, copies, copy_count, ci)) - continue; - ra_emit_phi_copy(ir, new_instrs, wp, pool_wp, &copies[ci], records, record_count); - copies[ci].emitted = 1; - emitted++; - progress = 1; - break; - } - - if (!progress) { - int ci; - for (ci = 0; ci < copy_count; ci++) - if (!copies[ci].emitted) - break; - if (ci >= copy_count) - break; - - int32_t saved_src = copies[ci].src_vreg; - int32_t tmp_vreg = ra_create_phi_temp(ir, copies[ci].btype, *wp, - *wp + copy_count - emitted, - phi_spill_cursor); - if (tmp_vreg < 0) { - RA_DBG("SSA phi resolver: failed to allocate cycle temp in block %d", block); - break; - } - - RAPhiCopy save = { - .dest_vreg = tmp_vreg, - .src_vreg = saved_src, - .btype = copies[ci].btype, - .emitted = 0, - }; - RA_DBG("SSA phi resolver: breaking cyclic parallel copy in block %d with T%d", - block, TCCIR_DECODE_VREG_POSITION(tmp_vreg)); - if (TCC_LOG_LS) { - stats->cycle_temporaries++; - ra_phi_stats_add_participant(stats, tmp_vreg); - } - ra_emit_phi_copy(ir, new_instrs, wp, pool_wp, &save, records, record_count); - copies[ci].src_vreg = tmp_vreg; - } - } - if (TCC_LOG_LS) - stats->parallel_emitted += *wp - start_wp; -} - -static void ra_build_live_regs_bitmap(TCCIRState *ir); - -static void ra_resolve_phis(TCCIRState *ir, IRCFG *cfg, IRSSAState *ssa, - RAPhiStats *stats) -{ - int nb = cfg->num_blocks; - int old_n = ir->next_instruction_index; - - /* Count copies needed per predecessor block. - * - * Single-pass aggregation: walk every phi node once (O(total phi - * operands)) and bucket each operand into its predecessor block's - * counter. The previous version called ra_count_phi_copies_for_pred - * per predecessor, which itself looped over every block, producing - * O(blocks^2) work even when no phis existed — pathological for huge - * branch-heavy functions (compile/20001226-1 has 16K blocks). */ - int *copies_per_block = tcc_mallocz(nb * sizeof(int)); - int total_copies = 0; - /* For modified-JUMPIF detection we need the per-(pred,succ) count too, - * but only for blocks that ended in a JUMPIF AND have any copies on the - * target edge. Build a bitmap of (pred -> succ-block) edges that carry - * at least one phi copy. We use a simple flat array indexed by pred. */ - int *copies_to_jumpif_succ = tcc_mallocz(nb * sizeof(int)); - for (int b = 0; b < nb; b++) copies_to_jumpif_succ[b] = -1; - /* First, identify the JUMPIF-target succ_block per pred (if any). */ - for (int b = 0; b < nb; b++) { - IRBasicBlock *bb = &cfg->blocks[b]; - int last_instr = bb->end_idx - 1; - if (last_instr < bb->start_idx) continue; - if (ir->compact_instructions[last_instr].op != TCCIR_OP_JUMPIF) continue; - IROperand dest = tcc_ir_op_get_dest(ir, &ir->compact_instructions[last_instr]); - int old_target = (int)irop_get_imm64_ex(ir, dest); - int target_block = (old_target >= 0 && old_target < old_n) ? cfg->instr_to_block[old_target] : -1; - copies_to_jumpif_succ[b] = target_block; /* may be -1 */ - } - /* Now walk phis once. Per operand: increment copies_per_block[pred], - * and if pred's JUMPIF target == this phi's succ block, also note that - * the JUMPIF edge carries a copy. */ - int extra_jumps = 0; - int modified_jumpifs = 0; - uint8_t *jumpif_edge_has_copy = tcc_mallocz(nb); - for (int sb = 0; sb < nb; sb++) { - for (IRPhiNode *phi = ssa->block_phis[sb]; phi; phi = phi->next) { - for (int pi = 0; pi < phi->num_operands; pi++) { - if (TCC_LOG_LS) - stats->phi_operands++; - int pred = phi->operands[pi].pred_block; - if (pred < 0 || pred >= nb) continue; - if (phi->operands[pi].vreg < 0) continue; - if (!ra_phi_copy_needed(ir, phi, pi)) continue; - copies_per_block[pred]++; - total_copies++; - if (TCC_LOG_LS) { - stats->parallel_requested++; - ra_phi_stats_add_participant(stats, phi->dest_vreg); - ra_phi_stats_add_participant(stats, phi->operands[pi].vreg); - } - if (copies_to_jumpif_succ[pred] == sb) - jumpif_edge_has_copy[pred] = 1; - } - } - } - for (int b = 0; b < nb; b++) { - if (jumpif_edge_has_copy[b]) { - extra_jumps++; - modified_jumpifs++; - } - } - tcc_free(copies_to_jumpif_succ); - tcc_free(jumpif_edge_has_copy); - - RA_DBG("SSA phi resolver: total_copies=%d extra_jumps=%d", total_copies, extra_jumps); - for (int b = 0; b < nb; b++) { - if (copies_per_block[b] > 0) - RA_DBG(" block %d: %d copies", b, copies_per_block[b]); - } - - /* Dump all phi nodes for debugging */ - for (int sb = 0; sb < nb; sb++) { - for (IRPhiNode *phi = ssa->block_phis[sb]; phi; phi = phi->next) { - IRLiveInterval *dest_li = tcc_ir_vreg_live_interval(ir, phi->dest_vreg); - RA_DBG(" phi in block %d: dest=T%d (r0=%d off=%d) ops=%d", - sb, TCCIR_DECODE_VREG_POSITION(phi->dest_vreg), - dest_li ? dest_li->allocation.r0 : -99, - dest_li ? dest_li->allocation.offset : -99, - phi->num_operands); - for (int pi = 0; pi < phi->num_operands; pi++) { - IRLiveInterval *src_li = tcc_ir_vreg_is_valid(ir, phi->operands[pi].vreg) ? - tcc_ir_vreg_live_interval(ir, phi->operands[pi].vreg) : NULL; - int needed = ra_phi_copy_needed(ir, phi, pi); - RA_DBG(" op[%d]: src=T%d pred=%d (r0=%d off=%d) needed=%d", - pi, TCCIR_DECODE_VREG_POSITION(phi->operands[pi].vreg), - phi->operands[pi].pred_block, - src_li ? src_li->allocation.r0 : -99, - src_li ? src_li->allocation.offset : -99, - needed); - } - } - } - - if (total_copies == 0) { - tcc_free(copies_per_block); - return; - } - - /* Track emitted phi copies so we can extend source vreg live intervals */ - RAPhiCopyRecord *copy_records = tcc_mallocz(sizeof(RAPhiCopyRecord) * RA_MAX_PHI_COPY_RECORDS); - int copy_record_count = 0; - - /* Build new instruction array with phi copies inserted */ - int new_n = old_n + total_copies + extra_jumps; - int new_cap = new_n + 16; - new_cap += total_copies; - IRQuadCompact *new_instrs = tcc_mallocz(new_cap * sizeof(IRQuadCompact)); - int *old_to_new = tcc_malloc(old_n * sizeof(int)); - - /* Grow operand pool */ - int pool_base = ir->iroperand_pool_count; - int needed_pool = total_copies * 4 + extra_jumps + modified_jumpifs * 2; - while (pool_base + needed_pool > ir->iroperand_pool_capacity) { - int nc = ir->iroperand_pool_capacity ? ir->iroperand_pool_capacity * 2 : 256; - ir->iroperand_pool = tcc_realloc(ir->iroperand_pool, nc * sizeof(IROperand)); - ir->iroperand_pool_capacity = nc; - } - - RAPhiPredIndex phi_pred_idx; - ra_phi_pred_index_build(cfg, ssa, &phi_pred_idx); - - int wp = 0; - int pool_wp = pool_base; - int phi_spill_cursor = ra_find_phi_spill_cursor(ir); - int first_phi_temp_pos = ir->next_temporary_variable; - - for (int b = 0; b < nb; b++) { - IRBasicBlock *bb = &cfg->blocks[b]; - int last_instr = bb->end_idx - 1; - int insert_before = bb->end_idx; - if (last_instr >= bb->start_idx) { - TccIrOp op = ir->compact_instructions[last_instr].op; - if (op == TCCIR_OP_JUMP || op == TCCIR_OP_JUMPIF || - op == TCCIR_OP_RETURNVALUE || op == TCCIR_OP_RETURNVOID || - op == TCCIR_OP_IJUMP || op == TCCIR_OP_SWITCH_TABLE) - insert_before = last_instr; - } - - if (copies_per_block[b] > 0 && last_instr >= bb->start_idx && - ir->compact_instructions[last_instr].op == TCCIR_OP_JUMPIF) { - IRQuadCompact *term = &ir->compact_instructions[last_instr]; - IROperand old_dest = tcc_ir_op_get_dest(ir, term); - int old_target = (int)irop_get_imm64_ex(ir, old_dest); - int target_block = (old_target >= 0 && old_target < old_n) ? cfg->instr_to_block[old_target] : -1; - int fallthrough_block = (b + 1 < nb) ? b + 1 : -1; - int target_count = (target_block >= 0) ? ra_count_phi_copies_for_pred(ir, cfg, ssa, b, target_block) : 0; - int fallthrough_count = - (fallthrough_block >= 0) ? ra_count_phi_copies_for_pred(ir, cfg, ssa, b, fallthrough_block) : 0; - /* Every copy counted for this pred must land on the target or the - * fall-through edge. A shortfall means the CFG no longer matches - * ssa->block_phis (a pass retargeted a jump while phis were live) and - * a phi arm would be SILENTLY dropped — a guaranteed miscompile. - * Surface it loudly instead. */ - if (target_count + fallthrough_count < copies_per_block[b]) - fprintf(stderr, - "tcc: internal warning: %d phi cop%s dropped at block %d " - "(JUMPIF target %d -> block %d, fallthrough block %d) — " - "CFG/phi desync, expect wrong code\n", - copies_per_block[b] - target_count - fallthrough_count, - (copies_per_block[b] - target_count - fallthrough_count) == 1 ? "y" : "ies", - b, old_target, target_block, fallthrough_block); - - for (int i = bb->start_idx; i < last_instr; i++) { - old_to_new[i] = wp; - new_instrs[wp++] = ir->compact_instructions[i]; - } - - old_to_new[last_instr] = wp; - if (target_count > 0) { - IROperand cond = tcc_ir_op_get_src1(ir, term); - cond.u.imm32 = ra_invert_cond((int)irop_get_imm64_ex(ir, cond)); - - IROperand skip_dest = old_dest; - skip_dest.u.imm32 = -(wp + 1 + target_count + 1 + 1); - int skip_dest_pool_idx = pool_wp; - ir->iroperand_pool[pool_wp] = skip_dest; - ir->iroperand_pool[pool_wp + 1] = cond; - new_instrs[wp] = *term; - new_instrs[wp].operand_base = pool_wp; - wp++; - pool_wp += 2; - - RAPhiCopy *copies = tcc_malloc(sizeof(RAPhiCopy) * target_count); - int copy_count = ra_collect_phi_copies_for_pred(ir, cfg, ssa, b, target_block, &phi_pred_idx, copies); - ra_emit_scheduled_phi_copies(ir, new_instrs, &wp, &pool_wp, copies, copy_count, b, - copy_records, ©_record_count, &phi_spill_cursor, stats); - tcc_free(copies); - - ir->iroperand_pool[pool_wp] = old_dest; - new_instrs[wp].op = TCCIR_OP_JUMP; - new_instrs[wp].operand_base = pool_wp; - new_instrs[wp].line_num = term->line_num; - new_instrs[wp].is_jump_target = 0; - wp++; - pool_wp++; - - /* The inverted JUMPIF skips over the phi copies AND this back-edge JUMP, - * landing on the instruction right after the JUMP we just wrote (== wp - * now). Compute the skip target HERE, after the JUMP write, using the - * current wp — NOT before emitting the copies/JUMP. Reading wp earlier - * (the old `-(wp + 2)` before the copy emit) is fragile: the value used - * must reflect the copies just emitted via ra_emit_scheduled_phi_copies - * (which advances wp through &wp). Encode as the negative sentinel the - * "Fix jump targets" pass below decodes with `-old_target - 1`, so a - * target of `wp` is stored as `-(wp + 1)`. */ - skip_dest = ir->iroperand_pool[skip_dest_pool_idx]; - skip_dest.u.imm32 = -(wp + 1); - ir->iroperand_pool[skip_dest_pool_idx] = skip_dest; - } else { - new_instrs[wp++] = *term; - } - - if (fallthrough_count > 0) { - RAPhiCopy *copies = tcc_malloc(sizeof(RAPhiCopy) * fallthrough_count); - int copy_count = ra_collect_phi_copies_for_pred(ir, cfg, ssa, b, fallthrough_block, &phi_pred_idx, copies); - ra_emit_scheduled_phi_copies(ir, new_instrs, &wp, &pool_wp, copies, copy_count, b, - copy_records, ©_record_count, &phi_spill_cursor, stats); - tcc_free(copies); - } - continue; - } - - /* Keep conditional phi copies off the compare/test -> JUMPIF edge: - * physical-register copies can otherwise clobber the condition flags. */ - for (int i = bb->start_idx; i < insert_before; i++) { - old_to_new[i] = wp; - new_instrs[wp++] = ir->compact_instructions[i]; - } - - int pre_copy_wp = wp; - - /* Insert phi copies for this block's successors */ - if (copies_per_block[b] > 0) { - RAPhiCopy *copies = tcc_malloc(sizeof(RAPhiCopy) * copies_per_block[b]); - int copy_count = ra_collect_phi_copies_for_pred(ir, cfg, ssa, b, -1, &phi_pred_idx, copies); - ra_emit_scheduled_phi_copies(ir, new_instrs, &wp, &pool_wp, copies, copy_count, b, - copy_records, ©_record_count, &phi_spill_cursor, stats); - tcc_free(copies); - } - - /* Copy terminator and remaining instructions */ - for (int i = insert_before; i < bb->end_idx; i++) { - if (i == bb->start_idx && copies_per_block[b] > 0) - old_to_new[i] = pre_copy_wp; - else - old_to_new[i] = wp; - new_instrs[wp++] = ir->compact_instructions[i]; - } - } - - ir->iroperand_pool_count = pool_wp; - - ra_phi_pred_index_free(&phi_pred_idx); - - /* Fix jump targets */ - for (int i = 0; i < wp; i++) { - IRQuadCompact *q = &new_instrs[i]; - if (q->op == TCCIR_OP_JUMP || q->op == TCCIR_OP_JUMPIF) { - IROperand dest = tcc_ir_op_get_dest(ir, q); - int old_target = (int)irop_get_imm64_ex(ir, dest); - if (old_target < 0) { - dest.u.imm32 = -old_target - 1; - ir->iroperand_pool[q->operand_base] = dest; - } else if (old_target >= 0 && old_target < old_n) { - dest.u.imm32 = old_to_new[old_target]; - ir->iroperand_pool[q->operand_base] = dest; - } else if (old_target >= old_n) { - dest.u.imm32 = wp + (old_target - old_n); - ir->iroperand_pool[q->operand_base] = dest; - } - } - } - - /* Fix switch table targets */ - for (int t = 0; t < ir->num_switch_tables; t++) { - TCCIRSwitchTable *table = &ir->switch_tables[t]; - for (int ti = 0; ti < table->num_entries; ti++) { - int ot = table->targets[ti]; - if (ot >= 0 && ot < old_n) table->targets[ti] = old_to_new[ot]; - } - if (table->default_target >= 0 && table->default_target < old_n) - table->default_target = old_to_new[table->default_target]; - } - - /* Replace instruction array */ - tcc_free(ir->compact_instructions); - ir->compact_instructions = new_instrs; - ir->compact_instructions_size = new_cap; - ir->next_instruction_index = wp; - - /* Rebuild is_jump_target flags */ - for (int i = 0; i < wp; i++) - new_instrs[i].is_jump_target = 0; - for (int i = 0; i < wp; i++) { - IRQuadCompact *q = &new_instrs[i]; - if (q->op == TCCIR_OP_JUMP || q->op == TCCIR_OP_JUMPIF) { - IROperand dest = tcc_ir_op_get_dest(ir, q); - int tgt = (int)irop_get_imm64_ex(ir, dest); - if (tgt >= 0 && tgt < wp) new_instrs[tgt].is_jump_target = 1; - } - } - for (int t = 0; t < ir->num_switch_tables; t++) { - TCCIRSwitchTable *table = &ir->switch_tables[t]; - for (int ti = 0; ti < table->num_entries; ti++) { - int tgt = table->targets[ti]; - if (tgt >= 0 && tgt < wp) new_instrs[tgt].is_jump_target = 1; - } - } - - /* The remaining steps (live-interval remap/extend, live_regs bitmap) - * only apply when phi resolution runs after register allocation. - * Pre-RA, no LS intervals or bitmap exist yet — skip. ra_build_intervals - * will scan the freshly emitted ASSIGN copies and produce correct - * intervals from scratch. - * - * Clear ssa->block_phis: the explicit copies we just inserted are now - * the source of truth. Leaving phi nodes active confuses the interval - * builder (it tries to extend phi-dest intervals as if the phi were - * still semantically active, on top of the now-explicit defs). */ - if (ra_phi_resolve_pre_ra_mode) { - /* Free each block's phi list before detaching it — the explicit copies are - * now the source of truth, so these nodes are dead. Merely NULLing the - * heads (as before) orphaned every phi node + operand array: tcc_ir_ssa_free - * later sees an empty block_phis and frees nothing, leaking on every compile. */ - for (int b = 0; b < nb; b++) { - IRPhiNode *phi = ssa->block_phis[b]; - while (phi) { - IRPhiNode *next = phi->next; - tcc_free(phi->operands); - tcc_free(phi); - phi = next; - } - ssa->block_phis[b] = NULL; - } - tcc_free(old_to_new); - tcc_free(copies_per_block); - tcc_free(copy_records); - return; - } - - /* Remap live interval start/end */ - for (int i = 0; i < ir->ls.next_interval_index; i++) { - LSLiveInterval *lsi = &ir->ls.intervals[i]; - int32_t vreg = lsi->vreg; - IRLiveInterval *li = tcc_ir_vreg_live_interval(ir, vreg); - if (!li) continue; - if (TCCIR_DECODE_VREG_TYPE(vreg) == TCCIR_VREG_TYPE_TEMP && - TCCIR_DECODE_VREG_POSITION(vreg) >= first_phi_temp_pos) - continue; - /* Remap start */ - if (li->start < (uint32_t)old_n) { - li->start = old_to_new[li->start]; - lsi->start = li->start; - } - /* Remap end */ - if (li->end < (uint32_t)old_n) { - li->end = old_to_new[li->end]; - lsi->end = li->end; - } - } - - tcc_free(old_to_new); - tcc_free(copies_per_block); - - /* Extend source vreg live intervals to cover phi copy instructions. - * Phi copies read the source vreg at the copy's instruction index, - * so the source must be alive there. Without this extension, the - * scratch register allocator may reuse the source's register. */ - for (int i = 0; i < copy_record_count; i++) { - int32_t sv = copy_records[i].src_vreg; - int ci = copy_records[i].new_instr_idx; - IRLiveInterval *li = tcc_ir_vreg_live_interval(ir, sv); - if (!li) continue; - if ((int)li->end < ci) { - RA_DBG(" phi copy: extending vreg 0x%x end from %d to %d", sv, (int)li->end, ci); - li->end = ci; - } - for (int j = 0; j < ir->ls.next_interval_index; j++) { - LSLiveInterval *lsi = &ir->ls.intervals[j]; - if (lsi->vreg == sv && (int)lsi->end < ci) { - lsi->end = ci; - break; - } - } - } - tcc_free(copy_records); - - /* Re-extend intervals for backward jumps in the post-phi instruction stream. - * The original backward-jump extension was computed on pre-phi indices; phi - * copies inserted before a backward jump are therefore not covered. Any - * register live at the jump target must also be considered live during the - * phi copies so that the scratch-register allocator does not clobber it. */ - { - int new_n = ir->next_instruction_index; - for (int i = 0; i < new_n; i++) { - IRQuadCompact *q = &ir->compact_instructions[i]; - if (q->op == TCCIR_OP_JUMP || q->op == TCCIR_OP_JUMPIF) { - int target = (int)tcc_ir_op_get_dest(ir, q).u.imm32; - if (target >= 0 && target < new_n && target < i) { - for (int j = 0; j < ir->ls.next_interval_index; j++) { - LSLiveInterval *lsi = &ir->ls.intervals[j]; - IRLiveInterval *li = tcc_ir_vreg_live_interval(ir, lsi->vreg); - if (!li) continue; - if ((int)li->start <= target && (int)li->end >= target && - (int)li->end < i) { - RA_DBG(" post-phi back-edge extend: vreg 0x%x [%d,%d] -> [%d,%d]", - lsi->vreg, (int)li->start, (int)li->end, (int)li->start, i); - li->end = i; - lsi->end = i; - } - } - } - } - } - } - - /* Build live_regs_by_instruction table */ - ra_build_live_regs_bitmap(ir); -} - -static void ra_build_live_regs_bitmap(TCCIRState *ir) -{ - uint32_t max_end = 0; - for (int i = 0; i < ir->ls.next_interval_index; i++) { - LSLiveInterval *lsi = &ir->ls.intervals[i]; - if (lsi->stack_location != 0 || lsi->r0 < 0) continue; - if (lsi->reg_type != LS_REG_TYPE_INT && lsi->reg_type != LS_REG_TYPE_LLONG && - lsi->reg_type != LS_REG_TYPE_DOUBLE_SOFT && lsi->reg_type != LS_REG_TYPE_COMPLEX_FLOAT) - continue; - if (lsi->end > max_end) max_end = lsi->end; - } - int sz = (int)max_end + 1; - if (sz < ir->next_instruction_index) sz = ir->next_instruction_index; - if (sz > 0) { - if (ir->ls.live_regs_by_instruction) - tcc_free(ir->ls.live_regs_by_instruction); - ir->ls.live_regs_by_instruction = tcc_mallocz(sizeof(uint32_t) * sz); - ir->ls.live_regs_by_instruction_size = sz; - RA_DBG("SSA live_regs_by_instruction build (sz=%d)", sz); - for (int i = 0; i < ir->ls.next_interval_index; i++) { - LSLiveInterval *lsi = &ir->ls.intervals[i]; - if (lsi->stack_location != 0 || lsi->r0 < 0) continue; - if (lsi->reg_type != LS_REG_TYPE_INT && lsi->reg_type != LS_REG_TYPE_LLONG && - lsi->reg_type != LS_REG_TYPE_DOUBLE_SOFT && lsi->reg_type != LS_REG_TYPE_COMPLEX_FLOAT) - continue; - uint32_t mask = 0; - if (lsi->r0 >= 0 && lsi->r0 < 16) mask |= (1u << lsi->r0); - if (lsi->r1 >= 0 && lsi->r1 < 16) mask |= (1u << lsi->r1); - if (!mask) continue; - int s = (int)lsi->start, e = (int)lsi->end; - if (s < 0) s = 0; - if (e >= sz) e = sz - 1; - RA_DBG(" interval vreg=0x%x r0=R%d r1=%d range=[%d,%d] mask=0x%x", - lsi->vreg, lsi->r0, lsi->r1, s, e, mask); - for (int k = s; k <= e; k++) - ir->ls.live_regs_by_instruction[k] |= mask; - } - - if (TCC_LOG_LS) { - for (int k = 0; k < sz; k++) - RA_DBG(" instr[%d] live=0x%x", k, ir->ls.live_regs_by_instruction[k]); - } - } -} - -/* ============================================================================ - * Graph-based register coalescing - * - * The linear scan's per-decision coalescing (boundary / loop-phi / exit-phi - * "transfer") merges only simple 2-vreg copy chains; it cannot merge a - * multi-predecessor merge-phi (e.g. an induction variable that, after loop - * rotation, enters the next loop via two edges). And ra_build_intervals' - * single-[start,end] model reports FALSE overlaps for SSA versions that are - * live on mutually-exclusive paths, so they look like they interfere. - * - * This pass builds ACCURATE liveness (backward dataflow over a fresh CFG) and a - * real interference graph, then conservatively coalesces copy-related - * non-interfering vregs (union-find). Merged classes are applied via the - * `coalesce_to` interval-merge: one representative interval covers the union of - * the class' ranges, non-reps are skipped by the scan and inherit the rep's - * register, and the residual `mov R,R` is erased by tcc_ir_move_coalescing. - * - * On by default at -O1+ (level 2 = apply, INT single-register, pure copies). - * Overrides: TCC_NO_COALESCE disables it; TCC_COALESCE=N forces level N - * (1 = compute only, 2 = apply, 3 = also coalesce RMW two-address edges). - * Bails on functions with un-enumerated CFG edges (IJUMP / SWITCH_*) or any - * 64-bit (LLONG) value. - * ============================================================================ */ - -TCC_DBG_ENV_FLAG(ra_no_coalesce, "TCC_NO_COALESCE") -TCC_DBG_ENV_INT(ra_coalesce_env_level, "TCC_COALESCE", 2) /* default: apply, pure copies */ -TCC_DBG_ENV_FLAG(ra_dump_ir_co, "DUMP_IR_CO") -TCC_DBG_ENV_FLAG(ra_no_coalesce_llong, "TCC_NO_COALESCE_LLONG") -/* Memory-SSA is opt-in; see the comment at its use site in tcc_ir_ssa_regalloc. */ -TCC_DBG_ENV_FLAG(ra_mem_ssa_fwd, "TCC_MEM_SSA_FWD") -TCC_DBG_ENV_FLAG(ra_mem_ssa_dump, "TCC_MEM_SSA_DUMP") -TCC_DBG_ENV_FLAG(ra_mem_ssa_verify_on, "TCC_MEM_SSA") - -static int ra_coalesce_level(void) -{ - return ra_no_coalesce() ? 0 : ra_coalesce_env_level(); -} - -/* Collect an instruction's def and use vreg operands into out-params. - * STORE-class dest is a USE (it is the address); MLA accumulator is a USE. - * Matches ra_build_intervals' operand semantics exactly. */ -static void ra_co_ops(TCCIRState *ir, IRQuadCompact *q, - int32_t *out_def, int *has_def, int32_t uses[4], int *nuse) -{ - *has_def = 0; - *nuse = 0; - if (q->op == TCCIR_OP_NOP) return; - int store_class = (q->op == TCCIR_OP_STORE || q->op == TCCIR_OP_STORE_INDEXED || - q->op == TCCIR_OP_STORE_POSTINC); - if (irop_config[q->op].has_src1) { - IROperand s = tcc_ir_op_get_src1(ir, q); - if (irop_has_vreg(s) && !irop_is_immediate(s)) uses[(*nuse)++] = irop_get_vreg(s); - } - if (irop_config[q->op].has_src2) { - IROperand s = tcc_ir_op_get_src2(ir, q); - if (irop_has_vreg(s) && !irop_is_immediate(s)) uses[(*nuse)++] = irop_get_vreg(s); - } - if (q->op == TCCIR_OP_MLA) { - IROperand s = tcc_ir_op_get_accum(ir, q); - if (irop_has_vreg(s) && !irop_is_immediate(s)) uses[(*nuse)++] = irop_get_vreg(s); - } - if (irop_config[q->op].has_dest) { - IROperand d = tcc_ir_op_get_dest(ir, q); - if (irop_has_vreg(d)) { - /* A plain STORE whose TEMP dest has is_lval clear is SSA renaming's - * in-place register write of a promoted var (a pointer-deref store - * keeps is_lval) — a DEF, matching ra_build_intervals. Full-width - * only: a narrow store leaves the old upper bytes live-before. */ - int inplace_def = (q->op == TCCIR_OP_STORE && !d.is_lval && !d.is_local && - TCCIR_DECODE_VREG_TYPE(irop_get_vreg(d)) == TCCIR_VREG_TYPE_TEMP && - (d.btype == IROP_BTYPE_INT32 || d.btype == IROP_BTYPE_FLOAT32 || - d.btype == IROP_BTYPE_INT64 || d.btype == IROP_BTYPE_FLOAT64)); - if (store_class && !inplace_def) uses[(*nuse)++] = irop_get_vreg(d); - else { *out_def = irop_get_vreg(d); *has_def = 1; } - } - } -} - -/* Static counts of references from 16-bit-encodable ops; docs/regalloc_narrow_pref.md */ -static void ra_build_narrow_weights(TCCIRState *ir, const RegAllocTarget *target, - SSAInterval *intervals, int count, int max_vreg_pos) -{ - if (!target->op_narrow_capable || tcc_state->optimize < 1 || count <= 0 || max_vreg_pos <= 0) - return; - if (tcc_ir_opt_pass_disabled("ra:narrow_pref")) - return; - int tbl = 4 * max_vreg_pos; - int32_t *iv_of = tcc_malloc(sizeof(int32_t) * tbl); - for (int i = 0; i < tbl; i++) iv_of[i] = -1; - for (int i = 0; i < count; i++) { - int idx = TCCIR_DECODE_VREG_TYPE(intervals[i].vreg) * max_vreg_pos + - TCCIR_DECODE_VREG_POSITION(intervals[i].vreg); - if (idx >= 0 && idx < tbl) iv_of[idx] = i; - } - for (int i = 0; i < ir->next_instruction_index; i++) { - IRQuadCompact *q = &ir->compact_instructions[i]; - if (q->op == TCCIR_OP_NOP) continue; - int src2_imm = 0, scale = 0; - if (irop_config[q->op].has_src2) - src2_imm = irop_is_immediate(tcc_ir_op_get_src2(ir, q)); - if (q->op == TCCIR_OP_LOAD_INDEXED || q->op == TCCIR_OP_STORE_INDEXED) { - IROperand s = tcc_ir_op_get_scale(ir, q); - if (irop_get_tag(s) == IROP_TAG_IMM32) scale = s.u.imm32; - } - if (!target->op_narrow_capable(q->op, src2_imm, scale)) continue; - int32_t def, uses[4]; - int has_def, nuse; - ra_co_ops(ir, q, &def, &has_def, uses, &nuse); - int32_t ops[5]; - int nops = 0; - if (has_def) ops[nops++] = def; - for (int k = 0; k < nuse; k++) ops[nops++] = uses[k]; - for (int k = 0; k < nops; k++) { - int idx = TCCIR_DECODE_VREG_TYPE(ops[k]) * max_vreg_pos + - TCCIR_DECODE_VREG_POSITION(ops[k]); - if (idx < 0 || idx >= tbl || iv_of[idx] < 0) continue; - SSAInterval *iv = &intervals[iv_of[idx]]; - if (iv->narrow_uses < UINT16_MAX) iv->narrow_uses++; - } - } - tcc_free(iv_of); -} - -/* Backward-liveness bit matrix: nb blocks x tbl vreg-slots, one allocation, - * auto-freed on every exit path. Rows feed the tcc_bitspan_* word kernels. */ -TCC_BIT_MATRIX_DEFINE(RaLiveMatrix) - -/* Refine live_regs_by_instruction (the interval-derived approximation the - * scratch-register picker consults) with ACCURATE per-instruction liveness from - * a real CFG backward dataflow. - * - * The interval bitmap models each value as one contiguous [start,end] range. - * For a loop-carried value (defined inside a rotated loop body and live across - * the back-edge into the next iteration) that single range does NOT span the - * loop-header prefix where the value is still live, so the bitmap under-reports - * the value's register as free there. The scratch picker then hands it out and - * clobbers the loop-carried value (random-C O2 wrong-code / HardFault once loop - * rotation is enabled — Finding #15 follow-up, seeds 244 et al). - * - * This dataflow (same ra_co_ops def/use model the graph-coalescer trusts) marks - * every register holding a genuinely-live, register-resident vreg. It is - * strictly conservative for the picker: it can only ADD live bits, never remove - * them, so it can never introduce a new clobber — it only prevents real ones. - * Bails (leaving the interval bitmap as-is) on functions with un-enumerated - * edges (IJUMP / SWITCH_TABLE), matching the coalescer's own guard. */ -static void ra_refine_live_regs_accurate(TCCIRState *ir) -{ - int n = ir->next_instruction_index; - if (n <= 0) return; - int has_backedge = 0; - for (int i = 0; i < n; i++) { - IRQuadCompact *q = &ir->compact_instructions[i]; - int op = q->op; - if (op == TCCIR_OP_IJUMP || op == TCCIR_OP_SWITCH_TABLE || op == TCCIR_OP_SWITCH_LOAD) - return; - if (op == TCCIR_OP_JUMP || op == TCCIR_OP_JUMPIF) { - int t = (int)tcc_ir_op_get_dest(ir, q).u.imm32; - if (t >= 0 && t < i) - has_backedge = 1; - } - } - if (!has_backedge) return; - IRCFG *cfg = tcc_ir_cfg_build(ir); - if (!cfg) return; - tcc_ir_cfg_compute_rpo(cfg); /* the dataflow needs only the RPO ordering */ - int nb = cfg->num_blocks; - if (nb <= 0) { tcc_ir_cfg_free(cfg); return; } - /* vreg index space */ - int maxpos = 1; - for (int j = 0; j < ir->ls.next_interval_index; j++) { - int p = TCCIR_DECODE_VREG_POSITION(ir->ls.intervals[j].vreg); - if (p + 1 > maxpos) maxpos = p + 1; - } - int tbl = 4 * maxpos; - #define DVIDX(vr) ((TCCIR_DECODE_VREG_TYPE(vr) * maxpos) + TCCIR_DECODE_VREG_POSITION(vr)) - /* vreg -> physical regs */ - int8_t *vr0 = tcc_malloc(tbl); int8_t *vr1 = tcc_malloc(tbl); - for (int i = 0; i < tbl; i++) { vr0[i] = -1; vr1[i] = -1; } - for (int j = 0; j < ir->ls.next_interval_index; j++) { - LSLiveInterval *iv = &ir->ls.intervals[j]; - if (iv->stack_location != 0) continue; - int vi = DVIDX(iv->vreg); - if (vi < 0 || vi >= tbl) continue; - vr0[vi] = (int8_t)iv->r0; vr1[vi] = (int8_t)iv->r1; - } - /* Only register-resident slots can contribute to the final mask; liveness is - * per-vreg independent, so restricting the dataflow universe to them is exact. */ - int *dense = tcc_malloc(sizeof(int) * tbl); - int ndense = 0; - for (int i = 0; i < tbl; i++) { - int tracked = (vr0[i] >= 0 && vr0[i] < 16) || (vr1[i] >= 0 && vr1[i] < 16); - dense[i] = tracked ? ndense++ : -1; - } - if (ndense == 0) { - tcc_free(vr0); tcc_free(vr1); tcc_free(dense); - tcc_ir_cfg_free(cfg); - return; - } - int nw = (ndense + 63) / 64; - bit_matrix(RaLiveMatrix) useb = {0}; - bit_matrix(RaLiveMatrix) defbk = {0}; - bit_matrix(RaLiveMatrix) livein = {0}; - bit_matrix(RaLiveMatrix) liveout = {0}; - RaLiveMatrix_init(&useb, nb, ndense); - RaLiveMatrix_init(&defbk, nb, ndense); - RaLiveMatrix_init(&livein, nb, ndense); - RaLiveMatrix_init(&liveout, nb, ndense); - for (int b = 0; b < nb; b++) { - uint64_t *ub = RaLiveMatrix_row(&useb, b), *db = RaLiveMatrix_row(&defbk, b); - int s = cfg->blocks[b].start_idx, e = cfg->blocks[b].end_idx; - for (int i = s; i < e && i < n; i++) { - int32_t def=-1, hd=0, uses[4], nu=0; - ra_co_ops(ir, &ir->compact_instructions[i], &def, &hd, uses, &nu); - for (int k=0;k=tbl)continue; int ud=dense[u]; if(ud<0)continue; if(!tcc_bitspan_test(db,ud)) tcc_bitspan_set(ub,ud);} - if (hd && tcc_ir_vreg_is_valid(ir,def)){int d=DVIDX(def); if(d>=0&&d=0) tcc_bitspan_set(db,dense[d]);} - } - } - int changed=1, guard=0; - while (changed && guard++ < nb+4) { - changed=0; - for (int ri=cfg->rpo_count-1; ri>=0; ri--) { - int b = cfg->rpo_order ? cfg->rpo_order[ri] : ri; - if (b<0||b>=nb) continue; - uint64_t *lo = RaLiveMatrix_row(&liveout, b), *li = RaLiveMatrix_row(&livein, b); - uint64_t *ub = RaLiveMatrix_row(&useb, b), *db = RaLiveMatrix_row(&defbk, b); - tcc_bitspan_zero(lo, nw); - for (int si=0;siblocks[b].num_succs;si++){int sb=cfg->blocks[b].succs[si]; if(sb<0||sb>=nb)continue; tcc_bitspan_or(lo, RaLiveMatrix_row(&livein, sb), nw);} - changed |= tcc_bitspan_or_andnot(li, ub, lo, db, nw); - } - } - /* Loop-liveness completion. A value live at a loop header is live throughout - * the ENTIRE loop body (it round-trips the back-edge), but the interval model - * gives it a single [def,last-use] range that leaves the loop-header prefix - * uncovered — the scratch picker then reuses its register inside the loop and - * clobbers the loop-carried value (seed 244). For each back-edge, OR the - * registers live-IN at the loop header across the whole loop body [header, - * back-edge]. Scoped to loop bodies on purpose: a blanket per-instruction - * live-out refinement also marks straight-line liveness the interval model - * intentionally omits, which over-constrains the scratch picker and perturbs - * unrelated functions into latent-bug territory (seed 221). */ - for (int bi = 0; bi < n; bi++) { - IRQuadCompact *q = &ir->compact_instructions[bi]; - if (q->op != TCCIR_OP_JUMP && q->op != TCCIR_OP_JUMPIF) continue; - int t = (int)tcc_ir_op_get_dest(ir, q).u.imm32; - if (t < 0 || t >= bi) continue; /* not a back-edge */ - /* live-in at the loop header t: find the block starting at t. */ - int hb = -1; - for (int b = 0; b < nb; b++) if (cfg->blocks[b].start_idx == t) { hb = b; break; } - if (hb < 0) continue; - uint64_t *hli = RaLiveMatrix_row(&livein, hb); - uint32_t mask = 0; - for (int vi=0; vi=0 && vr0[vi]<16) mask |= (1u<=0 && vr1[vi]<16) mask |= (1u<ls.live_regs_by_instruction) continue; - int e = bi; if (e >= ir->ls.live_regs_by_instruction_size) e = ir->ls.live_regs_by_instruction_size - 1; - for (int k = t; k <= e; k++) - ir->ls.live_regs_by_instruction[k] |= mask; - } - #undef DVIDX - tcc_free(vr0);tcc_free(vr1);tcc_free(dense); - tcc_ir_cfg_free(cfg); -} - -static void ra_coalesce_graph(TCCIRState *ir, SSAInterval *intervals, int count, - int max_vreg_pos) -{ - int level = ra_coalesce_level(); - if (level <= 0 || tcc_state->optimize < 1 || count <= 1 || max_vreg_pos <= 0) - return; - int n = ir->next_instruction_index; - if (n <= 0) return; - TCC_DBG_BLOCK(ra_dump_ir_co) { printf("==== IR AT GRAPH-COALESCE ====\n"); tcc_ir_show(ir); fflush(stdout); } - - /* ---- Stage 1: fresh CFG (entry cfg is stale after phi resolution). ---- */ - IRCFG *cfg = tcc_ir_cfg_build(ir); - if (!cfg) return; - tcc_ir_cfg_compute_dominators(cfg); /* populates rpo_order/rpo_count for the dataflow */ - if (cfg->rpo_count <= 0) { tcc_ir_cfg_free(cfg); return; } - for (int i = 0; i < n; i++) { - int op = ir->compact_instructions[i].op; - /* IJUMP (computed goto) has un-enumerable successors — live-out is - * untrustworthy, bail. SWITCH_TABLE is fine: tcc_ir_cfg_build models - * every dispatch edge (targets[] is pre-filled with the default target - * for gap values, and out-of-range never reaches the dispatch — the - * frontend guards it with an explicit JUMPIF). SWITCH_LOAD is a plain - * table-value LOAD, not a jump. Bailing on switches was the single - * blocker for the 64-bit switch-merge family: every case temp kept its - * own spill slot + str/str/ldr/ldr round-trip (ashrdi-1 constant_shift: - * 488-byte frame vs GCC's zero). */ - if (op == TCCIR_OP_IJUMP) { - tcc_ir_cfg_free(cfg); - return; - } - } - int nb = cfg->num_blocks; - int tbl = 4 * max_vreg_pos; - int nw = (tbl + 63) / 64; - /* Guard pathologically large functions (compile-time / memory). */ - if (nb <= 0 || (long)nb * nw > (4L << 20)) { tcc_ir_cfg_free(cfg); return; } - - /* Historical note: this used to bail on any function containing a 64-bit - * (LLONG) interval, blaming the UMULL half-operand spill path for a - * bug_ull_mul miscompile under coalescing. The actual culprits were the - * deferred-PARAM liveness hole and the backedge_phi_hoist side-entry - * retarget (both since fixed); with the pair-aware pressure gate below, - * LLONG functions coalesce safely — including LLONG-LLONG pair copies - * (admitted in the union gate). TCC_NO_COALESCE_LLONG restores the old - * bail for attribution. */ - if (ra_no_coalesce_llong()) - for (int i = 0; i < count; i++) - if (intervals[i].reg_type == LS_REG_TYPE_LLONG) { tcc_ir_cfg_free(cfg); return; } - - #define VIDX(vr) ((TCCIR_DECODE_VREG_TYPE(vr) * max_vreg_pos) + TCCIR_DECODE_VREG_POSITION(vr)) - - int *iv_of = tcc_malloc(sizeof(int) * tbl); - for (int i = 0; i < tbl; i++) iv_of[i] = -1; - for (int i = 0; i < count; i++) { - int idx = VIDX(intervals[i].vreg); - if (idx >= 0 && idx < tbl) iv_of[idx] = i; - } - - /* ---- Deferred-PARAM uses: model FUNCPARAMVAL sources at their CALL. ---- - * Argument marshaling happens at the call site, so a FUNCPARAMVAL source - * is read again at the matching FUNCCALL — after any instructions between - * the PARAM quad and the CALL. ra_build_intervals extends interval ends - * accordingly; this liveness must do the same or the interference graph - * misses the overlap and a coalesced def between PARAM and CALL clobbers - * the argument before the call reads it (bug_stride_minimal: the CSE'd - * `i+1` loop increment merged with `i` while `printf`'s deferred PARAM - * still needed the old `i`). cid matching mirrors ra_build_intervals. */ - int *call_param_head = tcc_malloc(sizeof(int) * n); - int *param_sibling = tcc_malloc(sizeof(int) * n); - for (int i = 0; i < n; i++) { call_param_head[i] = -1; param_sibling[i] = -1; } - for (int i = 0; i < n; i++) { - IRQuadCompact *q = &ir->compact_instructions[i]; - if (q->op != TCCIR_OP_FUNCPARAMVAL) continue; - IROperand s1 = tcc_ir_op_get_src1(ir, q); - if (!irop_has_vreg(s1) || irop_is_immediate(s1)) continue; - int cid = TCCIR_DECODE_CALL_ID(irop_get_imm64_ex(ir, tcc_ir_op_get_src2(ir, q))); - if (cid < 0) continue; - for (int j = i + 1; j < n; j++) { - IRQuadCompact *qq = &ir->compact_instructions[j]; - if (qq->op != TCCIR_OP_FUNCCALLVOID && qq->op != TCCIR_OP_FUNCCALLVAL) continue; - int ccid = TCCIR_DECODE_CALL_ID(irop_get_imm64_ex(ir, tcc_ir_op_get_src2(ir, qq))); - if (ccid == cid) { - param_sibling[i] = call_param_head[j]; - call_param_head[j] = i; - break; - } - } - } - /* Iterate a CALL's deferred param-source vregs: body receives vreg in `vr_`. */ - #define RA_CO_FOR_CALL_PARAM_USES(call_idx, vr_, body) \ - do { \ - for (int p_ = call_param_head[(call_idx)]; p_ >= 0; p_ = param_sibling[p_]) { \ - IROperand ps_ = tcc_ir_op_get_src1(ir, &ir->compact_instructions[p_]); \ - int32_t vr_ = irop_get_vreg(ps_); \ - if (!tcc_ir_vreg_is_valid(ir, vr_)) continue; \ - body \ - } \ - } while (0) - - /* ---- Stage 2: backward liveness dataflow (live_in/live_out per block). ---- */ - bit_matrix(RaLiveMatrix) useb = {0}; - bit_matrix(RaLiveMatrix) defbk = {0}; - bit_matrix(RaLiveMatrix) livein = {0}; - bit_matrix(RaLiveMatrix) liveout = {0}; - RaLiveMatrix_init(&useb, nb, tbl); - RaLiveMatrix_init(&defbk, nb, tbl); - RaLiveMatrix_init(&livein, nb, tbl); - RaLiveMatrix_init(&liveout, nb, tbl); - - for (int b = 0; b < nb; b++) { - uint64_t *ub = RaLiveMatrix_row(&useb, b), *db = RaLiveMatrix_row(&defbk, b); - int s = cfg->blocks[b].start_idx, e = cfg->blocks[b].end_idx; - for (int i = s; i < e && i < n; i++) { - int32_t def = -1, hd = 0, uses[4], nu = 0; - ra_co_ops(ir, &ir->compact_instructions[i], &def, &hd, uses, &nu); - for (int k = 0; k < nu; k++) { - if (!tcc_ir_vreg_is_valid(ir, uses[k])) continue; - int u = VIDX(uses[k]); - if (u < 0 || u >= tbl) continue; - if (!tcc_bitspan_test(db, u)) tcc_bitspan_set(ub, u); /* upward-exposed use */ - } - RA_CO_FOR_CALL_PARAM_USES(i, vr, { - int u = VIDX(vr); - if (u >= 0 && u < tbl && !tcc_bitspan_test(db, u)) tcc_bitspan_set(ub, u); - }); - if (hd && tcc_ir_vreg_is_valid(ir, def)) { - int d = VIDX(def); - if (d >= 0 && d < tbl) tcc_bitspan_set(db, d); - } - } - } - - /* Fixpoint over reverse-RPO order. */ - int changed = 1, guard = 0; - while (changed && guard++ < nb + 4) { - changed = 0; - for (int ri = cfg->rpo_count - 1; ri >= 0; ri--) { - int b = cfg->rpo_order ? cfg->rpo_order[ri] : ri; - if (b < 0 || b >= nb) continue; - uint64_t *lo = RaLiveMatrix_row(&liveout, b), *li = RaLiveMatrix_row(&livein, b); - uint64_t *ub = RaLiveMatrix_row(&useb, b), *db = RaLiveMatrix_row(&defbk, b); - /* live_out = union of successors' live_in */ - tcc_bitspan_zero(lo, nw); - for (int si = 0; si < cfg->blocks[b].num_succs; si++) { - int sb = cfg->blocks[b].succs[si]; - if (sb < 0 || sb >= nb) continue; - tcc_bitspan_or(lo, RaLiveMatrix_row(&livein, sb), nw); - } - /* live_in = use ∪ (live_out − def) */ - changed |= tcc_bitspan_or_andnot(li, ub, lo, db, nw); - } - } - - /* ---- Register-pressure gate. ---- - * Coalescing can only ever ADD instructions (vs the baseline) by forcing a - * spill: merging two non-interfering values reduces distinct values at every - * point EXCEPT a liveness hole, where the merged interval occupies the - * register and raises pressure by one. If the function's peak INT pressure - * leaves headroom (< K allocatable int regs), no spill can result, so - * coalescing is a pure win (it only removes copies). When pressure already - * reaches K (spilling territory), coalescing's longer intervals can perturb - * the linear scan into worse spills (observed: large high-pressure functions - * regress). Bail in that case — it costs only the high-pressure functions, - * never the small loop kernels (e.g. memclr) this is built for. */ - { - int K = tcc_state->registers_for_allocator; - if (K <= 0 || K > 13) K = 13; - /* Weight per value: how many INT registers it occupies while live. - * LLONG / soft-double / complex-float take a pair; complex-double takes - * four. Counting pairs as 1 (the old behavior) under-approximates - * pressure in 64-bit-heavy functions and lets coalescing push them into - * spill territory. */ - uint8_t *iw = tcc_mallocz(tbl); - for (int i = 0; i < count; i++) { - int idx = VIDX(intervals[i].vreg); - if (idx < 0 || idx >= tbl) continue; - switch (intervals[i].reg_type) { - case LS_REG_TYPE_INT: - if (intervals[i].r1 < 0) iw[idx] = 1; - break; - case LS_REG_TYPE_LLONG: - case LS_REG_TYPE_DOUBLE_SOFT: - case LS_REG_TYPE_COMPLEX_FLOAT: - iw[idx] = 2; - break; - case LS_REG_TYPE_COMPLEX_DOUBLE: - iw[idx] = 4; - break; - default: - break; /* VFP-resident: no INT pressure */ - } - } - uint64_t *live = tcc_malloc(sizeof(uint64_t) * nw); - int maxp = 0; - for (int b = 0; b < nb && maxp < K; b++) { - uint64_t *lo = RaLiveMatrix_row(&liveout, b); - int p = 0; - tcc_bitspan_copy(live, lo, nw); - tcc_bitspan_for_each_set(lo, nw, tbl, vi) { - p += iw[vi]; - } - int s = cfg->blocks[b].start_idx, e = cfg->blocks[b].end_idx; - for (int i = e - 1; i >= s && i < n; i--) { - if (p > maxp) maxp = p; - IRQuadCompact *q = &ir->compact_instructions[i]; - int32_t def = -1, hd = 0, uses[4], nu = 0; - ra_co_ops(ir, q, &def, &hd, uses, &nu); - if (hd && tcc_ir_vreg_is_valid(ir, def)) { - int d = VIDX(def); - if (d >= 0 && d < tbl && tcc_bitspan_test(live, d)) { p -= iw[d]; tcc_bitspan_reset(live, d); } - } - for (int k = 0; k < nu; k++) { - if (!tcc_ir_vreg_is_valid(ir, uses[k])) continue; - int u = VIDX(uses[k]); - if (u >= 0 && u < tbl && !tcc_bitspan_test(live, u)) { p += iw[u]; tcc_bitspan_set(live, u); } - } - RA_CO_FOR_CALL_PARAM_USES(i, vr, { - int u = VIDX(vr); - if (u >= 0 && u < tbl && !tcc_bitspan_test(live, u)) { p += iw[u]; tcc_bitspan_set(live, u); } - }); - } - } - tcc_free(iw); tcc_free(live); - if (maxp >= K) { - RA_DBG("coalesce: skip — peak INT pressure %d >= K=%d", maxp, K); - tcc_free(iv_of); - tcc_free(call_param_head); tcc_free(param_sibling); - tcc_ir_cfg_free(cfg); - return; - } - } - - /* ---- Build per-block def bitmap to detect phi-related copies. ---- - * A copy dest that is defined in more than one block is a phi result - * (explicit copies inserted after SSA phi resolution). Coalescing such - * a dest with its source can overwrite the source's value on a sibling - * phi arm when the source is still live across the merge (seed 860). */ - uint8_t *def_block_count = tcc_mallocz(tbl); - int *def_last_block = tcc_malloc(sizeof(int) * tbl); - for (int i = 0; i < tbl; i++) def_last_block[i] = -1; - int *instr_block = tcc_malloc(sizeof(int) * n); - for (int i = 0; i < n; i++) instr_block[i] = -1; - /* Per-value linked list of def positions (head/next), so per-edge admission - * checks walk only that value's defs instead of rescanning all n - * instructions (quadratic on big functions — 101_cleanup 0.3s -> 19s). */ - int *def_head = tcc_malloc(sizeof(int) * tbl); - int *def_next = tcc_malloc(sizeof(int) * n); - for (int i = 0; i < tbl; i++) def_head[i] = -1; - for (int i = 0; i < n; i++) def_next[i] = -1; - for (int b = 0; b < nb; b++) { - int s = cfg->blocks[b].start_idx, e = cfg->blocks[b].end_idx; - for (int i = s; i < e && i < n; i++) { - instr_block[i] = b; - IRQuadCompact *q = &ir->compact_instructions[i]; - int32_t def = -1, hd = 0, uses[4], nu = 0; - ra_co_ops(ir, q, &def, &hd, uses, &nu); - if (hd && tcc_ir_vreg_is_valid(ir, def)) { - int d = VIDX(def); - if (d >= 0 && d < tbl && def_last_block[d] != b) { - def_last_block[d] = b; - if (def_block_count[d] < 2) def_block_count[d]++; - } - } - } - } - tcc_free(def_last_block); - /* Def list covers ALL instructions (not just CFG-block ranges) so the - * admission walks see exactly the defs the previous full scans saw — - * a def outside any block (instr_block -1) must keep rejecting. */ - for (int i = 0; i < n; i++) { - IRQuadCompact *q = &ir->compact_instructions[i]; - int32_t def = -1, hd = 0, uses[4], nu = 0; - ra_co_ops(ir, q, &def, &hd, uses, &nu); - if (hd && tcc_ir_vreg_is_valid(ir, def)) { - int d = VIDX(def); - if (d >= 0 && d < tbl) { - def_next[i] = def_head[d]; - def_head[d] = i; - } - } - } - - /* ---- Collect copy edges + candidate set (Stage 4 prep). ---- */ - /* Copy edge kinds: ASSIGN dst<-src; two-address dst<-src OP imm (ADD/SUB). */ - int *cand_id = tcc_malloc(sizeof(int) * tbl); - for (int i = 0; i < tbl; i++) cand_id[i] = -1; - int ncand = 0; - int ecap = 16, ne = 0; - int32_t *edge_d = tcc_malloc(sizeof(int32_t) * ecap); - int32_t *edge_s = tcc_malloc(sizeof(int32_t) * ecap); - #define ADD_CAND(vidx) do { if (cand_id[vidx] < 0) cand_id[vidx] = ncand++; } while (0) - for (int i = 0; i < n; i++) { - IRQuadCompact *q = &ir->compact_instructions[i]; - int32_t dv = -1, sv = -1; - if (q->op == TCCIR_OP_ASSIGN) { - IROperand d = tcc_ir_op_get_dest(ir, q), s = tcc_ir_op_get_src1(ir, q); - /* Pure register copy only: a dereferenced operand means this ASSIGN is a - * load/store (`T1 = *T0`), NOT a copy — coalescing its operands is wrong. */ - if (irop_has_vreg(d) && irop_has_vreg(s) && !irop_is_immediate(s) && - !d.is_lval && !s.is_lval) { - dv = irop_get_vreg(d); sv = irop_get_vreg(s); - } - } else if ((q->op == TCCIR_OP_ADD || q->op == TCCIR_OP_SUB) && level >= 3) { - IROperand d = tcc_ir_op_get_dest(ir, q), s1 = tcc_ir_op_get_src1(ir, q), - s2 = tcc_ir_op_get_src2(ir, q); - if (irop_has_vreg(d) && irop_has_vreg(s1) && !irop_is_immediate(s1) && - irop_is_immediate(s2)) { /* dst <- src OP imm (RMW, two-address) */ - dv = irop_get_vreg(d); sv = irop_get_vreg(s1); - } - } - if (dv < 0 || sv < 0 || dv == sv) continue; - if (!tcc_ir_vreg_is_valid(ir, dv) || !tcc_ir_vreg_is_valid(ir, sv)) continue; - /* A volatile VAR must stay memory-resident (every access a real ldr/str); - * coalescing it into a register would elide its mandated loads/stores. */ - { - IRLiveInterval *dvi = tcc_ir_vreg_live_interval(ir, dv); - IRLiveInterval *svi = tcc_ir_vreg_live_interval(ir, sv); - if ((dvi && dvi->is_volatile) || (svi && svi->is_volatile)) - continue; - } - int di = VIDX(dv), si = VIDX(sv); - if (di < 0 || di >= tbl || si < 0 || si >= tbl) continue; - if (iv_of[di] < 0 || iv_of[si] < 0) continue; /* both must have intervals */ - /* Reject unsafe phi-result copies: the dest is defined on multiple - * incoming edges. The dangerous case is when the source temp is itself - * a copy of a VAR that is live-out of the merge block; coalescing the - * phi result with that source (transitively with the VAR) lets a sibling - * phi arm overwrite the still-live VAR (seed 860). Latch-style loop - * phis, where the source is computed in the latch, are unaffected. */ - { - if (def_block_count[di] > 1) { - /* Allow phi-copy coalescing only for loop phis. Conditional-merge - * phis can have a source equivalent to a variable live across the - * merge; coalescing them lets the sibling arm overwrite that variable - * (seed 860). Loop phis are safe because the latch source is computed - * fresh each iteration and dies at the copy. Two placements qualify: - * - the copy sits in the loop header itself (a predecessor is a back - * edge: the copy's block dominates that predecessor), or - * - the copy sits in a latch (a successor is reached by a back edge: - * that successor dominates the copy's block). This is where phi - * resolution places the `IV <- IV+1` copy of a rotated or - * bottom-tested loop, previously rejected here — the single - * largest source of residual `adds rX,rY,#i; mov rY,rX` pairs. */ - int bi = instr_block[i]; - int is_loop_phi = 0; - if (bi >= 0 && bi < nb) { - for (int pi = 0; pi < cfg->blocks[bi].num_preds; pi++) { - int pb = cfg->blocks[bi].preds[pi]; - if (pb >= 0 && pb < nb && tcc_ir_cfg_dominates(cfg, bi, pb)) { - is_loop_phi = 1; - break; - } - } - for (int sx = 0; sx < cfg->blocks[bi].num_succs && !is_loop_phi; sx++) { - int sb = cfg->blocks[bi].succs[sx]; - if (sb >= 0 && sb < nb && sb != bi && tcc_ir_cfg_dominates(cfg, sb, bi)) - is_loop_phi = 1; - } - /* Loop-ENTRY copy (split critical edge into a loop header): the - * copy block has a unique successor that dominates every OTHER - * def block of the dest — i.e. all sibling defs (the latch - * copies) execute strictly after this copy, so they can only - * overwrite a value the copy already delivered. The parallel - * arms of a conditional merge (seed 860) fail this test: a - * sibling arm's def block is never dominated by the merge. */ - if (!is_loop_phi && cfg->blocks[bi].num_succs == 1) { - int sb = cfg->blocks[bi].succs[0]; - if (sb >= 0 && sb < nb && sb != bi) { - int all_dominated = 1; - for (int k = def_head[di]; k >= 0 && all_dominated; k = def_next[k]) { - if (k == i) continue; - int kb = instr_block[k]; - if (kb == bi) continue; /* same-block sibling: executes after us */ - if (kb < 0 || kb >= nb || !tcc_ir_cfg_dominates(cfg, sb, kb)) - all_dominated = 0; - } - if (all_dominated) - is_loop_phi = 1; - } - } - /* Diamond-arm copy whose source is dead at every sibling def of - * the dest: no OTHER def of the dest falls inside the source's - * live interval [start, end), so overwriting the class register - * on a parallel arm can never clobber a value that arm (or any - * path from it) still reads. Interval ranges are back-edge - * extended by ra_build_intervals, so linear containment is a - * conservative test under loops. Catches the SETIF-temp -> - * arm-copy -> phi-dest chain (960402-1: class inherits the r0 - * RETURNVALUE preference) and BOTH arms of a value-select diamond - * (arith-1::sat_add — admitting only one arm is worse than none: - * co_member disables the in-scan hint transfer and the surviving - * real copy blocks post_ra_forward_diamond). The seed-860 hazard - * (source equivalent to a value live ACROSS the merge) fails this - * test at the sibling def; its transitive form (source already - * coalesced with a live-through value) is refused by the - * class-level interference check at union time. */ - if (!is_loop_phi && bi >= 0 && iv_of[si] >= 0) { - uint32_t s_start = intervals[iv_of[si]].start; - uint32_t s_end = intervals[iv_of[si]].end; - int safe = 1; - for (int k = def_head[di]; k >= 0 && safe; k = def_next[k]) { - if (k == i) continue; - if ((uint32_t)k >= s_start && (uint32_t)k < s_end) - safe = 0; - } - if (safe) - is_loop_phi = 1; - } - } - if (!is_loop_phi) { - continue; - } - } - } - ADD_CAND(di); ADD_CAND(si); - if (ne >= ecap) { ecap *= 2; edge_d = tcc_realloc(edge_d, sizeof(int32_t)*ecap); - edge_s = tcc_realloc(edge_s, sizeof(int32_t)*ecap); } - edge_d[ne] = di; edge_s[ne] = si; ne++; - } - - tcc_free(def_block_count); - tcc_free(def_head); - tcc_free(def_next); - - if (ncand < 2 || ne == 0) { - tcc_free(iv_of); tcc_free(instr_block); - tcc_free(cand_id); tcc_free(edge_d); tcc_free(edge_s); - tcc_free(call_param_head); tcc_free(param_sibling); - tcc_ir_cfg_free(cfg); - return; - } - - /* Reverse map: candidate index -> VIDX. */ - int *cand_vidx = tcc_malloc(sizeof(int) * ncand); - for (int i = 0; i < tbl; i++) if (cand_id[i] >= 0) cand_vidx[cand_id[i]] = i; - - /* ---- Stage 3: interference among candidates (per-def live-out). ---- */ - /* Two passes to build CSR adjacency: count degrees, then fill. */ - int *deg = tcc_mallocz(sizeof(int) * ncand); - uint64_t *live = tcc_malloc(sizeof(uint64_t) * nw); - - for (int pass = 0; pass < 2; pass++) { - int *adj_start = NULL, *adj = NULL, total = 0; - if (pass == 1) { - adj_start = tcc_malloc(sizeof(int) * (ncand + 1)); - adj_start[0] = 0; - for (int c = 0; c < ncand; c++) adj_start[c + 1] = adj_start[c] + deg[c]; - total = adj_start[ncand]; - adj = total ? tcc_malloc(sizeof(int) * total) : tcc_malloc(1); - for (int c = 0; c < ncand; c++) deg[c] = adj_start[c]; /* reuse as write cursor */ - } - for (int b = 0; b < nb; b++) { - uint64_t *lo = RaLiveMatrix_row(&liveout, b); - tcc_bitspan_copy(live, lo, nw); - int s = cfg->blocks[b].start_idx, e = cfg->blocks[b].end_idx; - for (int i = e - 1; i >= s && i < n; i--) { - IRQuadCompact *q = &ir->compact_instructions[i]; - int32_t def = -1, hd = 0, uses[4], nu = 0; - ra_co_ops(ir, q, &def, &hd, uses, &nu); - /* Copy source to exclude from interference for this def. ONLY for a - * pure copy `D <- S` (D == S after, so they don't interfere here). For - * an RMW `D <- S OP imm` the result differs from S, so if S is live-out - * it genuinely interferes with D — must NOT be excluded. */ - int32_t copy_src = -1; - if (q->op == TCCIR_OP_ASSIGN) { - IROperand s1 = tcc_ir_op_get_src1(ir, q); - IROperand dd = tcc_ir_op_get_dest(ir, q); - /* Only a pure register copy `D <- S` (no deref) makes D and S equal; - * a deref ASSIGN is a load/store, so its operands genuinely interfere. */ - if (irop_has_vreg(s1) && !irop_is_immediate(s1) && !s1.is_lval && !dd.is_lval) - copy_src = irop_get_vreg(s1); - } - if (hd && tcc_ir_vreg_is_valid(ir, def)) { - int d = VIDX(def); - if (d >= 0 && d < tbl && cand_id[d] >= 0) { - int cd = cand_id[d]; - int csrc = (copy_src >= 0 && tcc_ir_vreg_is_valid(ir, copy_src)) ? VIDX(copy_src) : -1; - tcc_bitspan_for_each_set(live, nw, tbl, vi) { - int c = cand_id[vi]; - if (c < 0) continue; - if (vi == d) continue; - if (csrc >= 0 && vi == csrc) continue; - if (pass == 0) { deg[cd]++; deg[c]++; } - else { adj[deg[cd]++] = c; adj[deg[c]++] = cd; } - } - } - } - /* transition live: kill def, gen uses */ - if (hd && tcc_ir_vreg_is_valid(ir, def)) { - int d = VIDX(def); if (d >= 0 && d < tbl) tcc_bitspan_reset(live, d); - } - for (int k = 0; k < nu; k++) { - if (!tcc_ir_vreg_is_valid(ir, uses[k])) continue; - int u = VIDX(uses[k]); if (u >= 0 && u < tbl) tcc_bitspan_set(live, u); - } - RA_CO_FOR_CALL_PARAM_USES(i, vr, { - int u = VIDX(vr); if (u >= 0 && u < tbl) tcc_bitspan_set(live, u); - }); - } - } - if (pass == 1) { - /* ---- Stage 4: union-find conservative coalescing. ---- */ - int *parent = tcc_malloc(sizeof(int) * ncand); - int *rank = tcc_mallocz(sizeof(int) * ncand); - int *mnext = tcc_malloc(sizeof(int) * ncand); /* member linked list per root */ - int *seen = tcc_mallocz(sizeof(int) * ncand); /* generation-stamped neighbor set */ - int gen = 0; - for (int c = 0; c < ncand; c++) { parent[c] = c; mnext[c] = -1; } - #define UF_FIND(x) ({ int _r = (x); while (parent[_r] != _r) { parent[_r] = parent[parent[_r]]; _r = parent[_r]; } _r; }) - - /* K for the Briggs degree test = number of allocatable int regs. */ - int K = tcc_state->registers_for_allocator; - if (K <= 0 || K > 13) K = 13; - - int merged = 0; - for (int ei = 0; ei < ne; ei++) { - int ca = cand_id[edge_d[ei]], cb = cand_id[edge_s[ei]]; - if (ca < 0 || cb < 0) continue; - int ra = UF_FIND(ca), rb = UF_FIND(cb); - if (ra == rb) continue; - SSAInterval *ia = &intervals[iv_of[cand_vidx[ra]]]; - SSAInterval *ib = &intervals[iv_of[cand_vidx[rb]]]; - /* gate: same-type classes only — INT single-reg with INT, LLONG pair - * with LLONG pair. A mixed edge is a truncating/widening copy (e.g. - * `T9 <- T8` extracting the low half of a 64-bit value): the interval - * model cannot express half-pair aliasing, so those never merge. - * DOUBLE_SOFT pairs were tried and measured NET WORSE (+220 corpus, - * cdivchk* +11): merged classes extend across __aeabi_d* calls and - * defeat the in-scan return-pair r0:r1 preference (the pr58574 - * lever), which handles soft-float chains better than a class can. - * Complex types stay out too (rare; pair-sym marshaling subtleties). - * Not precolored/addrtaken/param/spill-fixed. */ - if (ia->reg_type != ib->reg_type) continue; - if (ia->reg_type != LS_REG_TYPE_INT && ia->reg_type != LS_REG_TYPE_LLONG) continue; - if (ia->r1 >= 0 || ib->r1 >= 0) continue; - if (ia->addrtaken || ib->addrtaken || ia->precolored >= 0 || ib->precolored >= 0) continue; - if (ia->is_param || ib->is_param) continue; - /* non-interference: no member of class ra interferes with class rb */ - int interferes = 0; - for (int m = ra; m >= 0 && !interferes; m = mnext[m]) { - for (int a = adj_start[m]; a < adj_start[m + 1]; a++) { - if (UF_FIND(adj[a]) == rb) { interferes = 1; break; } - } - } - if (interferes) continue; - /* Conservative pressure test: reject if the merged class would have >= K - * DISTINCT interference-neighbor classes — such a node may be impossible - * to color, so coalescing it risks forcing a spill (a size regression). - * Counting all distinct neighbor classes (not just high-degree ones) is a - * safe over-approximation of Briggs; the IV web has few neighbors so it - * still coalesces, while high-pressure webs are correctly left alone. */ - { - /* Neighbor classes are weighted by register footprint (INT = 1, - * LLONG pair = 2), and a pair-class merge needs a free adjacent - * pair, so its headroom threshold drops by one. DOUBLE_SOFT - * neighbors deliberately stay weight 1: weighting them 2 measured - * +264 corpus — soft-double values rarely stay live across the - * merged range (they cycle through __aeabi_d* r0:r1 returns), so - * counting them as resident pairs over-blocks INT/LLONG merges. */ - int limit = (ia->reg_type == LS_REG_TYPE_LLONG) ? K - 1 : K; - int over = 0, distinct = 0; - gen++; - for (int side = 0; side < 2 && !over; side++) { - int r = side ? rb : ra; - for (int m = r; m >= 0 && !over; m = mnext[m]) { - for (int a = adj_start[m]; a < adj_start[m + 1]; a++) { - int nr = UF_FIND(adj[a]); - if (nr == ra || nr == rb) continue; - if (seen[nr] != gen) { - seen[nr] = gen; - SSAInterval *niv = &intervals[iv_of[cand_vidx[nr]]]; - distinct += (niv->reg_type == LS_REG_TYPE_LLONG) ? 2 : 1; - if (distinct >= limit) { over = 1; break; } - } - } - } - } - if (over) continue; - } - /* union (rank) + splice member lists */ - if (rank[ra] < rank[rb]) { int t = ra; ra = rb; rb = t; } - parent[rb] = ra; - if (rank[ra] == rank[rb]) rank[ra]++; - int tail = ra; while (mnext[tail] >= 0) tail = mnext[tail]; - mnext[tail] = rb; - merged++; - } - - /* ---- Stage 5: apply via coalesce_to interval-merge. ---- */ - if (level >= 2 && merged > 0) { - /* For each root with >1 member, choose rep = earliest-start interval, - * extend its range to the union, and flag the rest. */ - for (int c = 0; c < ncand; c++) { - if (UF_FIND(c) != c) continue; /* roots only */ - if (mnext[c] < 0) continue; /* singleton */ - /* gather members, pick rep */ - int rep_iv = -1; uint32_t lo_s = 0xffffffffu, hi_e = 0; - int xcall = 0; uint32_t uc = 0, nuc = 0, sum_len = 0; - int pref = -1; uint32_t pref_end = 0; - for (int m = c; m >= 0; m = mnext[m]) { - int ivi = iv_of[cand_vidx[m]]; - SSAInterval *iv = &intervals[ivi]; - if (iv->start < lo_s) lo_s = iv->start; - if (iv->end > hi_e) hi_e = iv->end; - xcall |= iv->crosses_call; - uc += iv->use_count; - nuc += iv->narrow_uses; - sum_len += iv->end - iv->start + 1; - /* Keep the LATEST-ending member's soft register preference: the - * r0-for-RETURNVALUE hint matters at the class's end boundary, - * and the member feeding the return is the one that ends last. - * Without this the merged class loses the hint (co_member skips - * the in-scan hint transfer) and both diamond arms compute into - * a scratch reg plus a final `mov r0, rX` (960402-1 +2). */ - if (iv->pref_reg >= 0 && (pref < 0 || iv->end >= pref_end)) { - pref = iv->pref_reg; pref_end = iv->end; - } - if (rep_iv < 0 || iv->start < intervals[rep_iv].start) rep_iv = ivi; - } - if (rep_iv < 0) continue; - /* Density gate: the rep covers the CONTIGUOUS union [lo,hi], occupying - * the register across any gaps between members. If the union is much - * larger than the members' combined live length, those gaps are holes - * the merge fills — raising register pressure and risking spills (a - * size regression). Members of one value have OVERLAPPING (false) - * ranges, so sum_len >= union_len for the cases worth merging; reject - * when the union exceeds the members' total (a real hole). */ - if ((hi_e - lo_s + 1) > sum_len) - continue; - intervals[rep_iv].start = lo_s; - intervals[rep_iv].end = hi_e; - intervals[rep_iv].crosses_call = xcall ? 1 : 0; - intervals[rep_iv].use_count = (uc > 65535) ? 65535 : (uint16_t)uc; - intervals[rep_iv].narrow_uses = (nuc > 65535) ? 65535 : (uint16_t)nuc; - if (pref >= 0 && intervals[rep_iv].pref_reg < 0) - intervals[rep_iv].pref_reg = (int8_t)pref; - /* Store the representative's VREG (stable across the scan's qsort), - * not its array index. */ - int32_t rep_vreg = intervals[rep_iv].vreg; - intervals[rep_iv].co_member = 1; - for (int m = c; m >= 0; m = mnext[m]) { - int ivi = iv_of[cand_vidx[m]]; - intervals[ivi].co_member = 1; - if (ivi != rep_iv) intervals[ivi].coalesce_to = rep_vreg; - } - RA_DBG("coalesce: root class -> rep T%d [%u,%u] xcall=%d", - TCCIR_DECODE_VREG_POSITION(intervals[rep_iv].vreg), lo_s, hi_e, xcall); - } - } - RA_DBG("coalesce: %d candidates, %d edges, %d unions (level=%d)", ncand, ne, merged, level); - - tcc_free(parent); tcc_free(rank); tcc_free(mnext); tcc_free(seen); - #undef UF_FIND - } - if (pass == 1) { tcc_free(adj_start); tcc_free(adj); } - } - - #undef ADD_CAND - #undef VIDX - tcc_free(deg); tcc_free(live); - tcc_free(iv_of); - tcc_free(instr_block); - tcc_free(cand_id); tcc_free(cand_vidx); tcc_free(edge_d); tcc_free(edge_s); - tcc_free(call_param_head); tcc_free(param_sibling); - #undef RA_CO_FOR_CALL_PARAM_USES - tcc_ir_cfg_free(cfg); -} - -/* ============================================================================ - * Entry Point - * ============================================================================ */ - -/* Promote multiply-block-defined TEMPs to fresh VARs so SSA construction places - * phis for them. The frontend emits a single TEMP written on BOTH arms of a - * branch-lowered ternary (`cond ? a : b` where an arm has a side effect / call, - * so it cannot lower to SELECT) — e.g. `T323 <- a` in one block and `T323 <- b` - * in another, then a merge-block use. That violates the SSA-by-construction - * assumption the renamer makes for TEMPs (it renames only VARs and leaves such a - * TEMP untouched), so the merge use resolves to ONE arm's definition - * unconditionally — random-C O1/O2 wrong-code, seeds 100/118 (the value reached a - * later inlined-csmix use as the else-arm value regardless of the condition). - * Converting the TEMP to a VAR routes it through the normal var→SSA promotion, - * which inserts the phi. VAR and TEMP operands share the IROP_TAG_VREG encoding - * and differ only in the type bits, so irop_set_vreg suffices; tcc_ir_vreg_alloc_var - * grows the live-interval array. Only fires for the rare multi-block-def TEMP. */ -/* The def-side operand filter shared by the passes below: returns the TEMP - * index this instruction defines, or -1 when it defines no plain TEMP. STORE - * dests are addresses (a use), FUNCPARAM* dests are not TEMP defs, and an - * is_lval dest is a deref store target rather than a plain TEMP def. */ -static int ra_def_temp_of(TCCIRState *ir, IRQuadCompact *q, int ntmp) -{ - if (q->op == TCCIR_OP_NOP || !irop_config[q->op].has_dest) - return -1; - if (q->op == TCCIR_OP_STORE || q->op == TCCIR_OP_STORE_INDEXED || - q->op == TCCIR_OP_STORE_POSTINC || q->op == TCCIR_OP_FUNCPARAMVAL || - q->op == TCCIR_OP_FUNCPARAMVOID) - return -1; - IROperand d = tcc_ir_op_get_dest(ir, q); - int32_t vr = irop_get_vreg(d); - if (vr < 0 || d.is_lval || TCCIR_DECODE_VREG_TYPE(vr) != TCCIR_VREG_TYPE_TEMP) - return -1; - int t = TCCIR_DECODE_VREG_POSITION(vr); - return (t >= 0 && t < ntmp) ? t : -1; -} - -/* Membership set over (TEMP, block) pairs, used below to answer "does block b - * define TEMP t?" without a per-TEMP rescan. Open addressing, power-of-two - * capacity, linear probe; slot 0 means empty, so keys are stored biased by one. - * The caller sizes the table at >= 2x the number of insertions, which bounds - * the load factor at 0.5 and guarantees the probe terminates. */ -static inline uint64_t ra_blockset_mix(uint64_t k) -{ - k *= 0x9E3779B97F4A7C15ull; - return k ^ (k >> 29); -} - -static void ra_blockset_add(uint64_t *set, int cap, uint64_t key) -{ - const uint64_t k = key + 1; - int i = (int)(ra_blockset_mix(k) & (uint64_t)(cap - 1)); - while (set[i] && set[i] != k) - i = (i + 1) & (cap - 1); - set[i] = k; -} - -static int ra_blockset_has(const uint64_t *set, int cap, uint64_t key) -{ - const uint64_t k = key + 1; - int i = (int)(ra_blockset_mix(k) & (uint64_t)(cap - 1)); - while (set[i]) { - if (set[i] == k) - return 1; - i = (i + 1) & (cap - 1); - } - return 0; -} - -static void ra_promote_multidef_temps_to_vars(TCCIRState *ir, IRCFG *cfg) -{ - int n = ir->next_instruction_index; - int ntmp = ir->next_temporary_variable; - if (n <= 0 || ntmp <= 0 || !cfg || cfg->num_blocks <= 1) - return; - - /* Skip functions that take label addresses (GCC labels-as-values, `&&label`): - * their exact machine-code layout is observable at runtime via the label-offset - * map, so the phi-resolution copies this promotion introduces would shift those - * offsets (96_nodata_wanted measures code size with `&&label` arithmetic). - * Such functions also have inlining disabled (tccgen gates auto-inline on - * !func_has_label_addr), so they never hit the inlined-ternary miscompile this - * promotion fixes — skipping them is free of correctness cost. */ - if (ir->func_has_label_addr) - return; - - /* Only run when SSA construction will actually proceed and rename the new VARs - * back into SSA temps. SSA construction BAILS on un-enumerable control flow - * (IJUMP / computed goto, SETJMP); if we promoted there, the converted VARs - * would be left as unpromoted stack slots and change codegen for the worse - * (96_nodata_wanted's `&&label` arithmetic). Mirror ssa_has_unsupported_ops. */ - for (int i = 0; i < n; i++) { - TccIrOp op = ir->compact_instructions[i].op; - if (op == TCCIR_OP_IJUMP || op == TCCIR_OP_SETJMP || op == TCCIR_OP_NL_SETJMP) - return; - } - - /* def_block[t] = the block of t's first def, or -2 = multi-block, -1 = none. */ - int *def_block = tcc_malloc(sizeof(int) * ntmp); - for (int t = 0; t < ntmp; t++) def_block[t] = -1; - - for (int i = 0; i < n; i++) { - int t = ra_def_temp_of(ir, &ir->compact_instructions[i], ntmp); - if (t < 0) continue; - int blk = cfg->instr_to_block[i]; - if (def_block[t] == -1) def_block[t] = blk; - else if (def_block[t] != blk) def_block[t] = -2; /* multi-block */ - } - - /* A multi-block-defined TEMP only needs a phi (and only then is its renaming - * actually wrong) when it has a USE in a block that does not itself define it — - * a value flowing across a merge. A TEMP whose uses are all in its own - * def-blocks reaches each use from the local def and is already correct; - * promoting it would insert needless phi-copies and grow code (96_nodata_wanted - * measures code size via `&&label` arithmetic and is sensitive to this). For - * each multi-block TEMP, mark its def-blocks and require a use elsewhere. */ - int32_t *temp_to_var = tcc_malloc(sizeof(int32_t) * ntmp); - for (int t = 0; t < ntmp; t++) temp_to_var[t] = -1; - uint8_t *needs_phi = tcc_mallocz(ntmp); - { - /* Rescanning every instruction per multi-block TEMP -- once to collect its - * def-blocks, once more to look for a use outside them -- is - * O(multi-block TEMPs x instructions), with an operand decode at every step - * and an isdef[] clear per candidate on top. strlen-5 spent 24.7% of its - * whole compile here. Two passes over the instructions answer the same - * question: the first records every (TEMP, block) pair that is a def, the - * second asks that set about every use. */ - const uint64_t nblk = (uint64_t)cfg->num_blocks; - /* Nothing multi-block means nothing to decide -- and no table to pay for. - * Size it from the number of defs that actually go in, not from n: most - * functions here have a handful, and a table sized for every instruction - * cost more to allocate and zero than it saved (20040709-2). */ - int any_multi = 0; - for (int t = 0; t < ntmp && !any_multi; t++) - any_multi = (def_block[t] == -2); - if (!any_multi) - goto promote_decided; /* nothing to decide: don't walk the instructions */ - - int ndefs = 0; - for (int i = 0; i < n; i++) { - int t = ra_def_temp_of(ir, &ir->compact_instructions[i], ntmp); - if (t >= 0 && def_block[t] == -2) ndefs++; - } - if (!ndefs) - goto promote_decided; - - int dcap = 16; - while (dcap < ndefs * 2) dcap <<= 1; - uint64_t *defset = tcc_mallocz(sizeof(uint64_t) * dcap); - - for (int i = 0; i < n; i++) { - int t = ra_def_temp_of(ir, &ir->compact_instructions[i], ntmp); - if (t < 0 || def_block[t] != -2) continue; - ra_blockset_add(defset, dcap, (uint64_t)t * nblk + (uint64_t)cfg->instr_to_block[i]); - } - - /* a use in a non-def block => needs a phi */ - for (int i = 0; i < n; i++) { - IRQuadCompact *q = &ir->compact_instructions[i]; - if (q->op == TCCIR_OP_NOP) continue; - int32_t uses[5]; int nu = 0; - if (irop_config[q->op].has_src1) uses[nu++] = irop_get_vreg(tcc_ir_op_get_src1(ir, q)); - if (irop_config[q->op].has_src2) uses[nu++] = irop_get_vreg(tcc_ir_op_get_src2(ir, q)); - if (q->op == TCCIR_OP_MLA) uses[nu++] = irop_get_vreg(tcc_ir_op_get_accum(ir, q)); - if (q->op == TCCIR_OP_STORE || q->op == TCCIR_OP_STORE_INDEXED || q->op == TCCIR_OP_STORE_POSTINC) - uses[nu++] = irop_get_vreg(tcc_ir_op_get_dest(ir, q)); - if (!nu) continue; - const uint64_t blk = (uint64_t)cfg->instr_to_block[i]; - for (int u = 0; u < nu; u++) { - if (uses[u] < 0 || TCCIR_DECODE_VREG_TYPE(uses[u]) != TCCIR_VREG_TYPE_TEMP) continue; - int t = TCCIR_DECODE_VREG_POSITION(uses[u]); - if (t < 0 || t >= ntmp || def_block[t] != -2 || needs_phi[t]) continue; - if (!ra_blockset_has(defset, dcap, (uint64_t)t * nblk + blk)) - needs_phi[t] = 1; - } - } - tcc_free(defset); - promote_decided:; - } - int any = 0; - for (int t = 0; t < ntmp; t++) { - if (needs_phi[t]) { temp_to_var[t] = tcc_ir_vreg_alloc_var(ir); any = 1; } - } - tcc_free(needs_phi); - if (!any) { tcc_free(def_block); tcc_free(temp_to_var); return; } - - /* Rewrite every operand referencing a promoted TEMP to its VAR (type bits only; - * is_local/is_lval/tag are preserved). */ - #define REMAP(getter, setter) \ - do { \ - IROperand o = getter(ir, q); \ - int32_t ovr = irop_get_vreg(o); \ - if (ovr >= 0 && TCCIR_DECODE_VREG_TYPE(ovr) == TCCIR_VREG_TYPE_TEMP) { \ - int op_t = TCCIR_DECODE_VREG_POSITION(ovr); \ - if (op_t >= 0 && op_t < ntmp && temp_to_var[op_t] >= 0) { \ - irop_set_vreg(&o, temp_to_var[op_t]); \ - setter(ir, q, o); \ - } \ - } \ - } while (0) - - for (int i = 0; i < n; i++) { - IRQuadCompact *q = &ir->compact_instructions[i]; - if (q->op == TCCIR_OP_NOP) continue; - if (irop_config[q->op].has_dest) REMAP(tcc_ir_op_get_dest, tcc_ir_op_set_dest); - if (irop_config[q->op].has_src1) REMAP(tcc_ir_op_get_src1, tcc_ir_op_set_src1); - if (irop_config[q->op].has_src2) REMAP(tcc_ir_op_get_src2, tcc_ir_op_set_src2); - if (q->op == TCCIR_OP_MLA) REMAP(tcc_ir_op_get_accum, tcc_ir_op_set_accum); - } - #undef REMAP - - tcc_free(def_block); - tcc_free(temp_to_var); -} - -/* Mark single-def TEMP intervals whose value is a plain int32 constant as - * rematerializable: when such a value gets spilled, machine_op recomputes it - * with `mov reg, #imm` at each use instead of a stack reload — cheaper (no - * memory) and it removes a spill load per use, directly shrinking the - * spill-reload traffic that widens tcc's load count vs gcc. Runs on the final - * post-allocation IR so the recorded immediate matches what codegen sees. */ -static void ra_mark_rematerializable(TCCIRState *ir) -{ - int nt = ir->temporary_variables_live_intervals_size; - if (nt <= 0) - return; - for (int t = 0; t < nt; t++) - ir->temporary_variables_live_intervals[t].remat_kind = 0; - - int n = ir->next_instruction_index; - int *def_count = tcc_mallocz(nt * sizeof(int)); - int *def_idx = tcc_malloc(nt * sizeof(int)); - for (int t = 0; t < nt; t++) - def_idx[t] = -1; - - for (int i = 0; i < n; i++) { - IRQuadCompact *q = &ir->compact_instructions[i]; - if (q->op == TCCIR_OP_NOP || !irop_config[q->op].has_dest) - continue; - int32_t vr = irop_get_vreg(tcc_ir_op_get_dest(ir, q)); - if (vr < 0 || TCCIR_DECODE_VREG_TYPE(vr) != TCCIR_VREG_TYPE_TEMP) - continue; - int p = TCCIR_DECODE_VREG_POSITION(vr); - if (p >= nt) - continue; - def_count[p]++; - def_idx[p] = i; - } - - /* Slot-reading use: an is_lval deref of the value (machine_op reloads it — see - * the use_llocal path); such an appearance needs the spill store to stay. - * Includes the DEST (a deref store `*T = x` reads T as the address). */ - uint8_t *needs_slot = tcc_mallocz(nt); - for (int i = 0; i < n; i++) { - IRQuadCompact *q = &ir->compact_instructions[i]; - if (q->op == TCCIR_OP_NOP) - continue; - for (int slot = 0; slot < 4; slot++) { - IROperand s; - if (slot == 0) { if (!irop_config[q->op].has_dest) continue; s = tcc_ir_op_get_dest(ir, q); } - else if (slot == 1) { if (!irop_config[q->op].has_src1) continue; s = tcc_ir_op_get_src1(ir, q); } - else if (slot == 2) { if (!irop_config[q->op].has_src2) continue; s = tcc_ir_op_get_src2(ir, q); } - else { if (q->op != TCCIR_OP_MLA) continue; s = tcc_ir_op_get_accum(ir, q); } - if (!(s.is_lval && !s.is_local && !s.is_llocal)) - continue; - int32_t vr = irop_get_vreg(s); - if (vr < 0 || TCCIR_DECODE_VREG_TYPE(vr) != TCCIR_VREG_TYPE_TEMP) - continue; - int p = TCCIR_DECODE_VREG_POSITION(vr); - if (p < nt) - needs_slot[p] = 1; - } - } - - for (int t = 0; t < nt; t++) { - if (def_count[t] != 1) - continue; - IRQuadCompact *dq = &ir->compact_instructions[def_idx[t]]; - if (dq->op != TCCIR_OP_ASSIGN) - continue; - IROperand dd = tcc_ir_op_get_dest(ir, dq); - if (dd.is_lval || irop_get_btype(dd) != IROP_BTYPE_INT32) - continue; - IROperand s = tcc_ir_op_get_src1(ir, dq); - if (s.tag != IROP_TAG_IMM32 || s.is_lval || s.is_sym || s.is_local || s.is_llocal) - continue; - IRLiveInterval *iv = &ir->temporary_variables_live_intervals[t]; - iv->remat_kind = 1; - iv->remat_imm = irop_get_imm32(s); - - /* Do NOT drop the def: rematerialization only happens for the one - * machine_op operand shape (spilled + need_lval value read, see - * machine_op.c) — a PARAM src or a plain binop src2 of the spilled temp - * takes the slot/frame path instead and would read garbage once the - * def's mov+store is gone (fuzz seed longlong:6393; the LLONG - * pair classes raised pressure enough to spill the const). The - * redundant mov+store for a remat'd spilled constant is noise compared - * to wrong code. */ - (void)needs_slot; - } - - tcc_free(needs_slot); - tcc_free(def_count); - tcc_free(def_idx); -} - -/* Gate + time + dump one flat-region pass, the shape every entry in the - * pre-CFG cluster below shares. `changed` receives the pass's return value so - * the timing dump can report productivity (Phase 0.4 of - * docs/plans/opt_pass_dedup_and_perf.md). */ -#define RA_FLAT_PASS(changed, name, cond, call) \ - do \ - { \ - (changed) = 0; \ - if (tcc_state && (cond) && !tcc_ir_opt_pass_disabled(name)) \ - TCC_PASS_TIMED(changed, name, (call)); \ - tcc_ir_dump_after_pass(ir, name); \ - } while (0) - -/* ssa:cfg_cleanup body — see the call site for why it runs where it does. */ -static int ra_cfg_cleanup(TCCIRState *ir) -{ - const IRPassGroup *groups; - int group_count; - int total = 0; - tcc_ir_opt_get_pipeline(IR_OPT_LEVEL_2, &groups, &group_count); - const IRPassGroup *cleanup_group = &groups[group_count - 1]; - for (int outer = 0; outer < 3; outer++) { - int ch = 0; - for (int i = 0; i < 8; i++) { - int c = tcc_ir_opt_jump_threading(ir); - /* Inside the cascade, not beside it: retargeting the constant arm's edge - * turns the true arm's trailing JUMP into a fall-through and orphans the - * merge label, and only eliminate_fallthrough + dce below can collect - * that -- which is what exposes `T <-- (cond)` to setif fusion. */ - c += tcc_ir_opt_bool_diamond_branch(ir); - c += tcc_ir_opt_eliminate_fallthrough(ir); - c += tcc_ir_opt_jumpif_invert(ir, 0); - if (c) - tcc_ir_opt_compact_nops(ir); - if (tcc_state->opt_dce) { - c += tcc_ir_opt_orphan_cmp_elim(ir); - c += tcc_ir_opt_dce(ir); - } - ch += c; - if (!c) - break; - } - if (outer == 0 && !ch) - break; - IROptCtx cl_ctx; - tcc_ir_opt_ctx_init(&cl_ctx, ir); - if (tcc_state->opt_dead_store) - ch += tcc_ir_opt_gens_call_result_ex(&cl_ctx); - ch += tcc_ir_opt_run_group(&cl_ctx, cleanup_group); - tcc_ir_opt_ctx_free(&cl_ctx); - total += ch; - if (!ch) - break; - } - return total; -} - -void tcc_ir_ssa_regalloc(TCCIRState *ir, const RegAllocTarget *target, int spill_base) -{ - if (!ir || !target) return; - int ra_ch; - RAPhiStats phi_stats = {0}; - dbg_scan_overlap(ir, "ssa_regalloc_entry"); - - /* ssa:mem_init — frontend memory-init lowering (memset(0)+const stores → - * rodata BLOCK_COPY; small stack/global zero-memset → direct STORE #0). - * Runs first on raw flat IR, before cfg_cleanup and the loop transforms - * below, matching its former position at the head of the tccgen.c pipeline. - * Ungated by -O (the lowering also fires at -O0); knob - * TCC_DISABLE_PASS=ssa:mem_init. See docs/plan_legacy_flat_ir_ssa_retire.md. */ - RA_FLAT_PASS(ra_ch, "ssa:mem_init", 1, ssa_opt_mem_init(ir)); - - /* ssa:cfg_cleanup — GCC-style cleanup_cfg at SSA-pipeline entry: thread jump - * chains / drop fall-through jumps / NOP orphan flag-setters left by the flat - * pipeline, so the flat-region loop transforms below (reroll/licm/loop_rotate) - * see normalized control flow instead of trampoline chains they would rotate - * around and duplicate tests for. When the cascade fired, re-run call-result - * demotion + the flat late_cleanup group (self-gated passes): collapsed - * diamonds expose dead pure calls / VLA allocs / stores those passes could - * not see on their earlier run. */ - RA_FLAT_PASS(ra_ch, "ssa:cfg_cleanup", - tcc_state->optimize >= 1 && tcc_state->opt_jump_threading, - ra_cfg_cleanup(ir)); - - /* ssa:struct_copy_roundtrip — drop the memmove(B,A);memmove(A,B) pair left - * by an inlined identity `y = retme(y)` helper. The matcher needs the two - * copies in one straight line; the inline expansion leaves a fall-through - * JMP + target marker between them that only cfg_cleanup removes, so the - * tccgen-time call misses the pattern (20040709-2 fn1* family). */ - RA_FLAT_PASS(ra_ch, "ssa:struct_copy_roundtrip", tcc_state->opt_redundant_store, - tcc_ir_opt_struct_copy_roundtrip_elim(ir)); - - /* ssa:or_bool_diamond — fold `acc |= (cond ? 1 : 0)` stack-slot - * materialization into per-arm ORs. Needs the STORE-slot/OR adjacency that - * cfg_cleanup's eliminate_fallthrough just created. Gate matches the legacy - * tccgen.c call (opt_const_prop); knob TCC_DISABLE_PASS=ssa:or_bool_diamond. */ - RA_FLAT_PASS(ra_ch, "ssa:or_bool_diamond", tcc_state->opt_const_prop, - ssa_opt_or_bool_diamond(ir)); - - /* ssa:stack_addr_simplify — deref-of-known-stack-addr → direct StackLoc; legacy gate; knob TCC_DISABLE_PASS=ssa:stack_addr_simplify */ - RA_FLAT_PASS(ra_ch, "ssa:stack_addr_simplify", tcc_state->opt_const_prop, - tcc_ir_opt_stack_addr_simplify(ir)); - - /* ssa:reroll — re-roll runs of identical macro-unrolled blocks into a counted - * loop. Runs first in the flat region (preserving reroll's legacy "earliest - * loop transform" order) but now post-propagation: foldable runs are already - * collapsed by downstream const-prop, so only non-foldable repetition survives - * to re-roll, which fixes the legacy pre-propagation counterproductivity. - * Gated opt_reroll (-O2); knob TCC_DISABLE_PASS=ssa:reroll. - * See docs/plan_legacy_loop_reroll_ssa.md. */ - /* compact the (N-1)*P NOPs a successful re-roll leaves so CFG/SSA below don't iterate them */ - RA_FLAT_PASS(ra_ch, "ssa:reroll", tcc_state->opt_reroll, - ssa_opt_reroll(ir) ? (tcc_ir_opt_compact_nops(ir), 1) : 0); - - /* ssa:licm — hoist loop invariants (arithmetic + pure/const calls) to the - * preheader before the other loop transforms and ssa:iv_strength_reduction, - * preserving the legacy "LICM before IV-SR" order now that both left tccgen. - * Reuses the proven licm.c engine; gated opt_licm (-O2); knob - * TCC_DISABLE_PASS=ssa:licm. */ - RA_FLAT_PASS(ra_ch, "ssa:licm", tcc_state->opt_licm, ssa_opt_licm(ir)); - - /* Loop rotation (ssa:loop_rotate): convert safe top-tested loops to - * bottom-tested on flat IR, before the CFG/SSA below are built, so the - * downstream SSA passes and regalloc see the rotated shape. CFG/dominator - * based natural-loop detection driving the proven flat-IR rewrite; gated to - * -O1+ (matching the legacy pass and the SSA opt tier) and disableable via - * TCC_DISABLE_PASS=ssa:loop_rotate. */ - RA_FLAT_PASS(ra_ch, "ssa:loop_rotate", tcc_state->optimize >= 1, - ssa_opt_loop_rotate(ir)); - - /* ssa:licm ran BEFORE rotation and so saw the un-rotated header/latch/body - * shape, whose preheader is not the header's immediate predecessor — the - * invariant-global-load hoist's insertion safety demands exactly that, so it - * declined every ordinary `for` loop. Rotation provides the shape; give the - * hoist a second chance on it. */ - RA_FLAT_PASS(ra_ch, "ssa:licm_global_load", tcc_state->opt_licm, - ssa_opt_licm_global_load(ir)); - - /* Zero-trip guard elimination (ssa:loop_guard_elim): drop the pre-loop - * `CMP iv,#lim / JUMPIF` that rotation leaves in front of a bottom-tested - * loop when the IV's entry value is a constant carried from the exit of a - * preceding counted loop over the same variable. Runs immediately after - * ssa:loop_rotate — rotation is the shape provider, and everything below - * must already see the single-exit form. Const prop covers the literal-init - * case on its own; only the carried case needs this pass (SSA sees a phi at - * the header and cannot conclude i == A). Gated -O1+ to match - * ssa:loop_rotate; disableable via TCC_DISABLE_PASS=ssa:loop_guard_elim, - * which also makes rotation stop admitting the shapes that depend on it. */ - RA_FLAT_PASS(ra_ch, "ssa:loop_guard_elim", tcc_state->optimize >= 1, - tcc_ir_opt_loop_guard_elim(ir)); - - /* First-iteration-exit peeling (ssa:first_iter_exit): eliminate top-tested - * loops whose header exit test is provably true on first entry, on flat IR - * after rotation (rotation declines these shapes; a rotated loop's guard is - * outside the loop so this pass declines rotated shapes — no overlap). - * Gate matches the legacy tccgen.c pass exactly (-O1+ with const-prop, so - * -fno-const-prop keeps its meaning for bisection); disableable via - * TCC_DISABLE_PASS=ssa:first_iter_exit. */ - RA_FLAT_PASS(ra_ch, "ssa:first_iter_exit", - tcc_state->optimize >= 1 && tcc_state->opt_const_prop, - ssa_opt_first_iter_exit(ir)); - - /* Pointer-IV exit-value substitution (ssa:ptr_iv_exit_subst): rewrite - * post-loop pointer-IV reads to the closed-form exit address and fold the - * consuming `p != &a[N]` compares (pass-owned; nothing downstream folds - * them). Gate matches the legacy tccgen.c pass (-O1+ with const-prop). */ - RA_FLAT_PASS(ra_ch, "ssa:ptr_iv_exit_subst", - tcc_state->optimize >= 1 && tcc_state->opt_const_prop, - ssa_opt_ptr_iv_exit_subst(ir)); - - /* Loop constant simulation (ssa:loop_const_sim): collapse register-only - * bounded-trip loops to residual final values on flat IR. Gate matches the - * legacy tccgen.c Phase 4e pass exactly (opt_loop_unroll, -O2 default; - * -floop-unroll reaches it at lower levels; -fno-loop-unroll disables both - * it and the unroller, keeping the shared bisection knob). Disableable via - * TCC_DISABLE_PASS=ssa:loop_const_sim. */ - RA_FLAT_PASS(ra_ch, "ssa:loop_const_sim", tcc_state->opt_loop_unroll, - ssa_opt_loop_const_sim(ir)); - - /* Loop unrolling / constant-trip elimination (ssa:loop_unroll): fully unroll - * or close-form-eliminate small constant/symbolic-trip register-only loops on - * flat IR after const_sim has starved it of its own candidates. Same gate as - * the legacy tccgen.c Phase 5a pass (opt_loop_unroll); the SSA/regalloc - * pipeline folds the residual arithmetic (no post-unroll cascade replicated). - * Disableable via TCC_DISABLE_PASS=ssa:loop_unroll. */ - RA_FLAT_PASS(ra_ch, "ssa:loop_unroll", tcc_state->opt_loop_unroll, - ssa_opt_loop_unroll(ir)); - - /* Induction-variable strength reduction (ssa:iv_strength_reduction): transform - * array-indexing recurrences base + i*stride into a maintained stride pointer - * (enabling post-increment addressing) and optionally eliminate the counter IV - * against a hoisted end pointer, on flat IR after rotation/const_sim/unroll - * have normalized and starved the loops. Runs before ssa:decrement_to_zero, - * preserving the legacy "decrement_to_zero after IV-SR" order. Gate matches - * the legacy tccgen.c Phase 6 pass (opt_iv_strength_red, -O1+); disableable via - * TCC_DISABLE_PASS=ssa:iv_strength_reduction. */ - RA_FLAT_PASS(ra_ch, "ssa:iv_strength_reduction", tcc_state->opt_iv_strength_red, - ssa_opt_iv_strength_reduction(ir)); - - /* Decrement-to-zero (ssa:decrement_to_zero): rewrite count-up pure-counter - * loops that ssa:loop_rotate turned bottom-tested (and const_sim/unroll left - * as side-effecting survivors) into count-down-to-zero so the backend fuses - * the latch SUB+CMP#0 into a flag-setting SUBS. Flat IR before the CFG build - * below, so the NOPed guard's control-flow change is reflected downstream. - * Gated -O1+ (matches ssa:loop_rotate, the shape provider); disableable via - * TCC_DISABLE_PASS=ssa:decrement_to_zero. */ - RA_FLAT_PASS(ra_ch, "ssa:decrement_to_zero", tcc_state->optimize >= 1, - ssa_opt_decrement_to_zero(ir)); - - /* Switch-value IPCP: fold calls to single-arg pure dispatchers whose arg is - * constant into ASSIGN #const, replaying any captured global stores at the - * call site. Runs as the last flat transform, immediately before the CFG - * build, so the replayed stores are only ever seen by the SSA pipeline - * (which handles them correctly) and not by the legacy cfg_cleanup DSE, - * which drops a store still read by a following CMP. (const_call_replace is - * store-free and runs early in gen_function so its constant cascades.) */ - if (tcc_state && tcc_state->opt_ipc) { - int ipc_ch = 0; - if (!tcc_ir_opt_pass_disabled("ssa:switch_call_replace")) - ipc_ch += tcc_ir_opt_switch_call_replace(ir); - /* Re-run the flat propagation group so the freshly folded ASSIGN #const - * cascades through the caller. switch_call_replace runs here (not early - * with const_call_replace) because it replays captured global stores that - * must bypass the legacy cfg_cleanup DSE; this re-run recovers the caller - * cascade it would otherwise lose. Gated on an actual rewrite so non-IPCP - * functions pay nothing. */ - if (ipc_ch && tcc_state->optimize >= 1) { - const IRPassGroup *groups; - int group_count; - tcc_ir_opt_get_pipeline(IR_OPT_LEVEL_2, &groups, &group_count); - IROptCtx ipc_ctx; - tcc_ir_opt_ctx_init(&ipc_ctx, ir); - tcc_ir_opt_run_group(&ipc_ctx, &groups[0]); - tcc_ir_opt_ctx_free(&ipc_ctx); - } - tcc_ir_dump_after_pass(ir, "ssa:switch_call_replace"); - } - - /* Sweep NOPs (and their stale jump-target marks) left by the flat pipeline - * and the flat transforms above before building the CFG: a NOP that was a - * jump target introduces a spurious block boundary that blocks var->temp - * promotion in ssa_rename (cmp_offset_common_base). The flat groups only - * compact when a pass reported changes, so a clean run can still arrive - * here with NOPs in the stream. */ - tcc_ir_opt_compact_nops(ir); - - /* Build CFG + dominators */ - TCCPassTimer ra2_pt; - tcc_pass_timing_begin(&ra2_pt, "ra2:cfg_ssa"); - IRCFG *cfg = tcc_ir_cfg_build(ir); - if (!cfg) { - /* Fallback: no CFG means trivial function, use old allocator path */ - tcc_pass_timing_end(&ra2_pt, -1); - return; - } - tcc_ir_cfg_compute_dominators(cfg); - tcc_ir_cfg_compute_dom_frontiers(cfg); - - ra_promote_multidef_temps_to_vars(ir, cfg); - tcc_ir_dump_after_pass(ir, "ssa_promote"); - - /* Construct SSA. - * - * NOTE (docs/plans/o0_compile_perf.md §9): this is NOT skippable at -O0, - * even though DCE-on-phis is the only -O0 *optimization* downstream of it. - * SSA renaming is what gives every definition its own vreg, which is what - * makes the allocator's one-interval-per-vreg model sound; a value defined - * on two paths otherwise gets a single interval spanning both. Forcing the - * no-promotable fallback at -O0 was measured at -20% compile time and -210 B - * of code, and miscompiled 4 gcc-torture execute tests (990404-1, pr125291, - * pr34415, pending-4) -- with dominators and promotion still running, so it - * is the renaming itself that is load-bearing. Making -O0 cheaper here - * needs a different interval model, not a skip. */ - IRSSAState *ssa = tcc_ir_ssa_construct(ir, cfg); - int had_promotable = (ssa != NULL); - if (!ssa) { - /* No promotable variables; still build intervals from flat IR */ - ssa = tcc_mallocz(sizeof(IRSSAState)); - ssa->cfg = cfg; - ssa->block_phis = tcc_mallocz(cfg->num_blocks * sizeof(IRPhiNode *)); - ssa->num_vars = ir->next_local_variable; - } else { - tcc_ir_ssa_rename(ir, ssa); - } - tcc_ir_dump_after_pass(ir, "ssa_rename"); - /* The same state with the joins shown -- `ssa_rename`'s flat listing reads - * phi destinations that nothing in it defines. */ - tcc_ir_dump_ssa_after_pass(ir, ssa, "ssa_phi"); - dbg_scan_imm_dest(ir, "ssa_rename"); dbg_scan_overlap(ir, "ssa_rename"); - tcc_pass_timing_end(&ra2_pt, -1); - tcc_pass_timing_begin(&ra2_pt, "ra2:ssaopt"); - - /* Phase 2 (memory-SSA): store->load forwarding over the reaching-def walk, - * surviving joins (MemoryPhi) and provably-non-aliasing intervening stores - * that the dom-spine load_cse cannot. Currently OPT-IN (TCC_MEM_SSA_FWD): - * only full-word (4/8-byte) stores forward soundly; the high-value sub-word - * bitfield idiom needs a consumer-mask demand analysis before it can be - * forwarded safely (a raw sub-word forward truncates — see pr78477), so this - * is not yet on by default. TCC_MEM_SSA / TCC_MEM_SSA_DUMP verify / dump the - * constructed memory-SSA. */ - { - const int do_fwd = ra_mem_ssa_fwd(); - const int do_dump = ra_mem_ssa_dump(); - const int do_validate = ra_mem_ssa_verify_on() || do_dump; - if (do_fwd || do_validate) { - MemSSAState *msa = tcc_ir_mem_ssa_build(ir, cfg); - if (msa) { - if (do_validate && !tcc_ir_mem_ssa_verify(ir, msa)) - fprintf(stderr, "[mem-ssa] INVARIANT VIOLATION (%d instrs)\n", - ir->next_instruction_index); - if (do_dump) - tcc_ir_mem_ssa_dump(ir, msa); - if (do_fwd) - tcc_ir_mem_ssa_load_fwd(ir, msa); - tcc_ir_mem_ssa_free(msa); - } - } - } - - /* SSA optimization passes. - * At -O0: only run DCE to remove dead phi definitions that could - * confuse phi resolution. Skip copy propagation and target generators - * which can break VLA/alignment code in unoptimized IR. - * At -O1+: run the full optimization engine. When no variables were - * promoted (all address-taken), run only load CSE and branch folding - * which operate safely on TEMP vregs without phi nodes. */ - { - IRSSAOptCtx ssa_opt_ctx; - tcc_ir_ssa_opt_init(&ssa_opt_ctx, ir, ssa, cfg); - if (tcc_state->optimize >= 1) { - if (had_promotable) { - tcc_ir_ssa_opt_run(&ssa_opt_ctx); - } else { - /* Run a pass, then make it observable to -dump-ir-passes= - * golden snapshots (same names as the tcc_ir_ssa_opt_run driver). */ -#define RUN_SSA(name, call) \ - do \ - { \ - if (!tcc_ir_opt_pass_disabled(name)) \ - { \ - int _c; \ - TCC_PASS_TIMED(_c, name, (call)); \ - np_changes += _c; \ - } \ - tcc_ir_dump_after_pass(ir, name); \ - } while (0) - ssa_opt_ctx.no_stack_fwd = 0; - for (int np_iter = 0; np_iter < 5; np_iter++) { - int np_changes = 0; - RUN_SSA("ssa:var_const_fold", ssa_opt_var_const_fold(&ssa_opt_ctx)); - RUN_SSA("ssa:var_forward", ssa_opt_var_forward(&ssa_opt_ctx)); - RUN_SSA("ssa:sccp", ssa_opt_sccp(&ssa_opt_ctx)); - RUN_SSA("ssa:load_cse", ssa_opt_load_cse(&ssa_opt_ctx)); - RUN_SSA("ssa:diamond_store_fwd", ssa_opt_diamond_store_fwd(&ssa_opt_ctx)); - RUN_SSA("ssa:const_string_fold", tcc_ir_ssa_opt_const_string_fold(&ssa_opt_ctx)); - RUN_SSA("ssa:bitop_const_fold", tcc_ir_ssa_opt_bitop_const_fold(&ssa_opt_ctx)); - RUN_SSA("ssa:ptr_store_dse", tcc_ir_ssa_opt_ptr_store_dse(&ssa_opt_ctx)); - RUN_SSA("ssa:cprop", ssa_opt_cprop(&ssa_opt_ctx)); - RUN_SSA("ssa:fold", ssa_opt_fold(&ssa_opt_ctx)); - RUN_SSA("ssa:var_imm_prop", ssa_opt_var_imm_prop(&ssa_opt_ctx)); - RUN_SSA("ssa:const_prop_tmp", ssa_opt_const_prop_tmp(&ssa_opt_ctx)); - RUN_SSA("ssa:branch", ssa_opt_branch(&ssa_opt_ctx)); - RUN_SSA("ssa:vrp", (tcc_state && tcc_state->opt_vrp) ? ssa_opt_vrp(&ssa_opt_ctx) : 0); - RUN_SSA("ssa:setif_or_taut", - (tcc_state && tcc_state->opt_const_prop) ? ssa_opt_setif_or_taut(&ssa_opt_ctx) : 0); - RUN_SSA("ssa:setif_mask_fold", - (tcc_state && tcc_state->opt_const_prop) ? ssa_opt_setif_mask_fold(&ssa_opt_ctx) : 0); - RUN_SSA("ssa:cmp_offset_fold", - (tcc_state && tcc_state->opt_const_prop) ? ssa_opt_cmp_offset_fold(&ssa_opt_ctx) : 0); - RUN_SSA("ssa:reassoc", ssa_opt_reassoc(&ssa_opt_ctx)); - RUN_SSA("ssa:strength", ssa_opt_strength(&ssa_opt_ctx)); - RUN_SSA("ssa:narrow", ssa_opt_narrow(&ssa_opt_ctx)); - RUN_SSA("ssa:gvn", ssa_opt_gvn(&ssa_opt_ctx)); - RUN_SSA("ssa:phi_simplify", ssa_opt_phi_simplify(&ssa_opt_ctx)); - RUN_SSA("ssa:dce", ssa_opt_dce(&ssa_opt_ctx)); - np_changes += tcc_ir_ssa_opt_guard_collapse(&ssa_opt_ctx); - if (!np_changes) - break; - } - /* After the loop, never inside it: bool_norm deletes the `CMP b,#0` - * that setif_mask_fold matches on (see the SSA driver's tail). */ - if (tcc_state && tcc_state->opt_const_prop && - !tcc_ir_opt_pass_disabled("ssa:bool_norm")) { - int _c; - TCC_PASS_TIMED(_c, "ssa:bool_norm", ssa_opt_bool_norm(&ssa_opt_ctx)); - tcc_ir_dump_after_pass(ir, "ssa:bool_norm"); - if (_c) - ssa_opt_dce(&ssa_opt_ctx); - } - /* Target-specific fusions (MLA, LOAD/STORE_INDEXED on ARM). These - * don't need promotable vars or phi nodes — they pattern-match on - * existing TEMP vregs. */ - tcc_ir_ssa_opt_run_target(&ssa_opt_ctx); -#undef RUN_SSA - } - } else { - /* -O0. `ssa_opt_dce` here is *not* an optimization the level is meant to - * turn off: at optimize == 0 its body is `dce_temp_worklist` + - * `dce_unreachable` (everything else in it is gated on optimize >= 1), so - * what it removes is a dead compiler-generated temp and a block nothing - * can reach. Unreachable code is not code, and dropping it is - * load-bearing at -O0 -- gcc-torture's `link_error` tests - * (`medce-1`, `20030330-1`, `ieee/fp-cmp-7`) fold their condition in the - * *frontend* and are left with a `JMP` over a call that must not reach - * the linker. gcc -O0 drops it too. - * - * `ssa_opt_cprop` used to run here and has been removed, because that one - * is an optimization. Its immediate dispatcher (`ssa_gen_cprop_imm`) - * forwarded `T <- #k [ASSIGN]` into every use, so - * `int k = 4 * 2; t = n * k;` reached the encoder at -O0 as a single - * `MUL #8` -- which the backend then turned into a shifted add, making - * -O0 output indistinguishable from -O2 on any function small enough to - * read. Propagating a value is exactly what -O0 promises not to do; with - * it gone the same function keeps its `movs r1, #8` and a real `mul`. - * - * If something else ever *has* to run at -O0, it is lowering, and it - * belongs in the lowering path rather than in the else arm of the - * optimizer's gate. */ - ssa_opt_dce(&ssa_opt_ctx); - } - if (tcc_state && tcc_state->optimize >= 1 && tcc_state->opt_redundant_store && - !tcc_ir_opt_pass_disabled("ssa:rmw_byte_clear")) { - int c = tcc_ir_opt_rmw_byte_clear(ir); - if (c && tcc_state->opt_dce) - ssa_opt_dce(&ssa_opt_ctx); - } - tcc_ir_dump_after_pass(ir, "ssa:rmw_byte_clear"); - /* ssa:memmove_global_fwd — rerun of the tccgen-time init-copy-from-global - * forwarding. Its read-only-slot precondition only becomes true here: - * ssa:struct_copy_roundtrip removes the retme pair and ssa:dce ret_store - * kills the write-back, both inside this pipeline — after the tccgen call - * already ran (20040709-2 fn1* family). */ - if (tcc_state && tcc_state->optimize >= 1 && tcc_state->opt_redundant_store && - !tcc_ir_opt_pass_disabled("ssa:memmove_global_fwd")) { - if (tcc_ir_opt_memmove_global_load_fwd(ir) > 0 && tcc_state->opt_dce) { - /* Flat pass: rebuild the SSA use chains it left stale, or the DCE - * worklist can't cascade the now-dead dest-LEA/ADD address temps. - * Phi operand uses must be re-added too — dropping them lets DCE gut - * every loop-carried def (33_ternary_op loop body). */ - for (int p = 0; p < ssa_opt_ctx.vinfo_cap; p++) - ssa_opt_ctx.vinfo[p].use_count = 0; - for (int i = 0; i < ir->next_instruction_index; i++) { - IRQuadCompact *q = &ir->compact_instructions[i]; - if (q->op != TCCIR_OP_NOP) - ssa_opt_scan_instr_uses(&ssa_opt_ctx, i, q); - } - if (ssa->block_phis) { - for (int b = 0; b < cfg->num_blocks; b++) { - for (IRPhiNode *phi = ssa->block_phis[b]; phi; phi = phi->next) { - for (int pi = 0; pi < phi->num_operands; pi++) { - IRSSAVregInfo *pvi = ssa_opt_vinfo(&ssa_opt_ctx, phi->operands[pi].vreg); - if (pvi) - ssa_opt_add_use_phi(pvi, b, pi); - } - } - } - } - ssa_opt_dce(&ssa_opt_ctx); - } - } - tcc_ir_dump_after_pass(ir, "ssa:memmove_global_fwd"); - /* Park a global address that is reloaded across calls in a callee-saved - * register for its whole live range, instead of reloading it from the - * literal pool at each call-separated use. Runs last so target fusions - * have already consumed the load/store-indexed bases they hoist; -O2 only - * (GCC likewise only parks globals at -O2). - * docs/plans/gap_a_ssa_var_index_addr_prop.md (gap ②). */ - tcc_ir_ssa_opt_free(&ssa_opt_ctx); - } - /* And once the passes have had it: which phis survived them. */ - tcc_ir_dump_ssa_after_pass(ir, ssa, "ssa_phi_opt"); - dbg_scan_imm_dest(ir, "ssa_opt_block"); dbg_scan_overlap(ir, "ssa_opt_block"); - - /* Set types from operand btypes (same as tcc_ir_live_analysis). - * Skip lvalue operands: when is_lval=1 the vreg holds a pointer (32-bit) - * and the btype describes the pointed-to value, not the pointer itself. */ - for (int i = 0; i < ir->next_instruction_index; i++) { - IRQuadCompact *q = &ir->compact_instructions[i]; - IROperand dest = tcc_ir_op_get_dest(ir, q); - if (irop_config[q->op].has_dest && tcc_ir_vreg_is_valid(ir, irop_get_vreg(dest)) && !dest.is_lval) { - int btype = irop_get_btype(dest); - if (btype == IROP_BTYPE_FLOAT32 || btype == IROP_BTYPE_FLOAT64) - tcc_ir_vreg_type_set_fp(ir, irop_get_vreg(dest), 1, btype == IROP_BTYPE_FLOAT64); - else if (btype == IROP_BTYPE_INT64) - tcc_ir_vreg_type_set_64bit(ir, irop_get_vreg(dest)); - if (dest.is_complex) - tcc_ir_vreg_type_set_complex(ir, irop_get_vreg(dest)); - } - IROperand src1 = tcc_ir_op_get_src1(ir, q); - if (irop_config[q->op].has_src1 && tcc_ir_vreg_is_valid(ir, irop_get_vreg(src1)) && !src1.is_lval) { - int btype = irop_get_btype(src1); - if (btype == IROP_BTYPE_FLOAT32 || btype == IROP_BTYPE_FLOAT64) - tcc_ir_vreg_type_set_fp(ir, irop_get_vreg(src1), 1, btype == IROP_BTYPE_FLOAT64); - else if (btype == IROP_BTYPE_INT64) - tcc_ir_vreg_type_set_64bit(ir, irop_get_vreg(src1)); - if (src1.is_complex) - tcc_ir_vreg_type_set_complex(ir, irop_get_vreg(src1)); - } - IROperand src2 = tcc_ir_op_get_src2(ir, q); - if (irop_config[q->op].has_src2 && tcc_ir_vreg_is_valid(ir, irop_get_vreg(src2)) && !src2.is_lval) { - int btype = irop_get_btype(src2); - if (btype == IROP_BTYPE_FLOAT32 || btype == IROP_BTYPE_FLOAT64) - tcc_ir_vreg_type_set_fp(ir, irop_get_vreg(src2), 1, btype == IROP_BTYPE_FLOAT64); - else if (btype == IROP_BTYPE_INT64) - tcc_ir_vreg_type_set_64bit(ir, irop_get_vreg(src2)); - if (src2.is_complex) - tcc_ir_vreg_type_set_complex(ir, irop_get_vreg(src2)); - } - } - - /* Propagate types from phi nodes to their dest AND operand vregs. - * SSA rename creates new TEMPs that may only appear with INT32 btype - * in their defining instruction, but the phi btype reflects the - * original variable's type. Phi resolution will insert ASSIGN copies - * with the phi btype, so codegen will expect 64-bit values from these - * vregs even if their defs used INT32 btype. */ - if (ssa->block_phis) { - for (int b = 0; b < cfg->num_blocks; b++) { - for (IRPhiNode *phi = ssa->block_phis[b]; phi; phi = phi->next) { - int is_fp = (phi->btype == IROP_BTYPE_FLOAT32 || phi->btype == IROP_BTYPE_FLOAT64); - int is_i64 = (phi->btype == IROP_BTYPE_INT64); - int is_dbl = (phi->btype == IROP_BTYPE_FLOAT64); - /* Check original variable type as fallback */ - if (!is_fp && !is_i64 && phi->orig_vreg >= 0 && - tcc_ir_vreg_is_valid(ir, phi->orig_vreg)) { - IRLiveInterval *orig_li = tcc_ir_vreg_live_interval(ir, phi->orig_vreg); - if (orig_li) { - if (orig_li->is_llong) is_i64 = 1; - if (orig_li->is_float) { is_fp = 1; is_dbl = orig_li->is_double; } - } - } - if (!is_fp && !is_i64) continue; - - /* Propagate to dest vreg */ - int32_t dv = phi->dest_vreg; - if (dv >= 0 && tcc_ir_vreg_is_valid(ir, dv)) { - if (is_fp) tcc_ir_vreg_type_set_fp(ir, dv, 1, is_dbl); - if (is_i64) tcc_ir_vreg_type_set_64bit(ir, dv); - } - /* Propagate to all operand vregs (phi sources) */ - for (int pi = 0; pi < phi->num_operands; pi++) { - int32_t ov = phi->operands[pi].vreg; - if (ov >= 0 && tcc_ir_vreg_is_valid(ir, ov)) { - if (is_fp) tcc_ir_vreg_type_set_fp(ir, ov, 1, is_dbl); - if (is_i64) tcc_ir_vreg_type_set_64bit(ir, ov); - } - } - } - } - } - - /* Resolve phis BEFORE register allocation: insert ASSIGN copies at - * predecessor block ends so phi-DSTs and phi-SRCs become regular SSA - * temps with non-overlapping intervals. Without this, the linear scan - * sees phi-DST intervals that span the whole loop body alongside their - * phi-SRC operands' intervals (also spanning the body), creating - * artificial register pressure across loops. After this pass the IR - * is no longer in SSA form; ra_build_intervals scans the explicit - * copies and produces concrete intervals. */ - ra_phi_resolve_pre_ra_mode = 1; - tcc_pass_timing_end(&ra2_pt, -1); - tcc_pass_timing_begin(&ra2_pt, "ra2:phis_folds"); - ra_resolve_phis(ir, cfg, ssa, &phi_stats); - ra_phi_resolve_pre_ra_mode = 0; - /* The one edit that makes a phi disappear: every dump before this one can - * have phis in it (with -dump-ir-passes=ssa_phi to see them) and no dump - * after it can, because there is no phi left to show. */ - tcc_ir_dump_after_pass(ir, "ra_resolve_phis"); - dbg_scan_imm_dest(ir,"ra_resolve_phis"); - - /* Collapse "TMP <- const; T_phi <- TMP" chains the phi resolver leaves - * behind in dense switch case bodies. Each fold drops one ASSIGN and one - * SSA temp from the case body, cutting both the per-case instruction count - * and the phi-temp live ranges that drive the linear scan into spills. */ - ra_fold_phi_const_chain(ir); - dbg_scan_imm_dest(ir,"ra_fold_phi_const_chain"); - - /* Once the per-case bodies are canonicalised to "T_phi <- const; JMP merge", - * try to rewrite the entire SWITCH_TABLE dispatch into a single SWITCH_LOAD - * against an inline value table. Must run after the phi-const fold above - * (which produces the canonical body shape) and before live-interval - * construction (which would otherwise see the now-dead case bodies). */ - tcc_ir_opt_switch_to_data(ir); - dbg_scan_imm_dest(ir,"switch_to_data"); - - /* Fold CMP + JUMPIF where both operands resolve to constants within the - * same basic block. Phi resolution often materializes the entry-path - * constant of a loop counter right before its bound check; folding the - * dead skip-loop block removes the carrier-vreg copies it contains and - * cuts the carriers' live ranges, easing register pressure. */ - ra_fold_const_branches(ir); - dbg_scan_imm_dest(ir,"ra_fold_const_branches"); - - /* const_memcpy_fwd: by this point the SSA opt fold (ssa_opt_fold's - * bit-complement / SCCP / GVN) has materialised compile-time-constant - * aggregate values (e.g. pr60502's `*x |= *x ^ {-1,...}` → all-0xFF), and - * the IR is de-SSA'd flat form. Rewrite a constant-filled non-escaping - * stack buffer copied by an aligned AEABI mem* helper into direct wide - * constant stores to the destination, dropping the buffer + the call. Must - * run AFTER the SSA fold (which produces the constants) and BEFORE call - * prefix / interval construction (which must see the call/stores removed). - * The codegen STRD-imm peephole then pairs the word stores into `strd`. */ - if (tcc_state->optimize >= 1) - tcc_ir_opt_const_memcpy_to_dest(ir); - - /* Park a global address reloaded across calls in a callee-saved register for - * its whole live range instead of reloading it from the literal pool at each - * call-separated use. Runs here — after phi resolution de-SSA'd the IR — so - * prepending entry materializations can't desync phi resolution; block_phis - * is emptied, so rebuilding the CFG (which the inserts invalidate) is safe. - * -O2 only (GCC likewise only parks globals at -O2). - * docs/plans/gap_a_ssa_var_index_addr_prop.md (gap ②). */ - if (tcc_state && tcc_state->optimize >= 2) { - int addr_changes = 0; - /* Before the address hoists, so it matches operands the frontend built - * (which carry IROP_AUX_NONVOLATILE) rather than bases synthesized by a - * pass. Here rather than in the flat pipeline because the two reads it - * collapses only name the same address temp once GVN has run — see the - * header of source/opt/flat/memory/deref_operand_cse.c. */ - addr_changes += tcc_ir_opt_deref_operand_cse(ir); - if (!tcc_ir_opt_pass_disabled("ssa:global_addr_hoist")) - addr_changes += tcc_ir_ssa_opt_global_addr_hoist(ir); - if (!tcc_ir_opt_pass_disabled("ssa:loop_addr_hoist")) - addr_changes += tcc_ir_ssa_opt_loop_addr_hoist(ir); - if (!tcc_ir_opt_pass_disabled("ssa:local_addr_cse")) - addr_changes += tcc_ir_ssa_opt_local_addr_cse(ir); - /* Same reason these three run here: an indexed access's STACKOFF base is a - * DIRECT frame reference that slot-based alias analysis reads precisely, so - * parking it in a register may only happen once every alias-sensitive pass - * is done (early it miscompiles gcc.c-torture pr51466). */ - if (!tcc_ir_opt_pass_disabled("ssa:stackoff_indexed_base_cse")) - addr_changes += tcc_ir_opt_stackoff_indexed_base_cse(ir); - if (addr_changes > 0) { - tcc_ir_cfg_free(cfg); - cfg = tcc_ir_cfg_build(ir); - tcc_ir_cfg_compute_dominators(cfg); - ssa->cfg = cfg; - /* block_phis is sized to the old block count and consumed by - * ra_build_intervals; the fresh CFG may have a different count. It is - * all-NULL post-resolution, so resize and zero it to the new count. */ - ssa->block_phis = tcc_realloc(ssa->block_phis, cfg->num_blocks * sizeof(IRPhiNode *)); - memset(ssa->block_phis, 0, cfg->num_blocks * sizeof(IRPhiNode *)); - } - } - tcc_ir_dump_after_pass(ir, "ssa:global_addr_hoist"); - - /* Build call prefix for call-crossing detection */ - int *call_prefix = ra_build_call_prefix(ir); - - /* Build SSA live intervals */ - SSAInterval *intervals = NULL; - int interval_count = 0; - int max_vreg_pos = 0; - tcc_pass_timing_end(&ra2_pt, -1); - tcc_pass_timing_begin(&ra2_pt, "ra2:intervals"); - ra_build_intervals(ir, cfg, ssa, &intervals, &interval_count, call_prefix, &max_vreg_pos); - tcc_pass_timing_end(&ra2_pt, -1); - tcc_pass_timing_begin(&ra2_pt, "ra2:hints"); - ra_build_narrow_weights(ir, target, intervals, interval_count, max_vreg_pos); - - /* Build phi register hints. block_phis is empty after pre-RA resolution, - * so the phi-based pass is a no-op; the assign-based pass picks up - * the explicit copies emitted at predecessor block ends. */ - ra_build_phi_hints(intervals, interval_count, ssa, cfg, max_vreg_pos); - ra_build_assign_hints(intervals, interval_count, ir, max_vreg_pos); - ra_build_load_param_hints(intervals, interval_count, ir, max_vreg_pos); - ra_build_bfi_hints(intervals, interval_count, ir, max_vreg_pos); - ra_build_outgoing_param_hints(intervals, interval_count, ir, max_vreg_pos); - - /* Graph-based coalescing (accurate liveness + interference) — merges - * copy-related non-interfering vregs, including multi-predecessor merge-phis - * the in-scan transfer cannot handle. Gated by TCC_COALESCE. */ - tcc_pass_timing_end(&ra2_pt, -1); - tcc_pass_timing_begin(&ra2_pt, "ra2:coalesce_g"); - ra_coalesce_graph(ir, intervals, interval_count, max_vreg_pos); - tcc_pass_timing_end(&ra2_pt, -1); - tcc_pass_timing_begin(&ra2_pt, "ra2:scan"); - - /* Run linear scan */ - uint64_t dirty_int = 0, dirty_fp = 0; - int n_instr = ir->next_instruction_index; - int has_call = call_prefix && n_instr > 0 && call_prefix[n_instr] > 0; - RaAliveInfo alive; - memset(&alive, 0, sizeof alive); - if (ra_alive_worth_building(ir, intervals, interval_count)) - ra_alive_build(ir, &alive, max_vreg_pos); - ra_linear_scan(ir, intervals, interval_count, target, spill_base, &dirty_int, &dirty_fp, - max_vreg_pos, has_call, &alive); - ra_alive_free(&alive); - tcc_pass_timing_end(&ra2_pt, -1); - tcc_pass_timing_begin(&ra2_pt, "ra2:finish"); - - /* Propagate each coalesced member's allocation from its representative (which - * the scan allocated; members were skipped). coalesce_to holds the rep's - * vreg (stable across the scan's qsort); resolve via a vreg->interval search. */ - { - int any = 0; - for (int i = 0; i < interval_count && !any; i++) - if (intervals[i].coalesce_to >= 0) any = 1; - if (any) { - for (int i = 0; i < interval_count; i++) { - if (intervals[i].coalesce_to < 0) continue; - for (int j = 0; j < interval_count; j++) { - if (intervals[j].vreg != intervals[i].coalesce_to || intervals[j].coalesce_to >= 0) - continue; - intervals[i].r0 = intervals[j].r0; - intervals[i].r1 = intervals[j].r1; - intervals[i].stack_location = intervals[j].stack_location; - break; - } - } - } - } - - if (TCC_LOG_LS) { - for (int i = 0; i < interval_count; i++) { - if (intervals[i].stack_location != 0 && - ra_phi_stats_has_participant(&phi_stats, intervals[i].vreg)) - phi_stats.participant_spills++; - } - LOG_LS("phi_stats operands=%d parallel_requested=%d parallel_emitted=%d cycle_temporaries=%d participant_spills=%d", - phi_stats.phi_operands, phi_stats.parallel_requested, - phi_stats.parallel_emitted, phi_stats.cycle_temporaries, - phi_stats.participant_spills); - } - - /* Write results to IR + LS state */ - ra_write_results(ir, intervals, interval_count); - ir->ls.dirty_registers = dirty_int; - ir->ls.dirty_float_registers = dirty_fp; - - /* Phi resolution already happened before ra_build_intervals (above). - * The instruction stream now has explicit ASSIGN copies; ssa->block_phis - * is cleared. We just need to build the live_regs bitmap from the - * intervals the linear scan produced. */ - ra_build_live_regs_bitmap(ir); - ra_refine_live_regs_accurate(ir); - - /* Strip dead phi copies now that allocation and every liveness/scratch - * bitmap are final. A phi copy into a never-read temp is dead code whose - * register the scan may have reused for a value live across the merge; left - * in, it clobbers that value (seed 198468). Running it here — after the - * bitmaps — keeps the allocation identical and only drops dead instructions. */ - ra_eliminate_dead_reg_copies(ir); - - ra_repair_incomplete_calls(ir); - - ra_mark_rematerializable(ir); - - tcc_pass_timing_end(&ra2_pt, -1); - - /* Cleanup */ - tcc_free(phi_stats.participants); - tcc_free(intervals); - if (call_prefix) tcc_free(call_prefix); - tcc_ir_ssa_free(ssa); - tcc_ir_cfg_free(cfg); -} - -/* ============================================================================ - * Post-Allocation Redundant Reload Elimination - * - * Every soft-float routine opens by punning its arguments through a union: - * - * StackLoc[-8] <-- R0(P0) [STORE] ; ua.d = a - * R0(V0) <-- StackLoc[-8] [LOAD] ; a_bits = ua.u - * - * which lowers to `strd r0,r1,[sp,#N]` immediately followed by - * `ldrd r0,r1,[sp,#N]` -- a reload of registers that already hold the value. - * sl_forward refuses to forward it because the two ends have different types - * (FLOAT64 -> INT64), and forwarding it BEFORE allocation was measured and - * reverted: keeping the punned value in a register stretched a 64-bit live - * range and the linear-scan allocator cascaded into spills (dadd +13.4%). - * - * After allocation that trade no longer exists. The registers are already - * chosen, so dropping a load that rewrites a register with the value it - * already holds cannot lengthen anything -- it only removes a memory access. - * - * The analysis is a straight-line available-value table: remember which - * register pair was last stored to each frame slot, drop a later load of that - * slot into exactly those registers, and invalidate on anything that could - * disturb either side. - * ==========================================================================*/ - -/* An allocation half carries its physical register in the low five bits and a - * spill flag at 0x20, but an UNALLOCATED half is the all-ones sentinel -- and - * that sentinel has the spill bit set. So "is this half spilled" is only a - * question once the half names a register; asked unconditionally it answers - * yes for the absent high half of every 32-bit value. Codegen already reads - * it this way (ir/codegen.c takes pr1_reg and pr1_spilled apart separately). - */ -static int ra_alloc_half_unset(uint16_t half) -{ - return (half & PREG_REG_NONE) == PREG_REG_NONE; -} - -static int ra_alloc_half_spilled(uint16_t half) -{ - return !ra_alloc_half_unset(half) && (half & PREG_SPILLED) != 0; -} - -/* A direct frame slot: `StackLoc[off]` as a memory operand (an `Addr[...]` - * form has is_lval clear and is an address, not a location). */ -static int ra_rl_slot_offset(IROperand op, int *off) -{ - if (op.tag != IROP_TAG_STACKOFF || op.is_llocal || op.is_sym) - return 0; - if (irop_get_vreg(op) != -1 || !op.is_lval) - return 0; - *off = (int)irop_get_stack_offset(op); - return 1; -} - -/* Physical registers behind a plain register operand, as codegen resolves it. */ -static int ra_rl_operand_regs(TCCIRState *ir, IROperand op, int *r0, int *r1) -{ - int32_t vr = irop_get_vreg(op); - if (vr < 0 || !tcc_ir_vreg_is_valid(ir, vr)) - return 0; - IRLiveInterval *li = tcc_ir_vreg_live_interval(ir, vr); - if (!li || li->allocation.offset != 0) - return 0; - if (ra_alloc_half_spilled(li->allocation.r0) || ra_alloc_half_spilled(li->allocation.r1)) - return 0; - int a0 = li->allocation.r0 & PREG_REG_NONE; - int a1 = li->allocation.r1 & PREG_REG_NONE; - if (a0 == PREG_REG_NONE) - return 0; - *r0 = a0; - *r1 = (a1 == PREG_REG_NONE) ? -1 : a1; - return 1; -} - -/* Bytes an access moves when the register ends up holding EXACTLY the slot's - * contents. A sub-word access does not: `strb r2,[sp,#N]` writes only r2's - * low byte and `ldrb r2,[sp,#N]` brings it back zero-extended, so neither end - * of such a pair may be matched against the other or against a word access. - * FLOAT64/INT64 (and FLOAT32/INT32) share a width because a 64-bit soft-float - * value lives in the same GPR pair as the integer it was punned from -- that - * equivalence is the whole point of the pass. */ -static int ra_rl_exact_width(int btype) -{ - switch (btype) - { - case IROP_BTYPE_INT32: - case IROP_BTYPE_FLOAT32: - return 4; - case IROP_BTYPE_INT64: - case IROP_BTYPE_FLOAT64: - return 8; - default: - return 0; - } -} - -typedef struct RARelSlot -{ - int off; /* frame offset of the slot */ - int size; /* bytes the access covered (4 or 8; exact, see above) */ - int r0, r1; /* registers whose value it holds (r1 < 0 when 32-bit) */ - int def_idx; /* the store that put it there */ - int valid; -} RARelSlot; - -#define RA_REL_MAX_TRACKED 16 - -/* Keep `regs` occupied over [lo,hi] so the encoder cannot hand them out. - * - * A register outside every live interval holds nothing the allocator is - * accounting for, and the encoder helps itself to exactly those whenever it - * has to materialise an operand: `R0 <- StackLoc[-12] ADD StackLoc[-36]` is - * three instructions, two of them loads of the operands into whatever happens - * to be free. Neither load exists at IR level, so nothing in this pass sees - * them -- it would go on believing a slot's value is in a register the - * encoder had already reused (nested_struct_return printed dx + dy where it - * meant p.y + dy). - * - * Refusing whenever the register is dead somewhere in between is correct but - * throws away most of what this pass is for. Reserving it instead makes the - * assumption true: both scratch pickers -- tcc_ls_find_free_scratch_reg and - * scratch_pushed_dead_reg -- union live_regs_by_instruction[] into what they - * consider taken, and codegen has not run yet. Nothing else owns the - * register over this window, or it would have been live and there would be - * nothing to reserve. */ -static void ra_rl_reserve_regs(LSLiveIntervalState *ls, int r0, int r1, int lo, int hi) -{ - if (!ls->live_regs_by_instruction) - return; - for (int k = lo; k <= hi && k < ls->live_regs_by_instruction_size; ++k) - { - if (k < 0) - continue; - if (r0 >= 0) - ls->live_regs_by_instruction[k] |= (1u << r0); - if (r1 >= 0) - ls->live_regs_by_instruction[k] |= (1u << r1); - } -} - -static int ra_redundant_reload_elim(TCCIRState *ir) -{ - LSLiveIntervalState *ls = &ir->ls; - const int n = ir->next_instruction_index; - int removed = 0; - - if (tcc_state->optimize < 1) - return 0; - if (tcc_ir_opt_pass_disabled("ra:reload_elim")) - return 0; - - /* A slot whose address is taken anywhere can be written through that - * pointer, so it is never tracked. */ - int escaped[RA_REL_MAX_TRACKED]; - int nescaped = 0; - int escape_overflow = 0; - for (int i = 0; i < n; ++i) - { - IRQuadCompact *q = &ir->compact_instructions[i]; - if (q->op == TCCIR_OP_NOP) - continue; - for (int k = 0; k < 4; ++k) - { - IROperand op = (k == 0) ? (irop_config[q->op].has_dest ? tcc_ir_op_get_dest(ir, q) : IROP_NONE) - : (k == 1) ? (irop_config[q->op].has_src1 ? tcc_ir_op_get_src1(ir, q) : IROP_NONE) - : (k == 2) ? (irop_config[q->op].has_src2 ? tcc_ir_op_get_src2(ir, q) : IROP_NONE) - : (q->op == TCCIR_OP_MLA ? tcc_ir_op_get_accum(ir, q) : IROP_NONE); - if (op.tag != IROP_TAG_STACKOFF || op.is_llocal || op.is_lval) - continue; - if (irop_get_vreg(op) != -1) - continue; - int off = (int)irop_get_stack_offset(op); /* Addr[StackLoc[off]] */ - if (nescaped < RA_REL_MAX_TRACKED) - escaped[nescaped++] = off; - else - escape_overflow = 1; - } - } - - RARelSlot tracked[RA_REL_MAX_TRACKED]; - int ntracked = 0; - - for (int i = 0; i < n; ++i) - { - IRQuadCompact *q = &ir->compact_instructions[i]; - - /* Control can arrive here from anywhere else; nothing carries over. */ - if (q->is_jump_target) - ntracked = 0; - if (q->op == TCCIR_OP_NOP) - continue; - - switch (q->op) - { - /* A call, a write through a pointer or an indexed write can land on any - * slot, and inline asm can do anything at all. */ - case TCCIR_OP_FUNCCALLVAL: - case TCCIR_OP_FUNCCALLVOID: - case TCCIR_OP_STORE_INDEXED: - case TCCIR_OP_STORE_POSTINC: - case TCCIR_OP_INLINE_ASM: - case TCCIR_OP_ASM_INPUT: - case TCCIR_OP_ASM_OUTPUT: - case TCCIR_OP_SETJMP: - case TCCIR_OP_NL_SETJMP: - case TCCIR_OP_VLA_ALLOC: - case TCCIR_OP_VLA_SP_RESTORE: - ntracked = 0; - continue; - default: - break; - } - - int slot_off = 0; - - /* Drop a load that rewrites the very registers already holding the slot. */ - if (q->op == TCCIR_OP_LOAD && ntracked && - ra_rl_slot_offset(tcc_ir_op_get_src1(ir, q), &slot_off)) - { - int d0 = 0, d1 = 0; - IROperand dest = tcc_ir_op_get_dest(ir, q); - int width = ra_rl_exact_width(irop_get_btype(tcc_ir_op_get_src1(ir, q))); - if (width && !dest.is_lval && ra_rl_operand_regs(ir, dest, &d0, &d1) && - ((d1 >= 0) == (width == 8))) - { - for (int t = 0; t < ntracked; ++t) - { - if (!tracked[t].valid || tracked[t].off != slot_off || - tracked[t].size != width) - continue; - if (tracked[t].r0 != d0 || tracked[t].r1 != d1) - break; - ra_rl_reserve_regs(ls, tracked[t].r0, tracked[t].r1, tracked[t].def_idx, i); - q->op = TCCIR_OP_NOP; - removed++; - RA_DBG("reload_elim @%d: slot %d already in R%d/R%d", i, slot_off, d0, d1); - break; - } - } - if (q->op == TCCIR_OP_NOP) - continue; - } - - /* Record a store, after invalidating whatever it overlaps. */ - if (q->op == TCCIR_OP_STORE && - ra_rl_slot_offset(tcc_ir_op_get_dest(ir, q), &slot_off)) - { - IROperand val = tcc_ir_op_get_src1(ir, q); - int s0 = 0, s1 = 0; - int dst_bt = irop_get_btype(tcc_ir_op_get_dest(ir, q)); - int size = ra_rl_exact_width(dst_bt); - int inval_size = size; - if (!size) - { - /* A sub-word store writes at most two bytes, so invalidating a whole - * word over-covers it; it leaves no register holding the slot, so it - * records nothing. Anything else (a struct copy) has a width this - * code cannot see -- assume it reached everything. */ - if (dst_bt != IROP_BTYPE_INT8 && dst_bt != IROP_BTYPE_INT16) - { - ntracked = 0; - continue; - } - inval_size = 4; - } - - for (int t = 0; t < ntracked; ++t) - if (tracked[t].valid && slot_off < tracked[t].off + tracked[t].size && - tracked[t].off < slot_off + inval_size) - tracked[t].valid = 0; - - int is_escaped = escape_overflow; - for (int e = 0; e < nescaped && !is_escaped; ++e) - if (escaped[e] == slot_off) - is_escaped = 1; - - if (size && !is_escaped && !val.is_lval && ra_rl_operand_regs(ir, val, &s0, &s1) && - ((s1 >= 0) == (size == 8)) && ra_rl_exact_width(irop_get_btype(val)) == size && - ntracked < RA_REL_MAX_TRACKED) - { - tracked[ntracked].off = slot_off; - tracked[ntracked].size = size; - tracked[ntracked].r0 = s0; - tracked[ntracked].r1 = s1; - tracked[ntracked].def_idx = i; - tracked[ntracked].valid = 1; - ntracked++; - } - continue; - } - - /* A store to a slot we could not decode, or through a vreg, may hit - * anything we are holding. */ - if (q->op == TCCIR_OP_STORE) - { - ntracked = 0; - continue; - } - - /* Any other write to a register drops the entries that named it. */ - if (irop_config[q->op].has_dest) - { - IROperand dest = tcc_ir_op_get_dest(ir, q); - int d0 = 0, d1 = 0; - if (!dest.is_lval && ra_rl_operand_regs(ir, dest, &d0, &d1)) - { - for (int t = 0; t < ntracked; ++t) - if (tracked[t].valid && - (tracked[t].r0 == d0 || tracked[t].r1 == d0 || - (d1 >= 0 && (tracked[t].r0 == d1 || tracked[t].r1 == d1)))) - tracked[t].valid = 0; - } - else if (!dest.is_lval && irop_get_vreg(dest) >= 0) - { - /* Destination register unknown -- assume it hit everything. */ - ntracked = 0; - } - } - } - - return removed; -} - -/* ============================================================================ - * Post-Allocation Dead Frame-Store Elimination - * - * The reload elimination above is what leaves these behind. Every soft-float - * routine opens by punning its arguments through a union: - * - * StackLoc[-8] <-- R0(P0) [STORE] ; ua.d = a - * R0(V0) <-- StackLoc[-8] [LOAD] ; a_bits = ua.u - * - * Once the reload is gone nothing in the function reads that slot again, and - * the store is `strd r0,r1,[sp,#N]` writing two words no one will ever look at. - * __aeabi_dadd opens with two such slots, __aeabi_dmul with four. The IR-level - * dead-store pass (dce_dead_stackloc_stores) cannot reach them: it runs before - * allocation, while the reload is still a reader. - * - * The rule is deliberately blunt -- a store dies only when NO instruction - * anywhere in the function names an overlapping byte of its slot -- so it needs - * no ordering or dominance reasoning and stays correct however control flows. - * Anything that could reach a frame slot without naming it rules the whole - * function out: an address taken of any slot, an indexed access based on one - * (which reaches past the operand's own width), inline asm, setjmp, a VLA - * moving sp, or a nested function's static chain. - * ==========================================================================*/ - -/* Bytes an access to a frame slot covers, or 0 when the width is not - * statically known -- a struct copy or a complex pair reaches further than its - * component btype says. */ -static int ra_dfs_width(IROperand op) -{ - if (op.is_complex) - return 0; - switch (irop_get_btype(op)) - { - case IROP_BTYPE_INT8: - return 1; - case IROP_BTYPE_INT16: - return 2; - case IROP_BTYPE_INT32: - case IROP_BTYPE_FLOAT32: - return 4; - case IROP_BTYPE_INT64: - case IROP_BTYPE_FLOAT64: - return 8; - default: - return 0; - } -} - -/* A plain frame slot operand: `StackLoc[off]` naming memory directly. */ -static int ra_dfs_is_slot(IROperand op) -{ - return op.tag == IROP_TAG_STACKOFF && !op.is_sym && irop_get_vreg(op) == -1; -} - -#define RA_DFS_MAX_READS 256 - -static int ra_dead_frame_store_elim(TCCIRState *ir) -{ - const int n = ir->next_instruction_index; - int removed = 0; - - if (tcc_state->optimize < 1) - return 0; - if (tcc_ir_opt_pass_disabled("ra:dead_frame_store")) - return 0; - if (ir->has_static_chain) - return 0; - - struct - { - int off; - int size; - } reads[RA_DFS_MAX_READS]; - int nreads = 0; - - for (int i = 0; i < n; ++i) - { - IRQuadCompact *q = &ir->compact_instructions[i]; - if (q->op == TCCIR_OP_NOP) - continue; - - switch (q->op) - { - case TCCIR_OP_INLINE_ASM: - case TCCIR_OP_ASM_INPUT: - case TCCIR_OP_ASM_OUTPUT: - case TCCIR_OP_SETJMP: - case TCCIR_OP_NL_SETJMP: - case TCCIR_OP_VLA_ALLOC: - case TCCIR_OP_VLA_SP_RESTORE: - case TCCIR_OP_SET_CHAIN: - case TCCIR_OP_INIT_CHAIN_SLOT: - return 0; - default: - break; - } - - /* An indexed or post-incrementing access based on a slot runs off the end - * of the operand's own btype, so its width says nothing about what it - * touched. */ - const int reaches_past_width = - (q->op == TCCIR_OP_LOAD_INDEXED || q->op == TCCIR_OP_STORE_INDEXED || - q->op == TCCIR_OP_LOAD_POSTINC || q->op == TCCIR_OP_STORE_POSTINC); - - for (int k = 0; k < 4; ++k) - { - IROperand op = (k == 0) ? (irop_config[q->op].has_dest ? tcc_ir_op_get_dest(ir, q) : IROP_NONE) - : (k == 1) ? (irop_config[q->op].has_src1 ? tcc_ir_op_get_src1(ir, q) : IROP_NONE) - : (k == 2) ? (irop_config[q->op].has_src2 ? tcc_ir_op_get_src2(ir, q) : IROP_NONE) - : (q->op == TCCIR_OP_MLA ? tcc_ir_op_get_accum(ir, q) : IROP_NONE); - if (!ra_dfs_is_slot(op)) - continue; - /* `Addr[StackLoc[..]]` hands the address to something this scan cannot - * follow; is_llocal reaches memory through a pointer held in the slot. */ - if (!op.is_lval || op.is_llocal || reaches_past_width) - return 0; - /* The destination of a plain STORE is the write under consideration, not - * a read. Every other appearance keeps the slot alive. */ - if (k == 0 && q->op == TCCIR_OP_STORE) - continue; - int w = ra_dfs_width(op); - if (w == 0 || nreads >= RA_DFS_MAX_READS) - return 0; - reads[nreads].off = (int)irop_get_stack_offset(op); - reads[nreads].size = w; - nreads++; - } - } - - for (int i = 0; i < n; ++i) - { - IRQuadCompact *q = &ir->compact_instructions[i]; - if (q->op != TCCIR_OP_STORE) - continue; - IROperand dest = tcc_ir_op_get_dest(ir, q); - if (!ra_dfs_is_slot(dest) || !dest.is_lval || dest.is_llocal) - continue; - if (tcc_ir_access_is_volatile(ir, dest)) - continue; - const int w = ra_dfs_width(dest); - if (w == 0) - continue; - const int off = (int)irop_get_stack_offset(dest); - int is_read = 0; - for (int r = 0; r < nreads && !is_read; ++r) - if (off < reads[r].off + reads[r].size && reads[r].off < off + w) - is_read = 1; - if (is_read) - continue; - q->op = TCCIR_OP_NOP; - removed++; - RA_DBG("dead_frame_store @%d: slot %d (%d bytes) is never read", i, off, w); - } - - return removed; -} - -/* ============================================================================ - * Post-Allocation Copy Propagation - * - * Move coalescing (below) removes a register copy by REASSIGNING one of its - * endpoints so the move degenerates to `mov rX,rX`. Both of its directions - * need one endpoint's register to be free across the other's live range, so - * neither can fire when the source OUTLIVES the copy: the source is still - * sitting in its own register, which is precisely why that register is not - * free. - * - * That is the dominant shape in soft float, where one union-punned 64-bit - * value feeds a dozen inlined accessors: - * - * R4(T8) <-- ... ; a_bits, live to the end of the function - * R8(T18) <-- &R4(T8) [ASSIGN] ; mov r8,r4 / mov r9,r5 - * R8(T19) <-- &R8(T18) SHR #52 ; reads only the high half - * - * The complementary transform needs no free register at all: rewrite T18's - * reads back to T8 and drop the copy. That is legal exactly when T8's - * register still holds T8 everywhere T18 is live, and the allocator's own - * invariant -- one live interval per physical register at a time -- makes - * that checkable by scanning the interval list for another claimant. - * - * Nothing here changes an allocation, so unlike every hint- or - * forwarding-based attempt on these same copies, it cannot stretch a live - * range and trade a move for a spill. Each removed quad is a removed - * instruction. - * ==========================================================================*/ - -/* Ops whose operands are pinned to particular physical registers (ABI slots, - * asm constraints, the static chain) or whose destination must not alias a - * source: a read inside one of these cannot be redirected to another - * register. */ -static int ra_cp_use_is_redirectable(int op) -{ - switch (op) - { - case TCCIR_OP_RETURNVALUE: - case TCCIR_OP_FUNCPARAMVAL: - case TCCIR_OP_FUNCPARAMVOID: - case TCCIR_OP_FUNCCALLVAL: - case TCCIR_OP_FUNCCALLVOID: - case TCCIR_OP_SET_CHAIN: - case TCCIR_OP_ASM_INPUT: - case TCCIR_OP_ASM_OUTPUT: - case TCCIR_OP_INLINE_ASM: - case TCCIR_OP_IJUMP: - case TCCIR_OP_SWITCH_TABLE: - case TCCIR_OP_SWITCH_LOAD: - case TCCIR_OP_SETJMP: - case TCCIR_OP_NL_SETJMP: - case TCCIR_OP_UMULL: - case TCCIR_OP_MLA: - case TCCIR_OP_VLA_ALLOC: - case TCCIR_OP_VLA_SP_SAVE: - case TCCIR_OP_VLA_SP_RESTORE: - return 0; - default: - return 1; - } -} - -/* Does any interval other than `keep` occupy one of `keep`'s registers at some - * point in [lo,hi]? If not, `keep`'s registers still hold `keep`'s value - * across the whole window and reads of a copy of it can be redirected there. */ -static int ra_cp_src_regs_clobbered(const LSLiveIntervalState *ls, - const LSLiveInterval *keep, - uint32_t lo, uint32_t hi) -{ - int regs[2]; - int nregs = 0; - regs[nregs++] = keep->r0; - if (keep->r1 >= 0 && keep->r1 < PREG_NONE && keep->r1 != keep->r0) - regs[nregs++] = keep->r1; - - for (int j = 0; j < ls->next_interval_index; ++j) - { - const LSLiveInterval *x = &ls->intervals[j]; - if (x == keep || x->stack_location != 0) - continue; - if (x->start > hi || x->end < lo) - continue; - for (int r = 0; r < nregs; ++r) - if (x->r0 == regs[r] || x->r1 == regs[r]) - { - return 1; - } - } - return 0; -} - -/* Rewrite the reads of a copy's destination back to its source and drop the - * copy. Returns the number of copies removed. */ -static int ra_copy_propagate(TCCIRState *ir) -{ - LSLiveIntervalState *ls = &ir->ls; - const int n = ir->next_instruction_index; - int removed = 0; - - if (tcc_state->optimize < 1) - return 0; - if (!ls->live_regs_by_instruction || ls->live_regs_by_instruction_size <= 0) - return 0; - if (tcc_ir_opt_pass_disabled("ra:copy_prop")) - return 0; - - for (int i = 0; i < n; ++i) - { - IRQuadCompact *q = &ir->compact_instructions[i]; - if (q->op != TCCIR_OP_ASSIGN && q->op != TCCIR_OP_LOAD) - continue; - - IROperand dest = tcc_ir_op_get_dest(ir, q); - IROperand src = tcc_ir_op_get_src1(ir, q); - /* A LOAD off a vreg whose interval ended up in a register performs no - * memory access -- it is a register copy at codegen, and the IR generator - * emits the inlined `temp = var` binding in exactly that form. Skip - * is_llocal (a real dereference through a pointer) and is_sym (a global). - * Sub-word LOADs emit a UXTB/SXTH alongside the move to narrow the value, - * so only full-width copies qualify. */ - int is_copy_load = 0; - if (q->op == TCCIR_OP_LOAD) - { - int dbt = irop_get_btype(dest), sbt = irop_get_btype(src); - /* is_lval on a STACKOFF source is the VAR's own slot being read, which - * for a register-resident VAR is a register copy. On a VREG source it - * is a DEREFERENCE OF THAT REGISTER -- `R4 <- R3***DEREF***` reads the - * memory R3 points at, and rewriting its readers to R3 hands them the - * address instead of the value (pr90311: `b -= c < (unsigned char) a` - * subtracted from &b). is_llocal is not the flag that separates them; - * a plain pointer dereference does not set it. */ - int deref_through_reg = (src.tag == IROP_TAG_VREG && src.is_lval); - if ((src.tag == IROP_TAG_VREG || (src.tag == IROP_TAG_STACKOFF && src.is_local)) && - !deref_through_reg && !src.is_llocal && !src.is_sym && dbt == sbt && - (dbt == IROP_BTYPE_INT32 || dbt == IROP_BTYPE_INT64 || dbt == IROP_BTYPE_FUNC)) - is_copy_load = 1; - if (!is_copy_load) - continue; - } - /* Plain register-to-register only; an lval on either side is memory -- - * except a copy-LOAD, whose source carries is_lval for the VAR read. */ - if (dest.is_lval || (src.is_lval && !is_copy_load)) - continue; - if (irop_get_btype(dest) != irop_get_btype(src)) - continue; - - int32_t dv = irop_get_vreg(dest); - int32_t sv = irop_get_vreg(src); - if (dv < 0 || sv < 0 || dv == sv) - continue; - if (!tcc_ir_vreg_is_valid(ir, dv) || !tcc_ir_vreg_is_valid(ir, sv)) - continue; - /* The destination must be a temp: a VAR has a home beyond its register. - * The source may be a VAR -- a register-resident one is just a register, - * and it is the punned `a_bits` that every inlined accessor re-reads -- - * but then every write to it counts as a redefinition (below). */ - int src_is_var = (TCCIR_DECODE_VREG_TYPE(sv) == TCCIR_VREG_TYPE_VAR); - if (TCCIR_DECODE_VREG_TYPE(dv) != TCCIR_VREG_TYPE_TEMP) - continue; - if (TCCIR_DECODE_VREG_TYPE(sv) != TCCIR_VREG_TYPE_TEMP && !src_is_var) - continue; - - /* One interval per endpoint. A vreg split across several intervals can - * hold a DIFFERENT register in each, and the range reasoning below reads - * a single interval's bounds -- picking one of several would let a read - * be redirected into the register the other half was living in. */ - LSLiveInterval *src_iv = NULL, *dst_iv = NULL; - int src_ivs = 0, dst_ivs = 0; - for (int j = 0; j < ls->next_interval_index; ++j) - { - if (ls->intervals[j].vreg == (uint32_t)sv) { src_iv = &ls->intervals[j]; src_ivs++; } - if (ls->intervals[j].vreg == (uint32_t)dv) { dst_iv = &ls->intervals[j]; dst_ivs++; } - } - if (!src_iv || !dst_iv || src_ivs != 1 || dst_ivs != 1) - continue; - /* PREG_NONE/PREG_SPILLED are >= 0 but are not registers. */ - if (src_iv->r0 < 0 || src_iv->r0 >= PREG_NONE) - continue; - if (src_iv->stack_location != 0) - continue; - /* A SPILLED destination is the better case, not a disqualifying one: the - * copy the allocator could not keep in a register costs a store and a - * reload at every read, and redirecting those reads to the source removes - * all of it. __aeabi_dadd's `b_bits` is exactly this -- one 64-bit ASSIGN - * off the parameter, spilled, then loaded back four bytes at a time. The - * legality argument below never mentions the destination's register, only - * that the SOURCE's registers still hold the source across the reads, so - * it carries over unchanged. */ - const int dst_spilled = (dst_iv->r0 < 0 || dst_iv->r0 >= PREG_NONE || dst_iv->stack_location != 0); - /* Both ends must sit in the same register class, and a 64-bit class means - * a register PAIR: redirecting a paired read to an interval that only owns - * r0 hands the backend a half-formed operand (it asserts on r1 == 31). A - * register-resident VAR can be exactly that. */ - if (src_iv->reg_type != dst_iv->reg_type) - continue; - if (!dst_spilled) - { - int src_pair = (src_iv->r1 >= 0 && src_iv->r1 < PREG_NONE); - int dst_pair = (dst_iv->r1 >= 0 && dst_iv->r1 < PREG_NONE); - if (src_pair != dst_pair) - continue; - } - if (src_iv->addrtaken || dst_iv->addrtaken) - continue; - /* dst's register is shared with a graph-coalesced class that expects this - * write to land; src is only read, so its class membership is harmless. */ - if (dst_iv->co_member) - continue; - /* The LS intervals above drive the allocator; codegen resolves an operand - * through the IR-level interval's `allocation` instead, and the two are - * not interchangeable -- a VAR can be register-resident there with no high - * half, which the backend rejects the moment a 64-bit read names it. - * Validate against the structure codegen will actually consult. */ - { - IRLiveInterval *src_li = tcc_ir_vreg_live_interval(ir, sv); - IRLiveInterval *dst_li = tcc_ir_vreg_live_interval(ir, dv); - if (!src_li || !dst_li) - continue; - /* A phi-pinned dest shares its register with an identity-elided phi - * partner (post_ra_forward_diamond / ra_phi_copy_needed): the partner's - * reads continue PAST dst's own interval, and this copy is the only - * write that puts the value there. Removing it leaves the shared - * register stale on the path the elided copy used to cover — the - * ra_refine_live_regs_accurate clamp-diamond self-host break. */ - if (dst_li->phi_pinned) - continue; - if (src_li->allocation.offset != 0) - continue; - if (ra_alloc_half_spilled(src_li->allocation.r0) || - ra_alloc_half_spilled(src_li->allocation.r1)) - continue; - if (ra_alloc_half_unset(src_li->allocation.r0)) - continue; - /* The source must carry the full width the reads expect. With a - * register-resident destination that is "both are pairs or neither is"; - * with a spilled one there is no destination pair to compare against, so - * the operand btype -- already required equal above -- decides. */ - const int src_paired = ((src_li->allocation.r1 & PREG_REG_NONE) != PREG_REG_NONE); - if (!dst_spilled) - { - if (dst_li->allocation.offset != 0) - continue; - if (ra_alloc_half_spilled(dst_li->allocation.r0) || - ra_alloc_half_spilled(dst_li->allocation.r1)) - continue; - if (ra_alloc_half_unset(dst_li->allocation.r0)) - continue; - if (src_paired != ((dst_li->allocation.r1 & PREG_REG_NONE) != PREG_REG_NONE)) - continue; - } - else if (src_paired != (irop_get_btype(dest) == IROP_BTYPE_INT64 || - irop_get_btype(dest) == IROP_BTYPE_FLOAT64)) - continue; - } - - /* Already the same register: codegen elides it, nothing to remove. */ - if (src_iv->r0 == dst_iv->r0 && src_iv->r1 == dst_iv->r1) - continue; - /* src must already be live at the copy. */ - if (src_iv->start > (uint32_t)i) - continue; - - /* Collect dst's reads. Require exactly one definition (this copy), every - * read after it, and every read in an op whose operands are not pinned. */ - int ok = 1; - int nuses = 0; - int last_use = i; - for (int k = 0; k < n; ++k) - { - IRQuadCompact *qk = &ir->compact_instructions[k]; - if (k == i || qk->op == TCCIR_OP_NOP) - continue; - if (irop_config[qk->op].has_dest && - irop_get_vreg(tcc_ir_op_get_dest(ir, qk)) == dv) - { ok = 0; break; } /* redefined, or written through as an address */ - - int reads = 0; - if (irop_config[qk->op].has_src1 && - irop_get_vreg(tcc_ir_op_get_src1(ir, qk)) == dv) - reads = 1; - if (!reads && irop_config[qk->op].has_src2 && - irop_get_vreg(tcc_ir_op_get_src2(ir, qk)) == dv) - reads = 1; - if (!reads && qk->op == TCCIR_OP_MLA && - irop_get_vreg(tcc_ir_op_get_accum(ir, qk)) == dv) - reads = 1; - if (!reads) - continue; - - if (k < i || !ra_cp_use_is_redirectable(qk->op)) - { ok = 0; break; } - nuses++; - if (k > last_use) - last_use = k; - } - /* nuses == 0 is a dead copy; ra_eliminate_dead_reg_copies owns that. */ - if (!ok || nuses == 0) - continue; - - /* src's value must survive to the last redirected read. */ - if (src_iv->end < (uint32_t)last_use) - continue; - /* ...and no other interval may claim src's registers over that window. */ - if (ra_cp_src_regs_clobbered(ls, src_iv, (uint32_t)i, (uint32_t)last_use)) - continue; - /* ...and src itself must not be redefined under the reads. */ - for (int k = i + 1; k <= last_use; ++k) - { - IRQuadCompact *qk = &ir->compact_instructions[k]; - if (qk->op == TCCIR_OP_NOP || !irop_config[qk->op].has_dest) - continue; - IROperand dk = tcc_ir_op_get_dest(ir, qk); - /* For a temp, an lval destination is a store THROUGH it (an address - * use), not a new value. For a VAR it is a write to the variable, so - * there is no exemption. */ - if (dk.is_lval && !src_is_var) - continue; - if (irop_get_vreg(dk) == sv) - { ok = 0; break; } - } - if (!ok) - continue; - - for (int k = i + 1; k <= last_use; ++k) - { - IRQuadCompact *qk = &ir->compact_instructions[k]; - if (qk->op == TCCIR_OP_NOP) - continue; - if (irop_config[qk->op].has_src1) - { - IROperand o = tcc_ir_op_get_src1(ir, qk); - if (irop_get_vreg(o) == dv) - { irop_set_vreg(&o, sv); tcc_ir_set_src1(ir, k, o); } - } - if (irop_config[qk->op].has_src2) - { - IROperand o = tcc_ir_op_get_src2(ir, qk); - if (irop_get_vreg(o) == dv) - { irop_set_vreg(&o, sv); tcc_ir_set_src2(ir, k, o); } - } - } - q->op = TCCIR_OP_NOP; - removed++; - RA_DBG("copy_prop @%d: T%d -> T%d (R%d), %d use(s) to %d", i, - (int)(dv & 0xffffff), (int)(sv & 0xffffff), src_iv->r0, nuses, last_use); - } - - return removed; -} - -/* ============================================================================ - * Post-Allocation Move Coalescing - * - * Eliminates register-to-register copies (ASSIGN dest = src) by making both - * sides share the same physical register. The source must die at the ASSIGN - * instruction, the new register must be free for the destination's entire - * live range, and call-crossing safety must be preserved. - * - * The code generator already elides identity moves (mov rX, rX), so a - * successful coalescing eliminates the copy without modifying the IR. - * ============================================================================ */ - -static int ra_count_copies_reaching_codegen(TCCIRState *ir) -{ - int copies = 0; - - for (int i = 0; i < ir->next_instruction_index; i++) { - IRQuadCompact *q = &ir->compact_instructions[i]; - if (q->op != TCCIR_OP_ASSIGN) - continue; - - IROperand dest = tcc_ir_op_get_dest(ir, q); - IROperand src = tcc_ir_op_get_src1(ir, q); - int32_t dest_vreg = irop_get_vreg(dest); - int32_t src_vreg = irop_get_vreg(src); - if (dest.is_lval || src.is_lval || dest_vreg < 0 || src_vreg < 0) - continue; - if (irop_get_btype(dest) != irop_get_btype(src)) { - copies++; - continue; - } - - IRLiveInterval *dest_li = tcc_ir_vreg_live_interval(ir, dest_vreg); - IRLiveInterval *src_li = tcc_ir_vreg_live_interval(ir, src_vreg); - if (!ra_phi_copy_is_identity(dest_li, src_li)) - copies++; - } - - return copies; -} - -/* Is `vr` mentioned by any instruction at all? */ -static int ra_rp_vreg_mentioned(TCCIRState *ir, uint32_t vr) -{ - const int n = ir->next_instruction_index; - for (int k = 0; k < n; ++k) - { - IRQuadCompact *qk = &ir->compact_instructions[k]; - if (qk->op == TCCIR_OP_NOP) - continue; - if (irop_config[qk->op].has_dest && - (uint32_t)irop_get_vreg(tcc_ir_op_get_dest(ir, qk)) == vr) - return 1; - if (irop_config[qk->op].has_src1 && - (uint32_t)irop_get_vreg(tcc_ir_op_get_src1(ir, qk)) == vr) - return 1; - if (irop_config[qk->op].has_src2 && - (uint32_t)irop_get_vreg(tcc_ir_op_get_src2(ir, qk)) == vr) - return 1; - if (qk->op == TCCIR_OP_MLA && - (uint32_t)irop_get_vreg(tcc_ir_op_get_accum(ir, qk)) == vr) - return 1; - } - return 0; -} - -/* Is any of `keep`'s registers claimed by another interval over [lo,hi]? - * - * ra_cp_src_regs_clobbered answers the same question, but every pass that - * deletes an instruction leaves the interval of the value it computed behind: - * the vreg is then mentioned nowhere, yet its interval still claims its - * registers over the range it used to occupy. Such a phantom holds no value - * and cannot clobber anything -- and because it sits exactly where the deleted - * instruction was, it lands on the one-instruction window this pass asks - * about far more often than its share. So a claim is only believed once its - * vreg is shown to still exist in the instruction stream. - */ -static int ra_rp_dst_regs_busy(TCCIRState *ir, const LSLiveIntervalState *ls, - const LSLiveInterval *keep, uint32_t lo, uint32_t hi) -{ - int regs[2]; - int nregs = 0; - regs[nregs++] = keep->r0; - if (keep->r1 >= 0 && keep->r1 < PREG_NONE && keep->r1 != keep->r0) - regs[nregs++] = keep->r1; - - for (int j = 0; j < ls->next_interval_index; ++j) - { - const LSLiveInterval *x = &ls->intervals[j]; - if (x == keep || x->stack_location != 0) - continue; - if (x->start > hi || x->end < lo) - continue; - int hits = 0; - for (int r = 0; r < nregs; ++r) - if (x->r0 == regs[r] || x->r1 == regs[r]) - hits = 1; - if (!hits) - continue; - if (ra_rp_vreg_mentioned(ir, x->vreg)) - return 1; - } - return 0; -} - -/* Which producers may have their destination register changed. This is an - * ALLOW list, not a deny list: an op earns a place only when its destination - * is a plain register write the encoder is free to direct anywhere. A fixed - * destination (a call's r0, UMULL's pair, the division helpers), a - * two-address one (BFI presets its dest to the host word) and one the encoder - * also reads (the postinc forms write back through their base, LDRD against - * its own base is unpredictable) all stay off it. */ -static int ra_rp_def_is_retargetable(int op) -{ - switch (op) - { - case TCCIR_OP_ADD: - case TCCIR_OP_SUB: - case TCCIR_OP_MUL: - case TCCIR_OP_AND: - case TCCIR_OP_OR: - case TCCIR_OP_XOR: - case TCCIR_OP_SHL: - case TCCIR_OP_SAR: - case TCCIR_OP_SHR: - case TCCIR_OP_UBFX: - case TCCIR_OP_SBFX: - case TCCIR_OP_ZEXT: - case TCCIR_OP_SETIF: - case TCCIR_OP_ASSIGN: - return 1; - default: - return 0; - } -} - -/* Retarget the producer of a dying copy source, and drop the copy. - * - * ubfx r9, r1, #20, #11 ubfx sl, r1, #20, #11 - * mov sl, r9 -> - * - * This is the backward sibling of ra_copy_propagate. Copy propagation - * rewrites the reads of a copy's DESTINATION back to its source, so it needs - * the source to still be the occupant of its register across every one of - * them -- a long-lived destination therefore rules it out. Retargeting goes - * the other way: the producer writes the destination's register directly, the - * copy disappears, and whatever reads of the source remain are rewritten to - * the destination, which holds the identical value from the producer onwards. - * - * Nothing here depends on how long the destination lives. What it costs is - * that the destination is born one instruction earlier, so its register has - * to be free over that one instruction -- and, when the source outlives the - * copy, that the source's last read still falls inside the destination's own - * live range, so no range grows at the far end either. - * - * The move coalescer's forward direction covers the same shape by giving the - * destination the SOURCE's register, but only when that register is free - * across the destination's whole live range, which for a value computed early - * and used late it usually is not. Retargeting asks for far less: the - * destination's register must be free over the single instruction the value's - * definition moves back by. - * - * Every candidate is checked against the whole function rather than against - * the live intervals alone, so an interval that over-approximates cannot make - * a read of the source disappear. - */ -static int ra_retarget_producer(TCCIRState *ir) -{ - LSLiveIntervalState *ls = &ir->ls; - const int n = ir->next_instruction_index; - int retargeted = 0; - - if (tcc_state->optimize < 1) - return 0; - if (!ls->live_regs_by_instruction || ls->live_regs_by_instruction_size <= 0) - return 0; - if (tcc_ir_opt_pass_disabled("ra:retarget_producer")) - return 0; - const int tbl_size = ls->live_regs_by_instruction_size; - - for (int i = 0; i < n; ++i) - { - IRQuadCompact *q = &ir->compact_instructions[i]; - if (q->op != TCCIR_OP_ASSIGN) - continue; - /* Control may reach the copy without passing the producer, in which case - * the copy is the only thing establishing the destination on that path. */ - if (q->is_jump_target) - continue; - - IROperand dest = tcc_ir_op_get_dest(ir, q); - IROperand src = tcc_ir_op_get_src1(ir, q); - if (dest.is_lval || src.is_lval) - continue; - if (irop_get_btype(dest) != irop_get_btype(src)) - continue; - - int32_t dv = irop_get_vreg(dest); - int32_t sv = irop_get_vreg(src); - if (dv < 0 || sv < 0 || dv == sv) - continue; - if (!tcc_ir_vreg_is_valid(ir, dv) || !tcc_ir_vreg_is_valid(ir, sv)) - continue; - /* Both ends must be temps. A VAR has a home beyond its register, and the - * source's disappears entirely here. */ - if (TCCIR_DECODE_VREG_TYPE(dv) != TCCIR_VREG_TYPE_TEMP || - TCCIR_DECODE_VREG_TYPE(sv) != TCCIR_VREG_TYPE_TEMP) - continue; - - /* The producer is the instruction before the copy; NOPs left by earlier - * passes do not count as instructions, but a NOP that is a jump target - * still lets control in between the two. */ - int d = -1; - for (int k = i - 1; k >= 0; --k) - { - IRQuadCompact *qk = &ir->compact_instructions[k]; - if (qk->op != TCCIR_OP_NOP) { d = k; break; } - if (qk->is_jump_target) break; - } - if (d < 0) - continue; - IRQuadCompact *pq = &ir->compact_instructions[d]; - if (!ra_rp_def_is_retargetable(pq->op) || !irop_config[pq->op].has_dest) - continue; - IROperand pd = tcc_ir_op_get_dest(ir, pq); - if (pd.is_lval || irop_get_vreg(pd) != sv) - continue; - /* The width the encoder lowers by is the DESTINATION's, and passes that - * narrow a value retype operands in place -- so the producer's own view of - * its destination must be the width the copy moves. */ - if (irop_get_btype(pd) != irop_get_btype(dest)) - continue; - - /* Exactly one interval per endpoint: a split vreg holds a different - * register in each half, and the reasoning below reads one pair of - * bounds. */ - LSLiveInterval *src_iv = NULL, *dst_iv = NULL; - int src_ivs = 0, dst_ivs = 0; - for (int j = 0; j < ls->next_interval_index; ++j) - { - if (ls->intervals[j].vreg == (uint32_t)sv) { src_iv = &ls->intervals[j]; src_ivs++; } - if (ls->intervals[j].vreg == (uint32_t)dv) { dst_iv = &ls->intervals[j]; dst_ivs++; } - } - if (!src_iv || !dst_iv || src_ivs != 1 || dst_ivs != 1) - continue; - /* PREG_NONE/PREG_SPILLED are >= 0 but are not registers. Both ends must - * be register-resident: a spilled destination would turn the producer's - * write into a spill store, which is not what this is measuring. */ - if (src_iv->r0 < 0 || src_iv->r0 >= PREG_NONE || src_iv->stack_location != 0) - continue; - if (dst_iv->r0 < 0 || dst_iv->r0 >= PREG_NONE || dst_iv->stack_location != 0) - continue; - if (src_iv->reg_type != dst_iv->reg_type) - continue; - /* A 64-bit class means a register PAIR; handing the encoder a destination - * that owns only its low half aborts it (r1 == 31). */ - { - int src_pair = (src_iv->r1 >= 0 && src_iv->r1 < PREG_NONE); - int dst_pair = (dst_iv->r1 >= 0 && dst_iv->r1 < PREG_NONE); - if (src_pair != dst_pair) - continue; - } - if (src_iv->addrtaken || dst_iv->addrtaken) - continue; - /* A graph-coalesced interval shares one register with its whole class. */ - if (src_iv->co_member || dst_iv->co_member) - continue; - /* Already the same register: codegen elides the move, nothing to remove. */ - if (src_iv->r0 == dst_iv->r0 && src_iv->r1 == dst_iv->r1) - continue; - - /* Codegen resolves an operand through the IR-level interval's allocation, - * not through the LS one, and the two are not interchangeable. */ - { - IRLiveInterval *src_li = tcc_ir_vreg_live_interval(ir, sv); - IRLiveInterval *dst_li = tcc_ir_vreg_live_interval(ir, dv); - if (!src_li || !dst_li) - continue; - /* Retargeting moves the producer's write off src's register. A - * phi-pinned src shares that register with an identity-elided phi - * partner whose reads outlive src's interval — the write must stay. - * (Mirror of the ra_copy_propagate guard.) */ - if (src_li->phi_pinned) - continue; - if (dst_li->allocation.offset != 0) - continue; - if (ra_alloc_half_spilled(dst_li->allocation.r0) || - ra_alloc_half_spilled(dst_li->allocation.r1)) - continue; - if (ra_alloc_half_unset(dst_li->allocation.r0)) - continue; - if (ra_alloc_half_unset(src_li->allocation.r1) != - ra_alloc_half_unset(dst_li->allocation.r1)) - continue; - } - - /* The source is born at the producer, the destination at the copy, and - * the source does not outlive the destination -- the value ends up in the - * destination's register, which is guaranteed to be the destination's own - * only up to where the allocator said it ends. */ - if (src_iv->start != (uint32_t)d) - continue; - if (dst_iv->start != (uint32_t)i) - continue; - if (src_iv->end > dst_iv->end) - continue; - - /* The destination's registers must be free from the producer to the last - * read that will be redirected onto them. Below the copy that window is - * inside the destination's own range and the check is a formality; above - * it, it is what makes moving the definition back safe -- and it is also - * what keeps the producer's operands out of the way, since a source of - * the producer is live at the producer, and a multi-instruction lowering - * need not read all of its operands before writing its first half. */ - { - uint32_t hi = src_iv->end > (uint32_t)i ? src_iv->end : (uint32_t)i; - if (ra_rp_dst_regs_busy(ir, ls, dst_iv, (uint32_t)d, hi)) - continue; - } - - /* Now against the instruction stream rather than the intervals, so that - * an interval which over-approximates cannot hide a mention. Outside the - * producer and the copy, the source may only be READ, only below the copy - * and inside the destination's range, and only by an op whose operands are - * not pinned. A mention in a DEST operand is refused whatever it means -- - * a redefinition invalidates the rewrite, and a store THROUGH the source - * is a read this pass does not rewrite. The destination, in turn, must - * not be written anywhere but the copy. */ - int ok = 1; - int last_use = i; - for (int k = 0; k < n; ++k) - { - if (k == d || k == i) - continue; - IRQuadCompact *qk = &ir->compact_instructions[k]; - if (qk->op == TCCIR_OP_NOP) - continue; - /* Control must not be able to enter at the copy: on such an edge the - * copy is the only thing that establishes the destination, and the - * producer that would replace it never ran. `is_jump_target` says this - * already and is checked above, but it is a bit many passes maintain by - * hand, so the branches are also read directly. A computed target is - * refused outright because it names no index to compare against. */ - if (qk->op == TCCIR_OP_IJUMP || qk->op == TCCIR_OP_SWITCH_TABLE || - qk->op == TCCIR_OP_SWITCH_LOAD || qk->op == TCCIR_OP_SETJMP || - qk->op == TCCIR_OP_NL_SETJMP) - { ok = 0; break; } - if ((qk->op == TCCIR_OP_JUMP || qk->op == TCCIR_OP_JUMPIF) && - (int)irop_get_imm64_ex(ir, tcc_ir_op_get_dest(ir, qk)) == i) - { ok = 0; break; } - if (irop_config[qk->op].has_dest && - irop_get_vreg(tcc_ir_op_get_dest(ir, qk)) == sv) - { ok = 0; break; } - int reads = 0; - if (irop_config[qk->op].has_src1 && - irop_get_vreg(tcc_ir_op_get_src1(ir, qk)) == sv) - reads = 1; - if (!reads && irop_config[qk->op].has_src2 && - irop_get_vreg(tcc_ir_op_get_src2(ir, qk)) == sv) - reads = 1; - if (!reads && qk->op == TCCIR_OP_MLA && - irop_get_vreg(tcc_ir_op_get_accum(ir, qk)) == sv) - reads = 1; - if (!reads) - continue; - if (k < i || (uint32_t)k > dst_iv->end || !ra_cp_use_is_redirectable(qk->op)) - { ok = 0; break; } - if (k > last_use) - last_use = k; - } - if (!ok) - continue; - /* The redirected reads must all see the value the producer wrote, so the - * destination may not be given a new one underneath them. Only the span - * that actually carries a redirected read matters: a temp reassigned - * beyond the last of them is none of this transform's business. */ - for (int k = i + 1; k <= last_use; ++k) - { - IRQuadCompact *qk = &ir->compact_instructions[k]; - if (qk->op == TCCIR_OP_NOP) - continue; - if (irop_config[qk->op].has_dest && - irop_get_vreg(tcc_ir_op_get_dest(ir, qk)) == dv) - { ok = 0; break; } - } - if (!ok) - continue; - - irop_set_vreg(&pd, dv); - tcc_ir_set_dest(ir, d, pd); - q->op = TCCIR_OP_NOP; - for (int k = i + 1; k <= last_use; ++k) - { - IRQuadCompact *qk = &ir->compact_instructions[k]; - if (qk->op == TCCIR_OP_NOP) - continue; - if (irop_config[qk->op].has_src1) - { - IROperand o = tcc_ir_op_get_src1(ir, qk); - if (irop_get_vreg(o) == sv) - { irop_set_vreg(&o, dv); tcc_ir_set_src1(ir, k, o); } - } - if (irop_config[qk->op].has_src2) - { - IROperand o = tcc_ir_op_get_src2(ir, qk); - if (irop_get_vreg(o) == sv) - { irop_set_vreg(&o, dv); tcc_ir_set_src2(ir, k, o); } - } - } - - /* The destination is now born at the producer. The source's interval is - * left claiming its own registers over [d,i]; nothing writes them any - * more, but pretending they are still busy only costs later passes an - * opportunity, where clearing them wrongly would cost correctness. */ - dst_iv->start = (uint32_t)d; - { - IRLiveInterval *dst_li = tcc_ir_vreg_live_interval(ir, dv); - if (dst_li && dst_li->start > (uint32_t)d) - dst_li->start = (uint32_t)d; - } - for (int k = d; k <= i && k < tbl_size; ++k) - { - ls->live_regs_by_instruction[k] |= (1u << dst_iv->r0); - if (dst_iv->r1 >= 0 && dst_iv->r1 < PREG_NONE) - ls->live_regs_by_instruction[k] |= (1u << dst_iv->r1); - } - retargeted++; - RA_DBG("retarget_producer @%d: %s dest T%d -> T%d (R%d), reads to %d", d, - tcc_ir_get_op_name((TccIrOp)pq->op), (int)(sv & 0xffffff), - (int)(dv & 0xffffff), dst_iv->r0, last_use); - } - - return retargeted; -} - -/* ── ra:load_postinc / ra:store_postinc ── - * - * `D <- P***DEREF***` immediately followed (in the same straight-line region) - * by `P <- P + #k` is ARM's post-indexed load, `ldr D,[P],#k`; the mirrored - * `P***DEREF*** <- V; P <- P + #k` is the post-indexed store, `str V,[P],#k`. - * The backend has emitted both forms since forever - * (tcc_gen_machine_load/store_postinc_mop), but until now NOTHING in the - * pipeline ever produced the ops, so the whole addressing mode was dead code. - * Every IV-strength-reduced pointer walk paid a separate `adds P,#k`; on the - * M33 the post-increment is also a cycle cheaper than the separate add (docs, - * and tcc-m33-addressing-mode-cycle-costs). - * - * The store arm only sees IV-strength-reduced walks (`a[i] = v` loops turned - * into a pointer walk). A source-level `*p++ = v` does NOT lower to this - * shape but to a copy of the pointer followed by the increment - * (`T <- P; P <- T + k; T***DEREF*** <- V`); that needs its own matcher plus - * a liveness proof that T is dead, and is still absent. - * - * This runs POST-allocation deliberately. Pre-RA the fused op writes P without - * the allocator being told (irop_config says the POSTINC ops read P as a USE - * only), and P is exactly the loop-carried value the allocator most needs to - * model. After allocation both registers are already fixed and folding the - * add changes no allocation at all. - * - * The operand layout is the indexed-op one: two value slots, unused, increment - * at operand_base+3 (codegen reads it via tcc_ir_op_get_scale), so the fused - * instruction takes four fresh pool slots rather than reusing the original - * op's two. - */ -static int ra_load_postinc_fuse(TCCIRState *ir) -{ - const int n = ir->next_instruction_index; - int fused = 0; - - if (tcc_state->optimize < 1) - return 0; - int load_disabled = tcc_ir_opt_pass_disabled("ra:load_postinc"); - int store_disabled = tcc_ir_opt_pass_disabled("ra:store_postinc"); - if (load_disabled && store_disabled) - return 0; - - for (int i = 0; i < n; ++i) - { - IRQuadCompact *q = &ir->compact_instructions[i]; - int is_load = (q->op == TCCIR_OP_LOAD); - int is_store = (q->op == TCCIR_OP_STORE); - if (!is_load && !is_store) - continue; - if (is_load ? load_disabled : store_disabled) - continue; - - /* LOAD: `value <- ptr***DEREF***` — ptr is src1, an lvalue. - * STORE: `ptr***DEREF*** <- value` — ptr is the dest, an lvalue. */ - IROperand ptr = is_load ? tcc_ir_op_get_src1(ir, q) : tcc_ir_op_get_dest(ir, q); - IROperand value = is_load ? tcc_ir_op_get_dest(ir, q) : tcc_ir_op_get_src1(ir, q); - - /* A dereference THROUGH a register: `R4 <- R3***DEREF***`. is_local / - * is_llocal / is_sym are the frame-slot and global forms, which name memory - * some other way and have no pointer register to write back. */ - if (irop_get_tag(ptr) != IROP_TAG_VREG || !ptr.is_lval) - continue; - if (ptr.is_local || ptr.is_llocal || ptr.is_sym || ptr.is_complex) - continue; - /* No volatility guard is needed: the fusion neither removes, duplicates nor - * reorders the access. `ldr D,[P],#k` performs exactly the access the - * original did, in the same place; only the address arithmetic moves. */ - if (value.is_lval) - continue; - /* load_postinc_mop takes the access width from the DEST operand (and - * store_postinc_mop from the VALUE operand), while a plain LOAD/STORE - * takes it from the lvalue. Requiring the two to agree makes the forms - * equivalent; a widening load (`int c = *bytep`, dest INT32 over an INT8 - * access) would otherwise silently become LDR. Signedness only matters - * for loads — a store of width N writes the low N bytes either way. - * 64-bit is LDRD/STRD-with-writeback and stays out until it has its own - * test. */ - { - int vbt = irop_get_btype(value), pbt = irop_get_btype(ptr); - if (vbt != pbt || (is_load && value.is_unsigned != ptr.is_unsigned)) - continue; - if (vbt != IROP_BTYPE_INT32 && vbt != IROP_BTYPE_INT16 && vbt != IROP_BTYPE_INT8) - continue; - } - - int32_t pv = irop_get_vreg(ptr); - int32_t dv = irop_get_vreg(value); - if (pv < 0 || dv < 0 || pv == dv) - continue; - - /* The pointer must live in a real register: load_postinc_mop writes the - * incremented address back to the register it found the base in, and never - * to a spill slot, so a spilled P would silently lose the increment. */ - { - IRLiveInterval *pli = tcc_ir_vreg_live_interval(ir, pv); - if (!pli || pli->allocation.offset != 0) - continue; - if (ra_alloc_half_unset(pli->allocation.r0) || - ra_alloc_half_spilled(pli->allocation.r0)) - continue; - } - - /* Scan forward for `P <- P + #k`, bailing on anything that would change - * what the fusion means. The jump-target check comes BEFORE the NOP - * skip: a NOPed quad that is still a jump target means a path enters the - * window that never executed the access, and once the add is folded away - * that path would leave P un-incremented. */ - int add_idx = -1; - int assign_idx = -1; /* the `P <- T` half of a renamed split, if any */ - for (int j = i + 1; j < n; ++j) - { - IRQuadCompact *jq = &ir->compact_instructions[j]; - /* Leaving the region: an instruction reached from elsewhere may arrive - * with P un-incremented, and a branch may skip the add entirely. */ - if (jq->is_jump_target) - break; - if (jq->op == TCCIR_OP_NOP) - continue; - if (jq->op == TCCIR_OP_JUMP || jq->op == TCCIR_OP_JUMPIF || - jq->op == TCCIR_OP_IJUMP || jq->op == TCCIR_OP_SWITCH_TABLE || - jq->op == TCCIR_OP_RETURNVALUE || jq->op == TCCIR_OP_RETURNVOID) - break; - - if (jq->op == TCCIR_OP_ADD) - { - IROperand ad = tcc_ir_op_get_dest(ir, jq); - IROperand a1 = tcc_ir_op_get_src1(ir, jq); - IROperand a2 = tcc_ir_op_get_src2(ir, jq); - if (!ad.is_lval && irop_get_vreg(a1) == pv && irop_is_immediate(a2)) - { - int64_t k = irop_get_imm64_ex(ir, a2); - int32_t av = irop_get_vreg(ad); - if (k >= 1 && k <= 255 && av == pv) - { - add_idx = j; /* in-place: P <- P + #k */ - } - else if (k >= 1 && k <= 255 && av >= 0) - { - /* SSA rename splits the in-place bump into `T <- P + #k` followed - * by `P <- T [ASSIGN]`. That is still the same increment when T - * is born at the ADD and dies at the copy — prove it with the - * allocator's own intervals, then fold and NOP both halves. - * Between the two halves only non-jump-target NOPs may sit. */ - for (int m = j + 1; m < n; ++m) - { - IRQuadCompact *mq = &ir->compact_instructions[m]; - if (mq->is_jump_target) - break; - if (mq->op == TCCIR_OP_NOP) - continue; - if (mq->op == TCCIR_OP_ASSIGN) - { - IROperand cd = tcc_ir_op_get_dest(ir, mq); - IROperand cs = tcc_ir_op_get_src1(ir, mq); - if (!cd.is_lval && !cs.is_lval && irop_get_vreg(cd) == pv && - irop_get_vreg(cs) == av) - { - IRLiveInterval *tli = tcc_ir_vreg_live_interval(ir, av); - if (tli && tli->start == (uint32_t)j && tli->end == (uint32_t)m) - { - add_idx = j; - assign_idx = m; - } - } - } - break; /* first non-NOP decides either way */ - } - } - } - if (add_idx >= 0) - break; - } - - /* Any other read or write of P in between would see the wrong value - * once the increment moves up to the load. */ - { - int touches = 0; - for (int k = 0; k < 3 && !touches; ++k) - { - IROperand o; - if (k == 0) - { - if (!irop_config[jq->op].has_dest) - continue; - o = tcc_ir_op_get_dest(ir, jq); - } - else if (k == 1) - { - if (!irop_config[jq->op].has_src1) - continue; - o = tcc_ir_op_get_src1(ir, jq); - } - else - { - if (!irop_config[jq->op].has_src2) - continue; - o = tcc_ir_op_get_src2(ir, jq); - } - if (irop_get_vreg(o) == pv) - touches = 1; - } - if (jq->op == TCCIR_OP_MLA && irop_get_vreg(tcc_ir_op_get_accum(ir, jq)) == pv) - touches = 1; - if (touches) - break; - } - } - if (add_idx < 0) - continue; - - IRQuadCompact *aq = &ir->compact_instructions[add_idx]; - IROperand incr = tcc_ir_op_get_src2(ir, aq); - - /* Four fresh pool slots: dest, ptr, unused, increment (see the header). */ - int nb = ir->iroperand_pool_count; - tcc_ir_pool_add(ir, IROP_NONE); - tcc_ir_pool_add(ir, IROP_NONE); - tcc_ir_pool_add(ir, IROP_NONE); - tcc_ir_pool_add(ir, IROP_NONE); - if (nb + 3 >= ir->iroperand_pool_capacity) - continue; - - q->op = is_load ? TCCIR_OP_LOAD_POSTINC : TCCIR_OP_STORE_POSTINC; - q->operand_base = (uint32_t)nb; - /* The backend wants the POINTER VALUE here, not the lvalue it dereferences: - * mach_ensure_in_reg on an lval operand emits a load OF the pointer from - * the memory it names, and the writeback then lands in a scratch. */ - IROperand base = ptr; - base.is_lval = 0; - base.aux = 0; - /* LOAD_POSTINC decodes {dest=value, src1=base}; STORE_POSTINC decodes - * {dest=base, src1=value} — the same slots as the plain ops they replace. */ - ir->iroperand_pool[nb + 0] = is_load ? value : base; - ir->iroperand_pool[nb + 1] = is_load ? base : value; - ir->iroperand_pool[nb + 2] = IROP_NONE; - ir->iroperand_pool[nb + 3] = incr; - aq->op = TCCIR_OP_NOP; - if (assign_idx >= 0) - ir->compact_instructions[assign_idx].op = TCCIR_OP_NOP; /* the `P <- T` half */ - fused++; - RA_DBG("%s_postinc @%d: folded +%d from @%d", is_load ? "load" : "store", i, - (int)irop_get_imm64_ex(ir, incr), add_idx); - } - - return fused; -} - -int tcc_ir_move_coalescing(TCCIRState *ir) -{ - LSLiveIntervalState *ls = &ir->ls; - - /* Copies whose source outlives them cannot be coalesced by reassignment; - * propagate those away first so the loop below only sees the rest. */ - int propagated = ra_copy_propagate(ir); - /* Then the copies whose source is born one instruction earlier, which have a - * producer to retarget instead of reads to redirect. */ - int retargeted = ra_retarget_producer(ir); - /* Drop reloads of a slot into the registers that already hold it -- the - * union punning every soft-float routine opens with. */ - int reloads = ra_redundant_reload_elim(ir); - /* And then the stores those reloads were the only reader of. */ - reloads += ra_dead_frame_store_elim(ir); - - if (!ls->live_regs_by_instruction || ls->live_regs_by_instruction_size <= 0) { - if (TCC_LOG_LS) - LOG_LS("copy_stats coalesced=0 reaching_codegen=%d", - ra_count_copies_reaching_codegen(ir)); - return 0; - } - - int coalesced = 0; - const int n = ir->next_instruction_index; - const int tbl_size = ls->live_regs_by_instruction_size; - - /* Track vregs already reverse-coalesced to prevent chains where a src - * gets moved to register A, then a later ASSIGN moves it back to B. */ - uint32_t *rev_done = NULL; - int rev_done_size = 0; - - for (int i = 0; i < n; ++i) - { - IRQuadCompact *q = &ir->compact_instructions[i]; - /* LOAD with a VREG source where the underlying interval ended up in a - * register (no spill) is a register copy at codegen — treat like ASSIGN - * for coalescing. Catches inlined "temp = var" patterns the IR - * generator emits as LOAD even when no memory access is involved. - * Accept is_lval=1 for local VAR reads as long as the VAR is reg-only; - * skip is_llocal (true memory load via pointer) and is_sym (global). */ - /* LOAD from a vreg whose interval ended up in a register (no spill) - * is a register copy at codegen — treat like ASSIGN for coalescing. - * Catches inlined "temp = var" patterns where the IR generator emits - * LOAD with VREG/STACKOFF source even though no memory access happens. - * Skip is_llocal (double-indirection via pointer) and is_sym (global). - * Skip sub-word btypes: those LOADs emit UXTB/SXTB/UXTH/SXTH alongside - * the mov to truncate; coalescing them away would skip the narrowing - * and yield wrong values (see pr69447, fp-cmp-8 sub-word args). */ - int is_copy_load = 0; - if (q->op == TCCIR_OP_LOAD) { - IROperand ts = tcc_ir_op_get_src1(ir, q); - int valid_tag = (ts.tag == IROP_TAG_VREG || - (ts.tag == IROP_TAG_STACKOFF && ts.is_local)); - int dest_bt = irop_get_btype(tcc_ir_op_get_dest(ir, q)); - int src_bt = irop_get_btype(ts); - int width_safe = (dest_bt == src_bt) && - (dest_bt == IROP_BTYPE_INT32 || - dest_bt == IROP_BTYPE_INT64 || - dest_bt == IROP_BTYPE_FUNC); - if (width_safe && valid_tag && !ts.is_llocal && !ts.is_sym) { - int32_t tsv = irop_get_vreg(ts); - if (tsv >= 0 && tcc_ir_vreg_is_valid(ir, tsv)) { - for (int j = 0; j < ls->next_interval_index; ++j) { - if (ls->intervals[j].vreg == (uint32_t)tsv) { - /* Only a source in a REAL register is a register copy. PREG_NONE - * (0x1F) is >= 0 but means "not allocated" — e.g. a stack-passed - * parameter that lives in the caller's frame, not a register. - * Coalescing the LOAD result into such a source would make it - * inherit PREG_NONE and be mis-lowered as a spill at frame offset - * 0 (clobbering the saved frame pointer). */ - if (ls->intervals[j].r0 >= 0 && ls->intervals[j].r0 < PREG_NONE && - ls->intervals[j].stack_location == 0) - is_copy_load = 1; - break; - } - } - } - } - } - if (q->op != TCCIR_OP_ASSIGN && !is_copy_load) - continue; - - const IROperand src1 = tcc_ir_op_get_src1(ir, q); - const IROperand dest = tcc_ir_op_get_dest(ir, q); - /* For is_copy_load (LOAD from in-register VAR) the src has is_lval=1 - * but it behaves as a register copy — don't skip it on that basis. */ - if ((src1.is_lval && !is_copy_load) || dest.is_lval) continue; - int32_t sv = irop_get_vreg(src1); - if (sv < 0 || !tcc_ir_vreg_is_valid(ir, sv)) - continue; - int32_t dv = irop_get_vreg(dest); - if (dv < 0 || !tcc_ir_vreg_is_valid(ir, dv)) - continue; - - LSLiveInterval *src_iv = NULL, *dst_iv = NULL; - for (int j = 0; j < ls->next_interval_index; ++j) - { - if (ls->intervals[j].vreg == (uint32_t)sv) src_iv = &ls->intervals[j]; - if (ls->intervals[j].vreg == (uint32_t)dv) dst_iv = &ls->intervals[j]; - if (src_iv && dst_iv) break; - } - if (!src_iv || !dst_iv) continue; - /* Both endpoints must live in a REAL register. PREG_NONE (0x1F) and - * PREG_SPILLED (0x20) are >= 0 but are NOT registers (e.g. a stack-passed - * parameter resident in the caller's frame). Coalescing onto such an - * endpoint propagates PREG_NONE into a live value, which is then mis-lowered - * as a spill at frame offset 0 — clobbering a saved register at [FP,#0]. */ - if (src_iv->r0 < 0 || src_iv->r0 >= PREG_NONE || - dst_iv->r0 < 0 || dst_iv->r0 >= PREG_NONE) continue; - if (src_iv->stack_location != 0 || dst_iv->stack_location != 0) continue; - if (src_iv->r0 == dst_iv->r0) continue; - /* Never reassign a graph-coalesced interval: it shares one register with - * its whole class, and reassigning one member here would split the class - * (the other members keep the class register), corrupting the value. */ - if (src_iv->co_member || dst_iv->co_member) continue; - - /* Nor a phi-pinned one. post_ra_forward_diamond / ra_phi_copy_needed - * delete an identity phi copy precisely BECAUSE its two vregs already sit - * in one register, and set phi_pinned to say the pairing is now load - * bearing: no instruction is left that would re-establish it. Moving - * either endpoint to a different register here silently un-pairs them, and - * the merge block reads a register the elided-copy path never wrote. - * (Same guard as ra_copy_propagate / ra_retarget_producer above; this loop - * reassigns registers rather than rewriting operands, so it needs it too.) */ - { - IRLiveInterval *src_ir_iv = tcc_ir_vreg_live_interval(ir, sv); - IRLiveInterval *dst_ir_iv = tcc_ir_vreg_live_interval(ir, dv); - if ((src_ir_iv && src_ir_iv->phi_pinned) || (dst_ir_iv && dst_ir_iv->phi_pinned)) - continue; - } - - /* Forward direction: reassign dest to use src's register. - * Requires src to die at this ASSIGN. */ - if (src_iv->end == (uint32_t)i) { - int src_reg = src_iv->r0; - if (dst_iv->crosses_call && !(src_reg >= 4 && src_reg <= 11)) - goto try_reverse; - - int conflict = 0; - for (int k = i + 1; k <= (int)dst_iv->end && k < tbl_size; ++k) - { - if (ls->live_regs_by_instruction[k] & (1u << src_reg)) { - /* Two-address relaxation: at k == dst.end the dst's last use is - * an instruction that consumes dst and writes a fresh result. - * If that result lands in src_reg (some interval starts at k in - * src_reg) AND the same instruction reads dst, sharing src_reg - * is safe — ARM ops read all sources before writing dest, so - * `OP src_reg, ..., src_reg` is a valid two-operand form. */ - int safe = 0; - if (k == (int)dst_iv->end) { - IRQuadCompact *kq = &ir->compact_instructions[k]; - int reads_dst = 0; - if (irop_config[kq->op].has_src1 && - irop_get_vreg(tcc_ir_op_get_src1(ir, kq)) == dv) - reads_dst = 1; - if (!reads_dst && irop_config[kq->op].has_src2 && - irop_get_vreg(tcc_ir_op_get_src2(ir, kq)) == dv) - reads_dst = 1; - if (reads_dst) { - for (int j2 = 0; j2 < ls->next_interval_index; j2++) { - LSLiveInterval *xi = &ls->intervals[j2]; - if (xi->r0 == src_reg && xi->stack_location == 0 && - xi->start == (uint32_t)k) { - safe = 1; - break; - } - } - } - } - if (!safe) { conflict = 1; break; } - } - } - if (!conflict) { - for (int k = (int)dst_iv->start; k < i && k < tbl_size; ++k) - { - if (ls->live_regs_by_instruction[k] & (1u << src_reg)) - { conflict = 1; break; } - } - } - if (!conflict) { - int old_reg = dst_iv->r0; - dst_iv->r0 = src_reg; - for (int k = (int)dst_iv->start; k <= (int)dst_iv->end && k < tbl_size; ++k) - { - /* old_reg's bit may be shared with another interval that coalesced - * onto it earlier (in-place two-address ops overlap on purpose) — - * only clear positions where no other claimant is still live. */ - if (!tcc_ls_reg_held_by_other(ls, old_reg, k, dst_iv)) - ls->live_regs_by_instruction[k] &= ~(1u << old_reg); - ls->live_regs_by_instruction[k] |= (1u << src_reg); - } - RA_DBG("move_coalesce fwd @%d: T%d R%d->R%d [%u,%u]", i, - (int)(dv & 0xffffff), old_reg, src_reg, dst_iv->start, dst_iv->end); - coalesced++; - continue; - } - } - - /* Reverse direction: reassign src to use dest's register. - * Works for loop-carried phi copies where src = f(dest, ...) and - * dest's register is only occupied by dest during src's range. - * Safety: src must not be redefined between the ASSIGN and dest's - * last use, otherwise the shared register would get clobbered. */ -try_reverse:; - /* Skip if this src vreg was already reverse-coalesced */ - { - int already = 0; - for (int ri = 0; ri < rev_done_size; ri++) { - if (rev_done[ri] == (uint32_t)sv) { already = 1; break; } - } - if (already) continue; - } - int dest_reg = dst_iv->r0; - if (src_iv->crosses_call && !(dest_reg >= 4 && dest_reg <= 11)) - continue; - - int conflict = 0; - - /* Conservative: src must be defined directly FROM dest (reads dest - * as src1), like `src = dest + 1` or `src = dest * x + acc`. - * This guarantees ARM's read-before-write makes the in-place - * operation correct. */ - { - int def_idx = (int)src_iv->start; - if (def_idx < 0 || def_idx >= n) { conflict = 1; goto rev_check_done; } - IRQuadCompact *qdef = &ir->compact_instructions[def_idx]; - if (!irop_config[qdef->op].has_src1) { conflict = 1; goto rev_check_done; } - IROperand s1 = tcc_ir_op_get_src1(ir, qdef); - if (irop_get_vreg(s1) != dv) { conflict = 1; goto rev_check_done; } - } - - /* Check src is not redefined while dest is still live */ - for (int k = i + 1; k <= (int)dst_iv->end && k < n; ++k) - { - IRQuadCompact *qk = &ir->compact_instructions[k]; - if (qk->op == TCCIR_OP_NOP) continue; - if (irop_config[qk->op].has_dest) { - IROperand dk = tcc_ir_op_get_dest(ir, qk); - int is_mem_store = (qk->op == TCCIR_OP_STORE || qk->op == TCCIR_OP_STORE_INDEXED || - qk->op == TCCIR_OP_STORE_POSTINC) && dk.is_lval; - if (!is_mem_store) { - int32_t dkvr = irop_get_vreg(dk); - if (dkvr == sv) { conflict = 1; break; } - } - } - } - if (conflict) goto rev_check_done; - - /* Symmetric guard (dest side): after this copy src and dest share - * dest_reg holding the same value. If dest is given a NEW, independent - * value while src is still live, that write clobbers dest_reg and src's - * remaining uses read the wrong value. The loop-carried phi copy this - * pass targets has src dying at the copy (src_iv->end == i), so the range - * below is empty and legitimate coalescing is unaffected; the guard only - * fires when src OUTLIVES the copy and dest is re-defined underneath it - * (bitfield 40979: `u4 = u3` copy, then `u4 = const` clobbers the shared - * register while `u3` is still read). A redefinition at exactly src's - * last use that also reads src is the two-address read-before-write case - * and stays safe. */ - for (int k = i + 1; k <= (int)src_iv->end && k < n; ++k) - { - IRQuadCompact *qk = &ir->compact_instructions[k]; - if (qk->op == TCCIR_OP_NOP) continue; - if (!irop_config[qk->op].has_dest) continue; - IROperand dk = tcc_ir_op_get_dest(ir, qk); - int is_mem_store = (qk->op == TCCIR_OP_STORE || qk->op == TCCIR_OP_STORE_INDEXED || - qk->op == TCCIR_OP_STORE_POSTINC) && dk.is_lval; - if (is_mem_store) continue; - if (irop_get_vreg(dk) != dv) continue; - if (k == (int)src_iv->end) { - int reads_src = 0; - if (irop_config[qk->op].has_src1 && - irop_get_vreg(tcc_ir_op_get_src1(ir, qk)) == sv) reads_src = 1; - if (!reads_src && irop_config[qk->op].has_src2 && - irop_get_vreg(tcc_ir_op_get_src2(ir, qk)) == sv) reads_src = 1; - if (!reads_src && qk->op == TCCIR_OP_MLA && - irop_get_vreg(tcc_ir_op_get_accum(ir, qk)) == sv) reads_src = 1; - if (reads_src) continue; - } - conflict = 1; - break; - } - if (conflict) goto rev_check_done; - - /* Check dest not used between src's def and the ASSIGN. - * src's def overwrites dest_reg; any intervening use of dest - * would read the wrong value. */ - for (int k = (int)src_iv->start + 1; k < i && k < n; ++k) - { - IRQuadCompact *qk = &ir->compact_instructions[k]; - if (qk->op == TCCIR_OP_NOP) continue; - if (irop_config[qk->op].has_src1) { - if (irop_get_vreg(tcc_ir_op_get_src1(ir, qk)) == dv) { conflict = 1; break; } - } - if (!conflict && irop_config[qk->op].has_src2) { - if (irop_get_vreg(tcc_ir_op_get_src2(ir, qk)) == dv) { conflict = 1; break; } - } - if (!conflict && irop_config[qk->op].has_dest) { - IROperand dk = tcc_ir_op_get_dest(ir, qk); - if (dk.is_lval && irop_get_vreg(dk) == dv) { conflict = 1; break; } - } - if (!conflict && qk->op == TCCIR_OP_MLA) { - if (irop_get_vreg(tcc_ir_op_get_accum(ir, qk)) == dv) { conflict = 1; break; } - } - } - if (conflict) goto rev_check_done; - - /* Check no control-flow escape between src's def and the ASSIGN. - * src's def overwrites dest_reg; the ASSIGN re-establishes dest's value - * only on the path that reaches it. A JUMP/JUMPIF in (def, ASSIGN) that - * targets outside [def, ASSIGN] lets control reach later uses of dest - * with dest_reg clobbered and the restoring copy skipped — e.g. a - * top-tested pointer-chase loop (`while (p->next) p = p->next;`) whose - * exit edge branches past the back-edge copy while `p` is still live. */ - for (int k = (int)src_iv->start; k < i && k < n; ++k) - { - IRQuadCompact *qk = &ir->compact_instructions[k]; - if (qk->op == TCCIR_OP_NOP) continue; - if (qk->op == TCCIR_OP_IJUMP || qk->op == TCCIR_OP_SWITCH_TABLE || - qk->op == TCCIR_OP_SWITCH_LOAD) { conflict = 1; break; } - if (qk->op == TCCIR_OP_JUMP || qk->op == TCCIR_OP_JUMPIF) { - int jt = (int)irop_get_imm64_ex(ir, tcc_ir_op_get_dest(ir, qk)); - if (jt < (int)src_iv->start || jt > i) { conflict = 1; break; } - } - } -rev_check_done: - if (conflict) continue; - - /* Check dest_reg not occupied by other intervals during src's range. - * Identity-based: earlier coalesces may have moved a third interval onto - * dest_reg inside dst_iv's range, so "position within dst_iv's range" is - * not proof the claim is dst_iv's own. */ - for (int k = (int)src_iv->start; k <= (int)src_iv->end && k < tbl_size; ++k) - { - if (ls->live_regs_by_instruction[k] & (1u << dest_reg)) - { - if (tcc_ls_reg_held_by_other(ls, dest_reg, k, dst_iv)) - { conflict = 1; break; } - /* dest_reg is live here — only OK if it's from dest_iv itself */ - if (k < (int)dst_iv->start || k > (int)dst_iv->end) - { conflict = 1; break; } - } - } - if (conflict) continue; - - int old_reg = src_iv->r0; - src_iv->r0 = dest_reg; - for (int k = (int)src_iv->start; k <= (int)src_iv->end && k < tbl_size; ++k) - { - /* old_reg's bit may be shared with another interval that coalesced - * onto it earlier — only clear positions with no other live claimant - * (volatile 36818: T175 leaving R5 wiped T212's in-place-XOR claim, - * and the phase-3 scratch fixup then put the outer loop counter there). */ - if (!tcc_ls_reg_held_by_other(ls, old_reg, k, src_iv)) - ls->live_regs_by_instruction[k] &= ~(1u << old_reg); - ls->live_regs_by_instruction[k] |= (1u << dest_reg); - } - RA_DBG("move_coalesce rev @%d: T%d R%d->R%d [%u,%u]", i, - (int)(sv & 0xffffff), old_reg, dest_reg, src_iv->start, src_iv->end); - /* Record this src vreg as reverse-coalesced */ - rev_done = tcc_realloc(rev_done, sizeof(uint32_t) * (rev_done_size + 1)); - rev_done[rev_done_size++] = (uint32_t)sv; - coalesced++; - } - - if (rev_done) - tcc_free(rev_done); - - if (coalesced > 0) - tcc_ls_recompute_dirty_registers(ls); - - /* Last: the IR is in its final shape, so an increment folded into a load - * cannot be moved apart again. */ - int postinc = ra_load_postinc_fuse(ir); - - if (TCC_LOG_LS) - LOG_LS("copy_stats propagated=%d reloads=%d coalesced=%d retargeted=%d reaching_codegen=%d", - propagated, reloads, coalesced, retargeted, - ra_count_copies_reaching_codegen(ir)); - - return coalesced + propagated + reloads + retargeted + postinc; -} diff --git a/source/ir/regalloc.h b/source/ir/regalloc.h index 5fbf36d48..c6b999fbe 100644 --- a/source/ir/regalloc.h +++ b/source/ir/regalloc.h @@ -61,4 +61,5 @@ int tcc_ir_move_coalescing(struct TCCIRState *ir); int ra_repair_incomplete_calls(struct TCCIRState *ir); int ra_fold_const_branches(struct TCCIRState *ir); int ra_fold_phi_const_chain(struct TCCIRState *ir); +int ra_fuse_umaal(struct TCCIRState *ir); diff --git a/source/ir/regalloc_entry.c b/source/ir/regalloc_entry.c new file mode 100644 index 000000000..c874c2387 --- /dev/null +++ b/source/ir/regalloc_entry.c @@ -0,0 +1,1834 @@ +/* + * TCC IR - SSA-Aware Register Allocator + * + * Operates directly on SSA-renamed IR with phi nodes. + * Replaces the tccls.c linear scan when -fssa-regalloc is enabled. + * + * Copyright (c) 2025 Mateusz Stadnik + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +/* SSA register allocator: the entry point (tcc_ir_ssa_regalloc) and + * spill-driven live-range splitting. */ + +#include "regalloc_priv.h" + +/* ============================================================================ + * Entry Point + * ============================================================================ */ + +/* Promote multiply-block-defined TEMPs to fresh VARs so SSA construction places + * phis for them. The frontend emits a single TEMP written on BOTH arms of a + * branch-lowered ternary (`cond ? a : b` where an arm has a side effect / call, + * so it cannot lower to SELECT) — e.g. `T323 <- a` in one block and `T323 <- b` + * in another, then a merge-block use. That violates the SSA-by-construction + * assumption the renamer makes for TEMPs (it renames only VARs and leaves such a + * TEMP untouched), so the merge use resolves to ONE arm's definition + * unconditionally — random-C O1/O2 wrong-code, seeds 100/118 (the value reached a + * later inlined-csmix use as the else-arm value regardless of the condition). + * Converting the TEMP to a VAR routes it through the normal var→SSA promotion, + * which inserts the phi. VAR and TEMP operands share the IROP_TAG_VREG encoding + * and differ only in the type bits, so irop_set_vreg suffices; tcc_ir_vreg_alloc_var + * grows the live-interval array. Only fires for the rare multi-block-def TEMP. */ +/* The def-side operand filter shared by the passes below: returns the TEMP + * index this instruction defines, or -1 when it defines no plain TEMP. STORE + * dests are addresses (a use), FUNCPARAM* dests are not TEMP defs, and an + * is_lval dest is a deref store target rather than a plain TEMP def. */ +static int ra_def_temp_of(TCCIRState *ir, IRQuadCompact *q, int ntmp) +{ + if (q->op == TCCIR_OP_NOP || !irop_config[q->op].has_dest) + return -1; + if (q->op == TCCIR_OP_STORE || q->op == TCCIR_OP_STORE_INDEXED || + q->op == TCCIR_OP_STORE_POSTINC || q->op == TCCIR_OP_FUNCPARAMVAL || + q->op == TCCIR_OP_FUNCPARAMVOID) + return -1; + IROperand d = tcc_ir_op_get_dest(ir, q); + int32_t vr = irop_get_vreg(d); + if (vr < 0 || d.is_lval || TCCIR_DECODE_VREG_TYPE(vr) != TCCIR_VREG_TYPE_TEMP) + return -1; + int t = TCCIR_DECODE_VREG_POSITION(vr); + return (t >= 0 && t < ntmp) ? t : -1; +} + +/* Membership set over (TEMP, block) pairs, used below to answer "does block b + * define TEMP t?" without a per-TEMP rescan. Open addressing, power-of-two + * capacity, linear probe; slot 0 means empty, so keys are stored biased by one. + * The caller sizes the table at >= 2x the number of insertions, which bounds + * the load factor at 0.5 and guarantees the probe terminates. */ +static inline uint64_t ra_blockset_mix(uint64_t k) +{ + k *= 0x9E3779B97F4A7C15ull; + return k ^ (k >> 29); +} + +static void ra_blockset_add(uint64_t *set, int cap, uint64_t key) +{ + const uint64_t k = key + 1; + int i = (int)(ra_blockset_mix(k) & (uint64_t)(cap - 1)); + while (set[i] && set[i] != k) + i = (i + 1) & (cap - 1); + set[i] = k; +} + +static int ra_blockset_has(const uint64_t *set, int cap, uint64_t key) +{ + const uint64_t k = key + 1; + int i = (int)(ra_blockset_mix(k) & (uint64_t)(cap - 1)); + while (set[i]) { + if (set[i] == k) + return 1; + i = (i + 1) & (cap - 1); + } + return 0; +} + +static void ra_promote_multidef_temps_to_vars(TCCIRState *ir, IRCFG *cfg) +{ + int n = ir->next_instruction_index; + int ntmp = ir->next_temporary_variable; + if (n <= 0 || ntmp <= 0 || !cfg || cfg->num_blocks <= 1) + return; + + /* Skip functions that take label addresses (GCC labels-as-values, `&&label`): + * their exact machine-code layout is observable at runtime via the label-offset + * map, so the phi-resolution copies this promotion introduces would shift those + * offsets (96_nodata_wanted measures code size with `&&label` arithmetic). + * Such functions also have inlining disabled (tccgen gates auto-inline on + * !func_has_label_addr), so they never hit the inlined-ternary miscompile this + * promotion fixes — skipping them is free of correctness cost. */ + if (ir->func_has_label_addr) + return; + + /* Only run when SSA construction will actually proceed and rename the new VARs + * back into SSA temps. SSA construction BAILS on un-enumerable control flow + * (IJUMP / computed goto, SETJMP); if we promoted there, the converted VARs + * would be left as unpromoted stack slots and change codegen for the worse + * (96_nodata_wanted's `&&label` arithmetic). Mirror ssa_has_unsupported_ops. */ + for (int i = 0; i < n; i++) { + TccIrOp op = ir->compact_instructions[i].op; + if (op == TCCIR_OP_IJUMP || op == TCCIR_OP_SETJMP || op == TCCIR_OP_NL_SETJMP) + return; + } + + /* def_block[t] = the block of t's first def, or -2 = multi-block, -1 = none. */ + int *def_block = tcc_malloc(sizeof(int) * ntmp); + for (int t = 0; t < ntmp; t++) def_block[t] = -1; + + for (int i = 0; i < n; i++) { + int t = ra_def_temp_of(ir, &ir->compact_instructions[i], ntmp); + if (t < 0) continue; + int blk = cfg->instr_to_block[i]; + if (def_block[t] == -1) def_block[t] = blk; + else if (def_block[t] != blk) def_block[t] = -2; /* multi-block */ + } + + /* A multi-block-defined TEMP only needs a phi (and only then is its renaming + * actually wrong) when it has a USE in a block that does not itself define it — + * a value flowing across a merge. A TEMP whose uses are all in its own + * def-blocks reaches each use from the local def and is already correct; + * promoting it would insert needless phi-copies and grow code (96_nodata_wanted + * measures code size via `&&label` arithmetic and is sensitive to this). For + * each multi-block TEMP, mark its def-blocks and require a use elsewhere. */ + int32_t *temp_to_var = tcc_malloc(sizeof(int32_t) * ntmp); + for (int t = 0; t < ntmp; t++) temp_to_var[t] = -1; + uint8_t *needs_phi = tcc_mallocz(ntmp); + { + /* Rescanning every instruction per multi-block TEMP -- once to collect its + * def-blocks, once more to look for a use outside them -- is + * O(multi-block TEMPs x instructions), with an operand decode at every step + * and an isdef[] clear per candidate on top. strlen-5 spent 24.7% of its + * whole compile here. Two passes over the instructions answer the same + * question: the first records every (TEMP, block) pair that is a def, the + * second asks that set about every use. */ + const uint64_t nblk = (uint64_t)cfg->num_blocks; + /* Nothing multi-block means nothing to decide -- and no table to pay for. + * Size it from the number of defs that actually go in, not from n: most + * functions here have a handful, and a table sized for every instruction + * cost more to allocate and zero than it saved (20040709-2). */ + int any_multi = 0; + for (int t = 0; t < ntmp && !any_multi; t++) + any_multi = (def_block[t] == -2); + if (!any_multi) + goto promote_decided; /* nothing to decide: don't walk the instructions */ + + int ndefs = 0; + for (int i = 0; i < n; i++) { + int t = ra_def_temp_of(ir, &ir->compact_instructions[i], ntmp); + if (t >= 0 && def_block[t] == -2) ndefs++; + } + if (!ndefs) + goto promote_decided; + + int dcap = 16; + while (dcap < ndefs * 2) dcap <<= 1; + uint64_t *defset = tcc_mallocz(sizeof(uint64_t) * dcap); + + for (int i = 0; i < n; i++) { + int t = ra_def_temp_of(ir, &ir->compact_instructions[i], ntmp); + if (t < 0 || def_block[t] != -2) continue; + ra_blockset_add(defset, dcap, (uint64_t)t * nblk + (uint64_t)cfg->instr_to_block[i]); + } + + /* a use in a non-def block => needs a phi */ + for (int i = 0; i < n; i++) { + IRQuadCompact *q = &ir->compact_instructions[i]; + if (q->op == TCCIR_OP_NOP) continue; + int32_t uses[5]; int nu = 0; + if (irop_config[q->op].has_src1) uses[nu++] = irop_get_vreg(tcc_ir_op_get_src1(ir, q)); + if (irop_config[q->op].has_src2) uses[nu++] = irop_get_vreg(tcc_ir_op_get_src2(ir, q)); + if (tcc_ir_op_is_mac(q->op)) uses[nu++] = irop_get_vreg(tcc_ir_op_get_accum(ir, q)); + if (q->op == TCCIR_OP_STORE || q->op == TCCIR_OP_STORE_INDEXED || q->op == TCCIR_OP_STORE_POSTINC) + uses[nu++] = irop_get_vreg(tcc_ir_op_get_dest(ir, q)); + if (!nu) continue; + const uint64_t blk = (uint64_t)cfg->instr_to_block[i]; + for (int u = 0; u < nu; u++) { + if (uses[u] < 0 || TCCIR_DECODE_VREG_TYPE(uses[u]) != TCCIR_VREG_TYPE_TEMP) continue; + int t = TCCIR_DECODE_VREG_POSITION(uses[u]); + if (t < 0 || t >= ntmp || def_block[t] != -2 || needs_phi[t]) continue; + if (!ra_blockset_has(defset, dcap, (uint64_t)t * nblk + blk)) + needs_phi[t] = 1; + } + } + tcc_free(defset); + promote_decided:; + } + int any = 0; + for (int t = 0; t < ntmp; t++) { + if (needs_phi[t]) { temp_to_var[t] = tcc_ir_vreg_alloc_var(ir); any = 1; } + } + tcc_free(needs_phi); + if (!any) { tcc_free(def_block); tcc_free(temp_to_var); return; } + + /* Rewrite every operand referencing a promoted TEMP to its VAR (type bits only; + * is_local/is_lval/tag are preserved). */ + #define REMAP(getter, setter) \ + do { \ + IROperand o = getter(ir, q); \ + int32_t ovr = irop_get_vreg(o); \ + if (ovr >= 0 && TCCIR_DECODE_VREG_TYPE(ovr) == TCCIR_VREG_TYPE_TEMP) { \ + int op_t = TCCIR_DECODE_VREG_POSITION(ovr); \ + if (op_t >= 0 && op_t < ntmp && temp_to_var[op_t] >= 0) { \ + irop_set_vreg(&o, temp_to_var[op_t]); \ + setter(ir, q, o); \ + } \ + } \ + } while (0) + + for (int i = 0; i < n; i++) { + IRQuadCompact *q = &ir->compact_instructions[i]; + if (q->op == TCCIR_OP_NOP) continue; + if (irop_config[q->op].has_dest) REMAP(tcc_ir_op_get_dest, tcc_ir_op_set_dest); + if (irop_config[q->op].has_src1) REMAP(tcc_ir_op_get_src1, tcc_ir_op_set_src1); + if (irop_config[q->op].has_src2) REMAP(tcc_ir_op_get_src2, tcc_ir_op_set_src2); + if (tcc_ir_op_is_mac(q->op)) REMAP(tcc_ir_op_get_accum, tcc_ir_op_set_accum); + } + #undef REMAP + + tcc_free(def_block); + tcc_free(temp_to_var); +} + +/* Mark single-def TEMP intervals whose value is a plain int32 constant as + * rematerializable: when such a value gets spilled, machine_op recomputes it + * with `mov reg, #imm` at each use instead of a stack reload — cheaper (no + * memory) and it removes a spill load per use, directly shrinking the + * spill-reload traffic that widens tcc's load count vs gcc. Runs on the final + * post-allocation IR so the recorded immediate matches what codegen sees. */ +static void ra_mark_rematerializable(TCCIRState *ir) +{ + int nt = ir->temporary_variables_live_intervals_size; + if (nt <= 0) + return; + for (int t = 0; t < nt; t++) + ir->temporary_variables_live_intervals[t].remat_kind = 0; + + int n = ir->next_instruction_index; + int *def_count = tcc_mallocz(nt * sizeof(int)); + int *def_idx = tcc_malloc(nt * sizeof(int)); + for (int t = 0; t < nt; t++) + def_idx[t] = -1; + + for (int i = 0; i < n; i++) { + IRQuadCompact *q = &ir->compact_instructions[i]; + if (q->op == TCCIR_OP_NOP || !irop_config[q->op].has_dest) + continue; + int32_t vr = irop_get_vreg(tcc_ir_op_get_dest(ir, q)); + if (vr < 0 || TCCIR_DECODE_VREG_TYPE(vr) != TCCIR_VREG_TYPE_TEMP) + continue; + int p = TCCIR_DECODE_VREG_POSITION(vr); + if (p >= nt) + continue; + def_count[p]++; + def_idx[p] = i; + } + + /* Slot-reading use: an is_lval deref of the value (machine_op reloads it — see + * the use_llocal path); such an appearance needs the spill store to stay. + * Includes the DEST (a deref store `*T = x` reads T as the address). */ + uint8_t *needs_slot = tcc_mallocz(nt); + for (int i = 0; i < n; i++) { + IRQuadCompact *q = &ir->compact_instructions[i]; + if (q->op == TCCIR_OP_NOP) + continue; + for (int slot = 0; slot < 4; slot++) { + IROperand s; + if (slot == 0) { if (!irop_config[q->op].has_dest) continue; s = tcc_ir_op_get_dest(ir, q); } + else if (slot == 1) { if (!irop_config[q->op].has_src1) continue; s = tcc_ir_op_get_src1(ir, q); } + else if (slot == 2) { if (!irop_config[q->op].has_src2) continue; s = tcc_ir_op_get_src2(ir, q); } + else { if (!tcc_ir_op_is_mac(q->op)) continue; s = tcc_ir_op_get_accum(ir, q); } + if (!(s.is_lval && !s.is_local && !s.is_llocal)) + continue; + int32_t vr = irop_get_vreg(s); + if (vr < 0 || TCCIR_DECODE_VREG_TYPE(vr) != TCCIR_VREG_TYPE_TEMP) + continue; + int p = TCCIR_DECODE_VREG_POSITION(vr); + if (p < nt) + needs_slot[p] = 1; + } + } + + for (int t = 0; t < nt; t++) { + if (def_count[t] != 1) + continue; + IRQuadCompact *dq = &ir->compact_instructions[def_idx[t]]; + if (dq->op != TCCIR_OP_ASSIGN) + continue; + IROperand dd = tcc_ir_op_get_dest(ir, dq); + if (dd.is_lval || irop_get_btype(dd) != IROP_BTYPE_INT32) + continue; + IROperand s = tcc_ir_op_get_src1(ir, dq); + if (s.tag != IROP_TAG_IMM32 || s.is_lval || s.is_sym || s.is_local || s.is_llocal) + continue; + IRLiveInterval *iv = &ir->temporary_variables_live_intervals[t]; + iv->remat_kind = 1; + iv->remat_imm = irop_get_imm32(s); + + /* Do NOT drop the def: rematerialization only happens for the one + * machine_op operand shape (spilled + need_lval value read, see + * machine_op.c) — a PARAM src or a plain binop src2 of the spilled temp + * takes the slot/frame path instead and would read garbage once the + * def's mov+store is gone (fuzz seed longlong:6393; the LLONG + * pair classes raised pressure enough to spill the const). The + * redundant mov+store for a remat'd spilled constant is noise compared + * to wrong code. */ + (void)needs_slot; + } + + tcc_free(needs_slot); + tcc_free(def_count); + tcc_free(def_idx); +} + +/* ssa:cfg_cleanup body — see the call site for why it runs where it does. */ +static int ra_cfg_cleanup(TCCIRState *ir) +{ + const IRPassGroup *groups; + int group_count; + int total = 0; + tcc_ir_opt_get_pipeline(IR_OPT_LEVEL_2, &groups, &group_count); + const IRPassGroup *cleanup_group = &groups[group_count - 1]; + for (int outer = 0; outer < 3; outer++) { + int ch = 0; + for (int i = 0; i < 8; i++) { + int c = tcc_ir_opt_jump_threading(ir); + /* Inside the cascade, not beside it: retargeting the constant arm's edge + * turns the true arm's trailing JUMP into a fall-through and orphans the + * merge label, and only eliminate_fallthrough + dce below can collect + * that -- which is what exposes `T <-- (cond)` to setif fusion. */ + c += tcc_ir_opt_bool_diamond_branch(ir); + c += tcc_ir_opt_eliminate_fallthrough(ir); + c += tcc_ir_opt_jumpif_invert(ir, 0); + if (c) + tcc_ir_opt_compact_nops(ir); + if (tcc_state->opt_dce) { + c += tcc_ir_opt_orphan_cmp_elim(ir); + c += tcc_ir_opt_dce(ir); + } + ch += c; + if (!c) + break; + } + if (outer == 0 && !ch) + break; + IROptCtx cl_ctx; + tcc_ir_opt_ctx_init(&cl_ctx, ir); + if (tcc_state->opt_dead_store) + ch += tcc_ir_opt_gens_call_result_ex(&cl_ctx); + ch += tcc_ir_opt_run_group(&cl_ctx, cleanup_group); + tcc_ir_opt_ctx_free(&cl_ctx); + total += ch; + if (!ch) + break; + } + return total; +} + +static int ra_split_flag_reader(TccIrOp op) +{ + return op == TCCIR_OP_JUMPIF || op == TCCIR_OP_SETIF || op == TCCIR_OP_SELECT || op == TCCIR_OP_ADC_USE || + op == TCCIR_OP_SUBC_USE; +} + +static int ra_split_flag_setter(TccIrOp op) +{ + return op == TCCIR_OP_CMP || op == TCCIR_OP_TEST_ZERO || op == TCCIR_OP_FCMP || op == TCCIR_OP_ADC_GEN || + op == TCCIR_OP_SUBC_GEN; +} + +/* A copy lowers to MOV/LDR/STR; a 16-bit MOVS would clobber the flags, so no + * copy goes where a later flag reader still needs them. */ +static int ra_split_flags_live_before(TCCIRState *ir, int i) +{ + int n = ir->next_instruction_index; + for (int k = i; k < n; k++) { + IRQuadCompact *q = &ir->compact_instructions[k]; + if (k > i && q->is_jump_target) + return 0; + if (ra_split_flag_reader(q->op)) + return 1; + if (ra_split_flag_setter(q->op)) + return 0; + if (q->op == TCCIR_OP_JUMP || q->op == TCCIR_OP_RETURNVALUE || q->op == TCCIR_OP_RETURNVOID) + return 0; + } + return 0; +} + +/* Ops a split value must not appear in: operands beyond dest/src1/src2/accum, + * operands the op also writes, register pairs, or positions the backend pins. */ +static int ra_split_op_excluded(TccIrOp op) +{ + switch (op) { + case TCCIR_OP_LOAD_POSTINC: case TCCIR_OP_STORE_POSTINC: case TCCIR_OP_INLINE_ASM: + case TCCIR_OP_ASM_INPUT: case TCCIR_OP_ASM_OUTPUT: case TCCIR_OP_CALLSEQ_BEGIN: + case TCCIR_OP_CALLARG_REG: case TCCIR_OP_CALLARG_STACK: case TCCIR_OP_CALLSEQ_END: + case TCCIR_OP_SWITCH_TABLE: case TCCIR_OP_SWITCH_LOAD: case TCCIR_OP_IJUMP: + case TCCIR_OP_BLOCK_COPY: case TCCIR_OP_VLA_ALLOC: case TCCIR_OP_VLA_SP_SAVE: + case TCCIR_OP_VLA_SP_RESTORE: case TCCIR_OP_SETJMP: case TCCIR_OP_LONGJMP: + case TCCIR_OP_NL_SETJMP: case TCCIR_OP_NL_LONGJMP: case TCCIR_OP_BUILTIN_APPLY_ARGS: + case TCCIR_OP_BUILTIN_APPLY: case TCCIR_OP_BUILTIN_RETURN: case TCCIR_OP_SET_CHAIN: + case TCCIR_OP_INIT_CHAIN_SLOT: case TCCIR_OP_UMAAL: case TCCIR_OP_UMULL: case TCCIR_OP_SMULL: + case TCCIR_OP_PACK64: case TCCIR_OP_ZEXT: case TCCIR_OP_BFI: case TCCIR_OP_ADC_USE: + case TCCIR_OP_ADC_GEN: case TCCIR_OP_SUBC_USE: case TCCIR_OP_SUBC_GEN: case TCCIR_OP_RETURN_ADDRESS: + return 1; + default: + return 0; + } +} + +static int ra_split_is_call(TCCIRState *ir, IRQuadCompact *q) +{ + TccIrOp op = q->op; + if (op == TCCIR_OP_FUNCCALLVOID || op == TCCIR_OP_FUNCCALLVAL || op == TCCIR_OP_BUILTIN_APPLY || + ir_op_is_implicit_call_ra(op)) + return 1; + if (op == TCCIR_OP_BLOCK_COPY) + return (int)irop_get_imm64_ex(ir, tcc_ir_op_get_src2(ir, q)) >= TCCIR_BLOCK_COPY_MEMCPY_MIN_BYTES; + return 0; +} + +/* How an operand naming a split candidate reads it, and what the fresh temp's + * operand becomes. A promoted VAR is referenced either as a plain vreg or + * through its spill encoding (STACKOFF, is_local + is_lval = its VALUE); a + * plain vreg with is_lval dereferences the value. Returns 0 for shapes a + * temp cannot stand in for (the var's address, double indirection). */ +static int ra_split_use_operand(IROperand s, IROperand *out, int32_t to) +{ + if (s.is_llocal || s.btype == IROP_BTYPE_STRUCT || s.is_complex) + return 0; + IROperand t = s; + if (s.is_local) { + if (!s.is_lval) + return 0; /* address of the var's home */ + t.is_local = 0; + t.is_lval = 0; + } else if (s.tag != IROP_TAG_VREG) + return 0; + t.tag = IROP_TAG_VREG; + t.is_param = 0; + t.u.imm32 = 0; + irop_set_vreg(&t, to); + if (out) + *out = t; + return 1; +} + +/* Rename every USE of `from` in q (the slots ra_co_ops reads) to `to`. */ +static void ra_split_rename_uses(TCCIRState *ir, IRQuadCompact *q, int32_t from, int32_t to) +{ + IROperand s, t; + if (irop_config[q->op].has_src1) { + s = tcc_ir_op_get_src1(ir, q); + if (irop_has_vreg(s) && !irop_is_immediate(s) && irop_get_vreg(s) == from && ra_split_use_operand(s, &t, to)) + tcc_ir_op_set_src1(ir, q, t); + } + if (irop_config[q->op].has_src2) { + s = tcc_ir_op_get_src2(ir, q); + if (irop_has_vreg(s) && !irop_is_immediate(s) && irop_get_vreg(s) == from && ra_split_use_operand(s, &t, to)) + tcc_ir_op_set_src2(ir, q, t); + } + if (tcc_ir_op_is_mac(q->op)) { + s = tcc_ir_op_get_accum(ir, q); + if (irop_has_vreg(s) && !irop_is_immediate(s) && irop_get_vreg(s) == from && ra_split_use_operand(s, &t, to)) + tcc_ir_op_set_accum(ir, q, t); + } + if (q->op == TCCIR_OP_STORE || q->op == TCCIR_OP_STORE_INDEXED) { + s = tcc_ir_op_get_dest(ir, q); + if (irop_has_vreg(s) && irop_get_vreg(s) == from && ra_split_use_operand(s, &t, to)) + tcc_ir_op_set_dest(ir, q, t); + } +} + +/* Make q, the single-register def of a candidate, write `to` instead. A + * STORE-form def (a promoted VAR's spill-encoded store, or a temp's in-place + * store) becomes the ASSIGN it stands for: same dest/src1 layout. */ +static void ra_split_rename_def(TCCIRState *ir, IRQuadCompact *q, int32_t to) +{ + IROperand d = tcc_ir_op_get_dest(ir, q); + if (q->op == TCCIR_OP_STORE) { + IROperand t; + memset(&t, 0, sizeof t); + t.tag = IROP_TAG_VREG; + t.btype = IROP_BTYPE_INT32; + irop_set_vreg(&t, to); + q->op = TCCIR_OP_ASSIGN; + tcc_ir_op_set_dest(ir, q, t); + return; + } + irop_set_vreg(&d, to); + tcc_ir_op_set_dest(ir, q, d); +} + +/* Insert the planned copies (sorted by `before`, skip-copies first) into the + * instruction stream, remapping branch and switch targets. */ +static void ra_split_insert(TCCIRState *ir, RaSplitCopy *cp, int ncp) +{ + int old_n = ir->next_instruction_index; + int new_cap = old_n + ncp + 16; + IRQuadCompact *ni = tcc_mallocz(new_cap * sizeof(IRQuadCompact)); + int *old_to_new = tcc_malloc((old_n + 1) * sizeof(int)); + while (ir->iroperand_pool_count + 2 * ncp > ir->iroperand_pool_capacity) { + int nc = ir->iroperand_pool_capacity ? ir->iroperand_pool_capacity * 2 : 256; + ir->iroperand_pool = tcc_realloc(ir->iroperand_pool, nc * sizeof(IROperand)); + ir->iroperand_pool_capacity = nc; + } + int wp = 0, c = 0; + for (int i = 0; i <= old_n; i++) { + int landing = -1; + int line = i < old_n ? ir->compact_instructions[i].line_num : 0; + for (; c < ncp && cp[c].before == i; c++) { + IROperand d, s; + memset(&d, 0, sizeof d); + memset(&s, 0, sizeof s); + irop_set_vreg(&d, cp[c].dest); + d.tag = IROP_TAG_VREG; + d.btype = cp[c].btype; + d.is_unsigned = cp[c].is_unsigned; + irop_set_vreg(&s, cp[c].src); + s.tag = IROP_TAG_VREG; + s.btype = cp[c].btype; + s.is_unsigned = cp[c].is_unsigned; + int pb = ir->iroperand_pool_count; + ir->iroperand_pool[pb] = d; + ir->iroperand_pool[pb + 1] = s; + ir->iroperand_pool_count += 2; + if (cp[c].land && landing < 0) + landing = wp; + ni[wp].op = TCCIR_OP_ASSIGN; + ni[wp].operand_base = pb; + ni[wp].line_num = line; + ni[wp].orig_index = ++ir->max_orig_index; + wp++; + } + if (i == old_n) { + old_to_new[i] = landing >= 0 ? landing : wp; + break; + } + ni[wp] = ir->compact_instructions[i]; + old_to_new[i] = landing >= 0 ? landing : wp; + if (landing >= 0 && ni[wp].is_jump_target) { + ni[wp].is_jump_target = 0; + ni[landing].is_jump_target = 1; + } + wp++; + } + for (int i = 0; i < wp; i++) { + IRQuadCompact *q = &ni[i]; + if (q->op != TCCIR_OP_JUMP && q->op != TCCIR_OP_JUMPIF) + continue; + IROperand dest = ir->iroperand_pool[q->operand_base]; + int t = (int)irop_get_imm64_ex(ir, dest); + if (t >= 0 && t <= old_n) { + dest.u.imm32 = old_to_new[t]; + ir->iroperand_pool[q->operand_base] = dest; + } + } + for (int t = 0; t < ir->num_switch_tables; t++) { + TCCIRSwitchTable *table = &ir->switch_tables[t]; + for (int ti = 0; ti < table->num_entries; ti++) + if (table->targets[ti] >= 0 && table->targets[ti] <= old_n) + table->targets[ti] = old_to_new[table->targets[ti]]; + if (table->default_target >= 0 && table->default_target <= old_n) + table->default_target = old_to_new[table->default_target]; + } + tcc_free(ir->compact_instructions); + ir->compact_instructions = ni; + ir->compact_instructions_size = new_cap; + ir->next_instruction_index = wp; + tcc_free(old_to_new); +} + +static int ra_split_copy_cmp(const void *a, const void *b) +{ + const RaSplitCopy *x = a, *y = b; + if (x->before != y->before) + return x->before < y->before ? -1 : 1; + if (x->land != y->land) + return x->land - y->land; /* skip-copies (end of the previous block) first */ + return 0; +} + +/* The def of candidate `v` at q, or 0 when q does not define it. 2: a + * STORE-form def (rewritten to ASSIGN when split). -1: a def shape the split + * cannot rename (narrow, address, pair, excluded op). */ +static int ra_split_def_kind(TCCIRState *ir, IRQuadCompact *q, int32_t v) +{ + if (!irop_config[q->op].has_dest) + return 0; + IROperand d = tcc_ir_op_get_dest(ir, q); + if (!irop_has_vreg(d) || irop_get_vreg(d) != v) + return 0; + int vt = TCCIR_DECODE_VREG_TYPE(v); + if (q->op == TCCIR_OP_STORE) { + if (irop_get_btype(d) != IROP_BTYPE_INT32 || d.is_complex) + return -1; + /* promoted VAR through its spill encoding: STORE StackLoc(V) = V's def */ + if (vt == TCCIR_VREG_TYPE_VAR && d.tag == IROP_TAG_STACKOFF && d.is_local && d.is_lval && !d.is_llocal) + return 2; + /* a temp's in-place def (SSA renaming of a promoted var's slot store) */ + if (vt == TCCIR_VREG_TYPE_TEMP && d.tag == IROP_TAG_VREG && !d.is_lval && !d.is_local && !d.is_llocal) + return 2; + return 0; /* a store THROUGH the value: a use, classified by the caller */ + } + if (q->op == TCCIR_OP_STORE_INDEXED || q->op == TCCIR_OP_STORE_POSTINC) + return 0; + int bt = irop_get_btype(d); + if (ra_split_op_excluded(q->op) || d.tag != IROP_TAG_VREG || d.is_lval || d.is_local || d.is_llocal || + d.is_complex || !(bt == IROP_BTYPE_INT32 || bt == IROP_BTYPE_INT8 || bt == IROP_BTYPE_INT16)) + return -1; + return 1; +} + +/* Returns the number of IR edits (0: IR untouched). */ +static int ra_split_spilled(TCCIRState *ir, IRCFG *cfg, SSAInterval *intervals, int count, int max_vreg_pos, + const int *call_prefix) +{ + int n = ir->next_instruction_index; + (void)cfg; /* may predate the flat edits after phi resolution; built fresh below */ + if (n <= 0 || !call_prefix) + return 0; + if (ir->func_has_label_addr || ir->inline_asm_count > 0 || tcc_ir_calls_returns_twice(ir)) + return 0; + + int tbl = 4 * (max_vreg_pos > 0 ? max_vreg_pos : 1); + int *cand = tcc_malloc(sizeof(int) * tbl); + for (int i = 0; i < tbl; i++) + cand[i] = -1; + int ncand = 0; + int32_t *cvreg = tcc_malloc(sizeof(int32_t) * (count > 0 ? count : 1)); +#define SPLIT_IDX(vr) (TCCIR_DECODE_VREG_TYPE(vr) * max_vreg_pos + TCCIR_DECODE_VREG_POSITION(vr)) +#define SPLIT_CAND(vr) ((vr) >= 0 && SPLIT_IDX(vr) >= 0 && SPLIT_IDX(vr) < tbl ? cand[SPLIT_IDX(vr)] : -1) + for (int i = 0; i < count; i++) { + SSAInterval *iv = &intervals[i]; + if (iv->stack_location == 0 || iv->r0 >= 0 || iv->addrtaken || iv->is_volatile || iv->is_param || + iv->reg_type != LS_REG_TYPE_INT || iv->precolored >= 0 || iv->coalesce_to >= 0 || iv->co_member) + continue; + int t = TCCIR_DECODE_VREG_TYPE(iv->vreg); + if (t != TCCIR_VREG_TYPE_TEMP && t != TCCIR_VREG_TYPE_VAR) + continue; + IRLiveInterval *li = tcc_ir_vreg_live_interval(ir, iv->vreg); + if (!li || li->addrtaken || li->is_volatile || li->is_lvalue || tcc_ir_vreg_is_ignored(ir, iv->vreg)) + continue; + int idx = SPLIT_IDX(iv->vreg); + if (idx < 0 || idx >= tbl) + continue; + cand[idx] = ncand; + cvreg[ncand++] = iv->vreg; + } + if (ncand == 0) { + tcc_free(cand); + tcc_free(cvreg); + return 0; + } + cfg = tcc_ir_cfg_build(ir); + if (!cfg || !cfg->instr_to_block || cfg->num_instrs < n) { + if (cfg) + tcc_ir_cfg_free(cfg); + tcc_free(cand); + tcc_free(cvreg); + return 0; + } + tcc_ir_cfg_compute_dominators(cfg); + + uint8_t *bad = tcc_mallocz(ncand); + int *nuse = tcc_mallocz(sizeof(int) * (ncand + 1)); + int *ndef = tcc_mallocz(sizeof(int) * (ncand + 1)); + + /* Paired CALL of every FUNCPARAMVAL (see ra_build_intervals). */ + int *param_call = tcc_malloc(sizeof(int) * n); + { + int max_cid = -1; + for (int i = 0; i < n; i++) { + IRQuadCompact *q = &ir->compact_instructions[i]; + if (q->op == TCCIR_OP_FUNCCALLVOID || q->op == TCCIR_OP_FUNCCALLVAL) { + int ccid = TCCIR_DECODE_CALL_ID(irop_get_imm64_ex(ir, tcc_ir_op_get_src2(ir, q))); + if (ccid > max_cid) + max_cid = ccid; + } + } + int *next_call = max_cid >= 0 ? tcc_malloc(sizeof(int) * (max_cid + 1)) : NULL; + for (int k = 0; k <= max_cid; k++) + next_call[k] = -1; + for (int i = n - 1; i >= 0; i--) { + IRQuadCompact *q = &ir->compact_instructions[i]; + param_call[i] = -1; + if (q->op == TCCIR_OP_FUNCCALLVOID || q->op == TCCIR_OP_FUNCCALLVAL) { + int ccid = TCCIR_DECODE_CALL_ID(irop_get_imm64_ex(ir, tcc_ir_op_get_src2(ir, q))); + if (ccid >= 0 && ccid <= max_cid) + next_call[ccid] = i; + } else if (q->op == TCCIR_OP_FUNCPARAMVAL && next_call) { + int cid = TCCIR_DECODE_CALL_ID(irop_get_imm64_ex(ir, tcc_ir_op_get_src2(ir, q))); + if (cid >= 0 && cid <= max_cid) + param_call[i] = next_call[cid]; + } + } + if (next_call) + tcc_free(next_call); + } + + /* Pass 1: count uses and defs; disqualify shapes a temp can't replace. */ + for (int i = 0; i < n; i++) { + IRQuadCompact *q = &ir->compact_instructions[i]; + if (q->op == TCCIR_OP_NOP) + continue; + int excluded = ra_split_op_excluded(q->op); + IROperand ops[4]; + int no = 0; + if (irop_config[q->op].has_src1) ops[no++] = tcc_ir_op_get_src1(ir, q); + if (irop_config[q->op].has_src2) ops[no++] = tcc_ir_op_get_src2(ir, q); + if (tcc_ir_op_is_mac(q->op)) ops[no++] = tcc_ir_op_get_accum(ir, q); + for (int k = 0; k < no; k++) { + if (!irop_has_vreg(ops[k]) || irop_is_immediate(ops[k])) + continue; + int c = SPLIT_CAND(irop_get_vreg(ops[k])); + if (c < 0) + continue; + nuse[c]++; + if (!bad[c] && (excluded || !ra_split_use_operand(ops[k], NULL, 0))) + bad[c] = 1; + } + if (!irop_config[q->op].has_dest) + continue; + IROperand d = tcc_ir_op_get_dest(ir, q); + if (!irop_has_vreg(d)) + continue; + int32_t dv = irop_get_vreg(d); + int c = SPLIT_CAND(dv); + if (c < 0) + continue; + int dk = ra_split_def_kind(ir, q, dv); + if (dk > 0) { + ndef[c]++; + } else if (dk < 0) { + bad[c] = 1; + } else { + /* A store through the value. STORE_INDEXED's base is a use for a + * temp; for a VAR ra_build_intervals reads it as a def, so leave it. */ + int ok = (q->op == TCCIR_OP_STORE && TCCIR_DECODE_VREG_TYPE(dv) == TCCIR_VREG_TYPE_TEMP && d.is_lval && + !d.is_local && !d.is_llocal && d.tag == IROP_TAG_VREG) || + (q->op == TCCIR_OP_STORE_INDEXED && TCCIR_DECODE_VREG_TYPE(dv) == TCCIR_VREG_TYPE_TEMP && + d.tag == IROP_TAG_VREG && !d.is_local && !d.is_llocal); + nuse[c]++; + if (!ok) + bad[c] = 1; + } + } + + /* Use and def lists in instruction order, bucketed by candidate. */ + int total = 0, dtotal = 0; + for (int c = 0; c < ncand; c++) { + int k = nuse[c]; + nuse[c] = total; + total += k; + k = ndef[c]; + ndef[c] = dtotal; + dtotal += k; + } + nuse[ncand] = total; + ndef[ncand] = dtotal; + RaSplitUse *uses = tcc_malloc(sizeof(RaSplitUse) * (total > 0 ? total : 1)); + int *defs = tcc_malloc(sizeof(int) * (dtotal > 0 ? dtotal : 1)); + int *ufill = tcc_malloc(sizeof(int) * ncand); + int *dfill = tcc_malloc(sizeof(int) * ncand); + for (int c = 0; c < ncand; c++) { + ufill[c] = nuse[c]; + dfill[c] = ndef[c]; + } + for (int i = 0; i < n; i++) { + IRQuadCompact *q = &ir->compact_instructions[i]; + if (q->op == TCCIR_OP_NOP) + continue; + int e = (q->op == TCCIR_OP_FUNCPARAMVAL && param_call[i] >= 0) ? param_call[i] : i; + IROperand ops[4]; + int no = 0; + if (irop_config[q->op].has_src1) ops[no++] = tcc_ir_op_get_src1(ir, q); + if (irop_config[q->op].has_src2) ops[no++] = tcc_ir_op_get_src2(ir, q); + if (tcc_ir_op_is_mac(q->op)) ops[no++] = tcc_ir_op_get_accum(ir, q); + for (int k = 0; k < no; k++) { + if (!irop_has_vreg(ops[k]) || irop_is_immediate(ops[k])) + continue; + int c = SPLIT_CAND(irop_get_vreg(ops[k])); + if (c < 0) + continue; + uses[ufill[c]].instr = i; + uses[ufill[c]++].eff = e; + } + if (!irop_config[q->op].has_dest) + continue; + IROperand d = tcc_ir_op_get_dest(ir, q); + if (!irop_has_vreg(d)) + continue; + int32_t dv = irop_get_vreg(d); + int c = SPLIT_CAND(dv); + if (c < 0) + continue; + int dk = ra_split_def_kind(ir, q, dv); + if (dk > 0) + defs[dfill[c]++] = i; + else if (dk == 0) { + uses[ufill[c]].instr = i; + uses[ufill[c]++].eff = e; + } + } + + /* maxsrc[t]: furthest backward jump landing on t; jtp: jump-target prefix. */ + int *maxsrc = tcc_mallocz(sizeof(int) * (n + 1)); + int *jtp = tcc_malloc(sizeof(int) * (n + 2)); + jtp[0] = 0; + for (int i = 0; i < n; i++) { + IRQuadCompact *q = &ir->compact_instructions[i]; + jtp[i + 1] = jtp[i] + (q->is_jump_target ? 1 : 0); + if (q->op != TCCIR_OP_JUMP && q->op != TCCIR_OP_JUMPIF) + continue; + int t = (int)irop_get_imm64_ex(ir, tcc_ir_op_get_dest(ir, q)); + if (t >= 0 && t <= i && i > maxsrc[t]) + maxsrc[t] = i; + } + jtp[n + 1] = jtp[n]; + + RaSplitCopy *cps = NULL; + int ncp = 0, cap = 0; + uint8_t *grouped = tcc_mallocz(total > 0 ? total : 1); + int dead_stores = 0; + + /* Call-free, and no loop that could leave the range past its end and come back. */ +#define SPLIT_REGION_OK(start, e) \ + ({ \ + int ok_ = (e) >= (start) && call_prefix[(e)] == call_prefix[(start)] && (e) - (start) < 4096; \ + for (int t_ = (start); ok_ && t_ <= (e); t_++) \ + if (maxsrc[t_] > (e)) \ + ok_ = 0; \ + ok_; \ + }) + /* No jump target in (a, b]: b is reached from a only by falling through. */ +#define SPLIT_STRAIGHT(a, b) (jtp[(b) + 1] - jtp[(a) + 1] == 0) +#define SPLIT_PUSH(b, l, dv, sv, bt, uns) \ + do { \ + if (ncp == cap) { \ + cap = cap ? 2 * cap : 64; \ + cps = tcc_realloc(cps, sizeof(RaSplitCopy) * cap); \ + } \ + cps[ncp].before = (b); \ + cps[ncp].land = (l); \ + cps[ncp].dest = (dv); \ + cps[ncp].src = (sv); \ + cps[ncp].btype = (bt); \ + cps[ncp].is_unsigned = (uns); \ + ncp++; \ + } while (0) + + for (int c = 0; c < ncand; c++) { + int lo = nuse[c], hi = nuse[c + 1], dlo = ndef[c], dhi = ndef[c + 1]; + int nd = dhi - dlo; + if (bad[c] || nd == 0 || hi - lo < 1) + continue; + int32_t V = cvreg[c]; + int single = nd == 1; + + /* A def of V strictly between a and b (linear)? */ +#define SPLIT_DEF_BETWEEN(a, b) \ + ({ \ + int f_ = 0; \ + for (int x_ = dlo; x_ < dhi && !f_; x_++) \ + if (defs[x_] > (a) && defs[x_] < (b)) \ + f_ = 1; \ + f_; \ + }) + + /* Def groups: uses the def reaches in a call-free region read a temp the + * def now writes. */ + for (int x = dlo; x < dhi; x++) { + int d = defs[x], db = cfg->instr_to_block[d]; + if (db < 0 || d + 1 >= n || ra_split_flags_live_before(ir, d + 1)) + continue; + int start = ra_split_is_call(ir, &ir->compact_instructions[d]) ? d + 1 : d; + int members = 0, rest = 0; + for (int k = lo; k < hi; k++) { + if (grouped[k]) + continue; + int ui = uses[k].instr, ub = cfg->instr_to_block[ui]; + int reach = ui > d && ub >= 0 && + (single ? (ub == db || tcc_ir_cfg_dominates(cfg, db, ub)) + : (SPLIT_STRAIGHT(d, ui) && !SPLIT_DEF_BETWEEN(d, ui))); + if (reach && SPLIT_REGION_OK(start, uses[k].eff)) + members++; + else + rest++; + } + if (members == 0) + continue; + int32_t T = tcc_ir_vreg_alloc_temp(ir); + if (T < 0) + break; + int last = -1; + for (int k = lo; k < hi; k++) { + if (grouped[k]) + continue; + int ui = uses[k].instr, ub = cfg->instr_to_block[ui]; + int reach = ui > d && ub >= 0 && + (single ? (ub == db || tcc_ir_cfg_dominates(cfg, db, ub)) + : (SPLIT_STRAIGHT(d, ui) && !SPLIT_DEF_BETWEEN(d, ui))); + if (!(reach && SPLIT_REGION_OK(start, uses[k].eff))) + continue; + grouped[k] = 1; + if (ui != last) { + ra_split_rename_uses(ir, &ir->compact_instructions[ui], V, T); + last = ui; + } + } + /* The spill store of a narrow def extends it into the slot; the copy + * carries the def's width so it still does. */ + IROperand dop = tcc_ir_op_get_dest(ir, &ir->compact_instructions[d]); + int dbt = ir->compact_instructions[d].op == TCCIR_OP_STORE ? IROP_BTYPE_INT32 : irop_get_btype(dop); + int duns = dbt == IROP_BTYPE_INT32 ? 0 : dop.is_unsigned; + ra_split_rename_def(ir, &ir->compact_instructions[d], T); + if (!single || rest > 0) + SPLIT_PUSH(d + 1, 0, V, T, dbt, duns); + else + dead_stores++; + } + + /* Reload groups: uses after a first use of V in one call-free region share + * its reload. Single def: the def dominates the leader and the leader the + * members. Several defs: straight-line, no def of V in [leader, member). */ + for (int k = lo; k < hi; k++) { + if (grouped[k]) + continue; + int u0 = uses[k].instr, b0 = cfg->instr_to_block[u0]; + if (b0 < 0) + continue; + if (single) { + int d = defs[dlo], db = cfg->instr_to_block[d]; + if (db < 0 || !(b0 == db ? d < u0 : tcc_ir_cfg_dominates(cfg, db, b0))) + continue; + } + if (ra_split_flags_live_before(ir, u0)) + continue; + int distinct = 0, last = -1; +#define SPLIT_MEMBER(j) \ + ({ \ + int ui_ = uses[(j)].instr, ub_ = cfg->instr_to_block[ui_]; \ + (j) == k || (ub_ >= 0 && ui_ >= u0 && \ + (single ? (ub_ == b0 || tcc_ir_cfg_dominates(cfg, b0, ub_)) \ + : (SPLIT_STRAIGHT(u0, ui_) && !SPLIT_DEF_BETWEEN(u0 - 1, ui_))) && \ + SPLIT_REGION_OK(u0, uses[(j)].eff)); \ + }) + for (int j = k; j < hi; j++) { + if (grouped[j] || !SPLIT_MEMBER(j)) + continue; + if (uses[j].instr != last) { + distinct++; + last = uses[j].instr; + } + } + if (distinct < 2) + continue; + int32_t T = tcc_ir_vreg_alloc_temp(ir); + if (T < 0) + break; + last = -1; + for (int j = k; j < hi; j++) { + if (grouped[j] || !SPLIT_MEMBER(j)) + continue; + grouped[j] = 1; + if (uses[j].instr != last) { + ra_split_rename_uses(ir, &ir->compact_instructions[uses[j].instr], V, T); + last = uses[j].instr; + } + } +#undef SPLIT_MEMBER + SPLIT_PUSH(u0, 1, T, V, IROP_BTYPE_INT32, 0); + } +#undef SPLIT_DEF_BETWEEN + } +#undef SPLIT_REGION_OK +#undef SPLIT_STRAIGHT +#undef SPLIT_PUSH +#undef SPLIT_CAND +#undef SPLIT_IDX + + int changed = ncp + dead_stores; + if (ncp > 0) { + qsort(cps, ncp, sizeof(RaSplitCopy), ra_split_copy_cmp); + ra_split_insert(ir, cps, ncp); + } + tcc_free(cps); + tcc_free(grouped); + tcc_free(maxsrc); + tcc_free(jtp); + tcc_free(uses); + tcc_free(defs); + tcc_free(ufill); + tcc_free(dfill); + tcc_free(param_call); + tcc_free(bad); + tcc_free(nuse); + tcc_free(ndef); + tcc_free(cand); + tcc_free(cvreg); + tcc_ir_cfg_free(cfg); + return changed; +} + +void tcc_ir_ssa_regalloc(TCCIRState *ir, const RegAllocTarget *target, int spill_base) +{ + if (!ir || !target) return; + int ra_ch; + RAPhiStats phi_stats = {0}; + dbg_scan_overlap(ir, "ssa_regalloc_entry"); + + /* ssa:mem_init — frontend memory-init lowering (memset(0)+const stores → + * rodata BLOCK_COPY; small stack/global zero-memset → direct STORE #0). + * Runs first on raw flat IR, before cfg_cleanup and the loop transforms + * below, matching its former position at the head of the tccgen.c pipeline. + * Ungated by -O (the lowering also fires at -O0); knob + * TCC_DISABLE_PASS=ssa:mem_init. See docs/plan_legacy_flat_ir_ssa_retire.md. */ + /* sret_nrvo -- a struct returned through the hidden pointer is built in the + * caller's buffer instead of a local copied out at the return. Before + * mem_init, which turns its initializer into a BLOCK_COPY that only writes + * the frame. */ + RA_FLAT_PASS(ra_ch, "sret_nrvo", tcc_state->optimize > 0, tcc_ir_opt_sret_nrvo(ir)); + RA_FLAT_PASS(ra_ch, "ssa:mem_init", 1, ssa_opt_mem_init(ir)); + + /* ssa:cfg_cleanup — GCC-style cleanup_cfg at SSA-pipeline entry: thread jump + * chains / drop fall-through jumps / NOP orphan flag-setters left by the flat + * pipeline, so the flat-region loop transforms below (reroll/licm/loop_rotate) + * see normalized control flow instead of trampoline chains they would rotate + * around and duplicate tests for. When the cascade fired, re-run call-result + * demotion + the flat late_cleanup group (self-gated passes): collapsed + * diamonds expose dead pure calls / VLA allocs / stores those passes could + * not see on their earlier run. */ + RA_FLAT_PASS(ra_ch, "ssa:cfg_cleanup", + tcc_state->optimize >= 1 && tcc_state->opt_jump_threading, + ra_cfg_cleanup(ir)); + + /* ssa:struct_copy_roundtrip — drop the memmove(B,A);memmove(A,B) pair left + * by an inlined identity `y = retme(y)` helper. The matcher needs the two + * copies in one straight line; the inline expansion leaves a fall-through + * JMP + target marker between them that only cfg_cleanup removes, so the + * tccgen-time call misses the pattern (20040709-2 fn1* family). */ + RA_FLAT_PASS(ra_ch, "ssa:struct_copy_roundtrip", tcc_state->opt_redundant_store, + tcc_ir_opt_struct_copy_roundtrip_elim(ir)); + + /* ssa:or_bool_diamond — fold `acc |= (cond ? 1 : 0)` stack-slot + * materialization into per-arm ORs. Needs the STORE-slot/OR adjacency that + * cfg_cleanup's eliminate_fallthrough just created. Gate matches the legacy + * tccgen.c call (opt_const_prop); knob TCC_DISABLE_PASS=ssa:or_bool_diamond. */ + RA_FLAT_PASS(ra_ch, "ssa:or_bool_diamond", tcc_state->opt_const_prop, + ssa_opt_or_bool_diamond(ir)); + + /* ssa:stack_addr_simplify — deref-of-known-stack-addr → direct StackLoc; legacy gate; knob TCC_DISABLE_PASS=ssa:stack_addr_simplify */ + RA_FLAT_PASS(ra_ch, "ssa:stack_addr_simplify", tcc_state->opt_const_prop, + tcc_ir_opt_stack_addr_simplify(ir)); + + /* ssa:reroll — re-roll runs of identical macro-unrolled blocks into a counted + * loop. Runs first in the flat region (preserving reroll's legacy "earliest + * loop transform" order) but now post-propagation: foldable runs are already + * collapsed by downstream const-prop, so only non-foldable repetition survives + * to re-roll, which fixes the legacy pre-propagation counterproductivity. + * Gated opt_reroll (-O2); knob TCC_DISABLE_PASS=ssa:reroll. + * See docs/plan_legacy_loop_reroll_ssa.md. */ + /* compact the (N-1)*P NOPs a successful re-roll leaves so CFG/SSA below don't iterate them */ + RA_FLAT_PASS(ra_ch, "ssa:reroll", tcc_state->opt_reroll, + ssa_opt_reroll(ir) ? (tcc_ir_opt_compact_nops(ir), 1) : 0); + + /* ssa:licm — hoist loop invariants (arithmetic + pure/const calls) to the + * preheader before the other loop transforms and ssa:iv_strength_reduction, + * preserving the legacy "LICM before IV-SR" order now that both left tccgen. + * Reuses the proven licm.c engine; gated opt_licm (-O2); knob + * TCC_DISABLE_PASS=ssa:licm. */ + RA_FLAT_PASS(ra_ch, "ssa:licm", tcc_state->opt_licm, ssa_opt_licm(ir)); + + /* Loop rotation (ssa:loop_rotate): convert safe top-tested loops to + * bottom-tested on flat IR, before the CFG/SSA below are built, so the + * downstream SSA passes and regalloc see the rotated shape. CFG/dominator + * based natural-loop detection driving the proven flat-IR rewrite; gated to + * -O1+ (matching the legacy pass and the SSA opt tier) and disableable via + * TCC_DISABLE_PASS=ssa:loop_rotate. */ + RA_FLAT_PASS(ra_ch, "ssa:loop_rotate", tcc_state->optimize >= 1, + ssa_opt_loop_rotate(ir)); + + /* ssa:licm ran BEFORE rotation and so saw the un-rotated header/latch/body + * shape, whose preheader is not the header's immediate predecessor — the + * invariant-global-load hoist's insertion safety demands exactly that, so it + * declined every ordinary `for` loop. Rotation provides the shape; give the + * hoist a second chance on it. */ + RA_FLAT_PASS(ra_ch, "ssa:licm_global_load", tcc_state->opt_licm, + ssa_opt_licm_global_load(ir)); + + /* Zero-trip guard elimination (ssa:loop_guard_elim): drop the pre-loop + * `CMP iv,#lim / JUMPIF` that rotation leaves in front of a bottom-tested + * loop when the IV's entry value is a constant carried from the exit of a + * preceding counted loop over the same variable. Runs immediately after + * ssa:loop_rotate — rotation is the shape provider, and everything below + * must already see the single-exit form. Const prop covers the literal-init + * case on its own; only the carried case needs this pass (SSA sees a phi at + * the header and cannot conclude i == A). Gated -O1+ to match + * ssa:loop_rotate; disableable via TCC_DISABLE_PASS=ssa:loop_guard_elim, + * which also makes rotation stop admitting the shapes that depend on it. */ + RA_FLAT_PASS(ra_ch, "ssa:loop_guard_elim", tcc_state->optimize >= 1, + tcc_ir_opt_loop_guard_elim(ir)); + + /* First-iteration-exit peeling (ssa:first_iter_exit): eliminate top-tested + * loops whose header exit test is provably true on first entry, on flat IR + * after rotation (rotation declines these shapes; a rotated loop's guard is + * outside the loop so this pass declines rotated shapes — no overlap). + * Gate matches the legacy tccgen.c pass exactly (-O1+ with const-prop, so + * -fno-const-prop keeps its meaning for bisection); disableable via + * TCC_DISABLE_PASS=ssa:first_iter_exit. */ + RA_FLAT_PASS(ra_ch, "ssa:first_iter_exit", + tcc_state->optimize >= 1 && tcc_state->opt_const_prop, + ssa_opt_first_iter_exit(ir)); + + /* Pointer-IV exit-value substitution (ssa:ptr_iv_exit_subst): rewrite + * post-loop pointer-IV reads to the closed-form exit address and fold the + * consuming `p != &a[N]` compares (pass-owned; nothing downstream folds + * them). Gate matches the legacy tccgen.c pass (-O1+ with const-prop). */ + RA_FLAT_PASS(ra_ch, "ssa:ptr_iv_exit_subst", + tcc_state->optimize >= 1 && tcc_state->opt_const_prop, + ssa_opt_ptr_iv_exit_subst(ir)); + + /* Loop constant simulation (ssa:loop_const_sim): collapse register-only + * bounded-trip loops to residual final values on flat IR. Gate matches the + * legacy tccgen.c Phase 4e pass exactly (opt_loop_unroll, -O2 default; + * -floop-unroll reaches it at lower levels; -fno-loop-unroll disables both + * it and the unroller, keeping the shared bisection knob). Disableable via + * TCC_DISABLE_PASS=ssa:loop_const_sim. */ + RA_FLAT_PASS(ra_ch, "ssa:loop_const_sim", tcc_state->opt_loop_unroll, + ssa_opt_loop_const_sim(ir)); + + /* Loop unrolling / constant-trip elimination (ssa:loop_unroll): fully unroll + * or close-form-eliminate small constant/symbolic-trip register-only loops on + * flat IR after const_sim has starved it of its own candidates. Same gate as + * the legacy tccgen.c Phase 5a pass (opt_loop_unroll); the SSA/regalloc + * pipeline folds the residual arithmetic (no post-unroll cascade replicated). + * Disableable via TCC_DISABLE_PASS=ssa:loop_unroll. */ + RA_FLAT_PASS(ra_ch, "ssa:loop_unroll", tcc_state->opt_loop_unroll, + ssa_opt_loop_unroll(ir)); + /* A table read `tab[i]` in an unrolled body is a load at a constant offset + * now: the propagation group folds the addresses and the loads to the + * table's values (symref_prop, global_init), which SSA does not. Only for + * a function whose unrolled loop read a const table: rerun on every + * unrolled body, the group lost a packed-bitfield RMW through an address + * alias (fuzz 231). */ + if (ra_ch && ir->unrolled_table_loads && !tcc_ir_opt_pass_disabled("ssa:unroll_cascade")) + { + { + const IRPassGroup *groups; + int group_count; + tcc_ir_opt_get_pipeline(IR_OPT_LEVEL_2, &groups, &group_count); + IROptCtx uc_ctx; + tcc_ir_opt_ctx_init(&uc_ctx, ir); + tcc_ir_opt_run_group(&uc_ctx, &groups[0]); + tcc_ir_opt_ctx_free(&uc_ctx); + } + ir->unrolled_table_loads = 0; + tcc_ir_dump_after_pass(ir, "ssa:unroll_cascade"); + } + + /* Induction-variable strength reduction (ssa:iv_strength_reduction): transform + * array-indexing recurrences base + i*stride into a maintained stride pointer + * (enabling post-increment addressing) and optionally eliminate the counter IV + * against a hoisted end pointer, on flat IR after rotation/const_sim/unroll + * have normalized and starved the loops. Runs before ssa:decrement_to_zero, + * preserving the legacy "decrement_to_zero after IV-SR" order. Gate matches + * the legacy tccgen.c Phase 6 pass (opt_iv_strength_red, -O1+); disableable via + * TCC_DISABLE_PASS=ssa:iv_strength_reduction. */ + RA_FLAT_PASS(ra_ch, "ssa:iv_strength_reduction", tcc_state->opt_iv_strength_red, + ssa_opt_iv_strength_reduction(ir)); + + /* Bottom-test the walks IVSR just produced (ssa:loop_bottom_test): copy the + * header CMP in front of the back-edge and make the back-edge the inverted + * conditional branch, leaving the header pair as the zero-trip guard. Runs + * HERE, after ssa:iv_strength_reduction, because counter elimination is what + * creates the `p != end` top test this rewrites; ssa:loop_rotate ran long + * before and its matcher cannot see a body that falls through from the + * header. Flat IR before the CFG build, like the transforms above. Gated + * -O1+ (matches ssa:loop_rotate); disableable via + * TCC_DISABLE_PASS=ssa:loop_bottom_test, level knob TCC_BOTTOM_TEST. */ + RA_FLAT_PASS(ra_ch, "ssa:loop_bottom_test", tcc_state->optimize >= 1, + ssa_opt_loop_bottom_test(ir)); + + /* Decrement-to-zero (ssa:decrement_to_zero): rewrite count-up pure-counter + * loops that ssa:loop_rotate turned bottom-tested (and const_sim/unroll left + * as side-effecting survivors) into count-down-to-zero so the backend fuses + * the latch SUB+CMP#0 into a flag-setting SUBS. Flat IR before the CFG build + * below, so the NOPed guard's control-flow change is reflected downstream. + * Gated -O1+ (matches ssa:loop_rotate, the shape provider); disableable via + * TCC_DISABLE_PASS=ssa:decrement_to_zero. */ + RA_FLAT_PASS(ra_ch, "ssa:decrement_to_zero", tcc_state->optimize >= 1, + ssa_opt_decrement_to_zero(ir)); + + /* Switch-value IPCP: fold calls to single-arg pure dispatchers whose arg is + * constant into ASSIGN #const, replaying any captured global stores at the + * call site. Runs as the last flat transform, immediately before the CFG + * build, so the replayed stores are only ever seen by the SSA pipeline + * (which handles them correctly) and not by the legacy cfg_cleanup DSE, + * which drops a store still read by a following CMP. (const_call_replace is + * store-free and runs early in gen_function so its constant cascades.) */ + if (tcc_state && tcc_state->opt_ipc) { + int ipc_ch = 0; + if (!tcc_ir_opt_pass_disabled("ssa:switch_call_replace")) + ipc_ch += tcc_ir_opt_switch_call_replace(ir); + /* Re-run the flat propagation group so the freshly folded ASSIGN #const + * cascades through the caller. switch_call_replace runs here (not early + * with const_call_replace) because it replays captured global stores that + * must bypass the legacy cfg_cleanup DSE; this re-run recovers the caller + * cascade it would otherwise lose. Gated on an actual rewrite so non-IPCP + * functions pay nothing. */ + if (ipc_ch && tcc_state->optimize >= 1) { + const IRPassGroup *groups; + int group_count; + tcc_ir_opt_get_pipeline(IR_OPT_LEVEL_2, &groups, &group_count); + IROptCtx ipc_ctx; + tcc_ir_opt_ctx_init(&ipc_ctx, ir); + tcc_ir_opt_run_group(&ipc_ctx, &groups[0]); + tcc_ir_opt_ctx_free(&ipc_ctx); + } + tcc_ir_dump_after_pass(ir, "ssa:switch_call_replace"); + } + + /* Sweep NOPs (and their stale jump-target marks) left by the flat pipeline + * and the flat transforms above before building the CFG: a NOP that was a + * jump target introduces a spurious block boundary that blocks var->temp + * promotion in ssa_rename (cmp_offset_common_base). The flat groups only + * compact when a pass reported changes, so a clean run can still arrive + * here with NOPs in the stream. */ + tcc_ir_opt_compact_nops(ir); + + /* Small frame objects touched only a word at a time become VARs, which the + * SSA construction below renames like any other scalar. */ + ir->sra_var_lo = ir->sra_var_hi = -1; + if (tcc_state && tcc_state->optimize > 0 && !tcc_ir_opt_pass_disabled("sra")) + { + tcc_ir_opt_sra(ir); + tcc_ir_dump_after_pass(ir, "sra"); + } + if (tcc_state && tcc_state->optimize > 0 && !tcc_ir_opt_pass_disabled("param_home_fwd")) + { + tcc_ir_opt_param_home_fwd(ir); + tcc_ir_dump_after_pass(ir, "param_home_fwd"); + } + + /* Build CFG + dominators */ + TCCPassTimer ra2_pt; + tcc_pass_timing_begin(&ra2_pt, "ra2:cfg_ssa"); + IRCFG *cfg = tcc_ir_cfg_build(ir); + if (!cfg) { + /* Fallback: no CFG means trivial function, use old allocator path */ + tcc_pass_timing_end(&ra2_pt, -1); + return; + } + tcc_ir_cfg_compute_dominators(cfg); + tcc_ir_cfg_compute_dom_frontiers(cfg); + + ra_promote_multidef_temps_to_vars(ir, cfg); + tcc_ir_dump_after_pass(ir, "ssa_promote"); + + /* Construct SSA. + * + * NOTE (docs/plans/o0_compile_perf.md §9): this is NOT skippable at -O0, + * even though DCE-on-phis is the only -O0 *optimization* downstream of it. + * SSA renaming is what gives every definition its own vreg, which is what + * makes the allocator's one-interval-per-vreg model sound; a value defined + * on two paths otherwise gets a single interval spanning both. Forcing the + * no-promotable fallback at -O0 was measured at -20% compile time and -210 B + * of code, and miscompiled 4 gcc-torture execute tests (990404-1, pr125291, + * pr34415, pending-4) -- with dominators and promotion still running, so it + * is the renaming itself that is load-bearing. Making -O0 cheaper here + * needs a different interval model, not a skip. */ + IRSSAState *ssa = tcc_ir_ssa_construct(ir, cfg); + int had_promotable = (ssa != NULL); + if (!ssa) { + /* No promotable variables; still build intervals from flat IR */ + ssa = tcc_mallocz(sizeof(IRSSAState)); + ssa->cfg = cfg; + ssa->block_phis = tcc_mallocz(cfg->num_blocks * sizeof(IRPhiNode *)); + ssa->num_vars = ir->next_local_variable; + } else { + tcc_ir_ssa_rename(ir, ssa); + } + tcc_ir_dump_after_pass(ir, "ssa_rename"); + /* The same state with the joins shown -- `ssa_rename`'s flat listing reads + * phi destinations that nothing in it defines. */ + tcc_ir_dump_ssa_after_pass(ir, ssa, "ssa_phi"); + dbg_scan_imm_dest(ir, "ssa_rename"); dbg_scan_overlap(ir, "ssa_rename"); + tcc_pass_timing_end(&ra2_pt, -1); + tcc_pass_timing_begin(&ra2_pt, "ra2:ssaopt"); + + /* Phase 2 (memory-SSA): store->load forwarding over the reaching-def walk, + * surviving joins (MemoryPhi) and provably-non-aliasing intervening stores + * that the dom-spine load_cse cannot. Currently OPT-IN (TCC_MEM_SSA_FWD): + * only full-word (4/8-byte) stores forward soundly; the high-value sub-word + * bitfield idiom needs a consumer-mask demand analysis before it can be + * forwarded safely (a raw sub-word forward truncates — see pr78477), so this + * is not yet on by default. TCC_MEM_SSA / TCC_MEM_SSA_DUMP verify / dump the + * constructed memory-SSA. */ + { + const int do_fwd = ra_mem_ssa_fwd(); + const int do_dump = ra_mem_ssa_dump(); + const int do_validate = ra_mem_ssa_verify_on() || do_dump; + if (do_fwd || do_validate) { + MemSSAState *msa = tcc_ir_mem_ssa_build(ir, cfg); + if (msa) { + if (do_validate && !tcc_ir_mem_ssa_verify(ir, msa)) + fprintf(stderr, "[mem-ssa] INVARIANT VIOLATION (%d instrs)\n", + ir->next_instruction_index); + if (do_dump) + tcc_ir_mem_ssa_dump(ir, msa); + if (do_fwd) + tcc_ir_mem_ssa_load_fwd(ir, msa); + tcc_ir_mem_ssa_free(msa); + } + } + } + + /* SSA optimization passes. + * At -O0: only run DCE to remove dead phi definitions that could + * confuse phi resolution. Skip copy propagation and target generators + * which can break VLA/alignment code in unoptimized IR. + * At -O1+: run the full optimization engine. When no variables were + * promoted (all address-taken), run only load CSE and branch folding + * which operate safely on TEMP vregs without phi nodes. */ + { + IRSSAOptCtx ssa_opt_ctx; + tcc_ir_ssa_opt_init(&ssa_opt_ctx, ir, ssa, cfg); + if (tcc_state->optimize >= 1) { + if (had_promotable) { + tcc_ir_ssa_opt_run(&ssa_opt_ctx); + } else { + /* Run a pass, then make it observable to -dump-ir-passes= + * golden snapshots (same names as the tcc_ir_ssa_opt_run driver). */ +#define RUN_SSA(name, call) \ + do \ + { \ + if (!tcc_ir_opt_pass_disabled(name)) \ + { \ + int _c; \ + TCC_PASS_TIMED(_c, name, (call)); \ + np_changes += _c; \ + } \ + tcc_ir_dump_after_pass(ir, name); \ + } while (0) + ssa_opt_ctx.no_stack_fwd = 0; + for (int np_iter = 0; np_iter < 5; np_iter++) { + int np_changes = 0; + RUN_SSA("ssa:var_const_fold", ssa_opt_var_const_fold(&ssa_opt_ctx)); + RUN_SSA("ssa:var_forward", ssa_opt_var_forward(&ssa_opt_ctx)); + RUN_SSA("ssa:sccp", ssa_opt_sccp(&ssa_opt_ctx)); + RUN_SSA("ssa:load_cse", ssa_opt_load_cse(&ssa_opt_ctx)); + RUN_SSA("ssa:diamond_store_fwd", ssa_opt_diamond_store_fwd(&ssa_opt_ctx)); + RUN_SSA("ssa:const_string_fold", tcc_ir_ssa_opt_const_string_fold(&ssa_opt_ctx)); + RUN_SSA("ssa:bitop_const_fold", tcc_ir_ssa_opt_bitop_const_fold(&ssa_opt_ctx)); + RUN_SSA("ssa:ptr_store_dse", tcc_ir_ssa_opt_ptr_store_dse(&ssa_opt_ctx)); + RUN_SSA("ssa:cprop", ssa_opt_cprop(&ssa_opt_ctx)); + RUN_SSA("ssa:fold", ssa_opt_fold(&ssa_opt_ctx)); + RUN_SSA("ssa:var_imm_prop", ssa_opt_var_imm_prop(&ssa_opt_ctx)); + RUN_SSA("ssa:const_prop_tmp", ssa_opt_const_prop_tmp(&ssa_opt_ctx)); + RUN_SSA("ssa:branch", ssa_opt_branch(&ssa_opt_ctx)); + RUN_SSA("ssa:vrp", (tcc_state && tcc_state->opt_vrp) ? ssa_opt_vrp(&ssa_opt_ctx) : 0); + RUN_SSA("ssa:setif_or_taut", + (tcc_state && tcc_state->opt_const_prop) ? ssa_opt_setif_or_taut(&ssa_opt_ctx) : 0); + RUN_SSA("ssa:setif_mask_fold", + (tcc_state && tcc_state->opt_const_prop) ? ssa_opt_setif_mask_fold(&ssa_opt_ctx) : 0); + RUN_SSA("ssa:cmp_offset_fold", + (tcc_state && tcc_state->opt_const_prop) ? ssa_opt_cmp_offset_fold(&ssa_opt_ctx) : 0); + RUN_SSA("ssa:reassoc", ssa_opt_reassoc(&ssa_opt_ctx)); + RUN_SSA("ssa:strength", ssa_opt_strength(&ssa_opt_ctx)); + RUN_SSA("ssa:narrow", ssa_opt_narrow(&ssa_opt_ctx)); + RUN_SSA("ssa:gvn", ssa_opt_gvn(&ssa_opt_ctx)); + RUN_SSA("ssa:phi_simplify", ssa_opt_phi_simplify(&ssa_opt_ctx)); + RUN_SSA("ssa:dce", ssa_opt_dce(&ssa_opt_ctx)); + np_changes += tcc_ir_ssa_opt_guard_collapse(&ssa_opt_ctx); + if (!np_changes) + break; + } + /* After the loop, never inside it: bool_norm deletes the `CMP b,#0` + * that setif_mask_fold matches on (see the SSA driver's tail). */ + if (tcc_state && tcc_state->opt_const_prop && + !tcc_ir_opt_pass_disabled("ssa:bool_norm")) { + int _c; + TCC_PASS_TIMED(_c, "ssa:bool_norm", ssa_opt_bool_norm(&ssa_opt_ctx)); + tcc_ir_dump_after_pass(ir, "ssa:bool_norm"); + if (_c) + ssa_opt_dce(&ssa_opt_ctx); + } + /* Target-specific fusions (MLA, LOAD/STORE_INDEXED on ARM). These + * don't need promotable vars or phi nodes — they pattern-match on + * existing TEMP vregs. */ + tcc_ir_ssa_opt_run_target(&ssa_opt_ctx); +#undef RUN_SSA + } + } else { + /* -O0. `ssa_opt_dce` here is *not* an optimization the level is meant to + * turn off: at optimize == 0 its body is `dce_temp_worklist` + + * `dce_unreachable` (everything else in it is gated on optimize >= 1), so + * what it removes is a dead compiler-generated temp and a block nothing + * can reach. Unreachable code is not code, and dropping it is + * load-bearing at -O0 -- gcc-torture's `link_error` tests + * (`medce-1`, `20030330-1`, `ieee/fp-cmp-7`) fold their condition in the + * *frontend* and are left with a `JMP` over a call that must not reach + * the linker. gcc -O0 drops it too. + * + * `ssa_opt_cprop` used to run here and has been removed, because that one + * is an optimization. Its immediate dispatcher (`ssa_gen_cprop_imm`) + * forwarded `T <- #k [ASSIGN]` into every use, so + * `int k = 4 * 2; t = n * k;` reached the encoder at -O0 as a single + * `MUL #8` -- which the backend then turned into a shifted add, making + * -O0 output indistinguishable from -O2 on any function small enough to + * read. Propagating a value is exactly what -O0 promises not to do; with + * it gone the same function keeps its `movs r1, #8` and a real `mul`. + * + * If something else ever *has* to run at -O0, it is lowering, and it + * belongs in the lowering path rather than in the else arm of the + * optimizer's gate. */ + ssa_opt_dce(&ssa_opt_ctx); + } + if (tcc_state && tcc_state->optimize >= 1 && tcc_state->opt_redundant_store && + !tcc_ir_opt_pass_disabled("ssa:rmw_byte_clear")) { + int c = tcc_ir_opt_rmw_byte_clear(ir); + if (c && tcc_state->opt_dce) + ssa_opt_dce(&ssa_opt_ctx); + } + tcc_ir_dump_after_pass(ir, "ssa:rmw_byte_clear"); + /* ssa:memmove_global_fwd — rerun of the tccgen-time init-copy-from-global + * forwarding. Its read-only-slot precondition only becomes true here: + * ssa:struct_copy_roundtrip removes the retme pair and ssa:dce ret_store + * kills the write-back, both inside this pipeline — after the tccgen call + * already ran (20040709-2 fn1* family). */ + if (tcc_state && tcc_state->optimize >= 1 && tcc_state->opt_redundant_store && + !tcc_ir_opt_pass_disabled("ssa:memmove_global_fwd")) { + if (tcc_ir_opt_memmove_global_load_fwd(ir) > 0 && tcc_state->opt_dce) { + /* Flat pass: rebuild the SSA use chains it left stale, or the DCE + * worklist can't cascade the now-dead dest-LEA/ADD address temps. + * Phi operand uses must be re-added too — dropping them lets DCE gut + * every loop-carried def (33_ternary_op loop body). */ + for (int p = 0; p < ssa_opt_ctx.vinfo_cap; p++) + ssa_opt_ctx.vinfo[p].use_count = 0; + for (int i = 0; i < ir->next_instruction_index; i++) { + IRQuadCompact *q = &ir->compact_instructions[i]; + if (q->op != TCCIR_OP_NOP) + ssa_opt_scan_instr_uses(&ssa_opt_ctx, i, q); + } + if (ssa->block_phis) { + for (int b = 0; b < cfg->num_blocks; b++) { + for (IRPhiNode *phi = ssa->block_phis[b]; phi; phi = phi->next) { + for (int pi = 0; pi < phi->num_operands; pi++) { + IRSSAVregInfo *pvi = ssa_opt_vinfo(&ssa_opt_ctx, phi->operands[pi].vreg); + if (pvi) + ssa_opt_add_use_phi(pvi, b, pi); + } + } + } + } + ssa_opt_dce(&ssa_opt_ctx); + } + } + tcc_ir_dump_after_pass(ir, "ssa:memmove_global_fwd"); + /* Park a global address that is reloaded across calls in a callee-saved + * register for its whole live range, instead of reloading it from the + * literal pool at each call-separated use. Runs last so target fusions + * have already consumed the load/store-indexed bases they hoist; -O2 only + * (GCC likewise only parks globals at -O2). + * docs/plans/gap_a_ssa_var_index_addr_prop.md (gap ②). */ + tcc_ir_ssa_opt_free(&ssa_opt_ctx); + } + /* And once the passes have had it: which phis survived them. */ + tcc_ir_dump_ssa_after_pass(ir, ssa, "ssa_phi_opt"); + dbg_scan_imm_dest(ir, "ssa_opt_block"); dbg_scan_overlap(ir, "ssa_opt_block"); + + /* Set types from operand btypes (same as tcc_ir_live_analysis). + * Skip lvalue operands: when is_lval=1 the vreg holds a pointer (32-bit) + * and the btype describes the pointed-to value, not the pointer itself. */ + for (int i = 0; i < ir->next_instruction_index; i++) { + IRQuadCompact *q = &ir->compact_instructions[i]; + IROperand dest = tcc_ir_op_get_dest(ir, q); + if (irop_config[q->op].has_dest && tcc_ir_vreg_is_valid(ir, irop_get_vreg(dest)) && !dest.is_lval) { + int btype = irop_get_btype(dest); + if (btype == IROP_BTYPE_FLOAT32 || btype == IROP_BTYPE_FLOAT64) + tcc_ir_vreg_type_set_fp(ir, irop_get_vreg(dest), 1, btype == IROP_BTYPE_FLOAT64); + else if (btype == IROP_BTYPE_INT64) + tcc_ir_vreg_type_set_64bit(ir, irop_get_vreg(dest)); + if (dest.is_complex) + tcc_ir_vreg_type_set_complex(ir, irop_get_vreg(dest)); + } + IROperand src1 = tcc_ir_op_get_src1(ir, q); + if (irop_config[q->op].has_src1 && tcc_ir_vreg_is_valid(ir, irop_get_vreg(src1)) && !src1.is_lval) { + int btype = irop_get_btype(src1); + if (btype == IROP_BTYPE_FLOAT32 || btype == IROP_BTYPE_FLOAT64) + tcc_ir_vreg_type_set_fp(ir, irop_get_vreg(src1), 1, btype == IROP_BTYPE_FLOAT64); + else if (btype == IROP_BTYPE_INT64) + tcc_ir_vreg_type_set_64bit(ir, irop_get_vreg(src1)); + if (src1.is_complex) + tcc_ir_vreg_type_set_complex(ir, irop_get_vreg(src1)); + } + IROperand src2 = tcc_ir_op_get_src2(ir, q); + if (irop_config[q->op].has_src2 && tcc_ir_vreg_is_valid(ir, irop_get_vreg(src2)) && !src2.is_lval) { + int btype = irop_get_btype(src2); + if (btype == IROP_BTYPE_FLOAT32 || btype == IROP_BTYPE_FLOAT64) + tcc_ir_vreg_type_set_fp(ir, irop_get_vreg(src2), 1, btype == IROP_BTYPE_FLOAT64); + else if (btype == IROP_BTYPE_INT64) + tcc_ir_vreg_type_set_64bit(ir, irop_get_vreg(src2)); + if (src2.is_complex) + tcc_ir_vreg_type_set_complex(ir, irop_get_vreg(src2)); + } + } + + /* Propagate types from phi nodes to their dest AND operand vregs. + * SSA rename creates new TEMPs that may only appear with INT32 btype + * in their defining instruction, but the phi btype reflects the + * original variable's type. Phi resolution will insert ASSIGN copies + * with the phi btype, so codegen will expect 64-bit values from these + * vregs even if their defs used INT32 btype. */ + if (ssa->block_phis) { + for (int b = 0; b < cfg->num_blocks; b++) { + for (IRPhiNode *phi = ssa->block_phis[b]; phi; phi = phi->next) { + int is_fp = (phi->btype == IROP_BTYPE_FLOAT32 || phi->btype == IROP_BTYPE_FLOAT64); + int is_i64 = (phi->btype == IROP_BTYPE_INT64); + int is_dbl = (phi->btype == IROP_BTYPE_FLOAT64); + /* Check original variable type as fallback */ + if (!is_fp && !is_i64 && phi->orig_vreg >= 0 && + tcc_ir_vreg_is_valid(ir, phi->orig_vreg)) { + IRLiveInterval *orig_li = tcc_ir_vreg_live_interval(ir, phi->orig_vreg); + if (orig_li) { + if (orig_li->is_llong) is_i64 = 1; + if (orig_li->is_float) { is_fp = 1; is_dbl = orig_li->is_double; } + } + } + if (!is_fp && !is_i64) continue; + + /* Propagate to dest vreg */ + int32_t dv = phi->dest_vreg; + if (dv >= 0 && tcc_ir_vreg_is_valid(ir, dv)) { + if (is_fp) tcc_ir_vreg_type_set_fp(ir, dv, 1, is_dbl); + if (is_i64) tcc_ir_vreg_type_set_64bit(ir, dv); + } + /* Propagate to all operand vregs (phi sources) */ + for (int pi = 0; pi < phi->num_operands; pi++) { + int32_t ov = phi->operands[pi].vreg; + if (ov >= 0 && tcc_ir_vreg_is_valid(ir, ov)) { + if (is_fp) tcc_ir_vreg_type_set_fp(ir, ov, 1, is_dbl); + if (is_i64) tcc_ir_vreg_type_set_64bit(ir, ov); + } + } + } + } + } + + /* Resolve phis BEFORE register allocation: insert ASSIGN copies at + * predecessor block ends so phi-DSTs and phi-SRCs become regular SSA + * temps with non-overlapping intervals. Without this, the linear scan + * sees phi-DST intervals that span the whole loop body alongside their + * phi-SRC operands' intervals (also spanning the body), creating + * artificial register pressure across loops. After this pass the IR + * is no longer in SSA form; ra_build_intervals scans the explicit + * copies and produces concrete intervals. */ + ra_phi_resolve_pre_ra_mode = 1; + tcc_pass_timing_end(&ra2_pt, -1); + tcc_pass_timing_begin(&ra2_pt, "ra2:phis_folds"); + ra_resolve_phis(ir, cfg, ssa, &phi_stats); + ra_phi_resolve_pre_ra_mode = 0; + /* The one edit that makes a phi disappear: every dump before this one can + * have phis in it (with -dump-ir-passes=ssa_phi to see them) and no dump + * after it can, because there is no phi left to show. */ + tcc_ir_dump_after_pass(ir, "ra_resolve_phis"); + dbg_scan_imm_dest(ir,"ra_resolve_phis"); + + /* Collapse "TMP <- const; T_phi <- TMP" chains the phi resolver leaves + * behind in dense switch case bodies. Each fold drops one ASSIGN and one + * SSA temp from the case body, cutting both the per-case instruction count + * and the phi-temp live ranges that drive the linear scan into spills. */ + ra_fold_phi_const_chain(ir); + dbg_scan_imm_dest(ir,"ra_fold_phi_const_chain"); + + /* Once the per-case bodies are canonicalised to "T_phi <- const; JMP merge", + * try to rewrite the entire SWITCH_TABLE dispatch into a single SWITCH_LOAD + * against an inline value table. Must run after the phi-const fold above + * (which produces the canonical body shape) and before live-interval + * construction (which would otherwise see the now-dead case bodies). */ + tcc_ir_opt_switch_to_data(ir); + dbg_scan_imm_dest(ir,"switch_to_data"); + + /* Fold CMP + JUMPIF where both operands resolve to constants within the + * same basic block. Phi resolution often materializes the entry-path + * constant of a loop counter right before its bound check; folding the + * dead skip-loop block removes the carrier-vreg copies it contains and + * cuts the carriers' live ranges, easing register pressure. */ + ra_fold_const_branches(ir); + dbg_scan_imm_dest(ir,"ra_fold_const_branches"); + + /* const_memcpy_fwd: by this point the SSA opt fold (ssa_opt_fold's + * bit-complement / SCCP / GVN) has materialised compile-time-constant + * aggregate values (e.g. pr60502's `*x |= *x ^ {-1,...}` → all-0xFF), and + * the IR is de-SSA'd flat form. Rewrite a constant-filled non-escaping + * stack buffer copied by an aligned AEABI mem* helper into direct wide + * constant stores to the destination, dropping the buffer + the call. Must + * run AFTER the SSA fold (which produces the constants) and BEFORE call + * prefix / interval construction (which must see the call/stores removed). + * The codegen STRD-imm peephole then pairs the word stores into `strd`. */ + if (tcc_state->optimize >= 1) + tcc_ir_opt_const_memcpy_to_dest(ir); + + /* Park a global address reloaded across calls in a callee-saved register for + * its whole live range instead of reloading it from the literal pool at each + * call-separated use. Runs here — after phi resolution de-SSA'd the IR — so + * prepending entry materializations can't desync phi resolution; block_phis + * is emptied, so rebuilding the CFG (which the inserts invalidate) is safe. + * -O2 only (GCC likewise only parks globals at -O2). + * docs/plans/gap_a_ssa_var_index_addr_prop.md (gap ②). */ + if (tcc_state && tcc_state->optimize >= 2) { + int addr_changes = 0; + /* Before the address hoists, so it matches operands the frontend built + * (which carry IROP_AUX_NONVOLATILE) rather than bases synthesized by a + * pass. Here rather than in the flat pipeline because the two reads it + * collapses only name the same address temp once GVN has run — see the + * header of source/opt/flat/memory/deref_operand_cse.c. */ + addr_changes += tcc_ir_opt_deref_operand_cse(ir); + if (!tcc_ir_opt_pass_disabled("ssa:global_addr_hoist")) + addr_changes += tcc_ir_ssa_opt_global_addr_hoist(ir); + if (!tcc_ir_opt_pass_disabled("ssa:loop_addr_hoist")) + addr_changes += tcc_ir_ssa_opt_loop_addr_hoist(ir); + if (!tcc_ir_opt_pass_disabled("ssa:local_addr_cse")) + addr_changes += tcc_ir_ssa_opt_local_addr_cse(ir); + /* Same reason these three run here: an indexed access's STACKOFF base is a + * DIRECT frame reference that slot-based alias analysis reads precisely, so + * parking it in a register may only happen once every alias-sensitive pass + * is done (early it miscompiles gcc.c-torture pr51466). */ + if (!tcc_ir_opt_pass_disabled("ssa:stackoff_indexed_base_cse")) + addr_changes += tcc_ir_opt_stackoff_indexed_base_cse(ir); + if (addr_changes > 0) { + tcc_ir_cfg_free(cfg); + cfg = tcc_ir_cfg_build(ir); + tcc_ir_cfg_compute_dominators(cfg); + ssa->cfg = cfg; + /* block_phis is sized to the old block count and consumed by + * ra_build_intervals; the fresh CFG may have a different count. It is + * all-NULL post-resolution, so resize and zero it to the new count. */ + ssa->block_phis = tcc_realloc(ssa->block_phis, cfg->num_blocks * sizeof(IRPhiNode *)); + memset(ssa->block_phis, 0, cfg->num_blocks * sizeof(IRPhiNode *)); + } + } + tcc_ir_dump_after_pass(ir, "ssa:global_addr_hoist"); + + /* `CMP; t <- SETIF; TEST_ZERO t; JUMPIF` -> `CMP; JUMPIF`, again. The flat + * pipeline's setif_fuse runs before SSA, when the Zig C backend's one + * `bool t` per function is still a single VAR with copies and several + * reads; SSA renaming gives each check its own single-use temp, and those + * quartets used to reach codegen as ITE/MOV/MOV/CMP/branch. Here phis are + * explicit copies, so the use scan sees every read. */ + if (tcc_state->optimize >= 1 && !tcc_ir_opt_pass_disabled("ra:setif_fuse")) + tcc_ir_opt_setif_branch_fuse(ir); + tcc_ir_dump_after_pass(ir, "ra:setif_fuse"); + + /* Product + zero-extended words -> UMAAL. Last of the IR rewrites: every + * pass before this point is unaware of TCCIR_OP_UMAAL; the allocator and + * everything after it read its accumulator via tcc_ir_op_is_mac. */ + if (tcc_state->optimize >= 1 && !tcc_ir_opt_pass_disabled("ra:umaal")) + ra_fuse_umaal(ir); + tcc_ir_dump_after_pass(ir, "ra:umaal"); + + /* TEMP definitions nothing reads, or overwritten before a read -- the SSA + * passes above leave them (an entry-block VAR renamed to one TEMP keeps its + * zero fill). Here, before the intervals: graph coalescing below erases + * the copies between vregs that share a register, after which a value can + * be read under another vreg's name (dead_def.c). */ + if (tcc_state->optimize >= 1 && !tcc_ir_opt_pass_disabled("dead_def") && tcc_ir_dead_def(ir)) + tcc_ir_dump_after_pass(ir, "dead_def"); + + int *call_prefix = NULL; + SSAInterval *intervals = NULL; + int interval_count = 0; + int max_vreg_pos = 0; + uint64_t dirty_int = 0, dirty_fp = 0; + for (int round = 0;; round++) { + /* Build call prefix for call-crossing detection */ + call_prefix = ra_build_call_prefix(ir); + + /* Build SSA live intervals */ + intervals = NULL; + interval_count = 0; + max_vreg_pos = 0; + tcc_pass_timing_end(&ra2_pt, -1); + tcc_pass_timing_begin(&ra2_pt, "ra2:intervals"); + ra_build_intervals(ir, cfg, ssa, &intervals, &interval_count, call_prefix, &max_vreg_pos); + tcc_pass_timing_end(&ra2_pt, -1); + tcc_pass_timing_begin(&ra2_pt, "ra2:hints"); + ra_build_narrow_weights(ir, target, intervals, interval_count, max_vreg_pos); + + /* Build phi register hints. block_phis is empty after pre-RA resolution, + * so the phi-based pass is a no-op; the assign-based pass picks up + * the explicit copies emitted at predecessor block ends. */ + ra_build_phi_hints(intervals, interval_count, ssa, cfg, max_vreg_pos); + ra_build_assign_hints(intervals, interval_count, ir, max_vreg_pos); + ra_build_load_param_hints(intervals, interval_count, ir, max_vreg_pos); + ra_build_bfi_hints(intervals, interval_count, ir, max_vreg_pos); + ra_build_outgoing_param_hints(intervals, interval_count, ir, max_vreg_pos); + + /* Graph-based coalescing (accurate liveness + interference) — merges + * copy-related non-interfering vregs, including multi-predecessor merge-phis + * the in-scan transfer cannot handle. Gated by TCC_COALESCE. */ + tcc_pass_timing_end(&ra2_pt, -1); + tcc_pass_timing_begin(&ra2_pt, "ra2:coalesce_g"); + ra_coalesce_graph(ir, intervals, interval_count, max_vreg_pos); + tcc_pass_timing_end(&ra2_pt, -1); + tcc_pass_timing_begin(&ra2_pt, "ra2:scan"); + + /* Run linear scan */ + dirty_int = 0; + dirty_fp = 0; + int n_instr = ir->next_instruction_index; + int has_call = call_prefix && n_instr > 0 && call_prefix[n_instr] > 0; + RaAliveInfo alive; + memset(&alive, 0, sizeof alive); + if (ra_alive_worth_building(ir, intervals, interval_count)) + ra_alive_build(ir, &alive, max_vreg_pos); + /* Drop the frame objects nothing references any more so spills go right + * below the live ones (frame.c). Here, after the SSA passes: some of them + * add direct frame references (loop_const_sim's residual stores), and the + * layout must see every reference the code will make. */ + if (round == 0 && tcc_state && tcc_state->optimize > 0 && !tcc_ir_opt_pass_disabled("frame_relayout")) + tcc_ir_frame_relayout(ir, &spill_base); + ra_linear_scan(ir, intervals, interval_count, target, spill_base, &dirty_int, &dirty_fp, + max_vreg_pos, has_call, &alive); + ra_alive_free(&alive); + tcc_pass_timing_end(&ra2_pt, -1); + tcc_pass_timing_begin(&ra2_pt, "ra2:finish"); + + /* Propagate each coalesced member's allocation from its representative (which + * the scan allocated; members were skipped). coalesce_to holds the rep's + * vreg (stable across the scan's qsort); resolve via a vreg->interval search. */ + { + int any = 0; + for (int i = 0; i < interval_count && !any; i++) + if (intervals[i].coalesce_to >= 0) any = 1; + if (any) { + for (int i = 0; i < interval_count; i++) { + if (intervals[i].coalesce_to < 0) continue; + for (int j = 0; j < interval_count; j++) { + if (intervals[j].vreg != intervals[i].coalesce_to || intervals[j].coalesce_to >= 0) + continue; + intervals[i].r0 = intervals[j].r0; + intervals[i].r1 = intervals[j].r1; + intervals[i].stack_location = intervals[j].stack_location; + break; + } + } + } + } + + /* Spill-driven splitting: rewrite the spilled values' uses, then allocate + * again from scratch (see ra_split_spilled). */ + if (round == 0 && tcc_state->optimize >= 1 && !tcc_ir_opt_pass_disabled("ra:spill_split") && + ra_split_spilled(ir, cfg, intervals, interval_count, max_vreg_pos, call_prefix) > 0) { + tcc_free(intervals); + intervals = NULL; + tcc_free(call_prefix); + call_prefix = NULL; + tcc_ir_cfg_free(cfg); + cfg = tcc_ir_cfg_build(ir); + tcc_ir_cfg_compute_dominators(cfg); + ssa->cfg = cfg; + ssa->block_phis = tcc_realloc(ssa->block_phis, cfg->num_blocks * sizeof(IRPhiNode *)); + memset(ssa->block_phis, 0, cfg->num_blocks * sizeof(IRPhiNode *)); + continue; + } + break; + } + + if (TCC_LOG_LS) { + for (int i = 0; i < interval_count; i++) { + if (intervals[i].stack_location != 0 && + ra_phi_stats_has_participant(&phi_stats, intervals[i].vreg)) + phi_stats.participant_spills++; + } + LOG_LS("phi_stats operands=%d parallel_requested=%d parallel_emitted=%d cycle_temporaries=%d participant_spills=%d", + phi_stats.phi_operands, phi_stats.parallel_requested, + phi_stats.parallel_emitted, phi_stats.cycle_temporaries, + phi_stats.participant_spills); + } + + /* Write results to IR + LS state */ + ra_write_results(ir, intervals, interval_count); + ir->ls.dirty_registers = dirty_int; + ir->ls.dirty_float_registers = dirty_fp; + + /* Phi resolution already happened before ra_build_intervals (above). + * The instruction stream now has explicit ASSIGN copies; ssa->block_phis + * is cleared. We just need to build the live_regs bitmap from the + * intervals the linear scan produced. */ + ra_build_live_regs_bitmap(ir); + ra_refine_live_regs_accurate(ir); + + /* Strip dead phi copies now that allocation and every liveness/scratch + * bitmap are final. A phi copy into a never-read temp is dead code whose + * register the scan may have reused for a value live across the merge; left + * in, it clobbers that value (seed 198468). Running it here — after the + * bitmaps — keeps the allocation identical and only drops dead instructions. */ + ra_eliminate_dead_reg_copies(ir); + + ra_repair_incomplete_calls(ir); + + ra_mark_rematerializable(ir); + + tcc_pass_timing_end(&ra2_pt, -1); + + /* Cleanup */ + tcc_free(phi_stats.participants); + tcc_free(intervals); + if (call_prefix) tcc_free(call_prefix); + tcc_ir_ssa_free(ssa); + tcc_ir_cfg_free(cfg); +} diff --git a/source/ir/regalloc_phi.c b/source/ir/regalloc_phi.c new file mode 100644 index 000000000..a2bb7e022 --- /dev/null +++ b/source/ir/regalloc_phi.c @@ -0,0 +1,2149 @@ +/* + * TCC IR - SSA-Aware Register Allocator + * + * Operates directly on SSA-renamed IR with phi nodes. + * Replaces the tccls.c linear scan when -fssa-regalloc is enabled. + * + * Copyright (c) 2025 Mateusz Stadnik + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +/* SSA register allocator: phi resolution and graph-based register + * coalescing. */ + +#include "regalloc_priv.h" + +static void ra_phi_stats_add_participant(RAPhiStats *stats, int32_t vreg) +{ + if (!TCC_LOG_LS || !stats || vreg < 0) + return; + + for (int i = 0; i < stats->participant_count; i++) + if (stats->participants[i] == vreg) + return; + + if (stats->participant_count >= stats->participant_cap) { + int cap = stats->participant_cap ? stats->participant_cap * 2 : 16; + stats->participants = tcc_realloc(stats->participants, + cap * sizeof(*stats->participants)); + stats->participant_cap = cap; + } + stats->participants[stats->participant_count++] = vreg; +} +int ra_phi_stats_has_participant(RAPhiStats *stats, int32_t vreg) +{ + if (!TCC_LOG_LS || !stats) + return 0; + for (int i = 0; i < stats->participant_count; i++) + if (stats->participants[i] == vreg) + return 1; + return 0; +} + +static int ra_interval_reg_count(IRLiveInterval *li, int regs[2]) +{ + int n = 0; + if (!li || li->allocation.offset != 0) + return 0; + int r0 = li->allocation.r0; + if ((r0 & PREG_SPILLED) == 0 && r0 != PREG_NONE && r0 != 0xffff) + regs[n++] = r0; + int r1 = li->allocation.r1; + if ((r1 & PREG_SPILLED) == 0 && r1 != PREG_NONE && r1 != 0xffff) + regs[n++] = r1; + return n; +} + +static int ra_interval_regs_overlap(IRLiveInterval *a, IRLiveInterval *b) +{ + int ar[2], br[2]; + int an = ra_interval_reg_count(a, ar); + int bn = ra_interval_reg_count(b, br); + for (int i = 0; i < an; i++) + for (int j = 0; j < bn; j++) + if (ar[i] == br[j]) + return 1; + return 0; +} + +static int ra_interval_locations_overlap(IRLiveInterval *a, IRLiveInterval *b) +{ + if (!a || !b) + return 0; + if (a->allocation.offset != 0 || b->allocation.offset != 0) + return a->allocation.offset != 0 && a->allocation.offset == b->allocation.offset; + return ra_interval_regs_overlap(a, b); +} + +/* Set to 1 while running ra_resolve_phis BEFORE register allocation, where + * no allocation info exists. In that mode, copy-elision (identity check) + * and physical-register clobber detection must fall back to vreg-level + * reasoning. */ +int ra_phi_resolve_pre_ra_mode = 0; + +/* Post-allocation dead register-copy elimination. + * + * Phi resolution (ra_resolve_phis) emits an ASSIGN copy for every phi operand, + * including phi destinations that are never read. Such a copy is dead code — + * but because its destination has no uses (an empty live range), the linear + * scan freely gives it a register that simultaneously holds a DIFFERENT value + * which is live across the merge (the two do not interfere). The dead copy + * then overwrites that register, destroying the live value. + * switch fuzz seed 198468 -O1: at a loop-exit merge the dead exit-phi copy + * `R6(T121) <- T130` clobbered R6, which still held the live u6 read one + * instruction later by `T106 <- R6(T104)`, so u6 read back as 0. + * + * These copies must NOT be suppressed before allocation: removing them changes + * register pressure and perturbs every allocation decision, regressing + * unrelated code by shifting the scan onto latent bugs. They are NOP'd here, + * after allocation AND after every liveness / scratch bitmap has been built, so + * the allocation is byte-identical to before and only the emitted code changes. + * + * A copy is removable when its destination is an SSA TEMP — never + * address-taken, so every read is an explicit operand — that appears as a read + * operand nowhere. Reads are enumerated exactly as ra_build_intervals counts + * uses: src1, src2, the MLA accumulator, and the address of a STORE-class op. + * Iterate to a fixed point so a chain of dead copies (each read only by the + * next) collapses completely. */ +void ra_eliminate_dead_reg_copies(TCCIRState *ir) +{ + int max_pos = ir->next_local_variable; + if (ir->next_temporary_variable > max_pos) max_pos = ir->next_temporary_variable; + if (ir->next_parameter > max_pos) max_pos = ir->next_parameter; + if (max_pos <= 0) + return; + size_t map_size = (size_t)4 * max_pos; + uint8_t *read_map = tcc_malloc(map_size); + int n = ir->next_instruction_index; + + #define RA_DEADCOPY_IDX(vr) \ + (TCCIR_DECODE_VREG_TYPE(vr) * max_pos + TCCIR_DECODE_VREG_POSITION(vr)) + #define RA_DEADCOPY_MARK(vr) do { \ + int32_t _v = (vr); \ + if (_v >= 0) { \ + long _i = RA_DEADCOPY_IDX(_v); \ + if (_i >= 0 && (size_t)_i < map_size) read_map[_i] = 1; \ + } \ + } while (0) + + for (;;) { + memset(read_map, 0, map_size); + + for (int i = 0; i < n; i++) { + IRQuadCompact *q = &ir->compact_instructions[i]; + if (q->op == TCCIR_OP_NOP) + continue; + if (irop_config[q->op].has_src1) { + IROperand s = tcc_ir_op_get_src1(ir, q); + if (irop_has_vreg(s)) RA_DEADCOPY_MARK(irop_get_vreg(s)); + } + if (irop_config[q->op].has_src2) { + IROperand s = tcc_ir_op_get_src2(ir, q); + if (irop_has_vreg(s)) RA_DEADCOPY_MARK(irop_get_vreg(s)); + } + if (tcc_ir_op_is_mac(q->op)) { + IROperand s = tcc_ir_op_get_accum(ir, q); + if (irop_has_vreg(s)) RA_DEADCOPY_MARK(irop_get_vreg(s)); + } + /* STORE-class ops read their "dest" operand (the memory address). */ + if (q->op == TCCIR_OP_STORE || q->op == TCCIR_OP_STORE_INDEXED || + q->op == TCCIR_OP_STORE_POSTINC) { + IROperand d = tcc_ir_op_get_dest(ir, q); + if (irop_has_vreg(d)) RA_DEADCOPY_MARK(irop_get_vreg(d)); + } + } + + int changed = 0; + for (int i = 0; i < n; i++) { + IRQuadCompact *q = &ir->compact_instructions[i]; + if (q->op != TCCIR_OP_ASSIGN) + continue; + IROperand d = tcc_ir_op_get_dest(ir, q); + if (!irop_has_vreg(d)) + continue; + int32_t dv = irop_get_vreg(d); + if (TCCIR_DECODE_VREG_TYPE(dv) != TCCIR_VREG_TYPE_TEMP) + continue; + long idx = RA_DEADCOPY_IDX(dv); + if (idx < 0 || (size_t)idx >= map_size || read_map[idx]) + continue; /* destination is read somewhere — live copy, keep */ + /* Dead copy into a never-read temp: NOP it. The slot (and any + * is_jump_target flag on it) is preserved so branch targets still + * resolve to a valid index; codegen skips NOPs. */ + q->op = TCCIR_OP_NOP; + changed = 1; + } + + if (!changed) + break; + } + + #undef RA_DEADCOPY_MARK + #undef RA_DEADCOPY_IDX + tcc_free(read_map); +} +int ra_phi_copy_is_identity(IRLiveInterval *dest_li, IRLiveInterval *src_li) +{ + if (!dest_li || !src_li) + return 0; + /* Pre-RA: never collapse; allocation may still place them in the same + * register, in which case the post-RA move-coalescing pass will erase + * the redundant copy. */ + if (ra_phi_resolve_pre_ra_mode) + return 0; + if (dest_li->allocation.offset != 0 || src_li->allocation.offset != 0) + return dest_li->allocation.offset == src_li->allocation.offset; + return dest_li->allocation.r0 == src_li->allocation.r0 && + dest_li->allocation.r1 == src_li->allocation.r1; +} + +static int ra_phi_copy_dest_clobbers_pending_source(TCCIRState *ir, RAPhiCopy *copies, + int copy_count, int copy_idx) +{ + /* Pre-RA: clobber means "copy_idx writes a vreg that another pending + * copy still needs to read". Compare by vreg, not by physical + * register (which doesn't exist yet). */ + if (ra_phi_resolve_pre_ra_mode) { + int32_t dest_vr = copies[copy_idx].dest_vreg; + for (int i = 0; i < copy_count; i++) { + if (i == copy_idx || copies[i].emitted) + continue; + if (copies[i].src_vreg == dest_vr) + return 1; + } + return 0; + } + + IRLiveInterval *dest_li = tcc_ir_vreg_live_interval(ir, copies[copy_idx].dest_vreg); + if (!dest_li) + return 0; + + for (int i = 0; i < copy_count; i++) { + if (i == copy_idx || copies[i].emitted) + continue; + if (!tcc_ir_vreg_is_valid(ir, copies[i].src_vreg)) + continue; + IRLiveInterval *src_li = tcc_ir_vreg_live_interval(ir, copies[i].src_vreg); + if (ra_interval_locations_overlap(dest_li, src_li)) + return 1; + } + return 0; +} + +static int ra_invert_cond(int tok) +{ + switch (tok) { + case TOK_ULT: return TOK_UGE; + case TOK_UGE: return TOK_ULT; + case TOK_EQ: return TOK_NE; + case TOK_NE: return TOK_EQ; + case TOK_ULE: return TOK_UGT; + case TOK_UGT: return TOK_ULE; + case TOK_LT: return TOK_GE; + case TOK_GE: return TOK_LT; + case TOK_LE: return TOK_GT; + case TOK_GT: return TOK_LE; + default: return tok ^ 1; + } +} + +static int ra_phi_copy_needed(TCCIRState *ir, IRPhiNode *phi, int operand_idx) +{ + IRLiveInterval *dest_li = tcc_ir_vreg_live_interval(ir, phi->dest_vreg); + if (!dest_li) + return 0; + /* Pre-RA: emit a copy for every operand. Post-RA coalescing will drop + * any that turn out to land in the same physical register. Copies into a + * phi destination that is never read are NOT suppressed here: doing so + * pre-RA changes register pressure and perturbs the whole allocation. They + * are stripped after allocation instead — see ra_eliminate_dead_reg_copies. */ + if (ra_phi_resolve_pre_ra_mode) { + if (!tcc_ir_vreg_is_valid(ir, phi->operands[operand_idx].vreg)) + return 0; + return 1; + } + if (dest_li->allocation.r0 == PREG_NONE && dest_li->allocation.offset == 0) + return 0; + IRLiveInterval *src_li = NULL; + if (tcc_ir_vreg_is_valid(ir, phi->operands[operand_idx].vreg)) + src_li = tcc_ir_vreg_live_interval(ir, phi->operands[operand_idx].vreg); + if (ra_phi_copy_is_identity(dest_li, src_li)) { + if (src_li) + src_li->phi_pinned = 1; + return 0; + } + return 1; +} + +static int ra_phi_pred_candidate(const IRPhiNode *phi, int pi, int nb) +{ + int pred = phi->operands[pi].pred_block; + if (pred < 0 || pred >= nb || phi->operands[pi].vreg < 0) + return -1; + return pred; +} + +static void ra_phi_pred_index_build(IRCFG *cfg, IRSSAState *ssa, RAPhiPredIndex *idx) +{ + const int nb = cfg->num_blocks; + + idx->off = tcc_mallocz(sizeof(int) * (nb + 1)); + idx->phi = NULL; + idx->opidx = NULL; + + for (int sb = 0; sb < nb; sb++) + for (IRPhiNode *phi = ssa->block_phis[sb]; phi; phi = phi->next) + for (int pi = 0; pi < phi->num_operands; pi++) { + int pred = ra_phi_pred_candidate(phi, pi, nb); + if (pred >= 0) + idx->off[pred + 1]++; + } + for (int b = 0; b < nb; b++) + idx->off[b + 1] += idx->off[b]; + + const int total = idx->off[nb]; + if (!total) + return; + + idx->phi = tcc_malloc(sizeof(IRPhiNode *) * total); + idx->opidx = tcc_malloc(sizeof(int) * total); + + int *cursor = tcc_malloc(sizeof(int) * nb); + memcpy(cursor, idx->off, sizeof(int) * nb); + for (int sb = 0; sb < nb; sb++) + for (IRPhiNode *phi = ssa->block_phis[sb]; phi; phi = phi->next) + for (int pi = 0; pi < phi->num_operands; pi++) { + int pred = ra_phi_pred_candidate(phi, pi, nb); + if (pred < 0) + continue; + idx->phi[cursor[pred]] = phi; + idx->opidx[cursor[pred]] = pi; + cursor[pred]++; + } + tcc_free(cursor); +} + +static void ra_phi_pred_index_free(RAPhiPredIndex *idx) +{ + tcc_free(idx->off); + tcc_free(idx->phi); + tcc_free(idx->opidx); + idx->off = NULL; + idx->phi = NULL; + idx->opidx = NULL; +} + +/* Resolve a successor filter to the block range the scanners below must walk. + * + * Both took the filter as a per-iteration equality test, so a call that wanted + * ONE block still walked all of them -- and ra_resolve_phis calls them once per + * predecessor, which is O(blocks^2). pr34093's 1000-case switch spent 32% of + * its entire compile in ra_collect_phi_copies_for_pred that way. Same fix, and + * same reason, as the copies_per_block aggregation in ra_resolve_phis. */ +static void ra_phi_scan_range(const IRCFG *cfg, int succ_filter, int *first, int *last) +{ + if (succ_filter < 0) { + *first = 0; + *last = cfg->num_blocks - 1; + } else if (succ_filter < cfg->num_blocks) { + *first = *last = succ_filter; + } else { + /* Names no block: scan nothing, as the equality test used to. */ + *first = 0; + *last = -1; + } +} + +static int ra_count_phi_copies_for_pred(TCCIRState *ir, IRCFG *cfg, IRSSAState *ssa, + int pred_block, int succ_filter) +{ + int count = 0; + int first, last; + ra_phi_scan_range(cfg, succ_filter, &first, &last); + for (int sb = first; sb <= last; sb++) { + for (IRPhiNode *phi = ssa->block_phis[sb]; phi; phi = phi->next) { + for (int pi = 0; pi < phi->num_operands; pi++) { + if (phi->operands[pi].pred_block != pred_block || phi->operands[pi].vreg < 0) + continue; + if (ra_phi_copy_needed(ir, phi, pi)) + count++; + } + } + } + return count; +} + +static int ra_collect_phi_copies_for_pred(TCCIRState *ir, IRCFG *cfg, IRSSAState *ssa, + int pred_block, int succ_filter, + const RAPhiPredIndex *idx, RAPhiCopy *copies) +{ + int copy_count = 0; + int first, last; + + if (succ_filter < 0 && idx->phi && pred_block >= 0 && pred_block < cfg->num_blocks) { + for (int e = idx->off[pred_block]; e < idx->off[pred_block + 1]; e++) { + IRPhiNode *phi = idx->phi[e]; + int pi = idx->opidx[e]; + if (!ra_phi_copy_needed(ir, phi, pi)) + continue; + copies[copy_count].dest_vreg = phi->dest_vreg; + copies[copy_count].src_vreg = phi->operands[pi].vreg; + copies[copy_count].btype = phi->btype; + copies[copy_count].emitted = 0; + copy_count++; + } + return copy_count; + } + + ra_phi_scan_range(cfg, succ_filter, &first, &last); + for (int sb = first; sb <= last; sb++) { + for (IRPhiNode *phi = ssa->block_phis[sb]; phi; phi = phi->next) { + for (int pi = 0; pi < phi->num_operands; pi++) { + if (phi->operands[pi].pred_block != pred_block || phi->operands[pi].vreg < 0) + continue; + if (!ra_phi_copy_needed(ir, phi, pi)) + continue; + copies[copy_count].dest_vreg = phi->dest_vreg; + copies[copy_count].src_vreg = phi->operands[pi].vreg; + copies[copy_count].btype = phi->btype; + copies[copy_count].emitted = 0; + copy_count++; + } + } + } + return copy_count; +} + +static void ra_emit_phi_copy(TCCIRState *ir, IRQuadCompact *new_instrs, int *wp, + int *pool_wp, const RAPhiCopy *copy, + RAPhiCopyRecord *records, int *record_count) +{ + IROperand dest_op; + memset(&dest_op, 0, sizeof(dest_op)); + irop_set_vreg(&dest_op, copy->dest_vreg); + dest_op.tag = IROP_TAG_VREG; + dest_op.btype = copy->btype; + + IROperand src_op; + memset(&src_op, 0, sizeof(src_op)); + irop_set_vreg(&src_op, copy->src_vreg); + src_op.tag = IROP_TAG_VREG; + src_op.btype = copy->btype; + + ir->iroperand_pool[*pool_wp] = dest_op; + ir->iroperand_pool[*pool_wp + 1] = src_op; + + if (records && record_count && *record_count < RA_MAX_PHI_COPY_RECORDS) { + records[*record_count].src_vreg = copy->src_vreg; + records[*record_count].new_instr_idx = *wp; + (*record_count)++; + } + + new_instrs[*wp].op = TCCIR_OP_ASSIGN; + new_instrs[*wp].operand_base = *pool_wp; + new_instrs[*wp].line_num = 0; + new_instrs[*wp].is_jump_target = 0; + (*wp)++; + *pool_wp += 2; +} + +static int ra_btype_stack_size(int btype) +{ + switch (btype) { + case IROP_BTYPE_INT64: + case IROP_BTYPE_FLOAT64: + return 8; + default: + return 4; + } +} + +static void ra_note_vreg_type_for_btype(TCCIRState *ir, int32_t vreg, int btype) +{ + if (!tcc_ir_vreg_is_valid(ir, vreg)) + return; + if (btype == IROP_BTYPE_FLOAT32 || btype == IROP_BTYPE_FLOAT64) + tcc_ir_vreg_type_set_fp(ir, vreg, 1, btype == IROP_BTYPE_FLOAT64); + else if (btype == IROP_BTYPE_INT64) + tcc_ir_vreg_type_set_64bit(ir, vreg); +} + +static void ra_record_stack_operand_min(IROperand op, int *min_offset) +{ + if (op.tag != IROP_TAG_STACKOFF) + return; + int off = irop_get_stack_offset(op); + if (off < *min_offset) + *min_offset = off; +} + +static int ra_find_phi_spill_cursor(TCCIRState *ir) +{ + int min_offset = 0; + for (int i = 0; i < ir->ls.next_interval_index; i++) { + LSLiveInterval *lsi = &ir->ls.intervals[i]; + if (lsi->stack_location < min_offset) + min_offset = lsi->stack_location; + } + for (int i = 0; i < ir->next_instruction_index; i++) { + IRQuadCompact *q = &ir->compact_instructions[i]; + if (irop_config[q->op].has_dest) + ra_record_stack_operand_min(tcc_ir_op_get_dest(ir, q), &min_offset); + if (irop_config[q->op].has_src1) + ra_record_stack_operand_min(tcc_ir_op_get_src1(ir, q), &min_offset); + if (irop_config[q->op].has_src2) + ra_record_stack_operand_min(tcc_ir_op_get_src2(ir, q), &min_offset); + if (tcc_ir_op_is_mac(q->op)) + ra_record_stack_operand_min(tcc_ir_op_get_accum(ir, q), &min_offset); + } + return min_offset; +} + +static int32_t ra_create_phi_temp(TCCIRState *ir, int btype, int start, int end, + int *phi_spill_cursor) +{ + int32_t tmp_vreg = tcc_ir_vreg_alloc_temp(ir); + if (tmp_vreg < 0) + return -1; + + int size = ra_btype_stack_size(btype); + *phi_spill_cursor -= size; + if (size > 4 && (*phi_spill_cursor & 7)) + *phi_spill_cursor &= ~7; + + ra_note_vreg_type_for_btype(ir, tmp_vreg, btype); + tcc_ls_add_live_interval(&ir->ls, tmp_vreg, start, end, 0, 0, + tcc_ir_vreg_type_get(ir, tmp_vreg), 0, -1); + LSLiveInterval *lsi = &ir->ls.intervals[ir->ls.next_interval_index - 1]; + lsi->r0 = -1; + lsi->r1 = -1; + lsi->stack_location = *phi_spill_cursor; + + tcc_ir_stack_reg_assign(ir, tmp_vreg, *phi_spill_cursor, -1, -1); + IRLiveInterval *li = tcc_ir_vreg_live_interval(ir, tmp_vreg); + if (li) { + li->start = (uint32_t)start; + li->end = (uint32_t)end; + } + + return tmp_vreg; +} + +static void ra_emit_scheduled_phi_copies(TCCIRState *ir, IRQuadCompact *new_instrs, + int *wp, int *pool_wp, RAPhiCopy *copies, + int copy_count, int block, + RAPhiCopyRecord *records, int *record_count, + int *phi_spill_cursor, RAPhiStats *stats) +{ + int start_wp = *wp; + int emitted = 0; + while (emitted < copy_count) { + int progress = 0; + for (int ci = 0; ci < copy_count; ci++) { + if (copies[ci].emitted) + continue; + if (ra_phi_copy_dest_clobbers_pending_source(ir, copies, copy_count, ci)) + continue; + ra_emit_phi_copy(ir, new_instrs, wp, pool_wp, &copies[ci], records, record_count); + copies[ci].emitted = 1; + emitted++; + progress = 1; + break; + } + + if (!progress) { + int ci; + for (ci = 0; ci < copy_count; ci++) + if (!copies[ci].emitted) + break; + if (ci >= copy_count) + break; + + int32_t saved_src = copies[ci].src_vreg; + int32_t tmp_vreg = ra_create_phi_temp(ir, copies[ci].btype, *wp, + *wp + copy_count - emitted, + phi_spill_cursor); + if (tmp_vreg < 0) { + RA_DBG("SSA phi resolver: failed to allocate cycle temp in block %d", block); + break; + } + + RAPhiCopy save = { + .dest_vreg = tmp_vreg, + .src_vreg = saved_src, + .btype = copies[ci].btype, + .emitted = 0, + }; + RA_DBG("SSA phi resolver: breaking cyclic parallel copy in block %d with T%d", + block, TCCIR_DECODE_VREG_POSITION(tmp_vreg)); + if (TCC_LOG_LS) { + stats->cycle_temporaries++; + ra_phi_stats_add_participant(stats, tmp_vreg); + } + ra_emit_phi_copy(ir, new_instrs, wp, pool_wp, &save, records, record_count); + copies[ci].src_vreg = tmp_vreg; + } + } + if (TCC_LOG_LS) + stats->parallel_emitted += *wp - start_wp; +} +void ra_resolve_phis(TCCIRState *ir, IRCFG *cfg, IRSSAState *ssa, + RAPhiStats *stats) +{ + int nb = cfg->num_blocks; + int old_n = ir->next_instruction_index; + + /* Count copies needed per predecessor block. + * + * Single-pass aggregation: walk every phi node once (O(total phi + * operands)) and bucket each operand into its predecessor block's + * counter. The previous version called ra_count_phi_copies_for_pred + * per predecessor, which itself looped over every block, producing + * O(blocks^2) work even when no phis existed — pathological for huge + * branch-heavy functions (compile/20001226-1 has 16K blocks). */ + int *copies_per_block = tcc_mallocz(nb * sizeof(int)); + int total_copies = 0; + /* For modified-JUMPIF detection we need the per-(pred,succ) count too, + * but only for blocks that ended in a JUMPIF AND have any copies on the + * target edge. Build a bitmap of (pred -> succ-block) edges that carry + * at least one phi copy. We use a simple flat array indexed by pred. */ + int *copies_to_jumpif_succ = tcc_mallocz(nb * sizeof(int)); + for (int b = 0; b < nb; b++) copies_to_jumpif_succ[b] = -1; + /* First, identify the JUMPIF-target succ_block per pred (if any). */ + for (int b = 0; b < nb; b++) { + IRBasicBlock *bb = &cfg->blocks[b]; + int last_instr = bb->end_idx - 1; + if (last_instr < bb->start_idx) continue; + if (ir->compact_instructions[last_instr].op != TCCIR_OP_JUMPIF) continue; + IROperand dest = tcc_ir_op_get_dest(ir, &ir->compact_instructions[last_instr]); + int old_target = (int)irop_get_imm64_ex(ir, dest); + int target_block = (old_target >= 0 && old_target < old_n) ? cfg->instr_to_block[old_target] : -1; + copies_to_jumpif_succ[b] = target_block; /* may be -1 */ + } + /* Now walk phis once. Per operand: increment copies_per_block[pred], + * and if pred's JUMPIF target == this phi's succ block, also note that + * the JUMPIF edge carries a copy. */ + int extra_jumps = 0; + int modified_jumpifs = 0; + uint8_t *jumpif_edge_has_copy = tcc_mallocz(nb); + for (int sb = 0; sb < nb; sb++) { + for (IRPhiNode *phi = ssa->block_phis[sb]; phi; phi = phi->next) { + for (int pi = 0; pi < phi->num_operands; pi++) { + if (TCC_LOG_LS) + stats->phi_operands++; + int pred = phi->operands[pi].pred_block; + if (pred < 0 || pred >= nb) continue; + if (phi->operands[pi].vreg < 0) continue; + if (!ra_phi_copy_needed(ir, phi, pi)) continue; + copies_per_block[pred]++; + total_copies++; + if (TCC_LOG_LS) { + stats->parallel_requested++; + ra_phi_stats_add_participant(stats, phi->dest_vreg); + ra_phi_stats_add_participant(stats, phi->operands[pi].vreg); + } + if (copies_to_jumpif_succ[pred] == sb) + jumpif_edge_has_copy[pred] = 1; + } + } + } + for (int b = 0; b < nb; b++) { + if (jumpif_edge_has_copy[b]) { + extra_jumps++; + modified_jumpifs++; + } + } + tcc_free(copies_to_jumpif_succ); + tcc_free(jumpif_edge_has_copy); + + RA_DBG("SSA phi resolver: total_copies=%d extra_jumps=%d", total_copies, extra_jumps); + for (int b = 0; b < nb; b++) { + if (copies_per_block[b] > 0) + RA_DBG(" block %d: %d copies", b, copies_per_block[b]); + } + + /* Dump all phi nodes for debugging */ + for (int sb = 0; sb < nb; sb++) { + for (IRPhiNode *phi = ssa->block_phis[sb]; phi; phi = phi->next) { + IRLiveInterval *dest_li = tcc_ir_vreg_live_interval(ir, phi->dest_vreg); + RA_DBG(" phi in block %d: dest=T%d (r0=%d off=%d) ops=%d", + sb, TCCIR_DECODE_VREG_POSITION(phi->dest_vreg), + dest_li ? dest_li->allocation.r0 : -99, + dest_li ? dest_li->allocation.offset : -99, + phi->num_operands); + for (int pi = 0; pi < phi->num_operands; pi++) { + IRLiveInterval *src_li = tcc_ir_vreg_is_valid(ir, phi->operands[pi].vreg) ? + tcc_ir_vreg_live_interval(ir, phi->operands[pi].vreg) : NULL; + int needed = ra_phi_copy_needed(ir, phi, pi); + RA_DBG(" op[%d]: src=T%d pred=%d (r0=%d off=%d) needed=%d", + pi, TCCIR_DECODE_VREG_POSITION(phi->operands[pi].vreg), + phi->operands[pi].pred_block, + src_li ? src_li->allocation.r0 : -99, + src_li ? src_li->allocation.offset : -99, + needed); + } + } + } + + if (total_copies == 0) { + tcc_free(copies_per_block); + return; + } + + /* Track emitted phi copies so we can extend source vreg live intervals */ + RAPhiCopyRecord *copy_records = tcc_mallocz(sizeof(RAPhiCopyRecord) * RA_MAX_PHI_COPY_RECORDS); + int copy_record_count = 0; + + /* Build new instruction array with phi copies inserted */ + int new_n = old_n + total_copies + extra_jumps; + int new_cap = new_n + 16; + new_cap += total_copies; + IRQuadCompact *new_instrs = tcc_mallocz(new_cap * sizeof(IRQuadCompact)); + int *old_to_new = tcc_malloc(old_n * sizeof(int)); + + /* Grow operand pool */ + int pool_base = ir->iroperand_pool_count; + int needed_pool = total_copies * 4 + extra_jumps + modified_jumpifs * 2; + while (pool_base + needed_pool > ir->iroperand_pool_capacity) { + int nc = ir->iroperand_pool_capacity ? ir->iroperand_pool_capacity * 2 : 256; + ir->iroperand_pool = tcc_realloc(ir->iroperand_pool, nc * sizeof(IROperand)); + ir->iroperand_pool_capacity = nc; + } + + RAPhiPredIndex phi_pred_idx; + ra_phi_pred_index_build(cfg, ssa, &phi_pred_idx); + + int wp = 0; + int pool_wp = pool_base; + int phi_spill_cursor = ra_find_phi_spill_cursor(ir); + int first_phi_temp_pos = ir->next_temporary_variable; + + for (int b = 0; b < nb; b++) { + IRBasicBlock *bb = &cfg->blocks[b]; + int last_instr = bb->end_idx - 1; + int insert_before = bb->end_idx; + if (last_instr >= bb->start_idx) { + TccIrOp op = ir->compact_instructions[last_instr].op; + if (op == TCCIR_OP_JUMP || op == TCCIR_OP_JUMPIF || + op == TCCIR_OP_RETURNVALUE || op == TCCIR_OP_RETURNVOID || + op == TCCIR_OP_IJUMP || op == TCCIR_OP_SWITCH_TABLE) + insert_before = last_instr; + } + + if (copies_per_block[b] > 0 && last_instr >= bb->start_idx && + ir->compact_instructions[last_instr].op == TCCIR_OP_JUMPIF) { + IRQuadCompact *term = &ir->compact_instructions[last_instr]; + IROperand old_dest = tcc_ir_op_get_dest(ir, term); + int old_target = (int)irop_get_imm64_ex(ir, old_dest); + int target_block = (old_target >= 0 && old_target < old_n) ? cfg->instr_to_block[old_target] : -1; + int fallthrough_block = (b + 1 < nb) ? b + 1 : -1; + int target_count = (target_block >= 0) ? ra_count_phi_copies_for_pred(ir, cfg, ssa, b, target_block) : 0; + int fallthrough_count = + (fallthrough_block >= 0) ? ra_count_phi_copies_for_pred(ir, cfg, ssa, b, fallthrough_block) : 0; + /* Every copy counted for this pred must land on the target or the + * fall-through edge. A shortfall means the CFG no longer matches + * ssa->block_phis (a pass retargeted a jump while phis were live) and + * a phi arm would be SILENTLY dropped — a guaranteed miscompile. + * Surface it loudly instead. */ + if (target_count + fallthrough_count < copies_per_block[b]) + fprintf(stderr, + "tcc: internal warning: %d phi cop%s dropped at block %d " + "(JUMPIF target %d -> block %d, fallthrough block %d) — " + "CFG/phi desync, expect wrong code\n", + copies_per_block[b] - target_count - fallthrough_count, + (copies_per_block[b] - target_count - fallthrough_count) == 1 ? "y" : "ies", + b, old_target, target_block, fallthrough_block); + + for (int i = bb->start_idx; i < last_instr; i++) { + old_to_new[i] = wp; + new_instrs[wp++] = ir->compact_instructions[i]; + } + + old_to_new[last_instr] = wp; + if (target_count > 0) { + IROperand cond = tcc_ir_op_get_src1(ir, term); + cond.u.imm32 = ra_invert_cond((int)irop_get_imm64_ex(ir, cond)); + + IROperand skip_dest = old_dest; + skip_dest.u.imm32 = -(wp + 1 + target_count + 1 + 1); + int skip_dest_pool_idx = pool_wp; + ir->iroperand_pool[pool_wp] = skip_dest; + ir->iroperand_pool[pool_wp + 1] = cond; + new_instrs[wp] = *term; + new_instrs[wp].operand_base = pool_wp; + wp++; + pool_wp += 2; + + RAPhiCopy *copies = tcc_malloc(sizeof(RAPhiCopy) * target_count); + int copy_count = ra_collect_phi_copies_for_pred(ir, cfg, ssa, b, target_block, &phi_pred_idx, copies); + ra_emit_scheduled_phi_copies(ir, new_instrs, &wp, &pool_wp, copies, copy_count, b, + copy_records, ©_record_count, &phi_spill_cursor, stats); + tcc_free(copies); + + ir->iroperand_pool[pool_wp] = old_dest; + new_instrs[wp].op = TCCIR_OP_JUMP; + new_instrs[wp].operand_base = pool_wp; + new_instrs[wp].line_num = term->line_num; + new_instrs[wp].is_jump_target = 0; + wp++; + pool_wp++; + + /* The inverted JUMPIF skips over the phi copies AND this back-edge JUMP, + * landing on the instruction right after the JUMP we just wrote (== wp + * now). Compute the skip target HERE, after the JUMP write, using the + * current wp — NOT before emitting the copies/JUMP. Reading wp earlier + * (the old `-(wp + 2)` before the copy emit) is fragile: the value used + * must reflect the copies just emitted via ra_emit_scheduled_phi_copies + * (which advances wp through &wp). Encode as the negative sentinel the + * "Fix jump targets" pass below decodes with `-old_target - 1`, so a + * target of `wp` is stored as `-(wp + 1)`. */ + skip_dest = ir->iroperand_pool[skip_dest_pool_idx]; + skip_dest.u.imm32 = -(wp + 1); + ir->iroperand_pool[skip_dest_pool_idx] = skip_dest; + } else { + new_instrs[wp++] = *term; + } + + if (fallthrough_count > 0) { + RAPhiCopy *copies = tcc_malloc(sizeof(RAPhiCopy) * fallthrough_count); + int copy_count = ra_collect_phi_copies_for_pred(ir, cfg, ssa, b, fallthrough_block, &phi_pred_idx, copies); + ra_emit_scheduled_phi_copies(ir, new_instrs, &wp, &pool_wp, copies, copy_count, b, + copy_records, ©_record_count, &phi_spill_cursor, stats); + tcc_free(copies); + } + continue; + } + + /* Keep conditional phi copies off the compare/test -> JUMPIF edge: + * physical-register copies can otherwise clobber the condition flags. */ + for (int i = bb->start_idx; i < insert_before; i++) { + old_to_new[i] = wp; + new_instrs[wp++] = ir->compact_instructions[i]; + } + + int pre_copy_wp = wp; + + /* Insert phi copies for this block's successors */ + if (copies_per_block[b] > 0) { + RAPhiCopy *copies = tcc_malloc(sizeof(RAPhiCopy) * copies_per_block[b]); + int copy_count = ra_collect_phi_copies_for_pred(ir, cfg, ssa, b, -1, &phi_pred_idx, copies); + ra_emit_scheduled_phi_copies(ir, new_instrs, &wp, &pool_wp, copies, copy_count, b, + copy_records, ©_record_count, &phi_spill_cursor, stats); + tcc_free(copies); + } + + /* Copy terminator and remaining instructions */ + for (int i = insert_before; i < bb->end_idx; i++) { + if (i == bb->start_idx && copies_per_block[b] > 0) + old_to_new[i] = pre_copy_wp; + else + old_to_new[i] = wp; + new_instrs[wp++] = ir->compact_instructions[i]; + } + } + + ir->iroperand_pool_count = pool_wp; + + ra_phi_pred_index_free(&phi_pred_idx); + + /* Fix jump targets */ + for (int i = 0; i < wp; i++) { + IRQuadCompact *q = &new_instrs[i]; + if (q->op == TCCIR_OP_JUMP || q->op == TCCIR_OP_JUMPIF) { + IROperand dest = tcc_ir_op_get_dest(ir, q); + int old_target = (int)irop_get_imm64_ex(ir, dest); + if (old_target < 0) { + dest.u.imm32 = -old_target - 1; + ir->iroperand_pool[q->operand_base] = dest; + } else if (old_target >= 0 && old_target < old_n) { + dest.u.imm32 = old_to_new[old_target]; + ir->iroperand_pool[q->operand_base] = dest; + } else if (old_target >= old_n) { + dest.u.imm32 = wp + (old_target - old_n); + ir->iroperand_pool[q->operand_base] = dest; + } + } + } + + /* Fix switch table targets */ + for (int t = 0; t < ir->num_switch_tables; t++) { + TCCIRSwitchTable *table = &ir->switch_tables[t]; + for (int ti = 0; ti < table->num_entries; ti++) { + int ot = table->targets[ti]; + if (ot >= 0 && ot < old_n) table->targets[ti] = old_to_new[ot]; + } + if (table->default_target >= 0 && table->default_target < old_n) + table->default_target = old_to_new[table->default_target]; + } + + /* Replace instruction array */ + tcc_free(ir->compact_instructions); + ir->compact_instructions = new_instrs; + ir->compact_instructions_size = new_cap; + ir->next_instruction_index = wp; + + /* Rebuild is_jump_target flags */ + for (int i = 0; i < wp; i++) + new_instrs[i].is_jump_target = 0; + for (int i = 0; i < wp; i++) { + IRQuadCompact *q = &new_instrs[i]; + if (q->op == TCCIR_OP_JUMP || q->op == TCCIR_OP_JUMPIF) { + IROperand dest = tcc_ir_op_get_dest(ir, q); + int tgt = (int)irop_get_imm64_ex(ir, dest); + if (tgt >= 0 && tgt < wp) new_instrs[tgt].is_jump_target = 1; + } + } + for (int t = 0; t < ir->num_switch_tables; t++) { + TCCIRSwitchTable *table = &ir->switch_tables[t]; + for (int ti = 0; ti < table->num_entries; ti++) { + int tgt = table->targets[ti]; + if (tgt >= 0 && tgt < wp) new_instrs[tgt].is_jump_target = 1; + } + } + + /* The remaining steps (live-interval remap/extend, live_regs bitmap) + * only apply when phi resolution runs after register allocation. + * Pre-RA, no LS intervals or bitmap exist yet — skip. ra_build_intervals + * will scan the freshly emitted ASSIGN copies and produce correct + * intervals from scratch. + * + * Clear ssa->block_phis: the explicit copies we just inserted are now + * the source of truth. Leaving phi nodes active confuses the interval + * builder (it tries to extend phi-dest intervals as if the phi were + * still semantically active, on top of the now-explicit defs). */ + if (ra_phi_resolve_pre_ra_mode) { + /* Free each block's phi list before detaching it — the explicit copies are + * now the source of truth, so these nodes are dead. Merely NULLing the + * heads (as before) orphaned every phi node + operand array: tcc_ir_ssa_free + * later sees an empty block_phis and frees nothing, leaking on every compile. */ + for (int b = 0; b < nb; b++) { + IRPhiNode *phi = ssa->block_phis[b]; + while (phi) { + IRPhiNode *next = phi->next; + tcc_free(phi->operands); + tcc_free(phi); + phi = next; + } + ssa->block_phis[b] = NULL; + } + tcc_free(old_to_new); + tcc_free(copies_per_block); + tcc_free(copy_records); + return; + } + + /* Remap live interval start/end */ + for (int i = 0; i < ir->ls.next_interval_index; i++) { + LSLiveInterval *lsi = &ir->ls.intervals[i]; + int32_t vreg = lsi->vreg; + IRLiveInterval *li = tcc_ir_vreg_live_interval(ir, vreg); + if (!li) continue; + if (TCCIR_DECODE_VREG_TYPE(vreg) == TCCIR_VREG_TYPE_TEMP && + TCCIR_DECODE_VREG_POSITION(vreg) >= first_phi_temp_pos) + continue; + /* Remap start */ + if (li->start < (uint32_t)old_n) { + li->start = old_to_new[li->start]; + lsi->start = li->start; + } + /* Remap end */ + if (li->end < (uint32_t)old_n) { + li->end = old_to_new[li->end]; + lsi->end = li->end; + } + } + + tcc_free(old_to_new); + tcc_free(copies_per_block); + + /* Extend source vreg live intervals to cover phi copy instructions. + * Phi copies read the source vreg at the copy's instruction index, + * so the source must be alive there. Without this extension, the + * scratch register allocator may reuse the source's register. */ + for (int i = 0; i < copy_record_count; i++) { + int32_t sv = copy_records[i].src_vreg; + int ci = copy_records[i].new_instr_idx; + IRLiveInterval *li = tcc_ir_vreg_live_interval(ir, sv); + if (!li) continue; + if ((int)li->end < ci) { + RA_DBG(" phi copy: extending vreg 0x%x end from %d to %d", sv, (int)li->end, ci); + li->end = ci; + } + for (int j = 0; j < ir->ls.next_interval_index; j++) { + LSLiveInterval *lsi = &ir->ls.intervals[j]; + if (lsi->vreg == sv && (int)lsi->end < ci) { + lsi->end = ci; + break; + } + } + } + tcc_free(copy_records); + + /* Re-extend intervals for backward jumps in the post-phi instruction stream. + * The original backward-jump extension was computed on pre-phi indices; phi + * copies inserted before a backward jump are therefore not covered. Any + * register live at the jump target must also be considered live during the + * phi copies so that the scratch-register allocator does not clobber it. */ + { + int new_n = ir->next_instruction_index; + for (int i = 0; i < new_n; i++) { + IRQuadCompact *q = &ir->compact_instructions[i]; + if (q->op == TCCIR_OP_JUMP || q->op == TCCIR_OP_JUMPIF) { + int target = (int)tcc_ir_op_get_dest(ir, q).u.imm32; + if (target >= 0 && target < new_n && target < i) { + for (int j = 0; j < ir->ls.next_interval_index; j++) { + LSLiveInterval *lsi = &ir->ls.intervals[j]; + IRLiveInterval *li = tcc_ir_vreg_live_interval(ir, lsi->vreg); + if (!li) continue; + if ((int)li->start <= target && (int)li->end >= target && + (int)li->end < i) { + RA_DBG(" post-phi back-edge extend: vreg 0x%x [%d,%d] -> [%d,%d]", + lsi->vreg, (int)li->start, (int)li->end, (int)li->start, i); + li->end = i; + lsi->end = i; + } + } + } + } + } + } + + /* Build live_regs_by_instruction table */ + ra_build_live_regs_bitmap(ir); +} +void ra_build_live_regs_bitmap(TCCIRState *ir) +{ + uint32_t max_end = 0; + for (int i = 0; i < ir->ls.next_interval_index; i++) { + LSLiveInterval *lsi = &ir->ls.intervals[i]; + if (lsi->stack_location != 0 || lsi->r0 < 0) continue; + if (lsi->reg_type != LS_REG_TYPE_INT && lsi->reg_type != LS_REG_TYPE_LLONG && + lsi->reg_type != LS_REG_TYPE_DOUBLE_SOFT && lsi->reg_type != LS_REG_TYPE_COMPLEX_FLOAT) + continue; + if (lsi->end > max_end) max_end = lsi->end; + } + int sz = (int)max_end + 1; + if (sz < ir->next_instruction_index) sz = ir->next_instruction_index; + if (sz > 0) { + if (ir->ls.live_regs_by_instruction) + tcc_free(ir->ls.live_regs_by_instruction); + ir->ls.live_regs_by_instruction = tcc_mallocz(sizeof(uint32_t) * sz); + ir->ls.live_regs_by_instruction_size = sz; + RA_DBG("SSA live_regs_by_instruction build (sz=%d)", sz); + for (int i = 0; i < ir->ls.next_interval_index; i++) { + LSLiveInterval *lsi = &ir->ls.intervals[i]; + if (lsi->stack_location != 0 || lsi->r0 < 0) continue; + if (lsi->reg_type != LS_REG_TYPE_INT && lsi->reg_type != LS_REG_TYPE_LLONG && + lsi->reg_type != LS_REG_TYPE_DOUBLE_SOFT && lsi->reg_type != LS_REG_TYPE_COMPLEX_FLOAT) + continue; + uint32_t mask = 0; + if (lsi->r0 >= 0 && lsi->r0 < 16) mask |= (1u << lsi->r0); + if (lsi->r1 >= 0 && lsi->r1 < 16) mask |= (1u << lsi->r1); + if (!mask) continue; + int s = (int)lsi->start, e = (int)lsi->end; + if (s < 0) s = 0; + if (e >= sz) e = sz - 1; + RA_DBG(" interval vreg=0x%x r0=R%d r1=%d range=[%d,%d] mask=0x%x", + lsi->vreg, lsi->r0, lsi->r1, s, e, mask); + for (int k = s; k <= e; k++) + ir->ls.live_regs_by_instruction[k] |= mask; + } + + if (TCC_LOG_LS) { + for (int k = 0; k < sz; k++) + RA_DBG(" instr[%d] live=0x%x", k, ir->ls.live_regs_by_instruction[k]); + } + } +} + +/* ============================================================================ + * Graph-based register coalescing + * + * The linear scan's per-decision coalescing (boundary / loop-phi / exit-phi + * "transfer") merges only simple 2-vreg copy chains; it cannot merge a + * multi-predecessor merge-phi (e.g. an induction variable that, after loop + * rotation, enters the next loop via two edges). And ra_build_intervals' + * single-[start,end] model reports FALSE overlaps for SSA versions that are + * live on mutually-exclusive paths, so they look like they interfere. + * + * This pass builds ACCURATE liveness (backward dataflow over a fresh CFG) and a + * real interference graph, then conservatively coalesces copy-related + * non-interfering vregs (union-find). Merged classes are applied via the + * `coalesce_to` interval-merge: one representative interval covers the union of + * the class' ranges, non-reps are skipped by the scan and inherit the rep's + * register, and the residual `mov R,R` is erased by tcc_ir_move_coalescing. + * + * On by default at -O1+ (level 2 = apply, INT single-register, pure copies). + * Overrides: TCC_NO_COALESCE disables it; TCC_COALESCE=N forces level N + * (1 = compute only, 2 = apply, 3 = also coalesce RMW two-address edges). + * Bails on functions with un-enumerated CFG edges (IJUMP / SWITCH_*) or any + * 64-bit (LLONG) value. + * ============================================================================ */ + +TCC_DBG_ENV_FLAG(ra_no_coalesce, "TCC_NO_COALESCE") +TCC_DBG_ENV_INT(ra_coalesce_env_level, "TCC_COALESCE", 2) /* default: apply, pure copies */ +TCC_DBG_ENV_FLAG(ra_dump_ir_co, "DUMP_IR_CO") +TCC_DBG_ENV_FLAG(ra_no_coalesce_llong, "TCC_NO_COALESCE_LLONG") +TCC_DBG_ENV_FLAG(ra_no_coalesce_gapless, "TCC_NO_COALESCE_GAPLESS") + +static int ra_coalesce_level(void) +{ + return ra_no_coalesce() ? 0 : ra_coalesce_env_level(); +} + +/* Collect an instruction's def and use vreg operands into out-params. + * STORE-class dest is a USE (it is the address); MLA accumulator is a USE. + * Matches ra_build_intervals' operand semantics exactly. */ +void ra_co_ops(TCCIRState *ir, IRQuadCompact *q, + int32_t *out_def, int *has_def, int32_t uses[4], int *nuse) +{ + *has_def = 0; + *nuse = 0; + if (q->op == TCCIR_OP_NOP) return; + int store_class = (q->op == TCCIR_OP_STORE || q->op == TCCIR_OP_STORE_INDEXED || + q->op == TCCIR_OP_STORE_POSTINC); + if (irop_config[q->op].has_src1) { + IROperand s = tcc_ir_op_get_src1(ir, q); + if (irop_has_vreg(s) && !irop_is_immediate(s)) uses[(*nuse)++] = irop_get_vreg(s); + } + if (irop_config[q->op].has_src2) { + IROperand s = tcc_ir_op_get_src2(ir, q); + if (irop_has_vreg(s) && !irop_is_immediate(s)) uses[(*nuse)++] = irop_get_vreg(s); + } + if (tcc_ir_op_is_mac(q->op)) { + IROperand s = tcc_ir_op_get_accum(ir, q); + if (irop_has_vreg(s) && !irop_is_immediate(s)) uses[(*nuse)++] = irop_get_vreg(s); + } + if (irop_config[q->op].has_dest) { + IROperand d = tcc_ir_op_get_dest(ir, q); + if (irop_has_vreg(d)) { + /* A plain STORE whose TEMP dest has is_lval clear is SSA renaming's + * in-place register write of a promoted var (a pointer-deref store + * keeps is_lval) — a DEF, matching ra_build_intervals. Full-width + * only: a narrow store leaves the old upper bytes live-before. */ + int inplace_def = (q->op == TCCIR_OP_STORE && !d.is_lval && !d.is_local && + TCCIR_DECODE_VREG_TYPE(irop_get_vreg(d)) == TCCIR_VREG_TYPE_TEMP && + (d.btype == IROP_BTYPE_INT32 || d.btype == IROP_BTYPE_FLOAT32 || + d.btype == IROP_BTYPE_INT64 || d.btype == IROP_BTYPE_FLOAT64)); + if (store_class && !inplace_def) uses[(*nuse)++] = irop_get_vreg(d); + else { *out_def = irop_get_vreg(d); *has_def = 1; } + } + } +} + +/* Static counts of references from 16-bit-encodable ops; docs/regalloc_narrow_pref.md */ +void ra_build_narrow_weights(TCCIRState *ir, const RegAllocTarget *target, + SSAInterval *intervals, int count, int max_vreg_pos) +{ + if (!target->op_narrow_capable || tcc_state->optimize < 1 || count <= 0 || max_vreg_pos <= 0) + return; + if (tcc_ir_opt_pass_disabled("ra:narrow_pref")) + return; + int tbl = 4 * max_vreg_pos; + int32_t *iv_of = tcc_malloc(sizeof(int32_t) * tbl); + for (int i = 0; i < tbl; i++) iv_of[i] = -1; + for (int i = 0; i < count; i++) { + int idx = TCCIR_DECODE_VREG_TYPE(intervals[i].vreg) * max_vreg_pos + + TCCIR_DECODE_VREG_POSITION(intervals[i].vreg); + if (idx >= 0 && idx < tbl) iv_of[idx] = i; + } + for (int i = 0; i < ir->next_instruction_index; i++) { + IRQuadCompact *q = &ir->compact_instructions[i]; + if (q->op == TCCIR_OP_NOP) continue; + int src2_imm = 0, scale = 0; + if (irop_config[q->op].has_src2) + src2_imm = irop_is_immediate(tcc_ir_op_get_src2(ir, q)); + if (q->op == TCCIR_OP_LOAD_INDEXED || q->op == TCCIR_OP_STORE_INDEXED) { + IROperand s = tcc_ir_op_get_scale(ir, q); + if (irop_get_tag(s) == IROP_TAG_IMM32) scale = s.u.imm32; + } + if (!target->op_narrow_capable(q->op, src2_imm, scale)) continue; + int32_t def, uses[4]; + int has_def, nuse; + ra_co_ops(ir, q, &def, &has_def, uses, &nuse); + int32_t ops[5]; + int nops = 0; + if (has_def) ops[nops++] = def; + for (int k = 0; k < nuse; k++) ops[nops++] = uses[k]; + for (int k = 0; k < nops; k++) { + int idx = TCCIR_DECODE_VREG_TYPE(ops[k]) * max_vreg_pos + + TCCIR_DECODE_VREG_POSITION(ops[k]); + if (idx < 0 || idx >= tbl || iv_of[idx] < 0) continue; + SSAInterval *iv = &intervals[iv_of[idx]]; + if (iv->narrow_uses < UINT16_MAX) iv->narrow_uses++; + } + } + tcc_free(iv_of); +} + +/* Backward-liveness bit matrix: nb blocks x tbl vreg-slots, one allocation, + * auto-freed on every exit path. Rows feed the tcc_bitspan_* word kernels. */ +TCC_BIT_MATRIX_DEFINE(RaLiveMatrix) + +/* Refine live_regs_by_instruction (the interval-derived approximation the + * scratch-register picker consults) with ACCURATE per-instruction liveness from + * a real CFG backward dataflow. + * + * The interval bitmap models each value as one contiguous [start,end] range. + * For a loop-carried value (defined inside a rotated loop body and live across + * the back-edge into the next iteration) that single range does NOT span the + * loop-header prefix where the value is still live, so the bitmap under-reports + * the value's register as free there. The scratch picker then hands it out and + * clobbers the loop-carried value (random-C O2 wrong-code / HardFault once loop + * rotation is enabled — Finding #15 follow-up, seeds 244 et al). + * + * This dataflow (same ra_co_ops def/use model the graph-coalescer trusts) marks + * every register holding a genuinely-live, register-resident vreg. It is + * strictly conservative for the picker: it can only ADD live bits, never remove + * them, so it can never introduce a new clobber — it only prevents real ones. + * Bails (leaving the interval bitmap as-is) on functions with un-enumerated + * edges (IJUMP / SWITCH_TABLE), matching the coalescer's own guard. */ +void ra_refine_live_regs_accurate(TCCIRState *ir) +{ + int n = ir->next_instruction_index; + if (n <= 0) return; + int has_backedge = 0; + for (int i = 0; i < n; i++) { + IRQuadCompact *q = &ir->compact_instructions[i]; + int op = q->op; + if (op == TCCIR_OP_IJUMP || op == TCCIR_OP_SWITCH_TABLE || op == TCCIR_OP_SWITCH_LOAD) + return; + if (op == TCCIR_OP_JUMP || op == TCCIR_OP_JUMPIF) { + int t = (int)tcc_ir_op_get_dest(ir, q).u.imm32; + if (t >= 0 && t < i) + has_backedge = 1; + } + } + if (!has_backedge) return; + IRCFG *cfg = tcc_ir_cfg_build(ir); + if (!cfg) return; + tcc_ir_cfg_compute_rpo(cfg); /* the dataflow needs only the RPO ordering */ + int nb = cfg->num_blocks; + if (nb <= 0) { tcc_ir_cfg_free(cfg); return; } + /* vreg index space */ + int maxpos = 1; + for (int j = 0; j < ir->ls.next_interval_index; j++) { + int p = TCCIR_DECODE_VREG_POSITION(ir->ls.intervals[j].vreg); + if (p + 1 > maxpos) maxpos = p + 1; + } + int tbl = 4 * maxpos; + #define DVIDX(vr) ((TCCIR_DECODE_VREG_TYPE(vr) * maxpos) + TCCIR_DECODE_VREG_POSITION(vr)) + /* vreg -> physical regs */ + int8_t *vr0 = tcc_malloc(tbl); int8_t *vr1 = tcc_malloc(tbl); + for (int i = 0; i < tbl; i++) { vr0[i] = -1; vr1[i] = -1; } + for (int j = 0; j < ir->ls.next_interval_index; j++) { + LSLiveInterval *iv = &ir->ls.intervals[j]; + if (iv->stack_location != 0) continue; + int vi = DVIDX(iv->vreg); + if (vi < 0 || vi >= tbl) continue; + vr0[vi] = (int8_t)iv->r0; vr1[vi] = (int8_t)iv->r1; + } + /* Only register-resident slots can contribute to the final mask, and only a + * slot read before any def in some block (upward-exposed) can ever be live-in + * at a block; liveness is per-vreg independent, so restricting the dataflow + * universe to slots that are both is exact. dense[] first holds each slot's + * last defining block while the blocks are scanned for upward-exposed uses. */ + int *dense = tcc_malloc(sizeof(int) * tbl); + uint8_t *upward = tcc_mallocz(tbl); + for (int i = 0; i < tbl; i++) dense[i] = -1; + for (int b = 0; b < nb; b++) { + int s = cfg->blocks[b].start_idx, e = cfg->blocks[b].end_idx; + for (int i = s; i < e && i < n; i++) { + int32_t def=-1, hd=0, uses[4], nu=0; + ra_co_ops(ir, &ir->compact_instructions[i], &def, &hd, uses, &nu); + for (int k=0;k=0&&u=0&&d= 0 && vr0[i] < 16) || (vr1[i] >= 0 && vr1[i] < 16); + dense[i] = tracked && upward[i] ? ndense++ : -1; + } + tcc_free(upward); + if (ndense == 0) { + tcc_free(vr0); tcc_free(vr1); tcc_free(dense); + tcc_ir_cfg_free(cfg); + return; + } + int nw = (ndense + 63) / 64; + bit_matrix(RaLiveMatrix) useb = {0}; + bit_matrix(RaLiveMatrix) defbk = {0}; + bit_matrix(RaLiveMatrix) livein = {0}; + bit_matrix(RaLiveMatrix) liveout = {0}; + RaLiveMatrix_init(&useb, nb, ndense); + RaLiveMatrix_init(&defbk, nb, ndense); + RaLiveMatrix_init(&livein, nb, ndense); + RaLiveMatrix_init(&liveout, nb, ndense); + for (int b = 0; b < nb; b++) { + uint64_t *ub = RaLiveMatrix_row(&useb, b), *db = RaLiveMatrix_row(&defbk, b); + int s = cfg->blocks[b].start_idx, e = cfg->blocks[b].end_idx; + for (int i = s; i < e && i < n; i++) { + int32_t def=-1, hd=0, uses[4], nu=0; + ra_co_ops(ir, &ir->compact_instructions[i], &def, &hd, uses, &nu); + for (int k=0;k=tbl)continue; int ud=dense[u]; if(ud<0)continue; if(!tcc_bitspan_test(db,ud)) tcc_bitspan_set(ub,ud);} + if (hd && tcc_ir_vreg_is_valid(ir,def)){int d=DVIDX(def); if(d>=0&&d=0) tcc_bitspan_set(db,dense[d]);} + } + } + int changed=1, guard=0; + while (changed && guard++ < nb+4) { + changed=0; + for (int ri=cfg->rpo_count-1; ri>=0; ri--) { + int b = cfg->rpo_order ? cfg->rpo_order[ri] : ri; + if (b<0||b>=nb) continue; + uint64_t *lo = RaLiveMatrix_row(&liveout, b), *li = RaLiveMatrix_row(&livein, b); + uint64_t *ub = RaLiveMatrix_row(&useb, b), *db = RaLiveMatrix_row(&defbk, b); + tcc_bitspan_zero(lo, nw); + for (int si=0;siblocks[b].num_succs;si++){int sb=cfg->blocks[b].succs[si]; if(sb<0||sb>=nb)continue; tcc_bitspan_or(lo, RaLiveMatrix_row(&livein, sb), nw);} + changed |= tcc_bitspan_or_andnot(li, ub, lo, db, nw); + } + } + /* Only live_in is read from here on. */ + RaLiveMatrix_cleanup(&useb); + RaLiveMatrix_cleanup(&defbk); + RaLiveMatrix_cleanup(&liveout); + /* Loop-liveness completion. A value live at a loop header is live throughout + * the ENTIRE loop body (it round-trips the back-edge), but the interval model + * gives it a single [def,last-use] range that leaves the loop-header prefix + * uncovered — the scratch picker then reuses its register inside the loop and + * clobbers the loop-carried value (seed 244). For each back-edge, OR the + * registers live-IN at the loop header across the whole loop body [header, + * back-edge]. Scoped to loop bodies on purpose: a blanket per-instruction + * live-out refinement also marks straight-line liveness the interval model + * intentionally omits, which over-constrains the scratch picker and perturbs + * unrelated functions into latent-bug territory (seed 221). */ + for (int bi = 0; bi < n; bi++) { + IRQuadCompact *q = &ir->compact_instructions[bi]; + if (q->op != TCCIR_OP_JUMP && q->op != TCCIR_OP_JUMPIF) continue; + int t = (int)tcc_ir_op_get_dest(ir, q).u.imm32; + if (t < 0 || t >= bi) continue; /* not a back-edge */ + /* live-in at the loop header t: find the block starting at t. */ + int hb = -1; + for (int b = 0; b < nb; b++) if (cfg->blocks[b].start_idx == t) { hb = b; break; } + if (hb < 0) continue; + uint64_t *hli = RaLiveMatrix_row(&livein, hb); + uint32_t mask = 0; + for (int vi=0; vi=0 && vr0[vi]<16) mask |= (1u<=0 && vr1[vi]<16) mask |= (1u<ls.live_regs_by_instruction) continue; + int e = bi; if (e >= ir->ls.live_regs_by_instruction_size) e = ir->ls.live_regs_by_instruction_size - 1; + for (int k = t; k <= e; k++) + ir->ls.live_regs_by_instruction[k] |= mask; + } + #undef DVIDX + tcc_free(vr0);tcc_free(vr1);tcc_free(dense); + tcc_ir_cfg_free(cfg); +} + +/* 1 when the member intervals of the class rooted at `root` cover their + * union [min start, max end] without a gap. */ +static int ra_co_class_gapless(const SSAInterval *intervals, const int *iv_of, const int *cand_vidx, + const int *mnext, int root) +{ + int nm = 0; + for (int m = root; m >= 0; m = mnext[m]) + nm++; + uint32_t *st = tcc_malloc(sizeof(uint32_t) * 2 * nm); + int k = 0; + for (int m = root; m >= 0; m = mnext[m], k++) + { + const SSAInterval *iv = &intervals[iv_of[cand_vidx[m]]]; + st[2 * k] = iv->start; + st[2 * k + 1] = iv->end; + } + /* insertion sort by start: classes are small */ + for (int a = 1; a < nm; a++) + for (int b = a; b > 0 && st[2 * b] < st[2 * (b - 1)]; b--) + { + uint32_t t0 = st[2 * b], t1 = st[2 * b + 1]; + st[2 * b] = st[2 * (b - 1)]; + st[2 * b + 1] = st[2 * (b - 1) + 1]; + st[2 * (b - 1)] = t0; + st[2 * (b - 1) + 1] = t1; + } + int ok = 1; + uint32_t reach = st[1]; + for (int a = 1; a < nm && ok; a++) + { + if (st[2 * a] > reach + 1) + ok = 0; + if (st[2 * a + 1] > reach) + reach = st[2 * a + 1]; + } + tcc_free(st); + return ok; +} +void ra_coalesce_graph(TCCIRState *ir, SSAInterval *intervals, int count, + int max_vreg_pos) +{ + int level = ra_coalesce_level(); + if (level <= 0 || tcc_state->optimize < 1 || count <= 1 || max_vreg_pos <= 0) + return; + int n = ir->next_instruction_index; + if (n <= 0) return; + TCC_DBG_BLOCK(ra_dump_ir_co) { printf("==== IR AT GRAPH-COALESCE ====\n"); tcc_ir_show(ir); fflush(stdout); } + + /* ---- Stage 1: fresh CFG (entry cfg is stale after phi resolution). ---- */ + IRCFG *cfg = tcc_ir_cfg_build(ir); + if (!cfg) return; + tcc_ir_cfg_compute_dominators(cfg); /* populates rpo_order/rpo_count for the dataflow */ + if (cfg->rpo_count <= 0) { tcc_ir_cfg_free(cfg); return; } + for (int i = 0; i < n; i++) { + int op = ir->compact_instructions[i].op; + /* IJUMP (computed goto) has un-enumerable successors — live-out is + * untrustworthy, bail. SWITCH_TABLE is fine: tcc_ir_cfg_build models + * every dispatch edge (targets[] is pre-filled with the default target + * for gap values, and out-of-range never reaches the dispatch — the + * frontend guards it with an explicit JUMPIF). SWITCH_LOAD is a plain + * table-value LOAD, not a jump. Bailing on switches was the single + * blocker for the 64-bit switch-merge family: every case temp kept its + * own spill slot + str/str/ldr/ldr round-trip (ashrdi-1 constant_shift: + * 488-byte frame vs GCC's zero). */ + if (op == TCCIR_OP_IJUMP) { + tcc_ir_cfg_free(cfg); + return; + } + } + int nb = cfg->num_blocks; + int tbl = 4 * max_vreg_pos; + int nw = (tbl + 63) / 64; + /* Guard pathologically large functions (compile-time / memory). */ + if (nb <= 0 || (long)nb * nw > (4L << 20)) { tcc_ir_cfg_free(cfg); return; } + + /* Historical note: this used to bail on any function containing a 64-bit + * (LLONG) interval, blaming the UMULL half-operand spill path for a + * bug_ull_mul miscompile under coalescing. The actual culprits were the + * deferred-PARAM liveness hole and the backedge_phi_hoist side-entry + * retarget (both since fixed); with the pair-aware pressure gate below, + * LLONG functions coalesce safely — including LLONG-LLONG pair copies + * (admitted in the union gate). TCC_NO_COALESCE_LLONG restores the old + * bail for attribution. */ + if (ra_no_coalesce_llong()) + for (int i = 0; i < count; i++) + if (intervals[i].reg_type == LS_REG_TYPE_LLONG) { tcc_ir_cfg_free(cfg); return; } + + #define VIDX(vr) ((TCCIR_DECODE_VREG_TYPE(vr) * max_vreg_pos) + TCCIR_DECODE_VREG_POSITION(vr)) + + int *iv_of = tcc_malloc(sizeof(int) * tbl); + for (int i = 0; i < tbl; i++) iv_of[i] = -1; + for (int i = 0; i < count; i++) { + int idx = VIDX(intervals[i].vreg); + if (idx >= 0 && idx < tbl) iv_of[idx] = i; + } + + /* ---- Deferred-PARAM uses: model FUNCPARAMVAL sources at their CALL. ---- + * Argument marshaling happens at the call site, so a FUNCPARAMVAL source + * is read again at the matching FUNCCALL — after any instructions between + * the PARAM quad and the CALL. ra_build_intervals extends interval ends + * accordingly; this liveness must do the same or the interference graph + * misses the overlap and a coalesced def between PARAM and CALL clobbers + * the argument before the call reads it (bug_stride_minimal: the CSE'd + * `i+1` loop increment merged with `i` while `printf`'s deferred PARAM + * still needed the old `i`). cid matching mirrors ra_build_intervals. */ + int *call_param_head = tcc_malloc(sizeof(int) * n); + int *param_sibling = tcc_malloc(sizeof(int) * n); + for (int i = 0; i < n; i++) { call_param_head[i] = -1; param_sibling[i] = -1; } + for (int i = 0; i < n; i++) { + IRQuadCompact *q = &ir->compact_instructions[i]; + if (q->op != TCCIR_OP_FUNCPARAMVAL) continue; + IROperand s1 = tcc_ir_op_get_src1(ir, q); + if (!irop_has_vreg(s1) || irop_is_immediate(s1)) continue; + int cid = TCCIR_DECODE_CALL_ID(irop_get_imm64_ex(ir, tcc_ir_op_get_src2(ir, q))); + if (cid < 0) continue; + for (int j = i + 1; j < n; j++) { + IRQuadCompact *qq = &ir->compact_instructions[j]; + if (qq->op != TCCIR_OP_FUNCCALLVOID && qq->op != TCCIR_OP_FUNCCALLVAL) continue; + int ccid = TCCIR_DECODE_CALL_ID(irop_get_imm64_ex(ir, tcc_ir_op_get_src2(ir, qq))); + if (ccid == cid) { + param_sibling[i] = call_param_head[j]; + call_param_head[j] = i; + break; + } + } + } + /* Iterate a CALL's deferred param-source vregs: body receives vreg in `vr_`. */ + #define RA_CO_FOR_CALL_PARAM_USES(call_idx, vr_, body) \ + do { \ + for (int p_ = call_param_head[(call_idx)]; p_ >= 0; p_ = param_sibling[p_]) { \ + IROperand ps_ = tcc_ir_op_get_src1(ir, &ir->compact_instructions[p_]); \ + int32_t vr_ = irop_get_vreg(ps_); \ + if (!tcc_ir_vreg_is_valid(ir, vr_)) continue; \ + body \ + } \ + } while (0) + + /* ---- Stage 2: backward liveness dataflow (live_in/live_out per block). ---- + * Only a vreg with an upward-exposed use in some block (read there before + * any def in that block) can be live across a block boundary: live_in(b) = + * use(b) ∪ (live_out(b) − def(b)) and live_out is the union of successor + * live_ins, so a vreg in no use(b) is in no live_in and no live_out. The + * dataflow runs over those "global" vregs only (dense index g, glob_vidx: + * g -> VIDX), which is exact and keeps the four nb-row matrices a fraction + * of nb x tbl bits -- over the whole VIDX space they were 4 x 7.8 MB for one + * function of tinycc's own ir/codegen.c, the peak of that compile. */ + int *glob_of = tcc_malloc(sizeof(int) * tbl); + int *def_stamp = tcc_malloc(sizeof(int) * tbl); + for (int i = 0; i < tbl; i++) { glob_of[i] = -1; def_stamp[i] = -1; } + int ng = 0; + #define RA_CO_NOTE_USE(u) \ + do { \ + if ((u) >= 0 && (u) < tbl && def_stamp[(u)] != b && glob_of[(u)] < 0) \ + glob_of[(u)] = ng++; \ + } while (0) + for (int b = 0; b < nb; b++) { + int s = cfg->blocks[b].start_idx, e = cfg->blocks[b].end_idx; + for (int i = s; i < e && i < n; i++) { + int32_t def = -1, hd = 0, uses[4], nu = 0; + ra_co_ops(ir, &ir->compact_instructions[i], &def, &hd, uses, &nu); + for (int k = 0; k < nu; k++) + if (tcc_ir_vreg_is_valid(ir, uses[k])) RA_CO_NOTE_USE(VIDX(uses[k])); + RA_CO_FOR_CALL_PARAM_USES(i, vr, { RA_CO_NOTE_USE(VIDX(vr)); }); + if (hd && tcc_ir_vreg_is_valid(ir, def)) { + int d = VIDX(def); + if (d >= 0 && d < tbl) def_stamp[d] = b; + } + } + } + #undef RA_CO_NOTE_USE + tcc_free(def_stamp); + int *glob_vidx = tcc_malloc(sizeof(int) * (ng + 1)); + for (int i = 0; i < tbl; i++) + if (glob_of[i] >= 0) glob_vidx[glob_of[i]] = i; + int ngw = (ng + 63) / 64; + + /* use(b) / def(b) are a handful of globals per block, so they are kept as + * per-block lists (CSR: blk_off[b] .. blk_off[b+1] into use_g / def_g), and + * live_in(s) = use(s) ∪ (live_out(s) − def(s)) is rebuilt from them when a + * predecessor reads it: live_out is the only nb-row matrix. */ + int *use_off = tcc_malloc(sizeof(int) * (nb + 1)); + int *def_off = tcc_malloc(sizeof(int) * (nb + 1)); + int use_cap = 64, def_cap = 64, nuse = 0, ndef = 0; + int *use_g = tcc_malloc(sizeof(int) * use_cap); + int *def_g = tcc_malloc(sizeof(int) * def_cap); + int *g_def_blk = tcc_malloc(sizeof(int) * (ng + 1)); + int *g_use_blk = tcc_malloc(sizeof(int) * (ng + 1)); + for (int g = 0; g < ng; g++) { g_def_blk[g] = -1; g_use_blk[g] = -1; } + /* upward-exposed use of global g in block b (read before any def in b) */ + #define RA_CO_ADD_USE(g_) \ + do { \ + int gg_ = (g_); \ + if (gg_ >= 0 && g_def_blk[gg_] != b && g_use_blk[gg_] != b) { \ + g_use_blk[gg_] = b; \ + if (nuse == use_cap) use_g = tcc_realloc(use_g, sizeof(int) * (use_cap *= 2)); \ + use_g[nuse++] = gg_; \ + } \ + } while (0) + for (int b = 0; b < nb; b++) { + use_off[b] = nuse; + def_off[b] = ndef; + int s = cfg->blocks[b].start_idx, e = cfg->blocks[b].end_idx; + for (int i = s; i < e && i < n; i++) { + int32_t def = -1, hd = 0, uses[4], nu = 0; + ra_co_ops(ir, &ir->compact_instructions[i], &def, &hd, uses, &nu); + for (int k = 0; k < nu; k++) { + if (!tcc_ir_vreg_is_valid(ir, uses[k])) continue; + int u = VIDX(uses[k]); + if (u >= 0 && u < tbl) RA_CO_ADD_USE(glob_of[u]); + } + RA_CO_FOR_CALL_PARAM_USES(i, vr, { + int u = VIDX(vr); + if (u >= 0 && u < tbl) RA_CO_ADD_USE(glob_of[u]); + }); + if (hd && tcc_ir_vreg_is_valid(ir, def)) { + int d = VIDX(def); + int g = (d >= 0 && d < tbl) ? glob_of[d] : -1; + if (g >= 0 && g_def_blk[g] != b) { + g_def_blk[g] = b; + if (ndef == def_cap) def_g = tcc_realloc(def_g, sizeof(int) * (def_cap *= 2)); + def_g[ndef++] = g; + } + } + } + } + use_off[nb] = nuse; + def_off[nb] = ndef; + #undef RA_CO_ADD_USE + tcc_free(g_def_blk); + tcc_free(g_use_blk); + tcc_free(glob_of); + + bit_matrix(RaLiveMatrix) liveout = {0}; + RaLiveMatrix_init(&liveout, nb, ng); + uint64_t *in_row = tcc_malloc(sizeof(uint64_t) * (ngw + 1)); + uint64_t *out_row = tcc_malloc(sizeof(uint64_t) * (ngw + 1)); + + /* Fixpoint over reverse-RPO order. */ + int changed = 1, guard = 0; + while (changed && guard++ < nb + 4) { + changed = 0; + for (int ri = cfg->rpo_count - 1; ri >= 0; ri--) { + int b = cfg->rpo_order ? cfg->rpo_order[ri] : ri; + if (b < 0 || b >= nb) continue; + /* live_out = union of successors' live_in */ + tcc_bitspan_zero(out_row, ngw); + for (int si = 0; si < cfg->blocks[b].num_succs; si++) { + int sb = cfg->blocks[b].succs[si]; + if (sb < 0 || sb >= nb) continue; + /* live_in(sb) = use ∪ (live_out − def) */ + tcc_bitspan_copy(in_row, RaLiveMatrix_row(&liveout, sb), ngw); + for (int k = def_off[sb]; k < def_off[sb + 1]; k++) tcc_bitspan_reset(in_row, def_g[k]); + for (int k = use_off[sb]; k < use_off[sb + 1]; k++) tcc_bitspan_set(in_row, use_g[k]); + tcc_bitspan_or(out_row, in_row, ngw); + } + uint64_t *lo = RaLiveMatrix_row(&liveout, b); + if (ngw && memcmp(lo, out_row, sizeof(uint64_t) * ngw)) { + tcc_bitspan_copy(lo, out_row, ngw); + changed = 1; + } + } + } + tcc_free(in_row); tcc_free(out_row); + tcc_free(use_off); tcc_free(def_off); tcc_free(use_g); tcc_free(def_g); + #define RA_CO_LIVE_OUT(live_, b_) \ + do { \ + tcc_bitspan_zero((live_), nw); \ + tcc_bitspan_for_each_set(RaLiveMatrix_row(&liveout, (b_)), ngw, ng, g_) \ + tcc_bitspan_set((live_), glob_vidx[g_]); \ + } while (0) + + /* ---- Register-pressure gate. ---- + * Coalescing can only ever ADD instructions (vs the baseline) by forcing a + * spill: merging two non-interfering values reduces distinct values at every + * point EXCEPT a liveness hole, where the merged interval occupies the + * register and raises pressure by one. If the function's peak INT pressure + * leaves headroom (< K allocatable int regs), no spill can result, so + * coalescing is a pure win (it only removes copies). When pressure already + * reaches K (spilling territory), coalescing's longer intervals can perturb + * the linear scan into worse spills (observed: large high-pressure functions + * regress). In that case only a class whose members' ranges leave no gap + * is applied (high_pressure, checked at Stage 5): it occupies the register + * exactly where one of its members already did, so no point of the scan + * sees one more live interval. That still takes a loop's `i <- i - 1` and + * `acc <- acc << 8 | b` latch copies, which lie inside the phi's range -- + * the whole-function bail used to drop them in every function with one + * high-pressure point (Zig's Wyhash.hash: a u128 multiply). */ + int high_pressure = 0; + { + int K = tcc_state->registers_for_allocator; + if (K <= 0 || K > 13) K = 13; + /* Weight per value: how many INT registers it occupies while live. + * LLONG / soft-double / complex-float take a pair; complex-double takes + * four. Counting pairs as 1 (the old behavior) under-approximates + * pressure in 64-bit-heavy functions and lets coalescing push them into + * spill territory. */ + uint8_t *iw = tcc_mallocz(tbl); + for (int i = 0; i < count; i++) { + int idx = VIDX(intervals[i].vreg); + if (idx < 0 || idx >= tbl) continue; + switch (intervals[i].reg_type) { + case LS_REG_TYPE_INT: + if (intervals[i].r1 < 0) iw[idx] = 1; + break; + case LS_REG_TYPE_LLONG: + case LS_REG_TYPE_DOUBLE_SOFT: + case LS_REG_TYPE_COMPLEX_FLOAT: + iw[idx] = 2; + break; + case LS_REG_TYPE_COMPLEX_DOUBLE: + iw[idx] = 4; + break; + default: + break; /* VFP-resident: no INT pressure */ + } + } + uint64_t *live = tcc_malloc(sizeof(uint64_t) * nw); + int maxp = 0; + for (int b = 0; b < nb && maxp < K; b++) { + int p = 0; + RA_CO_LIVE_OUT(live, b); + tcc_bitspan_for_each_set(live, nw, tbl, vi) { + p += iw[vi]; + } + int s = cfg->blocks[b].start_idx, e = cfg->blocks[b].end_idx; + for (int i = e - 1; i >= s && i < n; i--) { + if (p > maxp) maxp = p; + IRQuadCompact *q = &ir->compact_instructions[i]; + int32_t def = -1, hd = 0, uses[4], nu = 0; + ra_co_ops(ir, q, &def, &hd, uses, &nu); + if (hd && tcc_ir_vreg_is_valid(ir, def)) { + int d = VIDX(def); + if (d >= 0 && d < tbl && tcc_bitspan_test(live, d)) { p -= iw[d]; tcc_bitspan_reset(live, d); } + } + for (int k = 0; k < nu; k++) { + if (!tcc_ir_vreg_is_valid(ir, uses[k])) continue; + int u = VIDX(uses[k]); + if (u >= 0 && u < tbl && !tcc_bitspan_test(live, u)) { p += iw[u]; tcc_bitspan_set(live, u); } + } + RA_CO_FOR_CALL_PARAM_USES(i, vr, { + int u = VIDX(vr); + if (u >= 0 && u < tbl && !tcc_bitspan_test(live, u)) { p += iw[u]; tcc_bitspan_set(live, u); } + }); + } + } + tcc_free(iw); tcc_free(live); + if (maxp >= K) { + RA_DBG("coalesce: peak INT pressure %d >= K=%d, gapless classes only", maxp, K); + if (ra_no_coalesce_gapless()) { + tcc_free(iv_of); tcc_free(glob_vidx); + tcc_free(call_param_head); tcc_free(param_sibling); + tcc_ir_cfg_free(cfg); + return; + } + high_pressure = 1; + } + } + + /* ---- Build per-block def bitmap to detect phi-related copies. ---- + * A copy dest that is defined in more than one block is a phi result + * (explicit copies inserted after SSA phi resolution). Coalescing such + * a dest with its source can overwrite the source's value on a sibling + * phi arm when the source is still live across the merge (seed 860). */ + uint8_t *def_block_count = tcc_mallocz(tbl); + int *def_last_block = tcc_malloc(sizeof(int) * tbl); + for (int i = 0; i < tbl; i++) def_last_block[i] = -1; + int *instr_block = tcc_malloc(sizeof(int) * n); + for (int i = 0; i < n; i++) instr_block[i] = -1; + /* Per-value linked list of def positions (head/next), so per-edge admission + * checks walk only that value's defs instead of rescanning all n + * instructions (quadratic on big functions — 101_cleanup 0.3s -> 19s). */ + int *def_head = tcc_malloc(sizeof(int) * tbl); + int *def_next = tcc_malloc(sizeof(int) * n); + for (int i = 0; i < tbl; i++) def_head[i] = -1; + for (int i = 0; i < n; i++) def_next[i] = -1; + for (int b = 0; b < nb; b++) { + int s = cfg->blocks[b].start_idx, e = cfg->blocks[b].end_idx; + for (int i = s; i < e && i < n; i++) { + instr_block[i] = b; + IRQuadCompact *q = &ir->compact_instructions[i]; + int32_t def = -1, hd = 0, uses[4], nu = 0; + ra_co_ops(ir, q, &def, &hd, uses, &nu); + if (hd && tcc_ir_vreg_is_valid(ir, def)) { + int d = VIDX(def); + if (d >= 0 && d < tbl && def_last_block[d] != b) { + def_last_block[d] = b; + if (def_block_count[d] < 2) def_block_count[d]++; + } + } + } + } + tcc_free(def_last_block); + /* Def list covers ALL instructions (not just CFG-block ranges) so the + * admission walks see exactly the defs the previous full scans saw — + * a def outside any block (instr_block -1) must keep rejecting. */ + for (int i = 0; i < n; i++) { + IRQuadCompact *q = &ir->compact_instructions[i]; + int32_t def = -1, hd = 0, uses[4], nu = 0; + ra_co_ops(ir, q, &def, &hd, uses, &nu); + if (hd && tcc_ir_vreg_is_valid(ir, def)) { + int d = VIDX(def); + if (d >= 0 && d < tbl) { + def_next[i] = def_head[d]; + def_head[d] = i; + } + } + } + + /* ---- Collect copy edges + candidate set (Stage 4 prep). ---- */ + /* Copy edge kinds: ASSIGN dst<-src; two-address dst<-src OP imm (ADD/SUB). */ + int *cand_id = tcc_malloc(sizeof(int) * tbl); + for (int i = 0; i < tbl; i++) cand_id[i] = -1; + int ncand = 0; + int ecap = 16, ne = 0; + int32_t *edge_d = tcc_malloc(sizeof(int32_t) * ecap); + int32_t *edge_s = tcc_malloc(sizeof(int32_t) * ecap); + #define ADD_CAND(vidx) do { if (cand_id[vidx] < 0) cand_id[vidx] = ncand++; } while (0) + for (int i = 0; i < n; i++) { + IRQuadCompact *q = &ir->compact_instructions[i]; + int32_t dv = -1, sv = -1; + if (q->op == TCCIR_OP_ASSIGN) { + IROperand d = tcc_ir_op_get_dest(ir, q), s = tcc_ir_op_get_src1(ir, q); + /* Pure register copy only: a dereferenced operand means this ASSIGN is a + * load/store (`T1 = *T0`), NOT a copy — coalescing its operands is wrong. */ + if (irop_has_vreg(d) && irop_has_vreg(s) && !irop_is_immediate(s) && + !d.is_lval && !s.is_lval) { + dv = irop_get_vreg(d); sv = irop_get_vreg(s); + } + } else if ((q->op == TCCIR_OP_ADD || q->op == TCCIR_OP_SUB) && level >= 3) { + IROperand d = tcc_ir_op_get_dest(ir, q), s1 = tcc_ir_op_get_src1(ir, q), + s2 = tcc_ir_op_get_src2(ir, q); + if (irop_has_vreg(d) && irop_has_vreg(s1) && !irop_is_immediate(s1) && + irop_is_immediate(s2)) { /* dst <- src OP imm (RMW, two-address) */ + dv = irop_get_vreg(d); sv = irop_get_vreg(s1); + } + } + if (dv < 0 || sv < 0 || dv == sv) continue; + if (!tcc_ir_vreg_is_valid(ir, dv) || !tcc_ir_vreg_is_valid(ir, sv)) continue; + /* A volatile VAR must stay memory-resident (every access a real ldr/str); + * coalescing it into a register would elide its mandated loads/stores. */ + { + IRLiveInterval *dvi = tcc_ir_vreg_live_interval(ir, dv); + IRLiveInterval *svi = tcc_ir_vreg_live_interval(ir, sv); + if ((dvi && dvi->is_volatile) || (svi && svi->is_volatile)) + continue; + } + int di = VIDX(dv), si = VIDX(sv); + if (di < 0 || di >= tbl || si < 0 || si >= tbl) continue; + if (iv_of[di] < 0 || iv_of[si] < 0) continue; /* both must have intervals */ + /* Reject unsafe phi-result copies: the dest is defined on multiple + * incoming edges. The dangerous case is when the source temp is itself + * a copy of a VAR that is live-out of the merge block; coalescing the + * phi result with that source (transitively with the VAR) lets a sibling + * phi arm overwrite the still-live VAR (seed 860). Latch-style loop + * phis, where the source is computed in the latch, are unaffected. */ + { + if (def_block_count[di] > 1) { + /* Allow phi-copy coalescing only for loop phis. Conditional-merge + * phis can have a source equivalent to a variable live across the + * merge; coalescing them lets the sibling arm overwrite that variable + * (seed 860). Loop phis are safe because the latch source is computed + * fresh each iteration and dies at the copy. Two placements qualify: + * - the copy sits in the loop header itself (a predecessor is a back + * edge: the copy's block dominates that predecessor), or + * - the copy sits in a latch (a successor is reached by a back edge: + * that successor dominates the copy's block). This is where phi + * resolution places the `IV <- IV+1` copy of a rotated or + * bottom-tested loop, previously rejected here — the single + * largest source of residual `adds rX,rY,#i; mov rY,rX` pairs. */ + int bi = instr_block[i]; + int is_loop_phi = 0; + if (bi >= 0 && bi < nb) { + for (int pi = 0; pi < cfg->blocks[bi].num_preds; pi++) { + int pb = cfg->blocks[bi].preds[pi]; + if (pb >= 0 && pb < nb && tcc_ir_cfg_dominates(cfg, bi, pb)) { + is_loop_phi = 1; + break; + } + } + for (int sx = 0; sx < cfg->blocks[bi].num_succs && !is_loop_phi; sx++) { + int sb = cfg->blocks[bi].succs[sx]; + if (sb >= 0 && sb < nb && sb != bi && tcc_ir_cfg_dominates(cfg, sb, bi)) + is_loop_phi = 1; + } + /* Loop-ENTRY copy (split critical edge into a loop header): the + * copy block has a unique successor that dominates every OTHER + * def block of the dest — i.e. all sibling defs (the latch + * copies) execute strictly after this copy, so they can only + * overwrite a value the copy already delivered. The parallel + * arms of a conditional merge (seed 860) fail this test: a + * sibling arm's def block is never dominated by the merge. */ + if (!is_loop_phi && cfg->blocks[bi].num_succs == 1) { + int sb = cfg->blocks[bi].succs[0]; + if (sb >= 0 && sb < nb && sb != bi) { + int all_dominated = 1; + for (int k = def_head[di]; k >= 0 && all_dominated; k = def_next[k]) { + if (k == i) continue; + int kb = instr_block[k]; + if (kb == bi) continue; /* same-block sibling: executes after us */ + if (kb < 0 || kb >= nb || !tcc_ir_cfg_dominates(cfg, sb, kb)) + all_dominated = 0; + } + if (all_dominated) + is_loop_phi = 1; + } + } + /* Diamond-arm copy whose source is dead at every sibling def of + * the dest: no OTHER def of the dest falls inside the source's + * live interval [start, end), so overwriting the class register + * on a parallel arm can never clobber a value that arm (or any + * path from it) still reads. Interval ranges are back-edge + * extended by ra_build_intervals, so linear containment is a + * conservative test under loops. Catches the SETIF-temp -> + * arm-copy -> phi-dest chain (960402-1: class inherits the r0 + * RETURNVALUE preference) and BOTH arms of a value-select diamond + * (arith-1::sat_add — admitting only one arm is worse than none: + * co_member disables the in-scan hint transfer and the surviving + * real copy blocks post_ra_forward_diamond). The seed-860 hazard + * (source equivalent to a value live ACROSS the merge) fails this + * test at the sibling def; its transitive form (source already + * coalesced with a live-through value) is refused by the + * class-level interference check at union time. */ + if (!is_loop_phi && bi >= 0 && iv_of[si] >= 0) { + uint32_t s_start = intervals[iv_of[si]].start; + uint32_t s_end = intervals[iv_of[si]].end; + int safe = 1; + for (int k = def_head[di]; k >= 0 && safe; k = def_next[k]) { + if (k == i) continue; + if ((uint32_t)k >= s_start && (uint32_t)k < s_end) + safe = 0; + } + if (safe) + is_loop_phi = 1; + } + } + if (!is_loop_phi) { + continue; + } + } + } + ADD_CAND(di); ADD_CAND(si); + if (ne >= ecap) { ecap *= 2; edge_d = tcc_realloc(edge_d, sizeof(int32_t)*ecap); + edge_s = tcc_realloc(edge_s, sizeof(int32_t)*ecap); } + edge_d[ne] = di; edge_s[ne] = si; ne++; + } + + tcc_free(def_block_count); + tcc_free(def_head); + tcc_free(def_next); + + if (ncand < 2 || ne == 0) { + tcc_free(iv_of); tcc_free(glob_vidx); tcc_free(instr_block); + tcc_free(cand_id); tcc_free(edge_d); tcc_free(edge_s); + tcc_free(call_param_head); tcc_free(param_sibling); + tcc_ir_cfg_free(cfg); + return; + } + + /* Reverse map: candidate index -> VIDX. */ + int *cand_vidx = tcc_malloc(sizeof(int) * ncand); + for (int i = 0; i < tbl; i++) if (cand_id[i] >= 0) cand_vidx[cand_id[i]] = i; + + /* ---- Stage 3: interference among candidates (per-def live-out). ---- */ + /* Two passes to build CSR adjacency: count degrees, then fill. */ + int *deg = tcc_mallocz(sizeof(int) * ncand); + uint64_t *live = tcc_malloc(sizeof(uint64_t) * nw); + + for (int pass = 0; pass < 2; pass++) { + int *adj_start = NULL, *adj = NULL, total = 0; + if (pass == 1) { + adj_start = tcc_malloc(sizeof(int) * (ncand + 1)); + adj_start[0] = 0; + for (int c = 0; c < ncand; c++) adj_start[c + 1] = adj_start[c] + deg[c]; + total = adj_start[ncand]; + adj = total ? tcc_malloc(sizeof(int) * total) : tcc_malloc(1); + for (int c = 0; c < ncand; c++) deg[c] = adj_start[c]; /* reuse as write cursor */ + } + for (int b = 0; b < nb; b++) { + RA_CO_LIVE_OUT(live, b); + int s = cfg->blocks[b].start_idx, e = cfg->blocks[b].end_idx; + for (int i = e - 1; i >= s && i < n; i--) { + IRQuadCompact *q = &ir->compact_instructions[i]; + int32_t def = -1, hd = 0, uses[4], nu = 0; + ra_co_ops(ir, q, &def, &hd, uses, &nu); + /* Copy source to exclude from interference for this def. ONLY for a + * pure copy `D <- S` (D == S after, so they don't interfere here). For + * an RMW `D <- S OP imm` the result differs from S, so if S is live-out + * it genuinely interferes with D — must NOT be excluded. */ + int32_t copy_src = -1; + if (q->op == TCCIR_OP_ASSIGN) { + IROperand s1 = tcc_ir_op_get_src1(ir, q); + IROperand dd = tcc_ir_op_get_dest(ir, q); + /* Only a pure register copy `D <- S` (no deref) makes D and S equal; + * a deref ASSIGN is a load/store, so its operands genuinely interfere. */ + if (irop_has_vreg(s1) && !irop_is_immediate(s1) && !s1.is_lval && !dd.is_lval) + copy_src = irop_get_vreg(s1); + } + if (hd && tcc_ir_vreg_is_valid(ir, def)) { + int d = VIDX(def); + if (d >= 0 && d < tbl && cand_id[d] >= 0) { + int cd = cand_id[d]; + int csrc = (copy_src >= 0 && tcc_ir_vreg_is_valid(ir, copy_src)) ? VIDX(copy_src) : -1; + tcc_bitspan_for_each_set(live, nw, tbl, vi) { + int c = cand_id[vi]; + if (c < 0) continue; + if (vi == d) continue; + if (csrc >= 0 && vi == csrc) continue; + if (pass == 0) { deg[cd]++; deg[c]++; } + else { adj[deg[cd]++] = c; adj[deg[c]++] = cd; } + } + } + } + /* transition live: kill def, gen uses */ + if (hd && tcc_ir_vreg_is_valid(ir, def)) { + int d = VIDX(def); if (d >= 0 && d < tbl) tcc_bitspan_reset(live, d); + } + for (int k = 0; k < nu; k++) { + if (!tcc_ir_vreg_is_valid(ir, uses[k])) continue; + int u = VIDX(uses[k]); if (u >= 0 && u < tbl) tcc_bitspan_set(live, u); + } + RA_CO_FOR_CALL_PARAM_USES(i, vr, { + int u = VIDX(vr); if (u >= 0 && u < tbl) tcc_bitspan_set(live, u); + }); + } + } + if (pass == 1) { + /* ---- Stage 4: union-find conservative coalescing. ---- */ + int *parent = tcc_malloc(sizeof(int) * ncand); + int *rank = tcc_mallocz(sizeof(int) * ncand); + int *mnext = tcc_malloc(sizeof(int) * ncand); /* member linked list per root */ + int *seen = tcc_mallocz(sizeof(int) * ncand); /* generation-stamped neighbor set */ + int gen = 0; + for (int c = 0; c < ncand; c++) { parent[c] = c; mnext[c] = -1; } + #define UF_FIND(x) ({ int _r = (x); while (parent[_r] != _r) { parent[_r] = parent[parent[_r]]; _r = parent[_r]; } _r; }) + + /* K for the Briggs degree test = number of allocatable int regs. */ + int K = tcc_state->registers_for_allocator; + if (K <= 0 || K > 13) K = 13; + + int merged = 0; + for (int ei = 0; ei < ne; ei++) { + int ca = cand_id[edge_d[ei]], cb = cand_id[edge_s[ei]]; + if (ca < 0 || cb < 0) continue; + int ra = UF_FIND(ca), rb = UF_FIND(cb); + if (ra == rb) continue; + SSAInterval *ia = &intervals[iv_of[cand_vidx[ra]]]; + SSAInterval *ib = &intervals[iv_of[cand_vidx[rb]]]; + /* gate: same-type classes only — INT single-reg with INT, LLONG pair + * with LLONG pair. A mixed edge is a truncating/widening copy (e.g. + * `T9 <- T8` extracting the low half of a 64-bit value): the interval + * model cannot express half-pair aliasing, so those never merge. + * DOUBLE_SOFT pairs were tried and measured NET WORSE (+220 corpus, + * cdivchk* +11): merged classes extend across __aeabi_d* calls and + * defeat the in-scan return-pair r0:r1 preference (the pr58574 + * lever), which handles soft-float chains better than a class can. + * Complex types stay out too (rare; pair-sym marshaling subtleties). + * Not precolored/addrtaken/param/spill-fixed. */ + if (ia->reg_type != ib->reg_type) continue; + if (ia->reg_type != LS_REG_TYPE_INT && ia->reg_type != LS_REG_TYPE_LLONG) continue; + if (ia->r1 >= 0 || ib->r1 >= 0) continue; + if (ia->addrtaken || ib->addrtaken || ia->precolored >= 0 || ib->precolored >= 0) continue; + if (ia->is_param || ib->is_param) continue; + /* non-interference: no member of class ra interferes with class rb */ + int interferes = 0; + for (int m = ra; m >= 0 && !interferes; m = mnext[m]) { + for (int a = adj_start[m]; a < adj_start[m + 1]; a++) { + if (UF_FIND(adj[a]) == rb) { interferes = 1; break; } + } + } + if (interferes) continue; + /* Conservative pressure test: reject if the merged class would have >= K + * DISTINCT interference-neighbor classes — such a node may be impossible + * to color, so coalescing it risks forcing a spill (a size regression). + * Counting all distinct neighbor classes (not just high-degree ones) is a + * safe over-approximation of Briggs; the IV web has few neighbors so it + * still coalesces, while high-pressure webs are correctly left alone. */ + { + /* Neighbor classes are weighted by register footprint (INT = 1, + * LLONG pair = 2), and a pair-class merge needs a free adjacent + * pair, so its headroom threshold drops by one. DOUBLE_SOFT + * neighbors deliberately stay weight 1: weighting them 2 measured + * +264 corpus — soft-double values rarely stay live across the + * merged range (they cycle through __aeabi_d* r0:r1 returns), so + * counting them as resident pairs over-blocks INT/LLONG merges. */ + int limit = (ia->reg_type == LS_REG_TYPE_LLONG) ? K - 1 : K; + int over = 0, distinct = 0; + gen++; + for (int side = 0; side < 2 && !over; side++) { + int r = side ? rb : ra; + for (int m = r; m >= 0 && !over; m = mnext[m]) { + for (int a = adj_start[m]; a < adj_start[m + 1]; a++) { + int nr = UF_FIND(adj[a]); + if (nr == ra || nr == rb) continue; + if (seen[nr] != gen) { + seen[nr] = gen; + SSAInterval *niv = &intervals[iv_of[cand_vidx[nr]]]; + distinct += (niv->reg_type == LS_REG_TYPE_LLONG) ? 2 : 1; + if (distinct >= limit) { over = 1; break; } + } + } + } + } + if (over) continue; + } + /* union (rank) + splice member lists */ + if (rank[ra] < rank[rb]) { int t = ra; ra = rb; rb = t; } + parent[rb] = ra; + if (rank[ra] == rank[rb]) rank[ra]++; + int tail = ra; while (mnext[tail] >= 0) tail = mnext[tail]; + mnext[tail] = rb; + merged++; + } + + /* ---- Stage 5: apply via coalesce_to interval-merge. ---- */ + if (level >= 2 && merged > 0) { + /* For each root with >1 member, choose rep = earliest-start interval, + * extend its range to the union, and flag the rest. */ + for (int c = 0; c < ncand; c++) { + if (UF_FIND(c) != c) continue; /* roots only */ + if (mnext[c] < 0) continue; /* singleton */ + /* gather members, pick rep */ + int rep_iv = -1; uint32_t lo_s = 0xffffffffu, hi_e = 0; + int xcall = 0; uint32_t uc = 0, nuc = 0, sum_len = 0; + int pref = -1; uint32_t pref_end = 0; + for (int m = c; m >= 0; m = mnext[m]) { + int ivi = iv_of[cand_vidx[m]]; + SSAInterval *iv = &intervals[ivi]; + if (iv->start < lo_s) lo_s = iv->start; + if (iv->end > hi_e) hi_e = iv->end; + xcall |= iv->crosses_call; + uc += iv->use_count; + nuc += iv->narrow_uses; + sum_len += iv->end - iv->start + 1; + /* Keep the LATEST-ending member's soft register preference: the + * r0-for-RETURNVALUE hint matters at the class's end boundary, + * and the member feeding the return is the one that ends last. + * Without this the merged class loses the hint (co_member skips + * the in-scan hint transfer) and both diamond arms compute into + * a scratch reg plus a final `mov r0, rX` (960402-1 +2). */ + if (iv->pref_reg >= 0 && (pref < 0 || iv->end >= pref_end)) { + pref = iv->pref_reg; pref_end = iv->end; + } + if (rep_iv < 0 || iv->start < intervals[rep_iv].start) rep_iv = ivi; + } + if (rep_iv < 0) continue; + /* Density gate: the rep covers the CONTIGUOUS union [lo,hi], occupying + * the register across any gaps between members. If the union is much + * larger than the members' combined live length, those gaps are holes + * the merge fills — raising register pressure and risking spills (a + * size regression). Members of one value have OVERLAPPING (false) + * ranges, so sum_len >= union_len for the cases worth merging; reject + * when the union exceeds the members' total (a real hole). */ + if ((hi_e - lo_s + 1) > sum_len) + continue; + if (high_pressure && !ra_co_class_gapless(intervals, iv_of, cand_vidx, mnext, c)) + continue; + intervals[rep_iv].start = lo_s; + intervals[rep_iv].end = hi_e; + intervals[rep_iv].crosses_call = xcall ? 1 : 0; + intervals[rep_iv].use_count = (uc > 65535) ? 65535 : (uint16_t)uc; + intervals[rep_iv].narrow_uses = (nuc > 65535) ? 65535 : (uint16_t)nuc; + if (pref >= 0 && intervals[rep_iv].pref_reg < 0) + intervals[rep_iv].pref_reg = (int8_t)pref; + /* Store the representative's VREG (stable across the scan's qsort), + * not its array index. */ + int32_t rep_vreg = intervals[rep_iv].vreg; + intervals[rep_iv].co_member = 1; + for (int m = c; m >= 0; m = mnext[m]) { + int ivi = iv_of[cand_vidx[m]]; + intervals[ivi].co_member = 1; + if (ivi != rep_iv) intervals[ivi].coalesce_to = rep_vreg; + } + RA_DBG("coalesce: root class -> rep T%d [%u,%u] xcall=%d", + TCCIR_DECODE_VREG_POSITION(intervals[rep_iv].vreg), lo_s, hi_e, xcall); + } + } + RA_DBG("coalesce: %d candidates, %d edges, %d unions (level=%d)", ncand, ne, merged, level); + + tcc_free(parent); tcc_free(rank); tcc_free(mnext); tcc_free(seen); + #undef UF_FIND + } + if (pass == 1) { tcc_free(adj_start); tcc_free(adj); } + } + + #undef ADD_CAND + #undef VIDX + #undef RA_CO_LIVE_OUT + tcc_free(deg); tcc_free(live); + tcc_free(iv_of); tcc_free(glob_vidx); + tcc_free(instr_block); + tcc_free(cand_id); tcc_free(cand_vidx); tcc_free(edge_d); tcc_free(edge_s); + tcc_free(call_param_head); tcc_free(param_sibling); + #undef RA_CO_FOR_CALL_PARAM_USES + tcc_ir_cfg_free(cfg); +} diff --git a/source/ir/regalloc_post.c b/source/ir/regalloc_post.c new file mode 100644 index 000000000..6b6b45803 --- /dev/null +++ b/source/ir/regalloc_post.c @@ -0,0 +1,2018 @@ +/* + * TCC IR - SSA-Aware Register Allocator + * + * Operates directly on SSA-renamed IR with phi nodes. + * Replaces the tccls.c linear scan when -fssa-regalloc is enabled. + * + * Copyright (c) 2025 Mateusz Stadnik + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +/* SSA register allocator: post-allocation passes -- redundant reload and + * dead frame-store elimination, copy propagation and move coalescing. */ + +#include "regalloc_priv.h" + +/* ============================================================================ + * Post-Allocation Redundant Reload Elimination + * + * Every soft-float routine opens by punning its arguments through a union: + * + * StackLoc[-8] <-- R0(P0) [STORE] ; ua.d = a + * R0(V0) <-- StackLoc[-8] [LOAD] ; a_bits = ua.u + * + * which lowers to `strd r0,r1,[sp,#N]` immediately followed by + * `ldrd r0,r1,[sp,#N]` -- a reload of registers that already hold the value. + * sl_forward refuses to forward it because the two ends have different types + * (FLOAT64 -> INT64), and forwarding it BEFORE allocation was measured and + * reverted: keeping the punned value in a register stretched a 64-bit live + * range and the linear-scan allocator cascaded into spills (dadd +13.4%). + * + * After allocation that trade no longer exists. The registers are already + * chosen, so dropping a load that rewrites a register with the value it + * already holds cannot lengthen anything -- it only removes a memory access. + * + * The analysis is a straight-line available-value table: remember which + * register pair was last stored to each frame slot, drop a later load of that + * slot into exactly those registers, and invalidate on anything that could + * disturb either side. + * ==========================================================================*/ + +/* An allocation half carries its physical register in the low five bits and a + * spill flag at 0x20, but an UNALLOCATED half is the all-ones sentinel -- and + * that sentinel has the spill bit set. So "is this half spilled" is only a + * question once the half names a register; asked unconditionally it answers + * yes for the absent high half of every 32-bit value. Codegen already reads + * it this way (ir/codegen.c takes pr1_reg and pr1_spilled apart separately). + */ +static int ra_alloc_half_unset(uint16_t half) +{ + return (half & PREG_REG_NONE) == PREG_REG_NONE; +} + +static int ra_alloc_half_spilled(uint16_t half) +{ + return !ra_alloc_half_unset(half) && (half & PREG_SPILLED) != 0; +} + +/* A direct frame slot: `StackLoc[off]` as a memory operand (an `Addr[...]` + * form has is_lval clear and is an address, not a location). */ +static int ra_rl_slot_offset(IROperand op, int *off) +{ + if (op.tag != IROP_TAG_STACKOFF || op.is_llocal || op.is_sym) + return 0; + if (irop_get_vreg(op) != -1 || !op.is_lval) + return 0; + *off = (int)irop_get_stack_offset(op); + return 1; +} + +/* Physical registers behind a plain register operand, as codegen resolves it. */ +static int ra_rl_operand_regs(TCCIRState *ir, IROperand op, int *r0, int *r1) +{ + int32_t vr = irop_get_vreg(op); + if (vr < 0 || !tcc_ir_vreg_is_valid(ir, vr)) + return 0; + IRLiveInterval *li = tcc_ir_vreg_live_interval(ir, vr); + if (!li || li->allocation.offset != 0) + return 0; + if (ra_alloc_half_spilled(li->allocation.r0) || ra_alloc_half_spilled(li->allocation.r1)) + return 0; + int a0 = li->allocation.r0 & PREG_REG_NONE; + int a1 = li->allocation.r1 & PREG_REG_NONE; + if (a0 == PREG_REG_NONE) + return 0; + *r0 = a0; + *r1 = (a1 == PREG_REG_NONE) ? -1 : a1; + return 1; +} + +/* Bytes an access moves when the register ends up holding EXACTLY the slot's + * contents. A sub-word access does not: `strb r2,[sp,#N]` writes only r2's + * low byte and `ldrb r2,[sp,#N]` brings it back zero-extended, so neither end + * of such a pair may be matched against the other or against a word access. + * FLOAT64/INT64 (and FLOAT32/INT32) share a width because a 64-bit soft-float + * value lives in the same GPR pair as the integer it was punned from -- that + * equivalence is the whole point of the pass. */ +static int ra_rl_exact_width(int btype) +{ + switch (btype) + { + case IROP_BTYPE_INT32: + case IROP_BTYPE_FLOAT32: + return 4; + case IROP_BTYPE_INT64: + case IROP_BTYPE_FLOAT64: + return 8; + default: + return 0; + } +} + +/* Keep `regs` occupied over [lo,hi] so the encoder cannot hand them out. + * + * A register outside every live interval holds nothing the allocator is + * accounting for, and the encoder helps itself to exactly those whenever it + * has to materialise an operand: `R0 <- StackLoc[-12] ADD StackLoc[-36]` is + * three instructions, two of them loads of the operands into whatever happens + * to be free. Neither load exists at IR level, so nothing in this pass sees + * them -- it would go on believing a slot's value is in a register the + * encoder had already reused (nested_struct_return printed dx + dy where it + * meant p.y + dy). + * + * Refusing whenever the register is dead somewhere in between is correct but + * throws away most of what this pass is for. Reserving it instead makes the + * assumption true: both scratch pickers -- tcc_ls_find_free_scratch_reg and + * scratch_pushed_dead_reg -- union live_regs_by_instruction[] into what they + * consider taken, and codegen has not run yet. Nothing else owns the + * register over this window, or it would have been live and there would be + * nothing to reserve. */ +static void ra_rl_reserve_regs(LSLiveIntervalState *ls, int r0, int r1, int lo, int hi) +{ + if (!ls->live_regs_by_instruction) + return; + for (int k = lo; k <= hi && k < ls->live_regs_by_instruction_size; ++k) + { + if (k < 0) + continue; + if (r0 >= 0) + ls->live_regs_by_instruction[k] |= (1u << r0); + if (r1 >= 0) + ls->live_regs_by_instruction[k] |= (1u << r1); + } +} + +static int ra_redundant_reload_elim(TCCIRState *ir) +{ + LSLiveIntervalState *ls = &ir->ls; + const int n = ir->next_instruction_index; + int removed = 0; + + if (tcc_state->optimize < 1) + return 0; + if (tcc_ir_opt_pass_disabled("ra:reload_elim")) + return 0; + + /* A slot whose address is taken anywhere can be written through that + * pointer, so it is never tracked. */ + int escaped[RA_REL_MAX_TRACKED]; + int nescaped = 0; + int escape_overflow = 0; + for (int i = 0; i < n; ++i) + { + IRQuadCompact *q = &ir->compact_instructions[i]; + if (q->op == TCCIR_OP_NOP) + continue; + for (int k = 0; k < 4; ++k) + { + IROperand op = (k == 0) ? (irop_config[q->op].has_dest ? tcc_ir_op_get_dest(ir, q) : IROP_NONE) + : (k == 1) ? (irop_config[q->op].has_src1 ? tcc_ir_op_get_src1(ir, q) : IROP_NONE) + : (k == 2) ? (irop_config[q->op].has_src2 ? tcc_ir_op_get_src2(ir, q) : IROP_NONE) + : (tcc_ir_op_is_mac(q->op) ? tcc_ir_op_get_accum(ir, q) : IROP_NONE); + if (op.tag != IROP_TAG_STACKOFF || op.is_llocal || op.is_lval) + continue; + if (irop_get_vreg(op) != -1) + continue; + int off = (int)irop_get_stack_offset(op); /* Addr[StackLoc[off]] */ + if (nescaped < RA_REL_MAX_TRACKED) + escaped[nescaped++] = off; + else + escape_overflow = 1; + } + } + + RARelSlot tracked[RA_REL_MAX_TRACKED]; + int ntracked = 0; + + for (int i = 0; i < n; ++i) + { + IRQuadCompact *q = &ir->compact_instructions[i]; + + /* Control can arrive here from anywhere else; nothing carries over. */ + if (q->is_jump_target) + ntracked = 0; + if (q->op == TCCIR_OP_NOP) + continue; + + switch (q->op) + { + /* A call, a write through a pointer or an indexed write can land on any + * slot, and inline asm can do anything at all. */ + case TCCIR_OP_FUNCCALLVAL: + case TCCIR_OP_FUNCCALLVOID: + case TCCIR_OP_STORE_INDEXED: + case TCCIR_OP_STORE_POSTINC: + case TCCIR_OP_INLINE_ASM: + case TCCIR_OP_ASM_INPUT: + case TCCIR_OP_ASM_OUTPUT: + case TCCIR_OP_SETJMP: + case TCCIR_OP_NL_SETJMP: + case TCCIR_OP_VLA_ALLOC: + case TCCIR_OP_VLA_SP_RESTORE: + ntracked = 0; + continue; + default: + break; + } + + int slot_off = 0; + + /* Drop a load that rewrites the very registers already holding the slot. */ + if (q->op == TCCIR_OP_LOAD && ntracked && + ra_rl_slot_offset(tcc_ir_op_get_src1(ir, q), &slot_off)) + { + int d0 = 0, d1 = 0; + IROperand dest = tcc_ir_op_get_dest(ir, q); + int width = ra_rl_exact_width(irop_get_btype(tcc_ir_op_get_src1(ir, q))); + if (width && !dest.is_lval && ra_rl_operand_regs(ir, dest, &d0, &d1) && + ((d1 >= 0) == (width == 8))) + { + for (int t = 0; t < ntracked; ++t) + { + if (!tracked[t].valid || tracked[t].off != slot_off || + tracked[t].size != width) + continue; + if (tracked[t].r0 != d0 || tracked[t].r1 != d1) + break; + ra_rl_reserve_regs(ls, tracked[t].r0, tracked[t].r1, tracked[t].def_idx, i); + /* With the load gone the destination has no definition of its own: + * it holds the slot's value only because it shares a register with + * the stored value. Pin both, as ra_phi_copy_needed does for an + * elided identity copy, or codegen's scratch-conflict fixup + * (try_reassign_scratch_conflict) moves the destination to a + * register nothing ever writes. */ + tcc_ir_vreg_live_interval(ir, irop_get_vreg(dest))->phi_pinned = 1; + tcc_ir_vreg_live_interval(ir, tracked[t].vreg)->phi_pinned = 1; + q->op = TCCIR_OP_NOP; + removed++; + RA_DBG("reload_elim @%d: slot %d already in R%d/R%d", i, slot_off, d0, d1); + break; + } + } + if (q->op == TCCIR_OP_NOP) + continue; + } + + /* Record a store, after invalidating whatever it overlaps. */ + if (q->op == TCCIR_OP_STORE && + ra_rl_slot_offset(tcc_ir_op_get_dest(ir, q), &slot_off)) + { + IROperand val = tcc_ir_op_get_src1(ir, q); + int s0 = 0, s1 = 0; + int dst_bt = irop_get_btype(tcc_ir_op_get_dest(ir, q)); + int size = ra_rl_exact_width(dst_bt); + int inval_size = size; + if (!size) + { + /* A sub-word store writes at most two bytes, so invalidating a whole + * word over-covers it; it leaves no register holding the slot, so it + * records nothing. Anything else (a struct copy) has a width this + * code cannot see -- assume it reached everything. */ + if (dst_bt != IROP_BTYPE_INT8 && dst_bt != IROP_BTYPE_INT16) + { + ntracked = 0; + continue; + } + inval_size = 4; + } + + for (int t = 0; t < ntracked; ++t) + if (tracked[t].valid && slot_off < tracked[t].off + tracked[t].size && + tracked[t].off < slot_off + inval_size) + tracked[t].valid = 0; + + int is_escaped = escape_overflow; + for (int e = 0; e < nescaped && !is_escaped; ++e) + if (escaped[e] == slot_off) + is_escaped = 1; + + if (size && !is_escaped && !val.is_lval && ra_rl_operand_regs(ir, val, &s0, &s1) && + ((s1 >= 0) == (size == 8)) && ra_rl_exact_width(irop_get_btype(val)) == size && + ntracked < RA_REL_MAX_TRACKED) + { + tracked[ntracked].off = slot_off; + tracked[ntracked].size = size; + tracked[ntracked].r0 = s0; + tracked[ntracked].r1 = s1; + tracked[ntracked].vreg = irop_get_vreg(val); + tracked[ntracked].def_idx = i; + tracked[ntracked].valid = 1; + ntracked++; + } + continue; + } + + /* A store to a slot we could not decode, or through a vreg, may hit + * anything we are holding. */ + if (q->op == TCCIR_OP_STORE) + { + ntracked = 0; + continue; + } + + /* Any other write to a register drops the entries that named it. */ + if (irop_config[q->op].has_dest) + { + IROperand dest = tcc_ir_op_get_dest(ir, q); + int d0 = 0, d1 = 0; + if (!dest.is_lval && ra_rl_operand_regs(ir, dest, &d0, &d1)) + { + for (int t = 0; t < ntracked; ++t) + if (tracked[t].valid && + (tracked[t].r0 == d0 || tracked[t].r1 == d0 || + (d1 >= 0 && (tracked[t].r0 == d1 || tracked[t].r1 == d1)))) + tracked[t].valid = 0; + } + else if (!dest.is_lval && irop_get_vreg(dest) >= 0) + { + /* Destination register unknown -- assume it hit everything. */ + ntracked = 0; + } + } + } + + return removed; +} + +/* ============================================================================ + * Post-Allocation Dead Frame-Store Elimination + * + * The reload elimination above is what leaves these behind. Every soft-float + * routine opens by punning its arguments through a union: + * + * StackLoc[-8] <-- R0(P0) [STORE] ; ua.d = a + * R0(V0) <-- StackLoc[-8] [LOAD] ; a_bits = ua.u + * + * Once the reload is gone nothing in the function reads that slot again, and + * the store is `strd r0,r1,[sp,#N]` writing two words no one will ever look at. + * __aeabi_dadd opens with two such slots, __aeabi_dmul with four. The IR-level + * dead-store pass (dce_dead_stackloc_stores) cannot reach them: it runs before + * allocation, while the reload is still a reader. + * + * The rule is deliberately blunt -- a store dies only when NO instruction + * anywhere in the function names an overlapping byte of its slot -- so it needs + * no ordering or dominance reasoning and stays correct however control flows. + * Anything that could reach a frame slot without naming it rules the whole + * function out: an address taken of any slot, an indexed access based on one + * (which reaches past the operand's own width), inline asm, setjmp, a VLA + * moving sp, or a nested function's static chain. + * ==========================================================================*/ + +/* Bytes an access to a frame slot covers, or 0 when the width is not + * statically known -- a struct copy or a complex pair reaches further than its + * component btype says. */ +static int ra_dfs_width(IROperand op) +{ + if (op.is_complex) + return 0; + switch (irop_get_btype(op)) + { + case IROP_BTYPE_INT8: + return 1; + case IROP_BTYPE_INT16: + return 2; + case IROP_BTYPE_INT32: + case IROP_BTYPE_FLOAT32: + return 4; + case IROP_BTYPE_INT64: + case IROP_BTYPE_FLOAT64: + return 8; + default: + return 0; + } +} + +/* A plain frame slot operand: `StackLoc[off]` naming memory directly -- the + * frontend's temporary slots included, which carry a negative vreg + * (VR_TEMP_LOCAL). Excluding those hid `Addr[temporary]` from the escape + * check below, and a store to the same slot through a -1 operand (lea_fold + * makes those) was deleted although a call read the slot. */ +static int ra_dfs_is_slot(IROperand op) +{ + return op.tag == IROP_TAG_STACKOFF && !op.is_sym && irop_get_vreg(op) < 0; +} + +/* True when the store at `at`, writing [off, off+w), is dead because control + * runs straight from it to a RETURN with nothing reading the slot on the way. + * + * This is the case the whole-function scan above cannot take. A soft-float + * routine ends `ur.u = ; return ur.d;` -- and once post-allocation + * reload elimination has dropped the matching load, because the value is + * already in the return register, the store writes eight bytes that the + * epilogue is about to abandon. The blunt rule keeps it, because the SAME + * slot is read on the cold NaN and zero paths that returned earlier. Those + * reads are unreachable from here, which is exactly what a straight-line walk + * to the return can prove without any dataflow. + * + * Everything that could get control somewhere else -- a jump, a call, an + * indirect branch -- ends the walk conservatively. A jump TARGET does not: + * it only adds a way in, and every way in still runs the same read-free tail + * to the same return. The caller has already ruled out the whole function on + * an escaping address, inline asm, setjmp, a VLA or a static chain, so a slot + * can only be reached here by being named. */ +static int ra_dfs_dead_to_return(TCCIRState *ir, int at, int off, int w) +{ + const int n = ir->next_instruction_index; + int steps = 0; + + for (int j = at + 1; j < n && steps < RA_DFS_TAIL_LIMIT; ++j) + { + IRQuadCompact *q = &ir->compact_instructions[j]; + if (q->op == TCCIR_OP_NOP) + continue; + steps++; + + if (q->op == TCCIR_OP_RETURNVALUE || q->op == TCCIR_OP_RETURNVOID) + return 1; + + /* Anything that can transfer control elsewhere, or that could read the + * frame through something other than a named slot. */ + switch (q->op) + { + case TCCIR_OP_STORE: + case TCCIR_OP_LOAD: + case TCCIR_OP_ASSIGN: + break; + default: + if (irop_config[q->op].has_dest && irop_get_vreg(tcc_ir_op_get_dest(ir, q)) >= 0) + break; /* a plain computation into a register */ + return 0; + } + + for (int k = 0; k < 3; ++k) + { + IROperand op = (k == 0) ? (irop_config[q->op].has_dest ? tcc_ir_op_get_dest(ir, q) : IROP_NONE) + : (k == 1) ? (irop_config[q->op].has_src1 ? tcc_ir_op_get_src1(ir, q) : IROP_NONE) + : (irop_config[q->op].has_src2 ? tcc_ir_op_get_src2(ir, q) : IROP_NONE); + if (!ra_dfs_is_slot(op)) + continue; + /* A plain STORE's destination overwrites the slot, it does not read it; + * anything else naming an overlapping byte keeps this store alive. */ + if (k == 0 && q->op == TCCIR_OP_STORE) + continue; + const int ow = ra_dfs_width(op); + if (ow == 0) + return 0; + const int ooff = (int)irop_get_stack_offset(op); + if (off < ooff + ow && ooff < off + w) + return 0; + } + } + return 0; +} + +static int ra_dead_frame_store_elim(TCCIRState *ir) +{ + const int n = ir->next_instruction_index; + int removed = 0; + + if (tcc_state->optimize < 1) + return 0; + if (tcc_ir_opt_pass_disabled("ra:dead_frame_store")) + return 0; + if (ir->has_static_chain) + return 0; + + struct + { + int off; + int size; + } reads[RA_DFS_MAX_READS]; + int nreads = 0; + + for (int i = 0; i < n; ++i) + { + IRQuadCompact *q = &ir->compact_instructions[i]; + if (q->op == TCCIR_OP_NOP) + continue; + + switch (q->op) + { + case TCCIR_OP_INLINE_ASM: + case TCCIR_OP_ASM_INPUT: + case TCCIR_OP_ASM_OUTPUT: + case TCCIR_OP_SETJMP: + case TCCIR_OP_NL_SETJMP: + case TCCIR_OP_VLA_ALLOC: + case TCCIR_OP_VLA_SP_RESTORE: + case TCCIR_OP_SET_CHAIN: + case TCCIR_OP_INIT_CHAIN_SLOT: + return 0; + default: + break; + } + + /* An indexed or post-incrementing access based on a slot runs off the end + * of the operand's own btype, so its width says nothing about what it + * touched. */ + const int reaches_past_width = + (q->op == TCCIR_OP_LOAD_INDEXED || q->op == TCCIR_OP_STORE_INDEXED || + q->op == TCCIR_OP_LOAD_POSTINC || q->op == TCCIR_OP_STORE_POSTINC); + + for (int k = 0; k < 4; ++k) + { + IROperand op = (k == 0) ? (irop_config[q->op].has_dest ? tcc_ir_op_get_dest(ir, q) : IROP_NONE) + : (k == 1) ? (irop_config[q->op].has_src1 ? tcc_ir_op_get_src1(ir, q) : IROP_NONE) + : (k == 2) ? (irop_config[q->op].has_src2 ? tcc_ir_op_get_src2(ir, q) : IROP_NONE) + : (tcc_ir_op_is_mac(q->op) ? tcc_ir_op_get_accum(ir, q) : IROP_NONE); + if (!ra_dfs_is_slot(op)) + continue; + /* `Addr[StackLoc[..]]` hands the address to something this scan cannot + * follow; is_llocal reaches memory through a pointer held in the slot. */ + if (!op.is_lval || op.is_llocal || reaches_past_width) + return 0; + /* The destination of a plain STORE is the write under consideration, not + * a read. Every other appearance keeps the slot alive. */ + if (k == 0 && q->op == TCCIR_OP_STORE) + continue; + int w = ra_dfs_width(op); + if (w == 0 || nreads >= RA_DFS_MAX_READS) + return 0; + reads[nreads].off = (int)irop_get_stack_offset(op); + reads[nreads].size = w; + nreads++; + } + } + + for (int i = 0; i < n; ++i) + { + IRQuadCompact *q = &ir->compact_instructions[i]; + if (q->op != TCCIR_OP_STORE) + continue; + IROperand dest = tcc_ir_op_get_dest(ir, q); + if (!ra_dfs_is_slot(dest) || !dest.is_lval || dest.is_llocal) + continue; + if (tcc_ir_access_is_volatile(ir, dest)) + continue; + const int w = ra_dfs_width(dest); + if (w == 0) + continue; + const int off = (int)irop_get_stack_offset(dest); + int is_read = 0; + for (int r = 0; r < nreads && !is_read; ++r) + if (off < reads[r].off + reads[r].size && reads[r].off < off + w) + is_read = 1; + if (is_read && !ra_dfs_dead_to_return(ir, i, off, w)) + continue; + q->op = TCCIR_OP_NOP; + removed++; + RA_DBG("dead_frame_store @%d: slot %d (%d bytes) is dead (%s)", i, off, w, + is_read ? "straight line to the return" : "never read"); + } + + return removed; +} + +/* ============================================================================ + * Post-Allocation Copy Propagation + * + * Move coalescing (below) removes a register copy by REASSIGNING one of its + * endpoints so the move degenerates to `mov rX,rX`. Both of its directions + * need one endpoint's register to be free across the other's live range, so + * neither can fire when the source OUTLIVES the copy: the source is still + * sitting in its own register, which is precisely why that register is not + * free. + * + * That is the dominant shape in soft float, where one union-punned 64-bit + * value feeds a dozen inlined accessors: + * + * R4(T8) <-- ... ; a_bits, live to the end of the function + * R8(T18) <-- &R4(T8) [ASSIGN] ; mov r8,r4 / mov r9,r5 + * R8(T19) <-- &R8(T18) SHR #52 ; reads only the high half + * + * The complementary transform needs no free register at all: rewrite T18's + * reads back to T8 and drop the copy. That is legal exactly when T8's + * register still holds T8 everywhere T18 is live, and the allocator's own + * invariant -- one live interval per physical register at a time -- makes + * that checkable by scanning the interval list for another claimant. + * + * Nothing here changes an allocation, so unlike every hint- or + * forwarding-based attempt on these same copies, it cannot stretch a live + * range and trade a move for a spill. Each removed quad is a removed + * instruction. + * ==========================================================================*/ + +/* Ops whose operands are pinned to particular physical registers (ABI slots, + * asm constraints, the static chain) or whose destination must not alias a + * source: a read inside one of these cannot be redirected to another + * register. */ +static int ra_cp_use_is_redirectable(int op) +{ + switch (op) + { + case TCCIR_OP_RETURNVALUE: + case TCCIR_OP_FUNCPARAMVAL: + case TCCIR_OP_FUNCPARAMVOID: + case TCCIR_OP_FUNCCALLVAL: + case TCCIR_OP_FUNCCALLVOID: + case TCCIR_OP_SET_CHAIN: + case TCCIR_OP_ASM_INPUT: + case TCCIR_OP_ASM_OUTPUT: + case TCCIR_OP_INLINE_ASM: + case TCCIR_OP_IJUMP: + case TCCIR_OP_SWITCH_TABLE: + case TCCIR_OP_SWITCH_LOAD: + case TCCIR_OP_SETJMP: + case TCCIR_OP_NL_SETJMP: + case TCCIR_OP_UMULL: + case TCCIR_OP_MLA: + case TCCIR_OP_UMAAL: + case TCCIR_OP_VLA_ALLOC: + case TCCIR_OP_VLA_SP_SAVE: + case TCCIR_OP_VLA_SP_RESTORE: + return 0; + default: + return 1; + } +} + +/* Does any interval other than `keep` occupy one of `keep`'s registers at some + * point in [lo,hi]? If not, `keep`'s registers still hold `keep`'s value + * across the whole window and reads of a copy of it can be redirected there. */ +static int ra_cp_src_regs_clobbered(const LSLiveIntervalState *ls, + const LSLiveInterval *keep, + uint32_t lo, uint32_t hi) +{ + int regs[2]; + int nregs = 0; + regs[nregs++] = keep->r0; + if (keep->r1 >= 0 && keep->r1 < PREG_NONE && keep->r1 != keep->r0) + regs[nregs++] = keep->r1; + + for (int j = 0; j < ls->next_interval_index; ++j) + { + const LSLiveInterval *x = &ls->intervals[j]; + if (x == keep || x->stack_location != 0) + continue; + if (x->start > hi || x->end < lo) + continue; + for (int r = 0; r < nregs; ++r) + if (x->r0 == regs[r] || x->r1 == regs[r]) + { + return 1; + } + } + return 0; +} + +/* Rewrite the reads of a copy's destination back to its source and drop the + * copy. Returns the number of copies removed. */ +static int ra_copy_propagate(TCCIRState *ir) +{ + LSLiveIntervalState *ls = &ir->ls; + const int n = ir->next_instruction_index; + int removed = 0; + + if (tcc_state->optimize < 1) + return 0; + if (!ls->live_regs_by_instruction || ls->live_regs_by_instruction_size <= 0) + return 0; + if (tcc_ir_opt_pass_disabled("ra:copy_prop")) + return 0; + + for (int i = 0; i < n; ++i) + { + IRQuadCompact *q = &ir->compact_instructions[i]; + if (q->op != TCCIR_OP_ASSIGN && q->op != TCCIR_OP_LOAD) + continue; + + IROperand dest = tcc_ir_op_get_dest(ir, q); + IROperand src = tcc_ir_op_get_src1(ir, q); + /* A LOAD off a vreg whose interval ended up in a register performs no + * memory access -- it is a register copy at codegen, and the IR generator + * emits the inlined `temp = var` binding in exactly that form. Skip + * is_llocal (a real dereference through a pointer) and is_sym (a global). + * Sub-word LOADs emit a UXTB/SXTH alongside the move to narrow the value, + * so only full-width copies qualify. */ + int is_copy_load = 0; + if (q->op == TCCIR_OP_LOAD) + { + int dbt = irop_get_btype(dest), sbt = irop_get_btype(src); + /* is_lval on a STACKOFF source is the VAR's own slot being read, which + * for a register-resident VAR is a register copy. On a VREG source it + * is a DEREFERENCE OF THAT REGISTER -- `R4 <- R3***DEREF***` reads the + * memory R3 points at, and rewriting its readers to R3 hands them the + * address instead of the value (pr90311: `b -= c < (unsigned char) a` + * subtracted from &b). is_llocal is not the flag that separates them; + * a plain pointer dereference does not set it. */ + int deref_through_reg = (src.tag == IROP_TAG_VREG && src.is_lval); + if ((src.tag == IROP_TAG_VREG || (src.tag == IROP_TAG_STACKOFF && src.is_local)) && + !deref_through_reg && !src.is_llocal && !src.is_sym && dbt == sbt && + (dbt == IROP_BTYPE_INT32 || dbt == IROP_BTYPE_INT64 || dbt == IROP_BTYPE_FUNC)) + is_copy_load = 1; + if (!is_copy_load) + continue; + } + /* Plain register-to-register only; an lval on either side is memory -- + * except a copy-LOAD, whose source carries is_lval for the VAR read. */ + if (dest.is_lval || (src.is_lval && !is_copy_load)) + continue; + if (irop_get_btype(dest) != irop_get_btype(src)) + continue; + + int32_t dv = irop_get_vreg(dest); + int32_t sv = irop_get_vreg(src); + if (dv < 0 || sv < 0 || dv == sv) + continue; + if (!tcc_ir_vreg_is_valid(ir, dv) || !tcc_ir_vreg_is_valid(ir, sv)) + continue; + /* The destination must be a temp: a VAR has a home beyond its register. + * The source may be a VAR -- a register-resident one is just a register, + * and it is the punned `a_bits` that every inlined accessor re-reads -- + * but then every write to it counts as a redefinition (below). */ + int src_is_var = (TCCIR_DECODE_VREG_TYPE(sv) == TCCIR_VREG_TYPE_VAR); + if (TCCIR_DECODE_VREG_TYPE(dv) != TCCIR_VREG_TYPE_TEMP) + continue; + if (TCCIR_DECODE_VREG_TYPE(sv) != TCCIR_VREG_TYPE_TEMP && !src_is_var) + continue; + + /* One interval per endpoint. A vreg split across several intervals can + * hold a DIFFERENT register in each, and the range reasoning below reads + * a single interval's bounds -- picking one of several would let a read + * be redirected into the register the other half was living in. */ + LSLiveInterval *src_iv = NULL, *dst_iv = NULL; + int src_ivs = 0, dst_ivs = 0; + for (int j = 0; j < ls->next_interval_index; ++j) + { + if (ls->intervals[j].vreg == (uint32_t)sv) { src_iv = &ls->intervals[j]; src_ivs++; } + if (ls->intervals[j].vreg == (uint32_t)dv) { dst_iv = &ls->intervals[j]; dst_ivs++; } + } + if (!src_iv || !dst_iv || src_ivs != 1 || dst_ivs != 1) + continue; + /* PREG_NONE/PREG_SPILLED are >= 0 but are not registers. */ + if (src_iv->r0 < 0 || src_iv->r0 >= PREG_NONE) + continue; + if (src_iv->stack_location != 0) + continue; + /* A SPILLED destination is the better case, not a disqualifying one: the + * copy the allocator could not keep in a register costs a store and a + * reload at every read, and redirecting those reads to the source removes + * all of it. __aeabi_dadd's `b_bits` is exactly this -- one 64-bit ASSIGN + * off the parameter, spilled, then loaded back four bytes at a time. The + * legality argument below never mentions the destination's register, only + * that the SOURCE's registers still hold the source across the reads, so + * it carries over unchanged. */ + const int dst_spilled = (dst_iv->r0 < 0 || dst_iv->r0 >= PREG_NONE || dst_iv->stack_location != 0); + /* Both ends must sit in the same register class, and a 64-bit class means + * a register PAIR: redirecting a paired read to an interval that only owns + * r0 hands the backend a half-formed operand (it asserts on r1 == 31). A + * register-resident VAR can be exactly that. */ + if (src_iv->reg_type != dst_iv->reg_type) + continue; + if (!dst_spilled) + { + int src_pair = (src_iv->r1 >= 0 && src_iv->r1 < PREG_NONE); + int dst_pair = (dst_iv->r1 >= 0 && dst_iv->r1 < PREG_NONE); + if (src_pair != dst_pair) + continue; + } + if (src_iv->addrtaken || dst_iv->addrtaken) + continue; + /* dst's register is shared with a graph-coalesced class that expects this + * write to land; src is only read, so its class membership is harmless. */ + if (dst_iv->co_member) + continue; + /* The LS intervals above drive the allocator; codegen resolves an operand + * through the IR-level interval's `allocation` instead, and the two are + * not interchangeable -- a VAR can be register-resident there with no high + * half, which the backend rejects the moment a 64-bit read names it. + * Validate against the structure codegen will actually consult. */ + { + IRLiveInterval *src_li = tcc_ir_vreg_live_interval(ir, sv); + IRLiveInterval *dst_li = tcc_ir_vreg_live_interval(ir, dv); + if (!src_li || !dst_li) + continue; + /* A phi-pinned dest shares its register with an identity-elided phi + * partner (post_ra_forward_diamond / ra_phi_copy_needed): the partner's + * reads continue PAST dst's own interval, and this copy is the only + * write that puts the value there. Removing it leaves the shared + * register stale on the path the elided copy used to cover — the + * ra_refine_live_regs_accurate clamp-diamond self-host break. */ + if (dst_li->phi_pinned) + continue; + if (src_li->allocation.offset != 0) + continue; + if (ra_alloc_half_spilled(src_li->allocation.r0) || + ra_alloc_half_spilled(src_li->allocation.r1)) + continue; + if (ra_alloc_half_unset(src_li->allocation.r0)) + continue; + /* The source must carry the full width the reads expect. With a + * register-resident destination that is "both are pairs or neither is"; + * with a spilled one there is no destination pair to compare against, so + * the operand btype -- already required equal above -- decides. */ + const int src_paired = ((src_li->allocation.r1 & PREG_REG_NONE) != PREG_REG_NONE); + if (!dst_spilled) + { + if (dst_li->allocation.offset != 0) + continue; + if (ra_alloc_half_spilled(dst_li->allocation.r0) || + ra_alloc_half_spilled(dst_li->allocation.r1)) + continue; + if (ra_alloc_half_unset(dst_li->allocation.r0)) + continue; + if (src_paired != ((dst_li->allocation.r1 & PREG_REG_NONE) != PREG_REG_NONE)) + continue; + } + else if (src_paired != (irop_get_btype(dest) == IROP_BTYPE_INT64 || + irop_get_btype(dest) == IROP_BTYPE_FLOAT64)) + continue; + } + + /* Already the same register: codegen elides it, nothing to remove. */ + if (src_iv->r0 == dst_iv->r0 && src_iv->r1 == dst_iv->r1) + continue; + /* src must already be live at the copy. */ + if (src_iv->start > (uint32_t)i) + continue; + + /* Collect dst's reads. Require exactly one definition (this copy), every + * read after it, and every read in an op whose operands are not pinned. */ + int ok = 1; + int nuses = 0; + int last_use = i; + for (int k = 0; k < n; ++k) + { + IRQuadCompact *qk = &ir->compact_instructions[k]; + if (k == i || qk->op == TCCIR_OP_NOP) + continue; + if (irop_config[qk->op].has_dest && + irop_get_vreg(tcc_ir_op_get_dest(ir, qk)) == dv) + { ok = 0; break; } /* redefined, or written through as an address */ + + int reads = 0; + if (irop_config[qk->op].has_src1 && + irop_get_vreg(tcc_ir_op_get_src1(ir, qk)) == dv) + reads = 1; + if (!reads && irop_config[qk->op].has_src2 && + irop_get_vreg(tcc_ir_op_get_src2(ir, qk)) == dv) + reads = 1; + if (!reads && tcc_ir_op_is_mac(qk->op) && + irop_get_vreg(tcc_ir_op_get_accum(ir, qk)) == dv) + reads = 1; + if (!reads) + continue; + + if (k < i || !ra_cp_use_is_redirectable(qk->op)) + { ok = 0; break; } + nuses++; + if (k > last_use) + last_use = k; + } + /* nuses == 0 is a dead copy; ra_eliminate_dead_reg_copies owns that. */ + if (!ok || nuses == 0) + continue; + + /* src's value must survive to the last redirected read. */ + if (src_iv->end < (uint32_t)last_use) + continue; + /* ...and no other interval may claim src's registers over that window. */ + if (ra_cp_src_regs_clobbered(ls, src_iv, (uint32_t)i, (uint32_t)last_use)) + continue; + /* ...and src itself must not be redefined under the reads. */ + for (int k = i + 1; k <= last_use; ++k) + { + IRQuadCompact *qk = &ir->compact_instructions[k]; + if (qk->op == TCCIR_OP_NOP || !irop_config[qk->op].has_dest) + continue; + IROperand dk = tcc_ir_op_get_dest(ir, qk); + /* For a temp, an lval destination is a store THROUGH it (an address + * use), not a new value. For a VAR it is a write to the variable, so + * there is no exemption. */ + if (dk.is_lval && !src_is_var) + continue; + if (irop_get_vreg(dk) == sv) + { ok = 0; break; } + } + if (!ok) + continue; + + for (int k = i + 1; k <= last_use; ++k) + { + IRQuadCompact *qk = &ir->compact_instructions[k]; + if (qk->op == TCCIR_OP_NOP) + continue; + if (irop_config[qk->op].has_src1) + { + IROperand o = tcc_ir_op_get_src1(ir, qk); + if (irop_get_vreg(o) == dv) + { irop_set_vreg(&o, sv); tcc_ir_set_src1(ir, k, o); } + } + if (irop_config[qk->op].has_src2) + { + IROperand o = tcc_ir_op_get_src2(ir, qk); + if (irop_get_vreg(o) == dv) + { irop_set_vreg(&o, sv); tcc_ir_set_src2(ir, k, o); } + } + } + q->op = TCCIR_OP_NOP; + removed++; + RA_DBG("copy_prop @%d: T%d -> T%d (R%d), %d use(s) to %d", i, + (int)(dv & 0xffffff), (int)(sv & 0xffffff), src_iv->r0, nuses, last_use); + } + + return removed; +} + +/* ============================================================================ + * Post-Allocation Move Coalescing + * + * Eliminates register-to-register copies (ASSIGN dest = src) by making both + * sides share the same physical register. The source must die at the ASSIGN + * instruction, the new register must be free for the destination's entire + * live range, and call-crossing safety must be preserved. + * + * The code generator already elides identity moves (mov rX, rX), so a + * successful coalescing eliminates the copy without modifying the IR. + * ============================================================================ */ + +static int ra_count_copies_reaching_codegen(TCCIRState *ir) +{ + int copies = 0; + + for (int i = 0; i < ir->next_instruction_index; i++) { + IRQuadCompact *q = &ir->compact_instructions[i]; + if (q->op != TCCIR_OP_ASSIGN) + continue; + + IROperand dest = tcc_ir_op_get_dest(ir, q); + IROperand src = tcc_ir_op_get_src1(ir, q); + int32_t dest_vreg = irop_get_vreg(dest); + int32_t src_vreg = irop_get_vreg(src); + if (dest.is_lval || src.is_lval || dest_vreg < 0 || src_vreg < 0) + continue; + if (irop_get_btype(dest) != irop_get_btype(src)) { + copies++; + continue; + } + + IRLiveInterval *dest_li = tcc_ir_vreg_live_interval(ir, dest_vreg); + IRLiveInterval *src_li = tcc_ir_vreg_live_interval(ir, src_vreg); + if (!ra_phi_copy_is_identity(dest_li, src_li)) + copies++; + } + + return copies; +} + +/* Is `vr` mentioned by any instruction at all? */ +static int ra_rp_vreg_mentioned(TCCIRState *ir, uint32_t vr) +{ + const int n = ir->next_instruction_index; + for (int k = 0; k < n; ++k) + { + IRQuadCompact *qk = &ir->compact_instructions[k]; + if (qk->op == TCCIR_OP_NOP) + continue; + if (irop_config[qk->op].has_dest && + (uint32_t)irop_get_vreg(tcc_ir_op_get_dest(ir, qk)) == vr) + return 1; + if (irop_config[qk->op].has_src1 && + (uint32_t)irop_get_vreg(tcc_ir_op_get_src1(ir, qk)) == vr) + return 1; + if (irop_config[qk->op].has_src2 && + (uint32_t)irop_get_vreg(tcc_ir_op_get_src2(ir, qk)) == vr) + return 1; + if (tcc_ir_op_is_mac(qk->op) && + (uint32_t)irop_get_vreg(tcc_ir_op_get_accum(ir, qk)) == vr) + return 1; + } + return 0; +} + +/* Is any of `keep`'s registers claimed by another interval over [lo,hi]? + * + * ra_cp_src_regs_clobbered answers the same question, but every pass that + * deletes an instruction leaves the interval of the value it computed behind: + * the vreg is then mentioned nowhere, yet its interval still claims its + * registers over the range it used to occupy. Such a phantom holds no value + * and cannot clobber anything -- and because it sits exactly where the deleted + * instruction was, it lands on the one-instruction window this pass asks + * about far more often than its share. So a claim is only believed once its + * vreg is shown to still exist in the instruction stream. + */ +static int ra_rp_dst_regs_busy(TCCIRState *ir, const LSLiveIntervalState *ls, + const LSLiveInterval *keep, uint32_t lo, uint32_t hi) +{ + int regs[2]; + int nregs = 0; + regs[nregs++] = keep->r0; + if (keep->r1 >= 0 && keep->r1 < PREG_NONE && keep->r1 != keep->r0) + regs[nregs++] = keep->r1; + + for (int j = 0; j < ls->next_interval_index; ++j) + { + const LSLiveInterval *x = &ls->intervals[j]; + if (x == keep || x->stack_location != 0) + continue; + if (x->start > hi || x->end < lo) + continue; + int hits = 0; + for (int r = 0; r < nregs; ++r) + if (x->r0 == regs[r] || x->r1 == regs[r]) + hits = 1; + if (!hits) + continue; + if (ra_rp_vreg_mentioned(ir, x->vreg)) + return 1; + } + return 0; +} + +/* Which producers may have their destination register changed. This is an + * ALLOW list, not a deny list: an op earns a place only when its destination + * is a plain register write the encoder is free to direct anywhere. A fixed + * destination (a call's r0, UMULL's pair, the division helpers), a + * two-address one (BFI presets its dest to the host word) and one the encoder + * also reads (the postinc forms write back through their base, LDRD against + * its own base is unpredictable) all stay off it. */ +static int ra_rp_def_is_retargetable(int op) +{ + switch (op) + { + case TCCIR_OP_ADD: + case TCCIR_OP_SUB: + case TCCIR_OP_MUL: + case TCCIR_OP_AND: + case TCCIR_OP_OR: + case TCCIR_OP_XOR: + case TCCIR_OP_SHL: + case TCCIR_OP_SAR: + case TCCIR_OP_SHR: + case TCCIR_OP_UBFX: + case TCCIR_OP_SBFX: + case TCCIR_OP_ZEXT: + case TCCIR_OP_SETIF: + case TCCIR_OP_ASSIGN: + return 1; + default: + return 0; + } +} + +/* Retarget the producer of a dying copy source, and drop the copy. + * + * ubfx r9, r1, #20, #11 ubfx sl, r1, #20, #11 + * mov sl, r9 -> + * + * This is the backward sibling of ra_copy_propagate. Copy propagation + * rewrites the reads of a copy's DESTINATION back to its source, so it needs + * the source to still be the occupant of its register across every one of + * them -- a long-lived destination therefore rules it out. Retargeting goes + * the other way: the producer writes the destination's register directly, the + * copy disappears, and whatever reads of the source remain are rewritten to + * the destination, which holds the identical value from the producer onwards. + * + * Nothing here depends on how long the destination lives. What it costs is + * that the destination is born one instruction earlier, so its register has + * to be free over that one instruction -- and, when the source outlives the + * copy, that the source's last read still falls inside the destination's own + * live range, so no range grows at the far end either. + * + * The move coalescer's forward direction covers the same shape by giving the + * destination the SOURCE's register, but only when that register is free + * across the destination's whole live range, which for a value computed early + * and used late it usually is not. Retargeting asks for far less: the + * destination's register must be free over the single instruction the value's + * definition moves back by. + * + * Every candidate is checked against the whole function rather than against + * the live intervals alone, so an interval that over-approximates cannot make + * a read of the source disappear. + */ +static int ra_retarget_producer(TCCIRState *ir) +{ + LSLiveIntervalState *ls = &ir->ls; + const int n = ir->next_instruction_index; + int retargeted = 0; + + if (tcc_state->optimize < 1) + return 0; + if (!ls->live_regs_by_instruction || ls->live_regs_by_instruction_size <= 0) + return 0; + if (tcc_ir_opt_pass_disabled("ra:retarget_producer")) + return 0; + const int tbl_size = ls->live_regs_by_instruction_size; + + for (int i = 0; i < n; ++i) + { + IRQuadCompact *q = &ir->compact_instructions[i]; + if (q->op != TCCIR_OP_ASSIGN) + continue; + /* Control may reach the copy without passing the producer, in which case + * the copy is the only thing establishing the destination on that path. */ + if (q->is_jump_target) + continue; + + IROperand dest = tcc_ir_op_get_dest(ir, q); + IROperand src = tcc_ir_op_get_src1(ir, q); + if (dest.is_lval || src.is_lval) + continue; + if (irop_get_btype(dest) != irop_get_btype(src)) + continue; + + int32_t dv = irop_get_vreg(dest); + int32_t sv = irop_get_vreg(src); + if (dv < 0 || sv < 0 || dv == sv) + continue; + if (!tcc_ir_vreg_is_valid(ir, dv) || !tcc_ir_vreg_is_valid(ir, sv)) + continue; + /* Both ends must be temps. A VAR has a home beyond its register, and the + * source's disappears entirely here. */ + if (TCCIR_DECODE_VREG_TYPE(dv) != TCCIR_VREG_TYPE_TEMP || + TCCIR_DECODE_VREG_TYPE(sv) != TCCIR_VREG_TYPE_TEMP) + continue; + + /* The producer is the instruction before the copy; NOPs left by earlier + * passes do not count as instructions, but a NOP that is a jump target + * still lets control in between the two. */ + int d = -1; + for (int k = i - 1; k >= 0; --k) + { + IRQuadCompact *qk = &ir->compact_instructions[k]; + if (qk->op != TCCIR_OP_NOP) { d = k; break; } + if (qk->is_jump_target) break; + } + if (d < 0) + continue; + IRQuadCompact *pq = &ir->compact_instructions[d]; + if (!ra_rp_def_is_retargetable(pq->op) || !irop_config[pq->op].has_dest) + continue; + IROperand pd = tcc_ir_op_get_dest(ir, pq); + if (pd.is_lval || irop_get_vreg(pd) != sv) + continue; + /* The width the encoder lowers by is the DESTINATION's, and passes that + * narrow a value retype operands in place -- so the producer's own view of + * its destination must be the width the copy moves. */ + if (irop_get_btype(pd) != irop_get_btype(dest)) + continue; + + /* Exactly one interval per endpoint: a split vreg holds a different + * register in each half, and the reasoning below reads one pair of + * bounds. */ + LSLiveInterval *src_iv = NULL, *dst_iv = NULL; + int src_ivs = 0, dst_ivs = 0; + for (int j = 0; j < ls->next_interval_index; ++j) + { + if (ls->intervals[j].vreg == (uint32_t)sv) { src_iv = &ls->intervals[j]; src_ivs++; } + if (ls->intervals[j].vreg == (uint32_t)dv) { dst_iv = &ls->intervals[j]; dst_ivs++; } + } + if (!src_iv || !dst_iv || src_ivs != 1 || dst_ivs != 1) + continue; + /* PREG_NONE/PREG_SPILLED are >= 0 but are not registers. Both ends must + * be register-resident: a spilled destination would turn the producer's + * write into a spill store, which is not what this is measuring. */ + if (src_iv->r0 < 0 || src_iv->r0 >= PREG_NONE || src_iv->stack_location != 0) + continue; + if (dst_iv->r0 < 0 || dst_iv->r0 >= PREG_NONE || dst_iv->stack_location != 0) + continue; + if (src_iv->reg_type != dst_iv->reg_type) + continue; + /* A 64-bit class means a register PAIR; handing the encoder a destination + * that owns only its low half aborts it (r1 == 31). */ + { + int src_pair = (src_iv->r1 >= 0 && src_iv->r1 < PREG_NONE); + int dst_pair = (dst_iv->r1 >= 0 && dst_iv->r1 < PREG_NONE); + if (src_pair != dst_pair) + continue; + } + if (src_iv->addrtaken || dst_iv->addrtaken) + continue; + /* A graph-coalesced interval shares one register with its whole class. */ + if (src_iv->co_member || dst_iv->co_member) + continue; + /* Already the same register: codegen elides the move, nothing to remove. */ + if (src_iv->r0 == dst_iv->r0 && src_iv->r1 == dst_iv->r1) + continue; + + /* Codegen resolves an operand through the IR-level interval's allocation, + * not through the LS one, and the two are not interchangeable. */ + { + IRLiveInterval *src_li = tcc_ir_vreg_live_interval(ir, sv); + IRLiveInterval *dst_li = tcc_ir_vreg_live_interval(ir, dv); + if (!src_li || !dst_li) + continue; + /* Retargeting moves the producer's write off src's register. A + * phi-pinned src shares that register with an identity-elided phi + * partner whose reads outlive src's interval — the write must stay. + * (Mirror of the ra_copy_propagate guard.) */ + if (src_li->phi_pinned) + continue; + if (dst_li->allocation.offset != 0) + continue; + if (ra_alloc_half_spilled(dst_li->allocation.r0) || + ra_alloc_half_spilled(dst_li->allocation.r1)) + continue; + if (ra_alloc_half_unset(dst_li->allocation.r0)) + continue; + if (ra_alloc_half_unset(src_li->allocation.r1) != + ra_alloc_half_unset(dst_li->allocation.r1)) + continue; + } + + /* The source is born at the producer, the destination at the copy, and + * the source does not outlive the destination -- the value ends up in the + * destination's register, which is guaranteed to be the destination's own + * only up to where the allocator said it ends. */ + if (src_iv->start != (uint32_t)d) + continue; + if (dst_iv->start != (uint32_t)i) + continue; + if (src_iv->end > dst_iv->end) + continue; + + /* The destination's registers must be free from the producer to the last + * read that will be redirected onto them. Below the copy that window is + * inside the destination's own range and the check is a formality; above + * it, it is what makes moving the definition back safe -- and it is also + * what keeps the producer's operands out of the way, since a source of + * the producer is live at the producer, and a multi-instruction lowering + * need not read all of its operands before writing its first half. */ + { + uint32_t hi = src_iv->end > (uint32_t)i ? src_iv->end : (uint32_t)i; + if (ra_rp_dst_regs_busy(ir, ls, dst_iv, (uint32_t)d, hi)) + continue; + } + + /* Now against the instruction stream rather than the intervals, so that + * an interval which over-approximates cannot hide a mention. Outside the + * producer and the copy, the source may only be READ, only below the copy + * and inside the destination's range, and only by an op whose operands are + * not pinned. A mention in a DEST operand is refused whatever it means -- + * a redefinition invalidates the rewrite, and a store THROUGH the source + * is a read this pass does not rewrite. The destination, in turn, must + * not be written anywhere but the copy. */ + int ok = 1; + int last_use = i; + for (int k = 0; k < n; ++k) + { + if (k == d || k == i) + continue; + IRQuadCompact *qk = &ir->compact_instructions[k]; + if (qk->op == TCCIR_OP_NOP) + continue; + /* Control must not be able to enter at the copy: on such an edge the + * copy is the only thing that establishes the destination, and the + * producer that would replace it never ran. `is_jump_target` says this + * already and is checked above, but it is a bit many passes maintain by + * hand, so the branches are also read directly. A computed target is + * refused outright because it names no index to compare against. */ + if (qk->op == TCCIR_OP_IJUMP || qk->op == TCCIR_OP_SWITCH_TABLE || + qk->op == TCCIR_OP_SWITCH_LOAD || qk->op == TCCIR_OP_SETJMP || + qk->op == TCCIR_OP_NL_SETJMP) + { ok = 0; break; } + if ((qk->op == TCCIR_OP_JUMP || qk->op == TCCIR_OP_JUMPIF) && + (int)irop_get_imm64_ex(ir, tcc_ir_op_get_dest(ir, qk)) == i) + { ok = 0; break; } + if (irop_config[qk->op].has_dest && + irop_get_vreg(tcc_ir_op_get_dest(ir, qk)) == sv) + { ok = 0; break; } + int reads = 0; + if (irop_config[qk->op].has_src1 && + irop_get_vreg(tcc_ir_op_get_src1(ir, qk)) == sv) + reads = 1; + if (!reads && irop_config[qk->op].has_src2 && + irop_get_vreg(tcc_ir_op_get_src2(ir, qk)) == sv) + reads = 1; + if (!reads && tcc_ir_op_is_mac(qk->op) && + irop_get_vreg(tcc_ir_op_get_accum(ir, qk)) == sv) + reads = 1; + if (!reads) + continue; + if (k < i || (uint32_t)k > dst_iv->end || !ra_cp_use_is_redirectable(qk->op)) + { ok = 0; break; } + if (k > last_use) + last_use = k; + } + if (!ok) + continue; + /* The redirected reads must all see the value the producer wrote, so the + * destination may not be given a new one underneath them. Only the span + * that actually carries a redirected read matters: a temp reassigned + * beyond the last of them is none of this transform's business. */ + for (int k = i + 1; k <= last_use; ++k) + { + IRQuadCompact *qk = &ir->compact_instructions[k]; + if (qk->op == TCCIR_OP_NOP) + continue; + if (irop_config[qk->op].has_dest && + irop_get_vreg(tcc_ir_op_get_dest(ir, qk)) == dv) + { ok = 0; break; } + } + if (!ok) + continue; + + irop_set_vreg(&pd, dv); + tcc_ir_set_dest(ir, d, pd); + q->op = TCCIR_OP_NOP; + for (int k = i + 1; k <= last_use; ++k) + { + IRQuadCompact *qk = &ir->compact_instructions[k]; + if (qk->op == TCCIR_OP_NOP) + continue; + if (irop_config[qk->op].has_src1) + { + IROperand o = tcc_ir_op_get_src1(ir, qk); + if (irop_get_vreg(o) == sv) + { irop_set_vreg(&o, dv); tcc_ir_set_src1(ir, k, o); } + } + if (irop_config[qk->op].has_src2) + { + IROperand o = tcc_ir_op_get_src2(ir, qk); + if (irop_get_vreg(o) == sv) + { irop_set_vreg(&o, dv); tcc_ir_set_src2(ir, k, o); } + } + } + + /* The destination is now born at the producer. The source's interval is + * left claiming its own registers over [d,i]; nothing writes them any + * more, but pretending they are still busy only costs later passes an + * opportunity, where clearing them wrongly would cost correctness. */ + dst_iv->start = (uint32_t)d; + { + IRLiveInterval *dst_li = tcc_ir_vreg_live_interval(ir, dv); + if (dst_li && dst_li->start > (uint32_t)d) + dst_li->start = (uint32_t)d; + } + for (int k = d; k <= i && k < tbl_size; ++k) + { + ls->live_regs_by_instruction[k] |= (1u << dst_iv->r0); + if (dst_iv->r1 >= 0 && dst_iv->r1 < PREG_NONE) + ls->live_regs_by_instruction[k] |= (1u << dst_iv->r1); + } + retargeted++; + RA_DBG("retarget_producer @%d: %s dest T%d -> T%d (R%d), reads to %d", d, + tcc_ir_get_op_name((TccIrOp)pq->op), (int)(sv & 0xffffff), + (int)(dv & 0xffffff), dst_iv->r0, last_use); + } + + return retargeted; +} + +/* ── ra:load_postinc / ra:store_postinc ── + * + * `D <- P***DEREF***` immediately followed (in the same straight-line region) + * by `P <- P + #k` is ARM's post-indexed load, `ldr D,[P],#k`; the mirrored + * `P***DEREF*** <- V; P <- P + #k` is the post-indexed store, `str V,[P],#k`. + * The backend has emitted both forms since forever + * (tcc_gen_machine_load/store_postinc_mop), but until now NOTHING in the + * pipeline ever produced the ops, so the whole addressing mode was dead code. + * Every IV-strength-reduced pointer walk paid a separate `adds P,#k`; on the + * M33 the post-increment is also a cycle cheaper than the separate add (docs, + * and tcc-m33-addressing-mode-cycle-costs). + * + * The store arm only sees IV-strength-reduced walks (`a[i] = v` loops turned + * into a pointer walk). A source-level `*p++ = v` does NOT lower to this + * shape but to a copy of the pointer followed by the increment + * (`T <- P; P <- T + k; T***DEREF*** <- V`); that needs its own matcher plus + * a liveness proof that T is dead, and is still absent. + * + * This runs POST-allocation deliberately. Pre-RA the fused op writes P without + * the allocator being told (irop_config says the POSTINC ops read P as a USE + * only), and P is exactly the loop-carried value the allocator most needs to + * model. After allocation both registers are already fixed and folding the + * add changes no allocation at all. + * + * The operand layout is the indexed-op one: two value slots, unused, increment + * at operand_base+3 (codegen reads it via tcc_ir_op_get_scale), so the fused + * instruction takes four fresh pool slots rather than reusing the original + * op's two. + */ +static int ra_load_postinc_fuse(TCCIRState *ir) +{ + const int n = ir->next_instruction_index; + int fused = 0; + + if (tcc_state->optimize < 1) + return 0; + int load_disabled = tcc_ir_opt_pass_disabled("ra:load_postinc"); + int store_disabled = tcc_ir_opt_pass_disabled("ra:store_postinc"); + if (load_disabled && store_disabled) + return 0; + + for (int i = 0; i < n; ++i) + { + IRQuadCompact *q = &ir->compact_instructions[i]; + int is_load = (q->op == TCCIR_OP_LOAD); + int is_store = (q->op == TCCIR_OP_STORE); + if (!is_load && !is_store) + continue; + if (is_load ? load_disabled : store_disabled) + continue; + + /* LOAD: `value <- ptr***DEREF***` — ptr is src1, an lvalue. + * STORE: `ptr***DEREF*** <- value` — ptr is the dest, an lvalue. */ + IROperand ptr = is_load ? tcc_ir_op_get_src1(ir, q) : tcc_ir_op_get_dest(ir, q); + IROperand value = is_load ? tcc_ir_op_get_dest(ir, q) : tcc_ir_op_get_src1(ir, q); + + /* A dereference THROUGH a register: `R4 <- R3***DEREF***`. is_local / + * is_llocal / is_sym are the frame-slot and global forms, which name memory + * some other way and have no pointer register to write back. */ + if (irop_get_tag(ptr) != IROP_TAG_VREG || !ptr.is_lval) + continue; + if (ptr.is_local || ptr.is_llocal || ptr.is_sym || ptr.is_complex) + continue; + /* No volatility guard is needed: the fusion neither removes, duplicates nor + * reorders the access. `ldr D,[P],#k` performs exactly the access the + * original did, in the same place; only the address arithmetic moves. */ + if (value.is_lval) + continue; + /* load_postinc_mop takes the access width from the DEST operand (and + * store_postinc_mop from the VALUE operand), while a plain LOAD/STORE + * takes it from the lvalue. Requiring the two to agree makes the forms + * equivalent; a widening load (`int c = *bytep`, dest INT32 over an INT8 + * access) would otherwise silently become LDR. Signedness only matters + * for loads — a store of width N writes the low N bytes either way. + * 64-bit is LDRD/STRD-with-writeback and stays out until it has its own + * test. */ + { + int vbt = irop_get_btype(value), pbt = irop_get_btype(ptr); + if (vbt != pbt || (is_load && value.is_unsigned != ptr.is_unsigned)) + continue; + if (vbt != IROP_BTYPE_INT32 && vbt != IROP_BTYPE_INT16 && vbt != IROP_BTYPE_INT8) + continue; + } + + int32_t pv = irop_get_vreg(ptr); + int32_t dv = irop_get_vreg(value); + if (pv < 0 || dv < 0 || pv == dv) + continue; + + /* The pointer must live in a real register: load_postinc_mop writes the + * incremented address back to the register it found the base in, and never + * to a spill slot, so a spilled P would silently lose the increment. */ + { + IRLiveInterval *pli = tcc_ir_vreg_live_interval(ir, pv); + if (!pli || pli->allocation.offset != 0) + continue; + if (ra_alloc_half_unset(pli->allocation.r0) || + ra_alloc_half_spilled(pli->allocation.r0)) + continue; + } + + /* Scan forward for `P <- P + #k`, bailing on anything that would change + * what the fusion means. The jump-target check comes BEFORE the NOP + * skip: a NOPed quad that is still a jump target means a path enters the + * window that never executed the access, and once the add is folded away + * that path would leave P un-incremented. */ + int add_idx = -1; + int assign_idx = -1; /* the `P <- T` half of a renamed split, if any */ + for (int j = i + 1; j < n; ++j) + { + IRQuadCompact *jq = &ir->compact_instructions[j]; + /* Leaving the region: an instruction reached from elsewhere may arrive + * with P un-incremented, and a branch may skip the add entirely. */ + if (jq->is_jump_target) + break; + if (jq->op == TCCIR_OP_NOP) + continue; + if (jq->op == TCCIR_OP_JUMP || jq->op == TCCIR_OP_JUMPIF || + jq->op == TCCIR_OP_IJUMP || jq->op == TCCIR_OP_SWITCH_TABLE || + jq->op == TCCIR_OP_RETURNVALUE || jq->op == TCCIR_OP_RETURNVOID) + break; + + if (jq->op == TCCIR_OP_ADD) + { + IROperand ad = tcc_ir_op_get_dest(ir, jq); + IROperand a1 = tcc_ir_op_get_src1(ir, jq); + IROperand a2 = tcc_ir_op_get_src2(ir, jq); + if (!ad.is_lval && irop_get_vreg(a1) == pv && irop_is_immediate(a2)) + { + int64_t k = irop_get_imm64_ex(ir, a2); + int32_t av = irop_get_vreg(ad); + if (k >= 1 && k <= 255 && av == pv) + { + add_idx = j; /* in-place: P <- P + #k */ + } + else if (k >= 1 && k <= 255 && av >= 0) + { + /* SSA rename splits the in-place bump into `T <- P + #k` followed + * by `P <- T [ASSIGN]`. That is still the same increment when T + * is born at the ADD and dies at the copy — prove it with the + * allocator's own intervals, then fold and NOP both halves. + * Between the two halves only non-jump-target NOPs may sit. */ + for (int m = j + 1; m < n; ++m) + { + IRQuadCompact *mq = &ir->compact_instructions[m]; + if (mq->is_jump_target) + break; + if (mq->op == TCCIR_OP_NOP) + continue; + if (mq->op == TCCIR_OP_ASSIGN) + { + IROperand cd = tcc_ir_op_get_dest(ir, mq); + IROperand cs = tcc_ir_op_get_src1(ir, mq); + if (!cd.is_lval && !cs.is_lval && irop_get_vreg(cd) == pv && + irop_get_vreg(cs) == av) + { + IRLiveInterval *tli = tcc_ir_vreg_live_interval(ir, av); + if (tli && tli->start == (uint32_t)j && tli->end == (uint32_t)m) + { + add_idx = j; + assign_idx = m; + } + } + } + break; /* first non-NOP decides either way */ + } + } + } + if (add_idx >= 0) + break; + } + + /* Any other read or write of P in between would see the wrong value + * once the increment moves up to the load. */ + { + int touches = 0; + for (int k = 0; k < 3 && !touches; ++k) + { + IROperand o; + if (k == 0) + { + if (!irop_config[jq->op].has_dest) + continue; + o = tcc_ir_op_get_dest(ir, jq); + } + else if (k == 1) + { + if (!irop_config[jq->op].has_src1) + continue; + o = tcc_ir_op_get_src1(ir, jq); + } + else + { + if (!irop_config[jq->op].has_src2) + continue; + o = tcc_ir_op_get_src2(ir, jq); + } + if (irop_get_vreg(o) == pv) + touches = 1; + } + if (tcc_ir_op_is_mac(jq->op) && irop_get_vreg(tcc_ir_op_get_accum(ir, jq)) == pv) + touches = 1; + /* A call reads its arguments where it is made, not where their + * FUNCPARAMVALs sit, and those can lie ABOVE the access: in + * `printf("%c%d", *p, ext(*p))` the `*p` for printf is a param quad + * before the load feeding ext, yet it is evaluated at the printf + * call, below the load. Walk back to the call's own params (an + * earlier call with the same id ends them). */ + if (!touches && (jq->op == TCCIR_OP_FUNCCALLVOID || jq->op == TCCIR_OP_FUNCCALLVAL)) + { + int cid = TCCIR_DECODE_CALL_ID(irop_get_imm64_ex(ir, tcc_ir_op_get_src2(ir, jq))); + for (int m = j - 1; m >= 0 && !touches; --m) + { + IRQuadCompact *mq = &ir->compact_instructions[m]; + if (mq->op == TCCIR_OP_FUNCCALLVOID || mq->op == TCCIR_OP_FUNCCALLVAL) + { + if (TCCIR_DECODE_CALL_ID(irop_get_imm64_ex(ir, tcc_ir_op_get_src2(ir, mq))) == cid) + break; + continue; + } + if (mq->op != TCCIR_OP_FUNCPARAMVAL || + TCCIR_DECODE_CALL_ID(irop_get_imm64_ex(ir, tcc_ir_op_get_src2(ir, mq))) != cid) + continue; + if (irop_get_vreg(tcc_ir_op_get_src1(ir, mq)) == pv) + touches = 1; + } + } + if (touches) + break; + } + } + if (add_idx < 0) + continue; + + IRQuadCompact *aq = &ir->compact_instructions[add_idx]; + IROperand incr = tcc_ir_op_get_src2(ir, aq); + + /* Four fresh pool slots: dest, ptr, unused, increment (see the header). */ + int nb = ir->iroperand_pool_count; + tcc_ir_pool_add(ir, IROP_NONE); + tcc_ir_pool_add(ir, IROP_NONE); + tcc_ir_pool_add(ir, IROP_NONE); + tcc_ir_pool_add(ir, IROP_NONE); + if (nb + 3 >= ir->iroperand_pool_capacity) + continue; + + q->op = is_load ? TCCIR_OP_LOAD_POSTINC : TCCIR_OP_STORE_POSTINC; + q->operand_base = (uint32_t)nb; + /* The backend wants the POINTER VALUE here, not the lvalue it dereferences: + * mach_ensure_in_reg on an lval operand emits a load OF the pointer from + * the memory it names, and the writeback then lands in a scratch. */ + IROperand base = ptr; + base.is_lval = 0; + base.aux = 0; + /* LOAD_POSTINC decodes {dest=value, src1=base}; STORE_POSTINC decodes + * {dest=base, src1=value} — the same slots as the plain ops they replace. */ + ir->iroperand_pool[nb + 0] = is_load ? value : base; + ir->iroperand_pool[nb + 1] = is_load ? base : value; + ir->iroperand_pool[nb + 2] = IROP_NONE; + ir->iroperand_pool[nb + 3] = incr; + aq->op = TCCIR_OP_NOP; + if (assign_idx >= 0) + ir->compact_instructions[assign_idx].op = TCCIR_OP_NOP; /* the `P <- T` half */ + fused++; + RA_DBG("%s_postinc @%d: folded +%d from @%d", is_load ? "load" : "store", i, + (int)irop_get_imm64_ex(ir, incr), add_idx); + } + + return fused; +} + +int tcc_ir_move_coalescing(TCCIRState *ir) +{ + LSLiveIntervalState *ls = &ir->ls; + + /* Copies whose source outlives them cannot be coalesced by reassignment; + * propagate those away first so the loop below only sees the rest. */ + int propagated = ra_copy_propagate(ir); + /* Then the copies whose source is born one instruction earlier, which have a + * producer to retarget instead of reads to redirect. */ + int retargeted = ra_retarget_producer(ir); + /* Drop reloads of a slot into the registers that already hold it -- the + * union punning every soft-float routine opens with. */ + int reloads = ra_redundant_reload_elim(ir); + /* And then the stores those reloads were the only reader of. */ + reloads += ra_dead_frame_store_elim(ir); + + if (!ls->live_regs_by_instruction || ls->live_regs_by_instruction_size <= 0) { + if (TCC_LOG_LS) + LOG_LS("copy_stats coalesced=0 reaching_codegen=%d", + ra_count_copies_reaching_codegen(ir)); + return 0; + } + + int coalesced = 0; + const int n = ir->next_instruction_index; + const int tbl_size = ls->live_regs_by_instruction_size; + + /* Track vregs already reverse-coalesced to prevent chains where a src + * gets moved to register A, then a later ASSIGN moves it back to B. */ + uint32_t *rev_done = NULL; + int rev_done_size = 0; + + for (int i = 0; i < n; ++i) + { + IRQuadCompact *q = &ir->compact_instructions[i]; + /* LOAD with a VREG source where the underlying interval ended up in a + * register (no spill) is a register copy at codegen — treat like ASSIGN + * for coalescing. Catches inlined "temp = var" patterns the IR + * generator emits as LOAD even when no memory access is involved. + * Accept is_lval=1 for local VAR reads as long as the VAR is reg-only; + * skip is_llocal (true memory load via pointer) and is_sym (global). */ + /* LOAD from a vreg whose interval ended up in a register (no spill) + * is a register copy at codegen — treat like ASSIGN for coalescing. + * Catches inlined "temp = var" patterns where the IR generator emits + * LOAD with VREG/STACKOFF source even though no memory access happens. + * Skip is_llocal (double-indirection via pointer) and is_sym (global). + * Skip sub-word btypes: those LOADs emit UXTB/SXTB/UXTH/SXTH alongside + * the mov to truncate; coalescing them away would skip the narrowing + * and yield wrong values (see pr69447, fp-cmp-8 sub-word args). */ + int is_copy_load = 0; + if (q->op == TCCIR_OP_LOAD) { + IROperand ts = tcc_ir_op_get_src1(ir, q); + int valid_tag = (ts.tag == IROP_TAG_VREG || + (ts.tag == IROP_TAG_STACKOFF && ts.is_local)); + int dest_bt = irop_get_btype(tcc_ir_op_get_dest(ir, q)); + int src_bt = irop_get_btype(ts); + int width_safe = (dest_bt == src_bt) && + (dest_bt == IROP_BTYPE_INT32 || + dest_bt == IROP_BTYPE_INT64 || + dest_bt == IROP_BTYPE_FUNC); + if (width_safe && valid_tag && !ts.is_llocal && !ts.is_sym) { + int32_t tsv = irop_get_vreg(ts); + if (tsv >= 0 && tcc_ir_vreg_is_valid(ir, tsv)) { + for (int j = 0; j < ls->next_interval_index; ++j) { + if (ls->intervals[j].vreg == (uint32_t)tsv) { + /* Only a source in a REAL register is a register copy. PREG_NONE + * (0x1F) is >= 0 but means "not allocated" — e.g. a stack-passed + * parameter that lives in the caller's frame, not a register. + * Coalescing the LOAD result into such a source would make it + * inherit PREG_NONE and be mis-lowered as a spill at frame offset + * 0 (clobbering the saved frame pointer). */ + if (ls->intervals[j].r0 >= 0 && ls->intervals[j].r0 < PREG_NONE && + ls->intervals[j].stack_location == 0) + is_copy_load = 1; + break; + } + } + } + } + } + if (q->op != TCCIR_OP_ASSIGN && !is_copy_load) + continue; + + const IROperand src1 = tcc_ir_op_get_src1(ir, q); + const IROperand dest = tcc_ir_op_get_dest(ir, q); + /* For is_copy_load (LOAD from in-register VAR) the src has is_lval=1 + * but it behaves as a register copy — don't skip it on that basis. */ + if ((src1.is_lval && !is_copy_load) || dest.is_lval) continue; + int32_t sv = irop_get_vreg(src1); + if (sv < 0 || !tcc_ir_vreg_is_valid(ir, sv)) + continue; + int32_t dv = irop_get_vreg(dest); + if (dv < 0 || !tcc_ir_vreg_is_valid(ir, dv)) + continue; + + LSLiveInterval *src_iv = NULL, *dst_iv = NULL; + for (int j = 0; j < ls->next_interval_index; ++j) + { + if (ls->intervals[j].vreg == (uint32_t)sv) src_iv = &ls->intervals[j]; + if (ls->intervals[j].vreg == (uint32_t)dv) dst_iv = &ls->intervals[j]; + if (src_iv && dst_iv) break; + } + if (!src_iv || !dst_iv) continue; + /* Both endpoints must live in a REAL register. PREG_NONE (0x1F) and + * PREG_SPILLED (0x20) are >= 0 but are NOT registers (e.g. a stack-passed + * parameter resident in the caller's frame). Coalescing onto such an + * endpoint propagates PREG_NONE into a live value, which is then mis-lowered + * as a spill at frame offset 0 — clobbering a saved register at [FP,#0]. */ + if (src_iv->r0 < 0 || src_iv->r0 >= PREG_NONE || + dst_iv->r0 < 0 || dst_iv->r0 >= PREG_NONE) continue; + if (src_iv->stack_location != 0 || dst_iv->stack_location != 0) continue; + if (src_iv->r0 == dst_iv->r0) continue; + /* Never reassign a graph-coalesced interval: it shares one register with + * its whole class, and reassigning one member here would split the class + * (the other members keep the class register), corrupting the value. */ + if (src_iv->co_member || dst_iv->co_member) continue; + + /* Nor a phi-pinned one. post_ra_forward_diamond / ra_phi_copy_needed + * delete an identity phi copy precisely BECAUSE its two vregs already sit + * in one register, and set phi_pinned to say the pairing is now load + * bearing: no instruction is left that would re-establish it. Moving + * either endpoint to a different register here silently un-pairs them, and + * the merge block reads a register the elided-copy path never wrote. + * (Same guard as ra_copy_propagate / ra_retarget_producer above; this loop + * reassigns registers rather than rewriting operands, so it needs it too.) */ + { + IRLiveInterval *src_ir_iv = tcc_ir_vreg_live_interval(ir, sv); + IRLiveInterval *dst_ir_iv = tcc_ir_vreg_live_interval(ir, dv); + if ((src_ir_iv && src_ir_iv->phi_pinned) || (dst_ir_iv && dst_ir_iv->phi_pinned)) + continue; + } + + /* Forward direction: reassign dest to use src's register. + * Requires src to die at this ASSIGN. */ + if (src_iv->end == (uint32_t)i) { + int src_reg = src_iv->r0; + if (dst_iv->crosses_call && !(src_reg >= 4 && src_reg <= 11)) + goto try_reverse; + + int conflict = 0; + for (int k = i + 1; k <= (int)dst_iv->end && k < tbl_size; ++k) + { + if (ls->live_regs_by_instruction[k] & (1u << src_reg)) { + /* Two-address relaxation: at k == dst.end the dst's last use is + * an instruction that consumes dst and writes a fresh result. + * If that result lands in src_reg (some interval starts at k in + * src_reg) AND the same instruction reads dst, sharing src_reg + * is safe — ARM ops read all sources before writing dest, so + * `OP src_reg, ..., src_reg` is a valid two-operand form. */ + int safe = 0; + if (k == (int)dst_iv->end) { + IRQuadCompact *kq = &ir->compact_instructions[k]; + int reads_dst = 0; + if (irop_config[kq->op].has_src1 && + irop_get_vreg(tcc_ir_op_get_src1(ir, kq)) == dv) + reads_dst = 1; + if (!reads_dst && irop_config[kq->op].has_src2 && + irop_get_vreg(tcc_ir_op_get_src2(ir, kq)) == dv) + reads_dst = 1; + if (reads_dst) { + for (int j2 = 0; j2 < ls->next_interval_index; j2++) { + LSLiveInterval *xi = &ls->intervals[j2]; + if (xi->r0 == src_reg && xi->stack_location == 0 && + xi->start == (uint32_t)k) { + safe = 1; + break; + } + } + } + } + if (!safe) { conflict = 1; break; } + } + } + if (!conflict) { + for (int k = (int)dst_iv->start; k < i && k < tbl_size; ++k) + { + if (ls->live_regs_by_instruction[k] & (1u << src_reg)) + { conflict = 1; break; } + } + } + if (!conflict) { + int old_reg = dst_iv->r0; + dst_iv->r0 = src_reg; + for (int k = (int)dst_iv->start; k <= (int)dst_iv->end && k < tbl_size; ++k) + { + /* old_reg's bit may be shared with another interval that coalesced + * onto it earlier (in-place two-address ops overlap on purpose) — + * only clear positions where no other claimant is still live. */ + if (!tcc_ls_reg_held_by_other(ls, old_reg, k, dst_iv)) + ls->live_regs_by_instruction[k] &= ~(1u << old_reg); + ls->live_regs_by_instruction[k] |= (1u << src_reg); + } + RA_DBG("move_coalesce fwd @%d: T%d R%d->R%d [%u,%u]", i, + (int)(dv & 0xffffff), old_reg, src_reg, dst_iv->start, dst_iv->end); + coalesced++; + continue; + } + } + + /* Reverse direction: reassign src to use dest's register. + * Works for loop-carried phi copies where src = f(dest, ...) and + * dest's register is only occupied by dest during src's range. + * Safety: src must not be redefined between the ASSIGN and dest's + * last use, otherwise the shared register would get clobbered. */ +try_reverse:; + /* Skip if this src vreg was already reverse-coalesced */ + { + int already = 0; + for (int ri = 0; ri < rev_done_size; ri++) { + if (rev_done[ri] == (uint32_t)sv) { already = 1; break; } + } + if (already) continue; + } + int dest_reg = dst_iv->r0; + if (src_iv->crosses_call && !(dest_reg >= 4 && dest_reg <= 11)) + continue; + + int conflict = 0; + + /* Conservative: src must be defined directly FROM dest (reads dest + * as src1), like `src = dest + 1` or `src = dest * x + acc`. + * This guarantees ARM's read-before-write makes the in-place + * operation correct. */ + { + int def_idx = (int)src_iv->start; + if (def_idx < 0 || def_idx >= n) { conflict = 1; goto rev_check_done; } + IRQuadCompact *qdef = &ir->compact_instructions[def_idx]; + if (!irop_config[qdef->op].has_src1) { conflict = 1; goto rev_check_done; } + IROperand s1 = tcc_ir_op_get_src1(ir, qdef); + if (irop_get_vreg(s1) != dv) { conflict = 1; goto rev_check_done; } + } + + /* Check src is not redefined while dest is still live */ + for (int k = i + 1; k <= (int)dst_iv->end && k < n; ++k) + { + IRQuadCompact *qk = &ir->compact_instructions[k]; + if (qk->op == TCCIR_OP_NOP) continue; + if (irop_config[qk->op].has_dest) { + IROperand dk = tcc_ir_op_get_dest(ir, qk); + int is_mem_store = (qk->op == TCCIR_OP_STORE || qk->op == TCCIR_OP_STORE_INDEXED || + qk->op == TCCIR_OP_STORE_POSTINC) && dk.is_lval; + if (!is_mem_store) { + int32_t dkvr = irop_get_vreg(dk); + if (dkvr == sv) { conflict = 1; break; } + } + } + } + if (conflict) goto rev_check_done; + + /* Symmetric guard (dest side): after this copy src and dest share + * dest_reg holding the same value. If dest is given a NEW, independent + * value while src is still live, that write clobbers dest_reg and src's + * remaining uses read the wrong value. The loop-carried phi copy this + * pass targets has src dying at the copy (src_iv->end == i), so the range + * below is empty and legitimate coalescing is unaffected; the guard only + * fires when src OUTLIVES the copy and dest is re-defined underneath it + * (bitfield 40979: `u4 = u3` copy, then `u4 = const` clobbers the shared + * register while `u3` is still read). A redefinition at exactly src's + * last use that also reads src is the two-address read-before-write case + * and stays safe. */ + for (int k = i + 1; k <= (int)src_iv->end && k < n; ++k) + { + IRQuadCompact *qk = &ir->compact_instructions[k]; + if (qk->op == TCCIR_OP_NOP) continue; + if (!irop_config[qk->op].has_dest) continue; + IROperand dk = tcc_ir_op_get_dest(ir, qk); + int is_mem_store = (qk->op == TCCIR_OP_STORE || qk->op == TCCIR_OP_STORE_INDEXED || + qk->op == TCCIR_OP_STORE_POSTINC) && dk.is_lval; + if (is_mem_store) continue; + if (irop_get_vreg(dk) != dv) continue; + if (k == (int)src_iv->end) { + int reads_src = 0; + if (irop_config[qk->op].has_src1 && + irop_get_vreg(tcc_ir_op_get_src1(ir, qk)) == sv) reads_src = 1; + if (!reads_src && irop_config[qk->op].has_src2 && + irop_get_vreg(tcc_ir_op_get_src2(ir, qk)) == sv) reads_src = 1; + if (!reads_src && tcc_ir_op_is_mac(qk->op) && + irop_get_vreg(tcc_ir_op_get_accum(ir, qk)) == sv) reads_src = 1; + if (reads_src) continue; + } + conflict = 1; + break; + } + if (conflict) goto rev_check_done; + + /* Check dest not used between src's def and the ASSIGN. + * src's def overwrites dest_reg; any intervening use of dest + * would read the wrong value. */ + for (int k = (int)src_iv->start + 1; k < i && k < n; ++k) + { + IRQuadCompact *qk = &ir->compact_instructions[k]; + if (qk->op == TCCIR_OP_NOP) continue; + if (irop_config[qk->op].has_src1) { + if (irop_get_vreg(tcc_ir_op_get_src1(ir, qk)) == dv) { conflict = 1; break; } + } + if (!conflict && irop_config[qk->op].has_src2) { + if (irop_get_vreg(tcc_ir_op_get_src2(ir, qk)) == dv) { conflict = 1; break; } + } + if (!conflict && irop_config[qk->op].has_dest) { + IROperand dk = tcc_ir_op_get_dest(ir, qk); + if (dk.is_lval && irop_get_vreg(dk) == dv) { conflict = 1; break; } + } + if (!conflict && tcc_ir_op_is_mac(qk->op)) { + if (irop_get_vreg(tcc_ir_op_get_accum(ir, qk)) == dv) { conflict = 1; break; } + } + } + if (conflict) goto rev_check_done; + + /* Check no control-flow escape between src's def and the ASSIGN. + * src's def overwrites dest_reg; the ASSIGN re-establishes dest's value + * only on the path that reaches it. A JUMP/JUMPIF in (def, ASSIGN) that + * targets outside [def, ASSIGN] lets control reach later uses of dest + * with dest_reg clobbered and the restoring copy skipped — e.g. a + * top-tested pointer-chase loop (`while (p->next) p = p->next;`) whose + * exit edge branches past the back-edge copy while `p` is still live. */ + for (int k = (int)src_iv->start; k < i && k < n; ++k) + { + IRQuadCompact *qk = &ir->compact_instructions[k]; + if (qk->op == TCCIR_OP_NOP) continue; + if (qk->op == TCCIR_OP_IJUMP || qk->op == TCCIR_OP_SWITCH_TABLE || + qk->op == TCCIR_OP_SWITCH_LOAD) { conflict = 1; break; } + if (qk->op == TCCIR_OP_JUMP || qk->op == TCCIR_OP_JUMPIF) { + int jt = (int)irop_get_imm64_ex(ir, tcc_ir_op_get_dest(ir, qk)); + if (jt < (int)src_iv->start || jt > i) { conflict = 1; break; } + } + } +rev_check_done: + if (conflict) continue; + + /* Check dest_reg not occupied by other intervals during src's range. + * Identity-based: earlier coalesces may have moved a third interval onto + * dest_reg inside dst_iv's range, so "position within dst_iv's range" is + * not proof the claim is dst_iv's own. */ + for (int k = (int)src_iv->start; k <= (int)src_iv->end && k < tbl_size; ++k) + { + if (ls->live_regs_by_instruction[k] & (1u << dest_reg)) + { + if (tcc_ls_reg_held_by_other(ls, dest_reg, k, dst_iv)) + { conflict = 1; break; } + /* dest_reg is live here — only OK if it's from dest_iv itself */ + if (k < (int)dst_iv->start || k > (int)dst_iv->end) + { conflict = 1; break; } + } + } + if (conflict) continue; + + int old_reg = src_iv->r0; + src_iv->r0 = dest_reg; + for (int k = (int)src_iv->start; k <= (int)src_iv->end && k < tbl_size; ++k) + { + /* old_reg's bit may be shared with another interval that coalesced + * onto it earlier — only clear positions with no other live claimant + * (volatile 36818: T175 leaving R5 wiped T212's in-place-XOR claim, + * and the phase-3 scratch fixup then put the outer loop counter there). */ + if (!tcc_ls_reg_held_by_other(ls, old_reg, k, src_iv)) + ls->live_regs_by_instruction[k] &= ~(1u << old_reg); + ls->live_regs_by_instruction[k] |= (1u << dest_reg); + } + RA_DBG("move_coalesce rev @%d: T%d R%d->R%d [%u,%u]", i, + (int)(sv & 0xffffff), old_reg, dest_reg, src_iv->start, src_iv->end); + /* Record this src vreg as reverse-coalesced */ + rev_done = tcc_realloc(rev_done, sizeof(uint32_t) * (rev_done_size + 1)); + rev_done[rev_done_size++] = (uint32_t)sv; + coalesced++; + } + + if (rev_done) + tcc_free(rev_done); + + if (coalesced > 0) + tcc_ls_recompute_dirty_registers(ls); + + /* Last: the IR is in its final shape, so an increment folded into a load + * cannot be moved apart again. */ + int postinc = ra_load_postinc_fuse(ir); + + if (TCC_LOG_LS) + LOG_LS("copy_stats propagated=%d reloads=%d coalesced=%d retargeted=%d reaching_codegen=%d", + propagated, reloads, coalesced, retargeted, + ra_count_copies_reaching_codegen(ir)); + + return coalesced + propagated + reloads + retargeted + postinc; +} diff --git a/source/ir/regalloc_priv.h b/source/ir/regalloc_priv.h new file mode 100644 index 000000000..5afb82322 --- /dev/null +++ b/source/ir/regalloc_priv.h @@ -0,0 +1,355 @@ +/* + * TCC IR - SSA-Aware Register Allocator + * + * Operates directly on SSA-renamed IR with phi nodes. + * Replaces the tccls.c linear scan when -fssa-regalloc is enabled. + * + * Copyright (c) 2025 Mateusz Stadnik + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +/* Declarations shared by the SSA register allocator's source files + * (regalloc.c and regalloc_{scan,phi,entry,post}.c), which used to be one + * 9.5k-line TU. Each function or variable declared at the bottom is + * defined in the file its comment names. The allocator's public interface + * is regalloc.h. */ + +#ifndef REGALLOC_PRIV_H +#define REGALLOC_PRIV_H + +#define USING_GLOBALS +#include "ir.h" +#include "regalloc.h" +#include "cfg.h" +#include "ssa.h" +#include "mem_ssa.h" +#include "ssa_opt.h" +#include "opt/ssa/branch.h" +#include "const_string_fold.h" +#include "bitop_const_fold.h" +#include "opt/ssa/branch.h" +#include "opt/ssa/fold.h" +#include "opt/ssa/var_imm_prop.h" +#include "opt/ssa/cprop.h" +#include "global_addr_hoist.h" +#include "load_cse.h" +#include "diamond_store_fwd.h" + +#include "opt/ssa/strength.h" +#include "opt/ssa/reassoc.h" +#include "opt/ssa/gvn.h" +#include "opt/ssa/vrp.h" +#include "opt/ssa/setif_or_taut.h" +#include "opt/ssa/bool_norm.h" +#include "opt/ssa/cmp_offset_fold.h" +#include "opt_pipeline.h" +#include "opt.h" +#include "licm.h" +#include "opt_loop_utils.h" + +#include "memory/bitspan.h" +#include "memory/bit_matrix.h" + +#define RA_DBG(fmt, ...) LOG_LS(fmt, ##__VA_ARGS__) + +/* ============================================================================ + * SSA Live Interval + * ============================================================================ */ + +typedef struct SSAInterval { + int32_t vreg; + uint32_t start; + uint32_t end; + int8_t r0; + int8_t r1; + int32_t stack_location; + uint8_t crosses_call : 1; + uint8_t crosses_real_call : 1; /* crosses a real FUNCCALL (VFP caller-saved clobber) — excludes native FP-op implicit calls */ + uint8_t addrtaken : 1; + uint8_t is_volatile : 1; /* volatile-qualified local: force to a stack slot so every access is a real ldr/str */ + uint8_t is_param : 1; + uint8_t reg_shared : 1; /* cur shares hr with another active interval (return-block tail); skip expire-free and active push */ + uint8_t alive_shared : 1; /* holds a pair BORROWED from a still-live owner (ra_alive_reg_shareable proved the owner dead only across THIS interval's range) — must not be passed on */ + uint8_t loop_phi_locked : 1; /* absorbed a loop-phi partner (carries a loop-carried value across the whole loop body); must not be evicted — spilling it mid-loop would not reload the partner's uses and corrupts the IV */ + uint8_t reg_type; + uint16_t use_count; + uint16_t narrow_uses; /* static count of references from ops with 16-bit encodings (want r0-r7) */ + int8_t precolored; + int8_t pref_reg; /* soft hint: prefer this physical reg if available (e.g. r0 for RETURNVALUE feeders) */ + int32_t hint_vreg; + int32_t coalesce_to; /* graph coalescing: vreg of the representative this one merged into (-1 = rep / not merged) */ + uint8_t co_member; /* 1 if part of a graph-coalesced class (rep or member) — the in-scan transfer must leave it alone */ +} SSAInterval; + +/* Pre-allocation frame-pointer prediction. The real decision is made after + * allocation — tcc_ir_codegen_generate forces FP for VLA, soft-float ops and + * static chain; the Thumb prologue for variadic functions and force_lr_save — + * so freeing the FP register for allocation is safe only when none of those + * can fire. This must remain a SUPERSET of every post-allocation forcing + * site: add the condition HERE FIRST when adding one there. Inline asm does + * not force a frame pointer, but its text can name r7 outright, which the + * allocator cannot see — keep the register reserved in such functions. */ +/* ============================================================================ + * Accurate per-vreg liveness, for the allocator's last-resort register share + * + * ra_build_intervals models every value as ONE contiguous [start,end] range. + * That is a strict over-approximation across a diamond: a value defined above + * an if/else and read only in ONE arm looks live in the other arm too, so the + * linear scan sees a false conflict and spills a value that could have had a + * register for free. `__aeabi_dadd`'s alignment block is the shape: `exp_diff` + * is read in the `exp_diff > 0` arm and again in the `exp_diff < 0` arm, so its + * register looks busy in BOTH -- and the 64-bit temp defined there was written + * to the frame and read back at the very next instruction. + * + * This computes real liveness (backward dataflow over the CFG, the same + * def/use model the graph coalescer trusts, deferred PARAM sources included) + * and records, per vreg, every instruction index at which it is live-in or + * live-out. ra_linear_scan consults it ONLY when it is about to spill: if an + * active interval is provably dead across the whole of the new interval's + * range, the two cannot conflict on any path and can share one register. + * + * Unknown answers are reported BUSY, so a failure to build (un-enumerated CFG + * edges, a vreg outside the table, an oversized function) can only cost a + * sharing opportunity, never create a clobber. + * ========================================================================== */ +typedef struct RaAliveInfo +{ + int valid; + int n; /* instruction indices covered: [0, n) */ + int tbl; /* vreg slot table size */ + int maxpos; + int words; /* uint64 words per vreg row */ + int ndense; + int *dense; /* tbl entries: slot -> row index, or -1 */ + uint64_t *rows; /* ndense rows of `words` words */ +} RaAliveInfo; + +#define RA_ALIVE_VIDX(info, vr) \ + ((TCCIR_DECODE_VREG_TYPE(vr) * (info)->maxpos) + TCCIR_DECODE_VREG_POSITION(vr)) + +/* Cap: a row per vreg over the whole instruction range. Beyond this the + * bitmap is not worth the compile time or the memory, and the allocator just + * keeps its previous behaviour. */ +#define RA_ALIVE_MAX_WORDS (1 << 18) /* 2 MiB */ + +typedef struct +{ + int off, size; + uint32_t released; /* end of the interval that last held the slot */ +} RaFreeSpillSlot; + +typedef struct +{ + SSAInterval *iv; + int size; +} RaActiveSpill; + +typedef struct +{ + RaFreeSpillSlot *free_slots; + int free_count; + RaActiveSpill *active; /* spilled intervals still live */ + int active_count; + uint32_t active_min_end; + /* No reuse when the function calls setjmp, vfork, ...: a slot whose interval + * ended on one return may still be read on the other. */ + int no_reuse; +} RaSpillPool; + +/* A live-range edge for the callee-saved pressure sweep in ra_linear_scan. */ +typedef struct { + uint32_t pos; + int delta; +} RaPressureEv; + +/* ============================================================================ + * Phi Resolution + * ============================================================================ */ + +typedef struct RAPhiCopy { + int32_t dest_vreg; + int32_t src_vreg; + int btype; + uint8_t emitted; +} RAPhiCopy; + +typedef struct RAPhiCopyRecord { + int32_t src_vreg; + int new_instr_idx; +} RAPhiCopyRecord; + +typedef struct RAPhiStats { + int phi_operands; + int parallel_requested; + int parallel_emitted; + int cycle_temporaries; + int participant_spills; + int32_t *participants; + int participant_count; + int participant_cap; +} RAPhiStats; + +#define RA_MAX_PHI_COPY_RECORDS 512 + +/* Candidate (phi, operand) pairs bucketed by predecessor block, in exactly the + * order a block-major scan would visit them. + * + * ra_collect_phi_copies_for_pred() with no successor filter wants every phi + * operand naming one predecessor, and scanning all blocks' phis to find them + * is O(blocks x phi operands) -- once per predecessor, so quadratic. pr34093's + * 1000-case switch still spent ~25% of its compile there after the filtered + * scans were fixed. The buckets hold CANDIDATES, not decisions: + * ra_phi_copy_needed() is still asked about each one, in the same order as + * before, so its phi_pinned side effect is unchanged. */ +typedef struct RAPhiPredIndex +{ + int *off; /* num_blocks + 1 bucket bounds */ + IRPhiNode **phi; /* off[nb] entries */ + int *opidx; +} RAPhiPredIndex; +/* Memory-SSA is opt-in; see the comment at its use site in tcc_ir_ssa_regalloc. */ +TCC_DBG_ENV_FLAG(ra_mem_ssa_fwd, "TCC_MEM_SSA_FWD") +TCC_DBG_ENV_FLAG(ra_mem_ssa_dump, "TCC_MEM_SSA_DUMP") +TCC_DBG_ENV_FLAG(ra_mem_ssa_verify_on, "TCC_MEM_SSA") + +/* Gate + time + dump one flat-region pass, the shape every entry in the + * pre-CFG cluster below shares. `changed` receives the pass's return value so + * the timing dump can report productivity (Phase 0.4 of + * docs/plans/opt_pass_dedup_and_perf.md). */ +#define RA_FLAT_PASS(changed, name, cond, call) \ + do \ + { \ + (changed) = 0; \ + if (tcc_state && (cond) && !tcc_ir_opt_pass_disabled(name)) \ + TCC_PASS_TIMED(changed, name, (call)); \ + tcc_ir_dump_after_pass(ir, name); \ + } while (0) + +/* ============================================================================ + * Spill-driven live-range splitting + * + * Linear scan without splitting spills a value for its whole range: its def + * stores it and EVERY use reloads it, even a use two instructions after the + * def or right after another reload of the same slot. Most spilled values in + * call-heavy code (the Zig compiler: 88% of its spilled intervals cross a + * real call) are forced to memory only by the calls; between two calls a + * caller-saved register would hold them fine. + * + * After the first scan, each spilled 32-bit (or narrower) value V is split + * where a register copy is provably enough: + * - def group: uses a def of V reaches within one call-free region read a + * fresh temp the def now writes; `V <- T` after the def is the store the + * spill needed anyway (dropped when V has one def and no other use); + * - reload groups: uses after an earlier use of V in one call-free region + * share `T <- V` (one reload) placed before that first use. + * Then the allocator runs again on the rewritten IR. The new temps are short + * and cross no call, so they mostly land in caller-saved registers. + * zig.c at -O2: -22 KB of .text; about half of the reloads it removes were + * already elided by the encoder's slot cache, and in the giant functions + * some temps spill again. TCC_DISABLE_PASS=ra:spill_split turns it off. + * + * Reaching, for V with ONE definition d: dominance. d dominates the leader + * and the leader (or d) dominates each member, so a path from the copy to a + * member that does not pass the copy again cannot pass d either, and V still + * holds the copied value there. For V with several definitions: straight + * line only -- no jump target between the copy and the member, and no def of + * V in between. + * + * "Call-free" is only a profitability filter (the second scan gives a temp + * that crosses a call a callee-saved register or a slot, like any value): no + * call in [start, member) on linear order, and no backward jump from beyond + * the member into the range. + * ==========================================================================*/ + +typedef struct +{ + int before; /* old index of the instruction the copy goes in front of */ + int land; /* 1: branches to `before` now land on the copy; 0: they skip it */ + int32_t dest, src; + uint8_t btype, is_unsigned; /* the def's width for a def-group store copy */ +} RaSplitCopy; + +typedef struct +{ + int instr; + int eff; /* where the value is consumed: the paired CALL for a FUNCPARAMVAL */ +} RaSplitUse; + +typedef struct RARelSlot +{ + int off; /* frame offset of the slot */ + int size; /* bytes the access covered (4 or 8; exact, see above) */ + int r0, r1; /* registers whose value it holds (r1 < 0 when 32-bit) */ + int32_t vreg; /* the stored value */ + int def_idx; /* the store that put it there */ + int valid; +} RARelSlot; + +#define RA_REL_MAX_TRACKED 16 + +/* How far the straight-line walk below will look before giving up. A store + * this far from its return is not the shape being targeted. */ +#define RA_DFS_TAIL_LIMIT 64 + +#define RA_DFS_MAX_READS 256 + +/* Defined in regalloc.c. */ +int ra_reg_has_other_holder(SSAInterval **active, int active_count, + const SSAInterval *skip, int hr); +int ir_op_is_implicit_call_ra(TccIrOp op); +int *ra_build_call_prefix(TCCIRState *ir); +const char *ra_vreg_type_char(int type); +void ra_build_intervals(TCCIRState *ir, IRCFG *cfg, IRSSAState *ssa, + SSAInterval **out_intervals, int *out_count, + const int *call_prefix, int *out_max_vreg_pos); +void ra_build_phi_hints(SSAInterval *intervals, int count, + IRSSAState *ssa, IRCFG *cfg, int max_vreg_pos); +void ra_build_assign_hints(SSAInterval *intervals, int count, + TCCIRState *ir, int max_vreg_pos); +void ra_build_load_param_hints(SSAInterval *intervals, int count, + TCCIRState *ir, int max_vreg_pos); +void ra_build_bfi_hints(SSAInterval *intervals, int count, + TCCIRState *ir, int max_vreg_pos); +void ra_build_outgoing_param_hints(SSAInterval *intervals, int count, + TCCIRState *ir, int max_vreg_pos); + +/* Defined in regalloc_scan.c. */ +void ra_alive_free(RaAliveInfo *info); +int ra_alive_worth_building(TCCIRState *ir, const SSAInterval *intervals, int count); +void ra_alive_build(TCCIRState *ir, RaAliveInfo *info, int max_vreg_pos); +void ra_linear_scan(TCCIRState *ir, SSAInterval *intervals, int count, + const RegAllocTarget *target, int spill_base, + uint64_t *out_dirty_int, uint64_t *out_dirty_fp, + int max_vreg_pos, int has_call, const RaAliveInfo *alive); +void ra_write_results(TCCIRState *ir, SSAInterval *intervals, int count); + +/* Defined in regalloc_phi.c. */ +extern int ra_phi_resolve_pre_ra_mode; +int ra_phi_stats_has_participant(RAPhiStats *stats, int32_t vreg); +void ra_eliminate_dead_reg_copies(TCCIRState *ir); +int ra_phi_copy_is_identity(IRLiveInterval *dest_li, IRLiveInterval *src_li); +void ra_resolve_phis(TCCIRState *ir, IRCFG *cfg, IRSSAState *ssa, + RAPhiStats *stats); +void ra_build_live_regs_bitmap(TCCIRState *ir); +void ra_co_ops(TCCIRState *ir, IRQuadCompact *q, + int32_t *out_def, int *has_def, int32_t uses[4], int *nuse); +void ra_build_narrow_weights(TCCIRState *ir, const RegAllocTarget *target, + SSAInterval *intervals, int count, int max_vreg_pos); +void ra_refine_live_regs_accurate(TCCIRState *ir); +void ra_coalesce_graph(TCCIRState *ir, SSAInterval *intervals, int count, + int max_vreg_pos); + +#endif /* REGALLOC_PRIV_H */ diff --git a/source/ir/regalloc_scan.c b/source/ir/regalloc_scan.c new file mode 100644 index 000000000..a1c6e22ab --- /dev/null +++ b/source/ir/regalloc_scan.c @@ -0,0 +1,1990 @@ +/* + * TCC IR - SSA-Aware Register Allocator + * + * Operates directly on SSA-renamed IR with phi nodes. + * Replaces the tccls.c linear scan when -fssa-regalloc is enabled. + * + * Copyright (c) 2025 Mateusz Stadnik + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +/* SSA register allocator: linear-scan allocation, accurate per-vreg + * liveness for the last-resort register share, and writing the results + * back to the IR. */ + +#include "regalloc_priv.h" + +/* ============================================================================ + * Linear Scan Allocation + * ============================================================================ */ + +static int sort_by_start(const void *a, const void *b) +{ + const SSAInterval *ia = (const SSAInterval *)a; + const SSAInterval *ib = (const SSAInterval *)b; + if (ia->is_param && !ib->is_param) return -1; + if (!ia->is_param && ib->is_param) return 1; + if (ia->start < ib->start) return -1; + if (ia->start > ib->start) return 1; + /* Equal starts: precolored intervals first, so their fixed registers are + * claimed before the scan hands the same register to a non-precolored + * interval (the precolored assignment does not check int_free). Then by + * vreg — qsort is not stable, and leaving ties unspecified makes the + * allocation depend on the libc's qsort (host glibc and the device libc + * order equal elements differently). */ + if (ia->precolored >= 0 && ib->precolored < 0) return -1; + if (ia->precolored < 0 && ib->precolored >= 0) return 1; + if (ia->vreg < ib->vreg) return -1; + if (ia->vreg > ib->vreg) return 1; + return 0; +} + +/* Safety check for loop-carried phi coalescing. + * + * Returns 1 if cur can take partner's physical register even though their + * live intervals overlap. The case: cur is defined inside the partner's + * live range by an instruction that reads partner as a source, and + * partner's only remaining use past that def is a single ASSIGN with + * dest = partner, src1 = cur (i.e. the explicit back-edge phi copy left + * over after SSA destruction). Under that pattern partner's value is + * "killed" at cur's def: after the defining instruction R holds cur's + * value, the back-edge ASSIGN becomes mov R, R (elided), and R remains + * the carrier for the same logical loop variable across iterations. + * + * Note: cur->start may have been pulled back to the loop entry by the + * back-edge extension (so cur's "official" start is before its actual + * def). We locate the def position by scanning for the first instruction + * in [cur->start, partner->end] whose dest is cur. + * + * Reads of partner *before* the def are fine: at those positions the + * register holds partner's value (which equals what cur will be assigned + * from in the previous iteration via the back-edge ASSIGN). Reads + * *after* the def — except for the back-edge ASSIGN itself — are not + * fine: they would see cur's value, not partner's. */ +static int ra_safe_loop_phi_coalesce(TCCIRState *ir, SSAInterval *cur, SSAInterval *partner) +{ + int n = ir->next_instruction_index; + int cur_start = (int)cur->start; + int partner_end = (int)partner->end; + int32_t cur_vreg = cur->vreg; + int32_t partner_vreg = partner->vreg; + + if (cur_start < 0 || cur_start >= n || partner_end < cur_start || partner_end >= n) + return 0; + + /* cur (the loop update) must be consumed by the back-edge copy partner<-cur + * at partner_end, so cur must NOT outlive partner. If cur->end > partner_end + * then cur and partner are two DISTINCT values that both span the loop body + * (they interfere), and sharing one register conflates them. This was a + * self-host miscompile: the cross coalesced a pointer-holding interval with + * an index-holding one in ra_coalesce_graph (cur.end > partner.end), yielding + * a register used as both index and pointer, which corrupted that pass's own + * coalescing decisions on later compiles. Legitimate loop-IV updates have + * cur.end <= partner.end (the update is dead after the back-edge copy). */ + if ((int)cur->end > partner_end) + return 0; + + /* Locate cur's def: first instruction in [cur_start, partner_end] whose + * non-STORE dest is cur AND which reads partner as a source. */ + int def_pos = -1; + for (int j = cur_start; j <= partner_end; j++) { + IRQuadCompact *q = &ir->compact_instructions[j]; + if (q->op == TCCIR_OP_NOP) continue; + if (!irop_config[q->op].has_dest) continue; + if (q->op == TCCIR_OP_STORE || q->op == TCCIR_OP_STORE_INDEXED || + q->op == TCCIR_OP_STORE_POSTINC) continue; + IROperand d = tcc_ir_op_get_dest(ir, q); + if (!irop_has_vreg(d) || irop_get_vreg(d) != cur_vreg) continue; + + int reads_partner = 0; + if (irop_config[q->op].has_src1) { + IROperand s = tcc_ir_op_get_src1(ir, q); + if (irop_has_vreg(s) && irop_get_vreg(s) == partner_vreg) reads_partner = 1; + } + if (!reads_partner && irop_config[q->op].has_src2) { + IROperand s = tcc_ir_op_get_src2(ir, q); + if (irop_has_vreg(s) && irop_get_vreg(s) == partner_vreg) reads_partner = 1; + } + if (!reads_partner && tcc_ir_op_is_mac(q->op)) { + IROperand s = tcc_ir_op_get_accum(ir, q); + if (irop_has_vreg(s) && irop_get_vreg(s) == partner_vreg) reads_partner = 1; + } + if (!reads_partner) return 0; + def_pos = j; + break; + } + if (def_pos < 0) return 0; + + int found_back_copy = 0; + for (int j = def_pos + 1; j <= partner_end; j++) { + IRQuadCompact *q = &ir->compact_instructions[j]; + if (q->op == TCCIR_OP_NOP) continue; + + /* cur must be defined only at def_pos. The override's correctness rests on + * "after def_pos the register holds cur's value and the back-edge copy is + * mov R,R"; a *second* def of cur before the back-edge breaks that — the + * register then carries an intermediate value while partner is still + * (textually) live, and coalescing conflates two distinct values. This + * happens when def_pos is a copy `cur <- partner` at the top of an OUTER + * loop body and cur is then re-assigned inside a nested (rotated) inner + * loop before the outer back-edge copy `partner <- cur` (longlong seed 218: + * g12-carried hash T160<-T161, re-defined inside the rotated g16 loop). The + * linear scan cannot model the inner back-edge, so reject conservatively. */ + if (irop_config[q->op].has_dest) { + IROperand cd = tcc_ir_op_get_dest(ir, q); + if (irop_has_vreg(cd) && irop_get_vreg(cd) == cur_vreg) + return 0; + } + + int uses_partner_as_src = 0; + if (irop_config[q->op].has_src1) { + IROperand s = tcc_ir_op_get_src1(ir, q); + if (irop_has_vreg(s) && irop_get_vreg(s) == partner_vreg) + uses_partner_as_src = 1; + } + if (!uses_partner_as_src && irop_config[q->op].has_src2) { + IROperand s = tcc_ir_op_get_src2(ir, q); + if (irop_has_vreg(s) && irop_get_vreg(s) == partner_vreg) + uses_partner_as_src = 1; + } + if (!uses_partner_as_src && tcc_ir_op_is_mac(q->op)) { + IROperand s = tcc_ir_op_get_accum(ir, q); + if (irop_has_vreg(s) && irop_get_vreg(s) == partner_vreg) + uses_partner_as_src = 1; + } + if (uses_partner_as_src) { + if (q->op != TCCIR_OP_ASSIGN) return 0; + IROperand d = tcc_ir_op_get_dest(ir, q); + if (!irop_has_vreg(d) || irop_get_vreg(d) != partner_vreg) return 0; + IROperand s = tcc_ir_op_get_src1(ir, q); + if (!irop_has_vreg(s) || irop_get_vreg(s) != cur_vreg) return 0; + if (found_back_copy) return 0; + found_back_copy = 1; + continue; + } + + if (irop_config[q->op].has_dest) { + int dest_is_use = (q->op == TCCIR_OP_STORE || q->op == TCCIR_OP_STORE_INDEXED || + q->op == TCCIR_OP_STORE_POSTINC); + if (dest_is_use) { + IROperand d = tcc_ir_op_get_dest(ir, q); + if (irop_has_vreg(d) && irop_get_vreg(d) == partner_vreg) + return 0; + } else { + IROperand d = tcc_ir_op_get_dest(ir, q); + if (irop_has_vreg(d) && irop_get_vreg(d) == partner_vreg) { + if (q->op != TCCIR_OP_ASSIGN) return 0; + IROperand s = tcc_ir_op_get_src1(ir, q); + if (!irop_has_vreg(s) || irop_get_vreg(s) != cur_vreg) return 0; + if (found_back_copy) return 0; + found_back_copy = 1; + } + } + } + } + return found_back_copy; +} + +/* Returns 1 if instruction q references `vreg` as any source or destination + * operand (read or write). Mirrors the operand enumeration ra_build_intervals + * uses (src1, src2, MLA accumulator, dest including STORE-class). */ +static int ra_instr_touches_vreg(TCCIRState *ir, IRQuadCompact *q, int32_t vreg) +{ + if (q->op == TCCIR_OP_NOP) + return 0; + if (irop_config[q->op].has_src1) { + IROperand s = tcc_ir_op_get_src1(ir, q); + if (irop_has_vreg(s) && irop_get_vreg(s) == vreg) return 1; + } + if (irop_config[q->op].has_src2) { + IROperand s = tcc_ir_op_get_src2(ir, q); + if (irop_has_vreg(s) && irop_get_vreg(s) == vreg) return 1; + } + if (tcc_ir_op_is_mac(q->op)) { + IROperand s = tcc_ir_op_get_accum(ir, q); + if (irop_has_vreg(s) && irop_get_vreg(s) == vreg) return 1; + } + if (irop_config[q->op].has_dest) { + IROperand d = tcc_ir_op_get_dest(ir, q); + if (irop_has_vreg(d) && irop_get_vreg(d) == vreg) return 1; + } + return 0; +} + +/* Control-flow-aware safety check for "exit-phi" coalescing. + * + * cur is defined by a pure copy `cur <- partner` (an ASSIGN whose only source + * is partner — the SSA-destruction copy left at a loop-exit / block-merge + * edge). Linear scan models partner with a single [start,end] interval whose + * end is partner's last *textual* use. Under TCC's inverted-header loop + * layout the loop body is emitted *after* the loop's exit edge, so partner's + * body uses sit textually below the copy and the interval spuriously overlaps + * cur — even though, in the actual control flow, reaching the copy means the + * loop has exited and partner is dead. ra_safe_loop_phi_coalesce only covers + * the back-edge phi (`partner <- cur`), not this forward exit phi. + * + * Returns 1 when no instruction reachable from *after* the copy reads or + * writes partner — i.e. partner is genuinely dead past the copy. cur and + * partner are then the same logical value over disjoint control-flow regions + * and may share a register; the residual `mov R,R` is erased by the post-RA + * move-coalescing pass. + * + * Reachability is computed directly on the current instruction stream (jump + * targets define the edges). The shared `cfg` cannot be used here: it is + * built before ra_resolve_phis inserts these very copies and is stale by the + * time the linear scan runs. Any indirect/multi-target transfer (IJUMP, + * SWITCH_*) whose successors we cannot enumerate forces a conservative + * reject. */ +static int ra_safe_exit_phi_coalesce(TCCIRState *ir, SSAInterval *cur, SSAInterval *partner) +{ + int n = ir->next_instruction_index; + int32_t cur_vreg = cur->vreg; + int32_t partner_vreg = partner->vreg; + if (cur->start < 0 || (int)cur->start >= n) + return 0; + + /* cur must be partner's forward continuation, which means it outlives + * partner: cur->end >= partner->end. When partner outlives cur, partner is + * still needed on some path after cur dies (e.g. a parameter live across all + * arms of a switch while cur is a copy in one arm); transferring partner's + * register to cur and dropping partner from the active set would then free + * the register while partner is still live elsewhere, corrupting it. */ + if ((int)partner->end > (int)cur->end) + return 0; + + /* Locate cur's first def and require it to be a pure copy `cur <- partner`. */ + int def_pos = -1; + for (int j = (int)cur->start; j < n; j++) { + IRQuadCompact *q = &ir->compact_instructions[j]; + if (q->op == TCCIR_OP_NOP) continue; + if (!irop_config[q->op].has_dest) continue; + IROperand d = tcc_ir_op_get_dest(ir, q); + if (!irop_has_vreg(d) || irop_get_vreg(d) != cur_vreg) continue; + if (q->op != TCCIR_OP_ASSIGN) return 0; + IROperand s = tcc_ir_op_get_src1(ir, q); + if (!irop_has_vreg(s) || irop_get_vreg(s) != partner_vreg) return 0; + def_pos = j; + break; + } + if (def_pos < 0) + return 0; + + /* DFS over instructions reachable from after the copy. Each instruction is + * marked visited *when pushed*, so it is enqueued at most once and the stack + * never exceeds n entries. If a back-edge re-enters def_pos itself, the + * copy's own read of partner is seen and the case is conservatively rejected. */ + uint8_t *visited = tcc_mallocz(n); + int *stack = tcc_malloc(sizeof(int) * n); + int sp = 0, dead = 1; +#define RA_EXITPHI_PUSH(x) do { int _x = (x); \ + if (_x >= 0 && _x < n && !visited[_x]) { visited[_x] = 1; stack[sp++] = _x; } } while (0) + RA_EXITPHI_PUSH(def_pos + 1); + + while (sp > 0) { + int i = stack[--sp]; + IRQuadCompact *q = &ir->compact_instructions[i]; + if (q->op == TCCIR_OP_NOP) { + RA_EXITPHI_PUSH(i + 1); + continue; + } + if (ra_instr_touches_vreg(ir, q, partner_vreg)) { dead = 0; break; } + + if (q->op == TCCIR_OP_RETURNVOID || q->op == TCCIR_OP_RETURNVALUE || + q->op == TCCIR_OP_TRAP) + continue; /* no successor */ + if (q->op == TCCIR_OP_IJUMP || q->op == TCCIR_OP_SWITCH_TABLE || + q->op == TCCIR_OP_SWITCH_LOAD) { dead = 0; break; } /* unknown targets */ + if (q->op == TCCIR_OP_JUMP) { + RA_EXITPHI_PUSH((int)irop_get_imm64_ex(ir, tcc_ir_op_get_dest(ir, q))); + continue; + } + if (q->op == TCCIR_OP_JUMPIF) { + RA_EXITPHI_PUSH((int)irop_get_imm64_ex(ir, tcc_ir_op_get_dest(ir, q))); + RA_EXITPHI_PUSH(i + 1); + continue; + } + /* Ordinary instruction (including calls — keep the fall-through; a + * noreturn call's dead fall-through only makes the check more + * conservative). */ + RA_EXITPHI_PUSH(i + 1); + } +#undef RA_EXITPHI_PUSH + + /* The reachable set is now complete in `visited` (the loop only breaks early + * when dead is already 0). cur may legitimately take partner's register + * only if every definition of cur is one of: + * (a) the copy itself (def_pos), or + * (b) reachable from the copy — cur is the forward continuation of partner + * (e.g. the loop increment), or + * (c) another pure copy `cur <- partner` from the SAME partner — a parallel + * SSA phi copy on a different incoming edge of the same merge; it also + * collapses to mov R,R once cur and partner share R, so it is harmless. + * A def from a DIFFERENT source on a sibling path (a true merge phi, e.g. a + * return-value phi `ret <- a` / `ret <- b`) makes cur hold a value unrelated + * to partner there; coalescing cur into partner's register would corrupt it. */ + if (dead) { + for (int j = 0; j < n; j++) { + if (j == def_pos) continue; + IRQuadCompact *q = &ir->compact_instructions[j]; + if (q->op == TCCIR_OP_NOP || !irop_config[q->op].has_dest) continue; + if (q->op == TCCIR_OP_STORE || q->op == TCCIR_OP_STORE_INDEXED || + q->op == TCCIR_OP_STORE_POSTINC) continue; + IROperand d = tcc_ir_op_get_dest(ir, q); + if (!irop_has_vreg(d) || irop_get_vreg(d) != cur_vreg) continue; + if (visited[j]) continue; /* (b) forward continuation */ + /* (c) a parallel `cur <- partner` copy is fine; anything else rejects. */ + int parallel_copy = 0; + if (q->op == TCCIR_OP_ASSIGN) { + IROperand s = tcc_ir_op_get_src1(ir, q); + if (irop_has_vreg(s) && irop_get_vreg(s) == partner_vreg) + parallel_copy = 1; + } + if (!parallel_copy) { dead = 0; break; } + } + } + + tcc_free(visited); + tcc_free(stack); + return dead; +} + +/* Row of instruction indices at which `vreg` is live (live-in or live-out), + * or NULL when the answer is not known. */ +static const uint64_t *ra_alive_row(const RaAliveInfo *info, int32_t vreg) +{ + if (!info || !info->valid) + return NULL; + int vi = RA_ALIVE_VIDX(info, vreg); + if (vi < 0 || vi >= info->tbl) + return NULL; + int d = info->dense[vi]; + if (d < 0) + return NULL; + return info->rows + (size_t)d * (size_t)info->words; +} + +/* 1 if `vreg` may be live anywhere in [lo,hi]. Conservative: unknown -> 1. + * + * The range asked about is the NEW interval's [start,end] -- the window over + * which the allocator guarantees it the register -- and NOT its accurate live + * set. Those differ: the interval over-approximates, and a donor live inside + * one of its holes has still had its value destroyed by the def at `start`. + * Testing the two live SETS against each other instead looks stronger and is + * wrong; it miscompiles bench_double.c's Mandelbrot kernel. */ +static int ra_alive_busy(const RaAliveInfo *info, int32_t vreg, uint32_t lo, uint32_t hi) +{ + const uint64_t *row = ra_alive_row(info, vreg); + if (!row) + return 1; + if (hi >= (uint32_t)info->n) + hi = (uint32_t)info->n - 1; + if (lo > hi) + return 1; + for (uint32_t k = lo; k <= hi; k++) + { + uint32_t w = k >> 6; + if ((k & 63) == 0 && row[w] == 0 && k + 63 <= hi) + { + k += 63; + continue; + } + if (row[w] & (1ull << (k & 63))) + return 1; + } + return 0; +} +void ra_alive_free(RaAliveInfo *info) +{ + if (!info) + return; + tcc_free(info->dense); + tcc_free(info->rows); + memset(info, 0, sizeof *info); +} + +/* Cheap pre-check so the dataflow is not built for a function that could never + * use it. A graph-coalesced class is disqualifying by construction: the + * representative's register carries the whole class, which its own vreg's + * liveness does not describe. The return-block share is per-interval and is + * handled at the donor (`reg_shared`) plus an interlock in the scan. */ +int ra_alive_worth_building(TCCIRState *ir, const SSAInterval *intervals, int count) +{ + if (tcc_state->optimize < 1) + return 0; + if (tcc_ir_opt_pass_disabled("ra:alive_share")) + return 0; + for (int i = 0; i < count; i++) + if (intervals[i].coalesce_to >= 0 || intervals[i].co_member) + return 0; + /* A back-edge is disqualifying, and MEASURED so: with loops admitted + * double_add improves a further 1.9% and bench_double.c's Mandelbrot kernel + * MISCOMPILES. A loop puts values on a register that no single vreg's + * liveness describes -- the loop-phi absorb is only the visible half -- so + * the donor filters are not sufficient there. */ + const int n = ir->next_instruction_index; + for (int i = 0; i < n; i++) + { + IRQuadCompact *q = &ir->compact_instructions[i]; + if (q->op != TCCIR_OP_JUMP && q->op != TCCIR_OP_JUMPIF) + continue; + int t = (int)irop_get_imm64_ex(ir, tcc_ir_op_get_dest(ir, q)); + if (t >= 0 && t <= i) + return 0; + } + return 1; +} +void ra_alive_build(TCCIRState *ir, RaAliveInfo *info, int max_vreg_pos) +{ + memset(info, 0, sizeof *info); + const int n = ir->next_instruction_index; + if (n <= 0 || max_vreg_pos <= 0) + return; + if (tcc_state->optimize < 1) + return; + /* Un-enumerated edges make the dataflow unsound; same guard the coalescer + * and the scratch-liveness refinement use. */ + for (int i = 0; i < n; i++) + { + int op = ir->compact_instructions[i].op; + if (op == TCCIR_OP_IJUMP || op == TCCIR_OP_SWITCH_TABLE || op == TCCIR_OP_SWITCH_LOAD) + return; + } + + IRCFG *cfg = tcc_ir_cfg_build(ir); + if (!cfg) + return; + tcc_ir_cfg_compute_rpo(cfg); + const int nb = cfg->num_blocks; + if (nb <= 0) + { + tcc_ir_cfg_free(cfg); + return; + } + + const int maxpos = max_vreg_pos; + const int tbl = 4 * maxpos; + #define AVIDX(vr) ((TCCIR_DECODE_VREG_TYPE(vr) * maxpos) + TCCIR_DECODE_VREG_POSITION(vr)) + + int *dense = tcc_malloc(sizeof(int) * tbl); + for (int i = 0; i < tbl; i++) + dense[i] = -1; + int ndense = 0; + for (int i = 0; i < n; i++) + { + int32_t def = -1, hd = 0, uses[4], nu = 0; + ra_co_ops(ir, &ir->compact_instructions[i], &def, &hd, uses, &nu); + for (int k = 0; k < nu; k++) + { + if (!tcc_ir_vreg_is_valid(ir, uses[k])) continue; + int u = AVIDX(uses[k]); + if (u >= 0 && u < tbl && dense[u] < 0) dense[u] = ndense++; + } + if (hd && tcc_ir_vreg_is_valid(ir, def)) + { + int d = AVIDX(def); + if (d >= 0 && d < tbl && dense[d] < 0) dense[d] = ndense++; + } + } + if (ndense == 0) + { + tcc_free(dense); + tcc_ir_cfg_free(cfg); + return; + } + + const int words = (n + 63) / 64; + if ((size_t)ndense * (size_t)words > RA_ALIVE_MAX_WORDS) + { + tcc_free(dense); + tcc_ir_cfg_free(cfg); + return; + } + + /* Deferred PARAM sources are read again at their CALL, after anything + * between the PARAM quad and the call; ra_build_intervals extends ends for + * exactly this, and the liveness has to model it or a share would clobber an + * argument already marshalled. */ + int *call_param_head = tcc_malloc(sizeof(int) * n); + int *param_sibling = tcc_malloc(sizeof(int) * n); + for (int i = 0; i < n; i++) { call_param_head[i] = -1; param_sibling[i] = -1; } + for (int i = 0; i < n; i++) + { + IRQuadCompact *q = &ir->compact_instructions[i]; + if (q->op != TCCIR_OP_FUNCPARAMVAL) continue; + IROperand s1 = tcc_ir_op_get_src1(ir, q); + if (!irop_has_vreg(s1) || irop_is_immediate(s1)) continue; + int cid = TCCIR_DECODE_CALL_ID(irop_get_imm64_ex(ir, tcc_ir_op_get_src2(ir, q))); + if (cid < 0) continue; + for (int j = i + 1; j < n; j++) + { + IRQuadCompact *qq = &ir->compact_instructions[j]; + if (qq->op != TCCIR_OP_FUNCCALLVOID && qq->op != TCCIR_OP_FUNCCALLVAL) continue; + if (TCCIR_DECODE_CALL_ID(irop_get_imm64_ex(ir, tcc_ir_op_get_src2(ir, qq))) == cid) + { + param_sibling[i] = call_param_head[j]; + call_param_head[j] = i; + break; + } + } + } + #define AV_FOR_PARAM_USES(call_idx, vr_, body) \ + do { \ + for (int p_ = call_param_head[(call_idx)]; p_ >= 0; p_ = param_sibling[p_]) { \ + IROperand ps_ = tcc_ir_op_get_src1(ir, &ir->compact_instructions[p_]); \ + int32_t vr_ = irop_get_vreg(ps_); \ + if (!tcc_ir_vreg_is_valid(ir, vr_)) continue; \ + body \ + } \ + } while (0) + + const int bw = (ndense + 63) / 64; + uint64_t *use_b = tcc_mallocz(sizeof(uint64_t) * (size_t)nb * bw); + uint64_t *def_b = tcc_mallocz(sizeof(uint64_t) * (size_t)nb * bw); + uint64_t *in_b = tcc_mallocz(sizeof(uint64_t) * (size_t)nb * bw); + uint64_t *out_b = tcc_mallocz(sizeof(uint64_t) * (size_t)nb * bw); + + for (int b = 0; b < nb; b++) + { + uint64_t *ub = use_b + (size_t)b * bw, *db = def_b + (size_t)b * bw; + int s = cfg->blocks[b].start_idx, e = cfg->blocks[b].end_idx; + for (int i = s; i < e && i < n; i++) + { + int32_t def = -1, hd = 0, uses[4], nu = 0; + ra_co_ops(ir, &ir->compact_instructions[i], &def, &hd, uses, &nu); + for (int k = 0; k < nu; k++) + { + if (!tcc_ir_vreg_is_valid(ir, uses[k])) continue; + int u = AVIDX(uses[k]); if (u < 0 || u >= tbl || dense[u] < 0) continue; + if (!tcc_bitspan_test(db, dense[u])) tcc_bitspan_set(ub, dense[u]); + } + AV_FOR_PARAM_USES(i, vr, { + int u = AVIDX(vr); if (u < 0 || u >= tbl || dense[u] < 0) continue; + if (!tcc_bitspan_test(db, dense[u])) tcc_bitspan_set(ub, dense[u]); + }); + if (hd && tcc_ir_vreg_is_valid(ir, def)) + { + int d = AVIDX(def); if (d >= 0 && d < tbl && dense[d] >= 0) tcc_bitspan_set(db, dense[d]); + } + } + } + + int changed = 1, guard = 0; + while (changed && guard++ < nb + 4) + { + changed = 0; + for (int ri = cfg->rpo_count - 1; ri >= 0; ri--) + { + int b = cfg->rpo_order ? cfg->rpo_order[ri] : ri; + if (b < 0 || b >= nb) continue; + uint64_t *lo = out_b + (size_t)b * bw, *li = in_b + (size_t)b * bw; + uint64_t *ub = use_b + (size_t)b * bw, *db = def_b + (size_t)b * bw; + tcc_bitspan_zero(lo, bw); + for (int si = 0; si < cfg->blocks[b].num_succs; si++) + { + int sb = cfg->blocks[b].succs[si]; + if (sb < 0 || sb >= nb) continue; + tcc_bitspan_or(lo, in_b + (size_t)sb * bw, bw); + } + changed |= tcc_bitspan_or_andnot(li, ub, lo, db, bw); + } + } + + uint64_t *rows = tcc_mallocz(sizeof(uint64_t) * (size_t)ndense * words); + uint64_t *live = tcc_malloc(sizeof(uint64_t) * bw); + for (int b = 0; b < nb; b++) + { + int s = cfg->blocks[b].start_idx, e = cfg->blocks[b].end_idx; + tcc_bitspan_copy(live, out_b + (size_t)b * bw, bw); + for (int i = (e < n ? e : n) - 1; i >= s; i--) + { + /* Mark live-OUT of i, then live-IN of i: a vreg counted at either end + * must keep its register across i. */ + tcc_bitspan_for_each_set(live, bw, ndense, di) + { + rows[(size_t)di * words + (i >> 6)] |= (1ull << (i & 63)); + } + int32_t def = -1, hd = 0, uses[4], nu = 0; + ra_co_ops(ir, &ir->compact_instructions[i], &def, &hd, uses, &nu); + if (hd && tcc_ir_vreg_is_valid(ir, def)) + { + int d = AVIDX(def); + if (d >= 0 && d < tbl && dense[d] >= 0) tcc_bitspan_reset(live, dense[d]); + } + for (int k = 0; k < nu; k++) + { + if (!tcc_ir_vreg_is_valid(ir, uses[k])) continue; + int u = AVIDX(uses[k]); if (u < 0 || u >= tbl || dense[u] < 0) continue; + tcc_bitspan_set(live, dense[u]); + } + AV_FOR_PARAM_USES(i, vr, { + int u = AVIDX(vr); if (u < 0 || u >= tbl || dense[u] < 0) continue; + tcc_bitspan_set(live, dense[u]); + }); + tcc_bitspan_for_each_set(live, bw, ndense, di) + { + rows[(size_t)di * words + (i >> 6)] |= (1ull << (i & 63)); + } + } + } + + tcc_free(live); + tcc_free(use_b); tcc_free(def_b); tcc_free(in_b); tcc_free(out_b); + tcc_free(call_param_head); tcc_free(param_sibling); + tcc_ir_cfg_free(cfg); + #undef AV_FOR_PARAM_USES + #undef AVIDX + + info->valid = 1; + info->n = n; + info->tbl = tbl; + info->maxpos = maxpos; + info->words = words; + info->ndense = ndense; + info->dense = dense; + info->rows = rows; +} + +static int ra_may_need_frame_pointer(const TCCIRState *ir) +{ + if (tcc_state->force_frame_pointer) + return 1; + if (func_var) + return 1; + if (ir->has_static_chain || tcc_state->nb_nested_funcs > 0) + return 1; + for (int i = 0; i < ir->next_instruction_index; i++) + { + const int op = ir->compact_instructions[i].op; + if (op == TCCIR_OP_VLA_ALLOC) + return 1; + if (op >= TCCIR_OP_FADD && op <= TCCIR_OP_CVT_FTOI) + return 1; + if (op == TCCIR_OP_ASM_INPUT || op == TCCIR_OP_INLINE_ASM || op == TCCIR_OP_ASM_OUTPUT) + return 1; + } + return 0; +} + +/* Can physical register `r` be handed to `cur` even though an active interval + * still owns it? Yes exactly when EVERY active interval holding it is provably + * dead across the whole of cur's range: then no execution reads that register's + * old value from inside the range, and on the paths where the old value IS read + * later, cur was never defined and never wrote it. + * + * Every donor property that would make its vreg's own liveness an incomplete + * description of what the register holds -- a coalesced class, a loop-phi + * absorb, a return-block share, an address-taken home -- disqualifies it. */ +static int ra_alive_reg_shareable(const RaAliveInfo *alive, SSAInterval *cur, + SSAInterval **active, int active_count, int r, + const RegAllocTarget *target) +{ + if (!alive || !alive->valid) + return 0; + if (r < 0 || r >= tcc_state->registers_for_allocator) + return 0; + if (cur->crosses_call) + { + int ok = 0; + for (int ci = 0; ci < target->int_class.num_callee_saved; ci++) + if (target->int_class.callee_saved[ci] == r) { ok = 1; break; } + if (!ok) + return 0; + } + int holders = 0; + for (int j = 0; j < active_count; j++) + { + SSAInterval *a = active[j]; + if (a == cur || a->stack_location != 0) + continue; + if (a->r0 != r && a->r1 != r) + continue; + holders++; + if (a->precolored >= 0 || a->co_member || a->loop_phi_locked || a->reg_shared || a->addrtaken) + return 0; + if (a->reg_type == LS_REG_TYPE_FLOAT || a->reg_type == LS_REG_TYPE_DOUBLE) + return 0; + if (ra_alive_busy(alive, (int32_t)a->vreg, cur->start, cur->end)) + return 0; + } + return holders > 0; +} + +/* Spill-slot reuse. A spilled interval only touches its slot inside + * [start, end], so once it has expired the slot can hold a later interval of + * the same size. Without this every spill took a fresh slot for the whole + * function, and a frame grew with the TOTAL number of spills rather than the + * most that are live at once -- Zig's generated C, one long function body of + * short-lived temporaries after another, had median frames of 104 bytes where + * gcc needs 36, with 123k stack accesses past the 16-bit LDR/STR [sp] reach. + * + * A slot is reused only if its previous owner ended before the new owner + * STARTS. That is automatic for an interval spilled when the scan reaches it, + * but not for an eviction victim: it had a register since its start and is + * spilled for the whole interval, so a slot freed after that start is still + * shared. TCC_NO_SPILL_REUSE=1 restores the fresh-slot behaviour. */ +TCC_DBG_ENV_FLAG(ra_no_spill_reuse, "TCC_NO_SPILL_REUSE") + +static void ra_spill_pool_expire(RaSpillPool *p, uint32_t pos) +{ + if (p->active_count == 0 || p->active_min_end >= pos) + return; + int w = 0; + p->active_min_end = UINT32_MAX; + for (int j = 0; j < p->active_count; j++) + { + SSAInterval *a = p->active[j].iv; + if (a->end < pos) + { + RaFreeSpillSlot *f = &p->free_slots[p->free_count++]; + f->off = a->stack_location; + f->size = p->active[j].size; + f->released = a->end; + } + else + { + p->active[w++] = p->active[j]; + if (a->end < p->active_min_end) + p->active_min_end = a->end; + } + } + p->active_count = w; +} + +/* Give `iv` a `size`-byte spill slot: a free one released before iv->start, + * else a fresh one below *spill_loc. */ +static void ra_spill_pool_assign(RaSpillPool *p, SSAInterval *iv, int size, int *spill_loc) +{ + int off = 0, found = 0; + if (!p->no_reuse) + for (int k = p->free_count - 1; k >= 0; k--) + if (p->free_slots[k].size == size && p->free_slots[k].released < iv->start) + { + off = p->free_slots[k].off; + p->free_slots[k] = p->free_slots[--p->free_count]; + found = 1; + break; + } + if (!found) + { + *spill_loc -= size; + off = *spill_loc; + } + iv->stack_location = off; + p->active[p->active_count].iv = iv; + p->active[p->active_count++].size = size; + if (iv->end < p->active_min_end) + p->active_min_end = iv->end; +} + +/* What spilling `iv` for its whole range costs: a reload per (loop-weighted) + * use plus the store of its definition. */ +static uint32_t ra_spill_cost(const SSAInterval *iv) +{ + return (uint32_t)iv->use_count + 1; +} + +static int ra_pressure_ev_cmp(const void *a, const void *b) +{ + const RaPressureEv *x = a, *y = b; + if (x->pos != y->pos) + return x->pos < y->pos ? -1 : 1; + return y->delta - x->delta; /* starts before ends at one position: the upper bound */ +} +void ra_linear_scan(TCCIRState *ir, SSAInterval *intervals, int count, + const RegAllocTarget *target, int spill_base, + uint64_t *out_dirty_int, uint64_t *out_dirty_fp, + int max_vreg_pos, int has_call, const RaAliveInfo *alive) +{ + if (count <= 0) return; + + qsort(intervals, count, sizeof(SSAInterval), sort_by_start); + + /* Accurate-liveness register sharing; ra_alive_worth_building did the + * gating, and a failed build leaves `valid` clear. + * + * The return-block share below (`reg_shared`) is the one other mechanism + * that puts a second owner on a register, and it leaves that owner OUT of + * `active` -- so neither mechanism can see the other's extra holder. The + * two are interlocked: whichever fires first in a function locks the other + * out for the rest of it. */ + int share_ok = (alive && alive->valid); + int alive_shared_used = 0; + int reg_shared_used = 0; + const int evict_density = tcc_state->optimize >= 1 && !tcc_ir_opt_pass_disabled("ra:evict_density"); + const int low_saved_pref = tcc_state->optimize >= 1 && !tcc_ir_opt_pass_disabled("ra:low_saved_pref"); + + /* Build vreg -> interval lookup for phi hint resolution */ + int hint_tbl_size = (max_vreg_pos > 0) ? 4 * max_vreg_pos : 1; + SSAInterval **vreg_to_iv = tcc_mallocz(sizeof(SSAInterval *) * hint_tbl_size); + #define HINT_IDX(vr) \ + ((TCCIR_DECODE_VREG_TYPE(vr) * max_vreg_pos) + TCCIR_DECODE_VREG_POSITION(vr)) + for (int i = 0; i < count; i++) { + int idx = HINT_IDX(intervals[i].vreg); + if (idx >= 0 && idx < hint_tbl_size) + vreg_to_iv[idx] = &intervals[i]; + } + + /* Register availability bitmaps */ + uint64_t int_avail = 0; + for (int i = 0; i < target->int_class.num_caller_saved; i++) + int_avail |= (1ull << target->int_class.caller_saved[i]); + for (int i = 0; i < target->int_class.num_callee_saved; i++) + int_avail |= (1ull << target->int_class.callee_saved[i]); + + uint64_t fp_avail = 0; + for (int i = 0; i < target->fp_class.num_caller_saved; i++) + fp_avail |= (1ull << target->fp_class.caller_saved[i]); + for (int i = 0; i < target->fp_class.num_callee_saved; i++) + fp_avail |= (1ull << target->fp_class.callee_saved[i]); + + /* Respect tcc_state->registers_for_allocator limit */ + uint64_t int_allowed = int_avail & tcc_state->registers_map_for_allocator; + uint64_t fp_allowed = fp_avail & tcc_state->float_registers_map_for_allocator; + + /* Nested-function trampolines load the static chain into the static-chain + * register (R10) and tail-jump to the nested function, clobbering R10 + * without restoring it — even though R10 is AAPCS callee-saved. A parent + * that defines nested functions calls them (directly, or via a function + * pointer through an ABI-compliant helper) and would see any value the + * allocator parked in R10 corrupted across that call (nestfunc-2: the + * i/j/k loop counters lived in R10 and the inner foo() call clobbered it → + * infinite loop). Reserve R10 in such parents. Nested functions + * themselves (has_static_chain) hold the live chain in R10 and are handled + * separately, so leave their allocation untouched. */ + if (tcc_state->nb_nested_funcs > 0 && !ir->has_static_chain) + int_allowed &= ~(1ull << (uint64_t)architecture_config.static_chain_reg); + + /* The frame-pointer register (R7 on Thumb) is held out of the target's + * allocator map because the backend needs it as the frame base — but only + * in the minority of functions that actually get a frame pointer. The FP + * decision is made AFTER allocation (tcc_ir_codegen_generate and the + * prologue), so free the register here only when no forcing condition can + * fire. ra_may_need_frame_pointer must stay a SUPERSET of those + * conditions: predicting FP when none materializes just leaves the + * register unused, but the reverse would hand out the frame base. The + * prologue hard-fails if they ever disagree. A low register matters + * doubly on Thumb: it takes 16-bit encodings that R8/R10/R11 cannot, and + * every value it holds is one fewer spill. */ + if (target->frame_pointer_reg >= 0 && !ra_may_need_frame_pointer(ir)) + int_allowed |= int_avail & (1ull << (uint64_t)target->frame_pointer_reg); + + /* Withhold one register in functions that address shared .rodata often. + * Each such reference otherwise reloads the segment's runtime base from the + * GOT through a scratch register it has to push and pop; parking the base in + * a callee-saved register for the whole body turns every one of them into a + * bare ADD. The backend claims the register after allocation, from whatever + * this pass left free (tcc_gen_machine_rodata_anchor_claim), so all this has + * to do is leave one unallocated — and only where the references pay for it, + * since a withheld register is one fewer for values. Purely a size trade: + * if the backend declines to claim, the register is merely unused. */ + if (target->rodata_anchor_reg >= 0 && target->rodata_anchor_sites && + target->rodata_anchor_sites(ir, RA_RODATA_ANCHOR_MIN_SITES) >= RA_RODATA_ANCHOR_MIN_SITES) + int_allowed &= ~(1ull << (uint64_t)target->rodata_anchor_reg); + + uint64_t int_free = int_allowed; + uint64_t fp_free = fp_allowed; + uint64_t dirty_int = 0; + uint64_t dirty_fp = 0; + + /* Active set sorted by end point */ + SSAInterval **active = tcc_malloc(sizeof(SSAInterval *) * count); + int active_count = 0; + + /* Active addrtaken intervals — tracked separately because the main `active` + * set is for register-resident intervals; addrtaken intervals always spill + * and were previously dropped on the floor. We track them so their stack + * slots can be returned to a free list once the interval ends. */ + SSAInterval **active_addrtaken = tcc_malloc(sizeof(SSAInterval *) * count); + int active_addrtaken_count = 0; + uint32_t active_addrtaken_min_end = UINT32_MAX; + + /* Free list of expired 4-byte addrtaken stack slots, available for reuse + * by later addrtaken intervals. Only 4-byte slots are tracked here; other + * sizes fall through to fresh allocation, matching the legacy behavior of + * always assigning `spill_loc -= 4` regardless of value width. */ + int *free_slots_4 = tcc_malloc(sizeof(int) * count); + int free_slots_4_count = 0; + + /* When more call-crossing INT values are live at once than there are free + * low callee-saved registers, r8-r11 get pushed whatever the order, so the + * question is only which values take r4-r7. First come first served gave + * them to whatever started earliest -- a pointer passed from call to call + * -- and the value compared with #0 after every call landed in r9: every + * CMP #imm, CBZ, LDR/STR [Rn,#imm] and ADDS on it 32-bit (9,400 CMP.W on a + * high register in zig.c against gcc's 3,400). Then a call-crossing value + * at most one 16-bit-capable op touches takes a free high register first, + * leaving r4-r7 to the values that use them (ra:hi_callee_pref). Measured + * on zig.c -Os: narrow_uses <= 1 -11.5 KB, == 0 -1.4, <= 2 -4.6; density + * gates (as the non-crossing preference uses) all lost. */ + int hi_callee_pref = 0; + if (tcc_state->optimize >= 1 && !tcc_ir_opt_pass_disabled("ra:hi_callee_pref")) { + int low_cs = __builtin_popcountll(int_free & 0xf0ull); + int nev = 0; + RaPressureEv *ev = tcc_malloc(sizeof(RaPressureEv) * 2 * (count > 0 ? count : 1)); + for (int i = 0; i < count; i++) { + SSAInterval *iv = &intervals[i]; + if (iv->coalesce_to >= 0 || !iv->crosses_call || iv->addrtaken) + continue; + const int w = iv->reg_type == LS_REG_TYPE_INT ? 1 : iv->reg_type == LS_REG_TYPE_LLONG ? 2 : 0; + if (!w) + continue; + ev[nev].pos = iv->start, ev[nev].delta = w, nev++; + ev[nev].pos = iv->end + 1, ev[nev].delta = -w, nev++; + } + qsort(ev, nev, sizeof(RaPressureEv), ra_pressure_ev_cmp); + int live = 0, peak = 0; + for (int k = 0; k < nev; k++) { + live += ev[k].delta; + if (live > peak) + peak = live; + } + tcc_free(ev); + hi_callee_pref = peak > low_cs; + } + + int spill_loc = spill_base; + RaSpillPool spool = {0}; + spool.free_slots = tcc_malloc(sizeof(RaFreeSpillSlot) * count); + spool.active = tcc_malloc(sizeof(RaActiveSpill) * count); + spool.active_min_end = UINT32_MAX; + spool.no_reuse = ra_no_spill_reuse() || tcc_ir_calls_returns_twice(ir); + + for (int i = 0; i < count; i++) { + SSAInterval *cur = &intervals[i]; + + /* Graph coalescing: non-representative members are merged into their + * representative's interval and inherit its register after the scan. Skip + * them so they neither consume a register nor enter the active set. */ + if (cur->coalesce_to >= 0) + continue; + + /* Expire old intervals. + * + * First compute which hard registers the SURVIVING (still-live) intervals + * still occupy. A register may be held by two active intervals at once + * when a phi/merge copy coalesced them onto the same register: e.g. an + * if/else merge temp for a variable inherits the register from one arm + * (via its single hint_vreg) while the other arm's interval is still live + * and also holds that register. If the shorter of the two expires first, + * returning its register to the free pool would let a later allocation + * (a loop-body temp, say) clobber the value the longer, still-live + * interval needs after the loop. Only free a register when no surviving + * interval still holds it. */ + uint64_t survivor_int = 0, survivor_fp = 0; + for (int j = 0; j < active_count; j++) { + SSAInterval *a = active[j]; + if (a->end >= cur->start && a->r0 >= 0 && a->stack_location == 0) { + if (a->reg_type == LS_REG_TYPE_FLOAT || a->reg_type == LS_REG_TYPE_DOUBLE) { + survivor_fp |= (1ull << a->r0); + if (a->r1 >= 0) survivor_fp |= (1ull << a->r1); + } else { + survivor_int |= (1ull << a->r0); + if (a->r1 >= 0) survivor_int |= (1ull << a->r1); + } + } + } + int w = 0; + for (int j = 0; j < active_count; j++) { + SSAInterval *a = active[j]; + if (a->end < cur->start) { + /* Free register — but not if cur shared hr with another active + * interval (reg_shared): the partner still logically owns hr, + * so freeing it here would let a later allocation clobber the + * loop body's view of partner. Likewise skip any register a + * surviving active interval still holds (coalesced sharing). */ + if (a->r0 >= 0 && a->stack_location == 0 && !a->reg_shared) { + if (a->reg_type == LS_REG_TYPE_FLOAT || a->reg_type == LS_REG_TYPE_DOUBLE) { + if (!(survivor_fp & (1ull << a->r0))) fp_free |= (1ull << a->r0); + if (a->r1 >= 0 && !(survivor_fp & (1ull << a->r1))) fp_free |= (1ull << a->r1); + } else { + if (!(survivor_int & (1ull << a->r0))) int_free |= (1ull << a->r0); + if (a->r1 >= 0 && !(survivor_int & (1ull << a->r1))) int_free |= (1ull << a->r1); + } + } + } else { + active[w++] = a; + } + } + active_count = w; + + /* Expire old addrtaken intervals — return their 4-byte slots to the + * free list so later non-overlapping addrtaken intervals can reuse them. + * Skip the scan when no active entry can expire (min_end >= cur->start). */ + if (active_addrtaken_count > 0 && active_addrtaken_min_end < cur->start) { + int wa = 0; + active_addrtaken_min_end = UINT32_MAX; + for (int j = 0; j < active_addrtaken_count; j++) { + SSAInterval *a = active_addrtaken[j]; + if (a->end < cur->start) { + free_slots_4[free_slots_4_count++] = a->stack_location; + } else { + active_addrtaken[wa++] = a; + if (a->end < active_addrtaken_min_end) + active_addrtaken_min_end = a->end; + } + } + active_addrtaken_count = wa; + } + + ra_spill_pool_expire(&spool, cur->start); + + /* Address-taken: force spill. + * Reuse an expired addrtaken slot when one is available; otherwise grow + * the spill area. Slot reuse is correct here because the addrtaken + * extension pass in ra_build_intervals has already pushed V's end past + * the death of any pointer derived from V — so two intervals with + * non-overlapping (extended) lifetimes truly access disjoint memory + * windows. Track in active_addrtaken so the slot returns to the free + * list when cur expires. + * A volatile local is forced to memory the same way: every access must + * be a real ldr/str, so it must never occupy a register. */ + if (cur->addrtaken || cur->is_volatile) { + if (free_slots_4_count > 0 && !spool.no_reuse) { + cur->stack_location = free_slots_4[--free_slots_4_count]; + } else { + spill_loc -= 4; + cur->stack_location = spill_loc; + } + active_addrtaken[active_addrtaken_count++] = cur; + if (cur->end < active_addrtaken_min_end) + active_addrtaken_min_end = cur->end; + continue; + } + + /* Precolored: assign fixed register */ + if (cur->precolored >= 0) { + int reg = cur->precolored; + cur->r0 = reg; + if (LS_IS_VFP_REG(reg)) { + /* Hard-float float param precolored to its incoming VFP register. */ + fp_free &= ~(1ull << (uint64_t)LS_VFP_REG_NUM(reg)); + dirty_fp |= (1ull << (uint64_t)LS_VFP_REG_NUM(reg)); + } else { + int_free &= ~(1ull << reg); + dirty_int |= (1ull << reg); + } + /* Insert into active set */ + active[active_count++] = cur; + continue; + } + + /* Float/double: use FP class */ + if (cur->reg_type == LS_REG_TYPE_FLOAT) { + int reg = -1; + /* Every allocatable single-precision VFP register (s0-s13) is caller-saved + * and there are no callee-saved VFP registers in the allocation map, so a + * float that is live across a REAL call cannot stay in a VFP register — + * force it to a stack slot. Native FP-op "implicit calls" (a single- + * precision vadd.f32) clobber nothing, so they do not force a spill. */ + if (!cur->crosses_real_call && fp_free) { + reg = __builtin_ctzll(fp_free); + fp_free &= ~(1ull << reg); + dirty_fp |= (1ull << reg); + } + if (reg >= 0) { + cur->r0 = LS_VFP_REG_BASE + reg; + active[active_count++] = cur; + } else { + ra_spill_pool_assign(&spool, cur, 4, &spill_loc); + } + continue; + } + + if (cur->reg_type == LS_REG_TYPE_DOUBLE) { + /* Need even-aligned pair */ + int reg = -1; + for (int r = 0; r < 30; r += 2) { + if ((fp_free & (3ull << r)) == (3ull << r)) { + reg = r; + break; + } + } + if (reg >= 0) { + fp_free &= ~(3ull << reg); + dirty_fp |= (3ull << reg); + cur->r0 = LS_VFP_REG_BASE + reg; + cur->r1 = LS_VFP_REG_BASE + reg + 1; + active[active_count++] = cur; + } else { + ra_spill_pool_assign(&spool, cur, 8, &spill_loc); + } + continue; + } + + /* 64-bit integer or soft-float double: need register pair */ + if (cur->reg_type == LS_REG_TYPE_LLONG || cur->reg_type == LS_REG_TYPE_DOUBLE_SOFT || + cur->reg_type == LS_REG_TYPE_COMPLEX_FLOAT) { + int r0 = -1, r1 = -1; + + /* Return-pair preference for 64-bit call results. + * + * A call returns its 64-bit result in r0:r1. When that result does not + * cross another call (it is consumed before the next call — e.g. it + * feeds that call's first argument), keeping it in r0:r1 avoids a move + * from the return pair to a callee-saved/other pair. This is the + * dominant cost in chained soft-float expressions: in pr58574's Horner + * polynomials each __aeabi_dmul / __aeabi_dadd result feeds the next + * helper call, and parking the result anywhere but r0:r1 forces a dead + * mov of the return value to its home pair. + * + * Boundary reuse: the pair may still be held by the just-returned call's + * last argument(s), whose intervals end exactly at this call. Those + * values were copied into their AAPCS argument registers before the call + * and are dead afterwards (the call clobbered r0:r1 with the result), so + * the pair is free to take. Evict any such boundary occupant — the same + * idea as the single-reg pref_reg boundary eviction below. */ + /* The mirror case: a pair the function RETURNS. AAPCS requires it in + * r0:r1 at the return, so computing it there directly avoids + * `mov r0,r4 / mov r1,r5` AND the push/pop of the callee-saved pair it + * would otherwise occupy — `long long add64(a,b){return a+b;}` goes from + * 7 instructions to 3, matching GCC. Same !crosses_call requirement: a + * call in between would clobber r0:r1. */ + int wants_return_pair = 0; + if (!cur->crosses_call && 1 < tcc_state->registers_for_allocator) { + if ((int)cur->start < ir->next_instruction_index && + ir->compact_instructions[cur->start].op == TCCIR_OP_FUNCCALLVAL) { + wants_return_pair = 1; /* incoming: this pair IS a call's result */ + } else if ((int)cur->end < ir->next_instruction_index) { + IRQuadCompact *eq = &ir->compact_instructions[cur->end]; + if (eq->op == TCCIR_OP_RETURNVALUE && + irop_get_vreg(tcc_ir_op_get_src1(ir, eq)) == cur->vreg) + wants_return_pair = 1; /* outgoing: this pair is what we return */ + } + } + if (wants_return_pair) { + int want0 = 0, want1 = 1; /* r0:r1 */ + int avail0 = (int_free & (1ull << want0)) != 0; + int avail1 = (int_free & (1ull << want1)) != 0; + int evict0 = -1, evict1 = -1; + for (int k = 0; k < active_count && (!avail0 || !avail1); k++) { + SSAInterval *a = active[k]; + if (a->stack_location != 0 || a->end != cur->start) + continue; + /* Same co-holder rule: an interval whose registers someone else + * still holds is not a victim — freeing them frees nothing. */ + if (ra_reg_has_other_holder(active, active_count, a, a->r0) || + (a->r1 >= 0 && ra_reg_has_other_holder(active, active_count, a, a->r1))) + continue; + if (!avail0 && (a->r0 == want0 || a->r1 == want0)) { evict0 = k; avail0 = 1; } + if (!avail1 && (a->r0 == want1 || a->r1 == want1)) { evict1 = k; avail1 = 1; } + } + if (avail0 && avail1) { + int idxs[2], ni = 0; + if (evict0 >= 0) idxs[ni++] = evict0; + if (evict1 >= 0 && evict1 != evict0) idxs[ni++] = evict1; + if (ni == 2 && idxs[0] < idxs[1]) { int t = idxs[0]; idxs[0] = idxs[1]; idxs[1] = t; } + for (int e = 0; e < ni; e++) { + SSAInterval *a = active[idxs[e]]; + int_free |= (1ull << a->r0); + if (a->r1 >= 0) int_free |= (1ull << a->r1); + active[idxs[e]] = active[--active_count]; + } + r0 = want0; r1 = want1; + } + } + + /* In-place pair inheritance. UMAAL accumulates into its destination + * pair, and ZEXT / PACK64 build theirs from words that usually die at + * that very instruction: taking the dying operands' registers makes the + * moves the emitter would otherwise need disappear. A donor qualifies + * only when this instruction is its last use (end == cur->start), and + * only on the terms the pair boundary reuse below sets for donors. */ + if (r0 < 0 && cur->reg_type == LS_REG_TYPE_LLONG && !cur->co_member && + tcc_state->optimize >= 1 && max_vreg_pos > 0 && + (int)cur->start < ir->next_instruction_index) { + IRQuadCompact *dq = &ir->compact_instructions[cur->start]; + SSAInterval *don[2] = {NULL, NULL}; + if (dq->op == TCCIR_OP_UMAAL) { + int32_t av = irop_get_vreg(tcc_ir_op_get_accum(ir, dq)); + int ai = av >= 0 ? HINT_IDX(av) : -1; + SSAInterval *p = (ai >= 0 && ai < hint_tbl_size) ? vreg_to_iv[ai] : NULL; + if (p && p->r1 >= 0) + don[0] = don[1] = p; + } else if (dq->op == TCCIR_OP_ZEXT || dq->op == TCCIR_OP_PACK64) { + for (int w = 0; w < (dq->op == TCCIR_OP_PACK64 ? 2 : 1); w++) { + IROperand so = w ? tcc_ir_op_get_src2(ir, dq) : tcc_ir_op_get_src1(ir, dq); + int32_t sv = (!so.is_lval && irop_has_vreg(so)) ? irop_get_vreg(so) : -1; + int si = sv >= 0 ? HINT_IDX(sv) : -1; + SSAInterval *p = (si >= 0 && si < hint_tbl_size) ? vreg_to_iv[si] : NULL; + /* A word-typed source naming a pair reads its low register. */ + if (p && (p->r1 < 0 || !irop_needs_pair(so))) + don[w] = p; + } + } + int want[2] = {-1, -1}; + for (int w = 0; w < 2; w++) { + SSAInterval *p = don[w]; + if (!p || p->end != cur->start || p->r0 < 0 || p->stack_location != 0 || + p->co_member || p->loop_phi_locked || p->alive_shared || + (p->reg_type != LS_REG_TYPE_INT && p->reg_type != LS_REG_TYPE_LLONG)) + continue; + /* UMAAL inherits the accumulator pair word for word; ZEXT / PACK64 + * read each source's low register. */ + int hr = (dq->op == TCCIR_OP_UMAAL && w == 1) ? p->r1 : p->r0; + /* Retiring the donor frees both of its registers: neither may have + * another holder (an alive_share borrower). */ + if (hr >= tcc_state->registers_for_allocator || + ra_reg_has_other_holder(active, active_count, p, p->r0) || + (p->r1 >= 0 && ra_reg_has_other_holder(active, active_count, p, p->r1))) + continue; + if (cur->crosses_call) { + int cs = 0; + for (int ci = 0; ci < target->int_class.num_callee_saved; ci++) + if (target->int_class.callee_saved[ci] == hr) cs = 1; + if (!cs) + continue; + } + want[w] = hr; + } + if (want[0] >= 0 && want[0] == want[1]) + want[1] = -1; + /* Every other path hands out pairs with r0 < r1, and code downstream + * leans on it: the return-pair eviction takes r0:r1 from a boundary + * donor, so a donor pair r4:r0 made a 64-bit ADD write r0 (its low + * word) before reading r0 (the source's high word). Keep the order; + * a crossed pair of donors keeps just the low one. */ + if (want[0] >= 0 && want[1] >= 0 && want[0] > want[1]) + want[1] = -1; + if (want[0] >= 0 || want[1] >= 0) { + /* The other word (ZEXT's high half, or an operand that stays live) + * takes any free register the pair rules allow. */ + for (int w = 0; w < 2; w++) { + if (want[w] >= 0) + continue; + for (int r = 0; r < 13; r++) { + if (!(int_free & (1ull << r)) || r == want[1 - w] || + r >= tcc_state->registers_for_allocator) + continue; + if (want[1 - w] >= 0 && (w == 0 ? r > want[1] : r < want[0])) + continue; + if (cur->crosses_call) { + int cs = 0; + for (int ci = 0; ci < target->int_class.num_callee_saved; ci++) + if (target->int_class.callee_saved[ci] == r) cs = 1; + if (!cs) + continue; + } + want[w] = r; + break; + } + } + if (want[0] >= 0 && want[1] >= 0 && want[0] < want[1]) { + /* Commit: retire the donors that supplied a register (they end + * here, and nothing else holds their registers), so the + * registers are ours. */ + for (int w = 0; w < 2; w++) { + SSAInterval *p = don[w]; + if (!p || (w == 1 && p == don[0])) + continue; + int took = 0; + for (int x = 0; x < 2; x++) + took |= want[x] == p->r0 || (p->r1 >= 0 && want[x] == p->r1); + if (!took) + continue; + for (int k = 0; k < active_count; k++) { + if (active[k] == p && p->end == cur->start) { + if (p->r0 >= 0) int_free |= (1ull << p->r0); + if (p->r1 >= 0) int_free |= (1ull << p->r1); + active[k] = active[--active_count]; + break; + } + } + } + r0 = want[0]; + r1 = want[1]; + } + } + } + + if (r0 < 0 && cur->crosses_call && target->int_class.pair_align) { + /* Prefer callee-saved even-aligned pairs (R4:R5, R6:R7, R8:R9, R10:R11) */ + for (int r = 4; r < 12; r += 2) { + if ((int_free & (3ull << r)) == (3ull << r)) { + r0 = r; r1 = r + 1; + break; + } + } + } + if (r0 < 0 && !cur->crosses_call && target->int_class.pair_align) { + /* Try any even-aligned pair (only for non-call-crossing) */ + for (int r = 0; r < 12; r += 2) { + if ((int_free & (3ull << r)) == (3ull << r)) { + r0 = r; r1 = r + 1; + break; + } + } + } + if (r0 < 0 && !cur->crosses_call) { + /* Fallback: any two registers (only for non-call-crossing) */ + int first = -1; + for (int r = 0; r < 13; r++) { + if (int_free & (1ull << r)) { + if (first < 0) first = r; + else { r0 = first; r1 = r; break; } + } + } + } + if (r0 < 0 && cur->crosses_call && cur->reg_type == LS_REG_TYPE_LLONG) { + /* Fallback: any two callee-saved registers (need not be an aligned + * pair). Thumb-2 LDRD/STRD accept any two distinct r0..r12/r14 + * (try_ldrd_pair / try_strd_pair in arm-thumb-gen.c only enforce + * lo != hi and Rn != SP/PC). Other 64-bit ops (cmp+sbcs, adds+adcs, + * load/store as two 32-bit halves) likewise have no pair-alignment + * requirement. Without this fallback, a call-crossing 64-bit value + * spills whenever the small set of aligned callee-saved pairs is + * busy — e.g. when R7 is excluded as Thumb FP, R4:R5/R6:R7/R8:R9/ + * R10:R11 quickly collide with the loop IV, array base, and another + * live 64-bit value. + * + * If two free callee-saved are not immediately available, evict up to + * two single-INT victims (lowest-use, end > cur->end, currently in a + * callee-saved reg) to free the pair. Mirrors the eviction policy + * used for single-INT spill below, just iterated to fill both slots. + * Bail out if a candidate victim has use_count >= cur->use_count to + * avoid spilling a hotter interval. + * + * Restricted to LS_REG_TYPE_LLONG to avoid evicting INT base pointers + * for soft-double / complex-float pairs, where the cost/benefit shifts + * (softfloat helpers reload args anyway, and complex temporaries are + * often short-lived). Enabling it for those types regressed + * gcc-execute/20021120-1::foo by 200+ insns in benchmarks. */ + for (int round = 0; r0 < 0 && round < 3; round++) { + int first = -1; + for (int ci = 0; ci < target->int_class.num_callee_saved; ci++) { + int r = target->int_class.callee_saved[ci]; + if (int_free & (1ull << r)) { + if (first < 0) first = r; + else { r0 = first; r1 = r; break; } + } + } + if (r0 >= 0) break; + SSAInterval *victim = NULL; + int victim_idx = -1; + uint16_t victim_uses = UINT16_MAX; + for (int j = 0; j < active_count; j++) { + SSAInterval *a = active[j]; + if (a->precolored >= 0) continue; + if (a->reg_type != LS_REG_TYPE_INT) continue; + if (a->end <= cur->end) continue; + if (a->r0 < 0 || a->stack_location != 0) continue; + /* Never evict a loop-phi-locked interval: it holds a loop-carried + * value whose register is SHARED with a coalesce partner that stays + * live (see the loop-phi coalescing above and the identical guard in + * the single-register spill victim scan below). Spilling it here + * frees a register the partner still occupies, so a later pair + * allocation would double-book it and clobber the loop-carried value + * (combo_num seed 84127: the g16 loop counter lost to a 64-bit OR's + * high half). */ + if (a->loop_phi_locked) continue; + int is_callee = 0; + for (int ci = 0; ci < target->int_class.num_callee_saved; ci++) { + if (target->int_class.callee_saved[ci] == a->r0) { is_callee = 1; break; } + } + if (!is_callee) continue; + if (a->use_count < victim_uses) { + victim_uses = a->use_count; + victim = a; + victim_idx = j; + } + } + if (!victim || victim_uses >= cur->use_count) break; + int_free |= (1ull << victim->r0); + ra_spill_pool_assign(&spool, victim, 4, &spill_loc); + victim->r0 = -1; + active[victim_idx] = active[--active_count]; + } + } + if (r0 >= 0) { + int_free &= ~((1ull << r0) | (1ull << r1)); + dirty_int |= ((1ull << r0) | (1ull << r1)); + cur->r0 = r0; + cur->r1 = r1; + active[active_count++] = cur; + } else { + /* Boundary reuse for 64-bit pairs. + * + * An interval whose end == cur->start is read (at the latest) by the + * very instruction that defines cur, so its pair is free the moment + * that instruction retires. The strict `<` expiration above still + * counts it as active, which is why the pair paths saw 0-1 free + * registers and spilled temps that live for a single instruction. + * Handing cur exactly that pair makes dest == that source, and the + * dest-equals-source case is safe for the ops below: + * + * ADD/SUB/AND/OR/XOR : thumb_emit_data_processing_mop64 writes the + * low half then reads the high half, so an EXACT pair match is + * fine (partial overlap is not — we never create one, since we + * take both registers of one interval). + * SHL/SHR/SAR : thumb_emit_shift64_mop is already alias + * aware (it computes the cross-shift into a temp first and + * reorders the halves when dst aliases src). + * ASSIGN : a copy onto itself; both movs become + * mov rX, rX and are elided. + * + * MUL/UMULL/MLA are excluded: UMULL requires its destination pair not + * to overlap the sources at all. Calls are excluded because the + * result lands in fixed registers. This is the pair analogue of the + * single-INT phi-hint boundary case further below, whose comment + * blanket-excluded pairs; the shift emitter turned out to be safe. */ + int def_op = (cur->start < ir->next_instruction_index) + ? ir->compact_instructions[cur->start].op : -1; + int op_ok = (def_op == TCCIR_OP_ADD || def_op == TCCIR_OP_SUB || def_op == TCCIR_OP_AND || + def_op == TCCIR_OP_OR || def_op == TCCIR_OP_XOR || def_op == TCCIR_OP_SHL || + def_op == TCCIR_OP_SHR || def_op == TCCIR_OP_SAR || def_op == TCCIR_OP_ASSIGN); + int reused = 0; + if (op_ok && cur->reg_type == LS_REG_TYPE_LLONG) { + for (int k = 0; k < active_count; k++) { + SSAInterval *p = active[k]; + if (p->end != cur->start || p->r0 < 0 || p->r1 < 0 || p->stack_location != 0) + continue; + if (p->co_member || cur->co_member || p->loop_phi_locked) + continue; + /* A pair the donor itself only BORROWED is not the donor's to pass + * on. ra_alive_reg_shareable proved the real owner dead across the + * DONOR's range and no further, so inheriting the pair here extends + * the loan over a range nothing has checked -- and the owner, still + * live, finds its register rewritten under it. + * + * __aeabi_dadd built exactly that chain: alive_share lent r4:r5 to + * a temp over [140,148] while the aligned-mantissa phi owned them + * across [131,180], boundary reuse passed them to [148,149] and + * then to the sticky-bit value over [149,160], and the phi's + * mantissa came back as the other operand's. 1.0 + 12.0 returned + * 24.0. */ + if (p->alive_shared) + continue; + if (p->r0 >= tcc_state->registers_for_allocator || + p->r1 >= tcc_state->registers_for_allocator) + continue; + if (cur->crosses_call) { + int c0 = 0, c1 = 0; + for (int ci = 0; ci < target->int_class.num_callee_saved; ci++) { + if (target->int_class.callee_saved[ci] == p->r0) c0 = 1; + if (target->int_class.callee_saved[ci] == p->r1) c1 = 1; + } + if (!c0 || !c1) + continue; + } + cur->r0 = p->r0; + cur->r1 = p->r1; + ir->ls.dirty_registers |= (1ull << p->r0) | (1ull << p->r1); + active[k] = active[--active_count]; + active[active_count++] = cur; + reused = 1; + break; + } + } + if (reused) + continue; + + /* Share two registers whose current owners are dead across cur's whole + * range, rather than writing a 64-bit temp to the frame and reading it + * straight back. */ + if (share_ok && !reg_shared_used) + { + int p0 = -1, p1 = -1; + for (int r = 0; r < 13 && p1 < 0; r++) + { + if (int_free & (1ull << r)) + continue; /* a free register would have been taken above */ + if (!ra_alive_reg_shareable(alive, cur, active, active_count, r, target)) + continue; + if (p0 < 0) p0 = r; else p1 = r; + } + if (p1 >= 0) + { + dirty_int |= ((1ull << p0) | (1ull << p1)); + cur->r0 = p0; + cur->r1 = p1; + cur->alive_shared = 1; + alive_shared_used = 1; + active[active_count++] = cur; + RA_DBG(" alive_share pair T%d [%u,%u] -> R%d:R%d", + TCCIR_DECODE_VREG_POSITION(cur->vreg), cur->start, cur->end, p0, p1); + continue; + } + } + + ra_spill_pool_assign(&spool, cur, 8, &spill_loc); + } + continue; + } + + /* Complex double: always spill (128-bit) */ + if (cur->reg_type == LS_REG_TYPE_COMPLEX_DOUBLE) { + ra_spill_pool_assign(&spool, cur, 16, &spill_loc); + continue; + } + + /* Integer: single register */ + int reg = -1; + RA_DBG(" alloc T%d [%u,%u] xcall=%d narrow=%u int_free=0x%llx active=%d", + TCCIR_DECODE_VREG_POSITION(cur->vreg), cur->start, cur->end, + cur->crosses_call, cur->narrow_uses, (unsigned long long)int_free, active_count); + + /* Phi-coalescing: try the hinted partner's register first. + * ra_build_phi_hints set hint_vreg to a vreg this interval used to + * share a phi edge with (now resolved into an explicit ASSIGN copy). + * If the partner has expired and freed its register, taking that + * same register here turns the explicit copy into mov rX, rX which + * the post-RA move-coalescing pass erases. + * + * Boundary case: when partner->end == cur->start, the standard `<` + * expiration kept partner active, so its register looks busy here. + * But within a single ARM 3-operand instruction (or ASSIGN/`mov`), + * sources are read before the dest is written — so the same register + * can serve both. Allow the hint to take that register and force + * partner out of the active set so subsequent allocations see it as + * free. Restricted to single-register INT to avoid corrupting register + * pairs (umull, ll-shift, etc.) where the architecture forbids dest/ + * source overlap. */ + if (cur->hint_vreg >= 0 && max_vreg_pos > 0 && !cur->co_member) { + int hint_idx = HINT_IDX(cur->hint_vreg); + if (hint_idx >= 0 && hint_idx < hint_tbl_size) { + SSAInterval *partner = vreg_to_iv[hint_idx]; + if (partner && !partner->co_member && partner->r0 >= 0 && partner->r1 < 0 && + partner->stack_location == 0) { + int hr = partner->r0; + int hr_free = (int_free & (1ull << hr)) != 0; + int boundary = !hr_free && partner->end == cur->start && + cur->reg_type == LS_REG_TYPE_INT && + partner->reg_type == LS_REG_TYPE_INT; + /* Partner expiring at cur->start does NOT free hr while an + * alive_share borrower's interval runs past it: taking hr here + * would clobber the borrower, and marking it free would let the + * next allocation clobber it again. */ + if (boundary && ra_reg_has_other_holder(active, active_count, partner, hr)) + boundary = 0; + if ((hr_free || boundary) && + hr < tcc_state->registers_for_allocator) { + int ok = 1; + if (cur->crosses_call) { + ok = 0; + for (int ci = 0; ci < target->int_class.num_callee_saved; ci++) { + if (target->int_class.callee_saved[ci] == hr) { ok = 1; break; } + } + } + if (ok) { + reg = hr; + if (boundary) { + /* Force partner out of active so its register doesn't + * appear taken to subsequent intervals. */ + for (int k = 0; k < active_count; k++) { + if (active[k] == partner) { + int_free |= (1ull << hr); + active[k] = active[--active_count]; + break; + } + } + } + } + } + } + } + } + + /* Loop-carried phi coalescing. + * + * The standard boundary case above fires when partner expires exactly + * at cur->start. A loop-carried phi (sum_array's `sum + arr[i]`, + * for example) extends the partner's live interval to the back-edge + * JMP — past cur's def — so partner's register looks taken at the + * point we want to coalesce. If cur's defining instruction reads + * partner and the only remaining use of partner before partner->end + * is an ASSIGN `partner <-- cur` (the back-edge copy), the two share + * a value from cur->start onward and can share the register. + * + * Transfer ownership of R from partner to cur in the active set: + * extend cur->end to cover partner->end, remove partner from active, + * and let cur represent R there. Both vregs keep r0 = R so codegen + * resolves either name to the same register, and the back-edge + * ASSIGN becomes mov R, R which post-RA cleanup elides. Without + * this transfer R would be double-counted and the expire phase would + * free it as soon as partner's original end lapses while cur is still + * alive in R. */ + int coalesced_loop_phi = 0; + if (reg < 0 && cur->hint_vreg >= 0 && max_vreg_pos > 0 && !cur->co_member && + cur->reg_type == LS_REG_TYPE_INT && tcc_state->optimize >= 1) { + int hint_idx = HINT_IDX(cur->hint_vreg); + if (hint_idx >= 0 && hint_idx < hint_tbl_size) { + SSAInterval *partner = vreg_to_iv[hint_idx]; + if (partner && !partner->co_member && partner->r0 >= 0 && partner->r1 < 0 && + partner->stack_location == 0 && + partner->reg_type == LS_REG_TYPE_INT && + (int)partner->end > (int)cur->start) { + int hr = partner->r0; + int partner_active_idx = -1; + for (int k = 0; k < active_count; k++) { + if (active[k] == partner) { partner_active_idx = k; break; } + } + if (partner_active_idx >= 0 && !(int_free & (1ull << hr)) && + hr < tcc_state->registers_for_allocator) { + int ok = 1; + if (cur->crosses_call) { + ok = 0; + for (int ci = 0; ci < target->int_class.num_callee_saved; ci++) { + if (target->int_class.callee_saved[ci] == hr) { ok = 1; break; } + } + } + if (ok && (ra_safe_loop_phi_coalesce(ir, cur, partner) || + ra_safe_exit_phi_coalesce(ir, cur, partner))) { + reg = hr; + coalesced_loop_phi = 1; + if (partner->end > cur->end) + cur->end = partner->end; + cur->r0 = reg; + /* cur now carries partner's loop-carried value over the extended + * range; the partner is gone from active, so cur is the sole + * holder of hr that the partner's remaining uses depend on. + * Evicting cur mid-loop would spill it without reloading those + * partner uses → IV corruption. Lock it against eviction. */ + cur->loop_phi_locked = 1; + active[partner_active_idx] = active[--active_count]; + } + } + } + } + } + + /* Soft preference: try iv->pref_reg first (e.g., r0 for vregs that + * feed RETURNVALUE). If free, take it. If a still-active interval + * ends exactly at cur->start and holds that register, evict it and + * take it (boundary case — same logic as the phi-coalescing hint). */ + if (reg < 0 && cur->pref_reg >= 0 && cur->reg_type == LS_REG_TYPE_INT) { + int hr = (int)cur->pref_reg; + if (hr < tcc_state->registers_for_allocator) { + int hr_free = (int_free & (1ull << hr)) != 0; + int ok = 1; + if (cur->crosses_call) { + ok = 0; + for (int ci = 0; ci < target->int_class.num_callee_saved; ci++) { + if (target->int_class.callee_saved[ci] == hr) { ok = 1; break; } + } + } + if (ok) { + if (hr_free) { + reg = hr; + } else { + /* Boundary: an active INT interval ending at cur->start in hr. */ + for (int k = 0; k < active_count; k++) { + SSAInterval *a = active[k]; + if (a->r0 == hr && a->r1 < 0 && a->end == cur->start && + a->stack_location == 0 && a->reg_type == LS_REG_TYPE_INT && + !ra_reg_has_other_holder(active, active_count, a, hr)) { + int_free |= (1ull << hr); + active[k] = active[--active_count]; + reg = hr; + break; + } + } + } + /* Return-block register sharing: cur is a single-use RETURNVALUE + * feeder whose def at cur->start and consumer at cur->end form + * a return tail. Control returns to the caller at cur->end, so + * we can SHARE hr with whatever interval `partner` is holding + * it — cur's def overwrites hr, but we never re-execute past + * the return on this control-flow path. + * + * Safety conditions: + * 1. cur->end is RETURNVALUE/RETURNVOID. + * 2. partner's vreg is not READ at any instruction in + * [cur->start, cur->end] — cur's def would clobber it. + * + * Sharing model: cur->r0 = hr, but we do NOT remove partner + * from active, do NOT mark hr free in int_free, and set + * cur->reg_shared so cur's expire phase does not free hr + * (partner still logically owns it). Other CFG paths through + * partner's live range emit their own reads of hr unaffected — + * those paths never execute cur's def, so partner's value + * remains intact in hr along them. */ + if (reg < 0 && !alive_shared_used && cur->end > cur->start && + (int)cur->end < ir->next_instruction_index) { + IRQuadCompact *eq = &ir->compact_instructions[cur->end]; + if (eq->op == TCCIR_OP_RETURNVALUE || eq->op == TCCIR_OP_RETURNVOID) { + for (int k = 0; k < active_count; k++) { + SSAInterval *a = active[k]; + if (a->r0 != hr || a->r1 >= 0 || a->stack_location != 0 || + a->reg_type != LS_REG_TYPE_INT) + continue; + int conflict = 0; + for (int p = (int)cur->start; p <= (int)cur->end && !conflict; p++) { + IRQuadCompact *pq = &ir->compact_instructions[p]; + /* Any operand reference to the partner clobbers the share: + * cur's def at cur->start overwrites hr, so partner must not be + * needed anywhere in the range. Use ra_instr_touches_vreg so a + * STORE-class op's dest (its base *pointer*, which the store + * READS) and an MLA accumulator count — a naive src1/src2 scan + * missed a partner used as a store base and shared hr anyway, + * emitting `str rX, [rX]` (value written through itself). */ + if (ra_instr_touches_vreg(ir, pq, a->vreg)) { conflict = 1; break; } + } + if (!conflict) { + cur->r0 = hr; + cur->reg_shared = 1; + reg_shared_used = 1; + dirty_int |= (1ull << hr); + reg = hr; + break; + } + } + } + } + } + } + } + + if (reg < 0 && cur->crosses_call && hi_callee_pref && cur->narrow_uses <= 1) { + for (int ci = 0; ci < target->int_class.num_callee_saved && reg < 0; ci++) { + int r = target->int_class.callee_saved[ci]; + if (r >= 8 && r < tcc_state->registers_for_allocator && (int_free & (1ull << r))) + reg = r; + } + } + if (reg < 0 && cur->crosses_call) { + /* Prefer callee-saved */ + for (int ci = 0; ci < target->int_class.num_callee_saved; ci++) { + int r = target->int_class.callee_saved[ci]; + if (int_free & (1ull << r)) { reg = r; break; } + } + } + if (reg < 0 && !cur->crosses_call && + (int)cur->start < ir->next_instruction_index) { + IRQuadCompact *dq = &ir->compact_instructions[cur->start]; + if (tcc_ir_op_is_mac(dq->op) && + irop_get_vreg(tcc_ir_op_get_dest(ir, dq)) == cur->vreg) { + IROperand accum = tcc_ir_op_get_accum(ir, dq); + if (accum.is_sym) { + static const int sym_mla_order[] = {10, 11, 8, 9}; + for (int oi = 0; oi < 4; oi++) { + int r = sym_mla_order[oi]; + if (r < tcc_state->registers_for_allocator && + (int_free & (1ull << r))) { + reg = r; + break; + } + } + } + } + } + if (reg < 0 && !cur->crosses_call) { + /* Narrow-dense intervals prefer r4-r7 over r12 for 16-bit encodings; gates in docs/regalloc_narrow_pref.md */ + static const int alloc_order[] = {0, 1, 2, 3, 12, 4, 5, 6, 7, 8, 9, 10, 11}; + static const int alloc_order_narrow[] = {0, 1, 2, 3, 4, 5, 6, 7, 12, 8, 9, 10, 11}; + int narrow_dense = cur->narrow_uses > 0 && + (uint32_t)cur->narrow_uses * 8 >= (cur->end - cur->start); + const int *order = narrow_dense ? alloc_order_narrow : alloc_order; + /* A low callee-saved register the prologue already saves beats R12: every + * access through R12 is a 32-bit encoding, and the push costs nothing + * more. zig.c -O2: -9 KB (TCC_DISABLE_PASS=ra:low_saved_pref). */ + if (low_saved_pref && order == alloc_order) { + uint64_t lowdirty = int_free & dirty_int & 0xf0ull; + for (int r = 4; r <= 7 && reg < 0; r++) + if ((lowdirty & (1ull << r)) && r < tcc_state->registers_for_allocator) + reg = r; + } + for (int pass = 0; pass < 2 && reg < 0; pass++) { + for (int oi = 0; oi < 13; oi++) { + int r = order[oi]; + if (r >= tcc_state->registers_for_allocator) continue; + if (!(int_free & (1ull << r))) continue; + if (pass == 0 && order == alloc_order_narrow) { + if (r >= 4 && r <= 7 && !(dirty_int & (1ull << r)) && !has_call && + cur->narrow_uses < 2) + continue; + if (r >= 8 && r != 12) + continue; + } + reg = r; + break; + } + } + } + + if (cur->reg_shared) { + /* Return-block share: cur->r0 was set in the pref_reg path. + * Don't touch int_free (partner still owns hr) and don't add cur + * to active (cur's expire would otherwise hit the !reg_shared + * guard but adding it is just bookkeeping; the simpler invariant + * is "shared cur never enters active"). */ + } else if (coalesced_loop_phi) { + /* Partner was removed from active above; transfer R's ownership + * to cur. int_free stays unchanged (R was taken via partner, + * now taken via cur). */ + dirty_int |= (1ull << reg); + active[active_count++] = cur; + } else if (reg >= 0) { + int_free &= ~(1ull << reg); + dirty_int |= (1ull << reg); + cur->r0 = reg; + active[active_count++] = cur; + } else { + /* Before spilling, look for a register whose current owner is provably + * dead across cur's whole range: sharing it costs nothing, where a spill + * costs a store plus a reload at every use. */ + if (share_ok && !reg_shared_used) { + int sh = -1; + for (int r = 0; r < 13; r++) { + if (int_free & (1ull << r)) continue; /* a free one was taken above */ + if (!ra_alive_reg_shareable(alive, cur, active, active_count, r, target)) continue; + sh = r; + break; + } + if (sh >= 0) { + dirty_int |= (1ull << sh); + cur->r0 = sh; + alive_shared_used = 1; + active[active_count++] = cur; + RA_DBG(" alive_share T%d [%u,%u] -> R%d", + TCCIR_DECODE_VREG_POSITION(cur->vreg), cur->start, cur->end, sh); + continue; + } + } + /* Spill: among intervals that extend past cur, evict the one + * with the lowest spill cost (fewest loop-weighted uses). + * use_count is already weighted by loop depth (4^depth per use), + * so this prefers evicting intervals with few loop-hot uses. + * + * ra:evict_density weighs that cost by the range the eviction frees: + * cost per remaining instruction. Without splitting a victim is gone + * for the rest of its range, so a long, sparsely used one frees its + * register for every later interval, where a short hot one frees it + * for a few. zig.c -O2: -21 KB. Refusing the eviction when cur is + * the colder one (spill cur instead) was measured +47 KB: the freed + * register is what keeps the next intervals out of memory. */ + SSAInterval *victim = NULL; + int victim_idx = -1; + uint16_t victim_uses = UINT16_MAX; + for (int j = 0; j < active_count; j++) { + SSAInterval *a = active[j]; + if (a->precolored >= 0) continue; + if (a->reg_type != LS_REG_TYPE_INT) continue; + if (a->end <= cur->end) continue; + /* Never evict a loop-phi-locked interval: it holds a loop-carried value + * (its absorbed partner's uses still read this register across the loop + * body) and spilling it here would not reload those uses. */ + if (a->loop_phi_locked) continue; + /* Its register has to be ITS ALONE. alive_share puts a second live + * holder on a register the allocator still shows as taken by the first, + * so spilling the one we picked does NOT free it -- the other is still + * living there, and handing the register to cur destroys its value. + * The expire path states the same invariant ("only free a register when + * no surviving interval still holds it"); eviction had no equivalent. + * + * Measured: alive_share lends R12 to T509 over [105,122] while T411 owns + * it across [102,126]; eviction then spills T411 and gives R12 to T510 + * over [115,119], overwriting T509 between its def and its use. fuzz + * longlong 197 and 1449 at -O2, combo_num 1103 at -O1. */ + { + int co_held = 0; + for (int k = 0; k < active_count; k++) { + SSAInterval *b = active[k]; + if (b != a && b != cur && b->stack_location == 0 && + (b->r0 == a->r0 || b->r1 == a->r0)) { co_held = 1; break; } + } + if (co_held) continue; + } + if (evict_density) { + if (!victim || + (uint64_t)ra_spill_cost(a) * (victim->end - cur->start + 1) < + (uint64_t)ra_spill_cost(victim) * (a->end - cur->start + 1)) { + victim_uses = a->use_count; + victim = a; + victim_idx = j; + } + } else if (a->use_count < victim_uses || + (a->use_count == victim_uses && victim && a->end > victim->end)) { + victim_uses = a->use_count; + victim = a; + victim_idx = j; + } + } + if (victim) { + /* Evict victim, give its register to cur */ + reg = victim->r0; + victim->r0 = -1; + ra_spill_pool_assign(&spool, victim, 4, &spill_loc); + /* Remove victim from active */ + active[victim_idx] = active[--active_count]; + cur->r0 = reg; + active[active_count++] = cur; + } else { + ra_spill_pool_assign(&spool, cur, 4, &spill_loc); + } + } + } + + tcc_free(spool.free_slots); + tcc_free(spool.active); + tcc_free(active); + tcc_free(active_addrtaken); + tcc_free(free_slots_4); + tcc_free(vreg_to_iv); + #undef HINT_IDX + + if (TCC_LOG_LS) { + RA_DBG("SSA ra_linear_scan: allocation results (%d intervals)", count); + for (int i = 0; i < count; i++) { + SSAInterval *iv = &intervals[i]; + int type = TCCIR_DECODE_VREG_TYPE(iv->vreg); + int pos = TCCIR_DECODE_VREG_POSITION(iv->vreg); + if (iv->stack_location != 0) { + RA_DBG(" %s%d [%u,%u] -> spill(%d)", ra_vreg_type_char(type), pos, + iv->start, iv->end, iv->stack_location); + } else if (iv->r1 >= 0) { + RA_DBG(" %s%d [%u,%u] -> R%d:R%d", ra_vreg_type_char(type), pos, + iv->start, iv->end, iv->r0, iv->r1); + } else { + RA_DBG(" %s%d [%u,%u] -> R%d", ra_vreg_type_char(type), pos, + iv->start, iv->end, iv->r0); + } + } + RA_DBG(" dirty_int=0x%llx dirty_fp=0x%llx", (unsigned long long)dirty_int, (unsigned long long)dirty_fp); + } + + *out_dirty_int = dirty_int; + *out_dirty_fp = dirty_fp; +} + +/* ============================================================================ + * Write Results to IR + * ============================================================================ */ + +void ra_write_results(TCCIRState *ir, SSAInterval *intervals, int count) +{ + /* Clear LS intervals and repopulate from SSA results */ + tcc_ls_clear_live_intervals(&ir->ls); + + for (int i = 0; i < count; i++) { + SSAInterval *iv = &intervals[i]; + tcc_ls_add_live_interval(&ir->ls, iv->vreg, iv->start, iv->end, + iv->crosses_call, iv->addrtaken, iv->reg_type, + 0, iv->precolored); + LSLiveInterval *lsi = &ir->ls.intervals[ir->ls.next_interval_index - 1]; + lsi->r0 = iv->r0; + lsi->r1 = iv->r1; + lsi->stack_location = iv->stack_location; + lsi->co_member = iv->co_member; + + /* Also write to IRLiveInterval for codegen */ + tcc_ir_stack_reg_assign(ir, iv->vreg, iv->stack_location, iv->r0, iv->r1); + IRLiveInterval *li = tcc_ir_vreg_live_interval(ir, iv->vreg); + if (li) { + li->start = iv->start; + li->end = iv->end; + li->crosses_call = iv->crosses_call; + } + } +} diff --git a/source/ir/self_store.c b/source/ir/self_store.c new file mode 100644 index 000000000..0a6dce664 --- /dev/null +++ b/source/ir/self_store.c @@ -0,0 +1,275 @@ +/* + * TinyCC - Tiny C Compiler + * + * Self-store removal: stores that write back the value just loaded from the + * same frame bytes. + * + * Frame relayout (frame.c) gives objects with disjoint lifetimes the same + * bytes. A copy `u = t` where t dies at the copy and u starts there is such a + * pair, so after relayout the word-by-word copy reads and writes the same slot: + * + * T1 <-- StackLoc[-16] [LOAD] + * T2 <-- StackLoc[-12] [LOAD] + * StackLoc[-16] <-- T1 [STORE] + * StackLoc[-12] <-- T2 [STORE] + * + * The stores change nothing. In the Zig compiler built as C these are 5,367 + * groups, 56 KB of code (22 KB in main_buildOutputType alone). + * + * A STORE to a frame slot is deleted when its value is a TEMP loaded from the + * same slot earlier in the same straight run of code, the store writes no more + * bytes than the load read, and nothing between them can write memory there: + * no call, no store through a pointer, no store overlapping the slot, no inline + * asm, and no jump target (another path could arrive with other bytes in the + * slot). The load is deleted too when the store was its value's only use. + * + * Runs on the final IR, after register allocation, when every frame operand + * carries its final offset. + */ + +#include "ir.h" + +#define SC_MAX_DIST 64 + +static int sc_width(IROperand op) +{ + switch (irop_get_btype(op)) + { + case IROP_BTYPE_INT8: + return 1; + case IROP_BTYPE_INT16: + return 2; + case IROP_BTYPE_INT32: + case IROP_BTYPE_FLOAT32: + return 4; + case IROP_BTYPE_INT64: + case IROP_BTYPE_FLOAT64: + return 8; + default: + return 0; + } +} + +/* A direct access to frame bytes: an lvalue stack operand without a vreg. + * Sets its offset and width. */ +static int sc_frame_access(TCCIRState *ir, IROperand op, int *off, int *width) +{ + if (irop_is_none(op) || irop_get_vreg(op) >= 0 || op.is_param || !op.is_lval) + return 0; + int tag = irop_get_tag(op); + if (tag != IROP_TAG_STACKOFF && !(tag == IROP_TAG_VREG && (op.is_local || op.is_llocal))) + return 0; + if (tcc_ir_access_is_volatile(ir, op)) + return 0; + *off = irop_get_stack_offset(op); + *width = sc_width(op); + return *width > 0; +} + +/* A TEMP read as a plain value (not dereferenced). */ +static int32_t sc_temp_value(IROperand op) +{ + int32_t vr = irop_get_vreg(op); + if (vr < 0 || op.is_lval || op.is_llocal || TCCIR_DECODE_VREG_TYPE(vr) != TCCIR_VREG_TYPE_TEMP) + return -1; + if (irop_get_tag(op) != IROP_TAG_VREG) + return -1; + return vr; +} + +static int sc_has_slot3(int op) +{ + return tcc_ir_op_is_mac(op) || op == TCCIR_OP_LOAD_INDEXED || op == TCCIR_OP_STORE_INDEXED || op == TCCIR_OP_SELECT; +} + +static IROperand sc_slot(TCCIRState *ir, IRQuadCompact *q, int s) +{ + if (s == 0 && irop_config[q->op].has_dest) + return tcc_ir_op_get_dest(ir, q); + if (s == 1 && irop_config[q->op].has_src1) + return tcc_ir_op_get_src1(ir, q); + if (s == 2 && irop_config[q->op].has_src2) + return tcc_ir_op_get_src2(ir, q); + if (s == 3 && sc_has_slot3(q->op)) + return ir->iroperand_pool[q->operand_base + 3]; + return IROP_NONE; +} + +/* Ops that neither transfer control nor write memory other than through + * their destination operand. */ +static int sc_plain_op(int op) +{ + switch (op) + { + case TCCIR_OP_ADD: + case TCCIR_OP_ADC_USE: + case TCCIR_OP_ADC_GEN: + case TCCIR_OP_SUB: + case TCCIR_OP_SUBC_USE: + case TCCIR_OP_SUBC_GEN: + case TCCIR_OP_MUL: + case TCCIR_OP_MLA: + case TCCIR_OP_UMULL: + case TCCIR_OP_DIV: + case TCCIR_OP_UMOD: + case TCCIR_OP_IMOD: + case TCCIR_OP_AND: + case TCCIR_OP_OR: + case TCCIR_OP_XOR: + case TCCIR_OP_SHL: + case TCCIR_OP_SAR: + case TCCIR_OP_SHR: + case TCCIR_OP_PDIV: + case TCCIR_OP_UDIV: + case TCCIR_OP_CMP: + case TCCIR_OP_SETIF: + case TCCIR_OP_TEST_ZERO: + case TCCIR_OP_LOAD: + case TCCIR_OP_STORE: + case TCCIR_OP_ASSIGN: + case TCCIR_OP_LEA: + case TCCIR_OP_LOAD_INDEXED: + case TCCIR_OP_UBFX: + case TCCIR_OP_SBFX: + case TCCIR_OP_BFI: + case TCCIR_OP_FADD: + case TCCIR_OP_FSUB: + case TCCIR_OP_FMUL: + case TCCIR_OP_FDIV: + case TCCIR_OP_FNEG: + case TCCIR_OP_FCMP: + case TCCIR_OP_CVT_FTOF: + case TCCIR_OP_CVT_ITOF: + case TCCIR_OP_CVT_FTOI: + case TCCIR_OP_ZEXT: + case TCCIR_OP_PACK64: + case TCCIR_OP_BOOL_OR: + case TCCIR_OP_BOOL_AND: + case TCCIR_OP_SELECT: + case TCCIR_OP_ROR: + case TCCIR_OP_NOP: + return 1; + default: + return tcc_ir_op_is_mac(op); + } +} + +/* Can instruction q write any of the frame bytes [off, off + width), or + * redefine `vr`? */ +static int sc_clobbers(TCCIRState *ir, IRQuadCompact *q, int off, int width, int32_t vr) +{ + if (!sc_plain_op(q->op)) + return 1; + if (q->op == TCCIR_OP_NOP || !irop_config[q->op].has_dest) + return 0; + IROperand d = tcc_ir_op_get_dest(ir, q); + if (irop_is_none(d)) + return 0; + int32_t dv = irop_get_vreg(d); + if (dv == vr) + return 1; + int doff, dwidth; + if (sc_frame_access(ir, d, &doff, &dwidth)) + return doff < off + width && off < doff + dwidth; + if (dv < 0) + return d.is_lval; /* some other memory destination */ + if (q->op == TCCIR_OP_STORE) + return 1; /* through a pointer */ + int t = TCCIR_DECODE_VREG_TYPE(dv); + if (t == TCCIR_VREG_TYPE_TEMP) + return d.is_lval; /* a TEMP destination is a register; lval = through it */ + return 0; /* a VAR or PARAM lives in its own register or spill slot */ +} + +/* A side annotation that changes what the instruction reads or writes. */ +static int sc_annotated(TCCIRState *ir, IRQuadCompact *q) +{ + return tcc_ir_barrel_shift_at(ir, q) || tcc_ir_shift64_dead_half_at(ir, q) || tcc_ir_zero_half64_at(ir, q) || + tcc_ir_bfi_params_at(ir, q); +} + +int tcc_ir_self_store(TCCIRState *ir) +{ + const int n = ir->next_instruction_index; + if (n < 2 || ir->func_has_label_addr) + return 0; + const int nt = ir->next_temporary_variable; + if (nt <= 0) + return 0; + /* Operand occurrences per TEMP, to tell when a load loses its last use. */ + int *occ = tcc_mallocz(sizeof(int) * nt); + for (int i = 0; i < n; i++) + { + IRQuadCompact *q = &ir->compact_instructions[i]; + if (q->op == TCCIR_OP_NOP) + continue; + for (int s = 0; s < 4; s++) + { + int32_t vr = irop_get_vreg(sc_slot(ir, q, s)); + if (vr < 0 || TCCIR_DECODE_VREG_TYPE(vr) != TCCIR_VREG_TYPE_TEMP) + continue; + int pos = TCCIR_DECODE_VREG_POSITION(vr); + if (pos < nt) + occ[pos]++; + } + } + + int removed = 0; + for (int i = 0; i < n; i++) + { + IRQuadCompact *st = &ir->compact_instructions[i]; + if (st->op != TCCIR_OP_STORE) + continue; + int off, width; + if (!sc_frame_access(ir, tcc_ir_op_get_dest(ir, st), &off, &width)) + continue; + int32_t vr = sc_temp_value(tcc_ir_op_get_src1(ir, st)); + if (vr < 0) + continue; + int pos = TCCIR_DECODE_VREG_POSITION(vr); + if (pos >= nt) + continue; + if (sc_annotated(ir, st) || st->is_jump_target) + continue; + + /* Walk back to the value's definition through a straight run. */ + int ld = -1; + for (int j = i - 1, dist = 0; j >= 0 && dist < SC_MAX_DIST; j--) + { + IRQuadCompact *q = &ir->compact_instructions[j]; + if (q->op == TCCIR_OP_NOP) + { + if (q->is_jump_target) + break; + continue; + } + dist++; + if (q->op == TCCIR_OP_LOAD && irop_get_vreg(tcc_ir_op_get_dest(ir, q)) == vr) + { + IROperand d = tcc_ir_op_get_dest(ir, q); + int loff, lwidth; + if (!d.is_lval && sc_frame_access(ir, tcc_ir_op_get_src1(ir, q), &loff, &lwidth) && loff == off && + lwidth >= width && !sc_annotated(ir, q)) + ld = j; + break; + } + if (sc_clobbers(ir, q, off, width, vr)) + break; + if (q->is_jump_target) + break; + } + if (ld < 0) + continue; + + st->op = TCCIR_OP_NOP; + removed++; + occ[pos]--; + if (occ[pos] == 1) + { + ir->compact_instructions[ld].op = TCCIR_OP_NOP; + occ[pos] = 0; + } + } + tcc_free(occ); + return removed; +} diff --git a/source/ir/ssa.c b/source/ir/ssa.c index 683d71eb2..96219c53a 100644 --- a/source/ir/ssa.c +++ b/source/ir/ssa.c @@ -52,6 +52,16 @@ typedef struct { SSABitset *multi_block_def; SSABitset *global; /* upward-exposed somewhere: a read may observe a cross-block value */ SSABitset *nonstore_def; /* has at least one def that is not a slot STORE */ + SSABitset *value_def; /* a def names it as a VALUE (is_lval 0): SRA's field definitions, which have + * no memory form -- codegen resolves such a dest as the slot's address -- so + * the VAR must be renamed whatever else the rules say */ + SSABitset *off_mixed; /* some operand names it at an offset other than its own */ + SSABitset *def_outside_entry; /* a definition outside the entry block */ + SSABitset *dom_def; /* written exactly once, by a definition that dominates every read */ + int sra_lo, sra_hi; /* the field VARs SRA made for this function (ir->sra_var_lo/hi) */ + int prom_sra; /* promote SRA's field VARs past the STORE-only rule */ + int prom_single; /* promote entry-block-only single-def VARs */ + int prom_domdef; /* promote VARs whose one definition dominates every read (dom_def) */ int *var_btype; const uint8_t *store_def_ok; /* per VAR: an INT32 slot STORE covers every read */ int block_bitset_bytes; @@ -62,7 +72,17 @@ typedef struct { * in-place rename (multi-def temp). Read once — the callers sit in inner loops. */ TCC_DBG_ENV_FLAG(ssa_no_store2assign, "TCC_NO_STORE2ASSIGN") -/* A full-width var-SLOT STORE of a plain VALUE completely redefines the +/* The constant a narrow slot STORE of `c` leaves in a var of dest's type. */ +static int32_t ssa_narrow_store_imm(IROperand d, int32_t c) +{ + if (d.btype == IROP_BTYPE_INT8) + return d.is_unsigned ? (int32_t)(uint8_t)c : (int32_t)(int8_t)c; + if (d.btype == IROP_BTYPE_INT16) + return d.is_unsigned ? (int32_t)(uint16_t)c : (int32_t)(int16_t)c; + return c; +} + +/* A var-SLOT STORE of a plain VALUE covering every read completely redefines the * variable, so the renamer turns it into an ASSIGN with a FRESH SSA name. * Phi placement must agree, or a fresh name defined on one arm is lost at the * join (wrong code) — hence this single predicate shared by both. Returns the @@ -80,14 +100,30 @@ static int ssa_store_slot_def_pos(TCCIRState *ir, IRQuadCompact *q, int32_t dvr = irop_get_vreg(d); if (dvr < 0 || TCCIR_DECODE_VREG_TYPE(dvr) != TCCIR_VREG_TYPE_VAR) return -1; - if (!d.is_local || d.btype != IROP_BTYPE_INT32) + /* A word or a 64-bit integer write, covering every read of the var. An + * inlined helper's int64_t parameter is bound with a 64-bit STORE; updated + * in place it stayed one multi-def TEMP that no constant reaches. A narrow + * STORE truncates, so it defines the var only when it stores a constant, + * which the renamer writes as the var holds it (ssa_narrow_store_imm). */ + IROperand s = tcc_ir_op_get_src1(ir, q); + int bytes = d.btype == IROP_BTYPE_INT32 ? 4 : d.btype == IROP_BTYPE_INT64 ? 8 : 0; + if ((d.btype == IROP_BTYPE_INT8 || d.btype == IROP_BTYPE_INT16) && + irop_get_tag(s) == IROP_TAG_IMM32 && !s.is_lval) + bytes = d.btype == IROP_BTYPE_INT8 ? 1 : 2; + if (!d.is_local || !bytes) return -1; int pos = TCCIR_DECODE_VREG_POSITION(dvr); - if (pos >= num_vars || !store_def_ok[pos]) + if (pos >= num_vars || store_def_ok[pos] > bytes) return -1; - IROperand s = tcc_ir_op_get_src1(ir, q); + /* A symbol's ADDRESS (a SYMREF without is_lval) is a constant like an + * immediate; only an is_lval SYMREF reads the global. Left a memory write, + * the address store kept the SRA field VAR's previous def under the same + * TEMP name, and the zero a compound literal's fill stored first reached the + * code: `movs r0, #0; ldr r0, =msg` for every Zig panic message slice. */ int src_is_value = (irop_get_tag(s) == IROP_TAG_VREG && !s.is_lval) || - irop_get_tag(s) == IROP_TAG_IMM32; + irop_get_tag(s) == IROP_TAG_IMM32 || + (irop_get_tag(s) == IROP_TAG_I64 && !s.is_lval && bytes == 8) || + (irop_get_tag(s) == IROP_TAG_SYMREF && !s.is_lval && bytes == 4); return src_is_value ? pos : -1; } @@ -174,6 +210,8 @@ static int ssa_scan_var_defs(TCCIRState *ir, IRCFG *cfg, SSAVarInfo *info) continue; int blk = cfg->instr_to_block[i]; + if (blk != 0) + SSABitset_set(info->def_outside_entry, pos); uint8_t *def_bits = &info->def_blocks[pos * bitset_bytes]; if (SSABitset_test(&has_def, pos)) { if (!raw_bitset_test(def_bits, blk)) @@ -183,10 +221,36 @@ static int ssa_scan_var_defs(TCCIRState *ir, IRCFG *cfg, SSAVarInfo *info) raw_bitset_set(def_bits, blk); if (q->op != TCCIR_OP_STORE) SSABitset_set(info->nonstore_def, pos); + if (q->op != TCCIR_OP_STORE && !dest.is_lval) + SSABitset_set(info->value_def, pos); if (dest.btype != IROP_BTYPE_INT32) info->var_btype[pos] = dest.btype; } + /* Renaming clears an operand's offset (ssa_rename_use), so a VAR named + * anywhere at an offset other than its own home (a _Complex double read at + * +0 and +8 -- ieee/cdivchk) must keep its memory under any widened rule. */ + for (int i = 0; i < n; i++) { + IRQuadCompact *q = &ir->compact_instructions[i]; + if (q->op == TCCIR_OP_NOP) + continue; + int nops = irop_config[q->op].has_dest + irop_config[q->op].has_src1 + irop_config[q->op].has_src2; + for (int k = 0; k < nops; k++) { + uint32_t pi = q->operand_base + k; + if (pi >= (uint32_t)ir->iroperand_pool_count) + continue; + IROperand o = ir->iroperand_pool[pi]; + int32_t vr = irop_get_vreg(o); + if (vr < 0 || TCCIR_DECODE_VREG_TYPE(vr) != TCCIR_VREG_TYPE_VAR || irop_get_tag(o) != IROP_TAG_STACKOFF) + continue; + int pos = TCCIR_DECODE_VREG_POSITION(vr); + if (pos >= num_vars || pos >= ir->variables_live_intervals_size) + continue; + if (o.u.imm32 != ir->variables_live_intervals[pos].original_offset) + SSABitset_set(info->off_mixed, pos); + } + } + for (int v = 0; v < num_vars; v++) { if (SSABitset_test(info->addrtaken, v)) continue; @@ -203,6 +267,88 @@ static int ssa_scan_var_defs(TCCIRState *ir, IRCFG *cfg, SSAVarInfo *info) return 0; } +/* dom_def: VARs written by exactly one instruction whose value reaches every + * read, because the write dominates it (an earlier instruction of the same + * block, or a block dominating the reader's). Renaming such a VAR places no + * phi -- nothing joins -- so it costs no copy, which is what the STORE-only + * rule of ssa_var_promotable guards against. It is the shape of a called-once + * function's parameter after inlining (`p = arg;` once, read throughout the + * body), of which zig.c kept thousands in memory, reloading each at every use. + * Any other write -- a narrow store that only updates the name in place, a + * post-increment's write-back -- and any read the write does not dominate (a + * loop-carried use, a path that reads it uninitialized, a read by the defining + * instruction itself) leaves the VAR to the other rules. */ +static void ssa_scan_dominating_defs(TCCIRState *ir, IRCFG *cfg, SSAVarInfo *info) +{ + const int n = ir->next_instruction_index, num_vars = info->num_vars; + int32_t *def_at = tcc_malloc(sizeof(int32_t) * (num_vars > 0 ? num_vars : 1)); + for (int v = 0; v < num_vars; v++) + def_at[v] = -1; /* -1 none, -2 more than one or not a clean definition */ +#define DD_VAR(o) (irop_get_vreg(o) >= 0 && TCCIR_DECODE_VREG_TYPE(irop_get_vreg(o)) == TCCIR_VREG_TYPE_VAR && \ + TCCIR_DECODE_VREG_POSITION(irop_get_vreg(o)) < num_vars ? TCCIR_DECODE_VREG_POSITION(irop_get_vreg(o)) : -1) + for (int i = 0; i < n; i++) + { + IRQuadCompact *q = &ir->compact_instructions[i]; + if (q->op == TCCIR_OP_NOP) + continue; + int v; + if (q->op == TCCIR_OP_LOAD_POSTINC && (v = DD_VAR(tcc_ir_op_get_src1(ir, q))) >= 0) + def_at[v] = -2; /* its pointer is written back */ + if (!irop_config[q->op].has_dest || q->op == TCCIR_OP_FUNCPARAMVAL || q->op == TCCIR_OP_FUNCPARAMVOID) + continue; + IROperand d = tcc_ir_op_get_dest(ir, q); + if ((v = DD_VAR(d)) < 0) + continue; + if (q->op == TCCIR_OP_STORE_INDEXED || ((q->op == TCCIR_OP_STORE) && !d.is_local)) + continue; /* a store THROUGH the VAR's value reads it */ + int clean = q->op != TCCIR_OP_STORE_POSTINC && + (q->op != TCCIR_OP_STORE || ssa_store_slot_def_pos(ir, q, info->store_def_ok, num_vars) >= 0); + def_at[v] = def_at[v] == -1 && clean ? i : -2; + } + for (int i = 0; i < n; i++) + { + IRQuadCompact *q = &ir->compact_instructions[i]; + if (q->op == TCCIR_OP_NOP) + continue; + const int ub = cfg->instr_to_block[i]; + IROperand ops[4]; + int nops = 0; + if (irop_config[q->op].has_dest) + { + IROperand d = tcc_ir_op_get_dest(ir, q); + /* A store through the VAR, a post-increment base, or any dest that is + * not the VAR's own definition counts as a read here. */ + if (DD_VAR(d) >= 0 && def_at[DD_VAR(d)] != i) + ops[nops++] = d; + } + if (irop_config[q->op].has_src1) + ops[nops++] = tcc_ir_op_get_src1(ir, q); + if (irop_config[q->op].has_src2) + ops[nops++] = tcc_ir_op_get_src2(ir, q); + if (tcc_ir_op_is_mac(q->op) || q->op == TCCIR_OP_LOAD_INDEXED || q->op == TCCIR_OP_STORE_INDEXED || + q->op == TCCIR_OP_SELECT) + { + if (q->operand_base + 3 < (uint32_t)ir->iroperand_pool_count) + ops[nops++] = ir->iroperand_pool[q->operand_base + 3]; + } + for (int k = 0; k < nops; k++) + { + int v = DD_VAR(ops[k]); + if (v < 0 || def_at[v] < 0) + continue; + const int d = def_at[v], db = cfg->instr_to_block[d]; + int ok = ub >= 0 && db >= 0 && (db == ub ? d < i : tcc_ir_cfg_dominates(cfg, db, ub)); + if (!ok) + def_at[v] = -2; + } + } + for (int v = 0; v < num_vars; v++) + if (def_at[v] >= 0) + SSABitset_set(info->dom_def, v); +#undef DD_VAR + tcc_free(def_at); +} + /* Briggs semi-pruned SSA: classify each VAR as "global" (upward-exposed — * some read may observe a value defined in another block) or block-local * (every read is preceded, in its own block, by a definition that fully @@ -521,11 +667,25 @@ static uint8_t *ssa_compute_live_in(TCCIRState *ir, IRCFG *cfg, * defined on one arm of a branch and read after the merge is the same shape. * Promoting it is always safe: the phi resolver drops undef (vreg<0) operands, * so a path that leaves the var genuinely uninitialized is unchanged. */ +/* SRA's field VARs are promoted past the STORE-only rule only in functions + * where SRA made at most this many: promoting all 2-4 thousand of Zig's + * main_buildOutputType turned a 141 KB function into 206 KB of spills. + * Measured on zig.c: 1000 -> -30 KB, 2000 -> -33 KB, 4000 -> +32 KB. */ +TCC_DBG_ENV_INT(ssa_prom_sra_max, "TCC_SSA_PROM_SRA_MAX", 2000) + static int ssa_var_promotable(const SSAVarInfo *info, IRCFG *cfg, int nb, int v, int single_block) { if (SSABitset_test(info->addrtaken, v)) return 0; + /* One definition that dominates every read: no phi, no copy (dom_def). */ + if (info->prom_domdef && SSABitset_test(info->dom_def, v) && !SSABitset_test(info->off_mixed, v)) + return 1; + /* SRA's field VARs are defined as values (`&V <- x`); left in memory such a + * definition writes nothing (zig.c: an error code built in one block of a + * 3000-object function, past the SRA exemption's cap, read back as 0). */ + if (SSABitset_test(info->value_def, v)) + return 1; /* Block-local (non-global) var: no value ever crosses a CFG edge, so it is * renameable with no phis at all — including the single-block-def shape * rejected below, which otherwise stays a memory-resident VAR. */ @@ -541,10 +701,24 @@ static int ssa_var_promotable(const SSAVarInfo *info, IRCFG *cfg, int nb, int v, * loop-15 +12 insns / +74 cycles, bug_switch_in_loop +122 cycles). Vars that * mix ASSIGN and STORE defs — the shape this whole path exists for — keep * being promoted. */ - if (!SSABitset_test(info->nonstore_def, v)) + /* SRA's field VARs are exempt: SRA saw every access to the object they + * replace, so they are plain scalars whose slot STOREs are all their defs -- + * not the loop-carried source variables the rule's cycles cost is about. + * Promoted, the value passes see through them (zig.c -33 KB). */ + if (!SSABitset_test(info->nonstore_def, v) && + !(info->prom_sra && v >= info->sra_lo && v < info->sra_hi && + info->sra_hi - info->sra_lo <= ssa_prom_sra_max() && !SSABitset_test(info->off_mixed, v))) return 0; if (SSABitset_test(info->multi_block_def, v)) return 1; + /* A VAR defined only in the entry block, which nothing jumps back to, + * needs no phi: the entry runs once and dominates every block. Renamed, the + * value passes see through it (zig.c -61 KB). NOT "its def block has an + * empty dominance frontier": the frontier under-reports on the un-rotated + * loop shape, and promoting on it dropped a value (gcc-torture pr78675). */ + if (info->prom_single && !SSABitset_test(info->def_outside_entry, v) && cfg->blocks[0].num_preds == 0 && + !SSABitset_test(info->off_mixed, v)) + return 1; /* Single-block-def: promote iff a phi would actually be placed, i.e. some * def-block has a non-empty dominance frontier. */ const uint8_t *def_bits = &info->def_blocks[v * info->block_bitset_bytes]; @@ -555,6 +729,19 @@ static int ssa_var_promotable(const SSAVarInfo *info, IRCFG *cfg, int nb, int v, return 0; } +/* TCC_SSA_PROM_DBG: one line per VAR the promoter refuses, with the reason and + * whether SRA made it. */ +TCC_DBG_ENV_FLAG(ssa_prom_dbg, "TCC_SSA_PROM_DBG") + +static const char *ssa_refuse_reason(const SSAVarInfo *info, IRCFG *cfg, int nb, int v) +{ + if (SSABitset_test(info->addrtaken, v)) + return "addrtaken"; + if (!SSABitset_test(info->nonstore_def, v)) + return "store-only-global"; + return "single-def-no-df"; +} + static uint8_t *ssa_build_promotable(const SSAVarInfo *info, IRCFG *cfg, int nb, int *out_count) { @@ -564,6 +751,16 @@ static uint8_t *ssa_build_promotable(const SSAVarInfo *info, IRCFG *cfg, int nb, for (int v = 0; v < num_vars; v++) { if (ssa_var_promotable(info, cfg, nb, v, single_block)) count++; + else + TCC_DBG_BLOCK(ssa_prom_dbg) + { + extern const char *funcname; + int has_def = 0; + for (int b = 0; b < info->block_bitset_bytes && !has_def; b++) + has_def = info->def_blocks[v * info->block_bitset_bytes + b] != 0; + fprintf(stderr, "[PROM] %s v=%d sra=%d defs=%d %s\n", funcname, v, + v >= info->sra_lo && v < info->sra_hi, has_def, ssa_refuse_reason(info, cfg, nb, v)); + } } *out_count = count; if (count == 0) @@ -654,13 +851,21 @@ IRSSAState *tcc_ir_ssa_construct(TCCIRState *ir, IRCFG *cfg) dynamic_bitset(SSABitset) multi_block_def = {0}; dynamic_bitset(SSABitset) global = {0}; dynamic_bitset(SSABitset) nonstore_def = {0}; + dynamic_bitset(SSABitset) value_def = {0}; + dynamic_bitset(SSABitset) off_mixed = {0}; + dynamic_bitset(SSABitset) def_outside_entry = {0}; + dynamic_bitset(SSABitset) dom_def = {0}; if (vector_resize(&def_blocks_owner, (size_t)num_vars * bitset_bytes) != 0 || vector_resize(&var_btype_owner, num_vars) != 0 || SSABitset_init(&addrtaken, num_vars) != 0 || SSABitset_init(&multi_block_def, num_vars) != 0 || SSABitset_init(&global, num_vars) != 0 || - SSABitset_init(&nonstore_def, num_vars) != 0) + SSABitset_init(&nonstore_def, num_vars) != 0 || + SSABitset_init(&value_def, num_vars) != 0 || + SSABitset_init(&off_mixed, num_vars) != 0 || + SSABitset_init(&def_outside_entry, num_vars) != 0 || + SSABitset_init(&dom_def, num_vars) != 0) return NULL; SSAVarInfo info = { @@ -669,25 +874,34 @@ IRSSAState *tcc_ir_ssa_construct(TCCIRState *ir, IRCFG *cfg) .multi_block_def = &multi_block_def, .global = &global, .nonstore_def = &nonstore_def, + .value_def = &value_def, + .off_mixed = &off_mixed, + .def_outside_entry = &def_outside_entry, + .dom_def = &dom_def, + .sra_lo = ir->sra_var_lo, + .sra_hi = ir->sra_var_hi, + .prom_sra = !tcc_ir_opt_pass_disabled("ssa_prom_sra"), + .prom_single = !tcc_ir_opt_pass_disabled("ssa_prom_single"), + .prom_domdef = !tcc_ir_opt_pass_disabled("ssa_prom_domdef"), .var_btype = vector_data(&var_btype_owner), .block_bitset_bytes = bitset_bytes, .num_vars = num_vars, }; /* Read widths first: whether a slot STORE counts as a fresh definition is a * per-var property the def scan below needs, and the upward-exposure scan - * does not depend on def_blocks. Fold the decision into one byte per var — - * a 4-byte write must cover every read of the var (the 0xFF non-scalar - * sentinel and 64-bit reads fail this). */ + * does not depend on def_blocks. One byte per var, its widest read: a + * STORE at least that wide covers every read (the 0xFF non-scalar sentinel + * fails every width). */ uint8_t *var_max_read = tcc_mallocz(num_vars); ssa_scan_var_global(ir, cfg, &info, var_max_read); - for (int v = 0; v < num_vars; v++) - var_max_read[v] = (var_max_read[v] <= 4) ? 1 : 0; info.store_def_ok = var_max_read; if (ssa_scan_var_defs(ir, cfg, &info) != 0) { tcc_free(var_max_read); return NULL; } + if (info.prom_domdef) + ssa_scan_dominating_defs(ir, cfg, &info); int promotable_count; uint8_t *is_promotable = ssa_build_promotable(&info, cfg, nb, &promotable_count); @@ -748,10 +962,17 @@ IRSSAState *tcc_ir_ssa_construct(TCCIRState *ir, IRCFG *cfg) * ============================================================================ */ typedef vector(int32_t) VRegStack; - -static void vstack_push(VRegStack *s, int32_t v) +/* Undo log for the dominator-tree walk: the variable index of every push, in + * order. A frame remembers only the log length on entry and pops exactly the + * pushes made since then on exit -- the same stack contents as snapshotting + * every variable's depth per frame, without the nb x num_vars depth table + * (8 MB for one function of tinycc's own ir/codegen.c). */ +typedef vector(int) VRegPushLog; + +static void vstack_push(VRegStack *stacks, VRegPushLog *log, int pos, int32_t v) { - vector_push_back(s, v); + vector_push_back(&stacks[pos], v); + vector_push_back(log, pos); } static int32_t vstack_top(VRegStack *s) @@ -790,7 +1011,8 @@ static int ssa_rename_use(IROperand *op, int num_vars, const uint8_t *is_promota return deref_through_value ? 2 : 1; } -static void ssa_rename_phi_defs(IRSSAState *ssa, int b, VRegStack *stacks, int num_vars) +static void ssa_rename_phi_defs(IRSSAState *ssa, int b, VRegStack *stacks, VRegPushLog *log, + int num_vars) { for (IRPhiNode *phi = ssa->block_phis[b]; phi; phi = phi->next) { int32_t orig = phi->orig_vreg; @@ -798,12 +1020,13 @@ static void ssa_rename_phi_defs(IRSSAState *ssa, int b, VRegStack *stacks, int n continue; int pos = TCCIR_DECODE_VREG_POSITION(orig); if (pos < num_vars && raw_bitset_test(ssa->is_promotable, pos)) - vstack_push(&stacks[pos], phi->dest_vreg); + vstack_push(stacks, log, pos, phi->dest_vreg); } } static void ssa_rename_block_instrs(TCCIRState *ir, IRSSAState *ssa, IRBasicBlock *bb, - VRegStack *stacks, int num_vars, int *next_temp_pos) + VRegStack *stacks, VRegPushLog *log, int num_vars, + int *next_temp_pos) { for (int i = bb->start_idx; i < bb->end_idx; i++) { IRQuadCompact *q = &ir->compact_instructions[i]; @@ -844,8 +1067,14 @@ static void ssa_rename_block_instrs(TCCIRState *ir, IRSSAState *ssa, IRBasicBloc * allocator's def scan. */ if (q->op == TCCIR_OP_STORE) { int pos = ssa_store_slot_def_pos(ir, q, ssa->var_store_def_ok, num_vars); - if (pos >= 0 && raw_bitset_test(ssa->is_promotable, pos)) + if (pos >= 0 && raw_bitset_test(ssa->is_promotable, pos)) { q->op = TCCIR_OP_ASSIGN; /* falls through to fresh-def dest handling */ + IROperand d = tcc_ir_op_get_dest(ir, q), s = tcc_ir_op_get_src1(ir, q); + if (d.btype == IROP_BTYPE_INT8 || d.btype == IROP_BTYPE_INT16) { + s.u.imm32 = ssa_narrow_store_imm(d, s.u.imm32); + tcc_ir_op_set_src1(ir, q, s); + } + } } if (q->op == TCCIR_OP_STORE || q->op == TCCIR_OP_STORE_INDEXED || @@ -864,7 +1093,7 @@ static void ssa_rename_block_instrs(TCCIRState *ir, IRSSAState *ssa, IRBasicBloc int pos = TCCIR_DECODE_VREG_POSITION(vr); if (pos < num_vars && raw_bitset_test(ssa->is_promotable, pos)) { int32_t new_name = TCCIR_ENCODE_VREG(TCCIR_VREG_TYPE_TEMP, (*next_temp_pos)++); - vstack_push(&stacks[pos], new_name); + vstack_push(stacks, log, pos, new_name); irop_set_vreg(&d, new_name); d.tag = IROP_TAG_VREG; d.is_lval = 0; @@ -923,36 +1152,32 @@ void tcc_ir_ssa_rename(TCCIRState *ir, IRSSAState *ssa) for (int v = 0; v < num_vars; v++) { if (raw_bitset_test(ssa->is_promotable, v)) { int32_t init_name = TCCIR_ENCODE_VREG(TCCIR_VREG_TYPE_TEMP, next_temp_pos++); - vstack_push(&stacks[v], init_name); + vector_push_back(&stacks[v], init_name); /* below every frame: never popped */ } } - typedef struct { int block; int child_idx; } DomFrame; + typedef struct { int block; int child_idx; int log_mark; } DomFrame; scoped_vector(DomFrame) dom_stack_owner = {0}; - scoped_vector(int) saved_depths_owner = {0}; - if (vector_resize(&dom_stack_owner, nb) != 0 || - vector_resize(&saved_depths_owner, (size_t)nb * num_vars) != 0) { + __attribute__((cleanup(tcc_vector_cleanup_storage))) VRegPushLog push_log_owner = {0}; + if (vector_resize(&dom_stack_owner, nb) != 0) { for (int v = 0; v < num_vars; v++) vector_cleanup(&stacks[v]); return; } DomFrame *dom_stack = vector_data(&dom_stack_owner); - int *saved_depths = vector_data(&saved_depths_owner); + VRegPushLog *push_log = &push_log_owner; int dsp = 0; - dom_stack[dsp++] = (DomFrame){0, 0}; + dom_stack[dsp++] = (DomFrame){0, 0, 0}; while (dsp > 0) { DomFrame *top = &dom_stack[dsp - 1]; int b = top->block; if (top->child_idx == 0) { - int *frame_depths = &saved_depths[(dsp - 1) * num_vars]; - for (int v = 0; v < num_vars; v++) - frame_depths[v] = stacks[v].size; - - ssa_rename_phi_defs(ssa, b, stacks, num_vars); - ssa_rename_block_instrs(ir, ssa, &cfg->blocks[b], stacks, num_vars, &next_temp_pos); + top->log_mark = (int)push_log->size; + ssa_rename_phi_defs(ssa, b, stacks, push_log, num_vars); + ssa_rename_block_instrs(ir, ssa, &cfg->blocks[b], stacks, push_log, num_vars, &next_temp_pos); ssa_fill_successor_phis(ssa, cfg, b, stacks, num_vars); } @@ -960,12 +1185,11 @@ void tcc_ir_ssa_rename(TCCIRState *ir, IRSSAState *ssa) if (top->child_idx < bb->num_dom_children) { int child = bb->dom_children[top->child_idx]; top->child_idx++; - dom_stack[dsp++] = (DomFrame){child, 0}; + dom_stack[dsp++] = (DomFrame){child, 0, 0}; } else { - int *frame_depths = &saved_depths[(dsp - 1) * num_vars]; - for (int v = 0; v < num_vars; v++) - stacks[v].size = frame_depths[v]; + while ((int)push_log->size > top->log_mark) + stacks[push_log->data[--push_log->size]].size--; dsp--; } } diff --git a/source/ir/ssa.h b/source/ir/ssa.h index a3e98f7d4..eda3e4586 100644 --- a/source/ir/ssa.h +++ b/source/ir/ssa.h @@ -28,13 +28,13 @@ typedef struct IRSSAState IRPhiNode **block_phis; /* array[num_blocks]: linked list of phis per block */ int32_t next_ssa_vreg; /* next available SSA vreg position (TEMP type) */ uint8_t *is_promotable; /* bitset indexed by VAR position */ - uint8_t *var_store_def_ok; /* per VAR: 1 = a full-width INT32 slot STORE of - * this var may be rewritten to a fresh-name - * ASSIGN def during rename. Requires every read - * to be ≤ 4 bytes so the write covers it; phi - * placement consults the same flag, so an + uint8_t *var_store_def_ok; /* per VAR: its widest read in bytes (0xFF for a + * non-scalar access). An INT32 or INT64 slot + * STORE at least that wide covers every read, so + * rename rewrites it to a fresh-name ASSIGN def; + * phi placement consults the same test, so an * upward-exposed (global) var gets phis for its - * STORE defs too. */ + * STORE defs too (ssa_store_slot_def_pos). */ int num_vars; /* size of VAR namespace at construct time */ } IRSSAState; diff --git a/source/ir/tccir.h b/source/ir/tccir.h index 2a601dd85..44e404f5e 100644 --- a/source/ir/tccir.h +++ b/source/ir/tccir.h @@ -242,8 +242,28 @@ typedef enum TccIrOp TCCIR_OP_RBIT, /* reverse bit order (rbit) */ TCCIR_OP_REV, /* reverse byte order in a word (rev) == bswap32 */ TCCIR_OP_REV16, /* reverse byte order in each halfword (rev16) */ + /* dest = this function's return address: __builtin_return_address(0). Read + * from the slot the prologue saved LR to, SP-relative, so it needs no frame + * pointer. No sources; its value depends on the call site, so a function + * using it is never pure. */ + TCCIR_OP_RETURN_ADDRESS, + /* Unsigned multiply accumulate-accumulate (ARM UMAAL), 64-bit dest: + * dest = (uint32)src1 * (uint32)src2 + lo32(accum) + hi32(accum) + * accum is pool[operand_base+3] like MLA's: a 64-bit value whose two words + * are the two 32-bit addends (the sum cannot overflow 64 bits). Created + * only by the late umaal fusion right before register allocation, so only + * the allocator, the post-RA passes and codegen see it; everything that + * reads MLA's fourth operand reads this one via tcc_ir_op_is_mac(). */ + TCCIR_OP_UMAAL, } TccIrOp; +/* Ops with a fourth (accumulator) operand at pool[operand_base+3] that is a + * value read by the instruction. */ +static inline int tcc_ir_op_is_mac(int op) +{ + return op == TCCIR_OP_MLA || op == TCCIR_OP_UMAAL; +} + /* Size (in bytes) at or above which the backend lowers a TCCIR_OP_BLOCK_COPY to * a real memcpy() call instead of an inline LDM/STM sequence (see * tcc_gen_machine_block_copy_mop in arm-thumb-gen.c). A memcpy call clobbers @@ -332,6 +352,9 @@ typedef struct IRLiveInterval uint8_t is_volatile : 1; // whether the source object has volatile-qualified type uint8_t crosses_call : 1; // whether interval spans a function call uint8_t phi_pinned : 1; // register relied upon by identity phi — do not reassign + uint8_t incoming_stack : 1; // param the ABI placed wholly on the caller's stack (incoming_reg0 < 0 is also "unset") + uint8_t is_struct : 1; // param of struct type: its vreg names memory, never a value + uint8_t nested_home : 1; // param captured by a nested function: it keeps a parent-frame home uint32_t start; // start instruction index uint32_t end; // end instruction index IRVregReplacement allocation; @@ -454,9 +477,30 @@ typedef struct IRQuadCompact int orig_index; /* Original IR index (stable across DCE) */ TccIrOp op; /* Operation code */ uint32_t operand_base; /* Index into svalue_pool */ - uint32_t line_num : 30; /* Source line for debug info (non-negative, 30 bits = up to 1B lines) */ + uint32_t line_num : 28; /* Source line for debug info (non-negative, 28 bits = up to 268M lines) */ + /* TRAP only: __builtin_unreachable(). Emits no code, and a path that can + * only end here may be assumed never taken (tcc_ir_opt_unreachable_fold). + * Every other pass sees an ordinary TRAP -- a terminator with no successor + * and a side effect -- which is the conservative reading. */ + uint32_t unreachable : 1; uint32_t is_jump_target : 1; /* Set when at least one JUMP/JUMPIF targets this instruction */ uint32_t no_unroll : 1; /* Set on rerolled back-edges to prevent re-unrolling */ + /* Emit a 2-byte NOP ahead of this instruction when the code offset is not + * 4-aligned. Measured on RP2350 silicon: a BACKWARD-branch target at + * `addr % 4 == 2` costs about a cycle per iteration, because the M33 fetches + * a word at a time and the first fetch after the refill then delivers half as + * much. It turned bench_memcpy's one-instruction-shorter checksum loop into + * a 13.8% LOSS and handed it back in full (2,066,226 -> 1,812,226) once the + * head was word-aligned. + * + * The head's WIDTH is irrelevant and must not be filtered on: 16-bit heads + * benefit just as much (all six of mibench_sha's are `cmp`, worth -4.98% on + * their own), and filtering to 32-bit heads took the rig set win from + * -0.944% to -0.059%. + * + * Set by ssa:loop_bottom_test on the heads it makes bottom-tested, and by + * tcc_ir_codegen_generate on every backward-branch target (TCC_ALIGN_LOOPS). */ + uint32_t align_target : 1; } IRQuadCompact; /* Per-operation operand configuration (defined in tccir.c) */ @@ -472,15 +516,37 @@ extern const IRRegistersConfig irop_config[]; /* Forward declaration for FP materialization cache */ typedef struct TCCFPMatCache TCCFPMatCache; +/* How a parameter reaches the function body (pure-forwarder detection). */ +typedef struct IRParamForm +{ + uint8_t kind; /* IR_PF_* */ + int8_t reg_base; /* IR_PF_HOME: first argument register */ + int32_t vreg; /* IR_PF_REG / IR_PF_MEM: the PARAM vreg */ + int32_t off; /* IR_PF_MEM: parameter offset; IR_PF_HOME: home slot */ + int32_t words; /* IR_PF_HOME: words stored into the home slot */ +} IRParamForm; +#define IR_PF_NONE 0 /* unknown: never forwarded untouched */ +#define IR_PF_REG 1 /* a value in the PARAM vreg */ +#define IR_PF_MEM 2 /* memory at a parameter offset (stack scalar, in-place struct) */ +#define IR_PF_HOME 3 /* a register-passed struct the prologue stores into a home slot */ + typedef struct TCCIRState { // number of function parameters int8_t parameters_count; + IRParamForm *param_forms; /* [parameters_count] */ /* Named-argument usage for variadic prolog (AAPCS). */ int named_arg_reg_bytes; int named_arg_stack_bytes; uint8_t is_variadic : 1; + /* A parameter straddles r3 and the stack: the prologue pushes r0-r3 below + * the stack arguments, as for a variadic function, so it lies contiguous in + * memory and is used in place. */ + uint8_t push_arg_regs : 1; + /* The body only forwards its parameters, untouched, to one call of the same + * signature: emitted as a branch to it (tcc_ir_backend_fold_pure_forward). */ + uint8_t pure_forward : 1; uint8_t leaffunc : 1; uint8_t tail_call_only : 1; uint8_t naked : 1; @@ -505,6 +571,28 @@ typedef struct TCCIRState int32_t captured_count; /* number of captured variables */ int32_t loc; int32_t parent_loc; /* parent's loc value (for nested function offset validation) */ + /* Frontend stack objects as (frame offset, size incl. alignment padding, + * FRAME_OBJ_* flags) triples, for frame relayout (frame.c). Filled while + * the body is parsed. */ + int32_t *frame_objs; + int frame_obj_count, frame_obj_cap; + /* Padding of the frame objects whose type the frontend knew, as (start, + * low word, high word) triples of a byte mask: bit b = byte b of the object + * is no member's (tcc_ir_frame_note_type). */ + int32_t *frame_pads; + int frame_pad_count, frame_pad_cap; + /* By call id: the size of the struct written through parameter 0, the + * struct-return buffer, or 0 (frame.c). */ + int32_t *sret_calls; + int sret_calls_size; + /* Sorted, merged [start, end) extents of frame_objs for lookups, built for + * frame_obj_count objects; frame_relaid once relayout moved them. */ + int32_t *frame_index; + int frame_index_n, frame_index_objs; + uint8_t frame_relaid; + /* ssa:loop_unroll copied loads from a const table with the IV as index: + * the constant-index loads it left fold (ssa:unroll_cascade). */ + uint8_t unrolled_table_loads; /* Nested function tracking (for parent functions that contain nested functions) */ NestedFunc **nested_funcs; /* array of pointers to nested function descriptors */ @@ -567,6 +655,9 @@ typedef struct TCCIRState * 0 means "legacy/unknown" and falls back to nested-scan binding. */ int next_call_id; + /* The VARs the last SRA run on this function created, [lo, hi); -1 when it + * made none or did not run. ssa_var_promotable relaxes its rules for them. */ + int sra_var_lo, sra_var_hi; /* Current instruction index during code generation - used for scratch register allocation */ int codegen_instruction_idx; @@ -576,6 +667,11 @@ typedef struct TCCIRState */ int call_outgoing_base; /* frame offset (typically negative) */ int call_outgoing_size; /* bytes reserved (may include alignment padding) */ + /* Calls whose stack arguments do not fit the reserved area: per call id, the + * bytes SP drops below it around that call alone (a `sub sp` ... `add sp` + * window, see ir/codegen.c). NULL when no call needs one. */ + int32_t *call_dyn_extra; + int call_dyn_extra_size; /* Nested-call register save area: reserved in the frame for saving R0-R3 * (and R9/R12 for alignment) across nested function calls without PUSH/POP. @@ -764,6 +860,41 @@ int tcc_ir_get_vreg_static_chain(TCCIRState *ir); void tcc_ir_set_float_type(TCCIRState *ir, int vreg, int is_float, int is_double); void tcc_ir_set_llong_type(TCCIRState *ir, int vreg); void tcc_ir_set_original_offset(TCCIRState *ir, int vreg, int offset); +/* frame.c */ +/* `(loc - size) & mask`, recording the object in the current function's IR + * for tcc_ir_frame_relayout. */ +int tcc_ir_frame_alloc(int loc, int size, int mask); +void tcc_ir_frame_scope_end(struct TCCIRState *ir, int first_obj, int insn, int keep); +/* The same for the temporaries that carry a struct into or out of a call: its + * address reaching the call does not let the callee keep it. An argument's + * copy is never visible to the program; a call's struct result is, through + * an array member (`f().arr`), until the end of the full expression. */ +int tcc_ir_frame_alloc_arg_copy(int loc, int size, int mask); +int tcc_ir_frame_alloc_ret_temp(int loc, int size, int mask); +/* Parameter 0 of call `call_id` is the struct-return buffer. */ +void tcc_ir_frame_note_sret_call(int call_id, int size); +/* The frontend object containing frame offset `off`, as [*lo, *hi), while the + * frame still has the frontend's layout. 0 when unknown. */ +int tcc_ir_frame_object_at(TCCIRState *ir, int off, int *lo, int *hi); +/* Recorded size of the frame object starting exactly at `start`, or -1 + * (before frame relayout only). */ +int tcc_ir_frame_object_size_at(TCCIRState *ir, int start); +/* The object just allocated at `off` has C type `type`: note the bytes of it + * no member covers. */ +void tcc_ir_frame_note_type(int off, CType *type); +/* Bytes of [lo, hi) that are padding in every object recorded there, bit b = + * byte lo + b; 0 when any object overlapping it has no noted type or starts + * elsewhere (before frame relayout only). */ +uint64_t tcc_ir_frame_padding(TCCIRState *ir, int lo, int hi); +int tcc_ir_frame_relayout(TCCIRState *ir, int *ploc); +/* The function calls setjmp, vfork or another function that returns twice, so + * a frame slot live only on one return may still be read on the other. */ +int tcc_ir_calls_returns_twice(TCCIRState *ir); +int tcc_ir_cross_jump(TCCIRState *ir); +int tcc_ir_cross_jump_alloc(TCCIRState *ir); +int tcc_ir_region_merge(TCCIRState *ir); +int tcc_ir_self_store(TCCIRState *ir); +int tcc_ir_dead_def(TCCIRState *ir); int tcc_ir_get_reg_type(TCCIRState *ir, int vreg); void tcc_ir_register_allocation_params(TCCIRState *ir); @@ -772,6 +903,8 @@ void tcc_ir_register_allocation_params(TCCIRState *ir); * incoming stack home for the duration of the call. */ void tcc_ir_mark_return_value_incoming_regs(TCCIRState *ir); void tcc_ir_avoid_spilling_stack_passed_params(TCCIRState *ir); +int tcc_ir_param_is_captured_by_nested(int vreg); +void tcc_ir_mark_nested_captured_params(TCCIRState *ir); void tcc_ir_build_stack_layout(TCCIRState *ir); const TCCStackSlot *tcc_ir_stack_slot_by_vreg(const TCCIRState *ir, int vreg); const TCCStackSlot *tcc_ir_stack_slot_by_offset(const TCCIRState *ir, int frame_offset); diff --git a/source/ir/tccir_operand.c b/source/ir/tccir_operand.c index 88930efcf..ecf9a2e93 100644 --- a/source/ir/tccir_operand.c +++ b/source/ir/tccir_operand.c @@ -579,6 +579,10 @@ IROperand svalue_to_iroperand(TCCIRState *ir, const SValue *sv) { /* Stack offset: store directly in aux_data (±32KB range) */ int32_t offset = result.u.imm32; + /* tcc_ir_put names far struct sources through a pointer; anything + * still out of range here would be silently truncated. */ + if (offset < -32768 || offset > 32767) + tcc_error("internal: struct stack operand at offset %d is out of the 16-bit encoding range", offset); result.u.s.ctype_idx = (uint16_t)ctype_idx; result.u.s.aux_data = (int16_t)offset; /* store offset directly, no alignment assumption */ } @@ -1073,6 +1077,10 @@ static int compute_aapcs_member_alignment(CType *ct) * alignment is 1 (packed overrides natural alignment). */ if (f->a.packed || s->a.packed) member_align = 1; + /* An alignment given on the member itself counts, packed or not -- see + * compute_aapcs_natural_alignment, which this mirrors. */ + if (f->a.aligned && (1 << (f->a.aligned - 1)) > member_align) + member_align = 1 << (f->a.aligned - 1); if (member_align > max_align) max_align = member_align; f = next; diff --git a/source/ir/tccir_operand.h b/source/ir/tccir_operand.h index 9ceb429f4..530866849 100644 --- a/source/ir/tccir_operand.h +++ b/source/ir/tccir_operand.h @@ -420,6 +420,24 @@ uint32_t irop_get_pool_idx(const IROperand op); /* Check if operand is an lvalue (needs dereference) - uses bitfield */ int irop_op_is_lval(const IROperand op); +/* Whether an instruction's destination operand (re)defines its vreg. A VREG + * defines it as a plain value; as an lvalue (`T***DEREF*** <-- v`) it is a + * write through the pointer it holds. A VAR or PARAM lives in a stack slot + * and appears as a STACKOFF lvalue naming the variable itself -- a definition, + * whatever the op (an inlined pointer parameter lands as `V <-- x [STORE]`) -- + * or, is_llocal, as a write through the pointer it holds. */ +static inline int irop_dest_defines_vreg(IROperand d) +{ + if (irop_get_vreg(d) < 0) + return 0; + int tag = irop_get_tag(d); + if (tag == IROP_TAG_VREG) + return !d.is_lval; + if (tag == IROP_TAG_STACKOFF) + return d.is_lval && !d.is_llocal; + return 0; +} + /* Check if operand has VT_LOCAL semantics - uses bitfield */ int irop_op_is_local(const IROperand op); diff --git a/source/machine/Makefile b/source/machine/Makefile index 9eb605ebb..2fe482f08 100644 --- a/source/machine/Makefile +++ b/source/machine/Makefile @@ -17,7 +17,7 @@ MACHINE_HDRS = \ source/machine/tccmachine.h \ source/machine/tcc_target.h -MACHINE_OBJ = $(patsubst %.c,$(X)%.o,$(MACHINE_SRC)) +MACHINE_OBJ = $(call lib-objs,machine,$(MACHINE_SRC)) MACHINE_LIB = $(X)source/machine/libmachine.a $(X)source/machine/%.o: source/machine/%.c $(MACHINE_HDRS) $(TCC_CORE_DEPS) diff --git a/source/machine/tccabi.h b/source/machine/tccabi.h index 5628333d0..648e96771 100644 --- a/source/machine/tccabi.h +++ b/source/machine/tccabi.h @@ -68,8 +68,6 @@ typedef struct TCCAbiCallLayout TCCAbiArgDesc *args_effective; uint8_t *arg_flags; -/* arg_flags bits */ -#define TCC_ABI_ARG_FLAG_INVISIBLE_REF 0x01 /* large composite passed as hidden pointer */ /* Optional per-argument descriptors recorded as classification happens. * Useful for debugging and for re-running ABI decisions later. */ TCCAbiArgDesc *args; diff --git a/source/machine/tccls.c b/source/machine/tccls.c index cd18fef9f..b4317e494 100644 --- a/source/machine/tccls.c +++ b/source/machine/tccls.c @@ -144,6 +144,33 @@ static int tcc_ls_reg_type_stack_size(int reg_type) } void tcc_ls_compact_stack_locations(LSLiveIntervalState *ls, int spill_base) +{ + tcc_ls_compact_stack_locations_weighted(ls, spill_base, NULL); +} + +typedef struct +{ + int old_offset; + int size; + int new_offset; + uint32_t weight; +} SlotMapEntry; + +static const SlotMapEntry *slot_order_map; + +/* Coldest per byte first, so the hottest end up lowest: nearest SP. Ties + * keep the order the slots were first met in. */ +static int slot_cold_first(const void *a, const void *b) +{ + const int i = *(const int *)a, j = *(const int *)b; + const SlotMapEntry *x = &slot_order_map[i], *y = &slot_order_map[j]; + const uint64_t lx = (uint64_t)x->weight * (uint32_t)y->size, ly = (uint64_t)y->weight * (uint32_t)x->size; + if (lx != ly) + return lx < ly ? -1 : 1; + return i < j ? -1 : i > j; +} + +void tcc_ls_compact_stack_locations_weighted(LSLiveIntervalState *ls, int spill_base, const uint32_t *weights) { if (!ls) return; @@ -159,13 +186,6 @@ void tcc_ls_compact_stack_locations(LSLiveIntervalState *ls, int spill_base) * multiple intervals sharing a slot (from regalloc slot reuse) continue * to share after compaction. Without this mapping, each interval would * be assigned a fresh slot here, undoing addrtaken slot coalescing. */ - typedef struct - { - int old_offset; - int size; - int new_offset; - } SlotMapEntry; - SlotMapEntry *map = tcc_malloc(sizeof(SlotMapEntry) * n); int map_count = 0; @@ -177,9 +197,9 @@ void tcc_ls_compact_stack_locations(LSLiveIntervalState *ls, int spill_base) ht[i] = -1; #define SLOT_HASH(off) (((uint32_t)(off) * 2654435761u) & (ht_size - 1)) - /* Pass 1: collect distinct old offsets in first-encounter order (pass 2 - * assigns new offsets in that order) and track the max size required at - * each (so a slot shared by a 4-byte and an 8-byte interval gets 8). */ + /* Pass 1: collect distinct old offsets in first-encounter order, the max + * size required at each (so a slot shared by a 4-byte and an 8-byte + * interval gets 8) and the summed weight. */ for (int i = 0; i < n; ++i) { LSLiveInterval *it = &ls->intervals[i]; @@ -190,33 +210,46 @@ void tcc_ls_compact_stack_locations(LSLiveIntervalState *ls, int spill_base) unsigned h = SLOT_HASH(it->stack_location); while (ht[h] >= 0 && map[ht[h]].old_offset != (int)it->stack_location) h = (h + 1) & (ht_size - 1); - if (ht[h] >= 0) - { - if (size > map[ht[h]].size) - map[ht[h]].size = size; - } - else + if (ht[h] < 0) { map[map_count].old_offset = it->stack_location; map[map_count].size = size; map[map_count].new_offset = 0; + map[map_count].weight = 0; ht[h] = map_count; map_count++; } + SlotMapEntry *e = &map[ht[h]]; + if (size > e->size) + e->size = size; + if (weights) + e->weight += weights[i]; } - /* Pass 2: assign new offsets in the same order as old offsets were - * encountered. This preserves any relative ordering the codegen - * relied on (e.g. adjacent spill slots for LDRD pairs). */ - int loc = spill_base; + /* Pass 2: assign new offsets downwards from spill_base. Unweighted, in + * the order the slots were met; weighted, coldest per byte first, so the + * most-used slots sit nearest SP, where [sp, #imm] has its 16-bit form (up + * to 1020). The STRD/LDRD spill-pair peepholes test adjacency themselves. */ + int *order = tcc_malloc(sizeof(int) * (map_count > 0 ? map_count : 1)); for (int j = 0; j < map_count; ++j) + order[j] = j; + if (weights) + { + slot_order_map = map; + qsort(order, map_count, sizeof(int), slot_cold_first); + slot_order_map = NULL; + } + int loc = spill_base; + for (int k = 0; k < map_count; ++k) { - const int size = map[j].size; + SlotMapEntry *e = &map[order[k]]; + const int size = e->size; loc = (loc - size) & -size; if (loc == 0) loc = -size; - map[j].new_offset = loc; + e->new_offset = loc; } + tcc_free(order); /* Pass 3: rewrite each interval's stack_location through the map. */ for (int i = 0; i < n; ++i) diff --git a/source/machine/tccls.h b/source/machine/tccls.h index b9de44e34..882296f23 100644 --- a/source/machine/tccls.h +++ b/source/machine/tccls.h @@ -91,6 +91,9 @@ void tcc_ls_add_live_interval(LSLiveIntervalState *ls, int vreg, int start, int int reg_type, int lvalue, int precolored_reg); void tcc_ls_compact_stack_locations(LSLiveIntervalState *ls, int spill_base); +/* The same, with the slots ordered by weights[i] (per interval, summed over + * the intervals sharing a slot) per byte: the heaviest nearest SP. */ +void tcc_ls_compact_stack_locations_weighted(LSLiveIntervalState *ls, int spill_base, const uint32_t *weights); void tcc_ls_reset_scratch_cache(LSLiveIntervalState *ls); diff --git a/source/memory/Makefile b/source/memory/Makefile index 3ab170621..4b48d6e77 100644 --- a/source/memory/Makefile +++ b/source/memory/Makefile @@ -19,7 +19,7 @@ $(X)source/memory/%.o: source/memory/%.c $(MEMORY_HDRS) $(TCC_CORE_DEPS) @mkdir -p $(dir $@) $S$(CC) -o $@ -c $< $(addsuffix ,$(DEFINES) $(CFLAGS)) -MEMORY_OBJ = $(patsubst %.c,$(X)%.o,$(MEMORY_SRC)) +MEMORY_OBJ = $(call lib-objs,memory,$(MEMORY_SRC)) MEMORY_LIB = $(X)source/memory/libmemory.a $(MEMORY_LIB): $(MEMORY_OBJ) diff --git a/source/obj/Makefile b/source/obj/Makefile index 77532b4e6..72cf55119 100644 --- a/source/obj/Makefile +++ b/source/obj/Makefile @@ -9,6 +9,12 @@ OBJ_INC = -I$(TOP)/source/obj OBJ_SRC = \ source/obj/tccelf.c \ + source/obj/tccelf_sym.c \ + source/obj/tccelf_reloc.c \ + source/obj/tccelf_layout.c \ + source/obj/tccelf_output.c \ + source/obj/tccelf_load.c \ + source/obj/tccelf_ld.c \ source/obj/tccld.c \ source/obj/tccyaff.c \ source/obj/tccdbg.c @@ -24,7 +30,7 @@ OBJ_HDRS = \ # set even though it is not a .h file. OBJ_HDRS += source/obj/stab.def -OBJ_OBJ = $(patsubst %.c,$(X)%.o,$(OBJ_SRC)) +OBJ_OBJ = $(call lib-objs,obj,$(OBJ_SRC)) OBJ_LIB = $(X)source/obj/libobj.a $(X)source/obj/%.o: source/obj/%.c $(OBJ_HDRS) $(TCC_CORE_DEPS) diff --git a/source/obj/elf.h b/source/obj/elf.h index 76f3efc5a..9cf88084b 100644 --- a/source/obj/elf.h +++ b/source/obj/elf.h @@ -35,8 +35,6 @@ typedef unsigned long long int uint64_t; #endif #endif -/* Standard ELF types. */ - /* Type for a 16-bit quantity. */ typedef uint16_t Elf32_Half; typedef uint16_t Elf64_Half; @@ -111,174 +109,45 @@ typedef struct { array. The macros under each EI_* macro are the values the byte may have. */ -#define EI_MAG0 0 /* File identification byte 0 index */ #define ELFMAG0 0x7f /* Magic number byte 0 */ -#define EI_MAG1 1 /* File identification byte 1 index */ #define ELFMAG1 'E' /* Magic number byte 1 */ -#define EI_MAG2 2 /* File identification byte 2 index */ #define ELFMAG2 'L' /* Magic number byte 2 */ -#define EI_MAG3 3 /* File identification byte 3 index */ #define ELFMAG3 'F' /* Magic number byte 3 */ /* Conglomeration of the identification bytes, for easy testing as a word. */ #define ELFMAG "\177ELF" -#define SELFMAG 4 #define EI_CLASS 4 /* File class byte index */ -#define ELFCLASSNONE 0 /* Invalid class */ #define ELFCLASS32 1 /* 32-bit objects */ #define ELFCLASS64 2 /* 64-bit objects */ -#define ELFCLASSNUM 3 -#define EI_DATA 5 /* Data encoding byte index */ -#define ELFDATANONE 0 /* Invalid data encoding */ #define ELFDATA2LSB 1 /* 2's complement, little endian */ -#define ELFDATA2MSB 2 /* 2's complement, big endian */ -#define ELFDATANUM 3 -#define EI_VERSION 6 /* File version byte index */ /* Value must be EV_CURRENT */ #define EI_OSABI 7 /* OS ABI identification */ #define ELFOSABI_NONE 0 /* UNIX System V ABI */ -#define ELFOSABI_SYSV 0 /* Alias. */ -#define ELFOSABI_HPUX 1 /* HP-UX */ -#define ELFOSABI_NETBSD 2 /* NetBSD. */ -#define ELFOSABI_GNU 3 /* Object uses GNU ELF extensions. */ -#define ELFOSABI_LINUX ELFOSABI_GNU /* Compatibility alias. */ -#define ELFOSABI_SOLARIS 6 /* Sun Solaris. */ -#define ELFOSABI_AIX 7 /* IBM AIX. */ -#define ELFOSABI_IRIX 8 /* SGI Irix. */ #define ELFOSABI_FREEBSD 9 /* FreeBSD. */ -#define ELFOSABI_TRU64 10 /* Compaq TRU64 UNIX. */ -#define ELFOSABI_MODESTO 11 /* Novell Modesto. */ -#define ELFOSABI_OPENBSD 12 /* OpenBSD. */ -#define ELFOSABI_OPENVMS 13 -#define ELFOSABI_NSK 14 /* Hewlett-Packard Non-Stop Kernel. */ -#define ELFOSABI_AROS 15 /* Amiga Research OS. */ -#define ELFOSABI_FENIXOS 16 /* FenixOS. */ -#define ELFOSABI_ARM_AEABI 64 /* ARM EABI. */ -#define ELFOSABI_C6000_LINUX 65 /* Linux TMS320C6000. */ #define ELFOSABI_ARM 97 /* ARM */ -#define ELFOSABI_STANDALONE 255 /* Standalone (embedded) application */ - -#define EI_ABIVERSION 8 /* ABI version */ - -#define EI_PAD 9 /* Byte index of padding bytes */ /* Legal values for e_type (object file type). */ -#define ET_NONE 0 /* No file type */ #define ET_REL 1 /* Relocatable file */ #define ET_EXEC 2 /* Executable file */ #define ET_DYN 3 /* Shared object file */ -#define ET_CORE 4 /* Core file */ -#define ET_NUM 5 /* Number of defined types */ -#define ET_LOOS 0xfe00 /* OS-specific range start */ -#define ET_HIOS 0xfeff /* OS-specific range end */ -#define ET_LOPROC 0xff00 /* Processor-specific range start */ -#define ET_HIPROC 0xffff /* Processor-specific range end */ /* Legal values for e_machine (architecture). */ -#define EM_NONE 0 /* No machine */ -#define EM_M32 1 /* AT&T WE 32100 */ -#define EM_SPARC 2 /* SUN SPARC */ #define EM_386 3 /* Intel 80386 */ -#define EM_68K 4 /* Motorola m68k family */ -#define EM_88K 5 /* Motorola m88k family */ -#define EM_860 7 /* Intel 80860 */ -#define EM_MIPS 8 /* MIPS R3000 big-endian */ -#define EM_S370 9 /* IBM System/370 */ -#define EM_MIPS_RS3_LE 10 /* MIPS R3000 little-endian */ - -#define EM_PARISC 15 /* HPPA */ -#define EM_VPP500 17 /* Fujitsu VPP500 */ -#define EM_SPARC32PLUS 18 /* Sun's "v8plus" */ -#define EM_960 19 /* Intel 80960 */ -#define EM_PPC 20 /* PowerPC */ -#define EM_PPC64 21 /* PowerPC 64-bit */ -#define EM_S390 22 /* IBM S390 */ - -#define EM_V800 36 /* NEC V800 series */ -#define EM_FR20 37 /* Fujitsu FR20 */ -#define EM_RH32 38 /* TRW RH-32 */ -#define EM_RCE 39 /* Motorola RCE */ + #define EM_ARM 40 /* ARM */ -#define EM_FAKE_ALPHA 41 /* Digital Alpha */ -#define EM_SH 42 /* Hitachi SH */ -#define EM_SPARCV9 43 /* SPARC v9 64-bit */ -#define EM_TRICORE 44 /* Siemens Tricore */ -#define EM_ARC 45 /* Argonaut RISC Core */ -#define EM_H8_300 46 /* Hitachi H8/300 */ -#define EM_H8_300H 47 /* Hitachi H8/300H */ -#define EM_H8S 48 /* Hitachi H8S */ -#define EM_H8_500 49 /* Hitachi H8/500 */ -#define EM_IA_64 50 /* Intel Merced */ -#define EM_MIPS_X 51 /* Stanford MIPS-X */ -#define EM_COLDFIRE 52 /* Motorola Coldfire */ -#define EM_68HC12 53 /* Motorola M68HC12 */ -#define EM_MMA 54 /* Fujitsu MMA Multimedia Accelerator*/ -#define EM_PCP 55 /* Siemens PCP */ -#define EM_NCPU 56 /* Sony nCPU embedded RISC */ -#define EM_NDR1 57 /* Denso NDR1 microprocessor */ -#define EM_STARCORE 58 /* Motorola Start*Core processor */ -#define EM_ME16 59 /* Toyota ME16 processor */ -#define EM_ST100 60 /* STMicroelectronic ST100 processor */ -#define EM_TINYJ 61 /* Advanced Logic Corp. Tinyj emb.fam*/ -#define EM_X86_64 62 /* AMD x86-64 architecture */ -#define EM_PDSP 63 /* Sony DSP Processor */ - -#define EM_FX66 66 /* Siemens FX66 microcontroller */ -#define EM_ST9PLUS 67 /* STMicroelectronics ST9+ 8/16 mc */ -#define EM_ST7 68 /* STMicroelectronics ST7 8 bit mc */ -#define EM_68HC16 69 /* Motorola MC68HC16 microcontroller */ -#define EM_68HC11 70 /* Motorola MC68HC11 microcontroller */ -#define EM_68HC08 71 /* Motorola MC68HC08 microcontroller */ -#define EM_68HC05 72 /* Motorola MC68HC05 microcontroller */ -#define EM_SVX 73 /* Silicon Graphics SVx */ -#define EM_ST19 74 /* STMicroelectronics ST19 8 bit mc */ -#define EM_VAX 75 /* Digital VAX */ -#define EM_CRIS 76 /* Axis Communications 32-bit embedded processor */ -#define EM_JAVELIN 77 /* Infineon Technologies 32-bit embedded processor */ -#define EM_FIREPATH 78 /* Element 14 64-bit DSP Processor */ -#define EM_ZSP 79 /* LSI Logic 16-bit DSP Processor */ -#define EM_MMIX 80 /* Donald Knuth's educational 64-bit processor */ -#define EM_HUANY 81 /* Harvard University machine-independent object files */ -#define EM_PRISM 82 /* SiTera Prism */ -#define EM_AVR 83 /* Atmel AVR 8-bit microcontroller */ -#define EM_FR30 84 /* Fujitsu FR30 */ -#define EM_D10V 85 /* Mitsubishi D10V */ -#define EM_D30V 86 /* Mitsubishi D30V */ -#define EM_V850 87 /* NEC v850 */ -#define EM_M32R 88 /* Mitsubishi M32R */ -#define EM_MN10300 89 /* Matsushita MN10300 */ -#define EM_MN10200 90 /* Matsushita MN10200 */ -#define EM_PJ 91 /* picoJava */ -#define EM_OPENRISC 92 /* OpenRISC 32-bit embedded processor */ -#define EM_ARC_A5 93 /* ARC Cores Tangent-A5 */ -#define EM_XTENSA 94 /* Tensilica Xtensa Architecture */ -#define EM_AARCH64 183 /* ARM AARCH64 */ -#define EM_TILEPRO 188 /* Tilera TILEPro */ -#define EM_TILEGX 191 /* Tilera TILE-Gx */ -#define EM_RISCV 243 /* RISC-V */ -#define EM_NUM 253 - -/* If it is necessary to assign new unofficial EM_* values, please - pick large random numbers (0x8523, 0xa7f2, etc.) to minimize the - chances of collision with official or non-GNU unofficial values. */ - -#define EM_ALPHA 0x9026 -#define EM_C60 0x9c60 /* Legal values for e_version (version). */ -#define EV_NONE 0 /* Invalid ELF version */ #define EV_CURRENT 1 /* Current version */ -#define EV_NUM 2 /* Section header. */ @@ -312,20 +181,8 @@ typedef struct { #define SHN_UNDEF 0 /* Undefined section */ #define SHN_LORESERVE 0xff00 /* Start of reserved indices */ -#define SHN_LOPROC 0xff00 /* Start of processor-specific */ -#define SHN_BEFORE \ - 0xff00 /* Order section before all others \ - (Solaris). */ -#define SHN_AFTER \ - 0xff01 /* Order section after all others \ - (Solaris). */ -#define SHN_HIPROC 0xff1f /* End of processor-specific */ -#define SHN_LOOS 0xff20 /* Start of OS-specific */ -#define SHN_HIOS 0xff3f /* End of OS-specific */ #define SHN_ABS 0xfff1 /* Associated symbol is absolute */ #define SHN_COMMON 0xfff2 /* Associated symbol is common */ -#define SHN_XINDEX 0xffff /* Index is in extra table. */ -#define SHN_HIRESERVE 0xffff /* End of reserved indices */ /* Legal values for sh_type (section type). */ @@ -339,32 +196,15 @@ typedef struct { #define SHT_NOTE 7 /* Notes */ #define SHT_NOBITS 8 /* Program space with no data (bss) */ #define SHT_REL 9 /* Relocation entries, no addends */ -#define SHT_SHLIB 10 /* Reserved */ #define SHT_DYNSYM 11 /* Dynamic linker symbol table */ #define SHT_INIT_ARRAY 14 /* Array of constructors */ #define SHT_FINI_ARRAY 15 /* Array of destructors */ #define SHT_PREINIT_ARRAY 16 /* Array of pre-constructors */ -#define SHT_GROUP 17 /* Section group */ -#define SHT_SYMTAB_SHNDX 18 /* Extended section indices */ -#define SHT_NUM 19 /* Number of defined types. */ -#define SHT_LOOS 0x60000000 /* Start OS-specific. */ -#define SHT_GNU_ATTRIBUTES 0x6ffffff5 /* Object attributes. */ #define SHT_GNU_HASH 0x6ffffff6 /* GNU-style hash table. */ -#define SHT_GNU_LIBLIST 0x6ffffff7 /* Prelink library list */ -#define SHT_CHECKSUM 0x6ffffff8 /* Checksum for DSO content. */ -#define SHT_LOSUNW 0x6ffffffa /* Sun-specific low bound. */ -#define SHT_SUNW_move 0x6ffffffa -#define SHT_SUNW_COMDAT 0x6ffffffb -#define SHT_SUNW_syminfo 0x6ffffffc #define SHT_GNU_verdef 0x6ffffffd /* Version definition section. */ #define SHT_GNU_verneed 0x6ffffffe /* Version needs section. */ #define SHT_GNU_versym 0x6fffffff /* Version symbol table. */ -#define SHT_HISUNW 0x6fffffff /* Sun-specific high bound. */ -#define SHT_HIOS 0x6fffffff /* End OS-specific type */ #define SHT_LOPROC 0x70000000 /* Start of processor-specific */ -#define SHT_HIPROC 0x7fffffff /* End of processor-specific */ -#define SHT_LOUSER 0x80000000 /* Start of application-specific */ -#define SHT_HIUSER 0x8fffffff /* End of application-specific */ /* Legal values for sh_flags (section flags). */ @@ -373,25 +213,9 @@ typedef struct { #define SHF_EXECINSTR (1 << 2) /* Executable */ #define SHF_MERGE (1 << 4) /* Might be merged */ #define SHF_STRINGS (1 << 5) /* Contains nul-terminated strings */ -#define SHF_INFO_LINK (1 << 6) /* `sh_info' contains SHT index */ -#define SHF_LINK_ORDER (1 << 7) /* Preserve order after combining */ -#define SHF_OS_NONCONFORMING \ - (1 << 8) /* Non-standard OS specific handling \ - required */ #define SHF_GROUP (1 << 9) /* Section is member of a group. */ #define SHF_TLS (1 << 10) /* Section hold thread-local data. */ #define SHF_COMPRESSED (1 << 11) /* Section with compressed data. */ -#define SHF_MASKOS 0x0ff00000 /* OS-specific. */ -#define SHF_MASKPROC 0xf0000000 /* Processor-specific */ -#define SHF_ORDERED \ - (1 << 30) /* Special ordering requirement \ - (Solaris). */ -#define SHF_EXCLUDE \ - (1U << 31) /* Section is excluded unless \ - referenced or allocated (Solaris).*/ - -/* Section group handling. */ -#define GRP_COMDAT 0x1 /* Mark group as COMDAT. */ /* Symbol table entry. */ @@ -426,23 +250,6 @@ typedef struct { Elf64_Half si_flags; /* Per symbol flags */ } Elf64_Syminfo; -/* Possible values for si_boundto. */ -#define SYMINFO_BT_SELF 0xffff /* Symbol bound to self */ -#define SYMINFO_BT_PARENT 0xfffe /* Symbol bound to parent */ -#define SYMINFO_BT_LOWRESERVE 0xff00 /* Beginning of reserved entries */ - -/* Possible bitmasks for si_flags. */ -#define SYMINFO_FLG_DIRECT 0x0001 /* Direct bound symbol */ -#define SYMINFO_FLG_PASSTHRU 0x0002 /* Pass-thru symbol for translator */ -#define SYMINFO_FLG_COPY 0x0004 /* Symbol is a copy-reloc */ -#define SYMINFO_FLG_LAZYLOAD \ - 0x0008 /* Symbol bound to object to be lazy \ - loaded */ -/* Syminfo version values. */ -#define SYMINFO_NONE 0 -#define SYMINFO_CURRENT 1 -#define SYMINFO_NUM 2 - /* How to extract and insert information held in the st_info field. */ #define ELF32_ST_BIND(val) (((unsigned char)(val)) >> 4) @@ -459,12 +266,6 @@ typedef struct { #define STB_LOCAL 0 /* Local symbol */ #define STB_GLOBAL 1 /* Global symbol */ #define STB_WEAK 2 /* Weak symbol */ -#define STB_NUM 3 /* Number of defined types. */ -#define STB_LOOS 10 /* Start of OS-specific */ -#define STB_GNU_UNIQUE 10 /* Unique symbol. */ -#define STB_HIOS 12 /* End of OS-specific */ -#define STB_LOPROC 13 /* Start of processor-specific */ -#define STB_HIPROC 15 /* End of processor-specific */ /* Legal values for ST_TYPE subfield of st_info (symbol type). */ @@ -473,20 +274,7 @@ typedef struct { #define STT_FUNC 2 /* Symbol is a code object */ #define STT_SECTION 3 /* Symbol associated with a section */ #define STT_FILE 4 /* Symbol's name is file name */ -#define STT_COMMON 5 /* Symbol is a common data object */ -#define STT_TLS 6 /* Symbol is thread-local data object*/ -#define STT_NUM 7 /* Number of defined types. */ -#define STT_LOOS 10 /* Start of OS-specific */ #define STT_GNU_IFUNC 10 /* Symbol is indirect code object */ -#define STT_HIOS 12 /* End of OS-specific */ -#define STT_LOPROC 13 /* Start of processor-specific */ -#define STT_HIPROC 15 /* End of processor-specific */ - -/* Symbol table indices are found in the hash buckets and chain table - of a symbol hash table section. This special index value indicates - the end of a chain, meaning no further symbols are found in that bucket. */ - -#define STN_UNDEF 0 /* End of a chain. */ /* How to extract and insert information held in the st_other field. */ @@ -508,9 +296,6 @@ typedef struct { Elf32_Word r_info; /* Relocation type and symbol index */ } Elf32_Rel; -/* I have seen two different definitions of the Elf64_Rel and - Elf64_Rela structures, so we'll leave them out until Novell (or - whoever) gets their act together. */ /* The following, at least, is used on Sparc v9, MIPS, and Alpha. */ typedef struct { @@ -566,12 +351,6 @@ typedef struct { Elf64_Xword p_align; /* Segment alignment */ } Elf64_Phdr; -/* Special value for e_phnum. This indicates that the real number of - program headers is too large to fit into e_phnum. Instead the real - value is in the field sh_info of section 0. */ - -#define PN_XNUM 0xffff - /* Legal values for p_type (segment type). */ #define PT_NULL 0 /* Program header table entry unused */ @@ -579,71 +358,17 @@ typedef struct { #define PT_DYNAMIC 2 /* Dynamic linking information */ #define PT_INTERP 3 /* Program interpreter */ #define PT_NOTE 4 /* Auxiliary information */ -#define PT_SHLIB 5 /* Reserved */ #define PT_PHDR 6 /* Entry for header table itself */ #define PT_TLS 7 /* Thread-local storage segment */ -#define PT_NUM 8 /* Number of defined types */ -#define PT_LOOS 0x60000000 /* Start of OS-specific */ #define PT_GNU_EH_FRAME 0x6474e550 /* GCC .eh_frame_hdr segment */ #define PT_GNU_STACK 0x6474e551 /* Indicates stack executability */ #define PT_GNU_RELRO 0x6474e552 /* Read-only after relocation */ -#define PT_LOSUNW 0x6ffffffa -#define PT_SUNWBSS 0x6ffffffa /* Sun Specific segment */ -#define PT_SUNWSTACK 0x6ffffffb /* Stack segment */ -#define PT_HISUNW 0x6fffffff -#define PT_HIOS 0x6fffffff /* End of OS-specific */ -#define PT_LOPROC 0x70000000 /* Start of processor-specific */ -#define PT_HIPROC 0x7fffffff /* End of processor-specific */ /* Legal values for p_flags (segment flags). */ #define PF_X (1 << 0) /* Segment is executable */ #define PF_W (1 << 1) /* Segment is writable */ #define PF_R (1 << 2) /* Segment is readable */ -#define PF_MASKOS 0x0ff00000 /* OS-specific */ -#define PF_MASKPROC 0xf0000000 /* Processor-specific */ - -/* Legal values for note segment descriptor types for core files. */ - -#define NT_PRSTATUS 1 /* Contains copy of prstatus struct */ -#define NT_FPREGSET 2 /* Contains copy of fpregset struct */ -#define NT_PRPSINFO 3 /* Contains copy of prpsinfo struct */ -#define NT_PRXREG 4 /* Contains copy of prxregset struct */ -#define NT_TASKSTRUCT 4 /* Contains copy of task structure */ -#define NT_PLATFORM 5 /* String from sysinfo(SI_PLATFORM) */ -#define NT_AUXV 6 /* Contains copy of auxv array */ -#define NT_GWINDOWS 7 /* Contains copy of gwindows struct */ -#define NT_ASRS 8 /* Contains copy of asrset struct */ -#define NT_PSTATUS 10 /* Contains copy of pstatus struct */ -#define NT_PSINFO 13 /* Contains copy of psinfo struct */ -#define NT_PRCRED 14 /* Contains copy of prcred struct */ -#define NT_UTSNAME 15 /* Contains copy of utsname struct */ -#define NT_LWPSTATUS 16 /* Contains copy of lwpstatus struct */ -#define NT_LWPSINFO 17 /* Contains copy of lwpinfo struct */ -#define NT_PRFPXREG 20 /* Contains copy of fprxregset struct */ -#define NT_PRXFPREG 0x46e62b7f /* Contains copy of user_fxsr_struct */ -#define NT_PPC_VMX 0x100 /* PowerPC Altivec/VMX registers */ -#define NT_PPC_SPE 0x101 /* PowerPC SPE/EVR registers */ -#define NT_PPC_VSX 0x102 /* PowerPC VSX registers */ -#define NT_386_TLS 0x200 /* i386 TLS slots (struct user_desc) */ -#define NT_386_IOPERM 0x201 /* x86 io permission bitmap (1=deny) */ -#define NT_X86_XSTATE 0x202 /* x86 extended state using xsave */ -#define NT_S390_HIGH_GPRS 0x300 /* s390 upper register halves */ -#define NT_S390_TIMER 0x301 /* s390 timer register */ -#define NT_S390_TODCMP 0x302 /* s390 TOD clock comparator register */ -#define NT_S390_TODPREG 0x303 /* s390 TOD programmable register */ -#define NT_S390_CTRS 0x304 /* s390 control registers */ -#define NT_S390_PREFIX 0x305 /* s390 prefix register */ -#define NT_S390_LAST_BREAK 0x306 /* s390 breaking event address */ -#define NT_S390_SYSTEM_CALL 0x307 /* s390 system call restart data */ -#define NT_ARM_VFP 0x400 /* ARM VFP/NEON registers */ -#define NT_ARM_TLS 0x401 /* ARM TLS register */ -#define NT_ARM_HW_BREAK 0x402 /* ARM hardware breakpoint registers */ -#define NT_ARM_HW_WATCH 0x403 /* ARM hardware watchpoint registers */ - -/* Legal values for the note segment descriptor types for object files. */ - -#define NT_VERSION 1 /* Contains a version string. */ /* Dynamic section entry. */ @@ -689,43 +414,24 @@ typedef struct { #define DT_DEBUG 21 /* For debugging; unspecified */ #define DT_TEXTREL 22 /* Reloc might modify .text */ #define DT_JMPREL 23 /* Address of PLT relocs */ -#define DT_BIND_NOW 24 /* Process relocations of object */ #define DT_INIT_ARRAY 25 /* Array with addresses of init fct */ #define DT_FINI_ARRAY 26 /* Array with addresses of fini fct */ #define DT_INIT_ARRAYSZ 27 /* Size in bytes of DT_INIT_ARRAY */ #define DT_FINI_ARRAYSZ 28 /* Size in bytes of DT_FINI_ARRAY */ #define DT_RUNPATH 29 /* Library search path */ #define DT_FLAGS 30 /* Flags for the object being loaded */ -#define DT_ENCODING 32 /* Start of encoded range */ #define DT_PREINIT_ARRAY 32 /* Array with addresses of preinit fct*/ #define DT_PREINIT_ARRAYSZ 33 /* size in bytes of DT_PREINIT_ARRAY */ -#define DT_NUM 34 /* Number used */ -#define DT_LOOS 0x6000000d /* Start of OS-specific */ -#define DT_HIOS 0x6ffff000 /* End of OS-specific */ -#define DT_LOPROC 0x70000000 /* Start of processor-specific */ -#define DT_HIPROC 0x7fffffff /* End of processor-specific */ -#define DT_PROCNUM DT_MIPS_NUM /* Most used by any processor */ /* DT_* entries which fall between DT_VALRNGHI & DT_VALRNGLO use the Dyn.d_un.d_val field of the Elf*_Dyn structure. This follows Sun's approach. */ #define DT_VALRNGLO 0x6ffffd00 -#define DT_GNU_PRELINKED 0x6ffffdf5 /* Prelinking timestamp */ -#define DT_GNU_CONFLICTSZ 0x6ffffdf6 /* Size of conflict section */ -#define DT_GNU_LIBLISTSZ 0x6ffffdf7 /* Size of library list */ -#define DT_CHECKSUM 0x6ffffdf8 -#define DT_PLTPADSZ 0x6ffffdf9 -#define DT_MOVEENT 0x6ffffdfa -#define DT_MOVESZ 0x6ffffdfb #define DT_FEATURE_1 0x6ffffdfc /* Feature selection (DTF_*). */ #define DT_POSFLAG_1 \ 0x6ffffdfd /* Flags for DT_* entries, effecting \ the following DT_* entry. */ -#define DT_SYMINSZ 0x6ffffdfe /* Size of syminfo table (in bytes) */ -#define DT_SYMINENT 0x6ffffdff /* Entry size of syminfo */ #define DT_VALRNGHI 0x6ffffdff -#define DT_VALTAGIDX(tag) (DT_VALRNGHI - (tag)) /* Reverse order! */ -#define DT_VALNUM 12 /* DT_* entries which fall between DT_ADDRRNGHI & DT_ADDRRNGLO use the Dyn.d_un.d_ptr field of the Elf*_Dyn structure. @@ -734,19 +440,7 @@ typedef struct { built these entries will need to be adjusted. */ #define DT_ADDRRNGLO 0x6ffffe00 #define DT_GNU_HASH 0x6ffffef5 /* GNU-style hash table. */ -#define DT_TLSDESC_PLT 0x6ffffef6 -#define DT_TLSDESC_GOT 0x6ffffef7 -#define DT_GNU_CONFLICT 0x6ffffef8 /* Start of conflict section */ -#define DT_GNU_LIBLIST 0x6ffffef9 /* Library list */ -#define DT_CONFIG 0x6ffffefa /* Configuration information. */ -#define DT_DEPAUDIT 0x6ffffefb /* Dependency auditing. */ -#define DT_AUDIT 0x6ffffefc /* Object auditing. */ -#define DT_PLTPAD 0x6ffffefd /* PLT padding. */ -#define DT_MOVETAB 0x6ffffefe /* Move table. */ -#define DT_SYMINFO 0x6ffffeff /* Syminfo table. */ #define DT_ADDRRNGHI 0x6ffffeff -#define DT_ADDRTAGIDX(tag) (DT_ADDRRNGHI - (tag)) /* Reverse order! */ -#define DT_ADDRNUM 11 /* The versioning entry types. The next are defined as part of the GNU extension. */ @@ -757,71 +451,19 @@ typedef struct { /* These were chosen by Sun. */ #define DT_FLAGS_1 0x6ffffffb /* State flags, see DF_1_* below. */ -#define DT_VERDEF \ - 0x6ffffffc /* Address of version definition \ - table */ -#define DT_VERDEFNUM 0x6ffffffd /* Number of version definitions */ #define DT_VERNEED \ 0x6ffffffe /* Address of table with needed \ versions */ #define DT_VERNEEDNUM 0x6fffffff /* Number of needed versions */ -#define DT_VERSIONTAGIDX(tag) (DT_VERNEEDNUM - (tag)) /* Reverse order! */ -#define DT_VERSIONTAGNUM 16 - -/* Sun added these machine-independent extensions in the "processor-specific" - range. Be compatible. */ -#define DT_AUXILIARY 0x7ffffffd /* Shared object to load before self */ -#define DT_FILTER 0x7fffffff /* Shared object to get values from */ -#define DT_EXTRATAGIDX(tag) ((Elf32_Word) - ((Elf32_Sword)(tag) << 1 >> 1) - 1) -#define DT_EXTRANUM 3 /* Values of `d_un.d_val' in the DT_FLAGS entry. */ -#define DF_ORIGIN 0x00000001 /* Object may use DF_ORIGIN */ -#define DF_SYMBOLIC 0x00000002 /* Symbol resolutions starts here */ -#define DF_TEXTREL 0x00000004 /* Object contains text relocations */ #define DF_BIND_NOW 0x00000008 /* No lazy binding for this object */ -#define DF_STATIC_TLS 0x00000010 /* Module uses the static TLS model */ /* State flags selectable in the `d_un.d_val' element of the DT_FLAGS_1 entry in the dynamic section. */ #define DF_1_NOW 0x00000001 /* Set RTLD_NOW for this object. */ -#define DF_1_GLOBAL 0x00000002 /* Set RTLD_GLOBAL for this object. */ -#define DF_1_GROUP 0x00000004 /* Set RTLD_GROUP for this object. */ -#define DF_1_NODELETE 0x00000008 /* Set RTLD_NODELETE for this object.*/ -#define DF_1_LOADFLTR 0x00000010 /* Trigger filtee loading at runtime.*/ -#define DF_1_INITFIRST 0x00000020 /* Set RTLD_INITFIRST for this object*/ -#define DF_1_NOOPEN 0x00000040 /* Set RTLD_NOOPEN for this object. */ -#define DF_1_ORIGIN 0x00000080 /* $ORIGIN must be handled. */ -#define DF_1_DIRECT 0x00000100 /* Direct binding enabled. */ -#define DF_1_TRANS 0x00000200 -#define DF_1_INTERPOSE 0x00000400 /* Object is used to interpose. */ -#define DF_1_NODEFLIB 0x00000800 /* Ignore default lib search path. */ -#define DF_1_NODUMP 0x00001000 /* Object can't be dldump'ed. */ -#define DF_1_CONFALT 0x00002000 /* Configuration alternative created.*/ -#define DF_1_ENDFILTEE 0x00004000 /* Filtee terminates filters search. */ -#define DF_1_DISPRELDNE 0x00008000 /* Disp reloc applied at build time. */ -#define DF_1_DISPRELPND 0x00010000 /* Disp reloc applied at run-time. */ -#define DF_1_NODIRECT 0x00020000 /* Object has no-direct binding. */ -#define DF_1_IGNMULDEF 0x00040000 -#define DF_1_NOKSYMS 0x00080000 -#define DF_1_NOHDR 0x00100000 -#define DF_1_EDITED 0x00200000 /* Object is modified after built. */ -#define DF_1_NORELOC 0x00400000 -#define DF_1_SYMINTPOSE 0x00800000 /* Object has individual interposers. */ -#define DF_1_GLOBAUDIT 0x01000000 /* Global auditing required. */ -#define DF_1_SINGLETON 0x02000000 /* Singleton symbols are used. */ #define DF_1_PIE 0x08000000 -/* Flags for the feature selection in DT_FEATURE_1. */ -#define DTF_1_PARINIT 0x00000001 -#define DTF_1_CONFEXP 0x00000002 - -/* Flags in the DT_POSFLAG_1 entry effecting only the next DT_* entry. */ -#define DF_P1_LAZYLOAD 0x00000001 /* Lazyload following object. */ -#define DF_P1_GROUPPERM \ - 0x00000002 /* Symbols from next object are not \ - generally available. */ - /* Version definition sections. */ typedef struct { @@ -846,21 +488,9 @@ typedef struct { entry */ } Elf64_Verdef; -/* Legal values for vd_version (version revision). */ -#define VER_DEF_NONE 0 /* No version */ -#define VER_DEF_CURRENT 1 /* Current version */ -#define VER_DEF_NUM 2 /* Given version number */ - /* Legal values for vd_flags (version information flags). */ -#define VER_FLG_BASE 0x1 /* Version definition of file itself */ #define VER_FLG_WEAK 0x2 /* Weak version identifier */ -/* Versym symbol index values. */ -#define VER_NDX_LOCAL 0 /* Symbol is local. */ -#define VER_NDX_GLOBAL 1 /* Symbol is global. */ -#define VER_NDX_LORESERVE 0xff00 /* Beginning of reserved entries. */ -#define VER_NDX_ELIMINATE 0xff01 /* Symbol is to be eliminated. */ - /* Auxiliary version information. */ typedef struct { @@ -897,11 +527,6 @@ typedef struct { entry */ } Elf64_Verneed; -/* Legal values for vn_version (version revision). */ -#define VER_NEED_NONE 0 /* No version */ -#define VER_NEED_CURRENT 1 /* Current version */ -#define VER_NEED_NUM 2 /* Given version number */ - /* Auxiliary needed version information. */ typedef struct { @@ -925,8 +550,6 @@ typedef struct { /* Legal values for vna_flags. */ #define VER_FLG_WEAK 0x2 /* Weak version identifier */ -/* Auxiliary vector. */ - /* This vector is normally only used by the program interpreter. The usual definition in an ABI supplement uses the name auxv_t. The vector is not usually defined in a standard file, but it @@ -954,64 +577,6 @@ typedef struct { } a_un; } Elf64_auxv_t; -/* Legal values for a_type (entry type). */ - -#define AT_NULL 0 /* End of vector */ -#define AT_IGNORE 1 /* Entry should be ignored */ -#define AT_EXECFD 2 /* File descriptor of program */ -#define AT_PHDR 3 /* Program headers for program */ -#define AT_PHENT 4 /* Size of program header entry */ -#define AT_PHNUM 5 /* Number of program headers */ -#define AT_PAGESZ 6 /* System page size */ -#define AT_BASE 7 /* Base address of interpreter */ -#define AT_FLAGS 8 /* Flags */ -#define AT_ENTRY 9 /* Entry point of program */ -#define AT_NOTELF 10 /* Program is not ELF */ -#define AT_UID 11 /* Real uid */ -#define AT_EUID 12 /* Effective uid */ -#define AT_GID 13 /* Real gid */ -#define AT_EGID 14 /* Effective gid */ -#define AT_CLKTCK 17 /* Frequency of times() */ - -/* Some more special a_type values describing the hardware. */ -#define AT_PLATFORM 15 /* String identifying platform. */ -#define AT_HWCAP \ - 16 /* Machine dependent hints about \ - processor capabilities. */ - -/* This entry gives some information about the FPU initialization - performed by the kernel. */ -#define AT_FPUCW 18 /* Used FPU control word. */ - -/* Cache block sizes. */ -#define AT_DCACHEBSIZE 19 /* Data cache block size. */ -#define AT_ICACHEBSIZE 20 /* Instruction cache block size. */ -#define AT_UCACHEBSIZE 21 /* Unified cache block size. */ - -/* A special ignored value for PPC, used by the kernel to control the - interpretation of the AUXV. Must be > 16. */ -#define AT_IGNOREPPC 22 /* Entry should be ignored. */ - -#define AT_SECURE 23 /* Boolean, was exec setuid-like? */ - -#define AT_BASE_PLATFORM 24 /* String identifying real platforms.*/ - -#define AT_RANDOM 25 /* Address of 16 random bytes. */ - -#define AT_EXECFN 31 /* Filename of executable. */ - -/* Pointer to the global system page used for system calls and other - nice things. */ -#define AT_SYSINFO 32 -#define AT_SYSINFO_EHDR 33 - -/* Shapes of the caches. Bits 0-3 contains associativity; bits 4-7 contains - log2 of line size; mask those to get cache size. */ -#define AT_L1I_CACHESHAPE 34 -#define AT_L1D_CACHESHAPE 35 -#define AT_L2_CACHESHAPE 36 -#define AT_L3_CACHESHAPE 37 - /* Note section contents. Each entry in the note section begins with a header of a fixed form. */ @@ -1027,21 +592,6 @@ typedef struct { Elf64_Word n_type; /* Type of the note. */ } Elf64_Nhdr; -/* Known names of notes. */ - -/* Solaris entries in the note section have this name. */ -#define ELF_NOTE_SOLARIS "SUNW Solaris" - -/* Note entries for GNU systems have this name. */ -#define ELF_NOTE_GNU "GNU" - -/* Defined types of notes for Solaris. */ - -/* Value of descriptor (one word) is desired pagesize for the binary. */ -#define ELF_NOTE_PAGESIZE_HINT 1 - -/* Defined note types for GNU systems. */ - /* ABI information. The descriptor consists of words: word 0: OS descriptor word 1: major version of the ABI @@ -1049,29 +599,10 @@ typedef struct { word 3: subminor version of the ABI */ #define NT_GNU_ABI_TAG 1 -#define ELF_NOTE_ABI NT_GNU_ABI_TAG /* Old name. */ /* Known OSes. These values can appear in word 0 of an NT_GNU_ABI_TAG note section entry. */ -#define ELF_NOTE_OS_LINUX 0 #define ELF_NOTE_OS_GNU 1 -#define ELF_NOTE_OS_SOLARIS2 2 -#define ELF_NOTE_OS_FREEBSD 3 - -/* Synthetic hwcap information. The descriptor begins with two words: - word 0: number of entries - word 1: bitmask of enabled entries - Then follow variable-length entries, one byte followed by a - '\0'-terminated hwcap name string. The byte gives the bit - number to test if enabled, (1U << bit) & bitmask. */ -#define NT_GNU_HWCAP 2 - -/* Build ID bits as generated by ld --build-id. - The descriptor consists of any nonzero number of bytes. */ -#define NT_GNU_BUILD_ID 3 - -/* Version note generated by GNU gold containing a version string. */ -#define NT_GNU_GOLD_VERSION 4 /* Move records. */ typedef struct { @@ -1090,383 +621,19 @@ typedef struct { Elf64_Half m_stride; /* Stride info. */ } Elf64_Move; -/* Macro to construct move records. */ -#define ELF32_M_SYM(info) ((info) >> 8) -#define ELF32_M_SIZE(info) ((unsigned char)(info)) -#define ELF32_M_INFO(sym, size) (((sym) << 8) + (unsigned char)(size)) - -#define ELF64_M_SYM(info) ELF32_M_SYM(info) -#define ELF64_M_SIZE(info) ELF32_M_SIZE(info) -#define ELF64_M_INFO(sym, size) ELF32_M_INFO(sym, size) - -/* Motorola 68k specific definitions. */ - -/* Values for Elf32_Ehdr.e_flags. */ -#define EF_CPU32 0x00810000 - -/* m68k relocs. */ - -#define R_68K_NONE 0 /* No reloc */ -#define R_68K_32 1 /* Direct 32 bit */ -#define R_68K_16 2 /* Direct 16 bit */ -#define R_68K_8 3 /* Direct 8 bit */ -#define R_68K_PC32 4 /* PC relative 32 bit */ -#define R_68K_PC16 5 /* PC relative 16 bit */ -#define R_68K_PC8 6 /* PC relative 8 bit */ -#define R_68K_GOT32 7 /* 32 bit PC relative GOT entry */ -#define R_68K_GOT16 8 /* 16 bit PC relative GOT entry */ -#define R_68K_GOT8 9 /* 8 bit PC relative GOT entry */ -#define R_68K_GOT32O 10 /* 32 bit GOT offset */ -#define R_68K_GOT16O 11 /* 16 bit GOT offset */ -#define R_68K_GOT8O 12 /* 8 bit GOT offset */ -#define R_68K_PLT32 13 /* 32 bit PC relative PLT address */ -#define R_68K_PLT16 14 /* 16 bit PC relative PLT address */ -#define R_68K_PLT8 15 /* 8 bit PC relative PLT address */ -#define R_68K_PLT32O 16 /* 32 bit PLT offset */ -#define R_68K_PLT16O 17 /* 16 bit PLT offset */ -#define R_68K_PLT8O 18 /* 8 bit PLT offset */ -#define R_68K_COPY 19 /* Copy symbol at runtime */ -#define R_68K_GLOB_DAT 20 /* Create GOT entry */ -#define R_68K_JMP_SLOT 21 /* Create PLT entry */ -#define R_68K_RELATIVE 22 /* Adjust by program base */ -#define R_68K_TLS_GD32 25 /* 32 bit GOT offset for GD */ -#define R_68K_TLS_GD16 26 /* 16 bit GOT offset for GD */ -#define R_68K_TLS_GD8 27 /* 8 bit GOT offset for GD */ -#define R_68K_TLS_LDM32 28 /* 32 bit GOT offset for LDM */ -#define R_68K_TLS_LDM16 29 /* 16 bit GOT offset for LDM */ -#define R_68K_TLS_LDM8 30 /* 8 bit GOT offset for LDM */ -#define R_68K_TLS_LDO32 31 /* 32 bit module-relative offset */ -#define R_68K_TLS_LDO16 32 /* 16 bit module-relative offset */ -#define R_68K_TLS_LDO8 33 /* 8 bit module-relative offset */ -#define R_68K_TLS_IE32 34 /* 32 bit GOT offset for IE */ -#define R_68K_TLS_IE16 35 /* 16 bit GOT offset for IE */ -#define R_68K_TLS_IE8 36 /* 8 bit GOT offset for IE */ -#define R_68K_TLS_LE32 \ - 37 /* 32 bit offset relative to \ - static TLS block */ -#define R_68K_TLS_LE16 \ - 38 /* 16 bit offset relative to \ - static TLS block */ -#define R_68K_TLS_LE8 \ - 39 /* 8 bit offset relative to \ - static TLS block */ -#define R_68K_TLS_DTPMOD32 40 /* 32 bit module number */ -#define R_68K_TLS_DTPREL32 41 /* 32 bit module-relative offset */ -#define R_68K_TLS_TPREL32 42 /* 32 bit TP-relative offset */ -/* Keep this the last entry. */ -#define R_68K_NUM 43 - -/* Intel 80386 specific definitions. */ - /* i386 relocs. */ -#define R_386_NONE 0 /* No reloc */ #define R_386_32 1 /* Direct 32 bit */ #define R_386_PC32 2 /* PC relative 32 bit */ -#define R_386_GOT32 3 /* 32 bit GOT entry */ -#define R_386_PLT32 4 /* 32 bit PLT address */ -#define R_386_COPY 5 /* Copy symbol at runtime */ -#define R_386_GLOB_DAT 6 /* Create GOT entry */ -#define R_386_JMP_SLOT 7 /* Create PLT entry */ #define R_386_RELATIVE 8 /* Adjust by program base */ -#define R_386_GOTOFF 9 /* 32 bit offset to GOT */ -#define R_386_GOTPC 10 /* 32 bit PC relative offset to GOT */ -#define R_386_32PLT 11 -#define R_386_TLS_TPOFF 14 /* Offset in static TLS block */ -#define R_386_TLS_IE \ - 15 /* Address of GOT entry for static TLS \ - block offset */ -#define R_386_TLS_GOTIE \ - 16 /* GOT entry for static TLS block \ - offset */ -#define R_386_TLS_LE \ - 17 /* Offset relative to static TLS \ - block */ -#define R_386_TLS_GD \ - 18 /* Direct 32 bit for GNU version of \ - general dynamic thread local data */ -#define R_386_TLS_LDM \ - 19 /* Direct 32 bit for GNU version of \ - local dynamic thread local data \ - in LE code */ -#define R_386_16 20 -#define R_386_PC16 21 -#define R_386_8 22 -#define R_386_PC8 23 -#define R_386_TLS_GD_32 \ - 24 /* Direct 32 bit for general dynamic \ - thread local data */ -#define R_386_TLS_GD_PUSH 25 /* Tag for pushl in GD TLS code */ -#define R_386_TLS_GD_CALL \ - 26 /* Relocation for call to \ - __tls_get_addr() */ -#define R_386_TLS_GD_POP 27 /* Tag for popl in GD TLS code */ -#define R_386_TLS_LDM_32 \ - 28 /* Direct 32 bit for local dynamic \ - thread local data in LE code */ -#define R_386_TLS_LDM_PUSH 29 /* Tag for pushl in LDM TLS code */ -#define R_386_TLS_LDM_CALL \ - 30 /* Relocation for call to \ - __tls_get_addr() in LDM code */ -#define R_386_TLS_LDM_POP 31 /* Tag for popl in LDM TLS code */ -#define R_386_TLS_LDO_32 32 /* Offset relative to TLS block */ -#define R_386_TLS_IE_32 \ - 33 /* GOT entry for negated static TLS \ - block offset */ -#define R_386_TLS_LE_32 \ - 34 /* Negated offset relative to static \ - TLS block */ -#define R_386_TLS_DTPMOD32 35 /* ID of module containing symbol */ -#define R_386_TLS_DTPOFF32 36 /* Offset in TLS block */ -#define R_386_TLS_TPOFF32 37 /* Negated offset in static TLS block */ -/* 38? */ -#define R_386_TLS_GOTDESC 39 /* GOT offset for TLS descriptor. */ -#define R_386_TLS_DESC_CALL \ - 40 /* Marker of call through TLS \ - descriptor for \ - relaxation. */ -#define R_386_TLS_DESC \ - 41 /* TLS descriptor containing \ - pointer to code and to \ - argument, returning the TLS \ - offset for the symbol. */ -#define R_386_IRELATIVE 42 /* Adjust indirectly by program base */ -#define R_386_GOT32X 43 /* 32 bit GOT entry, relaxable */ -/* Keep this the last entry. */ -#define R_386_NUM 44 - -/* SUN SPARC specific definitions. */ - -/* Legal values for ST_TYPE subfield of st_info (symbol type). */ - -#define STT_SPARC_REGISTER 13 /* Global register reserved to app. */ - -/* Values for Elf64_Ehdr.e_flags. */ - -#define EF_SPARCV9_MM 3 -#define EF_SPARCV9_TSO 0 -#define EF_SPARCV9_PSO 1 -#define EF_SPARCV9_RMO 2 -#define EF_SPARC_LEDATA 0x800000 /* little endian data */ -#define EF_SPARC_EXT_MASK 0xFFFF00 -#define EF_SPARC_32PLUS 0x000100 /* generic V8+ features */ -#define EF_SPARC_SUN_US1 0x000200 /* Sun UltraSPARC1 extensions */ -#define EF_SPARC_HAL_R1 0x000400 /* HAL R1 extensions */ -#define EF_SPARC_SUN_US3 0x000800 /* Sun UltraSPARCIII extensions */ - -/* SPARC relocs. */ - -#define R_SPARC_NONE 0 /* No reloc */ -#define R_SPARC_8 1 /* Direct 8 bit */ -#define R_SPARC_16 2 /* Direct 16 bit */ -#define R_SPARC_32 3 /* Direct 32 bit */ -#define R_SPARC_DISP8 4 /* PC relative 8 bit */ -#define R_SPARC_DISP16 5 /* PC relative 16 bit */ -#define R_SPARC_DISP32 6 /* PC relative 32 bit */ -#define R_SPARC_WDISP30 7 /* PC relative 30 bit shifted */ -#define R_SPARC_WDISP22 8 /* PC relative 22 bit shifted */ -#define R_SPARC_HI22 9 /* High 22 bit */ -#define R_SPARC_22 10 /* Direct 22 bit */ -#define R_SPARC_13 11 /* Direct 13 bit */ -#define R_SPARC_LO10 12 /* Truncated 10 bit */ -#define R_SPARC_GOT10 13 /* Truncated 10 bit GOT entry */ -#define R_SPARC_GOT13 14 /* 13 bit GOT entry */ -#define R_SPARC_GOT22 15 /* 22 bit GOT entry shifted */ -#define R_SPARC_PC10 16 /* PC relative 10 bit truncated */ -#define R_SPARC_PC22 17 /* PC relative 22 bit shifted */ -#define R_SPARC_WPLT30 18 /* 30 bit PC relative PLT address */ -#define R_SPARC_COPY 19 /* Copy symbol at runtime */ -#define R_SPARC_GLOB_DAT 20 /* Create GOT entry */ -#define R_SPARC_JMP_SLOT 21 /* Create PLT entry */ -#define R_SPARC_RELATIVE 22 /* Adjust by program base */ -#define R_SPARC_UA32 23 /* Direct 32 bit unaligned */ - -/* Additional Sparc64 relocs. */ - -#define R_SPARC_PLT32 24 /* Direct 32 bit ref to PLT entry */ -#define R_SPARC_HIPLT22 25 /* High 22 bit PLT entry */ -#define R_SPARC_LOPLT10 26 /* Truncated 10 bit PLT entry */ -#define R_SPARC_PCPLT32 27 /* PC rel 32 bit ref to PLT entry */ -#define R_SPARC_PCPLT22 28 /* PC rel high 22 bit PLT entry */ -#define R_SPARC_PCPLT10 29 /* PC rel trunc 10 bit PLT entry */ -#define R_SPARC_10 30 /* Direct 10 bit */ -#define R_SPARC_11 31 /* Direct 11 bit */ -#define R_SPARC_64 32 /* Direct 64 bit */ -#define R_SPARC_OLO10 33 /* 10bit with secondary 13bit addend */ -#define R_SPARC_HH22 34 /* Top 22 bits of direct 64 bit */ -#define R_SPARC_HM10 35 /* High middle 10 bits of ... */ -#define R_SPARC_LM22 36 /* Low middle 22 bits of ... */ -#define R_SPARC_PC_HH22 37 /* Top 22 bits of pc rel 64 bit */ -#define R_SPARC_PC_HM10 38 /* High middle 10 bit of ... */ -#define R_SPARC_PC_LM22 39 /* Low middle 22 bits of ... */ -#define R_SPARC_WDISP16 40 /* PC relative 16 bit shifted */ -#define R_SPARC_WDISP19 41 /* PC relative 19 bit shifted */ -#define R_SPARC_GLOB_JMP 42 /* was part of v9 ABI but was removed */ -#define R_SPARC_7 43 /* Direct 7 bit */ -#define R_SPARC_5 44 /* Direct 5 bit */ -#define R_SPARC_6 45 /* Direct 6 bit */ -#define R_SPARC_DISP64 46 /* PC relative 64 bit */ -#define R_SPARC_PLT64 47 /* Direct 64 bit ref to PLT entry */ -#define R_SPARC_HIX22 48 /* High 22 bit complemented */ -#define R_SPARC_LOX10 49 /* Truncated 11 bit complemented */ -#define R_SPARC_H44 50 /* Direct high 12 of 44 bit */ -#define R_SPARC_M44 51 /* Direct mid 22 of 44 bit */ -#define R_SPARC_L44 52 /* Direct low 10 of 44 bit */ -#define R_SPARC_REGISTER 53 /* Global register usage */ -#define R_SPARC_UA64 54 /* Direct 64 bit unaligned */ -#define R_SPARC_UA16 55 /* Direct 16 bit unaligned */ -#define R_SPARC_TLS_GD_HI22 56 -#define R_SPARC_TLS_GD_LO10 57 -#define R_SPARC_TLS_GD_ADD 58 -#define R_SPARC_TLS_GD_CALL 59 -#define R_SPARC_TLS_LDM_HI22 60 -#define R_SPARC_TLS_LDM_LO10 61 -#define R_SPARC_TLS_LDM_ADD 62 -#define R_SPARC_TLS_LDM_CALL 63 -#define R_SPARC_TLS_LDO_HIX22 64 -#define R_SPARC_TLS_LDO_LOX10 65 -#define R_SPARC_TLS_LDO_ADD 66 -#define R_SPARC_TLS_IE_HI22 67 -#define R_SPARC_TLS_IE_LO10 68 -#define R_SPARC_TLS_IE_LD 69 -#define R_SPARC_TLS_IE_LDX 70 -#define R_SPARC_TLS_IE_ADD 71 -#define R_SPARC_TLS_LE_HIX22 72 -#define R_SPARC_TLS_LE_LOX10 73 -#define R_SPARC_TLS_DTPMOD32 74 -#define R_SPARC_TLS_DTPMOD64 75 -#define R_SPARC_TLS_DTPOFF32 76 -#define R_SPARC_TLS_DTPOFF64 77 -#define R_SPARC_TLS_TPOFF32 78 -#define R_SPARC_TLS_TPOFF64 79 -#define R_SPARC_GOTDATA_HIX22 80 -#define R_SPARC_GOTDATA_LOX10 81 -#define R_SPARC_GOTDATA_OP_HIX22 82 -#define R_SPARC_GOTDATA_OP_LOX10 83 -#define R_SPARC_GOTDATA_OP 84 -#define R_SPARC_H34 85 -#define R_SPARC_SIZE32 86 -#define R_SPARC_SIZE64 87 -#define R_SPARC_WDISP10 88 -#define R_SPARC_JMP_IREL 248 -#define R_SPARC_IRELATIVE 249 -#define R_SPARC_GNU_VTINHERIT 250 -#define R_SPARC_GNU_VTENTRY 251 -#define R_SPARC_REV32 252 -/* Keep this the last entry. */ -#define R_SPARC_NUM 253 - -/* For Sparc64, legal values for d_tag of Elf64_Dyn. */ - -#define DT_SPARC_REGISTER 0x70000001 -#define DT_SPARC_NUM 2 - -/* MIPS R3000 specific definitions. */ - -/* Legal values for e_flags field of Elf32_Ehdr. */ - -#define EF_MIPS_NOREORDER 1 /* A .noreorder directive was used */ -#define EF_MIPS_PIC 2 /* Contains PIC code */ -#define EF_MIPS_CPIC 4 /* Uses PIC calling sequence */ -#define EF_MIPS_XGOT 8 -#define EF_MIPS_64BIT_WHIRL 16 -#define EF_MIPS_ABI2 32 -#define EF_MIPS_ABI_ON32 64 -#define EF_MIPS_ARCH 0xf0000000 /* MIPS architecture level */ - -/* Legal values for MIPS architecture level. */ - -#define EF_MIPS_ARCH_1 0x00000000 /* -mips1 code. */ -#define EF_MIPS_ARCH_2 0x10000000 /* -mips2 code. */ -#define EF_MIPS_ARCH_3 0x20000000 /* -mips3 code. */ -#define EF_MIPS_ARCH_4 0x30000000 /* -mips4 code. */ -#define EF_MIPS_ARCH_5 0x40000000 /* -mips5 code. */ -#define EF_MIPS_ARCH_32 0x60000000 /* MIPS32 code. */ -#define EF_MIPS_ARCH_64 0x70000000 /* MIPS64 code. */ - -/* The following are non-official names and should not be used. */ - -#define E_MIPS_ARCH_1 0x00000000 /* -mips1 code. */ -#define E_MIPS_ARCH_2 0x10000000 /* -mips2 code. */ -#define E_MIPS_ARCH_3 0x20000000 /* -mips3 code. */ -#define E_MIPS_ARCH_4 0x30000000 /* -mips4 code. */ -#define E_MIPS_ARCH_5 0x40000000 /* -mips5 code. */ -#define E_MIPS_ARCH_32 0x60000000 /* MIPS32 code. */ -#define E_MIPS_ARCH_64 0x70000000 /* MIPS64 code. */ - -/* Special section indices. */ - -#define SHN_MIPS_ACOMMON 0xff00 /* Allocated common symbols */ -#define SHN_MIPS_TEXT 0xff01 /* Allocated test symbols. */ -#define SHN_MIPS_DATA 0xff02 /* Allocated data symbols. */ -#define SHN_MIPS_SCOMMON 0xff03 /* Small common symbols */ -#define SHN_MIPS_SUNDEFINED 0xff04 /* Small undefined symbols */ /* Legal values for sh_type field of Elf32_Shdr. */ #define SHT_MIPS_LIBLIST 0x70000000 /* Shared objects used in link */ -#define SHT_MIPS_MSYM 0x70000001 #define SHT_MIPS_CONFLICT 0x70000002 /* Conflicting symbols */ #define SHT_MIPS_GPTAB 0x70000003 /* Global data area sizes */ -#define SHT_MIPS_UCODE 0x70000004 /* Reserved for SGI/MIPS compilers */ -#define SHT_MIPS_DEBUG 0x70000005 /* MIPS ECOFF debugging information*/ #define SHT_MIPS_REGINFO 0x70000006 /* Register usage information */ -#define SHT_MIPS_PACKAGE 0x70000007 -#define SHT_MIPS_PACKSYM 0x70000008 -#define SHT_MIPS_RELD 0x70000009 -#define SHT_MIPS_IFACE 0x7000000b -#define SHT_MIPS_CONTENT 0x7000000c #define SHT_MIPS_OPTIONS 0x7000000d /* Miscellaneous options. */ -#define SHT_MIPS_SHDR 0x70000010 -#define SHT_MIPS_FDESC 0x70000011 -#define SHT_MIPS_EXTSYM 0x70000012 -#define SHT_MIPS_DENSE 0x70000013 -#define SHT_MIPS_PDESC 0x70000014 -#define SHT_MIPS_LOCSYM 0x70000015 -#define SHT_MIPS_AUXSYM 0x70000016 -#define SHT_MIPS_OPTSYM 0x70000017 -#define SHT_MIPS_LOCSTR 0x70000018 -#define SHT_MIPS_LINE 0x70000019 -#define SHT_MIPS_RFDESC 0x7000001a -#define SHT_MIPS_DELTASYM 0x7000001b -#define SHT_MIPS_DELTAINST 0x7000001c -#define SHT_MIPS_DELTACLASS 0x7000001d -#define SHT_MIPS_DWARF 0x7000001e /* DWARF debugging information. */ -#define SHT_MIPS_DELTADECL 0x7000001f -#define SHT_MIPS_SYMBOL_LIB 0x70000020 -#define SHT_MIPS_EVENTS 0x70000021 /* Event section. */ -#define SHT_MIPS_TRANSLATE 0x70000022 -#define SHT_MIPS_PIXIE 0x70000023 -#define SHT_MIPS_XLATE 0x70000024 -#define SHT_MIPS_XLATE_DEBUG 0x70000025 -#define SHT_MIPS_WHIRL 0x70000026 -#define SHT_MIPS_EH_REGION 0x70000027 -#define SHT_MIPS_XLATE_OLD 0x70000028 -#define SHT_MIPS_PDR_EXCEPTION 0x70000029 - -/* Legal values for sh_flags field of Elf32_Shdr. */ - -#define SHF_MIPS_GPREL 0x10000000 /* Must be part of global data area */ -#define SHF_MIPS_MERGE 0x20000000 -#define SHF_MIPS_ADDR 0x40000000 -#define SHF_MIPS_STRINGS 0x80000000 -#define SHF_MIPS_NOSTRIP 0x08000000 -#define SHF_MIPS_LOCAL 0x04000000 -#define SHF_MIPS_NAMES 0x02000000 -#define SHF_MIPS_NODUPE 0x01000000 - -/* Symbol tables. */ - -/* MIPS specific values for `st_other'. */ -#define STO_MIPS_DEFAULT 0x0 -#define STO_MIPS_INTERNAL 0x1 -#define STO_MIPS_HIDDEN 0x2 -#define STO_MIPS_PROTECTED 0x3 -#define STO_MIPS_PLT 0x8 -#define STO_MIPS_SC_ALIGN_UNUSED 0xff - -/* MIPS specific values for `st_info'. */ -#define STB_MIPS_SPLIT_COMMON 13 /* Entries found in sections of type SHT_MIPS_GPTAB. */ @@ -1502,43 +669,11 @@ typedef struct { /* Values for `kind' field in Elf_Options. */ -#define ODK_NULL 0 /* Undefined. */ -#define ODK_REGINFO 1 /* Register usage information. */ #define ODK_EXCEPTIONS 2 /* Exception processing options. */ -#define ODK_PAD 3 /* Section padding options. */ #define ODK_HWPATCH 4 /* Hardware workarounds performed */ -#define ODK_FILL 5 /* record the fill value used by the linker. */ -#define ODK_TAGS 6 /* reserve space for desktop tools to write. */ #define ODK_HWAND 7 /* HW workarounds. 'AND' bits when merging. */ #define ODK_HWOR 8 /* HW workarounds. 'OR' bits when merging. */ -/* Values for `info' in Elf_Options for ODK_EXCEPTIONS entries. */ - -#define OEX_FPU_MIN 0x1f /* FPE's which MUST be enabled. */ -#define OEX_FPU_MAX 0x1f00 /* FPE's which MAY be enabled. */ -#define OEX_PAGE0 0x10000 /* page zero must be mapped. */ -#define OEX_SMM 0x20000 /* Force sequential memory mode? */ -#define OEX_FPDBUG 0x40000 /* Force floating point debug mode? */ -#define OEX_PRECISEFP OEX_FPDBUG -#define OEX_DISMISS 0x80000 /* Dismiss invalid address faults? */ - -#define OEX_FPU_INVAL 0x10 -#define OEX_FPU_DIV0 0x08 -#define OEX_FPU_OFLO 0x04 -#define OEX_FPU_UFLO 0x02 -#define OEX_FPU_INEX 0x01 - -/* Masks for `info' in Elf_Options for an ODK_HWPATCH entry. */ - -#define OHW_R4KEOP 0x1 /* R4000 end-of-page patch. */ -#define OHW_R8KPFETCH 0x2 /* may need R8000 prefetch patch. */ -#define OHW_R5KEOP 0x4 /* R5000 end-of-page patch. */ -#define OHW_R5KCVTL 0x8 /* R5000 cvt.[ds].l bug. clean=1. */ - -#define OPAD_PREFIX 0x1 -#define OPAD_POSTFIX 0x2 -#define OPAD_SYMBOL 0x4 - /* Entry found in `.options' section. */ typedef struct { @@ -1546,171 +681,9 @@ typedef struct { Elf32_Word hwp_flags2; /* Extra flags. */ } Elf_Options_Hw; -/* Masks for `info' in ElfOptions for ODK_HWAND and ODK_HWOR entries. */ - -#define OHWA0_R4KEOP_CHECKED 0x00000001 -#define OHWA1_R4KEOP_CLEAN 0x00000002 - -/* MIPS relocs. */ - -#define R_MIPS_NONE 0 /* No reloc */ -#define R_MIPS_16 1 /* Direct 16 bit */ -#define R_MIPS_32 2 /* Direct 32 bit */ -#define R_MIPS_REL32 3 /* PC relative 32 bit */ -#define R_MIPS_26 4 /* Direct 26 bit shifted */ -#define R_MIPS_HI16 5 /* High 16 bit */ -#define R_MIPS_LO16 6 /* Low 16 bit */ -#define R_MIPS_GPREL16 7 /* GP relative 16 bit */ -#define R_MIPS_LITERAL 8 /* 16 bit literal entry */ -#define R_MIPS_GOT16 9 /* 16 bit GOT entry */ -#define R_MIPS_PC16 10 /* PC relative 16 bit */ -#define R_MIPS_CALL16 11 /* 16 bit GOT entry for function */ -#define R_MIPS_GPREL32 12 /* GP relative 32 bit */ - -#define R_MIPS_SHIFT5 16 -#define R_MIPS_SHIFT6 17 -#define R_MIPS_64 18 -#define R_MIPS_GOT_DISP 19 -#define R_MIPS_GOT_PAGE 20 -#define R_MIPS_GOT_OFST 21 -#define R_MIPS_GOT_HI16 22 -#define R_MIPS_GOT_LO16 23 -#define R_MIPS_SUB 24 -#define R_MIPS_INSERT_A 25 -#define R_MIPS_INSERT_B 26 -#define R_MIPS_DELETE 27 -#define R_MIPS_HIGHER 28 -#define R_MIPS_HIGHEST 29 -#define R_MIPS_CALL_HI16 30 -#define R_MIPS_CALL_LO16 31 -#define R_MIPS_SCN_DISP 32 -#define R_MIPS_REL16 33 -#define R_MIPS_ADD_IMMEDIATE 34 -#define R_MIPS_PJUMP 35 -#define R_MIPS_RELGOT 36 -#define R_MIPS_JALR 37 -#define R_MIPS_TLS_DTPMOD32 38 /* Module number 32 bit */ -#define R_MIPS_TLS_DTPREL32 39 /* Module-relative offset 32 bit */ -#define R_MIPS_TLS_DTPMOD64 40 /* Module number 64 bit */ -#define R_MIPS_TLS_DTPREL64 41 /* Module-relative offset 64 bit */ -#define R_MIPS_TLS_GD 42 /* 16 bit GOT offset for GD */ -#define R_MIPS_TLS_LDM 43 /* 16 bit GOT offset for LDM */ -#define R_MIPS_TLS_DTPREL_HI16 44 /* Module-relative offset, high 16 bits */ -#define R_MIPS_TLS_DTPREL_LO16 45 /* Module-relative offset, low 16 bits */ -#define R_MIPS_TLS_GOTTPREL 46 /* 16 bit GOT offset for IE */ -#define R_MIPS_TLS_TPREL32 47 /* TP-relative offset, 32 bit */ -#define R_MIPS_TLS_TPREL64 48 /* TP-relative offset, 64 bit */ -#define R_MIPS_TLS_TPREL_HI16 49 /* TP-relative offset, high 16 bits */ -#define R_MIPS_TLS_TPREL_LO16 50 /* TP-relative offset, low 16 bits */ -#define R_MIPS_GLOB_DAT 51 -#define R_MIPS_COPY 126 -#define R_MIPS_JUMP_SLOT 127 -/* Keep this the last entry. */ -#define R_MIPS_NUM 128 - -/* Legal values for p_type field of Elf32_Phdr. */ - -#define PT_MIPS_REGINFO 0x70000000 /* Register usage information */ -#define PT_MIPS_RTPROC 0x70000001 /* Runtime procedure table. */ -#define PT_MIPS_OPTIONS 0x70000002 - -/* Special program header types. */ - -#define PF_MIPS_LOCAL 0x10000000 - /* Legal values for d_tag field of Elf32_Dyn. */ -#define DT_MIPS_RLD_VERSION 0x70000001 /* Runtime linker interface version */ -#define DT_MIPS_TIME_STAMP 0x70000002 /* Timestamp */ -#define DT_MIPS_ICHECKSUM 0x70000003 /* Checksum */ -#define DT_MIPS_IVERSION 0x70000004 /* Version string (string tbl index) */ #define DT_MIPS_FLAGS 0x70000005 /* Flags */ -#define DT_MIPS_BASE_ADDRESS 0x70000006 /* Base address */ -#define DT_MIPS_MSYM 0x70000007 -#define DT_MIPS_CONFLICT 0x70000008 /* Address of CONFLICT section */ -#define DT_MIPS_LIBLIST 0x70000009 /* Address of LIBLIST section */ -#define DT_MIPS_LOCAL_GOTNO 0x7000000a /* Number of local GOT entries */ -#define DT_MIPS_CONFLICTNO 0x7000000b /* Number of CONFLICT entries */ -#define DT_MIPS_LIBLISTNO 0x70000010 /* Number of LIBLIST entries */ -#define DT_MIPS_SYMTABNO 0x70000011 /* Number of DYNSYM entries */ -#define DT_MIPS_UNREFEXTNO 0x70000012 /* First external DYNSYM */ -#define DT_MIPS_GOTSYM 0x70000013 /* First GOT entry in DYNSYM */ -#define DT_MIPS_HIPAGENO 0x70000014 /* Number of GOT page table entries */ -#define DT_MIPS_RLD_MAP 0x70000016 /* Address of run time loader map. */ -#define DT_MIPS_DELTA_CLASS 0x70000017 /* Delta C++ class definition. */ -#define DT_MIPS_DELTA_CLASS_NO \ - 0x70000018 /* Number of entries in \ - DT_MIPS_DELTA_CLASS. */ -#define DT_MIPS_DELTA_INSTANCE 0x70000019 /* Delta C++ class instances. */ -#define DT_MIPS_DELTA_INSTANCE_NO \ - 0x7000001a /* Number of entries in \ - DT_MIPS_DELTA_INSTANCE. */ -#define DT_MIPS_DELTA_RELOC 0x7000001b /* Delta relocations. */ -#define DT_MIPS_DELTA_RELOC_NO \ - 0x7000001c /* Number of entries in \ - DT_MIPS_DELTA_RELOC. */ -#define DT_MIPS_DELTA_SYM \ - 0x7000001d /* Delta symbols that Delta \ - relocations refer to. */ -#define DT_MIPS_DELTA_SYM_NO \ - 0x7000001e /* Number of entries in \ - DT_MIPS_DELTA_SYM. */ -#define DT_MIPS_DELTA_CLASSSYM \ - 0x70000020 /* Delta symbols that hold the \ - class declaration. */ -#define DT_MIPS_DELTA_CLASSSYM_NO \ - 0x70000021 /* Number of entries in \ - DT_MIPS_DELTA_CLASSSYM. */ -#define DT_MIPS_CXX_FLAGS 0x70000022 /* Flags indicating for C++ flavor. */ -#define DT_MIPS_PIXIE_INIT 0x70000023 -#define DT_MIPS_SYMBOL_LIB 0x70000024 -#define DT_MIPS_LOCALPAGE_GOTIDX 0x70000025 -#define DT_MIPS_LOCAL_GOTIDX 0x70000026 -#define DT_MIPS_HIDDEN_GOTIDX 0x70000027 -#define DT_MIPS_PROTECTED_GOTIDX 0x70000028 -#define DT_MIPS_OPTIONS 0x70000029 /* Address of .options. */ -#define DT_MIPS_INTERFACE 0x7000002a /* Address of .interface. */ -#define DT_MIPS_DYNSTR_ALIGN 0x7000002b -#define DT_MIPS_INTERFACE_SIZE \ - 0x7000002c /* Size of the .interface section. \ - */ -#define DT_MIPS_RLD_TEXT_RESOLVE_ADDR \ - 0x7000002d /* Address of rld_text_rsolve \ - function stored in GOT. */ -#define DT_MIPS_PERF_SUFFIX \ - 0x7000002e /* Default suffix of dso to be added \ - by rld on dlopen() calls. */ -#define DT_MIPS_COMPACT_SIZE \ - 0x7000002f /* (O32)Size of compact rel section. \ - */ -#define DT_MIPS_GP_VALUE 0x70000030 /* GP value for aux GOTs. */ -#define DT_MIPS_AUX_DYNAMIC 0x70000031 /* Address of aux .dynamic. */ -/* The address of .got.plt in an executable using the new non-PIC ABI. */ -#define DT_MIPS_PLTGOT 0x70000032 -/* The base of the PLT in an executable using the new non-PIC ABI if that - PLT is writable. For a non-writable PLT, this is omitted or has a zero - value. */ -#define DT_MIPS_RWPLT 0x70000034 -#define DT_MIPS_NUM 0x35 - -/* Legal values for DT_MIPS_FLAGS Elf32_Dyn entry. */ - -#define RHF_NONE 0 /* No flags */ -#define RHF_QUICKSTART (1 << 0) /* Use quickstart */ -#define RHF_NOTPOT (1 << 1) /* Hash size not power of 2 */ -#define RHF_NO_LIBRARY_REPLACEMENT (1 << 2) /* Ignore LD_LIBRARY_PATH */ -#define RHF_NO_MOVE (1 << 3) -#define RHF_SGI_ONLY (1 << 4) -#define RHF_GUARANTEE_INIT (1 << 5) -#define RHF_DELTA_C_PLUS_PLUS (1 << 6) -#define RHF_GUARANTEE_START_INIT (1 << 7) -#define RHF_PIXIE (1 << 8) -#define RHF_DEFAULT_DELAY_LOAD (1 << 9) -#define RHF_REQUICKSTART (1 << 10) -#define RHF_REQUICKSTARTED (1 << 11) -#define RHF_CORD (1 << 12) -#define RHF_NO_UNRES_UNDEF (1 << 13) -#define RHF_RLD_ORDER_SAFE (1 << 14) /* Entries found in sections of type SHT_MIPS_LIBLIST. */ @@ -1730,731 +703,31 @@ typedef struct { Elf64_Word l_flags; /* Flags */ } Elf64_Lib; -/* Legal values for l_flags. */ - -#define LL_NONE 0 -#define LL_EXACT_MATCH (1 << 0) /* Require exact match */ -#define LL_IGNORE_INT_VER (1 << 1) /* Ignore interface version */ -#define LL_REQUIRE_MINOR (1 << 2) -#define LL_EXPORTS (1 << 3) -#define LL_DELAY_LOAD (1 << 4) -#define LL_DELTA (1 << 5) - /* Entries found in sections of type SHT_MIPS_CONFLICT. */ typedef Elf32_Addr Elf32_Conflict; -/* HPPA specific definitions. */ - /* Legal values for e_flags field of Elf32_Ehdr. */ -#define EF_PARISC_TRAPNIL 0x00010000 /* Trap nil pointer dereference. */ -#define EF_PARISC_EXT 0x00020000 /* Program uses arch. extensions. */ -#define EF_PARISC_LSB 0x00040000 /* Program expects little endian. */ -#define EF_PARISC_WIDE 0x00080000 /* Program expects wide mode. */ -#define EF_PARISC_NO_KABP \ - 0x00100000 /* No kernel assisted branch \ - prediction. */ -#define EF_PARISC_LAZYSWAP 0x00400000 /* Allow lazy swapping. */ #define EF_PARISC_ARCH 0x0000ffff /* Architecture version. */ -/* Defined values for `e_flags & EF_PARISC_ARCH' are: */ - -#define EFA_PARISC_1_0 0x020b /* PA-RISC 1.0 big-endian. */ -#define EFA_PARISC_1_1 0x0210 /* PA-RISC 1.1 big-endian. */ -#define EFA_PARISC_2_0 0x0214 /* PA-RISC 2.0 big-endian. */ - -/* Additional section indices. */ - -#define SHN_PARISC_ANSI_COMMON \ - 0xff00 /* Section for tentatively declared \ - symbols in ANSI C. */ -#define SHN_PARISC_HUGE_COMMON 0xff01 /* Common blocks in huge model. */ - -/* Legal values for sh_type field of Elf32_Shdr. */ - -#define SHT_PARISC_EXT 0x70000000 /* Contains product specific ext. */ -#define SHT_PARISC_UNWIND 0x70000001 /* Unwind information. */ -#define SHT_PARISC_DOC 0x70000002 /* Debug info for optimized code. */ - -/* Legal values for sh_flags field of Elf32_Shdr. */ - -#define SHF_PARISC_SHORT 0x20000000 /* Section with short addressing. */ -#define SHF_PARISC_HUGE 0x40000000 /* Section far from gp. */ -#define SHF_PARISC_SBP 0x80000000 /* Static branch prediction code. */ - -/* Legal values for ST_TYPE subfield of st_info (symbol type). */ - -#define STT_PARISC_MILLICODE 13 /* Millicode function entry point. */ - -#define STT_HP_OPAQUE (STT_LOOS + 0x1) -#define STT_HP_STUB (STT_LOOS + 0x2) - -/* HPPA relocs. */ - -#define R_PARISC_NONE 0 /* No reloc. */ -#define R_PARISC_DIR32 1 /* Direct 32-bit reference. */ -#define R_PARISC_DIR21L 2 /* Left 21 bits of eff. address. */ -#define R_PARISC_DIR17R 3 /* Right 17 bits of eff. address. */ -#define R_PARISC_DIR17F 4 /* 17 bits of eff. address. */ -#define R_PARISC_DIR14R 6 /* Right 14 bits of eff. address. */ -#define R_PARISC_PCREL32 9 /* 32-bit rel. address. */ -#define R_PARISC_PCREL21L 10 /* Left 21 bits of rel. address. */ -#define R_PARISC_PCREL17R 11 /* Right 17 bits of rel. address. */ -#define R_PARISC_PCREL17F 12 /* 17 bits of rel. address. */ -#define R_PARISC_PCREL14R 14 /* Right 14 bits of rel. address. */ -#define R_PARISC_DPREL21L 18 /* Left 21 bits of rel. address. */ -#define R_PARISC_DPREL14R 22 /* Right 14 bits of rel. address. */ -#define R_PARISC_GPREL21L 26 /* GP-relative, left 21 bits. */ -#define R_PARISC_GPREL14R 30 /* GP-relative, right 14 bits. */ -#define R_PARISC_LTOFF21L 34 /* LT-relative, left 21 bits. */ -#define R_PARISC_LTOFF14R 38 /* LT-relative, right 14 bits. */ -#define R_PARISC_SECREL32 41 /* 32 bits section rel. address. */ -#define R_PARISC_SEGBASE 48 /* No relocation, set segment base. */ -#define R_PARISC_SEGREL32 49 /* 32 bits segment rel. address. */ -#define R_PARISC_PLTOFF21L 50 /* PLT rel. address, left 21 bits. */ -#define R_PARISC_PLTOFF14R 54 /* PLT rel. address, right 14 bits. */ -#define R_PARISC_LTOFF_FPTR32 57 /* 32 bits LT-rel. function pointer. */ -#define R_PARISC_LTOFF_FPTR21L 58 /* LT-rel. fct ptr, left 21 bits. */ -#define R_PARISC_LTOFF_FPTR14R 62 /* LT-rel. fct ptr, right 14 bits. */ -#define R_PARISC_FPTR64 64 /* 64 bits function address. */ -#define R_PARISC_PLABEL32 65 /* 32 bits function address. */ -#define R_PARISC_PLABEL21L 66 /* Left 21 bits of fdesc address. */ -#define R_PARISC_PLABEL14R 70 /* Right 14 bits of fdesc address. */ -#define R_PARISC_PCREL64 72 /* 64 bits PC-rel. address. */ -#define R_PARISC_PCREL22F 74 /* 22 bits PC-rel. address. */ -#define R_PARISC_PCREL14WR 75 /* PC-rel. address, right 14 bits. */ -#define R_PARISC_PCREL14DR 76 /* PC rel. address, right 14 bits. */ -#define R_PARISC_PCREL16F 77 /* 16 bits PC-rel. address. */ -#define R_PARISC_PCREL16WF 78 /* 16 bits PC-rel. address. */ -#define R_PARISC_PCREL16DF 79 /* 16 bits PC-rel. address. */ -#define R_PARISC_DIR64 80 /* 64 bits of eff. address. */ -#define R_PARISC_DIR14WR 83 /* 14 bits of eff. address. */ -#define R_PARISC_DIR14DR 84 /* 14 bits of eff. address. */ -#define R_PARISC_DIR16F 85 /* 16 bits of eff. address. */ -#define R_PARISC_DIR16WF 86 /* 16 bits of eff. address. */ -#define R_PARISC_DIR16DF 87 /* 16 bits of eff. address. */ -#define R_PARISC_GPREL64 88 /* 64 bits of GP-rel. address. */ -#define R_PARISC_GPREL14WR 91 /* GP-rel. address, right 14 bits. */ -#define R_PARISC_GPREL14DR 92 /* GP-rel. address, right 14 bits. */ -#define R_PARISC_GPREL16F 93 /* 16 bits GP-rel. address. */ -#define R_PARISC_GPREL16WF 94 /* 16 bits GP-rel. address. */ -#define R_PARISC_GPREL16DF 95 /* 16 bits GP-rel. address. */ -#define R_PARISC_LTOFF64 96 /* 64 bits LT-rel. address. */ -#define R_PARISC_LTOFF14WR 99 /* LT-rel. address, right 14 bits. */ -#define R_PARISC_LTOFF14DR 100 /* LT-rel. address, right 14 bits. */ -#define R_PARISC_LTOFF16F 101 /* 16 bits LT-rel. address. */ -#define R_PARISC_LTOFF16WF 102 /* 16 bits LT-rel. address. */ -#define R_PARISC_LTOFF16DF 103 /* 16 bits LT-rel. address. */ -#define R_PARISC_SECREL64 104 /* 64 bits section rel. address. */ -#define R_PARISC_SEGREL64 112 /* 64 bits segment rel. address. */ -#define R_PARISC_PLTOFF14WR 115 /* PLT-rel. address, right 14 bits. */ -#define R_PARISC_PLTOFF14DR 116 /* PLT-rel. address, right 14 bits. */ -#define R_PARISC_PLTOFF16F 117 /* 16 bits LT-rel. address. */ -#define R_PARISC_PLTOFF16WF 118 /* 16 bits PLT-rel. address. */ -#define R_PARISC_PLTOFF16DF 119 /* 16 bits PLT-rel. address. */ -#define R_PARISC_LTOFF_FPTR64 120 /* 64 bits LT-rel. function ptr. */ -#define R_PARISC_LTOFF_FPTR14WR 123 /* LT-rel. fct. ptr., right 14 bits. */ -#define R_PARISC_LTOFF_FPTR14DR 124 /* LT-rel. fct. ptr., right 14 bits. */ -#define R_PARISC_LTOFF_FPTR16F 125 /* 16 bits LT-rel. function ptr. */ -#define R_PARISC_LTOFF_FPTR16WF 126 /* 16 bits LT-rel. function ptr. */ -#define R_PARISC_LTOFF_FPTR16DF 127 /* 16 bits LT-rel. function ptr. */ -#define R_PARISC_LORESERVE 128 -#define R_PARISC_COPY 128 /* Copy relocation. */ -#define R_PARISC_IPLT 129 /* Dynamic reloc, imported PLT */ -#define R_PARISC_EPLT 130 /* Dynamic reloc, exported PLT */ -#define R_PARISC_TPREL32 153 /* 32 bits TP-rel. address. */ -#define R_PARISC_TPREL21L 154 /* TP-rel. address, left 21 bits. */ -#define R_PARISC_TPREL14R 158 /* TP-rel. address, right 14 bits. */ -#define R_PARISC_LTOFF_TP21L 162 /* LT-TP-rel. address, left 21 bits. */ -#define R_PARISC_LTOFF_TP14R 166 /* LT-TP-rel. address, right 14 bits.*/ -#define R_PARISC_LTOFF_TP14F 167 /* 14 bits LT-TP-rel. address. */ -#define R_PARISC_TPREL64 216 /* 64 bits TP-rel. address. */ -#define R_PARISC_TPREL14WR 219 /* TP-rel. address, right 14 bits. */ -#define R_PARISC_TPREL14DR 220 /* TP-rel. address, right 14 bits. */ -#define R_PARISC_TPREL16F 221 /* 16 bits TP-rel. address. */ -#define R_PARISC_TPREL16WF 222 /* 16 bits TP-rel. address. */ -#define R_PARISC_TPREL16DF 223 /* 16 bits TP-rel. address. */ -#define R_PARISC_LTOFF_TP64 224 /* 64 bits LT-TP-rel. address. */ -#define R_PARISC_LTOFF_TP14WR 227 /* LT-TP-rel. address, right 14 bits.*/ -#define R_PARISC_LTOFF_TP14DR 228 /* LT-TP-rel. address, right 14 bits.*/ -#define R_PARISC_LTOFF_TP16F 229 /* 16 bits LT-TP-rel. address. */ -#define R_PARISC_LTOFF_TP16WF 230 /* 16 bits LT-TP-rel. address. */ -#define R_PARISC_LTOFF_TP16DF 231 /* 16 bits LT-TP-rel. address. */ -#define R_PARISC_GNU_VTENTRY 232 -#define R_PARISC_GNU_VTINHERIT 233 -#define R_PARISC_TLS_GD21L 234 /* GD 21-bit left. */ -#define R_PARISC_TLS_GD14R 235 /* GD 14-bit right. */ -#define R_PARISC_TLS_GDCALL 236 /* GD call to __t_g_a. */ -#define R_PARISC_TLS_LDM21L 237 /* LD module 21-bit left. */ -#define R_PARISC_TLS_LDM14R 238 /* LD module 14-bit right. */ -#define R_PARISC_TLS_LDMCALL 239 /* LD module call to __t_g_a. */ -#define R_PARISC_TLS_LDO21L 240 /* LD offset 21-bit left. */ -#define R_PARISC_TLS_LDO14R 241 /* LD offset 14-bit right. */ -#define R_PARISC_TLS_DTPMOD32 242 /* DTP module 32-bit. */ -#define R_PARISC_TLS_DTPMOD64 243 /* DTP module 64-bit. */ -#define R_PARISC_TLS_DTPOFF32 244 /* DTP offset 32-bit. */ -#define R_PARISC_TLS_DTPOFF64 245 /* DTP offset 32-bit. */ -#define R_PARISC_TLS_LE21L R_PARISC_TPREL21L -#define R_PARISC_TLS_LE14R R_PARISC_TPREL14R -#define R_PARISC_TLS_IE21L R_PARISC_LTOFF_TP21L -#define R_PARISC_TLS_IE14R R_PARISC_LTOFF_TP14R -#define R_PARISC_TLS_TPREL32 R_PARISC_TPREL32 -#define R_PARISC_TLS_TPREL64 R_PARISC_TPREL64 -#define R_PARISC_HIRESERVE 255 - -/* Legal values for p_type field of Elf32_Phdr/Elf64_Phdr. */ - -#define PT_HP_TLS (PT_LOOS + 0x0) -#define PT_HP_CORE_NONE (PT_LOOS + 0x1) -#define PT_HP_CORE_VERSION (PT_LOOS + 0x2) -#define PT_HP_CORE_KERNEL (PT_LOOS + 0x3) -#define PT_HP_CORE_COMM (PT_LOOS + 0x4) -#define PT_HP_CORE_PROC (PT_LOOS + 0x5) -#define PT_HP_CORE_LOADABLE (PT_LOOS + 0x6) -#define PT_HP_CORE_STACK (PT_LOOS + 0x7) -#define PT_HP_CORE_SHM (PT_LOOS + 0x8) -#define PT_HP_CORE_MMF (PT_LOOS + 0x9) -#define PT_HP_PARALLEL (PT_LOOS + 0x10) -#define PT_HP_FASTBIND (PT_LOOS + 0x11) -#define PT_HP_OPT_ANNOT (PT_LOOS + 0x12) -#define PT_HP_HSL_ANNOT (PT_LOOS + 0x13) -#define PT_HP_STACK (PT_LOOS + 0x14) - -#define PT_PARISC_ARCHEXT 0x70000000 -#define PT_PARISC_UNWIND 0x70000001 - -/* Legal values for p_flags field of Elf32_Phdr/Elf64_Phdr. */ - -#define PF_PARISC_SBP 0x08000000 - -#define PF_HP_PAGE_SIZE 0x00100000 -#define PF_HP_FAR_SHARED 0x00200000 -#define PF_HP_NEAR_SHARED 0x00400000 -#define PF_HP_CODE 0x01000000 -#define PF_HP_MODIFY 0x02000000 -#define PF_HP_LAZYSWAP 0x04000000 -#define PF_HP_SBP 0x08000000 - -/* Alpha specific definitions. */ - -/* Legal values for e_flags field of Elf64_Ehdr. */ - -#define EF_ALPHA_32BIT 1 /* All addresses must be < 2GB. */ -#define EF_ALPHA_CANRELAX 2 /* Relocations for relaxing exist. */ - -/* Legal values for sh_type field of Elf64_Shdr. */ - -/* These two are primarily concerned with ECOFF debugging info. */ -#define SHT_ALPHA_DEBUG 0x70000001 -#define SHT_ALPHA_REGINFO 0x70000002 - -/* Legal values for sh_flags field of Elf64_Shdr. */ - -#define SHF_ALPHA_GPREL 0x10000000 - -/* Legal values for st_other field of Elf64_Sym. */ -#define STO_ALPHA_NOPV 0x80 /* No PV required. */ -#define STO_ALPHA_STD_GPLOAD 0x88 /* PV only used for initial ldgp. */ - -/* Alpha relocs. */ - -#define R_ALPHA_NONE 0 /* No reloc */ -#define R_ALPHA_REFLONG 1 /* Direct 32 bit */ -#define R_ALPHA_REFQUAD 2 /* Direct 64 bit */ -#define R_ALPHA_GPREL32 3 /* GP relative 32 bit */ -#define R_ALPHA_LITERAL 4 /* GP relative 16 bit w/optimization */ -#define R_ALPHA_LITUSE 5 /* Optimization hint for LITERAL */ -#define R_ALPHA_GPDISP 6 /* Add displacement to GP */ -#define R_ALPHA_BRADDR 7 /* PC+4 relative 23 bit shifted */ -#define R_ALPHA_HINT 8 /* PC+4 relative 16 bit shifted */ -#define R_ALPHA_SREL16 9 /* PC relative 16 bit */ -#define R_ALPHA_SREL32 10 /* PC relative 32 bit */ -#define R_ALPHA_SREL64 11 /* PC relative 64 bit */ -#define R_ALPHA_GPRELHIGH 17 /* GP relative 32 bit, high 16 bits */ -#define R_ALPHA_GPRELLOW 18 /* GP relative 32 bit, low 16 bits */ -#define R_ALPHA_GPREL16 19 /* GP relative 16 bit */ -#define R_ALPHA_COPY 24 /* Copy symbol at runtime */ -#define R_ALPHA_GLOB_DAT 25 /* Create GOT entry */ -#define R_ALPHA_JMP_SLOT 26 /* Create PLT entry */ -#define R_ALPHA_RELATIVE 27 /* Adjust by program base */ -#define R_ALPHA_TLS_GD_HI 28 -#define R_ALPHA_TLSGD 29 -#define R_ALPHA_TLS_LDM 30 -#define R_ALPHA_DTPMOD64 31 -#define R_ALPHA_GOTDTPREL 32 -#define R_ALPHA_DTPREL64 33 -#define R_ALPHA_DTPRELHI 34 -#define R_ALPHA_DTPRELLO 35 -#define R_ALPHA_DTPREL16 36 -#define R_ALPHA_GOTTPREL 37 -#define R_ALPHA_TPREL64 38 -#define R_ALPHA_TPRELHI 39 -#define R_ALPHA_TPRELLO 40 -#define R_ALPHA_TPREL16 41 -/* Keep this the last entry. */ -#define R_ALPHA_NUM 46 - -/* Magic values of the LITUSE relocation addend. */ -#define LITUSE_ALPHA_ADDR 0 -#define LITUSE_ALPHA_BASE 1 -#define LITUSE_ALPHA_BYTOFF 2 -#define LITUSE_ALPHA_JSR 3 -#define LITUSE_ALPHA_TLS_GD 4 -#define LITUSE_ALPHA_TLS_LDM 5 - -/* Legal values for d_tag of Elf64_Dyn. */ -#define DT_ALPHA_PLTRO (DT_LOPROC + 0) -#define DT_ALPHA_NUM 1 - -/* PowerPC specific declarations */ - -/* Values for Elf32/64_Ehdr.e_flags. */ -#define EF_PPC_EMB 0x80000000 /* PowerPC embedded flag */ - -/* Cygnus local bits below */ -#define EF_PPC_RELOCATABLE 0x00010000 /* PowerPC -mrelocatable flag*/ -#define EF_PPC_RELOCATABLE_LIB \ - 0x00008000 /* PowerPC -mrelocatable-lib \ - flag */ - -/* PowerPC relocations defined by the ABIs */ -#define R_PPC_NONE 0 -#define R_PPC_ADDR32 1 /* 32bit absolute address */ -#define R_PPC_ADDR24 2 /* 26bit address, 2 bits ignored. */ -#define R_PPC_ADDR16 3 /* 16bit absolute address */ -#define R_PPC_ADDR16_LO 4 /* lower 16bit of absolute address */ -#define R_PPC_ADDR16_HI 5 /* high 16bit of absolute address */ -#define R_PPC_ADDR16_HA 6 /* adjusted high 16bit */ -#define R_PPC_ADDR14 7 /* 16bit address, 2 bits ignored */ -#define R_PPC_ADDR14_BRTAKEN 8 -#define R_PPC_ADDR14_BRNTAKEN 9 -#define R_PPC_REL24 10 /* PC relative 26 bit */ -#define R_PPC_REL14 11 /* PC relative 16 bit */ -#define R_PPC_REL14_BRTAKEN 12 -#define R_PPC_REL14_BRNTAKEN 13 -#define R_PPC_GOT16 14 -#define R_PPC_GOT16_LO 15 -#define R_PPC_GOT16_HI 16 -#define R_PPC_GOT16_HA 17 -#define R_PPC_PLTREL24 18 -#define R_PPC_COPY 19 -#define R_PPC_GLOB_DAT 20 -#define R_PPC_JMP_SLOT 21 -#define R_PPC_RELATIVE 22 -#define R_PPC_LOCAL24PC 23 -#define R_PPC_UADDR32 24 -#define R_PPC_UADDR16 25 -#define R_PPC_REL32 26 -#define R_PPC_PLT32 27 -#define R_PPC_PLTREL32 28 -#define R_PPC_PLT16_LO 29 -#define R_PPC_PLT16_HI 30 -#define R_PPC_PLT16_HA 31 -#define R_PPC_SDAREL16 32 -#define R_PPC_SECTOFF 33 -#define R_PPC_SECTOFF_LO 34 -#define R_PPC_SECTOFF_HI 35 -#define R_PPC_SECTOFF_HA 36 - -/* PowerPC relocations defined for the TLS access ABI. */ -#define R_PPC_TLS 67 /* none (sym+add)@tls */ -#define R_PPC_DTPMOD32 68 /* word32 (sym+add)@dtpmod */ -#define R_PPC_TPREL16 69 /* half16* (sym+add)@tprel */ -#define R_PPC_TPREL16_LO 70 /* half16 (sym+add)@tprel@l */ -#define R_PPC_TPREL16_HI 71 /* half16 (sym+add)@tprel@h */ -#define R_PPC_TPREL16_HA 72 /* half16 (sym+add)@tprel@ha */ -#define R_PPC_TPREL32 73 /* word32 (sym+add)@tprel */ -#define R_PPC_DTPREL16 74 /* half16* (sym+add)@dtprel */ -#define R_PPC_DTPREL16_LO 75 /* half16 (sym+add)@dtprel@l */ -#define R_PPC_DTPREL16_HI 76 /* half16 (sym+add)@dtprel@h */ -#define R_PPC_DTPREL16_HA 77 /* half16 (sym+add)@dtprel@ha */ -#define R_PPC_DTPREL32 78 /* word32 (sym+add)@dtprel */ -#define R_PPC_GOT_TLSGD16 79 /* half16* (sym+add)@got@tlsgd */ -#define R_PPC_GOT_TLSGD16_LO 80 /* half16 (sym+add)@got@tlsgd@l */ -#define R_PPC_GOT_TLSGD16_HI 81 /* half16 (sym+add)@got@tlsgd@h */ -#define R_PPC_GOT_TLSGD16_HA 82 /* half16 (sym+add)@got@tlsgd@ha */ -#define R_PPC_GOT_TLSLD16 83 /* half16* (sym+add)@got@tlsld */ -#define R_PPC_GOT_TLSLD16_LO 84 /* half16 (sym+add)@got@tlsld@l */ -#define R_PPC_GOT_TLSLD16_HI 85 /* half16 (sym+add)@got@tlsld@h */ -#define R_PPC_GOT_TLSLD16_HA 86 /* half16 (sym+add)@got@tlsld@ha */ -#define R_PPC_GOT_TPREL16 87 /* half16* (sym+add)@got@tprel */ -#define R_PPC_GOT_TPREL16_LO 88 /* half16 (sym+add)@got@tprel@l */ -#define R_PPC_GOT_TPREL16_HI 89 /* half16 (sym+add)@got@tprel@h */ -#define R_PPC_GOT_TPREL16_HA 90 /* half16 (sym+add)@got@tprel@ha */ -#define R_PPC_GOT_DTPREL16 91 /* half16* (sym+add)@got@dtprel */ -#define R_PPC_GOT_DTPREL16_LO 92 /* half16* (sym+add)@got@dtprel@l */ -#define R_PPC_GOT_DTPREL16_HI 93 /* half16* (sym+add)@got@dtprel@h */ -#define R_PPC_GOT_DTPREL16_HA 94 /* half16* (sym+add)@got@dtprel@ha */ - -/* The remaining relocs are from the Embedded ELF ABI, and are not - in the SVR4 ELF ABI. */ -#define R_PPC_EMB_NADDR32 101 -#define R_PPC_EMB_NADDR16 102 -#define R_PPC_EMB_NADDR16_LO 103 -#define R_PPC_EMB_NADDR16_HI 104 -#define R_PPC_EMB_NADDR16_HA 105 -#define R_PPC_EMB_SDAI16 106 -#define R_PPC_EMB_SDA2I16 107 -#define R_PPC_EMB_SDA2REL 108 -#define R_PPC_EMB_SDA21 109 /* 16 bit offset in SDA */ -#define R_PPC_EMB_MRKREF 110 -#define R_PPC_EMB_RELSEC16 111 -#define R_PPC_EMB_RELST_LO 112 -#define R_PPC_EMB_RELST_HI 113 -#define R_PPC_EMB_RELST_HA 114 -#define R_PPC_EMB_BIT_FLD 115 -#define R_PPC_EMB_RELSDA 116 /* 16 bit relative offset in SDA */ - -/* Diab tool relocations. */ -#define R_PPC_DIAB_SDA21_LO 180 /* like EMB_SDA21, but lower 16 bit */ -#define R_PPC_DIAB_SDA21_HI 181 /* like EMB_SDA21, but high 16 bit */ -#define R_PPC_DIAB_SDA21_HA 182 /* like EMB_SDA21, adjusted high 16 */ -#define R_PPC_DIAB_RELSDA_LO 183 /* like EMB_RELSDA, but lower 16 bit */ -#define R_PPC_DIAB_RELSDA_HI 184 /* like EMB_RELSDA, but high 16 bit */ -#define R_PPC_DIAB_RELSDA_HA 185 /* like EMB_RELSDA, adjusted high 16 */ - -/* GNU extension to support local ifunc. */ -#define R_PPC_IRELATIVE 248 - -/* GNU relocs used in PIC code sequences. */ -#define R_PPC_REL16 249 /* half16 (sym+add-.) */ -#define R_PPC_REL16_LO 250 /* half16 (sym+add-.)@l */ -#define R_PPC_REL16_HI 251 /* half16 (sym+add-.)@h */ -#define R_PPC_REL16_HA 252 /* half16 (sym+add-.)@ha */ - -/* This is a phony reloc to handle any old fashioned TOC16 references - that may still be in object files. */ -#define R_PPC_TOC16 255 - -/* PowerPC specific values for the Dyn d_tag field. */ -#define DT_PPC_GOT (DT_LOPROC + 0) -#define DT_PPC_NUM 1 - -/* PowerPC64 relocations defined by the ABIs */ -#define R_PPC64_NONE R_PPC_NONE -#define R_PPC64_ADDR32 R_PPC_ADDR32 /* 32bit absolute address */ -#define R_PPC64_ADDR24 R_PPC_ADDR24 /* 26bit address, word aligned */ -#define R_PPC64_ADDR16 R_PPC_ADDR16 /* 16bit absolute address */ -#define R_PPC64_ADDR16_LO R_PPC_ADDR16_LO /* lower 16bits of address */ -#define R_PPC64_ADDR16_HI R_PPC_ADDR16_HI /* high 16bits of address. */ -#define R_PPC64_ADDR16_HA R_PPC_ADDR16_HA /* adjusted high 16bits. */ -#define R_PPC64_ADDR14 R_PPC_ADDR14 /* 16bit address, word aligned */ -#define R_PPC64_ADDR14_BRTAKEN R_PPC_ADDR14_BRTAKEN -#define R_PPC64_ADDR14_BRNTAKEN R_PPC_ADDR14_BRNTAKEN -#define R_PPC64_REL24 R_PPC_REL24 /* PC-rel. 26 bit, word aligned */ -#define R_PPC64_REL14 R_PPC_REL14 /* PC relative 16 bit */ -#define R_PPC64_REL14_BRTAKEN R_PPC_REL14_BRTAKEN -#define R_PPC64_REL14_BRNTAKEN R_PPC_REL14_BRNTAKEN -#define R_PPC64_GOT16 R_PPC_GOT16 -#define R_PPC64_GOT16_LO R_PPC_GOT16_LO -#define R_PPC64_GOT16_HI R_PPC_GOT16_HI -#define R_PPC64_GOT16_HA R_PPC_GOT16_HA - -#define R_PPC64_COPY R_PPC_COPY -#define R_PPC64_GLOB_DAT R_PPC_GLOB_DAT -#define R_PPC64_JMP_SLOT R_PPC_JMP_SLOT -#define R_PPC64_RELATIVE R_PPC_RELATIVE - -#define R_PPC64_UADDR32 R_PPC_UADDR32 -#define R_PPC64_UADDR16 R_PPC_UADDR16 -#define R_PPC64_REL32 R_PPC_REL32 -#define R_PPC64_PLT32 R_PPC_PLT32 -#define R_PPC64_PLTREL32 R_PPC_PLTREL32 -#define R_PPC64_PLT16_LO R_PPC_PLT16_LO -#define R_PPC64_PLT16_HI R_PPC_PLT16_HI -#define R_PPC64_PLT16_HA R_PPC_PLT16_HA - -#define R_PPC64_SECTOFF R_PPC_SECTOFF -#define R_PPC64_SECTOFF_LO R_PPC_SECTOFF_LO -#define R_PPC64_SECTOFF_HI R_PPC_SECTOFF_HI -#define R_PPC64_SECTOFF_HA R_PPC_SECTOFF_HA -#define R_PPC64_ADDR30 37 /* word30 (S + A - P) >> 2 */ -#define R_PPC64_ADDR64 38 /* doubleword64 S + A */ -#define R_PPC64_ADDR16_HIGHER 39 /* half16 #higher(S + A) */ -#define R_PPC64_ADDR16_HIGHERA 40 /* half16 #highera(S + A) */ -#define R_PPC64_ADDR16_HIGHEST 41 /* half16 #highest(S + A) */ -#define R_PPC64_ADDR16_HIGHESTA 42 /* half16 #highesta(S + A) */ -#define R_PPC64_UADDR64 43 /* doubleword64 S + A */ -#define R_PPC64_REL64 44 /* doubleword64 S + A - P */ -#define R_PPC64_PLT64 45 /* doubleword64 L + A */ -#define R_PPC64_PLTREL64 46 /* doubleword64 L + A - P */ -#define R_PPC64_TOC16 47 /* half16* S + A - .TOC */ -#define R_PPC64_TOC16_LO 48 /* half16 #lo(S + A - .TOC.) */ -#define R_PPC64_TOC16_HI 49 /* half16 #hi(S + A - .TOC.) */ -#define R_PPC64_TOC16_HA 50 /* half16 #ha(S + A - .TOC.) */ -#define R_PPC64_TOC 51 /* doubleword64 .TOC */ -#define R_PPC64_PLTGOT16 52 /* half16* M + A */ -#define R_PPC64_PLTGOT16_LO 53 /* half16 #lo(M + A) */ -#define R_PPC64_PLTGOT16_HI 54 /* half16 #hi(M + A) */ -#define R_PPC64_PLTGOT16_HA 55 /* half16 #ha(M + A) */ - -#define R_PPC64_ADDR16_DS 56 /* half16ds* (S + A) >> 2 */ -#define R_PPC64_ADDR16_LO_DS 57 /* half16ds #lo(S + A) >> 2 */ -#define R_PPC64_GOT16_DS 58 /* half16ds* (G + A) >> 2 */ -#define R_PPC64_GOT16_LO_DS 59 /* half16ds #lo(G + A) >> 2 */ -#define R_PPC64_PLT16_LO_DS 60 /* half16ds #lo(L + A) >> 2 */ -#define R_PPC64_SECTOFF_DS 61 /* half16ds* (R + A) >> 2 */ -#define R_PPC64_SECTOFF_LO_DS 62 /* half16ds #lo(R + A) >> 2 */ -#define R_PPC64_TOC16_DS 63 /* half16ds* (S + A - .TOC.) >> 2 */ -#define R_PPC64_TOC16_LO_DS 64 /* half16ds #lo(S + A - .TOC.) >> 2 */ -#define R_PPC64_PLTGOT16_DS 65 /* half16ds* (M + A) >> 2 */ -#define R_PPC64_PLTGOT16_LO_DS 66 /* half16ds #lo(M + A) >> 2 */ - -/* PowerPC64 relocations defined for the TLS access ABI. */ -#define R_PPC64_TLS 67 /* none (sym+add)@tls */ -#define R_PPC64_DTPMOD64 68 /* doubleword64 (sym+add)@dtpmod */ -#define R_PPC64_TPREL16 69 /* half16* (sym+add)@tprel */ -#define R_PPC64_TPREL16_LO 70 /* half16 (sym+add)@tprel@l */ -#define R_PPC64_TPREL16_HI 71 /* half16 (sym+add)@tprel@h */ -#define R_PPC64_TPREL16_HA 72 /* half16 (sym+add)@tprel@ha */ -#define R_PPC64_TPREL64 73 /* doubleword64 (sym+add)@tprel */ -#define R_PPC64_DTPREL16 74 /* half16* (sym+add)@dtprel */ -#define R_PPC64_DTPREL16_LO 75 /* half16 (sym+add)@dtprel@l */ -#define R_PPC64_DTPREL16_HI 76 /* half16 (sym+add)@dtprel@h */ -#define R_PPC64_DTPREL16_HA 77 /* half16 (sym+add)@dtprel@ha */ -#define R_PPC64_DTPREL64 78 /* doubleword64 (sym+add)@dtprel */ -#define R_PPC64_GOT_TLSGD16 79 /* half16* (sym+add)@got@tlsgd */ -#define R_PPC64_GOT_TLSGD16_LO 80 /* half16 (sym+add)@got@tlsgd@l */ -#define R_PPC64_GOT_TLSGD16_HI 81 /* half16 (sym+add)@got@tlsgd@h */ -#define R_PPC64_GOT_TLSGD16_HA 82 /* half16 (sym+add)@got@tlsgd@ha */ -#define R_PPC64_GOT_TLSLD16 83 /* half16* (sym+add)@got@tlsld */ -#define R_PPC64_GOT_TLSLD16_LO 84 /* half16 (sym+add)@got@tlsld@l */ -#define R_PPC64_GOT_TLSLD16_HI 85 /* half16 (sym+add)@got@tlsld@h */ -#define R_PPC64_GOT_TLSLD16_HA 86 /* half16 (sym+add)@got@tlsld@ha */ -#define R_PPC64_GOT_TPREL16_DS 87 /* half16ds* (sym+add)@got@tprel */ -#define R_PPC64_GOT_TPREL16_LO_DS 88 /* half16ds (sym+add)@got@tprel@l */ -#define R_PPC64_GOT_TPREL16_HI 89 /* half16 (sym+add)@got@tprel@h */ -#define R_PPC64_GOT_TPREL16_HA 90 /* half16 (sym+add)@got@tprel@ha */ -#define R_PPC64_GOT_DTPREL16_DS 91 /* half16ds* (sym+add)@got@dtprel */ -#define R_PPC64_GOT_DTPREL16_LO_DS 92 /* half16ds (sym+add)@got@dtprel@l */ -#define R_PPC64_GOT_DTPREL16_HI 93 /* half16 (sym+add)@got@dtprel@h */ -#define R_PPC64_GOT_DTPREL16_HA 94 /* half16 (sym+add)@got@dtprel@ha */ -#define R_PPC64_TPREL16_DS 95 /* half16ds* (sym+add)@tprel */ -#define R_PPC64_TPREL16_LO_DS 96 /* half16ds (sym+add)@tprel@l */ -#define R_PPC64_TPREL16_HIGHER 97 /* half16 (sym+add)@tprel@higher */ -#define R_PPC64_TPREL16_HIGHERA 98 /* half16 (sym+add)@tprel@highera */ -#define R_PPC64_TPREL16_HIGHEST 99 /* half16 (sym+add)@tprel@highest */ -#define R_PPC64_TPREL16_HIGHESTA 100 /* half16 (sym+add)@tprel@highesta */ -#define R_PPC64_DTPREL16_DS 101 /* half16ds* (sym+add)@dtprel */ -#define R_PPC64_DTPREL16_LO_DS 102 /* half16ds (sym+add)@dtprel@l */ -#define R_PPC64_DTPREL16_HIGHER 103 /* half16 (sym+add)@dtprel@higher */ -#define R_PPC64_DTPREL16_HIGHERA 104 /* half16 (sym+add)@dtprel@highera */ -#define R_PPC64_DTPREL16_HIGHEST 105 /* half16 (sym+add)@dtprel@highest */ -#define R_PPC64_DTPREL16_HIGHESTA 106 /* half16 (sym+add)@dtprel@highesta */ - -/* GNU extension to support local ifunc. */ -#define R_PPC64_JMP_IREL 247 -#define R_PPC64_IRELATIVE 248 -#define R_PPC64_REL16 249 /* half16 (sym+add-.) */ -#define R_PPC64_REL16_LO 250 /* half16 (sym+add-.)@l */ -#define R_PPC64_REL16_HI 251 /* half16 (sym+add-.)@h */ -#define R_PPC64_REL16_HA 252 /* half16 (sym+add-.)@ha */ - -/* PowerPC64 specific values for the Dyn d_tag field. */ -#define DT_PPC64_GLINK (DT_LOPROC + 0) -#define DT_PPC64_OPD (DT_LOPROC + 1) -#define DT_PPC64_OPDSZ (DT_LOPROC + 2) -#define DT_PPC64_NUM 3 - -/* ARM specific declarations */ - /* Processor specific flags for the ELF header e_flags field. */ -#define EF_ARM_RELEXEC 0x01 -#define EF_ARM_HASENTRY 0x02 -#define EF_ARM_INTERWORK 0x04 -#define EF_ARM_APCS_26 0x08 -#define EF_ARM_APCS_FLOAT 0x10 -#define EF_ARM_PIC 0x20 -#define EF_ARM_ALIGN8 0x40 /* 8-bit structure alignment is in use */ -#define EF_ARM_NEW_ABI 0x80 -#define EF_ARM_OLD_ABI 0x100 #define EF_ARM_SOFT_FLOAT 0x200 #define EF_ARM_VFP_FLOAT 0x400 -#define EF_ARM_MAVERICK_FLOAT 0x800 - -#define EF_ARM_ABI_FLOAT_SOFT 0x200 /* NB conflicts with EF_ARM_SOFT_FLOAT */ -#define EF_ARM_ABI_FLOAT_HARD 0x400 /* NB conflicts with EF_ARM_VFP_FLOAT */ - -/* Other constants defined in the ARM ELF spec. version B-01. */ -/* NB. These conflict with values defined above. */ -#define EF_ARM_SYMSARESORTED 0x04 -#define EF_ARM_DYNSYMSUSESEGIDX 0x08 -#define EF_ARM_MAPSYMSFIRST 0x10 -#define EF_ARM_EABIMASK 0XFF000000 /* Constants defined in AAELF. */ -#define EF_ARM_BE8 0x00800000 -#define EF_ARM_LE8 0x00400000 - -#define EF_ARM_EABI_VERSION(flags) ((flags) & EF_ARM_EABIMASK) -#define EF_ARM_EABI_UNKNOWN 0x00000000 -#define EF_ARM_EABI_VER1 0x01000000 -#define EF_ARM_EABI_VER2 0x02000000 -#define EF_ARM_EABI_VER3 0x03000000 -#define EF_ARM_EABI_VER4 0x04000000 -#define EF_ARM_EABI_VER5 0x05000000 - -/* Additional symbol types for Thumb. */ -#define STT_ARM_TFUNC STT_LOPROC /* A Thumb function. */ -#define STT_ARM_16BIT STT_HIPROC /* A Thumb label. */ - -/* ARM-specific values for sh_flags */ -#define SHF_ARM_ENTRYSECT 0x10000000 /* Section contains an entry point */ -#define SHF_ARM_COMDEF \ - 0x80000000 /* Section may be multiply defined \ - in the input to a link step. */ -/* ARM-specific program header flags */ -#define PF_ARM_SB \ - 0x10000000 /* Segment contains the location \ - addressed by the static base. */ -#define PF_ARM_PI 0x20000000 /* Position-independent segment. */ -#define PF_ARM_ABS 0x40000000 /* Absolute segment. */ - -/* Processor specific values for the Phdr p_type field. */ -#define PT_ARM_EXIDX (PT_LOPROC + 1) /* ARM unwind segment. */ +#define EF_ARM_EABI_VER5 0x05000000 /* Processor specific values for the Shdr sh_type field. */ #define SHT_ARM_EXIDX (SHT_LOPROC + 1) /* ARM unwind section. */ -#define SHT_ARM_PREEMPTMAP (SHT_LOPROC + 2) /* Preemption details. */ #define SHT_ARM_ATTRIBUTES (SHT_LOPROC + 3) /* ARM attributes section. */ /* AArch64 relocs. */ -#define R_AARCH64_NONE 0 /* No relocation. */ #define R_AARCH64_ABS64 257 /* Direct 64 bit. */ #define R_AARCH64_ABS32 258 /* Direct 32 bit. */ -#define R_AARCH64_ABS16 259 /* Direct 16-bit. */ -#define R_AARCH64_PREL64 260 /* PC-relative 64-bit. */ #define R_AARCH64_PREL32 261 /* PC-relative 32-bit. */ -#define R_AARCH64_PREL16 262 /* PC-relative 16-bit. */ -#define R_AARCH64_MOVW_UABS_G0 263 /* Dir. MOVZ imm. from bits 15:0. */ -#define R_AARCH64_MOVW_UABS_G0_NC 264 /* Likewise for MOVK; no check. */ -#define R_AARCH64_MOVW_UABS_G1 265 /* Dir. MOVZ imm. from bits 31:16. */ -#define R_AARCH64_MOVW_UABS_G1_NC 266 /* Likewise for MOVK; no check. */ -#define R_AARCH64_MOVW_UABS_G2 267 /* Dir. MOVZ imm. from bits 47:32. */ -#define R_AARCH64_MOVW_UABS_G2_NC 268 /* Likewise for MOVK; no check. */ -#define R_AARCH64_MOVW_UABS_G3 269 /* Dir. MOV{K,Z} imm. from 63:48. */ -#define R_AARCH64_MOVW_SABS_G0 270 /* Dir. MOV{N,Z} imm. from 15:0. */ -#define R_AARCH64_MOVW_SABS_G1 271 /* Dir. MOV{N,Z} imm. from 31:16. */ -#define R_AARCH64_MOVW_SABS_G2 272 /* Dir. MOV{N,Z} imm. from 47:32. */ -#define R_AARCH64_LD_PREL_LO19 273 /* PC-rel. LD imm. from bits 20:2. */ -#define R_AARCH64_ADR_PREL_LO21 274 /* PC-rel. ADR imm. from bits 20:0. */ -#define R_AARCH64_ADR_PREL_PG_HI21 275 /* Page-rel. ADRP imm. from 32:12. */ -#define R_AARCH64_ADR_PREL_PG_HI21_NC 276 /* Likewise; no overflow check. */ -#define R_AARCH64_ADD_ABS_LO12_NC 277 /* Dir. ADD imm. from bits 11:0. */ -#define R_AARCH64_LDST8_ABS_LO12_NC 278 /* Likewise for LD/ST; no check. */ -#define R_AARCH64_TSTBR14 279 /* PC-rel. TBZ/TBNZ imm. from 15:2. */ -#define R_AARCH64_CONDBR19 280 /* PC-rel. cond. br. imm. from 20:2. */ -#define R_AARCH64_JUMP26 282 /* PC-rel. B imm. from bits 27:2. */ -#define R_AARCH64_CALL26 283 /* Likewise for CALL. */ -#define R_AARCH64_LDST16_ABS_LO12_NC 284 /* Dir. ADD imm. from bits 11:1. */ -#define R_AARCH64_LDST32_ABS_LO12_NC 285 /* Likewise for bits 11:2. */ -#define R_AARCH64_LDST64_ABS_LO12_NC 286 /* Likewise for bits 11:3. */ -#define R_AARCH64_MOVW_PREL_G0 287 /* PC-rel. MOV{N,Z} imm. from 15:0. */ -#define R_AARCH64_MOVW_PREL_G0_NC 288 /* Likewise for MOVK; no check. */ -#define R_AARCH64_MOVW_PREL_G1 289 /* PC-rel. MOV{N,Z} imm. from 31:16. */ -#define R_AARCH64_MOVW_PREL_G1_NC 290 /* Likewise for MOVK; no check. */ -#define R_AARCH64_MOVW_PREL_G2 291 /* PC-rel. MOV{N,Z} imm. from 47:32. */ -#define R_AARCH64_MOVW_PREL_G2_NC 292 /* Likewise for MOVK; no check. */ -#define R_AARCH64_MOVW_PREL_G3 293 /* PC-rel. MOV{N,Z} imm. from 63:48. */ -#define R_AARCH64_LDST128_ABS_LO12_NC 299 /* Dir. ADD imm. from bits 11:4. */ -#define R_AARCH64_MOVW_GOTOFF_G0 300 /* GOT-rel. off. MOV{N,Z} imm. 15:0. */ -#define R_AARCH64_MOVW_GOTOFF_G0_NC 301 /* Likewise for MOVK; no check. */ -#define R_AARCH64_MOVW_GOTOFF_G1 302 /* GOT-rel. o. MOV{N,Z} imm. 31:16. */ -#define R_AARCH64_MOVW_GOTOFF_G1_NC 303 /* Likewise for MOVK; no check. */ -#define R_AARCH64_MOVW_GOTOFF_G2 304 /* GOT-rel. o. MOV{N,Z} imm. 47:32. */ -#define R_AARCH64_MOVW_GOTOFF_G2_NC 305 /* Likewise for MOVK; no check. */ -#define R_AARCH64_MOVW_GOTOFF_G3 306 /* GOT-rel. o. MOV{N,Z} imm. 63:48. */ -#define R_AARCH64_GOTREL64 307 /* GOT-relative 64-bit. */ -#define R_AARCH64_GOTREL32 308 /* GOT-relative 32-bit. */ -#define R_AARCH64_GOT_LD_PREL19 309 /* PC-rel. GOT off. load imm. 20:2. */ -#define R_AARCH64_LD64_GOTOFF_LO15 310 /* GOT-rel. off. LD/ST imm. 14:3. */ -#define R_AARCH64_ADR_GOT_PAGE 311 /* P-page-rel. GOT off. ADRP 32:12. */ -#define R_AARCH64_LD64_GOT_LO12_NC 312 /* Dir. GOT off. LD/ST imm. 11:3. */ -#define R_AARCH64_LD64_GOTPAGE_LO15 \ - 313 /* GOT-page-rel. GOT off. LD/ST 14:3 \ - */ -#define R_AARCH64_TLSGD_ADR_PREL21 512 /* PC-relative ADR imm. 20:0. */ -#define R_AARCH64_TLSGD_ADR_PAGE21 513 /* page-rel. ADRP imm. 32:12. */ -#define R_AARCH64_TLSGD_ADD_LO12_NC 514 /* direct ADD imm. from 11:0. */ -#define R_AARCH64_TLSGD_MOVW_G1 515 /* GOT-rel. MOV{N,Z} 31:16. */ -#define R_AARCH64_TLSGD_MOVW_G0_NC 516 /* GOT-rel. MOVK imm. 15:0. */ -#define R_AARCH64_TLSLD_ADR_PREL21 517 /* Like 512; local dynamic model. */ -#define R_AARCH64_TLSLD_ADR_PAGE21 518 /* Like 513; local dynamic model. */ -#define R_AARCH64_TLSLD_ADD_LO12_NC 519 /* Like 514; local dynamic model. */ -#define R_AARCH64_TLSLD_MOVW_G1 520 /* Like 515; local dynamic model. */ -#define R_AARCH64_TLSLD_MOVW_G0_NC 521 /* Like 516; local dynamic model. */ -#define R_AARCH64_TLSLD_LD_PREL19 522 /* TLS PC-rel. load imm. 20:2. */ -#define R_AARCH64_TLSLD_MOVW_DTPREL_G2 523 /* TLS DTP-rel. MOV{N,Z} 47:32. */ -#define R_AARCH64_TLSLD_MOVW_DTPREL_G1 524 /* TLS DTP-rel. MOV{N,Z} 31:16. */ -#define R_AARCH64_TLSLD_MOVW_DTPREL_G1_NC 525 /* Likewise; MOVK; no check. */ -#define R_AARCH64_TLSLD_MOVW_DTPREL_G0 526 /* TLS DTP-rel. MOV{N,Z} 15:0. */ -#define R_AARCH64_TLSLD_MOVW_DTPREL_G0_NC 527 /* Likewise; MOVK; no check. */ -#define R_AARCH64_TLSLD_ADD_DTPREL_HI12 \ - 528 /* DTP-rel. ADD imm. from 23:12. \ - */ -#define R_AARCH64_TLSLD_ADD_DTPREL_LO12 529 /* DTP-rel. ADD imm. from 11:0. */ -#define R_AARCH64_TLSLD_ADD_DTPREL_LO12_NC 530 /* Likewise; no ovfl. check. */ -#define R_AARCH64_TLSLD_LDST8_DTPREL_LO12 531 /* DTP-rel. LD/ST imm. 11:0. */ -#define R_AARCH64_TLSLD_LDST8_DTPREL_LO12_NC 532 /* Likewise; no check. */ -#define R_AARCH64_TLSLD_LDST16_DTPREL_LO12 533 /* DTP-rel. LD/ST imm. 11:1. */ -#define R_AARCH64_TLSLD_LDST16_DTPREL_LO12_NC 534 /* Likewise; no check. */ -#define R_AARCH64_TLSLD_LDST32_DTPREL_LO12 535 /* DTP-rel. LD/ST imm. 11:2. */ -#define R_AARCH64_TLSLD_LDST32_DTPREL_LO12_NC 536 /* Likewise; no check. */ -#define R_AARCH64_TLSLD_LDST64_DTPREL_LO12 537 /* DTP-rel. LD/ST imm. 11:3. */ -#define R_AARCH64_TLSLD_LDST64_DTPREL_LO12_NC 538 /* Likewise; no check. */ -#define R_AARCH64_TLSIE_MOVW_GOTTPREL_G1 539 /* GOT-rel. MOV{N,Z} 31:16. */ -#define R_AARCH64_TLSIE_MOVW_GOTTPREL_G0_NC 540 /* GOT-rel. MOVK 15:0. */ -#define R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21 541 /* Page-rel. ADRP 32:12. */ -#define R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC 542 /* Direct LD off. 11:3. */ -#define R_AARCH64_TLSIE_LD_GOTTPREL_PREL19 543 /* PC-rel. load imm. 20:2. */ -#define R_AARCH64_TLSLE_MOVW_TPREL_G2 544 /* TLS TP-rel. MOV{N,Z} 47:32. */ -#define R_AARCH64_TLSLE_MOVW_TPREL_G1 545 /* TLS TP-rel. MOV{N,Z} 31:16. */ -#define R_AARCH64_TLSLE_MOVW_TPREL_G1_NC 546 /* Likewise; MOVK; no check. */ -#define R_AARCH64_TLSLE_MOVW_TPREL_G0 547 /* TLS TP-rel. MOV{N,Z} 15:0. */ -#define R_AARCH64_TLSLE_MOVW_TPREL_G0_NC 548 /* Likewise; MOVK; no check. */ -#define R_AARCH64_TLSLE_ADD_TPREL_HI12 549 /* TP-rel. ADD imm. 23:12. */ -#define R_AARCH64_TLSLE_ADD_TPREL_LO12 550 /* TP-rel. ADD imm. 11:0. */ -#define R_AARCH64_TLSLE_ADD_TPREL_LO12_NC 551 /* Likewise; no ovfl. check. */ -#define R_AARCH64_TLSLE_LDST8_TPREL_LO12 552 /* TP-rel. LD/ST off. 11:0. */ -#define R_AARCH64_TLSLE_LDST8_TPREL_LO12_NC \ - 553 /* Likewise; no ovfl. check. \ - */ -#define R_AARCH64_TLSLE_LDST16_TPREL_LO12 554 /* TP-rel. LD/ST off. 11:1. */ -#define R_AARCH64_TLSLE_LDST16_TPREL_LO12_NC 555 /* Likewise; no check. */ -#define R_AARCH64_TLSLE_LDST32_TPREL_LO12 556 /* TP-rel. LD/ST off. 11:2. */ -#define R_AARCH64_TLSLE_LDST32_TPREL_LO12_NC 557 /* Likewise; no check. */ -#define R_AARCH64_TLSLE_LDST64_TPREL_LO12 558 /* TP-rel. LD/ST off. 11:3. */ -#define R_AARCH64_TLSLE_LDST64_TPREL_LO12_NC 559 /* Likewise; no check. */ -#define R_AARCH64_TLSDESC_LD_PREL19 560 /* PC-rel. load immediate 20:2. */ -#define R_AARCH64_TLSDESC_ADR_PREL21 561 /* PC-rel. ADR immediate 20:0. */ -#define R_AARCH64_TLSDESC_ADR_PAGE21 562 /* Page-rel. ADRP imm. 32:12. */ -#define R_AARCH64_TLSDESC_LD64_LO12 563 /* Direct LD off. from 11:3. */ -#define R_AARCH64_TLSDESC_ADD_LO12 564 /* Direct ADD imm. from 11:0. */ -#define R_AARCH64_TLSDESC_OFF_G1 565 /* GOT-rel. MOV{N,Z} imm. 31:16. */ -#define R_AARCH64_TLSDESC_OFF_G0_NC 566 /* GOT-rel. MOVK imm. 15:0; no ck. */ -#define R_AARCH64_TLSDESC_LDR 567 /* Relax LDR. */ -#define R_AARCH64_TLSDESC_ADD 568 /* Relax ADD. */ -#define R_AARCH64_TLSDESC_CALL 569 /* Relax BLR. */ -#define R_AARCH64_TLSLE_LDST128_TPREL_LO12 570 /* TP-rel. LD/ST off. 11:4. */ -#define R_AARCH64_TLSLE_LDST128_TPREL_LO12_NC 571 /* Likewise; no check. */ -#define R_AARCH64_TLSLD_LDST128_DTPREL_LO12 \ - 572 /* DTP-rel. LD/ST imm. 11:4. \ - */ -#define R_AARCH64_TLSLD_LDST128_DTPREL_LO12_NC 573 /* Likewise; no check. */ -#define R_AARCH64_COPY 1024 /* Copy symbol at runtime. */ -#define R_AARCH64_GLOB_DAT 1025 /* Create GOT entry. */ -#define R_AARCH64_JUMP_SLOT 1026 /* Create PLT entry. */ -#define R_AARCH64_RELATIVE 1027 /* Adjust by program base. */ -#define R_AARCH64_TLS_DTPMOD64 1028 /* Module number, 64 bit. */ -#define R_AARCH64_TLS_DTPREL64 1029 /* Module-relative offset, 64 bit. */ -#define R_AARCH64_TLS_TPREL64 1030 /* TP-relative offset, 64 bit. */ -#define R_AARCH64_TLSDESC 1031 /* TLS Descriptor. */ -#define R_AARCH64_IRELATIVE 1032 /* STT_GNU_IFUNC relocation. */ -/* Keep this the last entry. */ -#define R_AARCH64_NUM 1033 /* ARM relocs. */ @@ -2462,23 +735,8 @@ typedef Elf32_Addr Elf32_Conflict; #define R_ARM_PC24 1 /* PC relative 26 bit branch */ #define R_ARM_ABS32 2 /* Direct 32 bit */ #define R_ARM_REL32 3 /* PC relative 32 bit */ -#define R_ARM_PC13 4 -#define R_ARM_ABS16 5 /* Direct 16 bit */ -#define R_ARM_ABS12 6 /* Direct 12 bit */ -#define R_ARM_THM_ABS5 7 -#define R_ARM_ABS8 8 /* Direct 8 bit */ -#define R_ARM_SBREL32 9 #define R_ARM_THM_PC22 10 #define R_ARM_THM_PC8 11 -#define R_ARM_AMP_VCALL9 12 -#define R_ARM_SWI24 13 /* Obsolete static relocation. */ -#define R_ARM_TLS_DESC 13 /* Dynamic relocation. */ -#define R_ARM_THM_SWI8 14 -#define R_ARM_XPC25 15 -#define R_ARM_THM_XPC22 16 -#define R_ARM_TLS_DTPMOD32 17 /* ID of module containing symbol */ -#define R_ARM_TLS_DTPOFF32 18 /* Offset in TLS block */ -#define R_ARM_TLS_TPOFF32 19 /* Offset in static TLS block */ #define R_ARM_COPY 20 /* Copy symbol at runtime */ #define R_ARM_GLOB_DAT 21 /* Create GOT entry */ #define R_ARM_JUMP_SLOT 22 /* Create PLT entry */ @@ -2490,15 +748,7 @@ typedef Elf32_Addr Elf32_Conflict; #define R_ARM_CALL 28 #define R_ARM_JUMP24 29 #define R_ARM_THM_JUMP24 30 -#define R_ARM_BASE_ABS 31 /* Adjust by program base. */ -#define R_ARM_ALU_PCREL_7_0 32 -#define R_ARM_ALU_PCREL_15_8 33 -#define R_ARM_ALU_PCREL_23_15 34 -#define R_ARM_LDR_SBREL_11_0 35 -#define R_ARM_ALU_SBREL_19_12 36 -#define R_ARM_ALU_SBREL_27_20 37 #define R_ARM_TARGET1 38 -#define R_ARM_SBREL31 39 /* Program base relative. */ #define R_ARM_V4BX 40 #define R_ARM_TARGET2 41 #define R_ARM_PREL31 42 @@ -2513,31 +763,7 @@ typedef Elf32_Addr Elf32_Conflict; #define R_ARM_THM_ALU_PREL_11_0 53 #define R_ARM_THM_PC12 54 /* Values from 49 to 89 are not yet used/handled by tcc. */ -#define R_ARM_TLS_GOTDESC 90 -#define R_ARM_TLS_CALL 91 -#define R_ARM_TLS_DESCSEQ 92 -#define R_ARM_THM_TLS_CALL 93 #define R_ARM_GOT_PREL 96 -#define R_ARM_GNU_VTENTRY 100 -#define R_ARM_GNU_VTINHERIT 101 -#define R_ARM_THM_PC11 102 /* thumb unconditional branch */ -#define R_ARM_THM_PC9 103 /* thumb conditional branch */ -#define R_ARM_TLS_GD32 \ - 104 /* PC-rel 32 bit for global dynamic \ - thread local data */ -#define R_ARM_TLS_LDM32 \ - 105 /* PC-rel 32 bit for local dynamic \ - thread local data */ -#define R_ARM_TLS_LDO32 \ - 106 /* 32 bit offset relative to TLS \ - block */ -#define R_ARM_TLS_IE32 \ - 107 /* PC-rel 32 bit for GOT entry of \ - static TLS block offset */ -#define R_ARM_TLS_LE32 \ - 108 /* 32 bit offset relative to static \ - TLS block */ -#define R_ARM_THM_TLS_DESCSEQ 129 /* YASOS RELRO: 32-bit offset of a symbol within .rodata (S - rodata base). * Emitted for references to shared (pure-const) .rodata symbols; the runtime * address is anchor(rodata base from a fixed GOT slot) + this offset. Resolved @@ -2549,405 +775,26 @@ typedef Elf32_Addr Elf32_Conflict; * Caps the GOT at 4096 bytes; -mno-sb-relative-got selects the literal path. * See docs/sb_relative_got.md. Uses a free value in the 130-159 ABI gap. */ #define R_ARM_GOT_SBREL12 138 -#define R_ARM_IRELATIVE 160 -#define R_ARM_RXPC25 249 -#define R_ARM_RSBREL32 250 -#define R_ARM_THM_RPC22 251 -#define R_ARM_RREL32 252 -#define R_ARM_RABS22 253 -#define R_ARM_RPC24 254 -#define R_ARM_RBASE 255 +/* YASOS module-local call marker: sits on a BL/B.W (next to its + * R_ARM_THM_JUMP24) whose R9 reload the compiler dropped because it assumed + * the callee binds inside this module (-fmodule-local-calls). Patches nothing; + * the linker errors if the call binds to the PLT, since a cross-module callee + * returns with its own R9 and nothing would restore ours. Never reaches the + * YAFF writer. Uses a free value in the 130-159 ABI gap. */ +#define R_ARM_YASOS_LOCAL_CALL 139 /* Keep this the last entry. */ #define R_ARM_NUM 256 -/* TMS320C67xx specific declarations */ - -/* XXX: no ELF standard yet*/ - -/* TMS320C67xx relocs. */ -#define R_C60_32 1 -#define R_C60_GOT32 3 /* 32 bit GOT entry */ -#define R_C60_PLT32 4 /* 32 bit PLT address */ -#define R_C60_COPY 5 /* Copy symbol at runtime */ -#define R_C60_GLOB_DAT 6 /* Create GOT entry */ -#define R_C60_JMP_SLOT 7 /* Create PLT entry */ -#define R_C60_RELATIVE 8 /* Adjust by program base */ -#define R_C60_GOTOFF 9 /* 32 bit offset to GOT */ -#define R_C60_GOTPC 10 /* 32 bit PC relative offset to GOT */ - -#define R_C60LO16 0x54 /* low 16 bit MVKL embedded */ -#define R_C60HI16 0x55 /* high 16 bit MVKH embedded */ -/* Keep this the last entry. */ -#define R_C60_NUM 0x56 - -/* IA-64 specific declarations. */ - -/* Processor specific flags for the Ehdr e_flags field. */ -#define EF_IA_64_MASKOS 0x0000000f /* os-specific flags */ -#define EF_IA_64_ABI64 0x00000010 /* 64-bit ABI */ -#define EF_IA_64_ARCH 0xff000000 /* arch. version mask */ - -/* Processor specific values for the Phdr p_type field. */ -#define PT_IA_64_ARCHEXT (PT_LOPROC + 0) /* arch extension bits */ -#define PT_IA_64_UNWIND (PT_LOPROC + 1) /* ia64 unwind bits */ -#define PT_IA_64_HP_OPT_ANOT (PT_LOOS + 0x12) -#define PT_IA_64_HP_HSL_ANOT (PT_LOOS + 0x13) -#define PT_IA_64_HP_STACK (PT_LOOS + 0x14) - -/* Processor specific flags for the Phdr p_flags field. */ -#define PF_IA_64_NORECOV 0x80000000 /* spec insns w/o recovery */ - -/* Processor specific values for the Shdr sh_type field. */ -#define SHT_IA_64_EXT (SHT_LOPROC + 0) /* extension bits */ -#define SHT_IA_64_UNWIND (SHT_LOPROC + 1) /* unwind bits */ - -/* Processor specific flags for the Shdr sh_flags field. */ -#define SHF_IA_64_SHORT 0x10000000 /* section near gp */ -#define SHF_IA_64_NORECOV 0x20000000 /* spec insns w/o recovery */ - -/* Processor specific values for the Dyn d_tag field. */ -#define DT_IA_64_PLT_RESERVE (DT_LOPROC + 0) -#define DT_IA_64_NUM 1 - -/* IA-64 relocations. */ -#define R_IA64_NONE 0x00 /* none */ -#define R_IA64_IMM14 0x21 /* symbol + addend, add imm14 */ -#define R_IA64_IMM22 0x22 /* symbol + addend, add imm22 */ -#define R_IA64_IMM64 0x23 /* symbol + addend, mov imm64 */ -#define R_IA64_DIR32MSB 0x24 /* symbol + addend, data4 MSB */ -#define R_IA64_DIR32LSB 0x25 /* symbol + addend, data4 LSB */ -#define R_IA64_DIR64MSB 0x26 /* symbol + addend, data8 MSB */ -#define R_IA64_DIR64LSB 0x27 /* symbol + addend, data8 LSB */ -#define R_IA64_GPREL22 0x2a /* @gprel(sym + add), add imm22 */ -#define R_IA64_GPREL64I 0x2b /* @gprel(sym + add), mov imm64 */ -#define R_IA64_GPREL32MSB 0x2c /* @gprel(sym + add), data4 MSB */ -#define R_IA64_GPREL32LSB 0x2d /* @gprel(sym + add), data4 LSB */ -#define R_IA64_GPREL64MSB 0x2e /* @gprel(sym + add), data8 MSB */ -#define R_IA64_GPREL64LSB 0x2f /* @gprel(sym + add), data8 LSB */ -#define R_IA64_LTOFF22 0x32 /* @ltoff(sym + add), add imm22 */ -#define R_IA64_LTOFF64I 0x33 /* @ltoff(sym + add), mov imm64 */ -#define R_IA64_PLTOFF22 0x3a /* @pltoff(sym + add), add imm22 */ -#define R_IA64_PLTOFF64I 0x3b /* @pltoff(sym + add), mov imm64 */ -#define R_IA64_PLTOFF64MSB 0x3e /* @pltoff(sym + add), data8 MSB */ -#define R_IA64_PLTOFF64LSB 0x3f /* @pltoff(sym + add), data8 LSB */ -#define R_IA64_FPTR64I 0x43 /* @fptr(sym + add), mov imm64 */ -#define R_IA64_FPTR32MSB 0x44 /* @fptr(sym + add), data4 MSB */ -#define R_IA64_FPTR32LSB 0x45 /* @fptr(sym + add), data4 LSB */ -#define R_IA64_FPTR64MSB 0x46 /* @fptr(sym + add), data8 MSB */ -#define R_IA64_FPTR64LSB 0x47 /* @fptr(sym + add), data8 LSB */ -#define R_IA64_PCREL60B 0x48 /* @pcrel(sym + add), brl */ -#define R_IA64_PCREL21B 0x49 /* @pcrel(sym + add), ptb, call */ -#define R_IA64_PCREL21M 0x4a /* @pcrel(sym + add), chk.s */ -#define R_IA64_PCREL21F 0x4b /* @pcrel(sym + add), fchkf */ -#define R_IA64_PCREL32MSB 0x4c /* @pcrel(sym + add), data4 MSB */ -#define R_IA64_PCREL32LSB 0x4d /* @pcrel(sym + add), data4 LSB */ -#define R_IA64_PCREL64MSB 0x4e /* @pcrel(sym + add), data8 MSB */ -#define R_IA64_PCREL64LSB 0x4f /* @pcrel(sym + add), data8 LSB */ -#define R_IA64_LTOFF_FPTR22 0x52 /* @ltoff(@fptr(s+a)), imm22 */ -#define R_IA64_LTOFF_FPTR64I 0x53 /* @ltoff(@fptr(s+a)), imm64 */ -#define R_IA64_LTOFF_FPTR32MSB 0x54 /* @ltoff(@fptr(s+a)), data4 MSB */ -#define R_IA64_LTOFF_FPTR32LSB 0x55 /* @ltoff(@fptr(s+a)), data4 LSB */ -#define R_IA64_LTOFF_FPTR64MSB 0x56 /* @ltoff(@fptr(s+a)), data8 MSB */ -#define R_IA64_LTOFF_FPTR64LSB 0x57 /* @ltoff(@fptr(s+a)), data8 LSB */ -#define R_IA64_SEGREL32MSB 0x5c /* @segrel(sym + add), data4 MSB */ -#define R_IA64_SEGREL32LSB 0x5d /* @segrel(sym + add), data4 LSB */ -#define R_IA64_SEGREL64MSB 0x5e /* @segrel(sym + add), data8 MSB */ -#define R_IA64_SEGREL64LSB 0x5f /* @segrel(sym + add), data8 LSB */ -#define R_IA64_SECREL32MSB 0x64 /* @secrel(sym + add), data4 MSB */ -#define R_IA64_SECREL32LSB 0x65 /* @secrel(sym + add), data4 LSB */ -#define R_IA64_SECREL64MSB 0x66 /* @secrel(sym + add), data8 MSB */ -#define R_IA64_SECREL64LSB 0x67 /* @secrel(sym + add), data8 LSB */ -#define R_IA64_REL32MSB 0x6c /* data 4 + REL */ -#define R_IA64_REL32LSB 0x6d /* data 4 + REL */ -#define R_IA64_REL64MSB 0x6e /* data 8 + REL */ -#define R_IA64_REL64LSB 0x6f /* data 8 + REL */ -#define R_IA64_LTV32MSB 0x74 /* symbol + addend, data4 MSB */ -#define R_IA64_LTV32LSB 0x75 /* symbol + addend, data4 LSB */ -#define R_IA64_LTV64MSB 0x76 /* symbol + addend, data8 MSB */ -#define R_IA64_LTV64LSB 0x77 /* symbol + addend, data8 LSB */ -#define R_IA64_PCREL21BI 0x79 /* @pcrel(sym + add), 21bit inst */ -#define R_IA64_PCREL22 0x7a /* @pcrel(sym + add), 22bit inst */ -#define R_IA64_PCREL64I 0x7b /* @pcrel(sym + add), 64bit inst */ -#define R_IA64_IPLTMSB 0x80 /* dynamic reloc, imported PLT, MSB */ -#define R_IA64_IPLTLSB 0x81 /* dynamic reloc, imported PLT, LSB */ -#define R_IA64_COPY 0x84 /* copy relocation */ -#define R_IA64_SUB 0x85 /* Addend and symbol difference */ -#define R_IA64_LTOFF22X 0x86 /* LTOFF22, relaxable. */ -#define R_IA64_LDXMOV 0x87 /* Use of LTOFF22X. */ -#define R_IA64_TPREL14 0x91 /* @tprel(sym + add), imm14 */ -#define R_IA64_TPREL22 0x92 /* @tprel(sym + add), imm22 */ -#define R_IA64_TPREL64I 0x93 /* @tprel(sym + add), imm64 */ -#define R_IA64_TPREL64MSB 0x96 /* @tprel(sym + add), data8 MSB */ -#define R_IA64_TPREL64LSB 0x97 /* @tprel(sym + add), data8 LSB */ -#define R_IA64_LTOFF_TPREL22 0x9a /* @ltoff(@tprel(s+a)), imm2 */ -#define R_IA64_DTPMOD64MSB 0xa6 /* @dtpmod(sym + add), data8 MSB */ -#define R_IA64_DTPMOD64LSB 0xa7 /* @dtpmod(sym + add), data8 LSB */ -#define R_IA64_LTOFF_DTPMOD22 0xaa /* @ltoff(@dtpmod(sym + add)), imm22 */ -#define R_IA64_DTPREL14 0xb1 /* @dtprel(sym + add), imm14 */ -#define R_IA64_DTPREL22 0xb2 /* @dtprel(sym + add), imm22 */ -#define R_IA64_DTPREL64I 0xb3 /* @dtprel(sym + add), imm64 */ -#define R_IA64_DTPREL32MSB 0xb4 /* @dtprel(sym + add), data4 MSB */ -#define R_IA64_DTPREL32LSB 0xb5 /* @dtprel(sym + add), data4 LSB */ -#define R_IA64_DTPREL64MSB 0xb6 /* @dtprel(sym + add), data8 MSB */ -#define R_IA64_DTPREL64LSB 0xb7 /* @dtprel(sym + add), data8 LSB */ -#define R_IA64_LTOFF_DTPREL22 0xba /* @ltoff(@dtprel(s+a)), imm22 */ - -/* SH specific declarations */ - -/* Processor specific flags for the ELF header e_flags field. */ -#define EF_SH_MACH_MASK 0x1f -#define EF_SH_UNKNOWN 0x0 -#define EF_SH1 0x1 -#define EF_SH2 0x2 -#define EF_SH3 0x3 -#define EF_SH_DSP 0x4 -#define EF_SH3_DSP 0x5 -#define EF_SH4AL_DSP 0x6 -#define EF_SH3E 0x8 -#define EF_SH4 0x9 -#define EF_SH2E 0xb -#define EF_SH4A 0xc -#define EF_SH2A 0xd -#define EF_SH4_NOFPU 0x10 -#define EF_SH4A_NOFPU 0x11 -#define EF_SH4_NOMMU_NOFPU 0x12 -#define EF_SH2A_NOFPU 0x13 -#define EF_SH3_NOMMU 0x14 -#define EF_SH2A_SH4_NOFPU 0x15 -#define EF_SH2A_SH3_NOFPU 0x16 -#define EF_SH2A_SH4 0x17 -#define EF_SH2A_SH3E 0x18 - -/* SH relocs. */ -#define R_SH_NONE 0 -#define R_SH_DIR32 1 -#define R_SH_REL32 2 -#define R_SH_DIR8WPN 3 -#define R_SH_IND12W 4 -#define R_SH_DIR8WPL 5 -#define R_SH_DIR8WPZ 6 -#define R_SH_DIR8BP 7 -#define R_SH_DIR8W 8 -#define R_SH_DIR8L 9 -#define R_SH_SWITCH16 25 -#define R_SH_SWITCH32 26 -#define R_SH_USES 27 -#define R_SH_COUNT 28 -#define R_SH_ALIGN 29 -#define R_SH_CODE 30 -#define R_SH_DATA 31 -#define R_SH_LABEL 32 -#define R_SH_SWITCH8 33 -#define R_SH_GNU_VTINHERIT 34 -#define R_SH_GNU_VTENTRY 35 -#define R_SH_TLS_GD_32 144 -#define R_SH_TLS_LD_32 145 -#define R_SH_TLS_LDO_32 146 -#define R_SH_TLS_IE_32 147 -#define R_SH_TLS_LE_32 148 -#define R_SH_TLS_DTPMOD32 149 -#define R_SH_TLS_DTPOFF32 150 -#define R_SH_TLS_TPOFF32 151 -#define R_SH_GOT32 160 -#define R_SH_PLT32 161 -#define R_SH_COPY 162 -#define R_SH_GLOB_DAT 163 -#define R_SH_JMP_SLOT 164 -#define R_SH_RELATIVE 165 -#define R_SH_GOTOFF 166 -#define R_SH_GOTPC 167 -/* Keep this the last entry. */ -#define R_SH_NUM 256 - -/* S/390 specific definitions. */ - -/* Valid values for the e_flags field. */ - -#define EF_S390_HIGH_GPRS 0x00000001 /* High GPRs kernel facility needed. */ - -/* Additional s390 relocs */ - -#define R_390_NONE 0 /* No reloc. */ -#define R_390_8 1 /* Direct 8 bit. */ -#define R_390_12 2 /* Direct 12 bit. */ -#define R_390_16 3 /* Direct 16 bit. */ -#define R_390_32 4 /* Direct 32 bit. */ -#define R_390_PC32 5 /* PC relative 32 bit. */ -#define R_390_GOT12 6 /* 12 bit GOT offset. */ -#define R_390_GOT32 7 /* 32 bit GOT offset. */ -#define R_390_PLT32 8 /* 32 bit PC relative PLT address. */ -#define R_390_COPY 9 /* Copy symbol at runtime. */ -#define R_390_GLOB_DAT 10 /* Create GOT entry. */ -#define R_390_JMP_SLOT 11 /* Create PLT entry. */ -#define R_390_RELATIVE 12 /* Adjust by program base. */ -#define R_390_GOTOFF32 13 /* 32 bit offset to GOT. */ -#define R_390_GOTPC 14 /* 32 bit PC relative offset to GOT. */ -#define R_390_GOT16 15 /* 16 bit GOT offset. */ -#define R_390_PC16 16 /* PC relative 16 bit. */ -#define R_390_PC16DBL 17 /* PC relative 16 bit shifted by 1. */ -#define R_390_PLT16DBL 18 /* 16 bit PC rel. PLT shifted by 1. */ -#define R_390_PC32DBL 19 /* PC relative 32 bit shifted by 1. */ -#define R_390_PLT32DBL 20 /* 32 bit PC rel. PLT shifted by 1. */ -#define R_390_GOTPCDBL 21 /* 32 bit PC rel. GOT shifted by 1. */ -#define R_390_64 22 /* Direct 64 bit. */ -#define R_390_PC64 23 /* PC relative 64 bit. */ -#define R_390_GOT64 24 /* 64 bit GOT offset. */ -#define R_390_PLT64 25 /* 64 bit PC relative PLT address. */ -#define R_390_GOTENT 26 /* 32 bit PC rel. to GOT entry >> 1. */ -#define R_390_GOTOFF16 27 /* 16 bit offset to GOT. */ -#define R_390_GOTOFF64 28 /* 64 bit offset to GOT. */ -#define R_390_GOTPLT12 29 /* 12 bit offset to jump slot. */ -#define R_390_GOTPLT16 30 /* 16 bit offset to jump slot. */ -#define R_390_GOTPLT32 31 /* 32 bit offset to jump slot. */ -#define R_390_GOTPLT64 32 /* 64 bit offset to jump slot. */ -#define R_390_GOTPLTENT 33 /* 32 bit rel. offset to jump slot. */ -#define R_390_PLTOFF16 34 /* 16 bit offset from GOT to PLT. */ -#define R_390_PLTOFF32 35 /* 32 bit offset from GOT to PLT. */ -#define R_390_PLTOFF64 36 /* 16 bit offset from GOT to PLT. */ -#define R_390_TLS_LOAD 37 /* Tag for load insn in TLS code. */ -#define R_390_TLS_GDCALL \ - 38 /* Tag for function call in general \ - dynamic TLS code. */ -#define R_390_TLS_LDCALL \ - 39 /* Tag for function call in local \ - dynamic TLS code. */ -#define R_390_TLS_GD32 \ - 40 /* Direct 32 bit for general dynamic \ - thread local data. */ -#define R_390_TLS_GD64 \ - 41 /* Direct 64 bit for general dynamic \ - thread local data. */ -#define R_390_TLS_GOTIE12 \ - 42 /* 12 bit GOT offset for static TLS \ - block offset. */ -#define R_390_TLS_GOTIE32 \ - 43 /* 32 bit GOT offset for static TLS \ - block offset. */ -#define R_390_TLS_GOTIE64 \ - 44 /* 64 bit GOT offset for static TLS \ - block offset. */ -#define R_390_TLS_LDM32 \ - 45 /* Direct 32 bit for local dynamic \ - thread local data in LE code. */ -#define R_390_TLS_LDM64 \ - 46 /* Direct 64 bit for local dynamic \ - thread local data in LE code. */ -#define R_390_TLS_IE32 \ - 47 /* 32 bit address of GOT entry for \ - negated static TLS block offset. */ -#define R_390_TLS_IE64 \ - 48 /* 64 bit address of GOT entry for \ - negated static TLS block offset. */ -#define R_390_TLS_IEENT \ - 49 /* 32 bit rel. offset to GOT entry for \ - negated static TLS block offset. */ -#define R_390_TLS_LE32 \ - 50 /* 32 bit negated offset relative to \ - static TLS block. */ -#define R_390_TLS_LE64 \ - 51 /* 64 bit negated offset relative to \ - static TLS block. */ -#define R_390_TLS_LDO32 \ - 52 /* 32 bit offset relative to TLS \ - block. */ -#define R_390_TLS_LDO64 \ - 53 /* 64 bit offset relative to TLS \ - block. */ -#define R_390_TLS_DTPMOD 54 /* ID of module containing symbol. */ -#define R_390_TLS_DTPOFF 55 /* Offset in TLS block. */ -#define R_390_TLS_TPOFF \ - 56 /* Negated offset in static TLS \ - block. */ -#define R_390_20 57 /* Direct 20 bit. */ -#define R_390_GOT20 58 /* 20 bit GOT offset. */ -#define R_390_GOTPLT20 59 /* 20 bit offset to jump slot. */ -#define R_390_TLS_GOTIE20 \ - 60 /* 20 bit GOT offset for static TLS \ - block offset. */ -#define R_390_IRELATIVE 61 /* STT_GNU_IFUNC relocation. */ -/* Keep this the last entry. */ -#define R_390_NUM 62 - -/* CRIS relocations. */ -#define R_CRIS_NONE 0 -#define R_CRIS_8 1 -#define R_CRIS_16 2 -#define R_CRIS_32 3 -#define R_CRIS_8_PCREL 4 -#define R_CRIS_16_PCREL 5 -#define R_CRIS_32_PCREL 6 -#define R_CRIS_GNU_VTINHERIT 7 -#define R_CRIS_GNU_VTENTRY 8 -#define R_CRIS_COPY 9 -#define R_CRIS_GLOB_DAT 10 -#define R_CRIS_JUMP_SLOT 11 -#define R_CRIS_RELATIVE 12 -#define R_CRIS_16_GOT 13 -#define R_CRIS_32_GOT 14 -#define R_CRIS_16_GOTPLT 15 -#define R_CRIS_32_GOTPLT 16 -#define R_CRIS_32_GOTREL 17 -#define R_CRIS_32_PLT_GOTREL 18 -#define R_CRIS_32_PLT_PCREL 19 - -#define R_CRIS_NUM 20 - /* AMD x86-64 relocations. */ -#define R_X86_64_NONE 0 /* No reloc */ #define R_X86_64_64 1 /* Direct 64 bit */ #define R_X86_64_PC32 2 /* PC relative 32 bit signed */ #define R_X86_64_GOT32 3 /* 32 bit GOT entry */ #define R_X86_64_PLT32 4 /* 32 bit PLT address */ -#define R_X86_64_COPY 5 /* Copy symbol at runtime */ -#define R_X86_64_GLOB_DAT 6 /* Create GOT entry */ -#define R_X86_64_JUMP_SLOT 7 /* Create PLT entry */ -#define R_X86_64_RELATIVE 8 /* Adjust by program base */ #define R_X86_64_GOTPCREL \ 9 /* 32 bit signed PC relative \ offset to GOT */ #define R_X86_64_32 10 /* Direct 32 bit zero extended */ #define R_X86_64_32S 11 /* Direct 32 bit sign extended */ -#define R_X86_64_16 12 /* Direct 16 bit zero extended */ -#define R_X86_64_PC16 13 /* 16 bit sign extended pc relative */ -#define R_X86_64_8 14 /* Direct 8 bit sign extended */ -#define R_X86_64_PC8 15 /* 8 bit sign extended pc relative */ -#define R_X86_64_DTPMOD64 16 /* ID of module containing symbol */ -#define R_X86_64_DTPOFF64 17 /* Offset in module's TLS block */ -#define R_X86_64_TPOFF64 18 /* Offset in initial TLS block */ -#define R_X86_64_TLSGD \ - 19 /* 32 bit signed PC relative offset \ - to two GOT entries for GD symbol */ -#define R_X86_64_TLSLD \ - 20 /* 32 bit signed PC relative offset \ - to two GOT entries for LD symbol */ -#define R_X86_64_DTPOFF32 21 /* Offset in TLS block */ -#define R_X86_64_GOTTPOFF \ - 22 /* 32 bit signed PC relative offset \ - to GOT entry for IE symbol */ -#define R_X86_64_TPOFF32 23 /* Offset in initial TLS block */ -#define R_X86_64_PC64 24 /* PC relative 64 bit */ -#define R_X86_64_GOTOFF64 25 /* 64 bit offset to GOT */ -#define R_X86_64_GOTPC32 \ - 26 /* 32 bit signed pc relative \ - offset to GOT */ -#define R_X86_64_GOT64 27 /* 64-bit GOT entry offset */ -#define R_X86_64_GOTPCREL64 \ - 28 /* 64-bit PC relative offset \ - to GOT entry */ -#define R_X86_64_GOTPC64 29 /* 64-bit PC relative offset to GOT */ -#define R_X86_64_GOTPLT64 30 /* like GOT64, says PLT entry needed */ -#define R_X86_64_PLTOFF64 \ - 31 /* 64-bit GOT relative offset \ - to PLT entry */ -#define R_X86_64_SIZE32 32 /* Size of symbol plus 32-bit addend */ -#define R_X86_64_SIZE64 33 /* Size of symbol plus 64-bit addend */ -#define R_X86_64_GOTPC32_TLSDESC 34 /* GOT offset for TLS descriptor. */ -#define R_X86_64_TLSDESC_CALL \ - 35 /* Marker for call through TLS \ - descriptor. */ -#define R_X86_64_TLSDESC 36 /* TLS descriptor. */ -#define R_X86_64_IRELATIVE 37 /* Adjust indirectly by program base */ -#define R_X86_64_RELATIVE64 38 /* 64-bit adjust by program base */ #define R_X86_64_GOTPCRELX \ 41 /* like GOTPCREL, but optionally with \ linker optimizations */ @@ -2955,455 +802,17 @@ typedef Elf32_Addr Elf32_Conflict; 42 /* like GOTPCRELX, but a REX prefix \ is present */ -#define R_X86_64_NUM 43 - /* x86-64 sh_type values. */ #define SHT_X86_64_UNWIND 0x70000001 /* Unwind information. */ -/* AM33 relocations. */ -#define R_MN10300_NONE 0 /* No reloc. */ -#define R_MN10300_32 1 /* Direct 32 bit. */ -#define R_MN10300_16 2 /* Direct 16 bit. */ -#define R_MN10300_8 3 /* Direct 8 bit. */ -#define R_MN10300_PCREL32 4 /* PC-relative 32-bit. */ -#define R_MN10300_PCREL16 5 /* PC-relative 16-bit signed. */ -#define R_MN10300_PCREL8 6 /* PC-relative 8-bit signed. */ -#define R_MN10300_GNU_VTINHERIT 7 /* Ancient C++ vtable garbage... */ -#define R_MN10300_GNU_VTENTRY 8 /* ... collection annotation. */ -#define R_MN10300_24 9 /* Direct 24 bit. */ -#define R_MN10300_GOTPC32 10 /* 32-bit PCrel offset to GOT. */ -#define R_MN10300_GOTPC16 11 /* 16-bit PCrel offset to GOT. */ -#define R_MN10300_GOTOFF32 12 /* 32-bit offset from GOT. */ -#define R_MN10300_GOTOFF24 13 /* 24-bit offset from GOT. */ -#define R_MN10300_GOTOFF16 14 /* 16-bit offset from GOT. */ -#define R_MN10300_PLT32 15 /* 32-bit PCrel to PLT entry. */ -#define R_MN10300_PLT16 16 /* 16-bit PCrel to PLT entry. */ -#define R_MN10300_GOT32 17 /* 32-bit offset to GOT entry. */ -#define R_MN10300_GOT24 18 /* 24-bit offset to GOT entry. */ -#define R_MN10300_GOT16 19 /* 16-bit offset to GOT entry. */ -#define R_MN10300_COPY 20 /* Copy symbol at runtime. */ -#define R_MN10300_GLOB_DAT 21 /* Create GOT entry. */ -#define R_MN10300_JMP_SLOT 22 /* Create PLT entry. */ -#define R_MN10300_RELATIVE 23 /* Adjust by program base. */ -#define R_MN10300_TLS_GD 24 /* 32-bit offset for global dynamic. */ -#define R_MN10300_TLS_LD 25 /* 32-bit offset for local dynamic. */ -#define R_MN10300_TLS_LDO 26 /* Module-relative offset. */ -#define R_MN10300_TLS_GOTIE \ - 27 /* GOT offset for static TLS block \ - offset. */ -#define R_MN10300_TLS_IE \ - 28 /* GOT address for static TLS block \ - offset. */ -#define R_MN10300_TLS_LE \ - 29 /* Offset relative to static TLS \ - block. */ -#define R_MN10300_TLS_DTPMOD 30 /* ID of module containing symbol. */ -#define R_MN10300_TLS_DTPOFF 31 /* Offset in module TLS block. */ -#define R_MN10300_TLS_TPOFF 32 /* Offset in static TLS block. */ -#define R_MN10300_SYM_DIFF \ - 33 /* Adjustment for next reloc as needed \ - by linker relaxation. */ -#define R_MN10300_ALIGN \ - 34 /* Alignment requirement for linker \ - relaxation. */ -#define R_MN10300_NUM 35 - -/* M32R relocs. */ -#define R_M32R_NONE 0 /* No reloc. */ -#define R_M32R_16 1 /* Direct 16 bit. */ -#define R_M32R_32 2 /* Direct 32 bit. */ -#define R_M32R_24 3 /* Direct 24 bit. */ -#define R_M32R_10_PCREL 4 /* PC relative 10 bit shifted. */ -#define R_M32R_18_PCREL 5 /* PC relative 18 bit shifted. */ -#define R_M32R_26_PCREL 6 /* PC relative 26 bit shifted. */ -#define R_M32R_HI16_ULO 7 /* High 16 bit with unsigned low. */ -#define R_M32R_HI16_SLO 8 /* High 16 bit with signed low. */ -#define R_M32R_LO16 9 /* Low 16 bit. */ -#define R_M32R_SDA16 10 /* 16 bit offset in SDA. */ -#define R_M32R_GNU_VTINHERIT 11 -#define R_M32R_GNU_VTENTRY 12 -/* M32R relocs use SHT_RELA. */ -#define R_M32R_16_RELA 33 /* Direct 16 bit. */ -#define R_M32R_32_RELA 34 /* Direct 32 bit. */ -#define R_M32R_24_RELA 35 /* Direct 24 bit. */ -#define R_M32R_10_PCREL_RELA 36 /* PC relative 10 bit shifted. */ -#define R_M32R_18_PCREL_RELA 37 /* PC relative 18 bit shifted. */ -#define R_M32R_26_PCREL_RELA 38 /* PC relative 26 bit shifted. */ -#define R_M32R_HI16_ULO_RELA 39 /* High 16 bit with unsigned low */ -#define R_M32R_HI16_SLO_RELA 40 /* High 16 bit with signed low */ -#define R_M32R_LO16_RELA 41 /* Low 16 bit */ -#define R_M32R_SDA16_RELA 42 /* 16 bit offset in SDA */ -#define R_M32R_RELA_GNU_VTINHERIT 43 -#define R_M32R_RELA_GNU_VTENTRY 44 -#define R_M32R_REL32 45 /* PC relative 32 bit. */ - -#define R_M32R_GOT24 48 /* 24 bit GOT entry */ -#define R_M32R_26_PLTREL 49 /* 26 bit PC relative to PLT shifted */ -#define R_M32R_COPY 50 /* Copy symbol at runtime */ -#define R_M32R_GLOB_DAT 51 /* Create GOT entry */ -#define R_M32R_JMP_SLOT 52 /* Create PLT entry */ -#define R_M32R_RELATIVE 53 /* Adjust by program base */ -#define R_M32R_GOTOFF 54 /* 24 bit offset to GOT */ -#define R_M32R_GOTPC24 55 /* 24 bit PC relative offset to GOT */ -#define R_M32R_GOT16_HI_ULO \ - 56 /* High 16 bit GOT entry with unsigned \ - low */ -#define R_M32R_GOT16_HI_SLO \ - 57 /* High 16 bit GOT entry with signed \ - low */ -#define R_M32R_GOT16_LO 58 /* Low 16 bit GOT entry */ -#define R_M32R_GOTPC_HI_ULO \ - 59 /* High 16 bit PC relative offset to \ - GOT with unsigned low */ -#define R_M32R_GOTPC_HI_SLO \ - 60 /* High 16 bit PC relative offset to \ - GOT with signed low */ -#define R_M32R_GOTPC_LO \ - 61 /* Low 16 bit PC relative offset to \ - GOT */ -#define R_M32R_GOTOFF_HI_ULO \ - 62 /* High 16 bit offset to GOT \ - with unsigned low */ -#define R_M32R_GOTOFF_HI_SLO \ - 63 /* High 16 bit offset to GOT \ - with signed low */ -#define R_M32R_GOTOFF_LO 64 /* Low 16 bit offset to GOT */ -#define R_M32R_NUM 256 /* Keep this the last entry. */ - -/* TILEPro relocations. */ -#define R_TILEPRO_NONE 0 /* No reloc */ -#define R_TILEPRO_32 1 /* Direct 32 bit */ -#define R_TILEPRO_16 2 /* Direct 16 bit */ -#define R_TILEPRO_8 3 /* Direct 8 bit */ -#define R_TILEPRO_32_PCREL 4 /* PC relative 32 bit */ -#define R_TILEPRO_16_PCREL 5 /* PC relative 16 bit */ -#define R_TILEPRO_8_PCREL 6 /* PC relative 8 bit */ -#define R_TILEPRO_LO16 7 /* Low 16 bit */ -#define R_TILEPRO_HI16 8 /* High 16 bit */ -#define R_TILEPRO_HA16 9 /* High 16 bit, adjusted */ -#define R_TILEPRO_COPY 10 /* Copy relocation */ -#define R_TILEPRO_GLOB_DAT 11 /* Create GOT entry */ -#define R_TILEPRO_JMP_SLOT 12 /* Create PLT entry */ -#define R_TILEPRO_RELATIVE 13 /* Adjust by program base */ -#define R_TILEPRO_BROFF_X1 14 /* X1 pipe branch offset */ -#define R_TILEPRO_JOFFLONG_X1 15 /* X1 pipe jump offset */ -#define R_TILEPRO_JOFFLONG_X1_PLT 16 /* X1 pipe jump offset to PLT */ -#define R_TILEPRO_IMM8_X0 17 /* X0 pipe 8-bit */ -#define R_TILEPRO_IMM8_Y0 18 /* Y0 pipe 8-bit */ -#define R_TILEPRO_IMM8_X1 19 /* X1 pipe 8-bit */ -#define R_TILEPRO_IMM8_Y1 20 /* Y1 pipe 8-bit */ -#define R_TILEPRO_MT_IMM15_X1 21 /* X1 pipe mtspr */ -#define R_TILEPRO_MF_IMM15_X1 22 /* X1 pipe mfspr */ -#define R_TILEPRO_IMM16_X0 23 /* X0 pipe 16-bit */ -#define R_TILEPRO_IMM16_X1 24 /* X1 pipe 16-bit */ -#define R_TILEPRO_IMM16_X0_LO 25 /* X0 pipe low 16-bit */ -#define R_TILEPRO_IMM16_X1_LO 26 /* X1 pipe low 16-bit */ -#define R_TILEPRO_IMM16_X0_HI 27 /* X0 pipe high 16-bit */ -#define R_TILEPRO_IMM16_X1_HI 28 /* X1 pipe high 16-bit */ -#define R_TILEPRO_IMM16_X0_HA 29 /* X0 pipe high 16-bit, adjusted */ -#define R_TILEPRO_IMM16_X1_HA 30 /* X1 pipe high 16-bit, adjusted */ -#define R_TILEPRO_IMM16_X0_PCREL 31 /* X0 pipe PC relative 16 bit */ -#define R_TILEPRO_IMM16_X1_PCREL 32 /* X1 pipe PC relative 16 bit */ -#define R_TILEPRO_IMM16_X0_LO_PCREL 33 /* X0 pipe PC relative low 16 bit */ -#define R_TILEPRO_IMM16_X1_LO_PCREL 34 /* X1 pipe PC relative low 16 bit */ -#define R_TILEPRO_IMM16_X0_HI_PCREL 35 /* X0 pipe PC relative high 16 bit */ -#define R_TILEPRO_IMM16_X1_HI_PCREL 36 /* X1 pipe PC relative high 16 bit */ -#define R_TILEPRO_IMM16_X0_HA_PCREL 37 /* X0 pipe PC relative ha() 16 bit */ -#define R_TILEPRO_IMM16_X1_HA_PCREL 38 /* X1 pipe PC relative ha() 16 bit */ -#define R_TILEPRO_IMM16_X0_GOT 39 /* X0 pipe 16-bit GOT offset */ -#define R_TILEPRO_IMM16_X1_GOT 40 /* X1 pipe 16-bit GOT offset */ -#define R_TILEPRO_IMM16_X0_GOT_LO 41 /* X0 pipe low 16-bit GOT offset */ -#define R_TILEPRO_IMM16_X1_GOT_LO 42 /* X1 pipe low 16-bit GOT offset */ -#define R_TILEPRO_IMM16_X0_GOT_HI 43 /* X0 pipe high 16-bit GOT offset */ -#define R_TILEPRO_IMM16_X1_GOT_HI 44 /* X1 pipe high 16-bit GOT offset */ -#define R_TILEPRO_IMM16_X0_GOT_HA 45 /* X0 pipe ha() 16-bit GOT offset */ -#define R_TILEPRO_IMM16_X1_GOT_HA 46 /* X1 pipe ha() 16-bit GOT offset */ -#define R_TILEPRO_MMSTART_X0 47 /* X0 pipe mm "start" */ -#define R_TILEPRO_MMEND_X0 48 /* X0 pipe mm "end" */ -#define R_TILEPRO_MMSTART_X1 49 /* X1 pipe mm "start" */ -#define R_TILEPRO_MMEND_X1 50 /* X1 pipe mm "end" */ -#define R_TILEPRO_SHAMT_X0 51 /* X0 pipe shift amount */ -#define R_TILEPRO_SHAMT_X1 52 /* X1 pipe shift amount */ -#define R_TILEPRO_SHAMT_Y0 53 /* Y0 pipe shift amount */ -#define R_TILEPRO_SHAMT_Y1 54 /* Y1 pipe shift amount */ -#define R_TILEPRO_DEST_IMM8_X1 55 /* X1 pipe destination 8-bit */ -/* Relocs 56-59 are currently not defined. */ -#define R_TILEPRO_TLS_GD_CALL 60 /* "jal" for TLS GD */ -#define R_TILEPRO_IMM8_X0_TLS_GD_ADD 61 /* X0 pipe "addi" for TLS GD */ -#define R_TILEPRO_IMM8_X1_TLS_GD_ADD 62 /* X1 pipe "addi" for TLS GD */ -#define R_TILEPRO_IMM8_Y0_TLS_GD_ADD 63 /* Y0 pipe "addi" for TLS GD */ -#define R_TILEPRO_IMM8_Y1_TLS_GD_ADD 64 /* Y1 pipe "addi" for TLS GD */ -#define R_TILEPRO_TLS_IE_LOAD 65 /* "lw_tls" for TLS IE */ -#define R_TILEPRO_IMM16_X0_TLS_GD 66 /* X0 pipe 16-bit TLS GD offset */ -#define R_TILEPRO_IMM16_X1_TLS_GD 67 /* X1 pipe 16-bit TLS GD offset */ -#define R_TILEPRO_IMM16_X0_TLS_GD_LO 68 /* X0 pipe low 16-bit TLS GD offset */ -#define R_TILEPRO_IMM16_X1_TLS_GD_LO 69 /* X1 pipe low 16-bit TLS GD offset */ -#define R_TILEPRO_IMM16_X0_TLS_GD_HI \ - 70 /* X0 pipe high 16-bit TLS GD offset \ - */ -#define R_TILEPRO_IMM16_X1_TLS_GD_HI \ - 71 /* X1 pipe high 16-bit TLS GD offset \ - */ -#define R_TILEPRO_IMM16_X0_TLS_GD_HA \ - 72 /* X0 pipe ha() 16-bit TLS GD offset \ - */ -#define R_TILEPRO_IMM16_X1_TLS_GD_HA \ - 73 /* X1 pipe ha() 16-bit TLS GD offset \ - */ -#define R_TILEPRO_IMM16_X0_TLS_IE 74 /* X0 pipe 16-bit TLS IE offset */ -#define R_TILEPRO_IMM16_X1_TLS_IE 75 /* X1 pipe 16-bit TLS IE offset */ -#define R_TILEPRO_IMM16_X0_TLS_IE_LO 76 /* X0 pipe low 16-bit TLS IE offset */ -#define R_TILEPRO_IMM16_X1_TLS_IE_LO 77 /* X1 pipe low 16-bit TLS IE offset */ -#define R_TILEPRO_IMM16_X0_TLS_IE_HI \ - 78 /* X0 pipe high 16-bit TLS IE offset \ - */ -#define R_TILEPRO_IMM16_X1_TLS_IE_HI \ - 79 /* X1 pipe high 16-bit TLS IE offset \ - */ -#define R_TILEPRO_IMM16_X0_TLS_IE_HA \ - 80 /* X0 pipe ha() 16-bit TLS IE offset \ - */ -#define R_TILEPRO_IMM16_X1_TLS_IE_HA \ - 81 /* X1 pipe ha() 16-bit TLS IE offset \ - */ -#define R_TILEPRO_TLS_DTPMOD32 82 /* ID of module containing symbol */ -#define R_TILEPRO_TLS_DTPOFF32 83 /* Offset in TLS block */ -#define R_TILEPRO_TLS_TPOFF32 84 /* Offset in static TLS block */ -#define R_TILEPRO_IMM16_X0_TLS_LE 85 /* X0 pipe 16-bit TLS LE offset */ -#define R_TILEPRO_IMM16_X1_TLS_LE 86 /* X1 pipe 16-bit TLS LE offset */ -#define R_TILEPRO_IMM16_X0_TLS_LE_LO 87 /* X0 pipe low 16-bit TLS LE offset */ -#define R_TILEPRO_IMM16_X1_TLS_LE_LO 88 /* X1 pipe low 16-bit TLS LE offset */ -#define R_TILEPRO_IMM16_X0_TLS_LE_HI \ - 89 /* X0 pipe high 16-bit TLS LE offset \ - */ -#define R_TILEPRO_IMM16_X1_TLS_LE_HI \ - 90 /* X1 pipe high 16-bit TLS LE offset \ - */ -#define R_TILEPRO_IMM16_X0_TLS_LE_HA \ - 91 /* X0 pipe ha() 16-bit TLS LE offset \ - */ -#define R_TILEPRO_IMM16_X1_TLS_LE_HA \ - 92 /* X1 pipe ha() 16-bit TLS LE offset \ - */ - -#define R_TILEPRO_GNU_VTINHERIT 128 /* GNU C++ vtable hierarchy */ -#define R_TILEPRO_GNU_VTENTRY 129 /* GNU C++ vtable member usage */ - -#define R_TILEPRO_NUM 130 - -/* TILE-Gx relocations. */ -#define R_TILEGX_NONE 0 /* No reloc */ -#define R_TILEGX_64 1 /* Direct 64 bit */ -#define R_TILEGX_32 2 /* Direct 32 bit */ -#define R_TILEGX_16 3 /* Direct 16 bit */ -#define R_TILEGX_8 4 /* Direct 8 bit */ -#define R_TILEGX_64_PCREL 5 /* PC relative 64 bit */ -#define R_TILEGX_32_PCREL 6 /* PC relative 32 bit */ -#define R_TILEGX_16_PCREL 7 /* PC relative 16 bit */ -#define R_TILEGX_8_PCREL 8 /* PC relative 8 bit */ -#define R_TILEGX_HW0 9 /* hword 0 16-bit */ -#define R_TILEGX_HW1 10 /* hword 1 16-bit */ -#define R_TILEGX_HW2 11 /* hword 2 16-bit */ -#define R_TILEGX_HW3 12 /* hword 3 16-bit */ -#define R_TILEGX_HW0_LAST 13 /* last hword 0 16-bit */ -#define R_TILEGX_HW1_LAST 14 /* last hword 1 16-bit */ -#define R_TILEGX_HW2_LAST 15 /* last hword 2 16-bit */ -#define R_TILEGX_COPY 16 /* Copy relocation */ -#define R_TILEGX_GLOB_DAT 17 /* Create GOT entry */ -#define R_TILEGX_JMP_SLOT 18 /* Create PLT entry */ -#define R_TILEGX_RELATIVE 19 /* Adjust by program base */ -#define R_TILEGX_BROFF_X1 20 /* X1 pipe branch offset */ -#define R_TILEGX_JUMPOFF_X1 21 /* X1 pipe jump offset */ -#define R_TILEGX_JUMPOFF_X1_PLT 22 /* X1 pipe jump offset to PLT */ -#define R_TILEGX_IMM8_X0 23 /* X0 pipe 8-bit */ -#define R_TILEGX_IMM8_Y0 24 /* Y0 pipe 8-bit */ -#define R_TILEGX_IMM8_X1 25 /* X1 pipe 8-bit */ -#define R_TILEGX_IMM8_Y1 26 /* Y1 pipe 8-bit */ -#define R_TILEGX_DEST_IMM8_X1 27 /* X1 pipe destination 8-bit */ -#define R_TILEGX_MT_IMM14_X1 28 /* X1 pipe mtspr */ -#define R_TILEGX_MF_IMM14_X1 29 /* X1 pipe mfspr */ -#define R_TILEGX_MMSTART_X0 30 /* X0 pipe mm "start" */ -#define R_TILEGX_MMEND_X0 31 /* X0 pipe mm "end" */ -#define R_TILEGX_SHAMT_X0 32 /* X0 pipe shift amount */ -#define R_TILEGX_SHAMT_X1 33 /* X1 pipe shift amount */ -#define R_TILEGX_SHAMT_Y0 34 /* Y0 pipe shift amount */ -#define R_TILEGX_SHAMT_Y1 35 /* Y1 pipe shift amount */ -#define R_TILEGX_IMM16_X0_HW0 36 /* X0 pipe hword 0 */ -#define R_TILEGX_IMM16_X1_HW0 37 /* X1 pipe hword 0 */ -#define R_TILEGX_IMM16_X0_HW1 38 /* X0 pipe hword 1 */ -#define R_TILEGX_IMM16_X1_HW1 39 /* X1 pipe hword 1 */ -#define R_TILEGX_IMM16_X0_HW2 40 /* X0 pipe hword 2 */ -#define R_TILEGX_IMM16_X1_HW2 41 /* X1 pipe hword 2 */ -#define R_TILEGX_IMM16_X0_HW3 42 /* X0 pipe hword 3 */ -#define R_TILEGX_IMM16_X1_HW3 43 /* X1 pipe hword 3 */ -#define R_TILEGX_IMM16_X0_HW0_LAST 44 /* X0 pipe last hword 0 */ -#define R_TILEGX_IMM16_X1_HW0_LAST 45 /* X1 pipe last hword 0 */ -#define R_TILEGX_IMM16_X0_HW1_LAST 46 /* X0 pipe last hword 1 */ -#define R_TILEGX_IMM16_X1_HW1_LAST 47 /* X1 pipe last hword 1 */ -#define R_TILEGX_IMM16_X0_HW2_LAST 48 /* X0 pipe last hword 2 */ -#define R_TILEGX_IMM16_X1_HW2_LAST 49 /* X1 pipe last hword 2 */ -#define R_TILEGX_IMM16_X0_HW0_PCREL 50 /* X0 pipe PC relative hword 0 */ -#define R_TILEGX_IMM16_X1_HW0_PCREL 51 /* X1 pipe PC relative hword 0 */ -#define R_TILEGX_IMM16_X0_HW1_PCREL 52 /* X0 pipe PC relative hword 1 */ -#define R_TILEGX_IMM16_X1_HW1_PCREL 53 /* X1 pipe PC relative hword 1 */ -#define R_TILEGX_IMM16_X0_HW2_PCREL 54 /* X0 pipe PC relative hword 2 */ -#define R_TILEGX_IMM16_X1_HW2_PCREL 55 /* X1 pipe PC relative hword 2 */ -#define R_TILEGX_IMM16_X0_HW3_PCREL 56 /* X0 pipe PC relative hword 3 */ -#define R_TILEGX_IMM16_X1_HW3_PCREL 57 /* X1 pipe PC relative hword 3 */ -#define R_TILEGX_IMM16_X0_HW0_LAST_PCREL 58 /* X0 pipe PC-rel last hword 0 */ -#define R_TILEGX_IMM16_X1_HW0_LAST_PCREL 59 /* X1 pipe PC-rel last hword 0 */ -#define R_TILEGX_IMM16_X0_HW1_LAST_PCREL 60 /* X0 pipe PC-rel last hword 1 */ -#define R_TILEGX_IMM16_X1_HW1_LAST_PCREL 61 /* X1 pipe PC-rel last hword 1 */ -#define R_TILEGX_IMM16_X0_HW2_LAST_PCREL 62 /* X0 pipe PC-rel last hword 2 */ -#define R_TILEGX_IMM16_X1_HW2_LAST_PCREL 63 /* X1 pipe PC-rel last hword 2 */ -#define R_TILEGX_IMM16_X0_HW0_GOT 64 /* X0 pipe hword 0 GOT offset */ -#define R_TILEGX_IMM16_X1_HW0_GOT 65 /* X1 pipe hword 0 GOT offset */ -#define R_TILEGX_IMM16_X0_HW0_PLT_PCREL 66 /* X0 pipe PC-rel PLT hword 0 */ -#define R_TILEGX_IMM16_X1_HW0_PLT_PCREL 67 /* X1 pipe PC-rel PLT hword 0 */ -#define R_TILEGX_IMM16_X0_HW1_PLT_PCREL 68 /* X0 pipe PC-rel PLT hword 1 */ -#define R_TILEGX_IMM16_X1_HW1_PLT_PCREL 69 /* X1 pipe PC-rel PLT hword 1 */ -#define R_TILEGX_IMM16_X0_HW2_PLT_PCREL 70 /* X0 pipe PC-rel PLT hword 2 */ -#define R_TILEGX_IMM16_X1_HW2_PLT_PCREL 71 /* X1 pipe PC-rel PLT hword 2 */ -#define R_TILEGX_IMM16_X0_HW0_LAST_GOT \ - 72 /* X0 pipe last hword 0 GOT offset \ - */ -#define R_TILEGX_IMM16_X1_HW0_LAST_GOT \ - 73 /* X1 pipe last hword 0 GOT offset \ - */ -#define R_TILEGX_IMM16_X0_HW1_LAST_GOT \ - 74 /* X0 pipe last hword 1 GOT offset \ - */ -#define R_TILEGX_IMM16_X1_HW1_LAST_GOT \ - 75 /* X1 pipe last hword 1 GOT offset \ - */ -#define R_TILEGX_IMM16_X0_HW3_PLT_PCREL 76 /* X0 pipe PC-rel PLT hword 3 */ -#define R_TILEGX_IMM16_X1_HW3_PLT_PCREL 77 /* X1 pipe PC-rel PLT hword 3 */ -#define R_TILEGX_IMM16_X0_HW0_TLS_GD 78 /* X0 pipe hword 0 TLS GD offset */ -#define R_TILEGX_IMM16_X1_HW0_TLS_GD 79 /* X1 pipe hword 0 TLS GD offset */ -#define R_TILEGX_IMM16_X0_HW0_TLS_LE 80 /* X0 pipe hword 0 TLS LE offset */ -#define R_TILEGX_IMM16_X1_HW0_TLS_LE 81 /* X1 pipe hword 0 TLS LE offset */ -#define R_TILEGX_IMM16_X0_HW0_LAST_TLS_LE 82 /* X0 pipe last hword 0 LE off */ -#define R_TILEGX_IMM16_X1_HW0_LAST_TLS_LE 83 /* X1 pipe last hword 0 LE off */ -#define R_TILEGX_IMM16_X0_HW1_LAST_TLS_LE 84 /* X0 pipe last hword 1 LE off */ -#define R_TILEGX_IMM16_X1_HW1_LAST_TLS_LE 85 /* X1 pipe last hword 1 LE off */ -#define R_TILEGX_IMM16_X0_HW0_LAST_TLS_GD 86 /* X0 pipe last hword 0 GD off */ -#define R_TILEGX_IMM16_X1_HW0_LAST_TLS_GD 87 /* X1 pipe last hword 0 GD off */ -#define R_TILEGX_IMM16_X0_HW1_LAST_TLS_GD 88 /* X0 pipe last hword 1 GD off */ -#define R_TILEGX_IMM16_X1_HW1_LAST_TLS_GD 89 /* X1 pipe last hword 1 GD off */ -/* Relocs 90-91 are currently not defined. */ -#define R_TILEGX_IMM16_X0_HW0_TLS_IE 92 /* X0 pipe hword 0 TLS IE offset */ -#define R_TILEGX_IMM16_X1_HW0_TLS_IE 93 /* X1 pipe hword 0 TLS IE offset */ -#define R_TILEGX_IMM16_X0_HW0_LAST_PLT_PCREL \ - 94 /* X0 pipe PC-rel PLT last hword 0 */ -#define R_TILEGX_IMM16_X1_HW0_LAST_PLT_PCREL \ - 95 /* X1 pipe PC-rel PLT last hword 0 */ -#define R_TILEGX_IMM16_X0_HW1_LAST_PLT_PCREL \ - 96 /* X0 pipe PC-rel PLT last hword 1 */ -#define R_TILEGX_IMM16_X1_HW1_LAST_PLT_PCREL \ - 97 /* X1 pipe PC-rel PLT last hword 1 */ -#define R_TILEGX_IMM16_X0_HW2_LAST_PLT_PCREL \ - 98 /* X0 pipe PC-rel PLT last hword 2 */ -#define R_TILEGX_IMM16_X1_HW2_LAST_PLT_PCREL \ - 99 /* X1 pipe PC-rel PLT last hword 2 */ -#define R_TILEGX_IMM16_X0_HW0_LAST_TLS_IE \ - 100 /* X0 pipe last hword 0 IE off \ - */ -#define R_TILEGX_IMM16_X1_HW0_LAST_TLS_IE \ - 101 /* X1 pipe last hword 0 IE off \ - */ -#define R_TILEGX_IMM16_X0_HW1_LAST_TLS_IE \ - 102 /* X0 pipe last hword 1 IE off \ - */ -#define R_TILEGX_IMM16_X1_HW1_LAST_TLS_IE \ - 103 /* X1 pipe last hword 1 IE off \ - */ -/* Relocs 104-105 are currently not defined. */ -#define R_TILEGX_TLS_DTPMOD64 106 /* 64-bit ID of symbol's module */ -#define R_TILEGX_TLS_DTPOFF64 107 /* 64-bit offset in TLS block */ -#define R_TILEGX_TLS_TPOFF64 108 /* 64-bit offset in static TLS block */ -#define R_TILEGX_TLS_DTPMOD32 109 /* 32-bit ID of symbol's module */ -#define R_TILEGX_TLS_DTPOFF32 110 /* 32-bit offset in TLS block */ -#define R_TILEGX_TLS_TPOFF32 111 /* 32-bit offset in static TLS block */ -#define R_TILEGX_TLS_GD_CALL 112 /* "jal" for TLS GD */ -#define R_TILEGX_IMM8_X0_TLS_GD_ADD 113 /* X0 pipe "addi" for TLS GD */ -#define R_TILEGX_IMM8_X1_TLS_GD_ADD 114 /* X1 pipe "addi" for TLS GD */ -#define R_TILEGX_IMM8_Y0_TLS_GD_ADD 115 /* Y0 pipe "addi" for TLS GD */ -#define R_TILEGX_IMM8_Y1_TLS_GD_ADD 116 /* Y1 pipe "addi" for TLS GD */ -#define R_TILEGX_TLS_IE_LOAD 117 /* "ld_tls" for TLS IE */ -#define R_TILEGX_IMM8_X0_TLS_ADD 118 /* X0 pipe "addi" for TLS GD/IE */ -#define R_TILEGX_IMM8_X1_TLS_ADD 119 /* X1 pipe "addi" for TLS GD/IE */ -#define R_TILEGX_IMM8_Y0_TLS_ADD 120 /* Y0 pipe "addi" for TLS GD/IE */ -#define R_TILEGX_IMM8_Y1_TLS_ADD 121 /* Y1 pipe "addi" for TLS GD/IE */ - -#define R_TILEGX_GNU_VTINHERIT 128 /* GNU C++ vtable hierarchy */ -#define R_TILEGX_GNU_VTENTRY 129 /* GNU C++ vtable member usage */ - -#define R_TILEGX_NUM 130 - /* RISC-V ELF Flags */ -#define EF_RISCV_RVC 0x0001 -#define EF_RISCV_FLOAT_ABI 0x0006 -#define EF_RISCV_FLOAT_ABI_SOFT 0x0000 -#define EF_RISCV_FLOAT_ABI_SINGLE 0x0002 #define EF_RISCV_FLOAT_ABI_DOUBLE 0x0004 -#define EF_RISCV_FLOAT_ABI_QUAD 0x0006 /* RISC-V relocations. */ -#define R_RISCV_NONE 0 #define R_RISCV_32 1 #define R_RISCV_64 2 -#define R_RISCV_RELATIVE 3 -#define R_RISCV_COPY 4 -#define R_RISCV_JUMP_SLOT 5 -#define R_RISCV_TLS_DTPMOD32 6 -#define R_RISCV_TLS_DTPMOD64 7 -#define R_RISCV_TLS_DTPREL32 8 -#define R_RISCV_TLS_DTPREL64 9 -#define R_RISCV_TLS_TPREL32 10 -#define R_RISCV_TLS_TPREL64 11 -#define R_RISCV_BRANCH 16 -#define R_RISCV_JAL 17 -#define R_RISCV_CALL 18 -#define R_RISCV_CALL_PLT 19 -#define R_RISCV_GOT_HI20 20 -#define R_RISCV_TLS_GOT_HI20 21 -#define R_RISCV_TLS_GD_HI20 22 -#define R_RISCV_PCREL_HI20 23 -#define R_RISCV_PCREL_LO12_I 24 -#define R_RISCV_PCREL_LO12_S 25 -#define R_RISCV_HI20 26 -#define R_RISCV_LO12_I 27 -#define R_RISCV_LO12_S 28 -#define R_RISCV_TPREL_HI20 29 -#define R_RISCV_TPREL_LO12_I 30 -#define R_RISCV_TPREL_LO12_S 31 -#define R_RISCV_TPREL_ADD 32 -#define R_RISCV_ADD8 33 -#define R_RISCV_ADD16 34 -#define R_RISCV_ADD32 35 -#define R_RISCV_ADD64 36 -#define R_RISCV_SUB8 37 -#define R_RISCV_SUB16 38 -#define R_RISCV_SUB32 39 -#define R_RISCV_SUB64 40 -#define R_RISCV_GNU_VTINHERIT 41 -#define R_RISCV_GNU_VTENTRY 42 #define R_RISCV_ALIGN 43 -#define R_RISCV_RVC_BRANCH 44 -#define R_RISCV_RVC_JUMP 45 -#define R_RISCV_RVC_LUI 46 -#define R_RISCV_GPREL_I 47 -#define R_RISCV_GPREL_S 48 -#define R_RISCV_TPREL_I 49 -#define R_RISCV_TPREL_S 50 #define R_RISCV_RELAX 51 -#define R_RISCV_SUB6 52 -#define R_RISCV_SET6 53 -#define R_RISCV_SET8 54 -#define R_RISCV_SET16 55 -#define R_RISCV_SET32 56 #define R_RISCV_32_PCREL 57 -#define R_RISCV_NUM 58 - #endif /* elf.h */ diff --git a/source/obj/tccdbg.c b/source/obj/tccdbg.c index 08f89c955..543ddd0b4 100644 --- a/source/obj/tccdbg.c +++ b/source/obj/tccdbg.c @@ -3037,16 +3037,29 @@ ST_FUNC void tcc_debug_funcend(TCCState *s1, int size) #endif func_sib = dwarf_info_section->data_offset; dwarf_data4(dwarf_info_section, 0); // sibling +#if defined TCC_TARGET_ARM_THUMB + if (s1->need_frame_pointer) + { + /* r7 is the frame's bottom; variables sit at their (negative) frame + * offsets from its top. */ + int top = tcc_gen_machine_frame_top_offset(); + dwarf_data1(dwarf_info_section, 1 + dwarf_sleb128_size(top)); + dwarf_data1(dwarf_info_section, DW_OP_breg7); + dwarf_sleb128(dwarf_info_section, top); + } + else + { + dwarf_data1(dwarf_info_section, 1); + dwarf_data1(dwarf_info_section, DW_OP_call_frame_cfa); + } +#else dwarf_data1(dwarf_info_section, 1); +#endif #if defined(TCC_TARGET_I386) dwarf_data1(dwarf_info_section, DW_OP_reg5); // ebp #elif defined(TCC_TARGET_X86_64) dwarf_data1(dwarf_info_section, DW_OP_reg6); // rbp #elif defined TCC_TARGET_ARM_THUMB - if (s1->need_frame_pointer) - dwarf_data1(dwarf_info_section, DW_OP_reg7); - else - dwarf_data1(dwarf_info_section, DW_OP_call_frame_cfa); #elif defined TCC_TARGET_ARM dwarf_data1(dwarf_info_section, DW_OP_reg13); // sp #elif defined TCC_TARGET_ARM64 diff --git a/source/obj/tccelf.c b/source/obj/tccelf.c index 81a74fd00..7c356cb35 100644 --- a/source/obj/tccelf.c +++ b/source/obj/tccelf.c @@ -18,46 +18,20 @@ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ -#include "tcc.h" -#include "tccld.h" -#include "tccyaff.h" +/* ELF: TCCState ELF setup and teardown, deferred and lazily loaded + * sections, and garbage-collection marking of referenced sections. The rest + * is in tccelf_{sym,reloc,layout,output,load,ld}.c; see tccelf_priv.h. */ -/********************************************************/ -/* global variables */ +#include "tccelf_priv.h" -/* elf version information */ -struct sym_version -{ - char *lib; - char *version; - int out_index; - int prev_same_lib; -}; - -#define nb_sym_versions s1->nb_sym_versions -#define sym_versions s1->sym_versions -#define nb_sym_to_version s1->nb_sym_to_version -#define sym_to_version s1->sym_to_version -#define dt_verneednum s1->dt_verneednum -#define versym_section s1->versym_section -#define verneed_section s1->verneed_section - -/* special flag to indicate that the section should not be linked to the other - * ones */ -#define SHF_PRIVATE 0x80000000 -/* section is dynsymtab_section */ -#define SHF_DYNSYM 0x40000000 -/* Larger initial ELF hash tables reduce rebuild churn without changing lookup semantics. */ -#define SYMTAB_INITIAL_HASH_BUCKETS 512 +static void apply_reloc_patches(Section *sec, unsigned char *data, size_t size); +static void sort_patches_by_offset(Section *sec); #if defined(TCC_TARGET_PE) -#define shf_RELRO SHF_ALLOC static const char rdata[] = ".rdata"; #elif defined(TCC_TARGET_ARM_THUMB) -#define shf_RELRO SHF_ALLOC static const char rdata[] = ".rodata"; #else -#define shf_RELRO SHF_ALLOC /* eventually made SHF_WRITE in sort_sections() */ static const char rdata[] = ".data.ro"; #endif @@ -124,7 +98,7 @@ ST_FUNC void tccelf_new(TCCState *s) Only defer debug sections — deferring everything else adds overhead (malloc + strdup + re-open) that exceeds the savings when most sections will be needed for linking anyway. */ -static int should_defer_section(const char *name, int sh_type) +int should_defer_section(const char *name, int sh_type) { (void)sh_type; /* Defer DWARF debug sections */ @@ -137,14 +111,8 @@ static int should_defer_section(const char *name, int sh_type) return 0; } -/* Forward declarations for lazy loading functions */ -static void free_reloc_patches(Section *s); -static void apply_reloc_patches(Section *sec, unsigned char *data, size_t size); -static void sort_patches_by_offset(Section *sec); -static void add_reloc_patch(Section *s, uint32_t offset, uint32_t value); - /* Free all deferred chunks for a section */ -static void free_deferred_chunks(Section *sec) +void free_deferred_chunks(Section *sec) { DeferredChunk *c = sec->deferred_head; while (c) @@ -160,7 +128,7 @@ static void free_deferred_chunks(Section *sec) } /* Add a deferred chunk to a section */ -static void section_add_deferred(TCCState *s1, Section *sec, const char *path, unsigned long file_off, +void section_add_deferred(TCCState *s1, Section *sec, const char *path, unsigned long file_off, unsigned long size, unsigned long dest_off) { DeferredChunk *chunk = tcc_mallocz(sizeof(DeferredChunk)); @@ -342,7 +310,7 @@ static int apply_patches_to_buffer(Section *sec, int *patch_idx, uint32_t buf_st * This avoids the memory allocation for sections that are only written to output. * Applies relocation patches inline during streaming if present. * Returns 0 on success, -1 on error. */ -static int section_write_streaming(TCCState *s1, Section *sec, FILE *f) +int section_write_streaming(TCCState *s1, Section *sec, FILE *f) { DeferredChunk *c; int fd; @@ -980,8 +948,6 @@ ST_FUNC void tccelf_begin_file(TCCState *s1) #endif } -static void update_relocs(TCCState *s1, Section *s, int *old_to_new_syms, int first_sym); - /* At the end of compilation, convert any UNDEF syms to global, and merge with previously existing symbols */ ST_FUNC void tccelf_end_file(TCCState *s1) @@ -1028,6271 +994,3 @@ ST_FUNC void tccelf_end_file(TCCState *s1) s1->total_output[i] += s->data_offset - s->sh_offset; } } - -/* ---- Global section name hash table ---- */ - -static unsigned int section_name_hash(const char *name) -{ - unsigned int h = 5381; - const unsigned char *p = (const unsigned char *)name; - while (*p) - h = ((h << 5) + h) ^ *p++; - return h; -} - -static void section_ht_grow(TCCState *s1) -{ - unsigned int new_size = s1->section_ht_mask ? (s1->section_ht_mask + 1) * 2 : 64; - Section **new_ht = tcc_mallocz(new_size * sizeof(Section *)); - unsigned int new_mask = new_size - 1; - if (s1->section_ht) - { - unsigned int i; - for (i = 0; i <= s1->section_ht_mask; i++) - { - Section *s = s1->section_ht[i]; - if (s) - { - unsigned int idx = section_name_hash(s->name) & new_mask; - while (new_ht[idx]) - idx = (idx + 1) & new_mask; - new_ht[idx] = s; - } - } - tcc_free(s1->section_ht); - } - s1->section_ht = new_ht; - s1->section_ht_mask = new_mask; -} - -static void section_ht_insert(TCCState *s1, Section *sec) -{ - unsigned int idx; - if (s1->section_ht_count * 2 >= s1->section_ht_mask) - section_ht_grow(s1); - idx = section_name_hash(sec->name) & s1->section_ht_mask; - while (s1->section_ht[idx]) - idx = (idx + 1) & s1->section_ht_mask; - s1->section_ht[idx] = sec; - s1->section_ht_count++; -} - -static Section *section_ht_find(TCCState *s1, const char *name) -{ - unsigned int idx; - Section *s; - if (!s1->section_ht) - return NULL; - idx = section_name_hash(name) & s1->section_ht_mask; - while ((s = s1->section_ht[idx]) != NULL) - { - if (!strcmp(s->name, name)) - return s; - idx = (idx + 1) & s1->section_ht_mask; - } - return NULL; -} - -/* ---- End section name hash table ---- */ - -ST_FUNC Section *new_section(TCCState *s1, const char *name, int sh_type, int sh_flags) -{ - Section *sec; - - sec = tcc_mallocz(sizeof(Section) + strlen(name)); - sec->s1 = s1; - strcpy(sec->name, name); - sec->sh_type = sh_type; - sec->sh_flags = sh_flags; - switch (sh_type) - { - case SHT_GNU_versym: - sec->sh_addralign = 2; - break; - case SHT_HASH: - case SHT_GNU_HASH: - case SHT_REL: - case SHT_RELA: - case SHT_DYNSYM: - case SHT_SYMTAB: - case SHT_DYNAMIC: - case SHT_GNU_verneed: - case SHT_GNU_verdef: - sec->sh_addralign = 8; // PTR_SIZE; - break; - case SHT_STRTAB: - sec->sh_addralign = 1; - break; - default: - sec->sh_addralign = 8; // PTR_SIZE; /* gcc/pcc default alignment */ - break; - } - - if (sh_flags & SHF_PRIVATE) - { - dynarray_add(&s1->priv_sections, &s1->nb_priv_sections, sec); - } - else - { - sec->sh_num = s1->nb_sections; - dynarray_add(&s1->sections, &s1->nb_sections, sec); - section_ht_insert(s1, sec); - } - - return sec; -} - -ST_FUNC void init_symtab(Section *s) -{ - int *ptr, nb_buckets = SYMTAB_INITIAL_HASH_BUCKETS; - put_elf_str(s->link, ""); - section_ptr_add(s, sizeof(ElfW(Sym))); - ptr = section_ptr_add(s->hash, (2 + nb_buckets + 1) * sizeof(int)); - ptr[0] = nb_buckets; - ptr[1] = 1; - memset(ptr + 2, 0, (nb_buckets + 1) * sizeof(int)); -} - -ST_FUNC Section *new_symtab(TCCState *s1, const char *symtab_name, int sh_type, int sh_flags, const char *strtab_name, - const char *hash_name, int hash_sh_flags) -{ - Section *symtab, *strtab, *hash; - symtab = new_section(s1, symtab_name, sh_type, sh_flags); - symtab->sh_entsize = sizeof(ElfW(Sym)); - strtab = new_section(s1, strtab_name, SHT_STRTAB, sh_flags); - symtab->link = strtab; - hash = new_section(s1, hash_name, SHT_HASH, hash_sh_flags); - hash->sh_entsize = sizeof(int); - symtab->hash = hash; - hash->link = symtab; - init_symtab(symtab); - return symtab; -} - -/* realloc section and set its content to zero */ -ST_FUNC void section_realloc(Section *sec, unsigned long new_size) -{ - unsigned long size; - unsigned char *data; - - size = sec->data_allocated; - if (size == 0) - { - /* First allocation: round up to power of 2 with minimum 256 bytes - to reduce future reallocations */ - size = 256; - while (size < new_size) - size = size * 2; - } - else - { - while (size < new_size) - size = size * 2; - } - data = tcc_realloc(sec->data, size); - memset(data + sec->data_allocated, 0, size - sec->data_allocated); - sec->data = data; - sec->data_allocated = size; -} - -/* reserve at least 'size' bytes aligned per 'align' in section - 'sec' from current offset, and return the aligned offset */ -ST_FUNC size_t section_add(Section *sec, addr_t size, int align) -{ - size_t offset, offset1; - - offset = (sec->data_offset + align - 1) & -align; - offset1 = offset + size; - if (sec->sh_type != SHT_NOBITS && offset1 > sec->data_allocated) - section_realloc(sec, offset1); - sec->data_offset = offset1; - if (align > sec->sh_addralign) - sec->sh_addralign = align; - return offset; -} - -/* reserve at least 'size' bytes in section 'sec' from - sec->data_offset. */ -ST_FUNC void *section_ptr_add(Section *sec, addr_t size) -{ - size_t offset = section_add(sec, size, 1); - return sec->data + offset; -} - -/* Pre-allocate section capacity without changing data_offset. - Use this when you know the total size needed to avoid multiple reallocations. */ -ST_FUNC void section_prealloc(Section *sec, unsigned long size) -{ - unsigned long needed = sec->data_offset + size; - if (needed > sec->data_allocated) - section_realloc(sec, needed); -} - -#ifndef ELF_OBJ_ONLY -/* reserve at least 'size' bytes from section start */ -static void section_reserve(Section *sec, unsigned long size) -{ - if (size > sec->data_allocated) - section_realloc(sec, size); - if (size > sec->data_offset) - sec->data_offset = size; -} -#endif - -static Section *have_section(TCCState *s1, const char *name) -{ - Section *sec; - int i; - for (i = 1; i < s1->nb_sections; i++) - { - sec = s1->sections[i]; - if (!strcmp(name, sec->name)) - return sec; - } - return NULL; -} - -/* return a reference to a section, and create it if it does not - exists */ -ST_FUNC Section *find_section(TCCState *s1, const char *name) -{ - Section *sec = have_section(s1, name); - if (sec) - return sec; - /* sections are created as PROGBITS */ - return new_section(s1, name, SHT_PROGBITS, SHF_ALLOC); -} - -ST_FUNC int put_elf_str(Section *s, const char *sym) -{ - int offset, len; - char *ptr; - - if (!sym) - sym = ""; - len = strlen(sym) + 1; - offset = s->data_offset; - ptr = section_ptr_add(s, len); - memmove(ptr, sym, len); - return offset; -} - -/* elf symbol hashing function */ -static ElfW(Word) elf_hash(const unsigned char *name) -{ - ElfW(Word) h = 0, g; - - while (*name) - { - h = (h << 4) + *name++; - g = h & 0xf0000000; - if (g) - h ^= g >> 24; - h &= ~g; - } - return h; -} - -/* rebuild hash table of section s */ -/* NOTE: we do factorize the hash table code to go faster */ -static void rebuild_hash(Section *s, unsigned int nb_buckets) -{ - ElfW(Sym) * sym; - int *ptr, *hash, nb_syms, sym_index, h; - unsigned char *strtab; - unsigned int full_hash; - unsigned int *hcache; - - strtab = s->link->data; - nb_syms = s->data_offset / sizeof(ElfW(Sym)); - - if (!nb_buckets) - nb_buckets = ((int *)s->hash->data)[0]; - - s->hash->data_offset = 0; - ptr = section_ptr_add(s->hash, (2 + nb_buckets + nb_syms) * sizeof(int)); - ptr[0] = nb_buckets; - ptr[1] = nb_syms; - ptr += 2; - hash = ptr; - memset(hash, 0, (nb_buckets + 1) * sizeof(int)); - ptr += nb_buckets + 1; - - /* Rebuild hash value cache */ - s->hash->hash_val_cache = tcc_realloc(s->hash->hash_val_cache, nb_syms * sizeof(unsigned int)); - s->hash->hash_val_alloc = nb_syms; - hcache = s->hash->hash_val_cache; - hcache[0] = 0; - - sym = (ElfW(Sym) *)s->data + 1; - for (sym_index = 1; sym_index < nb_syms; sym_index++) - { - if (ELFW(ST_BIND)(sym->st_info) != STB_LOCAL) - { - full_hash = elf_hash(strtab + sym->st_name); - h = full_hash % nb_buckets; - *ptr = hash[h]; - hash[h] = sym_index; - hcache[sym_index] = full_hash; - } - else - { - *ptr = 0; - hcache[sym_index] = 0; - } - ptr++; - sym++; - } -} - -/* return the symbol number */ -ST_FUNC int put_elf_sym(Section *s, addr_t value, unsigned long size, int info, int other, int shndx, const char *name) -{ - int name_offset, sym_index; - int nbuckets, h; - ElfW(Sym) * sym; - Section *hs; - - /* Validate name pointer - catch garbage early. - Accept printable ASCII and valid UTF-8 lead bytes (0xC2-0xF4). - Reject control characters and bare continuation bytes. */ - if (name && name[0]) - { - unsigned char first = (unsigned char)name[0]; - if (first < 0x20 || (first > 0x7e && first < 0xc2) || first > 0xf4) - { - name = NULL; - } - } - - sym = section_ptr_add(s, sizeof(ElfW(Sym))); - if (name && name[0]) - name_offset = put_elf_str(s->link, name); - else - name_offset = 0; - /* XXX: endianness */ - sym->st_name = name_offset; - sym->st_value = value; - sym->st_size = size; - sym->st_info = info; - sym->st_other = other; - sym->st_shndx = shndx; - sym_index = sym - (ElfW(Sym) *)s->data; - hs = s->hash; - if (hs) - { - int *ptr, *base; - ptr = section_ptr_add(hs, sizeof(int)); - base = (int *)hs->data; - /* only add global or weak symbols. */ - if (ELFW(ST_BIND)(info) != STB_LOCAL) - { - /* add another hashing entry */ - unsigned int full_hash; - nbuckets = base[0]; - full_hash = elf_hash((unsigned char *)s->link->data + name_offset); - h = full_hash % nbuckets; - *ptr = base[2 + h]; - base[2 + h] = sym_index; - base[1]++; - /* we resize the hash table */ - hs->nb_hashed_syms++; - if (hs->nb_hashed_syms > 2 * nbuckets) - { - rebuild_hash(s, 2 * nbuckets); - } - else - { - /* Store cached hash value (rebuild_hash handles this when triggered) */ - if (sym_index >= hs->hash_val_alloc) - { - int new_alloc = (sym_index + 16) & ~15; - hs->hash_val_cache = tcc_realloc(hs->hash_val_cache, new_alloc * sizeof(unsigned int)); - memset(hs->hash_val_cache + hs->hash_val_alloc, 0, (new_alloc - hs->hash_val_alloc) * sizeof(unsigned int)); - hs->hash_val_alloc = new_alloc; - } - hs->hash_val_cache[sym_index] = full_hash; - } - } - else - { - *ptr = 0; - base[1]++; - } - } - - /* Track non-local UNDEF symbols in symtab for fast alacarte lookup */ - if (shndx == SHN_UNDEF && ELFW(ST_BIND)(info) != STB_LOCAL) - { - TCCState *ts = s->s1; - if (s == ts->symtab) - { - if (ts->nb_undef_syms >= ts->undef_sym_alloc) - { - ts->undef_sym_alloc = ts->undef_sym_alloc ? ts->undef_sym_alloc * 2 : 64; - ts->undef_sym_list = tcc_realloc(ts->undef_sym_list, ts->undef_sym_alloc * sizeof(int)); - } - ts->undef_sym_list[ts->nb_undef_syms++] = sym_index; - } - } - - return sym_index; -} - -static int find_elf_sym_with_hash(Section *s, const char *name, unsigned int full_hash) -{ - ElfW(Sym) * sym; - Section *hs; - int nbuckets, sym_index; - const char *name1; - unsigned int *hcache; - - hs = s->hash; - if (!hs) - return 0; - nbuckets = ((int *)hs->data)[0]; - sym_index = ((int *)hs->data)[2 + full_hash % nbuckets]; - hcache = hs->hash_val_cache; - - while (sym_index != 0) - { - /* Compare cached hash value first to avoid expensive strcmp */ - if (!hcache || hcache[sym_index] == full_hash) - { - sym = &((ElfW(Sym) *)s->data)[sym_index]; - name1 = (char *)s->link->data + sym->st_name; - if (!strcmp(name, name1)) - return sym_index; - } - sym_index = ((int *)hs->data)[2 + nbuckets + sym_index]; - } - return 0; -} - -ST_FUNC int find_elf_sym(Section *s, const char *name) -{ - return find_elf_sym_with_hash(s, name, elf_hash((unsigned char *)name)); -} - -/* Resolve `name` against loaded libraries: first the already-interned - dynsymtab_section, then (on miss) the loaded YAFF libraries' on-disk hash - tables (tcc_yaff_resolve), which interns a hit into dynsymtab_section. - Returns the dynsymtab index, or 0 if no loaded library provides it. */ -ST_FUNC int tcc_dynsym_find(TCCState *s1, const char *name) -{ - int idx = find_elf_sym(s1->dynsymtab_section, name); - if (idx) - return idx; - return tcc_yaff_resolve(s1, name); -} - -/* return elf symbol value, signal error if 'err' is nonzero, decorate - name if FORC */ -ST_FUNC addr_t get_sym_addr(TCCState *s1, const char *name, int err, int forc) -{ - int sym_index; - ElfW(Sym) * sym; - char buf[256]; - if (forc && - s1->leading_underscore -#ifdef TCC_TARGET_PE - /* win32-32bit stdcall symbols always have _ already */ - && !strchr(name, '@') -#endif - ) - { - buf[0] = '_'; - pstrcpy(buf + 1, sizeof(buf) - 1, name); - name = buf; - } - sym_index = find_elf_sym(s1->symtab, name); - sym = &((ElfW(Sym) *)s1->symtab->data)[sym_index]; - if (!sym_index || sym->st_shndx == SHN_UNDEF) - { - if (err) - tcc_error_noabort("%s not defined", name); - return (addr_t)-1; - } - return sym->st_value; -} - -/* return elf symbol value */ -LIBTCCAPI void *tcc_get_symbol(TCCState *s, const char *name) -{ - addr_t addr = get_sym_addr(s, name, 0, 1); - return addr == -1 ? NULL : (void *)(uintptr_t)addr; -} - -/* list elf symbol names and values */ -ST_FUNC void list_elf_symbols(TCCState *s, void *ctx, void (*symbol_cb)(void *ctx, const char *name, const void *val)) -{ - ElfW(Sym) * sym; - Section *symtab; - int sym_index, end_sym; - const char *name; - unsigned char sym_vis, sym_bind; - - symtab = s->symtab; - end_sym = symtab->data_offset / sizeof(ElfSym); - for (sym_index = 0; sym_index < end_sym; ++sym_index) - { - sym = &((ElfW(Sym) *)symtab->data)[sym_index]; - if (sym->st_value) - { - name = (char *)symtab->link->data + sym->st_name; - sym_bind = ELFW(ST_BIND)(sym->st_info); - sym_vis = ELFW(ST_VISIBILITY)(sym->st_other); - if (sym_bind == STB_GLOBAL && sym_vis == STV_DEFAULT) - symbol_cb(ctx, name, (void *)(uintptr_t)sym->st_value); - } - } -} - -/* list elf symbol names and values */ -LIBTCCAPI void tcc_list_symbols(TCCState *s, void *ctx, void (*symbol_cb)(void *ctx, const char *name, const void *val)) -{ - list_elf_symbols(s, ctx, symbol_cb); -} - -#ifndef ELF_OBJ_ONLY -static void version_add(TCCState *s1) -{ - int i; - ElfW(Sym) * sym; - ElfW(Verneed) *vn = NULL; - Section *symtab; - int sym_index, end_sym, nb_versions = 2, nb_entries = 0; - ElfW(Half) * versym; - const char *name; - - if (0 == nb_sym_versions) - return; - versym_section = new_section(s1, ".gnu.version", SHT_GNU_versym, SHF_ALLOC); - versym_section->sh_entsize = sizeof(ElfW(Half)); - versym_section->link = s1->dynsym; - - /* add needed symbols */ - symtab = s1->dynsym; - end_sym = symtab->data_offset / sizeof(ElfSym); - versym = section_ptr_add(versym_section, end_sym * sizeof(ElfW(Half))); - for (sym_index = 1; sym_index < end_sym; ++sym_index) - { - int dllindex, verndx; - sym = &((ElfW(Sym) *)symtab->data)[sym_index]; - name = (char *)symtab->link->data + sym->st_name; - dllindex = tcc_dynsym_find(s1, name); - verndx = (dllindex && dllindex < nb_sym_to_version) ? sym_to_version[dllindex] : -1; - if (verndx >= 0 - /* XXX: on android, clang refuses to link with a libtcc.so made by tcc - when defined symbols have a version > 1 or when the version is '0'. - Whereas version '1' for example for 'signal' in an exe defeats - bcheck's signal_redir. */ - && (sym->st_shndx == SHN_UNDEF || (s1->output_type & TCC_OUTPUT_EXE))) - { - if (!sym_versions[verndx].out_index) - sym_versions[verndx].out_index = nb_versions++; - versym[sym_index] = sym_versions[verndx].out_index; - } - else - { - versym[sym_index] = 1; /* (*global*) */ - } - // printf("SYM %d %s\n", versym[sym_index], name); - } - /* generate verneed section, but not when it will be empty. Some - dynamic linkers look at their contents even when DTVERNEEDNUM and - section size is zero. */ - if (nb_versions > 2) - { - verneed_section = new_section(s1, ".gnu.version_r", SHT_GNU_verneed, SHF_ALLOC); - verneed_section->link = s1->dynsym->link; - for (i = nb_sym_versions; i-- > 0;) - { - struct sym_version *sv = &sym_versions[i]; - int n_same_libs = 0, prev; - size_t vnofs; - ElfW(Vernaux) *vna = 0; - if (sv->out_index < 1) - continue; - - /* make sure that a DT_NEEDED tag is put */ - /* abitest-tcc fails on older i386-linux with "ld-linux.so.2" DT_NEEDED - ret_int_test... Inconsistency detected by ld.so: dl-minimal.c: 148: - realloc: Assertion `ptr == alloc_last_block' failed! */ - if (strcmp(sv->lib, "ld-linux.so.2")) - tcc_add_dllref(s1, sv->lib, 0); - - vnofs = section_add(verneed_section, sizeof(*vn), 1); - vn = (ElfW(Verneed) *)(verneed_section->data + vnofs); - vn->vn_version = 1; - vn->vn_file = put_elf_str(verneed_section->link, sv->lib); - vn->vn_aux = sizeof(*vn); - do - { - prev = sv->prev_same_lib; - if (sv->out_index > 0) - { - vna = section_ptr_add(verneed_section, sizeof(*vna)); - vna->vna_hash = elf_hash((const unsigned char *)sv->version); - vna->vna_flags = 0; - vna->vna_other = sv->out_index; - sv->out_index = -2; - vna->vna_name = put_elf_str(verneed_section->link, sv->version); - vna->vna_next = sizeof(*vna); - // printf("LIB %d %s %s\n", vna->vna_other, sv->lib, - // verneed_section->link->data + vna->vna_name); - n_same_libs++; - } - if (prev >= 0) - sv = &sym_versions[prev]; - } while (prev >= 0); - vna->vna_next = 0; - vn = (ElfW(Verneed) *)(verneed_section->data + vnofs); - vn->vn_cnt = n_same_libs; - vn->vn_next = sizeof(*vn) + n_same_libs * sizeof(*vna); - nb_entries++; - } - if (vn) - vn->vn_next = 0; - verneed_section->sh_info = nb_entries; - } - dt_verneednum = nb_entries; -} -#endif /* ndef ELF_OBJ_ONLY */ - -/* add an elf symbol : check if it is already defined and patch - it. Return symbol index. NOTE that sh_num can be SHN_UNDEF. */ -ST_FUNC int set_elf_sym(Section *s, addr_t value, unsigned long size, int info, int other, int shndx, const char *name) -{ - TCCState *s1 = s->s1; - ElfW(Sym) * esym; - int sym_bind, sym_index, sym_type, esym_bind; - unsigned char sym_vis, esym_vis, new_vis; - - sym_bind = ELFW(ST_BIND)(info); - sym_type = ELFW(ST_TYPE)(info); - sym_vis = ELFW(ST_VISIBILITY)(other); - - if (sym_bind != STB_LOCAL) - { - /* we search global or weak symbols */ - sym_index = find_elf_sym_with_hash(s, name, elf_hash((const unsigned char *)name)); - if (!sym_index) - goto do_def; - esym = &((ElfW(Sym) *)s->data)[sym_index]; - if (esym->st_value == value && esym->st_size == size && esym->st_info == info && esym->st_other == other && - esym->st_shndx == shndx) - return sym_index; - if (esym->st_shndx != SHN_UNDEF) - { - esym_bind = ELFW(ST_BIND)(esym->st_info); - /* propagate the most constraining visibility */ - /* STV_DEFAULT(0)st_other); - if (esym_vis == STV_DEFAULT) - { - new_vis = sym_vis; - } - else if (sym_vis == STV_DEFAULT) - { - new_vis = esym_vis; - } - else - { - new_vis = (esym_vis < sym_vis) ? esym_vis : sym_vis; - } - esym->st_other = (esym->st_other & ~ELFW(ST_VISIBILITY)(-1)) | new_vis; - if (shndx == SHN_UNDEF) - { - /* ignore adding of undefined symbol if the - corresponding symbol is already defined */ - } - else if (sym_bind == STB_GLOBAL && esym_bind == STB_WEAK) - { - /* global overrides weak, so patch */ - goto do_patch; - } - else if (sym_bind == STB_WEAK && esym_bind == STB_GLOBAL) - { - /* weak is ignored if already global */ - } - else if (sym_bind == STB_WEAK && esym_bind == STB_WEAK) - { - /* keep first-found weak definition, ignore subsequents */ - } - else if (sym_vis == STV_HIDDEN || sym_vis == STV_INTERNAL) - { - /* ignore hidden symbols after */ - } - else if ((esym->st_shndx == SHN_COMMON || esym->st_shndx == bss_section->sh_num) && - (shndx < SHN_LORESERVE && shndx != bss_section->sh_num)) - { - /* data symbol gets precedence over common/bss */ - goto do_patch; - } - else if (shndx == SHN_COMMON || shndx == bss_section->sh_num) - { - /* data symbol keeps precedence over common/bss */ - } - else if (s->sh_flags & SHF_DYNSYM) - { - /* we accept that two DLL define the same symbol */ - } - else if (esym->st_other & ST_ASM_SET) - { - /* If the existing symbol came from an asm .set - we can override. */ - goto do_patch; - } - else - { -#if 0 - printf("new_bind=%x new_shndx=%x new_vis=%x old_bind=%x old_shndx=%x old_vis=%x\n", - sym_bind, shndx, new_vis, esym_bind, esym->st_shndx, esym_vis); -#endif - tcc_error_noabort("'%s' defined twice", name); - } - } - else - { - esym->st_other = other; - do_patch: - esym->st_info = ELFW(ST_INFO)(sym_bind, sym_type); - esym->st_shndx = shndx; - esym->st_value = value; - esym->st_size = size; - } - } - else - { - do_def: - sym_index = put_elf_sym(s, value, size, ELFW(ST_INFO)(sym_bind, sym_type), other, shndx, name); - /* Signal the --start-group rescan loop only when a genuinely NEW - undefined symbol appears. Resolving an existing undef (do_patch) - does not create new work for other archives. */ - if (shndx == SHN_UNDEF) - s1->new_undef_sym = 1; - } - return sym_index; -} - -/* put relocation */ -ST_FUNC void put_elf_reloca(Section *symtab, Section *s, unsigned long offset, int type, int symbol, addr_t addend) -{ - TCCState *s1 = s->s1; - char buf[256]; - Section *sr; - ElfW_Rel *rel; - - /* Validate symbol index. - Skip the check for R_RELATIVE relocations to the GOT: put_got_entry() - smuggles a symtab_section index through a relocation nominally linked - to s1->dynsym, so the index is valid for symtab_section, not for the - symtab passed here. See the "Hack alarm" comment in put_got_entry. */ - if (type != R_RELATIVE) - { - int num_syms = symtab->data_offset / sizeof(ElfW(Sym)); - if (symbol < 0 || symbol >= num_syms) - { - return; /* Skip invalid symbol index */ - } - } - - sr = s->reloc; - if (!sr) - { - /* if no relocation section, create it */ - snprintf(buf, sizeof(buf), REL_SECTION_FMT, s->name); - /* if the symtab is allocated, then we consider the relocation - are also */ - sr = new_section(s->s1, buf, SHT_RELX, symtab->sh_flags); - sr->sh_entsize = sizeof(ElfW_Rel); - sr->link = symtab; - sr->sh_info = s->sh_num; - s->reloc = sr; - } - rel = section_ptr_add(sr, sizeof(ElfW_Rel)); - rel->r_offset = offset; - rel->r_info = ELFW(R_INFO)(symbol, type); -#if SHT_RELX == SHT_RELA - rel->r_addend = addend; -#endif - if (SHT_RELX != SHT_RELA && addend) - tcc_error_noabort("non-zero addend on REL architecture"); -} - -ST_FUNC void put_elf_reloc(Section *symtab, Section *s, unsigned long offset, int type, int symbol) -{ - put_elf_reloca(symtab, s, offset, type, symbol, 0); -} - -ST_FUNC struct sym_attr *get_sym_attr(TCCState *s1, int index, int alloc) -{ - int n; - struct sym_attr *tab; - - if (index >= s1->nb_sym_attrs) - { - if (!alloc) - return s1->sym_attrs; - /* find immediately bigger power of 2 and reallocate array */ - n = 1; - while (index >= n) - n *= 2; - tab = tcc_realloc(s1->sym_attrs, n * sizeof(*s1->sym_attrs)); - s1->sym_attrs = tab; - memset(s1->sym_attrs + s1->nb_sym_attrs, 0, (n - s1->nb_sym_attrs) * sizeof(*s1->sym_attrs)); - s1->nb_sym_attrs = n; - } - return &s1->sym_attrs[index]; -} - -static void update_relocs(TCCState *s1, Section *s, int *old_to_new_syms, int first_sym) -{ - int i, type, sym_index; - Section *sr; - ElfW_Rel *rel; - - for (i = 1; i < s1->nb_sections; i++) - { - sr = s1->sections[i]; - if (sr->sh_type == SHT_RELX && sr->link == s) - { - for_each_elem(sr, 0, rel, ElfW_Rel) - { - sym_index = ELFW(R_SYM)(rel->r_info); - type = ELFW(R_TYPE)(rel->r_info); - if ((sym_index -= first_sym) < 0) - continue; /* zero sym_index in reloc (can happen with asm) */ - sym_index = old_to_new_syms[sym_index]; - rel->r_info = ELFW(R_INFO)(sym_index, type); - } - } - } -} - -/* In an ELF file symbol table, the local symbols must appear below - the global and weak ones. Since TCC cannot sort it while generating - the code, we must do it after. All the relocation tables are also - modified to take into account the symbol table sorting */ -ST_FUNC void tcc_elf_sort_syms(TCCState *s1, Section *s) -{ - int *old_to_new_syms; - ElfW(Sym) * new_syms; - int nb_syms, i; - ElfW(Sym) * p, *q; - - nb_syms = s->data_offset / sizeof(ElfW(Sym)); - new_syms = tcc_malloc(nb_syms * sizeof(ElfW(Sym))); - old_to_new_syms = tcc_malloc(nb_syms * sizeof(int)); - - /* first pass for local symbols */ - p = (ElfW(Sym) *)s->data; - q = new_syms; - for (i = 0; i < nb_syms; i++) - { - if (ELFW(ST_BIND)(p->st_info) == STB_LOCAL) - { - old_to_new_syms[i] = q - new_syms; - *q++ = *p; - } - p++; - } - /* save the number of local symbols in section header */ - if (s->sh_size) /* this 'if' makes IDA happy */ - s->sh_info = q - new_syms; - - /* then second pass for non local symbols */ - p = (ElfW(Sym) *)s->data; - for (i = 0; i < nb_syms; i++) - { - if (ELFW(ST_BIND)(p->st_info) != STB_LOCAL) - { - old_to_new_syms[i] = q - new_syms; - *q++ = *p; - } - p++; - } - - /* we copy the new symbols to the old */ - memcpy(s->data, new_syms, nb_syms * sizeof(ElfW(Sym))); - tcc_free(new_syms); - - update_relocs(s1, s, old_to_new_syms, 0); - tcc_free(old_to_new_syms); -} - -#ifndef ELF_OBJ_ONLY -/* See: https://flapenguin.me/elf-dt-gnu-hash */ -#define ELFCLASS_BITS (PTR_SIZE * 8) - -static Section *create_gnu_hash(TCCState *s1) -{ - int nb_syms, i, ndef, nbuckets, symoffset, bloom_size, bloom_shift; - ElfW(Sym) * p; - Section *gnu_hash; - Section *dynsym = s1->dynsym; - Elf32_Word *ptr; - - gnu_hash = new_section(s1, ".gnu.hash", SHT_GNU_HASH, SHF_ALLOC); - gnu_hash->link = dynsym->hash->link; - - nb_syms = dynsym->data_offset / sizeof(ElfW(Sym)); - - /* count def symbols */ - ndef = 0; - p = (ElfW(Sym) *)dynsym->data; - for (i = 0; i < nb_syms; i++, p++) - ndef += p->st_shndx != SHN_UNDEF; - - /* calculate gnu hash sizes and fill header */ - nbuckets = ndef / 4 + 1; - symoffset = nb_syms - ndef; - bloom_shift = PTR_SIZE == 8 ? 6 : 5; - bloom_size = 1; /* must be power of two */ - while (ndef >= bloom_size * (1 << (bloom_shift - 3))) - bloom_size *= 2; - ptr = section_ptr_add(gnu_hash, 4 * 4 + PTR_SIZE * bloom_size + nbuckets * 4 + ndef * 4); - ptr[0] = nbuckets; - ptr[1] = symoffset; - ptr[2] = bloom_size; - ptr[3] = bloom_shift; - return gnu_hash; -} - -static Elf32_Word elf_gnu_hash(const unsigned char *name) -{ - Elf32_Word h = 5381; - unsigned char c; - - while ((c = *name++)) - h = h * 33 + c; - return h; -} - -static void update_gnu_hash(TCCState *s1, Section *gnu_hash) -{ - int *old_to_new_syms; - ElfW(Sym) * new_syms; - int nb_syms, i, nbuckets, bloom_size, bloom_shift; - ElfW(Sym) * p, *q; - Section *vs; - Section *dynsym = s1->dynsym; - Elf32_Word *ptr, *buckets, *chain, *hash; - unsigned int *nextbuck; - addr_t *bloom; - unsigned char *strtab; - struct - { - int first, last; - } *buck; - - strtab = dynsym->link->data; - nb_syms = dynsym->data_offset / sizeof(ElfW(Sym)); - new_syms = tcc_malloc(nb_syms * sizeof(ElfW(Sym))); - old_to_new_syms = tcc_malloc(nb_syms * sizeof(int)); - hash = tcc_malloc(nb_syms * sizeof(Elf32_Word)); - nextbuck = tcc_malloc(nb_syms * sizeof(int)); - - /* calculate hashes and copy undefs */ - p = (ElfW(Sym) *)dynsym->data; - q = new_syms; - for (i = 0; i < nb_syms; i++, p++) - { - if (p->st_shndx == SHN_UNDEF) - { - old_to_new_syms[i] = q - new_syms; - *q++ = *p; - } - else - hash[i] = elf_gnu_hash(strtab + p->st_name); - } - - ptr = (Elf32_Word *)gnu_hash->data; - nbuckets = ptr[0]; - bloom_size = ptr[2]; - bloom_shift = ptr[3]; - bloom = (addr_t *)(void *)&ptr[4]; - buckets = (Elf32_Word *)(void *)&bloom[bloom_size]; - chain = &buckets[nbuckets]; - buck = tcc_malloc(nbuckets * sizeof(*buck)); - - if (gnu_hash->data_offset != 4 * 4 + PTR_SIZE * bloom_size + nbuckets * 4 + (nb_syms - (q - new_syms)) * 4) - tcc_error_noabort("gnu_hash size incorrect"); - - /* find buckets */ - for (i = 0; i < nbuckets; i++) - buck[i].first = -1; - - p = (ElfW(Sym) *)dynsym->data; - for (i = 0; i < nb_syms; i++, p++) - if (p->st_shndx != SHN_UNDEF) - { - int bucket = hash[i] % nbuckets; - - if (buck[bucket].first == -1) - buck[bucket].first = buck[bucket].last = i; - else - { - nextbuck[buck[bucket].last] = i; - buck[bucket].last = i; - } - } - - /* fill buckets/chains/bloom and sort symbols */ - p = (ElfW(Sym) *)dynsym->data; - for (i = 0; i < nbuckets; i++) - { - int cur = buck[i].first; - - if (cur != -1) - { - buckets[i] = q - new_syms; - for (;;) - { - old_to_new_syms[cur] = q - new_syms; - *q++ = p[cur]; - *chain++ = hash[cur] & ~1; - bloom[(hash[cur] / ELFCLASS_BITS) % bloom_size] |= - (addr_t)1 << (hash[cur] % ELFCLASS_BITS) | (addr_t)1 << ((hash[cur] >> bloom_shift) % ELFCLASS_BITS); - if (cur == buck[i].last) - break; - cur = nextbuck[cur]; - } - chain[-1] |= 1; - } - } - - memcpy(dynsym->data, new_syms, nb_syms * sizeof(ElfW(Sym))); - tcc_free(new_syms); - tcc_free(hash); - tcc_free(buck); - tcc_free(nextbuck); - - update_relocs(s1, dynsym, old_to_new_syms, 0); - - /* modify the versions */ - vs = versym_section; - if (vs) - { - ElfW(Half) * newver, *versym = (ElfW(Half) *)vs->data; - - if (1 /*versym*/) - { - newver = tcc_malloc(nb_syms * sizeof(*newver)); - for (i = 0; i < nb_syms; i++) - newver[old_to_new_syms[i]] = versym[i]; - memcpy(vs->data, newver, nb_syms * sizeof(*newver)); - tcc_free(newver); - } - } - - tcc_free(old_to_new_syms); - - /* rebuild hash */ - ptr = (Elf32_Word *)dynsym->hash->data; - rebuild_hash(dynsym, ptr[0]); -} -#endif /* ELF_OBJ_ONLY */ - -/* relocate symbol table, resolve undefined symbols if do_resolve is - true and output error if undefined symbol. */ -ST_FUNC void relocate_syms(TCCState *s1, Section *symtab, int do_resolve) -{ - ElfW(Sym) * sym; - int sym_bind, sh_num; - const char *name; - int sym_idx = 0; - - for_each_elem(symtab, 1, sym, ElfW(Sym)) - { - sym_idx++; - sh_num = sym->st_shndx; - if (sh_num == SHN_UNDEF) - { - if (do_resolve == 2) /* relocating dynsym */ - continue; - /* Validate st_name offset before using it */ - if (sym->st_name >= s1->symtab->link->data_offset) - { - tcc_error_noabort("internal error: symbol %d has invalid st_name offset 0x%x (strtab size: 0x%lx)", sym_idx, - sym->st_name, (unsigned long)s1->symtab->link->data_offset); - continue; - } - name = (char *)s1->symtab->link->data + sym->st_name; - /* Debug: print symbol info when name is empty or looks wrong */ - /* Use ld.so to resolve symbol for us (for tcc -run) */ - if (do_resolve) - { -#if defined TCC_IS_NATIVE && !defined TCC_TARGET_PE - /* dlsym() needs the undecorated name. */ - void *addr = dlsym(RTLD_DEFAULT, &name[s1->leading_underscore]); -#if TARGETOS_OpenBSD || TARGETOS_FreeBSD || TARGETOS_NetBSD || TARGETOS_ANDROID || TARGETOS_YasOS - if (addr == NULL) - { - int i; - for (i = 0; i < s1->nb_loaded_dlls; i++) - if ((addr = dlsym(s1->loaded_dlls[i]->handle, name))) - break; - } -#endif - if (addr) - { - sym->st_value = (addr_t)addr; - LOG_RELOC("relocate_sym: %s -> 0x%lx", name, sym->st_value); - goto found; - } -#endif - /* if dynamic symbol exist, it will be used in relocate_section */ - } - else if (s1->dynsym && find_elf_sym(s1->dynsym, name)) - goto found; - /* XXX: _fp_hw seems to be part of the ABI, so we ignore - it */ - if (!strcmp(name, "_fp_hw")) - goto found; - /* only weak symbols are accepted to be undefined. Their - value is zero */ - sym_bind = ELFW(ST_BIND)(sym->st_info); - if (sym_bind == STB_WEAK) - sym->st_value = 0; - else - tcc_error_noabort("undefined symbol '%s'", name); - } - else if (sh_num < SHN_LORESERVE) - { - /* add section base */ - sym->st_value += s1->sections[sym->st_shndx]->sh_addr; - } - found:; - } -} -/* Add a relocation patch for lazy section streaming. - * Uses dynamic arrays instead of linked list for memory efficiency. - * Each patch is 8 bytes (2 x uint32_t) vs 24 bytes with linked list. */ -static void add_reloc_patch(Section *s, uint32_t offset, uint32_t value) -{ - /* Ensure capacity */ - if (s->nb_reloc_patches >= s->alloc_reloc_patches) - { - int new_alloc = s->alloc_reloc_patches ? s->alloc_reloc_patches * 2 : 16; - s->reloc_patch_offsets = tcc_realloc(s->reloc_patch_offsets, new_alloc * sizeof(uint32_t)); - s->reloc_patch_values = tcc_realloc(s->reloc_patch_values, new_alloc * sizeof(uint32_t)); - s->alloc_reloc_patches = new_alloc; - } - /* Append patch */ - s->reloc_patch_offsets[s->nb_reloc_patches] = offset; - s->reloc_patch_values[s->nb_reloc_patches] = value; - s->nb_reloc_patches++; -} - -/* Free all relocation patches for a section */ -static void free_reloc_patches(Section *s) -{ - tcc_free(s->reloc_patch_offsets); - tcc_free(s->reloc_patch_values); - s->reloc_patch_offsets = NULL; - s->reloc_patch_values = NULL; - s->nb_reloc_patches = 0; - s->alloc_reloc_patches = 0; -} - -/* relocate a given section (CPU dependent) by applying the relocations - in the associated relocation section */ -static void relocate_section(TCCState *s1, Section *s, Section *sr) -{ - ElfW_Rel *rel; - ElfW(Sym) * sym; - int type, sym_index; - unsigned char *ptr; - addr_t tgt, addr; - int is_dwarf = s->sh_num >= s1->dwlo && s->sh_num < s1->dwhi; - - /* Always materialize non-debug sections */ - if (!is_dwarf) - section_ensure_loaded(s1, s); - - section_ensure_loaded(s1, sr); - section_ensure_loaded(s1, symtab_section); - - /* For lazy debug sections, we store patches instead of materializing */ - if (is_dwarf && s->lazy && !s->materialized) - { - for_each_elem(sr, 0, rel, ElfW_Rel) - { - sym_index = ELFW(R_SYM)(rel->r_info); - sym = &((ElfW(Sym) *)symtab_section->data)[sym_index]; - type = ELFW(R_TYPE)(rel->r_info); - tgt = sym->st_value; -#if SHT_RELX == SHT_RELA - tgt += rel->r_addend; -#endif - if (type == R_DATA_32DW) - { - uint32_t value; - if (sym->st_shndx >= s1->dwlo && sym->st_shndx < s1->dwhi) - /* dwarf-to-dwarf section relocation (e.g., .debug_info -> .debug_str) */ - value = tgt - s1->sections[sym->st_shndx]->sh_addr; - else - /* code/data reference from debug section (e.g., DW_AT_low_pc -> .text) */ - value = tgt; - add_reloc_patch(s, (uint32_t)rel->r_offset, value); - } - } - return; - } - - qrel = (ElfW_Rel *)sr->data; - - for_each_elem(sr, 0, rel, ElfW_Rel) - { - ptr = s->data + rel->r_offset; - sym_index = ELFW(R_SYM)(rel->r_info); - sym = &((ElfW(Sym) *)symtab_section->data)[sym_index]; - type = ELFW(R_TYPE)(rel->r_info); - tgt = sym->st_value; -#if SHT_RELX == SHT_RELA - tgt += rel->r_addend; -#endif - if (is_dwarf && type == R_DATA_32DW && sym->st_shndx >= s1->dwlo && sym->st_shndx < s1->dwhi) - { - /* dwarf section relocation to each other */ - add32le(ptr, tgt - s1->sections[sym->st_shndx]->sh_addr); - continue; - } - addr = s->sh_addr + rel->r_offset; - relocate(s1, rel, type, ptr, addr, tgt); - } -#ifndef ELF_OBJ_ONLY - /* if the relocation is allocated, we change its symbol table */ - if (sr->sh_flags & SHF_ALLOC) - { - sr->link = s1->dynsym; - if (s1->output_type & TCC_OUTPUT_DYN) - { - size_t r = (uint8_t *)qrel - sr->data; - sr->data_offset = sr->sh_size = r; -#ifdef CONFIG_TCC_PIE - if (r && (s->sh_flags & SHF_EXECINSTR)) - tcc_warning("%d relocations to %s", (unsigned)(r / sizeof *qrel), s->name); -#endif - } - } -#endif -} - -/* relocate all sections */ -ST_FUNC void relocate_sections(TCCState *s1) -{ - int i; - Section *s, *sr; - - for (i = 1; i < s1->nb_sections; ++i) - { - sr = s1->sections[i]; - if (sr->sh_type != SHT_RELX) - continue; - s = s1->sections[sr->sh_info]; -#ifdef TCC_TARGET_ARM - /* Skip relocations for suppressed ARM exception index sections. - set_sec_sizes() clears SHF_ALLOC on .ARM.exidx (stack unwinding - not used on bare-metal), but relocation sections survive. If we - still process them, R_ARM_PREL31 entries that reference orphan - .ARM.extab (placed far away in RAM) overflow the 31-bit range. */ - if (s->sh_type == SHT_ARM_EXIDX && !(s->sh_flags & SHF_ALLOC)) - continue; -#endif -#ifndef TCC_TARGET_MACHO - if (s != s1->got || s1->static_link || s1->output_type == TCC_OUTPUT_MEMORY) -#endif - { - relocate_section(s1, s, sr); - } -#ifndef ELF_OBJ_ONLY - if (sr->sh_flags & SHF_ALLOC) - { - ElfW_Rel *rel; - /* relocate relocation table in 'sr' */ - for_each_elem(sr, 0, rel, ElfW_Rel) rel->r_offset += s->sh_addr; - } -#endif - } -} - -#ifndef ELF_OBJ_ONLY -/* count the number of dynamic relocations so that we can reserve - their space */ -static int prepare_dynamic_rel(TCCState *s1, Section *sr) -{ - int count = 0; -#if defined(TCC_TARGET_I386) || defined(TCC_TARGET_X86_64) || defined(TCC_TARGET_ARM) || defined(TCC_TARGET_ARM64) || \ - defined(TCC_TARGET_RISCV64) - ElfW_Rel *rel; - for_each_elem(sr, 0, rel, ElfW_Rel) - { - int sym_index = ELFW(R_SYM)(rel->r_info); - int type = ELFW(R_TYPE)(rel->r_info); - switch (type) - { -#if defined(TCC_TARGET_I386) - case R_386_32: - if (!get_sym_attr(s1, sym_index, 0)->dyn_index && - ((ElfW(Sym) *)symtab_section->data + sym_index)->st_shndx == SHN_UNDEF) - { - /* don't fixup unresolved (weak) symbols */ - rel->r_info = ELFW(R_INFO)(sym_index, R_386_RELATIVE); - break; - } -#elif defined(TCC_TARGET_X86_64) - case R_X86_64_32: - case R_X86_64_32S: - case R_X86_64_64: -#elif defined(TCC_TARGET_ARM) - case R_ARM_ABS32: - case R_ARM_TARGET1: -#elif defined(TCC_TARGET_ARM64) - case R_AARCH64_ABS32: - case R_AARCH64_ABS64: -#elif defined(TCC_TARGET_RISCV64) - case R_RISCV_32: - case R_RISCV_64: -#endif - count++; - break; -#if defined(TCC_TARGET_I386) - case R_386_PC32: -#elif defined(TCC_TARGET_X86_64) - case R_X86_64_PC32: - { - ElfW(Sym) *sym = &((ElfW(Sym) *)symtab_section->data)[sym_index]; - /* Hidden defined symbols can and must be resolved locally. - We're misusing a PLT32 reloc for this, as that's always - resolved to its address even in shared libs. */ - if (sym->st_shndx != SHN_UNDEF && ELFW(ST_VISIBILITY)(sym->st_other) == STV_HIDDEN) - { - rel->r_info = ELFW(R_INFO)(sym_index, R_X86_64_PLT32); - break; - } - } -#elif defined(TCC_TARGET_ARM64) - case R_AARCH64_PREL32: -#endif - if (s1->output_type != TCC_OUTPUT_DLL) - break; - if (get_sym_attr(s1, sym_index, 0)->dyn_index) - count++; - break; - default: - break; - } - } -#endif - return count; -} -#endif - -#ifdef NEED_BUILD_GOT -int build_got(TCCState *s1) -{ - /* if no got, then create it */ - s1->got = new_section(s1, ".got", SHT_PROGBITS, SHF_ALLOC | SHF_WRITE); - s1->got->sh_entsize = 8; - /* keep space for _DYNAMIC pointer and two dummy got entries */ -#if defined(TCC_TARGET_YASOS) - /* + a reserved slot (index YAFF_RODATA_ANCHOR_GOT_INDEX = 3) holding the - * runtime base of the shared .rodata segment. Reserved up front so its GOT - * offset is a compile-time constant the codegen addresses as [R9,#24], - * independent of the final GOT layout; the loader fills it (the YAFF writer - * emits its relocation when -share-rodata is active). */ - section_ptr_add(s1->got, 4 * PTR_SIZE * 2); -#else - section_ptr_add(s1->got, 3 * PTR_SIZE * 2); -#endif - return set_elf_sym(symtab_section, 0, 0, ELFW(ST_INFO)(STB_GLOBAL, STT_OBJECT), 0, s1->got->sh_num, - "_GLOBAL_OFFSET_TABLE_"); -} - -/* Create a GOT and (for function call) a PLT entry corresponding to a symbol - in s1->symtab. When creating the dynamic symbol table entry for the GOT - relocation, use 'size' and 'info' for the corresponding symbol metadata. - Returns the offset of the GOT or (if any) PLT entry. */ -static struct sym_attr *put_got_entry(TCCState *s1, int dyn_reloc_type, int sym_index) -{ - int need_plt_entry; - const char *name; - ElfW(Sym) * sym; - struct sym_attr *attr; - unsigned got_offset; - char plt_name[200]; - int len; - Section *s_rel; - - need_plt_entry = (dyn_reloc_type == R_JMP_SLOT); - attr = get_sym_attr(s1, sym_index, 1); - - /* In case a function is both called and its address taken 2 GOT entries - are created, one for taking the address (GOT) and the other for the PLT - entry (PLTGOT). */ - if (need_plt_entry ? attr->plt_offset : attr->got_offset) - return attr; - - s_rel = s1->got; - if (need_plt_entry) - { - if (!s1->plt) - { - s1->plt = new_section(s1, ".plt", SHT_PROGBITS, SHF_ALLOC | SHF_EXECINSTR); - s1->plt->sh_entsize = 4; - } - s_rel = s1->plt; - } - - /* create the GOT entry */ - got_offset = s1->got->data_offset; - section_ptr_add(s1->got, PTR_SIZE * 2); - - /* Create the GOT relocation that will insert the address of the object or - function of interest in the GOT entry. This is a static relocation for - memory output (dlsym will give us the address of symbols) and dynamic - relocation otherwise (executable and DLLs). The relocation should be - done lazily for GOT entry with *_JUMP_SLOT relocation type (the one - associated to a PLT entry) but is currently done at load time for an - unknown reason. */ - - sym = &((ElfW(Sym) *)symtab_section->data)[sym_index]; - name = (char *)symtab_section->link->data + sym->st_name; - // printf("sym %d %s\n", need_plt_entry, name); - - if (s1->dynsym) - { - if (ELFW(ST_BIND)(sym->st_info) == STB_LOCAL) - { - /* Hack alarm. We don't want to emit dynamic symbols - and symbol based relocs for STB_LOCAL symbols, but rather - want to resolve them directly. At this point the symbol - values aren't final yet, so we must defer this. We will later - have to create a RELATIVE reloc anyway, so we misuse the - relocation slot to smuggle the symbol reference until - fill_local_got_entries. Not that the sym_index is - relative to symtab_section, not s1->dynsym! Nevertheless - we use s1->dyn_sym so that if this is the first call - that got->reloc is correctly created. Also note that - RELATIVE relocs are not normally created for the .got, - so the types serves as a marker for later (and is retained - also for the final output, which is okay because then the - got is just normal data). */ - put_elf_reloc(s1->dynsym, s1->got, got_offset, R_RELATIVE, sym_index); - } - else - { - if (0 == attr->dyn_index) - attr->dyn_index = set_elf_sym(s1->dynsym, sym->st_value, sym->st_size, sym->st_info, 0, sym->st_shndx, name); - put_elf_reloc(s1->dynsym, s_rel, got_offset, dyn_reloc_type, attr->dyn_index); - } - } - else - { - put_elf_reloc(symtab_section, s1->got, got_offset, dyn_reloc_type, sym_index); - } - - if (need_plt_entry) - { - attr->plt_offset = create_plt_entry(s1, got_offset, attr); - - /* create a symbol 'sym@plt' for the PLT jump vector */ - len = strlen(name); - if (len > sizeof plt_name - 5) - len = sizeof plt_name - 5; - memcpy(plt_name, name, len); - strcpy(plt_name + len, "@plt"); - attr->plt_sym = - put_elf_sym(s1->symtab, attr->plt_offset, 0, ELFW(ST_INFO)(STB_GLOBAL, STT_FUNC), 0, s1->plt->sh_num, plt_name); - } - else - { - attr->got_offset = got_offset; - } - - return attr; -} - -/* build GOT and PLT entries */ -/* Two passes because R_JMP_SLOT should become first. Some targets - (arm, arm64) do not allow mixing R_JMP_SLOT and R_GLOB_DAT. */ -ST_FUNC void build_got_entries(TCCState *s1, int got_sym) -{ - Section *s; - ElfW_Rel *rel; - ElfW(Sym) * sym; - int i, type, gotplt_entry, reloc_type, sym_index; - struct sym_attr *attr; - int pass = 0; -redo: - for (i = 1; i < s1->nb_sections; i++) - { - s = s1->sections[i]; - if (s->sh_type != SHT_RELX) - continue; - /* no need to handle got relocations */ - if (s->link != symtab_section) - continue; - /* Relocations from non-allocated sections (DWARF) never execute at - * runtime and must not fabricate GOT/PLT entries. Besides being pure - * GOT bloat, under --gc-sections they reference the SHN_ABS 0xFFFFFFFF - * tombstones of collected code, and AUTO_GOTPLT_ENTRY would turn each - * into an out-of-range GOT slot that the yasld loader refuses to map - * (boot-time "Can't find section for GOT[n]: OffsetOutOfRange"). */ - if (s->sh_info < s1->nb_sections && s1->sections[s->sh_info] && - !(s1->sections[s->sh_info]->sh_flags & SHF_ALLOC)) - continue; - for_each_elem(s, 0, rel, ElfW_Rel) - { - type = ELFW(R_TYPE)(rel->r_info); - gotplt_entry = gotplt_entry_type(type); - if (gotplt_entry == -1) - { - tcc_error_noabort("Unknown relocation type for got: %d", type); - continue; - } - sym_index = ELFW(R_SYM)(rel->r_info); - sym = &((ElfW(Sym) *)symtab_section->data)[sym_index]; - - if (gotplt_entry == NO_GOTPLT_ENTRY) - { - continue; - } - - /* Automatically create PLT/GOT [entry] if it is an undefined - reference (resolved at runtime), or the symbol is absolute, - probably created by tcc_add_symbol, and thus on 64-bit - targets might be too far from application code. */ - if (gotplt_entry == AUTO_GOTPLT_ENTRY) - { - if (sym->st_shndx == SHN_UNDEF) - { - ElfW(Sym) * esym; - int dynindex; - if (!PCRELATIVE_DLLPLT && (s1->output_type & TCC_OUTPUT_DYN)) - continue; - /* Relocations for UNDEF symbols would normally need - to be transferred into the executable or shared object. - If that were done AUTO_GOTPLT_ENTRY wouldn't exist. - But TCC doesn't do that (at least for exes), so we - need to resolve all such relocs locally. And that - means PLT slots for functions in DLLs and COPY relocs for - data symbols. COPY relocs were generated in - bind_exe_dynsyms (and the symbol adjusted to be defined), - and for functions we were generated a dynamic symbol - of function type. */ - if (s1->dynsym) - { - /* dynsym isn't set for -run :-/ */ - dynindex = get_sym_attr(s1, sym_index, 0)->dyn_index; - esym = (ElfW(Sym) *)s1->dynsym->data + dynindex; - if (dynindex && (ELFW(ST_TYPE)(esym->st_info) == STT_FUNC || - (ELFW(ST_TYPE)(esym->st_info) == STT_NOTYPE && ELFW(ST_TYPE)(sym->st_info) == STT_FUNC))) - goto jmp_slot; - } - } - else if (sym->st_shndx == SHN_ABS) - { - if (sym->st_value == 0) /* from tcc_add_btstub() */ - continue; -#ifndef TCC_TARGET_ARM - if (PTR_SIZE != 8) - continue; -#endif - /* from tcc_add_symbol(): on 64 bit platforms these - need to go through .got */ - } - else - continue; - } - -#ifdef TCC_TARGET_X86_64 - if ((type == R_X86_64_PLT32 || type == R_X86_64_PC32) && sym->st_shndx != SHN_UNDEF && - (ELFW(ST_VISIBILITY)(sym->st_other) != STV_DEFAULT || ELFW(ST_BIND)(sym->st_info) == STB_LOCAL || - s1->output_type & TCC_OUTPUT_EXE)) - { - if (pass != 0) - continue; - rel->r_info = ELFW(R_INFO)(sym_index, R_X86_64_PC32); - continue; - } -#endif - reloc_type = code_reloc(type); - if (reloc_type == -1) - { - tcc_error_noabort("Unknown relocation type: %d", type); - continue; - } - - if (reloc_type != 0) - { - jmp_slot: - if (pass != 0) - continue; - reloc_type = R_JMP_SLOT; - } - else - { - if (pass != 1) - continue; - reloc_type = R_GLOB_DAT; - } - - if (!s1->got) - got_sym = build_got(s1); - - if (gotplt_entry == BUILD_GOT_ONLY) - continue; - - attr = put_got_entry(s1, reloc_type, sym_index); - - if (reloc_type == R_JMP_SLOT) - rel->r_info = ELFW(R_INFO)(attr->plt_sym, type); - } - } - if (++pass < 2) - goto redo; - /* .rel.plt refers to .got actually */ - if (s1->plt && s1->plt->reloc) - s1->plt->reloc->sh_info = s1->got->sh_num; - if (got_sym) /* set size */ - ((ElfW(Sym) *)symtab_section->data)[got_sym].st_size = s1->got->data_offset; -} -#endif /* def NEED_BUILD_GOT */ - -ST_FUNC int set_global_sym(TCCState *s1, const char *name, Section *sec, addr_t offs) -{ - int shn = sec ? sec->sh_num : offs || !name ? SHN_ABS : SHN_UNDEF; - if (sec && offs == -1) - offs = sec->data_offset; - return set_elf_sym(symtab_section, offs, 0, ELFW(ST_INFO)(name ? STB_GLOBAL : STB_LOCAL, STT_NOTYPE), 0, shn, name); -} - -static void add_init_array_defines(TCCState *s1, const char *section_name) -{ - Section *s; - addr_t end_offset; - char buf[1024]; - s = have_section(s1, section_name); - if (!s || !(s->sh_flags & SHF_ALLOC)) - { - end_offset = 0; - s = text_section; - } - else - { - end_offset = s->data_offset; - } - snprintf(buf, sizeof(buf), "__%s_start", section_name + 1); - set_global_sym(s1, buf, s, 0); - snprintf(buf, sizeof(buf), "__%s_end", section_name + 1); - set_global_sym(s1, buf, s, end_offset); -} - -ST_FUNC void add_array(TCCState *s1, const char *sec, int c) -{ - Section *s; - s = find_section(s1, sec); - s->sh_flags = shf_RELRO; - s->sh_type = sec[1] == 'i' ? SHT_INIT_ARRAY : SHT_FINI_ARRAY; - put_elf_reloc(s1->symtab, s, s->data_offset, R_DATA_PTR, c); - section_ptr_add(s, PTR_SIZE); -} - -#ifdef CONFIG_TCC_BCHECK -ST_FUNC void tcc_add_bcheck(TCCState *s1) -{ - if (0 == s1->do_bounds_check) - return; - section_ptr_add(bounds_section, sizeof(addr_t)); -} -#endif - -/* set symbol to STB_LOCAL and resolve. The point is to not export it as - a dynamic symbol to allow so's to have one each with a different value. */ -static void set_local_sym(TCCState *s1, const char *name, Section *s, int offset) -{ - int c = find_elf_sym(s1->symtab, name); - if (c) - { - ElfW(Sym) *esym = (ElfW(Sym) *)s1->symtab->data + c; - esym->st_info = ELFW(ST_INFO)(STB_LOCAL, STT_NOTYPE); - esym->st_value = offset; - esym->st_shndx = s->sh_num; - } -} - -/* avoid generating debug/test_coverage code for stub functions */ -static void tcc_compile_string_no_debug(TCCState *s, const char *str) -{ - int save_do_debug = s->do_debug; - int save_test_coverage = s->test_coverage; - - s->do_debug = 0; - s->test_coverage = 0; - tcc_compile_string(s, str); - s->do_debug = save_do_debug; - s->test_coverage = save_test_coverage; -} - -#ifdef CONFIG_TCC_BACKTRACE -static void put_ptr(TCCState *s1, Section *s, int offs) -{ - int c; - c = set_global_sym(s1, NULL, s, offs); - s = data_section; - put_elf_reloc(s1->symtab, s, s->data_offset, R_DATA_PTR, c); - section_ptr_add(s, PTR_SIZE); -} - -ST_FUNC void tcc_add_btstub(TCCState *s1) -{ - Section *s; - int n, o, *p; - CString cstr; - const char *__rt_info = &"___rt_info"[!s1->leading_underscore]; - - s = data_section; - /* Align to PTR_SIZE */ - section_ptr_add(s, -s->data_offset & (PTR_SIZE - 1)); - o = s->data_offset; - /* create a struct rt_context (see tccrun.c) */ - if (s1->dwarf) - { - put_ptr(s1, dwarf_line_section, 0); - put_ptr(s1, dwarf_line_section, -1); - if (s1->dwarf >= 5) - put_ptr(s1, dwarf_line_str_section, 0); - else - put_ptr(s1, dwarf_str_section, 0); - } - else - { - /* stabs removed - emit zeroes as placeholder */ - section_ptr_add(s, 3 * PTR_SIZE); - } - - /* skip esym_start/esym_end/elf_str (not loaded) */ - section_ptr_add(s, 3 * PTR_SIZE); - - if (s1->output_type == TCC_OUTPUT_MEMORY && 0 == s1->dwarf) - { - put_ptr(s1, text_section, 0); - } - else - { - /* prog_base : local nameless symbol with offset 0 at SHN_ABS */ - put_ptr(s1, NULL, 0); -#if defined TCC_TARGET_MACHO - /* adjust for __PAGEZERO */ - if (s1->dwarf == 0 && s1->output_type == TCC_OUTPUT_EXE) - write64le(data_section->data + data_section->data_offset - PTR_SIZE, (uint64_t)1 << 32); -#endif - } - n = 3 * PTR_SIZE; -#ifdef CONFIG_TCC_BCHECK - if (s1->do_bounds_check) - { - put_ptr(s1, bounds_section, 0); - n -= PTR_SIZE; - } -#endif - section_ptr_add(s, n); - p = section_ptr_add(s, 2 * sizeof(int)); - p[0] = s1->rt_num_callers; - p[1] = s1->dwarf; - // if (s->data_offset - o != 10*PTR_SIZE + 2*sizeof (int)) exit(99); - - if (s1->output_type == TCC_OUTPUT_MEMORY) - { - set_global_sym(s1, __rt_info, s, o); - return; - } - - cstr_new(&cstr); - cstr_printf(&cstr, "extern void __bt_init(),__bt_exit(),__bt_init_dll();" - "static void *__rt_info[];" - "__attribute__((constructor)) static void __bt_init_rt(){"); -#ifdef TCC_TARGET_PE - if (s1->output_type == TCC_OUTPUT_DLL) -#ifdef CONFIG_TCC_BCHECK - cstr_printf(&cstr, "__bt_init_dll(%d);", s1->do_bounds_check); -#else - cstr_printf(&cstr, "__bt_init_dll(0);"); -#endif -#endif - cstr_printf(&cstr, "__bt_init(__rt_info,%d);}", s1->output_type != TCC_OUTPUT_DLL); - /* In case dlcose is called by application */ - cstr_printf(&cstr, "__attribute__((destructor)) static void __bt_exit_rt(){" - "__bt_exit(__rt_info);}"); - tcc_compile_string_no_debug(s1, cstr.data); - cstr_free(&cstr); - set_local_sym(s1, __rt_info, s, o); -} -#endif /* def CONFIG_TCC_BACKTRACE */ - -static void tcc_tcov_add_file(TCCState *s1, const char *filename) -{ - CString cstr; - void *ptr; - char wd[1024]; - - if (tcov_section == NULL) - return; - section_ptr_add(tcov_section, 1); - write32le(tcov_section->data, tcov_section->data_offset); - - cstr_new(&cstr); - if (filename[0] == '/') - cstr_printf(&cstr, "%s.tcov", filename); - else - { - getcwd(wd, sizeof(wd)); - cstr_printf(&cstr, "%s/%s.tcov", wd, filename); - } - ptr = section_ptr_add(tcov_section, cstr.size + 1); - strcpy((char *)ptr, cstr.data); - unlink((char *)ptr); -#ifdef _WIN32 - normalize_slashes((char *)ptr); -#endif - cstr_free(&cstr); - - cstr_new(&cstr); - cstr_printf(&cstr, "extern char *__tcov_data[];" - "extern void __store_test_coverage ();" - "__attribute__((destructor)) static void __tcov_exit() {" - "__store_test_coverage(__tcov_data);" - "}"); - tcc_compile_string_no_debug(s1, cstr.data); - cstr_free(&cstr); - set_local_sym(s1, &"___tcov_data"[!s1->leading_underscore], tcov_section, 0); -} - -#if !defined TCC_TARGET_PE && !defined TCC_TARGET_MACHO -/* add libc crt1/crti objects */ -ST_FUNC void tccelf_add_crtbegin(TCCState *s1) -{ -#if TARGETOS_OpenBSD - if (s1->output_type != TCC_OUTPUT_DLL) - tcc_add_crt(s1, "crt0.o"); - if (s1->output_type == TCC_OUTPUT_DLL) - tcc_add_crt(s1, "crtbeginS.o"); - else - tcc_add_crt(s1, "crtbegin.o"); -#elif TARGETOS_FreeBSD || TARGETOS_NetBSD - if (s1->output_type != TCC_OUTPUT_DLL) -#if TARGETOS_FreeBSD - tcc_add_crt(s1, "crt1.o"); -#else - tcc_add_crt(s1, "crt0.o"); -#endif - tcc_add_crt(s1, "crti.o"); - if (s1->static_link) - tcc_add_crt(s1, "crtbeginT.o"); - else if (s1->output_type == TCC_OUTPUT_DLL) - tcc_add_crt(s1, "crtbeginS.o"); - else - tcc_add_crt(s1, "crtbegin.o"); -#elif TARGETOS_ANDROID - if (s1->output_type == TCC_OUTPUT_DLL) - tcc_add_crt(s1, "crtbegin_so.o"); - else - tcc_add_crt(s1, "crtbegin_dynamic.o"); -#else - if (s1->output_type != TCC_OUTPUT_DLL) - tcc_add_crt(s1, "crt1.o"); - tcc_add_crt(s1, "crti.o"); -#endif -} - -ST_FUNC void tccelf_add_crtend(TCCState *s1) -{ -#if TARGETOS_OpenBSD - if (s1->output_type == TCC_OUTPUT_DLL) - tcc_add_crt(s1, "crtendS.o"); - else - tcc_add_crt(s1, "crtend.o"); -#elif TARGETOS_FreeBSD || TARGETOS_NetBSD - if (s1->output_type == TCC_OUTPUT_DLL) - tcc_add_crt(s1, "crtendS.o"); - else - tcc_add_crt(s1, "crtend.o"); - tcc_add_crt(s1, "crtn.o"); -#elif TARGETOS_ANDROID - if (s1->output_type == TCC_OUTPUT_DLL) - tcc_add_crt(s1, "crtend_so.o"); - else - tcc_add_crt(s1, "crtend_android.o"); -#else - tcc_add_crt(s1, "crtn.o"); -#endif -} -#endif /* !defined TCC_TARGET_PE && !defined TCC_TARGET_MACHO */ - -#if defined TCC_TARGET_ARM -/* Determine the FP library base name (without lib prefix / extension). - * Same name is used for both static (.a) and shared (.so) variants. - * Returns the name for use with tcc_add_library() or manual path construction. - * - * Returns NULL if hard float ABI (no FP library needed). - */ -static const char *tccelf_get_fp_lib_name(TCCState *s1) -{ - if (s1->float_abi == ARM_HARD_FLOAT) - return NULL; - - /* -mfloat-abi=soft means "no FP instructions anywhere in this image", and the - * runtime the image links is part of the image. Pairing it with a hardware - * -mfpu used to be the only way to say "hardware, behind a call" and so - * quietly linked librp2350fp -- DCP instructions, on a build that had just - * declared it emits none, with a YAFF header that asked the loader for - * nothing. That mode now has its own flag (-mfp-inline=none), which leaves - * this one free to mean what it says. */ - if (s1->float_abi == ARM_SOFT_FLOAT) - return "softfp"; - - if (s1->fpu_type) - { - switch (s1->fpu_type) - { - case ARM_FPU_VFPV4: - case ARM_FPU_FPV4_SP_D16: - case ARM_FPU_FPV5_SP_D16: - return "vfpv4sp"; - case ARM_FPU_RP2350: - /* RP2350 double coprocessor: doubles go through the DCP sequences in - * librp2350fp rather than the generic soft-float implementations. */ - return "rp2350fp"; - case ARM_FPU_FPV5_D16: - case ARM_FPU_NEON: - case ARM_FPU_NEON_VFPV4: - case ARM_FPU_NEON_FP_ARMV8: - return "vfpv5dp"; - default: - break; - } - } - - return "softfp"; -} - -/* Add ARM floating-point library based on compiler flags. - * Selects the correct FP library variant based on -mfpu and -mfloat-abi. - * - * Library names follow the short convention: - * libsoftfp.{a,so} - Pure software floating point - * libvfpv4sp.{a,so} - VFPv4 single-precision HW, double SW - * libvfpv5dp.{a,so} - VFPv5 full double-precision HW - * librp2350fp.{a,so} - RP2350 double coprocessor - * - * On YasOS native (dynamic linking): uses tcc_add_library() which searches - * library_paths for libsoftfp.so in /usr/lib/, etc. - * - * On cross-compiler / static: uses tcc_add_dll() with "fp/libsoftfp.a" path, - * resolved relative to tcc_lib_path (CONFIG_TCCDIR), e.g. lib/tcc/fp/. - * - * Hard float ABI needs no FP library (uses raw FP instructions). - */ -/* Is the __aeabi_ runtime bound as a shared object rather than copied in from - * an archive? -mfp-lib decides; AUTO keeps the historical split, where only - * the on-device compiler binds the shared object. */ -ST_FUNC int tccelf_arm_fp_lib_is_shared(TCCState *s1) -{ - if (s1->static_link) - return 0; - if (s1->fp_lib == ARM_FP_LIB_SHARED) - return 1; - if (s1->fp_lib == ARM_FP_LIB_STATIC) - return 0; -#if TARGETOS_YasOS && defined(TCC_IS_NATIVE) - return 1; -#else - return 0; -#endif -} - -ST_FUNC void tccelf_add_arm_fp_lib(TCCState *s1) -{ - const char *fp_lib = tccelf_get_fp_lib_name(s1); - if (!fp_lib) - { - if (s1->verbose) - printf("Hard float ABI: no FP library needed\n"); - return; - } - - if (tccelf_arm_fp_lib_is_shared(s1)) - { - /* Search library_paths for libXXX.so then libXXX.a (e.g. - * /usr/lib/libsoftfp.so), so the __aeabi_ entry points are resolved by the - * OS's dynamic loader at exec instead of copied into this module. */ - if (s1->verbose) - printf("Adding ARM FP shared library: lib%s\n", fp_lib); - tcc_add_library(s1, fp_lib); - return; - } - - /* Cross-compiler or static link: find via tcc lib path (fp/ subdirectory). - * tcc_add_dll() searches library_paths which includes {B} = tcc_lib_path, - * so "fp/libsoftfp.a" resolves to e.g. lib/tcc/fp/libsoftfp.a */ - { - char fp_path[64]; - snprintf(fp_path, sizeof(fp_path), "fp/lib%s.a", fp_lib); - if (s1->verbose) - printf("Adding ARM FP library: %s\n", fp_path); - tcc_add_dll(s1, fp_path, AFF_PRINT_ERROR); - } -} -#endif - -#ifndef TCC_TARGET_PE -/* add tcc runtime libraries - * - * Two levels of library suppression (matching GCC semantics): - * - * -nostdlib : skip standard libraries (libc, libpthread, libdl, ...) - * but still link compiler runtime (libtcc1, softfp, crt). - * - * -nodefaultlibs : skip everything including compiler runtime (libtcc1, - * softfp, crt). Used when building the runtime libs - * themselves to avoid circular dependencies. - */ -ST_FUNC void tcc_add_runtime(TCCState *s1) -{ - s1->filetype = 0; - - /* -nodefaultlibs: skip all implicit libraries and runtime support */ - if (s1->nodefaultlibs) - return; - -#ifdef CONFIG_TCC_BCHECK - tcc_add_bcheck(s1); -#endif - tcc_add_pragma_libs(s1); - - /* A shared FP runtime has to be bound BEFORE libc. - * - * Every shared object in this rootfs that was ever linked against the FP - * archive absorbed a copy of the __aeabi_ set and re-exports it -- libc.so - * alone exports fifteen doubles entry points. The loader resolves an import - * by walking the module's dependencies in the order the image lists them - * (Module.find_symbol over `children`, appended in import order), so an - * image that named libc first would silently run libc's copy of every - * operation and the -mfpu it was built with would decide nothing. Binding - * the runtime the link actually selected first is what makes the choice - * stick. The archive path keeps the old position: an archive is searched a - * la carte, so it has to come after the objects that reference it. */ -#if defined TCC_TARGET_ARM - if (tccelf_arm_fp_lib_is_shared(s1)) - tccelf_add_arm_fp_lib(s1); -#endif - - /* Standard libraries (skipped by -nostdlib) */ - if (!s1->nostdlib) - { - int lpthread = s1->option_pthread; - -#ifdef CONFIG_TCC_BCHECK - if (s1->do_bounds_check && s1->output_type != TCC_OUTPUT_DLL) - { - tcc_add_support(s1, "bcheck.o"); -#if !(TARGETOS_OpenBSD || TARGETOS_NetBSD) - tcc_add_library(s1, "dl"); -#endif - lpthread = 1; - } -#endif -#ifdef CONFIG_TCC_BACKTRACE - if (s1->do_backtrace) - { - if (s1->output_type & TCC_OUTPUT_EXE) - tcc_add_support(s1, "bt-exe.o"); - if (s1->output_type != TCC_OUTPUT_DLL) - tcc_add_support(s1, "bt-log.o"); - tcc_add_btstub(s1); - lpthread = 1; - } -#endif - if (lpthread) - tcc_add_library(s1, "pthread"); - tcc_add_library(s1, "c"); - tcc_add_library(s1, "m"); -#ifdef TCC_LIBGCC - if (!s1->static_link) - { - if (TCC_LIBGCC[0] == '/') - tcc_add_file(s1, TCC_LIBGCC); - else - tcc_add_dll(s1, TCC_LIBGCC, AFF_PRINT_ERROR); - } -#endif -#if defined TCC_TARGET_ARM && TARGETOS_FreeBSD - tcc_add_library(s1, "gcc_s"); // unwind code -#endif - } - - /* Compiler runtime (always linked unless -nodefaultlibs). - * Order matters: softfp objects reference libtcc1 symbols (__aeabi_memset, - * __aeabi_lcmp, etc.), so the FP library must come first so that its - * undefined references exist when libtcc1.a is processed à la carte. */ -#if defined TCC_TARGET_ARM - if (!tccelf_arm_fp_lib_is_shared(s1)) - tccelf_add_arm_fp_lib(s1); -#endif - if (TCC_LIBTCC1[0]) - tcc_add_support(s1, TCC_LIBTCC1); -#ifndef TCC_TARGET_MACHO - if (s1->output_type != TCC_OUTPUT_MEMORY && !s1->nostdlib) - tccelf_add_crtend(s1); -#endif -} -#endif /* ndef TCC_TARGET_PE */ - -/* add various standard linker symbols (must be done after the - sections are filled (for example after allocating common - symbols)) */ -static void tcc_add_linker_symbols(TCCState *s1) -{ - char buf[1024]; - int i; - Section *s; - - set_global_sym(s1, "_etext", text_section, -1); - set_global_sym(s1, "_edata", data_section, -1); - set_global_sym(s1, "_end", bss_section, -1); -#if TARGETOS_OpenBSD - set_global_sym(s1, "__executable_start", NULL, ELF_START_ADDR); -#endif -#ifdef TCC_TARGET_RISCV64 - /* XXX should be .sdata+0x800, not .data+0x800 */ - set_global_sym(s1, "__global_pointer$", data_section, 0x800); -#endif - /* horrible new standard ldscript defines */ - add_init_array_defines(s1, ".preinit_array"); - add_init_array_defines(s1, ".init_array"); - add_init_array_defines(s1, ".fini_array"); - /* add start and stop symbols for sections whose name can be - expressed in C */ - for (i = 1; i < s1->nb_sections; i++) - { - s = s1->sections[i]; - if ((s->sh_flags & SHF_ALLOC) && - (s->sh_type == SHT_PROGBITS || s->sh_type == SHT_NOBITS || s->sh_type == SHT_STRTAB)) - { - /* check if section name can be expressed in C */ - const char *p0, *p; - p0 = s->name; - if (*p0 == '.') - ++p0; - p = p0; - for (;;) - { - int c = *p; - if (!c) - break; - if (!isid(c) && !isnum(c)) - goto next_sec; - p++; - } - snprintf(buf, sizeof(buf), "__start_%s", p0); - set_global_sym(s1, buf, s, 0); - snprintf(buf, sizeof(buf), "__stop_%s", p0); - set_global_sym(s1, buf, s, -1); - } - next_sec:; - } -} - -ST_FUNC void resolve_common_syms(TCCState *s1) -{ - ElfW(Sym) * sym; - - /* Allocate common symbols in BSS. */ - for_each_elem(symtab_section, 1, sym, ElfW(Sym)) - { - if (sym->st_shndx == SHN_COMMON) - { - /* symbol alignment is in st_value for SHN_COMMONs */ - sym->st_value = section_add(bss_section, sym->st_size, sym->st_value); - sym->st_shndx = bss_section->sh_num; - } - } - - /* Now assign linker provided symbols their value. */ - tcc_add_linker_symbols(s1); -} - -#ifndef ELF_OBJ_ONLY -ST_FUNC void fill_got_entry(TCCState *s1, ElfW_Rel *rel) -{ - int sym_index = ELFW(R_SYM)(rel->r_info); - ElfW(Sym) *sym = &((ElfW(Sym) *)symtab_section->data)[sym_index]; - struct sym_attr *attr = get_sym_attr(s1, sym_index, 0); - unsigned offset = attr->got_offset; - - if (0 == offset) - return; - section_reserve(s1->got, offset + PTR_SIZE); -#if PTR_SIZE == 8 - write64le(s1->got->data + offset, sym->st_value); -#else - write32le(s1->got->data + offset, sym->st_value); -#endif -} - -/* Perform relocation to GOT or PLT entries */ -ST_FUNC void fill_got(TCCState *s1) -{ - Section *s; - ElfW_Rel *rel; - int i; - - for (i = 1; i < s1->nb_sections; i++) - { - s = s1->sections[i]; - if (s->sh_type != SHT_RELX) - continue; - /* no need to handle got relocations */ - if (s->link != symtab_section) - continue; - for_each_elem(s, 0, rel, ElfW_Rel) - { - switch (ELFW(R_TYPE)(rel->r_info)) - { - case R_X86_64_GOT32: - case R_X86_64_GOTPCREL: - case R_X86_64_GOTPCRELX: - case R_X86_64_REX_GOTPCRELX: - case R_X86_64_PLT32: - fill_got_entry(s1, rel); - break; - } - } - } -} - -/* See put_got_entry for a description. This is the second stage - where GOT references to local defined symbols are rewritten. */ -static void fill_local_got_entries(TCCState *s1) -{ - ElfW_Rel *rel; - if (!s1->got->reloc) - return; - for_each_elem(s1->got->reloc, 0, rel, ElfW_Rel) - { - if (ELFW(R_TYPE)(rel->r_info) == R_RELATIVE) - { - int sym_index = ELFW(R_SYM)(rel->r_info); - ElfW(Sym) *sym = &((ElfW(Sym) *)symtab_section->data)[sym_index]; - struct sym_attr *attr = get_sym_attr(s1, sym_index, 0); - unsigned offset = attr->got_offset; - if (offset != rel->r_offset - s1->got->sh_addr) - tcc_error_noabort("fill_local_got_entries: huh?"); - /* Store the ELF symbol type (e.g. STT_FUNC vs STT_NOTYPE) in the - second word of the 8-byte GOT entry. The YAFF writer reads this - to distinguish function pointers (which need thunks) from plain - code addresses such as labels used by goto *&&label. */ - write32le(s1->got->data + offset + PTR_SIZE, ELFW(ST_TYPE)(sym->st_info)); - rel->r_info = ELFW(R_INFO)(0, R_RELATIVE); -#if SHT_RELX == SHT_RELA - rel->r_addend = sym->st_value; -#else - /* All our REL architectures also happen to be 32bit LE. */ - write32le(s1->got->data + offset, sym->st_value); -#endif - } - } -} - -/* Bind symbols of executable: resolve undefined symbols from exported symbols - in shared libraries */ -static void bind_exe_dynsyms(TCCState *s1, int is_PIE) -{ - const char *name; - int sym_index, index; - ElfW(Sym) * sym, *esym; - int type; - - /* Resolve undefined symbols from dynamic symbols. When there is a match: - - if STT_FUNC or STT_GNU_IFUNC symbol -> add it in PLT - - if STT_OBJECT symbol -> add it in .bss section with suitable reloc */ - for_each_elem(symtab_section, 1, sym, ElfW(Sym)) - { - if (sym->st_shndx == SHN_UNDEF) - { - /* Validate st_name before using it */ - if (sym->st_name >= symtab_section->link->data_offset) - { - int sym_idx = sym - (ElfW(Sym) *)symtab_section->data; - tcc_error_noabort( - "internal error (bind_exe_dynsyms): symbol %d has invalid st_name offset 0x%x (strtab size: 0x%lx)", - sym_idx, sym->st_name, (unsigned long)symtab_section->link->data_offset); - continue; - } - name = (char *)symtab_section->link->data + sym->st_name; - sym_index = tcc_dynsym_find(s1, name); - if (sym_index) - { - if (is_PIE) - continue; - esym = &((ElfW(Sym) *)s1->dynsymtab_section->data)[sym_index]; - type = ELFW(ST_TYPE)(esym->st_info); - if ((type == STT_FUNC) || (type == STT_GNU_IFUNC)) - { - /* Indirect functions shall have STT_FUNC type in executable - * dynsym section. Indeed, a dlsym call following a lazy - * resolution would pick the symbol value from the - * executable dynsym entry which would contain the address - * of the function wanted by the caller of dlsym instead of - * the address of the function that would return that - * address */ - int dynindex = put_elf_sym(s1->dynsym, 0, esym->st_size, ELFW(ST_INFO)(STB_GLOBAL, STT_FUNC), 0, 0, name); - int index = sym - (ElfW(Sym) *)symtab_section->data; - get_sym_attr(s1, index, 1)->dyn_index = dynindex; - } - else if (type == STT_OBJECT) - { - unsigned long offset; - ElfW(Sym) * dynsym; - offset = bss_section->data_offset; - /* XXX: which alignment ? */ - offset = (offset + 16 - 1) & -16; - set_elf_sym(s1->symtab, offset, esym->st_size, esym->st_info, 0, bss_section->sh_num, name); - index = put_elf_sym(s1->dynsym, offset, esym->st_size, esym->st_info, 0, bss_section->sh_num, name); - - /* Ensure R_COPY works for weak symbol aliases */ - if (ELFW(ST_BIND)(esym->st_info) == STB_WEAK) - { - for_each_elem(s1->dynsymtab_section, 1, dynsym, ElfW(Sym)) - { - if ((dynsym->st_value == esym->st_value) && (ELFW(ST_BIND)(dynsym->st_info) == STB_GLOBAL)) - { - char *dynname = (char *)s1->dynsymtab_section->link->data + dynsym->st_name; - put_elf_sym(s1->dynsym, offset, dynsym->st_size, dynsym->st_info, 0, bss_section->sh_num, dynname); - break; - } - } - } - - put_elf_reloc(s1->dynsym, bss_section, offset, R_COPY, index); - offset += esym->st_size; - bss_section->data_offset = offset; - } - } - else - { - /* STB_WEAK undefined symbols are accepted */ - /* XXX: _fp_hw seems to be part of the ABI, so we ignore it */ - if (ELFW(ST_BIND)(sym->st_info) == STB_WEAK || !strcmp(name, "_fp_hw")) - { - } - else - { - tcc_error_noabort("undefined symbol '%s'", name); - } - } - } - } -} - -/* Bind symbols of libraries: export all non local symbols of executable that - are referenced by shared libraries. The reason is that the dynamic loader - search symbol first in executable and then in libraries. Therefore a - reference to a symbol already defined by a library can still be resolved by - a symbol in the executable. With -rdynamic, export all defined symbols */ -static void bind_libs_dynsyms(TCCState *s1) -{ - const char *name; - int dynsym_index; - ElfW(Sym) * sym, *esym; - - for_each_elem(symtab_section, 1, sym, ElfW(Sym)) - { - name = (char *)symtab_section->link->data + sym->st_name; - dynsym_index = tcc_dynsym_find(s1, name); - if (sym->st_shndx != SHN_UNDEF) - { - if (ELFW(ST_BIND)(sym->st_info) != STB_LOCAL && (dynsym_index || s1->rdynamic)) - set_elf_sym(s1->dynsym, sym->st_value, sym->st_size, sym->st_info, 0, sym->st_shndx, name); - } - else if (dynsym_index) - { - esym = (ElfW(Sym) *)s1->dynsymtab_section->data + dynsym_index; - if (esym->st_shndx == SHN_UNDEF) - { - /* weak symbols can stay undefined */ - if (ELFW(ST_BIND)(esym->st_info) != STB_WEAK) - tcc_warning("undefined dynamic symbol '%s'", name); - } - } - } -} - -/* Export all non local symbols. This is used by shared libraries so that the - non local symbols they define can resolve a reference in another shared - library or in the executable. Correspondingly, it allows undefined local - symbols to be resolved by other shared libraries or by the executable. */ -static void export_global_syms(TCCState *s1) -{ - int dynindex, index; - const char *name; - ElfW(Sym) * sym; - for_each_elem(symtab_section, 1, sym, ElfW(Sym)) - { - if (ELFW(ST_BIND)(sym->st_info) != STB_LOCAL) - { - name = (char *)symtab_section->link->data + sym->st_name; - dynindex = set_elf_sym(s1->dynsym, sym->st_value, sym->st_size, sym->st_info, 0, sym->st_shndx, name); - index = sym - (ElfW(Sym) *)symtab_section->data; - get_sym_attr(s1, index, 1)->dyn_index = dynindex; - } - } -} - -/* decide if an unallocated section should be output. */ -static int set_sec_sizes(TCCState *s1) -{ - int i; - Section *s; - int textrel = 0; - int file_type = s1->output_type; - - /* Allocate strings for section names */ - for (i = 1; i < s1->nb_sections; i++) - { - s = s1->sections[i]; - if (s->sh_type == SHT_RELX && !(s->sh_flags & SHF_ALLOC)) - { - /* when generating a DLL, we include relocations but - we may patch them */ - if ((file_type & TCC_OUTPUT_DYN) && (s1->sections[s->sh_info]->sh_flags & SHF_ALLOC)) - { - int count = prepare_dynamic_rel(s1, s); - if (count) - { - /* allocate the section */ - s->sh_flags |= SHF_ALLOC; - s->sh_size = count * sizeof(ElfW_Rel); - if (s1->sections[s->sh_info]->sh_flags & SHF_EXECINSTR) - textrel += count; - } - } -#ifdef TCC_TARGET_YAFF - else if (s1->output_format == TCC_OUTPUT_FORMAT_YAFF && (s1->sections[s->sh_info]->sh_flags & SHF_ALLOC)) - { - /* Preserve data relocation sections for YAFF output. - The YAFF writer needs R_ARM_ABS32 entries to generate - data relocations for the dynamic loader. Setting sh_size - ensures alloc_sec_names assigns a name, preventing - reorder_sections from pruning the section. */ - s->sh_size = s->data_offset; - } -#endif - } - else if ((s->sh_flags & SHF_ALLOC) -#ifdef TCC_TARGET_ARM - || s->sh_type == SHT_ARM_ATTRIBUTES -#endif - || s1->do_debug) - { - s->sh_size = s->data_offset; - } - -#ifdef TCC_TARGET_ARM - /* XXX: Suppress stack unwinding section. */ - if (s->sh_type == SHT_ARM_EXIDX) - { - s->sh_flags = 0; - s->sh_size = 0; - } -#endif - - /* Suppress legacy stabs sections. */ - if (!strcmp(s->name, ".stab") || !strcmp(s->name, ".stabstr") || !strncmp(s->name, ".rel.stab", 9) || - !strncmp(s->name, ".rela.stab", 10)) - { - s->sh_flags = 0; - s->sh_size = 0; - } - } - return textrel; -} - -/* various data used under elf_output_file() */ -struct dyn_inf -{ - Section *dynamic; - Section *dynstr; - struct - { - /* Info to be copied in dynamic section */ - unsigned long data_offset; - addr_t rel_addr; - addr_t rel_size; - }; - - ElfW(Phdr) * phdr; - int phnum; - int shnum; - Section *interp; - Section *note; - Section *gnu_hash; - - /* read only segment mapping for GNU_RELRO */ - Section _roinf, *roinf; -}; - -/* Find the linker script output section index and pattern index for a given - section name. Returns output section index via return value (-1 if not - found), and sets *pat_idx to the pattern index within that output section. - Patterns are checked first (in order) since they define the ordering within - the output section. If no pattern matches but the section name exactly - matches an output section name, pat_idx is set to a value after all patterns - to indicate it should come last within that output section. */ -static int ld_find_output_section_idx(TCCState *s1, const char *name, int *pat_idx) -{ - LDScript *ld = s1->ld_script; - int i, j; - - if (pat_idx) - *pat_idx = -1; - - if (!ld || ld->nb_output_sections == 0) - return -1; - - for (i = 0; i < ld->nb_output_sections; i++) - { - LDOutputSection *os = &ld->output_sections[i]; - /* Check patterns first - they define the ordering within the output section - */ - for (j = 0; j < os->nb_patterns; j++) - { - if (ld_section_matches_pattern(name, os->patterns[j].pattern)) - { - if (pat_idx) - *pat_idx = j; - return i; - } - } - /* Check exact name match - comes after all patterns */ - if (!strcmp(name, os->name)) - { - if (pat_idx) - *pat_idx = os->nb_patterns; /* after all patterns */ - return i; - } - } - return -1; -} - -/* Check if a section name matches a specific output section index */ -static int ld_section_matches_output(TCCState *s1, const char *name, int os_idx) -{ - int pat_idx = -1; - int found = ld_find_output_section_idx(s1, name, &pat_idx); - return found == os_idx; -} - -/* Decide the layout of sections loaded in memory. This must be done before - program headers are filled since they contain info about the layout. - We do the following ordering: interp, symbol tables, relocations, progbits, - nobits */ -static int sort_sections(TCCState *s1, int *sec_order, struct dyn_inf *d) -{ - Section *s; - int i, j, k, f, f0, n, ld_idx; - int nb_sections = s1->nb_sections; - int *sec_cls = sec_order + nb_sections; - - for (i = 1; i < nb_sections; i++) - { - s = s1->sections[i]; - if (0 == s->sh_name) - { - j = 0x900; /* no sh_name: won't go to file */ - } - else if (s->sh_flags & SHF_ALLOC) - { - j = 0x100; - } - else - { - j = 0x700; - } - if (j >= 0x700 && s1->output_format != TCC_OUTPUT_FORMAT_ELF) - s->sh_size = 0, j = 0x900; - - if (s->sh_type == SHT_SYMTAB || s->sh_type == SHT_DYNSYM) - { - k = 0xff; - } - else if (s->sh_type == SHT_STRTAB && strcmp(s->name, ".stabstr")) - { - k = 0xff; - if (i == nb_sections - 1) /* ".shstrtab" assumed to stay last */ - k = 0xff; - } - else if (s->sh_type == SHT_HASH || s->sh_type == SHT_GNU_HASH) - { - k = 0xff; - } - else if (s->sh_type == SHT_GNU_verdef || s->sh_type == SHT_GNU_verneed || s->sh_type == SHT_GNU_versym) - { - k = 0x13; - } - else if (s->sh_type == SHT_RELX) - { - k = 0x80; - if (s1->plt && s == s1->plt->reloc) - k = 0x81; - } - else if (s->sh_flags & SHF_EXECINSTR) - { - k = 0x30; - if (s == s1->plt) - { - k = 0x32; - } - /* RELRO sections --> */ - } - else if (s->sh_type == SHT_PREINIT_ARRAY) - { - k = 0x41; - } - else if (s->sh_type == SHT_INIT_ARRAY) - { - k = 0x42; - } - else if (s->sh_type == SHT_FINI_ARRAY) - { - k = 0x43; - } - else if (s->sh_type == SHT_DYNAMIC) - { - k = 0x280; - } - else if (s == s1->got) - { - k = 0x70; /* .got as RELRO needs BIND_NOW in DT_FLAGS */ - } - else if (s->reloc && (s->reloc->sh_flags & SHF_ALLOC) && j == 0x100) - { - if (s == rodata_section) - { - k = 0x43; - } - else - { - k = 0x44; - } - /* <-- */ - } - else if (s->sh_type == SHT_NOTE) - { - k = 0x60; - } - else if (s->sh_type == SHT_NOBITS) - { - k = 0x70; /* bss */ - } - else if (s == d->interp) - { - k = 0xff; - } - else if (s == rodata_section) - { - k = 0x40; /* rodata */ - } - else - { - k = 0x50; /* data */ - } - - k += j; - - /* Check for RELRO sections before potentially modifying k for linker - script ordering. RELRO sections are in range 0x141-0x14f. */ - if ((k & 0xfff0) == 0x140) - { - /* make RELRO section writable */ - s->sh_flags |= SHF_WRITE; - } - - /* If linker script has output sections defined, use linker script order - for ALLOC program sections (not relocation, symbol, or other special - sections). The linker script output section index becomes the primary - sort key, with the pattern index within the output section as the - secondary key. This ensures sections are ordered according to the - pattern order in the linker script (e.g., KEEP(*(.isr_vector)) before - *(.text)). - Sections not in the linker script are placed after all linker script - sections. - - Classification encoding for linker script sections (within 0x100-0x6ff): - - Upper nibble (0x100-0x600): output section index (up to 6 sections) - - Lower byte: pattern index * 2 (to leave room for sub-classes) - - For sections not in linker script, we use 0x6xx range. - - Skip relocation sections (SHT_RELX), symbol tables, string tables, - hash tables, and other special sections - they should keep their - default ordering. */ - if (j == 0x100 && s1->ld_script && s1->ld_script->nb_output_sections > 0 && s->sh_type != SHT_RELX && - s->sh_type != SHT_SYMTAB && s->sh_type != SHT_DYNSYM && s->sh_type != SHT_STRTAB && s->sh_type != SHT_HASH && - s->sh_type != SHT_GNU_HASH && s->sh_type != SHT_DYNAMIC) - { - int pat_idx = 0; - ld_idx = ld_find_output_section_idx(s1, s->name, &pat_idx); - if (ld_idx >= 0) - { - /* Section is in linker script: use ld_idx as primary key, - pattern index as secondary key. - pat_idx is the index of the matching pattern (0+), or nb_patterns - for exact name match (comes after all patterns). - Keep values in 0x100-0x5ff range to ensure proper handling. */ - /* Limit ld_idx to fit in 5 bits (0-31 output sections) */ - if (ld_idx > 31) - ld_idx = 31; - /* pat_idx in lower bits, ld_idx in upper bits, all within 0x100-0x6ff - */ - k = 0x100 + (ld_idx << 4) + (pat_idx & 0x0f); - } - else - { - /* Section not in linker script: place after all linker script sections - but still within ALLOC range. Use 0x6xx + original sub-class. */ - k = 0x600 + ((k & 0x7f) >> 4); - } - } - - for (n = i; n > 1 && k < (f = sec_cls[n - 1]); --n) - sec_cls[n] = f, sec_order[n] = sec_order[n - 1]; - sec_cls[n] = k, sec_order[n] = i; - } - sec_order[0] = 0; - d->shnum = 1; - - /* count PT_LOAD headers needed */ - n = f0 = 0; - for (i = 1; i < nb_sections; i++) - { - s = s1->sections[sec_order[i]]; - k = sec_cls[i]; - f = 0; - if (k < 0x900) - ++d->shnum; - if (k < 0x700) - { - f = s->sh_flags & (SHF_ALLOC | SHF_WRITE | SHF_EXECINSTR | SHF_TLS); -#if TARGETOS_NetBSD - /* NetBSD only supports 2 PT_LOAD sections. - See: https://blog.netbsd.org/tnf/entry/the_first_report_on_lld */ - if ((f & SHF_WRITE) == 0) - f |= SHF_EXECINSTR; -#else - if ((k & 0xfff0) == 0x240) /* RELRO sections */ - f |= 1 << 4; -#endif - /* start new header when flags changed or relro, but avoid zero memsz */ - if (f != f0 && s->sh_size) - f0 = f, ++n, f |= 1 << 8; - } - sec_cls[i] = f; - LOG_RELOC("ph %d sec %02d : %3X %3X %x %04X %s", (f > 0) * n, i, f, k, s->sh_type, (int)s->sh_size, s->name); - } - return n; -} - -static ElfW(Phdr) * fill_phdr(ElfW(Phdr) * ph, int type, Section *s) -{ - if (s) - { - ph->p_offset = s->sh_offset; - ph->p_vaddr = s->sh_addr; - ph->p_filesz = s->sh_size; - ph->p_align = s->sh_addralign; - } - ph->p_type = type; - ph->p_flags = PF_R; - ph->p_paddr = ph->p_vaddr; - ph->p_memsz = ph->p_filesz; - return ph; -} - -/* Assign sections to segments and decide how are sections laid out when loaded - in memory. This function also fills corresponding program headers. */ -static int layout_sections(TCCState *s1, int *sec_order, struct dyn_inf *d) -{ - Section *s; - addr_t addr, tmp, align, s_align, base; - ElfW(Phdr) *ph = NULL; - int i, f, n, phnum, phfill; - int file_offset; - int elf_header_offset = 0; - - /* compute number of program headers */ - phnum = sort_sections(s1, sec_order, d); - phfill = 0; /* set to 1 to have dll's with a PT_PHDR */ - if (d->interp) - phfill = 2; - phnum += phfill; - if (d->note) - ++phnum; - if (d->dynamic) - ++phnum; - if (eh_frame_hdr_section) - ++phnum; - if (d->roinf) - ++phnum; - /* Add extra segments for memory regions (each region may need new PT_LOAD) */ - if (s1->ld_script && s1->ld_script->nb_memory_regions > 1) - phnum += s1->ld_script->nb_memory_regions - 1; - d->phnum = phnum; - d->phdr = tcc_mallocz(phnum * sizeof(ElfW(Phdr))); - - file_offset = 0; - if (s1->output_format == TCC_OUTPUT_FORMAT_ELF) - { - file_offset = (sizeof(ElfW(Ehdr)) + phnum * sizeof(ElfW(Phdr)) + 3) & -4; - file_offset += d->shnum * sizeof(ElfW(Shdr)); - } - - s_align = ELF_PAGE_SIZE; - if (s1->section_align) - s_align = s1->section_align; - - addr = ELF_START_ADDR; - if (s1->output_type & TCC_OUTPUT_DYN) - addr = 0; - -#ifdef TCC_TARGET_YAFF - /* YAFF format uses 0-based offsets for all sections. - Force text address to 0 unless the user explicitly set one. */ - if (s1->output_format == TCC_OUTPUT_FORMAT_YAFF && !s1->has_text_addr) - addr = 0; -#endif - - /* Use linker script MEMORY origin if available */ - if (s1->ld_script && s1->ld_script->nb_memory_regions > 0) - { - addr = s1->ld_script->memory_regions[0].origin; - } - - if (s1->has_text_addr) - { - addr = s1->text_addr; - if (0) - { - int a_offset, p_offset; - /* we ensure that (addr % ELF_PAGE_SIZE) == file_offset % - ELF_PAGE_SIZE */ - a_offset = (int)(addr & (s_align - 1)); - p_offset = file_offset & (s_align - 1); - if (a_offset < p_offset) - a_offset += s_align; - file_offset += (a_offset - p_offset); - } - } - /* compute address after headers */ - // addr += file_offset; - elf_header_offset = file_offset; - - /* Track per-memory-region address counters for linker script placement */ - addr_t mr_addr[LD_MAX_MEMORY_REGIONS]; - int cur_mr = 0; - if (s1->ld_script && s1->ld_script->nb_memory_regions > 0) - { - for (int mr = 0; mr < s1->ld_script->nb_memory_regions; mr++) - { - LDMemoryRegion *region = &s1->ld_script->memory_regions[mr]; - mr_addr[mr] = region->origin; - } - addr = mr_addr[0]; - } - else - { - mr_addr[0] = addr; - } - - /* `base` is what the first PT_LOAD claims as its p_vaddr, so it has to be - where the first section actually lands -- which, when a linker script - supplies MEMORY regions, is the first region's origin and not the text - address above. A YASOS-targeted compiler defaults to text_addr 0 with - has_text_addr set (tcc_new), so taking `base` any earlier described a - script-placed image as starting at 0: p_vaddr 0 against sections at the - region origin, and p_memsz stretched from 0 to the end of the segment. - QEMU loads such a segment on top of the vector table it just wrote (the - 0x00000000 and 0x10000000 SSRAM aliases on mps2-an505 are the same - memory), so the CPU reset-fetched SP=0/PC=0 and locked up before the - first instruction. */ - base = addr; - - n = 0; - for (i = 1; i < s1->nb_sections; i++) - { - int sec_idx = sec_order[i]; - int sec_flags_idx = i + s1->nb_sections; - s = s1->sections[sec_idx]; - f = sec_order[sec_flags_idx]; - align = s->sh_addralign - 1; - - if (f == 0) - { /* no alloc */ - file_offset = (file_offset + align) & ~align; - s->sh_offset = file_offset; - if (s->sh_type != SHT_NOBITS) - file_offset += s->sh_size; - continue; - } - - /* Check if this section should be placed in a different memory region */ - if (s1->ld_script && s1->ld_script->nb_memory_regions > 0 && s1->ld_script->nb_output_sections > 0) - { - int pat_idx = -1; - int ld_idx = ld_find_output_section_idx(s1, s->name, &pat_idx); - if (ld_idx >= 0) - { - LDOutputSection *os = &s1->ld_script->output_sections[ld_idx]; - int new_mr = os->memory_region_idx; - if (new_mr >= 0 && new_mr < s1->ld_script->nb_memory_regions) - { - if (new_mr != cur_mr) - { - /* Save current region's address and switch to new region */ - mr_addr[cur_mr] = addr; - cur_mr = new_mr; - addr = mr_addr[cur_mr]; - /* Force new program header when changing memory regions */ - f |= 1 << 8; - } - } - } - } - - if ((f & 1 << 8) && n) - { - /* different rwx section flags */ - if (s1->output_format == TCC_OUTPUT_FORMAT_ELF) - { - /* if in the middle of a page, w e duplicate the page in - memory so that one copy is RX and the other is RW */ - if ((addr & (s_align - 1)) != 0) - addr += s_align; - } - else - { - align = s_align - 1; - } - } - - tmp = addr; - addr = (addr + align) & ~align; - file_offset += (int)(addr - tmp); - s->sh_offset = file_offset; - s->sh_addr = addr; - s->sh_size = (s->sh_size + align) & ~align; - - if (f & 1 << 8) - { - /* set new program header */ - int ph_idx = phfill + n; - ph = &d->phdr[ph_idx]; - ph->p_type = PT_LOAD; - ph->p_align = s_align; - ph->p_flags = PF_R; - if (f & SHF_WRITE) - ph->p_flags |= PF_W; - if (f & SHF_EXECINSTR) - ph->p_flags |= PF_X; - if (f & SHF_TLS) - { - ph->p_type = PT_TLS; - ph->p_align = align + 1; - } - - ph->p_offset = file_offset; - ph->p_vaddr = addr; - - if (n == 0) - { - /* Make the first PT_LOAD segment include the program - headers itself (and the ELF header as well), it'll - come out with same memory use but will make various - tools like binutils strip work better. */ - // ph->p_offset = 0; - ph->p_vaddr = base; - } - ph->p_paddr = ph->p_vaddr; - ++n; - } - - if (f & 1 << 4) - { - Section *roinf = &d->_roinf; - if (roinf->sh_size == 0) - { - roinf->sh_offset = s->sh_offset; - roinf->sh_addr = s->sh_addr; - roinf->sh_addralign = 1; - } - roinf->sh_size = (addr - roinf->sh_addr) + s->sh_size; - } - - addr += s->sh_size; - if (s->sh_type != SHT_NOBITS) - file_offset += s->sh_size; - - if (ph) - { - ph->p_filesz = file_offset - ph->p_offset; - ph->p_memsz = addr - ph->p_vaddr; - if (n == 1) - { - ph->p_memsz += elf_header_offset; - } - } - } - - /* Fill other headers */ - if (d->note) - fill_phdr(++ph, PT_NOTE, d->note); - if (d->dynamic) - { - ElfW(Phdr) *dph = fill_phdr(++ph, PT_DYNAMIC, d->dynamic); - dph->p_flags |= PF_W; - } - if (eh_frame_hdr_section) - fill_phdr(++ph, PT_GNU_EH_FRAME, eh_frame_hdr_section); - if (d->roinf) - { - ElfW(Phdr) *rph = fill_phdr(++ph, PT_GNU_RELRO, d->roinf); - rph->p_flags |= PF_W; - } - if (d->interp) - { - ElfW(Phdr) *iph = &d->phdr[1]; - fill_phdr(iph, PT_INTERP, d->interp); - } - if (phfill) - { - ph = &d->phdr[0]; - ph->p_offset = sizeof(ElfW(Ehdr)); - ph->p_vaddr = base + ph->p_offset; - { - int phdr_sz = phnum * sizeof(ElfW(Phdr)); - ph->p_filesz = phdr_sz; - } - ph->p_align = 4; - fill_phdr(ph, PT_PHDR, NULL); - } - return 0; -} - -/* put dynamic tag */ -static void put_dt(Section *dynamic, int dt, addr_t val) -{ - ElfW(Dyn) * dyn; - dyn = section_ptr_add(dynamic, sizeof(ElfW(Dyn))); - dyn->d_tag = dt; - dyn->d_un.d_val = val; -} - -/* Fill the dynamic section with tags describing the address and size of - sections */ -static void fill_dynamic(TCCState *s1, struct dyn_inf *dyninf) -{ - Section *dynamic = dyninf->dynamic; - Section *s; - - /* put dynamic section entries */ - put_dt(dynamic, DT_HASH, s1->dynsym->hash->sh_addr); - put_dt(dynamic, DT_GNU_HASH, dyninf->gnu_hash->sh_addr); - put_dt(dynamic, DT_STRTAB, dyninf->dynstr->sh_addr); - put_dt(dynamic, DT_SYMTAB, s1->dynsym->sh_addr); - put_dt(dynamic, DT_STRSZ, dyninf->dynstr->data_offset); - put_dt(dynamic, DT_SYMENT, sizeof(ElfW(Sym))); -#if PTR_SIZE == 8 - put_dt(dynamic, DT_RELA, dyninf->rel_addr); - put_dt(dynamic, DT_RELASZ, dyninf->rel_size); - put_dt(dynamic, DT_RELAENT, sizeof(ElfW_Rel)); - if (s1->plt && s1->plt->reloc) - { - put_dt(dynamic, DT_PLTGOT, s1->got->sh_addr); - put_dt(dynamic, DT_PLTRELSZ, s1->plt->reloc->data_offset); - put_dt(dynamic, DT_JMPREL, s1->plt->reloc->sh_addr); - put_dt(dynamic, DT_PLTREL, DT_RELA); - } - put_dt(dynamic, DT_RELACOUNT, 0); -#else - put_dt(dynamic, DT_REL, dyninf->rel_addr); - put_dt(dynamic, DT_RELSZ, dyninf->rel_size); - put_dt(dynamic, DT_RELENT, sizeof(ElfW_Rel)); - if (s1->plt && s1->plt->reloc) - { - put_dt(dynamic, DT_PLTGOT, s1->got->sh_addr); - put_dt(dynamic, DT_PLTRELSZ, s1->plt->reloc->data_offset); - put_dt(dynamic, DT_JMPREL, s1->plt->reloc->sh_addr); - put_dt(dynamic, DT_PLTREL, DT_REL); - } - put_dt(dynamic, DT_RELCOUNT, 0); -#endif - if (versym_section && verneed_section) - { - /* The dynamic linker can not handle VERSYM without VERNEED */ - put_dt(dynamic, DT_VERSYM, versym_section->sh_addr); - put_dt(dynamic, DT_VERNEED, verneed_section->sh_addr); - put_dt(dynamic, DT_VERNEEDNUM, dt_verneednum); - } - s = have_section(s1, ".preinit_array"); - if (s && s->data_offset) - { - put_dt(dynamic, DT_PREINIT_ARRAY, s->sh_addr); - put_dt(dynamic, DT_PREINIT_ARRAYSZ, s->data_offset); - } - s = have_section(s1, ".init_array"); - if (s && s->data_offset) - { - put_dt(dynamic, DT_INIT_ARRAY, s->sh_addr); - put_dt(dynamic, DT_INIT_ARRAYSZ, s->data_offset); - } - s = have_section(s1, ".fini_array"); - if (s && s->data_offset) - { - put_dt(dynamic, DT_FINI_ARRAY, s->sh_addr); - put_dt(dynamic, DT_FINI_ARRAYSZ, s->data_offset); - } - s = have_section(s1, ".init"); - if (s && s->data_offset) - { - put_dt(dynamic, DT_INIT, s->sh_addr); - } - s = have_section(s1, ".fini"); - if (s && s->data_offset) - { - put_dt(dynamic, DT_FINI, s->sh_addr); - } - if (s1->do_debug) - put_dt(dynamic, DT_DEBUG, 0); - put_dt(dynamic, DT_NULL, 0); -} - -/* Remove gaps between RELX sections. - These gaps are a result of final_sections_reloc. Here some relocs are - removed. The gaps are then filled with 0 in tcc_output_elf. The 0 is - intepreted as R_...NONE reloc. This does work on most targets but on - OpenBSD/arm64 this is illegal. OpenBSD/arm64 does not support R_...NONE - reloc. */ -static void update_reloc_sections(TCCState *s1, struct dyn_inf *dyninf) -{ - int i; - unsigned long file_offset = 0; - Section *s; - Section *relocplt = s1->plt ? s1->plt->reloc : NULL; - - /* dynamic relocation table information, for .dynamic section */ - dyninf->rel_addr = dyninf->rel_size = 0; - - for (i = 1; i < s1->nb_sections; i++) - { - s = s1->sections[i]; - if (s->sh_type == SHT_RELX && s != relocplt) - { - if (dyninf->rel_size == 0) - { - dyninf->rel_addr = s->sh_addr; - file_offset = s->sh_offset; - } - else - { - s->sh_addr = dyninf->rel_addr + dyninf->rel_size; - s->sh_offset = file_offset + dyninf->rel_size; - } - dyninf->rel_size += s->sh_size; - } - } -} -#endif /* ndef ELF_OBJ_ONLY */ - -/* Create an ELF file on disk. - This function handle ELF specific layout requirements */ -static int tcc_output_elf(TCCState *s1, FILE *f, int phnum, ElfW(Phdr) * phdr) -{ - int i, shnum, offset, size, file_type; - Section *s; - ElfW(Ehdr) ehdr; - ElfW(Shdr) shdr, *sh; - - file_type = s1->output_type; - shnum = s1->nb_sections; - - memset(&ehdr, 0, sizeof(ehdr)); - if (phnum > 0) - { - ehdr.e_phentsize = sizeof(ElfW(Phdr)); - ehdr.e_phnum = phnum; - ehdr.e_phoff = sizeof(ElfW(Ehdr)); - } - - /* fill header */ - ehdr.e_ident[0] = ELFMAG0; - ehdr.e_ident[1] = ELFMAG1; - ehdr.e_ident[2] = ELFMAG2; - ehdr.e_ident[3] = ELFMAG3; - ehdr.e_ident[4] = ELFCLASSW; - ehdr.e_ident[5] = ELFDATA2LSB; - ehdr.e_ident[6] = EV_CURRENT; - -#if TARGETOS_FreeBSD || TARGETOS_FreeBSD_kernel - ehdr.e_ident[EI_OSABI] = ELFOSABI_FREEBSD; -#elif defined TCC_TARGET_ARM && defined TCC_ARM_EABI - ehdr.e_flags = EF_ARM_EABI_VER5; - ehdr.e_flags |= s1->float_abi == ARM_HARD_FLOAT ? EF_ARM_VFP_FLOAT : EF_ARM_SOFT_FLOAT; -#elif defined TCC_TARGET_ARM - ehdr.e_ident[EI_OSABI] = ELFOSABI_ARM; -#elif defined TCC_TARGET_RISCV64 - /* XXX should be configurable */ - ehdr.e_flags = EF_RISCV_FLOAT_ABI_DOUBLE; -#endif - - if (file_type == TCC_OUTPUT_OBJ) - { - ehdr.e_type = ET_REL; - } - else - { - if (file_type & TCC_OUTPUT_DYN) - ehdr.e_type = ET_DYN; - else - ehdr.e_type = ET_EXEC; - if (s1->elf_entryname) - ehdr.e_entry = get_sym_addr(s1, s1->elf_entryname, 1, 0); - else - ehdr.e_entry = get_sym_addr(s1, "_start", !!(file_type & TCC_OUTPUT_EXE), 0); - if (ehdr.e_entry == (addr_t)-1) - ehdr.e_entry = text_section->sh_addr; - if (s1->nb_errors) - return -1; - } - - tcc_elf_sort_syms(s1, s1->symtab); - - ehdr.e_machine = EM_TCC_TARGET; - ehdr.e_version = EV_CURRENT; - ehdr.e_shoff = (sizeof(ElfW(Ehdr)) + phnum * sizeof(ElfW(Phdr)) + 3) & -4; - ehdr.e_ehsize = sizeof(ElfW(Ehdr)); - ehdr.e_shentsize = sizeof(ElfW(Shdr)); - ehdr.e_shnum = shnum; - ehdr.e_shstrndx = shnum - 1; - - offset = fwrite(&ehdr, 1, sizeof(ElfW(Ehdr)), f); - if (phdr) - offset += fwrite(phdr, 1, phnum * sizeof(ElfW(Phdr)), f); - - /* output section headers */ - while (offset < ehdr.e_shoff) - { - fputc(0, f); - offset++; - } - - for (i = 0; i < shnum; i++) - { - sh = &shdr; - memset(sh, 0, sizeof(ElfW(Shdr))); - if (i) - { - s = s1->sections[i]; - sh->sh_name = s->sh_name; - sh->sh_type = s->sh_type; - sh->sh_flags = s->sh_flags; - sh->sh_entsize = s->sh_entsize; - sh->sh_info = s->sh_info; - if (s->link) - sh->sh_link = s->link->sh_num; - sh->sh_addralign = s->sh_addralign; - sh->sh_addr = s->sh_addr; - sh->sh_offset = s->sh_offset; - sh->sh_size = s->sh_size; - } - offset += fwrite(sh, 1, sizeof(ElfW(Shdr)), f); - } - - /* output sections - use streaming for lazy sections to avoid memory allocation */ - for (i = 1; i < s1->nb_sections; i++) - { - s = s1->sections[i]; - if (s->sh_type != SHT_NOBITS) - { - while (offset < s->sh_offset) - { - fputc(0, f); - offset++; - } - size = s->sh_size; - if (size) - { - if (s->lazy && !s->materialized) - { - /* Stream directly from source files without loading into memory */ - section_write_streaming(s1, s, f); - offset += size; - } - else - { - /* Already materialized, write from memory */ - const int to_write = size < s->data_allocated ? size : s->data_allocated; - int written = fwrite(s->data, 1, to_write, f); - offset += written; - } - } - } - } - return 0; -} - -static int tcc_output_binary(TCCState *s1, FILE *f) -{ - Section *s; - int i, offset, size; - - offset = 0; - for (i = 1; i < s1->nb_sections; i++) - { - s = s1->sections[i]; - if (s->sh_type != SHT_NOBITS && (s->sh_flags & SHF_ALLOC)) - { - while (offset < s->sh_offset) - { - fputc(0, f); - offset++; - } - size = s->sh_size; - if (s->lazy && !s->materialized) - { - /* Stream directly from source files without loading into memory */ - section_write_streaming(s1, s, f); - } - else - { - fwrite(s->data, 1, size, f); - } - offset += size; - } - } - return 0; -} - -/* Write an elf, coff or "binary" file */ -static int tcc_write_elf_file(TCCState *s1, const char *filename, int phnum, ElfW(Phdr) * phdr) -{ - int fd, mode, file_type, ret; - FILE *f; - - file_type = s1->output_type; - if (file_type == TCC_OUTPUT_OBJ) - mode = 0666; - else - mode = 0777; - unlink(filename); - fd = open(filename, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, mode); - if (fd < 0 || (f = fdopen(fd, "wb")) == NULL) - return tcc_error_noabort("could not write '%s: %s'", filename, strerror(errno)); - if (s1->verbose) - printf("<- %s\n", filename); -#ifdef TCC_TARGET_COFF - if (s1->output_format == TCC_OUTPUT_FORMAT_COFF) - tcc_output_coff(s1, f); - else -#endif -#ifdef TCC_TARGET_YAFF - if (s1->output_format == TCC_OUTPUT_FORMAT_YAFF) - ret = tcc_output_yaff(s1, f, filename); - else -#endif - if (s1->output_format == TCC_OUTPUT_FORMAT_ELF) - ret = tcc_output_elf(s1, f, phnum, phdr); - else - ret = tcc_output_binary(s1, f); - fclose(f); - - return ret; -} - -#ifndef ELF_OBJ_ONLY -/* order sections according to sec_order, remove sections - that we aren't going to output. */ -static void reorder_sections(TCCState *s1, int *sec_order) -{ - int i, nnew, k, *backmap; - Section **snew, *s; - ElfW(Sym) * sym; - - backmap = tcc_malloc(s1->nb_sections * sizeof(backmap[0])); - for (i = 0, nnew = 0, snew = NULL; i < s1->nb_sections; i++) - { - k = sec_order[i]; - s = s1->sections[k]; - if (!i || s->sh_name) - { - backmap[k] = nnew; - dynarray_add(&snew, &nnew, s); - } - else - { - backmap[k] = 0; - /* just remember to free them later */ - dynarray_add(&s1->priv_sections, &s1->nb_priv_sections, s); - } - } - for (i = 1; i < nnew; i++) - { - s = snew[i]; - s->sh_num = i; - if (s->sh_type == SHT_RELX) - s->sh_info = backmap[s->sh_info]; - else if (s->sh_type == SHT_SYMTAB || s->sh_type == SHT_DYNSYM) - for_each_elem(s, 1, sym, ElfW(Sym)) if (sym->st_shndx < s1->nb_sections) sym->st_shndx = backmap[sym->st_shndx]; - } - tcc_free(s1->sections); - s1->sections = snew; - s1->nb_sections = nnew; - tcc_free(backmap); -} - -#ifdef TCC_TARGET_ARM -/* Emit the ARM EABI build-attributes section describing the code we produced. - * Consumers (ld, gdb, objdump) use it to check ABI compatibility between - * objects — most importantly Tag_ABI_VFP_args, which says whether FP arguments - * travel in VFP registers (-mfloat-abi=hard) or GPRs (soft/softfp). - * - * Layout (ABI addenda §2.2): - * 'A' | uint32 vendor_len | "aeabi\0" | 0x01 Tag_File | uint32 sub_len | pairs - * where vendor_len counts everything after the 'A' (including itself) and - * sub_len counts the Tag_File byte, its length field and the attribute pairs. - * Every tag and value we emit is < 128, so each is a one-byte ULEB128. */ -static void create_arm_attribute_section(TCCState *s1) -{ - ArmEabiAttrs a; - arm_get_eabi_attrs(s1, &a); - - unsigned char body[96]; - int n = 0; - - body[n++] = 5; /* Tag_CPU_name (NUL-terminated string) */ - for (const char *p = a.cpu_name; *p; p++) - body[n++] = (unsigned char)*p; - body[n++] = 0; - body[n++] = 6; - body[n++] = (unsigned char)a.cpu_arch; /* Tag_CPU_arch */ - body[n++] = 7; - body[n++] = 'M'; /* Tag_CPU_arch_profile: Microcontroller */ - body[n++] = 9; - body[n++] = 3; /* Tag_THUMB_ISA_use: Yes. No Tag_ARM_ISA_use: - * M-profile has no ARM instruction set. */ - if (a.fp_arch) - { - body[n++] = 10; - body[n++] = (unsigned char)a.fp_arch; /* Tag_FP_arch */ - } - body[n++] = 18; - body[n++] = 4; /* Tag_ABI_PCS_wchar_t: 4 */ - body[n++] = 20; - body[n++] = 1; /* Tag_ABI_FP_denormal: Needed */ - body[n++] = 21; - body[n++] = 1; /* Tag_ABI_FP_exceptions: Needed */ - body[n++] = 23; - body[n++] = 3; /* Tag_ABI_FP_number_model: IEEE 754 */ - body[n++] = 24; - body[n++] = 1; /* Tag_ABI_align_needed: 8-byte */ - body[n++] = 25; - body[n++] = 1; /* Tag_ABI_align_preserved: 8-byte, except leaf SP */ - body[n++] = 26; - body[n++] = 2; /* Tag_ABI_enum_size: int (arm-none-eabi-gcc defaults to - * 'small' instead — a deliberate, declared difference). */ - if (a.hardfp_use) - { - body[n++] = 27; - body[n++] = (unsigned char)a.hardfp_use; /* Tag_ABI_HardFP_use */ - } - if (a.vfp_args) - { - body[n++] = 28; - body[n++] = (unsigned char)a.vfp_args; /* Tag_ABI_VFP_args */ - } - body[n++] = 30; - body[n++] = 6; /* Tag_ABI_optimization_goals: Aggressive Debug */ - body[n++] = 34; - body[n++] = 1; /* Tag_CPU_unaligned_access: v6 */ - - const int sub_len = 1 + 4 + n; - const int vendor_len = 4 + 6 + sub_len; - - Section *attr = new_section(s1, ".ARM.attributes", SHT_ARM_ATTRIBUTES, 0); - attr->sh_addralign = 1; - unsigned char *ptr = section_ptr_add(attr, 1 + vendor_len); - int o = 0; - ptr[o++] = 'A'; - write32le(ptr + o, vendor_len); - o += 4; - memcpy(ptr + o, "aeabi", 6); - o += 6; - ptr[o++] = 1; /* Tag_File */ - write32le(ptr + o, sub_len); - o += 4; - memcpy(ptr + o, body, n); -} -#endif - -#if TARGETOS_OpenBSD || TARGETOS_NetBSD -static Section *create_bsd_note_section(TCCState *s1, const char *name, const char *value) -{ - Section *s = find_section(s1, name); - - if (s->data_offset == 0) - { - char *ptr = section_ptr_add(s, sizeof(ElfW(Nhdr)) + 8 + 4); - ElfW(Nhdr) *note = (ElfW(Nhdr) *)ptr; - - s->sh_type = SHT_NOTE; - note->n_namesz = 8; - note->n_descsz = 4; - note->n_type = ELF_NOTE_OS_GNU; - strcpy(ptr + sizeof(ElfW(Nhdr)), value); - } - return s; -} -#endif - -static void alloc_sec_names(TCCState *s1, int is_obj); -static void ld_apply_symbols(TCCState *s1, LDScript *ld); -static void ld_update_symbol_values(TCCState *s1, LDScript *ld); -ST_FUNC void ld_export_standard_symbols(TCCState *s1); - -/* --gc-sections implementation: remove unused sections */ -static void gc_sections(TCCState *s1) -{ - int i, sym_index, changed; - Section *s, *sr; - ElfW(Sym) * sym, *symtab; - ElfW_Rel *rel; - unsigned char *sec_used; - int nb_syms; - const char *name; - - /* Allocate array to track which sections are used */ - sec_used = tcc_mallocz(s1->nb_sections); - - /* Always keep certain essential sections */ - for (i = 1; i < s1->nb_sections; i++) - { - s = s1->sections[i]; - if (!s) - continue; - /* Keep symtab, strtab, shstrtab, and relocation sections */ - if (s->sh_type == SHT_SYMTAB || s->sh_type == SHT_STRTAB || s->sh_type == SHT_HASH || s->sh_type == SHT_DYNSYM || - s->sh_type == SHT_GNU_HASH || s->sh_type == SHT_GNU_versym || s->sh_type == SHT_GNU_verneed || - s->sh_type == SHT_GNU_verdef || s->sh_type == SHT_RELX || s->sh_type == SHT_DYNAMIC || s->sh_type == SHT_NOTE) - { - sec_used[i] = 1; - continue; - } - /* Keep init/fini arrays and special sections */ - if (!strcmp(s->name, ".init") || !strcmp(s->name, ".fini") || !strcmp(s->name, ".init_array") || - !strcmp(s->name, ".fini_array") || !strcmp(s->name, ".preinit_array") || !strcmp(s->name, ".ctors") || - !strcmp(s->name, ".dtors") || !strcmp(s->name, ".got") || !strcmp(s->name, ".got.plt") || - !strcmp(s->name, ".plt") || !strcmp(s->name, ".interp") || !strcmp(s->name, ".eh_frame") || - !strcmp(s->name, ".eh_frame_hdr") || !strcmp(s->name, ".ARM.attributes") || !strcmp(s->name, ".ARM.exidx")) - { - sec_used[i] = 1; - continue; - } - /* Keep sections marked with KEEP() in linker script */ - if (s1->ld_script && ld_section_should_keep(s1->ld_script, s->name)) - { - sec_used[i] = 1; - continue; - } - /* Keep debug sections if debugging enabled */ - if (s1->do_debug && !strncmp(s->name, ".debug", 6)) - { - sec_used[i] = 1; - continue; - } - /* Keep sections that are not SHF_ALLOC (like comments) if not allocatable - */ - if (!(s->sh_flags & SHF_ALLOC)) - { - sec_used[i] = 1; - continue; - } - } - - /* Mark sections containing entry point and other root symbols */ - symtab = (ElfW(Sym) *)symtab_section->data; - nb_syms = symtab_section->data_offset / sizeof(ElfW(Sym)); - - for (sym_index = 1; sym_index < nb_syms; sym_index++) - { - sym = &symtab[sym_index]; - if (sym->st_shndx == SHN_UNDEF || sym->st_shndx >= SHN_LORESERVE) - continue; - if (sym->st_shndx >= s1->nb_sections) - continue; - - name = (char *)symtab_section->link->data + sym->st_name; - - /* Mark entry point section */ - if (s1->elf_entryname && name[0] == s1->elf_entryname[0] && !strcmp(name, s1->elf_entryname)) - { - sec_used[sym->st_shndx] = 1; - continue; - } - if ((name[0] == '_' || name[0] == 'm') && - (!strcmp(name, "_start") || !strcmp(name, "main") || !strcmp(name, "_main") || !strcmp(name, "__start"))) - { - sec_used[sym->st_shndx] = 1; - continue; - } - /* Mark global/weak symbols that are exported */ - if (s1->rdynamic && ELFW(ST_BIND)(sym->st_info) != STB_LOCAL) - { - sec_used[sym->st_shndx] = 1; - continue; - } - /* Exported symbols are runtime roots regardless of -rdynamic: under - * -fvisibility=hidden the dynamic-symbol set is exactly the defined - * GLOBAL/WEAK symbols with default visibility, and other yasld modules - * resolve against them at runtime with no relocation in this image to - * witness the use. Collecting one would leave a .dynsym entry pointing - * at reclaimed bytes -- a silent wrong-address at runtime, not a link - * error. Hidden globals and statics stay collectable. */ - if (ELFW(ST_BIND)(sym->st_info) != STB_LOCAL && ELFW(ST_VISIBILITY)(sym->st_other) == STV_DEFAULT) - { - sec_used[sym->st_shndx] = 1; - continue; - } - } - - /* Iteratively mark sections referenced by relocations from used sections */ - do - { - changed = 0; - for (i = 1; i < s1->nb_sections; i++) - { - sr = s1->sections[i]; - if (!sr || sr->sh_type != SHT_RELX) - continue; - /* Get the section this relocation applies to */ - s = s1->sections[sr->sh_info]; - if (!s || !sec_used[sr->sh_info]) - continue; - /* References from non-allocated sections must not confer liveness: - * DWARF describes every function (each .text. is referenced by - * .rel.debug_info via its section symbol), so following these would - * pin the whole program and -g would silently disable GC. Dropped - * targets get the SHN_ABS tombstone in the sweep instead, which is - * how debug info for collected code is conventionally marked. */ - if (!(s->sh_flags & SHF_ALLOC)) - continue; - - /* Iterate through relocations */ - for_each_elem(sr, 0, rel, ElfW_Rel) - { - sym_index = ELFW(R_SYM)(rel->r_info); - if (sym_index == 0 || sym_index >= nb_syms) - continue; - sym = &symtab[sym_index]; - if (sym->st_shndx == SHN_UNDEF || sym->st_shndx >= SHN_LORESERVE) - continue; - if (sym->st_shndx >= s1->nb_sections) - continue; - if (!sec_used[sym->st_shndx]) - { - sec_used[sym->st_shndx] = 1; - changed = 1; - } - } - } - } while (changed); - - /* Also mark relocation sections for used sections */ - for (i = 1; i < s1->nb_sections; i++) - { - s = s1->sections[i]; - if (s && s->reloc && sec_used[i]) - { - sec_used[s->reloc->sh_num] = 1; - } - } - - /* Remove unused sections by zeroing their data */ - { - unsigned char *collected = tcc_mallocz(s1->nb_sections); - int any_collected = 0; - for (i = 1; i < s1->nb_sections; i++) - { - s = s1->sections[i]; - if (!s) - continue; - if (!sec_used[i] && (s->sh_flags & SHF_ALLOC) && s->data_offset > 0) - { - if (s1->verbose) - printf("GC: removing unused section '%s' (%d bytes)\n", s->name, (int)s->data_offset); - /* Zero the section - it will be skipped in output */ - s->data_offset = 0; - s->sh_size = 0; - /* Clearing SHF_ALLOC keeps the shell out of layout AND out of - * coalesce_function_sections, which must not fold a collected - * .text. back into .text. */ - s->sh_flags &= ~SHF_ALLOC; - collected[i] = 1; - any_collected = 1; - if (s->reloc) - { - s->reloc->data_offset = 0; - s->reloc->sh_size = 0; - } - /* Free deferred chunks for lazy sections to save memory */ - if (s->lazy && s->has_deferred_chunks) - { - free_deferred_chunks(s); - s->lazy = 0; - s->has_deferred_chunks = 0; - } - } - } - /* A symbol still pointing into a collected section would resolve to - * (stale base + offset). Exported symbols are rooted above, so only - * locals and hidden globals land here; give them the DWARF tombstone. - * SHN_ABS keeps relocate_syms' hands off the value, and 0xFFFFFFFF is - * unmistakably dead -- 0 is a real code address under -Ttext=0x0, so - * it must not be the marker. */ - if (any_collected) - { - for (sym_index = 1; sym_index < nb_syms; sym_index++) - { - sym = &symtab[sym_index]; - if (sym->st_shndx < s1->nb_sections && collected[sym->st_shndx]) - { - sym->st_value = 0xFFFFFFFF; - sym->st_shndx = SHN_ABS; - } - } - } - tcc_free(collected); - } - - tcc_free(sec_used); -} - -/* Fold every per-function .text.* input section (created under - * -ffunction-sections; unreferenced ones already emptied by gc_sections) - * back into the single text_section before GOT build and layout. - * - * Everything downstream of this point -- sbrel classification, layout, the - * export table and the YAFF writer -- addresses code through the - * text_section global (tccyaff.c writes text_section->data as the entire - * code payload), so coalescing here lets all of it stay ignorant of split - * text. Sections merge in creation order, which is emission order, so the - * resulting layout matches what a non-split build produces. - * - * Symbol fixup relies on two properties verified in this tree: - * - relocate_syms() adds a section base only for sh_num < SHN_LORESERVE, - * so retargeted symbols behave exactly like natively-.text ones; - * - DWARF references code via STT_SECTION symbols + addend - * (tcc_debug_funcstart records dwarf_register_text_section(cur_text_section)), - * so bumping the section symbol's st_value by the merge base keeps every - * debug address correct without touching the DWARF bytes. */ -static void coalesce_function_sections(TCCState *s1) -{ - int i, have_split = 0; - addr_t *base_of; - unsigned char *merged; - ElfW(Sym) * sym; - /* Snapshot the section count: put_elf_reloc below can create .rel.text - * mid-loop, growing s1->nb_sections past the size of the arrays. New - * sections are reloc sections, never merge candidates, so the snapshot - * loses nothing. */ - int nb_sections = s1->nb_sections; - - for (i = 1; i < nb_sections; i++) - { - Section *s = s1->sections[i]; - if (s && s != text_section && s->sh_type == SHT_PROGBITS && (s->sh_flags & SHF_EXECINSTR) && - (s->sh_flags & SHF_ALLOC) && !strncmp(s->name, ".text.", 6)) - { - have_split = 1; - break; - } - } - if (!have_split) - return; - - base_of = tcc_mallocz(sizeof(addr_t) * nb_sections); - merged = tcc_mallocz(nb_sections); - - for (i = 1; i < nb_sections; i++) - { - Section *s = s1->sections[i]; - if (!s || s == text_section || s->sh_type != SHT_PROGBITS) - continue; - if ((s->sh_flags & (SHF_EXECINSTR | SHF_ALLOC)) != (SHF_EXECINSTR | SHF_ALLOC)) - continue; - if (strncmp(s->name, ".text.", 6)) - continue; - - /* Thumb literal pools are PC-relative loads with PC aligned down to 4; - * every function must therefore start 4-aligned, exactly as it would - * have in a monolithic .text. */ - text_section->data_offset += -text_section->data_offset & (addr_t)3; - base_of[i] = text_section->data_offset; - merged[i] = 1; - if (s->data_offset) - { - void *p = section_ptr_add(text_section, s->data_offset); - memcpy(p, s->data, s->data_offset); - } - - /* Move the section's relocations into .rel.text, rebased. Addends live - * in the copied code bytes (REL format), so only r_offset changes; - * symbol indices are global to symtab and stay valid. */ - if (s->reloc && s->reloc->data_offset) - { - Section *sr = s->reloc; - int nb = sr->data_offset / sizeof(ElfW_Rel); - ElfW_Rel *rel = (ElfW_Rel *)sr->data; - int r; - for (r = 0; r < nb; r++) - put_elf_reloc(symtab_section, text_section, rel[r].r_offset + base_of[i], ELFW(R_TYPE)(rel[r].r_info), - ELFW(R_SYM)(rel[r].r_info)); - sr->data_offset = 0; - } - } - - /* Retarget every symbol defined in a merged section: function symbols - * (the thumb LSB tag survives the addition), $t/$d mapping markers, and - * the STT_SECTION symbols DWARF relocates against (st_value 0 -> base). */ - for_each_elem(symtab_section, 1, sym, ElfW(Sym)) - { - if (sym->st_shndx < nb_sections && merged[sym->st_shndx]) - { - sym->st_value += base_of[sym->st_shndx]; - sym->st_shndx = text_section->sh_num; - } - } - - /* Retire the merged shells: no ALLOC flag and no size means - * alloc_sec_names() skips them and they never reach the output. */ - for (i = 1; i < nb_sections; i++) - { - if (merged[i]) - { - Section *s = s1->sections[i]; - s->sh_flags &= ~SHF_ALLOC; - s->data_offset = 0; - s->sh_size = 0; - } - } - - tcc_free(base_of); - tcc_free(merged); -} - -/* Output an elf, coff or binary file */ -/* XXX: suppress unneeded sections */ -static int elf_output_file(TCCState *s1, const char *filename) -{ - int i, ret, file_type, *sec_order; - struct dyn_inf dyninf = {0}; - Section *interp, *dynstr, *dynamic; - int textrel, got_sym, dt_flags_1; - - file_type = s1->output_type; - s1->nb_errors = 0; - ret = -1; - interp = dynstr = dynamic = NULL; - sec_order = NULL; - dyninf.roinf = &dyninf._roinf; - - /* Load linker script if specified */ - if (s1->linker_script) - { - if (tcc_load_linker_script(s1, s1->linker_script) < 0) - return -1; - /* Apply linker script symbols early so they're available for resolution. - * Values for symbols in NOLOAD sections will be updated after layout. */ - ld_apply_symbols(s1, s1->ld_script); - } - -#ifdef TCC_TARGET_ARM - create_arm_attribute_section(s1); -#endif - -#if TARGETOS_OpenBSD - dyninf.note = create_bsd_note_section(s1, ".note.openbsd.ident", "OpenBSD"); -#endif - -#if TARGETOS_NetBSD - dyninf.note = create_bsd_note_section(s1, ".note.netbsd.ident", "NetBSD"); -#endif - -#if TARGETOS_FreeBSD || TARGETOS_NetBSD - dyninf.roinf = NULL; -#endif - /* if linking, also link in runtime libraries (libc, libgcc, etc.) */ - { - unsigned t = s1->do_bench ? tcc_getclock_us() : 0; - tcc_add_runtime(s1); - if (s1->do_bench) - s1->bench_output_runtime_us += tcc_getclock_us() - t; - } - resolve_common_syms(s1); - -#ifdef TCC_TARGET_YAFF - /* Merge .init_array / .fini_array into .data early — before - build_got_entries() — so that the __yaff_initfini symbol uses the - R_RELATIVE (local) GOT path, and relocations pointing into the - merged data are resolved naturally by relocate_sections(). */ - if (s1->output_format == TCC_OUTPUT_FORMAT_YAFF) - tcc_yaff_prepare_init_fini(s1); -#endif - - /* Phase 2: Garbage Collection During Loading - mark and load referenced sections */ - if (s1->gc_sections_aggressive) - { - tcc_gc_mark_phase(s1); - tcc_load_referenced_sections(s1); - tcc_free_lazy_objfiles(s1); - } - - /* Garbage collect unused sections if requested (skip if aggressive GC already ran) */ - if (s1->gc_sections && !s1->gc_sections_aggressive) - { - gc_sections(s1); - } - - /* Per-function .text.* sections (from -ffunction-sections objects) must be - * folded back into text_section before build_got/layout -- see the comment - * on coalesce_function_sections. Runs unconditionally: it is a no-op when - * no split-text input exists. */ - coalesce_function_sections(s1); - - if (!s1->static_link) - { - if (file_type & TCC_OUTPUT_EXE) - { - /* allow override the dynamic loader */ - const char *elfint = getenv("LD_SO"); - if (elfint == NULL) - elfint = DEFAULT_ELFINTERP(s1); - /* add interpreter section only if executable */ - // interp = new_section(s1, ".interp", SHT_PROGBITS, SHF_ALLOC); - // interp->sh_addralign = 1; - // ptr = section_ptr_add(interp, 1 + strlen(elfint)); - // strcpy(ptr, elfint); - dyninf.interp = interp; - } - - /* add dynamic symbol table */ - s1->dynsym = new_symtab(s1, ".dynsym", SHT_DYNSYM, SHF_ALLOC, ".dynstr", ".hash", SHF_ALLOC); - /* Number of local symbols (readelf complains if not set) */ - s1->dynsym->sh_info = 1; - dynstr = s1->dynsym->link; - /* add dynamic section */ - dynamic = new_section(s1, ".dynamic", SHT_DYNAMIC, SHF_ALLOC | SHF_WRITE); - dynamic->link = dynstr; - dynamic->sh_entsize = sizeof(ElfW(Dyn)); - - got_sym = build_got(s1); - if (file_type & TCC_OUTPUT_EXE) - { - bind_exe_dynsyms(s1, file_type & TCC_OUTPUT_DYN); - if (s1->nb_errors) - goto the_end; - } - { - unsigned t = s1->do_bench ? tcc_getclock_us() : 0; - build_got_entries(s1, got_sym); - if (s1->do_bench) - s1->bench_output_got_us += tcc_getclock_us() - t; - } - if (file_type & TCC_OUTPUT_EXE) - { - bind_libs_dynsyms(s1); - } - else - { - /* shared library case: simply export all global symbols */ - export_global_syms(s1); - } -#if TCC_EH_FRAME - /* fill with initial data */ - tcc_eh_frame_hdr(s1, 0); -#endif - dyninf.gnu_hash = create_gnu_hash(s1); - } - else - { - { - unsigned t = s1->do_bench ? tcc_getclock_us() : 0; - build_got_entries(s1, 0); - if (s1->do_bench) - s1->bench_output_got_us += tcc_getclock_us() - t; - } - } - version_add(s1); - - textrel = set_sec_sizes(s1); - - if (!s1->static_link) - { - /* add a list of needed dlls */ - for (i = 0; i < s1->nb_loaded_dlls; i++) - { - DLLReference *dllref = s1->loaded_dlls[i]; - if (dllref->level == 0) - put_dt(dynamic, DT_NEEDED, put_elf_str(dynstr, dllref->name)); - } - - if (s1->rpath) - put_dt(dynamic, s1->enable_new_dtags ? DT_RUNPATH : DT_RPATH, put_elf_str(dynstr, s1->rpath)); - - dt_flags_1 = DF_1_NOW; - if (file_type & TCC_OUTPUT_DYN) - { - if (s1->soname) - put_dt(dynamic, DT_SONAME, put_elf_str(dynstr, s1->soname)); - /* XXX: currently, since we do not handle PIC code, we - must relocate the readonly segments */ - if (textrel) - put_dt(dynamic, DT_TEXTREL, 0); - if (file_type & TCC_OUTPUT_EXE) - dt_flags_1 = DF_1_NOW | DF_1_PIE; - } - put_dt(dynamic, DT_FLAGS, DF_BIND_NOW); - put_dt(dynamic, DT_FLAGS_1, dt_flags_1); - if (s1->symbolic) - put_dt(dynamic, DT_SYMBOLIC, 0); - - dyninf.dynamic = dynamic; - dyninf.dynstr = dynstr; - /* remember offset and reserve space for 2nd call below */ - dyninf.data_offset = dynamic->data_offset; - fill_dynamic(s1, &dyninf); - dynamic->sh_size = dynamic->data_offset; - dynstr->sh_size = dynstr->data_offset; - } - - /* create and fill .shstrtab section */ - alloc_sec_names(s1, 0); - /* this array is used to reorder sections in the output file */ - sec_order = tcc_malloc(sizeof(int) * 2 * s1->nb_sections); - /* compute section to program header mapping */ - { - unsigned t = s1->do_bench ? tcc_getclock_us() : 0; - layout_sections(s1, sec_order, &dyninf); - if (s1->do_bench) - s1->bench_output_layout_us += tcc_getclock_us() - t; - } - - /* Export standard linker symbols after layout (addresses now known) */ - /* Skip if linker script is loaded - it provides its own symbol definitions */ - if (!s1->ld_script) - { - ld_export_standard_symbols(s1); - } - - /* Update and apply linker script symbols with final addresses */ - if (s1->ld_script) - { - ld_update_symbol_values(s1, s1->ld_script); - ld_apply_symbols(s1, s1->ld_script); - - /* Fix p_paddr for LOAD segments of sections with AT > (LMA != VMA). - The boot code copies .data from the LMA (Flash) to the VMA (RAM), - so the ELF loader must place the content at the LMA, not the VMA. */ - if (s1->ld_script->has_loadaddrs) - { - LDScript *ld = s1->ld_script; - int j, k; - for (j = 0; j < ld->nb_output_sections; j++) - { - if (ld->output_sections[j].load_memory_region_idx >= 0 && ld->output_sections[j].memory_region_idx >= 0 && - ld->output_sections[j].load_memory_region_idx != ld->output_sections[j].memory_region_idx) - { - /* This output section has AT > (LMA in different region than VMA). - Find the LOAD segment whose p_vaddr matches and fix p_paddr. */ - addr_t vma = 0; - /* Find the VMA of this output section from its first ELF section */ - for (k = 1; k < s1->nb_sections; k++) - { - Section *sec = s1->sections[k]; - if (sec->sh_addr && ld_section_matches_output(s1, sec->name, j)) - { - vma = sec->sh_addr; - break; - } - } - if (vma) - { - for (k = 0; k < dyninf.phnum; k++) - { - ElfW(Phdr) *ph = &dyninf.phdr[k]; - if (ph->p_type == PT_LOAD && ph->p_vaddr <= vma && vma < ph->p_vaddr + ph->p_memsz) - { - ph->p_paddr = ld->output_section_loadaddrs[j]; - break; - } - } - } - } - } - } - } - - if (dynamic) - { - /* put in GOT the dynamic section address and relocate PLT */ - write32le(s1->got->data, dynamic->sh_addr); - if (file_type == TCC_OUTPUT_EXE || (RELOCATE_DLLPLT && (file_type & TCC_OUTPUT_DYN))) - relocate_plt(s1); - /* relocate symbols in .dynsym now that final addresses are known */ - relocate_syms(s1, s1->dynsym, 2); - } - - /* if building executable or DLL, then relocate each section - except the GOT which is already relocated */ - { - unsigned t = s1->do_bench ? tcc_getclock_us() : 0; - relocate_syms(s1, s1->symtab, 0); - if (s1->nb_errors != 0) - { - if (s1->do_bench) - s1->bench_output_reloc_us += tcc_getclock_us() - t; - goto the_end; - } - relocate_sections(s1); - if (s1->do_bench) - s1->bench_output_reloc_us += tcc_getclock_us() - t; - } - if (dynamic) - { - update_reloc_sections(s1, &dyninf); - dynamic->data_offset = dyninf.data_offset; - fill_dynamic(s1, &dyninf); - } - /* Perform relocation to GOT or PLT entries */ - if (file_type == TCC_OUTPUT_EXE && s1->static_link) - fill_got(s1); - else if (s1->got) - fill_local_got_entries(s1); - - if (dyninf.gnu_hash) - update_gnu_hash(s1, dyninf.gnu_hash); - - reorder_sections(s1, sec_order); -#if TCC_EH_FRAME - /* fill with final data */ - tcc_eh_frame_hdr(s1, 1); -#endif - /* Create the ELF file with name 'filename' */ - { - unsigned t = s1->do_bench ? tcc_getclock_us() : 0; - ret = tcc_write_elf_file(s1, filename, dyninf.phnum, dyninf.phdr); - if (s1->do_bench) - s1->bench_output_write_us += tcc_getclock_us() - t; - } -the_end: - tcc_free(sec_order); - tcc_free(dyninf.phdr); - return ret; -} -#endif /* ndef ELF_OBJ_ONLY */ - -/* Allocate strings for section names */ -static void alloc_sec_names(TCCState *s1, int is_obj) -{ - int i; - Section *s, *strsec; - - strsec = new_section(s1, ".shstrtab", SHT_STRTAB, 0); - put_elf_str(strsec, ""); - for (i = 1; i < s1->nb_sections; i++) - { - s = s1->sections[i]; - if (is_obj) - s->sh_size = s->data_offset; - if (s->sh_size || s == strsec || (s->sh_flags & SHF_ALLOC) || is_obj) - s->sh_name = put_elf_str(strsec, s->name); - } - strsec->sh_size = strsec->data_offset; -} - -/* Output an elf .o file */ -static int elf_output_obj(TCCState *s1, const char *filename) -{ - Section *s; - int i, ret, file_offset; - s1->nb_errors = 0; - /* Allocate strings for section names */ - alloc_sec_names(s1, 1); - file_offset = (sizeof(ElfW(Ehdr)) + 3) & -4; - file_offset += s1->nb_sections * sizeof(ElfW(Shdr)); - for (i = 1; i < s1->nb_sections; i++) - { - s = s1->sections[i]; - file_offset = (file_offset + 15) & -16; - s->sh_offset = file_offset; - if (s->sh_type != SHT_NOBITS) - file_offset += s->sh_size; - } - /* Create the ELF file with name 'filename' */ - { - unsigned t = s1->do_bench ? tcc_getclock_us() : 0; - ret = tcc_write_elf_file(s1, filename, 0, NULL); - if (s1->do_bench) - s1->bench_output_write_us += tcc_getclock_us() - t; - } - return ret; -} - -LIBTCCAPI int tcc_output_file(TCCState *s, const char *filename) -{ - unsigned output_start = 0; - int ret; - - if (s->do_bench) - output_start = tcc_getclock_us(); - - if (s->test_coverage) - tcc_tcov_add_file(s, filename); - if (s->output_type == TCC_OUTPUT_OBJ) - ret = elf_output_obj(s, filename); -#ifdef TCC_TARGET_PE - else - ret = pe_output_file(s, filename); -#elif defined TCC_TARGET_MACHO - else - ret = macho_output_file(s, filename); -#else - else - ret = elf_output_file(s, filename); -#endif - if (s->do_bench) - { - unsigned elapsed = tcc_getclock_us() - output_start; - s->bench_output_time += elapsed; - s->bench_output_count++; - tcc_bench_log(s, "output", filename, elapsed); - } - return ret; -} - -ST_FUNC ssize_t full_read(int fd, void *buf, size_t count) -{ - char *cbuf = buf; - size_t rnum = 0; - for (;;) - { - ssize_t num = read(fd, cbuf, count - rnum); - if (num < 0) - return num; - if (num == 0) - break; - rnum += num; - cbuf += num; - } - return rnum; -} - -ST_FUNC void *load_data(int fd, unsigned long file_offset, unsigned long size) -{ - void *data; - - data = tcc_malloc(size); - lseek(fd, file_offset, SEEK_SET); - full_read(fd, data, size); - return data; -} - -/* Return the canonical section name for function/data sections. - * Merges .text.foo -> .text, .rodata.bar -> .rodata, .data.baz -> .data, .bss.qux -> .bss - * Returns original name if no match. - */ -static const char *get_merged_section_name(TCCState *s1, const char *name) -{ - /* Under --gc-sections, per-function text sections from input objects must - * keep their identity: fold them here and section-granularity GC has - * nothing left to collect (the .o path is exactly how the self-hosted tcc - * links). Same-named sections from different objects still merge, which - * is conservative. Survivors are folded back into one .text after GC by - * coalesce_function_sections(). Data/rodata folding stays unconditional - * -- their GC granularity is a separate project, and the YAFF writer - * reads those through fixed section globals. */ - int keep_text_split = s1->gc_sections; - if (name[0] != '.') - return name; - switch (name[1]) - { - case 't': - if (!strncmp(name, ".text.", 6)) - return keep_text_split ? name : ".text"; - break; - case 'r': - if (!strncmp(name, ".rodata.", 8)) - return ".rodata"; - /* Relocation sections MUST fold in lockstep with their targets. If - * .text.error folds into .text but .rel.text.error keeps its name, two - * disasters follow when the in-process compile (under - * -ffunction-sections) already created a section of that exact name: - * the member's entries -- rebased for .text -- merge into a reloc - * section whose sh_info points at the compiled 8-byte .text.error, and - * the loader's back-link update leaves text_section->reloc aimed at a - * random .rel.text.. Both corrupt relocation application. */ - if (!strncmp(name, ".rel.text.", 10)) - return keep_text_split ? name : ".rel.text"; - if (!strncmp(name, ".rel.rodata.", 12)) - return ".rel.rodata"; - if (!strncmp(name, ".rel.data.", 10)) - return ".rel.data"; - if (!strncmp(name, ".rela.text.", 11)) - return keep_text_split ? name : ".rela.text"; - if (!strncmp(name, ".rela.rodata.", 13)) - return ".rela.rodata"; - if (!strncmp(name, ".rela.data.", 11)) - return ".rela.data"; - break; - case 'd': - if (!strncmp(name, ".data.", 6)) - return ".data"; - break; - case 'b': - if (!strncmp(name, ".bss.", 5)) - return ".bss"; - break; - } - return name; -} - -typedef struct SectionMergeInfo -{ - Section *s; /* corresponding existing section */ - unsigned long offset; /* offset of the new section in the existing section */ - uint8_t new_section; /* true if section 's' was added */ - uint8_t link_once; /* true if link once section */ - const char *merged_to; /* canonical name if section was merged */ -} SectionMergeInfo; - -ST_FUNC int tcc_object_type(int fd, ElfW(Ehdr) * h) -{ - int size = full_read(fd, h, sizeof *h); - if (size == sizeof *h && 0 == memcmp(h, ELFMAG, 4) && - h->e_ident[EI_CLASS] == ELFCLASSW) - { - if (h->e_type == ET_REL) - return AFF_BINTYPE_REL; - if (h->e_type == ET_DYN) - return AFF_BINTYPE_DYN; - } - else if (size >= 8) - { - if (0 == memcmp(h, ARMAG, 8)) - return AFF_BINTYPE_AR; -#ifdef TCC_TARGET_COFF - if (((struct filehdr *)h)->f_magic == COFF_C67_MAGIC) - return AFF_BINTYPE_C67; -#endif - } - if (0 == memcmp(h, YAFFMAG, 4)) - { - return AFF_BINTYPE_YAFF; - } - - return 0; -} - -/* load an object file and merge it with current files */ -/* XXX: handle correctly stab (debug) info */ -static Section *find_existing_section(TCCState *s1, const char *name); - -ST_FUNC int tcc_load_object_file(TCCState *s1, int fd, unsigned long file_offset) -{ - ElfW(Ehdr) ehdr; - ElfW(Shdr) * shdr, *sh; - unsigned long size, offset, offseti; - int i, nb_syms, sym_index, ret, seencompressed; - char *strsec, *strtab; - int stab_index = 0, stabstr_index = 0; - (void)stab_index; - (void)stabstr_index; - int *old_to_new_syms; - char *sh_name, *name; - SectionMergeInfo *sm_table, *sm; - ElfW(Sym) * sym, *symtab; - ElfW_Rel *rel; - Section *s; - const char *last_lookup_name = NULL; - Section *last_lookup_section = NULL; - unsigned object_start = 0; - - if (s1->do_bench) - object_start = tcc_getclock_us(); - - /* Use lazy loading for aggressive GC mode */ - if (s1->gc_sections_aggressive) - { - return tcc_load_object_file_lazy(s1, fd, file_offset); - } - - lseek(fd, file_offset, SEEK_SET); - - if (tcc_object_type(fd, &ehdr) != AFF_BINTYPE_REL) - { - goto invalid; - } - /* test CPU specific stuff */ - - if (ehdr.e_ident[5] != ELFDATA2LSB || ehdr.e_machine != EM_TCC_TARGET) - { - invalid: - return tcc_error_noabort("invalid object file"); - } - /* read sections */ - shdr = load_data(fd, file_offset + ehdr.e_shoff, sizeof(ElfW(Shdr)) * ehdr.e_shnum); - sm_table = tcc_mallocz(sizeof(SectionMergeInfo) * ehdr.e_shnum); - - /* load section names */ - sh = &shdr[ehdr.e_shstrndx]; - strsec = load_data(fd, file_offset + sh->sh_offset, sh->sh_size); - - /* load symtab and strtab */ - old_to_new_syms = NULL; - symtab = NULL; - strtab = NULL; - nb_syms = 0; - seencompressed = 0; - stab_index = stabstr_index = 0; - ret = -1; - - for (i = 1; i < ehdr.e_shnum; i++) - { - sh = &shdr[i]; - if (sh->sh_type == SHT_SYMTAB) - { - if (symtab) - { - tcc_error_noabort("object must contain only one symtab"); - goto the_end; - } - nb_syms = sh->sh_size / sizeof(ElfW(Sym)); - symtab = load_data(fd, file_offset + sh->sh_offset, sh->sh_size); - sm_table[i].s = symtab_section; - - /* now load strtab */ - sh = &shdr[sh->sh_link]; - strtab = load_data(fd, file_offset + sh->sh_offset, sh->sh_size); - } - if (sh->sh_flags & SHF_COMPRESSED) - seencompressed = 1; - } - - /* now examine each section and try to merge its content with the - ones in memory */ - for (i = 1; i < ehdr.e_shnum; i++) - { - /* no need to examine section name strtab */ - if (i == ehdr.e_shstrndx) - continue; - sh = &shdr[i]; - if (sh->sh_type == SHT_RELX) - sh = &shdr[sh->sh_info]; - /* ignore sections types we do not handle (plus relocs to those) */ - sh_name = strsec + sh->sh_name; - if (sh_name[0] == '.' && (sh_name[1] == 'd' || sh_name[1] == 's') && - (0 == strncmp(sh_name, ".debug_", 7) || 0 == strncmp(sh_name, ".stab", 5))) - { - if (!s1->do_debug || seencompressed) - continue; -#if !(TARGETOS_OpenBSD || TARGETOS_FreeBSD || TARGETOS_NetBSD) - } - else if (sh_name[1] == 'e' && 0 == strncmp(sh_name, ".eh_frame", 9)) - { - if (NULL == eh_frame_section) - continue; -#endif - } - else if (sh->sh_type != SHT_PROGBITS && sh->sh_type != SHT_NOTE && sh->sh_type != SHT_NOBITS && - sh->sh_type != SHT_PREINIT_ARRAY && sh->sh_type != SHT_INIT_ARRAY && sh->sh_type != SHT_FINI_ARRAY -#ifdef TCC_ARM_EABI - && sh->sh_type != SHT_ARM_EXIDX -#endif -#if TARGETOS_OpenBSD || TARGETOS_FreeBSD || TARGETOS_NetBSD - && sh->sh_type != SHT_X86_64_UNWIND -#endif - ) - continue; - - sh = &shdr[i]; - sh_name = strsec + sh->sh_name; - if (sh->sh_addralign < 1) - sh->sh_addralign = 1; - /* find corresponding section, if any */ - /* Use merged name for .text.*, .rodata.*, .data.*, .bss.* sections */ - { - const char *lookup_name = get_merged_section_name(s1, sh_name); - s = NULL; - if (lookup_name == last_lookup_name) - { - s = last_lookup_section; - } - else - { - s = find_existing_section(s1, lookup_name); - } - last_lookup_name = lookup_name; - last_lookup_section = s; - if (s) - { - if (sh->sh_type != s->sh_type && strcmp(s->name, ".eh_frame")) - { - tcc_error_noabort("section type conflict: %s %02x <> %02x", s->name, sh->sh_type, s->sh_type); - goto the_end; - } - if (sh_name[1] == 'g' && !strncmp(sh_name, ".gnu.linkonce", 13)) - { - /* if a 'linkonce' section is already present, we - do not add it again. It is a little tricky as - symbols can still be defined in - it. */ - sm_table[i].link_once = 1; - goto next; - } - /* stab section tracking removed - DWARF only */ - /* Track if this section was merged (original name differs from lookup name) */ - if (sh_name != lookup_name) - sm_table[i].merged_to = lookup_name; - goto found; - } - /* not found: create new section with merged name */ - s = new_section(s1, lookup_name, sh->sh_type, sh->sh_flags & ~SHF_GROUP); - /* take as much info as possible from the section. sh_link and - sh_info will be updated later */ - s->sh_addralign = sh->sh_addralign; - s->sh_entsize = sh->sh_entsize; - sm_table[i].new_section = 1; - last_lookup_name = lookup_name; - last_lookup_section = s; - /* Track if this section was merged */ - if (sh_name != lookup_name) - sm_table[i].merged_to = lookup_name; - } - found: - /* align start of section */ - s->data_offset += -s->data_offset & (sh->sh_addralign - 1); - if (sh->sh_addralign > s->sh_addralign) - s->sh_addralign = sh->sh_addralign; - sm_table[i].offset = s->data_offset; - sm_table[i].s = s; - /* concatenate sections */ - size = sh->sh_size; - if (sh->sh_type != SHT_NOBITS) - { - if (should_defer_section(sh_name, sh->sh_type)) - { - /* Lazy loading: just record position for debug sections */ - unsigned long dest_off = s->data_offset; - s->data_offset += size; /* Reserve space without allocating */ - - /* Record where to load from later - include archive member offset if in archive */ - unsigned long abs_offset = - s1->current_archive_offset ? s1->current_archive_offset + sh->sh_offset : file_offset + sh->sh_offset; - /* Use archive path if loading from archive, otherwise use current file */ - const char *source_path = s1->current_archive_path ? s1->current_archive_path : s1->current_filename; - /* Track source path for materialization */ - section_add_deferred(s1, s, source_path, abs_offset, size, dest_off); - } - else - { - /* Immediate loading */ - unsigned char *ptr; - lseek(fd, file_offset + sh->sh_offset, SEEK_SET); - /* section_ptr_add will handle allocation as needed */ - ptr = section_ptr_add(s, size); - full_read(fd, ptr, size); - } - } - else - { - s->data_offset += size; - } - /* align end of section */ - /* This is needed if we compile a c file after this */ - if (s == text_section || s == data_section || s == rodata_section || s == bss_section || s == common_section) - s->data_offset += -s->data_offset & (s->sh_addralign - 1); - next:; - } - - /* stab string relocation removed - only DWARF debug info supported */ - - /* second short pass to update sh_link and sh_info fields of new - sections */ - for (i = 1; i < ehdr.e_shnum; i++) - { - s = sm_table[i].s; - if (!s || !sm_table[i].new_section) - continue; - sh = &shdr[i]; - if (sh->sh_link > 0) - s->link = sm_table[sh->sh_link].s; - if (sh->sh_type == SHT_RELX) - { - s->sh_info = sm_table[sh->sh_info].s->sh_num; - /* update backward link */ - s1->sections[s->sh_info]->reloc = s; - } - } - - /* resolve symbols */ - old_to_new_syms = tcc_mallocz(nb_syms * sizeof(int)); - - sym = symtab + 1; - for (i = 1; i < nb_syms; i++, sym++) - { - if (sym->st_shndx != SHN_UNDEF && sym->st_shndx < SHN_LORESERVE) - { - sm = &sm_table[sym->st_shndx]; - if (sm->link_once) - { - /* if a symbol is in a link once section, we use the - already defined symbol. It is very important to get - correct relocations */ - if (ELFW(ST_BIND)(sym->st_info) != STB_LOCAL) - { - name = strtab + sym->st_name; - sym_index = find_elf_sym(symtab_section, name); - if (sym_index) - old_to_new_syms[i] = sym_index; - } - continue; - } - /* if no corresponding section added, no need to add symbol */ - if (!sm->s) - continue; - /* convert section number */ - sym->st_shndx = sm->s->sh_num; - /* offset value */ - sym->st_value += sm->offset; - } - /* add symbol */ - name = strtab + sym->st_name; - sym_index = - set_elf_sym(symtab_section, sym->st_value, sym->st_size, sym->st_info, sym->st_other, sym->st_shndx, name); - old_to_new_syms[i] = sym_index; - } - - /* third pass to patch relocation entries */ - for (i = 1; i < ehdr.e_shnum; i++) - { - s = sm_table[i].s; - if (!s) - continue; - sh = &shdr[i]; - offset = sm_table[i].offset; - size = sh->sh_size; - switch (s->sh_type) - { - case SHT_RELX: - /* take relocation offset information */ - offseti = sm_table[sh->sh_info].offset; - for (rel = (ElfW_Rel *)s->data + (offset / sizeof(*rel)); - rel < (ElfW_Rel *)s->data + ((offset + size) / sizeof(*rel)); rel++) - { - int type; - unsigned sym_index; - /* convert symbol index */ - type = ELFW(R_TYPE)(rel->r_info); - sym_index = ELFW(R_SYM)(rel->r_info); - /* NOTE: only one symtab assumed */ - if (sym_index >= nb_syms) - goto invalid_reloc; - sym_index = old_to_new_syms[sym_index]; - /* ignore link_once in rel section. */ - if (!sym_index && !sm_table[sh->sh_info].link_once -#ifdef TCC_TARGET_ARM - && type != R_ARM_V4BX -#elif defined TCC_TARGET_RISCV64 - && type != R_RISCV_ALIGN && type != R_RISCV_RELAX -#endif - ) - { - invalid_reloc: - tcc_error_noabort("Invalid relocation entry [%2d] '%s' @ %.8x", i, strsec + sh->sh_name, (int)rel->r_offset); - goto the_end; - } - rel->r_info = ELFW(R_INFO)(sym_index, type); - /* offset the relocation offset */ - rel->r_offset += offseti; -#ifdef TCC_TARGET_ARM - /* Jumps and branches from a Thumb code to a PLT entry need - special handling since PLT entries are ARM code. - Unconditional bl instructions referencing PLT entries are - handled by converting these instructions into blx - instructions. Other case of instructions referencing a PLT - entry require to add a Thumb stub before the PLT entry to - switch to ARM mode. We set bit plt_thumb_stub of the - attribute of a symbol to indicate such a case. */ - if (type == R_ARM_THM_JUMP24) - get_sym_attr(s1, sym_index, 1)->plt_thumb_stub = 1; -#endif - } - break; - default: - break; - } - } - - ret = 0; -the_end: - if (s1->do_bench) - { - unsigned elapsed = tcc_getclock_us() - object_start; - s1->bench_object_load_time += elapsed; - s1->bench_object_load_count++; - if (s1->current_archive_offset) - s1->bench_archive_member_count++; - else - tcc_bench_log(s1, "load-obj", s1->current_filename, elapsed); - } - tcc_free(symtab); - tcc_free(strtab); - tcc_free(old_to_new_syms); - tcc_free(sm_table); - tcc_free(strsec); - tcc_free(shdr); - return ret; -} - -typedef struct ArchiveHeader -{ - char ar_name[16]; /* name of this member */ - char ar_date[12]; /* file mtime */ - char ar_uid[6]; /* owner uid; printed as decimal */ - char ar_gid[6]; /* owner gid; printed as decimal */ - char ar_mode[8]; /* file mode, printed as octal */ - char ar_size[10]; /* file size, printed as decimal */ - char ar_fmag[2]; /* should contain ARFMAG */ -} ArchiveHeader; - -#define ARFMAG "`\n" - -static unsigned long long get_be(const uint8_t *b, int n) -{ - unsigned long long ret = 0; - while (n) - ret = (ret << 8) | *b++, --n; - return ret; -} - -static int read_ar_header(int fd, int offset, ArchiveHeader *hdr) -{ - char *p, *e; - int len; - lseek(fd, offset, SEEK_SET); - len = full_read(fd, hdr, sizeof(ArchiveHeader)); - if (len != sizeof(ArchiveHeader)) - return len ? -1 : 0; - if (memcmp(hdr->ar_fmag, ARFMAG, sizeof hdr->ar_fmag)) - return -1; - p = hdr->ar_name; - for (e = p + sizeof hdr->ar_name; e > p && e[-1] == ' ';) - --e; - *e = '\0'; - hdr->ar_size[sizeof hdr->ar_size - 1] = 0; - return len; -} - -static int alacarte_member_seen(unsigned long long *loaded_members, unsigned int mask, unsigned long long offset) -{ - unsigned long long key = offset + 1; - unsigned int idx = ((unsigned int)offset ^ (unsigned int)(offset >> 32)) & mask; - - while (loaded_members[idx]) - { - if (loaded_members[idx] == key) - return 1; - idx = (idx + 1) & mask; - } - return 0; -} - -static void alacarte_mark_member(unsigned long long *loaded_members, unsigned int mask, unsigned long long offset) -{ - unsigned long long key = offset + 1; - unsigned int idx = ((unsigned int)offset ^ (unsigned int)(offset >> 32)) & mask; - - while (loaded_members[idx] && loaded_members[idx] != key) - { - idx = (idx + 1) & mask; - } - loaded_members[idx] = key; -} - -static Section *find_existing_section(TCCState *s1, const char *name) -{ - return section_ht_find(s1, name); -} - -/* Check whether any currently undefined symbol is satisfiable by cached - archive symbol tables. Returns 1 if at least one such undef exists, - meaning a group rescan may still be productive. */ -ST_FUNC int tcc_group_has_satisfiable_undefs(TCCState *s1) -{ - Section *s = symtab_section; - ElfW(Sym) *syms = (ElfW(Sym) *)s->data; - Section *hs = s->hash; - unsigned int *sym_hcache = hs ? hs->hash_val_cache : NULL; - int ci, ui, si; - if (s1->nb_archive_sym_caches == 0) - return 0; - for (ui = 0; ui < s1->nb_undef_syms; ui++) - { - unsigned int h; - int aidx; - si = s1->undef_sym_list[ui]; - if (syms[si].st_shndx != SHN_UNDEF) - continue; - const char *name = (char *)s->link->data + syms[si].st_name; - h = (sym_hcache && si < hs->hash_val_alloc) ? sym_hcache[si] : elf_hash((const unsigned char *)name); - for (ci = 0; ci < s1->nb_archive_sym_caches; ci++) - { - ArchiveSymbolCache *cache = &s1->archive_sym_caches[ci]; - for (aidx = cache->ar_ht_buckets[h & cache->ar_ht_mask]; aidx >= 0; aidx = cache->ar_ht_next[aidx]) - { - if (cache->name_hashes[aidx] == h && !strcmp(cache->sym_names[aidx], name)) - return 1; - } - } - } - return 0; -} - -/* Find an existing archive symbol cache entry by filename */ -static ArchiveSymbolCache *find_archive_sym_cache(TCCState *s1, const char *filename) -{ - int i; - for (i = 0; i < s1->nb_archive_sym_caches; i++) - { - if (!strcmp(s1->archive_sym_caches[i].filename, filename)) - return &s1->archive_sym_caches[i]; - } - return NULL; -} - -/* Create and populate a new archive symbol cache entry */ -static ArchiveSymbolCache *create_archive_sym_cache(TCCState *s1, const char *filename, int fd, int size, int entrysize) -{ - int i, nsyms, ar_ht_size; - unsigned int loaded_member_size; - uint8_t *data; - const uint8_t *ar_index; - const char *ar_names, *p; - ArchiveSymbolCache *cache; - - data = tcc_malloc(size); - if (full_read(fd, data, size) != size) - { - tcc_free(data); - return NULL; - } - nsyms = get_be(data, entrysize); - ar_index = data + entrysize; - ar_names = (char *)ar_index + nsyms * entrysize; - - /* Grow cache array */ - s1->archive_sym_caches = - tcc_realloc(s1->archive_sym_caches, (s1->nb_archive_sym_caches + 1) * sizeof(ArchiveSymbolCache)); - cache = &s1->archive_sym_caches[s1->nb_archive_sym_caches++]; - memset(cache, 0, sizeof(*cache)); - - cache->filename = tcc_strdup(filename); - cache->data = data; - cache->nsyms = nsyms; - cache->entrysize = entrysize; - - /* Allocate symbol arrays */ - cache->sym_names = tcc_malloc(nsyms * sizeof(const char *)); - cache->name_hashes = tcc_malloc(nsyms * sizeof(unsigned int)); - cache->member_offsets = tcc_malloc(nsyms * sizeof(unsigned long long)); - - /* Build chained hash table */ - ar_ht_size = 1; - while (ar_ht_size < nsyms * 2) - ar_ht_size <<= 1; - cache->ar_ht_mask = ar_ht_size - 1; - cache->ar_ht_buckets = tcc_malloc(ar_ht_size * sizeof(int)); - memset(cache->ar_ht_buckets, 0xff, ar_ht_size * sizeof(int)); - cache->ar_ht_next = tcc_malloc(nsyms * sizeof(int)); - memset(cache->ar_ht_next, 0xff, nsyms * sizeof(int)); - - /* Size loaded_members dedup set */ - loaded_member_size = 256; - if ((unsigned int)nsyms / 8 > loaded_member_size) - loaded_member_size = (unsigned int)nsyms / 8; - { - unsigned int v = loaded_member_size; - v--; - v |= v >> 1; - v |= v >> 2; - v |= v >> 4; - v |= v >> 8; - v |= v >> 16; - loaded_member_size = v + 1; - } - cache->loaded_member_mask = loaded_member_size - 1; - cache->loaded_members = tcc_mallocz(loaded_member_size * sizeof(unsigned long long)); - - /* Populate symbol arrays and hash table */ - for (p = ar_names, i = 0; i < nsyms; i++, p += strlen(p) + 1) - { - unsigned int h; - int bucket; - cache->sym_names[i] = p; - cache->name_hashes[i] = h = elf_hash((const unsigned char *)p); - cache->member_offsets[i] = get_be(ar_index + i * entrysize, entrysize); - bucket = h & cache->ar_ht_mask; - cache->ar_ht_next[i] = cache->ar_ht_buckets[bucket]; - cache->ar_ht_buckets[bucket] = i; - } - - return cache; -} - -/* Free all cached archive symbol tables */ -ST_FUNC void tcc_archive_cache_free(TCCState *s1) -{ - int i; - for (i = 0; i < s1->nb_archive_sym_caches; i++) - { - ArchiveSymbolCache *c = &s1->archive_sym_caches[i]; - tcc_free(c->filename); - tcc_free(c->loaded_members); - tcc_free(c->member_offsets); - tcc_free(c->name_hashes); - tcc_free(c->sym_names); - tcc_free(c->ar_ht_next); - tcc_free(c->ar_ht_buckets); - tcc_free(c->data); - } - tcc_free(s1->archive_sym_caches); - s1->archive_sym_caches = NULL; - s1->nb_archive_sym_caches = 0; -} - -/* load only the objects which resolve undefined symbols */ -static int tcc_load_alacarte(TCCState *s1, int fd, int size, int entrysize) -{ - int bound, len, ret = -1; - unsigned long long off; - ElfW(Sym) * sym; - ArchiveHeader hdr; - ArchiveSymbolCache *cache; - /* Save archive state for restoration */ - unsigned long saved_archive_offset = s1->current_archive_offset; - const char *saved_archive_path = s1->current_archive_path; - s1->current_archive_path = s1->current_filename; - - /* Look up or create cached archive symbol table. On --start-group - rescans this avoids re-reading the index, re-computing elf_hash - for all symbols, and re-allocating hash tables. */ - cache = find_archive_sym_cache(s1, s1->current_filename); - if (!cache) - { - cache = create_archive_sym_cache(s1, s1->current_filename, fd, size, entrysize); - if (!cache) - { - tcc_error_noabort("invalid archive"); - s1->current_archive_offset = saved_archive_offset; - s1->current_archive_path = saved_archive_path; - return -1; - } - } - - /* Inverted resolution: iterate tracked undefined symbols, - look up each in archive hash table. O(n_undef) per pass instead - of O(n_symtab) in the original forward scan. */ - do - { - Section *s = symtab_section; - Section *hs = s->hash; - unsigned int *sym_hcache = hs ? hs->hash_val_cache : NULL; - int ui, si; - bound = 0; - - for (ui = 0; ui < s1->nb_undef_syms; ui++) - { - unsigned int h; - int aidx; - - si = s1->undef_sym_list[ui]; - sym = &((ElfW(Sym) *)s->data)[si]; - if (sym->st_shndx != SHN_UNDEF) - continue; - - /* Use cached hash from symtab hash table when available, - avoiding elf_hash recomputation across archive calls. */ - h = (sym_hcache && si < hs->hash_val_alloc) - ? sym_hcache[si] - : elf_hash((const unsigned char *)((char *)s->link->data + sym->st_name)); - - for (aidx = cache->ar_ht_buckets[h & cache->ar_ht_mask]; aidx >= 0; aidx = cache->ar_ht_next[aidx]) - { - if (cache->name_hashes[aidx] != h) - continue; - { - const char *sym_name = (char *)s->link->data + sym->st_name; - if (strcmp(cache->sym_names[aidx], sym_name)) - continue; - } - off = cache->member_offsets[aidx]; - if (alacarte_member_seen(cache->loaded_members, cache->loaded_member_mask, off)) - break; - - len = read_ar_header(fd, off, &hdr); - if (len <= 0 || memcmp(hdr.ar_fmag, ARFMAG, 2)) - { - tcc_error_noabort("invalid archive"); - goto the_end; - } - off += len; - if (s1->verbose == 2) - printf(" -> %s\n", hdr.ar_name); - /* Set archive offset for lazy loading */ - s1->current_archive_offset = (unsigned long)off; - if (tcc_load_object_file(s1, fd, off) < 0) - goto the_end; - s1->current_archive_offset = saved_archive_offset; - alacarte_mark_member(cache->loaded_members, cache->loaded_member_mask, cache->member_offsets[aidx]); - ++bound; - ++s1->group_rescan_loaded; - /* symtab/hash may have been reallocated by tcc_load_object_file */ - hs = s->hash; - sym_hcache = hs ? hs->hash_val_cache : NULL; - break; - } - } - } while (bound); - /* Compact the list, removing symbols now defined. Shrinks iteration - cost for subsequent archive loads and group-rescan checks. */ - { - ElfW(Sym) *syms = (ElfW(Sym) *)symtab_section->data; - int ri, wi = 0; - for (ri = 0; ri < s1->nb_undef_syms; ri++) - { - int idx = s1->undef_sym_list[ri]; - if (syms[idx].st_shndx == SHN_UNDEF) - s1->undef_sym_list[wi++] = idx; - } - s1->nb_undef_syms = wi; - } - ret = 0; -the_end: - s1->current_archive_offset = saved_archive_offset; - s1->current_archive_path = saved_archive_path; - return ret; -} - -/* load a '.a' file */ -ST_FUNC int tcc_load_archive(TCCState *s1, int fd, int alacarte) -{ - ArchiveHeader hdr; - /* char magic[8]; */ - int size, len; - unsigned long file_offset; - ElfW(Ehdr) ehdr; - unsigned long saved_archive_offset; - const char *saved_archive_path; - unsigned archive_start = 0; - unsigned members_before = 0; - char archive_desc[1088]; - - if (s1->do_bench) - { - archive_start = tcc_getclock_us(); - members_before = s1->bench_archive_member_count; - } - - /* skip magic which was already checked */ - /* full_read(fd, magic, sizeof(magic)); */ - file_offset = sizeof ARMAG - 1; - - /* Save archive state for restoration */ - saved_archive_offset = s1->current_archive_offset; - saved_archive_path = s1->current_archive_path; - s1->current_archive_path = s1->current_filename; - - for (;;) - { - len = read_ar_header(fd, file_offset, &hdr); - if (len == 0) - { - if (s1->do_bench) - { - unsigned elapsed = tcc_getclock_us() - archive_start; - unsigned members_loaded = s1->bench_archive_member_count - members_before; - s1->bench_archive_load_time += elapsed; - s1->bench_archive_load_count++; - snprintf(archive_desc, sizeof(archive_desc), "%s (%u members)", - s1->current_filename ? s1->current_filename : "", members_loaded); - tcc_bench_log(s1, "load-archive", archive_desc, elapsed); - } - s1->current_archive_offset = saved_archive_offset; - s1->current_archive_path = saved_archive_path; - return 0; - } - if (len < 0) - { - s1->current_archive_offset = saved_archive_offset; - s1->current_archive_path = saved_archive_path; - return tcc_error_noabort("invalid archive"); - } - file_offset += len; - size = strtol(hdr.ar_size, NULL, 0); - if (alacarte) - { - /* coff symbol table : we handle it */ - if (!strcmp(hdr.ar_name, "/")) - { - int ret = tcc_load_alacarte(s1, fd, size, 4); - if (s1->do_bench) - { - unsigned elapsed = tcc_getclock_us() - archive_start; - unsigned members_loaded = s1->bench_archive_member_count - members_before; - s1->bench_archive_load_time += elapsed; - s1->bench_archive_load_count++; - snprintf(archive_desc, sizeof(archive_desc), "%s (%u members)", - s1->current_filename ? s1->current_filename : "", members_loaded); - tcc_bench_log(s1, "load-archive", archive_desc, elapsed); - } - s1->current_archive_offset = saved_archive_offset; - s1->current_archive_path = saved_archive_path; - return ret; - } - if (!strcmp(hdr.ar_name, "/SYM64/")) - { - int ret = tcc_load_alacarte(s1, fd, size, 8); - if (s1->do_bench) - { - unsigned elapsed = tcc_getclock_us() - archive_start; - unsigned members_loaded = s1->bench_archive_member_count - members_before; - s1->bench_archive_load_time += elapsed; - s1->bench_archive_load_count++; - snprintf(archive_desc, sizeof(archive_desc), "%s (%u members)", - s1->current_filename ? s1->current_filename : "", members_loaded); - tcc_bench_log(s1, "load-archive", archive_desc, elapsed); - } - s1->current_archive_offset = saved_archive_offset; - s1->current_archive_path = saved_archive_path; - return ret; - } - } - else if (tcc_object_type(fd, &ehdr) == AFF_BINTYPE_REL) - { - if (s1->verbose == 2) - printf(" -> %s\n", hdr.ar_name); - /* Set archive offset for lazy loading */ - s1->current_archive_offset = file_offset; - if (tcc_load_object_file(s1, fd, file_offset) < 0) - { - s1->current_archive_offset = saved_archive_offset; - s1->current_archive_path = saved_archive_path; - return -1; - } - s1->current_archive_offset = saved_archive_offset; - } - /* align to even */ - file_offset = (file_offset + size + 1) & ~1; - } - /* Unreachable for non-alacarte mode; required to silence - 'function might return no value' warning in TCC self-build. */ - s1->current_archive_offset = saved_archive_offset; - s1->current_archive_path = saved_archive_path; - return 0; -} - -#ifndef ELF_OBJ_ONLY -/* Set LV[I] to the global index of sym-version (LIB,VERSION). Maybe resizes - LV, maybe create a new entry for (LIB,VERSION). */ -static void set_ver_to_ver(TCCState *s1, int *n, int **lv, int i, char *lib, char *version) -{ - while (i >= *n) - { - *lv = tcc_realloc(*lv, (*n + 1) * sizeof(**lv)); - (*lv)[(*n)++] = -1; - } - if ((*lv)[i] == -1) - { - int v, prev_same_lib = -1; - for (v = 0; v < nb_sym_versions; v++) - { - if (strcmp(sym_versions[v].lib, lib)) - continue; - prev_same_lib = v; - if (!strcmp(sym_versions[v].version, version)) - break; - } - if (v == nb_sym_versions) - { - sym_versions = tcc_realloc(sym_versions, (v + 1) * sizeof(*sym_versions)); - sym_versions[v].lib = tcc_strdup(lib); - sym_versions[v].version = tcc_strdup(version); - sym_versions[v].out_index = 0; - sym_versions[v].prev_same_lib = prev_same_lib; - nb_sym_versions++; - } - (*lv)[i] = v; - } -} - -/* Associates symbol SYM_INDEX (in dynsymtab) with sym-version index - VERNDX. */ -static void set_sym_version(TCCState *s1, int sym_index, int verndx) -{ - if (sym_index >= nb_sym_to_version) - { - int newelems = sym_index ? sym_index * 2 : 1; - sym_to_version = tcc_realloc(sym_to_version, newelems * sizeof(*sym_to_version)); - memset(sym_to_version + nb_sym_to_version, -1, (newelems - nb_sym_to_version) * sizeof(*sym_to_version)); - nb_sym_to_version = newelems; - } - if (sym_to_version[sym_index] < 0) - sym_to_version[sym_index] = verndx; -} - -struct versym_info -{ - int nb_versyms; - ElfW(Verdef) * verdef; - ElfW(Verneed) * verneed; - ElfW(Half) * versym; - int nb_local_ver, *local_ver; -}; - -static void store_version(TCCState *s1, struct versym_info *v, char *dynstr) -{ - char *lib, *version; - uint32_t next; - int i; - -#define DEBUG_VERSION 0 - - if (v->versym && v->verdef) - { - ElfW(Verdef) *vdef = v->verdef; - lib = NULL; - do - { - ElfW(Verdaux) *verdaux = (ElfW(Verdaux) *)(((char *)vdef) + vdef->vd_aux); - -#if DEBUG_VERSION - printf("verdef: version:%u flags:%u index:%u, hash:%u\n", vdef->vd_version, vdef->vd_flags, vdef->vd_ndx, - vdef->vd_hash); -#endif - if (vdef->vd_cnt) - { - version = dynstr + verdaux->vda_name; - - if (lib == NULL) - lib = version; - else - set_ver_to_ver(s1, &v->nb_local_ver, &v->local_ver, vdef->vd_ndx, lib, version); -#if DEBUG_VERSION - printf(" verdaux(%u): %s\n", vdef->vd_ndx, version); -#endif - } - next = vdef->vd_next; - vdef = (ElfW(Verdef) *)(((char *)vdef) + next); - } while (next); - } - if (v->versym && v->verneed) - { - ElfW(Verneed) *vneed = v->verneed; - do - { - ElfW(Vernaux) *vernaux = (ElfW(Vernaux) *)(((char *)vneed) + vneed->vn_aux); - - lib = dynstr + vneed->vn_file; -#if DEBUG_VERSION - printf("verneed: %u %s\n", vneed->vn_version, lib); -#endif - for (i = 0; i < vneed->vn_cnt; i++) - { - if ((vernaux->vna_other & 0x8000) == 0) - { /* hidden */ - version = dynstr + vernaux->vna_name; - set_ver_to_ver(s1, &v->nb_local_ver, &v->local_ver, vernaux->vna_other, lib, version); -#if DEBUG_VERSION - printf(" vernaux(%u): %u %u %s\n", vernaux->vna_other, vernaux->vna_hash, vernaux->vna_flags, version); -#endif - } - vernaux = (ElfW(Vernaux) *)(((char *)vernaux) + vernaux->vna_next); - } - next = vneed->vn_next; - vneed = (ElfW(Verneed) *)(((char *)vneed) + next); - } while (next); - } - -#if DEBUG_VERSION - for (i = 0; i < v->nb_local_ver; i++) - { - if (v->local_ver[i] > 0) - { - printf("%d: lib: %s, version %s\n", i, sym_versions[v->local_ver[i]].lib, sym_versions[v->local_ver[i]].version); - } - } -#endif -} -/* load a library / DLL - 'level = 0' means that the DLL is referenced by the user - (so it should be added as DT_NEEDED in the generated ELF file) */ -ST_FUNC int tcc_load_dll(TCCState *s1, int fd, const char *filename, int level) -{ - ElfW(Ehdr) ehdr; - ElfW(Shdr) * shdr, *sh, *sh1; - int i, nb_syms, nb_dts, sym_bind, ret = -1; - ElfW(Sym) * sym, *dynsym; - ElfW(Dyn) * dt, *dynamic; - - char *dynstr; - int sym_index; - const char *name, *soname; - struct versym_info v; - unsigned dll_start = 0; - - if (s1->do_bench) - dll_start = tcc_getclock_us(); - - full_read(fd, &ehdr, sizeof(ehdr)); - - /* test CPU specific stuff */ - if (ehdr.e_ident[5] != ELFDATA2LSB || ehdr.e_machine != EM_TCC_TARGET) - { - return tcc_error_noabort("bad architecture"); - } - - /* read sections */ - shdr = load_data(fd, ehdr.e_shoff, sizeof(ElfW(Shdr)) * ehdr.e_shnum); - - /* load dynamic section and dynamic symbols */ - nb_syms = 0; - nb_dts = 0; - dynamic = NULL; - dynsym = NULL; /* avoid warning */ - dynstr = NULL; /* avoid warning */ - memset(&v, 0, sizeof v); - - for (i = 0, sh = shdr; i < ehdr.e_shnum; i++, sh++) - { - switch (sh->sh_type) - { - case SHT_DYNAMIC: - nb_dts = sh->sh_size / sizeof(ElfW(Dyn)); - dynamic = load_data(fd, sh->sh_offset, sh->sh_size); - break; - case SHT_DYNSYM: - nb_syms = sh->sh_size / sizeof(ElfW(Sym)); - dynsym = load_data(fd, sh->sh_offset, sh->sh_size); - sh1 = &shdr[sh->sh_link]; - dynstr = load_data(fd, sh1->sh_offset, sh1->sh_size); - break; - case SHT_GNU_verdef: - v.verdef = load_data(fd, sh->sh_offset, sh->sh_size); - break; - case SHT_GNU_verneed: - v.verneed = load_data(fd, sh->sh_offset, sh->sh_size); - break; - case SHT_GNU_versym: - v.nb_versyms = sh->sh_size / sizeof(ElfW(Half)); - v.versym = load_data(fd, sh->sh_offset, sh->sh_size); - break; - default: - break; - } - } - - if (!dynamic) - goto the_end; - - /* compute the real library name */ - soname = tcc_basename(filename); - for (i = 0, dt = dynamic; i < nb_dts; i++, dt++) - if (dt->d_tag == DT_SONAME) - soname = dynstr + dt->d_un.d_val; - - /* if the dll is already loaded, do not load it */ - if (tcc_add_dllref(s1, soname, level)->found) - goto ret_success; - - if (v.nb_versyms != nb_syms) - tcc_free(v.versym), v.versym = NULL; - else - store_version(s1, &v, dynstr); - - /* add dynamic symbols in dynsym_section */ - for (i = 1, sym = dynsym + 1; i < nb_syms; i++, sym++) - { - sym_bind = ELFW(ST_BIND)(sym->st_info); - if (sym_bind == STB_LOCAL) - continue; - name = dynstr + sym->st_name; - sym_index = set_elf_sym(s1->dynsymtab_section, sym->st_value, sym->st_size, sym->st_info, sym->st_other, - sym->st_shndx, name); - if (v.versym) - { - ElfW(Half) vsym = v.versym[i]; - if ((vsym & 0x8000) == 0 && vsym > 0 && vsym < v.nb_local_ver) - set_sym_version(s1, sym_index, v.local_ver[vsym]); - } - } - - /* do not load all referenced libraries - (recursive loading can break linking of libraries) */ - /* following DT_NEEDED is needed for the dynamic loader (libdl.so), - but it is no longer needed, when linking a library or a program. - When tcc output mode is OUTPUT_MEM, - tcc calls dlopen, which handles DT_NEEDED for us */ - -#if 0 - for(i = 0, dt = dynamic; i < nb_dts; i++, dt++) - if (dt->d_tag == DT_RPATH) - tcc_add_library_path(s1, dynstr + dt->d_un.d_val); - - /* load all referenced DLLs */ - for(i = 0, dt = dynamic; i < nb_dts; i++, dt++) { - switch(dt->d_tag) { - case DT_NEEDED: - name = dynstr + dt->d_un.d_val; - if (tcc_add_dllref(s1, name, -1)) - continue; - if (tcc_add_dll(s1, name, AFF_REFERENCED_DLL) < 0) { - ret = tcc_error_noabort("referenced dll '%s' not found", name); - goto the_end; - } - } - } -#endif - -ret_success: - ret = 0; -the_end: - if (s1->do_bench) - { - unsigned elapsed = tcc_getclock_us() - dll_start; - s1->bench_dll_load_time += elapsed; - s1->bench_dll_load_count++; - tcc_bench_log(s1, "load-dll", filename, elapsed); - } - tcc_free(dynstr); - tcc_free(dynsym); - tcc_free(dynamic); - tcc_free(shdr); - tcc_free(v.local_ver); - tcc_free(v.verdef); - tcc_free(v.verneed); - tcc_free(v.versym); - return ret; -} - -#define LD_TOK_NAME 256 -#define LD_TOK_EOF (-1) - -static int ld_inp(TCCState *s1) -{ - char b; - if (s1->cc != -1) - { - int c = s1->cc; - s1->cc = -1; - return c; - } - if (1 == read(s1->fd, &b, 1)) - return b; - return CH_EOF; -} - -/* return next ld script token */ -static int ld_next(TCCState *s1, char *name, int name_size) -{ - int c, d, ch; - char *q; - -redo: - ch = ld_inp(s1); - switch (ch) - { - case ' ': - case '\t': - case '\f': - case '\v': - case '\r': - case '\n': - goto redo; - case '/': - ch = ld_inp(s1); - if (ch == '*') - { /* comment */ - for (d = 0;; d = ch) - { - ch = ld_inp(s1); - if (ch == CH_EOF || (ch == '/' && d == '*')) - break; - } - goto redo; - } - else - { - q = name; - *q++ = '/'; - goto parse_name; - } - break; - case '\\': - /* case 'a' ... 'z': */ - case 'a': - case 'b': - case 'c': - case 'd': - case 'e': - case 'f': - case 'g': - case 'h': - case 'i': - case 'j': - case 'k': - case 'l': - case 'm': - case 'n': - case 'o': - case 'p': - case 'q': - case 'r': - case 's': - case 't': - case 'u': - case 'v': - case 'w': - case 'x': - case 'y': - case 'z': - /* case 'A' ... 'z': */ - case 'A': - case 'B': - case 'C': - case 'D': - case 'E': - case 'F': - case 'G': - case 'H': - case 'I': - case 'J': - case 'K': - case 'L': - case 'M': - case 'N': - case 'O': - case 'P': - case 'Q': - case 'R': - case 'S': - case 'T': - case 'U': - case 'V': - case 'W': - case 'X': - case 'Y': - case 'Z': - case '_': - case '.': - case '$': - case '~': - q = name; - parse_name: - for (;;) - { - if (!((ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || (ch >= '0' && ch <= '9') || - strchr("/.-_+=$:\\,~", ch))) - break; - if ((q - name) < name_size - 1) - { - *q++ = ch; - } - ch = ld_inp(s1); - } - s1->cc = ch; - *q = '\0'; - c = LD_TOK_NAME; - break; - case CH_EOF: - c = LD_TOK_EOF; - break; - default: - c = ch; - break; - } - return c; -} - -static int ld_add_file(TCCState *s1, const char filename[]) -{ - if (filename[0] == '/') - { - if (CONFIG_SYSROOT[0] == '\0' && tcc_add_file_internal(s1, filename, AFF_TYPE_BIN) == 0) - return 0; - filename = tcc_basename(filename); - } - return tcc_add_dll(s1, filename, AFF_PRINT_ERROR); -} - -static int ld_add_file_list(TCCState *s1, const char *cmd, int as_needed) -{ - char filename[1024], libname[1016]; - int t, group, nblibs = 0, ret = 0; - char **libs = NULL; - - group = !strcmp(cmd, "GROUP"); - if (!as_needed) - s1->new_undef_sym = 0; - t = ld_next(s1, filename, sizeof(filename)); - if (t != '(') - { - ret = tcc_error_noabort("( expected"); - goto lib_parse_error; - } - t = ld_next(s1, filename, sizeof(filename)); - for (;;) - { - libname[0] = '\0'; - if (t == LD_TOK_EOF) - { - ret = tcc_error_noabort("unexpected end of file"); - goto lib_parse_error; - } - else if (t == ')') - { - break; - } - else if (t == '-') - { - t = ld_next(s1, filename, sizeof(filename)); - if ((t != LD_TOK_NAME) || (filename[0] != 'l')) - { - ret = tcc_error_noabort("library name expected"); - goto lib_parse_error; - } - pstrcpy(libname, sizeof libname, &filename[1]); - if (s1->static_link) - { - snprintf(filename, sizeof filename, "lib%s.a", libname); - } - else - { - snprintf(filename, sizeof filename, "lib%s.so", libname); - } - } - else if (t != LD_TOK_NAME) - { - ret = tcc_error_noabort("filename expected"); - goto lib_parse_error; - } - if (!strcmp(filename, "AS_NEEDED")) - { - ret = ld_add_file_list(s1, cmd, 1); - if (ret) - goto lib_parse_error; - } - else - { - /* TODO: Implement AS_NEEDED support. */ - /* DT_NEEDED is not used any more so ignore as_needed */ - if (1 || !as_needed) - { - ret = ld_add_file(s1, filename); - if (ret) - goto lib_parse_error; - if (group) - { - /* Add the filename *and* the libname to avoid future conversions */ - dynarray_add(&libs, &nblibs, tcc_strdup(filename)); - if (libname[0] != '\0') - dynarray_add(&libs, &nblibs, tcc_strdup(libname)); - } - } - } - t = ld_next(s1, filename, sizeof(filename)); - if (t == ',') - { - t = ld_next(s1, filename, sizeof(filename)); - } - } - if (group && !as_needed) - { - /* Same fast check as the CLI --start-group loop: only continue - rescanning while some current undef is satisfiable by these - cached archives. */ - if (s1->new_undef_sym) - { - s1->new_undef_sym = tcc_group_has_satisfiable_undefs(s1); - } - while (s1->new_undef_sym) - { - int i; - s1->new_undef_sym = 0; - for (i = 0; i < nblibs; i++) - ld_add_file(s1, libs[i]); - } - } -lib_parse_error: - dynarray_reset(&libs, &nblibs); - return ret; -} - -/* interpret a subset of GNU ldscripts to handle the dummy libc.so - files */ -ST_FUNC int tcc_load_ldscript(TCCState *s1, int fd) -{ - char cmd[64]; - char filename[1024]; - int t, ret; - unsigned ldscript_start = 0; - - if (s1->do_bench) - ldscript_start = tcc_getclock_us(); - - s1->fd = fd; - s1->cc = -1; - for (;;) - { - t = ld_next(s1, cmd, sizeof(cmd)); - if (t == LD_TOK_EOF) - { - if (s1->do_bench) - { - unsigned elapsed = tcc_getclock_us() - ldscript_start; - s1->bench_ldscript_load_time += elapsed; - s1->bench_ldscript_load_count++; - tcc_bench_log(s1, "ldscript", s1->current_filename, elapsed); - } - return 0; - } - else if (t != LD_TOK_NAME) - return -1; - if (!strcmp(cmd, "INPUT") || !strcmp(cmd, "GROUP")) - { - ret = ld_add_file_list(s1, cmd, 0); - if (ret) - return ret; - } - else if (!strcmp(cmd, "OUTPUT_FORMAT") || !strcmp(cmd, "TARGET")) - { - /* ignore some commands */ - t = ld_next(s1, cmd, sizeof(cmd)); - if (t != '(') - return tcc_error_noabort("( expected"); - for (;;) - { - t = ld_next(s1, filename, sizeof(filename)); - if (t == LD_TOK_EOF) - { - return tcc_error_noabort("unexpected end of file"); - } - else if (t == ')') - { - break; - } - } - } - else - { - return -1; - } - } - return 0; -} - -/* Load and parse a linker script file */ -ST_FUNC int tcc_load_linker_script(TCCState *s1, const char *filename) -{ - int fd; - int ret; - - fd = open(filename, O_RDONLY | O_BINARY); - if (fd < 0) - { - return tcc_error_noabort("linker script '%s' not found", filename); - } - - /* Allocate linker script structure if not already done */ - if (!s1->ld_script) - { - s1->ld_script = tcc_mallocz(sizeof(LDScript)); - ld_script_init(s1->ld_script); - } - - ret = ld_script_parse(s1, s1->ld_script, fd); - close(fd); - -#if TCCELF_DUMP_LD_SCRIPT - if (ret == 0 && s1->verbose) - { - printf("Loaded linker script: %s\n", filename); - ld_script_dump(s1->ld_script); - } -#endif - - /* Add standard symbols */ - if (ret == 0) - { - ld_script_add_standard_symbols(s1, s1->ld_script); - } - - return ret; -} - -/* Apply linker script symbols to the ELF symbol table */ -static void ld_apply_symbols(TCCState *s1, LDScript *ld) -{ - int i; - for (i = 0; i < ld->nb_symbols; i++) - { - LDSymbol *sym = &ld->symbols[i]; - if (sym->defined) - { - int sym_idx; - int vis = - (sym->visibility == LD_SYM_HIDDEN || sym->visibility == LD_SYM_PROVIDE_HIDDEN) ? STV_HIDDEN : STV_DEFAULT; - - /* Look up once, reuse for both PROVIDE check and update */ - sym_idx = find_elf_sym(s1->symtab, sym->name); - - /* For PROVIDE symbols, only define if not already defined */ - if (sym->visibility == LD_SYM_PROVIDE || sym->visibility == LD_SYM_PROVIDE_HIDDEN) - { - if (sym_idx) - { - ElfW(Sym) *esym = &((ElfW(Sym) *)s1->symtab->data)[sym_idx]; - if (esym->st_shndx != SHN_UNDEF) - continue; /* Already defined, skip */ - } - } - - /* Update existing symbol, or create new */ - if (sym_idx) - { - ElfW(Sym) *esym = &((ElfW(Sym) *)s1->symtab->data)[sym_idx]; - esym->st_value = sym->value; - esym->st_shndx = SHN_ABS; - } - else - { - /* Use set_elf_sym directly with SHN_ABS to ensure symbols with value 0 - * are still defined as absolute (set_global_sym treats value 0 as - * UNDEF) - */ - set_elf_sym(s1->symtab, sym->value, 0, ELFW(ST_INFO)(STB_GLOBAL, STT_NOTYPE), vis, SHN_ABS, sym->name); - } - - /* Also update in dynsym if it exists there */ - if (s1->dynsym) - { - sym_idx = find_elf_sym(s1->dynsym, sym->name); - if (sym_idx) - { - ElfW(Sym) *esym = &((ElfW(Sym) *)s1->dynsym->data)[sym_idx]; - esym->st_value = sym->value; - esym->st_shndx = SHN_ABS; - } - } - } - } -} - -/* Update linker script symbol values based on actual section layout */ -static void ld_update_symbol_values(TCCState *s1, LDScript *ld) -{ - Section *s; - addr_t bss_start = 0, bss_end = 0; - addr_t data_start = 0, data_end = 0; - addr_t text_start = 0, text_end = 0; - addr_t rodata_start = 0, rodata_end = 0; - addr_t end_addr = 0; - addr_t sec_end; - int i, j; - addr_t output_section_addrs[LD_MAX_OUTPUT_SECTIONS] = {0}; - int output_section_has_addr[LD_MAX_OUTPUT_SECTIONS] = {0}; - addr_t output_section_loadaddrs[LD_MAX_OUTPUT_SECTIONS] = {0}; - addr_t output_section_sizes[LD_MAX_OUTPUT_SECTIONS] = {0}; - addr_t output_section_align[LD_MAX_OUTPUT_SECTIONS] = {0}; - addr_t output_section_vma_end[LD_MAX_OUTPUT_SECTIONS] = {0}; - - /* Find section addresses and map output sections to actual addresses */ - for (i = 1; i < s1->nb_sections; i++) - { - s = s1->sections[i]; - if (!s->sh_addr) - continue; - - sec_end = s->sh_addr + s->sh_size; - if (sec_end > end_addr) - end_addr = sec_end; - - /* Match .bss and .bss.* sections */ - if (!strcmp(s->name, ".bss") || !strncmp(s->name, ".bss.", 5)) - { - if (bss_start == 0 || s->sh_addr < bss_start) - bss_start = s->sh_addr; - if (sec_end > bss_end) - bss_end = sec_end; - } - /* Match .data and .data.* sections */ - else if (!strcmp(s->name, ".data") || !strncmp(s->name, ".data.", 6)) - { - if (data_start == 0 || s->sh_addr < data_start) - data_start = s->sh_addr; - if (sec_end > data_end) - data_end = sec_end; - } - /* Match .text and .text.* sections */ - else if (!strcmp(s->name, ".text") || !strncmp(s->name, ".text.", 6)) - { - if (text_start == 0 || s->sh_addr < text_start) - text_start = s->sh_addr; - if (sec_end > text_end) - text_end = sec_end; - } - /* Match .rodata and .rodata.* sections */ - else if (!strcmp(s->name, ".rodata") || !strncmp(s->name, ".rodata.", 8)) - { - if (rodata_start == 0 || s->sh_addr < rodata_start) - rodata_start = s->sh_addr; - if (sec_end > rodata_end) - rodata_end = sec_end; - } - - /* Map output section names to addresses */ - for (j = 0; j < ld->nb_output_sections; j++) - { - if (!strcmp(s->name, ld->output_sections[j].name)) - { - output_section_addrs[j] = s->sh_addr; - output_section_has_addr[j] = 1; - break; - } - } - - /* Accumulate sizes/alignments for load address computation */ - if (ld) - { - int pat_idx = -1; - int ld_idx = ld_find_output_section_idx(s1, s->name, &pat_idx); - if (ld_idx >= 0 && ld_idx < ld->nb_output_sections) - { - addr_t align = s->sh_addralign ? s->sh_addralign : 1; - if (align > output_section_align[ld_idx]) - output_section_align[ld_idx] = align; - if (s->sh_type != SHT_NOBITS) - output_section_sizes[ld_idx] += s->sh_size; - /* Track VMA end address (includes alignment between sections) */ - if (s->sh_addr + s->sh_size > output_section_vma_end[ld_idx]) - output_section_vma_end[ld_idx] = s->sh_addr + s->sh_size; - } - } - } - - /* Compute output section load addresses (LMA) */ - if (ld) - { - addr_t lma_cur[LD_MAX_MEMORY_REGIONS] = {0}; - if (ld->nb_memory_regions > 0) - { - for (i = 0; i < ld->nb_memory_regions; i++) - lma_cur[i] = ld->memory_regions[i].origin; - for (j = 0; j < ld->nb_output_sections; j++) - { - int load_mr = ld->output_sections[j].load_memory_region_idx; - int vma_mr = ld->output_sections[j].memory_region_idx; - int mr = (load_mr >= 0) ? load_mr : vma_mr; - if (mr < 0) - mr = 0; - if (mr >= 0 && mr < ld->nb_memory_regions) - { - addr_t align = output_section_align[j] ? output_section_align[j] : 1; - addr_t cur = lma_cur[mr]; - addr_t lma_start = (cur + align - 1) & ~(align - 1); - output_section_loadaddrs[j] = lma_start; - /* For sections where VMA region == LMA region (no AT > directive), - use the actual VMA end address which includes alignment padding - between input sections. Otherwise use the raw content size. */ - if (load_mr < 0 && output_section_vma_end[j] > lma_start) - lma_cur[mr] = output_section_vma_end[j]; - else - lma_cur[mr] = lma_start + output_section_sizes[j]; - } - } - } - else - { - for (j = 0; j < ld->nb_output_sections; j++) - output_section_loadaddrs[j] = output_section_addrs[j]; - } - - /* Save computed LMA values in LDScript for p_paddr fixup */ - for (j = 0; j < ld->nb_output_sections; j++) - ld->output_section_loadaddrs[j] = output_section_loadaddrs[j]; - ld->has_loadaddrs = 1; - } - - /* For NOLOAD sections (like .heap, .stack) that don't have actual ELF - * sections, compute their addresses based on the memory region they're - * assigned to in the linker script. Track per-region end addresses. */ - addr_t mr_end[LD_MAX_MEMORY_REGIONS] = {0}; - - /* Initialize memory region end addresses from laid-out sections */ - for (j = 0; j < ld->nb_output_sections; j++) - { - if (output_section_has_addr[j]) - { - int mr = ld->output_sections[j].memory_region_idx; - if (mr < 0) - mr = 0; - if (mr >= 0 && mr < ld->nb_memory_regions) - { - addr_t sec_end_addr = output_section_addrs[j] + ld->output_sections[j].current_offset; - if (sec_end_addr > mr_end[mr]) - mr_end[mr] = sec_end_addr; - } - } - } - - /* For regions with no sections, start from their origin */ - for (i = 0; i < ld->nb_memory_regions; i++) - { - if (mr_end[i] == 0) - mr_end[i] = ld->memory_regions[i].origin; - } - - /* Place NOLOAD sections in their assigned memory regions */ - for (j = 0; j < ld->nb_output_sections; j++) - { - if (output_section_addrs[j] == 0) - { - /* This output section has no matching ELF section (NOLOAD) */ - int mr = ld->output_sections[j].memory_region_idx; - if (mr < 0) - mr = 0; - if (mr >= 0 && mr < ld->nb_memory_regions) - { - output_section_addrs[j] = mr_end[mr]; - output_section_has_addr[j] = 1; - mr_end[mr] += ld->output_sections[j].current_offset; - } - } - } - - /* First pass: update symbols that are defined in output sections using - * section_offset. This handles all symbols generically. */ - for (j = 0; j < ld->nb_symbols; j++) - { - LDSymbol *sym = &ld->symbols[j]; - if (sym->defined && sym->section_idx >= 0 && sym->section_idx < ld->nb_output_sections) - { - addr_t section_addr = output_section_addrs[sym->section_idx]; - if (output_section_has_addr[sym->section_idx]) - { - /* Symbol value = section base address + offset within section */ - sym->value = section_addr + sym->section_offset; - } - } - } - - /* Resolve LOADADDR() symbols using computed LMA addresses */ - for (j = 0; j < ld->nb_symbols; j++) - { - LDSymbol *sym = &ld->symbols[j]; - if (sym->has_loadaddr && sym->loadaddr_section_idx >= 0 && sym->loadaddr_section_idx < ld->nb_output_sections) - { - addr_t lma = output_section_loadaddrs[sym->loadaddr_section_idx]; - sym->value = lma; - sym->defined = 1; - } - } - - /* Second pass: update standard section boundary symbols. - * For boundary symbols like __data_start__/__data_end__, always use - * the computed values based on actual section layout, because the - * linker script values are only relative offsets that don't account - * for all input sections. */ - for (j = 0; j < ld->nb_symbols; j++) - { - LDSymbol *sym = &ld->symbols[j]; - - /* Update standard section symbols - these ALWAYS use computed values */ - if (!strcmp(sym->name, "__bss_start__") || !strcmp(sym->name, "__bss_start")) - { - sym->value = bss_start; - sym->defined = 1; - } - else if (!strcmp(sym->name, "__bss_end__") || !strcmp(sym->name, "_bss_end__")) - { - sym->value = bss_end; - sym->defined = 1; - } - else if (!strcmp(sym->name, "__data_start__")) - { - sym->value = data_start; - sym->defined = 1; - } - else if (!strcmp(sym->name, "__data_end__") || !strcmp(sym->name, "_edata")) - { - sym->value = data_end; - sym->defined = 1; - } - else if (!strcmp(sym->name, "__text_start__") || !strcmp(sym->name, "_stext")) - { - sym->value = text_start; - sym->defined = 1; - } - else if (!strcmp(sym->name, "__text_end__") || !strcmp(sym->name, "_etext")) - { - sym->value = text_end; - sym->defined = 1; - } - else if (!strcmp(sym->name, "__rodata_start__")) - { - sym->value = rodata_start; - sym->defined = 1; - } - else if (!strcmp(sym->name, "__rodata_end__")) - { - sym->value = rodata_end; - sym->defined = 1; - } - else if (!strcmp(sym->name, "__end__") || !strcmp(sym->name, "_end") || !strcmp(sym->name, "end")) - { - sym->value = end_addr; - sym->defined = 1; - } - } -} - -/* Set or update a global symbol. If the symbol already exists, update its value - instead of trying to add it again (which would trigger "defined twice" error). */ -static void set_or_update_global_sym(TCCState *s1, const char *name, addr_t value) -{ - int sym_index = find_elf_sym(symtab_section, name); - if (sym_index) - { - ElfW(Sym) *esym = &((ElfW(Sym) *)symtab_section->data)[sym_index]; - if (esym->st_shndx != SHN_UNDEF) - { - /* Symbol already defined - update its value */ - esym->st_value = value; - esym->st_shndx = SHN_ABS; - return; - } - } - set_global_sym(s1, name, NULL, value); -} - -/* Export standard end/heap symbols based on section layout */ -ST_FUNC void ld_export_standard_symbols(TCCState *s1) -{ - Section *s; - addr_t bss_start = 0, bss_end = 0; - addr_t data_start = 0, data_end = 0; - addr_t text_start = 0, text_end = 0; - addr_t end_addr = 0; - addr_t sec_end; - int i; - - /* Find section addresses */ - for (i = 1; i < s1->nb_sections; i++) - { - s = s1->sections[i]; - if (!s->sh_addr) - continue; - - sec_end = s->sh_addr + s->sh_size; - if (sec_end > end_addr) - end_addr = sec_end; - - /* Match .bss and .bss.* sections */ - if (!strcmp(s->name, ".bss") || !strncmp(s->name, ".bss.", 5)) - { - if (bss_start == 0 || s->sh_addr < bss_start) - bss_start = s->sh_addr; - if (sec_end > bss_end) - bss_end = sec_end; - } - /* Match .data and .data.* sections */ - else if (!strcmp(s->name, ".data") || !strncmp(s->name, ".data.", 6)) - { - if (data_start == 0 || s->sh_addr < data_start) - data_start = s->sh_addr; - if (sec_end > data_end) - data_end = sec_end; - } - /* Match .text and .text.* sections */ - else if (!strcmp(s->name, ".text") || !strncmp(s->name, ".text.", 6)) - { - if (text_start == 0 || s->sh_addr < text_start) - text_start = s->sh_addr; - if (sec_end > text_end) - text_end = sec_end; - } - } - - /* Set standard symbols, updating existing ones if already defined - (e.g. by tcc_add_linker_symbols) */ - if (bss_start) - { - set_or_update_global_sym(s1, "__bss_start__", bss_start); - set_or_update_global_sym(s1, "__bss_start", bss_start); - } - if (bss_end) - { - set_or_update_global_sym(s1, "__bss_end__", bss_end); - set_or_update_global_sym(s1, "_bss_end__", bss_end); - } - if (data_start) - { - set_or_update_global_sym(s1, "__data_start__", data_start); - } - if (data_end) - { - set_or_update_global_sym(s1, "_edata", data_end); - set_or_update_global_sym(s1, "__data_end__", data_end); - } - if (text_start) - { - set_or_update_global_sym(s1, "__text_start__", text_start); - set_or_update_global_sym(s1, "_stext", text_start); - } - if (text_end) - { - set_or_update_global_sym(s1, "_etext", text_end); - set_or_update_global_sym(s1, "__text_end__", text_end); - } - if (end_addr) - { - set_or_update_global_sym(s1, "__end__", end_addr); - set_or_update_global_sym(s1, "_end", end_addr); - set_or_update_global_sym(s1, "end", end_addr); - /* Heap typically starts at end */ - set_or_update_global_sym(s1, "__heap_start__", end_addr); - } -} - -#endif /* !ELF_OBJ_ONLY */ diff --git a/source/obj/tccelf_layout.c b/source/obj/tccelf_layout.c new file mode 100644 index 000000000..f37a42d97 --- /dev/null +++ b/source/obj/tccelf_layout.c @@ -0,0 +1,991 @@ +/* + * ELF file handling for TCC + * + * Copyright (c) 2001-2004 Fabrice Bellard + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +/* ELF: filling the GOT, binding dynamic symbols, section sizes and order, + * section and program-header layout, and the dynamic section. */ + +#include "tccelf_priv.h" + +#ifndef ELF_OBJ_ONLY +ST_FUNC void fill_got_entry(TCCState *s1, ElfW_Rel *rel) +{ + int sym_index = ELFW(R_SYM)(rel->r_info); + ElfW(Sym) *sym = &((ElfW(Sym) *)symtab_section->data)[sym_index]; + struct sym_attr *attr = get_sym_attr(s1, sym_index, 0); + unsigned offset = attr->got_offset; + + if (0 == offset) + return; + section_reserve(s1->got, offset + PTR_SIZE); +#if PTR_SIZE == 8 + write64le(s1->got->data + offset, sym->st_value); +#else + write32le(s1->got->data + offset, sym->st_value); +#endif +} + +/* Perform relocation to GOT or PLT entries */ +ST_FUNC void fill_got(TCCState *s1) +{ + Section *s; + ElfW_Rel *rel; + int i; + + for (i = 1; i < s1->nb_sections; i++) + { + s = s1->sections[i]; + if (s->sh_type != SHT_RELX) + continue; + /* no need to handle got relocations */ + if (s->link != symtab_section) + continue; + for_each_elem(s, 0, rel, ElfW_Rel) + { + switch (ELFW(R_TYPE)(rel->r_info)) + { + case R_X86_64_GOT32: + case R_X86_64_GOTPCREL: + case R_X86_64_GOTPCRELX: + case R_X86_64_REX_GOTPCRELX: + case R_X86_64_PLT32: + fill_got_entry(s1, rel); + break; + } + } + } +} + +/* See put_got_entry for a description. This is the second stage + where GOT references to local defined symbols are rewritten. */ +void fill_local_got_entries(TCCState *s1) +{ + ElfW_Rel *rel; + if (!s1->got->reloc) + return; + for_each_elem(s1->got->reloc, 0, rel, ElfW_Rel) + { + if (ELFW(R_TYPE)(rel->r_info) == R_RELATIVE) + { + int sym_index = ELFW(R_SYM)(rel->r_info); + ElfW(Sym) *sym = &((ElfW(Sym) *)symtab_section->data)[sym_index]; + struct sym_attr *attr = get_sym_attr(s1, sym_index, 0); + unsigned offset = attr->got_offset; + if (offset != rel->r_offset - s1->got->sh_addr) + tcc_error_noabort("fill_local_got_entries: huh?"); + /* Store the ELF symbol type (e.g. STT_FUNC vs STT_NOTYPE) in the + second word of the 8-byte GOT entry. The YAFF writer reads this + to distinguish function pointers (which need thunks) from plain + code addresses such as labels used by goto *&&label. */ + write32le(s1->got->data + offset + PTR_SIZE, ELFW(ST_TYPE)(sym->st_info)); + rel->r_info = ELFW(R_INFO)(0, R_RELATIVE); +#if SHT_RELX == SHT_RELA + rel->r_addend = sym->st_value; +#else + /* All our REL architectures also happen to be 32bit LE. */ + write32le(s1->got->data + offset, sym->st_value); +#endif + } + } +} + +/* Bind symbols of executable: resolve undefined symbols from exported symbols + in shared libraries */ +void bind_exe_dynsyms(TCCState *s1, int is_PIE) +{ + const char *name; + int sym_index, index; + ElfW(Sym) * sym, *esym; + int type; + + /* Resolve undefined symbols from dynamic symbols. When there is a match: + - if STT_FUNC or STT_GNU_IFUNC symbol -> add it in PLT + - if STT_OBJECT symbol -> add it in .bss section with suitable reloc */ + for_each_elem(symtab_section, 1, sym, ElfW(Sym)) + { + if (sym->st_shndx == SHN_UNDEF) + { + /* Validate st_name before using it */ + if (sym->st_name >= symtab_section->link->data_offset) + { + int sym_idx = sym - (ElfW(Sym) *)symtab_section->data; + tcc_error_noabort( + "internal error (bind_exe_dynsyms): symbol %d has invalid st_name offset 0x%x (strtab size: 0x%lx)", + sym_idx, sym->st_name, (unsigned long)symtab_section->link->data_offset); + continue; + } + name = (char *)symtab_section->link->data + sym->st_name; + sym_index = tcc_dynsym_find(s1, name); + if (sym_index) + { + if (is_PIE) + continue; + esym = &((ElfW(Sym) *)s1->dynsymtab_section->data)[sym_index]; + type = ELFW(ST_TYPE)(esym->st_info); + if ((type == STT_FUNC) || (type == STT_GNU_IFUNC)) + { + /* Indirect functions shall have STT_FUNC type in executable + * dynsym section. Indeed, a dlsym call following a lazy + * resolution would pick the symbol value from the + * executable dynsym entry which would contain the address + * of the function wanted by the caller of dlsym instead of + * the address of the function that would return that + * address */ + int dynindex = put_elf_sym(s1->dynsym, 0, esym->st_size, ELFW(ST_INFO)(STB_GLOBAL, STT_FUNC), 0, 0, name); + int index = sym - (ElfW(Sym) *)symtab_section->data; + get_sym_attr(s1, index, 1)->dyn_index = dynindex; + } + else if (type == STT_OBJECT) + { + unsigned long offset; + ElfW(Sym) * dynsym; + offset = bss_section->data_offset; + /* XXX: which alignment ? */ + offset = (offset + 16 - 1) & -16; + set_elf_sym(s1->symtab, offset, esym->st_size, esym->st_info, 0, bss_section->sh_num, name); + index = put_elf_sym(s1->dynsym, offset, esym->st_size, esym->st_info, 0, bss_section->sh_num, name); + + /* Ensure R_COPY works for weak symbol aliases */ + if (ELFW(ST_BIND)(esym->st_info) == STB_WEAK) + { + for_each_elem(s1->dynsymtab_section, 1, dynsym, ElfW(Sym)) + { + if ((dynsym->st_value == esym->st_value) && (ELFW(ST_BIND)(dynsym->st_info) == STB_GLOBAL)) + { + char *dynname = (char *)s1->dynsymtab_section->link->data + dynsym->st_name; + put_elf_sym(s1->dynsym, offset, dynsym->st_size, dynsym->st_info, 0, bss_section->sh_num, dynname); + break; + } + } + } + + put_elf_reloc(s1->dynsym, bss_section, offset, R_COPY, index); + offset += esym->st_size; + bss_section->data_offset = offset; + } + } + else + { + /* STB_WEAK undefined symbols are accepted */ + /* XXX: _fp_hw seems to be part of the ABI, so we ignore it */ + if (ELFW(ST_BIND)(sym->st_info) == STB_WEAK || !strcmp(name, "_fp_hw")) + { + } + else + { + tcc_error_noabort("undefined symbol '%s'", name); + } + } + } + } +} + +/* Bind symbols of libraries: export all non local symbols of executable that + are referenced by shared libraries. The reason is that the dynamic loader + search symbol first in executable and then in libraries. Therefore a + reference to a symbol already defined by a library can still be resolved by + a symbol in the executable. With -rdynamic, export all defined symbols */ +void bind_libs_dynsyms(TCCState *s1) +{ + const char *name; + int dynsym_index; + ElfW(Sym) * sym, *esym; + + for_each_elem(symtab_section, 1, sym, ElfW(Sym)) + { + name = (char *)symtab_section->link->data + sym->st_name; + dynsym_index = tcc_dynsym_find(s1, name); + if (sym->st_shndx != SHN_UNDEF) + { + if (ELFW(ST_BIND)(sym->st_info) != STB_LOCAL && (dynsym_index || s1->rdynamic)) + set_elf_sym(s1->dynsym, sym->st_value, sym->st_size, sym->st_info, 0, sym->st_shndx, name); + } + else if (dynsym_index) + { + esym = (ElfW(Sym) *)s1->dynsymtab_section->data + dynsym_index; + if (esym->st_shndx == SHN_UNDEF) + { + /* weak symbols can stay undefined */ + if (ELFW(ST_BIND)(esym->st_info) != STB_WEAK) + tcc_warning("undefined dynamic symbol '%s'", name); + } + } + } +} + +/* Export all non local symbols. This is used by shared libraries so that the + non local symbols they define can resolve a reference in another shared + library or in the executable. Correspondingly, it allows undefined local + symbols to be resolved by other shared libraries or by the executable. */ +void export_global_syms(TCCState *s1) +{ + int dynindex, index; + const char *name; + ElfW(Sym) * sym; + for_each_elem(symtab_section, 1, sym, ElfW(Sym)) + { + if (ELFW(ST_BIND)(sym->st_info) != STB_LOCAL) + { + name = (char *)symtab_section->link->data + sym->st_name; + dynindex = set_elf_sym(s1->dynsym, sym->st_value, sym->st_size, sym->st_info, 0, sym->st_shndx, name); + index = sym - (ElfW(Sym) *)symtab_section->data; + get_sym_attr(s1, index, 1)->dyn_index = dynindex; + } + } +} + +/* decide if an unallocated section should be output. */ +int set_sec_sizes(TCCState *s1) +{ + int i; + Section *s; + int textrel = 0; + int file_type = s1->output_type; + + /* Allocate strings for section names */ + for (i = 1; i < s1->nb_sections; i++) + { + s = s1->sections[i]; + if (s->sh_type == SHT_RELX && !(s->sh_flags & SHF_ALLOC)) + { + /* when generating a DLL, we include relocations but + we may patch them */ + if ((file_type & TCC_OUTPUT_DYN) && (s1->sections[s->sh_info]->sh_flags & SHF_ALLOC)) + { + int count = prepare_dynamic_rel(s1, s); + if (count) + { + /* allocate the section */ + s->sh_flags |= SHF_ALLOC; + s->sh_size = count * sizeof(ElfW_Rel); + if (s1->sections[s->sh_info]->sh_flags & SHF_EXECINSTR) + textrel += count; + } + } +#ifdef TCC_TARGET_YAFF + else if (s1->output_format == TCC_OUTPUT_FORMAT_YAFF && (s1->sections[s->sh_info]->sh_flags & SHF_ALLOC)) + { + /* Preserve data relocation sections for YAFF output. + The YAFF writer needs R_ARM_ABS32 entries to generate + data relocations for the dynamic loader. Setting sh_size + ensures alloc_sec_names assigns a name, preventing + reorder_sections from pruning the section. */ + s->sh_size = s->data_offset; + } +#endif + } + else if ((s->sh_flags & SHF_ALLOC) +#ifdef TCC_TARGET_ARM + || s->sh_type == SHT_ARM_ATTRIBUTES +#endif + || s1->do_debug) + { + s->sh_size = s->data_offset; + } + +#ifdef TCC_TARGET_ARM + /* XXX: Suppress stack unwinding section. */ + if (s->sh_type == SHT_ARM_EXIDX) + { + s->sh_flags = 0; + s->sh_size = 0; + } +#endif + + /* Suppress legacy stabs sections. */ + if (!strcmp(s->name, ".stab") || !strcmp(s->name, ".stabstr") || !strncmp(s->name, ".rel.stab", 9) || + !strncmp(s->name, ".rela.stab", 10)) + { + s->sh_flags = 0; + s->sh_size = 0; + } + } + return textrel; +} + +/* Find the linker script output section index and pattern index for a given + section name. Returns output section index via return value (-1 if not + found), and sets *pat_idx to the pattern index within that output section. + Patterns are checked first (in order) since they define the ordering within + the output section. If no pattern matches but the section name exactly + matches an output section name, pat_idx is set to a value after all patterns + to indicate it should come last within that output section. */ +int ld_find_output_section_idx(TCCState *s1, const char *name, int *pat_idx) +{ + LDScript *ld = s1->ld_script; + int i, j; + + if (pat_idx) + *pat_idx = -1; + + if (!ld || ld->nb_output_sections == 0) + return -1; + + for (i = 0; i < ld->nb_output_sections; i++) + { + LDOutputSection *os = &ld->output_sections[i]; + /* Check patterns first - they define the ordering within the output section + */ + for (j = 0; j < os->nb_patterns; j++) + { + if (ld_section_matches_pattern(name, os->patterns[j].pattern)) + { + if (pat_idx) + *pat_idx = j; + return i; + } + } + /* Check exact name match - comes after all patterns */ + if (!strcmp(name, os->name)) + { + if (pat_idx) + *pat_idx = os->nb_patterns; /* after all patterns */ + return i; + } + } + return -1; +} + +/* Check if a section name matches a specific output section index */ +int ld_section_matches_output(TCCState *s1, const char *name, int os_idx) +{ + int pat_idx = -1; + int found = ld_find_output_section_idx(s1, name, &pat_idx); + return found == os_idx; +} + +/* Decide the layout of sections loaded in memory. This must be done before + program headers are filled since they contain info about the layout. + We do the following ordering: interp, symbol tables, relocations, progbits, + nobits */ +static int sort_sections(TCCState *s1, int *sec_order, struct dyn_inf *d) +{ + Section *s; + int i, j, k, f, f0, n, ld_idx; + int nb_sections = s1->nb_sections; + int *sec_cls = sec_order + nb_sections; + + for (i = 1; i < nb_sections; i++) + { + s = s1->sections[i]; + if (0 == s->sh_name) + { + j = 0x900; /* no sh_name: won't go to file */ + } + else if (s->sh_flags & SHF_ALLOC) + { + j = 0x100; + } + else + { + j = 0x700; + } + if (j >= 0x700 && s1->output_format != TCC_OUTPUT_FORMAT_ELF) + s->sh_size = 0, j = 0x900; + + if (s->sh_type == SHT_SYMTAB || s->sh_type == SHT_DYNSYM) + { + k = 0xff; + } + else if (s->sh_type == SHT_STRTAB && strcmp(s->name, ".stabstr")) + { + k = 0xff; + if (i == nb_sections - 1) /* ".shstrtab" assumed to stay last */ + k = 0xff; + } + else if (s->sh_type == SHT_HASH || s->sh_type == SHT_GNU_HASH) + { + k = 0xff; + } + else if (s->sh_type == SHT_GNU_verdef || s->sh_type == SHT_GNU_verneed || s->sh_type == SHT_GNU_versym) + { + k = 0x13; + } + else if (s->sh_type == SHT_RELX) + { + k = 0x80; + if (s1->plt && s == s1->plt->reloc) + k = 0x81; + } + else if (s->sh_flags & SHF_EXECINSTR) + { + k = 0x30; + if (s == s1->plt) + { + k = 0x32; + } + /* RELRO sections --> */ + } + else if (s->sh_type == SHT_PREINIT_ARRAY) + { + k = 0x41; + } + else if (s->sh_type == SHT_INIT_ARRAY) + { + k = 0x42; + } + else if (s->sh_type == SHT_FINI_ARRAY) + { + k = 0x43; + } + else if (s->sh_type == SHT_DYNAMIC) + { + k = 0x280; + } + else if (s == s1->got) + { + k = 0x70; /* .got as RELRO needs BIND_NOW in DT_FLAGS */ + } + else if (s->reloc && (s->reloc->sh_flags & SHF_ALLOC) && j == 0x100) + { + if (s == rodata_section) + { + k = 0x43; + } + else + { + k = 0x44; + } + /* <-- */ + } + else if (s->sh_type == SHT_NOTE) + { + k = 0x60; + } + else if (s->sh_type == SHT_NOBITS) + { + k = 0x70; /* bss */ + } + else if (s == d->interp) + { + k = 0xff; + } + else if (s == rodata_section) + { + k = 0x40; /* rodata */ + } + else + { + k = 0x50; /* data */ + } + + k += j; + + /* Check for RELRO sections before potentially modifying k for linker + script ordering. RELRO sections are in range 0x141-0x14f. */ + if ((k & 0xfff0) == 0x140) + { + /* make RELRO section writable */ + s->sh_flags |= SHF_WRITE; + } + + /* If linker script has output sections defined, use linker script order + for ALLOC program sections (not relocation, symbol, or other special + sections). The linker script output section index becomes the primary + sort key, with the pattern index within the output section as the + secondary key. This ensures sections are ordered according to the + pattern order in the linker script (e.g., KEEP(*(.isr_vector)) before + *(.text)). + Sections not in the linker script are placed after all linker script + sections. + + Classification encoding for linker script sections (within 0x100-0x6ff): + - Upper nibble (0x100-0x600): output section index (up to 6 sections) + - Lower byte: pattern index * 2 (to leave room for sub-classes) + + For sections not in linker script, we use 0x6xx range. + + Skip relocation sections (SHT_RELX), symbol tables, string tables, + hash tables, and other special sections - they should keep their + default ordering. */ + if (j == 0x100 && s1->ld_script && s1->ld_script->nb_output_sections > 0 && s->sh_type != SHT_RELX && + s->sh_type != SHT_SYMTAB && s->sh_type != SHT_DYNSYM && s->sh_type != SHT_STRTAB && s->sh_type != SHT_HASH && + s->sh_type != SHT_GNU_HASH && s->sh_type != SHT_DYNAMIC) + { + int pat_idx = 0; + ld_idx = ld_find_output_section_idx(s1, s->name, &pat_idx); + if (ld_idx >= 0) + { + /* Section is in linker script: use ld_idx as primary key, + pattern index as secondary key. + pat_idx is the index of the matching pattern (0+), or nb_patterns + for exact name match (comes after all patterns). + Keep values in 0x100-0x5ff range to ensure proper handling. */ + /* Limit ld_idx to fit in 5 bits (0-31 output sections) */ + if (ld_idx > 31) + ld_idx = 31; + /* pat_idx in lower bits, ld_idx in upper bits, all within 0x100-0x6ff + */ + k = 0x100 + (ld_idx << 4) + (pat_idx & 0x0f); + } + else + { + /* Section not in linker script: place after all linker script sections + but still within ALLOC range. Use 0x6xx + original sub-class. */ + k = 0x600 + ((k & 0x7f) >> 4); + } + } + + for (n = i; n > 1 && k < (f = sec_cls[n - 1]); --n) + sec_cls[n] = f, sec_order[n] = sec_order[n - 1]; + sec_cls[n] = k, sec_order[n] = i; + } + sec_order[0] = 0; + d->shnum = 1; + + /* count PT_LOAD headers needed */ + n = f0 = 0; + for (i = 1; i < nb_sections; i++) + { + s = s1->sections[sec_order[i]]; + k = sec_cls[i]; + f = 0; + if (k < 0x900) + ++d->shnum; + if (k < 0x700) + { + f = s->sh_flags & (SHF_ALLOC | SHF_WRITE | SHF_EXECINSTR | SHF_TLS); +#if TARGETOS_NetBSD + /* NetBSD only supports 2 PT_LOAD sections. + See: https://blog.netbsd.org/tnf/entry/the_first_report_on_lld */ + if ((f & SHF_WRITE) == 0) + f |= SHF_EXECINSTR; +#else + if ((k & 0xfff0) == 0x240) /* RELRO sections */ + f |= 1 << 4; +#endif + /* start new header when flags changed or relro, but avoid zero memsz */ + if (f != f0 && s->sh_size) + f0 = f, ++n, f |= 1 << 8; + } + sec_cls[i] = f; + LOG_RELOC("ph %d sec %02d : %3X %3X %x %04X %s", (f > 0) * n, i, f, k, s->sh_type, (int)s->sh_size, s->name); + } + return n; +} + +static ElfW(Phdr) * fill_phdr(ElfW(Phdr) * ph, int type, Section *s) +{ + if (s) + { + ph->p_offset = s->sh_offset; + ph->p_vaddr = s->sh_addr; + ph->p_filesz = s->sh_size; + ph->p_align = s->sh_addralign; + } + ph->p_type = type; + ph->p_flags = PF_R; + ph->p_paddr = ph->p_vaddr; + ph->p_memsz = ph->p_filesz; + return ph; +} + +/* Assign sections to segments and decide how are sections laid out when loaded + in memory. This function also fills corresponding program headers. */ +int layout_sections(TCCState *s1, int *sec_order, struct dyn_inf *d) +{ + Section *s; + addr_t addr, tmp, align, s_align, base; + ElfW(Phdr) *ph = NULL; + int i, f, n, phnum, phfill; + int file_offset; + int elf_header_offset = 0; + + /* compute number of program headers */ + phnum = sort_sections(s1, sec_order, d); + phfill = 0; /* set to 1 to have dll's with a PT_PHDR */ + if (d->interp) + phfill = 2; + phnum += phfill; + if (d->note) + ++phnum; + if (d->dynamic) + ++phnum; + if (eh_frame_hdr_section) + ++phnum; + if (d->roinf) + ++phnum; + /* Add extra segments for memory regions (each region may need new PT_LOAD) */ + if (s1->ld_script && s1->ld_script->nb_memory_regions > 1) + phnum += s1->ld_script->nb_memory_regions - 1; + d->phnum = phnum; + d->phdr = tcc_mallocz(phnum * sizeof(ElfW(Phdr))); + + file_offset = 0; + if (s1->output_format == TCC_OUTPUT_FORMAT_ELF) + { + file_offset = (sizeof(ElfW(Ehdr)) + phnum * sizeof(ElfW(Phdr)) + 3) & -4; + file_offset += d->shnum * sizeof(ElfW(Shdr)); + } + + s_align = ELF_PAGE_SIZE; + if (s1->section_align) + s_align = s1->section_align; + + addr = ELF_START_ADDR; + if (s1->output_type & TCC_OUTPUT_DYN) + addr = 0; + +#ifdef TCC_TARGET_YAFF + /* YAFF format uses 0-based offsets for all sections. + Force text address to 0 unless the user explicitly set one. */ + if (s1->output_format == TCC_OUTPUT_FORMAT_YAFF && !s1->has_text_addr) + addr = 0; +#endif + + /* Use linker script MEMORY origin if available */ + if (s1->ld_script && s1->ld_script->nb_memory_regions > 0) + { + addr = s1->ld_script->memory_regions[0].origin; + } + + if (s1->has_text_addr) + { + addr = s1->text_addr; + if (0) + { + int a_offset, p_offset; + /* we ensure that (addr % ELF_PAGE_SIZE) == file_offset % + ELF_PAGE_SIZE */ + a_offset = (int)(addr & (s_align - 1)); + p_offset = file_offset & (s_align - 1); + if (a_offset < p_offset) + a_offset += s_align; + file_offset += (a_offset - p_offset); + } + } + /* compute address after headers */ + // addr += file_offset; + elf_header_offset = file_offset; + + /* Track per-memory-region address counters for linker script placement */ + addr_t mr_addr[LD_MAX_MEMORY_REGIONS]; + int cur_mr = 0; + if (s1->ld_script && s1->ld_script->nb_memory_regions > 0) + { + for (int mr = 0; mr < s1->ld_script->nb_memory_regions; mr++) + { + LDMemoryRegion *region = &s1->ld_script->memory_regions[mr]; + mr_addr[mr] = region->origin; + } + addr = mr_addr[0]; + } + else + { + mr_addr[0] = addr; + } + + /* `base` is what the first PT_LOAD claims as its p_vaddr, so it has to be + where the first section actually lands -- which, when a linker script + supplies MEMORY regions, is the first region's origin and not the text + address above. A YASOS-targeted compiler defaults to text_addr 0 with + has_text_addr set (tcc_new), so taking `base` any earlier described a + script-placed image as starting at 0: p_vaddr 0 against sections at the + region origin, and p_memsz stretched from 0 to the end of the segment. + QEMU loads such a segment on top of the vector table it just wrote (the + 0x00000000 and 0x10000000 SSRAM aliases on mps2-an505 are the same + memory), so the CPU reset-fetched SP=0/PC=0 and locked up before the + first instruction. */ + base = addr; + + n = 0; + for (i = 1; i < s1->nb_sections; i++) + { + int sec_idx = sec_order[i]; + int sec_flags_idx = i + s1->nb_sections; + s = s1->sections[sec_idx]; + f = sec_order[sec_flags_idx]; + align = s->sh_addralign - 1; + + if (f == 0) + { /* no alloc */ + file_offset = (file_offset + align) & ~align; + s->sh_offset = file_offset; + if (s->sh_type != SHT_NOBITS) + file_offset += s->sh_size; + continue; + } + + /* Check if this section should be placed in a different memory region */ + if (s1->ld_script && s1->ld_script->nb_memory_regions > 0 && s1->ld_script->nb_output_sections > 0) + { + int pat_idx = -1; + int ld_idx = ld_find_output_section_idx(s1, s->name, &pat_idx); + if (ld_idx >= 0) + { + LDOutputSection *os = &s1->ld_script->output_sections[ld_idx]; + int new_mr = os->memory_region_idx; + if (new_mr >= 0 && new_mr < s1->ld_script->nb_memory_regions) + { + if (new_mr != cur_mr) + { + /* Save current region's address and switch to new region */ + mr_addr[cur_mr] = addr; + cur_mr = new_mr; + addr = mr_addr[cur_mr]; + /* Force new program header when changing memory regions */ + f |= 1 << 8; + } + } + } + } + + if ((f & 1 << 8) && n) + { + /* different rwx section flags */ + if (s1->output_format == TCC_OUTPUT_FORMAT_ELF) + { + /* if in the middle of a page, w e duplicate the page in + memory so that one copy is RX and the other is RW */ + if ((addr & (s_align - 1)) != 0) + addr += s_align; + } + else + { + align = s_align - 1; + } + } + + tmp = addr; + addr = (addr + align) & ~align; + file_offset += (int)(addr - tmp); + s->sh_offset = file_offset; + s->sh_addr = addr; + s->sh_size = (s->sh_size + align) & ~align; + + if (f & 1 << 8) + { + /* set new program header */ + int ph_idx = phfill + n; + ph = &d->phdr[ph_idx]; + ph->p_type = PT_LOAD; + ph->p_align = s_align; + ph->p_flags = PF_R; + if (f & SHF_WRITE) + ph->p_flags |= PF_W; + if (f & SHF_EXECINSTR) + ph->p_flags |= PF_X; + if (f & SHF_TLS) + { + ph->p_type = PT_TLS; + ph->p_align = align + 1; + } + + ph->p_offset = file_offset; + ph->p_vaddr = addr; + + if (n == 0) + { + /* Make the first PT_LOAD segment include the program + headers itself (and the ELF header as well), it'll + come out with same memory use but will make various + tools like binutils strip work better. */ + // ph->p_offset = 0; + ph->p_vaddr = base; + } + ph->p_paddr = ph->p_vaddr; + ++n; + } + + if (f & 1 << 4) + { + Section *roinf = &d->_roinf; + if (roinf->sh_size == 0) + { + roinf->sh_offset = s->sh_offset; + roinf->sh_addr = s->sh_addr; + roinf->sh_addralign = 1; + } + roinf->sh_size = (addr - roinf->sh_addr) + s->sh_size; + } + + addr += s->sh_size; + if (s->sh_type != SHT_NOBITS) + file_offset += s->sh_size; + + if (ph) + { + ph->p_filesz = file_offset - ph->p_offset; + ph->p_memsz = addr - ph->p_vaddr; + if (n == 1) + { + ph->p_memsz += elf_header_offset; + } + } + } + + /* Fill other headers */ + if (d->note) + fill_phdr(++ph, PT_NOTE, d->note); + if (d->dynamic) + { + ElfW(Phdr) *dph = fill_phdr(++ph, PT_DYNAMIC, d->dynamic); + dph->p_flags |= PF_W; + } + if (eh_frame_hdr_section) + fill_phdr(++ph, PT_GNU_EH_FRAME, eh_frame_hdr_section); + if (d->roinf) + { + ElfW(Phdr) *rph = fill_phdr(++ph, PT_GNU_RELRO, d->roinf); + rph->p_flags |= PF_W; + } + if (d->interp) + { + ElfW(Phdr) *iph = &d->phdr[1]; + fill_phdr(iph, PT_INTERP, d->interp); + } + if (phfill) + { + ph = &d->phdr[0]; + ph->p_offset = sizeof(ElfW(Ehdr)); + ph->p_vaddr = base + ph->p_offset; + { + int phdr_sz = phnum * sizeof(ElfW(Phdr)); + ph->p_filesz = phdr_sz; + } + ph->p_align = 4; + fill_phdr(ph, PT_PHDR, NULL); + } + return 0; +} + +/* put dynamic tag */ +void put_dt(Section *dynamic, int dt, addr_t val) +{ + ElfW(Dyn) * dyn; + dyn = section_ptr_add(dynamic, sizeof(ElfW(Dyn))); + dyn->d_tag = dt; + dyn->d_un.d_val = val; +} + +/* Fill the dynamic section with tags describing the address and size of + sections */ +void fill_dynamic(TCCState *s1, struct dyn_inf *dyninf) +{ + Section *dynamic = dyninf->dynamic; + Section *s; + + /* put dynamic section entries */ + put_dt(dynamic, DT_HASH, s1->dynsym->hash->sh_addr); + put_dt(dynamic, DT_GNU_HASH, dyninf->gnu_hash->sh_addr); + put_dt(dynamic, DT_STRTAB, dyninf->dynstr->sh_addr); + put_dt(dynamic, DT_SYMTAB, s1->dynsym->sh_addr); + put_dt(dynamic, DT_STRSZ, dyninf->dynstr->data_offset); + put_dt(dynamic, DT_SYMENT, sizeof(ElfW(Sym))); +#if PTR_SIZE == 8 + put_dt(dynamic, DT_RELA, dyninf->rel_addr); + put_dt(dynamic, DT_RELASZ, dyninf->rel_size); + put_dt(dynamic, DT_RELAENT, sizeof(ElfW_Rel)); + if (s1->plt && s1->plt->reloc) + { + put_dt(dynamic, DT_PLTGOT, s1->got->sh_addr); + put_dt(dynamic, DT_PLTRELSZ, s1->plt->reloc->data_offset); + put_dt(dynamic, DT_JMPREL, s1->plt->reloc->sh_addr); + put_dt(dynamic, DT_PLTREL, DT_RELA); + } + put_dt(dynamic, DT_RELACOUNT, 0); +#else + put_dt(dynamic, DT_REL, dyninf->rel_addr); + put_dt(dynamic, DT_RELSZ, dyninf->rel_size); + put_dt(dynamic, DT_RELENT, sizeof(ElfW_Rel)); + if (s1->plt && s1->plt->reloc) + { + put_dt(dynamic, DT_PLTGOT, s1->got->sh_addr); + put_dt(dynamic, DT_PLTRELSZ, s1->plt->reloc->data_offset); + put_dt(dynamic, DT_JMPREL, s1->plt->reloc->sh_addr); + put_dt(dynamic, DT_PLTREL, DT_REL); + } + put_dt(dynamic, DT_RELCOUNT, 0); +#endif + if (versym_section && verneed_section) + { + /* The dynamic linker can not handle VERSYM without VERNEED */ + put_dt(dynamic, DT_VERSYM, versym_section->sh_addr); + put_dt(dynamic, DT_VERNEED, verneed_section->sh_addr); + put_dt(dynamic, DT_VERNEEDNUM, dt_verneednum); + } + s = have_section(s1, ".preinit_array"); + if (s && s->data_offset) + { + put_dt(dynamic, DT_PREINIT_ARRAY, s->sh_addr); + put_dt(dynamic, DT_PREINIT_ARRAYSZ, s->data_offset); + } + s = have_section(s1, ".init_array"); + if (s && s->data_offset) + { + put_dt(dynamic, DT_INIT_ARRAY, s->sh_addr); + put_dt(dynamic, DT_INIT_ARRAYSZ, s->data_offset); + } + s = have_section(s1, ".fini_array"); + if (s && s->data_offset) + { + put_dt(dynamic, DT_FINI_ARRAY, s->sh_addr); + put_dt(dynamic, DT_FINI_ARRAYSZ, s->data_offset); + } + s = have_section(s1, ".init"); + if (s && s->data_offset) + { + put_dt(dynamic, DT_INIT, s->sh_addr); + } + s = have_section(s1, ".fini"); + if (s && s->data_offset) + { + put_dt(dynamic, DT_FINI, s->sh_addr); + } + if (s1->do_debug) + put_dt(dynamic, DT_DEBUG, 0); + put_dt(dynamic, DT_NULL, 0); +} + +/* Remove gaps between RELX sections. + These gaps are a result of final_sections_reloc. Here some relocs are + removed. The gaps are then filled with 0 in tcc_output_elf. The 0 is + intepreted as R_...NONE reloc. This does work on most targets but on + OpenBSD/arm64 this is illegal. OpenBSD/arm64 does not support R_...NONE + reloc. */ +void update_reloc_sections(TCCState *s1, struct dyn_inf *dyninf) +{ + int i; + unsigned long file_offset = 0; + Section *s; + Section *relocplt = s1->plt ? s1->plt->reloc : NULL; + + /* dynamic relocation table information, for .dynamic section */ + dyninf->rel_addr = dyninf->rel_size = 0; + + for (i = 1; i < s1->nb_sections; i++) + { + s = s1->sections[i]; + if (s->sh_type == SHT_RELX && s != relocplt) + { + if (dyninf->rel_size == 0) + { + dyninf->rel_addr = s->sh_addr; + file_offset = s->sh_offset; + } + else + { + s->sh_addr = dyninf->rel_addr + dyninf->rel_size; + s->sh_offset = file_offset + dyninf->rel_size; + } + dyninf->rel_size += s->sh_size; + } + } +} +#endif /* ndef ELF_OBJ_ONLY */ diff --git a/source/obj/tccelf_ld.c b/source/obj/tccelf_ld.c new file mode 100644 index 000000000..2341c9d19 --- /dev/null +++ b/source/obj/tccelf_ld.c @@ -0,0 +1,1098 @@ +/* + * ELF file handling for TCC + * + * Copyright (c) 2001-2004 Fabrice Bellard + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +/* ELF: loading shared libraries and linker scripts, and linker-script + * symbol assignment. */ + +#include "tccelf_priv.h" + +#ifndef ELF_OBJ_ONLY +/* Set LV[I] to the global index of sym-version (LIB,VERSION). Maybe resizes + LV, maybe create a new entry for (LIB,VERSION). */ +static void set_ver_to_ver(TCCState *s1, int *n, int **lv, int i, char *lib, char *version) +{ + while (i >= *n) + { + *lv = tcc_realloc(*lv, (*n + 1) * sizeof(**lv)); + (*lv)[(*n)++] = -1; + } + if ((*lv)[i] == -1) + { + int v, prev_same_lib = -1; + for (v = 0; v < nb_sym_versions; v++) + { + if (strcmp(sym_versions[v].lib, lib)) + continue; + prev_same_lib = v; + if (!strcmp(sym_versions[v].version, version)) + break; + } + if (v == nb_sym_versions) + { + sym_versions = tcc_realloc(sym_versions, (v + 1) * sizeof(*sym_versions)); + sym_versions[v].lib = tcc_strdup(lib); + sym_versions[v].version = tcc_strdup(version); + sym_versions[v].out_index = 0; + sym_versions[v].prev_same_lib = prev_same_lib; + nb_sym_versions++; + } + (*lv)[i] = v; + } +} + +/* Associates symbol SYM_INDEX (in dynsymtab) with sym-version index + VERNDX. */ +static void set_sym_version(TCCState *s1, int sym_index, int verndx) +{ + if (sym_index >= nb_sym_to_version) + { + int newelems = sym_index ? sym_index * 2 : 1; + sym_to_version = tcc_realloc(sym_to_version, newelems * sizeof(*sym_to_version)); + memset(sym_to_version + nb_sym_to_version, -1, (newelems - nb_sym_to_version) * sizeof(*sym_to_version)); + nb_sym_to_version = newelems; + } + if (sym_to_version[sym_index] < 0) + sym_to_version[sym_index] = verndx; +} + +struct versym_info +{ + int nb_versyms; + ElfW(Verdef) * verdef; + ElfW(Verneed) * verneed; + ElfW(Half) * versym; + int nb_local_ver, *local_ver; +}; + +static void store_version(TCCState *s1, struct versym_info *v, char *dynstr) +{ + char *lib, *version; + uint32_t next; + int i; + +#define DEBUG_VERSION 0 + + if (v->versym && v->verdef) + { + ElfW(Verdef) *vdef = v->verdef; + lib = NULL; + do + { + ElfW(Verdaux) *verdaux = (ElfW(Verdaux) *)(((char *)vdef) + vdef->vd_aux); + +#if DEBUG_VERSION + printf("verdef: version:%u flags:%u index:%u, hash:%u\n", vdef->vd_version, vdef->vd_flags, vdef->vd_ndx, + vdef->vd_hash); +#endif + if (vdef->vd_cnt) + { + version = dynstr + verdaux->vda_name; + + if (lib == NULL) + lib = version; + else + set_ver_to_ver(s1, &v->nb_local_ver, &v->local_ver, vdef->vd_ndx, lib, version); +#if DEBUG_VERSION + printf(" verdaux(%u): %s\n", vdef->vd_ndx, version); +#endif + } + next = vdef->vd_next; + vdef = (ElfW(Verdef) *)(((char *)vdef) + next); + } while (next); + } + if (v->versym && v->verneed) + { + ElfW(Verneed) *vneed = v->verneed; + do + { + ElfW(Vernaux) *vernaux = (ElfW(Vernaux) *)(((char *)vneed) + vneed->vn_aux); + + lib = dynstr + vneed->vn_file; +#if DEBUG_VERSION + printf("verneed: %u %s\n", vneed->vn_version, lib); +#endif + for (i = 0; i < vneed->vn_cnt; i++) + { + if ((vernaux->vna_other & 0x8000) == 0) + { /* hidden */ + version = dynstr + vernaux->vna_name; + set_ver_to_ver(s1, &v->nb_local_ver, &v->local_ver, vernaux->vna_other, lib, version); +#if DEBUG_VERSION + printf(" vernaux(%u): %u %u %s\n", vernaux->vna_other, vernaux->vna_hash, vernaux->vna_flags, version); +#endif + } + vernaux = (ElfW(Vernaux) *)(((char *)vernaux) + vernaux->vna_next); + } + next = vneed->vn_next; + vneed = (ElfW(Verneed) *)(((char *)vneed) + next); + } while (next); + } + +#if DEBUG_VERSION + for (i = 0; i < v->nb_local_ver; i++) + { + if (v->local_ver[i] > 0) + { + printf("%d: lib: %s, version %s\n", i, sym_versions[v->local_ver[i]].lib, sym_versions[v->local_ver[i]].version); + } + } +#endif +} +/* load a library / DLL + 'level = 0' means that the DLL is referenced by the user + (so it should be added as DT_NEEDED in the generated ELF file) */ +ST_FUNC int tcc_load_dll(TCCState *s1, int fd, const char *filename, int level) +{ + ElfW(Ehdr) ehdr; + ElfW(Shdr) * shdr, *sh, *sh1; + int i, nb_syms, nb_dts, sym_bind, ret = -1; + ElfW(Sym) * sym, *dynsym; + ElfW(Dyn) * dt, *dynamic; + + char *dynstr; + int sym_index; + const char *name, *soname; + struct versym_info v; + unsigned dll_start = 0; + + if (s1->do_bench) + dll_start = tcc_getclock_us(); + + full_read(fd, &ehdr, sizeof(ehdr)); + + /* test CPU specific stuff */ + if (ehdr.e_ident[5] != ELFDATA2LSB || ehdr.e_machine != EM_TCC_TARGET) + { + return tcc_error_noabort("bad architecture"); + } + + /* read sections */ + shdr = load_data(fd, ehdr.e_shoff, sizeof(ElfW(Shdr)) * ehdr.e_shnum); + + /* load dynamic section and dynamic symbols */ + nb_syms = 0; + nb_dts = 0; + dynamic = NULL; + dynsym = NULL; /* avoid warning */ + dynstr = NULL; /* avoid warning */ + memset(&v, 0, sizeof v); + + for (i = 0, sh = shdr; i < ehdr.e_shnum; i++, sh++) + { + switch (sh->sh_type) + { + case SHT_DYNAMIC: + nb_dts = sh->sh_size / sizeof(ElfW(Dyn)); + dynamic = load_data(fd, sh->sh_offset, sh->sh_size); + break; + case SHT_DYNSYM: + nb_syms = sh->sh_size / sizeof(ElfW(Sym)); + dynsym = load_data(fd, sh->sh_offset, sh->sh_size); + sh1 = &shdr[sh->sh_link]; + dynstr = load_data(fd, sh1->sh_offset, sh1->sh_size); + break; + case SHT_GNU_verdef: + v.verdef = load_data(fd, sh->sh_offset, sh->sh_size); + break; + case SHT_GNU_verneed: + v.verneed = load_data(fd, sh->sh_offset, sh->sh_size); + break; + case SHT_GNU_versym: + v.nb_versyms = sh->sh_size / sizeof(ElfW(Half)); + v.versym = load_data(fd, sh->sh_offset, sh->sh_size); + break; + default: + break; + } + } + + if (!dynamic) + goto the_end; + + /* compute the real library name */ + soname = tcc_basename(filename); + for (i = 0, dt = dynamic; i < nb_dts; i++, dt++) + if (dt->d_tag == DT_SONAME) + soname = dynstr + dt->d_un.d_val; + + /* if the dll is already loaded, do not load it */ + if (tcc_add_dllref(s1, soname, level)->found) + goto ret_success; + + if (v.nb_versyms != nb_syms) + tcc_free(v.versym), v.versym = NULL; + else + store_version(s1, &v, dynstr); + + /* add dynamic symbols in dynsym_section */ + for (i = 1, sym = dynsym + 1; i < nb_syms; i++, sym++) + { + sym_bind = ELFW(ST_BIND)(sym->st_info); + if (sym_bind == STB_LOCAL) + continue; + name = dynstr + sym->st_name; + sym_index = set_elf_sym(s1->dynsymtab_section, sym->st_value, sym->st_size, sym->st_info, sym->st_other, + sym->st_shndx, name); + if (v.versym) + { + ElfW(Half) vsym = v.versym[i]; + if ((vsym & 0x8000) == 0 && vsym > 0 && vsym < v.nb_local_ver) + set_sym_version(s1, sym_index, v.local_ver[vsym]); + } + } + + /* do not load all referenced libraries + (recursive loading can break linking of libraries) */ + /* following DT_NEEDED is needed for the dynamic loader (libdl.so), + but it is no longer needed, when linking a library or a program. + When tcc output mode is OUTPUT_MEM, + tcc calls dlopen, which handles DT_NEEDED for us */ + +#if 0 + for(i = 0, dt = dynamic; i < nb_dts; i++, dt++) + if (dt->d_tag == DT_RPATH) + tcc_add_library_path(s1, dynstr + dt->d_un.d_val); + + /* load all referenced DLLs */ + for(i = 0, dt = dynamic; i < nb_dts; i++, dt++) { + switch(dt->d_tag) { + case DT_NEEDED: + name = dynstr + dt->d_un.d_val; + if (tcc_add_dllref(s1, name, -1)) + continue; + if (tcc_add_dll(s1, name, AFF_REFERENCED_DLL) < 0) { + ret = tcc_error_noabort("referenced dll '%s' not found", name); + goto the_end; + } + } + } +#endif + +ret_success: + ret = 0; +the_end: + if (s1->do_bench) + { + unsigned elapsed = tcc_getclock_us() - dll_start; + s1->bench_dll_load_time += elapsed; + s1->bench_dll_load_count++; + tcc_bench_log(s1, "load-dll", filename, elapsed); + } + tcc_free(dynstr); + tcc_free(dynsym); + tcc_free(dynamic); + tcc_free(shdr); + tcc_free(v.local_ver); + tcc_free(v.verdef); + tcc_free(v.verneed); + tcc_free(v.versym); + return ret; +} + +#define LD_TOK_NAME 256 +#define LD_TOK_EOF (-1) + +static int ld_inp(TCCState *s1) +{ + char b; + if (s1->cc != -1) + { + int c = s1->cc; + s1->cc = -1; + return c; + } + if (1 == read(s1->fd, &b, 1)) + return b; + return CH_EOF; +} + +/* return next ld script token */ +static int ld_next(TCCState *s1, char *name, int name_size) +{ + int c, d, ch; + char *q; + +redo: + ch = ld_inp(s1); + switch (ch) + { + case ' ': + case '\t': + case '\f': + case '\v': + case '\r': + case '\n': + goto redo; + case '/': + ch = ld_inp(s1); + if (ch == '*') + { /* comment */ + for (d = 0;; d = ch) + { + ch = ld_inp(s1); + if (ch == CH_EOF || (ch == '/' && d == '*')) + break; + } + goto redo; + } + else + { + q = name; + *q++ = '/'; + goto parse_name; + } + break; + case '\\': + /* case 'a' ... 'z': */ + case 'a': + case 'b': + case 'c': + case 'd': + case 'e': + case 'f': + case 'g': + case 'h': + case 'i': + case 'j': + case 'k': + case 'l': + case 'm': + case 'n': + case 'o': + case 'p': + case 'q': + case 'r': + case 's': + case 't': + case 'u': + case 'v': + case 'w': + case 'x': + case 'y': + case 'z': + /* case 'A' ... 'z': */ + case 'A': + case 'B': + case 'C': + case 'D': + case 'E': + case 'F': + case 'G': + case 'H': + case 'I': + case 'J': + case 'K': + case 'L': + case 'M': + case 'N': + case 'O': + case 'P': + case 'Q': + case 'R': + case 'S': + case 'T': + case 'U': + case 'V': + case 'W': + case 'X': + case 'Y': + case 'Z': + case '_': + case '.': + case '$': + case '~': + q = name; + parse_name: + for (;;) + { + if (!((ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || (ch >= '0' && ch <= '9') || + strchr("/.-_+=$:\\,~", ch))) + break; + if ((q - name) < name_size - 1) + { + *q++ = ch; + } + ch = ld_inp(s1); + } + s1->cc = ch; + *q = '\0'; + c = LD_TOK_NAME; + break; + case CH_EOF: + c = LD_TOK_EOF; + break; + default: + c = ch; + break; + } + return c; +} + +static int ld_add_file(TCCState *s1, const char filename[]) +{ + if (filename[0] == '/') + { + if (CONFIG_SYSROOT[0] == '\0' && tcc_add_file_internal(s1, filename, AFF_TYPE_BIN) == 0) + return 0; + filename = tcc_basename(filename); + } + return tcc_add_dll(s1, filename, AFF_PRINT_ERROR); +} + +static int ld_add_file_list(TCCState *s1, const char *cmd, int as_needed) +{ + char filename[1024], libname[1016]; + int t, group, nblibs = 0, ret = 0; + char **libs = NULL; + + group = !strcmp(cmd, "GROUP"); + if (!as_needed) + s1->new_undef_sym = 0; + t = ld_next(s1, filename, sizeof(filename)); + if (t != '(') + { + ret = tcc_error_noabort("( expected"); + goto lib_parse_error; + } + t = ld_next(s1, filename, sizeof(filename)); + for (;;) + { + libname[0] = '\0'; + if (t == LD_TOK_EOF) + { + ret = tcc_error_noabort("unexpected end of file"); + goto lib_parse_error; + } + else if (t == ')') + { + break; + } + else if (t == '-') + { + t = ld_next(s1, filename, sizeof(filename)); + if ((t != LD_TOK_NAME) || (filename[0] != 'l')) + { + ret = tcc_error_noabort("library name expected"); + goto lib_parse_error; + } + pstrcpy(libname, sizeof libname, &filename[1]); + if (s1->static_link) + { + snprintf(filename, sizeof filename, "lib%s.a", libname); + } + else + { + snprintf(filename, sizeof filename, "lib%s.so", libname); + } + } + else if (t != LD_TOK_NAME) + { + ret = tcc_error_noabort("filename expected"); + goto lib_parse_error; + } + if (!strcmp(filename, "AS_NEEDED")) + { + ret = ld_add_file_list(s1, cmd, 1); + if (ret) + goto lib_parse_error; + } + else + { + /* TODO: Implement AS_NEEDED support. */ + /* DT_NEEDED is not used any more so ignore as_needed */ + if (1 || !as_needed) + { + ret = ld_add_file(s1, filename); + if (ret) + goto lib_parse_error; + if (group) + { + /* Add the filename *and* the libname to avoid future conversions */ + dynarray_add(&libs, &nblibs, tcc_strdup(filename)); + if (libname[0] != '\0') + dynarray_add(&libs, &nblibs, tcc_strdup(libname)); + } + } + } + t = ld_next(s1, filename, sizeof(filename)); + if (t == ',') + { + t = ld_next(s1, filename, sizeof(filename)); + } + } + if (group && !as_needed) + { + /* Same fast check as the CLI --start-group loop: only continue + rescanning while some current undef is satisfiable by these + cached archives. */ + if (s1->new_undef_sym) + { + s1->new_undef_sym = tcc_group_has_satisfiable_undefs(s1); + } + while (s1->new_undef_sym) + { + int i; + s1->new_undef_sym = 0; + for (i = 0; i < nblibs; i++) + ld_add_file(s1, libs[i]); + } + } +lib_parse_error: + dynarray_reset(&libs, &nblibs); + return ret; +} + +/* interpret a subset of GNU ldscripts to handle the dummy libc.so + files */ +ST_FUNC int tcc_load_ldscript(TCCState *s1, int fd) +{ + char cmd[64]; + char filename[1024]; + int t, ret; + unsigned ldscript_start = 0; + + if (s1->do_bench) + ldscript_start = tcc_getclock_us(); + + s1->fd = fd; + s1->cc = -1; + for (;;) + { + t = ld_next(s1, cmd, sizeof(cmd)); + if (t == LD_TOK_EOF) + { + if (s1->do_bench) + { + unsigned elapsed = tcc_getclock_us() - ldscript_start; + s1->bench_ldscript_load_time += elapsed; + s1->bench_ldscript_load_count++; + tcc_bench_log(s1, "ldscript", s1->current_filename, elapsed); + } + return 0; + } + else if (t != LD_TOK_NAME) + return -1; + if (!strcmp(cmd, "INPUT") || !strcmp(cmd, "GROUP")) + { + ret = ld_add_file_list(s1, cmd, 0); + if (ret) + return ret; + } + else if (!strcmp(cmd, "OUTPUT_FORMAT") || !strcmp(cmd, "TARGET")) + { + /* ignore some commands */ + t = ld_next(s1, cmd, sizeof(cmd)); + if (t != '(') + return tcc_error_noabort("( expected"); + for (;;) + { + t = ld_next(s1, filename, sizeof(filename)); + if (t == LD_TOK_EOF) + { + return tcc_error_noabort("unexpected end of file"); + } + else if (t == ')') + { + break; + } + } + } + else + { + return -1; + } + } + return 0; +} + +/* Load and parse a linker script file */ +ST_FUNC int tcc_load_linker_script(TCCState *s1, const char *filename) +{ + int fd; + int ret; + + fd = open(filename, O_RDONLY | O_BINARY); + if (fd < 0) + { + return tcc_error_noabort("linker script '%s' not found", filename); + } + + /* Allocate linker script structure if not already done */ + if (!s1->ld_script) + { + s1->ld_script = tcc_mallocz(sizeof(LDScript)); + ld_script_init(s1->ld_script); + } + + ret = ld_script_parse(s1, s1->ld_script, fd); + close(fd); + +#if TCCELF_DUMP_LD_SCRIPT + if (ret == 0 && s1->verbose) + { + printf("Loaded linker script: %s\n", filename); + ld_script_dump(s1->ld_script); + } +#endif + + /* Add standard symbols */ + if (ret == 0) + { + ld_script_add_standard_symbols(s1, s1->ld_script); + } + + return ret; +} + +/* Apply linker script symbols to the ELF symbol table */ +void ld_apply_symbols(TCCState *s1, LDScript *ld) +{ + int i; + for (i = 0; i < ld->nb_symbols; i++) + { + LDSymbol *sym = &ld->symbols[i]; + if (sym->defined) + { + int sym_idx; + int vis = + (sym->visibility == LD_SYM_HIDDEN || sym->visibility == LD_SYM_PROVIDE_HIDDEN) ? STV_HIDDEN : STV_DEFAULT; + + /* Look up once, reuse for both PROVIDE check and update */ + sym_idx = find_elf_sym(s1->symtab, sym->name); + + /* For PROVIDE symbols, only define if not already defined */ + if (sym->visibility == LD_SYM_PROVIDE || sym->visibility == LD_SYM_PROVIDE_HIDDEN) + { + if (sym_idx) + { + ElfW(Sym) *esym = &((ElfW(Sym) *)s1->symtab->data)[sym_idx]; + if (esym->st_shndx != SHN_UNDEF) + continue; /* Already defined, skip */ + } + } + + /* Update existing symbol, or create new */ + if (sym_idx) + { + ElfW(Sym) *esym = &((ElfW(Sym) *)s1->symtab->data)[sym_idx]; + esym->st_value = sym->value; + esym->st_shndx = SHN_ABS; + } + else + { + /* Use set_elf_sym directly with SHN_ABS to ensure symbols with value 0 + * are still defined as absolute (set_global_sym treats value 0 as + * UNDEF) + */ + set_elf_sym(s1->symtab, sym->value, 0, ELFW(ST_INFO)(STB_GLOBAL, STT_NOTYPE), vis, SHN_ABS, sym->name); + } + + /* Also update in dynsym if it exists there */ + if (s1->dynsym) + { + sym_idx = find_elf_sym(s1->dynsym, sym->name); + if (sym_idx) + { + ElfW(Sym) *esym = &((ElfW(Sym) *)s1->dynsym->data)[sym_idx]; + esym->st_value = sym->value; + esym->st_shndx = SHN_ABS; + } + } + } + } +} + +/* Update linker script symbol values based on actual section layout */ +void ld_update_symbol_values(TCCState *s1, LDScript *ld) +{ + Section *s; + addr_t bss_start = 0, bss_end = 0; + addr_t data_start = 0, data_end = 0; + addr_t text_start = 0, text_end = 0; + addr_t rodata_start = 0, rodata_end = 0; + addr_t end_addr = 0; + addr_t sec_end; + int i, j; + addr_t output_section_addrs[LD_MAX_OUTPUT_SECTIONS] = {0}; + int output_section_has_addr[LD_MAX_OUTPUT_SECTIONS] = {0}; + addr_t output_section_loadaddrs[LD_MAX_OUTPUT_SECTIONS] = {0}; + addr_t output_section_sizes[LD_MAX_OUTPUT_SECTIONS] = {0}; + addr_t output_section_align[LD_MAX_OUTPUT_SECTIONS] = {0}; + addr_t output_section_vma_end[LD_MAX_OUTPUT_SECTIONS] = {0}; + + /* Find section addresses and map output sections to actual addresses */ + for (i = 1; i < s1->nb_sections; i++) + { + s = s1->sections[i]; + if (!s->sh_addr) + continue; + + sec_end = s->sh_addr + s->sh_size; + if (sec_end > end_addr) + end_addr = sec_end; + + /* Match .bss and .bss.* sections */ + if (!strcmp(s->name, ".bss") || !strncmp(s->name, ".bss.", 5)) + { + if (bss_start == 0 || s->sh_addr < bss_start) + bss_start = s->sh_addr; + if (sec_end > bss_end) + bss_end = sec_end; + } + /* Match .data and .data.* sections */ + else if (!strcmp(s->name, ".data") || !strncmp(s->name, ".data.", 6)) + { + if (data_start == 0 || s->sh_addr < data_start) + data_start = s->sh_addr; + if (sec_end > data_end) + data_end = sec_end; + } + /* Match .text and .text.* sections */ + else if (!strcmp(s->name, ".text") || !strncmp(s->name, ".text.", 6)) + { + if (text_start == 0 || s->sh_addr < text_start) + text_start = s->sh_addr; + if (sec_end > text_end) + text_end = sec_end; + } + /* Match .rodata and .rodata.* sections */ + else if (!strcmp(s->name, ".rodata") || !strncmp(s->name, ".rodata.", 8)) + { + if (rodata_start == 0 || s->sh_addr < rodata_start) + rodata_start = s->sh_addr; + if (sec_end > rodata_end) + rodata_end = sec_end; + } + + /* Map output section names to addresses */ + for (j = 0; j < ld->nb_output_sections; j++) + { + if (!strcmp(s->name, ld->output_sections[j].name)) + { + output_section_addrs[j] = s->sh_addr; + output_section_has_addr[j] = 1; + break; + } + } + + /* Accumulate sizes/alignments for load address computation */ + if (ld) + { + int pat_idx = -1; + int ld_idx = ld_find_output_section_idx(s1, s->name, &pat_idx); + if (ld_idx >= 0 && ld_idx < ld->nb_output_sections) + { + addr_t align = s->sh_addralign ? s->sh_addralign : 1; + if (align > output_section_align[ld_idx]) + output_section_align[ld_idx] = align; + if (s->sh_type != SHT_NOBITS) + output_section_sizes[ld_idx] += s->sh_size; + /* Track VMA end address (includes alignment between sections) */ + if (s->sh_addr + s->sh_size > output_section_vma_end[ld_idx]) + output_section_vma_end[ld_idx] = s->sh_addr + s->sh_size; + } + } + } + + /* Compute output section load addresses (LMA) */ + if (ld) + { + addr_t lma_cur[LD_MAX_MEMORY_REGIONS] = {0}; + if (ld->nb_memory_regions > 0) + { + for (i = 0; i < ld->nb_memory_regions; i++) + lma_cur[i] = ld->memory_regions[i].origin; + for (j = 0; j < ld->nb_output_sections; j++) + { + int load_mr = ld->output_sections[j].load_memory_region_idx; + int vma_mr = ld->output_sections[j].memory_region_idx; + int mr = (load_mr >= 0) ? load_mr : vma_mr; + if (mr < 0) + mr = 0; + if (mr >= 0 && mr < ld->nb_memory_regions) + { + addr_t align = output_section_align[j] ? output_section_align[j] : 1; + addr_t cur = lma_cur[mr]; + addr_t lma_start = (cur + align - 1) & ~(align - 1); + output_section_loadaddrs[j] = lma_start; + /* For sections where VMA region == LMA region (no AT > directive), + use the actual VMA end address which includes alignment padding + between input sections. Otherwise use the raw content size. */ + if (load_mr < 0 && output_section_vma_end[j] > lma_start) + lma_cur[mr] = output_section_vma_end[j]; + else + lma_cur[mr] = lma_start + output_section_sizes[j]; + } + } + } + else + { + for (j = 0; j < ld->nb_output_sections; j++) + output_section_loadaddrs[j] = output_section_addrs[j]; + } + + /* Save computed LMA values in LDScript for p_paddr fixup */ + for (j = 0; j < ld->nb_output_sections; j++) + ld->output_section_loadaddrs[j] = output_section_loadaddrs[j]; + ld->has_loadaddrs = 1; + } + + /* For NOLOAD sections (like .heap, .stack) that don't have actual ELF + * sections, compute their addresses based on the memory region they're + * assigned to in the linker script. Track per-region end addresses. */ + addr_t mr_end[LD_MAX_MEMORY_REGIONS] = {0}; + + /* Initialize memory region end addresses from laid-out sections */ + for (j = 0; j < ld->nb_output_sections; j++) + { + if (output_section_has_addr[j]) + { + int mr = ld->output_sections[j].memory_region_idx; + if (mr < 0) + mr = 0; + if (mr >= 0 && mr < ld->nb_memory_regions) + { + addr_t sec_end_addr = output_section_addrs[j] + ld->output_sections[j].current_offset; + if (sec_end_addr > mr_end[mr]) + mr_end[mr] = sec_end_addr; + } + } + } + + /* For regions with no sections, start from their origin */ + for (i = 0; i < ld->nb_memory_regions; i++) + { + if (mr_end[i] == 0) + mr_end[i] = ld->memory_regions[i].origin; + } + + /* Place NOLOAD sections in their assigned memory regions */ + for (j = 0; j < ld->nb_output_sections; j++) + { + if (output_section_addrs[j] == 0) + { + /* This output section has no matching ELF section (NOLOAD) */ + int mr = ld->output_sections[j].memory_region_idx; + if (mr < 0) + mr = 0; + if (mr >= 0 && mr < ld->nb_memory_regions) + { + output_section_addrs[j] = mr_end[mr]; + output_section_has_addr[j] = 1; + mr_end[mr] += ld->output_sections[j].current_offset; + } + } + } + + /* First pass: update symbols that are defined in output sections using + * section_offset. This handles all symbols generically. */ + for (j = 0; j < ld->nb_symbols; j++) + { + LDSymbol *sym = &ld->symbols[j]; + if (sym->defined && sym->section_idx >= 0 && sym->section_idx < ld->nb_output_sections) + { + addr_t section_addr = output_section_addrs[sym->section_idx]; + if (output_section_has_addr[sym->section_idx]) + { + /* Symbol value = section base address + offset within section */ + sym->value = section_addr + sym->section_offset; + } + } + } + + /* Resolve LOADADDR() symbols using computed LMA addresses */ + for (j = 0; j < ld->nb_symbols; j++) + { + LDSymbol *sym = &ld->symbols[j]; + if (sym->has_loadaddr && sym->loadaddr_section_idx >= 0 && sym->loadaddr_section_idx < ld->nb_output_sections) + { + addr_t lma = output_section_loadaddrs[sym->loadaddr_section_idx]; + sym->value = lma; + sym->defined = 1; + } + } + + /* Second pass: update standard section boundary symbols. + * For boundary symbols like __data_start__/__data_end__, always use + * the computed values based on actual section layout, because the + * linker script values are only relative offsets that don't account + * for all input sections. */ + for (j = 0; j < ld->nb_symbols; j++) + { + LDSymbol *sym = &ld->symbols[j]; + + /* Update standard section symbols - these ALWAYS use computed values */ + if (!strcmp(sym->name, "__bss_start__") || !strcmp(sym->name, "__bss_start")) + { + sym->value = bss_start; + sym->defined = 1; + } + else if (!strcmp(sym->name, "__bss_end__") || !strcmp(sym->name, "_bss_end__")) + { + sym->value = bss_end; + sym->defined = 1; + } + else if (!strcmp(sym->name, "__data_start__")) + { + sym->value = data_start; + sym->defined = 1; + } + else if (!strcmp(sym->name, "__data_end__") || !strcmp(sym->name, "_edata")) + { + sym->value = data_end; + sym->defined = 1; + } + else if (!strcmp(sym->name, "__text_start__") || !strcmp(sym->name, "_stext")) + { + sym->value = text_start; + sym->defined = 1; + } + else if (!strcmp(sym->name, "__text_end__") || !strcmp(sym->name, "_etext")) + { + sym->value = text_end; + sym->defined = 1; + } + else if (!strcmp(sym->name, "__rodata_start__")) + { + sym->value = rodata_start; + sym->defined = 1; + } + else if (!strcmp(sym->name, "__rodata_end__")) + { + sym->value = rodata_end; + sym->defined = 1; + } + else if (!strcmp(sym->name, "__end__") || !strcmp(sym->name, "_end") || !strcmp(sym->name, "end")) + { + sym->value = end_addr; + sym->defined = 1; + } + } +} + +/* Set or update a global symbol. If the symbol already exists, update its value + instead of trying to add it again (which would trigger "defined twice" error). */ +static void set_or_update_global_sym(TCCState *s1, const char *name, addr_t value) +{ + int sym_index = find_elf_sym(symtab_section, name); + if (sym_index) + { + ElfW(Sym) *esym = &((ElfW(Sym) *)symtab_section->data)[sym_index]; + if (esym->st_shndx != SHN_UNDEF) + { + /* Symbol already defined - update its value */ + esym->st_value = value; + esym->st_shndx = SHN_ABS; + return; + } + } + set_global_sym(s1, name, NULL, value); +} + +/* Export standard end/heap symbols based on section layout */ +ST_FUNC void ld_export_standard_symbols(TCCState *s1) +{ + Section *s; + addr_t bss_start = 0, bss_end = 0; + addr_t data_start = 0, data_end = 0; + addr_t text_start = 0, text_end = 0; + addr_t end_addr = 0; + addr_t sec_end; + int i; + + /* Find section addresses */ + for (i = 1; i < s1->nb_sections; i++) + { + s = s1->sections[i]; + if (!s->sh_addr) + continue; + + sec_end = s->sh_addr + s->sh_size; + if (sec_end > end_addr) + end_addr = sec_end; + + /* Match .bss and .bss.* sections */ + if (!strcmp(s->name, ".bss") || !strncmp(s->name, ".bss.", 5)) + { + if (bss_start == 0 || s->sh_addr < bss_start) + bss_start = s->sh_addr; + if (sec_end > bss_end) + bss_end = sec_end; + } + /* Match .data and .data.* sections */ + else if (!strcmp(s->name, ".data") || !strncmp(s->name, ".data.", 6)) + { + if (data_start == 0 || s->sh_addr < data_start) + data_start = s->sh_addr; + if (sec_end > data_end) + data_end = sec_end; + } + /* Match .text and .text.* sections */ + else if (!strcmp(s->name, ".text") || !strncmp(s->name, ".text.", 6)) + { + if (text_start == 0 || s->sh_addr < text_start) + text_start = s->sh_addr; + if (sec_end > text_end) + text_end = sec_end; + } + } + + /* Set standard symbols, updating existing ones if already defined + (e.g. by tcc_add_linker_symbols) */ + if (bss_start) + { + set_or_update_global_sym(s1, "__bss_start__", bss_start); + set_or_update_global_sym(s1, "__bss_start", bss_start); + } + if (bss_end) + { + set_or_update_global_sym(s1, "__bss_end__", bss_end); + set_or_update_global_sym(s1, "_bss_end__", bss_end); + } + if (data_start) + { + set_or_update_global_sym(s1, "__data_start__", data_start); + } + if (data_end) + { + set_or_update_global_sym(s1, "_edata", data_end); + set_or_update_global_sym(s1, "__data_end__", data_end); + } + if (text_start) + { + set_or_update_global_sym(s1, "__text_start__", text_start); + set_or_update_global_sym(s1, "_stext", text_start); + } + if (text_end) + { + set_or_update_global_sym(s1, "_etext", text_end); + set_or_update_global_sym(s1, "__text_end__", text_end); + } + if (end_addr) + { + set_or_update_global_sym(s1, "__end__", end_addr); + set_or_update_global_sym(s1, "_end", end_addr); + set_or_update_global_sym(s1, "end", end_addr); + /* Heap typically starts at end */ + set_or_update_global_sym(s1, "__heap_start__", end_addr); + } +} + +#endif /* !ELF_OBJ_ONLY */ diff --git a/source/obj/tccelf_load.c b/source/obj/tccelf_load.c new file mode 100644 index 000000000..a740b327c --- /dev/null +++ b/source/obj/tccelf_load.c @@ -0,0 +1,969 @@ +/* + * ELF file handling for TCC + * + * Copyright (c) 2001-2004 Fabrice Bellard + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +/* ELF: loading object files and archives (whole and a la carte). */ + +#include "tccelf_priv.h" + +/* load an object file and merge it with current files */ +/* XXX: handle correctly stab (debug) info */ +static Section *find_existing_section(TCCState *s1, const char *name); + +ST_FUNC ssize_t full_read(int fd, void *buf, size_t count) +{ + char *cbuf = buf; + size_t rnum = 0; + for (;;) + { + ssize_t num = read(fd, cbuf, count - rnum); + if (num < 0) + return num; + if (num == 0) + break; + rnum += num; + cbuf += num; + } + return rnum; +} + +ST_FUNC void *load_data(int fd, unsigned long file_offset, unsigned long size) +{ + void *data; + + data = tcc_malloc(size); + lseek(fd, file_offset, SEEK_SET); + full_read(fd, data, size); + return data; +} + +/* Return the canonical section name for function/data sections. + * Merges .text.foo -> .text, .rodata.bar -> .rodata, .data.baz -> .data, .bss.qux -> .bss + * Returns original name if no match. + */ +static const char *get_merged_section_name(TCCState *s1, const char *name) +{ + /* Under --gc-sections, per-function text sections from input objects must + * keep their identity: fold them here and section-granularity GC has + * nothing left to collect (the .o path is exactly how the self-hosted tcc + * links). Same-named sections from different objects still merge, which + * is conservative. Survivors are folded back into one .text after GC by + * coalesce_function_sections(). Data/rodata folding stays unconditional + * -- their GC granularity is a separate project, and the YAFF writer + * reads those through fixed section globals. */ + int keep_text_split = s1->gc_sections; + if (name[0] != '.') + return name; + switch (name[1]) + { + case 't': + if (!strncmp(name, ".text.", 6)) + return keep_text_split ? name : ".text"; + break; + case 'r': + if (!strncmp(name, ".rodata.", 8)) + return ".rodata"; + /* Relocation sections MUST fold in lockstep with their targets. If + * .text.error folds into .text but .rel.text.error keeps its name, two + * disasters follow when the in-process compile (under + * -ffunction-sections) already created a section of that exact name: + * the member's entries -- rebased for .text -- merge into a reloc + * section whose sh_info points at the compiled 8-byte .text.error, and + * the loader's back-link update leaves text_section->reloc aimed at a + * random .rel.text.. Both corrupt relocation application. */ + if (!strncmp(name, ".rel.text.", 10)) + return keep_text_split ? name : ".rel.text"; + if (!strncmp(name, ".rel.rodata.", 12)) + return ".rel.rodata"; + if (!strncmp(name, ".rel.data.", 10)) + return ".rel.data"; + if (!strncmp(name, ".rela.text.", 11)) + return keep_text_split ? name : ".rela.text"; + if (!strncmp(name, ".rela.rodata.", 13)) + return ".rela.rodata"; + if (!strncmp(name, ".rela.data.", 11)) + return ".rela.data"; + break; + case 'd': + if (!strncmp(name, ".data.", 6)) + return ".data"; + break; + case 'b': + if (!strncmp(name, ".bss.", 5)) + return ".bss"; + break; + } + return name; +} + +ST_FUNC int tcc_object_type(int fd, ElfW(Ehdr) * h) +{ + int size = full_read(fd, h, sizeof *h); + if (size == sizeof *h && 0 == memcmp(h, ELFMAG, 4) && + h->e_ident[EI_CLASS] == ELFCLASSW) + { + if (h->e_type == ET_REL) + return AFF_BINTYPE_REL; + if (h->e_type == ET_DYN) + return AFF_BINTYPE_DYN; + } + else if (size >= 8) + { + if (0 == memcmp(h, ARMAG, 8)) + return AFF_BINTYPE_AR; +#ifdef TCC_TARGET_COFF + if (((struct filehdr *)h)->f_magic == COFF_C67_MAGIC) + return AFF_BINTYPE_C67; +#endif + } + if (0 == memcmp(h, YAFFMAG, 4)) + { + return AFF_BINTYPE_YAFF; + } + + return 0; +} + +ST_FUNC int tcc_load_object_file(TCCState *s1, int fd, unsigned long file_offset) +{ + ElfW(Ehdr) ehdr; + ElfW(Shdr) * shdr, *sh; + unsigned long size, offset, offseti; + int i, nb_syms, sym_index, ret, seencompressed; + char *strsec, *strtab; + int stab_index = 0, stabstr_index = 0; + (void)stab_index; + (void)stabstr_index; + int *old_to_new_syms; + char *sh_name, *name; + SectionMergeInfo *sm_table, *sm; + ElfW(Sym) * sym, *symtab; + ElfW_Rel *rel; + Section *s; + const char *last_lookup_name = NULL; + Section *last_lookup_section = NULL; + unsigned object_start = 0; + + if (s1->do_bench) + object_start = tcc_getclock_us(); + + /* Use lazy loading for aggressive GC mode */ + if (s1->gc_sections_aggressive) + { + return tcc_load_object_file_lazy(s1, fd, file_offset); + } + + lseek(fd, file_offset, SEEK_SET); + + if (tcc_object_type(fd, &ehdr) != AFF_BINTYPE_REL) + { + goto invalid; + } + /* test CPU specific stuff */ + + if (ehdr.e_ident[5] != ELFDATA2LSB || ehdr.e_machine != EM_TCC_TARGET) + { + invalid: + return tcc_error_noabort("invalid object file"); + } + /* read sections */ + shdr = load_data(fd, file_offset + ehdr.e_shoff, sizeof(ElfW(Shdr)) * ehdr.e_shnum); + sm_table = tcc_mallocz(sizeof(SectionMergeInfo) * ehdr.e_shnum); + + /* load section names */ + sh = &shdr[ehdr.e_shstrndx]; + strsec = load_data(fd, file_offset + sh->sh_offset, sh->sh_size); + + /* load symtab and strtab */ + old_to_new_syms = NULL; + symtab = NULL; + strtab = NULL; + nb_syms = 0; + seencompressed = 0; + stab_index = stabstr_index = 0; + ret = -1; + + for (i = 1; i < ehdr.e_shnum; i++) + { + sh = &shdr[i]; + if (sh->sh_type == SHT_SYMTAB) + { + if (symtab) + { + tcc_error_noabort("object must contain only one symtab"); + goto the_end; + } + nb_syms = sh->sh_size / sizeof(ElfW(Sym)); + symtab = load_data(fd, file_offset + sh->sh_offset, sh->sh_size); + sm_table[i].s = symtab_section; + + /* now load strtab */ + sh = &shdr[sh->sh_link]; + strtab = load_data(fd, file_offset + sh->sh_offset, sh->sh_size); + } + if (sh->sh_flags & SHF_COMPRESSED) + seencompressed = 1; + } + + /* now examine each section and try to merge its content with the + ones in memory */ + for (i = 1; i < ehdr.e_shnum; i++) + { + /* no need to examine section name strtab */ + if (i == ehdr.e_shstrndx) + continue; + sh = &shdr[i]; + if (sh->sh_type == SHT_RELX) + sh = &shdr[sh->sh_info]; + /* ignore sections types we do not handle (plus relocs to those) */ + sh_name = strsec + sh->sh_name; + if (sh_name[0] == '.' && (sh_name[1] == 'd' || sh_name[1] == 's') && + (0 == strncmp(sh_name, ".debug_", 7) || 0 == strncmp(sh_name, ".stab", 5))) + { + if (!s1->do_debug || seencompressed) + continue; +#if !(TARGETOS_OpenBSD || TARGETOS_FreeBSD || TARGETOS_NetBSD) + } + else if (sh_name[1] == 'e' && 0 == strncmp(sh_name, ".eh_frame", 9)) + { + if (NULL == eh_frame_section) + continue; +#endif + } + else if (sh->sh_type != SHT_PROGBITS && sh->sh_type != SHT_NOTE && sh->sh_type != SHT_NOBITS && + sh->sh_type != SHT_PREINIT_ARRAY && sh->sh_type != SHT_INIT_ARRAY && sh->sh_type != SHT_FINI_ARRAY +#ifdef TCC_ARM_EABI + && sh->sh_type != SHT_ARM_EXIDX +#endif +#if TARGETOS_OpenBSD || TARGETOS_FreeBSD || TARGETOS_NetBSD + && sh->sh_type != SHT_X86_64_UNWIND +#endif + ) + continue; + + sh = &shdr[i]; + sh_name = strsec + sh->sh_name; + if (sh->sh_addralign < 1) + sh->sh_addralign = 1; + /* find corresponding section, if any */ + /* Use merged name for .text.*, .rodata.*, .data.*, .bss.* sections */ + { + const char *lookup_name = get_merged_section_name(s1, sh_name); + s = NULL; + if (lookup_name == last_lookup_name) + { + s = last_lookup_section; + } + else + { + s = find_existing_section(s1, lookup_name); + } + last_lookup_name = lookup_name; + last_lookup_section = s; + if (s) + { + if (sh->sh_type != s->sh_type && strcmp(s->name, ".eh_frame")) + { + tcc_error_noabort("section type conflict: %s %02x <> %02x", s->name, sh->sh_type, s->sh_type); + goto the_end; + } + if (sh_name[1] == 'g' && !strncmp(sh_name, ".gnu.linkonce", 13)) + { + /* if a 'linkonce' section is already present, we + do not add it again. It is a little tricky as + symbols can still be defined in + it. */ + sm_table[i].link_once = 1; + goto next; + } + /* stab section tracking removed - DWARF only */ + /* Track if this section was merged (original name differs from lookup name) */ + if (sh_name != lookup_name) + sm_table[i].merged_to = lookup_name; + goto found; + } + /* not found: create new section with merged name */ + s = new_section(s1, lookup_name, sh->sh_type, sh->sh_flags & ~SHF_GROUP); + /* take as much info as possible from the section. sh_link and + sh_info will be updated later */ + s->sh_addralign = sh->sh_addralign; + s->sh_entsize = sh->sh_entsize; + sm_table[i].new_section = 1; + last_lookup_name = lookup_name; + last_lookup_section = s; + /* Track if this section was merged */ + if (sh_name != lookup_name) + sm_table[i].merged_to = lookup_name; + } + found: + /* align start of section */ + s->data_offset += -s->data_offset & (sh->sh_addralign - 1); + if (sh->sh_addralign > s->sh_addralign) + s->sh_addralign = sh->sh_addralign; + sm_table[i].offset = s->data_offset; + sm_table[i].s = s; + /* concatenate sections */ + size = sh->sh_size; + if (sh->sh_type != SHT_NOBITS) + { + if (should_defer_section(sh_name, sh->sh_type)) + { + /* Lazy loading: just record position for debug sections */ + unsigned long dest_off = s->data_offset; + s->data_offset += size; /* Reserve space without allocating */ + + /* Record where to load from later - include archive member offset if in archive */ + unsigned long abs_offset = + s1->current_archive_offset ? s1->current_archive_offset + sh->sh_offset : file_offset + sh->sh_offset; + /* Use archive path if loading from archive, otherwise use current file */ + const char *source_path = s1->current_archive_path ? s1->current_archive_path : s1->current_filename; + /* Track source path for materialization */ + section_add_deferred(s1, s, source_path, abs_offset, size, dest_off); + } + else + { + /* Immediate loading */ + unsigned char *ptr; + lseek(fd, file_offset + sh->sh_offset, SEEK_SET); + /* section_ptr_add will handle allocation as needed */ + ptr = section_ptr_add(s, size); + full_read(fd, ptr, size); + } + } + else + { + s->data_offset += size; + } + /* align end of section */ + /* This is needed if we compile a c file after this */ + if (s == text_section || s == data_section || s == rodata_section || s == bss_section || s == common_section) + s->data_offset += -s->data_offset & (s->sh_addralign - 1); + next:; + } + + /* stab string relocation removed - only DWARF debug info supported */ + + /* second short pass to update sh_link and sh_info fields of new + sections */ + for (i = 1; i < ehdr.e_shnum; i++) + { + s = sm_table[i].s; + if (!s || !sm_table[i].new_section) + continue; + sh = &shdr[i]; + if (sh->sh_link > 0) + s->link = sm_table[sh->sh_link].s; + if (sh->sh_type == SHT_RELX) + { + s->sh_info = sm_table[sh->sh_info].s->sh_num; + /* update backward link */ + s1->sections[s->sh_info]->reloc = s; + } + } + + /* resolve symbols */ + old_to_new_syms = tcc_mallocz(nb_syms * sizeof(int)); + + sym = symtab + 1; + for (i = 1; i < nb_syms; i++, sym++) + { + if (sym->st_shndx != SHN_UNDEF && sym->st_shndx < SHN_LORESERVE) + { + sm = &sm_table[sym->st_shndx]; + if (sm->link_once) + { + /* if a symbol is in a link once section, we use the + already defined symbol. It is very important to get + correct relocations */ + if (ELFW(ST_BIND)(sym->st_info) != STB_LOCAL) + { + name = strtab + sym->st_name; + sym_index = find_elf_sym(symtab_section, name); + if (sym_index) + old_to_new_syms[i] = sym_index; + } + continue; + } + /* if no corresponding section added, no need to add symbol */ + if (!sm->s) + continue; + /* convert section number */ + sym->st_shndx = sm->s->sh_num; + /* offset value */ + sym->st_value += sm->offset; + } + /* add symbol */ + name = strtab + sym->st_name; + sym_index = + set_elf_sym(symtab_section, sym->st_value, sym->st_size, sym->st_info, sym->st_other, sym->st_shndx, name); + old_to_new_syms[i] = sym_index; + } + + /* third pass to patch relocation entries */ + for (i = 1; i < ehdr.e_shnum; i++) + { + s = sm_table[i].s; + if (!s) + continue; + sh = &shdr[i]; + offset = sm_table[i].offset; + size = sh->sh_size; + switch (s->sh_type) + { + case SHT_RELX: + /* take relocation offset information */ + offseti = sm_table[sh->sh_info].offset; + for (rel = (ElfW_Rel *)s->data + (offset / sizeof(*rel)); + rel < (ElfW_Rel *)s->data + ((offset + size) / sizeof(*rel)); rel++) + { + int type; + unsigned sym_index; + /* convert symbol index */ + type = ELFW(R_TYPE)(rel->r_info); + sym_index = ELFW(R_SYM)(rel->r_info); + /* NOTE: only one symtab assumed */ + if (sym_index >= nb_syms) + goto invalid_reloc; + sym_index = old_to_new_syms[sym_index]; + /* ignore link_once in rel section. */ + if (!sym_index && !sm_table[sh->sh_info].link_once +#ifdef TCC_TARGET_ARM + && type != R_ARM_V4BX +#elif defined TCC_TARGET_RISCV64 + && type != R_RISCV_ALIGN && type != R_RISCV_RELAX +#endif + ) + { + invalid_reloc: + tcc_error_noabort("Invalid relocation entry [%2d] '%s' @ %.8x", i, strsec + sh->sh_name, (int)rel->r_offset); + goto the_end; + } + rel->r_info = ELFW(R_INFO)(sym_index, type); + /* offset the relocation offset */ + rel->r_offset += offseti; +#ifdef TCC_TARGET_ARM + /* Jumps and branches from a Thumb code to a PLT entry need + special handling since PLT entries are ARM code. + Unconditional bl instructions referencing PLT entries are + handled by converting these instructions into blx + instructions. Other case of instructions referencing a PLT + entry require to add a Thumb stub before the PLT entry to + switch to ARM mode. We set bit plt_thumb_stub of the + attribute of a symbol to indicate such a case. */ + if (type == R_ARM_THM_JUMP24) + get_sym_attr(s1, sym_index, 1)->plt_thumb_stub = 1; +#endif + } + break; + default: + break; + } + } + + ret = 0; +the_end: + if (s1->do_bench) + { + unsigned elapsed = tcc_getclock_us() - object_start; + s1->bench_object_load_time += elapsed; + s1->bench_object_load_count++; + if (s1->current_archive_offset) + s1->bench_archive_member_count++; + else + tcc_bench_log(s1, "load-obj", s1->current_filename, elapsed); + } + tcc_free(symtab); + tcc_free(strtab); + tcc_free(old_to_new_syms); + tcc_free(sm_table); + tcc_free(strsec); + tcc_free(shdr); + return ret; +} + +static unsigned long long get_be(const uint8_t *b, int n) +{ + unsigned long long ret = 0; + while (n) + ret = (ret << 8) | *b++, --n; + return ret; +} + +static int read_ar_header(int fd, int offset, ArchiveHeader *hdr) +{ + char *p, *e; + int len; + lseek(fd, offset, SEEK_SET); + len = full_read(fd, hdr, sizeof(ArchiveHeader)); + if (len != sizeof(ArchiveHeader)) + return len ? -1 : 0; + if (memcmp(hdr->ar_fmag, ARFMAG, sizeof hdr->ar_fmag)) + return -1; + p = hdr->ar_name; + for (e = p + sizeof hdr->ar_name; e > p && e[-1] == ' ';) + --e; + *e = '\0'; + hdr->ar_size[sizeof hdr->ar_size - 1] = 0; + return len; +} + +static int alacarte_member_seen(unsigned long long *loaded_members, unsigned int mask, unsigned long long offset) +{ + unsigned long long key = offset + 1; + unsigned int idx = ((unsigned int)offset ^ (unsigned int)(offset >> 32)) & mask; + + while (loaded_members[idx]) + { + if (loaded_members[idx] == key) + return 1; + idx = (idx + 1) & mask; + } + return 0; +} + +static void alacarte_mark_member(unsigned long long *loaded_members, unsigned int mask, unsigned long long offset) +{ + unsigned long long key = offset + 1; + unsigned int idx = ((unsigned int)offset ^ (unsigned int)(offset >> 32)) & mask; + + while (loaded_members[idx] && loaded_members[idx] != key) + { + idx = (idx + 1) & mask; + } + loaded_members[idx] = key; +} + +static Section *find_existing_section(TCCState *s1, const char *name) +{ + return section_ht_find(s1, name); +} + +/* Check whether any currently undefined symbol is satisfiable by cached + archive symbol tables. Returns 1 if at least one such undef exists, + meaning a group rescan may still be productive. */ +ST_FUNC int tcc_group_has_satisfiable_undefs(TCCState *s1) +{ + Section *s = symtab_section; + ElfW(Sym) *syms = (ElfW(Sym) *)s->data; + Section *hs = s->hash; + unsigned int *sym_hcache = hs ? hs->hash_val_cache : NULL; + int ci, ui, si; + if (s1->nb_archive_sym_caches == 0) + return 0; + for (ui = 0; ui < s1->nb_undef_syms; ui++) + { + unsigned int h; + int aidx; + si = s1->undef_sym_list[ui]; + if (syms[si].st_shndx != SHN_UNDEF) + continue; + const char *name = (char *)s->link->data + syms[si].st_name; + h = (sym_hcache && si < hs->hash_val_alloc) ? sym_hcache[si] : elf_hash((const unsigned char *)name); + for (ci = 0; ci < s1->nb_archive_sym_caches; ci++) + { + ArchiveSymbolCache *cache = &s1->archive_sym_caches[ci]; + for (aidx = cache->ar_ht_buckets[h & cache->ar_ht_mask]; aidx >= 0; aidx = cache->ar_ht_next[aidx]) + { + if (cache->name_hashes[aidx] == h && !strcmp(cache->sym_names[aidx], name)) + return 1; + } + } + } + return 0; +} + +/* Find an existing archive symbol cache entry by filename */ +static ArchiveSymbolCache *find_archive_sym_cache(TCCState *s1, const char *filename) +{ + int i; + for (i = 0; i < s1->nb_archive_sym_caches; i++) + { + if (!strcmp(s1->archive_sym_caches[i].filename, filename)) + return &s1->archive_sym_caches[i]; + } + return NULL; +} + +/* Create and populate a new archive symbol cache entry */ +static ArchiveSymbolCache *create_archive_sym_cache(TCCState *s1, const char *filename, int fd, int size, int entrysize) +{ + int i, nsyms, ar_ht_size; + unsigned int loaded_member_size; + uint8_t *data; + const uint8_t *ar_index; + const char *ar_names, *p; + ArchiveSymbolCache *cache; + + data = tcc_malloc(size); + if (full_read(fd, data, size) != size) + { + tcc_free(data); + return NULL; + } + nsyms = get_be(data, entrysize); + ar_index = data + entrysize; + ar_names = (char *)ar_index + nsyms * entrysize; + + /* Grow cache array */ + s1->archive_sym_caches = + tcc_realloc(s1->archive_sym_caches, (s1->nb_archive_sym_caches + 1) * sizeof(ArchiveSymbolCache)); + cache = &s1->archive_sym_caches[s1->nb_archive_sym_caches++]; + memset(cache, 0, sizeof(*cache)); + + cache->filename = tcc_strdup(filename); + cache->data = data; + cache->nsyms = nsyms; + cache->entrysize = entrysize; + + /* Allocate symbol arrays */ + cache->sym_names = tcc_malloc(nsyms * sizeof(const char *)); + cache->name_hashes = tcc_malloc(nsyms * sizeof(unsigned int)); + cache->member_offsets = tcc_malloc(nsyms * sizeof(unsigned long long)); + + /* Build chained hash table */ + ar_ht_size = 1; + while (ar_ht_size < nsyms * 2) + ar_ht_size <<= 1; + cache->ar_ht_mask = ar_ht_size - 1; + cache->ar_ht_buckets = tcc_malloc(ar_ht_size * sizeof(int)); + memset(cache->ar_ht_buckets, 0xff, ar_ht_size * sizeof(int)); + cache->ar_ht_next = tcc_malloc(nsyms * sizeof(int)); + memset(cache->ar_ht_next, 0xff, nsyms * sizeof(int)); + + /* Size loaded_members dedup set */ + loaded_member_size = 256; + if ((unsigned int)nsyms / 8 > loaded_member_size) + loaded_member_size = (unsigned int)nsyms / 8; + { + unsigned int v = loaded_member_size; + v--; + v |= v >> 1; + v |= v >> 2; + v |= v >> 4; + v |= v >> 8; + v |= v >> 16; + loaded_member_size = v + 1; + } + cache->loaded_member_mask = loaded_member_size - 1; + cache->loaded_members = tcc_mallocz(loaded_member_size * sizeof(unsigned long long)); + + /* Populate symbol arrays and hash table */ + for (p = ar_names, i = 0; i < nsyms; i++, p += strlen(p) + 1) + { + unsigned int h; + int bucket; + cache->sym_names[i] = p; + cache->name_hashes[i] = h = elf_hash((const unsigned char *)p); + cache->member_offsets[i] = get_be(ar_index + i * entrysize, entrysize); + bucket = h & cache->ar_ht_mask; + cache->ar_ht_next[i] = cache->ar_ht_buckets[bucket]; + cache->ar_ht_buckets[bucket] = i; + } + + return cache; +} + +ST_FUNC int tcc_archive_index_load(TCCState *s1, const char *path) +{ + char magic[sizeof ARMAG - 1]; + ArchiveHeader hdr; + int fd, len, ok = 0; + + if (find_archive_sym_cache(s1, path)) + return 1; + fd = open(path, O_RDONLY | O_BINARY); + if (fd < 0) + return 0; + /* The index is the first member, named "/" (32-bit offsets) or "/SYM64/". */ + if (full_read(fd, magic, sizeof magic) == sizeof magic && memcmp(magic, ARMAG, sizeof magic) == 0 && + (len = read_ar_header(fd, sizeof magic, &hdr)) > 0) + { + int entrysize = !strcmp(hdr.ar_name, "/") ? 4 : !strcmp(hdr.ar_name, "/SYM64/") ? 8 : 0; + if (entrysize) + ok = create_archive_sym_cache(s1, path, fd, strtol(hdr.ar_size, NULL, 0), entrysize) != NULL; + } + close(fd); + return ok; +} + +ST_FUNC int tcc_archive_index_has(TCCState *s1, const char *path, const char *name) +{ + ArchiveSymbolCache *cache = find_archive_sym_cache(s1, path); + unsigned int h; + int aidx; + if (!cache) + return 0; + h = elf_hash((const unsigned char *)name); + for (aidx = cache->ar_ht_buckets[h & cache->ar_ht_mask]; aidx >= 0; aidx = cache->ar_ht_next[aidx]) + if (cache->name_hashes[aidx] == h && strcmp(cache->sym_names[aidx], name) == 0) + return 1; + return 0; +} + +/* Free all cached archive symbol tables */ +ST_FUNC void tcc_archive_cache_free(TCCState *s1) +{ + int i; + for (i = 0; i < s1->nb_archive_sym_caches; i++) + { + ArchiveSymbolCache *c = &s1->archive_sym_caches[i]; + tcc_free(c->filename); + tcc_free(c->loaded_members); + tcc_free(c->member_offsets); + tcc_free(c->name_hashes); + tcc_free(c->sym_names); + tcc_free(c->ar_ht_next); + tcc_free(c->ar_ht_buckets); + tcc_free(c->data); + } + tcc_free(s1->archive_sym_caches); + s1->archive_sym_caches = NULL; + s1->nb_archive_sym_caches = 0; +} + +/* load only the objects which resolve undefined symbols */ +static int tcc_load_alacarte(TCCState *s1, int fd, int size, int entrysize) +{ + int bound, len, ret = -1; + unsigned long long off; + ElfW(Sym) * sym; + ArchiveHeader hdr; + ArchiveSymbolCache *cache; + /* Save archive state for restoration */ + unsigned long saved_archive_offset = s1->current_archive_offset; + const char *saved_archive_path = s1->current_archive_path; + s1->current_archive_path = s1->current_filename; + + /* Look up or create cached archive symbol table. On --start-group + rescans this avoids re-reading the index, re-computing elf_hash + for all symbols, and re-allocating hash tables. */ + cache = find_archive_sym_cache(s1, s1->current_filename); + if (!cache) + { + cache = create_archive_sym_cache(s1, s1->current_filename, fd, size, entrysize); + if (!cache) + { + tcc_error_noabort("invalid archive"); + s1->current_archive_offset = saved_archive_offset; + s1->current_archive_path = saved_archive_path; + return -1; + } + } + + /* Inverted resolution: iterate tracked undefined symbols, + look up each in archive hash table. O(n_undef) per pass instead + of O(n_symtab) in the original forward scan. */ + do + { + Section *s = symtab_section; + Section *hs = s->hash; + unsigned int *sym_hcache = hs ? hs->hash_val_cache : NULL; + int ui, si; + bound = 0; + + for (ui = 0; ui < s1->nb_undef_syms; ui++) + { + unsigned int h; + int aidx; + + si = s1->undef_sym_list[ui]; + sym = &((ElfW(Sym) *)s->data)[si]; + if (sym->st_shndx != SHN_UNDEF) + continue; + + /* Use cached hash from symtab hash table when available, + avoiding elf_hash recomputation across archive calls. */ + h = (sym_hcache && si < hs->hash_val_alloc) + ? sym_hcache[si] + : elf_hash((const unsigned char *)((char *)s->link->data + sym->st_name)); + + for (aidx = cache->ar_ht_buckets[h & cache->ar_ht_mask]; aidx >= 0; aidx = cache->ar_ht_next[aidx]) + { + if (cache->name_hashes[aidx] != h) + continue; + { + const char *sym_name = (char *)s->link->data + sym->st_name; + if (strcmp(cache->sym_names[aidx], sym_name)) + continue; + } + off = cache->member_offsets[aidx]; + if (alacarte_member_seen(cache->loaded_members, cache->loaded_member_mask, off)) + break; + + len = read_ar_header(fd, off, &hdr); + if (len <= 0 || memcmp(hdr.ar_fmag, ARFMAG, 2)) + { + tcc_error_noabort("invalid archive"); + goto the_end; + } + off += len; + if (s1->verbose == 2) + printf(" -> %s\n", hdr.ar_name); + /* Set archive offset for lazy loading */ + s1->current_archive_offset = (unsigned long)off; + if (tcc_load_object_file(s1, fd, off) < 0) + goto the_end; + s1->current_archive_offset = saved_archive_offset; + alacarte_mark_member(cache->loaded_members, cache->loaded_member_mask, cache->member_offsets[aidx]); + ++bound; + ++s1->group_rescan_loaded; + /* symtab/hash may have been reallocated by tcc_load_object_file */ + hs = s->hash; + sym_hcache = hs ? hs->hash_val_cache : NULL; + break; + } + } + } while (bound); + /* Compact the list, removing symbols now defined. Shrinks iteration + cost for subsequent archive loads and group-rescan checks. */ + { + ElfW(Sym) *syms = (ElfW(Sym) *)symtab_section->data; + int ri, wi = 0; + for (ri = 0; ri < s1->nb_undef_syms; ri++) + { + int idx = s1->undef_sym_list[ri]; + if (syms[idx].st_shndx == SHN_UNDEF) + s1->undef_sym_list[wi++] = idx; + } + s1->nb_undef_syms = wi; + } + ret = 0; +the_end: + s1->current_archive_offset = saved_archive_offset; + s1->current_archive_path = saved_archive_path; + return ret; +} + +/* load a '.a' file */ +ST_FUNC int tcc_load_archive(TCCState *s1, int fd, int alacarte) +{ + ArchiveHeader hdr; + /* char magic[8]; */ + int size, len; + unsigned long file_offset; + ElfW(Ehdr) ehdr; + unsigned long saved_archive_offset; + const char *saved_archive_path; + unsigned archive_start = 0; + unsigned members_before = 0; + char archive_desc[1088]; + + if (s1->do_bench) + { + archive_start = tcc_getclock_us(); + members_before = s1->bench_archive_member_count; + } + + /* skip magic which was already checked */ + /* full_read(fd, magic, sizeof(magic)); */ + file_offset = sizeof ARMAG - 1; + + /* Save archive state for restoration */ + saved_archive_offset = s1->current_archive_offset; + saved_archive_path = s1->current_archive_path; + s1->current_archive_path = s1->current_filename; + + for (;;) + { + len = read_ar_header(fd, file_offset, &hdr); + if (len == 0) + { + if (s1->do_bench) + { + unsigned elapsed = tcc_getclock_us() - archive_start; + unsigned members_loaded = s1->bench_archive_member_count - members_before; + s1->bench_archive_load_time += elapsed; + s1->bench_archive_load_count++; + snprintf(archive_desc, sizeof(archive_desc), "%s (%u members)", + s1->current_filename ? s1->current_filename : "", members_loaded); + tcc_bench_log(s1, "load-archive", archive_desc, elapsed); + } + s1->current_archive_offset = saved_archive_offset; + s1->current_archive_path = saved_archive_path; + return 0; + } + if (len < 0) + { + s1->current_archive_offset = saved_archive_offset; + s1->current_archive_path = saved_archive_path; + return tcc_error_noabort("invalid archive"); + } + file_offset += len; + size = strtol(hdr.ar_size, NULL, 0); + if (alacarte) + { + /* coff symbol table : we handle it */ + if (!strcmp(hdr.ar_name, "/")) + { + int ret = tcc_load_alacarte(s1, fd, size, 4); + if (s1->do_bench) + { + unsigned elapsed = tcc_getclock_us() - archive_start; + unsigned members_loaded = s1->bench_archive_member_count - members_before; + s1->bench_archive_load_time += elapsed; + s1->bench_archive_load_count++; + snprintf(archive_desc, sizeof(archive_desc), "%s (%u members)", + s1->current_filename ? s1->current_filename : "", members_loaded); + tcc_bench_log(s1, "load-archive", archive_desc, elapsed); + } + s1->current_archive_offset = saved_archive_offset; + s1->current_archive_path = saved_archive_path; + return ret; + } + if (!strcmp(hdr.ar_name, "/SYM64/")) + { + int ret = tcc_load_alacarte(s1, fd, size, 8); + if (s1->do_bench) + { + unsigned elapsed = tcc_getclock_us() - archive_start; + unsigned members_loaded = s1->bench_archive_member_count - members_before; + s1->bench_archive_load_time += elapsed; + s1->bench_archive_load_count++; + snprintf(archive_desc, sizeof(archive_desc), "%s (%u members)", + s1->current_filename ? s1->current_filename : "", members_loaded); + tcc_bench_log(s1, "load-archive", archive_desc, elapsed); + } + s1->current_archive_offset = saved_archive_offset; + s1->current_archive_path = saved_archive_path; + return ret; + } + } + else if (tcc_object_type(fd, &ehdr) == AFF_BINTYPE_REL) + { + if (s1->verbose == 2) + printf(" -> %s\n", hdr.ar_name); + /* Set archive offset for lazy loading */ + s1->current_archive_offset = file_offset; + if (tcc_load_object_file(s1, fd, file_offset) < 0) + { + s1->current_archive_offset = saved_archive_offset; + s1->current_archive_path = saved_archive_path; + return -1; + } + s1->current_archive_offset = saved_archive_offset; + } + /* align to even */ + file_offset = (file_offset + size + 1) & ~1; + } + /* Unreachable for non-alacarte mode; required to silence + 'function might return no value' warning in TCC self-build. */ + s1->current_archive_offset = saved_archive_offset; + s1->current_archive_path = saved_archive_path; + return 0; +} diff --git a/source/obj/tccelf_output.c b/source/obj/tccelf_output.c new file mode 100644 index 000000000..6b1707f06 --- /dev/null +++ b/source/obj/tccelf_output.c @@ -0,0 +1,1096 @@ +/* + * ELF file handling for TCC + * + * Copyright (c) 2001-2004 Fabrice Bellard + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +/* ELF: writing executables, shared objects, raw binaries and objects, + * including section garbage collection and function-section coalescing. */ + +#include "tccelf_priv.h" + +/* Create an ELF file on disk. + This function handle ELF specific layout requirements */ +static int tcc_output_elf(TCCState *s1, FILE *f, int phnum, ElfW(Phdr) * phdr) +{ + int i, shnum, offset, size, file_type; + Section *s; + ElfW(Ehdr) ehdr; + ElfW(Shdr) shdr, *sh; + + file_type = s1->output_type; + shnum = s1->nb_sections; + + memset(&ehdr, 0, sizeof(ehdr)); + if (phnum > 0) + { + ehdr.e_phentsize = sizeof(ElfW(Phdr)); + ehdr.e_phnum = phnum; + ehdr.e_phoff = sizeof(ElfW(Ehdr)); + } + + /* fill header */ + ehdr.e_ident[0] = ELFMAG0; + ehdr.e_ident[1] = ELFMAG1; + ehdr.e_ident[2] = ELFMAG2; + ehdr.e_ident[3] = ELFMAG3; + ehdr.e_ident[4] = ELFCLASSW; + ehdr.e_ident[5] = ELFDATA2LSB; + ehdr.e_ident[6] = EV_CURRENT; + +#if TARGETOS_FreeBSD || TARGETOS_FreeBSD_kernel + ehdr.e_ident[EI_OSABI] = ELFOSABI_FREEBSD; +#elif defined TCC_TARGET_ARM && defined TCC_ARM_EABI + ehdr.e_flags = EF_ARM_EABI_VER5; + ehdr.e_flags |= s1->float_abi == ARM_HARD_FLOAT ? EF_ARM_VFP_FLOAT : EF_ARM_SOFT_FLOAT; +#elif defined TCC_TARGET_ARM + ehdr.e_ident[EI_OSABI] = ELFOSABI_ARM; +#elif defined TCC_TARGET_RISCV64 + /* XXX should be configurable */ + ehdr.e_flags = EF_RISCV_FLOAT_ABI_DOUBLE; +#endif + + if (file_type == TCC_OUTPUT_OBJ) + { + ehdr.e_type = ET_REL; + } + else + { + if (file_type & TCC_OUTPUT_DYN) + ehdr.e_type = ET_DYN; + else + ehdr.e_type = ET_EXEC; + if (s1->elf_entryname) + ehdr.e_entry = get_sym_addr(s1, s1->elf_entryname, 1, 0); + else + ehdr.e_entry = get_sym_addr(s1, "_start", !!(file_type & TCC_OUTPUT_EXE), 0); + if (ehdr.e_entry == (addr_t)-1) + ehdr.e_entry = text_section->sh_addr; + if (s1->nb_errors) + return -1; + } + + tcc_elf_sort_syms(s1, s1->symtab); + + ehdr.e_machine = EM_TCC_TARGET; + ehdr.e_version = EV_CURRENT; + ehdr.e_shoff = (sizeof(ElfW(Ehdr)) + phnum * sizeof(ElfW(Phdr)) + 3) & -4; + ehdr.e_ehsize = sizeof(ElfW(Ehdr)); + ehdr.e_shentsize = sizeof(ElfW(Shdr)); + ehdr.e_shnum = shnum; + ehdr.e_shstrndx = shnum - 1; + + offset = fwrite(&ehdr, 1, sizeof(ElfW(Ehdr)), f); + if (phdr) + offset += fwrite(phdr, 1, phnum * sizeof(ElfW(Phdr)), f); + + /* output section headers */ + while (offset < ehdr.e_shoff) + { + fputc(0, f); + offset++; + } + + for (i = 0; i < shnum; i++) + { + sh = &shdr; + memset(sh, 0, sizeof(ElfW(Shdr))); + if (i) + { + s = s1->sections[i]; + sh->sh_name = s->sh_name; + sh->sh_type = s->sh_type; + sh->sh_flags = s->sh_flags; + sh->sh_entsize = s->sh_entsize; + sh->sh_info = s->sh_info; + if (s->link) + sh->sh_link = s->link->sh_num; + sh->sh_addralign = s->sh_addralign; + sh->sh_addr = s->sh_addr; + sh->sh_offset = s->sh_offset; + sh->sh_size = s->sh_size; + } + offset += fwrite(sh, 1, sizeof(ElfW(Shdr)), f); + } + + /* output sections - use streaming for lazy sections to avoid memory allocation */ + for (i = 1; i < s1->nb_sections; i++) + { + s = s1->sections[i]; + if (s->sh_type != SHT_NOBITS) + { + while (offset < s->sh_offset) + { + fputc(0, f); + offset++; + } + size = s->sh_size; + if (size) + { + if (s->lazy && !s->materialized) + { + /* Stream directly from source files without loading into memory */ + section_write_streaming(s1, s, f); + offset += size; + } + else + { + /* Already materialized, write from memory */ + const int to_write = size < s->data_allocated ? size : s->data_allocated; + int written = fwrite(s->data, 1, to_write, f); + offset += written; + } + } + } + } + return 0; +} + +static int tcc_output_binary(TCCState *s1, FILE *f) +{ + Section *s; + int i, offset, size; + + offset = 0; + for (i = 1; i < s1->nb_sections; i++) + { + s = s1->sections[i]; + if (s->sh_type != SHT_NOBITS && (s->sh_flags & SHF_ALLOC)) + { + while (offset < s->sh_offset) + { + fputc(0, f); + offset++; + } + size = s->sh_size; + if (s->lazy && !s->materialized) + { + /* Stream directly from source files without loading into memory */ + section_write_streaming(s1, s, f); + } + else + { + fwrite(s->data, 1, size, f); + } + offset += size; + } + } + return 0; +} + +/* Write an elf, coff or "binary" file */ +static int tcc_write_elf_file(TCCState *s1, const char *filename, int phnum, ElfW(Phdr) * phdr) +{ + int fd, mode, file_type, ret; + FILE *f; + + file_type = s1->output_type; + if (file_type == TCC_OUTPUT_OBJ) + mode = 0666; + else + mode = 0777; + unlink(filename); + fd = open(filename, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, mode); + if (fd < 0 || (f = fdopen(fd, "wb")) == NULL) + return tcc_error_noabort("could not write '%s: %s'", filename, strerror(errno)); + if (s1->verbose) + printf("<- %s\n", filename); +#ifdef TCC_TARGET_COFF + if (s1->output_format == TCC_OUTPUT_FORMAT_COFF) + tcc_output_coff(s1, f); + else +#endif +#ifdef TCC_TARGET_YAFF + if (s1->output_format == TCC_OUTPUT_FORMAT_YAFF) + ret = tcc_output_yaff(s1, f, filename); + else +#endif + if (s1->output_format == TCC_OUTPUT_FORMAT_ELF) + ret = tcc_output_elf(s1, f, phnum, phdr); + else + ret = tcc_output_binary(s1, f); + fclose(f); + + return ret; +} + +#ifndef ELF_OBJ_ONLY +/* order sections according to sec_order, remove sections + that we aren't going to output. */ +static void reorder_sections(TCCState *s1, int *sec_order) +{ + int i, nnew, k, *backmap; + Section **snew, *s; + ElfW(Sym) * sym; + + backmap = tcc_malloc(s1->nb_sections * sizeof(backmap[0])); + for (i = 0, nnew = 0, snew = NULL; i < s1->nb_sections; i++) + { + k = sec_order[i]; + s = s1->sections[k]; + if (!i || s->sh_name) + { + backmap[k] = nnew; + dynarray_add(&snew, &nnew, s); + } + else + { + backmap[k] = 0; + /* just remember to free them later */ + dynarray_add(&s1->priv_sections, &s1->nb_priv_sections, s); + } + } + for (i = 1; i < nnew; i++) + { + s = snew[i]; + s->sh_num = i; + if (s->sh_type == SHT_RELX) + s->sh_info = backmap[s->sh_info]; + else if (s->sh_type == SHT_SYMTAB || s->sh_type == SHT_DYNSYM) + for_each_elem(s, 1, sym, ElfW(Sym)) if (sym->st_shndx < s1->nb_sections) sym->st_shndx = backmap[sym->st_shndx]; + } + tcc_free(s1->sections); + s1->sections = snew; + s1->nb_sections = nnew; + tcc_free(backmap); +} + +#ifdef TCC_TARGET_ARM +/* Emit the ARM EABI build-attributes section describing the code we produced. + * Consumers (ld, gdb, objdump) use it to check ABI compatibility between + * objects — most importantly Tag_ABI_VFP_args, which says whether FP arguments + * travel in VFP registers (-mfloat-abi=hard) or GPRs (soft/softfp). + * + * Layout (ABI addenda §2.2): + * 'A' | uint32 vendor_len | "aeabi\0" | 0x01 Tag_File | uint32 sub_len | pairs + * where vendor_len counts everything after the 'A' (including itself) and + * sub_len counts the Tag_File byte, its length field and the attribute pairs. + * Every tag and value we emit is < 128, so each is a one-byte ULEB128. */ +static void create_arm_attribute_section(TCCState *s1) +{ + ArmEabiAttrs a; + arm_get_eabi_attrs(s1, &a); + + unsigned char body[96]; + int n = 0; + + body[n++] = 5; /* Tag_CPU_name (NUL-terminated string) */ + for (const char *p = a.cpu_name; *p; p++) + body[n++] = (unsigned char)*p; + body[n++] = 0; + body[n++] = 6; + body[n++] = (unsigned char)a.cpu_arch; /* Tag_CPU_arch */ + body[n++] = 7; + body[n++] = 'M'; /* Tag_CPU_arch_profile: Microcontroller */ + body[n++] = 9; + body[n++] = 3; /* Tag_THUMB_ISA_use: Yes. No Tag_ARM_ISA_use: + * M-profile has no ARM instruction set. */ + if (a.fp_arch) + { + body[n++] = 10; + body[n++] = (unsigned char)a.fp_arch; /* Tag_FP_arch */ + } + body[n++] = 18; + body[n++] = 4; /* Tag_ABI_PCS_wchar_t: 4 */ + body[n++] = 20; + body[n++] = 1; /* Tag_ABI_FP_denormal: Needed */ + body[n++] = 21; + body[n++] = 1; /* Tag_ABI_FP_exceptions: Needed */ + body[n++] = 23; + body[n++] = 3; /* Tag_ABI_FP_number_model: IEEE 754 */ + body[n++] = 24; + body[n++] = 1; /* Tag_ABI_align_needed: 8-byte */ + body[n++] = 25; + body[n++] = 1; /* Tag_ABI_align_preserved: 8-byte, except leaf SP */ + body[n++] = 26; + body[n++] = 2; /* Tag_ABI_enum_size: int (arm-none-eabi-gcc defaults to + * 'small' instead — a deliberate, declared difference). */ + if (a.hardfp_use) + { + body[n++] = 27; + body[n++] = (unsigned char)a.hardfp_use; /* Tag_ABI_HardFP_use */ + } + if (a.vfp_args) + { + body[n++] = 28; + body[n++] = (unsigned char)a.vfp_args; /* Tag_ABI_VFP_args */ + } + body[n++] = 30; + body[n++] = 6; /* Tag_ABI_optimization_goals: Aggressive Debug */ + body[n++] = 34; + body[n++] = 1; /* Tag_CPU_unaligned_access: v6 */ + + const int sub_len = 1 + 4 + n; + const int vendor_len = 4 + 6 + sub_len; + + Section *attr = new_section(s1, ".ARM.attributes", SHT_ARM_ATTRIBUTES, 0); + attr->sh_addralign = 1; + unsigned char *ptr = section_ptr_add(attr, 1 + vendor_len); + int o = 0; + ptr[o++] = 'A'; + write32le(ptr + o, vendor_len); + o += 4; + memcpy(ptr + o, "aeabi", 6); + o += 6; + ptr[o++] = 1; /* Tag_File */ + write32le(ptr + o, sub_len); + o += 4; + memcpy(ptr + o, body, n); +} +#endif + +#if TARGETOS_OpenBSD || TARGETOS_NetBSD +static Section *create_bsd_note_section(TCCState *s1, const char *name, const char *value) +{ + Section *s = find_section(s1, name); + + if (s->data_offset == 0) + { + char *ptr = section_ptr_add(s, sizeof(ElfW(Nhdr)) + 8 + 4); + ElfW(Nhdr) *note = (ElfW(Nhdr) *)ptr; + + s->sh_type = SHT_NOTE; + note->n_namesz = 8; + note->n_descsz = 4; + note->n_type = ELF_NOTE_OS_GNU; + strcpy(ptr + sizeof(ElfW(Nhdr)), value); + } + return s; +} +#endif + +static void alloc_sec_names(TCCState *s1, int is_obj); + +/* --gc-sections implementation: remove unused sections */ +void gc_sections(TCCState *s1) +{ + int i, sym_index, changed; + Section *s, *sr; + ElfW(Sym) * sym, *symtab; + ElfW_Rel *rel; + unsigned char *sec_used; + int nb_syms; + const char *name; + + /* Allocate array to track which sections are used */ + sec_used = tcc_mallocz(s1->nb_sections); + + /* Always keep certain essential sections */ + for (i = 1; i < s1->nb_sections; i++) + { + s = s1->sections[i]; + if (!s) + continue; + /* Keep symtab, strtab, shstrtab, and relocation sections */ + if (s->sh_type == SHT_SYMTAB || s->sh_type == SHT_STRTAB || s->sh_type == SHT_HASH || s->sh_type == SHT_DYNSYM || + s->sh_type == SHT_GNU_HASH || s->sh_type == SHT_GNU_versym || s->sh_type == SHT_GNU_verneed || + s->sh_type == SHT_GNU_verdef || s->sh_type == SHT_RELX || s->sh_type == SHT_DYNAMIC || s->sh_type == SHT_NOTE) + { + sec_used[i] = 1; + continue; + } + /* Keep init/fini arrays and special sections */ + if (!strcmp(s->name, ".init") || !strcmp(s->name, ".fini") || !strcmp(s->name, ".init_array") || + !strcmp(s->name, ".fini_array") || !strcmp(s->name, ".preinit_array") || !strcmp(s->name, ".ctors") || + !strcmp(s->name, ".dtors") || !strcmp(s->name, ".got") || !strcmp(s->name, ".got.plt") || + !strcmp(s->name, ".plt") || !strcmp(s->name, ".interp") || !strcmp(s->name, ".eh_frame") || + !strcmp(s->name, ".eh_frame_hdr") || !strcmp(s->name, ".ARM.attributes") || !strcmp(s->name, ".ARM.exidx")) + { + sec_used[i] = 1; + continue; + } + /* Keep sections marked with KEEP() in linker script */ + if (s1->ld_script && ld_section_should_keep(s1->ld_script, s->name)) + { + sec_used[i] = 1; + continue; + } + /* Keep debug sections if debugging enabled */ + if (s1->do_debug && !strncmp(s->name, ".debug", 6)) + { + sec_used[i] = 1; + continue; + } + /* Keep sections that are not SHF_ALLOC (like comments) if not allocatable + */ + if (!(s->sh_flags & SHF_ALLOC)) + { + sec_used[i] = 1; + continue; + } + } + + /* Mark sections containing entry point and other root symbols */ + symtab = (ElfW(Sym) *)symtab_section->data; + nb_syms = symtab_section->data_offset / sizeof(ElfW(Sym)); + + for (sym_index = 1; sym_index < nb_syms; sym_index++) + { + sym = &symtab[sym_index]; + if (sym->st_shndx == SHN_UNDEF || sym->st_shndx >= SHN_LORESERVE) + continue; + if (sym->st_shndx >= s1->nb_sections) + continue; + + name = (char *)symtab_section->link->data + sym->st_name; + + /* Mark entry point section */ + if (s1->elf_entryname && name[0] == s1->elf_entryname[0] && !strcmp(name, s1->elf_entryname)) + { + sec_used[sym->st_shndx] = 1; + continue; + } + if ((name[0] == '_' || name[0] == 'm') && + (!strcmp(name, "_start") || !strcmp(name, "main") || !strcmp(name, "_main") || !strcmp(name, "__start"))) + { + sec_used[sym->st_shndx] = 1; + continue; + } + /* Mark global/weak symbols that are exported */ + if (s1->rdynamic && ELFW(ST_BIND)(sym->st_info) != STB_LOCAL) + { + sec_used[sym->st_shndx] = 1; + continue; + } + /* Exported symbols are runtime roots regardless of -rdynamic: under + * -fvisibility=hidden the dynamic-symbol set is exactly the defined + * GLOBAL/WEAK symbols with default visibility, and other yasld modules + * resolve against them at runtime with no relocation in this image to + * witness the use. Collecting one would leave a .dynsym entry pointing + * at reclaimed bytes -- a silent wrong-address at runtime, not a link + * error. Hidden globals and statics stay collectable. */ + if (ELFW(ST_BIND)(sym->st_info) != STB_LOCAL && ELFW(ST_VISIBILITY)(sym->st_other) == STV_DEFAULT) + { + sec_used[sym->st_shndx] = 1; + continue; + } + } + + /* Iteratively mark sections referenced by relocations from used sections */ + do + { + changed = 0; + for (i = 1; i < s1->nb_sections; i++) + { + sr = s1->sections[i]; + if (!sr || sr->sh_type != SHT_RELX) + continue; + /* Get the section this relocation applies to */ + s = s1->sections[sr->sh_info]; + if (!s || !sec_used[sr->sh_info]) + continue; + /* References from non-allocated sections must not confer liveness: + * DWARF describes every function (each .text. is referenced by + * .rel.debug_info via its section symbol), so following these would + * pin the whole program and -g would silently disable GC. Dropped + * targets get the SHN_ABS tombstone in the sweep instead, which is + * how debug info for collected code is conventionally marked. */ + if (!(s->sh_flags & SHF_ALLOC)) + continue; + + /* Iterate through relocations */ + for_each_elem(sr, 0, rel, ElfW_Rel) + { + sym_index = ELFW(R_SYM)(rel->r_info); + if (sym_index == 0 || sym_index >= nb_syms) + continue; + sym = &symtab[sym_index]; + if (sym->st_shndx == SHN_UNDEF || sym->st_shndx >= SHN_LORESERVE) + continue; + if (sym->st_shndx >= s1->nb_sections) + continue; + if (!sec_used[sym->st_shndx]) + { + sec_used[sym->st_shndx] = 1; + changed = 1; + } + } + } + } while (changed); + + /* Also mark relocation sections for used sections */ + for (i = 1; i < s1->nb_sections; i++) + { + s = s1->sections[i]; + if (s && s->reloc && sec_used[i]) + { + sec_used[s->reloc->sh_num] = 1; + } + } + + /* Remove unused sections by zeroing their data */ + { + unsigned char *collected = tcc_mallocz(s1->nb_sections); + int any_collected = 0; + for (i = 1; i < s1->nb_sections; i++) + { + s = s1->sections[i]; + if (!s) + continue; + if (!sec_used[i] && (s->sh_flags & SHF_ALLOC) && s->data_offset > 0) + { + if (s1->verbose) + printf("GC: removing unused section '%s' (%d bytes)\n", s->name, (int)s->data_offset); + /* Zero the section - it will be skipped in output */ + s->data_offset = 0; + s->sh_size = 0; + /* Clearing SHF_ALLOC keeps the shell out of layout AND out of + * coalesce_function_sections, which must not fold a collected + * .text. back into .text. */ + s->sh_flags &= ~SHF_ALLOC; + collected[i] = 1; + any_collected = 1; + if (s->reloc) + { + s->reloc->data_offset = 0; + s->reloc->sh_size = 0; + } + /* Free deferred chunks for lazy sections to save memory */ + if (s->lazy && s->has_deferred_chunks) + { + free_deferred_chunks(s); + s->lazy = 0; + s->has_deferred_chunks = 0; + } + } + } + /* A symbol still pointing into a collected section would resolve to + * (stale base + offset). Exported symbols are rooted above, so only + * locals and hidden globals land here; give them the DWARF tombstone. + * SHN_ABS keeps relocate_syms' hands off the value, and 0xFFFFFFFF is + * unmistakably dead -- 0 is a real code address under -Ttext=0x0, so + * it must not be the marker. */ + if (any_collected) + { + for (sym_index = 1; sym_index < nb_syms; sym_index++) + { + sym = &symtab[sym_index]; + if (sym->st_shndx < s1->nb_sections && collected[sym->st_shndx]) + { + sym->st_value = 0xFFFFFFFF; + sym->st_shndx = SHN_ABS; + } + } + } + tcc_free(collected); + } + + tcc_free(sec_used); +} + +/* Fold every per-function .text.* input section (created under + * -ffunction-sections; unreferenced ones already emptied by gc_sections) + * back into the single text_section before GOT build and layout. + * + * Everything downstream of this point -- sbrel classification, layout, the + * export table and the YAFF writer -- addresses code through the + * text_section global (tccyaff.c writes text_section->data as the entire + * code payload), so coalescing here lets all of it stay ignorant of split + * text. Sections merge in creation order, which is emission order, so the + * resulting layout matches what a non-split build produces. + * + * Symbol fixup relies on two properties verified in this tree: + * - relocate_syms() adds a section base only for sh_num < SHN_LORESERVE, + * so retargeted symbols behave exactly like natively-.text ones; + * - DWARF references code via STT_SECTION symbols + addend + * (tcc_debug_funcstart records dwarf_register_text_section(cur_text_section)), + * so bumping the section symbol's st_value by the merge base keeps every + * debug address correct without touching the DWARF bytes. */ +static void coalesce_function_sections(TCCState *s1) +{ + int i, have_split = 0; + addr_t *base_of; + unsigned char *merged; + ElfW(Sym) * sym; + /* Snapshot the section count: put_elf_reloc below can create .rel.text + * mid-loop, growing s1->nb_sections past the size of the arrays. New + * sections are reloc sections, never merge candidates, so the snapshot + * loses nothing. */ + int nb_sections = s1->nb_sections; + + for (i = 1; i < nb_sections; i++) + { + Section *s = s1->sections[i]; + if (s && s != text_section && s->sh_type == SHT_PROGBITS && (s->sh_flags & SHF_EXECINSTR) && + (s->sh_flags & SHF_ALLOC) && !strncmp(s->name, ".text.", 6)) + { + have_split = 1; + break; + } + } + if (!have_split) + return; + + base_of = tcc_mallocz(sizeof(addr_t) * nb_sections); + merged = tcc_mallocz(nb_sections); + + for (i = 1; i < nb_sections; i++) + { + Section *s = s1->sections[i]; + if (!s || s == text_section || s->sh_type != SHT_PROGBITS) + continue; + if ((s->sh_flags & (SHF_EXECINSTR | SHF_ALLOC)) != (SHF_EXECINSTR | SHF_ALLOC)) + continue; + if (strncmp(s->name, ".text.", 6)) + continue; + + /* Thumb literal pools are PC-relative loads with PC aligned down to 4; + * every function must therefore start 4-aligned, exactly as it would + * have in a monolithic .text. */ + text_section->data_offset += -text_section->data_offset & (addr_t)3; + base_of[i] = text_section->data_offset; + merged[i] = 1; + if (s->data_offset) + { + void *p = section_ptr_add(text_section, s->data_offset); + memcpy(p, s->data, s->data_offset); + } + + /* Move the section's relocations into .rel.text, rebased. Addends live + * in the copied code bytes (REL format), so only r_offset changes; + * symbol indices are global to symtab and stay valid. */ + if (s->reloc && s->reloc->data_offset) + { + Section *sr = s->reloc; + int nb = sr->data_offset / sizeof(ElfW_Rel); + ElfW_Rel *rel = (ElfW_Rel *)sr->data; + int r; + for (r = 0; r < nb; r++) + put_elf_reloc(symtab_section, text_section, rel[r].r_offset + base_of[i], ELFW(R_TYPE)(rel[r].r_info), + ELFW(R_SYM)(rel[r].r_info)); + sr->data_offset = 0; + } + } + + /* Retarget every symbol defined in a merged section: function symbols + * (the thumb LSB tag survives the addition), $t/$d mapping markers, and + * the STT_SECTION symbols DWARF relocates against (st_value 0 -> base). */ + for_each_elem(symtab_section, 1, sym, ElfW(Sym)) + { + if (sym->st_shndx < nb_sections && merged[sym->st_shndx]) + { + sym->st_value += base_of[sym->st_shndx]; + sym->st_shndx = text_section->sh_num; + } + } + + /* Retire the merged shells: no ALLOC flag and no size means + * alloc_sec_names() skips them and they never reach the output. */ + for (i = 1; i < nb_sections; i++) + { + if (merged[i]) + { + Section *s = s1->sections[i]; + s->sh_flags &= ~SHF_ALLOC; + s->data_offset = 0; + s->sh_size = 0; + } + } + + tcc_free(base_of); + tcc_free(merged); +} + +/* Output an elf, coff or binary file */ +/* XXX: suppress unneeded sections */ +static int elf_output_file(TCCState *s1, const char *filename) +{ + int i, ret, file_type, *sec_order; + struct dyn_inf dyninf = {0}; + Section *interp, *dynstr, *dynamic; + int textrel, got_sym, dt_flags_1; + + file_type = s1->output_type; + s1->nb_errors = 0; + ret = -1; + interp = dynstr = dynamic = NULL; + sec_order = NULL; + dyninf.roinf = &dyninf._roinf; + + /* Load linker script if specified */ + if (s1->linker_script) + { + if (tcc_load_linker_script(s1, s1->linker_script) < 0) + return -1; + /* Apply linker script symbols early so they're available for resolution. + * Values for symbols in NOLOAD sections will be updated after layout. */ + ld_apply_symbols(s1, s1->ld_script); + } + +#ifdef TCC_TARGET_ARM + create_arm_attribute_section(s1); +#endif + +#if TARGETOS_OpenBSD + dyninf.note = create_bsd_note_section(s1, ".note.openbsd.ident", "OpenBSD"); +#endif + +#if TARGETOS_NetBSD + dyninf.note = create_bsd_note_section(s1, ".note.netbsd.ident", "NetBSD"); +#endif + +#if TARGETOS_FreeBSD || TARGETOS_NetBSD + dyninf.roinf = NULL; +#endif + /* if linking, also link in runtime libraries (libc, libgcc, etc.) */ + { + unsigned t = s1->do_bench ? tcc_getclock_us() : 0; + tcc_add_runtime(s1); + if (s1->do_bench) + s1->bench_output_runtime_us += tcc_getclock_us() - t; + } + resolve_common_syms(s1); + +#ifdef TCC_TARGET_YAFF + /* Merge .init_array / .fini_array into .data early — before + build_got_entries() — so that the __yaff_initfini symbol uses the + R_RELATIVE (local) GOT path, and relocations pointing into the + merged data are resolved naturally by relocate_sections(). */ + if (s1->output_format == TCC_OUTPUT_FORMAT_YAFF) + tcc_yaff_prepare_init_fini(s1); +#endif + + /* Phase 2: Garbage Collection During Loading - mark and load referenced sections */ + if (s1->gc_sections_aggressive) + { + tcc_gc_mark_phase(s1); + tcc_load_referenced_sections(s1); + tcc_free_lazy_objfiles(s1); + } + + /* Garbage collect unused sections if requested (skip if aggressive GC already ran) */ + if (s1->gc_sections && !s1->gc_sections_aggressive) + { + gc_sections(s1); + } + + /* Per-function .text.* sections (from -ffunction-sections objects) must be + * folded back into text_section before build_got/layout -- see the comment + * on coalesce_function_sections. Runs unconditionally: it is a no-op when + * no split-text input exists. */ + coalesce_function_sections(s1); + + if (!s1->static_link) + { + if (file_type & TCC_OUTPUT_EXE) + { + /* allow override the dynamic loader */ + const char *elfint = getenv("LD_SO"); + if (elfint == NULL) + elfint = DEFAULT_ELFINTERP(s1); + /* add interpreter section only if executable */ + // interp = new_section(s1, ".interp", SHT_PROGBITS, SHF_ALLOC); + // interp->sh_addralign = 1; + // ptr = section_ptr_add(interp, 1 + strlen(elfint)); + // strcpy(ptr, elfint); + dyninf.interp = interp; + } + + /* add dynamic symbol table */ + s1->dynsym = new_symtab(s1, ".dynsym", SHT_DYNSYM, SHF_ALLOC, ".dynstr", ".hash", SHF_ALLOC); + /* Number of local symbols (readelf complains if not set) */ + s1->dynsym->sh_info = 1; + dynstr = s1->dynsym->link; + /* add dynamic section */ + dynamic = new_section(s1, ".dynamic", SHT_DYNAMIC, SHF_ALLOC | SHF_WRITE); + dynamic->link = dynstr; + dynamic->sh_entsize = sizeof(ElfW(Dyn)); + + got_sym = build_got(s1); + if (file_type & TCC_OUTPUT_EXE) + { + bind_exe_dynsyms(s1, file_type & TCC_OUTPUT_DYN); + if (s1->nb_errors) + goto the_end; + } + { + unsigned t = s1->do_bench ? tcc_getclock_us() : 0; + build_got_entries(s1, got_sym); + if (s1->do_bench) + s1->bench_output_got_us += tcc_getclock_us() - t; + } + if (file_type & TCC_OUTPUT_EXE) + { + bind_libs_dynsyms(s1); + } + else + { + /* shared library case: simply export all global symbols */ + export_global_syms(s1); + } +#if TCC_EH_FRAME + /* fill with initial data */ + tcc_eh_frame_hdr(s1, 0); +#endif + dyninf.gnu_hash = create_gnu_hash(s1); + } + else + { + { + unsigned t = s1->do_bench ? tcc_getclock_us() : 0; + build_got_entries(s1, 0); + if (s1->do_bench) + s1->bench_output_got_us += tcc_getclock_us() - t; + } + } + version_add(s1); + + textrel = set_sec_sizes(s1); + + if (!s1->static_link) + { + /* add a list of needed dlls */ + for (i = 0; i < s1->nb_loaded_dlls; i++) + { + DLLReference *dllref = s1->loaded_dlls[i]; + if (dllref->level == 0) + put_dt(dynamic, DT_NEEDED, put_elf_str(dynstr, dllref->name)); + } + + if (s1->rpath) + put_dt(dynamic, s1->enable_new_dtags ? DT_RUNPATH : DT_RPATH, put_elf_str(dynstr, s1->rpath)); + + dt_flags_1 = DF_1_NOW; + if (file_type & TCC_OUTPUT_DYN) + { + if (s1->soname) + put_dt(dynamic, DT_SONAME, put_elf_str(dynstr, s1->soname)); + /* XXX: currently, since we do not handle PIC code, we + must relocate the readonly segments */ + if (textrel) + put_dt(dynamic, DT_TEXTREL, 0); + if (file_type & TCC_OUTPUT_EXE) + dt_flags_1 = DF_1_NOW | DF_1_PIE; + } + put_dt(dynamic, DT_FLAGS, DF_BIND_NOW); + put_dt(dynamic, DT_FLAGS_1, dt_flags_1); + if (s1->symbolic) + put_dt(dynamic, DT_SYMBOLIC, 0); + + dyninf.dynamic = dynamic; + dyninf.dynstr = dynstr; + /* remember offset and reserve space for 2nd call below */ + dyninf.data_offset = dynamic->data_offset; + fill_dynamic(s1, &dyninf); + dynamic->sh_size = dynamic->data_offset; + dynstr->sh_size = dynstr->data_offset; + } + + /* create and fill .shstrtab section */ + alloc_sec_names(s1, 0); + /* this array is used to reorder sections in the output file */ + sec_order = tcc_malloc(sizeof(int) * 2 * s1->nb_sections); + /* compute section to program header mapping */ + { + unsigned t = s1->do_bench ? tcc_getclock_us() : 0; + layout_sections(s1, sec_order, &dyninf); + if (s1->do_bench) + s1->bench_output_layout_us += tcc_getclock_us() - t; + } + + /* Export standard linker symbols after layout (addresses now known) */ + /* Skip if linker script is loaded - it provides its own symbol definitions */ + if (!s1->ld_script) + { + ld_export_standard_symbols(s1); + } + + /* Update and apply linker script symbols with final addresses */ + if (s1->ld_script) + { + ld_update_symbol_values(s1, s1->ld_script); + ld_apply_symbols(s1, s1->ld_script); + + /* Fix p_paddr for LOAD segments of sections with AT > (LMA != VMA). + The boot code copies .data from the LMA (Flash) to the VMA (RAM), + so the ELF loader must place the content at the LMA, not the VMA. */ + if (s1->ld_script->has_loadaddrs) + { + LDScript *ld = s1->ld_script; + int j, k; + for (j = 0; j < ld->nb_output_sections; j++) + { + if (ld->output_sections[j].load_memory_region_idx >= 0 && ld->output_sections[j].memory_region_idx >= 0 && + ld->output_sections[j].load_memory_region_idx != ld->output_sections[j].memory_region_idx) + { + /* This output section has AT > (LMA in different region than VMA). + Find the LOAD segment whose p_vaddr matches and fix p_paddr. */ + addr_t vma = 0; + /* Find the VMA of this output section from its first ELF section */ + for (k = 1; k < s1->nb_sections; k++) + { + Section *sec = s1->sections[k]; + if (sec->sh_addr && ld_section_matches_output(s1, sec->name, j)) + { + vma = sec->sh_addr; + break; + } + } + if (vma) + { + for (k = 0; k < dyninf.phnum; k++) + { + ElfW(Phdr) *ph = &dyninf.phdr[k]; + if (ph->p_type == PT_LOAD && ph->p_vaddr <= vma && vma < ph->p_vaddr + ph->p_memsz) + { + ph->p_paddr = ld->output_section_loadaddrs[j]; + break; + } + } + } + } + } + } + } + + if (dynamic) + { + /* put in GOT the dynamic section address and relocate PLT */ + write32le(s1->got->data, dynamic->sh_addr); + if (file_type == TCC_OUTPUT_EXE || (RELOCATE_DLLPLT && (file_type & TCC_OUTPUT_DYN))) + relocate_plt(s1); + /* relocate symbols in .dynsym now that final addresses are known */ + relocate_syms(s1, s1->dynsym, 2); + } + + /* if building executable or DLL, then relocate each section + except the GOT which is already relocated */ + { + unsigned t = s1->do_bench ? tcc_getclock_us() : 0; + relocate_syms(s1, s1->symtab, 0); + if (s1->nb_errors != 0) + { + if (s1->do_bench) + s1->bench_output_reloc_us += tcc_getclock_us() - t; + goto the_end; + } + relocate_sections(s1); + if (s1->do_bench) + s1->bench_output_reloc_us += tcc_getclock_us() - t; + } + if (dynamic) + { + update_reloc_sections(s1, &dyninf); + dynamic->data_offset = dyninf.data_offset; + fill_dynamic(s1, &dyninf); + } + /* Perform relocation to GOT or PLT entries */ + if (file_type == TCC_OUTPUT_EXE && s1->static_link) + fill_got(s1); + else if (s1->got) + fill_local_got_entries(s1); + + if (dyninf.gnu_hash) + update_gnu_hash(s1, dyninf.gnu_hash); + + reorder_sections(s1, sec_order); +#if TCC_EH_FRAME + /* fill with final data */ + tcc_eh_frame_hdr(s1, 1); +#endif + /* Create the ELF file with name 'filename' */ + { + unsigned t = s1->do_bench ? tcc_getclock_us() : 0; + ret = tcc_write_elf_file(s1, filename, dyninf.phnum, dyninf.phdr); + if (s1->do_bench) + s1->bench_output_write_us += tcc_getclock_us() - t; + } +the_end: + tcc_free(sec_order); + tcc_free(dyninf.phdr); + return ret; +} +#endif /* ndef ELF_OBJ_ONLY */ + +/* Allocate strings for section names */ +static void alloc_sec_names(TCCState *s1, int is_obj) +{ + int i; + Section *s, *strsec; + + strsec = new_section(s1, ".shstrtab", SHT_STRTAB, 0); + put_elf_str(strsec, ""); + for (i = 1; i < s1->nb_sections; i++) + { + s = s1->sections[i]; + if (is_obj) + s->sh_size = s->data_offset; + if (s->sh_size || s == strsec || (s->sh_flags & SHF_ALLOC) || is_obj) + s->sh_name = put_elf_str(strsec, s->name); + } + strsec->sh_size = strsec->data_offset; +} + +/* Output an elf .o file */ +static int elf_output_obj(TCCState *s1, const char *filename) +{ + Section *s; + int i, ret, file_offset; + s1->nb_errors = 0; + /* Allocate strings for section names */ + alloc_sec_names(s1, 1); + file_offset = (sizeof(ElfW(Ehdr)) + 3) & -4; + file_offset += s1->nb_sections * sizeof(ElfW(Shdr)); + for (i = 1; i < s1->nb_sections; i++) + { + s = s1->sections[i]; + file_offset = (file_offset + 15) & -16; + s->sh_offset = file_offset; + if (s->sh_type != SHT_NOBITS) + file_offset += s->sh_size; + } + /* Create the ELF file with name 'filename' */ + { + unsigned t = s1->do_bench ? tcc_getclock_us() : 0; + ret = tcc_write_elf_file(s1, filename, 0, NULL); + if (s1->do_bench) + s1->bench_output_write_us += tcc_getclock_us() - t; + } + return ret; +} + +LIBTCCAPI int tcc_output_file(TCCState *s, const char *filename) +{ + unsigned output_start = 0; + int ret; + + if (s->do_bench) + output_start = tcc_getclock_us(); + + if (s->test_coverage) + tcc_tcov_add_file(s, filename); + if (s->output_type == TCC_OUTPUT_OBJ) + ret = elf_output_obj(s, filename); +#ifdef TCC_TARGET_PE + else + ret = pe_output_file(s, filename); +#elif defined TCC_TARGET_MACHO + else + ret = macho_output_file(s, filename); +#else + else + ret = elf_output_file(s, filename); +#endif + if (s->do_bench) + { + unsigned elapsed = tcc_getclock_us() - output_start; + s->bench_output_time += elapsed; + s->bench_output_count++; + tcc_bench_log(s, "output", filename, elapsed); + } + return ret; +} diff --git a/source/obj/tccelf_priv.h b/source/obj/tccelf_priv.h new file mode 100644 index 000000000..bdbc1a008 --- /dev/null +++ b/source/obj/tccelf_priv.h @@ -0,0 +1,211 @@ +/* + * ELF file handling for TCC + * + * Copyright (c) 2001-2004 Fabrice Bellard + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +/* Declarations shared by the ELF writer/linker's source files (tccelf.c and + * tccelf_{sym,reloc,layout,output,load,ld}.c), which used to be one + * 7.3k-line TU. Each function declared at the bottom is defined in the + * file its comment names. */ + +#ifndef TCCELF_PRIV_H +#define TCCELF_PRIV_H + +#include "tcc.h" +#include "tccld.h" +#include "tccyaff.h" + +/********************************************************/ +/* global variables */ + +/* elf version information */ +struct sym_version +{ + char *lib; + char *version; + int out_index; + int prev_same_lib; +}; + +#define nb_sym_versions s1->nb_sym_versions +#define sym_versions s1->sym_versions +#define nb_sym_to_version s1->nb_sym_to_version +#define sym_to_version s1->sym_to_version +#define dt_verneednum s1->dt_verneednum +#define versym_section s1->versym_section +#define verneed_section s1->verneed_section + +/* special flag to indicate that the section should not be linked to the other + * ones */ +#define SHF_PRIVATE 0x80000000 +/* section is dynsymtab_section */ +#define SHF_DYNSYM 0x40000000 +/* Larger initial ELF hash tables reduce rebuild churn without changing lookup semantics. */ +#define SYMTAB_INITIAL_HASH_BUCKETS 512 + +#if defined(TCC_TARGET_PE) || defined(TCC_TARGET_ARM_THUMB) +#define shf_RELRO SHF_ALLOC +#else +#define shf_RELRO SHF_ALLOC /* eventually made SHF_WRITE in sort_sections() */ +#endif +#ifndef ELF_OBJ_ONLY +/* various data used under elf_output_file() */ +struct dyn_inf +{ + Section *dynamic; + Section *dynstr; + struct + { + /* Info to be copied in dynamic section */ + unsigned long data_offset; + addr_t rel_addr; + addr_t rel_size; + }; + + ElfW(Phdr) * phdr; + int phnum; + int shnum; + Section *interp; + Section *note; + Section *gnu_hash; + + /* read only segment mapping for GNU_RELRO */ + Section _roinf, *roinf; +}; +#endif + +typedef struct SectionMergeInfo +{ + Section *s; /* corresponding existing section */ + unsigned long offset; /* offset of the new section in the existing section */ + uint8_t new_section; /* true if section 's' was added */ + uint8_t link_once; /* true if link once section */ + const char *merged_to; /* canonical name if section was merged */ +} SectionMergeInfo; + +typedef struct ArchiveHeader +{ + char ar_name[16]; /* name of this member */ + char ar_date[12]; /* file mtime */ + char ar_uid[6]; /* owner uid; printed as decimal */ + char ar_gid[6]; /* owner gid; printed as decimal */ + char ar_mode[8]; /* file mode, printed as octal */ + char ar_size[10]; /* file size, printed as decimal */ + char ar_fmag[2]; /* should contain ARFMAG */ +} ArchiveHeader; + +#define ARFMAG "`\n" + +/* Defined in tccelf.c. */ +int should_defer_section(const char *name, int sh_type); +void free_deferred_chunks(Section *sec); +void section_add_deferred(TCCState *s1, Section *sec, const char *path, unsigned long file_off, + unsigned long size, unsigned long dest_off); +ST_FUNC void section_ensure_loaded(TCCState *s1, Section *sec); +int section_write_streaming(TCCState *s1, Section *sec, FILE *f); +ST_FUNC void tcc_free_lazy_objfiles(TCCState *s1); +ST_FUNC int tcc_load_object_file_lazy(TCCState *s1, int fd, unsigned long file_offset); +ST_FUNC void tcc_gc_mark_phase(TCCState *s1); +ST_FUNC void tcc_load_referenced_sections(TCCState *s1); + +/* Defined in tccelf_sym.c. */ +Section *section_ht_find(TCCState *s1, const char *name); +ST_FUNC Section *new_section(TCCState *s1, const char *name, int sh_type, int sh_flags); +ST_FUNC Section *new_symtab(TCCState *s1, const char *symtab_name, int sh_type, int sh_flags, const char *strtab_name, + const char *hash_name, int hash_sh_flags); +ST_FUNC void section_realloc(Section *sec, unsigned long new_size); +ST_FUNC size_t section_add(Section *sec, addr_t size, int align); +ST_FUNC void *section_ptr_add(Section *sec, addr_t size); +#ifndef ELF_OBJ_ONLY +void section_reserve(Section *sec, unsigned long size); +#endif +Section *have_section(TCCState *s1, const char *name); +ST_FUNC Section *find_section(TCCState *s1, const char *name); +ST_FUNC int put_elf_str(Section *s, const char *sym); +ElfW(Word) elf_hash(const unsigned char *name); +ST_FUNC int put_elf_sym(Section *s, addr_t value, unsigned long size, int info, int other, int shndx, const char *name); +ST_FUNC int find_elf_sym(Section *s, const char *name); +ST_FUNC int tcc_dynsym_find(TCCState *s1, const char *name); +ST_FUNC addr_t get_sym_addr(TCCState *s1, const char *name, int err, int forc); +#ifndef ELF_OBJ_ONLY +void version_add(TCCState *s1); +#endif +ST_FUNC int set_elf_sym(Section *s, addr_t value, unsigned long size, int info, int other, int shndx, const char *name); +ST_FUNC void put_elf_reloc(Section *symtab, Section *s, unsigned long offset, int type, int symbol); +ST_FUNC struct sym_attr *get_sym_attr(TCCState *s1, int index, int alloc); +void update_relocs(TCCState *s1, Section *s, int *old_to_new_syms, int first_sym); +ST_FUNC void tcc_elf_sort_syms(TCCState *s1, Section *s); +#ifndef ELF_OBJ_ONLY +Section *create_gnu_hash(TCCState *s1); +void update_gnu_hash(TCCState *s1, Section *gnu_hash); +#endif + +/* Defined in tccelf_reloc.c. */ +ST_FUNC void relocate_syms(TCCState *s1, Section *symtab, int do_resolve); +void free_reloc_patches(Section *s); +ST_FUNC void relocate_sections(TCCState *s1); +#ifndef ELF_OBJ_ONLY +int prepare_dynamic_rel(TCCState *s1, Section *sr); +#endif +#ifdef NEED_BUILD_GOT +int build_got(TCCState *s1); +ST_FUNC void build_got_entries(TCCState *s1, int got_sym); +#endif +ST_FUNC int set_global_sym(TCCState *s1, const char *name, Section *sec, addr_t offs); +void tcc_tcov_add_file(TCCState *s1, const char *filename); +#ifndef TCC_TARGET_PE +ST_FUNC void tcc_add_runtime(TCCState *s1); +#endif +ST_FUNC void resolve_common_syms(TCCState *s1); + +/* Defined in tccelf_layout.c. */ +#ifndef ELF_OBJ_ONLY +ST_FUNC void fill_got(TCCState *s1); +void fill_local_got_entries(TCCState *s1); +void bind_exe_dynsyms(TCCState *s1, int is_PIE); +void bind_libs_dynsyms(TCCState *s1); +void export_global_syms(TCCState *s1); +int set_sec_sizes(TCCState *s1); +int ld_find_output_section_idx(TCCState *s1, const char *name, int *pat_idx); +int ld_section_matches_output(TCCState *s1, const char *name, int os_idx); +int layout_sections(TCCState *s1, int *sec_order, struct dyn_inf *d); +void put_dt(Section *dynamic, int dt, addr_t val); +void fill_dynamic(TCCState *s1, struct dyn_inf *dyninf); +void update_reloc_sections(TCCState *s1, struct dyn_inf *dyninf); +#endif + +/* Defined in tccelf_output.c. */ +#ifndef ELF_OBJ_ONLY +void gc_sections(TCCState *s1); +#endif + +/* Defined in tccelf_load.c. */ +ST_FUNC ssize_t full_read(int fd, void *buf, size_t count); +ST_FUNC void *load_data(int fd, unsigned long file_offset, unsigned long size); +ST_FUNC int tcc_object_type(int fd, ElfW(Ehdr) * h); +ST_FUNC int tcc_group_has_satisfiable_undefs(TCCState *s1); + +/* Defined in tccelf_ld.c. */ +#ifndef ELF_OBJ_ONLY +ST_FUNC int tcc_load_linker_script(TCCState *s1, const char *filename); +void ld_apply_symbols(TCCState *s1, LDScript *ld); +void ld_update_symbol_values(TCCState *s1, LDScript *ld); +ST_FUNC void ld_export_standard_symbols(TCCState *s1); +#endif + +#endif /* TCCELF_PRIV_H */ diff --git a/source/obj/tccelf_reloc.c b/source/obj/tccelf_reloc.c new file mode 100644 index 000000000..742902941 --- /dev/null +++ b/source/obj/tccelf_reloc.c @@ -0,0 +1,1156 @@ +/* + * ELF file handling for TCC + * + * Copyright (c) 2001-2004 Fabrice Bellard + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +/* ELF: relocating symbols and sections, dynamic relocations, the GOT, and + * adding the runtime (crt objects, libtcc1, floating-point library). */ + +#include "tccelf_priv.h" + +static void add_reloc_patch(Section *s, uint32_t offset, uint32_t value); + +/* relocate symbol table, resolve undefined symbols if do_resolve is + true and output error if undefined symbol. */ +ST_FUNC void relocate_syms(TCCState *s1, Section *symtab, int do_resolve) +{ + ElfW(Sym) * sym; + int sym_bind, sh_num; + const char *name; + int sym_idx = 0; + + for_each_elem(symtab, 1, sym, ElfW(Sym)) + { + sym_idx++; + sh_num = sym->st_shndx; + if (sh_num == SHN_UNDEF) + { + if (do_resolve == 2) /* relocating dynsym */ + continue; + /* Validate st_name offset before using it */ + if (sym->st_name >= s1->symtab->link->data_offset) + { + tcc_error_noabort("internal error: symbol %d has invalid st_name offset 0x%x (strtab size: 0x%lx)", sym_idx, + sym->st_name, (unsigned long)s1->symtab->link->data_offset); + continue; + } + name = (char *)s1->symtab->link->data + sym->st_name; + /* Debug: print symbol info when name is empty or looks wrong */ + /* Use ld.so to resolve symbol for us (for tcc -run) */ + if (do_resolve) + { +#if defined TCC_IS_NATIVE && !defined TCC_TARGET_PE + /* dlsym() needs the undecorated name. */ + void *addr = dlsym(RTLD_DEFAULT, &name[s1->leading_underscore]); +#if TARGETOS_OpenBSD || TARGETOS_FreeBSD || TARGETOS_NetBSD || TARGETOS_ANDROID || TARGETOS_YasOS + if (addr == NULL) + { + int i; + for (i = 0; i < s1->nb_loaded_dlls; i++) + if ((addr = dlsym(s1->loaded_dlls[i]->handle, name))) + break; + } +#endif + if (addr) + { + sym->st_value = (addr_t)addr; + LOG_RELOC("relocate_sym: %s -> 0x%lx", name, sym->st_value); + goto found; + } +#endif + /* if dynamic symbol exist, it will be used in relocate_section */ + } + else if (s1->dynsym && find_elf_sym(s1->dynsym, name)) + goto found; + /* XXX: _fp_hw seems to be part of the ABI, so we ignore + it */ + if (!strcmp(name, "_fp_hw")) + goto found; + /* only weak symbols are accepted to be undefined. Their + value is zero */ + sym_bind = ELFW(ST_BIND)(sym->st_info); + if (sym_bind == STB_WEAK) + sym->st_value = 0; + else + tcc_error_noabort("undefined symbol '%s'", name); + } + else if (sh_num < SHN_LORESERVE) + { + /* add section base */ + sym->st_value += s1->sections[sym->st_shndx]->sh_addr; + } + found:; + } +} +/* Add a relocation patch for lazy section streaming. + * Uses dynamic arrays instead of linked list for memory efficiency. + * Each patch is 8 bytes (2 x uint32_t) vs 24 bytes with linked list. */ +static void add_reloc_patch(Section *s, uint32_t offset, uint32_t value) +{ + /* Ensure capacity */ + if (s->nb_reloc_patches >= s->alloc_reloc_patches) + { + int new_alloc = s->alloc_reloc_patches ? s->alloc_reloc_patches * 2 : 16; + s->reloc_patch_offsets = tcc_realloc(s->reloc_patch_offsets, new_alloc * sizeof(uint32_t)); + s->reloc_patch_values = tcc_realloc(s->reloc_patch_values, new_alloc * sizeof(uint32_t)); + s->alloc_reloc_patches = new_alloc; + } + /* Append patch */ + s->reloc_patch_offsets[s->nb_reloc_patches] = offset; + s->reloc_patch_values[s->nb_reloc_patches] = value; + s->nb_reloc_patches++; +} + +/* Free all relocation patches for a section */ +void free_reloc_patches(Section *s) +{ + tcc_free(s->reloc_patch_offsets); + tcc_free(s->reloc_patch_values); + s->reloc_patch_offsets = NULL; + s->reloc_patch_values = NULL; + s->nb_reloc_patches = 0; + s->alloc_reloc_patches = 0; +} + +/* relocate a given section (CPU dependent) by applying the relocations + in the associated relocation section */ +static void relocate_section(TCCState *s1, Section *s, Section *sr) +{ + ElfW_Rel *rel; + ElfW(Sym) * sym; + int type, sym_index; + unsigned char *ptr; + addr_t tgt, addr; + int is_dwarf = s->sh_num >= s1->dwlo && s->sh_num < s1->dwhi; + + /* Always materialize non-debug sections */ + if (!is_dwarf) + section_ensure_loaded(s1, s); + + section_ensure_loaded(s1, sr); + section_ensure_loaded(s1, symtab_section); + + /* For lazy debug sections, we store patches instead of materializing */ + if (is_dwarf && s->lazy && !s->materialized) + { + for_each_elem(sr, 0, rel, ElfW_Rel) + { + sym_index = ELFW(R_SYM)(rel->r_info); + sym = &((ElfW(Sym) *)symtab_section->data)[sym_index]; + type = ELFW(R_TYPE)(rel->r_info); + tgt = sym->st_value; +#if SHT_RELX == SHT_RELA + tgt += rel->r_addend; +#endif + if (type == R_DATA_32DW) + { + uint32_t value; + if (sym->st_shndx >= s1->dwlo && sym->st_shndx < s1->dwhi) + /* dwarf-to-dwarf section relocation (e.g., .debug_info -> .debug_str) */ + value = tgt - s1->sections[sym->st_shndx]->sh_addr; + else + /* code/data reference from debug section (e.g., DW_AT_low_pc -> .text) */ + value = tgt; + add_reloc_patch(s, (uint32_t)rel->r_offset, value); + } + } + return; + } + + qrel = (ElfW_Rel *)sr->data; + + for_each_elem(sr, 0, rel, ElfW_Rel) + { + ptr = s->data + rel->r_offset; + sym_index = ELFW(R_SYM)(rel->r_info); + sym = &((ElfW(Sym) *)symtab_section->data)[sym_index]; + type = ELFW(R_TYPE)(rel->r_info); + tgt = sym->st_value; +#if SHT_RELX == SHT_RELA + tgt += rel->r_addend; +#endif + if (is_dwarf && type == R_DATA_32DW && sym->st_shndx >= s1->dwlo && sym->st_shndx < s1->dwhi) + { + /* dwarf section relocation to each other */ + add32le(ptr, tgt - s1->sections[sym->st_shndx]->sh_addr); + continue; + } + addr = s->sh_addr + rel->r_offset; + relocate(s1, rel, type, ptr, addr, tgt); + } +#ifndef ELF_OBJ_ONLY + /* if the relocation is allocated, we change its symbol table */ + if (sr->sh_flags & SHF_ALLOC) + { + sr->link = s1->dynsym; + if (s1->output_type & TCC_OUTPUT_DYN) + { + size_t r = (uint8_t *)qrel - sr->data; + sr->data_offset = sr->sh_size = r; +#ifdef CONFIG_TCC_PIE + if (r && (s->sh_flags & SHF_EXECINSTR)) + tcc_warning("%d relocations to %s", (unsigned)(r / sizeof *qrel), s->name); +#endif + } + } +#endif +} + +/* relocate all sections */ +ST_FUNC void relocate_sections(TCCState *s1) +{ + int i; + Section *s, *sr; + + for (i = 1; i < s1->nb_sections; ++i) + { + sr = s1->sections[i]; + if (sr->sh_type != SHT_RELX) + continue; + s = s1->sections[sr->sh_info]; +#ifdef TCC_TARGET_ARM + /* Skip relocations for suppressed ARM exception index sections. + set_sec_sizes() clears SHF_ALLOC on .ARM.exidx (stack unwinding + not used on bare-metal), but relocation sections survive. If we + still process them, R_ARM_PREL31 entries that reference orphan + .ARM.extab (placed far away in RAM) overflow the 31-bit range. */ + if (s->sh_type == SHT_ARM_EXIDX && !(s->sh_flags & SHF_ALLOC)) + continue; +#endif +#ifndef TCC_TARGET_MACHO + if (s != s1->got || s1->static_link || s1->output_type == TCC_OUTPUT_MEMORY) +#endif + { + relocate_section(s1, s, sr); + } +#ifndef ELF_OBJ_ONLY + if (sr->sh_flags & SHF_ALLOC) + { + ElfW_Rel *rel; + /* relocate relocation table in 'sr' */ + for_each_elem(sr, 0, rel, ElfW_Rel) rel->r_offset += s->sh_addr; + } +#endif + } +} + +#ifndef ELF_OBJ_ONLY +/* count the number of dynamic relocations so that we can reserve + their space */ +int prepare_dynamic_rel(TCCState *s1, Section *sr) +{ + int count = 0; +#if defined(TCC_TARGET_I386) || defined(TCC_TARGET_X86_64) || defined(TCC_TARGET_ARM) || defined(TCC_TARGET_ARM64) || \ + defined(TCC_TARGET_RISCV64) + ElfW_Rel *rel; + for_each_elem(sr, 0, rel, ElfW_Rel) + { + int sym_index = ELFW(R_SYM)(rel->r_info); + int type = ELFW(R_TYPE)(rel->r_info); + switch (type) + { +#if defined(TCC_TARGET_I386) + case R_386_32: + if (!get_sym_attr(s1, sym_index, 0)->dyn_index && + ((ElfW(Sym) *)symtab_section->data + sym_index)->st_shndx == SHN_UNDEF) + { + /* don't fixup unresolved (weak) symbols */ + rel->r_info = ELFW(R_INFO)(sym_index, R_386_RELATIVE); + break; + } +#elif defined(TCC_TARGET_X86_64) + case R_X86_64_32: + case R_X86_64_32S: + case R_X86_64_64: +#elif defined(TCC_TARGET_ARM) + case R_ARM_ABS32: + case R_ARM_TARGET1: +#elif defined(TCC_TARGET_ARM64) + case R_AARCH64_ABS32: + case R_AARCH64_ABS64: +#elif defined(TCC_TARGET_RISCV64) + case R_RISCV_32: + case R_RISCV_64: +#endif + count++; + break; +#if defined(TCC_TARGET_I386) + case R_386_PC32: +#elif defined(TCC_TARGET_X86_64) + case R_X86_64_PC32: + { + ElfW(Sym) *sym = &((ElfW(Sym) *)symtab_section->data)[sym_index]; + /* Hidden defined symbols can and must be resolved locally. + We're misusing a PLT32 reloc for this, as that's always + resolved to its address even in shared libs. */ + if (sym->st_shndx != SHN_UNDEF && ELFW(ST_VISIBILITY)(sym->st_other) == STV_HIDDEN) + { + rel->r_info = ELFW(R_INFO)(sym_index, R_X86_64_PLT32); + break; + } + } +#elif defined(TCC_TARGET_ARM64) + case R_AARCH64_PREL32: +#endif + if (s1->output_type != TCC_OUTPUT_DLL) + break; + if (get_sym_attr(s1, sym_index, 0)->dyn_index) + count++; + break; + default: + break; + } + } +#endif + return count; +} +#endif + +#ifdef NEED_BUILD_GOT +int build_got(TCCState *s1) +{ + /* if no got, then create it */ + s1->got = new_section(s1, ".got", SHT_PROGBITS, SHF_ALLOC | SHF_WRITE); + s1->got->sh_entsize = 8; + /* keep space for _DYNAMIC pointer and two dummy got entries */ +#if defined(TCC_TARGET_YASOS) + /* + a reserved slot (index YAFF_RODATA_ANCHOR_GOT_INDEX = 3) holding the + * runtime base of the shared .rodata segment. Reserved up front so its GOT + * offset is a compile-time constant the codegen addresses as [R9,#24], + * independent of the final GOT layout; the loader fills it (the YAFF writer + * emits its relocation when -share-rodata is active). */ + section_ptr_add(s1->got, 4 * PTR_SIZE * 2); +#else + section_ptr_add(s1->got, 3 * PTR_SIZE * 2); +#endif + return set_elf_sym(symtab_section, 0, 0, ELFW(ST_INFO)(STB_GLOBAL, STT_OBJECT), 0, s1->got->sh_num, + "_GLOBAL_OFFSET_TABLE_"); +} + +/* Create a GOT and (for function call) a PLT entry corresponding to a symbol + in s1->symtab. When creating the dynamic symbol table entry for the GOT + relocation, use 'size' and 'info' for the corresponding symbol metadata. + Returns the offset of the GOT or (if any) PLT entry. */ +static struct sym_attr *put_got_entry(TCCState *s1, int dyn_reloc_type, int sym_index) +{ + int need_plt_entry; + const char *name; + ElfW(Sym) * sym; + struct sym_attr *attr; + unsigned got_offset; + char plt_name[200]; + int len; + Section *s_rel; + + need_plt_entry = (dyn_reloc_type == R_JMP_SLOT); + attr = get_sym_attr(s1, sym_index, 1); + + /* In case a function is both called and its address taken 2 GOT entries + are created, one for taking the address (GOT) and the other for the PLT + entry (PLTGOT). */ + if (need_plt_entry ? attr->plt_offset : attr->got_offset) + return attr; + + s_rel = s1->got; + if (need_plt_entry) + { + if (!s1->plt) + { + s1->plt = new_section(s1, ".plt", SHT_PROGBITS, SHF_ALLOC | SHF_EXECINSTR); + s1->plt->sh_entsize = 4; + } + s_rel = s1->plt; + } + + /* create the GOT entry */ + got_offset = s1->got->data_offset; + section_ptr_add(s1->got, PTR_SIZE * 2); + + /* Create the GOT relocation that will insert the address of the object or + function of interest in the GOT entry. This is a static relocation for + memory output (dlsym will give us the address of symbols) and dynamic + relocation otherwise (executable and DLLs). The relocation should be + done lazily for GOT entry with *_JUMP_SLOT relocation type (the one + associated to a PLT entry) but is currently done at load time for an + unknown reason. */ + + sym = &((ElfW(Sym) *)symtab_section->data)[sym_index]; + name = (char *)symtab_section->link->data + sym->st_name; + // printf("sym %d %s\n", need_plt_entry, name); + + if (s1->dynsym) + { + if (ELFW(ST_BIND)(sym->st_info) == STB_LOCAL) + { + /* Hack alarm. We don't want to emit dynamic symbols + and symbol based relocs for STB_LOCAL symbols, but rather + want to resolve them directly. At this point the symbol + values aren't final yet, so we must defer this. We will later + have to create a RELATIVE reloc anyway, so we misuse the + relocation slot to smuggle the symbol reference until + fill_local_got_entries. Not that the sym_index is + relative to symtab_section, not s1->dynsym! Nevertheless + we use s1->dyn_sym so that if this is the first call + that got->reloc is correctly created. Also note that + RELATIVE relocs are not normally created for the .got, + so the types serves as a marker for later (and is retained + also for the final output, which is okay because then the + got is just normal data). */ + put_elf_reloc(s1->dynsym, s1->got, got_offset, R_RELATIVE, sym_index); + } + else + { + if (0 == attr->dyn_index) + attr->dyn_index = set_elf_sym(s1->dynsym, sym->st_value, sym->st_size, sym->st_info, 0, sym->st_shndx, name); + put_elf_reloc(s1->dynsym, s_rel, got_offset, dyn_reloc_type, attr->dyn_index); + } + } + else + { + put_elf_reloc(symtab_section, s1->got, got_offset, dyn_reloc_type, sym_index); + } + + if (need_plt_entry) + { + attr->plt_offset = create_plt_entry(s1, got_offset, attr); + + /* create a symbol 'sym@plt' for the PLT jump vector */ + len = strlen(name); + if (len > sizeof plt_name - 5) + len = sizeof plt_name - 5; + memcpy(plt_name, name, len); + strcpy(plt_name + len, "@plt"); + attr->plt_sym = + put_elf_sym(s1->symtab, attr->plt_offset, 0, ELFW(ST_INFO)(STB_GLOBAL, STT_FUNC), 0, s1->plt->sh_num, plt_name); + } + else + { + attr->got_offset = got_offset; + } + + return attr; +} + +/* build GOT and PLT entries */ +/* Two passes because R_JMP_SLOT should become first. Some targets + (arm, arm64) do not allow mixing R_JMP_SLOT and R_GLOB_DAT. */ +ST_FUNC void build_got_entries(TCCState *s1, int got_sym) +{ + Section *s; + ElfW_Rel *rel; + ElfW(Sym) * sym; + int i, type, gotplt_entry, reloc_type, sym_index; + struct sym_attr *attr; + int pass = 0; +redo: + for (i = 1; i < s1->nb_sections; i++) + { + s = s1->sections[i]; + if (s->sh_type != SHT_RELX) + continue; + /* no need to handle got relocations */ + if (s->link != symtab_section) + continue; + /* Relocations from non-allocated sections (DWARF) never execute at + * runtime and must not fabricate GOT/PLT entries. Besides being pure + * GOT bloat, under --gc-sections they reference the SHN_ABS 0xFFFFFFFF + * tombstones of collected code, and AUTO_GOTPLT_ENTRY would turn each + * into an out-of-range GOT slot that the yasld loader refuses to map + * (boot-time "Can't find section for GOT[n]: OffsetOutOfRange"). */ + if (s->sh_info < s1->nb_sections && s1->sections[s->sh_info] && + !(s1->sections[s->sh_info]->sh_flags & SHF_ALLOC)) + continue; + for_each_elem(s, 0, rel, ElfW_Rel) + { + type = ELFW(R_TYPE)(rel->r_info); + gotplt_entry = gotplt_entry_type(type); + if (gotplt_entry == -1) + { + tcc_error_noabort("Unknown relocation type for got: %d", type); + continue; + } + sym_index = ELFW(R_SYM)(rel->r_info); + sym = &((ElfW(Sym) *)symtab_section->data)[sym_index]; + + if (gotplt_entry == NO_GOTPLT_ENTRY) + { + continue; + } + + /* Automatically create PLT/GOT [entry] if it is an undefined + reference (resolved at runtime), or the symbol is absolute, + probably created by tcc_add_symbol, and thus on 64-bit + targets might be too far from application code. */ + if (gotplt_entry == AUTO_GOTPLT_ENTRY) + { + if (sym->st_shndx == SHN_UNDEF) + { + ElfW(Sym) * esym; + int dynindex; + if (!PCRELATIVE_DLLPLT && (s1->output_type & TCC_OUTPUT_DYN)) + continue; + /* Relocations for UNDEF symbols would normally need + to be transferred into the executable or shared object. + If that were done AUTO_GOTPLT_ENTRY wouldn't exist. + But TCC doesn't do that (at least for exes), so we + need to resolve all such relocs locally. And that + means PLT slots for functions in DLLs and COPY relocs for + data symbols. COPY relocs were generated in + bind_exe_dynsyms (and the symbol adjusted to be defined), + and for functions we were generated a dynamic symbol + of function type. */ + if (s1->dynsym) + { + /* dynsym isn't set for -run :-/ */ + dynindex = get_sym_attr(s1, sym_index, 0)->dyn_index; + esym = (ElfW(Sym) *)s1->dynsym->data + dynindex; + if (dynindex && (ELFW(ST_TYPE)(esym->st_info) == STT_FUNC || + (ELFW(ST_TYPE)(esym->st_info) == STT_NOTYPE && ELFW(ST_TYPE)(sym->st_info) == STT_FUNC))) + goto jmp_slot; + } + } + else if (sym->st_shndx == SHN_ABS) + { + if (sym->st_value == 0) /* from tcc_add_btstub() */ + continue; +#ifndef TCC_TARGET_ARM + if (PTR_SIZE != 8) + continue; +#endif + /* from tcc_add_symbol(): on 64 bit platforms these + need to go through .got */ + } + else + continue; + } + +#ifdef TCC_TARGET_X86_64 + if ((type == R_X86_64_PLT32 || type == R_X86_64_PC32) && sym->st_shndx != SHN_UNDEF && + (ELFW(ST_VISIBILITY)(sym->st_other) != STV_DEFAULT || ELFW(ST_BIND)(sym->st_info) == STB_LOCAL || + s1->output_type & TCC_OUTPUT_EXE)) + { + if (pass != 0) + continue; + rel->r_info = ELFW(R_INFO)(sym_index, R_X86_64_PC32); + continue; + } +#endif + reloc_type = code_reloc(type); + if (reloc_type == -1) + { + tcc_error_noabort("Unknown relocation type: %d", type); + continue; + } + + if (reloc_type != 0) + { + jmp_slot: + if (pass != 0) + continue; + reloc_type = R_JMP_SLOT; + } + else + { + if (pass != 1) + continue; + reloc_type = R_GLOB_DAT; + } + + if (!s1->got) + got_sym = build_got(s1); + + if (gotplt_entry == BUILD_GOT_ONLY) + continue; + + attr = put_got_entry(s1, reloc_type, sym_index); + + if (reloc_type == R_JMP_SLOT) + rel->r_info = ELFW(R_INFO)(attr->plt_sym, type); + } + } + if (++pass < 2) + goto redo; + /* .rel.plt refers to .got actually */ + if (s1->plt && s1->plt->reloc) + s1->plt->reloc->sh_info = s1->got->sh_num; + if (got_sym) /* set size */ + ((ElfW(Sym) *)symtab_section->data)[got_sym].st_size = s1->got->data_offset; +} +#endif /* def NEED_BUILD_GOT */ + +ST_FUNC int set_global_sym(TCCState *s1, const char *name, Section *sec, addr_t offs) +{ + int shn = sec ? sec->sh_num : offs || !name ? SHN_ABS : SHN_UNDEF; + if (sec && offs == -1) + offs = sec->data_offset; + return set_elf_sym(symtab_section, offs, 0, ELFW(ST_INFO)(name ? STB_GLOBAL : STB_LOCAL, STT_NOTYPE), 0, shn, name); +} + +static void add_init_array_defines(TCCState *s1, const char *section_name) +{ + Section *s; + addr_t end_offset; + char buf[1024]; + s = have_section(s1, section_name); + if (!s || !(s->sh_flags & SHF_ALLOC)) + { + end_offset = 0; + s = text_section; + } + else + { + end_offset = s->data_offset; + } + snprintf(buf, sizeof(buf), "__%s_start", section_name + 1); + set_global_sym(s1, buf, s, 0); + snprintf(buf, sizeof(buf), "__%s_end", section_name + 1); + set_global_sym(s1, buf, s, end_offset); +} + +ST_FUNC void add_array(TCCState *s1, const char *sec, int c) +{ + Section *s; + s = find_section(s1, sec); + s->sh_flags = shf_RELRO; + s->sh_type = sec[1] == 'i' ? SHT_INIT_ARRAY : SHT_FINI_ARRAY; + put_elf_reloc(s1->symtab, s, s->data_offset, R_DATA_PTR, c); + section_ptr_add(s, PTR_SIZE); +} + +#ifdef CONFIG_TCC_BCHECK +ST_FUNC void tcc_add_bcheck(TCCState *s1) +{ + if (0 == s1->do_bounds_check) + return; + section_ptr_add(bounds_section, sizeof(addr_t)); +} +#endif + +/* set symbol to STB_LOCAL and resolve. The point is to not export it as + a dynamic symbol to allow so's to have one each with a different value. */ +static void set_local_sym(TCCState *s1, const char *name, Section *s, int offset) +{ + int c = find_elf_sym(s1->symtab, name); + if (c) + { + ElfW(Sym) *esym = (ElfW(Sym) *)s1->symtab->data + c; + esym->st_info = ELFW(ST_INFO)(STB_LOCAL, STT_NOTYPE); + esym->st_value = offset; + esym->st_shndx = s->sh_num; + } +} + +/* avoid generating debug/test_coverage code for stub functions */ +static void tcc_compile_string_no_debug(TCCState *s, const char *str) +{ + int save_do_debug = s->do_debug; + int save_test_coverage = s->test_coverage; + + s->do_debug = 0; + s->test_coverage = 0; + tcc_compile_string(s, str); + s->do_debug = save_do_debug; + s->test_coverage = save_test_coverage; +} + +#ifdef CONFIG_TCC_BACKTRACE +static void put_ptr(TCCState *s1, Section *s, int offs) +{ + int c; + c = set_global_sym(s1, NULL, s, offs); + s = data_section; + put_elf_reloc(s1->symtab, s, s->data_offset, R_DATA_PTR, c); + section_ptr_add(s, PTR_SIZE); +} + +ST_FUNC void tcc_add_btstub(TCCState *s1) +{ + Section *s; + int n, o, *p; + CString cstr; + const char *__rt_info = &"___rt_info"[!s1->leading_underscore]; + + s = data_section; + /* Align to PTR_SIZE */ + section_ptr_add(s, -s->data_offset & (PTR_SIZE - 1)); + o = s->data_offset; + /* create a struct rt_context (see tccrun.c) */ + if (s1->dwarf) + { + put_ptr(s1, dwarf_line_section, 0); + put_ptr(s1, dwarf_line_section, -1); + if (s1->dwarf >= 5) + put_ptr(s1, dwarf_line_str_section, 0); + else + put_ptr(s1, dwarf_str_section, 0); + } + else + { + /* stabs removed - emit zeroes as placeholder */ + section_ptr_add(s, 3 * PTR_SIZE); + } + + /* skip esym_start/esym_end/elf_str (not loaded) */ + section_ptr_add(s, 3 * PTR_SIZE); + + if (s1->output_type == TCC_OUTPUT_MEMORY && 0 == s1->dwarf) + { + put_ptr(s1, text_section, 0); + } + else + { + /* prog_base : local nameless symbol with offset 0 at SHN_ABS */ + put_ptr(s1, NULL, 0); +#if defined TCC_TARGET_MACHO + /* adjust for __PAGEZERO */ + if (s1->dwarf == 0 && s1->output_type == TCC_OUTPUT_EXE) + write64le(data_section->data + data_section->data_offset - PTR_SIZE, (uint64_t)1 << 32); +#endif + } + n = 3 * PTR_SIZE; +#ifdef CONFIG_TCC_BCHECK + if (s1->do_bounds_check) + { + put_ptr(s1, bounds_section, 0); + n -= PTR_SIZE; + } +#endif + section_ptr_add(s, n); + p = section_ptr_add(s, 2 * sizeof(int)); + p[0] = s1->rt_num_callers; + p[1] = s1->dwarf; + // if (s->data_offset - o != 10*PTR_SIZE + 2*sizeof (int)) exit(99); + + if (s1->output_type == TCC_OUTPUT_MEMORY) + { + set_global_sym(s1, __rt_info, s, o); + return; + } + + cstr_new(&cstr); + cstr_printf(&cstr, "extern void __bt_init(),__bt_exit(),__bt_init_dll();" + "static void *__rt_info[];" + "__attribute__((constructor)) static void __bt_init_rt(){"); +#ifdef TCC_TARGET_PE + if (s1->output_type == TCC_OUTPUT_DLL) +#ifdef CONFIG_TCC_BCHECK + cstr_printf(&cstr, "__bt_init_dll(%d);", s1->do_bounds_check); +#else + cstr_printf(&cstr, "__bt_init_dll(0);"); +#endif +#endif + cstr_printf(&cstr, "__bt_init(__rt_info,%d);}", s1->output_type != TCC_OUTPUT_DLL); + /* In case dlcose is called by application */ + cstr_printf(&cstr, "__attribute__((destructor)) static void __bt_exit_rt(){" + "__bt_exit(__rt_info);}"); + tcc_compile_string_no_debug(s1, cstr.data); + cstr_free(&cstr); + set_local_sym(s1, __rt_info, s, o); +} +#endif /* def CONFIG_TCC_BACKTRACE */ +void tcc_tcov_add_file(TCCState *s1, const char *filename) +{ + CString cstr; + void *ptr; + char wd[1024]; + + if (tcov_section == NULL) + return; + section_ptr_add(tcov_section, 1); + write32le(tcov_section->data, tcov_section->data_offset); + + cstr_new(&cstr); + if (filename[0] == '/') + cstr_printf(&cstr, "%s.tcov", filename); + else + { + getcwd(wd, sizeof(wd)); + cstr_printf(&cstr, "%s/%s.tcov", wd, filename); + } + ptr = section_ptr_add(tcov_section, cstr.size + 1); + strcpy((char *)ptr, cstr.data); + unlink((char *)ptr); +#ifdef _WIN32 + normalize_slashes((char *)ptr); +#endif + cstr_free(&cstr); + + cstr_new(&cstr); + cstr_printf(&cstr, "extern char *__tcov_data[];" + "extern void __store_test_coverage ();" + "__attribute__((destructor)) static void __tcov_exit() {" + "__store_test_coverage(__tcov_data);" + "}"); + tcc_compile_string_no_debug(s1, cstr.data); + cstr_free(&cstr); + set_local_sym(s1, &"___tcov_data"[!s1->leading_underscore], tcov_section, 0); +} + +#if !defined TCC_TARGET_PE && !defined TCC_TARGET_MACHO +/* add libc crt1/crti objects */ +ST_FUNC void tccelf_add_crtbegin(TCCState *s1) +{ +#if TARGETOS_OpenBSD + if (s1->output_type != TCC_OUTPUT_DLL) + tcc_add_crt(s1, "crt0.o"); + if (s1->output_type == TCC_OUTPUT_DLL) + tcc_add_crt(s1, "crtbeginS.o"); + else + tcc_add_crt(s1, "crtbegin.o"); +#elif TARGETOS_FreeBSD || TARGETOS_NetBSD + if (s1->output_type != TCC_OUTPUT_DLL) +#if TARGETOS_FreeBSD + tcc_add_crt(s1, "crt1.o"); +#else + tcc_add_crt(s1, "crt0.o"); +#endif + tcc_add_crt(s1, "crti.o"); + if (s1->static_link) + tcc_add_crt(s1, "crtbeginT.o"); + else if (s1->output_type == TCC_OUTPUT_DLL) + tcc_add_crt(s1, "crtbeginS.o"); + else + tcc_add_crt(s1, "crtbegin.o"); +#elif TARGETOS_ANDROID + if (s1->output_type == TCC_OUTPUT_DLL) + tcc_add_crt(s1, "crtbegin_so.o"); + else + tcc_add_crt(s1, "crtbegin_dynamic.o"); +#else + if (s1->output_type != TCC_OUTPUT_DLL) + tcc_add_crt(s1, "crt1.o"); + tcc_add_crt(s1, "crti.o"); +#endif +} + +ST_FUNC void tccelf_add_crtend(TCCState *s1) +{ +#if TARGETOS_OpenBSD + if (s1->output_type == TCC_OUTPUT_DLL) + tcc_add_crt(s1, "crtendS.o"); + else + tcc_add_crt(s1, "crtend.o"); +#elif TARGETOS_FreeBSD || TARGETOS_NetBSD + if (s1->output_type == TCC_OUTPUT_DLL) + tcc_add_crt(s1, "crtendS.o"); + else + tcc_add_crt(s1, "crtend.o"); + tcc_add_crt(s1, "crtn.o"); +#elif TARGETOS_ANDROID + if (s1->output_type == TCC_OUTPUT_DLL) + tcc_add_crt(s1, "crtend_so.o"); + else + tcc_add_crt(s1, "crtend_android.o"); +#else + tcc_add_crt(s1, "crtn.o"); +#endif +} +#endif /* !defined TCC_TARGET_PE && !defined TCC_TARGET_MACHO */ + +#if defined TCC_TARGET_ARM +/* Determine the FP library base name (without lib prefix / extension). + * Same name is used for both static (.a) and shared (.so) variants. + * Returns the name for use with tcc_add_library() or manual path construction. + * + * Returns NULL if hard float ABI (no FP library needed). + */ +ST_FUNC const char *tccelf_get_fp_lib_name(TCCState *s1) +{ + if (s1->float_abi == ARM_HARD_FLOAT) + return NULL; + + /* -mfloat-abi=soft means "no FP instructions anywhere in this image", and the + * runtime the image links is part of the image. Pairing it with a hardware + * -mfpu used to be the only way to say "hardware, behind a call" and so + * quietly linked librp2350fp -- DCP instructions, on a build that had just + * declared it emits none, with a YAFF header that asked the loader for + * nothing. That mode now has its own flag (-mfp-inline=none), which leaves + * this one free to mean what it says. */ + if (s1->float_abi == ARM_SOFT_FLOAT) + return "softfp"; + + if (s1->fpu_type) + { + switch (s1->fpu_type) + { + case ARM_FPU_VFPV4: + case ARM_FPU_FPV4_SP_D16: + case ARM_FPU_FPV5_SP_D16: + return "vfpv4sp"; + case ARM_FPU_RP2350: + /* RP2350 double coprocessor: doubles go through the DCP sequences in + * librp2350fp rather than the generic soft-float implementations. */ + return "rp2350fp"; + case ARM_FPU_FPV5_D16: + case ARM_FPU_NEON: + case ARM_FPU_NEON_VFPV4: + case ARM_FPU_NEON_FP_ARMV8: + return "vfpv5dp"; + default: + break; + } + } + + return "softfp"; +} + +/* Add ARM floating-point library based on compiler flags. + * Selects the correct FP library variant based on -mfpu and -mfloat-abi. + * + * Library names follow the short convention: + * libsoftfp.{a,so} - Pure software floating point + * libvfpv4sp.{a,so} - VFPv4 single-precision HW, double SW + * libvfpv5dp.{a,so} - VFPv5 full double-precision HW + * librp2350fp.{a,so} - RP2350 double coprocessor + * + * On YasOS native (dynamic linking): uses tcc_add_library() which searches + * library_paths for libsoftfp.so in /usr/lib/, etc. + * + * On cross-compiler / static: uses tcc_add_dll() with "fp/libsoftfp.a" path, + * resolved relative to tcc_lib_path (CONFIG_TCCDIR), e.g. lib/tcc/fp/. + * + * Hard float ABI needs no FP library (uses raw FP instructions). + */ +/* Is the __aeabi_ runtime bound as a shared object rather than copied in from + * an archive? -mfp-lib decides; AUTO keeps the historical split, where only + * the on-device compiler binds the shared object. */ +ST_FUNC int tccelf_arm_fp_lib_is_shared(TCCState *s1) +{ + if (s1->static_link) + return 0; + if (s1->fp_lib == ARM_FP_LIB_SHARED) + return 1; + if (s1->fp_lib == ARM_FP_LIB_STATIC) + return 0; +#if TARGETOS_YasOS && defined(TCC_IS_NATIVE) + return 1; +#else + return 0; +#endif +} + +ST_FUNC void tccelf_add_arm_fp_lib(TCCState *s1) +{ + const char *fp_lib = tccelf_get_fp_lib_name(s1); + if (!fp_lib) + { + if (s1->verbose) + printf("Hard float ABI: no FP library needed\n"); + return; + } + + if (tccelf_arm_fp_lib_is_shared(s1)) + { + /* Search library_paths for libXXX.so then libXXX.a (e.g. + * /usr/lib/libsoftfp.so), so the __aeabi_ entry points are resolved by the + * OS's dynamic loader at exec instead of copied into this module. */ + if (s1->verbose) + printf("Adding ARM FP shared library: lib%s\n", fp_lib); + tcc_add_library(s1, fp_lib); + return; + } + + /* Cross-compiler or static link: find via tcc lib path (fp/ subdirectory). + * tcc_add_dll() searches library_paths which includes {B} = tcc_lib_path, + * so "fp/libsoftfp.a" resolves to e.g. lib/tcc/fp/libsoftfp.a */ + { + char fp_path[64]; + snprintf(fp_path, sizeof(fp_path), "fp/lib%s.a", fp_lib); + if (s1->verbose) + printf("Adding ARM FP library: %s\n", fp_path); + tcc_add_dll(s1, fp_path, AFF_PRINT_ERROR); + } +} +#endif + +#ifndef TCC_TARGET_PE +/* add tcc runtime libraries + * + * Two levels of library suppression (matching GCC semantics): + * + * -nostdlib : skip standard libraries (libc, libpthread, libdl, ...) + * but still link compiler runtime (libtcc1, softfp, crt). + * + * -nodefaultlibs : skip everything including compiler runtime (libtcc1, + * softfp, crt). Used when building the runtime libs + * themselves to avoid circular dependencies. + */ +ST_FUNC void tcc_add_runtime(TCCState *s1) +{ + s1->filetype = 0; + + /* -nodefaultlibs: skip all implicit libraries and runtime support */ + if (s1->nodefaultlibs) + return; + +#ifdef CONFIG_TCC_BCHECK + tcc_add_bcheck(s1); +#endif + tcc_add_pragma_libs(s1); + + /* A shared FP runtime has to be bound BEFORE libc. + * + * Every shared object in this rootfs that was ever linked against the FP + * archive absorbed a copy of the __aeabi_ set and re-exports it -- libc.so + * alone exports fifteen doubles entry points. The loader resolves an import + * by walking the module's dependencies in the order the image lists them + * (Module.find_symbol over `children`, appended in import order), so an + * image that named libc first would silently run libc's copy of every + * operation and the -mfpu it was built with would decide nothing. Binding + * the runtime the link actually selected first is what makes the choice + * stick. The archive path keeps the old position: an archive is searched a + * la carte, so it has to come after the objects that reference it. */ +#if defined TCC_TARGET_ARM + if (tccelf_arm_fp_lib_is_shared(s1)) + tccelf_add_arm_fp_lib(s1); +#endif + + /* Standard libraries (skipped by -nostdlib) */ + if (!s1->nostdlib) + { + int lpthread = s1->option_pthread; + +#ifdef CONFIG_TCC_BCHECK + if (s1->do_bounds_check && s1->output_type != TCC_OUTPUT_DLL) + { + tcc_add_support(s1, "bcheck.o"); +#if !(TARGETOS_OpenBSD || TARGETOS_NetBSD) + tcc_add_library(s1, "dl"); +#endif + lpthread = 1; + } +#endif +#ifdef CONFIG_TCC_BACKTRACE + if (s1->do_backtrace) + { + if (s1->output_type & TCC_OUTPUT_EXE) + tcc_add_support(s1, "bt-exe.o"); + if (s1->output_type != TCC_OUTPUT_DLL) + tcc_add_support(s1, "bt-log.o"); + tcc_add_btstub(s1); + lpthread = 1; + } +#endif + if (lpthread) + tcc_add_library(s1, "pthread"); + tcc_add_library(s1, "c"); + tcc_add_library(s1, "m"); +#ifdef TCC_LIBGCC + if (!s1->static_link) + { + if (TCC_LIBGCC[0] == '/') + tcc_add_file(s1, TCC_LIBGCC); + else + tcc_add_dll(s1, TCC_LIBGCC, AFF_PRINT_ERROR); + } +#endif +#if defined TCC_TARGET_ARM && TARGETOS_FreeBSD + tcc_add_library(s1, "gcc_s"); // unwind code +#endif + } + + /* Compiler runtime (always linked unless -nodefaultlibs). + * Order matters: softfp objects reference libtcc1 symbols (__aeabi_memset, + * __aeabi_lcmp, etc.), so the FP library must come first so that its + * undefined references exist when libtcc1.a is processed à la carte. */ +#if defined TCC_TARGET_ARM + if (!tccelf_arm_fp_lib_is_shared(s1)) + tccelf_add_arm_fp_lib(s1); +#endif + if (TCC_LIBTCC1[0]) + tcc_add_support(s1, TCC_LIBTCC1); +#ifndef TCC_TARGET_MACHO + if (s1->output_type != TCC_OUTPUT_MEMORY && !s1->nostdlib) + tccelf_add_crtend(s1); +#endif +} +#endif /* ndef TCC_TARGET_PE */ + +/* add various standard linker symbols (must be done after the + sections are filled (for example after allocating common + symbols)) */ +static void tcc_add_linker_symbols(TCCState *s1) +{ + char buf[1024]; + int i; + Section *s; + + set_global_sym(s1, "_etext", text_section, -1); + set_global_sym(s1, "_edata", data_section, -1); + set_global_sym(s1, "_end", bss_section, -1); +#if TARGETOS_OpenBSD + set_global_sym(s1, "__executable_start", NULL, ELF_START_ADDR); +#endif +#ifdef TCC_TARGET_RISCV64 + /* XXX should be .sdata+0x800, not .data+0x800 */ + set_global_sym(s1, "__global_pointer$", data_section, 0x800); +#endif + /* horrible new standard ldscript defines */ + add_init_array_defines(s1, ".preinit_array"); + add_init_array_defines(s1, ".init_array"); + add_init_array_defines(s1, ".fini_array"); + /* add start and stop symbols for sections whose name can be + expressed in C */ + for (i = 1; i < s1->nb_sections; i++) + { + s = s1->sections[i]; + if ((s->sh_flags & SHF_ALLOC) && + (s->sh_type == SHT_PROGBITS || s->sh_type == SHT_NOBITS || s->sh_type == SHT_STRTAB)) + { + /* check if section name can be expressed in C */ + const char *p0, *p; + p0 = s->name; + if (*p0 == '.') + ++p0; + p = p0; + for (;;) + { + int c = *p; + if (!c) + break; + if (!isid(c) && !isnum(c)) + goto next_sec; + p++; + } + snprintf(buf, sizeof(buf), "__start_%s", p0); + set_global_sym(s1, buf, s, 0); + snprintf(buf, sizeof(buf), "__stop_%s", p0); + set_global_sym(s1, buf, s, -1); + } + next_sec:; + } +} + +ST_FUNC void resolve_common_syms(TCCState *s1) +{ + ElfW(Sym) * sym; + + /* Allocate common symbols in BSS. */ + for_each_elem(symtab_section, 1, sym, ElfW(Sym)) + { + if (sym->st_shndx == SHN_COMMON) + { + /* symbol alignment is in st_value for SHN_COMMONs */ + sym->st_value = section_add(bss_section, sym->st_size, sym->st_value); + sym->st_shndx = bss_section->sh_num; + } + } + + /* Now assign linker provided symbols their value. */ + tcc_add_linker_symbols(s1); +} diff --git a/source/obj/tccelf_sym.c b/source/obj/tccelf_sym.c new file mode 100644 index 000000000..0f351c4d4 --- /dev/null +++ b/source/obj/tccelf_sym.c @@ -0,0 +1,1085 @@ +/* + * ELF file handling for TCC + * + * Copyright (c) 2001-2004 Fabrice Bellard + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +/* ELF: sections, string and symbol tables, symbol lookup and versions, + * relocation entries and the GNU hash table. */ + +#include "tccelf_priv.h" + +/* ---- Global section name hash table ---- */ + +static unsigned int section_name_hash(const char *name) +{ + unsigned int h = 5381; + const unsigned char *p = (const unsigned char *)name; + while (*p) + h = ((h << 5) + h) ^ *p++; + return h; +} + +static void section_ht_grow(TCCState *s1) +{ + unsigned int new_size = s1->section_ht_mask ? (s1->section_ht_mask + 1) * 2 : 64; + Section **new_ht = tcc_mallocz(new_size * sizeof(Section *)); + unsigned int new_mask = new_size - 1; + if (s1->section_ht) + { + unsigned int i; + for (i = 0; i <= s1->section_ht_mask; i++) + { + Section *s = s1->section_ht[i]; + if (s) + { + unsigned int idx = section_name_hash(s->name) & new_mask; + while (new_ht[idx]) + idx = (idx + 1) & new_mask; + new_ht[idx] = s; + } + } + tcc_free(s1->section_ht); + } + s1->section_ht = new_ht; + s1->section_ht_mask = new_mask; +} + +static void section_ht_insert(TCCState *s1, Section *sec) +{ + unsigned int idx; + if (s1->section_ht_count * 2 >= s1->section_ht_mask) + section_ht_grow(s1); + idx = section_name_hash(sec->name) & s1->section_ht_mask; + while (s1->section_ht[idx]) + idx = (idx + 1) & s1->section_ht_mask; + s1->section_ht[idx] = sec; + s1->section_ht_count++; +} +Section *section_ht_find(TCCState *s1, const char *name) +{ + unsigned int idx; + Section *s; + if (!s1->section_ht) + return NULL; + idx = section_name_hash(name) & s1->section_ht_mask; + while ((s = s1->section_ht[idx]) != NULL) + { + if (!strcmp(s->name, name)) + return s; + idx = (idx + 1) & s1->section_ht_mask; + } + return NULL; +} + +/* ---- End section name hash table ---- */ + +ST_FUNC Section *new_section(TCCState *s1, const char *name, int sh_type, int sh_flags) +{ + Section *sec; + + sec = tcc_mallocz(sizeof(Section) + strlen(name)); + sec->s1 = s1; + strcpy(sec->name, name); + sec->sh_type = sh_type; + sec->sh_flags = sh_flags; + switch (sh_type) + { + case SHT_GNU_versym: + sec->sh_addralign = 2; + break; + case SHT_HASH: + case SHT_GNU_HASH: + case SHT_REL: + case SHT_RELA: + case SHT_DYNSYM: + case SHT_SYMTAB: + case SHT_DYNAMIC: + case SHT_GNU_verneed: + case SHT_GNU_verdef: + sec->sh_addralign = 8; // PTR_SIZE; + break; + case SHT_STRTAB: + sec->sh_addralign = 1; + break; + default: + sec->sh_addralign = 8; // PTR_SIZE; /* gcc/pcc default alignment */ + break; + } + + if (sh_flags & SHF_PRIVATE) + { + dynarray_add(&s1->priv_sections, &s1->nb_priv_sections, sec); + } + else + { + sec->sh_num = s1->nb_sections; + dynarray_add(&s1->sections, &s1->nb_sections, sec); + section_ht_insert(s1, sec); + } + + return sec; +} + +ST_FUNC void init_symtab(Section *s) +{ + int *ptr, nb_buckets = SYMTAB_INITIAL_HASH_BUCKETS; + put_elf_str(s->link, ""); + section_ptr_add(s, sizeof(ElfW(Sym))); + ptr = section_ptr_add(s->hash, (2 + nb_buckets + 1) * sizeof(int)); + ptr[0] = nb_buckets; + ptr[1] = 1; + memset(ptr + 2, 0, (nb_buckets + 1) * sizeof(int)); +} + +ST_FUNC Section *new_symtab(TCCState *s1, const char *symtab_name, int sh_type, int sh_flags, const char *strtab_name, + const char *hash_name, int hash_sh_flags) +{ + Section *symtab, *strtab, *hash; + symtab = new_section(s1, symtab_name, sh_type, sh_flags); + symtab->sh_entsize = sizeof(ElfW(Sym)); + strtab = new_section(s1, strtab_name, SHT_STRTAB, sh_flags); + symtab->link = strtab; + hash = new_section(s1, hash_name, SHT_HASH, hash_sh_flags); + hash->sh_entsize = sizeof(int); + symtab->hash = hash; + hash->link = symtab; + init_symtab(symtab); + return symtab; +} + +/* realloc section and set its content to zero */ +ST_FUNC void section_realloc(Section *sec, unsigned long new_size) +{ + unsigned long size; + unsigned char *data; + + size = sec->data_allocated; + if (size == 0) + { + /* First allocation: round up to power of 2 with minimum 256 bytes + to reduce future reallocations */ + size = 256; + while (size < new_size) + size = size * 2; + } + else + { + while (size < new_size) + size = size * 2; + } + data = tcc_realloc(sec->data, size); + memset(data + sec->data_allocated, 0, size - sec->data_allocated); + sec->data = data; + sec->data_allocated = size; +} + +/* reserve at least 'size' bytes aligned per 'align' in section + 'sec' from current offset, and return the aligned offset */ +ST_FUNC size_t section_add(Section *sec, addr_t size, int align) +{ + size_t offset, offset1; + + offset = (sec->data_offset + align - 1) & -align; + offset1 = offset + size; + if (sec->sh_type != SHT_NOBITS && offset1 > sec->data_allocated) + section_realloc(sec, offset1); + sec->data_offset = offset1; + if (align > sec->sh_addralign) + sec->sh_addralign = align; + return offset; +} + +/* reserve at least 'size' bytes in section 'sec' from + sec->data_offset. */ +ST_FUNC void *section_ptr_add(Section *sec, addr_t size) +{ + size_t offset = section_add(sec, size, 1); + return sec->data + offset; +} + +/* Pre-allocate section capacity without changing data_offset. + Use this when you know the total size needed to avoid multiple reallocations. */ +ST_FUNC void section_prealloc(Section *sec, unsigned long size) +{ + unsigned long needed = sec->data_offset + size; + if (needed > sec->data_allocated) + section_realloc(sec, needed); +} + +#ifndef ELF_OBJ_ONLY +/* reserve at least 'size' bytes from section start */ +void section_reserve(Section *sec, unsigned long size) +{ + if (size > sec->data_allocated) + section_realloc(sec, size); + if (size > sec->data_offset) + sec->data_offset = size; +} +#endif +Section *have_section(TCCState *s1, const char *name) +{ + Section *sec; + int i; + for (i = 1; i < s1->nb_sections; i++) + { + sec = s1->sections[i]; + if (!strcmp(name, sec->name)) + return sec; + } + return NULL; +} + +/* return a reference to a section, and create it if it does not + exists */ +ST_FUNC Section *find_section(TCCState *s1, const char *name) +{ + Section *sec = have_section(s1, name); + if (sec) + return sec; + /* sections are created as PROGBITS */ + return new_section(s1, name, SHT_PROGBITS, SHF_ALLOC); +} + +ST_FUNC int put_elf_str(Section *s, const char *sym) +{ + int offset, len; + char *ptr; + + if (!sym) + sym = ""; + len = strlen(sym) + 1; + offset = s->data_offset; + ptr = section_ptr_add(s, len); + memmove(ptr, sym, len); + return offset; +} + +/* elf symbol hashing function */ +ElfW(Word) elf_hash(const unsigned char *name) +{ + ElfW(Word) h = 0, g; + + while (*name) + { + h = (h << 4) + *name++; + g = h & 0xf0000000; + if (g) + h ^= g >> 24; + h &= ~g; + } + return h; +} + +/* rebuild hash table of section s */ +/* NOTE: we do factorize the hash table code to go faster */ +static void rebuild_hash(Section *s, unsigned int nb_buckets) +{ + ElfW(Sym) * sym; + int *ptr, *hash, nb_syms, sym_index, h; + unsigned char *strtab; + unsigned int full_hash; + unsigned int *hcache; + + strtab = s->link->data; + nb_syms = s->data_offset / sizeof(ElfW(Sym)); + + if (!nb_buckets) + nb_buckets = ((int *)s->hash->data)[0]; + + s->hash->data_offset = 0; + ptr = section_ptr_add(s->hash, (2 + nb_buckets + nb_syms) * sizeof(int)); + ptr[0] = nb_buckets; + ptr[1] = nb_syms; + ptr += 2; + hash = ptr; + memset(hash, 0, (nb_buckets + 1) * sizeof(int)); + ptr += nb_buckets + 1; + + /* Rebuild hash value cache */ + s->hash->hash_val_cache = tcc_realloc(s->hash->hash_val_cache, nb_syms * sizeof(unsigned int)); + s->hash->hash_val_alloc = nb_syms; + hcache = s->hash->hash_val_cache; + hcache[0] = 0; + + sym = (ElfW(Sym) *)s->data + 1; + for (sym_index = 1; sym_index < nb_syms; sym_index++) + { + if (ELFW(ST_BIND)(sym->st_info) != STB_LOCAL) + { + full_hash = elf_hash(strtab + sym->st_name); + h = full_hash % nb_buckets; + *ptr = hash[h]; + hash[h] = sym_index; + hcache[sym_index] = full_hash; + } + else + { + *ptr = 0; + hcache[sym_index] = 0; + } + ptr++; + sym++; + } +} + +/* return the symbol number */ +ST_FUNC int put_elf_sym(Section *s, addr_t value, unsigned long size, int info, int other, int shndx, const char *name) +{ + int name_offset, sym_index; + int nbuckets, h; + ElfW(Sym) * sym; + Section *hs; + + /* Validate name pointer - catch garbage early. + Accept printable ASCII and valid UTF-8 lead bytes (0xC2-0xF4). + Reject control characters and bare continuation bytes. */ + if (name && name[0]) + { + unsigned char first = (unsigned char)name[0]; + if (first < 0x20 || (first > 0x7e && first < 0xc2) || first > 0xf4) + { + name = NULL; + } + } + + sym = section_ptr_add(s, sizeof(ElfW(Sym))); + if (name && name[0]) + name_offset = put_elf_str(s->link, name); + else + name_offset = 0; + /* XXX: endianness */ + sym->st_name = name_offset; + sym->st_value = value; + sym->st_size = size; + sym->st_info = info; + sym->st_other = other; + sym->st_shndx = shndx; + sym_index = sym - (ElfW(Sym) *)s->data; + hs = s->hash; + if (hs) + { + int *ptr, *base; + ptr = section_ptr_add(hs, sizeof(int)); + base = (int *)hs->data; + /* only add global or weak symbols. */ + if (ELFW(ST_BIND)(info) != STB_LOCAL) + { + /* add another hashing entry */ + unsigned int full_hash; + nbuckets = base[0]; + full_hash = elf_hash((unsigned char *)s->link->data + name_offset); + h = full_hash % nbuckets; + *ptr = base[2 + h]; + base[2 + h] = sym_index; + base[1]++; + /* we resize the hash table */ + hs->nb_hashed_syms++; + if (hs->nb_hashed_syms > 2 * nbuckets) + { + rebuild_hash(s, 2 * nbuckets); + } + else + { + /* Store cached hash value (rebuild_hash handles this when triggered) */ + if (sym_index >= hs->hash_val_alloc) + { + int new_alloc = (sym_index + 16) & ~15; + hs->hash_val_cache = tcc_realloc(hs->hash_val_cache, new_alloc * sizeof(unsigned int)); + memset(hs->hash_val_cache + hs->hash_val_alloc, 0, (new_alloc - hs->hash_val_alloc) * sizeof(unsigned int)); + hs->hash_val_alloc = new_alloc; + } + hs->hash_val_cache[sym_index] = full_hash; + } + } + else + { + *ptr = 0; + base[1]++; + } + } + + /* Track non-local UNDEF symbols in symtab for fast alacarte lookup */ + if (shndx == SHN_UNDEF && ELFW(ST_BIND)(info) != STB_LOCAL) + { + TCCState *ts = s->s1; + if (s == ts->symtab) + { + if (ts->nb_undef_syms >= ts->undef_sym_alloc) + { + ts->undef_sym_alloc = ts->undef_sym_alloc ? ts->undef_sym_alloc * 2 : 64; + ts->undef_sym_list = tcc_realloc(ts->undef_sym_list, ts->undef_sym_alloc * sizeof(int)); + } + ts->undef_sym_list[ts->nb_undef_syms++] = sym_index; + } + } + + return sym_index; +} + +static int find_elf_sym_with_hash(Section *s, const char *name, unsigned int full_hash) +{ + ElfW(Sym) * sym; + Section *hs; + int nbuckets, sym_index; + const char *name1; + unsigned int *hcache; + + hs = s->hash; + if (!hs) + return 0; + nbuckets = ((int *)hs->data)[0]; + sym_index = ((int *)hs->data)[2 + full_hash % nbuckets]; + hcache = hs->hash_val_cache; + + while (sym_index != 0) + { + /* Compare cached hash value first to avoid expensive strcmp */ + if (!hcache || hcache[sym_index] == full_hash) + { + sym = &((ElfW(Sym) *)s->data)[sym_index]; + name1 = (char *)s->link->data + sym->st_name; + if (!strcmp(name, name1)) + return sym_index; + } + sym_index = ((int *)hs->data)[2 + nbuckets + sym_index]; + } + return 0; +} + +ST_FUNC int find_elf_sym(Section *s, const char *name) +{ + return find_elf_sym_with_hash(s, name, elf_hash((unsigned char *)name)); +} + +/* Resolve `name` against loaded libraries: first the already-interned + dynsymtab_section, then (on miss) the loaded YAFF libraries' on-disk hash + tables (tcc_yaff_resolve), which interns a hit into dynsymtab_section. + Returns the dynsymtab index, or 0 if no loaded library provides it. */ +ST_FUNC int tcc_dynsym_find(TCCState *s1, const char *name) +{ + int idx = find_elf_sym(s1->dynsymtab_section, name); + if (idx) + return idx; + return tcc_yaff_resolve(s1, name); +} + +/* return elf symbol value, signal error if 'err' is nonzero, decorate + name if FORC */ +ST_FUNC addr_t get_sym_addr(TCCState *s1, const char *name, int err, int forc) +{ + int sym_index; + ElfW(Sym) * sym; + char buf[256]; + if (forc && + s1->leading_underscore +#ifdef TCC_TARGET_PE + /* win32-32bit stdcall symbols always have _ already */ + && !strchr(name, '@') +#endif + ) + { + buf[0] = '_'; + pstrcpy(buf + 1, sizeof(buf) - 1, name); + name = buf; + } + sym_index = find_elf_sym(s1->symtab, name); + sym = &((ElfW(Sym) *)s1->symtab->data)[sym_index]; + if (!sym_index || sym->st_shndx == SHN_UNDEF) + { + if (err) + tcc_error_noabort("%s not defined", name); + return (addr_t)-1; + } + return sym->st_value; +} + +/* return elf symbol value */ +LIBTCCAPI void *tcc_get_symbol(TCCState *s, const char *name) +{ + addr_t addr = get_sym_addr(s, name, 0, 1); + return addr == -1 ? NULL : (void *)(uintptr_t)addr; +} + +/* list elf symbol names and values */ +ST_FUNC void list_elf_symbols(TCCState *s, void *ctx, void (*symbol_cb)(void *ctx, const char *name, const void *val)) +{ + ElfW(Sym) * sym; + Section *symtab; + int sym_index, end_sym; + const char *name; + unsigned char sym_vis, sym_bind; + + symtab = s->symtab; + end_sym = symtab->data_offset / sizeof(ElfSym); + for (sym_index = 0; sym_index < end_sym; ++sym_index) + { + sym = &((ElfW(Sym) *)symtab->data)[sym_index]; + if (sym->st_value) + { + name = (char *)symtab->link->data + sym->st_name; + sym_bind = ELFW(ST_BIND)(sym->st_info); + sym_vis = ELFW(ST_VISIBILITY)(sym->st_other); + if (sym_bind == STB_GLOBAL && sym_vis == STV_DEFAULT) + symbol_cb(ctx, name, (void *)(uintptr_t)sym->st_value); + } + } +} + +/* list elf symbol names and values */ +LIBTCCAPI void tcc_list_symbols(TCCState *s, void *ctx, void (*symbol_cb)(void *ctx, const char *name, const void *val)) +{ + list_elf_symbols(s, ctx, symbol_cb); +} + +#ifndef ELF_OBJ_ONLY +void version_add(TCCState *s1) +{ + int i; + ElfW(Sym) * sym; + ElfW(Verneed) *vn = NULL; + Section *symtab; + int sym_index, end_sym, nb_versions = 2, nb_entries = 0; + ElfW(Half) * versym; + const char *name; + + if (0 == nb_sym_versions) + return; + versym_section = new_section(s1, ".gnu.version", SHT_GNU_versym, SHF_ALLOC); + versym_section->sh_entsize = sizeof(ElfW(Half)); + versym_section->link = s1->dynsym; + + /* add needed symbols */ + symtab = s1->dynsym; + end_sym = symtab->data_offset / sizeof(ElfSym); + versym = section_ptr_add(versym_section, end_sym * sizeof(ElfW(Half))); + for (sym_index = 1; sym_index < end_sym; ++sym_index) + { + int dllindex, verndx; + sym = &((ElfW(Sym) *)symtab->data)[sym_index]; + name = (char *)symtab->link->data + sym->st_name; + dllindex = tcc_dynsym_find(s1, name); + verndx = (dllindex && dllindex < nb_sym_to_version) ? sym_to_version[dllindex] : -1; + if (verndx >= 0 + /* XXX: on android, clang refuses to link with a libtcc.so made by tcc + when defined symbols have a version > 1 or when the version is '0'. + Whereas version '1' for example for 'signal' in an exe defeats + bcheck's signal_redir. */ + && (sym->st_shndx == SHN_UNDEF || (s1->output_type & TCC_OUTPUT_EXE))) + { + if (!sym_versions[verndx].out_index) + sym_versions[verndx].out_index = nb_versions++; + versym[sym_index] = sym_versions[verndx].out_index; + } + else + { + versym[sym_index] = 1; /* (*global*) */ + } + // printf("SYM %d %s\n", versym[sym_index], name); + } + /* generate verneed section, but not when it will be empty. Some + dynamic linkers look at their contents even when DTVERNEEDNUM and + section size is zero. */ + if (nb_versions > 2) + { + verneed_section = new_section(s1, ".gnu.version_r", SHT_GNU_verneed, SHF_ALLOC); + verneed_section->link = s1->dynsym->link; + for (i = nb_sym_versions; i-- > 0;) + { + struct sym_version *sv = &sym_versions[i]; + int n_same_libs = 0, prev; + size_t vnofs; + ElfW(Vernaux) *vna = 0; + if (sv->out_index < 1) + continue; + + /* make sure that a DT_NEEDED tag is put */ + /* abitest-tcc fails on older i386-linux with "ld-linux.so.2" DT_NEEDED + ret_int_test... Inconsistency detected by ld.so: dl-minimal.c: 148: + realloc: Assertion `ptr == alloc_last_block' failed! */ + if (strcmp(sv->lib, "ld-linux.so.2")) + tcc_add_dllref(s1, sv->lib, 0); + + vnofs = section_add(verneed_section, sizeof(*vn), 1); + vn = (ElfW(Verneed) *)(verneed_section->data + vnofs); + vn->vn_version = 1; + vn->vn_file = put_elf_str(verneed_section->link, sv->lib); + vn->vn_aux = sizeof(*vn); + do + { + prev = sv->prev_same_lib; + if (sv->out_index > 0) + { + vna = section_ptr_add(verneed_section, sizeof(*vna)); + vna->vna_hash = elf_hash((const unsigned char *)sv->version); + vna->vna_flags = 0; + vna->vna_other = sv->out_index; + sv->out_index = -2; + vna->vna_name = put_elf_str(verneed_section->link, sv->version); + vna->vna_next = sizeof(*vna); + // printf("LIB %d %s %s\n", vna->vna_other, sv->lib, + // verneed_section->link->data + vna->vna_name); + n_same_libs++; + } + if (prev >= 0) + sv = &sym_versions[prev]; + } while (prev >= 0); + vna->vna_next = 0; + vn = (ElfW(Verneed) *)(verneed_section->data + vnofs); + vn->vn_cnt = n_same_libs; + vn->vn_next = sizeof(*vn) + n_same_libs * sizeof(*vna); + nb_entries++; + } + if (vn) + vn->vn_next = 0; + verneed_section->sh_info = nb_entries; + } + dt_verneednum = nb_entries; +} +#endif /* ndef ELF_OBJ_ONLY */ + +/* add an elf symbol : check if it is already defined and patch + it. Return symbol index. NOTE that sh_num can be SHN_UNDEF. */ +ST_FUNC int set_elf_sym(Section *s, addr_t value, unsigned long size, int info, int other, int shndx, const char *name) +{ + TCCState *s1 = s->s1; + ElfW(Sym) * esym; + int sym_bind, sym_index, sym_type, esym_bind; + unsigned char sym_vis, esym_vis, new_vis; + + sym_bind = ELFW(ST_BIND)(info); + sym_type = ELFW(ST_TYPE)(info); + sym_vis = ELFW(ST_VISIBILITY)(other); + + if (sym_bind != STB_LOCAL) + { + /* we search global or weak symbols */ + sym_index = find_elf_sym_with_hash(s, name, elf_hash((const unsigned char *)name)); + if (!sym_index) + goto do_def; + esym = &((ElfW(Sym) *)s->data)[sym_index]; + if (esym->st_value == value && esym->st_size == size && esym->st_info == info && esym->st_other == other && + esym->st_shndx == shndx) + return sym_index; + if (esym->st_shndx != SHN_UNDEF) + { + esym_bind = ELFW(ST_BIND)(esym->st_info); + /* propagate the most constraining visibility */ + /* STV_DEFAULT(0)st_other); + if (esym_vis == STV_DEFAULT) + { + new_vis = sym_vis; + } + else if (sym_vis == STV_DEFAULT) + { + new_vis = esym_vis; + } + else + { + new_vis = (esym_vis < sym_vis) ? esym_vis : sym_vis; + } + esym->st_other = (esym->st_other & ~ELFW(ST_VISIBILITY)(-1)) | new_vis; + if (shndx == SHN_UNDEF) + { + /* ignore adding of undefined symbol if the + corresponding symbol is already defined */ + } + else if (sym_bind == STB_GLOBAL && esym_bind == STB_WEAK) + { + /* global overrides weak, so patch */ + goto do_patch; + } + else if (sym_bind == STB_WEAK && esym_bind == STB_GLOBAL) + { + /* weak is ignored if already global */ + } + else if (sym_bind == STB_WEAK && esym_bind == STB_WEAK) + { + /* keep first-found weak definition, ignore subsequents */ + } + else if (sym_vis == STV_HIDDEN || sym_vis == STV_INTERNAL) + { + /* ignore hidden symbols after */ + } + else if ((esym->st_shndx == SHN_COMMON || esym->st_shndx == bss_section->sh_num) && + (shndx < SHN_LORESERVE && shndx != bss_section->sh_num)) + { + /* data symbol gets precedence over common/bss */ + goto do_patch; + } + else if (shndx == SHN_COMMON || shndx == bss_section->sh_num) + { + /* data symbol keeps precedence over common/bss */ + } + else if (s->sh_flags & SHF_DYNSYM) + { + /* we accept that two DLL define the same symbol */ + } + else if (esym->st_other & ST_ASM_SET) + { + /* If the existing symbol came from an asm .set + we can override. */ + goto do_patch; + } + else + { +#if 0 + printf("new_bind=%x new_shndx=%x new_vis=%x old_bind=%x old_shndx=%x old_vis=%x\n", + sym_bind, shndx, new_vis, esym_bind, esym->st_shndx, esym_vis); +#endif + tcc_error_noabort("'%s' defined twice", name); + } + } + else + { + esym->st_other = other; + do_patch: + esym->st_info = ELFW(ST_INFO)(sym_bind, sym_type); + esym->st_shndx = shndx; + esym->st_value = value; + esym->st_size = size; + } + } + else + { + do_def: + sym_index = put_elf_sym(s, value, size, ELFW(ST_INFO)(sym_bind, sym_type), other, shndx, name); + /* Signal the --start-group rescan loop only when a genuinely NEW + undefined symbol appears. Resolving an existing undef (do_patch) + does not create new work for other archives. */ + if (shndx == SHN_UNDEF) + s1->new_undef_sym = 1; + } + return sym_index; +} + +/* put relocation */ +ST_FUNC void put_elf_reloca(Section *symtab, Section *s, unsigned long offset, int type, int symbol, addr_t addend) +{ + TCCState *s1 = s->s1; + char buf[256]; + Section *sr; + ElfW_Rel *rel; + + /* Validate symbol index. + Skip the check for R_RELATIVE relocations to the GOT: put_got_entry() + smuggles a symtab_section index through a relocation nominally linked + to s1->dynsym, so the index is valid for symtab_section, not for the + symtab passed here. See the "Hack alarm" comment in put_got_entry. */ + if (type != R_RELATIVE) + { + int num_syms = symtab->data_offset / sizeof(ElfW(Sym)); + if (symbol < 0 || symbol >= num_syms) + { + return; /* Skip invalid symbol index */ + } + } + + sr = s->reloc; + if (!sr) + { + /* if no relocation section, create it */ + snprintf(buf, sizeof(buf), REL_SECTION_FMT, s->name); + /* if the symtab is allocated, then we consider the relocation + are also */ + sr = new_section(s->s1, buf, SHT_RELX, symtab->sh_flags); + sr->sh_entsize = sizeof(ElfW_Rel); + sr->link = symtab; + sr->sh_info = s->sh_num; + s->reloc = sr; + } + rel = section_ptr_add(sr, sizeof(ElfW_Rel)); + rel->r_offset = offset; + rel->r_info = ELFW(R_INFO)(symbol, type); +#if SHT_RELX == SHT_RELA + rel->r_addend = addend; +#endif + if (SHT_RELX != SHT_RELA && addend) + tcc_error_noabort("non-zero addend on REL architecture"); +} + +ST_FUNC void put_elf_reloc(Section *symtab, Section *s, unsigned long offset, int type, int symbol) +{ + put_elf_reloca(symtab, s, offset, type, symbol, 0); +} + +ST_FUNC struct sym_attr *get_sym_attr(TCCState *s1, int index, int alloc) +{ + int n; + struct sym_attr *tab; + + if (index >= s1->nb_sym_attrs) + { + if (!alloc) + return s1->sym_attrs; + /* find immediately bigger power of 2 and reallocate array */ + n = 1; + while (index >= n) + n *= 2; + tab = tcc_realloc(s1->sym_attrs, n * sizeof(*s1->sym_attrs)); + s1->sym_attrs = tab; + memset(s1->sym_attrs + s1->nb_sym_attrs, 0, (n - s1->nb_sym_attrs) * sizeof(*s1->sym_attrs)); + s1->nb_sym_attrs = n; + } + return &s1->sym_attrs[index]; +} +void update_relocs(TCCState *s1, Section *s, int *old_to_new_syms, int first_sym) +{ + int i, type, sym_index; + Section *sr; + ElfW_Rel *rel; + + for (i = 1; i < s1->nb_sections; i++) + { + sr = s1->sections[i]; + if (sr->sh_type == SHT_RELX && sr->link == s) + { + for_each_elem(sr, 0, rel, ElfW_Rel) + { + sym_index = ELFW(R_SYM)(rel->r_info); + type = ELFW(R_TYPE)(rel->r_info); + if ((sym_index -= first_sym) < 0) + continue; /* zero sym_index in reloc (can happen with asm) */ + sym_index = old_to_new_syms[sym_index]; + rel->r_info = ELFW(R_INFO)(sym_index, type); + } + } + } +} + +/* In an ELF file symbol table, the local symbols must appear below + the global and weak ones. Since TCC cannot sort it while generating + the code, we must do it after. All the relocation tables are also + modified to take into account the symbol table sorting */ +ST_FUNC void tcc_elf_sort_syms(TCCState *s1, Section *s) +{ + int *old_to_new_syms; + ElfW(Sym) * new_syms; + int nb_syms, i; + ElfW(Sym) * p, *q; + + nb_syms = s->data_offset / sizeof(ElfW(Sym)); + new_syms = tcc_malloc(nb_syms * sizeof(ElfW(Sym))); + old_to_new_syms = tcc_malloc(nb_syms * sizeof(int)); + + /* first pass for local symbols */ + p = (ElfW(Sym) *)s->data; + q = new_syms; + for (i = 0; i < nb_syms; i++) + { + if (ELFW(ST_BIND)(p->st_info) == STB_LOCAL) + { + old_to_new_syms[i] = q - new_syms; + *q++ = *p; + } + p++; + } + /* save the number of local symbols in section header */ + if (s->sh_size) /* this 'if' makes IDA happy */ + s->sh_info = q - new_syms; + + /* then second pass for non local symbols */ + p = (ElfW(Sym) *)s->data; + for (i = 0; i < nb_syms; i++) + { + if (ELFW(ST_BIND)(p->st_info) != STB_LOCAL) + { + old_to_new_syms[i] = q - new_syms; + *q++ = *p; + } + p++; + } + + /* we copy the new symbols to the old */ + memcpy(s->data, new_syms, nb_syms * sizeof(ElfW(Sym))); + tcc_free(new_syms); + + update_relocs(s1, s, old_to_new_syms, 0); + tcc_free(old_to_new_syms); +} + +#ifndef ELF_OBJ_ONLY +/* See: https://flapenguin.me/elf-dt-gnu-hash */ +#define ELFCLASS_BITS (PTR_SIZE * 8) +Section *create_gnu_hash(TCCState *s1) +{ + int nb_syms, i, ndef, nbuckets, symoffset, bloom_size, bloom_shift; + ElfW(Sym) * p; + Section *gnu_hash; + Section *dynsym = s1->dynsym; + Elf32_Word *ptr; + + gnu_hash = new_section(s1, ".gnu.hash", SHT_GNU_HASH, SHF_ALLOC); + gnu_hash->link = dynsym->hash->link; + + nb_syms = dynsym->data_offset / sizeof(ElfW(Sym)); + + /* count def symbols */ + ndef = 0; + p = (ElfW(Sym) *)dynsym->data; + for (i = 0; i < nb_syms; i++, p++) + ndef += p->st_shndx != SHN_UNDEF; + + /* calculate gnu hash sizes and fill header */ + nbuckets = ndef / 4 + 1; + symoffset = nb_syms - ndef; + bloom_shift = PTR_SIZE == 8 ? 6 : 5; + bloom_size = 1; /* must be power of two */ + while (ndef >= bloom_size * (1 << (bloom_shift - 3))) + bloom_size *= 2; + ptr = section_ptr_add(gnu_hash, 4 * 4 + PTR_SIZE * bloom_size + nbuckets * 4 + ndef * 4); + ptr[0] = nbuckets; + ptr[1] = symoffset; + ptr[2] = bloom_size; + ptr[3] = bloom_shift; + return gnu_hash; +} + +static Elf32_Word elf_gnu_hash(const unsigned char *name) +{ + Elf32_Word h = 5381; + unsigned char c; + + while ((c = *name++)) + h = h * 33 + c; + return h; +} +void update_gnu_hash(TCCState *s1, Section *gnu_hash) +{ + int *old_to_new_syms; + ElfW(Sym) * new_syms; + int nb_syms, i, nbuckets, bloom_size, bloom_shift; + ElfW(Sym) * p, *q; + Section *vs; + Section *dynsym = s1->dynsym; + Elf32_Word *ptr, *buckets, *chain, *hash; + unsigned int *nextbuck; + addr_t *bloom; + unsigned char *strtab; + struct + { + int first, last; + } *buck; + + strtab = dynsym->link->data; + nb_syms = dynsym->data_offset / sizeof(ElfW(Sym)); + new_syms = tcc_malloc(nb_syms * sizeof(ElfW(Sym))); + old_to_new_syms = tcc_malloc(nb_syms * sizeof(int)); + hash = tcc_malloc(nb_syms * sizeof(Elf32_Word)); + nextbuck = tcc_malloc(nb_syms * sizeof(int)); + + /* calculate hashes and copy undefs */ + p = (ElfW(Sym) *)dynsym->data; + q = new_syms; + for (i = 0; i < nb_syms; i++, p++) + { + if (p->st_shndx == SHN_UNDEF) + { + old_to_new_syms[i] = q - new_syms; + *q++ = *p; + } + else + hash[i] = elf_gnu_hash(strtab + p->st_name); + } + + ptr = (Elf32_Word *)gnu_hash->data; + nbuckets = ptr[0]; + bloom_size = ptr[2]; + bloom_shift = ptr[3]; + bloom = (addr_t *)(void *)&ptr[4]; + buckets = (Elf32_Word *)(void *)&bloom[bloom_size]; + chain = &buckets[nbuckets]; + buck = tcc_malloc(nbuckets * sizeof(*buck)); + + if (gnu_hash->data_offset != 4 * 4 + PTR_SIZE * bloom_size + nbuckets * 4 + (nb_syms - (q - new_syms)) * 4) + tcc_error_noabort("gnu_hash size incorrect"); + + /* find buckets */ + for (i = 0; i < nbuckets; i++) + buck[i].first = -1; + + p = (ElfW(Sym) *)dynsym->data; + for (i = 0; i < nb_syms; i++, p++) + if (p->st_shndx != SHN_UNDEF) + { + int bucket = hash[i] % nbuckets; + + if (buck[bucket].first == -1) + buck[bucket].first = buck[bucket].last = i; + else + { + nextbuck[buck[bucket].last] = i; + buck[bucket].last = i; + } + } + + /* fill buckets/chains/bloom and sort symbols */ + p = (ElfW(Sym) *)dynsym->data; + for (i = 0; i < nbuckets; i++) + { + int cur = buck[i].first; + + if (cur != -1) + { + buckets[i] = q - new_syms; + for (;;) + { + old_to_new_syms[cur] = q - new_syms; + *q++ = p[cur]; + *chain++ = hash[cur] & ~1; + bloom[(hash[cur] / ELFCLASS_BITS) % bloom_size] |= + (addr_t)1 << (hash[cur] % ELFCLASS_BITS) | (addr_t)1 << ((hash[cur] >> bloom_shift) % ELFCLASS_BITS); + if (cur == buck[i].last) + break; + cur = nextbuck[cur]; + } + chain[-1] |= 1; + } + } + + memcpy(dynsym->data, new_syms, nb_syms * sizeof(ElfW(Sym))); + tcc_free(new_syms); + tcc_free(hash); + tcc_free(buck); + tcc_free(nextbuck); + + update_relocs(s1, dynsym, old_to_new_syms, 0); + + /* modify the versions */ + vs = versym_section; + if (vs) + { + ElfW(Half) * newver, *versym = (ElfW(Half) *)vs->data; + + if (1 /*versym*/) + { + newver = tcc_malloc(nb_syms * sizeof(*newver)); + for (i = 0; i < nb_syms; i++) + newver[old_to_new_syms[i]] = versym[i]; + memcpy(vs->data, newver, nb_syms * sizeof(*newver)); + tcc_free(newver); + } + } + + tcc_free(old_to_new_syms); + + /* rebuild hash */ + ptr = (Elf32_Word *)dynsym->hash->data; + rebuild_hash(dynsym, ptr[0]); +} +#endif /* ELF_OBJ_ONLY */ diff --git a/source/obj/tccyaff.c b/source/obj/tccyaff.c index ca88baa18..107db07e8 100644 --- a/source/obj/tccyaff.c +++ b/source/obj/tccyaff.c @@ -21,10 +21,15 @@ #include "tcc.h" #include +#ifndef _WIN32 +#include +#endif #include "memory/unique_ptr.h" #include "tccyaff.h" +static const char *yaff_lib_read_exports(int fd, const YaffHeader *header, YaffLib *lib); + #define TCC_YAFF_MAX_SYMBOL_ENTRY_SIZE 255 #define SHF_DYNSYM 0x40000000 @@ -241,43 +246,21 @@ ST_FUNC int tcc_load_yaff(TCCState *s1, int fd, const char *filename, int level) full_read(fd, &header, sizeof(YaffHeader)); if (memcmp(header.magic, "YAFF", 4) != 0) return tcc_error_noabort("not a valid YAFF file"); + /* The loader refuses it, so linking against it would only move the error. */ + if (header.yaff_version != YAFF_VERSION) + return tcc_error_noabort("'%s': YAFF version %d, this toolchain writes %d; rebuild it", filename, + header.yaff_version, YAFF_VERSION); if (header.exported_symbols_amount > 0) { YaffLib *lib; - unsigned int region_size, hdr2[2], nbucket, nchain, lookup_size, chain_bytes; + const char *err; s1->yaff_libs = tcc_realloc(s1->yaff_libs, (s1->nb_yaff_libs + 1) * sizeof(YaffLib)); lib = &s1->yaff_libs[s1->nb_yaff_libs++]; - memset(lib, 0, sizeof(*lib)); - lib->nsyms = header.exported_symbols_amount; - - /* name/value region: [exported_symbols_offset, imported_symbols_lookup_offset) */ - region_size = header.imported_symbols_lookup_offset - header.exported_symbols_offset; - lib->region = tcc_malloc(region_size); - lseek(fd, header.exported_symbols_offset, SEEK_SET); - if ((unsigned)full_read(fd, lib->region, region_size) != region_size) - return tcc_error_noabort("short read of YAFF export region"); - - /* index -> region byte-offset lookup table (one u16 per exported symbol) */ - lookup_size = lib->nsyms * sizeof(unsigned short); - lib->lookup = tcc_malloc(lookup_size); - lseek(fd, header.exported_symbols_lookup_offset, SEEK_SET); - if ((unsigned)full_read(fd, lib->lookup, lookup_size) != lookup_size) - return tcc_error_noabort("short read of YAFF export lookup"); - - /* name hash: [nbucket, nchain, bucket[nbucket], chain[nchain]] (self-sized) */ - lseek(fd, header.exported_symbols_hash_table_offset, SEEK_SET); - if ((unsigned)full_read(fd, hdr2, sizeof(hdr2)) != sizeof(hdr2)) - return tcc_error_noabort("short read of YAFF hash header"); - nbucket = hdr2[0]; - nchain = hdr2[1]; - chain_bytes = (nbucket + nchain) * sizeof(unsigned int); - lib->hash = tcc_malloc((2 + nbucket + nchain) * sizeof(unsigned int)); - lib->hash[0] = nbucket; - lib->hash[1] = nchain; - if ((unsigned)full_read(fd, lib->hash + 2, chain_bytes) != chain_bytes) - return tcc_error_noabort("short read of YAFF hash table"); + err = yaff_lib_read_exports(fd, &header, lib); + if (err) + return tcc_error_noabort("%s", err); } /* if the dll is already loaded, do not load it */ @@ -286,6 +269,72 @@ ST_FUNC int tcc_load_yaff(TCCState *s1, int fd, const char *filename, int level) return 0; } +/* Read a library's exported-symbol tables into `lib` (zeroed here). Returns + NULL on success or a message naming the short read; the partial buffers are + left for yaff_lib_free. */ +static const char *yaff_lib_read_exports(int fd, const YaffHeader *header, YaffLib *lib) +{ + unsigned int region_size, hdr2[2], nbucket, nchain, lookup_size, chain_bytes; + + memset(lib, 0, sizeof(*lib)); + lib->nsyms = header->exported_symbols_amount; + + /* name/value region: [exported_symbols_offset, imported_symbols_lookup_offset) */ + region_size = header->imported_symbols_lookup_offset - header->exported_symbols_offset; + lib->region = tcc_malloc(region_size); + lseek(fd, header->exported_symbols_offset, SEEK_SET); + if ((unsigned)full_read(fd, lib->region, region_size) != region_size) + return "short read of YAFF export region"; + + /* index -> region byte-offset lookup table (one u16 per exported symbol) */ + lookup_size = lib->nsyms * sizeof(unsigned short); + lib->lookup = tcc_malloc(lookup_size); + lseek(fd, header->exported_symbols_lookup_offset, SEEK_SET); + if ((unsigned)full_read(fd, lib->lookup, lookup_size) != lookup_size) + return "short read of YAFF export lookup"; + + /* name hash: [nbucket, nchain, bucket[nbucket], chain[nchain]] (self-sized) */ + lseek(fd, header->exported_symbols_hash_table_offset, SEEK_SET); + if ((unsigned)full_read(fd, hdr2, sizeof(hdr2)) != sizeof(hdr2)) + return "short read of YAFF hash header"; + nbucket = hdr2[0]; + nchain = hdr2[1]; + chain_bytes = (nbucket + nchain) * sizeof(unsigned int); + lib->hash = tcc_malloc((2 + nbucket + nchain) * sizeof(unsigned int)); + lib->hash[0] = nbucket; + lib->hash[1] = nchain; + if ((unsigned)full_read(fd, lib->hash + 2, chain_bytes) != chain_bytes) + return "short read of YAFF hash table"; + return NULL; +} + +static void yaff_lib_free(YaffLib *lib) +{ + tcc_free(lib->region); + tcc_free(lib->lookup); + tcc_free(lib->hash); +} + +/* The export entry of `name` in `lib`, found through the on-disk name hash + (`h` = tcc_yaff_hash(name)), or NULL. */ +static YaffSymbolEntry *yaff_lib_find(const YaffLib *lib, unsigned int h, const char *name) +{ + unsigned int nbucket = lib->hash[0]; + unsigned int *bucket = lib->hash + 2; + unsigned int *chain = lib->hash + 2 + nbucket; + unsigned int i; + for (i = bucket[h % nbucket]; i != 0; i = chain[i]) + { + YaffSymbolEntry *e; + if (i >= lib->nsyms) + break; /* corrupt chain guard */ + e = (YaffSymbolEntry *)(lib->region + lib->lookup[i]); + if (strcmp((const char *)e + sizeof(YaffSymbolEntry), name) == 0) + return e; + } + return NULL; +} + /* Resolve `name` against the loaded YAFF libraries via their on-disk hash tables, interning a hit into dynsymtab_section. Returns the dynsymtab index (>0) or 0 if no loaded library exports it. */ @@ -295,27 +344,149 @@ ST_FUNC int tcc_yaff_resolve(TCCState *s1, const char *name) int li; for (li = 0; li < s1->nb_yaff_libs; li++) { - YaffLib *lib = &s1->yaff_libs[li]; - unsigned int nbucket = lib->hash[0]; - unsigned int *bucket = lib->hash + 2; - unsigned int *chain = lib->hash + 2 + nbucket; - unsigned int i; - for (i = bucket[h % nbucket]; i != 0; i = chain[i]) + YaffSymbolEntry *e = yaff_lib_find(&s1->yaff_libs[li], h, name); + if (e) + return set_elf_sym(s1->dynsymtab_section, e->offset, 1, + ELFW(ST_INFO)(e->weak ? STB_WEAK : STB_GLOBAL, + e->section == YAFF_SECTION_CODE ? STT_FUNC : STT_NOTYPE), + STV_DEFAULT, 1, (const char *)e + sizeof(YaffSymbolEntry)); + } + return 0; +} + +/* -fmodule-local-calls import set: which undefined functions will a link bind + to a shared library? The compiler needs to know while it still emits the + call, so it reads the export tables of every lib*.so in the library paths + (the sysroot's /usr/lib for the cross and the device's /usr/lib for the + native tcc) the first time it asks. A name a library exports is an import + -- unless libtcc1.a (or a statically linked FP runtime) defines it too: + tcc_add_runtime pulls that archive before the shared libraries resolve + anything, so its member is what the call binds to (toybox's __tcc_memmove is in its own .text although libc.so + exports the name). Classifying an in-module function as an import only + keeps a reload the call did not need; the other direction is what + R_ARM_YASOS_LOCAL_CALL catches at link time. */ +static void yaff_import_set_add(TCCState *s1, const char *path) +{ + YaffHeader header; + YaffLib lib; + int fd = open(path, O_RDONLY | O_BINARY); + if (fd < 0) + return; + if (full_read(fd, &header, sizeof(header)) == sizeof(header) && memcmp(header.magic, "YAFF", 4) == 0 && + header.yaff_version == YAFF_VERSION && header.exported_symbols_amount > 0) + { + if (yaff_lib_read_exports(fd, &header, &lib) == NULL) + { + s1->import_libs = tcc_realloc(s1->import_libs, (s1->nb_import_libs + 1) * sizeof(YaffLib)); + s1->import_libs[s1->nb_import_libs++] = lib; + } + else + yaff_lib_free(&lib); + } + close(fd); +} + +static void yaff_import_set_build(TCCState *s1) +{ + s1->import_set_state = 2; +#ifndef _WIN32 + { + char path[1024]; + int i; +#if defined TCC_TARGET_ARM + const char *fp_name = tccelf_get_fp_lib_name(s1); +#endif + for (i = 0; i < s1->nb_library_paths; i++) { - YaffSymbolEntry *e; - const char *ename; - if (i >= lib->nsyms) - break; /* corrupt chain guard */ - e = (YaffSymbolEntry *)(lib->region + lib->lookup[i]); - ename = (const char *)e + sizeof(YaffSymbolEntry); - if (strcmp(ename, name) == 0) - return set_elf_sym(s1->dynsymtab_section, e->offset, 1, - ELFW(ST_INFO)(e->weak ? STB_WEAK : STB_GLOBAL, - e->section == YAFF_SECTION_CODE ? STT_FUNC : STT_NOTYPE), - STV_DEFAULT, 1, ename); + const char *dir = s1->library_paths[i]; + struct dirent *e; + DIR *d = opendir(dir); + if (d) + { + while ((e = readdir(d)) != NULL) + { + size_t n = strlen(e->d_name); + if (n < 7 || strncmp(e->d_name, "lib", 3) != 0 || strcmp(e->d_name + n - 3, ".so") != 0) + continue; + snprintf(path, sizeof(path), "%s/%s", dir, e->d_name); + yaff_import_set_add(s1, path); + } + closedir(d); + } + if (!s1->import_libtcc1) + { + snprintf(path, sizeof(path), "%s/%slibtcc1.a", dir, CONFIG_TCC_CROSSPREFIX); + if (tcc_archive_index_load(s1, path)) + s1->import_libtcc1 = tcc_strdup(path); + } +#if defined TCC_TARGET_ARM + /* A soft-float runtime copied in from fp/lib.a (-mfp-lib=static, + or the cross's default) is part of the module like libtcc1, so its + __aeabi_ helpers keep R9 although lib.so exports the same + names. Found the way tccelf_add_arm_fp_lib's tcc_add_dll finds it. */ + if (!s1->import_fp_archive && fp_name && !tccelf_arm_fp_lib_is_shared(s1)) + { + snprintf(path, sizeof(path), "%s/fp/lib%s.a", dir, fp_name); + if (tcc_archive_index_load(s1, path)) + s1->import_fp_archive = tcc_strdup(path); + } +#endif } } - return 0; +#endif + /* No shared library anywhere: nothing is known about imports, and callers + fall back to reloading R9 after every call out of the translation unit. */ + if (s1->nb_import_libs > 0) + s1->import_set_state = 1; +} + +ST_FUNC int tcc_yaff_import_set_has(TCCState *s1, const char *name) +{ + unsigned int h; + int li; + if (s1->import_set_state == 0) + yaff_import_set_build(s1); + if (s1->import_set_state != 1) + return -1; + h = tcc_yaff_hash(name); + for (li = 0; li < s1->nb_import_libs; li++) + if (yaff_lib_find(&s1->import_libs[li], h, name)) + break; + if (li == s1->nb_import_libs) + return 0; + /* libtcc1.a and the FP runtime are linked even under -nostdlib; only + -nodefaultlibs leaves them out (tcc_add_runtime), and then the shared + library's copy is what the call reaches. */ + if (!s1->nodefaultlibs) + { + if (s1->import_libtcc1 && tcc_archive_index_has(s1, s1->import_libtcc1, name)) + return 0; + if (s1->import_fp_archive && tcc_archive_index_has(s1, s1->import_fp_archive, name)) + return 0; + } + return 1; +} + +ST_FUNC int tcc_yaff_libtcc1_has(TCCState *s1, const char *name) +{ + if (s1->import_set_state == 0) + yaff_import_set_build(s1); + return !s1->nodefaultlibs && s1->import_libtcc1 && tcc_archive_index_has(s1, s1->import_libtcc1, name); +} + +ST_FUNC void tcc_yaff_import_set_free(TCCState *s1) +{ + int li; + for (li = 0; li < s1->nb_import_libs; li++) + yaff_lib_free(&s1->import_libs[li]); + tcc_free(s1->import_libs); + s1->import_libs = NULL; + s1->nb_import_libs = 0; + tcc_free(s1->import_libtcc1); + s1->import_libtcc1 = NULL; + tcc_free(s1->import_fp_archive); + s1->import_fp_archive = NULL; + s1->import_set_state = 0; } ST_FUNC void tcc_yaff_libs_free(TCCState *s1) @@ -332,6 +503,29 @@ ST_FUNC void tcc_yaff_libs_free(TCCState *s1) s1->nb_yaff_libs = 0; } +/* Offset of a linked address inside the module's data region. + * + * The loader lays that region out as [rodata][padding][data][padding][bss] + * [padding][got] and reproduces the linker's own distances: header.data_length + * is computed as rodata + rd_padding + data, and header.bss_length carries the + * padding on either side of .bss (see tcc_output_yaff). So every address from + * .rodata's base to the end of the GOT converts with a single subtraction. + * + * It has to be done this way rather than per section: rebasing as + * `addr - data_section->sh_addr + rodata_section->sh_size` silently drops + * whatever alignment padding the linker left between .rodata and .data. That + * padding is usually zero, which is why it went unnoticed -- but an image with + * a page-aligned object in .data gets a real gap (336 bytes on the Zig + * compiler, whose .data is 4096-aligned), and then every relocation whose patch + * site is in .data lands that many bytes short: it overwrites the first word of + * a neighbouring object and leaves the pointer that actually needed relocating + * holding its link-time value. The symptom is a function pointer in a vtable + * reading back as a string address. */ +static addr_t tcc_yaff_data_region_offset(TCCState *s1, addr_t address) +{ + return address - rodata_section->sh_addr; +} + /* Write local relocations for GOT entries that reference local symbols. * * When a local (STB_LOCAL) symbol's address is taken, put_got_entry() @@ -398,12 +592,12 @@ static int tcc_yaff_write_local_relocations(TCCState *s1, FILE *f) else if (sym_value >= data_section->sh_addr && sym_value < data_section->sh_addr + data_section->sh_size) { section = YAFF_SECTION_DATA; - target_offset = sym_value - data_section->sh_addr + rodata_section->sh_size; + target_offset = tcc_yaff_data_region_offset(s1, sym_value); } else if (sym_value >= bss_section->sh_addr && sym_value < bss_section->sh_addr + bss_section->sh_size) { section = YAFF_SECTION_DATA; - target_offset = sym_value - bss_section->sh_addr + rodata_section->sh_size + data_section->sh_size; + target_offset = tcc_yaff_data_region_offset(s1, sym_value); } else { @@ -520,18 +714,7 @@ static int tcc_yaff_write_data_relocations(TCCState *s1, FILE *f) Section *target_sec = s1->sections[s->sh_info]; imp_to += target_sec->sh_addr; } - if (imp_to >= data_section->sh_addr && imp_to < data_section->sh_addr + data_section->sh_size) - { - imp_to = (imp_to - data_section->sh_addr) + rodata_section->sh_size; - } - else if (imp_to >= bss_section->sh_addr && imp_to < bss_section->sh_addr + bss_section->sh_size) - { - imp_to = (imp_to - bss_section->sh_addr) + rodata_section->sh_size + data_section->sh_size; - } - else - { - imp_to -= rodata_section->sh_addr; - } + imp_to = tcc_yaff_data_region_offset(s1, imp_to); struct sym_attr *attr = get_sym_attr(s1, sym_idx, 0); uint32_t got_offset = 0; @@ -579,7 +762,7 @@ static int tcc_yaff_write_data_relocations(TCCState *s1, FILE *f) else if (abs_from_address >= data_section->sh_addr && abs_from_address < data_section->sh_addr + data_section->sh_size) { original_offset = *(uint32_t *)(data_section->data + (abs_from_address - data_section->sh_addr)); - from_address = (abs_from_address - data_section->sh_addr) + rodata_section->sh_size; + from_address = tcc_yaff_data_region_offset(s1, abs_from_address); } else if (abs_from_address >= bss_section->sh_addr && abs_from_address < bss_section->sh_addr + bss_section->sh_size) { @@ -606,18 +789,7 @@ static int tcc_yaff_write_data_relocations(TCCState *s1, FILE *f) towards_code = original_offset < rodata_section->sh_addr; if (!towards_code) { - if (original_offset >= bss_section->sh_addr && original_offset < bss_section->sh_addr + bss_section->sh_size) - { - original_offset = (original_offset - bss_section->sh_addr) + rodata_section->sh_size + data_section->sh_size; - } - else if (original_offset >= data_section->sh_addr && original_offset < data_section->sh_addr + data_section->sh_size) - { - original_offset = (original_offset - data_section->sh_addr) + rodata_section->sh_size; - } - else - { - original_offset -= rodata_section->sh_addr; - } + original_offset = tcc_yaff_data_region_offset(s1, original_offset); } entry = (YaffDataRelocationEntry){ @@ -642,6 +814,7 @@ static int tcc_yaff_write_data_relocations(TCCState *s1, FILE *f) case R_ARM_GOTOFF: case R_ARM_RODATA_OFF: case R_ARM_GOT_SBREL12: + case R_ARM_YASOS_LOCAL_CALL: case R_ARM_GOTPC: case R_ARM_GOT_PREL: case R_ARM_PC24: @@ -806,6 +979,7 @@ static int tcc_yaff_write_symbol_table_relocations(TCCState *s1, FILE *f) case R_ARM_NONE: case R_ARM_RODATA_OFF: case R_ARM_GOT_SBREL12: + case R_ARM_YASOS_LOCAL_CALL: { // relocations that are safe to ignore due to their PC relative // nature (R_ARM_RODATA_OFF is resolved at link time into the .text @@ -915,18 +1089,7 @@ static int tcc_yaff_write_exported_symbols(TCCState *s1, FILE *f, YaffHeader *h) offset = sym->st_value; if (section_code == YAFF_SECTION_DATA) { - if (sym->st_shndx == bss_section->sh_num) - { - offset = (offset - bss_section->sh_addr) + rodata_section->sh_size + data_section->sh_size; - } - else if (sym->st_shndx == data_section->sh_num) - { - offset = (offset - data_section->sh_addr) + rodata_section->sh_size; - } - else - { - offset -= rodata_section->sh_addr; - } + offset = tcc_yaff_data_region_offset(s1, offset); } entry = (YaffSymbolEntry){ .section = section_code, @@ -985,6 +1148,14 @@ static void tcc_yaff_write_imported_symbols_lookup(TCCState *s1, FILE *f, YaffHe name_len = strlen(name) + 1; aligned_name_len = tcc_yaff_align(h, name_len); current_offset += sizeof(uint32_t) + aligned_name_len; + /* YaffLookupEntry.symbol_offset is still 16 bits, so the symbol NAME table + * is capped at 64 KiB even though the header offsets around it are not + * (YAFF_VERSION 3). Say so rather than wrapping: a wrapped lookup entry + * resolves imports to the wrong name, which is a far worse failure than a + * refused link. */ + if (current_offset > 0xFFFF) + tcc_error_noabort("imported symbol table exceeds 64 KiB, which " + "YaffLookupEntry cannot address"); fwrite(&entry, sizeof(entry), 1, f); } } @@ -1021,6 +1192,10 @@ static void tcc_yaff_write_exported_symbols_lookup(TCCState *s1, FILE *f, YaffHe name_len = strlen(name) + 1; aligned_name_len = tcc_yaff_align(h, name_len); current_offset += sizeof(uint32_t) + aligned_name_len; + /* See the imported lookup above: 16-bit entry, 64 KiB of names. */ + if (current_offset > 0xFFFF) + tcc_error_noabort("exported symbol table exceeds 64 KiB, which " + "YaffLookupEntry cannot address"); fwrite(&entry, sizeof(entry), 1, f); } } @@ -1317,6 +1492,27 @@ ST_FUNC int tcc_output_yaff(TCCState *s1, FILE *f, const char *filename) * (rodata_size when -share-rodata, else 0). */ if (!s1->share_rodata) header.const_rodata_length = 0; + /* The per-process region is the linked range from .rodata's base plus the + * shared prefix to the end of the GOT, mapped with one subtraction + * (tcc_yaff_data_region_offset). Its alignment is the largest any section + * in it asked for, and the loader has to start it at the same phase. */ + { + addr_t region_start = rodata_section->sh_addr + header.const_rodata_length; + addr_t region_end = s1->got->sh_addr + s1->got->sh_size; + addr_t align = 1; + for (i = 1; i < s1->nb_sections; i++) + { + Section *s = s1->sections[i]; + if (!(s->sh_flags & SHF_ALLOC) || (s->sh_flags & SHF_EXECINSTR) || !s->sh_size) + continue; + if (s->sh_addr < region_start || s->sh_addr >= region_end) + continue; + if ((addr_t)s->sh_addralign > align) + align = s->sh_addralign; + } + header.data_alignment = (uint32_t)align; + header.data_alignment_offset = (uint32_t)(region_start & (align - 1)); + } fwrite(&header, 1, sizeof(YaffHeader), f); aligned_name_len = strlen(name) + 1; diff --git a/source/obj/tccyaff.h b/source/obj/tccyaff.h index 2a4429b6b..30ac0ac9b 100644 --- a/source/obj/tccyaff.h +++ b/source/obj/tccyaff.h @@ -10,8 +10,20 @@ * 1 -- initial layout. * 2 -- architecture section (YaffArchSection) between the module name and * the imported-library table; YaffHeader.arch carries a real YaffArch - * instead of a constant 1. */ -#define YAFF_VERSION 2 + * instead of a constant 1. + * 3 -- the table offsets and the relocation/symbol counts are 32-bit. They + * were 16-bit, which silently capped everything ahead of the code at + * 64 KiB: a module with more than ~8k relocations wrapped text_offset + * and the loader read its "code" out of the middle of the relocation + * table. Nothing reported it -- the link succeeded and the image was + * garbage. Found on a 5 MB module (the Zig compiler) whose pre-text + * tables came to 105,440 bytes and whose text_offset read back 39,904. + * 4 -- data_alignment and data_alignment_offset: where the per-process data + * region has to start. The loader put it wherever its allocator + * returned, so an object aligned past that -- `static int buf[4] + * __attribute__((aligned(64)))` -- was aligned only against a base the + * linker assumed and nothing reproduced. */ +#define YAFF_VERSION 4 typedef struct { uint32_t nbucket; @@ -118,27 +130,32 @@ typedef struct __attribute__((packed)) YaffHeader { uint8_t text_and_data_separation; uint16_t version_major; uint16_t version_minor; - uint16_t symbol_table_relocations_amount; - uint16_t local_relocations_amount; - uint16_t data_relocations_amount; - uint16_t copy_relocations_amount; - uint16_t exported_symbols_amount; - uint16_t imported_symbols_amount; + /* 32-bit since YAFF_VERSION 3: a multi-megabyte module has far more than + * 65,535 data relocations, and the old 16-bit counts wrapped in silence. */ + uint32_t symbol_table_relocations_amount; + uint32_t local_relocations_amount; + uint32_t data_relocations_amount; + uint32_t copy_relocations_amount; + uint32_t exported_symbols_amount; + uint32_t imported_symbols_amount; uint32_t got_length; uint32_t got_plt_length; uint32_t plt_length; /* Offset of the YaffArchSection, which sits between the module name and the * imported-library table. Never 0 from YAFF_VERSION 2 on. */ - uint16_t arch_section_offset; - uint16_t imported_libraries_offset; - uint16_t relocations_offset; - uint16_t imported_symbols_offset; - uint16_t exported_symbols_offset; - uint16_t text_offset; - uint16_t imported_symbols_lookup_offset; - uint16_t exported_symbols_lookup_offset; - uint16_t imported_symbols_hash_table_offset; - uint16_t exported_symbols_hash_table_offset; + uint32_t arch_section_offset; + uint32_t imported_libraries_offset; + uint32_t relocations_offset; + uint32_t imported_symbols_offset; + uint32_t exported_symbols_offset; + /* Where the code starts in the file. This is the field the 16-bit layout + * broke on first: it sits behind every table above, so it is the largest + * offset in the header and the first one to pass 64 KiB. */ + uint32_t text_offset; + uint32_t imported_symbols_lookup_offset; + uint32_t exported_symbols_lookup_offset; + uint32_t imported_symbols_hash_table_offset; + uint32_t exported_symbols_hash_table_offset; /* Per-image stack/heap profile in bytes. 0xFFFFFFFF = use the OS default * (kernel-driven stack size; heap free to grow in the shared paged pool). * A concrete value lets a program declare its footprint (e.g. shell applets @@ -153,6 +170,14 @@ typedef struct __attribute__((packed)) YaffHeader { * behaviour). When >0, the loader maps it once (XIP, ref-counted) and code * reaches it via the rodata anchor GOT slot + R_ARM_RODATA_OFF offsets. */ uint32_t const_rodata_length; + /* Placement of the per-process data region ([data][bss][got], and .rodata + * too when it is not shared): the loader starts it at an address A with + * A % data_alignment == data_alignment_offset. data_alignment is the + * largest alignment any section in the region asked for (a power of two); + * the offset is the region's linked start modulo it, so every object keeps + * the alignment it had at link time. */ + uint32_t data_alignment; + uint32_t data_alignment_offset; } YaffHeader; typedef enum YaffSectionCode { diff --git a/source/opt/Makefile b/source/opt/Makefile index cd4154af5..05e57b00e 100644 --- a/source/opt/Makefile +++ b/source/opt/Makefile @@ -62,7 +62,8 @@ CORE_OPT_HDRS = \ RA_OPT_SRC = \ source/opt/ra/const_branch_fold.c \ source/opt/ra/incomplete_calls.c \ - source/opt/ra/phi_const_chain.c + source/opt/ra/phi_const_chain.c \ + source/opt/ra/umaal_fusion.c $(X)source/opt/ra/%.o: source/opt/ra/%.c $(CORE_OPT_HDRS) source/ir/regalloc.h @mkdir -p $(dir $@) @@ -90,7 +91,7 @@ OPT_SRC = \ $(SSA_OPT_SRC) \ $(FLAT_OPT_SRC) -OPT_OBJ = $(patsubst %.c,$(X)%.o,$(OPT_SRC)) +OPT_OBJ = $(call lib-objs,opt,$(OPT_SRC)) OPT_LIB = $(X)source/opt/libopt.a $(OPT_LIB): $(OPT_OBJ) diff --git a/source/opt/analysis/du_chains.c b/source/opt/analysis/du_chains.c index 269ac0bdf..039787ebb 100644 --- a/source/opt/analysis/du_chains.c +++ b/source/opt/analysis/du_chains.c @@ -113,7 +113,7 @@ void ir_opt_du_build_mode(TCCIRState *ir, IROptDU *du, uint8_t mode) du->use[idx]++; } /* MLA's 4th accumulator operand is a use; missing it makes its result look dead (seed 4274). */ - if (q->op == TCCIR_OP_MLA) + if (tcc_ir_op_is_mac(q->op)) { int idx = ir_opt_du_idx(du, irop_get_vreg(tcc_ir_op_get_accum(ir, q))); if (idx >= 0 && du->use[idx] < 2) diff --git a/source/opt/engine/pipeline_table.c b/source/opt/engine/pipeline_table.c index c858836db..7beb5e9a1 100644 --- a/source/opt/engine/pipeline_table.c +++ b/source/opt/engine/pipeline_table.c @@ -109,7 +109,13 @@ static const IROptPass propagation_passes[] = { /* Runs first so later passes don't work on a body that is about to collapse. */ PASS_GATED("uninit_ub", tcc_ir_opt_uninit_local_ub_ex, 0, IR_PASS_INVALIDATES_ALL, FLAG(opt_dce)), PASS_GATED("uninit_dom_ret", tcc_ir_opt_uninit_dominates_return_ex, 0, IR_PASS_INVALIDATES_ALL, FLAG(opt_dce)), + /* Before dce, which then takes the compares and counters the folded branches read. */ + PASS_GATED("unreachable_fold", tcc_ir_opt_unreachable_fold_ex, 0, IR_PASS_INVALIDATES_ALL, FLAG(opt_jump_threading)), PASS_GATED("dce", tcc_ir_opt_dce_ex, 0, IR_PASS_INVALIDATES_DU, FLAG(opt_dce)), + /* A constant local table only read through its address is read from .rodata instead. */ + PASS_GATED("const_local_table", tcc_ir_opt_const_local_table_ex, 0, IR_PASS_INVALIDATES_ALL, FLAG(opt_const_prop)), + /* A local filled by a whole copy of a by-value struct parameter becomes the parameter's own memory. */ + PASS_GATED("param_copy_alias", tcc_ir_opt_param_copy_alias_ex, 0, IR_PASS_INVALIDATES_ALL, FLAG(opt_store_load_fwd)), /* Resolves derefs through single-def &local pointers before the const/forwarding cluster below sees them. */ PASS_GATED("ptr_local_fwd", tcc_ir_opt_ptr_local_fwd_ex, 0, IR_PASS_INVALIDATES_DU, FLAG(opt_store_load_fwd)), /* Must follow dce: its prologue clears stale `addrtaken` flags on VARs whose LEA was just DCE-ed. */ @@ -195,6 +201,8 @@ static const IROptPass late_cleanup_passes[] = { /* VLA_ALLOC(size=0) -> NOP, so dead_lea_store (which bails on any VLA_ALLOC) can clean the stack scaffolding. */ PASS_GATED("zero_vla", tcc_ir_opt_zero_vla_elim_ex, 0, IR_PASS_INVALIDATES_ALL, FLAG(opt_dead_store)), PASS_GATED("byte_store_merge", tcc_ir_opt_byte_store_merge_ex, 0, IR_PASS_INVALIDATES_DU, FLAG(opt_redundant_store)), + /* Byte/halfword field stores over a zero-filled local word fold into its word store. */ + PASS_GATED("slot_const_store_fold", tcc_ir_opt_slot_const_store_fold_ex, 0, IR_PASS_INVALIDATES_DU, FLAG(opt_redundant_store)), PASS_GATED("dse", tcc_ir_opt_dse_ex, 0, IR_PASS_INVALIDATES_DU, FLAG(opt_dead_store)), /* Only fires with ir_late_reopt_phase + sym->a.tu_no_readers; must precede the DCE cascade that frees its RHS. */ PASS_GATED("dead_static_store", tcc_ir_opt_dead_static_store_elim_ex, 0, IR_PASS_INVALIDATES_DU, FLAG(opt_dead_store)), diff --git a/source/opt/flat/Makefile b/source/opt/flat/Makefile index 831c75548..29d316c58 100644 --- a/source/opt/flat/Makefile +++ b/source/opt/flat/Makefile @@ -13,16 +13,13 @@ FLAT_OPT_INC = -I$(TOP)/source/opt/flat -I$(TOP)/source/opt/flat/include \ FLAT_OPT_SRC = \ source/opt/flat/scalar/bool.c \ source/opt/flat/scalar/branch.c \ - source/opt/flat/scalar/branch_fold.c \ source/opt/flat/scalar/stack_bool_diamond.c \ source/opt/flat/scalar/const_string_calls.c \ source/opt/flat/scalar/const_call_fold.c \ source/opt/flat/scalar/addrof_const_fold.c \ source/opt/flat/scalar/global_init_prop.c \ source/opt/flat/scalar/symref_const_prop.c \ - source/opt/flat/scalar/complex_const_param_fold.c \ source/opt/flat/scalar/stack_addr_fold.c \ - source/opt/flat/scalar/single_value_tmp.c \ source/opt/flat/scalar/call_result.c \ source/opt/flat/scalar/self_arith.c \ source/opt/flat/scalar/narrow_store.c \ @@ -38,12 +35,12 @@ FLAT_OPT_SRC = \ source/opt/flat/scalar/add_reassoc.c \ source/opt/flat/scalar/cmp_field_fuse.c \ source/opt/flat/scalar/cmp_expr_fold.c \ - source/opt/flat/scalar/float_narrow.c \ source/opt/flat/scalar/bitfield.c \ source/opt/flat/memory/symaddr_cse.c \ source/opt/flat/memory/dse.c \ source/opt/flat/memory/var_store_elim.c \ source/opt/flat/memory/entry_store_prop.c \ + source/opt/flat/memory/sret_nrvo.c \ source/opt/flat/memory/sl_forward.c \ source/opt/flat/memory/copy_source_load_fwd.c \ source/opt/flat/memory/dead_local_slot.c \ @@ -65,6 +62,7 @@ FLAT_OPT_SRC = \ source/opt/flat/fusion/pack64.c \ source/opt/flat/fusion/pack64_tautology.c \ source/opt/flat/fusion/cmp_narrow_64.c \ + source/opt/flat/fusion/cmp_hi_only.c \ source/opt/flat/fusion/cmp_imm_swap.c \ source/opt/flat/fusion/and64_narrow.c \ source/opt/flat/fusion/shl32_or_chain.c \ @@ -101,7 +99,6 @@ FLAT_OPT_SRC = \ source/opt/flat/loop/seq_guard_elim.c \ source/opt/flat/loop/decrement_to_zero.c \ source/opt/flat/loop/const_sim.c \ - source/opt/flat/scalar/neg_chain_cse.c \ source/opt/flat/loop/reroll.c \ source/opt/flat/cfg/switch_to_data.c \ source/opt/flat/dce/dead_vla.c \ @@ -117,12 +114,10 @@ FLAT_OPT_SRC = \ source/opt/flat/loop/licm.c \ source/opt/flat/loop/stack_param_promote.c \ source/opt/flat/memory/invariant_global_load_hoist.c \ - source/opt/flat/memory/invariant_temp_deref_hoist.c \ source/opt/flat/memory/global_deref_cse.c \ source/opt/flat/memory/deref_operand_cse.c \ source/opt/flat/memory/bitfield_unit_narrow.c \ source/opt/flat/memory/rmw_byte_clear.c \ - source/opt/flat/memory/local_copy_prop.c \ source/opt/flat/memory/struct_copy_roundtrip_elim.c \ source/opt/flat/fusion/barrel_shift.c \ source/opt/flat/fusion/shift_pair_ubfx.c \ @@ -137,11 +132,15 @@ FLAT_OPT_SRC = \ source/opt/flat/memory/dead_init_via_call.c \ source/opt/flat/memory/stack_addr_cse.c \ source/opt/flat/memory/block_copy_init.c \ + source/opt/flat/memory/const_local_table.c \ source/opt/flat/memory/small_memset_to_store.c \ source/opt/flat/memory/small_global_memset_to_store.c \ source/opt/flat/memory/mem_init.c \ source/opt/flat/memory/addrof_var_fwd.c \ source/opt/flat/memory/ptr_local_fwd.c \ + source/opt/flat/memory/param_copy_alias.c \ + source/opt/flat/memory/slot_const_store_fold.c \ + source/opt/flat/memory/sra.c \ source/opt/flat/memory/memmove_to_indexed_stores.c # Header set the units depend on (force a rebuild on interface edits). diff --git a/source/opt/flat/cfg/jump_thread.c b/source/opt/flat/cfg/jump_thread.c index dbe417eda..492434193 100644 --- a/source/opt/flat/cfg/jump_thread.c +++ b/source/opt/flat/cfg/jump_thread.c @@ -202,3 +202,133 @@ int tcc_ir_opt_eliminate_fallthrough(TCCIRState *ir) } int tcc_ir_opt_eliminate_fallthrough_ex(IROptCtx *ctx) { return tcc_ir_opt_eliminate_fallthrough(ctx->ir); } + +/* ---- Branches into __builtin_unreachable ---- + * + * A TRAP marked `unreachable` (the frontend's __builtin_unreachable) is never + * reached, and neither is any instruction from which every path runs into one + * without doing anything observable on the way: those are DOOMED. A + * conditional branch with one doomed side therefore always goes the other + * way. It is folded -- dropped when its target is doomed, made unconditional + * when its fall-through is -- and the compare, the loop counter or the switch + * range check that fed it is left to DCE. The Zig C backend ends every + * exhaustive switch with `default: zig_unreachable();` (3,400 in zig.c) and + * bounds loops whose exit is unreachable. + * + * "Observable" is read conservatively: only register-to-register value + * instructions and non-volatile loads are passed over; a store, a call, an + * atomic or anything else keeps its path. */ +static int unreach_passes_over(TCCIRState *ir, IRQuadCompact *q) +{ + switch (q->op) + { + case TCCIR_OP_NOP: + case TCCIR_OP_ASSIGN: + case TCCIR_OP_LEA: + case TCCIR_OP_CMP: + case TCCIR_OP_TEST_ZERO: + case TCCIR_OP_SETIF: + case TCCIR_OP_ADD: + case TCCIR_OP_SUB: + case TCCIR_OP_MUL: + case TCCIR_OP_AND: + case TCCIR_OP_OR: + case TCCIR_OP_XOR: + case TCCIR_OP_SHL: + case TCCIR_OP_SAR: + case TCCIR_OP_SHR: + case TCCIR_OP_ROR: + case TCCIR_OP_UBFX: + case TCCIR_OP_SBFX: + case TCCIR_OP_BFI: + case TCCIR_OP_ZEXT: + case TCCIR_OP_PACK64: + case TCCIR_OP_SELECT: + case TCCIR_OP_CLZ: + case TCCIR_OP_RBIT: + case TCCIR_OP_REV: + case TCCIR_OP_REV16: + break; + case TCCIR_OP_LOAD: + if (tcc_ir_access_is_volatile(ir, tcc_ir_op_get_src1(ir, q))) + return 0; + break; + default: + return 0; + } + /* The value lands in a register or a VAR: a store through a pointer is no + * value instruction. */ + if (irop_config[q->op].has_dest) + { + IROperand d = tcc_ir_op_get_dest(ir, q); + if (d.is_lval && !d.is_local) + return 0; + } + return 1; +} + +TCC_DBG_ENV_FLAG(unreach_dbg, "TCC_UNREACH_DBG") + +int tcc_ir_opt_unreachable_fold(TCCIRState *ir) +{ + const int n = ir->next_instruction_index; + int any = 0; + for (int i = 0; i < n && !any; i++) + any = ir->compact_instructions[i].op == TCCIR_OP_TRAP && ir->compact_instructions[i].unreachable; + if (!any) + return 0; + + uint8_t *doomed = tcc_mallocz(n + 1); /* doomed[n] = 0: the epilogue */ + for (int round = 0, changed = 1; changed && round < 64; round++) + { + changed = 0; + for (int p = n - 1; p >= 0; p--) + { + if (doomed[p]) + continue; + IRQuadCompact *q = &ir->compact_instructions[p]; + int d = 0; + if (q->op == TCCIR_OP_TRAP) + d = q->unreachable; + else if (q->op == TCCIR_OP_JUMP || q->op == TCCIR_OP_JUMPIF) + { + int t = (int)irop_get_imm64_ex(ir, tcc_ir_op_get_dest(ir, q)); + d = t >= 0 && t < n && doomed[t] && (q->op == TCCIR_OP_JUMP || doomed[p + 1]); + } + else if (unreach_passes_over(ir, q)) + d = doomed[p + 1]; + if (d) + doomed[p] = 1, changed = 1; + } + } + + int changes = 0; + for (int p = 0; p < n; p++) + { + IRQuadCompact *q = &ir->compact_instructions[p]; + if (q->op != TCCIR_OP_JUMPIF || doomed[p]) + continue; + int t = (int)irop_get_imm64_ex(ir, tcc_ir_op_get_dest(ir, q)); + if (t < 0 || t > n) + continue; + if (doomed[t]) + q->op = TCCIR_OP_NOP; /* never taken */ + else if (doomed[p + 1]) + q->op = TCCIR_OP_JUMP; /* always taken */ + else + continue; + changes++; + TCC_DBG_BLOCK(unreach_dbg) + { + extern const char *funcname; + fprintf(stderr, "[UNREACH] %s %d %s\n", funcname, p, q->op == TCCIR_OP_NOP ? "never-taken" : "always-taken"); + } + } + tcc_free(doomed); + return changes; +} + +int tcc_ir_opt_unreachable_fold_ex(IROptCtx *ctx) +{ + return tcc_ir_opt_unreachable_fold(ctx->ir); +} diff --git a/source/opt/flat/dce/body_essential.c b/source/opt/flat/dce/body_essential.c index 84dd389e0..4c6aac5df 100644 --- a/source/opt/flat/dce/body_essential.c +++ b/source/opt/flat/dce/body_essential.c @@ -1051,6 +1051,7 @@ int tcc_ir_opt_local_only_body_elide(TCCIRState *ir) case TCCIR_OP_BUILTIN_APPLY_ARGS: case TCCIR_OP_BUILTIN_APPLY: case TCCIR_OP_BUILTIN_RETURN: + case TCCIR_OP_RETURN_ADDRESS: case TCCIR_OP_VLA_ALLOC: case TCCIR_OP_VLA_SP_SAVE: case TCCIR_OP_VLA_SP_RESTORE: diff --git a/source/opt/flat/dce/dead_addrvar_elim.c b/source/opt/flat/dce/dead_addrvar_elim.c index cad8e4e8b..acc8f9d66 100644 --- a/source/opt/flat/dce/dead_addrvar_elim.c +++ b/source/opt/flat/dce/dead_addrvar_elim.c @@ -162,6 +162,18 @@ int tcc_ir_opt_dead_addrvar_elim(TCCIRState *ir) var_bit_set(var_read, pos); } } + + /* MLA accumulator is a read the src1/src2 helpers above don't cover. */ + if (q->op == TCCIR_OP_MLA) + { + int32_t vr = irop_get_vreg(tcc_ir_op_get_accum(ir, q)); + if (vr >= 0 && TCCIR_DECODE_VREG_TYPE(vr) == TCCIR_VREG_TYPE_VAR) + { + int pos = TCCIR_DECODE_VREG_POSITION(vr); + if (pos <= max_var) + var_bit_set(var_read, pos); + } + } } /* Pass 2: mark VARs whose LEA pointer escapes (used outside a STORE ptr-copy dest). */ @@ -213,6 +225,22 @@ int tcc_ir_opt_dead_addrvar_elim(TCCIRState *ir) } } } + + /* Uses outside src1/src2: an MLA accumulator, and the base of a + * STORE_POSTINC, which Pass 3 keeps (it also advances the pointer) and so + * must keep the LEA defining it. STORE_INDEXED's base is a store through + * the LEA like STORE's and dies with it in Pass 3. */ + int32_t use_vr = -1; + if (q->op == TCCIR_OP_MLA) + use_vr = irop_get_vreg(tcc_ir_op_get_accum(ir, q)); + else if (q->op == TCCIR_OP_STORE_POSTINC) + use_vr = irop_get_vreg(tcc_ir_op_get_dest(ir, q)); + if (use_vr >= 0 && TCCIR_DECODE_VREG_TYPE(use_vr) == TCCIR_VREG_TYPE_TEMP) + { + int tmp_pos = TCCIR_DECODE_VREG_POSITION(use_vr); + if (tmp_pos <= max_tmp && lea_map[tmp_pos] >= 0 && lea_map[tmp_pos] <= max_var) + var_bit_set(var_read, lea_map[tmp_pos]); + } } /* Pass 2b: mark VARs read when their pointer escapes via FUNCPARAMVAL. */ @@ -290,8 +318,12 @@ int tcc_ir_opt_dead_addrvar_elim(TCCIRState *ir) } } - /* STORE through LEA to dead VAR -> NOP */ - if (q->op == TCCIR_OP_STORE) + /* STORE through LEA to dead VAR -> NOP. STORE_INDEXED too: it carries the + * base in a plain dest, and left behind it would store through the TEMP + * whose LEA was just deleted -- through whatever register the allocator + * gave it (Zig's meta.eql zeroed a u64 local with a memset lowered to two + * STORE_INDEXEDs, and wrote the zeros through its caller's r4 and r5). */ + if (q->op == TCCIR_OP_STORE || q->op == TCCIR_OP_STORE_INDEXED) { IROperand dest = tcc_ir_op_get_dest(ir, q); int32_t vr = irop_get_vreg(dest); diff --git a/source/opt/flat/dce/orphan_cmp_elim.c b/source/opt/flat/dce/orphan_cmp_elim.c index fffc42c14..34ab6a3b2 100644 --- a/source/opt/flat/dce/orphan_cmp_elim.c +++ b/source/opt/flat/dce/orphan_cmp_elim.c @@ -106,7 +106,10 @@ int tcc_ir_opt_orphan_cmp_elim(TCCIRState *ir) else if (q->op != TCCIR_OP_CMP && q->op != TCCIR_OP_TEST_ZERO) continue; - if (q->is_jump_target) + /* A jump target is no reason to keep a compare nobody reads: the jump + * then lands on a NOP. The helper-call form keeps the old rule -- its + * parameters sit ahead of it, where a jump into the middle could land. */ + if (q->is_jump_target && (is_flag_cmp_call || tcc_ir_opt_pass_disabled("orphan_cmp_target"))) continue; /* Nobody reading the flags does not make the comparison free: reaching its diff --git a/source/opt/flat/dce/redundant_cmp.c b/source/opt/flat/dce/redundant_cmp.c index 195a9d827..d0882d504 100644 --- a/source/opt/flat/dce/redundant_cmp.c +++ b/source/opt/flat/dce/redundant_cmp.c @@ -70,10 +70,42 @@ static int rcmp_can_drop(TCCIRState *ir, int keep, int del) return 0; if (tcc_ir_instr_access_is_volatile(ir, qd)) return 0; + /* A 64-bit CMP has no single flag result: the backend lowers it by what its + * reader asks -- EQ/NE get `CMP hi; IT EQ; CMPEQ lo` or `ORRS t, lo, hi`, + * which leave only Z meaningful, the orderings get `CMP lo; SBCS hi`. Two + * identical 64-bit CMPs read under different conditions produce different + * flags (`sr != 0 && (sr < 0) != ...` read ORRS's N as the sign of sr). */ + IROperand k1 = tcc_ir_op_get_src1(ir, qk), k2 = tcc_ir_op_get_src2(ir, qk); + if (irop_is_64bit(k1) || irop_is_64bit(k2)) + return 0; return rcmp_operand_same(tcc_ir_op_get_src1(ir, qk), tcc_ir_op_get_src1(ir, qd)) && rcmp_operand_same(tcc_ir_op_get_src2(ir, qk), tcc_ir_op_get_src2(ir, qd)); } +/* Would codegen turn the compare at `keep` and its JUMPIF at `jmp` into a + * CBZ/CBNZ? It does for a 32-bit `CMP r, #0` branched on EQ/NE (codegen.c, + * TCCIR_OP_CMP), and CBZ sets no flags at all: a compare deleted in favour of + * this one would read whatever flags were left before it. `if (x != 0) { if + * (x < 0) ...` compiled to `cbz r0; bge` and took the branch on the caller's + * flags. The compare is refused as a flag source whenever that can happen -- + * codegen also needs a low register and a short forward branch, but neither + * is known here. */ +static int rcmp_may_become_cbz(TCCIRState *ir, int keep, int jmp) +{ + IRQuadCompact *qk = &ir->compact_instructions[keep]; + IROperand a = tcc_ir_op_get_src1(ir, qk), b = tcc_ir_op_get_src2(ir, qk); + if (irop_is_64bit(a) || irop_is_64bit(b)) + return 0; + if (!(irop_is_immediate(a) && irop_get_imm64_ex(ir, a) == 0) && + !(irop_is_immediate(b) && irop_get_imm64_ex(ir, b) == 0)) + return 0; + IROperand c = tcc_ir_op_get_src1(ir, &ir->compact_instructions[jmp]); + if (!irop_is_immediate(c)) + return 1; + int cond = (int)irop_get_imm64_ex(ir, c); + return cond == TOK_EQ || cond == TOK_NE; +} + /* First index at or after `from` holding something other than a NOP. */ static int rcmp_next_real(TCCIRState *ir, int from) { @@ -177,6 +209,8 @@ int tcc_ir_opt_redundant_cmp(TCCIRState *ir) blocked = 1; if (blocked) continue; + if (rcmp_may_become_cbz(ir, i, j)) + continue; IROperand tgt = tcc_ir_op_get_dest(ir, &ir->compact_instructions[j]); if (irop_is_none(tgt)) diff --git a/source/opt/flat/fusion/add_deref_fold.c b/source/opt/flat/fusion/add_deref_fold.c index 8a2fd173a..fc7612127 100644 --- a/source/opt/flat/fusion/add_deref_fold.c +++ b/source/opt/flat/fusion/add_deref_fold.c @@ -97,6 +97,20 @@ int tcc_ir_opt_add_deref_fold(TCCIRState *ir) int32_t cs1_vr = irop_get_vreg(cs1); if (cs1_vr < 0 || TCCIR_DECODE_VREG_TYPE(cs1_vr) != TCCIR_VREG_TYPE_PARAM) continue; + /* The load moves to the ADD and reads through the PARAM there: the + * PARAM must still hold what was copied (`q = p; p++; q[2]`). */ + int param_redefined = 0; + for (int j = copy_idx + 1; j < i && !param_redefined; j++) + { + IRQuadCompact *rq = &ir->compact_instructions[j]; + if (rq->op != TCCIR_OP_NOP && irop_config[rq->op].has_dest) + { + IROperand rd = tcc_ir_op_get_dest(ir, rq); + param_redefined = irop_get_vreg(rd) == cs1_vr && irop_dest_defines_vreg(rd); + } + } + if (param_redefined) + continue; /* Use the PARAM source as base; leave the copy for DCE. Extra readers * of the TEMP base are fine since the copy stays put. */ src1 = cs1; diff --git a/source/opt/flat/fusion/cmp_hi_only.c b/source/opt/flat/fusion/cmp_hi_only.c new file mode 100644 index 000000000..01521f65c --- /dev/null +++ b/source/opt/flat/fusion/cmp_hi_only.c @@ -0,0 +1,151 @@ +/* + * TCC IR - 64-bit compare against a constant with a zero low half + * + * Copyright (c) 2025 Mateusz Stadnik + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation. + */ + +/* + * A 64-bit compare costs two instructions and a scratch register: `cmp rlo, + * #imm_lo` for the low words, then `sbcs rt, rhi, #imm_hi` to fold in the + * borrow. Against a constant whose LOW 32 bits are zero, and for a STRICT + * order test, the low words cannot decide the answer at all: + * + * x x_hi =u C <=> the negation of the above + * + * and the same for the signed pair, where a 64-bit compare orders the high + * words signed and the low words unsigned. So the whole compare is one + * `cmp rhi, #imm_hi` -- one instruction, no borrow, no scratch. + * + * This is the same proof tcc_ir_opt_cmp_narrow_64 applies, moved from the + * VALUE to the CONSTANT. That pass narrows to 32 bits when both operands are + * known to fit in their low words; this one keeps the value 64-bit and throws + * away the half the constant makes irrelevant. The two are disjoint: a + * constant with a zero low half and a non-zero high half never satisfies + * cmp_narrow_64's "high word is provably zero" test. + * + * NON-strict tests are excluded and the asymmetry is the point: `x >u C` is + * true when x_hi == C_hi and x_lo is merely non-zero, which a high-word + * compare cannot see. Only <, >= (constant on the right) survive. EQ/NE are + * excluded for the same reason and are already handled by codegen's cmp_eq64 + * peephole. + * + * Where sfp_round_pack_double's two normalisation guards -- + * `while (mant >= (DOUBLE_NORM_BIT << 1))` and `while (mant < DOUBLE_NORM_BIT)` + * -- are each a compare against 1<<56 / 1<<55. They run 2,310 times over 996 + * __aeabi_dadd calls, so this is ~2.3 instructions and a register per call of + * the hottest routine in the soft-float library. + * + * PURE ANNOTATION, no IR mutation, and it must run LAST for the same reason + * zero_half64 does: the verdict is about the FINAL instruction stream. A pass + * that rewrote the compare's operands afterwards -- swapping them, or + * substituting a different immediate -- would leave the annotation describing + * a compare that no longer exists, and codegen would silently drop a half that + * had become load-bearing. + */ + +#define USING_GLOBALS + +#include "ir.h" +#include "opt_engine.h" +#include "opt_utils.h" + +/* Bit 6 of the byte tcc_ir_zero_half64_at returns -- kept in sync with the + * decoder in thumb_emit_data_processing_mop64, which reads it from bit 24 of + * its `barrel_shift` word. It shares the array with zero_half64.c's bits + * because both answer the same question ("which halves does this instruction + * really need?") for the same consumer; it does NOT share the pass, because + * zero_half64 clears the array when it runs and this must be applied after. */ +#define ZH64_CMP_HI 0x40u + +/* Conditions under which a strict high-word compare gives the same answer as + * the full 64-bit one, with the constant as src2. Both orders are covered: + * the 64-bit lowering compares high words signed for LT/GE and unsigned for + * ULT/UGE, and a single high-word CMP feeds exactly the same flags to exactly + * the same branch. */ +static int chi_cond_ok(int tok) +{ + return tok == TOK_ULT || tok == TOK_UGE || tok == TOK_LT || tok == TOK_GE; +} + +int tcc_ir_opt_cmp_hi_only(TCCIRState *ir) +{ + const int n = ir->next_instruction_index; + int changes = 0; + + if (n < 2 || ir->max_orig_index < 0) + return 0; + + for (int i = 0; i < n; i++) + { + IRQuadCompact *q = &ir->compact_instructions[i]; + if (q->op != TCCIR_OP_CMP) + continue; + if (q->orig_index < 0) + continue; + + IROperand s1 = tcc_ir_op_get_src1(ir, q); + IROperand s2 = tcc_ir_op_get_src2(ir, q); + if (irop_get_btype(s1) != IROP_BTYPE_INT64) + continue; + if (!irop_is_immediate(s2) || irop_get_btype(s2) != IROP_BTYPE_INT64) + continue; + /* The low half is what makes the transform exact. Anything else and the + * low words still decide ties. */ + if (((uint64_t)irop_get_imm64_ex(ir, s2) & 0xFFFFFFFFull) != 0) + continue; + + /* Every condition applied to these flags must be one of the four. The + * scan mirrors codegen's own notion of a flag consumer: NOPs and ASSIGNs + * are transparent (an ASSIGN lowers to `mov`, which preserves flags, and + * phi resolution schedules them between the compare and its branch), and + * anything else ends the run. */ + int readers = 0; + int all_ok = 1; + for (int j = i + 1; j < n && all_ok; j++) + { + IRQuadCompact *qj = &ir->compact_instructions[j]; + if (qj->op == TCCIR_OP_NOP || qj->op == TCCIR_OP_ASSIGN) + continue; + IROperand c; + if (qj->op == TCCIR_OP_SETIF || qj->op == TCCIR_OP_JUMPIF) + c = tcc_ir_op_get_src1(ir, qj); + else if (qj->op == TCCIR_OP_SELECT) + c = tcc_ir_op_get_cond(ir, qj); + else + break; + if (!irop_is_immediate(c) || !chi_cond_ok((int)irop_get_imm64_ex(ir, c))) + { + all_ok = 0; + break; + } + readers++; + } + if (!all_ok || readers == 0) + continue; + + /* Share zero_half64's side table. It has already run and either allocated + * and filled the array or left it NULL (its pass can be disabled on its + * own); allocate but never clear, or its verdicts go with it. */ + if (!ir->zero_half64) + { + ir->zero_half64 = tcc_mallocz((size_t)ir->max_orig_index + 1); + ir->zero_half64_len = ir->max_orig_index + 1; + } + if (q->orig_index >= ir->zero_half64_len) + continue; + + LOG_IR_GEN("OPTIMIZE: cmp_hi_only at i=%d (constant low half is zero)", i); + ir->zero_half64[q->orig_index] |= ZH64_CMP_HI; + changes++; + } + + return changes; +} + +int tcc_ir_opt_cmp_hi_only_ex(IROptCtx *ctx) { return tcc_ir_opt_cmp_hi_only(ctx->ir); } diff --git a/source/opt/flat/fusion/indexed_chain.c b/source/opt/flat/fusion/indexed_chain.c index f94a40772..74988993c 100644 --- a/source/opt/flat/fusion/indexed_chain.c +++ b/source/opt/flat/fusion/indexed_chain.c @@ -70,7 +70,10 @@ OPT_GEN_FLAT(indexed_chain, TCCIR_OP_LOAD_INDEXED) return 0; } - if (new_base.is_local || new_base.is_llocal) + /* An lval operand is a memory read: `T <- P***DEREF*** ADD #k` adds k to the + * value stored at P, not to P. Folding it would drop that load and address + * P + k -- `l.ptr[1].f` became a read of the frame slot `&l + 2096`. */ + if (new_base.is_local || new_base.is_llocal || new_base.is_lval) return 0; long long imm_total = (long long)imm1 + imm2; @@ -80,7 +83,6 @@ OPT_GEN_FLAT(indexed_chain, TCCIR_OP_LOAD_INDEXED) if (!ir_xform_same_block(ir, add_idx, i)) return 0; - new_base.is_lval = 0; new_base.btype = base_op.btype; /* The chained base addresses the same object: keep its access marks, so the * 64-bit lowering stays off LDRD/STRD and a volatile access stays visible. */ diff --git a/source/opt/flat/fusion/lea_fold.c b/source/opt/flat/fusion/lea_fold.c index 528682f1b..0ffbe153f 100644 --- a/source/opt/flat/fusion/lea_fold.c +++ b/source/opt/flat/fusion/lea_fold.c @@ -44,6 +44,159 @@ extern int gsym_cse_insert_before(TCCIRState *ir, int before_idx, IRQuadCompact * STORE dest through the pointer (indistinguishable from a direct stack store). */ +/* Where each TEMP occurs (as dest, src1 or src2), for the pass's use scans. + * The fold only ever takes vregs out of instructions -- it NOPs the LEA and + * the ADD and replaces the consumer's operand with a StackLoc -- so a TEMP can + * appear nowhere but the instructions listed here when the pass starts, and + * walking the list with the scans' own per-instruction tests gives exactly + * what walking every instruction did. Walking to the end of the function to + * prove a LEA's result has one use made the pass quadratic: 2.8 G operand + * checks on Zig's zig.c at -O2. */ +typedef struct LfOcc +{ + int *start; /* per TEMP position: its instructions are idx[start[p] .. start[p + 1]) */ + int *idx; + int npos; + int *jumps; /* jumps[k]: non-NOP JUMP/JUMPIF before k (the pass keeps them all) */ +} LfOcc; + +static int lf_temp_pos(int32_t vr, int npos) +{ + if (vr < 0 || TCCIR_DECODE_VREG_TYPE(vr) != TCCIR_VREG_TYPE_TEMP) + return -1; + int p = TCCIR_DECODE_VREG_POSITION(vr); + if (vr != TCCIR_ENCODE_VREG(TCCIR_VREG_TYPE_TEMP, p) || p >= npos) + return -1; + return p; +} + +/* The TEMP positions instruction `q` names, at most three. */ +static int lf_insn_temps(TCCIRState *ir, IRQuadCompact *q, int npos, int out[3]) +{ + const IRRegistersConfig *cfg = &irop_config[q->op]; + int32_t v[3] = {-1, -1, -1}; + int m = 0; + if (cfg->has_src1) + v[0] = irop_get_vreg(tcc_ir_op_get_src1(ir, q)); + if (cfg->has_src2) + v[1] = irop_get_vreg(tcc_ir_op_get_src2(ir, q)); + if (cfg->has_dest) + v[2] = irop_get_vreg(tcc_ir_op_get_dest(ir, q)); + for (int k = 0; k < 3; k++) + { + int p = lf_temp_pos(v[k], npos); + if (p < 0) + continue; + int dup = 0; + for (int e = 0; e < m; e++) + dup |= out[e] == p; + if (!dup) + out[m++] = p; + } + return m; +} + +static void lf_occ_build(TCCIRState *ir, int n, LfOcc *o) +{ + int npos = 0; + for (int i = 0; i < n; i++) + { + IRQuadCompact *q = &ir->compact_instructions[i]; + if (q->op == TCCIR_OP_NOP) + continue; + const IRRegistersConfig *cfg = &irop_config[q->op]; + int32_t v[3] = {cfg->has_src1 ? irop_get_vreg(tcc_ir_op_get_src1(ir, q)) : -1, + cfg->has_src2 ? irop_get_vreg(tcc_ir_op_get_src2(ir, q)) : -1, + cfg->has_dest ? irop_get_vreg(tcc_ir_op_get_dest(ir, q)) : -1}; + for (int k = 0; k < 3; k++) + if (v[k] >= 0 && TCCIR_DECODE_VREG_TYPE(v[k]) == TCCIR_VREG_TYPE_TEMP && + TCCIR_DECODE_VREG_POSITION(v[k]) >= npos) + npos = TCCIR_DECODE_VREG_POSITION(v[k]) + 1; + } + o->npos = npos; + o->start = tcc_mallocz(sizeof(int) * (npos + 1)); + o->jumps = tcc_malloc(sizeof(int) * (n + 1)); + int t[3]; + o->jumps[0] = 0; + for (int i = 0; i < n; i++) + { + IRQuadCompact *q = &ir->compact_instructions[i]; + o->jumps[i + 1] = o->jumps[i] + (q->op == TCCIR_OP_JUMP || q->op == TCCIR_OP_JUMPIF); + if (q->op == TCCIR_OP_NOP) + continue; + int m = lf_insn_temps(ir, q, npos, t); + for (int k = 0; k < m; k++) + o->start[t[k] + 1]++; + } + for (int p = 0; p < npos; p++) + o->start[p + 1] += o->start[p]; + o->idx = tcc_malloc(sizeof(int) * (o->start[npos] + 1)); + int *fill = tcc_malloc(sizeof(int) * (npos + 1)); + memcpy(fill, o->start, sizeof(int) * (npos + 1)); + for (int i = 0; i < n; i++) + { + IRQuadCompact *q = &ir->compact_instructions[i]; + if (q->op == TCCIR_OP_NOP) + continue; + int m = lf_insn_temps(ir, q, npos, t); + for (int k = 0; k < m; k++) + o->idx[fill[t[k]]++] = i; + } + tcc_free(fill); +} + +static void lf_occ_free(LfOcc *o) +{ + tcc_free(o->start); + tcc_free(o->idx); + tcc_free(o->jumps); +} + +/* Instructions after `after` that may name `vr`, in order: its listed ones, or + * every instruction for a vreg the index does not cover. */ +typedef struct LfCursor +{ + const int *p, *end; /* listed: the remaining entries */ + int k, n; /* unlisted: the next index and the bound */ +} LfCursor; + +static void lf_cursor_init(LfCursor *c, const LfOcc *o, int32_t vr, int after, int n) +{ + int pos = lf_temp_pos(vr, o->npos); + if (pos < 0) + { + c->p = c->end = NULL; + c->k = after + 1; + c->n = n; + return; + } + int lo = o->start[pos], hi = o->start[pos + 1]; + while (lo < hi) + { + int mid = (lo + hi) / 2; + if (o->idx[mid] <= after) + lo = mid + 1; + else + hi = mid; + } + c->p = o->idx + lo; + c->end = o->idx + o->start[pos + 1]; + c->k = c->n = 0; +} + +static int lf_cursor_next(LfCursor *c) +{ + if (c->p) + return c->p < c->end ? *c->p++ : -1; + return c->k < c->n ? c->k++ : -1; +} + +/* A JUMP or JUMPIF strictly between `a` and `b`. */ +static int lf_jump_between(const LfOcc *o, int a, int b) +{ + return b > a + 1 && o->jumps[b] - o->jumps[a + 1] > 0; +} + int tcc_ir_opt_lea_fold(TCCIRState *ir) { int n = ir->next_instruction_index; @@ -54,6 +207,8 @@ int tcc_ir_opt_lea_fold(TCCIRState *ir) IROptDU du; ir_opt_du_build(ir, &du); + LfOcc occ; + lf_occ_build(ir, n, &occ); LOG_IR_GEN("=== LEA FOLD START (n=%d) ===", n); @@ -106,7 +261,9 @@ int tcc_ir_opt_lea_fold(TCCIRState *ir) * explicit linear scan that treats is_lval=1 dest positions as uses. */ { int total_uses = 0; - for (int k = i + 1; k < n && total_uses < 2; k++) + LfCursor cur; + lf_cursor_init(&cur, &occ, lea_vr, i, n); + for (int k = lf_cursor_next(&cur); k >= 0 && total_uses < 2; k = lf_cursor_next(&cur)) { IRQuadCompact *uq = &ir->compact_instructions[k]; if (uq->op == TCCIR_OP_NOP) @@ -153,12 +310,16 @@ int tcc_ir_opt_lea_fold(TCCIRState *ir) /* Find the single use of the LEA result. */ int cur_idx = -1; - for (int j = i + 1; j < n; j++) + LfCursor ucur; + lf_cursor_init(&ucur, &occ, lea_vr, i, n); + for (int j = lf_cursor_next(&ucur); j >= 0; j = lf_cursor_next(&ucur)) { IRQuadCompact *uq = &ir->compact_instructions[j]; + /* Same-block check: the use must precede any control-flow edge. */ + if (lf_jump_between(&occ, i, j)) + break; if (uq->op == TCCIR_OP_NOP) continue; - /* Same-block check: the use must precede any control-flow edge. */ if (uq->op == TCCIR_OP_JUMP || uq->op == TCCIR_OP_JUMPIF) break; const IRRegistersConfig *cfg = &irop_config[uq->op]; @@ -221,7 +382,9 @@ int tcc_ir_opt_lea_fold(TCCIRState *ir) int add_uses_real = 0; if (add_vr >= 0 && TCCIR_DECODE_VREG_TYPE(add_vr) == TCCIR_VREG_TYPE_TEMP) { - for (int k = cur_idx + 1; k < n && add_uses_real < 2; k++) + LfCursor acur; + lf_cursor_init(&acur, &occ, add_vr, cur_idx, n); + for (int k = lf_cursor_next(&acur); k >= 0 && add_uses_real < 2; k = lf_cursor_next(&acur)) { IRQuadCompact *uq2 = &ir->compact_instructions[k]; if (uq2->op == TCCIR_OP_NOP) @@ -261,9 +424,13 @@ int tcc_ir_opt_lea_fold(TCCIRState *ir) /* Find the consumer of add_vr in the same block. */ int cons_idx = -1; - for (int k = add_idx + 1; k < n; k++) + LfCursor ccur; + lf_cursor_init(&ccur, &occ, add_vr, add_idx, n); + for (int k = lf_cursor_next(&ccur); k >= 0; k = lf_cursor_next(&ccur)) { IRQuadCompact *ck = &ir->compact_instructions[k]; + if (lf_jump_between(&occ, add_idx, k)) + break; if (ck->op == TCCIR_OP_NOP) continue; if (ck->op == TCCIR_OP_JUMP || ck->op == TCCIR_OP_JUMPIF) @@ -332,6 +499,8 @@ int tcc_ir_opt_lea_fold(TCCIRState *ir) width_op.btype); stack_op.is_unsigned = width_op.is_unsigned; stack_op.is_static = lea_src.is_static; + /* An indexed access keeps its marks on the base operand. */ + irop_carry_access_marks(&stack_op, base); IROperand orig_dest = tcc_ir_op_get_dest(ir, cq); cq->op = TCCIR_OP_LOAD; @@ -415,6 +584,10 @@ int tcc_ir_opt_lea_fold(TCCIRState *ir) /*is_param_flag*/ (int)lea_src.is_param, old_op.btype); new_op.is_unsigned = old_op.is_unsigned; new_op.is_static = lea_src.is_static; + /* The access is the same one: its volatility and alignment marks go + * with it. Left unmarked, it reads as volatile in any function that + * touches volatile memory at all (tcc_ir_access_is_volatile). */ + irop_carry_access_marks(&new_op, old_op); } if (which == 1) @@ -436,5 +609,6 @@ int tcc_ir_opt_lea_fold(TCCIRState *ir) LOG_IR_GEN("=== LEA FOLD END: %d folds ===", changes); tcc_free(du.def); + lf_occ_free(&occ); return changes; } diff --git a/source/opt/flat/fusion/lea_rmw_fold.c b/source/opt/flat/fusion/lea_rmw_fold.c index 1c4049b92..6946f2113 100644 --- a/source/opt/flat/fusion/lea_rmw_fold.c +++ b/source/opt/flat/fusion/lea_rmw_fold.c @@ -252,6 +252,7 @@ int tcc_ir_opt_lea_rmw_fold(TCCIRState *ir) /*is_param_flag*/ (int)lea_src.is_param, old_op.btype); new_op.is_unsigned = old_op.is_unsigned; new_op.is_static = lea_src.is_static; + irop_carry_access_marks(&new_op, old_op); /* same access: same volatility */ if (which == 1) tcc_ir_op_set_src1(ir, cq, new_op); else if (which == 2) diff --git a/source/opt/flat/fusion/pack64.c b/source/opt/flat/fusion/pack64.c index dd3841639..c0e099c5b 100644 --- a/source/opt/flat/fusion/pack64.c +++ b/source/opt/flat/fusion/pack64.c @@ -221,11 +221,13 @@ int tcc_ir_opt_pack64_from_stack_stores(TCCIRState *ir) continue; if (!irop_config[kq->op].has_dest) continue; - if (kq->op == TCCIR_OP_STORE || kq->op == TCCIR_OP_STORE_INDEXED || - kq->op == TCCIR_OP_STORE_POSTINC) - continue; /* memory store, doesn't redefine vregs */ + if (kq->op == TCCIR_OP_STORE_INDEXED) + continue; /* its destination is the base address */ + /* A store through a pointer defines nothing, but a VAR stored as + * itself (a STACKOFF lvalue) is redefined, and so is a post-increment + * store's base. */ IROperand kd = tcc_ir_op_get_dest(ir, kq); - if (kd.is_lval) + if (!irop_dest_defines_vreg(kd)) continue; int32_t kdvr = irop_get_vreg(kd); if (kdvr < 0) diff --git a/source/opt/flat/fusion/shift64_dead_half.c b/source/opt/flat/fusion/shift64_dead_half.c index 6ef071e03..e70938749 100644 --- a/source/opt/flat/fusion/shift64_dead_half.c +++ b/source/opt/flat/fusion/shift64_dead_half.c @@ -111,8 +111,10 @@ static int s64_mark_dead_hi(TCCIRState *ir, int n) if (q->orig_index < 0 || q->orig_index > ir->max_orig_index) continue; /* Single use keeps the consumer unambiguous (and rejects the MLA-accumulator - * blind spot, which tcc_ir_vreg_has_single_use reports as multi-use). */ - if (!tcc_ir_vreg_has_single_use(ir, dv, i)) + * blind spot, which tcc_ir_vreg_has_single_use reports as multi-use). A + * graph-coalesced value is also read under its class members' names, which + * no use count of its own sees (tcc_ir_vreg_coalesced). */ + if (!tcc_ir_vreg_has_single_use(ir, dv, i) || tcc_ir_vreg_coalesced(ir, dv)) continue; int low_only = 0; @@ -230,7 +232,7 @@ int tcc_ir_opt_shift64_dead_half(TCCIRState *ir) continue; if (irop_get_btype(tcc_ir_op_get_dest(ir, def)) != IROP_BTYPE_INT64) continue; - if (!tcc_ir_vreg_has_single_use(ir, s1_vr, dpos)) + if (!tcc_ir_vreg_has_single_use(ir, s1_vr, dpos) || tcc_ir_vreg_coalesced(ir, s1_vr)) continue; if (!ir->shift64_dead_half) { diff --git a/source/opt/flat/fusion/stackoff_addr_cse.c b/source/opt/flat/fusion/stackoff_addr_cse.c index aab78166b..9a095c7d9 100644 --- a/source/opt/flat/fusion/stackoff_addr_cse.c +++ b/source/opt/flat/fusion/stackoff_addr_cse.c @@ -174,8 +174,11 @@ int tcc_ir_opt_stackoff_addr_cse(TCCIRState *ir) * ------------------------------------------------------------------------- */ /* A prologue `ASSIGN Tn <- Addr[StackLoc[off]]` whose Tn is never redefined, * or -1. The single-definition scan covers the whole function, not just the - * prologue: a later write to Tn would make the reuse read a different address. */ -static int32_t sib_find_prologue_addr_temp(TCCIRState *ir, int prologue_end, int32_t off) + * prologue: a later write to Tn would make the reuse read a different address. + * `*def_idx` receives the ASSIGN's index -- the prologue dominates the whole + * function, but only from that point on, so a use AHEAD of it in the prologue + * is not covered and the caller must leave it alone. */ +static int32_t sib_find_prologue_addr_temp(TCCIRState *ir, int prologue_end, int32_t off, int *def_idx) { int n = ir->next_instruction_index; for (int i = 0; i < prologue_end; i++) @@ -212,7 +215,10 @@ static int32_t sib_find_prologue_addr_temp(TCCIRState *ir, int prologue_end, int defs++; } if (defs == 1) + { + *def_idx = i; return vr; + } } return -1; } @@ -229,6 +235,7 @@ int tcc_ir_opt_stackoff_indexed_base_cse(TCCIRState *ir) int count; int32_t hoisted_vreg; int32_t existing_vreg; /* prologue TEMP already holding this address, or -1 */ + int existing_def; /* index of that TEMP's ASSIGN; uses before it stay raw */ IROperand sample; } slots[SIB_MAX_OFFSETS]; int nslots = 0; @@ -280,7 +287,9 @@ int tcc_ir_opt_stackoff_indexed_base_cse(TCCIRState *ir) slots[slot].offset = off; slots[slot].count = 0; slots[slot].hoisted_vreg = -1; - slots[slot].existing_vreg = sib_find_prologue_addr_temp(ir, prologue_end, off); + slots[slot].existing_def = -1; + slots[slot].existing_vreg = + sib_find_prologue_addr_temp(ir, prologue_end, off, &slots[slot].existing_def); slots[slot].sample = base; } slots[slot].count++; @@ -324,6 +333,14 @@ int tcc_ir_opt_stackoff_indexed_base_cse(TCCIRState *ir) if (slots[s].offset == base.u.imm32) { slot = s; break; } if (slot < 0 || slots[slot].hoisted_vreg < 0) continue; + /* A reused prologue temp is only defined from its ASSIGN onwards. The + * prologue is one straight-line block, so an access EARLIER in it would + * read the temp before anything wrote it -- it keeps its own raw base + * (agg_deep fuzz seed 1637: `arr[i]` read at 0054 rewritten to a temp + * assigned at 0071, which at -O2 loaded through whatever the caller left + * in r4). */ + if (slots[slot].existing_vreg >= 0 && i < slots[slot].existing_def) + continue; IROperand rep = irop_make_vreg(slots[slot].hoisted_vreg, base.btype); rep.is_unsigned = base.is_unsigned; /* Keep the packed-access mark: the 64-bit indexed lowering reads it to diff --git a/source/opt/flat/fusion/zero_half64.c b/source/opt/flat/fusion/zero_half64.c index a20a8d6c0..87d136356 100644 --- a/source/opt/flat/fusion/zero_half64.c +++ b/source/opt/flat/fusion/zero_half64.c @@ -204,8 +204,14 @@ static int zh_use_ignores_half(TCCIRState *ir, IRQuadCompact *u, int32_t vreg, i IROperand s2 = tcc_ir_op_get_src2(ir, u); int m1 = (irop_get_vreg(s1) == vreg) && irop_needs_pair(s1); int m2 = (irop_get_vreg(s2) == vreg) && irop_needs_pair(s2); + /* A word-typed operand naming the pair reads its LOW register (machine + * operands take their width from the operand, not the vreg). */ + int n1 = (irop_get_vreg(s1) == vreg) && !irop_needs_pair(s1) && !s1.is_lval; + int n2 = (irop_get_vreg(s2) == vreg) && !irop_needs_pair(s2) && !s2.is_lval; + if ((n1 || n2) && half == ZK_LO) + return 0; if (!m1 && !m2) - return 1; /* named only in a narrow slot, or not at all: reads no half */ + return 1; /* named only in a narrow slot (low word, handled above), or not at all */ /* Every shortcut below lives in the 64-bit pair lowering. A narrow * destination sends the same opcode to thumb_emit_data_processing_mop32, @@ -336,6 +342,25 @@ int tcc_ir_opt_zero_half64(TCCIRState *ir) st.ok_t[p] = (uint8_t)(cnt_t[p] == 1); for (int p = 0; p <= max_v; p++) st.ok_v[p] = (uint8_t)(cnt_v[p] == 1); + /* A value in a graph-coalesced class shares its register with the other + * members, and the copies between them are gone: their defs write it and + * their uses read it under other names. Its halves are neither known from + * its own def nor dead for want of readers of its own name -- expr_preprocess + * in tccpp.c lost `c = 0` (an int64 whose only reader, the phi copy into + * `c`, had been coalesced away), so `#if defined X` was true for every X + * in the native tcc. */ + for (int k = 0; k < ir->ls.next_interval_index; k++) + { + const LSLiveInterval *li = &ir->ls.intervals[k]; + if (!li->co_member) + continue; + int32_t vr = (int32_t)li->vreg; + int t = TCCIR_DECODE_VREG_TYPE(vr), p = TCCIR_DECODE_VREG_POSITION(vr); + if (t == TCCIR_VREG_TYPE_TEMP && p >= 0 && p <= max_t) + st.ok_t[p] = 0; + else if (t == TCCIR_VREG_TYPE_VAR && p >= 0 && p <= max_v) + st.ok_v[p] = 0; + } /* zh_var_addr_taken walks the whole body, so ask it once per local. */ for (int i = 0; i < n && max_v >= 0; i++) { @@ -471,6 +496,12 @@ int tcc_ir_opt_zero_half64(TCCIRState *ir) IRQuadCompact *u = &ir->compact_instructions[j]; if (u->op == TCCIR_OP_NOP) continue; + /* An accumulator (MLA's UMLAL form, UMAAL) is read whole: both words. */ + if (tcc_ir_op_is_mac(u->op) && irop_get_vreg(tcc_ir_op_get_accum(ir, u)) == dv) + { + all_ignore = 0; + break; + } if (irop_get_vreg(tcc_ir_op_get_src1(ir, u)) != dv && irop_get_vreg(tcc_ir_op_get_src2(ir, u)) != dv) continue; diff --git a/source/opt/flat/ipa/tu_dead_statics.c b/source/opt/flat/ipa/tu_dead_statics.c index fc77bde4a..11d643fc9 100644 --- a/source/opt/flat/ipa/tu_dead_statics.c +++ b/source/opt/flat/ipa/tu_dead_statics.c @@ -14,7 +14,30 @@ #include "opt.h" #include "tu_summary.h" - +/* Which symbols a relocation of emitted DATA refers to (by ELF index): an + * object whose address is stored in data escapes where no function summary + * can see it, and a function named in data can be called through it. Code + * needs no such scan: every function's own address uses were classified by + * tcc_ir_collect_tu_static_reads_preopt (anything but an address derivation + * counts as a read). The frontend's addrtaken is no substitute -- it is set + * by every `&x`, and zig's C writes every field store as + * `*(&((T *)&x)->f) = v`. */ +static uint8_t *tu_named_in_data(int *count) +{ + *count = symtab_section ? symtab_section->data_offset / sizeof(ElfW(Sym)) : 0; + uint8_t *in_data = tcc_mallocz(*count ? *count : 1); + for (int sh = 1; sh < tcc_state->nb_sections; sh++) + { + Section *sec = tcc_state->sections[sh]; + if (!sec || !(sec->sh_flags & SHF_ALLOC) || (sec->sh_flags & SHF_EXECINSTR) || !sec->reloc) + continue; + ElfW_Rel *rel = (ElfW_Rel *)sec->reloc->data, *rel_end = (ElfW_Rel *)(sec->reloc->data + sec->reloc->data_offset); + for (; rel < rel_end; rel++) + if (ELFW(R_SYM)(rel->r_info) < (unsigned)*count) + in_data[ELFW(R_SYM)(rel->r_info)] = 1; + } + return in_data; +} /* Statics never read by a reachable function get tu_no_readers; writers get func_late_reopt. */ void tcc_ir_tu_analyze_dead_statics(void) @@ -25,6 +48,8 @@ void tcc_ir_tu_analyze_dead_statics(void) total++; if (total == 0) return; + int nsyms; + uint8_t *in_data = tu_named_in_data(&nsyms); /* Mark reachable: BFS over the call graph. */ for (TuFuncSummary *e = tu_summary_head; e; e = e->next) @@ -35,7 +60,7 @@ void tcc_ir_tu_analyze_dead_statics(void) continue; if (!(fs->type.t & VT_STATIC)) is_root = 1; - if (fs->a.addrtaken) + if (fs->a.addrtaken || (fs->c > 0 && fs->c < nsyms && in_data[fs->c])) is_root = 1; /* Constructors / destructors are entry points called by the runtime. */ if (fs->type.ref && (fs->type.ref->f.func_ctor || fs->type.ref->f.func_dtor)) @@ -81,8 +106,8 @@ void tcc_ir_tu_analyze_dead_statics(void) Sym *g = candidates.items[c]; if (!g) continue; - if (g->a.addrtaken) - continue; /* address escaped; cannot prove no readers */ + if (g->c <= 0 || g->c >= nsyms || in_data[g->c]) + continue; /* address stored in data; cannot prove no readers */ int read_by_reachable = tu_symset_contains(&tu_source_reads, g); for (TuFuncSummary *e = tu_summary_head; e && !read_by_reachable; e = e->next) { @@ -122,4 +147,5 @@ void tcc_ir_tu_analyze_dead_statics(void) } tu_symset_free(&candidates); + tcc_free(in_data); } diff --git a/source/opt/flat/ipa/tu_func_summary.c b/source/opt/flat/ipa/tu_func_summary.c index d1846d447..54fb72e29 100644 --- a/source/opt/flat/ipa/tu_func_summary.c +++ b/source/opt/flat/ipa/tu_func_summary.c @@ -108,7 +108,13 @@ static int tu_is_static_global_candidate(const Sym *sym) return 1; } +/* vreg -> static symbol map. Flow-insensitive, so a vreg maps to a symbol + * only while EVERY definition of it derives that symbol: one that derives + * another, or none, poisons the entry for good (the last definition in + * instruction order is not the one reaching a use in a loop). A map that + * fills up answers nothing. */ #define TU_VREG_MAP_MAX 128 +#define TU_VREG_MAP_OVERFLOW (TU_VREG_MAP_MAX + 1) typedef struct { int32_t vreg; @@ -117,19 +123,25 @@ typedef struct static Sym *tu_vreg_map_lookup(const TuVregSymEntry *map, int count, int32_t vr) { + if (count == TU_VREG_MAP_OVERFLOW) + return NULL; for (int i = 0; i < count; i++) if (map[i].vreg == vr) return map[i].sym; return NULL; } -static void tu_vreg_map_set(TuVregSymEntry *map, int *count, int32_t vr, Sym *sym) +/* Record one definition of vr, deriving sym (NULL: none). */ +static void tu_vreg_map_def(TuVregSymEntry *map, int *count, int32_t vr, Sym *sym) { + if (vr < 0 || *count == TU_VREG_MAP_OVERFLOW) + return; for (int i = 0; i < *count; i++) { if (map[i].vreg == vr) { - map[i].sym = sym; + if (map[i].sym != sym) + map[i].sym = NULL; /* poisoned: NULL never matches a later sym */ return; } } @@ -139,23 +151,77 @@ static void tu_vreg_map_set(TuVregSymEntry *map, int *count, int32_t vr, Sym *sy map[*count].sym = sym; (*count)++; } + else + *count = TU_VREG_MAP_OVERFLOW; } -static void tu_vreg_map_clear(TuVregSymEntry *map, int *count, int32_t vr) +/* The vreg an instruction (re)defines through its destination, or -1: a + * plain value, or a VAR stored as itself (a STACKOFF lvalue, + * `V <- &s [STORE]`) -- not a store through a pointer, nor an indexed store's + * base. A post-increment's base is moved: tu_note_postinc_def. */ +static int32_t tu_dest_def(TCCIRState *ir, IRQuadCompact *q) { - for (int i = 0; i < *count; i++) + if (!irop_config[q->op].has_dest || q->op == TCCIR_OP_STORE_INDEXED || q->op == TCCIR_OP_STORE_POSTINC) + return -1; + IROperand dest = tcc_ir_op_get_dest(ir, q); + return irop_dest_defines_vreg(dest) ? irop_get_vreg(dest) : -1; +} + +/* A post-increment moves its base pointer: a definition deriving nothing. */ +static void tu_note_postinc_def(TCCIRState *ir, IRQuadCompact *q, TuVregSymEntry *map, int *count) +{ + if (q->op == TCCIR_OP_LOAD_POSTINC) + tu_vreg_map_def(map, count, irop_get_vreg(tcc_ir_op_get_src1(ir, q)), NULL); + else if (q->op == TCCIR_OP_STORE_POSTINC) + tu_vreg_map_def(map, count, irop_get_vreg(tcc_ir_op_get_dest(ir, q)), NULL); +} + +/* Any pre-opt value read or unrecognized address use of a static blocks tu_no_readers; store-address plumbing `T=&g+i; *T=v` does not. */ +/* Which call ids call a block-copy/fill helper (memcpy, memmove, memset and + * their __aeabi_ forms): argument 0 of such a call is the destination the + * helper writes through, so a static's address there is a write -- the + * shape of a struct assignment into it -- not an escape. */ +static uint8_t *tu_block_copy_call_ids(TCCIRState *ir, int *count) +{ + int max_id = -1; + const int n = ir->next_instruction_index; + for (int i = 0; i < n; i++) { - if (map[i].vreg == vr) - { - map[i].sym = NULL; - return; - } + IRQuadCompact *q = &ir->compact_instructions[i]; + if (q->op != TCCIR_OP_FUNCCALLVAL && q->op != TCCIR_OP_FUNCCALLVOID) + continue; + int cid = TCCIR_DECODE_CALL_ID((int32_t)irop_get_imm64_ex(ir, tcc_ir_op_get_src2(ir, q))); + if (cid > max_id) + max_id = cid; + } + *count = max_id + 1; + uint8_t *ids = tcc_mallocz(*count ? *count : 1); + for (int i = 0; i < n; i++) + { + IRQuadCompact *q = &ir->compact_instructions[i]; + if (q->op != TCCIR_OP_FUNCCALLVAL && q->op != TCCIR_OP_FUNCCALLVOID) + continue; + Sym *callee = irop_get_sym_ex(ir, tcc_ir_op_get_src1(ir, q)); + int cid = TCCIR_DECODE_CALL_ID((int32_t)irop_get_imm64_ex(ir, tcc_ir_op_get_src2(ir, q))); + if (callee && cid >= 0 && tu_is_block_copy_helper(get_tok_str(callee->v, NULL))) + ids[cid] = 1; } + return ids; +} + +static int tu_is_block_copy_dest_param(TCCIRState *ir, IRQuadCompact *q, const uint8_t *ids, int count) +{ + if (q->op != TCCIR_OP_FUNCPARAMVAL) + return 0; + int32_t enc = (int32_t)irop_get_imm64_ex(ir, tcc_ir_op_get_src2(ir, q)); + int cid = TCCIR_DECODE_CALL_ID(enc); + return TCCIR_DECODE_PARAM_IDX(enc) == 0 && cid >= 0 && cid < count && ids[cid]; } -/* Any pre-opt value read or unrecognized address use of a static blocks tu_no_readers; store-address plumbing `T=&g+i; *T=v` does not. */ void tcc_ir_collect_tu_static_reads_preopt(TCCIRState *ir) { + int nids; + uint8_t *copy_ids = tu_block_copy_call_ids(ir, &nids); TuVregSymEntry map[TU_VREG_MAP_MAX]; int mc = 0; const int n = ir->next_instruction_index; @@ -165,8 +231,6 @@ void tcc_ir_collect_tu_static_reads_preopt(TCCIRState *ir) if (q->op == TCCIR_OP_NOP) continue; - int is_store = (q->op == TCCIR_OP_STORE || q->op == TCCIR_OP_STORE_INDEXED || - q->op == TCCIR_OP_STORE_POSTINC); int is_load = (q->op == TCCIR_OP_LOAD || q->op == TCCIR_OP_LOAD_INDEXED || q->op == TCCIR_OP_LOAD_POSTINC); int is_addr_derive = (q->op == TCCIR_OP_ASSIGN || q->op == TCCIR_OP_ADD || @@ -191,6 +255,8 @@ void tcc_ir_collect_tu_static_reads_preopt(TCCIRState *ir) sym = tu_vreg_map_lookup(map, mc, irop_get_vreg(op)); if (!sym || !tu_is_static_global_candidate(sym)) continue; + if (oi == 0 && !op.is_lval && tu_is_block_copy_dest_param(ir, q, copy_ids, nids)) + continue; /* written through by the helper */ if (op.is_lval || is_load) { tu_symset_add(&tu_source_reads, sym); continue; @@ -200,11 +266,10 @@ void tcc_ir_collect_tu_static_reads_preopt(TCCIRState *ir) tu_symset_add(&tu_source_reads, sym); } - if (irop_config[q->op].has_dest && !is_store) + tu_note_postinc_def(ir, q, map, &mc); { - IROperand dest = tcc_ir_op_get_dest(ir, q); - int32_t dvr = irop_get_vreg(dest); - if (dvr >= 0 && !dest.is_lval) + int32_t dvr = tu_dest_def(ir, q); + if (dvr >= 0) { Sym *derived = NULL; if (is_addr_derive && irop_config[q->op].has_src1) @@ -223,19 +288,17 @@ void tcc_ir_collect_tu_static_reads_preopt(TCCIRState *ir) else if (!acc.is_sym && !acc.is_lval && irop_get_vreg(acc) >= 0) derived = tu_vreg_map_lookup(map, mc, irop_get_vreg(acc)); } - if (derived && tu_is_static_global_candidate(derived)) - tu_vreg_map_set(map, &mc, dvr, derived); - else - tu_vreg_map_clear(map, &mc, dvr); + tu_vreg_map_def(map, &mc, dvr, derived && tu_is_static_global_candidate(derived) ? derived : NULL); } } } + tcc_free(copy_ids); } /* Block-copy / fill helpers whose only memory effect is through the destination * pointer (argument 0). Callers resolve that pointer at the call site, so the * mod-ref walk may skip these even though they have no TU summary. */ -static int tu_is_block_copy_helper(const char *nm) +int tu_is_block_copy_helper(const char *nm) { if (!nm) return 0; @@ -287,9 +350,11 @@ static int tu_modref_walk(Sym *callee, MemLoc L) TuFuncSummary *s = tu_summary_lookup(f); if (!s) { - /* No summary. A block-copy helper's effect was already resolved into the - * CALLER's summary from its destination argument, so it is not an unknown - * here; anything else (external, or not yet compiled) is. */ + /* No summary. A block-copy helper reached through a callee had its effect + * resolved into that callee's summary from its destination argument, so it + * is not an unknown here; anything else (external, or not yet compiled) is. + * A DIRECT block-copy call never gets here: tcc_ir_call_may_write resolves + * its destination at the call site. */ if (tu_is_block_copy_helper(get_tok_str(f->v, NULL))) continue; return 1; @@ -323,6 +388,34 @@ int tcc_ir_call_may_write(TCCIRState *ir, int call_idx, MemLoc L) Sym *callee = irop_get_sym_ex(ir, tcc_ir_op_get_src1(ir, q)); if (!callee) return 1; /* indirect call */ + /* A DIRECT block-copy/fill call writes its destination range, and that has to + * be resolved here, at the call site. tu_modref_walk skips these helpers + * because, reached through a callee, their effect is already folded into that + * callee's summary -- but for a direct call the calling function is the one + * being optimized, whose summary is never consulted. Skipping it here said + * `g = make()` (a memmove into g) does not write g, and copy-source forwarding + * then answered a read of an earlier copy of g with g's new value. */ + const char *cname = get_tok_str(callee->v, NULL); + if (tu_is_block_copy_helper(cname)) + { + IROperand dst, len; + if (!ir_opt_get_call_param_operand(ir, call_idx, 0, &dst)) + return 1; + MemLoc D = memloc_of_pointer(ir, dst, call_idx); + if (D.kind != MEMLOC_GLOBAL && D.kind != MEMLOC_FRAME) + return 1; + /* Length: (dst, src, n) for the copies and memset; (dst, n[, c]) for the + * AEABI set/clear helpers. Unknown -> everything from dst onwards. */ + int len_idx = (!strncmp(cname, "__aeabi_memset", 14) || !strncmp(cname, "__aeabi_memclr", 14)) ? 1 : 2; + D.size = 1 << 30; + if (ir_opt_get_call_param_operand(ir, call_idx, len_idx, &len) && irop_is_immediate(len)) + { + int64_t n = irop_get_imm64_ex(ir, len); + if (n > 0 && n < (1 << 30)) + D.size = (int)n; + } + return mem_may_alias(D, L); + } return tu_modref_walk(callee, L); } @@ -347,6 +440,8 @@ void tcc_ir_collect_tu_func_summary(TCCIRState *ir, Sym *func_sym) int vreg_map_count = 0; TuSymSet addr_only_syms = {0}; /* statics referenced only by address (non-lval) */ TuSymSet escaped_syms = {0}; /* address-of refs that escaped (call/store-as-value) */ + int ncopy_ids; + uint8_t *copy_ids = tu_block_copy_call_ids(ir, &ncopy_ids); for (int i = 0; i < n; i++) { @@ -354,14 +449,12 @@ void tcc_ir_collect_tu_func_summary(TCCIRState *ir, Sym *func_sym) if (q->op == TCCIR_OP_NOP) continue; - /* A STORE dest is an address operand, not a value definition, so it must not disturb the map. */ - if (irop_config[q->op].has_dest && - q->op != TCCIR_OP_STORE && q->op != TCCIR_OP_STORE_INDEXED && - q->op != TCCIR_OP_STORE_POSTINC) + /* A store through a pointer is an address use, not a definition; a VAR + * stored as itself is one (tu_dest_def). */ + tu_note_postinc_def(ir, q, vreg_map, &vreg_map_count); { - IROperand dest = tcc_ir_op_get_dest(ir, q); - int32_t dvr = irop_get_vreg(dest); - if (dvr >= 0 && !dest.is_lval) + int32_t dvr = tu_dest_def(ir, q); + if (dvr >= 0) { Sym *derived_sym = NULL; @@ -407,10 +500,7 @@ void tcc_ir_collect_tu_func_summary(TCCIRState *ir, Sym *func_sym) derived_sym = tu_vreg_map_lookup(vreg_map, vreg_map_count, avr); } - if (derived_sym) - tu_vreg_map_set(vreg_map, &vreg_map_count, dvr, derived_sym); - else - tu_vreg_map_clear(vreg_map, &vreg_map_count, dvr); + tu_vreg_map_def(vreg_map, &vreg_map_count, dvr, derived_sym); } } @@ -448,8 +538,9 @@ void tcc_ir_collect_tu_func_summary(TCCIRState *ir, Sym *func_sym) } } - /* A static address passed as a call argument escapes. */ - if (q->op == TCCIR_OP_FUNCPARAMVAL) + /* A static address passed as a call argument escapes -- except as the + * destination of a block-copy helper, which only writes through it. */ + if (q->op == TCCIR_OP_FUNCPARAMVAL && !tu_is_block_copy_dest_param(ir, q, copy_ids, ncopy_ids)) { if (irop_config[q->op].has_src1) { @@ -552,7 +643,14 @@ void tcc_ir_collect_tu_func_summary(TCCIRState *ir, Sym *func_sym) { MemLoc dl = memloc_of_pointer(ir, p0, i); if (dl.kind == MEMLOC_GLOBAL && dl.sym) + { tu_symset_add(&s->global_writes, dl.sym); + if (tu_is_static_global_candidate(dl.sym)) + { + tu_symset_add(&s->static_writes, dl.sym); + writes_any_static = 1; + } + } else if (dl.kind != MEMLOC_FRAME) s->writes_unknown = 1; } @@ -718,6 +816,7 @@ void tcc_ir_collect_tu_func_summary(TCCIRState *ir, Sym *func_sym) tu_symset_free(&addr_only_syms); tu_symset_free(&escaped_syms); + tcc_free(copy_ids); if (writes_any_static && func_sym->type.ref) func_sym->type.ref->f.tu_static_writer = 1; diff --git a/source/opt/flat/ipa/tu_summary.h b/source/opt/flat/ipa/tu_summary.h index 276a61ca7..e6b85a785 100644 --- a/source/opt/flat/ipa/tu_summary.h +++ b/source/opt/flat/ipa/tu_summary.h @@ -47,4 +47,8 @@ void tu_symset_add(TuSymSet *s, Sym *sym); int tu_symset_contains(const TuSymSet *s, const Sym *sym); void tu_symset_free(TuSymSet *s); +/* memcpy/memmove/memset and their __aeabi_ forms: write only through + * argument 0. */ +int tu_is_block_copy_helper(const char *nm); + #endif diff --git a/source/opt/flat/loop/backedge_phi_hoist.c b/source/opt/flat/loop/backedge_phi_hoist.c index 6962f112c..c4ddce705 100644 --- a/source/opt/flat/loop/backedge_phi_hoist.c +++ b/source/opt/flat/loop/backedge_phi_hoist.c @@ -128,7 +128,7 @@ int tcc_ir_opt_backedge_phi_hoist(TCCIRState *ir) if (irop_get_vreg(tcc_ir_op_get_src2(ir, eq)) == adst_vr) safe = 0; } - if (safe && eq->op == TCCIR_OP_MLA) { + if (safe && tcc_ir_op_is_mac(eq->op)) { if (irop_get_vreg(tcc_ir_op_get_accum(ir, eq)) == adst_vr) safe = 0; } diff --git a/source/opt/flat/loop/const_sim.c b/source/opt/flat/loop/const_sim.c index 224ea73ad..8540e6d76 100644 --- a/source/opt/flat/loop/const_sim.c +++ b/source/opt/flat/loop/const_sim.c @@ -1624,11 +1624,14 @@ static int lcs_generic_loop_is_stack_local(TCCIRState *ir, int start_idx, } } - if (irop_config[q->op].has_dest) + /* Any definition -- a VAR written as itself is a STACKOFF lvalue -- that + * stores a stack address marks the vreg for good: the flag is read + * flow-insensitively, so a later definition must not clear it. */ + if (irop_config[q->op].has_dest && q->op != TCCIR_OP_STORE_INDEXED) { IROperand dest = tcc_ir_op_get_dest(ir, q); int32_t dvr = irop_get_vreg(dest); - if (dvr >= 0 && !dest.is_lval) + if (irop_dest_defines_vreg(dest)) { int vt = TCCIR_DECODE_VREG_TYPE(dvr); int pos = TCCIR_DECODE_VREG_POSITION(dvr); @@ -1646,7 +1649,8 @@ static int lcs_generic_loop_is_stack_local(TCCIRState *ir, int start_idx, q->op == TCCIR_OP_LOAD) && src1.is_local && !src1.is_lval && irop_get_tag(src1) == IROP_TAG_STACKOFF; - *slot = addr_def ? 1 : 0; + if (addr_def) + *slot = 1; } } } diff --git a/source/opt/flat/loop/func_purity.c b/source/opt/flat/loop/func_purity.c index 0df9ddc6d..ee0387254 100644 --- a/source/opt/flat/loop/func_purity.c +++ b/source/opt/flat/loop/func_purity.c @@ -279,6 +279,11 @@ TCCFuncPurity tcc_ir_infer_func_purity(TCCIRState *ir, Sym *func_sym) LOG_LICM("PURITY: Function '%s' is IMPURE: VLA allocation", func_name); return TCC_FUNC_PURITY_IMPURE; + case TCCIR_OP_RETURN_ADDRESS: + /* Same arguments, different call site, different result. */ + LOG_LICM("PURITY: Function '%s' is IMPURE: reads its return address", func_name); + return TCC_FUNC_PURITY_IMPURE; + default: break; } diff --git a/source/opt/flat/loop/iv_analysis.c b/source/opt/flat/loop/iv_analysis.c index 384e621ba..1bb0de314 100644 --- a/source/opt/flat/loop/iv_analysis.c +++ b/source/opt/flat/loop/iv_analysis.c @@ -140,17 +140,188 @@ TCC_DBG_ENV_FLAG(dbg_mlaiv, "TCC_DBG_MLAIV") * MLA). Anything else means try_eliminate_iv_counter will fail and the loop * would carry BOTH the index and the new pointer -- pure added work, measured * at +14.6% on mibench_dijkstra before this gate existed. */ -static int iv_ctr_eliminable(TCCIRState *ir, int32_t iv_vr, int iv_def_idx, int allow_a, int allow_b) +static IROperand iv_canon_base(TCCIRState *ir, IROperand op) +{ + int32_t vr = irop_get_vreg(op); + if (vr < 0 || op.is_lval || TCCIR_DECODE_VREG_TYPE(vr) != TCCIR_VREG_TYPE_TEMP) + return op; + if (!tcc_ir_vreg_has_single_def(ir, vr)) + return op; + int def = tcc_ir_find_defining_instruction(ir, vr, ir->next_instruction_index); + if (def < 0) + return op; + IRQuadCompact *dq = &ir->compact_instructions[def]; + if (dq->op != TCCIR_OP_ASSIGN) + return op; + IROperand src = tcc_ir_op_get_src1(ir, dq); + if (irop_get_vreg(src) >= 0) + return op; /* only look through a copy of a constant address */ + return src; +} + +int iv_same_base(TCCIRState *ir, IROperand a, IROperand b) +{ + a = iv_canon_base(ir, a); + b = iv_canon_base(ir, b); + int tag_a = irop_get_tag(a); + int tag_b = irop_get_tag(b); + if (tag_a != tag_b || a.is_lval != b.is_lval) + return 0; + if (tag_a == IROP_TAG_IMM32 || tag_a == IROP_TAG_STACKOFF) + return a.u.imm32 == b.u.imm32; + if (tag_a == IROP_TAG_VREG) + { + int32_t va = irop_get_vreg(a); + int32_t vb = irop_get_vreg(b); + return va >= 0 && va == vb; + } + return 0; +} + +/* The loop's full index range: start..end plus a detached body (the un-rotated + * shape keeps the body past the back-edge). */ +static void iv_loop_range(const IRLoop *loop, int *lo, int *hi) +{ + *lo = loop->start_idx; + *hi = loop->end_idx; + if (loop->num_body_instrs > 0) + { + if (loop->body_instrs[0] < *lo) + *lo = loop->body_instrs[0]; + if (loop->body_instrs[loop->num_body_instrs - 1] > *hi) + *hi = loop->body_instrs[loop->num_body_instrs - 1]; + } +} + +int iv_read_reachable_outside(TCCIRState *ir, IRLoop *loop, int32_t iv_vr) +{ + const int n = ir->next_instruction_index; + int lo, hi; + iv_loop_range(loop, &lo, &hi); + if (ir_opt_vreg_address_taken_between(ir, iv_vr, 0, n - 1)) + return 1; + + uint8_t *seen = tcc_mallocz((size_t)n + 1); + int *wl = tcc_malloc(sizeof(int) * ((size_t)n + 2)); + int nwl = 0; + int result = 0; + + /* Every way out of the loop: a branch from inside to outside, and the fall + * through past its last instruction when that is not an unconditional jump. */ + for (int j = lo; j <= hi && j < n; j++) + { + IRQuadCompact *q = &ir->compact_instructions[j]; + if (q->op == TCCIR_OP_IJUMP || q->op == TCCIR_OP_SWITCH_TABLE) + { + result = 1; + break; + } + if (q->op == TCCIR_OP_JUMP || q->op == TCCIR_OP_JUMPIF) + { + int t = (int)irop_get_imm64_ex(ir, tcc_ir_op_get_dest(ir, q)); + if (t < lo || t > hi) + wl[nwl++] = t; + } + } + if (!result && hi + 1 < n && ir->compact_instructions[hi].op != TCCIR_OP_JUMP) + wl[nwl++] = hi + 1; + + while (!result && nwl > 0) + { + int k = wl[--nwl]; + if (k < 0 || k >= n || seen[k]) + continue; + seen[k] = 1; + if (k >= lo && k <= hi) + { + result = 1; /* a path back into the loop that is not its own back-edge */ + break; + } + IRQuadCompact *q = &ir->compact_instructions[k]; + if (q->op == TCCIR_OP_NOP) + { + wl[nwl++] = k + 1; + continue; + } + if (q->op == TCCIR_OP_IJUMP || q->op == TCCIR_OP_SWITCH_TABLE) + { + result = 1; + break; + } + /* reads */ + if ((irop_config[q->op].has_src1 && irop_get_vreg(tcc_ir_op_get_src1(ir, q)) == iv_vr) || + (irop_config[q->op].has_src2 && irop_get_vreg(tcc_ir_op_get_src2(ir, q)) == iv_vr) || + (q->op == TCCIR_OP_MLA && irop_get_vreg(tcc_ir_op_get_accum(ir, q)) == iv_vr)) + { + result = 1; + break; + } + if (irop_config[q->op].has_dest) + { + IROperand d = tcc_ir_op_get_dest(ir, q); + if (irop_get_vreg(d) == iv_vr) + { + int direct = !d.is_lval || (TCCIR_DECODE_VREG_TYPE(iv_vr) == TCCIR_VREG_TYPE_VAR && d.is_local); + if (!direct) + { + result = 1; /* the IV used as an address */ + break; + } + continue; /* redefined: nothing past here sees the loop's value */ + } + } + if (q->op == TCCIR_OP_RETURNVOID || q->op == TCCIR_OP_RETURNVALUE) + continue; + if (q->op == TCCIR_OP_JUMP || q->op == TCCIR_OP_JUMPIF) + { + wl[nwl++] = (int)irop_get_imm64_ex(ir, tcc_ir_op_get_dest(ir, q)); + if (q->op == TCCIR_OP_JUMPIF) + wl[nwl++] = k + 1; + continue; + } + wl[nwl++] = k + 1; + } + tcc_free(seen); + tcc_free(wl); + return result; +} + +/* `allow` lists instructions that read the IV only to compute an address this + * transform is about to take over -- a DIV's SHL/MUL, or the `iv - k` of every + * affine INDEXED-DIV in the loop. A loop like SHA-1's message expansion reads + * its counter in four such subtractions per iteration, so a fixed two-slot + * whitelist could never admit it. + * + * With `loop` given, reads are judged INSIDE the loop by the rules below and + * OUTSIDE it by reachability: SHA-1's six loops share one `i`, each starting + * from a literal, and a whole-function scan called every one of them + * non-eliminable because the next loop's `i = i + 1` was a read. */ +static int iv_ctr_eliminable_list(TCCIRState *ir, IRLoop *loop, int32_t iv_vr, int iv_def_idx, + const int *allow, int nallow) { int safe = 1; int cmp_count = 0; - for (int j = 0; j < ir->next_instruction_index && safe; j++) + int lo = 0, hi = ir->next_instruction_index - 1; + if (loop) + { + iv_loop_range(loop, &lo, &hi); + if (iv_read_reachable_outside(ir, loop, iv_vr)) + return 0; + } + for (int j = lo; j <= hi && safe; j++) { IRQuadCompact *uq = &ir->compact_instructions[j]; if (uq->op == TCCIR_OP_NOP) continue; - if (j == iv_def_idx || j == allow_a || j == allow_b) + if (j == iv_def_idx) continue; + { + int allowed = 0; + for (int a = 0; a < nallow && !allowed; a++) + allowed = (allow[a] == j); + if (allowed) + continue; + } if (uq->op == TCCIR_OP_LOAD_INDEXED || uq->op == TCCIR_OP_STORE_INDEXED) { @@ -183,6 +354,17 @@ static int iv_ctr_eliminable(TCCIRState *ir, int32_t iv_vr, int iv_def_idx, int return safe && cmp_count > 0; } +static int iv_ctr_eliminable(TCCIRState *ir, int32_t iv_vr, int iv_def_idx, int allow_a, int allow_b) +{ + int allow[2]; + int n = 0; + if (allow_a >= 0) + allow[n++] = allow_a; + if (allow_b >= 0) + allow[n++] = allow_b; + return iv_ctr_eliminable_list(ir, NULL, iv_vr, iv_def_idx, allow, n); +} + /* Does this derived address value reach a real memory access (rather than * being consumed by an indexed op's addressing mode)? Those are the DIVs the * escape scan in transform_derived_iv used to refuse outright. */ @@ -289,7 +471,15 @@ static int iv_scaled_deref_postinc_viable(TCCIRState *ir, IRLoop *loop, int32_t if (!in_loop_access) return 0; - return iv_ctr_eliminable(ir, iv->vreg, iv->def_idx, allow_a, allow_b); + { + int allow[2]; + int n = 0; + if (allow_a >= 0) + allow[n++] = allow_a; + if (allow_b >= 0) + allow[n++] = allow_b; + return iv_ctr_eliminable_list(ir, loop, iv->vreg, iv->def_idx, allow, n); + } } int find_derived_ivs(TCCIRState *ir, IRLoop *loop, InductionVar *ivs, int num_ivs, DerivedIV *divs, int max_divs) @@ -328,6 +518,73 @@ int find_derived_ivs(TCCIRState *ir, IRLoop *loop, InductionVar *ivs, int num_iv fprintf(stderr, " (MLA scan range: [%d..%d])\n", mla_scan_start, mla_scan_end); } + /* Affine INDEXED-DIV candidates, found before anything else because two of + * the passes below consult them: `T = iv +/- k` (a single-use temp) feeding + * the index of an indexed access in this loop. SHA-1's message expansion, + * `W[i] = W[i-3] ^ W[i-8] ^ W[i-14] ^ W[i-16]`, reached codegen as four + * `subs r3, r2, #k` and four scaled-index loads per iteration, because only + * `W[i]` itself was a DIV and the subtractions then counted as extra reads + * of the counter that made it non-eliminable -- so even that one was + * refused. Recognising the offset accesses as DIVs on the SAME base and + * stride lets the driver walk one pointer for all of them, each access an + * immediate off it (gcc's `ldr r0, [r3, #32]`), with the subtractions and + * the counter gone. On the M33 an immediate offset is also a cycle cheaper + * than a scaled register index. */ +#define IV_AFFINE_MAX 16 + int affine_def[IV_AFFINE_MAX]; + IROperand affine_base[IV_AFFINE_MAX]; + int affine_stride[IV_AFFINE_MAX]; + int naffine = 0; + for (int bi = 0; bi < loop->num_body_instrs && naffine < IV_AFFINE_MAX; bi++) + { + int i = loop->body_instrs[bi]; + IRQuadCompact *q = &ir->compact_instructions[i]; + if (q->op != TCCIR_OP_ADD && q->op != TCCIR_OP_SUB) + continue; + IROperand d = tcc_ir_op_get_dest(ir, q); + int32_t dv = irop_get_vreg(d); + if (dv < 0 || d.is_lval || TCCIR_DECODE_VREG_TYPE(dv) != TCCIR_VREG_TYPE_TEMP) + continue; + IROperand a = tcc_ir_op_get_src1(ir, q); + IROperand k = tcc_ir_op_get_src2(ir, q); + if (!irop_is_immediate(k) || irop_get_vreg(a) < 0) + continue; + int iv_k = -1; + for (int v = 0; v < num_ivs && iv_k < 0; v++) + if (ivs[v].vreg == irop_get_vreg(a)) + iv_k = v; + if (iv_k < 0) + continue; + if (!tcc_ir_vreg_has_single_use(ir, dv, -1)) + continue; + /* The one use must be an indexed access's INDEX slot inside this loop. */ + for (int bj = 0; bj < loop->num_body_instrs; bj++) + { + IRQuadCompact *uq = &ir->compact_instructions[loop->body_instrs[bj]]; + if ((uq->op != TCCIR_OP_LOAD_INDEXED && uq->op != TCCIR_OP_STORE_INDEXED) || + irop_get_vreg(tcc_ir_op_get_src2(ir, uq)) != dv) + continue; + IROperand sc = tcc_ir_op_get_scale(ir, uq); + if (!irop_is_immediate(sc)) + break; + int scale = (int)irop_get_imm64_ex(ir, sc); + if (scale < 0 || scale > 3) + break; + affine_def[naffine] = i; + affine_base[naffine] = (uq->op == TCCIR_OP_LOAD_INDEXED) ? tcc_ir_op_get_src1(ir, uq) + : tcc_ir_op_get_dest(ir, uq); + affine_stride[naffine] = ivs[iv_k].step * (1 << scale); + naffine++; + break; + } + } + /* The SHL of every deref-form DIV admitted into a group below joins the + * allow list, so the group's eliminability check sees all of its own reads. */ + int group_allow[IV_AFFINE_MAX + 8]; + int ngroup_allow = 0; + for (int a = 0; a < naffine; a++) + group_allow[ngroup_allow++] = affine_def[a]; + /* Scan the extended body for ADD instructions (DIV computation) */ for (int bi = 0; bi < loop->num_body_instrs && num_divs < max_divs; bi++) { @@ -551,8 +808,32 @@ int find_derived_ivs(TCCIRState *ir, IRLoop *loop, InductionVar *ivs, int num_iv * The unscaled case is different and is handled by the fourth pass below: * there the address computation genuinely stays in the loop, because a * byte access off a stack base never gets folded. */ + int joins_group = 0; if (iv_div_addr_is_dereffed(ir, dest_vr) && !iv_scaled_deref_postinc_viable(ir, loop, dest_vr, stride, &ivs[iv_idx], shl_idx, -1)) + { + /* Not worth a walk on its own -- but as a member of an affine group on + * the same base it reads through the group's pointer at an immediate, + * and the group is a win whether or not its own access post-increments + * (the offset subtractions and the scaled indexes go regardless). */ + IROperand base_here = tcc_ir_op_get_src1(ir, q); + for (int a = 0; a < naffine && !joins_group; a++) + if (affine_stride[a] == stride && iv_same_base(ir, affine_base[a], base_here)) + joins_group = 1; + if (joins_group && ngroup_allow < IV_AFFINE_MAX + 8) + { + int allow_tmp[IV_AFFINE_MAX + 8]; + int nt = ngroup_allow; + memcpy(allow_tmp, group_allow, sizeof(int) * (size_t)ngroup_allow); + allow_tmp[nt++] = shl_idx; + if (!iv_ctr_eliminable_list(ir, loop, ivs[iv_idx].vreg, ivs[iv_idx].def_idx, allow_tmp, nt)) + joins_group = 0; + else + group_allow[ngroup_allow++] = shl_idx; + } + } + if (iv_div_addr_is_dereffed(ir, dest_vr) && !joins_group && + !iv_scaled_deref_postinc_viable(ir, loop, dest_vr, stride, &ivs[iv_idx], shl_idx, -1)) { LOG_IV_SR("IV_SR: Skipping scaled deref DIV at idx=%d — address folds into the addressing mode anyway", i); continue; @@ -565,6 +846,9 @@ int find_derived_ivs(TCCIRState *ir, IRLoop *loop, InductionVar *ivs, int num_iv divs[num_divs].use_idx = i; divs[num_divs].shl_idx = shl_idx; divs[num_divs].share_with = -1; + divs[num_divs].offset = 0; + divs[num_divs].off_idx = -1; + divs[num_divs].origin = 0; num_divs++; LOG_IV_SR("IV_SR: Found DIV base+%d*VAR%d at ADD idx=%d (SHL idx=%d)", stride, TCCIR_DECODE_VREG_POSITION(iv_vr), i, @@ -702,6 +986,9 @@ int find_derived_ivs(TCCIRState *ir, IRLoop *loop, InductionVar *ivs, int num_iv divs[num_divs].use_idx = i; divs[num_divs].shl_idx = -1; /* fused into MLA — nothing to NOP */ divs[num_divs].share_with = -1; + divs[num_divs].offset = 0; + divs[num_divs].off_idx = -1; + divs[num_divs].origin = 0; num_divs++; if (dbg_mlaiv()) @@ -709,7 +996,19 @@ int find_derived_ivs(TCCIRState *ir, IRLoop *loop, InductionVar *ivs, int num_iv LOG_IV_SR("IV_SR: Found MLA-DIV base+%d*VAR%d at MLA idx=%d (fused)", stride, TCCIR_DECODE_VREG_POSITION(iv_vr), i); } - /* Third pass: LOAD_INDEXED/STORE_INDEXED DIV — index is a BIV, stride = iv.step*(1<num_body_instrs && num_divs < max_divs; bi++) { int i = loop->body_instrs[bi]; @@ -719,6 +1018,10 @@ int find_derived_ivs(TCCIRState *ir, IRLoop *loop, InductionVar *ivs, int num_iv int is_store = (q->op == TCCIR_OP_STORE_INDEXED); if (!is_load && !is_store) continue; + /* A fused-shift annotation on the index would be lost by substituting an + * immediate for it (tccir_operand.c refuses exactly that). */ + if (tcc_ir_barrel_shift_at(ir, q)) + continue; IROperand base_op = is_load ? tcc_ir_op_get_src1(ir, q) : tcc_ir_op_get_dest(ir, q); IROperand index_op = tcc_ir_op_get_src2(ir, q); @@ -746,7 +1049,9 @@ int find_derived_ivs(TCCIRState *ir, IRLoop *loop, InductionVar *ivs, int num_iv } } - /* Copy-through chase */ + int aff_off_idx = -1; + int aff_k = 0; /* index units; scaled into bytes once the scale is known */ + /* Copy-through chase, then the affine `iv +/- k` chase */ if (iv_idx < 0) { int def = tcc_ir_find_defining_instruction(ir, iv_vr, i); @@ -765,6 +1070,32 @@ int find_derived_ivs(TCCIRState *ir, IRLoop *loop, InductionVar *ivs, int num_iv } } } + else if (dq->op == TCCIR_OP_ADD || dq->op == TCCIR_OP_SUB) + { + int listed = 0; + for (int a = 0; a < naffine && !listed; a++) + listed = (affine_def[a] == def); + if (listed) + { + int32_t src = irop_get_vreg(tcc_ir_op_get_src1(ir, dq)); + int64_t k = irop_get_imm64_ex(ir, tcc_ir_op_get_src2(ir, dq)); + for (int v = 0; v < num_ivs; v++) + { + if (ivs[v].vreg == src) + { + iv_idx = v; + break; + } + } + if (iv_idx >= 0 && k >= -1024 && k <= 1024) + { + aff_off_idx = def; + aff_k = (dq->op == TCCIR_OP_SUB) ? -(int)k : (int)k; + } + else + iv_idx = -1; + } + } } } if (iv_idx < 0) @@ -801,12 +1132,24 @@ int find_derived_ivs(TCCIRState *ir, IRLoop *loop, InductionVar *ivs, int num_iv if (scale < 0 || scale > 3) continue; int stride = ivs[iv_idx].step * (1 << scale); - - /* Eliminability gate (see iv_ctr_eliminable). */ - if (!iv_ctr_eliminable(ir, iv_vr, ivs[iv_idx].def_idx, -1, -1)) + int byte_offset = aff_k * (1 << scale); + if (byte_offset < -4095 || byte_offset > 4095) + continue; /* not an LDR/STR immediate on this target */ + + /* Eliminability gate. With no affine `iv +/- k` access in the loop this is + * exactly the historical whole-function, no-whitelist check -- so plain + * `a[j]` walks (including one read twice, or under a `break`) behave as + * before. Only when the loop really has the sliding-window shape do the + * affine offset computations and group members' SHLs join the whitelist. */ + int elig; + if (naffine > 0) + elig = iv_ctr_eliminable_list(ir, loop, ivs[iv_idx].vreg, ivs[iv_idx].def_idx, group_allow, ngroup_allow); + else + elig = iv_ctr_eliminable(ir, ivs[iv_idx].vreg, ivs[iv_idx].def_idx, -1, -1); + if (!elig) { LOG_IV_SR("IV_SR: Skipping INDEXED-DIV at idx=%d — IV VAR%d not eliminable", i, - TCCIR_DECODE_VREG_POSITION(iv_vr)); + TCCIR_DECODE_VREG_POSITION(ivs[iv_idx].vreg)); continue; } @@ -817,6 +1160,9 @@ int find_derived_ivs(TCCIRState *ir, IRLoop *loop, InductionVar *ivs, int num_iv divs[num_divs].use_idx = i; divs[num_divs].shl_idx = -1; /* shift is encoded in the scale field */ divs[num_divs].share_with = -1; + divs[num_divs].offset = byte_offset; + divs[num_divs].off_idx = aff_off_idx; + divs[num_divs].origin = 0; num_divs++; LOG_IV_SR("IV_SR: Found INDEXED-DIV base+%d*VAR%d at %s idx=%d (scale=%d, stride=%d)", stride, @@ -930,6 +1276,9 @@ int find_derived_ivs(TCCIRState *ir, IRLoop *loop, InductionVar *ivs, int num_iv divs[num_divs].use_idx = i; divs[num_divs].shl_idx = -1; /* no shift to NOP */ divs[num_divs].share_with = -1; + divs[num_divs].offset = 0; + divs[num_divs].off_idx = -1; + divs[num_divs].origin = 0; num_divs++; LOG_IV_SR("IV_SR: Found unscaled DIV base+%d*VAR%d at ADD idx=%d", ivs[iv_idx].step, diff --git a/source/opt/flat/loop/iv_strength_reduction.c b/source/opt/flat/loop/iv_strength_reduction.c index d880842fd..f7870fcf9 100644 --- a/source/opt/flat/loop/iv_strength_reduction.c +++ b/source/opt/flat/loop/iv_strength_reduction.c @@ -148,6 +148,37 @@ static int sr_div_value_stays_in_regs(TCCIRState *ir, int lo, int hi, int32_t se return 1; } +/* Point an indexed access at the walked pointer: same op, base = ptr, index = + * the byte immediate `rel`, scale 0 -- the shape the emitter's constant- + * displacement fast path turns into `ldr rd, [ptr, #rel]`. Rebuilt as a fresh + * four-slot pool entry the way symaddr_cse does it; the old base's access marks + * (volatile, underalignment) ride along on the new one. */ +static void sr_rewrite_indexed_through(TCCIRState *ir, IRQuadCompact *q, IROperand ptr_op, int rel) +{ + int is_load = (q->op == TCCIR_OP_LOAD_INDEXED); + IROperand old_base = is_load ? tcc_ir_op_get_src1(ir, q) : tcc_ir_op_get_dest(ir, q); + IROperand value = is_load ? tcc_ir_op_get_dest(ir, q) : tcc_ir_op_get_src1(ir, q); + IROperand new_base = ptr_op; + new_base.is_lval = old_base.is_lval; + irop_carry_access_marks(&new_base, old_base); + new_base.aux |= old_base.aux; + tcc_ir_pool_ensure(ir, 4); + int operand_base = ir->iroperand_pool_count; + if (is_load) + { + tcc_ir_pool_add(ir, value); + tcc_ir_pool_add(ir, new_base); + } + else + { + tcc_ir_pool_add(ir, new_base); + tcc_ir_pool_add(ir, value); + } + tcc_ir_pool_add(ir, irop_make_imm32(-1, rel, IROP_BTYPE_INT32)); + tcc_ir_pool_add(ir, irop_make_imm32(-1, 0, IROP_BTYPE_INT32)); + q->operand_base = operand_base; +} + int transform_derived_iv(TCCIRState *ir, IRLoop *loop, InductionVar *iv, DerivedIV *div, int *out_ptr_vreg, int *out_idx_shift, int *out_postnop_origpos, int *out_stride_pos, int shared_ptr_vreg) { @@ -235,7 +266,7 @@ int transform_derived_iv(TCCIRState *ir, IRLoop *loop, InductionVar *iv, Derived /* Don't split a CMP→JUMPIF pair: an inserted ADD (init_offset != 0) would clobber the flags. */ { int element_size_check = div->stride / iv->step; - int init_offset_check = iv->init_val * element_size_check; + int init_offset_check = iv->init_val * element_size_check + div->origin; if (init_offset_check != 0 && insert_pos > 0 && ir->compact_instructions[insert_pos - 1].op == TCCIR_OP_CMP && insert_pos < ir->next_instruction_index && ir->compact_instructions[insert_pos].op == TCCIR_OP_JUMPIF) { @@ -301,9 +332,11 @@ int transform_derived_iv(TCCIRState *ir, IRLoop *loop, InductionVar *iv, Derived int idx_shift = 0; - /* element_size = stride/step; init_offset = init_val*element_size (NOT init_val*stride when step != 1). */ + /* element_size = stride/step; init_offset = init_val*element_size (NOT init_val*stride when step != 1). + * A group primary aims the pointer at the group's smallest offset (origin) + * so every member reads at a non-negative immediate off it. */ int element_size = div->stride / iv->step; - int init_offset = iv->init_val * element_size; + int init_offset = iv->init_val * element_size + div->origin; if (init_offset == 0) { @@ -334,14 +367,23 @@ int transform_derived_iv(TCCIRState *ir, IRLoop *loop, InductionVar *iv, Derived /* Insertion shifted all indices >= insert_pos; update our tracked indices. */ int new_use_idx = div->use_idx + idx_shift; int new_shl_idx = (div->shl_idx >= 0) ? div->shl_idx + idx_shift : -1; + int new_off_idx = (div->off_idx >= 0) ? div->off_idx + idx_shift : -1; int new_iv_def_idx = iv->def_idx; if (iv->def_idx >= insert_pos) new_iv_def_idx += idx_shift; - /* Step 2: rewrite the use site to consume ptr — ADD/MLA→ASSIGN, LOAD_INDEXED→LOAD, STORE_INDEXED→STORE. */ + /* Step 2: rewrite the use site to consume ptr — ADD/MLA→ASSIGN, LOAD_INDEXED→LOAD, STORE_INDEXED→STORE. + * An access that sits `rel` bytes from where the pointer points stays indexed, + * with the immediate as its index. */ IRQuadCompact *add_q = &ir->compact_instructions[new_use_idx]; int rewrote_to_load_or_store = 0; - if (add_q->op == TCCIR_OP_LOAD_INDEXED) + int rel = div->offset - div->origin; + if ((add_q->op == TCCIR_OP_LOAD_INDEXED || add_q->op == TCCIR_OP_STORE_INDEXED) && rel != 0) + { + sr_rewrite_indexed_through(ir, add_q, ptr_op, rel); + rewrote_to_load_or_store = 1; + } + else if (add_q->op == TCCIR_OP_LOAD_INDEXED) { IROperand ptr_lval = ptr_op; ptr_lval.is_lval = 1; @@ -359,6 +401,14 @@ int transform_derived_iv(TCCIRState *ir, IRLoop *loop, InductionVar *iv, Derived rewrote_to_load_or_store = 1; /* src1 (value to store) is preserved at slot 1. */ } + else if (rel != 0) + { + /* A deref-form member of a group: its address temp is the group pointer + * plus this access's distance from the pointer's origin. */ + add_q->op = TCCIR_OP_ADD; + tcc_ir_op_set_src1(ir, add_q, ptr_op); + tcc_ir_op_set_src2(ir, add_q, irop_make_imm32(-1, rel, IROP_BTYPE_INT32)); + } else { add_q->op = TCCIR_OP_ASSIGN; @@ -384,12 +434,25 @@ int transform_derived_iv(TCCIRState *ir, IRLoop *loop, InductionVar *iv, Derived } } - /* Step 3: NOP the SHL/MUL (skipped for fused MLA — no separate multiply instr). */ + /* Step 3: NOP the SHL/MUL (skipped for fused MLA — no separate multiply instr), + * and the `iv +/- k` an affine INDEXED-DIV used to compute its index. */ if (new_shl_idx >= 0) { IRQuadCompact *shl_q = &ir->compact_instructions[new_shl_idx]; shl_q->op = TCCIR_OP_NOP; } + if (new_off_idx >= 0) + { + if (postnop_inserted && new_off_idx > new_use_idx) + new_off_idx++; + /* Only ever the `iv +/- k` this DIV recorded -- guard against a DerivedIV + * that reached here with off_idx defaulted to 0 rather than -1 (a + * hand-built or memset-zeroed struct), which would otherwise blank the + * init ASSIGN at index 0/1. */ + int oop = ir->compact_instructions[new_off_idx].op; + if (oop == TCCIR_OP_ADD || oop == TCCIR_OP_SUB) + ir->compact_instructions[new_off_idx].op = TCCIR_OP_NOP; + } /* Step 4: insert ptr += stride after the IV increment, pushed past all uses of the derived address (copy-prop/coalescing can merge ptr with the address temp). */ int stride_insert_pos = new_iv_def_idx + 1; @@ -709,6 +772,10 @@ int try_eliminate_iv_counter(TCCIRState *ir, IRLoop *loop, InductionVar *iv, Der * end_value = limit * element_size */ int element_size = div->stride / iv->step; int end_offset = limit_val * element_size; + /* end is built from ptr when ptr still equals base + origin (init 0), and + * from base otherwise -- only the latter still owes the origin. */ + if (iv->init_val != 0) + end_offset += div->origin; /* Allocate a vreg for end_ptr */ int end_vreg = tcc_ir_vreg_alloc_temp(ir); @@ -967,23 +1034,11 @@ int iv_strength_reduction_core(TCCIRState *ir, IRLoops *loops) continue; /* only chain to primaries */ if (divs[dj].iv_idx != divs[dk].iv_idx || divs[dj].stride != divs[dk].stride) continue; - /* Compare base operands: same vreg, or same immediate value, or same stack offset. */ - IROperand a = divs[dj].base_op; - IROperand b = divs[dk].base_op; - int tag_a = irop_get_tag(a); - int tag_b = irop_get_tag(b); - if (tag_a != tag_b) - continue; - int eq = 0; - if (tag_a == IROP_TAG_IMM32 || tag_a == IROP_TAG_STACKOFF) - eq = (a.u.imm32 == b.u.imm32 && a.is_lval == b.is_lval); - else if (tag_a == IROP_TAG_VREG) - { - int32_t va = irop_get_vreg(a); - int32_t vb = irop_get_vreg(b); - if (va >= 0 && va == vb && a.is_lval == b.is_lval) - eq = 1; - } + /* Compare base operands: same vreg, or same immediate value, or same + * stack offset -- looking through a single-def copy of an address, so + * loads off `Addr[StackLoc[k]]` and a store through `T <- Addr[...]` + * are one base. */ + int eq = iv_same_base(ir, divs[dj].base_op, divs[dk].base_op); if (eq) { divs[dj].share_with = dk; @@ -1021,6 +1076,39 @@ int iv_strength_reduction_core(TCCIRState *ir, IRLoops *loops) continue; } } + /* A group is a primary INDEXED-DIV plus every INDEXED-DIV sharing its + * (iv, stride, base). Aim the one pointer at the group's smallest + * offset: every member then reads at a non-negative immediate, which + * the 16-bit `ldr rd, [rn, #imm5*4]` encoding covers up to 124 and the + * wide form up to 4095. A non-indexed primary (an ADD-form address + * temp) must keep the pointer at offset 0 -- its temp IS the address. */ + /* A group is worth walking through one pointer only when it has the + * sliding-window shape -- at least one `base[iv +/- k]` member. Without + * that (a plain `a[j]` read twice, a scaled address the fusion already + * handles) the members keep their own paths, exactly as before. */ + int group_affine = (divs[di].off_idx >= 0); + for (int dj = di + 1; dj < num_divs && !group_affine; dj++) + if (divs[dj].share_with == di && divs[dj].off_idx >= 0) + group_affine = 1; + divs[di].origin = 0; + if (shared < 0 && group_affine && + (ir->compact_instructions[divs[di].use_idx].op == TCCIR_OP_LOAD_INDEXED || + ir->compact_instructions[divs[di].use_idx].op == TCCIR_OP_STORE_INDEXED || + ir->compact_instructions[divs[di].use_idx].op == TCCIR_OP_ADD)) + { + int origin = divs[di].offset; + for (int dj = di + 1; dj < num_divs; dj++) + { + if (divs[dj].share_with != di) + continue; + int uop = ir->compact_instructions[divs[dj].use_idx].op; + if (uop != TCCIR_OP_LOAD_INDEXED && uop != TCCIR_OP_STORE_INDEXED) + continue; + if (divs[dj].offset < origin) + origin = divs[dj].offset; + } + divs[di].origin = origin; + } int changes = transform_derived_iv(ir, loop, &ivs[divs[di].iv_idx], &divs[di], &sr_ptr_vreg, &sr_idx_shift, &sr_postnop_origpos, &sr_stride_pos, shared); @@ -1080,6 +1168,8 @@ int iv_strength_reduction_core(TCCIRState *ir, IRLoops *loops) { APPLY_SHIFT(divs[dj].use_idx); APPLY_SHIFT(divs[dj].shl_idx); + if (divs[dj].off_idx >= 0) + APPLY_SHIFT(divs[dj].off_idx); } for (int ij = 0; ij < num_ivs; ij++) { @@ -1098,6 +1188,39 @@ int iv_strength_reduction_core(TCCIRState *ir, IRLoops *loops) #undef APPLY_SHIFT + /* Members of this primary's group read through its pointer. An + * in-place rewrite inserts nothing, so the index bookkeeping above is + * all that is needed; a member is only taken when it will have run + * before the bump (the stride ADD lands after every access of the + * iteration -- in the latch, or pushed past the primary's own use), + * and the `iv +/- k` that fed its index is NOPed with it. A member + * left alone keeps its counter read and the elimination below then + * declines, which is the same code as before. */ + if (changes >= 3 && sr_ptr_vreg >= 0 && group_affine) + { + IROperand ptr_op = irop_make_vreg(sr_ptr_vreg, IROP_BTYPE_INT32); + for (int dj = di + 1; dj < num_divs; dj++) + { + if (divs[dj].share_with != di) + continue; + IRQuadCompact *uq = &ir->compact_instructions[divs[dj].use_idx]; + if (uq->op != TCCIR_OP_LOAD_INDEXED && uq->op != TCCIR_OP_STORE_INDEXED) + continue; + if (stride_pos >= 0 && divs[dj].use_idx >= stride_pos) + continue; + int rel = divs[dj].offset - divs[di].origin; + if (rel < 0 || rel > 4095) + continue; + sr_rewrite_indexed_through(ir, uq, ptr_op, rel); + if (divs[dj].off_idx >= 0) + ir->compact_instructions[divs[dj].off_idx].op = TCCIR_OP_NOP; + div_ptr_vregs[dj] = sr_ptr_vreg; + div_changes[dj] = 0; /* not an elimination candidate of its own */ + total_changes++; + LOG_IV_SR("IV_SR: DIV %d reads through DIV %d's pointer at +%d", dj, di, rel); + } + } + /* Transform only ONE derived IV per loop per call, then bail to IV * elimination. Processing several DIVs in one call requires shifting * every remaining DIV's recorded indices (APPLY_SHIFT) by the exact diff --git a/source/opt/flat/loop/licm.c b/source/opt/flat/loop/licm.c index 8d4a1794f..839126f2e 100644 --- a/source/opt/flat/loop/licm.c +++ b/source/opt/flat/loop/licm.c @@ -1377,7 +1377,98 @@ static void licm_move_jump_target_off_nop(TCCIRState *ir, int nop_pos) ir->compact_instructions[nop_pos].is_jump_target = 0; } -static int licm_is_direct_var_read(TCCIRState *ir, IROperand op) +/* A set of vregs, one bit per (type, position), for facts the dom-LICM + * detection phase asks about again and again while the IR stands still: which + * variables have their address taken anywhere, which vregs are written outside + * the loop. Each was a walk of the function per candidate -- 1.7 G + * instructions of address-taken scans on Zig's zig.c at -O2. A set is built + * on first use for a loop and dropped with it; the hoisting that follows + * detection is what changes the IR. */ +typedef struct LicmVregSet +{ + uint8_t *bits; /* NULL until built */ + int stride; /* positions per type */ +} LicmVregSet; + +/* Index of an ordinary (type << 28 | position) vreg in a set of `stride`, or + * -1 for any other encoding, which the caller answers the slow way. */ +static int licm_vset_key(int32_t vr, int stride) +{ + if (vr < 0) + return -1; + int t = TCCIR_DECODE_VREG_TYPE(vr), p = TCCIR_DECODE_VREG_POSITION(vr); + if (t < 0 || t > 3 || vr != TCCIR_ENCODE_VREG(t, p) || p >= stride) + return -1; + return t * stride + p; +} + +static int licm_max_vreg_pos(TCCIRState *ir) +{ + int max = -1; + for (int i = 0; i < ir->next_instruction_index; i++) + { + IRQuadCompact *q = &ir->compact_instructions[i]; + if (q->op == TCCIR_OP_NOP) + continue; + IROperand ops[4]; + int m = 0; + if (irop_config[q->op].has_dest) + ops[m++] = tcc_ir_op_get_dest(ir, q); + if (irop_config[q->op].has_src1) + ops[m++] = tcc_ir_op_get_src1(ir, q); + if (irop_config[q->op].has_src2) + ops[m++] = tcc_ir_op_get_src2(ir, q); + if (q->op == TCCIR_OP_MLA) + ops[m++] = tcc_ir_op_get_accum(ir, q); + for (int k = 0; k < m; k++) + { + int32_t vr = irop_get_vreg(ops[k]); + if (vr >= 0 && TCCIR_DECODE_VREG_POSITION(vr) > max) + max = TCCIR_DECODE_VREG_POSITION(vr); + } + } + return max; +} + +/* vreg_addr_taken_anywhere() for every vreg at once. */ +static void licm_addr_taken_set_build(TCCIRState *ir, LicmVregSet *set) +{ + set->stride = licm_max_vreg_pos(ir) + 1; + set->bits = tcc_mallocz((size_t)4 * set->stride + 1); + for (int i = 0; i < ir->next_instruction_index; i++) + { + IRQuadCompact *q = &ir->compact_instructions[i]; + if (q->op == TCCIR_OP_NOP) + continue; + IROperand srcs[3]; + int nsrcs = 0; + if (irop_config[q->op].has_src1) + srcs[nsrcs++] = tcc_ir_op_get_src1(ir, q); + if (irop_config[q->op].has_src2) + srcs[nsrcs++] = tcc_ir_op_get_src2(ir, q); + if (q->op == TCCIR_OP_MLA) + srcs[nsrcs++] = tcc_ir_op_get_accum(ir, q); + for (int k = 0; k < nsrcs; k++) + { + int key; + if (irop_get_tag(srcs[k]) == IROP_TAG_STACKOFF && !srcs[k].is_lval && + (key = licm_vset_key(irop_get_vreg(srcs[k]), set->stride)) >= 0) + set->bits[key] = 1; + } + } +} + +static int licm_addr_taken(TCCIRState *ir, LicmVregSet *set, int32_t vr) +{ + if (!set->bits) + licm_addr_taken_set_build(ir, set); + int key = licm_vset_key(vr, set->stride); + if (key < 0) + return vreg_addr_taken_anywhere(ir, vr); + return set->bits[key]; +} + +static int licm_is_direct_var_read(TCCIRState *ir, LicmVregSet *addr_taken, IROperand op) { int32_t vr = irop_get_vreg(op); if (vr < 0 || !op.is_local || op.is_llocal) @@ -1386,7 +1477,28 @@ static int licm_is_direct_var_read(TCCIRState *ir, IROperand op) return 0; if (irop_access_is_volatile(op)) return 0; - return !vreg_addr_taken_anywhere(ir, vr); + return !licm_addr_taken(ir, addr_taken, vr); +} + +/* The dests of every instruction in the blocks outside the loop. */ +static void licm_outside_defs_build(TCCIRState *ir, IRCFG *cfg, const uint8_t *in_loop, LicmVregSet *set) +{ + set->stride = licm_max_vreg_pos(ir) + 1; + set->bits = tcc_mallocz((size_t)4 * set->stride + 1); + for (int obi = 0; obi < cfg->num_blocks; obi++) + { + if (in_loop[obi]) + continue; + for (int oi2 = cfg->blocks[obi].start_idx; oi2 < cfg->blocks[obi].end_idx; oi2++) + { + IRQuadCompact *oq = &ir->compact_instructions[oi2]; + if (oq->op == TCCIR_OP_NOP || !irop_config[oq->op].has_dest) + continue; + int key = licm_vset_key(irop_get_vreg(tcc_ir_op_get_dest(ir, oq)), set->stride); + if (key >= 0) + set->bits[key] = 1; + } + } } /* A direct read of a named local's slot. `Addr[StackLoc[off]]` (is_lval == 0) @@ -2016,6 +2128,7 @@ IRLoops *tcc_ir_opt_licm_ex(TCCIRState *ir) uint8_t *is_invariant = tcc_mallocz(ir->next_instruction_index); LicmSlotCtx slot_ctx; + LicmVregSet addr_taken = {0}, outside_defs = {0}; int inv_changed = 1; slot_ctx.writes_frame = -1; slot_ctx.n = 0; @@ -2052,7 +2165,12 @@ IRLoops *tcc_ir_opt_licm_ex(TCCIRState *ir) continue; /* Check if vreg is also defined outside the loop */ int outside_def = 0; - for (int obi = 0; obi < cfg->num_blocks && !outside_def; obi++) { + if (!outside_defs.bits) + licm_outside_defs_build(ir, cfg, in_loop, &outside_defs); + int od_key = licm_vset_key(dvr, outside_defs.stride); + if (od_key >= 0) + outside_def = outside_defs.bits[od_key]; + for (int obi = 0; od_key < 0 && obi < cfg->num_blocks && !outside_def; obi++) { if (in_loop[obi]) continue; for (int oi2 = cfg->blocks[obi].start_idx; oi2 < cfg->blocks[obi].end_idx; oi2++) { @@ -2076,7 +2194,7 @@ IRLoops *tcc_ir_opt_licm_ex(TCCIRState *ir) * side effect across iterations. This mirrors the guard * is_operand_loop_invariant_ex already applies on the READ * side (docs/bugs.md #7). */ - if (vreg_addr_taken_anywhere(ir, dvr) && + if (licm_addr_taken(ir, &addr_taken, dvr) && cfg_loop_may_clobber_memory(ir, cfg, in_loop)) continue; } @@ -2097,7 +2215,7 @@ IRLoops *tcc_ir_opt_licm_ex(TCCIRState *ir) if (licm_is_direct_slot_read(s) && licm_slot_read_is_invariant(ir, cfg, in_loop, s, &slot_ctx)) continue; - if (licm_is_direct_var_read(ir, s)) + if (licm_is_direct_var_read(ir, &addr_taken, s)) continue; /* named local, address never taken — def_count covers it */ has_deref = 1; } @@ -2163,6 +2281,9 @@ IRLoops *tcc_ir_opt_licm_ex(TCCIRState *ir) } } + tcc_free(addr_taken.bits); + tcc_free(outside_defs.bits); + /* Find exit blocks */ uint8_t *is_exit = tcc_mallocz(cfg->num_blocks); for (int bi = 0; bi < cfg->num_blocks; bi++) { diff --git a/source/opt/flat/loop/loop_detect.c b/source/opt/flat/loop/loop_detect.c index 3a0306081..a4d39318e 100644 --- a/source/opt/flat/loop/loop_detect.c +++ b/source/opt/flat/loop/loop_detect.c @@ -112,6 +112,9 @@ IRLoops *tcc_ir_detect_loops(TCCIRState *ir) tcc_free(loops); return NULL; } + loops->ramp = tcc_malloc(sizeof(int) * ir->next_instruction_index); + for (int k = 0; k < ir->next_instruction_index; k++) + loops->ramp[k] = k; for (int i = 0; i < ir->next_instruction_index; i++) { @@ -152,14 +155,10 @@ IRLoops *tcc_ir_detect_loops(TCCIRState *ir) int body_size = i - target + 1; loop->body_instrs_capacity = body_size; - loop->body_instrs = tcc_mallocz(sizeof(int) * body_size); + loop->body_instrs = loops->ramp + target; + loop->num_body_instrs = body_size; - if (loop->body_instrs) { - for (int j = target; j <= i; j++) - { - loop->body_instrs[loop->num_body_instrs++] = j; - } /* Forward jumps out of [target, i] can still land in the body; extend the range. */ int max_idx = i; @@ -184,14 +183,8 @@ IRLoops *tcc_ir_detect_loops(TCCIRState *ir) if (max_idx > i) { int new_body_size = max_idx - target + 1; - tcc_free(loop->body_instrs); - loop->body_instrs = tcc_mallocz(sizeof(int) * new_body_size); loop->body_instrs_capacity = new_body_size; - loop->num_body_instrs = 0; - for (int j = target; j <= max_idx; j++) - { - loop->body_instrs[loop->num_body_instrs++] = j; - } + loop->num_body_instrs = new_body_size; } loops->num_loops++; @@ -211,7 +204,6 @@ IRLoops *tcc_ir_detect_loops(TCCIRState *ir) IRLoop *lj = &loops->loops[j]; if (li->header_idx == lj->header_idx && li->end_idx < lj->end_idx) { - tcc_free(li->body_instrs); li->body_instrs = NULL; li->num_body_instrs = 0; li->header_idx = -1; /* sentinel: removed */ @@ -271,15 +263,8 @@ void tcc_ir_free_loops(IRLoops *loops) if (!loops) return; - if (loops->loops) - { - for (int i = 0; i < loops->num_loops; i++) - { - if (loops->loops[i].body_instrs) - tcc_free(loops->loops[i].body_instrs); - } - tcc_free(loops->loops); - } + tcc_free(loops->loops); + tcc_free(loops->ramp); tcc_free(loops); } diff --git a/source/opt/flat/loop/loop_exit_analysis.c b/source/opt/flat/loop/loop_exit_analysis.c index 756481f8a..8f9eb7693 100644 --- a/source/opt/flat/loop/loop_exit_analysis.c +++ b/source/opt/flat/loop/loop_exit_analysis.c @@ -227,9 +227,11 @@ int compute_trip_count(int init_val, int limit, int step, int cond_token) /* Collect body instructions to clone (excluding loop control flow and IV update). */ int collect_body_instructions(TCCIRState *ir, IRLoop *loop, int iv_vreg, int cmp_idx, int jmpif_idx, - int iv_def_idx, int *body_indices, int max_body) + int iv_def_idx, int *body_indices, int max_body, int *out_has_memory) { int count = 0; + if (out_has_memory) + *out_has_memory = 0; /* Scan the full [start..end]; truncating the body would unroll wrong code. */ for (int i = loop->start_idx; i <= loop->end_idx; i++) { @@ -250,13 +252,9 @@ int collect_body_instructions(TCCIRState *ir, IRLoop *loop, int iv_vreg, int cmp if (i == iv_def_idx) continue; - /* Skip the ASSIGN saving old IV for post-increment (T = Viv) */ - if (q->op == TCCIR_OP_ASSIGN && i == iv_def_idx - 1) - { - IROperand src1 = tcc_ir_op_get_src1(ir, q); - if (irop_get_vreg(src1) == iv_vreg) - continue; - } + /* The ASSIGN saving the old IV for a post-increment (T = Viv) stays in + * the body: every copy makes it `T = #k`, and a body that reads T -- the + * `buf[i++] = arg` store of 990525-1 -- needs it (dead copies go to DCE). */ /* Reject bodies with internal branches */ if (q->op == TCCIR_OP_JUMPIF) @@ -279,23 +277,43 @@ int collect_body_instructions(TCCIRState *ir, IRLoop *loop, int iv_vreg, int cmp return -1; } - /* Reject memory ops: cloned-body aliasing isn't modeled, so keep unroll register-only. */ - if (q->op == TCCIR_OP_LOAD || q->op == TCCIR_OP_STORE || - q->op == TCCIR_OP_LOAD_INDEXED || q->op == TCCIR_OP_STORE_INDEXED || - q->op == TCCIR_OP_LOAD_POSTINC || q->op == TCCIR_OP_STORE_POSTINC || - q->op == TCCIR_OP_BLOCK_COPY || - (irop_config[q->op].has_src1 && ir_xform_operand_reads_memory(tcc_ir_op_get_src1(ir, q))) || - (irop_config[q->op].has_src2 && ir_xform_operand_reads_memory(tcc_ir_op_get_src2(ir, q))) || - (q->op == TCCIR_OP_MLA && ir_xform_operand_reads_memory(tcc_ir_op_get_accum(ir, q)))) + /* Plain loads and stores (and their indexed forms) clone in order, so the + * unrolled body touches memory exactly as the loop did; the caller counts + * them against a smaller size cap. Post-increment forms update their + * pointer as a side effect and a BLOCK_COPY's size may be a whole object: + * those stay loops. */ + if (q->op == TCCIR_OP_LOAD_POSTINC || q->op == TCCIR_OP_STORE_POSTINC || q->op == TCCIR_OP_BLOCK_COPY) { LOG_LOOP_OPT("collect_body: REJECTED at [%d] memory access op=%d", i, q->op); return -1; } + if (out_has_memory && + (q->op == TCCIR_OP_LOAD || q->op == TCCIR_OP_STORE || q->op == TCCIR_OP_LOAD_INDEXED || + q->op == TCCIR_OP_STORE_INDEXED || + (irop_config[q->op].has_src1 && ir_xform_operand_reads_memory(tcc_ir_op_get_src1(ir, q))) || + (irop_config[q->op].has_src2 && ir_xform_operand_reads_memory(tcc_ir_op_get_src2(ir, q))) || + (tcc_ir_op_is_mac(q->op) && ir_xform_operand_reads_memory(tcc_ir_op_get_accum(ir, q))))) + *out_has_memory = 1; + + /* `V <- mem [STORE]` (an inlined parameter bound to a load) stays one + * in-place-renamed multi-def TEMP in SSA (ssa_store_slot_def_pos), which + * is sound for the single copy a loop body has, not for N copies of it in + * straight-line code: later SSA folds took every copy's value from the + * first (fuzz 308/191/212 family). */ + if (q->op == TCCIR_OP_STORE) + { + IROperand d = tcc_ir_op_get_dest(ir, q); + int32_t dvr = irop_get_vreg(d); + if (dvr >= 0 && TCCIR_DECODE_VREG_TYPE(dvr) == TCCIR_VREG_TYPE_VAR && !d.is_llocal && + ir_xform_operand_reads_memory(tcc_ir_op_get_src1(ir, q))) + { + LOG_LOOP_OPT("collect_body: REJECTED at [%d] memory load bound to a VAR by STORE", i); + return -1; + } + } - /* Reject 4-operand ops: write_instr_at_nop copies only 3 slots, dropping the 4th. */ - if (q->op == TCCIR_OP_LOAD_INDEXED || q->op == TCCIR_OP_STORE_INDEXED || - q->op == TCCIR_OP_LOAD_POSTINC || q->op == TCCIR_OP_STORE_POSTINC || - q->op == TCCIR_OP_MLA || q->op == TCCIR_OP_SELECT) + /* A SELECT's 4th slot is its condition token; the others the caller copies. */ + if (q->op == TCCIR_OP_SELECT) { LOG_LOOP_OPT("collect_body: REJECTED at [%d] 4-operand op=%d", i, q->op); return -1; diff --git a/source/opt/flat/loop/loop_ir_mutate.c b/source/opt/flat/loop/loop_ir_mutate.c index a2fc22e35..6505d6b01 100644 --- a/source/opt/flat/loop/loop_ir_mutate.c +++ b/source/opt/flat/loop/loop_ir_mutate.c @@ -115,6 +115,49 @@ void write_instr_at_nop(TCCIRState *ir, int pos, TccIrOp op, IROperand dest, IRO q->operand_base = base; } +/* Grow a side table keyed by orig_index to hold index `idx`. */ +static void *orig_table_grow(void *tab, int *len, int idx, size_t elem) +{ + if (idx < *len) + return tab; + int nlen = idx + 1 > 2 * *len ? idx + 1 : 2 * *len; + tab = tcc_realloc(tab, (size_t)nlen * elem); + memset((char *)tab + (size_t)*len * elem, 0, (size_t)(nlen - *len) * elem); + *len = nlen; + return tab; +} + +/* Give instruction orig_index `to` the codegen annotations of `from`: the + * side tables the backend reads just before codegen (barrel_shifts, + * shift64_dead_half, zero_half64, bfi_params) are keyed by orig_index, and a + * clone stamped with a fresh index by write_instr_at_nop has none -- an + * unrolled `add rd, rn, rm lsl #4` came back as a plain add. */ +void tcc_ir_copy_orig_annotations(TCCIRState *ir, int from, int to) +{ + if (from < 0 || to < 0 || from == to) + return; + if (ir->barrel_shifts && from < ir->barrel_shifts_len && ir->barrel_shifts[from]) + { + ir->barrel_shifts = orig_table_grow(ir->barrel_shifts, &ir->barrel_shifts_len, to, 1); + ir->barrel_shifts[to] = ir->barrel_shifts[from]; + } + if (ir->shift64_dead_half && from < ir->shift64_dead_half_len && ir->shift64_dead_half[from]) + { + ir->shift64_dead_half = orig_table_grow(ir->shift64_dead_half, &ir->shift64_dead_half_len, to, 1); + ir->shift64_dead_half[to] = ir->shift64_dead_half[from]; + } + if (ir->zero_half64 && from < ir->zero_half64_len && ir->zero_half64[from]) + { + ir->zero_half64 = orig_table_grow(ir->zero_half64, &ir->zero_half64_len, to, 1); + ir->zero_half64[to] = ir->zero_half64[from]; + } + if (ir->bfi_params && from < ir->bfi_params_len && ir->bfi_params[from]) + { + ir->bfi_params = orig_table_grow(ir->bfi_params, &ir->bfi_params_len, to, sizeof(uint16_t)); + ir->bfi_params[to] = ir->bfi_params[from]; + } +} + /* Write a SELECT into a NOP slot; 4 pool entries: dest, then, else, cond at base+3. */ void write_select_at_nop(TCCIRState *ir, int pos, IROperand dest, IROperand then_val, IROperand else_val, int cond_tok) diff --git a/source/opt/flat/loop/loop_unroll.c b/source/opt/flat/loop/loop_unroll.c index f052e6221..7acfe83b8 100644 --- a/source/opt/flat/loop/loop_unroll.c +++ b/source/opt/flat/loop/loop_unroll.c @@ -19,6 +19,145 @@ #include "opt_loop_utils.h" +/* Ops with a 4th pool slot at operand_base+3: a multiply-accumulate's + * accumulator, an indexed access's scale. */ +static int unroll_has_op4(int op) +{ + return tcc_ir_op_is_mac(op) || op == TCCIR_OP_LOAD_INDEXED || op == TCCIR_OP_STORE_INDEXED; +} + +/* How many of a body's instructions survive once the IV is a constant: an + * ALU op or copy computed only from immediates, frame or symbol addresses and + * earlier foldable values folds away, and so does a load through a foldable + * address into a const-qualified symbol (a table). Everything else is real + * code in every copy. */ +static int unroll_residual_insns(TCCIRState *ir, const int *ops, const IROperand *dests, const IROperand *src1s, + const IROperand *src2s, int count, int32_t iv_vr, int *table_loads) +{ +#define UNROLL_MAX_FOLD 64 + int32_t fold_vr[UNROLL_MAX_FOLD]; + uint8_t fold_tab[UNROLL_MAX_FOLD]; /* the value addresses a const table */ + int nfold = 0, residual = 0; + for (int b = 0; b < count; b++) + { + const int op = ops[b]; + int foldable = 0, tab = 0; + const IROperand srcs[2] = {src1s[b], src2s[b]}; + const int nsrc = irop_config[op].has_src2 ? 2 : 1; + if (op == TCCIR_OP_ADD || op == TCCIR_OP_SUB || op == TCCIR_OP_SHL || op == TCCIR_OP_SHR || op == TCCIR_OP_SAR || + op == TCCIR_OP_MUL || op == TCCIR_OP_AND || op == TCCIR_OP_OR || op == TCCIR_OP_XOR || + op == TCCIR_OP_ASSIGN || op == TCCIR_OP_LOAD) + { + foldable = 1; + for (int k = 0; k < nsrc && foldable; k++) + { + IROperand o = srcs[k]; + int32_t vr = irop_get_vreg(o); + int tag = irop_get_tag(o); + if (vr == iv_vr && vr >= 0 && !(o.is_lval && TCCIR_DECODE_VREG_TYPE(vr) == TCCIR_VREG_TYPE_TEMP)) + continue; + if (vr < 0 && !o.is_lval && (tag == IROP_TAG_IMM32 || tag == IROP_TAG_I64 || tag == IROP_TAG_STACKOFF)) + continue; + if (vr < 0 && !o.is_lval && tag == IROP_TAG_SYMREF) + { + IRPoolSymref *sr = irop_get_symref_ex(ir, o); + if (sr && sr->sym && (sr->sym->type.t & VT_CONSTANT)) + tab = 1; + continue; + } + int f = -1; + for (int j = 0; j < nfold; j++) + if (fold_vr[j] == vr) + f = j; + if (vr < 0 || f < 0) + { + foldable = 0; + break; + } + if (o.is_lval && TCCIR_DECODE_VREG_TYPE(vr) == TCCIR_VREG_TYPE_TEMP) + { + /* A load through a foldable address: folds only from a table. */ + if (!fold_tab[f] || op != TCCIR_OP_LOAD) + foldable = 0; + else + *table_loads = 1; + continue; + } + tab |= fold_tab[f]; + } + } + IROperand d = dests[b]; + if (foldable && irop_config[op].has_dest && !d.is_lval && irop_get_vreg(d) >= 0 && nfold < UNROLL_MAX_FOLD) + { + fold_vr[nfold] = irop_get_vreg(d); + fold_tab[nfold] = op == TCCIR_OP_LOAD ? 0 : tab; + nfold++; + } + else if ((op == TCCIR_OP_ASSIGN || op == TCCIR_OP_LOAD) && irop_get_vreg(src1s[b]) >= 0 && + !(src1s[b].is_lval && TCCIR_DECODE_VREG_TYPE(irop_get_vreg(src1s[b])) == TCCIR_VREG_TYPE_TEMP) && + !src1s[b].is_llocal) + ; /* a register copy: coalesced or a move */ + else + residual++; + } + return residual; +#undef UNROLL_MAX_FOLD +} + +/* The fused barrel shift on src2 of the instruction numbered ORIG, or 0. */ +static uint8_t unroll_src2_shift(TCCIRState *ir, int orig) +{ + if (!ir->barrel_shifts || orig < 0 || orig >= ir->barrel_shifts_len) + return 0; + return ir->barrel_shifts[orig]; +} + +/* V shifted as barrel_shifts encoding ENC says ((type << 5) | amount, type + * 1 LSL, 2 LSR, 3 ASR, 4 ROR), 32-bit. 0 for an encoding it does not know. */ +static int unroll_apply_shift(uint8_t enc, int32_t v, int32_t *out) +{ + unsigned amount = enc & 31u; + uint32_t x = (uint32_t)v; + switch (enc >> 5) + { + case 1: *out = (int32_t)(x << amount); return 1; + case 2: *out = (int32_t)(x >> amount); return 1; + case 3: *out = amount ? (int32_t)((int32_t)x >> amount) : v; return 1; + case 4: *out = (int32_t)(amount ? (x >> amount) | (x << (32 - amount)) : x); return 1; + } + return 0; +} + +/* Whether OP is a plain use of a TEMP whose only definition is + * `T <- GlobalSym` for a const-qualified symbol; *SYMVAL gets that operand. */ +static int unroll_const_symbol_temp(TCCIRState *ir, IROperand op, IROperand *symval) +{ + int32_t vr = irop_get_vreg(op); + if (vr < 0 || TCCIR_DECODE_VREG_TYPE(vr) != TCCIR_VREG_TYPE_TEMP || op.is_lval || op.is_llocal || + irop_get_tag(op) != IROP_TAG_VREG) + return 0; + int defs = 0; + IROperand src = {0}; + for (int i = 0; i < ir->next_instruction_index && defs < 2; i++) + { + IRQuadCompact *q = &ir->compact_instructions[i]; + if (q->op == TCCIR_OP_NOP || !irop_config[q->op].has_dest) + continue; + IROperand d = tcc_ir_op_get_dest(ir, q); + if (irop_get_vreg(d) != vr || !irop_dest_defines_vreg(d)) + continue; + defs++; + src = q->op == TCCIR_OP_ASSIGN ? tcc_ir_op_get_src1(ir, q) : (IROperand){0}; + } + if (defs != 1 || irop_get_tag(src) != IROP_TAG_SYMREF || src.is_lval || irop_get_vreg(src) >= 0) + return 0; + IRPoolSymref *sr = irop_get_symref_ex(ir, src); + if (!sr || !sr->sym || !(sr->sym->type.t & VT_CONSTANT) || irop_get_btype(src) != irop_get_btype(op)) + return 0; + *symval = src; + return 1; +} + /* loops/loop_idx let sibling loop records be patched when the IR grows; NULL/0 if unused. */ int try_unroll_loop_ex(TCCIRState *ir, IRLoop *loop, IRLoops *loops, int loop_idx) { @@ -65,17 +204,20 @@ int try_unroll_loop_ex(TCCIRState *ir, IRLoop *loop, IRLoops *loops, int loop_id } int ret = 0; - size_t _usz = UNROLL_MAX_BODY_INSNS * (2 * sizeof(int) + 3 * sizeof(IROperand)); + size_t _usz = UNROLL_MAX_BODY_INSNS * (3 * sizeof(int) + 4 * sizeof(IROperand)); char *_ubuf = (char *)tcc_mallocz(_usz); char *_up = _ubuf; int *body_indices = (int *)_up; _up += UNROLL_MAX_BODY_INSNS * sizeof(int); int *body_ops = (int *)_up; _up += UNROLL_MAX_BODY_INSNS * sizeof(int); + int *body_origs = (int *)_up; _up += UNROLL_MAX_BODY_INSNS * sizeof(int); IROperand *body_dests = (IROperand *)_up; _up += UNROLL_MAX_BODY_INSNS * sizeof(IROperand); IROperand *body_src1s = (IROperand *)_up; _up += UNROLL_MAX_BODY_INSNS * sizeof(IROperand); - IROperand *body_src2s = (IROperand *)_up; + IROperand *body_src2s = (IROperand *)_up; _up += UNROLL_MAX_BODY_INSNS * sizeof(IROperand); + IROperand *body_op4s = (IROperand *)_up; + int has_memory = 0; int body_count = collect_body_instructions(ir, loop, iv->vreg, cmp_idx, jmpif_idx, iv->def_idx, body_indices, - UNROLL_MAX_BODY_INSNS); + UNROLL_MAX_BODY_INSNS, &has_memory); if (body_count <= 0 || body_count > UNROLL_MAX_BODY_INSNS) { LOG_LOOP_OPT("try_unroll_loop: body_count=%d (invalid or > %d), giving up", body_count, UNROLL_MAX_BODY_INSNS); @@ -83,7 +225,7 @@ int try_unroll_loop_ex(TCCIRState *ir, IRLoop *loop, IRLoops *loops, int loop_id } int total_insns = trip_count * body_count; - if (total_insns > UNROLL_MAX_TOTAL_INSNS) + if (total_insns > (has_memory ? UNROLL_MAX_RAW_MEM_INSNS : UNROLL_MAX_TOTAL_INSNS)) { LOG_LOOP_OPT("try_unroll_loop: total_insns=%d > %d, giving up", total_insns, UNROLL_MAX_TOTAL_INSNS); goto unroll_cleanup; @@ -93,9 +235,13 @@ int try_unroll_loop_ex(TCCIRState *ir, IRLoop *loop, IRLoops *loops, int loop_id IRQuadCompact *bq = &ir->compact_instructions[body_indices[b]]; int op = bq->op; body_ops[b] = op; + body_origs[b] = bq->orig_index; body_dests[b] = (IROperand){0}; body_src1s[b] = (IROperand){0}; body_src2s[b] = (IROperand){0}; + body_op4s[b] = (IROperand){0}; + if (unroll_has_op4(op)) + body_op4s[b] = ir->iroperand_pool[bq->operand_base + 3]; if (irop_config[op].has_dest) body_dests[b] = ir->iroperand_pool[bq->operand_base]; if (irop_config[op].has_src1) @@ -104,6 +250,64 @@ int try_unroll_loop_ex(TCCIRState *ir, IRLoop *loop, IRLoops *loops, int loop_id body_src2s[b] = ir->iroperand_pool[bq->operand_base + irop_config[op].has_dest + irop_config[op].has_src1]; } + /* A constant table's address hoisted out of the loop (`T <- GlobalSym`, + * T's only definition) comes back as the symbol in every copy, so the + * copies' reads are table reads at constant offsets that fold (Zig's + * MultiArrayList.slice inlined into a big caller: the table addresses are + * shared, and the loads stayed). */ + for (int b = 0; b < body_count; b++) + { + IROperand *srcs[2] = {&body_src1s[b], &body_src2s[b]}; + for (int k = 0; k < 2; k++) + { + if (k == 0 ? !irop_config[body_ops[b]].has_src1 : !irop_config[body_ops[b]].has_src2) + continue; + IROperand symval; + /* A src2 carrying a fused barrel shift stays a register. */ + if (k == 1 && unroll_src2_shift(ir, body_origs[b])) + continue; + if (unroll_const_symbol_temp(ir, *srcs[k], &symval)) + *srcs[k] = symval; + } + } + + int table_loads = 0; + if (has_memory && + trip_count * unroll_residual_insns(ir, body_ops, body_dests, body_src1s, body_src2s, body_count, iv->vreg, + &table_loads) > + UNROLL_MAX_TOTAL_MEM_INSNS) + { + LOG_LOOP_OPT("try_unroll_loop: residual after folding too large, giving up"); + goto unroll_cleanup; + } + + /* The IV is replaced by a constant in every copy: only as a value. An IV + * that is written in the body, or dereferenced as a pointer TEMP, keeps + * the loop. (A VAR IV read by LOAD is a plain read; see below.) */ + for (int b = 0; b < body_count; b++) + { + IROperand ops[4] = {body_dests[b], body_src1s[b], body_src2s[b], body_op4s[b]}; + for (int k = 0; k < 4; k++) + { + if (k == 0 && !irop_config[body_ops[b]].has_dest) + continue; + if (irop_get_vreg(ops[k]) != iv->vreg) + continue; + if (k == 0 || (ops[k].is_lval && TCCIR_DECODE_VREG_TYPE(iv->vreg) == TCCIR_VREG_TYPE_TEMP)) + { + LOG_LOOP_OPT("try_unroll_loop: IV written or dereferenced in body, giving up"); + goto unroll_cleanup; + } + int32_t shifted; + uint8_t enc = k == 2 ? unroll_src2_shift(ir, body_origs[b]) : 0; + if (enc && !unroll_apply_shift(enc, 0, &shifted)) + { + LOG_LOOP_OPT("try_unroll_loop: IV under an unknown barrel shift, giving up"); + goto unroll_cleanup; + } + } + } + /* Use only [start_idx..end_idx]; the extended body_instrs can include post-loop instrs. */ int loop_end = loop->end_idx; @@ -247,7 +451,7 @@ int try_unroll_loop_ex(TCCIRState *ir, IRLoop *loop, IRLoops *loops, int loop_id IRQuadCompact *q = &ir->compact_instructions[i]; if (q->op == TCCIR_OP_NOP) continue; - for (int slot = 0; slot < 3; slot++) + for (int slot = 0; slot < 4; slot++) { IROperand op; if (slot == 0) @@ -260,6 +464,11 @@ int try_unroll_loop_ex(TCCIRState *ir, IRLoop *loop, IRLoops *loops, int loop_id if (!irop_config[q->op].has_src1) continue; op = tcc_ir_op_get_src1(ir, q); } + else if (slot == 3) + { + if (!unroll_has_op4(q->op)) continue; + op = ir->iroperand_pool[q->operand_base + 3]; + } else { if (!irop_config[q->op].has_src2) continue; @@ -289,14 +498,38 @@ int try_unroll_loop_ex(TCCIRState *ir, IRLoop *loop, IRLoops *loops, int loop_id IROperand dest = body_dests[b]; IROperand src1 = body_src1s[b]; IROperand src2 = body_src2s[b]; + IROperand op4 = body_op4s[b]; int iv_val = iv->init_val + k * iv->step; IROperand iv_const = irop_make_imm32(-1, iv_val, IROP_BTYPE_INT32); if (irop_get_vreg(src1) == iv->vreg) + { src1 = iv_const; + /* `X <- Viv [LOAD]` reads the IV variable: now a constant. */ + if (saved_op == TCCIR_OP_LOAD) + saved_op = TCCIR_OP_ASSIGN; + } + /* A fused barrel shift on src2 (barrel_shifts[orig_index]) applies to + * the IV there: every consumer ignores a shift on an immediate, so the + * copy gets the shifted value and no annotation -- `base + (j << 4)` + * came out as `base + j` (pr93434: stores at t2 + 17, 18, ...). */ + int shift_folded = 0; if (irop_get_vreg(src2) == iv->vreg) + { + uint8_t enc = unroll_src2_shift(ir, body_origs[b]); src2 = iv_const; + if (enc) + { + int32_t v; + if (!unroll_apply_shift(enc, iv_val, &v)) + goto unroll_cleanup; /* not reached: checked before the loop went */ + src2 = irop_make_imm32(-1, v, IROP_BTYPE_INT32); + shift_folded = 1; + } + } + if (unroll_has_op4(saved_op) && irop_get_vreg(op4) == iv->vreg) + op4 = iv_const; for (int r = 0; r < rename_count; r++) { @@ -323,6 +556,13 @@ int try_unroll_loop_ex(TCCIRState *ir, IRLoop *loop, IRLoops *loops, int loop_id TCCIR_DECODE_VREG_POSITION(vr) == old_pos) irop_set_vreg(&src2, new_vr); } + if (unroll_has_op4(saved_op)) + { + int32_t vr = irop_get_vreg(op4); + if (vr >= 0 && TCCIR_DECODE_VREG_TYPE(vr) == TCCIR_VREG_TYPE_TEMP && + TCCIR_DECODE_VREG_POSITION(vr) == old_pos) + irop_set_vreg(&op4, new_vr); + } } while (write_pos <= loop_end && ir->compact_instructions[write_pos].op != TCCIR_OP_NOP) @@ -332,6 +572,10 @@ int try_unroll_loop_ex(TCCIRState *ir, IRLoop *loop, IRLoops *loops, int loop_id goto unroll_cleanup; /* should not happen: avail_slots check prevents it */ write_instr_at_nop(ir, write_pos, saved_op, dest, src1, src2); + if (unroll_has_op4(saved_op)) + tcc_ir_pool_add(ir, op4); /* lands at operand_base + 3 */ + if (!shift_folded) + tcc_ir_copy_orig_annotations(ir, body_origs[b], ir->compact_instructions[write_pos].orig_index); write_pos++; } } @@ -400,6 +644,8 @@ int try_unroll_loop_ex(TCCIRState *ir, IRLoop *loop, IRLoops *loops, int loop_id } ret = 1; + if (table_loads) + ir->unrolled_table_loads = 1; unroll_cleanup: tcc_free(_ubuf); diff --git a/source/opt/flat/loop/reroll.c b/source/opt/flat/loop/reroll.c index 728fb2591..b76ac5550 100644 --- a/source/opt/flat/loop/reroll.c +++ b/source/opt/flat/loop/reroll.c @@ -123,6 +123,7 @@ static int op_is_unsafe_for_reroll(TccIrOp op) case TCCIR_OP_BUILTIN_APPLY_ARGS: case TCCIR_OP_BUILTIN_APPLY: case TCCIR_OP_BUILTIN_RETURN: + case TCCIR_OP_RETURN_ADDRESS: case TCCIR_OP_ASM_INPUT: case TCCIR_OP_INLINE_ASM: case TCCIR_OP_ASM_OUTPUT: diff --git a/source/opt/flat/memory/bitfield_unit_narrow.c b/source/opt/flat/memory/bitfield_unit_narrow.c index 5b3fb6752..97e59215d 100644 --- a/source/opt/flat/memory/bitfield_unit_narrow.c +++ b/source/opt/flat/memory/bitfield_unit_narrow.c @@ -204,10 +204,11 @@ static int bfun_sole_def(TCCIRState *ir, int32_t vr) for (int i = 0; i < ir->next_instruction_index; i++) { IRQuadCompact *q = &ir->compact_instructions[i]; - if (q->op == TCCIR_OP_NOP || !irop_config[q->op].has_dest) + if (q->op == TCCIR_OP_NOP || !irop_config[q->op].has_dest || q->op == TCCIR_OP_STORE_INDEXED) continue; + /* A VAR written as itself is a STACKOFF lvalue: still a definition. */ IROperand d = tcc_ir_op_get_dest(ir, q); - if (irop_get_vreg(d) != vr || d.is_lval) + if (irop_get_vreg(d) != vr || !irop_dest_defines_vreg(d)) continue; if (found >= 0) return -1; diff --git a/source/opt/flat/memory/block_copy_init.c b/source/opt/flat/memory/block_copy_init.c index 8cc02f658..35f53f12d 100644 --- a/source/opt/flat/memory/block_copy_init.c +++ b/source/opt/flat/memory/block_copy_init.c @@ -8,6 +8,17 @@ * License as published by the Free Software Foundation. */ +/* A memset(stack, 0, N) followed by a run of constant stores into the same + * object is an initializer: `struct T x = {...}` or a compound literal. The + * run is replaced by one BLOCK_COPY from an image of the initialized bytes in + * .rodata (or .data when it holds relocations and .rodata is shared), which + * the backend emits as LDM/STM or a memcpy call. Identical images are emitted + * once per section, so a table the source repeats costs its bytes once. + * + * Each store writes its own width into the image; a run ends at the first + * instruction that is not a constant store inside the object (a load, a + * volatile or unaligned address store, one overwriting a relocation). */ + #define USING_GLOBALS #include "ir.h" @@ -15,6 +26,111 @@ #include "opt_utils.h" +typedef struct BciImage +{ + struct BciImage *next; + TCCState *s1; + Section *sec; + addr_t sec_off; + int size; + uint32_t hash; + BciReloc *rel; + int nrel; + Sym *sym; +} BciImage; + +/* Images already emitted, for reuse by an identical initializer anywhere in + * the output. */ +static BciImage *bci_images; + +static uint32_t bci_hash(const uint8_t *p, int n, const BciReloc *rel, int nrel) +{ + uint32_t h = 2166136261u; + for (int i = 0; i < n; i++) + h = (h ^ p[i]) * 16777619u; + for (int i = 0; i < nrel; i++) + h = (h ^ (uint32_t)rel[i].off ^ (uint32_t)(uintptr_t)rel[i].sym) * 16777619u; + return h; +} + +/* Field by field: BciReloc has padding on a 64-bit host, and the arrays are + * filled one field at a time, so a memcmp of them compares heap garbage. */ +static int bci_rel_equal(const BciReloc *a, const BciReloc *b, int nrel) +{ + for (int i = 0; i < nrel; i++) + if (a[i].off != b[i].off || a[i].sym != b[i].sym || a[i].addend != b[i].addend) + return 0; + return 1; +} + +Sym *bci_image_sym(Section *sec, const uint8_t *bytes, int size, const BciReloc *rel, int nrel) +{ + uint32_t h = bci_hash(bytes, size, rel, nrel); + BciImage **pp = &bci_images; + while (*pp) + { + BciImage *e = *pp; + if (e->s1 != tcc_state) + { + /* A previous compilation state's images: its sections are gone. */ + *pp = e->next; + tcc_free(e->rel); + tcc_free(e); + continue; + } + if (e->sec == sec && e->size == size && e->hash == h && e->nrel == nrel && + !memcmp(sec->data + e->sec_off, bytes, size) && bci_rel_equal(e->rel, rel, nrel)) + return e->sym; + pp = &e->next; + } + + addr_t off = section_add(sec, size, 4); + memcpy(sec->data + off, bytes, size); + for (int i = 0; i < nrel; i++) + greloc(sec, rel[i].sym, off + rel[i].off, R_DATA_PTR); + CType ctype; + /* const-qualified: nothing ever writes an image, so loads from it fold + * (global_init_prop). */ + ctype.t = VT_PTR | VT_CONST | VT_CONSTANT; + ctype.ref = NULL; + Sym *sym = get_sym_ref(&ctype, sec, off, size); + + BciImage *e = tcc_mallocz(sizeof *e); + e->s1 = tcc_state; + e->sec = sec; + e->sec_off = off; + e->size = size; + e->hash = h; + e->nrel = nrel; + if (nrel) + { + e->rel = tcc_malloc(sizeof *rel * nrel); + memcpy(e->rel, rel, sizeof *rel * nrel); + } + e->sym = sym; + e->next = bci_images; + bci_images = e; + return sym; +} + +/* Bytes a STORE writes, from its destination's type. */ +static int bci_store_width(IROperand dest) +{ + if (irop_is_64bit(dest)) + return 8; + switch (irop_get_btype(dest)) + { + case IROP_BTYPE_INT8: + return 1; + case IROP_BTYPE_INT16: + return 2; + case IROP_BTYPE_INT32: + case IROP_BTYPE_FLOAT32: + return 4; + default: + return 0; + } +} int tcc_ir_opt_block_copy_init(TCCIRState *ir) { @@ -24,6 +140,12 @@ int tcc_ir_opt_block_copy_init(TCCIRState *ir) if (n == 0) return 0; + int cap = 0, nstores; + int *store_indices = NULL; + uint8_t *image = NULL, *is_reloc = NULL; + BciReloc *rel = NULL; + int image_cap = 0; + for (int i = 0; i < n; i++) { IRQuadCompact *q = &ir->compact_instructions[i]; @@ -56,20 +178,28 @@ int tcc_ir_opt_block_copy_init(TCCIRState *ir) if (irop_get_tag(param_size) != IROP_TAG_IMM32) continue; int total_size = (int)irop_get_imm64_ex(ir, param_size); - if (total_size <= 0 || (total_size & 3) || total_size > 1024) + if (total_size <= 0 || (total_size & 3)) continue; - if (irop_get_tag(param_dest) != IROP_TAG_STACKOFF) + if (irop_get_tag(param_dest) != IROP_TAG_STACKOFF || irop_get_vreg(param_dest) >= 0) continue; int base_offset = (int)irop_get_imm64_ex(ir, param_dest); + /* The inline copy moves words (LDM/STM, LDRD/STRD), which fault unaligned. */ + if (base_offset & 3) + continue; - int store_indices[256]; - int store_offsets[256]; /* byte offset relative to base */ - int store_sizes[256]; /* 4 or 8 bytes */ - IROperand store_values[256]; - int nstores = 0; + if (total_size > image_cap) + { + image_cap = total_size; + image = tcc_realloc(image, image_cap); + is_reloc = tcc_realloc(is_reloc, image_cap); + } + memset(image, 0, total_size); + memset(is_reloc, 0, total_size); + nstores = 0; + int nrel = 0; - for (int j = i + 1; j < n && nstores < 256; j++) + for (int j = i + 1; j < n; j++) { IRQuadCompact *sq = &ir->compact_instructions[j]; if (sq->op == TCCIR_OP_NOP) @@ -80,78 +210,70 @@ int tcc_ir_opt_block_copy_init(TCCIRState *ir) IROperand st_dest = tcc_ir_op_get_dest(ir, sq); IROperand st_src = tcc_ir_op_get_src1(ir, sq); - if (irop_get_tag(st_dest) != IROP_TAG_STACKOFF || !st_dest.is_local) + if (irop_get_tag(st_dest) != IROP_TAG_STACKOFF || !st_dest.is_local || !st_dest.is_lval || + st_dest.is_llocal || irop_get_vreg(st_dest) >= 0 || tcc_ir_access_is_volatile(ir, st_dest)) break; - int st_off = (int)irop_get_imm64_ex(ir, st_dest); - int rel_off = st_off - base_offset; - int is_wide = irop_is_64bit(st_dest); - int st_size = is_wide ? 8 : 4; - - if (rel_off < 0 || rel_off + st_size > total_size || (rel_off & 3)) + int rel_off = (int)irop_get_imm64_ex(ir, st_dest) - base_offset; + int st_size = bci_store_width(st_dest); + if (st_size == 0 || rel_off < 0 || rel_off + st_size > total_size) break; int src_tag = irop_get_tag(st_src); + if (st_src.is_lval || st_src.is_local || st_src.is_llocal || irop_get_vreg(st_src) >= 0) + break; if (src_tag != IROP_TAG_SYMREF && src_tag != IROP_TAG_IMM32 && src_tag != IROP_TAG_I64 && src_tag != IROP_TAG_F32 && src_tag != IROP_TAG_F64) break; - - store_indices[nstores] = j; - store_offsets[nstores] = rel_off; - store_sizes[nstores] = st_size; - store_values[nstores] = st_src; - nstores++; - } - - if (nstores < 2) - continue; - - /* A block holding a symref needs a relocation, so it cannot go in shared - * read-only .rodata; place it in the writable data segment instead. */ - int block_has_symref = 0; - for (int s = 0; s < nstores; s++) - { - if (irop_get_tag(store_values[s]) == IROP_TAG_SYMREF) - { - block_has_symref = 1; + if (src_tag == IROP_TAG_SYMREF && (st_size != 4 || (rel_off & 3))) + break; + IRPoolSymref *symref = src_tag == IROP_TAG_SYMREF ? irop_get_symref_ex(ir, st_src) : NULL; + if (src_tag == IROP_TAG_SYMREF && (!symref || !symref->sym)) break; - } - } - Section *block_sec = - (block_has_symref && tcc_state->share_rodata) ? data_section : rodata_section; - size_t rodata_offset = section_add(block_sec, total_size, 4); - uint8_t *rodata_ptr = block_sec->data + rodata_offset; - memset(rodata_ptr, 0, total_size); + /* A relocation's word is written only whole, by the symref itself. */ + int hits_reloc = 0; + for (int b = rel_off; b < rel_off + st_size; b++) + hits_reloc |= is_reloc[b]; + if (hits_reloc) + break; - for (int s = 0; s < nstores; s++) - { - int src_tag = irop_get_tag(store_values[s]); - if (src_tag == IROP_TAG_SYMREF) + if (nstores == cap) { - IRPoolSymref *symref = irop_get_symref_ex(ir, store_values[s]); - if (symref && symref->sym) - { - write32le(rodata_ptr + store_offsets[s], symref->addend); - greloc(block_sec, symref->sym, rodata_offset + store_offsets[s], R_DATA_PTR); - } + cap = cap ? cap * 2 : 256; + store_indices = tcc_realloc(store_indices, sizeof(int) * cap); + rel = tcc_realloc(rel, sizeof(BciReloc) * cap); } - else if (store_sizes[s] == 8) + store_indices[nstores++] = j; + + if (symref) { - int64_t val = irop_get_imm64_ex(ir, store_values[s]); - write64le(rodata_ptr + store_offsets[s], (uint64_t)val); + write32le(image + rel_off, symref->addend); + memset(is_reloc + rel_off, 1, 4); + rel[nrel].off = rel_off; + rel[nrel].sym = symref->sym; + rel[nrel].addend = symref->addend; + nrel++; } else { - int32_t val = (int32_t)irop_get_imm64_ex(ir, store_values[s]); - write32le(rodata_ptr + store_offsets[s], val); + uint64_t val = (uint64_t)irop_get_imm64_ex(ir, st_src); + for (int b = 0; b < st_size; b++) + image[rel_off + b] = (uint8_t)(val >> (8 * b)); } } - CType ctype; - ctype.t = VT_PTR | VT_CONST; - ctype.ref = NULL; - Sym *rodata_sym = get_sym_ref(&ctype, block_sec, rodata_offset, total_size); + if (nstores < 2) + continue; + /* Past the old 1 KiB limit, only when the image costs no more than the + * code it replaces: a constant store takes 4-10 bytes of Thumb-2. */ + if (total_size > 1024 && nstores * 6 < total_size) + continue; + + /* A block holding a symref needs a relocation, so it cannot go in shared + * read-only .rodata; place it in the writable data segment instead. */ + Section *block_sec = (nrel && tcc_state->share_rodata) ? data_section : rodata_section; + Sym *rodata_sym = bci_image_sym(block_sec, image, total_size, rel, nrel); IROperand bc_dest = irop_make_stackoff(-1, base_offset, 1, 0, 0, IROP_BTYPE_INT32); uint32_t sym_pool_idx = tcc_ir_pool_add_symref(ir, rodata_sym, 0, 0); @@ -175,5 +297,9 @@ int tcc_ir_opt_block_copy_init(TCCIRState *ir) changes++; } + tcc_free(store_indices); + tcc_free(rel); + tcc_free(image); + tcc_free(is_reloc); return changes; } diff --git a/source/opt/flat/memory/const_local_table.c b/source/opt/flat/memory/const_local_table.c new file mode 100644 index 000000000..3e1fd6656 --- /dev/null +++ b/source/opt/flat/memory/const_local_table.c @@ -0,0 +1,520 @@ +/* + * TCC IR - constant local tables read from .rodata (flat, pre-SSA) + * + * Copyright (c) 2026 Mateusz Stadnik + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation. + */ + +/* A frame object written only by one run of constant stores that covers every + * byte of it, and whose address is only ever read through, holds the same + * bytes at every read: reads through its address go to an image of those bytes + * in .rodata instead. The Zig C backend indexes an all-constant compound + * literal with a loop counter -- `(struct arr_8_usize){{4,4,...,2}}.array[i]`, + * in every MultiArrayList.slice -- which rebuilt the table on the stack on + * every iteration. + * + * The object is the frontend's extent containing the address (before frame + * relayout). Its address goes into TEMPs -- by LEA, or folded into an + * ASSIGN/ADD/SUB operand; every use of such a + * TEMP, and of TEMPs derived from it by ASSIGN/ADD/SUB, must be a read through + * it (a deref source, or the base of LOAD_INDEXED) or a further derivation. + * Any other reference to the object's bytes -- a store outside the run, its + * address passed or stored, an asm -- keeps it. A read before the run has + * executed reads indeterminate memory, so the image serves it too. Direct + * frame loads of the object are left alone, and the run with them; with none, + * the run is deleted. */ + +#define USING_GLOBALS + +#include "ir.h" +#include "opt.h" +#include "opt_engine.h" +#include "opt_alias.h" +#include "opt_utils.h" + +#define CLT_MAX_OBJ 512 +#define CLT_MAX_SIZE 65536 + +typedef struct +{ + int lo, hi; /* frame extent [lo, hi) */ + int first, last; /* instruction range of the store run; -1 when none */ + int nstores; + uint8_t bad; + uint8_t direct_read; + int zcall; /* memset(obj, 0, size) the run starts with; -1 when none */ + int zfirst; /* its earliest PARAM */ +} CltObj; + +static int clt_find(const CltObj *objs, int nobj, int off) +{ + for (int k = 0; k < nobj; k++) + if (off >= objs[k].lo && off < objs[k].hi) + return k; + return -1; +} + +static int clt_is_temp(int32_t vr) +{ + return vr >= 0 && TCCIR_DECODE_VREG_TYPE(vr) == TCCIR_VREG_TYPE_TEMP; +} + +/* Tracked address TEMPs: TEMP position -> object index, -1 untracked. */ +typedef struct +{ + int *obj; + int n; +} CltTemps; + +static int clt_temp_obj(const CltTemps *t, int32_t vr) +{ + int pos = TCCIR_DECODE_VREG_POSITION(vr); + return pos < t->n ? t->obj[pos] : -1; +} + +/* 1 when newly tracked, 0 when already tracked for `obj`, -1 when tracked for + * another object. */ +static int clt_temp_add(CltTemps *t, int32_t vr, int obj) +{ + int pos = TCCIR_DECODE_VREG_POSITION(vr); + if (pos >= t->n) + return -1; + if (t->obj[pos] >= 0) + return t->obj[pos] == obj ? 0 : -1; + t->obj[pos] = obj; + return 1; +} + +static int clt_store_width(IROperand dest) +{ + if (irop_is_64bit(dest)) + return 8; + switch (irop_get_btype(dest)) + { + case IROP_BTYPE_INT8: + return 1; + case IROP_BTYPE_INT16: + return 2; + case IROP_BTYPE_INT32: + case IROP_BTYPE_FLOAT32: + return 4; + default: + return 0; + } +} + +/* A direct frame operand (no vreg): the object's own bytes or address. */ +static int clt_direct_frame(IROperand op) +{ + return irop_get_tag(op) == IROP_TAG_STACKOFF && op.is_local && !op.is_llocal && irop_get_vreg(op) < 0; +} + +/* Operand k of q (0 dest, 1 src1, 2 src2, 3 accum), or 0 when absent. */ +static int clt_operand(TCCIRState *ir, IRQuadCompact *q, int k, IROperand *op) +{ + switch (k) + { + case 0: + if (!irop_config[q->op].has_dest) + return 0; + *op = tcc_ir_op_get_dest(ir, q); + return 1; + case 1: + if (!irop_config[q->op].has_src1) + return 0; + *op = tcc_ir_op_get_src1(ir, q); + return 1; + case 2: + if (!irop_config[q->op].has_src2) + return 0; + *op = tcc_ir_op_get_src2(ir, q); + return 1; + default: + if (!tcc_ir_op_is_mac(q->op)) + return 0; + *op = tcc_ir_op_get_accum(ir, q); + return 1; + } +} + +/* An instruction putting a frame address into a TEMP: `T <- &slot` (LEA or + * ASSIGN), or `T <- &slot +/- x` with the address folded into the ADD/SUB. + * Returns the operand slot holding the address (1 or 2), else 0. */ +static int clt_addr_operand(TCCIRState *ir, IRQuadCompact *q, IROperand *addr) +{ + if (q->op != TCCIR_OP_LEA && q->op != TCCIR_OP_ASSIGN && q->op != TCCIR_OP_ADD && q->op != TCCIR_OP_SUB) + return 0; + IROperand d = tcc_ir_op_get_dest(ir, q); + if (d.is_lval || !clt_is_temp(irop_get_vreg(d))) + return 0; + IROperand s1 = tcc_ir_op_get_src1(ir, q); + if (clt_direct_frame(s1) && !s1.is_lval) + { + *addr = s1; + return 1; + } + if (q->op == TCCIR_OP_ADD) + { + IROperand s2 = tcc_ir_op_get_src2(ir, q); + if (clt_direct_frame(s2) && !s2.is_lval) + { + *addr = s2; + return 2; + } + } + return 0; +} + +/* A constant STORE into a frame object: returns the object, else -1. */ +static int clt_const_store(TCCIRState *ir, IRQuadCompact *q, const CltObj *objs, int nobj) +{ + if (q->op != TCCIR_OP_STORE) + return -1; + IROperand d = tcc_ir_op_get_dest(ir, q), s = tcc_ir_op_get_src1(ir, q); + if (!clt_direct_frame(d) || !d.is_lval || tcc_ir_access_is_volatile(ir, d)) + return -1; + int k = clt_find(objs, nobj, irop_get_stack_offset(d)); + if (k < 0) + return -1; + int w = clt_store_width(d); + int off = irop_get_stack_offset(d); + if (!w || off + w > objs[k].hi) + return -1; + int tag = irop_get_tag(s); + if (s.is_lval || s.is_local || s.is_llocal || irop_get_vreg(s) >= 0) + return -1; + if (tag != IROP_TAG_IMM32 && tag != IROP_TAG_I64 && tag != IROP_TAG_F32 && tag != IROP_TAG_F64) + return -1; + return k; +} + +int tcc_ir_opt_const_local_table(TCCIRState *ir) +{ + const int n = ir->next_instruction_index; + if (n == 0 || ir->frame_relaid || !ir->frame_obj_count || ir->inline_asm_count || tcc_ir_calls_returns_twice(ir)) + return 0; + + /* Candidate objects: those whose address goes into a TEMP. */ + CltObj *objs = tcc_malloc(sizeof(CltObj) * CLT_MAX_OBJ); + int nobj = 0; + for (int i = 0; i < n && nobj < CLT_MAX_OBJ; i++) + { + IRQuadCompact *q = &ir->compact_instructions[i]; + IROperand s; + if (!clt_addr_operand(ir, q, &s)) + continue; + int off = irop_get_stack_offset(s), lo, hi; + if (clt_find(objs, nobj, off) >= 0 || !tcc_ir_frame_object_at(ir, off, &lo, &hi)) + continue; + /* The extent carries a guard byte and alignment padding; the image covers + * the object itself -- the largest recorded at that start, should a later + * scope's object reuse it. */ + int size = -1; + for (int k = 0; k < ir->frame_obj_count; k++) + if (ir->frame_objs[4 * k] == lo && ir->frame_objs[4 * k + 2] > size) + size = ir->frame_objs[4 * k + 2]; + hi = lo + size; + if (size <= 0 || size > CLT_MAX_SIZE || (lo & 3) || off >= hi) + continue; + objs[nobj++] = (CltObj){.lo = lo, .hi = hi, .first = -1, .last = -1, .zcall = -1, .zfirst = -1}; + } + if (!nobj) + { + tcc_free(objs); + return 0; + } + + /* The address TEMPs, to a fixpoint: a derivation may precede, in + * instruction order, the def of the TEMP it derives from. */ + /* Only objects with a constant store can be tables. */ + { + int any = 0; + for (int i = 0; i < n; i++) + { + int k = clt_const_store(ir, &ir->compact_instructions[i], objs, nobj); + if (k >= 0) + objs[k].nstores++; + } + for (int k = 0; k < nobj; k++) + { + if (!objs[k].nstores) + objs[k].bad = 1; + any |= !objs[k].bad; + objs[k].nstores = 0; + } + if (!any) + { + tcc_free(objs); + return 0; + } + } + + /* A big initializer is `memset(obj, 0, size)` and then stores of its + * non-zero bytes only: the zero fill is part of the run. Its address PARAM + * is the only reference to the object it makes. */ + uint8_t *zmark = tcc_mallocz(n); + for (int i = 0; i < n; i++) + { + IRQuadCompact *q = &ir->compact_instructions[i]; + if (q->op != TCCIR_OP_FUNCCALLVOID) + continue; + Sym *callee = irop_get_sym_ex(ir, tcc_ir_op_get_src1(ir, q)); + const char *name = callee ? get_tok_str(callee->v, NULL) : NULL; + if (!name || (strcmp(name, "__aeabi_memset") && strcmp(name, "memset"))) + continue; + IROperand pd, ps, pf; + if (!ir_opt_get_call_param_operand(ir, i, 0, &pd) || !ir_opt_get_call_param_operand(ir, i, 1, &ps) || + !ir_opt_get_call_param_operand(ir, i, 2, &pf)) + continue; + if (!clt_direct_frame(pd) || pd.is_lval || irop_get_tag(ps) != IROP_TAG_IMM32 || + irop_get_tag(pf) != IROP_TAG_IMM32 || irop_get_imm64_ex(ir, pf) != 0) + continue; + int k = clt_find(objs, nobj, irop_get_stack_offset(pd)); + if (k < 0 || irop_get_stack_offset(pd) != objs[k].lo || irop_get_imm64_ex(ir, ps) != objs[k].hi - objs[k].lo) + continue; + if (objs[k].zcall >= 0) + { + objs[k].bad = 1; /* zeroed twice: not one run */ + continue; + } + int lo_p = i, ok = 1; + for (int a = 0; a < 3; a++) + { + int pi = ir_opt_get_call_param_index(ir, i, a); + if (pi < 0) + ok = 0; + else if (pi < lo_p) + lo_p = pi; + } + if (!ok) + continue; + objs[k].zcall = i; + objs[k].zfirst = lo_p; + for (int a = 0; a < 3; a++) + zmark[ir_opt_get_call_param_index(ir, i, a)] = 1; + zmark[i] = 1; + } + + CltTemps temps; + temps.n = ir->next_temporary_variable + 1; + temps.obj = tcc_malloc(sizeof(int) * temps.n); + for (int k = 0; k < temps.n; k++) + temps.obj[k] = -1; + int changed = 1; + for (int rounds = 0; changed && rounds < 8; rounds++) + { + changed = 0; + for (int i = 0; i < n; i++) + { + IRQuadCompact *q = &ir->compact_instructions[i]; + if (!irop_config[q->op].has_dest || !irop_config[q->op].has_src1) + continue; + IROperand d = tcc_ir_op_get_dest(ir, q); + int32_t dv = irop_get_vreg(d); + if (d.is_lval || !clt_is_temp(dv)) + continue; + IROperand s1 = tcc_ir_op_get_src1(ir, q), a; + int k = -1; + if (clt_addr_operand(ir, q, &a)) + k = clt_find(objs, nobj, irop_get_stack_offset(a)); + else if (q->op == TCCIR_OP_ASSIGN || q->op == TCCIR_OP_ADD || q->op == TCCIR_OP_SUB) + { + if (!s1.is_lval && clt_is_temp(irop_get_vreg(s1))) + k = clt_temp_obj(&temps, irop_get_vreg(s1)); + if (k < 0 && q->op == TCCIR_OP_ADD) + { + IROperand s2 = tcc_ir_op_get_src2(ir, q); + if (!s2.is_lval && clt_is_temp(irop_get_vreg(s2))) + k = clt_temp_obj(&temps, irop_get_vreg(s2)); + } + } + if (k < 0) + continue; + int r = clt_temp_add(&temps, dv, k); + if (r < 0) + { + /* One TEMP holding addresses in two objects (or one past the + * TEMP table): keep both. */ + int other = clt_temp_obj(&temps, dv); + objs[k].bad = 1; + if (other >= 0) + objs[other].bad = 1; + } + changed |= r > 0; + } + } + + if (changed) + { + /* Not converged: an unchecked derived TEMP could still write. */ + tcc_free(temps.obj); + tcc_free(objs); + tcc_free(zmark); + return 0; + } + + /* Classify every reference. */ + for (int i = 0; i < n; i++) + { + IRQuadCompact *q = &ir->compact_instructions[i]; + if (q->op == TCCIR_OP_NOP || zmark[i]) + continue; + int sk = clt_const_store(ir, q, objs, nobj); + if (sk >= 0) + { + CltObj *o = &objs[sk]; + if (o->first < 0) + o->first = i; + o->last = i; + o->nstores++; + continue; + } + for (int k = 0; k < 4; k++) + { + IROperand op; + if (!clt_operand(ir, q, k, &op)) + continue; + if (clt_direct_frame(op)) + { + int ob = clt_find(objs, nobj, irop_get_stack_offset(op)); + if (ob < 0) + continue; + IROperand a; + if (k != 0 && clt_addr_operand(ir, q, &a) == k) + continue; /* the address TEMP's uses are checked below */ + if (k != 0 && op.is_lval) + objs[ob].direct_read = 1; + else + objs[ob].bad = 1; + continue; + } + int32_t vr = irop_get_vreg(op); + if (!clt_is_temp(vr)) + continue; + int ob = clt_temp_obj(&temps, vr); + if (ob < 0) + continue; + if (k == 0) + { + /* A def of the TEMP (by one of the derivations above) is fine; a + * store through it writes the object. */ + if (op.is_lval) + objs[ob].bad = 1; + continue; + } + if (op.is_lval) + continue; /* read through the address */ + if (q->op == TCCIR_OP_LOAD_INDEXED && k == 1) + continue; + if ((q->op == TCCIR_OP_ASSIGN || q->op == TCCIR_OP_ADD || q->op == TCCIR_OP_SUB) && + clt_temp_obj(&temps, irop_get_vreg(tcc_ir_op_get_dest(ir, q))) == ob && + !tcc_ir_op_get_dest(ir, q).is_lval && !(q->op == TCCIR_OP_SUB && k == 2)) + continue; + objs[ob].bad = 1; /* the address escapes, or is compared or stored */ + } + } + + int changes = 0; + uint8_t *image = tcc_malloc(CLT_MAX_SIZE), *written = tcc_malloc(CLT_MAX_SIZE); + for (int ob = 0; ob < nobj; ob++) + { + CltObj *o = &objs[ob]; + if (o->bad || o->first < 0) + continue; + const int size = o->hi - o->lo; + memset(image, 0, size); + memset(written, o->zcall >= 0, size); + /* One contiguous run: nothing but this object's constant stores between + * its first and last -- after its zero fill, when it has one. */ + int ok = o->zcall < 0 || o->zcall < o->first; + const int run_lo = o->zcall >= 0 ? o->zfirst : o->first; + for (int i = run_lo; i <= o->last && ok; i++) + { + IRQuadCompact *q = &ir->compact_instructions[i]; + if (q->op == TCCIR_OP_NOP) + continue; + if (zmark[i] && i <= o->zcall) + { + /* one of this fill's PARAMs or its CALL; another call's PARAM + * interleaved would be a different call nested in the run */ + if (i == o->zcall) + continue; + int own = 0; + for (int a = 0; a < 3; a++) + own |= ir_opt_get_call_param_index(ir, o->zcall, a) == i; + if (own) + continue; + ok = 0; + break; + } + if (clt_const_store(ir, q, objs, nobj) != ob) + { + ok = 0; + break; + } + IROperand d = tcc_ir_op_get_dest(ir, q), s = tcc_ir_op_get_src1(ir, q); + int rel = irop_get_stack_offset(d) - o->lo, w = clt_store_width(d); + uint64_t val = (uint64_t)irop_get_imm64_ex(ir, s); + for (int b = 0; b < w; b++) + { + image[rel + b] = (uint8_t)(val >> (8 * b)); + written[rel + b] = 1; + } + } + for (int b = 0; b < size && ok; b++) + ok = written[b]; + if (!ok) + continue; + + Sym *sym = bci_image_sym(rodata_section, image, size, NULL, 0); + for (int i = 0; i < n; i++) + { + IRQuadCompact *q = &ir->compact_instructions[i]; + IROperand s; + int slot = clt_addr_operand(ir, q, &s); + if (!slot || clt_find(objs, nobj, irop_get_stack_offset(s)) != ob) + continue; + uint32_t pidx = tcc_ir_pool_add_symref(ir, sym, irop_get_stack_offset(s) - o->lo, 0); + IROperand sref = irop_make_symref(-1, pidx, 0, 0, 1, IROP_BTYPE_INT32); + IROperand d = tcc_ir_op_get_dest(ir, q), s1 = tcc_ir_op_get_src1(ir, q); + const int has_src2 = q->op != TCCIR_OP_LEA && irop_config[q->op].has_src2; + IROperand s2 = has_src2 ? tcc_ir_op_get_src2(ir, q) : s1; + int base = tcc_ir_iroperand_pool_add(ir, d); + tcc_ir_iroperand_pool_add(ir, slot == 1 ? sref : s1); + if (has_src2) + tcc_ir_iroperand_pool_add(ir, slot == 2 ? sref : s2); + if (q->op == TCCIR_OP_LEA) + q->op = TCCIR_OP_ASSIGN; + q->operand_base = base; + changes++; + } + if (!o->direct_read) + { + for (int i = o->first; i <= o->last; i++) + ir->compact_instructions[i].op = TCCIR_OP_NOP; + if (o->zcall >= 0) + { + ir_opt_nop_call_params(ir, o->zcall); + ir->compact_instructions[o->zcall].op = TCCIR_OP_NOP; + } + } + } + tcc_free(image); + tcc_free(written); + tcc_free(zmark); + tcc_free(temps.obj); + tcc_free(objs); + return changes; +} + +TCC_DBG_ENV_FLAG(clt_disabled, "TCC_NO_CONST_LOCAL_TABLE") + +int tcc_ir_opt_const_local_table_ex(IROptCtx *ctx) +{ + if (clt_disabled()) + return 0; + return tcc_ir_opt_const_local_table(ctx->ir); +} diff --git a/source/opt/flat/memory/dead_static_store.c b/source/opt/flat/memory/dead_static_store.c index dd9290ea6..1a440c3ad 100644 --- a/source/opt/flat/memory/dead_static_store.c +++ b/source/opt/flat/memory/dead_static_store.c @@ -20,6 +20,8 @@ #include "opt_utils.h" #include "opt_alias.h" #include "opt_loop_utils.h" +#include "memref.h" +#include "../ipa/tu_summary.h" /* Eliminate stores to file-scope statics with no TU readers (late_reopt only). */ @@ -30,8 +32,13 @@ static Sym *dss_symref_sym(TCCIRState *ir, IROperand op) return ref ? ref->sym : NULL; } -/* vreg->sym forward-trace map: resolves STORE dests reached through temps. */ +/* vreg->sym forward-trace map: resolves STORE dests reached through temps. + * Flow-insensitive, so a vreg maps to a symbol only while EVERY definition of + * it derives that symbol: one that derives another, or none, poisons the + * entry for good (the last definition in instruction order is not the one + * reaching a use in a loop). A map that fills up answers nothing. */ #define DSS_VREG_MAP_MAX 128 +#define DSS_VREG_MAP_OVERFLOW (DSS_VREG_MAP_MAX + 1) typedef struct { int32_t vreg; @@ -40,19 +47,25 @@ typedef struct static Sym *dss_vreg_map_lookup(const DssVregEntry *map, int count, int32_t vr) { + if (count == DSS_VREG_MAP_OVERFLOW) + return NULL; for (int i = 0; i < count; i++) if (map[i].vreg == vr) return map[i].sym; return NULL; } -static void dss_vreg_map_set(DssVregEntry *map, int *count, int32_t vr, Sym *sym) +/* Record one definition of vr, deriving sym (NULL: none). */ +static void dss_vreg_map_def(DssVregEntry *map, int *count, int32_t vr, Sym *sym) { + if (*count == DSS_VREG_MAP_OVERFLOW) + return; for (int i = 0; i < *count; i++) { if (map[i].vreg == vr) { - map[i].sym = sym; + if (map[i].sym != sym) + map[i].sym = NULL; /* poisoned: NULL never matches a later sym */ return; } } @@ -62,18 +75,8 @@ static void dss_vreg_map_set(DssVregEntry *map, int *count, int32_t vr, Sym *sym map[*count].sym = sym; (*count)++; } -} - -static void dss_vreg_map_clear(DssVregEntry *map, int *count, int32_t vr) -{ - for (int i = 0; i < *count; i++) - { - if (map[i].vreg == vr) - { - map[i].sym = NULL; - return; - } - } + else + *count = DSS_VREG_MAP_OVERFLOW; } static void dss_build_vreg_map(TCCIRState *ir, DssVregEntry *map, int *count) @@ -85,16 +88,23 @@ static void dss_build_vreg_map(TCCIRState *ir, DssVregEntry *map, int *count) IRQuadCompact *q = &ir->compact_instructions[i]; if (q->op == TCCIR_OP_NOP) continue; - if (!irop_config[q->op].has_dest) + /* A post-increment moves its base pointer: a definition deriving nothing. */ + if (q->op == TCCIR_OP_LOAD_POSTINC) + dss_vreg_map_def(map, count, irop_get_vreg(tcc_ir_op_get_src1(ir, q)), NULL); + if (!irop_config[q->op].has_dest || q->op == TCCIR_OP_STORE_INDEXED) continue; - if (q->op == TCCIR_OP_STORE || q->op == TCCIR_OP_STORE_INDEXED || - q->op == TCCIR_OP_STORE_POSTINC) + if (q->op == TCCIR_OP_STORE_POSTINC) + { + dss_vreg_map_def(map, count, irop_get_vreg(tcc_ir_op_get_dest(ir, q)), NULL); continue; + } + /* A store through a pointer defines nothing; a VAR stored as itself (a + * STACKOFF lvalue, `V <- &s [STORE]`) is a definition like any other. */ IROperand dest = tcc_ir_op_get_dest(ir, q); - int32_t dvr = irop_get_vreg(dest); - if (dvr < 0 || dest.is_lval) + if (!irop_dest_defines_vreg(dest)) continue; + int32_t dvr = irop_get_vreg(dest); Sym *derived_sym = NULL; if (irop_config[q->op].has_src1) @@ -129,10 +139,7 @@ static void dss_build_vreg_map(TCCIRState *ir, DssVregEntry *map, int *count) derived_sym = dss_vreg_map_lookup(map, *count, avr); } - if (derived_sym) - dss_vreg_map_set(map, count, dvr, derived_sym); - else - dss_vreg_map_clear(map, count, dvr); + dss_vreg_map_def(map, count, dvr, derived_sym); } } @@ -222,6 +229,24 @@ int tcc_ir_opt_dead_static_store_elim(TCCIRState *ir) for (int i = 0; i < n; i++) { IRQuadCompact *q = &ir->compact_instructions[i]; + /* A struct assignment into the static: a block-copy helper call that + * writes only through its destination argument. */ + if (q->op == TCCIR_OP_FUNCCALLVOID) + { + Sym *callee = irop_get_sym_ex(ir, tcc_ir_op_get_src1(ir, q)); + IROperand dst; + if (!callee || !tu_is_block_copy_helper(get_tok_str(callee->v, NULL)) || + !ir_opt_get_call_param_operand(ir, i, 0, &dst)) + continue; + MemLoc dl = memloc_of_pointer(ir, dst, i); + if (dl.kind != MEMLOC_GLOBAL || !dl.sym || !dl.sym->a.tu_no_readers || (dl.sym->type.t & VT_VOLATILE)) + continue; + LOG_IR_GEN("DEAD_STATIC_STORE: NOPed block copy at i=%d -> %s", i, get_tok_str(dl.sym->v & ~SYM_FIELD, NULL)); + ir_opt_nop_call_params(ir, i); + q->op = TCCIR_OP_NOP; + changes++; + continue; + } if (q->op != TCCIR_OP_STORE && q->op != TCCIR_OP_STORE_INDEXED && q->op != TCCIR_OP_STORE_POSTINC) continue; @@ -229,10 +254,10 @@ int tcc_ir_opt_dead_static_store_elim(TCCIRState *ir) Sym *sym = dss_resolve_store_dest_sym(ir, q, i, vreg_map, vreg_map_count); if (!sym) continue; + /* tu_no_readers is set only for a static whose address escapes nowhere + * (tcc_ir_tu_analyze_dead_statics). */ if (!sym->a.tu_no_readers) continue; - if (sym->a.addrtaken) - continue; /* Volatile stores stay observable even with no C-level reader. */ if (sym->type.t & VT_VOLATILE) continue; diff --git a/source/opt/flat/memory/dead_temp_local.c b/source/opt/flat/memory/dead_temp_local.c index 7ff452f07..deed49f99 100644 --- a/source/opt/flat/memory/dead_temp_local.c +++ b/source/opt/flat/memory/dead_temp_local.c @@ -55,6 +55,11 @@ int tcc_ir_opt_dead_temp_local_elim(TCCIRState *ir) int32_t dest_vr = irop_get_vreg(dest); if (dest_vr > -2 || dest_vr < -9) continue; + /* A struct operand does not carry its width (dead_local_slot poisons such + * a slot for the same reason); the 4 bytes assumed below could drop a + * write whose tail is read. */ + if (irop_get_btype(dest) == IROP_BTYPE_STRUCT) + continue; int my_off = irop_get_stack_offset(dest); int my_w = ir_opt_store_btype_size_bytes(irop_get_btype(dest)); if (my_w <= 0) my_w = irop_is_64bit(dest) ? 8 : 4; @@ -72,6 +77,16 @@ int tcc_ir_opt_dead_temp_local_elim(TCCIRState *ir) IROperand po; if (!dtl_read_operand(ir, p, k, &po)) continue; if (irop_get_vreg(po) != dest_vr) continue; + /* A struct-typed read of the slot -- a by-value struct argument, `PARAM + * ?slot` -- covers the whole struct, but its width is not in the + * operand, and taking it as 4 bytes dropped the store of every later + * word: `f(c ? g() : h())` with an 8-byte struct passed only its first + * word (the self-hosted tcc emitted 0x0000 for `cond ? th_uxtb(..) : + * th_sxtb(..)`). Count it as a read of all of it. */ + if (k != 0 && irop_get_btype(po) == IROP_BTYPE_STRUCT) { + alive = 1; + continue; + } int po_off = irop_get_stack_offset(po); int po_w = ir_opt_store_btype_size_bytes(irop_get_btype(po)); if (po_w <= 0) po_w = irop_is_64bit(po) ? 8 : 4; diff --git a/source/opt/flat/memory/dse.c b/source/opt/flat/memory/dse.c index 2bcc40b65..dd46beb32 100644 --- a/source/opt/flat/memory/dse.c +++ b/source/opt/flat/memory/dse.c @@ -25,9 +25,51 @@ int tcc_ir_opt_dse(TCCIRState *ir) return r; } -static int dse_operand_is_wide(IROperand op) +/* A span of an anonymous stack object that may be read: [lo, hi). */ +typedef struct { - return op.btype == IROP_BTYPE_INT64 || op.btype == IROP_BTYPE_FLOAT64; + const Sym *sym; + int64_t lo, hi; +} DseRange; + +static void dse_add_range(DseRange **r, int *n, int *cap, const Sym *sym, int64_t lo, int64_t hi) +{ + if (hi <= lo) + return; + if (*n == *cap) + { + *cap = *cap ? *cap * 2 : 32; + *r = tcc_realloc(*r, sizeof(DseRange) * *cap); + } + (*r)[*n].sym = sym; + (*r)[*n].lo = lo; + (*r)[*n].hi = hi; + (*n)++; +} + +static int dse_ranges_overlap(const DseRange *r, int n, const Sym *sym, int64_t lo, int64_t hi) +{ + for (int k = 0; k < n; k++) + if (r[k].sym == sym && r[k].lo < hi && lo < r[k].hi) + return 1; + return 0; +} + +/* Whether `q` assigns its dest from its sources. The stores do not: their + * dest is the address written through (or, for STORE_INDEXED/POSTINC, the + * base pointer), and an address in src1 is a value leaving for memory. */ +static int dse_defines_dest(const IRQuadCompact *q, IROperand dest) +{ + switch (q->op) + { + case TCCIR_OP_STORE_INDEXED: + case TCCIR_OP_STORE_POSTINC: + return 0; + case TCCIR_OP_STORE: + return !dest.is_lval; + default: + return 1; + } } /* Pull (sym, off) out of a local StackLoc operand: SYMREF carries sym+addend, else a bare stack offset. */ @@ -52,20 +94,6 @@ static int tcc_ir_opt_dse__timed(TCCIRState *ir) if (n == 0) return 0; - for (int i = 0; i < n; i++) - { - IRQuadCompact *q = &ir->compact_instructions[i]; - if (q->op == TCCIR_OP_NOP) - continue; - if (irop_config[q->op].has_dest && dse_operand_is_wide(tcc_ir_op_get_dest(ir, q))) - return 0; - if (irop_config[q->op].has_src1 && dse_operand_is_wide(tcc_ir_op_get_src1(ir, q))) - return 0; - if (irop_config[q->op].has_src2 && dse_operand_is_wide(tcc_ir_op_get_src2(ir, q))) - return 0; - if (q->op == TCCIR_OP_MLA && dse_operand_is_wide(tcc_ir_op_get_accum(ir, q))) - return 0; - } /* NOP orphaned FUNCPARAM whose call_id has no matching FUNCCALL (left by inlining). */ { @@ -515,6 +543,13 @@ static int tcc_ir_opt_dse__timed(TCCIRState *ir) #define STACKLOC_HASH_SIZE 256 uint8_t stackloc_read[STACKLOC_HASH_SIZE]; memset(stackloc_read, 0, sizeof(stackloc_read)); + /* Spans read through an address or as a struct: bounded by the object + * holding the offset when the frame still has the frontend's layout (C + * does not let a pointer leave its object), else up to the highest + * store. Kept as spans, not hashed bytes, so a big object does not mark + * everything read. */ + DseRange *ranges = NULL; + int nranges = 0, ranges_cap = 0; /* Max StackLoc STORE offset — bounds how far an address-of range extends. */ int64_t max_stackloc_off = 0; @@ -594,7 +629,7 @@ static int tcc_ir_opt_dse__timed(TCCIRState *ir) { IROperand d = tcc_ir_op_get_dest(ir, q); int32_t dvr = irop_get_vreg(d); - if (dvr >= 0 && TCCIR_DECODE_VREG_TYPE(dvr) == TCCIR_VREG_TYPE_TEMP) + if (dvr >= 0 && TCCIR_DECODE_VREG_TYPE(dvr) == TCCIR_VREG_TYPE_TEMP && dse_defines_dest(q, d)) { int dpos = TCCIR_DECODE_VREG_POSITION(dvr); if (dpos <= max_tmp_stackloc) @@ -653,8 +688,26 @@ static int tcc_ir_opt_dse__timed(TCCIRState *ir) stype == TCCIR_VREG_TYPE_TEMP && dtype == TCCIR_VREG_TYPE_TEMP && spos <= max_tmp_stackloc && dpos <= max_tmp_stackloc) origin = prop_tmp[spos]; + /* Pointer arithmetic into or out of a VAR carries the address as + * well: Phase 3 counts every ADD/SUB of an address as safe, so one + * whose result is not followed hides the reads through it. Zig's + * `t11 = &t4.path; t12 = *t11` is `V3 <-- T33 ADD #8`, and the + * stores filling t4 died although the path was read through V3. */ + else if ((q->op == TCCIR_OP_ADD || q->op == TCCIR_OP_SUB) && + (stype == TCCIR_VREG_TYPE_VAR || dtype == TCCIR_VREG_TYPE_VAR)) + { + if (stype == TCCIR_VREG_TYPE_TEMP && spos <= max_tmp_stackloc) + origin = prop_tmp[spos]; + else if (stype == TCCIR_VREG_TYPE_VAR && prop_var && spos <= max_var_stackloc) + origin = prop_var[spos]; + if ((dtype == TCCIR_VREG_TYPE_TEMP && dpos > max_tmp_stackloc) || + (dtype == TCCIR_VREG_TYPE_VAR && (!prop_var || dpos > max_var_stackloc))) + origin = -1; + } - if (origin < 0) + /* -2 travels on: a copy of an ambiguous address is still an address, + * and the loads through it must reach MARK_ORIGIN_READ. */ + if (origin == -1) continue; if (dtype == TCCIR_VREG_TYPE_TEMP && dpos <= max_tmp_stackloc) @@ -827,10 +880,12 @@ static int tcc_ir_opt_dse__timed(TCCIRState *ir) break; \ case IROP_BTYPE_STRUCT: \ { \ - /* Struct access — conservatively mark range up to max store offset */ \ - int64_t _send = max_stackloc_off + 4; \ - for (int64_t _s = _off; _s <= _send; _s++) \ - STACKLOC_SET(_sym, _s); \ + /* Struct access: to the end of its object, else up to the max store offset */ \ + int64_t _shi = max_stackloc_off + 5; \ + int _olo, _ohi; \ + if (!_sym && tcc_ir_frame_object_at(ir, (int)_off, &_olo, &_ohi)) \ + _shi = _ohi; \ + dse_add_range(&ranges, &nranges, &ranges_cap, _sym, _off, _shi); \ _width = 0; /* already handled */ \ break; \ } \ @@ -845,17 +900,12 @@ static int tcc_ir_opt_dse__timed(TCCIRState *ir) } \ else \ { \ - int64_t _range_end = max_stackloc_off + 4; \ - int64_t _range_len = _range_end - _off + 1; \ - if (_range_len > STACKLOC_HASH_SIZE * 8) \ - { \ - memset(stackloc_read, 0xFF, sizeof(stackloc_read)); \ - } \ - else if (_range_len > 0) \ - { \ - for (int64_t _k = _off; _k <= _range_end; _k++) \ - STACKLOC_SET(_sym, _k); \ - } \ + /* Address taken: its whole object, else up to the max store offset */ \ + int64_t _rlo = _off, _rhi = max_stackloc_off + 5; \ + int _olo, _ohi; \ + if (!_sym && tcc_ir_frame_object_at(ir, (int)_off, &_olo, &_ohi)) \ + _rlo = _olo, _rhi = _ohi; \ + dse_add_range(&ranges, &nranges, &ranges_cap, _sym, _rlo, _rhi); \ } \ } \ } while (0) @@ -865,7 +915,8 @@ static int tcc_ir_opt_dse__timed(TCCIRState *ir) { IROperand s = tcc_ir_op_get_src1(ir, q); /* Skip range marking for address-of feeding a write-only TMP (writes only, no read). */ - if (s.is_local && !s.is_lval && irop_get_vreg(s) < 0 && addr_tmp != NULL) + if (s.is_local && !s.is_lval && irop_get_vreg(s) < 0 && addr_tmp != NULL && + dse_defines_dest(q, tcc_ir_op_get_dest(ir, q))) { IROperand d = tcc_ir_op_get_dest(ir, q); int32_t dvr = irop_get_vreg(d); @@ -961,7 +1012,7 @@ static int tcc_ir_opt_dse__timed(TCCIRState *ir) swidth = -1; /* unknown extent: keep the store */ else if (dest.is_complex) swidth *= 2; - int live = (swidth < 0); + int live = (swidth < 0) || dse_ranges_overlap(ranges, nranges, sym, off, off + swidth); for (int b = 0; !live && b < swidth; b++) if (STACKLOC_TEST(sym, off + b)) live = 1; @@ -1027,7 +1078,8 @@ static int tcc_ir_opt_dse__timed(TCCIRState *ir) int range_is_read = 0; int64_t range_end = max_stackloc_off + 4; int64_t range_len = range_end - addr_off + 1; - if (range_len > STACKLOC_HASH_SIZE * 8) + if (range_len > STACKLOC_HASH_SIZE * 8 || + dse_ranges_overlap(ranges, nranges, addr_sym, addr_off, range_end + 1)) { range_is_read = 1; /* Too large — conservatively assume read */ } @@ -1129,6 +1181,7 @@ static int tcc_ir_opt_dse__timed(TCCIRState *ir) tcc_free(addr_tmp); if (addr_tmp_read) tcc_free(addr_tmp_read); + tcc_free(ranges); #undef STACKLOC_HASH_SIZE #undef STACKLOC_HASH diff --git a/source/opt/flat/memory/entry_store_prop.c b/source/opt/flat/memory/entry_store_prop.c index 39334167d..a299a6029 100644 --- a/source/opt/flat/memory/entry_store_prop.c +++ b/source/opt/flat/memory/entry_store_prop.c @@ -37,8 +37,8 @@ static const uint8_t *ir_opt_get_rodata_bytes(TCCIRState *ir, IROperand op, size return NULL; sym = symref->sym; - if (!sym) - return NULL; + if (!sym || sym->a.tentative) + return NULL; /* a tentative definition's bytes are not its value yet */ esym = elfsym(sym); if (!esym) @@ -497,8 +497,11 @@ int tcc_ir_opt_entry_store_prop(TCCIRState *ir) } SimpleLeaEntry; /* Definition counts, used only to set `maybe` above. A write *through* a - pointer (`T***DEREF*** <-- v`) is not a definition of the pointer, hence the - is_lval test. */ + pointer (`T***DEREF*** <-- v`) is not a definition of the pointer; a VAR + itself as the destination is (a STACKOFF lvalue). Testing is_lval alone + dropped the latter: `for (p = y; p != e; p++) s += *p` counted one of p's + two definitions, and the walk got the last element's value forwarded into + every iteration. */ uint8_t *tmp_defs = tcc_mallocz(max_tmp + 1); uint8_t *var_defs = tcc_mallocz(max_var + 1); for (int i = 0; i < n; i++) @@ -508,7 +511,7 @@ int tcc_ir_opt_entry_store_prop(TCCIRState *ir) continue; IROperand d = tcc_ir_op_get_dest(ir, q); int32_t vr = irop_get_vreg(d); - if (vr < 0 || d.is_lval) + if (!irop_dest_defines_vreg(d)) continue; int p = TCCIR_DECODE_VREG_POSITION(vr); if (TCCIR_DECODE_VREG_TYPE(vr) == TCCIR_VREG_TYPE_TEMP && p <= max_tmp && tmp_defs[p] < 2) @@ -1210,3 +1213,168 @@ int tcc_ir_opt_entry_store_prop(TCCIRState *ir) return changes; } int tcc_ir_opt_entry_store_prop_ex(IROptCtx *ctx) { return tcc_ir_opt_entry_store_prop(ctx->ir); } + +/* A by-value struct parameter lives in a frame home the entry block fills a + * word at a time, `StackLoc[h] <-- Pn [STORE]`, and field reads load it. SRA + * splits the copies Zig makes of it (`t0 = a0; t1 = &t0`) but not the home, + * which is no frame object, and the reads lea folding turned into direct slot + * loads -- `a0.len` as `StackLoc[h+4] [LOAD]`, `a0.ptr[i]` as `StackLoc[h] ADD + * i` -- reloaded it on every loop turn (Zig's StaticStringMap.defaultEql kept + * both slices in memory with three free registers). When nothing but the entry + * store writes a home word and no address inside the homes is taken, every + * word read of it is Pn. The store stays; with no reads left DSE drops it. */ +#define PHF_MAX 32 +int tcc_ir_opt_param_home_fwd(TCCIRState *ir) +{ + const int n = ir->next_instruction_index; + if (n == 0 || ir->inline_asm_count || ir->func_has_label_addr || tcc_ir_calls_returns_twice(ir)) + return 0; + /* Leaf functions only. Across a call the parameter needs a callee-saved + * register or a slot of its own, and the home was that slot already: + * forwarded, Wyhash.hash spent a callee-saved register on its key slice + * through the u128 loop and ran 2% slower. */ + for (int i = 0; i < n; i++) + if (ir->compact_instructions[i].op == TCCIR_OP_FUNCCALLVOID || ir->compact_instructions[i].op == TCCIR_OP_FUNCCALLVAL) + return 0; + + struct PhfHome + { + int off, idx, bad; + IROperand p; + } h[PHF_MAX]; + int nh = 0; + for (int i = 0; i < n && nh < PHF_MAX; i++) + { + IRQuadCompact *q = &ir->compact_instructions[i]; + if (q->is_jump_target && i > 0) + break; + if (q->op == TCCIR_OP_JUMP || q->op == TCCIR_OP_JUMPIF || q->op == TCCIR_OP_IJUMP || + q->op == TCCIR_OP_SWITCH_TABLE || q->op == TCCIR_OP_RETURNVALUE || q->op == TCCIR_OP_RETURNVOID) + break; + if (q->op != TCCIR_OP_STORE) + continue; + IROperand d = tcc_ir_op_get_dest(ir, q), s = tcc_ir_op_get_src1(ir, q); + if (irop_get_tag(d) != IROP_TAG_STACKOFF || !d.is_local || !d.is_lval || d.is_llocal || + irop_get_vreg(d) >= 0 || irop_get_btype(d) != IROP_BTYPE_INT32 || irop_is_64bit(d) || + tcc_ir_access_is_volatile(ir, d)) + continue; + int32_t pv = irop_get_vreg(s); + if (pv < 0 || TCCIR_DECODE_VREG_TYPE(pv) != TCCIR_VREG_TYPE_PARAM || s.is_lval || s.is_local || s.is_llocal || + irop_get_btype(s) != IROP_BTYPE_INT32 || irop_is_64bit(s)) + continue; + const int off = irop_get_stack_offset(d); + if (off & 3) + continue; + int dup = 0; + for (int k = 0; k < nh; k++) + if (h[k].off == off) + dup = h[k].bad = 1; + if (!dup) + h[nh++] = (struct PhfHome){.off = off, .idx = i, .bad = 0, .p = s}; + } + if (!nh) + return 0; + int lo = h[0].off, hi = h[0].off + 4; + for (int k = 1; k < nh; k++) + { + if (h[k].off < lo) + lo = h[k].off; + if (h[k].off + 4 > hi) + hi = h[k].off + 4; + } + + /* Every other reference to the home area. */ + for (int i = 0; i < n; i++) + { + IRQuadCompact *q = &ir->compact_instructions[i]; + if (q->op == TCCIR_OP_NOP) + continue; + int entry = 0; + for (int k = 0; k < nh; k++) + entry |= h[k].idx == i; + if (entry) + continue; + const int store_class = q->op == TCCIR_OP_STORE || q->op == TCCIR_OP_STORE_INDEXED || + q->op == TCCIR_OP_STORE_POSTINC; + for (int s = 0; s < 4; s++) + { + IROperand op; + if (s == 0) { if (!irop_config[q->op].has_dest) continue; op = tcc_ir_op_get_dest(ir, q); } + else if (s == 1) { if (!irop_config[q->op].has_src1) continue; op = tcc_ir_op_get_src1(ir, q); } + else if (s == 2) { if (!irop_config[q->op].has_src2) continue; op = tcc_ir_op_get_src2(ir, q); } + else { if (!tcc_ir_op_is_mac(q->op)) continue; op = tcc_ir_op_get_accum(ir, q); } + /* A parameter vreg written anywhere is not the entry value. */ + int32_t vr = irop_get_vreg(op); + if (s == 0 && vr >= 0 && TCCIR_DECODE_VREG_TYPE(vr) == TCCIR_VREG_TYPE_PARAM && + q->op != TCCIR_OP_FUNCPARAMVAL && q->op != TCCIR_OP_FUNCPARAMVOID) + for (int k = 0; k < nh; k++) + if (irop_get_vreg(h[k].p) == vr) + h[k].bad = 1; + if (irop_get_tag(op) != IROP_TAG_STACKOFF || !op.is_local || op.is_llocal || vr >= 0) + continue; + const int o = irop_get_stack_offset(op); + if (!op.is_lval) + { + /* its address: anything could be done through it */ + if (o >= lo && o < hi) + return 0; + continue; + } + int w = irop_is_64bit(op) ? 8 : ir_opt_store_btype_size_bytes(irop_get_btype(op)); + if (w <= 0) + w = irop_get_btype(op) == IROP_BTYPE_STRUCT ? hi - lo : 4; + if (o + w <= lo || o >= hi) + continue; + if (s == 0 && (store_class || q->op == TCCIR_OP_BLOCK_COPY)) + { + if (q->op != TCCIR_OP_STORE) + return 0; /* an indexed or block write into the homes */ + for (int k = 0; k < nh; k++) + if (o < h[k].off + 4 && o + w > h[k].off) + h[k].bad = 1; + } + else if (s == 0) + return 0; /* some other write of the slot */ + } + } + + int changes = 0; + for (int i = 0; i < n; i++) + { + IRQuadCompact *q = &ir->compact_instructions[i]; + if (q->op == TCCIR_OP_NOP) + continue; + int entry = 0; + for (int k = 0; k < nh; k++) + entry |= h[k].idx == i; + if (entry) + continue; + for (int s = 1; s < 4; s++) + { + IROperand op; + if (s == 1) { if (!irop_config[q->op].has_src1) continue; op = tcc_ir_op_get_src1(ir, q); } + else if (s == 2) { if (!irop_config[q->op].has_src2) continue; op = tcc_ir_op_get_src2(ir, q); } + else { if (!tcc_ir_op_is_mac(q->op)) continue; op = tcc_ir_op_get_accum(ir, q); } + if (irop_get_tag(op) != IROP_TAG_STACKOFF || !op.is_local || !op.is_lval || op.is_llocal || + irop_get_vreg(op) >= 0 || irop_get_btype(op) != IROP_BTYPE_INT32 || irop_is_64bit(op) || + tcc_ir_access_is_volatile(ir, op)) + continue; + const int o = irop_get_stack_offset(op); + for (int k = 0; k < nh; k++) + { + if (h[k].bad || h[k].off != o) + continue; + IROperand p = h[k].p; + if (s == 1) + tcc_ir_op_set_src1(ir, q, p); + else if (s == 2) + tcc_ir_op_set_src2(ir, q, p); + else + tcc_ir_op_set_accum(ir, q, p); + changes++; + break; + } + } + } + return changes; +} diff --git a/source/opt/flat/memory/invariant_global_load_hoist.c b/source/opt/flat/memory/invariant_global_load_hoist.c index 8b93364ed..492d0825d 100644 --- a/source/opt/flat/memory/invariant_global_load_hoist.c +++ b/source/opt/flat/memory/invariant_global_load_hoist.c @@ -190,26 +190,26 @@ int tcc_ir_opt_invariant_global_load_hoist(TCCIRState *ir) is_load_like = 1; } - if (!is_load_like) + /* An op that (re)defines a vreg -- including a VAR written as itself, a + * STACKOFF lvalue, and a load into a vreg an entry already names -- + * leaves every entry holding that vreg as some global's value stale. */ + if (irop_config[q->op].has_dest) { - /* If this op redefines a tracked vreg, drop that entry. */ - if (irop_config[q->op].has_dest) + IROperand d = tcc_ir_op_get_dest(ir, q); + if (irop_dest_defines_vreg(d)) { - IROperand d = tcc_ir_op_get_dest(ir, q); int32_t dvr = irop_get_vreg(d); - if (dvr >= 0 && !d.is_lval) + for (int k = 0; k < num_tracked;) { - for (int k = 0; k < num_tracked;) - { - if (tracked[k].result_vr == dvr) - tracked[k] = tracked[--num_tracked]; - else - k++; - } + if (tracked[k].result_vr == dvr) + tracked[k] = tracked[--num_tracked]; + else + k++; } } - continue; } + if (!is_load_like) + continue; IROperand src1 = tcc_ir_op_get_src1(ir, q); IROperand dest = tcc_ir_op_get_dest(ir, q); diff --git a/source/opt/flat/memory/invariant_temp_deref_hoist.c b/source/opt/flat/memory/invariant_temp_deref_hoist.c deleted file mode 100644 index 04ebbece7..000000000 --- a/source/opt/flat/memory/invariant_temp_deref_hoist.c +++ /dev/null @@ -1,500 +0,0 @@ -/* - * TCC IR - Invariant TEMP-deref hoist (pre-SSA) - * - * Copyright (c) 2025 Mateusz Stadnik - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation. - */ - -#define USING_GLOBALS - -#include - -#include "ir.h" -#include "opt.h" -#include "opt_engine.h" -#include "opt_du.h" -#include "opt_xform.h" -#include "opt_utils.h" -#include "opt_alias.h" -#include "opt_loop_utils.h" - - -extern int gsym_cse_insert_before(TCCIRState *ir, int before_idx, IRQuadCompact *new_q); - -/* ============================================================================ - * Invariant TEMP-deref Hoist (tcc_ir_opt_invariant_temp_deref_hoist) - * ============================================================================ - * - * Companion to tcc_ir_opt_invariant_global_load_hoist. After that pass has - * collapsed cross-BB reloads of a global pointer T into a single - * T = ASSIGN GlobalSym(P)***DEREF*** - * the body of the function may still contain many `op T***DEREF***` uses - * (e.g. `CMP T***DEREF***, X` repeated per iteration of an unrolled chain). - * Each one re-emits an LDR before the operation. - * - * For TEMPs that are defined once (load of a pointer from memory) and never - * redefined, this pass inserts a single ASSIGN T_v <- T***DEREF*** right - * after T's definition and rewrites every `T***DEREF***` use within the same - * clobber-free single-entry region to T_v (non-lval). Subsequent codegen - * keeps T_v in a register, removing the per-use LDR. - * - * Safety conditions match the global LOAD hoist pass: forward control flow - * only, no aliasing stores or calls between def and last use, and no jump - * skipping over the def to land in the use range. - */ -static int optmem_temp_copy_btype_compat(int a, int b) -{ - int a_word = (a == IROP_BTYPE_INT32 || a == IROP_BTYPE_STRUCT || a == IROP_BTYPE_FUNC); - int b_word = (b == IROP_BTYPE_INT32 || b == IROP_BTYPE_STRUCT || b == IROP_BTYPE_FUNC); - return a == b || (a_word && b_word); -} - -static int32_t optmem_resolve_temp_copy_root(int32_t vr, int use_idx, - const int32_t *copy_src, - const int *copy_def, int max_tmp) -{ - for (int depth = 0; depth < 16; depth++) - { - if (vr < 0 || TCCIR_DECODE_VREG_TYPE(vr) != TCCIR_VREG_TYPE_TEMP) - return vr; - int pos = TCCIR_DECODE_VREG_POSITION(vr); - if (pos < 0 || pos >= max_tmp) - return vr; - int32_t src = copy_src[pos]; - if (src < 0 || copy_def[pos] < 0 || use_idx <= copy_def[pos]) - return vr; - vr = src; - } - return vr; -} - -int tcc_ir_opt_invariant_temp_deref_hoist(TCCIRState *ir) -{ - int n = ir->next_instruction_index; - int changes = 0; - if (n < 2) - return 0; - - /* Same control-flow preconditions as the global-load hoist. */ - for (int i = 0; i < n; i++) - { - IRQuadCompact *q = &ir->compact_instructions[i]; - switch (q->op) - { - case TCCIR_OP_IJUMP: - case TCCIR_OP_SWITCH_TABLE: - case TCCIR_OP_SETJMP: - case TCCIR_OP_LONGJMP: - case TCCIR_OP_NL_SETJMP: - case TCCIR_OP_NL_LONGJMP: - case TCCIR_OP_INLINE_ASM: - case TCCIR_OP_ASM_INPUT: - case TCCIR_OP_ASM_OUTPUT: - case TCCIR_OP_BUILTIN_APPLY: - case TCCIR_OP_BUILTIN_APPLY_ARGS: - case TCCIR_OP_BUILTIN_RETURN: - return 0; - case TCCIR_OP_JUMP: - case TCCIR_OP_JUMPIF: - { - IROperand dest = tcc_ir_op_get_dest(ir, q); - int target = (int)irop_get_imm64_ex(ir, dest); - if (target <= i) - return 0; - break; - } - default: - break; - } - } - - /* Clobber map (writes that could alias any pointer-derefed memory). */ - unsigned char *clobber = tcc_mallocz((size_t)n); - if (!clobber) - return 0; - for (int i = 0; i < n; i++) - { - IRQuadCompact *q = &ir->compact_instructions[i]; - switch (q->op) - { - case TCCIR_OP_FUNCCALLVAL: - case TCCIR_OP_FUNCCALLVOID: - case TCCIR_OP_STORE_INDEXED: - case TCCIR_OP_STORE_POSTINC: - case TCCIR_OP_BLOCK_COPY: - case TCCIR_OP_TRAP: - clobber[i] = 1; - break; - case TCCIR_OP_STORE: - { - IROperand dest = tcc_ir_op_get_dest(ir, q); - /* A direct store to any GlobalSym could alias an unknown pointer. - * A store to a local stack slot cannot. */ - if (dest.is_sym && dest.is_lval) - clobber[i] = 1; - else if (!dest.is_local) - clobber[i] = 1; - break; - } - default: - break; - } - } - - int max_tmp = ir->next_temporary_variable; - int32_t *copy_src = NULL; - int *copy_def = NULL; - int *def_count = NULL; - if (max_tmp > 0) - { - copy_src = tcc_mallocz((size_t)max_tmp * sizeof(int32_t)); - copy_def = tcc_mallocz((size_t)max_tmp * sizeof(int)); - def_count = tcc_mallocz((size_t)max_tmp * sizeof(int)); - for (int p = 0; p < max_tmp; p++) - { - copy_src[p] = -1; - copy_def[p] = -1; - } - for (int i = 0; i < n; i++) - { - IRQuadCompact *q = &ir->compact_instructions[i]; - if (q->op == TCCIR_OP_NOP || !irop_config[q->op].has_dest) - continue; - IROperand dest = tcc_ir_op_get_dest(ir, q); - int32_t dvr = irop_get_vreg(dest); - if (dvr < 0 || dest.is_lval || - TCCIR_DECODE_VREG_TYPE(dvr) != TCCIR_VREG_TYPE_TEMP) - continue; - int dpos = TCCIR_DECODE_VREG_POSITION(dvr); - if (dpos < 0 || dpos >= max_tmp) - continue; - def_count[dpos]++; - copy_def[dpos] = i; - copy_src[dpos] = -1; - if (q->op != TCCIR_OP_ASSIGN || !irop_config[q->op].has_src1) - continue; - IROperand src1 = tcc_ir_op_get_src1(ir, q); - int32_t svr = irop_get_vreg(src1); - if (svr >= 0 && !src1.is_lval && !src1.is_local && !src1.is_llocal && - TCCIR_DECODE_VREG_TYPE(svr) == TCCIR_VREG_TYPE_TEMP && - optmem_temp_copy_btype_compat(irop_get_btype(dest), irop_get_btype(src1))) - copy_src[dpos] = svr; - } - for (int p = 0; p < max_tmp; p++) - if (def_count[p] != 1) - copy_src[p] = -1; - } - -#define ITDH_MAX_CANDS 16 - struct - { - int32_t temp_vr; - int def_idx; - int first_use; - int last_use; - int use_count; - int btype; - int is_unsigned; - int32_t hoist_vr; - int hoist_idx; /* position of the inserted ASSIGN — its own src must not be rewritten */ - } cands[ITDH_MAX_CANDS]; - int num_cands = 0; - - /* Pass 1: collect candidate TEMPs - defined by an ASSIGN/LOAD whose source - * operand is an lval (so the TEMP holds a value loaded from memory). These - * are exactly the "loaded pointer" TEMPs whose subsequent T***DEREF*** uses - * become per-iteration LDRs in the backend. */ - for (int i = 0; i < n && num_cands < ITDH_MAX_CANDS; i++) - { - IRQuadCompact *q = &ir->compact_instructions[i]; - if (q->op != TCCIR_OP_ASSIGN && q->op != TCCIR_OP_LOAD) - continue; - if (!irop_config[q->op].has_dest || !irop_config[q->op].has_src1) - continue; - IROperand src1 = tcc_ir_op_get_src1(ir, q); - IROperand dest = tcc_ir_op_get_dest(ir, q); - if (!src1.is_lval || dest.is_lval) - continue; - int32_t dest_vr = irop_get_vreg(dest); - if (dest_vr < 0) - continue; - if (TCCIR_DECODE_VREG_TYPE(dest_vr) != TCCIR_VREG_TYPE_TEMP) - continue; - cands[num_cands].temp_vr = dest_vr; - cands[num_cands].def_idx = i; - cands[num_cands].first_use = -1; - cands[num_cands].last_use = -1; - cands[num_cands].use_count = 0; - cands[num_cands].btype = -1; - cands[num_cands].is_unsigned = 0; - cands[num_cands].hoist_vr = -1; - cands[num_cands].hoist_idx = -1; - num_cands++; - } - - if (num_cands == 0) - { - tcc_free(clobber); - tcc_free(copy_src); - tcc_free(copy_def); - tcc_free(def_count); - return 0; - } - - /* Pass 2: scan IR. For each candidate, check redefinitions and collect - * lval-deref uses. Mark candidates whose TEMP is redefined as invalid. */ - for (int i = 0; i < n; i++) - { - IRQuadCompact *q = &ir->compact_instructions[i]; - if (q->op == TCCIR_OP_NOP) - continue; - - /* Redefinition check (skip the candidate's own def). */ - if (irop_config[q->op].has_dest) - { - IROperand d = tcc_ir_op_get_dest(ir, q); - int32_t dvr = irop_get_vreg(d); - if (dvr >= 0 && !d.is_lval) - { - for (int c = 0; c < num_cands; c++) - { - if (cands[c].use_count < 0) - continue; - if (cands[c].temp_vr == dvr && i != cands[c].def_idx) - cands[c].use_count = -1; /* mark invalid */ - } - } - } - - /* Operand scan for lval uses of the candidate TEMP. */ - for (int s = 0; s < 3; s++) - { - int has; - IROperand op; - if (s == 0) - { - has = irop_config[q->op].has_dest; - if (!has) - continue; - op = tcc_ir_op_get_dest(ir, q); - } - else if (s == 1) - { - has = irop_config[q->op].has_src1; - if (!has) - continue; - op = tcc_ir_op_get_src1(ir, q); - } - else - { - has = irop_config[q->op].has_src2; - if (!has) - continue; - op = tcc_ir_op_get_src2(ir, q); - } - if (!op.is_lval) - continue; - int32_t vr = irop_get_vreg(op); - if (vr < 0) - continue; - vr = optmem_resolve_temp_copy_root(vr, i, copy_src, copy_def, max_tmp); - for (int c = 0; c < num_cands; c++) - { - if (cands[c].use_count < 0) - continue; - if (cands[c].temp_vr != vr) - continue; - if (i <= cands[c].def_idx) - continue; /* the def itself - ignore */ - int btype = irop_get_btype(op); - if (cands[c].first_use < 0) - { - cands[c].first_use = i; - cands[c].btype = btype; - cands[c].is_unsigned = op.is_unsigned; - } - else if (btype != cands[c].btype) - { - cands[c].use_count = -1; - continue; - } - cands[c].last_use = i; - cands[c].use_count++; - } - } - } - - /* Pass 3: safety filter — drop candidates whose use range has a clobber or - * whose anchor doesn't dominate the uses (some external jump skips the def). */ - for (int c = 0; c < num_cands; c++) - { - if (cands[c].use_count < 2) - continue; - int anchor = cands[c].def_idx; - int last = cands[c].last_use; - int safe = 1; - for (int j = anchor + 1; j <= last; j++) - { - if (clobber[j]) - { - safe = 0; - break; - } - } - if (safe) - { - for (int j = 0; j < anchor; j++) - { - IRQuadCompact *jq = &ir->compact_instructions[j]; - if (jq->op != TCCIR_OP_JUMP && jq->op != TCCIR_OP_JUMPIF) - continue; - IROperand jdest = tcc_ir_op_get_dest(ir, jq); - int tgt = (int)irop_get_imm64_ex(ir, jdest); - if (tgt > anchor && tgt <= last) - { - safe = 0; - break; - } - } - } - if (!safe) - cands[c].use_count = -1; - } - - tcc_free(clobber); - - /* Pass 4: insert hoist ASSIGNs. Process in REVERSE order of def_idx so - * earlier-positioned candidates' indices are unaffected by later insertions - * (each insertion shifts only positions at-or-after itself). */ - for (int pass = 0; pass < num_cands; pass++) - { - int best = -1; - int best_idx = -1; - for (int c = 0; c < num_cands; c++) - { - if (cands[c].use_count < 2 || cands[c].hoist_vr >= 0) - continue; - if (cands[c].def_idx > best_idx) - { - best_idx = cands[c].def_idx; - best = c; - } - } - if (best < 0) - break; - - int c = best; - int32_t t_new = tcc_ir_vreg_alloc_temp(ir); - if (t_new < 0) - continue; - - IROperand new_dest = irop_make_vreg(t_new, cands[c].btype); - new_dest.is_unsigned = cands[c].is_unsigned; - IROperand new_src = irop_make_vreg(cands[c].temp_vr, cands[c].btype); - new_src.is_lval = 1; - new_src.is_unsigned = cands[c].is_unsigned; - - if (ir->iroperand_pool_count + 2 > ir->iroperand_pool_capacity) - tcc_ir_pool_ensure(ir, 2); - - IRQuadCompact new_q = {0}; - new_q.op = TCCIR_OP_ASSIGN; - new_q.operand_base = tcc_ir_pool_add(ir, new_dest); - tcc_ir_pool_add(ir, new_src); - - int insert_pos = cands[c].def_idx + 1; - if (gsym_cse_insert_before(ir, insert_pos, &new_q) < 0) - continue; - - cands[c].hoist_vr = t_new; - cands[c].hoist_idx = insert_pos; - /* Other candidates' positions at-or-after insert_pos shift by 1. */ - for (int c2 = 0; c2 < num_cands; c2++) - { - if (c2 == c) - continue; - if (cands[c2].def_idx >= insert_pos) - cands[c2].def_idx++; - if (cands[c2].first_use >= insert_pos) - cands[c2].first_use++; - if (cands[c2].last_use >= insert_pos) - cands[c2].last_use++; - if (cands[c2].hoist_idx >= 0 && cands[c2].hoist_idx >= insert_pos) - cands[c2].hoist_idx++; - } - n++; - } - - /* Pass 5: rewrite all `T***DEREF***` uses to the hoisted TEMP (non-lval). - * Rewriting is by vreg identity so it's independent of position shifts. - * Skip the inserted ASSIGN itself (whose src1 is the only legitimate - * `T_old***DEREF***` use that must remain). */ - for (int c = 0; c < num_cands; c++) - { - if (cands[c].hoist_vr < 0) - continue; - for (int i = cands[c].def_idx + 1; i < n; i++) - { - if (i == cands[c].hoist_idx) - continue; - IRQuadCompact *q = &ir->compact_instructions[i]; - if (q->op == TCCIR_OP_NOP) - continue; - for (int s = 0; s < 3; s++) - { - int has; - IROperand op; - if (s == 0) - { - has = irop_config[q->op].has_dest; - if (!has) - continue; - op = tcc_ir_op_get_dest(ir, q); - } - else if (s == 1) - { - has = irop_config[q->op].has_src1; - if (!has) - continue; - op = tcc_ir_op_get_src1(ir, q); - } - else - { - has = irop_config[q->op].has_src2; - if (!has) - continue; - op = tcc_ir_op_get_src2(ir, q); - } - if (!op.is_lval) - continue; - int32_t use_vr = irop_get_vreg(op); - use_vr = optmem_resolve_temp_copy_root(use_vr, i, copy_src, copy_def, max_tmp); - if (use_vr != cands[c].temp_vr) - continue; - if (irop_get_btype(op) != cands[c].btype) - continue; - IROperand repl = irop_make_vreg(cands[c].hoist_vr, cands[c].btype); - repl.is_unsigned = cands[c].is_unsigned; - if (s == 0) - tcc_ir_op_set_dest(ir, q, repl); - else if (s == 1) - tcc_ir_set_src1(ir, i, repl); - else - tcc_ir_set_src2(ir, i, repl); - changes++; - } - } - } - -#undef ITDH_MAX_CANDS - tcc_free(copy_src); - tcc_free(copy_def); - tcc_free(def_count); - return changes; -} - -int tcc_ir_opt_invariant_temp_deref_hoist_ex(IROptCtx *ctx) { return tcc_ir_opt_invariant_temp_deref_hoist(ctx->ir); } - diff --git a/source/opt/flat/memory/local_copy_prop.c b/source/opt/flat/memory/local_copy_prop.c deleted file mode 100644 index dc1873468..000000000 --- a/source/opt/flat/memory/local_copy_prop.c +++ /dev/null @@ -1,377 +0,0 @@ -/* - * TCC IR - Local Copy Propagation (pre-SSA) - * - * Copyright (c) 2025 Mateusz Stadnik - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation. - */ - -#define USING_GLOBALS - -#include - -#include "ir.h" -#include "opt.h" -#include "opt_engine.h" -#include "opt_du.h" -#include "opt_xform.h" -#include "opt_utils.h" -#include "opt_alias.h" -#include "opt_loop_utils.h" - - -/* - * Eliminates redundant copies through temporary stack slots. When inline - * struct copy produces: - * - * [writes to temp slot A] e.g. memset(A, 0, 56); A[0] = val; - * LOAD T = A[0]; STORE B[0] = T; - * LOAD T = A[4]; STORE B[4] = T; - * ... (copy chain: A → B) - * [only B is used afterwards] - * - * This pass redirects all writes from A to B and NOPs the copy chain, - * producing: - * - * [writes to B directly] e.g. memset(B, 0, 56); B[0] = val; - * [B is used] - */ - -static int lcp_find_next(TCCIRState *ir, int start, int n) -{ - for (int j = start; j < n; j++) - if (ir->compact_instructions[j].op != TCCIR_OP_NOP) - return j; - return -1; -} - -int tcc_ir_opt_local_copy_prop(TCCIRState *ir) -{ - int n = ir->next_instruction_index; - if (n < 4) - return 0; - - int changes = 0; - - for (int i = 0; i < n;) { - IRQuadCompact *q = &ir->compact_instructions[i]; - if (q->op != TCCIR_OP_LOAD) { - i++; - continue; - } - - IROperand load_src = tcc_ir_op_get_src1(ir, q); - if (irop_get_tag(load_src) != IROP_TAG_STACKOFF || !load_src.is_local || - (load_src.btype != IROP_BTYPE_INT32 && load_src.btype != IROP_BTYPE_FLOAT32)) { - i++; - continue; - } - - int store_i = lcp_find_next(ir, i + 1, n); - if (store_i < 0) { - i++; - continue; - } - IRQuadCompact *sq = &ir->compact_instructions[store_i]; - if (sq->op != TCCIR_OP_STORE || sq->is_jump_target) { - i++; - continue; - } - - IROperand store_dest = tcc_ir_op_get_dest(ir, sq); - if (irop_get_tag(store_dest) != IROP_TAG_STACKOFF || !store_dest.is_local) { - i++; - continue; - } - - IROperand load_dest = tcc_ir_op_get_dest(ir, q); - IROperand store_src1 = tcc_ir_op_get_src1(ir, sq); - if (irop_get_vreg(load_dest) < 0 || - irop_get_vreg(load_dest) != irop_get_vreg(store_src1)) { - i++; - continue; - } - - int32_t src_base = irop_get_stack_offset(load_src); - int32_t dst_base = irop_get_stack_offset(store_dest); - if (src_base == dst_base) { - i++; - continue; - } - int32_t delta = dst_base - src_base; - - /* Extend: find more consecutive LOAD+STORE copy pairs */ - int pair_loads[64], pair_stores[64]; - pair_loads[0] = i; - pair_stores[0] = store_i; - int count = 1; - int last_store = store_i; - - while (count < 64) { - int nl = lcp_find_next(ir, last_store + 1, n); - if (nl < 0 || ir->compact_instructions[nl].op != TCCIR_OP_LOAD || - ir->compact_instructions[nl].is_jump_target) - break; - - IROperand nl_src = tcc_ir_op_get_src1(ir, &ir->compact_instructions[nl]); - if (irop_get_tag(nl_src) != IROP_TAG_STACKOFF || !nl_src.is_local) - break; - if (irop_get_stack_offset(nl_src) != src_base + count * 4) - break; - - int ns = lcp_find_next(ir, nl + 1, n); - if (ns < 0 || ir->compact_instructions[ns].op != TCCIR_OP_STORE || - ir->compact_instructions[ns].is_jump_target) - break; - - IROperand ns_dest = tcc_ir_op_get_dest(ir, &ir->compact_instructions[ns]); - if (irop_get_tag(ns_dest) != IROP_TAG_STACKOFF || !ns_dest.is_local) - break; - if (irop_get_stack_offset(ns_dest) != dst_base + count * 4) - break; - - IROperand nl_dest = tcc_ir_op_get_dest(ir, &ir->compact_instructions[nl]); - IROperand ns_src1 = tcc_ir_op_get_src1(ir, &ir->compact_instructions[ns]); - if (irop_get_vreg(nl_dest) < 0 || - irop_get_vreg(nl_dest) != irop_get_vreg(ns_src1)) - break; - - pair_loads[count] = nl; - pair_stores[count] = ns; - count++; - last_store = ns; - } - - if (count < 4) { - i++; - continue; - } - - int32_t src_end = src_base + count * 4; - - /* Bail out if a memset/memclr call initializes a range that overlaps - * the source. We can't redirect the call, so the dest would have - * uninitialized bytes after we NOP the copy chain. */ - int has_overlapping_call = 0; - for (int j = 0; j < n && !has_overlapping_call; j++) { - IRQuadCompact *cq = &ir->compact_instructions[j]; - if (cq->op != TCCIR_OP_FUNCCALLVOID && cq->op != TCCIR_OP_FUNCCALLVAL) - continue; - IROperand csrc1 = tcc_ir_op_get_src1(ir, cq); - Sym *callee = irop_get_sym_ex(ir, csrc1); - if (!callee) - continue; - const char *name = get_tok_str(callee->v, NULL); - if (strcmp(name, "__aeabi_memset") != 0 && strcmp(name, "memset") != 0 && - strcmp(name, "__aeabi_memclr") != 0 && strcmp(name, "__aeabi_memclr4") != 0 && - strcmp(name, "__aeabi_memclr8") != 0) - continue; - IROperand cenc = tcc_ir_op_get_src2(ir, cq); - int call_id = TCCIR_DECODE_CALL_ID((uint32_t)irop_get_imm64_ex(ir, cenc)); - for (int p = j - 1; p >= 0; p--) { - IRQuadCompact *pq = &ir->compact_instructions[p]; - if (pq->op == TCCIR_OP_NOP) - continue; - if (pq->op != TCCIR_OP_FUNCPARAMVAL && pq->op != TCCIR_OP_FUNCPARAMVOID) - break; - IROperand penc = tcc_ir_op_get_src2(ir, pq); - if (TCCIR_DECODE_CALL_ID((uint32_t)irop_get_imm64_ex(ir, penc)) != call_id) - continue; - IROperand pval = tcc_ir_op_get_src1(ir, pq); - if (irop_get_tag(pval) == IROP_TAG_STACKOFF && pval.is_local && !pval.is_lval) { - int32_t addr = irop_get_stack_offset(pval); - if (addr < src_end && addr + 128 > src_base) - has_overlapping_call = 1; - } - } - } - if (has_overlapping_call) { - i++; - continue; - } - - /* Check safety: the source range must not be read outside the copy chain, - * and its address must not escape (except to memset/memclr). */ - int safe = 1; - int memset_param_instrs[8]; - int memset_param_count = 0; - - for (int j = 0; j < n && safe; j++) { - IRQuadCompact *cq = &ir->compact_instructions[j]; - if (cq->op == TCCIR_OP_NOP) - continue; - - int is_chain = 0; - for (int k = 0; k < count; k++) { - if (j == pair_loads[k] || j == pair_stores[k]) { - is_chain = 1; - break; - } - } - if (is_chain) - continue; - - /* Check src1 for reads from the source range. Any STACKOFF src1 - * with is_lval=1 in the source range is a memory read. */ - if (irop_config[cq->op].has_src1) { - IROperand op = tcc_ir_op_get_src1(ir, cq); - if (irop_get_tag(op) == IROP_TAG_STACKOFF && op.is_local && op.is_lval) { - int32_t off = irop_get_stack_offset(op); - if (off >= src_base && off < src_end) { - safe = 0; - break; - } - } - /* Check for address-of source range in any src1 operand */ - if (irop_get_tag(op) == IROP_TAG_STACKOFF && op.is_local && !op.is_lval) { - int32_t off = irop_get_stack_offset(op); - if (off >= src_base && off < src_end) { - if (cq->op == TCCIR_OP_FUNCPARAMVAL || cq->op == TCCIR_OP_FUNCPARAMVOID) { - if (memset_param_count < 8) - memset_param_instrs[memset_param_count++] = j; - else - safe = 0; - } - } - } - } - - /* Check for LEA / address-of source range */ - if (irop_config[cq->op].has_dest && cq->op != TCCIR_OP_STORE && - cq->op != TCCIR_OP_STORE_INDEXED && cq->op != TCCIR_OP_STORE_POSTINC) { - IROperand op = tcc_ir_op_get_dest(ir, cq); - if (irop_get_tag(op) == IROP_TAG_STACKOFF && op.is_local && !op.is_lval) { - int32_t off = irop_get_stack_offset(op); - if (off >= src_base && off < src_end) - safe = 0; - } - } - - if (irop_config[cq->op].has_src2) { - IROperand op = tcc_ir_op_get_src2(ir, cq); - if (irop_get_tag(op) == IROP_TAG_STACKOFF && op.is_local) { - int32_t off = irop_get_stack_offset(op); - if (off >= src_base && off < src_end) { - if (op.is_lval) - safe = 0; - } - } - } - } - - if (!safe) { - i++; - continue; - } - - /* Verify address-of uses are only for memset/memclr calls */ - for (int m = 0; m < memset_param_count && safe; m++) { - int param_i = memset_param_instrs[m]; - IRQuadCompact *pq = &ir->compact_instructions[param_i]; - IROperand penc = tcc_ir_op_get_src2(ir, pq); - int call_id = TCCIR_DECODE_CALL_ID((uint32_t)irop_get_imm64_ex(ir, penc)); - int found_call = 0; - for (int j = param_i + 1; j < n; j++) { - IRQuadCompact *cq = &ir->compact_instructions[j]; - if (cq->op == TCCIR_OP_NOP) - continue; - if (cq->op == TCCIR_OP_FUNCPARAMVAL || cq->op == TCCIR_OP_FUNCPARAMVOID) - continue; - if (cq->op == TCCIR_OP_FUNCCALLVOID || cq->op == TCCIR_OP_FUNCCALLVAL) { - IROperand csrc2 = tcc_ir_op_get_src2(ir, cq); - if (TCCIR_DECODE_CALL_ID((uint32_t)irop_get_imm64_ex(ir, csrc2)) == call_id) { - IROperand csrc1 = tcc_ir_op_get_src1(ir, cq); - Sym *callee = irop_get_sym_ex(ir, csrc1); - if (callee) { - const char *name = get_tok_str(callee->v, NULL); - if (strcmp(name, "__aeabi_memset") == 0 || - strcmp(name, "memset") == 0 || - strcmp(name, "__aeabi_memclr") == 0 || - strcmp(name, "__aeabi_memclr4") == 0 || - strcmp(name, "__aeabi_memclr8") == 0) - found_call = 1; - } - } - break; - } - break; - } - if (!found_call) - safe = 0; - } - - if (!safe) { - i++; - continue; - } - - /* Apply: redirect all writes to source range → dest range */ - for (int j = 0; j < n; j++) { - IRQuadCompact *cq = &ir->compact_instructions[j]; - if (cq->op == TCCIR_OP_NOP) - continue; - - int is_chain = 0; - for (int k = 0; k < count; k++) { - if (j == pair_loads[k] || j == pair_stores[k]) { - is_chain = 1; - break; - } - } - if (is_chain) - continue; - - if (cq->op == TCCIR_OP_STORE || cq->op == TCCIR_OP_STORE_INDEXED || - cq->op == TCCIR_OP_STORE_POSTINC) { - if (irop_config[cq->op].has_dest) { - IROperand op = tcc_ir_op_get_dest(ir, cq); - if (irop_get_tag(op) == IROP_TAG_STACKOFF && op.is_local && op.is_lval) { - int32_t off = irop_get_stack_offset(op); - if (off >= src_base && off < src_end) { - if (op.btype == IROP_BTYPE_STRUCT) - op.u.s.aux_data = (uint32_t)(int32_t)(off + delta); - else - op.u.imm32 = off + delta; - tcc_ir_op_set_dest(ir, cq, op); - } - } - } - } - - /* Redirect any src1 address-of (is_lval=0) StackLoc in source range. - * Covers both direct FUNCPARAMVAL and LEA/ASSIGN instructions that - * compute Addr[StackLoc[off]]. */ - if (irop_config[cq->op].has_src1) { - IROperand op = tcc_ir_op_get_src1(ir, cq); - if (irop_get_tag(op) == IROP_TAG_STACKOFF && op.is_local && !op.is_lval) { - int32_t off = irop_get_stack_offset(op); - if (off >= src_base && off < src_end) { - if (op.btype == IROP_BTYPE_STRUCT) - op.u.s.aux_data = (uint32_t)(int32_t)(off + delta); - else - op.u.imm32 = off + delta; - tcc_ir_op_set_src1(ir, cq, op); - } - } - } - } - - for (int k = 0; k < count; k++) { - ir->compact_instructions[pair_loads[k]].op = TCCIR_OP_NOP; - ir->compact_instructions[pair_stores[k]].op = TCCIR_OP_NOP; - } - - changes += count; - i = last_store + 1; - } - - return changes; -} - -int tcc_ir_opt_local_copy_prop_ex(IROptCtx *ctx) -{ - return tcc_ir_opt_local_copy_prop(ctx->ir); -} - diff --git a/source/opt/flat/memory/memmove_to_indexed_stores.c b/source/opt/flat/memory/memmove_to_indexed_stores.c index e1cbee25e..126809e75 100644 --- a/source/opt/flat/memory/memmove_to_indexed_stores.c +++ b/source/opt/flat/memory/memmove_to_indexed_stores.c @@ -124,8 +124,10 @@ int tcc_ir_opt_memmove_to_indexed_stores(TCCIRState *ir) break; if (!irop_config[lq->op].has_dest) continue; + /* A VAR written as itself is a STACKOFF lvalue: still a definition + * (irop_dest_defines_vreg), not one to search past. */ IROperand ld = tcc_ir_op_get_dest(ir, lq); - if (!irop_has_vreg(ld) || irop_get_vreg(ld) != src_vr || ld.is_lval) + if (irop_get_vreg(ld) != src_vr || !irop_dest_defines_vreg(ld)) continue; /* The op writing our vreg must be a LEA/ASSIGN from Addr[StackLoc[X]]. */ if (lq->op != TCCIR_OP_LEA && lq->op != TCCIR_OP_ASSIGN) @@ -196,7 +198,7 @@ int tcc_ir_opt_memmove_to_indexed_stores(TCCIRState *ir) if (!irop_config[lq->op].has_dest) continue; IROperand ld = tcc_ir_op_get_dest(ir, lq); - if (!irop_has_vreg(ld) || irop_get_vreg(ld) != trace_vr || ld.is_lval) + if (irop_get_vreg(ld) != trace_vr || !irop_dest_defines_vreg(ld)) continue; trace_depth++; if (lq->op == TCCIR_OP_LEA || lq->op == TCCIR_OP_ASSIGN) @@ -421,7 +423,7 @@ int tcc_ir_opt_memmove_to_indexed_stores(TCCIRState *ir) if (kq->op == TCCIR_OP_JUMP || kq->op == TCCIR_OP_JUMPIF || kq->op == TCCIR_OP_IJUMP) break; if (!irop_config[kq->op].has_dest) continue; IROperand kd = tcc_ir_op_get_dest(ir, kq); - if (!irop_has_vreg(kd) || irop_get_vreg(kd) != trace_vr || kd.is_lval) continue; + if (irop_get_vreg(kd) != trace_vr || !irop_dest_defines_vreg(kd)) continue; trace_depth++; if (kq->op == TCCIR_OP_ADD) { @@ -473,7 +475,7 @@ int tcc_ir_opt_memmove_to_indexed_stores(TCCIRState *ir) if (kq->op == TCCIR_OP_JUMP || kq->op == TCCIR_OP_JUMPIF || kq->op == TCCIR_OP_IJUMP) break; if (!irop_config[kq->op].has_dest) continue; IROperand kd = tcc_ir_op_get_dest(ir, kq); - if (!irop_has_vreg(kd) || irop_get_vreg(kd) != trace_vr || kd.is_lval) continue; + if (irop_get_vreg(kd) != trace_vr || !irop_dest_defines_vreg(kd)) continue; trace_depth++; if (kq->op == TCCIR_OP_ADD) { @@ -679,7 +681,7 @@ int tcc_ir_opt_memmove_to_indexed_stores(TCCIRState *ir) if (!irop_config[sq->op].has_dest) continue; IROperand d = tcc_ir_op_get_dest(ir, sq); - if (irop_has_vreg(d) && irop_get_vreg(d) == dst_vr && !d.is_lval) + if (irop_get_vreg(d) == dst_vr && irop_dest_defines_vreg(d)) { dst_def_idx = j; break; @@ -841,6 +843,7 @@ int tcc_ir_opt_memmove_to_indexed_stores(TCCIRState *ir) IROperand new_dest = irop_make_stackoff(-1, dst_base + offset, /*is_lval*/ 1, /*is_llocal*/ 0, /*is_param*/ 0, store_btype); + new_dest.aux |= IROP_AUX_NONVOLATILE; /* a memmove's store: no volatile access */ tcc_ir_pool_ensure(ir, 2); int new_pool = ir->iroperand_pool_count; tcc_ir_pool_add(ir, new_dest); diff --git a/source/opt/flat/memory/param_copy_alias.c b/source/opt/flat/memory/param_copy_alias.c new file mode 100644 index 000000000..92d3e0ccb --- /dev/null +++ b/source/opt/flat/memory/param_copy_alias.c @@ -0,0 +1,569 @@ +/* + * TCC IR - Struct parameter copy aliasing (flat, pre-SSA) + * + * Copyright (c) 2026 Mateusz Stadnik + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation. + */ + +#define USING_GLOBALS + +#include "ir.h" +#include "opt.h" +#include "opt_engine.h" + +/* A by-value struct parameter belongs to the callee: its words sit in memory + * the callee may read and write at will -- the caller's outgoing area for a + * parameter passed on the stack (or split across r3 and the stack), a home slot + * the prologue stores the registers into for one passed in registers. The Zig + * C backend copies every such parameter into a local before using it: + * + * t0 = a1; t1 = &t0; ... t1->field ... + * + * which tcc lowers to a word-by-word LOAD/STORE chain, or a memmove call for a + * large one. When the copy fills the whole local in the entry block, before + * anything else touches the local, and nothing but the copy ever reads the + * parameter, the local can simply be the parameter: every reference to the + * local is renamed onto the parameter's memory and the copy is dropped. Writes + * through the local then land in the parameter, which nothing else reads. */ + +#define PCA_MAX_WORDS 256 + +typedef struct PcaCopy +{ + int first, last; /* instruction range of the copy (inclusive) */ + int32_t dst, size; /* local destination [dst, dst + size) */ + int32_t src; /* source offset */ + IROperand tmpl; /* a source operand: the parameter's encoding */ + int nins; + int entry_only; /* the copy sits in the entry block, where any shape is safe */ + int ins[3 * PCA_MAX_WORDS + 8]; /* instructions the copy consists of */ +} PcaCopy; + +static int pca_nops(int op) +{ + int n = irop_config[op].has_dest + irop_config[op].has_src1 + irop_config[op].has_src2; + if (op == TCCIR_OP_MLA || op == TCCIR_OP_LOAD_INDEXED || op == TCCIR_OP_STORE_INDEXED || op == TCCIR_OP_SELECT) + n++; + return n; +} + +/* A concrete frame slot of the function's own frame (no vreg, not a + * parameter). */ +static int pca_is_local_slot(IROperand op, int32_t *off) +{ + if (irop_is_none(op) || irop_get_tag(op) != IROP_TAG_STACKOFF || !op.is_local || op.is_param || + irop_get_vreg(op) >= 0) + return 0; + *off = irop_get_stack_offset(op); + return 1; +} + +/* The next non-NOP instruction at or after `j` before `limit`, or -1. */ +static int pca_next(TCCIRState *ir, int j, int limit) +{ + for (; j < limit; j++) + if (ir->compact_instructions[j].op != TCCIR_OP_NOP) + return j; + return -1; +} + +static int pca_word_btype(IROperand op) +{ + return op.btype == IROP_BTYPE_INT32 && !op.is_complex; +} + +/* Operand counts of the function's TEMP vregs, by position, kept exact across + * the pass's own edits: every candidate word asks for its TEMP's use count, and + * scanning the whole function for each one made the pass quadratic -- 78% of an + * -O1 compile of Zig's zig.c, whose functions copy thousands of words. */ +typedef struct PcaUses +{ + int *count; /* NULL until first asked */ + int size; +} PcaUses; + +static int pca_temp_pos(int32_t vr, const PcaUses *u) +{ + if (vr < 0 || TCCIR_DECODE_VREG_TYPE(vr) != TCCIR_VREG_TYPE_TEMP) + return -1; + int pos = TCCIR_DECODE_VREG_POSITION(vr); + if (vr != TCCIR_ENCODE_VREG(TCCIR_VREG_TYPE_TEMP, pos) || pos >= u->size) + return -1; + return pos; +} + +static void pca_uses_adjust(PcaUses *u, IROperand op, int delta) +{ + int pos; + if (u->count && (pos = pca_temp_pos(irop_get_vreg(op), u)) >= 0) + u->count[pos] += delta; +} + +static void pca_uses_build(TCCIRState *ir, PcaUses *u) +{ + int max = -1; + for (int i = 0; i < ir->next_instruction_index; i++) + { + IRQuadCompact *q = &ir->compact_instructions[i]; + if (q->op == TCCIR_OP_NOP) + continue; + int nops = pca_nops(q->op); + for (int k = 0; k < nops; k++) + { + int32_t vr = irop_get_vreg(ir->iroperand_pool[q->operand_base + k]); + if (vr >= 0 && TCCIR_DECODE_VREG_TYPE(vr) == TCCIR_VREG_TYPE_TEMP && TCCIR_DECODE_VREG_POSITION(vr) > max) + max = TCCIR_DECODE_VREG_POSITION(vr); + } + } + u->size = max + 1; + u->count = tcc_mallocz(sizeof(int) * (u->size > 0 ? u->size : 1)); + for (int i = 0; i < ir->next_instruction_index; i++) + { + IRQuadCompact *q = &ir->compact_instructions[i]; + if (q->op == TCCIR_OP_NOP) + continue; + int nops = pca_nops(q->op); + for (int k = 0; k < nops; k++) + pca_uses_adjust(u, ir->iroperand_pool[q->operand_base + k], 1); + } +} + +static int pca_temp_uses(TCCIRState *ir, PcaUses *u, int32_t vr) +{ + if (!u->count) + pca_uses_build(ir, u); + int pos = pca_temp_pos(vr, u); + if (pos >= 0) + return u->count[pos]; + int uses = 0; + for (int i = 0; i < ir->next_instruction_index; i++) + { + IRQuadCompact *q = &ir->compact_instructions[i]; + if (q->op == TCCIR_OP_NOP) + continue; + int nops = pca_nops(q->op); + for (int k = 0; k < nops; k++) + uses += irop_get_vreg(ir->iroperand_pool[q->operand_base + k]) == vr; + } + return uses; +} + +/* LOAD T <- S+4k ; STORE L+4k <- T, repeated, from `i`. */ +static int pca_match_chain(TCCIRState *ir, PcaUses *u, int i, int limit, PcaCopy *c) +{ + int n = 0; + int j = i, last = i; + while (n < PCA_MAX_WORDS) + { + int js; + if (j < 0 || (js = pca_next(ir, j + 1, limit)) < 0) + break; + IRQuadCompact *lq = &ir->compact_instructions[j], *sq = &ir->compact_instructions[js]; + if (lq->op != TCCIR_OP_LOAD || sq->op != TCCIR_OP_STORE) + break; + IROperand s = tcc_ir_op_get_src1(ir, lq), t = tcc_ir_op_get_dest(ir, lq); + IROperand d = tcc_ir_op_get_dest(ir, sq), v = tcc_ir_op_get_src1(ir, sq); + int32_t doff; + if (irop_get_tag(s) != IROP_TAG_STACKOFF || !s.is_local || !s.is_lval || s.is_llocal || !pca_word_btype(s)) + break; + if (!pca_is_local_slot(d, &doff) || !d.is_lval || !pca_word_btype(d)) + break; + int32_t tv = irop_get_vreg(t); + if (tv < 0 || TCCIR_DECODE_VREG_TYPE(tv) != TCCIR_VREG_TYPE_TEMP || irop_get_vreg(v) != tv || v.is_lval) + break; + int32_t soff = irop_get_stack_offset(s); + if (n == 0) + { + c->tmpl = s; + c->src = soff; + c->dst = doff; + } + else if (soff != c->src + 4 * n || doff != c->dst + 4 * n || s.is_param != c->tmpl.is_param || + irop_get_vreg(s) != irop_get_vreg(c->tmpl)) + break; + if (pca_temp_uses(ir, u, tv) != 2) + break; + c->ins[c->nins++] = j; + c->ins[c->nins++] = js; + n++; + last = js; + j = pca_next(ir, js + 1, limit); + } + if (n < 2) + return 0; + c->first = i; + c->last = last; + c->size = 4 * n; + return 1; +} + +/* A single-use TEMP defined at `k` as the address of a stack slot. */ +static int pca_addr_temp(TCCIRState *ir, int k, int32_t tv, IROperand *slot) +{ + IRQuadCompact *q = &ir->compact_instructions[k]; + if ((q->op != TCCIR_OP_LEA && q->op != TCCIR_OP_ASSIGN) || irop_get_vreg(tcc_ir_op_get_dest(ir, q)) != tv) + return 0; + IROperand a = tcc_ir_op_get_src1(ir, q); + if (irop_get_tag(a) != IROP_TAG_STACKOFF || !a.is_local || a.is_lval || a.is_llocal) + return 0; + *slot = a; + return 1; +} + +static int pca_is_copy_fn(TCCIRState *ir, IRQuadCompact *call) +{ + Sym *s = irop_get_sym_ex(ir, tcc_ir_op_get_src1(ir, call)); + if (!s) + return 0; + const char *nm = get_tok_str(s->v, NULL); + return !strcmp(nm, "memmove") || !strcmp(nm, "memcpy") || !strcmp(nm, "__aeabi_memmove") || + !strcmp(nm, "__aeabi_memmove4") || !strcmp(nm, "__aeabi_memmove8") || !strcmp(nm, "__aeabi_memcpy") || + !strcmp(nm, "__aeabi_memcpy4") || !strcmp(nm, "__aeabi_memcpy8"); +} + +/* Ta <- &L ; Tb <- &S ; PARAM0 Ta ; PARAM1 Tb ; PARAM2 #size ; CALL memmove + * (the two address definitions may come in either order). */ +static int pca_match_call(TCCIRState *ir, PcaUses *u, int i, int limit, PcaCopy *c) +{ + int at[6]; + at[0] = i; + for (int k = 1; k < 6; k++) + if ((at[k] = pca_next(ir, at[k - 1] + 1, limit)) < 0) + return 0; + IRQuadCompact *q = &ir->compact_instructions[i]; + if (q->op != TCCIR_OP_LEA && q->op != TCCIR_OP_ASSIGN) + return 0; + IRQuadCompact *p0 = &ir->compact_instructions[at[2]], *p1 = &ir->compact_instructions[at[3]], + *p2 = &ir->compact_instructions[at[4]], *call = &ir->compact_instructions[at[5]]; + if (p0->op != TCCIR_OP_FUNCPARAMVAL || p1->op != TCCIR_OP_FUNCPARAMVAL || p2->op != TCCIR_OP_FUNCPARAMVAL || + call->op != TCCIR_OP_FUNCCALLVOID || !pca_is_copy_fn(ir, call)) + return 0; + int call_id = TCCIR_DECODE_CALL_ID((uint32_t)irop_get_imm64_ex(ir, tcc_ir_op_get_src2(ir, call))); + IRQuadCompact *ps[3] = {p0, p1, p2}; + for (int k = 0; k < 3; k++) + { + uint32_t enc = (uint32_t)irop_get_imm64_ex(ir, tcc_ir_op_get_src2(ir, ps[k])); + if (TCCIR_DECODE_CALL_ID(enc) != call_id || TCCIR_DECODE_PARAM_IDX(enc) != k) + return 0; + } + IROperand a0 = tcc_ir_op_get_src1(ir, p0), a1 = tcc_ir_op_get_src1(ir, p1), a2 = tcc_ir_op_get_src1(ir, p2); + int32_t t0 = irop_get_vreg(a0), t1 = irop_get_vreg(a1); + if (t0 < 0 || t1 < 0 || a0.is_lval || a1.is_lval || irop_get_tag(a2) != IROP_TAG_IMM32) + return 0; + IROperand ld, ls; + if (!(pca_addr_temp(ir, at[0], t0, &ld) && pca_addr_temp(ir, at[1], t1, &ls)) && + !(pca_addr_temp(ir, at[1], t0, &ld) && pca_addr_temp(ir, at[0], t1, &ls))) + return 0; + if (pca_temp_uses(ir, u, t0) != 2 || pca_temp_uses(ir, u, t1) != 2) + return 0; + int32_t doff; + if (!pca_is_local_slot(ld, &doff)) + return 0; + int32_t size = irop_get_imm32(a2); + if (size <= 0 || (size & 3)) + return 0; + c->dst = doff; + c->src = irop_get_stack_offset(ls); + c->size = size; + c->tmpl = ls; + c->tmpl.is_lval = 1; + c->tmpl.btype = IROP_BTYPE_INT32; + c->first = i; + c->last = at[5]; + for (int k = 0; k < 6; k++) + c->ins[c->nins++] = at[k]; + return 1; +} + +static int pca_in(int32_t off, int32_t base, int32_t size) +{ + return off >= base && off < base + size; +} + +/* Does `op` name the copy's source? */ +static int pca_is_src_ref(IROperand op, const PcaCopy *c) +{ + if (irop_is_none(op)) + return 0; + int32_t svr = irop_get_vreg(c->tmpl); + if (svr >= 0 && irop_get_vreg(op) == svr) + return 1; + if (irop_get_tag(op) != IROP_TAG_STACKOFF || !op.is_local || op.is_param != c->tmpl.is_param) + return 0; + if (!c->tmpl.is_param && irop_get_vreg(op) >= 0) + return 0; + return pca_in(irop_get_stack_offset(op), c->src, c->size); +} + +static int pca_is_copy_ins(const PcaCopy *c, int i) +{ + if (i < c->first || i > c->last) + return 0; + for (int k = 0; k < c->nins; k++) + if (c->ins[k] == i) + return 1; + return 0; +} + +/* Operand `k` of `q` is the hidden pointer a struct-returning call writes its + * result through, and it points at the copy's source: the call fills exactly + * those bytes, which is what the copy then reads. The pointer is the call's + * parameter 0 and never outlives the call (tcc_ir_frame_note_sret_call + * records which calls have one, and frame.c already treats such a buffer's + * address as not escaping). */ +static int pca_is_sret_buffer_arg(TCCIRState *ir, IRQuadCompact *q, int k, IROperand op, const PcaCopy *c) +{ + if (q->op != TCCIR_OP_FUNCPARAMVAL || k != 0 || op.is_lval || op.is_llocal) + return 0; + if (irop_get_stack_offset(op) != c->src) + return 0; + const uint32_t enc = (uint32_t)irop_get_imm64_ex(ir, tcc_ir_op_get_src2(ir, q)); + if (TCCIR_DECODE_PARAM_IDX(enc) != 0) + return 0; + const int call_id = TCCIR_DECODE_CALL_ID(enc); + if (call_id < 0 || call_id >= ir->sret_calls_size) + return 0; + return ir->sret_calls[call_id] == c->size; +} + +/* The copy reads a buffer the call right before it filled, and that call is + * the only thing that writes it: the instructions between the hidden pointer + * and the copy are that call and its own arguments. Then the source and the + * destination hold the same bytes at every point after the copy, whatever the + * control flow around them -- each is written once, next to the other -- so + * the destination's references may be renamed onto the source wherever they + * are, not only in the entry block. Reading the destination before the copy + * has run reads an uninitialised object either way. */ +static int pca_sret_copy_follows_its_call(TCCIRState *ir, const PcaCopy *c, int arg_idx) +{ + const uint32_t enc = (uint32_t)irop_get_imm64_ex(ir, tcc_ir_op_get_src2(ir, &ir->compact_instructions[arg_idx])); + const int call_id = TCCIR_DECODE_CALL_ID(enc); + int saw_call = 0; + for (int i = arg_idx + 1; i < c->first; i++) + { + IRQuadCompact *q = &ir->compact_instructions[i]; + if (q->op == TCCIR_OP_NOP) + continue; + if (q->op == TCCIR_OP_FUNCPARAMVAL || q->op == TCCIR_OP_FUNCPARAMVOID || q->op == TCCIR_OP_FUNCCALLVAL || + q->op == TCCIR_OP_FUNCCALLVOID) + { + IROperand enc_op = q->op == TCCIR_OP_FUNCPARAMVAL || q->op == TCCIR_OP_FUNCPARAMVOID + ? tcc_ir_op_get_src2(ir, q) + : tcc_ir_op_get_src2(ir, q); + if (TCCIR_DECODE_CALL_ID((uint32_t)irop_get_imm64_ex(ir, enc_op)) != call_id) + return 0; + saw_call |= q->op == TCCIR_OP_FUNCCALLVAL || q->op == TCCIR_OP_FUNCCALLVOID; + continue; + } + return 0; + } + return saw_call; +} + +/* Can the copy's source be the buffer a call right before it filled? Outside + * the entry block that is the only shape pca_try accepts, and the call and its + * arguments -- all that may sit between the hidden pointer and the copy -- are + * found by walking back from the copy, without the whole-function scan. */ +static int pca_may_follow_sret_call(TCCIRState *ir, const PcaCopy *c) +{ + if (c->tmpl.is_param) + return 0; + for (int i = c->first - 1; i >= 0; i--) + { + IRQuadCompact *q = &ir->compact_instructions[i]; + if (q->op == TCCIR_OP_NOP || q->op == TCCIR_OP_FUNCPARAMVOID || q->op == TCCIR_OP_FUNCCALLVAL || + q->op == TCCIR_OP_FUNCCALLVOID) + continue; + if (q->op != TCCIR_OP_FUNCPARAMVAL) + return 0; + IROperand op = ir->iroperand_pool[q->operand_base]; + if (pca_is_src_ref(op, c) && pca_is_sret_buffer_arg(ir, q, 0, op, c) && pca_sret_copy_follows_its_call(ir, c, i)) + return 1; + } + return 0; +} + +static int pca_try(TCCIRState *ir, PcaUses *u, PcaCopy *c) +{ + const int n = ir->next_instruction_index; + int32_t obj_size = tcc_ir_frame_object_size_at(ir, c->dst); + if (obj_size <= 0 || obj_size > c->size || c->size - obj_size >= 4) + return 0; + /* A local source is itself a whole frame object the copy reads completely. */ + if (!c->tmpl.is_param) + { + int32_t ssz = tcc_ir_frame_object_size_at(ir, c->src); + if (ssz <= 0 || ssz > c->size || c->size - ssz >= 4 || pca_in(c->src, c->dst, c->size) || + pca_in(c->dst, c->src, c->size)) + return 0; + if (irop_get_stack_offset(c->tmpl) + c->size > 0) + return 0; + } + else if (c->tmpl.btype == IROP_BTYPE_STRUCT || c->src < 0 || c->src + c->size > 32767) + return 0; + + if (!c->entry_only && !pca_may_follow_sret_call(ir, c)) + return 0; + + int sret_src = 0; + for (int i = 0; i < n; i++) + { + IRQuadCompact *q = &ir->compact_instructions[i]; + if (q->op == TCCIR_OP_NOP || pca_is_copy_ins(c, i)) + continue; + int nops = pca_nops(q->op); + for (int k = 0; k < nops; k++) + { + IROperand op = ir->iroperand_pool[q->operand_base + k]; + int32_t off; + if (pca_is_src_ref(op, c)) + { + /* A local source may be written before the copy: the prologue storing + * register-passed words into it, or the call that returned it, which + * fills it whole through the hidden pointer it is handed (Zig's C + * copies a call's result into a second local before reading it). + * Nothing else may touch it. */ + if (c->tmpl.is_param || i > c->first) + return 0; + if (q->op == TCCIR_OP_STORE && k == 0 && op.is_lval) + continue; + if (pca_is_sret_buffer_arg(ir, q, k, op, c) && pca_sret_copy_follows_its_call(ir, c, i)) + { + sret_src = 1; + continue; + } + return 0; + } + /* A vreg-backed operand's offset is only the frontend's watermark (see + * compute_stack_layout); concrete slots are what can name the local. */ + if (irop_get_tag(op) == IROP_TAG_STACKOFF && op.is_local && !op.is_param && irop_get_vreg(op) < 0 && + pca_in(irop_get_stack_offset(op), c->dst, c->size)) + { + /* The destination: only as a plain slot, and -- unless the source is + * the call's own buffer, where the two agree from the copy onwards + * whatever the control flow -- only after the copy. */ + if ((i < c->first && !sret_src) || !pca_is_local_slot(op, &off) || op.is_llocal) + return 0; + if (q->op == TCCIR_OP_STORE && k == 0 && op.is_lval && sret_src) + return 0; /* written by something other than the copy */ + } + } + } + + if (!c->entry_only && !sret_src) + return 0; + + /* Rename every destination reference onto the source. */ + for (int i = 0; i < n; i++) + { + IRQuadCompact *q = &ir->compact_instructions[i]; + if (q->op == TCCIR_OP_NOP || pca_is_copy_ins(c, i)) + continue; + int nops = pca_nops(q->op); + for (int k = 0; k < nops; k++) + { + IROperand *p = &ir->iroperand_pool[q->operand_base + k]; + int32_t off; + if (!pca_is_local_slot(*p, &off) || !pca_in(off, c->dst, c->size)) + continue; + int32_t noff = c->src + (off - c->dst); + IROperand r; + if (c->tmpl.is_param) + { + r = c->tmpl; + r.btype = p->btype; + r.is_lval = p->is_lval; + r.is_unsigned = p->is_unsigned; + r.is_complex = p->is_complex; + r.aux = p->aux; + } + else + r = *p; + if (p->btype == IROP_BTYPE_STRUCT) + { + r.u.s.ctype_idx = p->u.s.ctype_idx; + r.u.s.aux_data = (int16_t)noff; + } + else + r.u.imm32 = noff; + pca_uses_adjust(u, *p, -1); + pca_uses_adjust(u, r, 1); + *p = r; + } + } + for (int k = 0; k < c->nins; k++) + { + IRQuadCompact *q = &ir->compact_instructions[c->ins[k]]; + int nops = pca_nops(q->op); + for (int j = 0; j < nops; j++) + pca_uses_adjust(u, ir->iroperand_pool[q->operand_base + j], -1); + q->op = TCCIR_OP_NOP; + } + return 1; +} + +int tcc_ir_opt_param_copy_alias(TCCIRState *ir) +{ + if (!ir || !tcc_state || tcc_state->do_debug || tcc_bounds_checking(tcc_state) || ir->has_static_chain || + ir->captured_count > 0 || tcc_state->nb_nested_funcs > 0 || ir->func_has_label_addr) + return 0; + const int n = ir->next_instruction_index; + /* The entry block: up to the first jump target or control transfer. */ + int entry_end = n; + for (int i = 0; i < n; i++) + { + IRQuadCompact *q = &ir->compact_instructions[i]; + if (i > 0 && q->is_jump_target) + { + entry_end = i; + break; + } + if (q->op == TCCIR_OP_JUMP || q->op == TCCIR_OP_JUMPIF || q->op == TCCIR_OP_IJUMP || + q->op == TCCIR_OP_SWITCH_TABLE || q->op == TCCIR_OP_SETJMP || q->op == TCCIR_OP_NL_SETJMP || + q->op == TCCIR_OP_VLA_ALLOC || q->op == TCCIR_OP_INLINE_ASM || q->op == TCCIR_OP_ASM_INPUT || + q->op == TCCIR_OP_ASM_OUTPUT) + { + entry_end = i; + break; + } + } + if (ir->compact_instructions[0].is_jump_target) + return 0; + + int changes = 0; + PcaUses u = {0}; + PcaCopy *c = tcc_malloc(sizeof(PcaCopy)); + for (int i = 0; i < n; i++) + { + if (ir->compact_instructions[i].op == TCCIR_OP_NOP) + continue; + /* Only ins[0, nins) is ever read, so the 3 KB array needs no clearing. */ + memset(c, 0, offsetof(PcaCopy, ins)); + /* Beyond the entry block only the call-buffer shape qualifies; pca_try + * rejects the rest, which the entry-block limit used to keep out. */ + const int limit = i < entry_end ? entry_end : n; + if (!pca_match_chain(ir, &u, i, limit, c)) + { + memset(c, 0, offsetof(PcaCopy, ins)); + if (!pca_match_call(ir, &u, i, limit, c)) + continue; + } + c->entry_only = i < entry_end; + if (pca_try(ir, &u, c)) + { + changes++; + i = c->last; + } + } + tcc_free(c); + tcc_free(u.count); + return changes; +} + +int tcc_ir_opt_param_copy_alias_ex(IROptCtx *ctx) +{ + return tcc_ir_opt_param_copy_alias(ctx->ir); +} diff --git a/source/opt/flat/memory/ptr_local_fwd.c b/source/opt/flat/memory/ptr_local_fwd.c index 31b89ff04..4f931870c 100644 --- a/source/opt/flat/memory/ptr_local_fwd.c +++ b/source/opt/flat/memory/ptr_local_fwd.c @@ -17,9 +17,12 @@ #include "opt_utils.h" #include "opt_du.h" -/* A pointer VAR P written exactly once, in the entry block, by `LEA P <-- &V` - * (V a non-volatile local VAR), and whose own address is never taken, holds &V - * at every use. Every deref through P -- or through any TEMP whose defs all +/* A pointer VAR P whose every definition is `LEA P <-- &V` for the same V (a + * non-volatile local VAR), and whose own address is never taken, holds &V at + * every use: a use no definition reaches would read an uninitialized local, + * which C leaves undefined anyway. (The Zig C backend reuses one C local per + * type, so `t1 = &t0` recurs, and it sets them in every block, not just the + * entry.) Every deref through P -- or through any TEMP whose defs all * copy P's value -- therefore denotes V's slot and is rewritten to a direct VAR * access. Valid regardless of escapes: while any LEA of V survives, V stays * memory-resident, so *P and the direct access read/write the same storage. @@ -29,7 +32,8 @@ * * The rewrite reuses a "template" operand cloned from an existing direct lval * access of V, so every encoding convention (tag/flags/aux) is preserved - * exactly; vars with no direct access are skipped. */ + * exactly. A V with no such access -- only ever set by a call, say -- gets + * the canonical slot form of a 32-bit VAR read instead. */ #define PLF_UNKNOWN (-1) #define PLF_CONFLICT (-2) @@ -59,39 +63,19 @@ int tcc_ir_opt_ptr_local_fwd(TCCIRState *ir) if (n < 4 || nvar <= 0) return 0; - - int dc_stride = 0; - uint8_t *dc = ir_opt_build_def_count(ir, n, &dc_stride); - - /* Entry block: everything before the first label or control transfer - * dominates the whole function (calls do not end the block). */ - int entry_end = n; - for (int i = 0; i < n; i++) - { - IRQuadCompact *q = &ir->compact_instructions[i]; - if (q->op == TCCIR_OP_NOP) - continue; - if (i > 0 && q->is_jump_target) - { - entry_end = i; - break; - } - if (q->op == TCCIR_OP_JUMP || q->op == TCCIR_OP_JUMPIF || q->op == TCCIR_OP_IJUMP || - q->op == TCCIR_OP_SWITCH_TABLE || q->op == TCCIR_OP_RETURNVALUE || - q->op == TCCIR_OP_RETURNVOID) - { - entry_end = i; - break; - } - } + /* A nested function writes a captured P through the static chain, a def + * this body never shows. */ + if ((tcc_state && tcc_state->nb_nested_funcs > 0) || ir->captured_count > 0) + return 0; /* ptr_target[pos(P)] = composite vreg of V, or PLF_UNKNOWN. */ int32_t *ptr_target = tcc_malloc(nvar * sizeof(int32_t)); for (int i = 0; i < nvar; i++) ptr_target[i] = PLF_UNKNOWN; + /* Every `LEA P <-- &V` must name the same V... */ int candidates = 0; - for (int i = 0; i < entry_end; i++) + for (int i = 0; i < n; i++) { IRQuadCompact *q = &ir->compact_instructions[i]; if (q->op != TCCIR_OP_LEA || irop_config[q->op].has_src2) @@ -102,23 +86,39 @@ int tcc_ir_opt_ptr_local_fwd(TCCIRState *ir) int32_t v_vr = irop_get_vreg(src); if (p_vr < 0 || TCCIR_DECODE_VREG_TYPE(p_vr) != TCCIR_VREG_TYPE_VAR) continue; - if (v_vr < 0 || TCCIR_DECODE_VREG_TYPE(v_vr) != TCCIR_VREG_TYPE_VAR || src.is_lval) - continue; - if (!DC_IS_SINGLE_DEF(dc, dc_stride, p_vr)) - continue; - if (plf_vreg_is_volatile(ir, p_vr) || plf_vreg_is_volatile(ir, v_vr)) - continue; int p_pos = TCCIR_DECODE_VREG_POSITION(p_vr); if (p_pos >= nvar) continue; - ptr_target[p_pos] = v_vr; - candidates++; + int ok = v_vr >= 0 && TCCIR_DECODE_VREG_TYPE(v_vr) == TCCIR_VREG_TYPE_VAR && !src.is_lval && + !plf_vreg_is_volatile(ir, p_vr) && !plf_vreg_is_volatile(ir, v_vr); + if (!ok) + ptr_target[p_pos] = PLF_CONFLICT; + else if (ptr_target[p_pos] == PLF_UNKNOWN) + ptr_target[p_pos] = v_vr, candidates++; + else if (ptr_target[p_pos] != v_vr) + ptr_target[p_pos] = PLF_CONFLICT; + } + /* ...and P may have no other definition. */ + for (int i = 0; i < n && candidates; i++) + { + IRQuadCompact *q = &ir->compact_instructions[i]; + if (q->op == TCCIR_OP_NOP || q->op == TCCIR_OP_LEA || !irop_config[q->op].has_dest) + continue; + int32_t dvr = irop_get_vreg(tcc_ir_op_get_dest(ir, q)); + if (dvr >= 0 && TCCIR_DECODE_VREG_TYPE(dvr) == TCCIR_VREG_TYPE_VAR && TCCIR_DECODE_VREG_POSITION(dvr) < nvar) + ptr_target[TCCIR_DECODE_VREG_POSITION(dvr)] = PLF_CONFLICT; + } + candidates = 0; + for (int i = 0; i < nvar; i++) + { + if (ptr_target[i] == PLF_CONFLICT) + ptr_target[i] = PLF_UNKNOWN; + candidates += ptr_target[i] != PLF_UNKNOWN; } if (!candidates) { tcc_free(ptr_target); - tcc_free(dc); return 0; } @@ -169,7 +169,6 @@ int tcc_ir_opt_ptr_local_fwd(TCCIRState *ir) if (!candidates) { tcc_free(ptr_target); - tcc_free(dc); return 0; } @@ -194,6 +193,52 @@ int tcc_ir_opt_ptr_local_fwd(TCCIRState *ir) tmpl[pos] = d; tmpl_ok[pos] = 1; } + /* A V without one -- set only by a call's result, say -- gets the canonical + * slot form of a VAR read, when every operand naming it is 32 bits wide. */ + { + int8_t *vw = tcc_mallocz(nvar); /* 0 unseen, 1 all 32-bit, -1 mixed */ + for (int i = 0; i < n; i++) + { + IRQuadCompact *q = &ir->compact_instructions[i]; + if (q->op == TCCIR_OP_NOP || q->op == TCCIR_OP_LEA) + continue; + for (int s = 0; s < 3; s++) + { + IROperand o; + if (s == 0 && irop_config[q->op].has_dest) + o = tcc_ir_op_get_dest(ir, q); + else if (s == 1 && irop_config[q->op].has_src1) + o = tcc_ir_op_get_src1(ir, q); + else if (s == 2 && irop_config[q->op].has_src2) + o = tcc_ir_op_get_src2(ir, q); + else + continue; + int32_t vr = irop_get_vreg(o); + if (vr < 0 || TCCIR_DECODE_VREG_TYPE(vr) != TCCIR_VREG_TYPE_VAR || TCCIR_DECODE_VREG_POSITION(vr) >= nvar) + continue; + int pos = TCCIR_DECODE_VREG_POSITION(vr); + int w32 = irop_get_btype(o) == IROP_BTYPE_INT32 && !o.is_llocal && !o.is_complex; + if (vw[pos] == 0) + vw[pos] = w32 ? 1 : -1; + else if (!w32) + vw[pos] = -1; + } + } + for (int pos = 0; pos < nvar; pos++) + { + if (tmpl_ok[pos] || vw[pos] != 1) + continue; + int32_t v_vr = TCCIR_ENCODE_VREG(TCCIR_VREG_TYPE_VAR, pos); + if (!tcc_ir_vreg_is_valid(ir, v_vr) || plf_vreg_is_volatile(ir, v_vr)) + continue; + IRLiveInterval *iv = tcc_ir_get_live_interval(ir, v_vr); + if (!iv) + continue; + tmpl[pos] = irop_make_stackoff(v_vr, iv->original_offset, 1, 0, 0, IROP_BTYPE_INT32); + tmpl_ok[pos] = 2; /* synthesized: take the access marks from each use */ + } + tcc_free(vw); + } /* temp_val[pos(T)]: composite vreg of the V whose address every def of T * yields, or PLF_UNKNOWN / PLF_CONFLICT. Iterated for copy chains. */ @@ -320,6 +365,8 @@ int tcc_ir_opt_ptr_local_fwd(TCCIRState *ir) IROperand rep = tmpl[v_pos]; rep.is_unsigned = o.is_unsigned; + if (tmpl_ok[v_pos] == 2) + rep.aux = o.aux; if (s == 0) tcc_ir_set_src1(ir, i, rep); else if (s == 1) @@ -419,7 +466,6 @@ int tcc_ir_opt_ptr_local_fwd(TCCIRState *ir) tcc_free(tmpl_ok); tcc_free(tmpl); tcc_free(ptr_target); - tcc_free(dc); return changes; } diff --git a/source/opt/flat/memory/sl_forward.c b/source/opt/flat/memory/sl_forward.c index e26c8896d..05d11ed4d 100644 --- a/source/opt/flat/memory/sl_forward.c +++ b/source/opt/flat/memory/sl_forward.c @@ -180,6 +180,59 @@ static int sl_fwd_narrow_demand_only(TCCIRState *ir, int32_t target_vr, int star return found_use ? 1 : 0; } +/* Are the bits of `op` at or above `bits` already zero, so that a store that + * truncated it to `bits` stored the value whole? + * + * NOT decided from the operand's own btype: a STORE's source operand carries + * the STORE's width, so a byte store of a word makes its source look like a + * byte. It is the value's DEFINITION that has to say so -- a mask by a subset + * of `bits`, a shift that leaves at most that many bits, or a narrower + * unsigned load. */ +static int sl_fwd_value_within_bits(TCCIRState *ir, IROperand op, int bits) +{ + if (bits <= 0 || bits >= 32 || irop_is_64bit(op)) + return 0; + const uint32_t lim = (1u << bits) - 1; + if (irop_is_immediate(op) && !op.is_sym) + return ((uint64_t)(uint32_t)(int32_t)irop_get_imm64_ex(ir, op) & ~(uint64_t)lim) == 0; + int32_t vr = irop_get_vreg(op); + if (vr < 0 || op.is_lval) + return 0; + IRQuadCompact *def = NULL; + for (int i = 0; i < ir->next_instruction_index; i++) + { + IRQuadCompact *q = &ir->compact_instructions[i]; + if (q->op == TCCIR_OP_NOP || !irop_config[q->op].has_dest) + continue; + IROperand d = tcc_ir_op_get_dest(ir, q); + if (d.is_lval || irop_get_vreg(d) != vr) + continue; + if (def) + return 0; /* more than one definition */ + def = q; + } + if (!def) + return 0; + if (def->op == TCCIR_OP_AND) + { + IROperand a = tcc_ir_op_get_src1(ir, def), b = tcc_ir_op_get_src2(ir, def); + if (irop_is_immediate(b) && !b.is_sym && + ((uint64_t)(uint32_t)(int32_t)irop_get_imm64_ex(ir, b) & ~(uint64_t)lim) == 0) + return 1; + return irop_is_immediate(a) && !a.is_sym && + ((uint64_t)(uint32_t)(int32_t)irop_get_imm64_ex(ir, a) & ~(uint64_t)lim) == 0; + } + if (def->op == TCCIR_OP_LOAD || def->op == TCCIR_OP_ASSIGN) + { + IROperand a = tcc_ir_op_get_src1(ir, def); + if (irop_is_64bit(a) || !a.is_unsigned) + return 0; + int ab = irop_get_btype(a) == IROP_BTYPE_INT8 ? 8 : irop_get_btype(a) == IROP_BTYPE_INT16 ? 16 : 32; + return ab <= bits; + } + return 0; +} + /* Returns 1 iff EVERY read of value_vr anywhere in the function is "self-narrowing": * an AND with an immediate mask that fits in load_bits, or a 32-bit SHL by at least * (32 - load_bits). Both produce a result that does not depend on value_vr's bits at @@ -1168,9 +1221,13 @@ static int tcc_ir_opt_sl_forward__timed(TCCIRState *ir) /* A volatile read must be performed, not answered from a tracked store * (`volatile int x; x = 1; x = 2;` — the reads are what keep both - * stores alive). */ + * stores alive). + * + * `goto`, not `continue`: giving up on forwarding INTO this load must + * not also skip the bookkeeping its DESTINATION owes (see + * skip_load_fwd). */ if (tcc_ir_access_is_volatile(ir, src1)) - continue; + goto skip_load_fwd; /* Only forward for stack locals or LEA-resolvable non-locals. */ int load_via_lea = 0; @@ -1200,7 +1257,7 @@ static int tcc_ir_opt_sl_forward__timed(TCCIRState *ir) { LOG_SL_FWD("LOAD@i=%d SKIP: src1 not is_local and not is_lval", i); } - continue; + goto skip_load_fwd; } /* Skip if address taken (may alias through a pointer). */ @@ -1210,7 +1267,7 @@ static int tcc_ir_opt_sl_forward__timed(TCCIRState *ir) if (interval && interval->addrtaken) { LOG_SL_FWD("LOAD@i=%d SKIP: addr vreg:%d addrtaken", i, addr_vr); - continue; + goto skip_load_fwd; } } @@ -1460,6 +1517,41 @@ static int tcc_ir_opt_sl_forward__timed(TCCIRState *ir) break; } } + /* The same when a use needs the narrow value itself: forward the + * stored value zero-extended, `dest <- value AND #mask`. Unsigned + * loads only (every such one in Zig's C: a struct of at most a + * word returned in r0, an error union `{ uint16_t error; }`, is + * stored to its return slot and read back as its field). The + * bits of r0 above such a result are unspecified (AAPCS), so the + * mask is needed anyway. */ + if (store_bits > 0 && load_bits > 0 && store_bits > load_bits && store_bits <= 32 && + (e->store_btype == IROP_BTYPE_INT32 || e->store_btype == IROP_BTYPE_INT16) && + (src1.btype == IROP_BTYPE_INT16 || src1.btype == IROP_BTYPE_INT8) && src1.is_unsigned && + q->op == TCCIR_OP_LOAD && irop_get_vreg(e->stored_value) >= 0 && + !irop_op_is_lval(e->stored_value) && irop_get_vreg(tcc_ir_op_get_dest(ir, q)) >= 0) + { + LOG_SL_FWD("LOAD@i=%d FORWARD-NARROW-MASK: store@i=%d store_bits=%d load_bits=%d", i, + e->instruction_idx, store_bits, load_bits); + IROperand load_dest = tcc_ir_op_get_dest(ir, q); + IROperand value = e->stored_value; + tcc_ir_pool_ensure(ir, 3); + int base = ir->iroperand_pool_count; + tcc_ir_pool_add(ir, load_dest); + tcc_ir_pool_add(ir, value); + tcc_ir_pool_add(ir, irop_make_imm32(-1, (int32_t)((1u << load_bits) - 1), IROP_BTYPE_INT32)); + q->op = TCCIR_OP_AND; + q->operand_base = base; + if (fwd_store_count < SL_FWD_MAX_DEAD_STORES && !e->addr_addrtaken && + irop_get_vreg(tcc_ir_op_get_dest(ir, &ir->compact_instructions[e->instruction_idx])) < 0) + { + fwd_stores[fwd_store_count].store_idx = e->instruction_idx; + fwd_stores[fwd_store_count].offset = e->local_offset; + fwd_stores[fwd_store_count].sym = e->local_sym; + fwd_store_count++; + } + changes++; + break; + } LOG_SL_FWD("LOAD@i=%d REJECT width: store btype=%d vs load btype=%d (store at i=%d)", i, (int)e->store_btype, (int)src1.btype, e->instruction_idx); rejected_width++; @@ -1532,6 +1624,49 @@ static int tcc_ir_opt_sl_forward__timed(TCCIRState *ir) } } + /* Sub-32-bit same-width forward of a VALUE: the store truncated it + * to the field's width, so what the load reads is the low bits -- + * handing on the register whole passes bits the store dropped. The + * frontend truncates ahead of a narrow store to a LOCAL but not to a + * struct FIELD (`s.a = x` stores x and lets the width cut it), so + * `s.a = x + i; use(s.a)` read x + i unmasked from -O1 up. Forward + * it masked instead. A signed field would need a sign-extend, and + * an ASSIGN has no room to emit one, so it keeps its load. */ + if (!irop_is_immediate(e->stored_value) && + (src1.btype == IROP_BTYPE_INT8 || src1.btype == IROP_BTYPE_INT16) && + !sl_fwd_value_within_bits(ir, e->stored_value, src1.btype == IROP_BTYPE_INT8 ? 8 : 16)) + { + const int nbits = src1.btype == IROP_BTYPE_INT8 ? 8 : 16; + if (q->op != TCCIR_OP_LOAD || !src1.is_unsigned || irop_get_vreg(e->stored_value) < 0 || + irop_op_is_lval(e->stored_value) || irop_get_vreg(tcc_ir_op_get_dest(ir, q)) < 0) + { + LOG_SL_FWD("LOAD@i=%d REJECT narrow store not truncated: store@i=%d bits=%d", i, + e->instruction_idx, nbits); + rejected_width++; + continue; + } + LOG_SL_FWD("LOAD@i=%d FORWARD-SAME-WIDTH-MASK: store@i=%d bits=%d", i, e->instruction_idx, nbits); + IROperand load_dest = tcc_ir_op_get_dest(ir, q); + IROperand value = e->stored_value; + tcc_ir_pool_ensure(ir, 3); + int base = ir->iroperand_pool_count; + tcc_ir_pool_add(ir, load_dest); + tcc_ir_pool_add(ir, value); + tcc_ir_pool_add(ir, irop_make_imm32(-1, (int32_t)((1u << nbits) - 1), IROP_BTYPE_INT32)); + q->op = TCCIR_OP_AND; + q->operand_base = base; + if (fwd_store_count < SL_FWD_MAX_DEAD_STORES && !e->addr_addrtaken && + irop_get_vreg(tcc_ir_op_get_dest(ir, &ir->compact_instructions[e->instruction_idx])) < 0) + { + fwd_stores[fwd_store_count].store_idx = e->instruction_idx; + fwd_stores[fwd_store_count].offset = e->local_offset; + fwd_stores[fwd_store_count].sym = e->local_sym; + fwd_store_count++; + } + changes++; + break; + } + /* Stale check: skip if the LOAD's address vreg was written after the store (unless LEA-resolved). */ if (!load_via_lea && addr_vr >= 0) { @@ -1827,6 +1962,7 @@ static int tcc_ir_opt_sl_forward__timed(TCCIRState *ir) (const void *)addr_sym, (long long)addr_offset, matched_any, rejected_width, rejected_stale_tracker, rejected_invalid); } + skip_load_fwd:; } /* LOAD_INDEXED with LEA-mapped base + const index: resolve to base_offset+imm and forward. */ else if (q->op == TCCIR_OP_LOAD_INDEXED) @@ -2944,6 +3080,12 @@ static int tcc_ir_opt_sl_forward__timed(TCCIRState *ir) /* Check if the store was already NOP'd (e.g. by a later store overwrite) */ if (ir->compact_instructions[store_idx].op == TCCIR_OP_NOP) continue; + /* Bytes the store writes: a reader starting inside them still reads it + * (the high half of a word, forwarded only in its low half). */ + int store_w = ir_opt_store_btype_size_bytes( + irop_get_btype(tcc_ir_op_get_dest(ir, &ir->compact_instructions[store_idx]))); + if (store_w <= 0) + store_w = 1024; for (int j = 0; j < n && !still_read; j++) { @@ -2970,6 +3112,8 @@ static int tcc_ir_opt_sl_forward__timed(TCCIRState *ir) if ((op).is_local && irop_get_sym_ex(ir, (op)) == sym) \ { \ int64_t _roff = irop_get_imm64_ex(ir, (op)); \ + if (_roff > off && _roff < off + store_w) \ + still_read = 1; \ if (_roff != off && _roff <= off) \ { \ int _w = 4; \ diff --git a/source/opt/flat/memory/slot_const_store_fold.c b/source/opt/flat/memory/slot_const_store_fold.c new file mode 100644 index 000000000..a83f140ef --- /dev/null +++ b/source/opt/flat/memory/slot_const_store_fold.c @@ -0,0 +1,209 @@ +/* + * TCC IR - Fold constant partial stores into a frame word (flat, pre-SSA) + * + * Copyright (c) 2026 Mateusz Stadnik + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation. + */ + +#define USING_GLOBALS + +#include "ir.h" +#include "opt.h" +#include "opt_engine.h" + +/* A compound literal or initializer zero-fills its local, then stores each + * field: + * + * StackLoc[-28] <-- #0 StackLoc[-16] <-- #0 + * StackLoc[-28] <-- #1 (u8) StackLoc[-16] <-- #2 (u8) StackLoc[-15] <-- #32 (u8) + * + * When the local was read back word by word (the old by-reference argument + * copy), store-to-load forwarding folded each word into one constant. Passed + * by value, it is read as a whole by the call, so nothing did: every bool or + * byte field cost its own STRB.W. Within a block, a constant byte/halfword + * store into a word an earlier constant word store fully set -- with nothing + * reading the word in between -- is folded into that store's immediate. */ + +#define SCSF_MAX 64 + +typedef struct ScsfWord +{ + int32_t off; /* word-aligned frame offset */ + uint32_t val; /* the word's bytes */ + int store; /* the word store holding them */ +} ScsfWord; + +static int scsf_width(int btype) +{ + switch (btype) + { + case IROP_BTYPE_INT8: + return 1; + case IROP_BTYPE_INT16: + return 2; + case IROP_BTYPE_INT32: + case IROP_BTYPE_FLOAT32: + return 4; + case IROP_BTYPE_INT64: + case IROP_BTYPE_FLOAT64: + return 8; + default: + return -1; /* struct and the like: unknown extent */ + } +} + +/* A concrete slot of this function's own frame. */ +static int scsf_slot(IROperand op, int32_t *off) +{ + if (irop_is_none(op) || irop_get_tag(op) != IROP_TAG_STACKOFF || !op.is_local || op.is_param || op.is_llocal || + irop_get_vreg(op) >= 0) + return 0; + *off = irop_get_stack_offset(op); + return 1; +} + +static int scsf_nops(int op) +{ + int n = irop_config[op].has_dest + irop_config[op].has_src1 + irop_config[op].has_src2; + if (op == TCCIR_OP_MLA || op == TCCIR_OP_LOAD_INDEXED || op == TCCIR_OP_STORE_INDEXED || op == TCCIR_OP_SELECT) + n++; + return n; +} + +static void scsf_forget(ScsfWord *w, int *nw, int32_t lo, int32_t hi) +{ + for (int k = 0; k < *nw;) + { + if (w[k].off < hi && w[k].off + 4 > lo) + w[k] = w[--*nw]; + else + k++; + } +} + +static int scsf_find(ScsfWord *w, int nw, int32_t off) +{ + for (int k = 0; k < nw; k++) + if (w[k].off == off) + return k; + return -1; +} + +int tcc_ir_opt_slot_const_store_fold(TCCIRState *ir) +{ + if (!ir) + return 0; + const int n = ir->next_instruction_index; + ScsfWord w[SCSF_MAX]; + int nw = 0, changes = 0; + + for (int i = 0; i < n; i++) + { + IRQuadCompact *q = &ir->compact_instructions[i]; + if (q->is_jump_target) + nw = 0; + if (q->op == TCCIR_OP_NOP) + continue; + + if (q->op == TCCIR_OP_STORE) + { + IROperand d = tcc_ir_op_get_dest(ir, q), v = tcc_ir_op_get_src1(ir, q); + int32_t off; + if (scsf_slot(d, &off) && d.is_lval && irop_get_tag(v) == IROP_TAG_IMM32 && + !tcc_ir_access_is_volatile(ir, d)) + { + const int width = scsf_width(d.btype); + const uint32_t imm = (uint32_t)irop_get_imm32(v); + if (width == 4 && d.btype == IROP_BTYPE_INT32 && !(off & 3)) + { + int k = scsf_find(w, nw, off); + if (k < 0 && nw < SCSF_MAX) + k = nw++; + if (k >= 0) + w[k] = (ScsfWord){.off = off, .val = imm, .store = i}; + continue; + } + if ((width == 1 || (width == 2 && !(off & 1)))) + { + int k = scsf_find(w, nw, off & ~3); + if (k >= 0) + { + const int sh = (off & 3) * 8; + const uint32_t mask = (width == 1 ? 0xffu : 0xffffu) << sh; + w[k].val = (w[k].val & ~mask) | ((imm << sh) & mask); + IRQuadCompact *sq = &ir->compact_instructions[w[k].store]; + IROperand sv = tcc_ir_op_get_src1(ir, sq); + sv.u.imm32 = (int32_t)w[k].val; + tcc_ir_op_set_src1(ir, sq, sv); + q->op = TCCIR_OP_NOP; + changes++; + continue; + } + } + } + } + + /* Control transfer, a call, or a store through a pointer: forget all. */ + switch (q->op) + { + case TCCIR_OP_JUMP: + case TCCIR_OP_JUMPIF: + case TCCIR_OP_IJUMP: + case TCCIR_OP_SWITCH_TABLE: + case TCCIR_OP_SWITCH_LOAD: + case TCCIR_OP_RETURNVALUE: + case TCCIR_OP_RETURNVOID: + case TCCIR_OP_FUNCCALLVAL: + case TCCIR_OP_FUNCCALLVOID: + case TCCIR_OP_SETJMP: + case TCCIR_OP_NL_SETJMP: + case TCCIR_OP_INLINE_ASM: + case TCCIR_OP_ASM_INPUT: + case TCCIR_OP_ASM_OUTPUT: + case TCCIR_OP_VLA_ALLOC: + case TCCIR_OP_VLA_SP_SAVE: + case TCCIR_OP_VLA_SP_RESTORE: + case TCCIR_OP_STORE_INDEXED: + case TCCIR_OP_STORE_POSTINC: + case TCCIR_OP_BLOCK_COPY: + nw = 0; + continue; + default: + break; + } + if (q->op == TCCIR_OP_STORE) + { + int32_t off; + if (!scsf_slot(tcc_ir_op_get_dest(ir, q), &off)) + { + nw = 0; + continue; + } + } + + /* Any other reference to a tracked word -- a read, its address, a + * non-constant or wider store -- ends what is known about it. */ + const int nops = scsf_nops(q->op); + for (int k = 0; k < nops && nw; k++) + { + IROperand op = ir->iroperand_pool[q->operand_base + k]; + int32_t off; + if (!scsf_slot(op, &off)) + continue; + int width = op.is_lval ? scsf_width(op.btype) : -1; + if (width < 0) + nw = 0; /* a whole struct, or an address: its extent is unknown */ + else + scsf_forget(w, &nw, off, off + width); + } + } + return changes; +} + +int tcc_ir_opt_slot_const_store_fold_ex(IROptCtx *ctx) +{ + return tcc_ir_opt_slot_const_store_fold(ctx->ir); +} diff --git a/source/opt/flat/memory/small_memset_to_store.c b/source/opt/flat/memory/small_memset_to_store.c index bdeb70204..2f2734da8 100644 --- a/source/opt/flat/memory/small_memset_to_store.c +++ b/source/opt/flat/memory/small_memset_to_store.c @@ -129,8 +129,10 @@ int tcc_ir_opt_small_memset_to_store(TCCIRState *ir) ir_opt_nop_call_params(ir, i); /* STORE pool layout is [dest, src1]. */ + /* A memset is no volatile access, whatever the object's type. */ IROperand st_dest0 = irop_make_stackoff(-1, chunk_off[0], /*is_lval*/ 1, /*is_llocal*/ 0, /*is_param*/ 0, chunk_btype[0]); + st_dest0.aux |= IROP_AUX_NONVOLATILE; IROperand st_src0 = irop_make_imm32(-1, 0, chunk_btype[0]); int pool_base0 = tcc_ir_iroperand_pool_add(ir, st_dest0); tcc_ir_iroperand_pool_add(ir, st_src0); @@ -141,6 +143,7 @@ int tcc_ir_opt_small_memset_to_store(TCCIRState *ir) { IROperand st_dest1 = irop_make_stackoff(-1, chunk_off[1], /*is_lval*/ 1, /*is_llocal*/ 0, /*is_param*/ 0, chunk_btype[1]); + st_dest1.aux |= IROP_AUX_NONVOLATILE; IROperand st_src1 = irop_make_imm32(-1, 0, chunk_btype[1]); int pool_base1 = tcc_ir_iroperand_pool_add(ir, st_dest1); tcc_ir_iroperand_pool_add(ir, st_src1); diff --git a/source/opt/flat/memory/sra.c b/source/opt/flat/memory/sra.c new file mode 100644 index 000000000..5d8660379 --- /dev/null +++ b/source/opt/flat/memory/sra.c @@ -0,0 +1,2886 @@ +/* + * TinyCC - Tiny C Compiler + * + * Scalar replacement of small frame objects. + * + * A struct local the frontend put in the frame stays there: every field read + * is a load from the stack and every field write a store, and each call that + * takes it by value copies it first. The Zig C backend builds everything out + * of such locals -- `Type`, `Value`, `InternPool.Index` are one-word structs + * -- so most of the Zig compiler's stack traffic is to objects whose address + * never leaves the function. + * + * An object qualifies when every reference to it is a LOAD or ASSIGN reading + * an 8-, 16- or 32-bit field of it, a STORE writing one, or the whole of a + * four-byte, float-free object passed to or returned from a call -- which the + * AAPCS moves in a core register exactly like an int -- and the fields it is + * accessed as do not overlap: each byte offset is always accessed with one + * width, and a narrow one with one signedness. A field may also be reached + * through an address TEMP (`T <- &s; *(T + 4) = x`, the frontend's way to a + * member) when that TEMP is used for nothing but such accesses. Each field + * then becomes a fresh VAR, which SSA renames like any other scalar (a narrow + * one like a `short` local), and the address arithmetic goes. A word-aligned + * block copy to or from another pointer, and a struct-returning call's result + * buffer, access it whole and become field copies (SraWhole). Any other + * reference -- an address that escapes, a 64-bit or overlapping access -- + * leaves the object in memory, untouched. + * + * The rule is kept narrow on purpose, and nothing is rewritten for an object + * that stays in memory. Accepting a field in any operand (an ALU op reading + * it straight from the frame) promoted more objects but made the Zig compiler + * larger, and so did folding `*(&obj + k)` into direct frame accesses for every + * object: the extra promoted values mostly end up spilled, with the spill + * slots nearest sp, and a direct access loses the short base-register form + * and some later forwarding. + * + * Runs on the flat IR just before SSA construction. + */ + +#include + +#include "ir.h" +#include "cfg.h" +#include "opt.h" +#include "opt_utils.h" + +#define SRA_MAX_BYTES 36 /* 32 bytes, plus the guard word frame_record adds */ + +/* TCC_SRA_PTRVAR=0 keeps objects reached through pointer VARs in memory. */ +TCC_DBG_ENV_INT(sra_ptrvar, "TCC_SRA_PTRVAR", 1) +/* TCC_SRA_PTRVAR_DF=0 follows pointer VARs within a block only. */ +TCC_DBG_ENV_INT(sra_ptrvar_df, "TCC_SRA_PTRVAR_DF", 1) +/* TCC_SRA_PARAM_MAX: the largest struct argument split into words, in bytes. */ +TCC_DBG_ENV_INT(sra_param_max, "TCC_SRA_PARAM_MAX", SRA_MAX_BYTES - 4) +/* TCC_SRA_NARROW_IDX_OFF=1 keeps objects with a narrow indexed store in memory. */ +TCC_DBG_ENV_FLAG(sra_narrow_idx_off, "TCC_SRA_NARROW_IDX_OFF") +/* TCC_SRA_MIXED_OFF=1 keeps objects whose narrow fields are also read wider in memory. */ +TCC_DBG_ENV_FLAG(sra_mixed_off, "TCC_SRA_MIXED_OFF") +/* TCC_SRA_COMBINE_OFF=1 keeps objects whose wide loads span several narrow fields in memory. */ +TCC_DBG_ENV_FLAG(sra_combine_off, "TCC_SRA_COMBINE_OFF") +/* TCC_SRA_PARTIAL_OFF=1 keeps objects with a field updated or read in part in memory. */ +TCC_DBG_ENV_FLAG(sra_partial_off, "TCC_SRA_PARTIAL_OFF") +/* TCC_SRA_PARTIAL_GATE: 0 always; 1 only objects a call copies whole; 2 (default) those, and objects whose + * partial stores are all constants. */ +TCC_DBG_ENV_INT(sra_partial_gate, "TCC_SRA_PARTIAL_GATE", 2) +/* TCC_SRA_BFI=0 inserts with AND/SHL/OR instead of BFI. */ +TCC_DBG_ENV_INT(sra_bfi, "TCC_SRA_BFI", 1) +/* TCC_SRA_PAD_OFF=1 keeps a partial store's other bytes even when they are padding. */ +TCC_DBG_ENV_FLAG(sra_pad_off, "TCC_SRA_PAD_OFF") +/* TCC_SRA_PAIR_OFF=1 keeps one object of a copy between two promotable ones in memory. */ +TCC_DBG_ENV_FLAG(sra_pair_off, "TCC_SRA_PAIR_OFF") +/* TCC_SRA_PARTIAL_ONLY=,,...: only objects at these frame offsets may use partial fields + * (bisecting). Not a tuning knob. */ +static int sra_partial_only(int lo) +{ + static const char *list; + static int init; + if (!init) + init = 1, list = getenv("TCC_SRA_PARTIAL_ONLY"); + if (!list) + return 1; + char buf[16]; + int n = snprintf(buf, sizeof buf, "%d", lo); + for (const char *p = list; (p = strstr(p, buf)) != NULL; p++) + if ((p == list || p[-1] == ',') && (p[n] == 0 || p[n] == ',')) + return 1; + return 0; +} +/* TCC_SRA_VAR_BUDGET: past this many field VARs in one function SSA promotes none of them + * (ssa.c TCC_SSA_PROM_SRA_MAX), and a field updated in part or copied VAR to VAR costs more in memory + * than the slot access it replaced; such objects then keep their memory. */ +TCC_DBG_ENV_INT(sra_var_budget, "TCC_SRA_VAR_BUDGET", 2000) + +typedef struct +{ + int lo, hi; /* the object's extent, from tcc_ir_frame_object_at */ + int refs; /* every operand a live instruction names it with */ + int ok_refs; /* those a field VAR can replace */ + int bad; /* two accesses overlap, or one byte offset is read two ways */ + uint8_t width[SRA_MAX_BYTES]; /* per field start: bytes accessed, 0 = none */ + uint8_t sign[SRA_MAX_BYTES]; /* per narrow field: 1 signed, 2 unsigned */ + uint8_t has64[SRA_MAX_BYTES]; /* a 64-bit access starts here */ + uint8_t split[SRA_MAX_BYTES]; /* a 64-bit field kept as two word VARs */ + int32_t var[SRA_MAX_BYTES]; + int param_words; /* words of it a call receives by value (sra_struct_param_words) */ + int field_refs; /* accesses of single fields */ + uint8_t whole_w; /* a call writes a range of it whole (SraWhole) */ + uint8_t whole_r; /* a call reads a range of it whole */ + const char *why; /* TCC_SRA_DBG: the first reason the unit keeps its memory */ + int why_op; + int why_at; /* TCC_SRA_DBG: that reference's instruction, or -1 */ + uint8_t acc[SRA_MAX_BYTES]; /* per offset: widths loaded (bits 0-2), stored (bits 3-5) */ + uint8_t sgn[SRA_MAX_BYTES]; /* per offset: narrow accesses signed (1) / unsigned (2), bytes then halves */ + uint8_t nofit[SRA_MAX_BYTES]; /* per offset: widths of narrow stores whose value may not fit the width */ + uint8_t stvar[SRA_MAX_BYTES]; /* per offset: widths of narrow stores of a non-constant */ + uint8_t partial; /* some access is a part of a wider field */ + uint8_t pmask[SRA_MAX_BYTES]; /* per field start: a narrow field stored in part -- its VAR's upper + * bits are not zero, so a read of the whole field masks them */ + uint16_t nvst1[SRA_MAX_BYTES]; /* per offset: byte stores of a non-constant */ + uint16_t nvst2[SRA_MAX_BYTES]; /* per offset: halfword stores of a non-constant */ + int nloads; /* field loads */ + int npart; /* partial stores of a non-constant (each a read-modify-write) */ + uint8_t fstart[SRA_MAX_BYTES]; /* per byte: the start of the field holding it, 0xFF none (sra_index_fields) */ + uint64_t pad; /* bytes no member of its type covers (tcc_ir_frame_padding) */ +} SraUnit; + +/* TCC_SRA_DBG: one line per frame object SRA leaves in memory, with the first + * reference that kept it there. */ +TCC_DBG_ENV_FLAG(sra_dbg, "TCC_SRA_DBG") + +static int sra_dbg_at = -1; +static void sra_why(SraUnit *u, const char *why, int op) +{ + if (!u->why) + u->why = why, u->why_op = op, u->why_at = sra_dbg_at; +} + +/* Why operand `o` in slot s of an instruction with opcode `op` is no field. */ +static const char *sra_ref_kind(int op, int s, IROperand o) +{ + int bt = irop_get_btype(o); + if (bt == IROP_BTYPE_STRUCT) + return op == TCCIR_OP_FUNCPARAMVAL ? "struct-param" + : op == TCCIR_OP_FUNCCALLVAL ? "struct-callret" + : op == TCCIR_OP_RETURNVALUE ? "struct-return" + : op == TCCIR_OP_BLOCK_COPY ? "block-copy" + : "struct-other"; + if (op == TCCIR_OP_LEA || !o.is_lval) + return "address"; + if (irop_is_64bit(o)) + return op == TCCIR_OP_LOAD || op == TCCIR_OP_ASSIGN || op == TCCIR_OP_STORE ? "i64-ldst" : "i64-alu"; + if (op == TCCIR_OP_LOAD || op == TCCIR_OP_ASSIGN || op == TCCIR_OP_STORE) + return s == 0 ? "ldst-dest?" : "ldst-src?"; + return "alu-operand"; +} + +/* A frame slot the backend addresses directly, the same test frame.c uses. */ +static int sra_concrete(IROperand op, int *off) +{ + if (irop_is_none(op) || irop_get_vreg(op) >= 0 || op.is_param) + return 0; + int tag = irop_get_tag(op); + if (tag != IROP_TAG_STACKOFF && !(tag == IROP_TAG_VREG && (op.is_local || op.is_llocal))) + return 0; + *off = irop_get_stack_offset(op); + return 1; +} + +static int sra_type_has_float(CType *t, int depth) +{ + if (depth > 8) + return 1; + int bt = t->t & VT_BTYPE; + if (bt == VT_FLOAT || bt == VT_DOUBLE || bt == VT_LDOUBLE) + return 1; + if ((t->t & VT_ARRAY) && t->ref) + return sra_type_has_float(&t->ref->type, depth + 1); + if (bt == VT_STRUCT && t->ref) + for (Sym *f = t->ref->next; f; f = f->next) + if (sra_type_has_float(&f->type, depth + 1)) + return 1; + return 0; +} + +/* A four-byte struct operand that travels in a core register like an int. */ +static int sra_struct_is_word(IROperand op) +{ + if (irop_get_btype(op) != IROP_BTYPE_STRUCT || op.is_llocal || op.is_complex) + return 0; + CType *ct = irop_get_ctype(op); + if (!ct) + return 0; + int align; + return type_size(ct, &align) == 4 && align <= 4 && !sra_type_has_float(ct, 0); +} + +/* A struct passed BY VALUE whose words can come from field VARs: whole words, + * 8 bytes up to TCC_SRA_PARAM_MAX, float-free, AAPCS alignment <= 4. Its + * words then occupy exactly the registers / stack words N consecutive word + * arguments would (arm_aapcs.c: a composite starts at the NCRN and straddles + * r3 onto the stack contiguously, like scalars; only an 8-aligned one skips to + * an even register). The number of words, or 0. */ +static int sra_struct_param_words(IROperand op) +{ + if (irop_get_btype(op) != IROP_BTYPE_STRUCT || !op.is_lval || op.is_llocal || op.is_complex) + return 0; + CType *ct = irop_get_ctype(op); + if (!ct) + return 0; + int align, size = type_size(ct, &align); + if (size < 8 || size > sra_param_max() || (size & 3) || irop_aapcs_alignment(op) > 4 || align > 4 || + sra_type_has_float(ct, 0)) + return 0; + return size / 4; +} + +static int sra_btype_bytes(int bt) +{ + return bt == IROP_BTYPE_INT32 ? 4 : bt == IROP_BTYPE_INT16 ? 2 : bt == IROP_BTYPE_INT8 ? 1 : 0; +} + +/* Record a `bytes`-wide access at `off` of the unit, a store or a load; the + * fields are laid out once every access has been seen (sra_resolve_fields). */ +static void sra_note_field(SraUnit *u, int off, int bytes, int is_unsigned, int is_store) +{ + int at = off - u->lo; + u->field_refs++; + if (at < 0 || at + bytes > SRA_MAX_BYTES || at + bytes > u->hi - u->lo || (at & (bytes - 1))) + { + u->bad = 1; + sra_why(u, at + bytes > SRA_MAX_BYTES ? "too-big" : "misaligned", 0); + return; + } + u->acc[at] |= bytes << (is_store ? 3 : 0); + if (bytes < 4) + u->sgn[at] |= (is_unsigned ? 2 : 1) << (bytes == 1 ? 0 : 2); + if (!is_store) + u->nloads++; +} + +/* The field starting at each accessed byte. Accesses all of one width make a + * field of it. So do stores all of one width with loads of other widths: + * - narrower loads read the field's low bytes (sra_extract); + * - wider ones, of a narrow field -- a 2-byte struct returned as a word, an + * optional `{ payload; is_null; }` of bytes returned the same way -- read + * its VAR, ORed with those of the other narrow fields they cover shifted + * into place (sra_combine). No whole-range call may touch the object: + * then the bytes no field covers are never written, so they hold nothing + * a read could depend on, and zero bits (the VARs hold their fields + * zero-extended) serve for them. + * Any other mix -- stores at two widths at one offset (an error union whose + * `u16 error` is written by itself and copied as a word), an access that + * starts inside another's bytes, a result buffer read at two widths -- makes + * the WIDEST access at the offset the field, and every access it does not + * match a PARTIAL one of that field's VAR: a narrower load extracts its bytes + * (a shift and a mask), a narrower store inserts them (mask, shift, OR -- + * one BFI once fused), each such store a fresh definition of the VAR. The + * bytes an access starts in must all be inside the field. + * A signed or two-way-signed narrow field keeps the object in memory. */ +/* The widths (bit 1 = byte, bit 2 = halfword) of a store at `b` into the word + * field at `at` that leave only padding unwritten: sra_partial_store_padded + * lowers them to a move, so they are no read-modify-write. */ +static int sra_padded_widths(const SraUnit *u, int at, int w, int b) +{ + int ws = 0; + if (w != 4 || at + 4 > 64) + return 0; + for (int sw = 1; sw <= 2; sw <<= 1) + { + int ok = b + sw <= at + 4; + for (int x = at; ok && x < at + 4; x++) + if ((x < b || x >= b + sw) && !((u->pad >> x) & 1)) + ok = 0; + if (ok) + ws |= sw; + } + return ws; +} + +static void sra_resolve_fields(SraUnit *u, int whole) +{ + uint8_t inner[SRA_MAX_BYTES] = {0}; /* bytes inside a field, accessed in part */ + for (int at = 0; at < SRA_MAX_BYTES && !u->bad; at++) + { + int ld = u->acc[at] & 7, st = u->acc[at] >> 3, all = ld | st; + if (!all || inner[at]) + continue; + int W = all & 4 ? 4 : all & 2 ? 2 : 1; + int w = W, combine = 0, partial = 0; + if (all & (all - 1)) + { + w = st; + int ok = st && !(st & (st - 1)) && !sra_mixed_off(); + if (ok && (ld & ~(2 * st - 1))) + { + /* Wider loads: in the bytes they add, nothing but narrow fields of + * their own, stored and loaded at one width (sra_combine). */ + int top = ld & 4 ? 4 : 2; + ok = !(ld & (st - 1)) && !whole; + for (int b = at + 1; ok && b < at + top; b++) + { + int fb = u->acc[b] >> 3; + ok = !u->has64[b] && (!u->acc[b] || (fb && !(fb & (fb - 1)) && !(u->acc[b] & 7 & ~fb) && + b + fb <= at + top && !sra_combine_off())); + } + combine = ok; + } + else + ok = 0; + if (!combine) + { + w = W; + partial = 1; + } + } + if (!combine) + for (int b = at + 1; b < at + w && b < SRA_MAX_BYTES; b++) + { + int a = u->acc[b], mw = a & 0x24 ? 4 : a & 0x12 ? 2 : a ? 1 : 0; + if (u->has64[b] || b + mw > at + w) + { + u->bad = 1; + sra_why(u, "overlap", 0); + return; + } + if (a) + inner[b] = 1, partial = 1; + } + if (partial) + { + u->partial = 1; + if (w < 4) + { + int pst = st & (w - 1); + for (int b = at + 1; b < at + w && b < SRA_MAX_BYTES; b++) + pst |= u->acc[b] >> 3; + u->pmask[at] = pst != 0; + } + for (int b = at; b < at + w && b < SRA_MAX_BYTES; b++) + { + const int pw = sra_pad_off() ? 0 : sra_padded_widths(u, at, w, b); + u->npart += ((b == at && w == 1) || (pw & 1) ? 0 : u->nvst1[b]) + ((b == at && w == 2) || (pw & 2) ? 0 : u->nvst2[b]); + } + /* Worth it when a call copies the object whole (the copy's loads and + * stores go), or when the partial stores are constants (they fold into + * the field's value); a variable stored in part costs a BFI where a + * narrow store cost the same, and its object's reads were already one + * instruction each. */ + int stores_var = 0; + for (int b = at; b < at + w && b < SRA_MAX_BYTES; b++) + stores_var |= u->stvar[b] & (b == at ? (uint8_t)~w : 0xFF) & + (uint8_t)~(sra_pad_off() ? 0 : sra_padded_widths(u, at, w, b)); + if (sra_partial_off() || !sra_partial_only(u->lo) || (sra_partial_gate() == 1 && !whole) || + (sra_partial_gate() == 2 && !whole && stores_var)) + { + u->bad = 1; + sra_why(u, "mixed-width", 0); + return; + } + } + u->width[at] = w; + if (w < 4) + { + int sg = (u->sgn[at] >> (w == 1 ? 0 : 2)) & 3; + if (sg == 3) + { + u->bad = 1; + sra_why(u, "narrow-sign-mix", 0); + return; + } + u->sign[at] = sg; + /* A narrow field's own-width store is its definition and must fit + * (sra_store_as_def); a partial one is masked. */ + if (u->nofit[at] & w) + { + u->bad = 1; + sra_why(u, "narrow-store", 0); + return; + } + } + } +} + +/* A partial store of a variable is a read-modify-write of the field's VAR -- + * a load, a BFI and a store when the allocator keeps the VAR in memory, as it + * does for a value defined in every arm of a large switch (Zig's + * analyzeBodyInner writes its result's error code in 100+ arms and reads it + * once) -- where the narrow store into the slot was one instruction. Such an + * object pays only when its loads (each a slot access saved) outnumber them. */ +static void sra_partial_cost(SraUnit *u, int whole) +{ + if (!u->bad && u->partial && u->npart > u->nloads + 4 * whole) + { + u->bad = 1; + sra_why(u, "partial-cost", 0); + } +} + +/* fstart: for every byte a field covers, the field's start. */ +static void sra_index_fields(SraUnit *u) +{ + memset(u->fstart, 0xFF, sizeof u->fstart); + for (int at = 0; at < SRA_MAX_BYTES; at++) + for (int b = at; u->width[at] && b < at + u->width[at] && b < SRA_MAX_BYTES; b++) + u->fstart[b] = (uint8_t)at; +} + +/* A 64-bit access at `off`: an 8-byte aligned doubleword of the unit. Whether + * it becomes one 64-bit VAR or two word VARs is settled once every access has + * been seen (sra_resolve_wide). */ +static void sra_note_wide(SraUnit *u, int off) +{ + int at = off - u->lo; + u->field_refs++; + if (at < 0 || at + 8 > SRA_MAX_BYTES || at + 8 > u->hi - u->lo || (at & 7)) + { + u->bad = 1; + sra_why(u, at + 8 > SRA_MAX_BYTES ? "too-big" : "misaligned", 0); + return; + } + u->has64[at] = 1; +} + +/* Lay out the doublewords. One touched only 64 bits at a time is a 64-bit + * field. One ALSO touched a word at a time -- the Zig C backend's zig_u128, + * `{ uint64_t lo, hi; }`, is zero-filled and copied a word at a time and its + * halves read and written whole -- becomes two word fields, and its 64-bit + * accesses are rewritten to pack or split them (sra_rewrite_wide). A narrow + * access inside a doubleword keeps the object in memory. */ +static void sra_resolve_wide(SraUnit *u) +{ + for (int at = 0; at + 8 <= SRA_MAX_BYTES; at += 8) + { + if (!u->has64[at]) + continue; + for (int b = 1; b < 8; b++) + if (b != 4 && u->width[at + b]) + { + u->bad = 1; + sra_why(u, "wide-overlap", 0); + return; + } + int w0 = u->width[at], w1 = u->width[at + 4]; + if ((w0 && w0 != 4) || (w1 && w1 != 4)) + { + u->bad = 1; + sra_why(u, "wide-narrow", 0); + return; + } + if (!w0 && !w1) + u->width[at] = 8; + else + { + u->width[at] = u->width[at + 4] = 4; + u->split[at] = 1; + } + } +} + +/* A narrow field's write has to be a DEFINITION of its VAR, and of the whole + * of it. + * + * SSA only takes a full-width slot STORE as a definition of a variable (ssa.c, + * ssa_store_slot_def_pos); a narrower one updates whatever name is current, in + * place. A field written only by narrow stores would therefore get a VAR that + * is never defined at all, while its reads -- which rename turns into + * whole-register copies -- take that undefined name (struct_byval fuzz seed + * 1962: a `struct { unsigned char a; }` built and passed by value read back 3 + * bytes of whatever was under it at -O2). The frontend's own narrow locals do + * not have this problem: it writes those with ASSIGN. + * + * So a narrow field's STORE is rewritten to `ASSIGN var <- src`, which is one, + * and every access to the field gets a 32-bit operand. That drops the + * truncation the store's width used to do -- the frontend does NOT truncate + * ahead of a narrow store, `s.a = x` stores x whole and lets the width cut it + * -- so the object is promoted only when every stored value ALREADY fits the + * field. There is nowhere to put a mask otherwise: a STORE's operand block + * holds two operands and an AND needs three, and widening it in place would + * write over the next instruction's. + * + * A SIGNED narrow field would need its reads sign-extended, which a plain + * register copy does not do, so those keep their memory too. */ +static int sra_narrow_fields_unsigned(const SraUnit *u) +{ + for (int at = 0; at < SRA_MAX_BYTES; at++) + if (u->width[at] && u->width[at] < 4 && u->sign[at] != 2) + return 0; + return 1; +} + +static int sra_temp_pos(IROperand o, int ntemp) +{ + int32_t vr = irop_get_vreg(o); + if (vr < 0 || TCCIR_DECODE_VREG_TYPE(vr) != TCCIR_VREG_TYPE_TEMP || TCCIR_DECODE_VREG_POSITION(vr) >= ntemp) + return -1; + return TCCIR_DECODE_VREG_POSITION(vr); +} + +/* The mask a `bytes`-wide unsigned field holds. */ +static int32_t sra_field_mask(int bytes) +{ + return bytes == 1 ? 0xff : bytes == 2 ? 0xffff : -1; +} + +/* Is every bit of `op` outside `mask` already zero, so that storing it whole + * is what the narrow store would have stored? `op` must also be a plain + * value: SSA only makes a slot store a definition for one of those + * (ssa_store_slot_def_pos). Only the forms the frontend puts ahead of a + * narrow store are recognised -- a constant, a mask by a subset of it, and a + * value that is itself a narrow unsigned one. `def` maps a TEMP to its only + * defining instruction, or -1 when it has none or several. */ +static int sra_narrow_store_fits(TCCIRState *ir, IRQuadCompact *q, int bytes, + const int32_t *def, int ntemp) +{ + const int32_t mask = sra_field_mask(bytes); + IROperand op = tcc_ir_op_get_src1(ir, q); + if (irop_get_tag(op) == IROP_TAG_IMM32 && !op.is_lval) + return (irop_get_imm32(op) & ~mask) == 0; + if (irop_get_tag(op) != IROP_TAG_VREG || op.is_lval || op.is_llocal) + return 0; + int ob = sra_btype_bytes(irop_get_btype(op)); + if (ob && ob <= bytes && op.is_unsigned) + return 1; + int t = sra_temp_pos(op, ntemp); + if (t < 0 || !def || def[t] < 0) + return 0; + IRQuadCompact *dq = &ir->compact_instructions[def[t]]; + if (dq->op == TCCIR_OP_AND) + { + IROperand a = tcc_ir_op_get_src1(ir, dq), b = tcc_ir_op_get_src2(ir, dq); + return (irop_get_tag(b) == IROP_TAG_IMM32 && !b.is_lval && (irop_get_imm32(b) & ~mask) == 0) || + (irop_get_tag(a) == IROP_TAG_IMM32 && !a.is_lval && (irop_get_imm32(a) & ~mask) == 0); + } + if (dq->op == TCCIR_OP_LOAD || dq->op == TCCIR_OP_ASSIGN) + { + IROperand a = tcc_ir_op_get_src1(ir, dq); + int ab = sra_btype_bytes(irop_get_btype(a)); + return ab && ab <= bytes && a.is_unsigned; + } + return 0; +} + +/* A narrow field's STORE becomes an ASSIGN -- a definition of the VAR -- with + * the same two operands and the VAR widened to 32 bits. */ +static void sra_store_as_def(TCCIRState *ir, int i, IROperand rep) +{ + rep.is_lval = 0; /* a value definition of the VAR, not a write through it */ + ir->compact_instructions[i].op = TCCIR_OP_ASSIGN; + tcc_ir_set_dest(ir, i, rep); +} + +/* No two fields share a byte. */ +static int sra_fields_disjoint(const SraUnit *u) +{ + int end = 0; + for (int at = 0; at < SRA_MAX_BYTES; at++) + { + if (!u->width[at]) + continue; + if (at < end) + return 0; + end = at + u->width[at]; + } + return 1; +} + +static int sra_find_unit(SraUnit *units, int nu, int off) +{ + int a = 0, b = nu - 1; + while (a <= b) + { + int m = (a + b) / 2; + if (off < units[m].lo) + b = m - 1; + else if (off >= units[m].hi) + a = m + 1; + else + return m; + } + return -1; +} + +/* Slot s (0 dest, 1 src1, 2 src2) of an instruction: the width in bytes of the + * field a VAR can stand in for there, or 0. A whole four-byte struct moved + * like an int counts as its 32-bit field at offset 0. */ +static int sra_slot_bytes(TCCIRState *ir, TccIrOp op, int s, IROperand o) +{ + if (!o.is_lval || o.is_llocal || o.is_complex || tcc_ir_access_is_volatile(ir, o)) + return 0; + int bt = irop_get_btype(o); + if (sra_btype_bytes(bt) && + ((s == 1 && (op == TCCIR_OP_LOAD || op == TCCIR_OP_ASSIGN)) || (s == 0 && op == TCCIR_OP_STORE))) + return sra_btype_bytes(bt); + if (bt == IROP_BTYPE_STRUCT && sra_struct_is_word(o) && + ((s == 1 && (op == TCCIR_OP_FUNCPARAMVAL || op == TCCIR_OP_RETURNVALUE)) || + (s == 0 && op == TCCIR_OP_FUNCCALLVAL))) + return 4; + return 0; +} + +/* The ops whose src1 / src2 is a plain 64-bit value read, one a field VAR (or + * a PACK64 of two) can stand in for. */ +static int sra_wide_reader(int op) +{ + switch (op) + { + case TCCIR_OP_ADD: + case TCCIR_OP_SUB: + case TCCIR_OP_MUL: + case TCCIR_OP_AND: + case TCCIR_OP_OR: + case TCCIR_OP_XOR: + case TCCIR_OP_SHL: + case TCCIR_OP_SAR: + case TCCIR_OP_SHR: + case TCCIR_OP_CMP: + case TCCIR_OP_FUNCPARAMVAL: + case TCCIR_OP_RETURNVALUE: + case TCCIR_OP_LOAD: + case TCCIR_OP_ASSIGN: + case TCCIR_OP_STORE: + return 1; + default: + return 0; + } +} + +/* A 64-bit integer access in slot s (0 dest, 1 src1, 2 src2) of q that a field + * can take over: a value read by sra_wide_reader, or a STORE of a plain value + * -- a TEMP or a constant, which the rewrite turns into a definition (or two) + * of the field. Doubles live in VFP registers and are left alone. */ +static int sra_wide_access(TCCIRState *ir, IRQuadCompact *q, int s, IROperand o) +{ + if (!o.is_lval || o.is_llocal || o.is_complex || irop_get_btype(o) != IROP_BTYPE_INT64 || + tcc_ir_access_is_volatile(ir, o)) + return 0; + if (s == 0) + { + if (q->op != TCCIR_OP_STORE) + return 0; + IROperand v = tcc_ir_op_get_src1(ir, q); + int tag = irop_get_tag(v); + if (v.is_lval || v.is_complex) + return 0; + return (tag == IROP_TAG_VREG && irop_get_vreg(v) >= 0) || tag == IROP_TAG_IMM32 || tag == IROP_TAG_I64; + } + return (s == 1 || s == 2) && sra_wide_reader(q->op); +} + +/* The ops that keep a fourth operand at operand_base + 3 (frame.c's list). */ +static int sra_op_has_slot3(int op) +{ + return op == TCCIR_OP_MLA || op == TCCIR_OP_LOAD_INDEXED || op == TCCIR_OP_STORE_INDEXED || op == TCCIR_OP_SELECT; +} + +/* Operand slot s (0-3) of q, or IROP_NONE. */ +static IROperand sra_operand(TCCIRState *ir, IRQuadCompact *q, int s) +{ + if (s == 0 && irop_config[q->op].has_dest) + return tcc_ir_op_get_dest(ir, q); + if (s == 1 && irop_config[q->op].has_src1) + return tcc_ir_op_get_src1(ir, q); + if (s == 2 && irop_config[q->op].has_src2) + return tcc_ir_op_get_src2(ir, q); + if (s == 3 && sra_op_has_slot3(q->op) && q->operand_base + 3 < (uint32_t)ir->iroperand_pool_count) + return ir->iroperand_pool[q->operand_base + 3]; + return IROP_NONE; +} + +static int sra_function_eligible(TCCIRState *ir) +{ + if (tcc_state->do_debug || tcc_bounds_checking(tcc_state)) + return 0; + if (ir->has_static_chain || ir->captured_count > 0 || tcc_state->nb_nested_funcs > 0) + return 0; + if (tcc_ir_calls_returns_twice(ir)) + return 0; + for (int i = 0; i < ir->next_instruction_index; i++) + { + switch (ir->compact_instructions[i].op) + { + case TCCIR_OP_ASM_INPUT: + case TCCIR_OP_INLINE_ASM: + case TCCIR_OP_ASM_OUTPUT: + case TCCIR_OP_SETJMP: + case TCCIR_OP_NL_SETJMP: + case TCCIR_OP_VLA_ALLOC: + case TCCIR_OP_VLA_SP_SAVE: + case TCCIR_OP_VLA_SP_RESTORE: + return 0; + default: + break; + } + } + return 1; +} + +#define SRA_NO_ADDR INT32_MIN + +/* The frame offset a by-value struct argument is read from: its slot, or, + * beyond the 16-bit reach of a struct slot operand, `T***DEREF***` through an + * address TEMP (toff) -- the form tcc_ir_put gives far struct sources. */ +static int sra_param_at(IROperand o, int ntemp, const int32_t *toff, int *off) +{ + if (sra_concrete(o, off)) + return 1; + int t; + if (!o.is_lval || o.is_llocal || o.is_local || irop_get_tag(o) != IROP_TAG_VREG || + (t = sra_temp_pos(o, ntemp)) < 0 || toff[t] == SRA_NO_ADDR) + return 0; + *off = toff[t]; + return 1; +} + +/* Instructions the rewrite of a split doubleword adds: a PACK64 ahead of a + * reader of the 64-bit value, the high half of a split store after it. They + * are collected and spliced in by one pass over the function at the end. */ +typedef struct +{ + int anchor; /* the instruction they belong to */ + int after; /* 0: ahead of it -- a jump to it runs them; 1: after it */ + IRQuadCompact q; +} SraIns; + +typedef struct +{ + SraIns *v; + int n, cap; +} SraInsList; + +/* A new instruction with only the operand slots `op` uses, as codegen reads + * them (write_instr_at_nop's layout). */ +static void sra_ins_add(TCCIRState *ir, SraInsList *l, int anchor, int after, TccIrOp op, IROperand dest, + IROperand src1, IROperand src2) +{ + if (l->n == l->cap) + { + l->cap = l->cap ? 2 * l->cap : 16; + l->v = tcc_realloc(l->v, sizeof(SraIns) * l->cap); + } + SraIns *e = &l->v[l->n++]; + memset(e, 0, sizeof(*e)); + e->anchor = anchor; + e->after = after; + e->q.op = op; + e->q.orig_index = ++ir->max_orig_index; + e->q.line_num = ir->compact_instructions[anchor].line_num; + e->q.operand_base = ir->iroperand_pool_count; + if (irop_config[op].has_dest) + tcc_ir_pool_add(ir, dest); + if (irop_config[op].has_src1) + tcc_ir_pool_add(ir, src1); + if (irop_config[op].has_src2) + tcc_ir_pool_add(ir, src2); + if (sra_op_has_slot3(op)) + tcc_ir_pool_add(ir, irop_make_imm32(-1, 0, IROP_BTYPE_INT32)); /* an indexed access's scale */ +} + +/* Give q a fresh operand block for `op`: the old one may be too short. */ +static void sra_relay(TCCIRState *ir, IRQuadCompact *q, TccIrOp op, IROperand dest, IROperand src1, + IROperand src2) +{ + q->op = op; + q->operand_base = ir->iroperand_pool_count; + if (irop_config[op].has_dest) + tcc_ir_pool_add(ir, dest); + if (irop_config[op].has_src1) + tcc_ir_pool_add(ir, src1); + if (irop_config[op].has_src2) + tcc_ir_pool_add(ir, src2); +} + +/* A `bytes`-wide load of a wider field's low bytes, from its 32-bit VAR `var`: + * `dest <- var AND #mask`, or, sign-extending, `T <- var SHL #s` ahead of it + * and `dest <- T SAR #s`. The load at i becomes the last of them. */ +static void sra_extract(TCCIRState *ir, SraInsList *ins, int i, IROperand var, int bytes, int is_unsigned) +{ + IRQuadCompact *q = &ir->compact_instructions[i]; + IROperand dest = tcc_ir_op_get_dest(ir, q); + if (is_unsigned) + { + sra_relay(ir, q, TCCIR_OP_AND, dest, var, irop_make_imm32(-1, sra_field_mask(bytes), IROP_BTYPE_INT32)); + return; + } + const int sh = 32 - 8 * bytes; + IROperand t = irop_make_vreg(tcc_ir_vreg_alloc_temp(ir), IROP_BTYPE_INT32); + sra_ins_add(ir, ins, i, 0, TCCIR_OP_SHL, t, var, irop_make_imm32(-1, sh, IROP_BTYPE_INT32)); + sra_relay(ir, &ir->compact_instructions[i], TCCIR_OP_SAR, dest, t, irop_make_imm32(-1, sh, IROP_BTYPE_INT32)); +} + +static IROperand sra_field_op(TCCIRState *ir, SraUnit *u, int at, int bt, int is_unsigned); +static IROperand sra_new_temp(TCCIRState *ir, int bt); + +/* A `bytes`-wide load into `dest` of bytes [d, d + bytes) of the field at `fs` + * of u, from its VAR: the bytes shifted down and masked (`dest <- V SHR #8d` + * alone when they are the field's top bytes, `dest <- V AND #mask` alone when + * its bottom ones), or, sign-extending, shifted up to the top and back down + * arithmetically. The load at i becomes the last instruction. */ +static void sra_partial_load(TCCIRState *ir, SraInsList *ins, int i, SraUnit *u, int fs, int d, int bytes, + int is_unsigned, IROperand dest) +{ + IROperand v = sra_field_op(ir, u, fs, IROP_BTYPE_INT32, 1); + IRQuadCompact *q = &ir->compact_instructions[i]; + const int W = u->width[fs]; + if (is_unsigned) + { + if (d == 0) + sra_relay(ir, q, TCCIR_OP_AND, dest, v, irop_make_imm32(-1, sra_field_mask(bytes), IROP_BTYPE_INT32)); + else if (d + bytes == W && W == 4) + sra_relay(ir, q, TCCIR_OP_SHR, dest, v, irop_make_imm32(-1, 8 * d, IROP_BTYPE_INT32)); + else + { + IROperand t = sra_new_temp(ir, IROP_BTYPE_INT32); + sra_ins_add(ir, ins, i, 0, TCCIR_OP_SHR, t, v, irop_make_imm32(-1, 8 * d, IROP_BTYPE_INT32)); + sra_relay(ir, &ir->compact_instructions[i], TCCIR_OP_AND, dest, t, + irop_make_imm32(-1, sra_field_mask(bytes), IROP_BTYPE_INT32)); + } + return; + } + const int up = 32 - 8 * (d + bytes), down = 32 - 8 * bytes; + IROperand src = v; + if (up) + { + IROperand t = irop_make_vreg(tcc_ir_vreg_alloc_temp(ir), IROP_BTYPE_INT32); + sra_ins_add(ir, ins, i, 0, TCCIR_OP_SHL, t, v, irop_make_imm32(-1, up, IROP_BTYPE_INT32)); + src = t; + } + sra_relay(ir, &ir->compact_instructions[i], TCCIR_OP_SAR, dest, src, irop_make_imm32(-1, down, IROP_BTYPE_INT32)); +} + +/* A store into bytes [d, d + bytes) of the word field at `fs` whose other + * bytes are all padding: it may leave anything in them (C11 6.2.6.1p6), so + * the field's VAR is just the value, shifted into place -- no read of the old + * word and no insert. A Zig error union's `u16 error` in the low half of a + * word copied whole: `ldr; mov; bfi` per store became a move. Word fields + * only: a narrow field's VAR is read zero-extended (sra_combine). */ +static int sra_partial_store_padded(TCCIRState *ir, int i, SraUnit *u, int fs, int d, int bytes, IROperand val) +{ + if (u->width[fs] != 4 || fs + 4 > 64) + return 0; + for (int b = fs; b < fs + 4; b++) + if ((b < fs + d || b >= fs + d + bytes) && !((u->pad >> b) & 1)) + return 0; + IROperand def = sra_field_op(ir, u, fs, IROP_BTYPE_INT32, 1); + def.is_lval = 0; + IROperand x; + if (irop_is_plain_imm(val)) + x = irop_make_imm32(-1, (int32_t)(((uint32_t)irop_get_imm64_ex(ir, val) & (uint32_t)sra_field_mask(bytes)) << (8 * d)), + IROP_BTYPE_INT32); + else + { + x = irop_retype_scalar(val, IROP_BTYPE_INT32); + x.is_unsigned = 1; + } + IRQuadCompact *q = &ir->compact_instructions[i]; + if (d && !irop_is_plain_imm(val)) + sra_relay(ir, q, TCCIR_OP_SHL, def, x, irop_make_imm32(-1, 8 * d, IROP_BTYPE_INT32)); + else + sra_relay(ir, q, TCCIR_OP_ASSIGN, def, x, IROP_NONE); + return 1; +} + +/* A `bytes`-wide store of `val` into bytes [d, d + bytes) of the field at `fs` + * of u: `T <- V AND #~mask` ahead of i, and i becomes `V <- T OR x`, a fresh + * definition of the VAR, where x is val masked to its width (unless it fits + * already) and shifted into place; a constant is folded on the spot. */ +static void sra_partial_store(TCCIRState *ir, SraInsList *ins, int i, SraUnit *u, int fs, int d, int bytes, + IROperand val, int fits) +{ + IROperand v = sra_field_op(ir, u, fs, IROP_BTYPE_INT32, 1); + IROperand def = v; + def.is_lval = 0; + const int32_t mask = (int32_t)((uint32_t)sra_field_mask(bytes) << (8 * d)); + IROperand x; + if (irop_is_plain_imm(val)) + x = irop_make_imm32(-1, (int32_t)(((uint32_t)irop_get_imm64_ex(ir, val) << (8 * d)) & (uint32_t)mask), + IROP_BTYPE_INT32); + else + { + x = irop_retype_scalar(val, IROP_BTYPE_INT32); + x.is_unsigned = 1; + if (!fits) + { + IROperand t = sra_new_temp(ir, IROP_BTYPE_INT32); + sra_ins_add(ir, ins, i, 0, TCCIR_OP_AND, t, x, irop_make_imm32(-1, sra_field_mask(bytes), IROP_BTYPE_INT32)); + x = t; + } + if (d) + { + IROperand t = sra_new_temp(ir, IROP_BTYPE_INT32); + sra_ins_add(ir, ins, i, 0, TCCIR_OP_SHL, t, x, irop_make_imm32(-1, 8 * d, IROP_BTYPE_INT32)); + x = t; + } + } + IROperand kept = sra_new_temp(ir, IROP_BTYPE_INT32); + sra_ins_add(ir, ins, i, 0, TCCIR_OP_AND, kept, v, irop_make_imm32(-1, ~mask, IROP_BTYPE_INT32)); + sra_relay(ir, &ir->compact_instructions[i], TCCIR_OP_OR, def, kept, x); +} + +/* The same as one BFI: `V <- BFI(V, val)` inserting val's low 8*bytes bits at + * bit 8d (the machine op takes only those bits, so no mask is needed). Its + * lsb and width travel in ir->bfi_params by orig_index (bitfield.c). */ +static void sra_partial_store_bfi(TCCIRState *ir, int i, SraUnit *u, int fs, int d, int bytes, IROperand val) +{ + IROperand v = sra_field_op(ir, u, fs, IROP_BTYPE_INT32, 1); + IROperand def = v; + def.is_lval = 0; + IROperand x = irop_retype_scalar(val, IROP_BTYPE_INT32); + x.is_unsigned = 1; + IRQuadCompact *q = &ir->compact_instructions[i]; + sra_relay(ir, q, TCCIR_OP_BFI, def, v, x); + const int need = ir->max_orig_index + 1; + if (!ir->bfi_params || ir->bfi_params_len < need) + { + int len = need + 64; + ir->bfi_params = tcc_realloc(ir->bfi_params, (size_t)len * sizeof(uint16_t)); + memset(ir->bfi_params + ir->bfi_params_len, 0, (size_t)(len - ir->bfi_params_len) * sizeof(uint16_t)); + ir->bfi_params_len = len; + } + ir->bfi_params[q->orig_index] = (uint16_t)(((8 * d) & 0xFF) | (((8 * bytes) & 0xFF) << 8)); +} + +static int sra_ins_cmp(const void *a, const void *b) +{ + const SraIns *x = a, *y = b; + if (x->anchor != y->anchor) + return x->anchor < y->anchor ? -1 : 1; + if (x->after != y->after) + return x->after - y->after; + return x->q.orig_index < y->q.orig_index ? -1 : x->q.orig_index > y->q.orig_index; +} + +/* Splice the collected instructions in. A jump to an instruction lands on the + * first one added ahead of it (a PACK64 its reader needs); one added after an + * instruction only runs after it. */ +static void sra_ins_apply(TCCIRState *ir, SraInsList *l) +{ + if (l->n == 0) + return; + qsort(l->v, l->n, sizeof(SraIns), sra_ins_cmp); + const int n = ir->next_instruction_index, nn = n + l->n; + IRQuadCompact *out = tcc_malloc(sizeof(IRQuadCompact) * (nn + 1)); + int32_t *lead = tcc_malloc(sizeof(int32_t) * (n + 1)); + int j = 0, e = 0; + for (int i = 0; i < n; i++) + { + lead[i] = j; + int flag = ir->compact_instructions[i].is_jump_target; + int first = 1; + for (; e < l->n && l->v[e].anchor == i && !l->v[e].after; e++, first = 0) + { + out[j] = l->v[e].q; + out[j++].is_jump_target = first ? flag : 0; + } + out[j] = ir->compact_instructions[i]; + if (!first) + out[j].is_jump_target = 0; + j++; + for (; e < l->n && l->v[e].anchor == i; e++) + out[j++] = l->v[e].q; + } + lead[n] = j; + for (int i = 0; i < nn; i++) + { + IRQuadCompact *q = &out[i]; + if (q->op == TCCIR_OP_JUMP || q->op == TCCIR_OP_JUMPIF) + { + int target = (int)irop_get_imm64_ex(ir, tcc_ir_op_get_dest(ir, q)); + if (target >= 0 && target <= n) + tcc_ir_op_set_dest(ir, q, irop_make_imm32(-1, lead[target], IROP_BTYPE_INT32)); + } + } + for (int t = 0; t < ir->num_switch_tables; t++) + { + TCCIRSwitchTable *table = &ir->switch_tables[t]; + if (table->default_target >= 0 && table->default_target <= n) + table->default_target = lead[table->default_target]; + for (int k = 0; table->targets && k < table->num_entries; k++) + if (table->targets[k] >= 0 && table->targets[k] <= n) + table->targets[k] = lead[table->targets[k]]; + } + if (ir->compact_instructions_size < nn + 1) + { + ir->compact_instructions = tcc_realloc(ir->compact_instructions, sizeof(IRQuadCompact) * (nn + 1)); + ir->compact_instructions_size = nn + 1; + } + memcpy(ir->compact_instructions, out, sizeof(IRQuadCompact) * nn); + ir->next_instruction_index = nn; + tcc_free(lead); + tcc_free(out); +} + +/* The field VAR at byte `at` of unit u, as an operand of type `bt`. */ +static IROperand sra_field_op(TCCIRState *ir, SraUnit *u, int at, int bt, int is_unsigned) +{ + if (u->var[at] < 0) + u->var[at] = tcc_ir_vreg_alloc_var(ir); + IRLiveInterval *iv = tcc_ir_get_live_interval(ir, u->var[at]); + IROperand rep = irop_make_stackoff(u->var[at], iv ? iv->original_offset : 0, 1, 0, 0, bt); + rep.is_unsigned = is_unsigned; + rep.aux = IROP_AUX_NONVOLATILE; + return rep; +} + +/* Rewrite the 64-bit access `o` in slot s of instruction i to the doubleword + * at byte `at` of u (sra_resolve_wide laid it out). + * + * A 64-bit field is a 64-bit VAR: a read names it, and a STORE becomes an + * ASSIGN -- SSA takes only a 32-bit slot STORE for a definition + * (ssa_store_slot_def_pos), exactly the narrow-field case. + * + * A split doubleword is two word VARs, lo at `at` and hi at `at + 4`: + * - a LOAD / ASSIGN of it becomes `dest <- lo PACK64 hi`; + * - any other reader gets `T <- lo PACK64 hi` ahead of it and reads T; + * - a STORE of a constant stores its two words, one of a value x becomes + * `lo <- x` (the ASSIGN truncates, as the frontend's own u64 -> u32 cast + * does) followed by `T <- x SHR #32; hi <- T`. */ +static void sra_rewrite_wide(TCCIRState *ir, SraInsList *ins, int i, int s, IROperand o, SraUnit *u, int at) +{ + IRQuadCompact *q = &ir->compact_instructions[i]; + if (u->width[at] == 8) + { + IROperand rep = sra_field_op(ir, u, at, IROP_BTYPE_INT64, o.is_unsigned); + if (s == 0) + sra_store_as_def(ir, i, rep); + else if (s == 1) + tcc_ir_set_src1(ir, i, rep); + else + tcc_ir_set_src2(ir, i, rep); + return; + } + IROperand lo = sra_field_op(ir, u, at, IROP_BTYPE_INT32, 1); + IROperand hi = sra_field_op(ir, u, at + 4, IROP_BTYPE_INT32, 1); + if (s == 0) + { + IROperand v = tcc_ir_op_get_src1(ir, q); + int tag = irop_get_tag(v); + if (tag == IROP_TAG_IMM32 || tag == IROP_TAG_I64) + { + int64_t c = irop_get_imm64_ex(ir, v); + tcc_ir_set_dest(ir, i, lo); + tcc_ir_set_src1(ir, i, irop_make_imm32(-1, (int32_t)(uint32_t)c, IROP_BTYPE_INT32)); + sra_ins_add(ir, ins, i, 1, TCCIR_OP_STORE, hi, irop_make_imm32(-1, (int32_t)(uint32_t)((uint64_t)c >> 32), IROP_BTYPE_INT32), + IROP_NONE); + return; + } + lo.is_lval = 0; + hi.is_lval = 0; + q->op = TCCIR_OP_ASSIGN; + tcc_ir_set_dest(ir, i, lo); + int32_t t = tcc_ir_vreg_alloc_temp(ir); + IROperand th = irop_make_vreg(t, IROP_BTYPE_INT64); + th.is_unsigned = 1; + IROperand x = v; + x.is_unsigned = 1; + sra_ins_add(ir, ins, i, 1, TCCIR_OP_SHR, th, x, irop_make_imm32(-1, 32, IROP_BTYPE_INT32)); + sra_ins_add(ir, ins, i, 1, TCCIR_OP_ASSIGN, hi, th, IROP_NONE); + return; + } + IROperand d = irop_config[q->op].has_dest ? tcc_ir_op_get_dest(ir, q) : IROP_NONE; + if (s == 1 && (q->op == TCCIR_OP_LOAD || q->op == TCCIR_OP_ASSIGN) && !d.is_lval && + irop_get_btype(d) == IROP_BTYPE_INT64) + { + sra_relay(ir, q, TCCIR_OP_PACK64, d, lo, hi); + return; + } + int32_t t = tcc_ir_vreg_alloc_temp(ir); + IROperand tp = irop_make_vreg(t, IROP_BTYPE_INT64); + tp.is_unsigned = o.is_unsigned; + sra_ins_add(ir, ins, i, 0, TCCIR_OP_PACK64, tp, lo, hi); + if (q->op == TCCIR_OP_LOAD) + q->op = TCCIR_OP_ASSIGN; /* reads the packed TEMP's value, not memory */ + if (s == 1) + tcc_ir_set_src1(ir, i, tp); + else + tcc_ir_set_src2(ir, i, tp); +} + +/* A LOAD or STORE through a TEMP base, plain or indexed by a constant: the + * base's TEMP, with the displacement and the operand that carries the access's + * width, sign and volatility (ssa:stack_deref_fold reads it the same way). + * -1 for anything else. */ +static int sra_access_through(TCCIRState *ir, IRQuadCompact *q, int ntemp, int *disp, IROperand *acc) +{ + int op = q->op; + if (op != TCCIR_OP_STORE && op != TCCIR_OP_LOAD && op != TCCIR_OP_STORE_INDEXED && op != TCCIR_OP_LOAD_INDEXED) + return -1; + int is_store = op == TCCIR_OP_STORE || op == TCCIR_OP_STORE_INDEXED; + int indexed = op == TCCIR_OP_STORE_INDEXED || op == TCCIR_OP_LOAD_INDEXED; + IROperand base = is_store ? tcc_ir_op_get_dest(ir, q) : tcc_ir_op_get_src1(ir, q); + int t = sra_temp_pos(base, ntemp); + if (t < 0 || irop_get_tag(base) != IROP_TAG_VREG || base.is_local || base.is_llocal || base.is_lval == indexed) + return -1; + *disp = 0; + *acc = base; + if (indexed) + { + IROperand idx = tcc_ir_op_get_src2(ir, q); + IROperand scale = tcc_ir_op_get_scale(ir, q); + if (irop_get_tag(idx) != IROP_TAG_IMM32 || idx.is_lval || + (!irop_is_none(scale) && (irop_get_tag(scale) != IROP_TAG_IMM32 || irop_get_imm32(scale) != 0))) + return -1; + *disp = irop_get_imm32(idx); + *acc = is_store ? tcc_ir_op_get_src1(ir, q) : tcc_ir_op_get_dest(ir, q); + acc->aux = base.aux; + } + if (irop_is_64bit(*acc) || acc->is_complex || !sra_btype_bytes(irop_get_btype(*acc)) || + tcc_ir_access_is_volatile(ir, *acc)) + return -1; + return t; +} + +/* A call that reads or writes a range of an object whole. + * + * The frontend copies an aggregate over 16 bytes with an __aeabi_memmove4/8 + * call, and a struct-returning call writes its result through a buffer + * address passed as argument 0. Either names the object's address in a + * call, which would keep it in memory. But a word-aligned copy between the + * object and another pointer is a set of field copies -- when the object is + * replaced it becomes one LOAD (copy in) or STORE (copy out) per field, at + * the call, and the call goes. And a result buffer is only written: the + * callee cannot keep its address, so the object keeps its slot as the buffer + * and each field VAR is loaded from it right after the call. + * + * Bytes a call writes and a later whole read passes on (a copy out, a by-value + * argument) need a field even when nothing accesses them one at a time; bytes + * only one side touches are indeterminate on the other and need none. An + * object reached by nothing but such calls stays in memory: moving it field + * by field costs more than the block copy it replaces. */ +typedef struct +{ + int call; /* the call instruction */ + int cid; /* its call id */ + int param; /* the argument naming the object's address */ + int unit, off; /* the object, and the range's frame offset */ + int n; /* the range's bytes */ + int dir; /* 0: the call writes the range, 1: it reads it */ + int sret; /* a result buffer: the call stays, the fields load after it */ + int other; /* a copy: the other pointer's argument */ + int other_t; /* its TEMP, or -1 */ + int t; /* the address TEMP the argument reads, or -1 */ + IROperand slot; /* a result buffer: an address operand of the object's slot */ + int pair; /* a copy between two replaced objects: the other's unit, or -1 */ + int pair_off; /* and the range's frame offset in it */ +} SraWhole; + +enum +{ + SRA_CALL_OTHER, + SRA_CALL_COPY, /* (dst, src, n), both word aligned */ +}; + +static int sra_call_kind(TCCIRState *ir, IRQuadCompact *q) +{ + if (q->op != TCCIR_OP_FUNCCALLVOID) + return SRA_CALL_OTHER; + Sym *cs = irop_get_sym_ex(ir, tcc_ir_op_get_src1(ir, q)); + const char *nm = cs ? get_tok_str(cs->v, NULL) : NULL; + if (nm && (!strcmp(nm, "__aeabi_memmove4") || !strcmp(nm, "__aeabi_memmove8") || + !strcmp(nm, "__aeabi_memcpy4") || !strcmp(nm, "__aeabi_memcpy8"))) + return SRA_CALL_COPY; + return SRA_CALL_OTHER; +} + +/* Argument i (a FUNCPARAMVAL) of a call that accesses a range at the address + * it passes whole: fills w's call, direction, length and copy partner. + * call_at / cparam index the calls by id: the instruction, and arguments 0-2 + * (-1 none, -2 twice). */ +static int sra_whole_use(TCCIRState *ir, int i, const int32_t *call_at, const int32_t (*cparam)[3], + const uint8_t *ckind, int ncall, SraWhole *w) +{ + IRQuadCompact *q = &ir->compact_instructions[i]; + uint32_t e = (uint32_t)irop_get_imm64_ex(ir, tcc_ir_op_get_src2(ir, q)); + int c = TCCIR_DECODE_CALL_ID(e), p = TCCIR_DECODE_PARAM_IDX(e); + if (c < 0 || c >= ncall || call_at[c] <= i) + return 0; + memset(w, 0, sizeof(*w)); + w->call = call_at[c]; + w->cid = c; + w->param = i; + w->other = w->other_t = w->t = w->pair = -1; + if (ckind[c] == SRA_CALL_COPY && (p == 0 || p == 1)) + { + if (cparam[c][0] < 0 || cparam[c][1] < 0 || cparam[c][2] < 0) + return 0; + IROperand nop = tcc_ir_op_get_src1(ir, &ir->compact_instructions[cparam[c][2]]); + if (!irop_is_plain_imm(nop)) + return 0; + int64_t nn = irop_get_imm64_ex(ir, nop); + if (nn <= 0 || nn > SRA_MAX_BYTES) + return 0; + /* The other pointer is a register value, read where the copy happens: + * nothing but the call's own arguments may sit between them. */ + IROperand o = tcc_ir_op_get_src1(ir, &ir->compact_instructions[cparam[c][1 - p]]); + if (irop_get_tag(o) != IROP_TAG_VREG || irop_get_vreg(o) < 0 || o.is_lval || o.is_llocal || o.is_local) + return 0; + int first = cparam[c][0]; + for (int k = 1; k < 3; k++) + if (cparam[c][k] < first) + first = cparam[c][k]; + for (int j = first; j < w->call; j++) + { + IRQuadCompact *jq = &ir->compact_instructions[j]; + if (jq->op != TCCIR_OP_NOP && j != cparam[c][0] && j != cparam[c][1] && j != cparam[c][2]) + return 0; + } + w->n = (int)nn; + w->dir = p; + w->other = cparam[c][1 - p]; + return 1; + } + if (p == 0 && c < ir->sret_calls_size && ir->sret_calls[c] > 4) + { + w->n = ir->sret_calls[c]; + w->sret = 1; + return 1; + } + return 0; +} + +/* Fields for the bytes of [at0, at0 + n) no access covers: aligned words, or + * narrow unsigned pieces where a word does not fit. */ +static void sra_fill_gaps(SraUnit *u, int at0, int n) +{ + uint8_t cov[SRA_MAX_BYTES] = {0}; + for (int at = 0; at < SRA_MAX_BYTES; at++) + for (int b = at; b < at + u->width[at] && b < SRA_MAX_BYTES; b++) + cov[b] = 1; + for (int b = at0; b < at0 + n;) + { + if (cov[b]) + { + b++; + continue; + } + int w = 1; + if (!(b & 3) && b + 4 <= at0 + n && !cov[b + 1] && !cov[b + 2] && !cov[b + 3]) + w = 4; + else if (!(b & 1) && b + 2 <= at0 + n && !cov[b + 1]) + w = 2; + u->width[b] = w; + if (w < 4) + u->sign[b] = 2; + for (int k = b; k < b + w; k++) + cov[k] = 1; + b += w; + } +} + +/* No field of u crosses an end of [at0, at0 + n). */ +static int sra_range_clean(const SraUnit *u, int at0, int n) +{ + for (int at = 0; at < SRA_MAX_BYTES; at++) + { + int end = at + u->width[at]; + if (u->width[at] && ((at < at0 && end > at0) || (at < at0 + n && end > at0 + n))) + return 0; + } + return 1; +} + +static int sra_width_btype(int w) +{ + return w == 1 ? IROP_BTYPE_INT8 : w == 2 ? IROP_BTYPE_INT16 : IROP_BTYPE_INT32; +} + +static IROperand sra_new_temp(TCCIRState *ir, int bt) +{ + IROperand t = irop_make_vreg(tcc_ir_vreg_alloc_temp(ir), bt); + t.is_unsigned = 1; + return t; +} + +/* The object's slot at frame offset off, as an lvalue of type bt. */ +static IROperand sra_slot_op(IROperand addr, int off, int bt) +{ + IROperand s = irop_retype_scalar(addr, bt); + s.u.imm32 = off; + s.is_lval = 1; + s.is_unsigned = 1; + s.aux = IROP_AUX_NONVOLATILE; + return s; +} + +static IROperand sra_field_op(TCCIRState *ir, SraUnit *u, int at, int bt, int is_unsigned); + +/* The `bytes`-wide value at byte `at` of u, whose narrow fields cover only + * part of it (sra_resolve_fields): their VARs shifted into place and ORed, by + * instructions ahead of i -- and a signed halfword's sign extended, since a + * field now fills its top byte. */ +static IROperand sra_combine(TCCIRState *ir, SraInsList *ins, int i, SraUnit *u, int at, int bytes, + int is_unsigned) +{ + IROperand v = sra_field_op(ir, u, at, IROP_BTYPE_INT32, 1); + if (u->pmask[at]) + { + IROperand m = sra_new_temp(ir, IROP_BTYPE_INT32); + sra_ins_add(ir, ins, i, 0, TCCIR_OP_AND, m, v, irop_make_imm32(-1, sra_field_mask(u->width[at]), IROP_BTYPE_INT32)); + v = m; + } + for (int b = at + 1; b < at + bytes && b < SRA_MAX_BYTES; b++) + { + if (!u->width[b]) + continue; + IROperand f = sra_field_op(ir, u, b, IROP_BTYPE_INT32, 1); + if (u->pmask[b]) + { + IROperand m = sra_new_temp(ir, IROP_BTYPE_INT32); + sra_ins_add(ir, ins, i, 0, TCCIR_OP_AND, m, f, irop_make_imm32(-1, sra_field_mask(u->width[b]), IROP_BTYPE_INT32)); + f = m; + } + IROperand t = sra_new_temp(ir, IROP_BTYPE_INT32), o = sra_new_temp(ir, IROP_BTYPE_INT32); + sra_ins_add(ir, ins, i, 0, TCCIR_OP_SHL, t, f, irop_make_imm32(-1, 8 * (b - at), IROP_BTYPE_INT32)); + sra_ins_add(ir, ins, i, 0, TCCIR_OP_OR, o, v, t); + v = o; + } + if (!is_unsigned && bytes < 4) + { + const int sh = 32 - 8 * bytes; + IROperand t = sra_new_temp(ir, IROP_BTYPE_INT32), o = sra_new_temp(ir, IROP_BTYPE_INT32); + t.is_unsigned = o.is_unsigned = 0; + sra_ins_add(ir, ins, i, 0, TCCIR_OP_SHL, t, v, irop_make_imm32(-1, sh, IROP_BTYPE_INT32)); + sra_ins_add(ir, ins, i, 0, TCCIR_OP_SAR, o, t, irop_make_imm32(-1, sh, IROP_BTYPE_INT32)); + v = o; + } + return v; +} + +/* Does a `bytes`-wide read at `at` of u cover a field past the one there? */ +static int sra_covers_more(const SraUnit *u, int at, int bytes) +{ + for (int b = at + 1; b < at + bytes && b < SRA_MAX_BYTES; b++) + if (u->width[b]) + return 1; + return 0; +} + +/* Copies of the fields of [s0, s0 + n) of unit src into those of [d0, d0 + n) + * of unit dst, laid out alike (checked where the copy was accepted), ahead of + * the copy call at `anchor`. A narrow or 64-bit field's definition is an + * ASSIGN (sra_store_as_def, sra_rewrite_wide). */ +static void sra_expand_pair(TCCIRState *ir, SraInsList *ins, SraUnit *dst, int d0, SraUnit *src, int s0, int n, + int anchor) +{ + for (int d = 0; d < n; d++) + { + const int w = dst->width[d0 + d]; + if (!w) + continue; + const int bt = w == 8 ? IROP_BTYPE_INT64 : IROP_BTYPE_INT32; + IROperand x = sra_new_temp(ir, bt); + sra_ins_add(ir, ins, anchor, 0, TCCIR_OP_LOAD, x, sra_field_op(ir, src, s0 + d, bt, 1), IROP_NONE); + if (w < 4 && src->pmask[s0 + d]) + { + IROperand m = sra_new_temp(ir, bt); + sra_ins_add(ir, ins, anchor, 0, TCCIR_OP_AND, m, x, irop_make_imm32(-1, sra_field_mask(w), IROP_BTYPE_INT32)); + x = m; + } + IROperand rep = sra_field_op(ir, dst, d0 + d, bt, 1); + if (w != 4) + rep.is_lval = 0; + sra_ins_add(ir, ins, anchor, 0, w != 4 ? TCCIR_OP_ASSIGN : TCCIR_OP_STORE, rep, x, IROP_NONE); + d += w - 1; + } +} + +/* The field copies that replace e: loads into the field VARs of the range + * (from the other pointer ahead of the call, or from the object's own slot + * after it), or stores of them through the other pointer. */ +static void sra_expand_whole(TCCIRState *ir, SraInsList *ins, SraUnit *u, const SraWhole *e, IROperand base) +{ + const int a0 = e->off - u->lo, anchor = e->call, after = e->sret; + for (int at = a0; at < a0 + e->n; at++) + { + const int w = u->width[at]; + if (!w) + continue; + const int d = at - a0; + IROperand imm = irop_make_imm32(-1, d, IROP_BTYPE_INT32); + IROperand imm4 = irop_make_imm32(-1, d + 4, IROP_BTYPE_INT32); + if (e->dir == 0 && w == 8) + { + IROperand x = sra_new_temp(ir, IROP_BTYPE_INT64); + if (e->sret) + sra_ins_add(ir, ins, anchor, after, TCCIR_OP_LOAD, x, sra_slot_op(e->slot, u->lo + at, IROP_BTYPE_INT64), + IROP_NONE); + else + { + IROperand lo = sra_new_temp(ir, IROP_BTYPE_INT32), hi = sra_new_temp(ir, IROP_BTYPE_INT32); + sra_ins_add(ir, ins, anchor, after, TCCIR_OP_LOAD_INDEXED, lo, base, imm); + sra_ins_add(ir, ins, anchor, after, TCCIR_OP_LOAD_INDEXED, hi, base, imm4); + sra_ins_add(ir, ins, anchor, after, TCCIR_OP_PACK64, x, lo, hi); + } + IROperand rep = sra_field_op(ir, u, at, IROP_BTYPE_INT64, 1); + rep.is_lval = 0; + sra_ins_add(ir, ins, anchor, after, TCCIR_OP_ASSIGN, rep, x, IROP_NONE); + } + else if (e->dir == 0) + { + IROperand x = sra_new_temp(ir, sra_width_btype(w)); + if (e->sret) + sra_ins_add(ir, ins, anchor, after, TCCIR_OP_LOAD, x, sra_slot_op(e->slot, u->lo + at, sra_width_btype(w)), + IROP_NONE); + else + sra_ins_add(ir, ins, anchor, after, TCCIR_OP_LOAD_INDEXED, x, base, imm); + /* A narrow field's definition is an ASSIGN (sra_store_as_def). */ + IROperand rep = sra_field_op(ir, u, at, IROP_BTYPE_INT32, 1); + if (w < 4) + rep.is_lval = 0; + sra_ins_add(ir, ins, anchor, after, w < 4 ? TCCIR_OP_ASSIGN : TCCIR_OP_STORE, rep, x, IROP_NONE); + } + else if (w == 8) + { + IROperand x = sra_new_temp(ir, IROP_BTYPE_INT64), th = sra_new_temp(ir, IROP_BTYPE_INT64); + IROperand lo = sra_new_temp(ir, IROP_BTYPE_INT32), hi = sra_new_temp(ir, IROP_BTYPE_INT32); + sra_ins_add(ir, ins, anchor, after, TCCIR_OP_LOAD, x, sra_field_op(ir, u, at, IROP_BTYPE_INT64, 1), IROP_NONE); + sra_ins_add(ir, ins, anchor, after, TCCIR_OP_ASSIGN, lo, x, IROP_NONE); + sra_ins_add(ir, ins, anchor, after, TCCIR_OP_SHR, th, x, irop_make_imm32(-1, 32, IROP_BTYPE_INT32)); + sra_ins_add(ir, ins, anchor, after, TCCIR_OP_ASSIGN, hi, th, IROP_NONE); + sra_ins_add(ir, ins, anchor, after, TCCIR_OP_STORE_INDEXED, base, lo, imm); + sra_ins_add(ir, ins, anchor, after, TCCIR_OP_STORE_INDEXED, base, hi, imm4); + } + else + { + IROperand x = sra_new_temp(ir, IROP_BTYPE_INT32); + sra_ins_add(ir, ins, anchor, after, TCCIR_OP_LOAD, x, sra_field_op(ir, u, at, IROP_BTYPE_INT32, 1), IROP_NONE); + IROperand v = irop_make_vreg(irop_get_vreg(x), sra_width_btype(w)); + v.is_unsigned = 1; + sra_ins_add(ir, ins, anchor, after, TCCIR_OP_STORE_INDEXED, base, v, imm); + } + at += w - 1; + } +} + +#define SRA_VMIX (INT32_MIN + 1) /* a pointer VAR with more than one value */ +#define SRA_UNKNOWN (INT32_MIN + 2) /* a value that may or may not be a frame address */ +#define SRA_NOT_ADDR (INT32_MIN + 3) /* a value that is no frame address */ +#define SRA_FIRST_ADDR (INT32_MIN + 4) +#define SRA_DF_MAX (4 << 20) /* blocks x pointer VARs the dataflow tracks */ + +/* The position of a VAR operand below nvar, or -1. */ +static int sra_var_pos(IROperand o, int nvar) +{ + int32_t vr = irop_get_vreg(o); + if (vr < 0 || TCCIR_DECODE_VREG_TYPE(vr) != TCCIR_VREG_TYPE_VAR || TCCIR_DECODE_VREG_POSITION(vr) >= nvar) + return -1; + return TCCIR_DECODE_VREG_POSITION(vr); +} + +/* The VAR q defines, or -1: a value dest, or a STORE into the VAR's own slot + * (an inlined parameter's binding, ssa.c's reading). */ +static int sra_var_def(TCCIRState *ir, IRQuadCompact *q, int nvar) +{ + if (!irop_config[q->op].has_dest || q->op == TCCIR_OP_STORE_INDEXED || q->op == TCCIR_OP_STORE_POSTINC) + return -1; + IROperand d = tcc_ir_op_get_dest(ir, q); + if (d.is_llocal || (q->op == TCCIR_OP_STORE ? !d.is_local : d.is_lval)) + return -1; + return sra_var_pos(d, nvar); +} + +/* An address copy or constant offset: LEA, ASSIGN, a LOAD of a VAR's value, or + * ADD / SUB (whose second operand sra_offset_by checks). */ +static int sra_is_copy_op(TCCIRState *ir, IRQuadCompact *q, int nvar) +{ + if (q->op == TCCIR_OP_LOAD) + { + IROperand a = tcc_ir_op_get_src1(ir, q); + return sra_var_pos(a, nvar) >= 0 && !a.is_llocal; + } + return q->op == TCCIR_OP_LEA || q->op == TCCIR_OP_ASSIGN || q->op == TCCIR_OP_ADD || q->op == TCCIR_OP_SUB; +} + +/* base +/- q's constant second operand, or SRA_VMIX when it is not one. */ +static int32_t sra_offset_by(TCCIRState *ir, IRQuadCompact *q, int32_t base) +{ + IROperand c = tcc_ir_op_get_src2(ir, q); + if (irop_get_tag(c) != IROP_TAG_IMM32 || c.is_lval) + return SRA_VMIX; + int64_t x = (int64_t)base + (q->op == TCCIR_OP_ADD ? 1 : -1) * (int64_t)irop_get_imm32(c); + return x > -(1 << 24) && x < (1 << 24) ? (int32_t)x : SRA_VMIX; +} + +/* Is q an address computation of a TEMP -- `T <- Addr[slot]`, a copy, or a + * constant offset of another TEMP? Returns T's position, or -1. */ +static int sra_addr_def(TCCIRState *ir, IRQuadCompact *q, int ntemp) +{ + /* A LOAD of a VAR's value is a copy too: `T <- V1 [LOAD]` (a pointer VAR). */ + if (q->op == TCCIR_OP_LOAD) + { + IROperand a = tcc_ir_op_get_src1(ir, q); + int32_t avr = irop_get_vreg(a); + if (avr < 0 || TCCIR_DECODE_VREG_TYPE(avr) != TCCIR_VREG_TYPE_VAR || a.is_llocal) + return -1; + } + else if (q->op != TCCIR_OP_LEA && q->op != TCCIR_OP_ASSIGN && q->op != TCCIR_OP_ADD && q->op != TCCIR_OP_SUB) + return -1; + IROperand d = tcc_ir_op_get_dest(ir, q); + return d.is_lval ? -1 : sra_temp_pos(d, ntemp); +} + +/* A VAR only ever written with one constant reads as that constant. + * + * Any read either follows a write, which stored that constant, or sees a value + * nothing wrote -- so no dominance question arises, and no phi. Two kinds of + * VAR are trusted to show every write: + * - the field VARs this run made: every access to a replaced object was + * rewritten to them, SRA leaving an object alone unless it saw all of them. + * The Zig C backend zero-fills each zig_u128 it builds and then writes both + * halves, so `zig_make_u128(0, x)`'s high word is written 0, then 0; + * - any other VAR written only by slot STOREs: an inlined function's + * parameter is bound that way, and zig.h passes `bits` (128, 64) to every + * u128 helper. VARs written by an ASSIGN are left out: the short-circuit + * boolean lowering writes one arm into a TEMP the allocator later merges + * with the VAR (const_var_prop.c), a write this scan cannot see. + * SSA would rename such a VAR only when its write's block has a non-empty + * dominance frontier (ssa_var_promotable), and a constant nothing forwards + * keeps the sign tests and shift-by-zero branches of every u128 helper alive. + * Address-taken and volatile VARs are skipped, and a VAR read at a width + * other than its writes' keeps its reads. */ +static int sra_forward_const_vars(TCCIRState *ir, int first_sra_var) +{ + const int nv = ir->next_local_variable, n = ir->next_instruction_index; + if (nv <= 0) + return 0; + uint8_t *st = tcc_mallocz(nv); /* 0 unwritten, 1 always val[k], 2 no */ + int64_t *val = tcc_malloc(sizeof(int64_t) * nv); + uint8_t *bt = tcc_mallocz(nv); + for (int k = 0; k < nv; k++) + { + IRLiveInterval *iv = tcc_ir_get_live_interval(ir, TCCIR_ENCODE_VREG(TCCIR_VREG_TYPE_VAR, k)); + if (!iv || iv->addrtaken || iv->is_volatile) + st[k] = 2; + } + for (int i = 0; i < n; i++) + { + IRQuadCompact *q = &ir->compact_instructions[i]; + if (q->op == TCCIR_OP_NOP) + continue; + /* A LEA takes the VAR's address: whatever is written through it is not + * seen here. */ + if (q->op == TCCIR_OP_LEA || q->op == TCCIR_OP_ASM_INPUT || q->op == TCCIR_OP_ASM_OUTPUT) + { + IROperand a = q->op == TCCIR_OP_ASM_OUTPUT ? tcc_ir_op_get_dest(ir, q) : tcc_ir_op_get_src1(ir, q); + int32_t avr = irop_get_vreg(a); + if (avr >= 0 && TCCIR_DECODE_VREG_TYPE(avr) == TCCIR_VREG_TYPE_VAR && TCCIR_DECODE_VREG_POSITION(avr) < nv) + st[TCCIR_DECODE_VREG_POSITION(avr)] = 2; + continue; + } + if (!irop_config[q->op].has_dest) + continue; + IROperand d = tcc_ir_op_get_dest(ir, q); + int32_t vr = irop_get_vreg(d); + if (vr < 0 || TCCIR_DECODE_VREG_TYPE(vr) != TCCIR_VREG_TYPE_VAR) + continue; + int k = TCCIR_DECODE_VREG_POSITION(vr); + if (k >= nv || st[k] == 2) + continue; + /* A STORE through the VAR's value writes somewhere else. */ + if (q->op == TCCIR_OP_STORE_INDEXED || q->op == TCCIR_OP_STORE_POSTINC || + (q->op == TCCIR_OP_STORE && k < first_sra_var && !d.is_local)) + continue; + IROperand v = irop_config[q->op].has_src1 ? tcc_ir_op_get_src1(ir, q) : IROP_NONE; + int tag = irop_get_tag(v); + int btype = irop_get_btype(d); + int ok_op = q->op == TCCIR_OP_STORE || (q->op == TCCIR_OP_ASSIGN && k >= first_sra_var); + if (!ok_op || v.is_lval || v.is_sym || (tag != IROP_TAG_IMM32 && tag != IROP_TAG_I64) || + (btype != IROP_BTYPE_INT8 && btype != IROP_BTYPE_INT16 && btype != IROP_BTYPE_INT32 && + btype != IROP_BTYPE_INT64) || + (st[k] == 1 && bt[k] != btype)) + { + st[k] = 2; + continue; + } + int64_t c = irop_get_imm64_ex(ir, v); + if (btype == IROP_BTYPE_INT8) + c = d.is_unsigned ? (int64_t)(uint8_t)c : (int64_t)(int8_t)c; + else if (btype == IROP_BTYPE_INT16) + c = d.is_unsigned ? (int64_t)(uint16_t)c : (int64_t)(int16_t)c; + else if (btype != IROP_BTYPE_INT64) + c = (int64_t)(int32_t)(uint32_t)c; + st[k] = st[k] == 0 || val[k] == c ? 1 : 2; + val[k] = c; + bt[k] = (uint8_t)btype; + } + /* Every read must be a plain value read at the writes' width. */ + for (int i = 0; i < n; i++) + { + IRQuadCompact *q = &ir->compact_instructions[i]; + if (q->op == TCCIR_OP_NOP) + continue; + for (int s = 0; s < 4; s++) + { + IROperand o = sra_operand(ir, q, s); + int32_t vr = irop_get_vreg(o); + if (vr < 0 || TCCIR_DECODE_VREG_TYPE(vr) != TCCIR_VREG_TYPE_VAR) + continue; + int k = TCCIR_DECODE_VREG_POSITION(vr); + if (k >= nv || st[k] != 1) + continue; + if (s == 0) + { + /* The writes seen above; a STORE through the VAR's value reads it. */ + if (q->op == TCCIR_OP_STORE_INDEXED || q->op == TCCIR_OP_STORE_POSTINC || + (q->op == TCCIR_OP_STORE && !o.is_local && k < first_sra_var)) + st[k] = 2; + continue; + } + if (s == 3 || !o.is_lval || o.is_llocal || irop_get_btype(o) != bt[k]) + st[k] = 2; + } + } + int changes = 0; + for (int i = 0; i < n; i++) + { + IRQuadCompact *q = &ir->compact_instructions[i]; + /* A fused barrel shift names its operand registers: it keeps its reads + * (and the VAR its STOREs). */ + if (q->op == TCCIR_OP_NOP || tcc_ir_barrel_shift_at(ir, q)) + continue; + for (int s = 1; s <= 2; s++) + { + if (!(s == 1 ? irop_config[q->op].has_src1 : irop_config[q->op].has_src2)) + continue; + IROperand o = s == 1 ? tcc_ir_op_get_src1(ir, q) : tcc_ir_op_get_src2(ir, q); + int32_t vr = irop_get_vreg(o); + if (vr < 0 || TCCIR_DECODE_VREG_TYPE(vr) != TCCIR_VREG_TYPE_VAR) + continue; + int k = TCCIR_DECODE_VREG_POSITION(vr); + if (k >= nv || st[k] != 1) + continue; + int64_t c = val[k]; + int obt = irop_get_btype(o); + IROperand imm = c == (int64_t)(int32_t)c ? irop_make_imm32(-1, (int32_t)c, obt) + : irop_make_i64(-1, tcc_ir_pool_add_i64(ir, c), obt); + imm.is_unsigned = o.is_unsigned; + if (s == 1 && q->op == TCCIR_OP_LOAD) + q->op = TCCIR_OP_ASSIGN; + if (s == 1) + tcc_ir_set_src1(ir, i, imm); + else + tcc_ir_set_src2(ir, i, imm); + changes++; + } + } + tcc_free(bt); + tcc_free(val); + tcc_free(st); + return changes; +} + +static int sra_objects(TCCIRState *ir) +{ + if (ir->frame_relaid || ir->frame_obj_count == 0) + return 0; + + /* Units: the frame objects' merged extents, sorted by start. */ + int lo, hi; + tcc_ir_frame_object_at(ir, ir->frame_objs[0], &lo, &hi); /* builds ir->frame_index */ + int nu = ir->frame_index_n; + if (nu <= 0) + return 0; + SraUnit *u = tcc_mallocz(sizeof(SraUnit) * nu); + for (int k = 0; k < nu; k++) + { + u[k].lo = ir->frame_index[2 * k]; + u[k].hi = ir->frame_index[2 * k + 1]; + u[k].pad = sra_pad_off() ? 0 : tcc_ir_frame_padding(ir, u[k].lo, u[k].hi); + for (int b = 0; b < SRA_MAX_BYTES; b++) + u[k].var[b] = -1; + } + const int n = ir->next_instruction_index; + /* A split doubleword's rewrite adds instructions; a computed goto's targets + * are addresses taken in the frontend, which that would move. */ + int no_wide = 0; + for (int i = 0; i < n && !no_wide; i++) + no_wide = ir->compact_instructions[i].op == TCCIR_OP_IJUMP; + + /* Address TEMPs. The frontend reaches a field through one + * (`T <- &s; *(T + 4) = x`). A TEMP defined once, from `Addr[slot]` or a + * copy / constant offset of such a TEMP, holds that frame address wherever it + * is read; when every use is the base of a load or store at a constant + * displacement, or feeds another such TEMP, the accesses are the object's + * fields. Nothing is rewritten unless the object is replaced. */ + /* By-value struct arguments may become word arguments (sra_struct_param_words). */ + const int struct_param = !tcc_ir_opt_pass_disabled("sra_struct_param"); + const int ntemp = ir->next_temporary_variable; + uint8_t *defs = ntemp > 0 ? tcc_mallocz(ntemp) : NULL; + /* Each TEMP's only defining instruction, or -1: what a narrow store's value + * has to be looked up through (sra_narrow_store_fits). */ + int32_t *def_idx = ntemp > 0 ? tcc_malloc(sizeof(int32_t) * ntemp) : NULL; + for (int t = 0; t < ntemp; t++) + def_idx[t] = -1; + int32_t *toff = ntemp > 0 ? tcc_malloc(sizeof(int32_t) * ntemp) : NULL; + int32_t *tfrom = ntemp > 0 ? tcc_malloc(sizeof(int32_t) * ntemp) : NULL; /* TEMP it was derived from, or -1 */ + int32_t *troot = ntemp > 0 ? tcc_malloc(sizeof(int32_t) * ntemp) : NULL; /* unit of the root Addr */ + uint8_t *tbad = ntemp > 0 ? tcc_mallocz(ntemp) : NULL; + int16_t *tbad_op = ntemp > 0 ? tcc_mallocz(sizeof(int16_t) * ntemp) : NULL; + int32_t *tbad_at = ntemp > 0 ? tcc_mallocz(sizeof(int32_t) * ntemp) : NULL; + for (int t = 0; t < ntemp; t++) + toff[t] = SRA_NO_ADDR, tfrom[t] = -1, troot[t] = -1; + for (int i = 0; i < n && ntemp > 0; i++) + { + IRQuadCompact *q = &ir->compact_instructions[i]; + if (q->op == TCCIR_OP_NOP) + continue; + int t; + /* Definitions: a value dest, or a pointer a post-increment writes back. A + * STORE_INDEXED dest is the base pointer, read and not written. */ + if (irop_config[q->op].has_dest && q->op != TCCIR_OP_STORE_INDEXED) + { + IROperand d = tcc_ir_op_get_dest(ir, q); + if ((!d.is_lval || q->op == TCCIR_OP_STORE_POSTINC) && (t = sra_temp_pos(d, ntemp)) >= 0 && defs[t] < 2) + { + def_idx[t] = defs[t] == 0 ? i : -1; + defs[t]++; + } + } + if (q->op == TCCIR_OP_LOAD_POSTINC && (t = sra_temp_pos(tcc_ir_op_get_src1(ir, q), ntemp)) >= 0) + defs[t] = 2, def_idx[t] = -1; + } + for (int round = 0, changed = 1; round < 8 && changed && ntemp > 0; round++) + { + changed = 0; + for (int i = 0; i < n; i++) + { + IRQuadCompact *q = &ir->compact_instructions[i]; + int t = sra_addr_def(ir, q, ntemp); + if (t < 0 || defs[t] != 1 || toff[t] != SRA_NO_ADDR) + continue; + IROperand a = tcc_ir_op_get_src1(ir, q); + int off, at, k; + if (q->op == TCCIR_OP_LEA || q->op == TCCIR_OP_ASSIGN) + { + if (!a.is_lval && !a.is_llocal && sra_concrete(a, &off) && (k = sra_find_unit(u, nu, off)) >= 0) + toff[t] = off, troot[t] = k; + else if (q->op == TCCIR_OP_ASSIGN && !a.is_lval && (at = sra_temp_pos(a, ntemp)) >= 0 && + toff[at] != SRA_NO_ADDR) + toff[t] = toff[at], troot[t] = troot[at], tfrom[t] = at; + } + else + { + IROperand c = tcc_ir_op_get_src2(ir, q); + if (!a.is_lval && (at = sra_temp_pos(a, ntemp)) >= 0 && toff[at] != SRA_NO_ADDR && + irop_get_tag(c) == IROP_TAG_IMM32 && !c.is_lval) + { + int64_t v = (int64_t)toff[at] + (q->op == TCCIR_OP_ADD ? 1 : -1) * (int64_t)irop_get_imm32(c); + if (v > -(1 << 24) && v < (1 << 24)) + toff[t] = (int32_t)v, troot[t] = troot[at], tfrom[t] = at; + } + } + changed |= toff[t] != SRA_NO_ADDR; + } + } + /* Pointer VARs. The Zig C backend keeps an object's address in a C local + * -- `t1 = &t0; t5 = &t1->len; x = *t5` -- which the frontend makes a VAR, + * copied into a TEMP at each use (`T <- V1 [ASSIGN]` or `[LOAD]`), and it + * reuses one local for unrelated pointers of the same type. + * + * A value is followed through such VARs in three states: a known frame + * address, NOT a frame address, or unknown. Within a block a VAR holds what + * its last definition there gave it -- `Addr[slot]`, a copy or constant + * offset of a known address, or, from any other source (a load from memory, + * a call's result, a parameter, arithmetic), not a frame address: an + * object's address reaches memory, a call or arithmetic only by a use that + * pins that object below, so such a value cannot point into an object that + * stays promotable. Across blocks a VAR that can never hold a frame + * address holds none, and one whose every definition gives the same address + * (`&obj`, a copy of such a VAR, a constant offset of one) holds it + * everywhere; otherwise its value there is unknown. + * + * A TEMP copied from a known address becomes an address root like one + * defined from the slot itself. A read of an unknown value, or of an + * address in any use but a copy or constant offset -- a compare, a call + * argument, a store, a dereference -- pins every object the VAR ever held + * ("ptrvar-*" in TCC_SRA_DBG). */ + const int nvar = sra_ptrvar() ? ir->next_local_variable : 0; + int32_t *vglob = nvar ? tcc_malloc(sizeof(int32_t) * nvar) : NULL; /* the one address, SRA_NO_ADDR, SRA_VMIX */ + uint8_t *vmay = nvar ? tcc_mallocz(nvar) : NULL; /* can hold a frame address at all */ + /* 1 read with no known value, 2 an address read other than as one, 3 address-taken or volatile */ + uint8_t *vbad = nvar ? tcc_mallocz(nvar) : NULL; + int16_t *vbad_op = nvar ? tcc_mallocz(sizeof(int16_t) * nvar) : NULL; + int32_t *vbad_at = nvar ? tcc_mallocz(sizeof(int32_t) * nvar) : NULL; /* TCC_SRA_DBG: that read's instruction */ + int32_t *vcur = nvar ? tcc_malloc(sizeof(int32_t) * nvar) : NULL; + int32_t *vcurroot = nvar ? tcc_malloc(sizeof(int32_t) * nvar) : NULL; + int *vcurgen = nvar ? tcc_mallocz(sizeof(int) * nvar) : NULL; + int32_t *vdef = nvar && n ? tcc_malloc(sizeof(int32_t) * n) : NULL; /* per instruction: the address a VAR def holds */ + int32_t *vdefroot = nvar && n ? tcc_malloc(sizeof(int32_t) * n) : NULL; + if (nvar) + { + for (int v = 0; v < nvar; v++) + vglob[v] = SRA_NO_ADDR; + for (int i = 0; i < n; i++) + vdef[i] = SRA_NO_ADDR, vdefroot[i] = -1; + /* vmay: a definition from a frame address, or a copy (or offset) of a + * VAR or TEMP that can hold one. TEMPs take part: `T <- V0; V3 <- T + 4` + * passes V0's address to V3 through a TEMP whose own address is only + * found by the walk below, so a VAR fed by it must count as a pointer + * VAR here already. */ + uint8_t *tmay = ntemp > 0 ? tcc_mallocz(ntemp) : NULL; + for (int t = 0; t < ntemp; t++) + if (toff[t] != SRA_NO_ADDR) + tmay[t] = 1; + for (int changed = 1; changed;) + { + changed = 0; + for (int i = 0; i < n; i++) + { + IRQuadCompact *q = &ir->compact_instructions[i]; + int v = sra_var_def(ir, q, nvar); + int t = v < 0 && ntemp > 0 ? sra_addr_def(ir, q, ntemp) : -1; + if ((v < 0 || vmay[v]) && (t < 0 || tmay[t])) + continue; + if (!sra_is_copy_op(ir, q, nvar)) + continue; + IROperand a = tcc_ir_op_get_src1(ir, q); + int off, at, w; + if ((!a.is_lval && !a.is_llocal && sra_concrete(a, &off) && sra_find_unit(u, nu, off) >= 0 && + q->op != TCCIR_OP_ADD && q->op != TCCIR_OP_SUB && q->op != TCCIR_OP_LOAD) || + (!a.is_lval && (at = sra_temp_pos(a, ntemp)) >= 0 && tmay[at]) || + (!a.is_llocal && (w = sra_var_pos(a, nvar)) >= 0 && vmay[w])) + { + if (v >= 0) + vmay[v] = 1; + else + tmay[t] = 1; + changed = 1; + } + } + } + /* vglob: the one address a VAR holds wherever it is read. Optimistic over + * copy cycles: a VAR that can hold an address but is not known yet + * contributes nothing this round. */ + for (int round = 0, changed = 1; changed && round < 16; round++) + { + changed = 0; + for (int i = 0; i < n; i++) + { + IRQuadCompact *q = &ir->compact_instructions[i]; + int v = sra_var_def(ir, q, nvar); + if (v < 0 || !vmay[v] || vglob[v] == SRA_VMIX) + continue; + int32_t base = SRA_VMIX; + if (sra_is_copy_op(ir, q, nvar)) + { + IROperand a = tcc_ir_op_get_src1(ir, q); + int off, at, w; + if (!a.is_lval && !a.is_llocal && sra_concrete(a, &off) && sra_find_unit(u, nu, off) >= 0 && + q->op != TCCIR_OP_ADD && q->op != TCCIR_OP_SUB && q->op != TCCIR_OP_LOAD) + base = off; + else if (!a.is_lval && (at = sra_temp_pos(a, ntemp)) >= 0 && toff[at] != SRA_NO_ADDR && troot[at] >= 0) + base = toff[at]; + else if (!a.is_llocal && (w = sra_var_pos(a, nvar)) >= 0 && vmay[w]) + { + if (vglob[w] == SRA_NO_ADDR) + continue; /* not known yet */ + base = vglob[w]; + } + if (base != SRA_VMIX && (q->op == TCCIR_OP_ADD || q->op == TCCIR_OP_SUB)) + base = sra_offset_by(ir, q, base); + if (base != SRA_VMIX && sra_find_unit(u, nu, base) < 0) + base = SRA_VMIX; + } + int32_t nv2 = vglob[v] == SRA_NO_ADDR ? base : vglob[v] == base ? base : SRA_VMIX; + if (nv2 != vglob[v]) + vglob[v] = nv2, changed = 1; + } + } + for (int v = 0; v < nvar; v++) + { + IRLiveInterval *iv = tcc_ir_get_live_interval(ir, TCCIR_ENCODE_VREG(TCCIR_VREG_TYPE_VAR, v)); + if (!iv || iv->addrtaken || iv->is_volatile) + vbad[v] = 3, vglob[v] = SRA_VMIX, vmay[v] = 1; + } + /* Across blocks, by dataflow: a pointer VAR's value entering a block is + * the meet of its value leaving every predecessor -- the same frame + * address, or no frame address, on all of them, else unknown. Blocks + * the entry does not reach start unknown. Past SRA_DF_MAX cells the + * block-local rule above stands. */ + int npv = 0; + int32_t *pv = tcc_malloc(sizeof(int32_t) * nvar); + for (int v = 0; v < nvar; v++) + pv[v] = vmay[v] ? npv++ : -1; + IRCFG *cfg = npv && sra_ptrvar_df() ? tcc_ir_cfg_build(ir) : NULL; + int32_t *entry = NULL; + if (cfg && cfg->num_blocks > 0 && (int64_t)cfg->num_blocks * npv <= SRA_DF_MAX) + { + tcc_ir_cfg_compute_rpo(cfg); + const int nb = cfg->num_blocks; + entry = tcc_malloc(sizeof(int32_t) * nb * npv); + for (int k = 0; k < nb * npv; k++) + entry[k] = SRA_NO_ADDR; /* top: no path seen yet */ + for (int k = 0; k < npv; k++) + entry[k] = SRA_NOT_ADDR; /* block 0: nothing assigned yet */ + int32_t *st = tcc_malloc(sizeof(int32_t) * npv); + int32_t *tv = ntemp > 0 ? tcc_malloc(sizeof(int32_t) * ntemp) : NULL; + int *tstamp = ntemp > 0 ? tcc_mallocz(sizeof(int) * ntemp) : NULL; + int stamp = 0, changed = 1; + for (int round = 0; changed && round < 64; round++) + { + changed = 0; + for (int r = 0; r < cfg->rpo_count; r++) + { + const int b = cfg->rpo_order[r]; + IRBasicBlock *bb = &cfg->blocks[b]; + memcpy(st, entry + (size_t)b * npv, sizeof(int32_t) * npv); + stamp++; + for (int i = bb->start_idx; i < bb->end_idx; i++) + { + IRQuadCompact *q = &ir->compact_instructions[i]; + if (q->op == TCCIR_OP_NOP) + continue; + int vd = sra_var_def(ir, q, nvar); + int td = ntemp > 0 ? sra_addr_def(ir, q, ntemp) : -1; + if (td >= 0 && defs[td] != 1) + td = -1; + if ((vd < 0 || pv[vd] < 0) && td < 0) + continue; + int32_t val = SRA_NOT_ADDR; + if (sra_is_copy_op(ir, q, nvar)) + { + IROperand a = tcc_ir_op_get_src1(ir, q); + int off, at, w; + int32_t base = SRA_NOT_ADDR; + if (!a.is_lval && !a.is_llocal && sra_concrete(a, &off) && q->op != TCCIR_OP_ADD && + q->op != TCCIR_OP_SUB && q->op != TCCIR_OP_LOAD) + base = off; + else if (!a.is_lval && (at = sra_temp_pos(a, ntemp)) >= 0 && toff[at] != SRA_NO_ADDR) + base = toff[at]; + else if (!a.is_lval && (at = sra_temp_pos(a, ntemp)) >= 0 && tmay[at]) + base = tstamp[at] == stamp ? tv[at] : SRA_UNKNOWN; + else if (!a.is_llocal && (w = sra_var_pos(a, nvar)) >= 0) + base = pv[w] >= 0 ? st[pv[w]] : SRA_NOT_ADDR; + if (base >= SRA_FIRST_ADDR && (q->op == TCCIR_OP_ADD || q->op == TCCIR_OP_SUB)) + base = sra_offset_by(ir, q, base); + val = base == SRA_VMIX ? SRA_UNKNOWN + : base >= SRA_FIRST_ADDR && sra_find_unit(u, nu, base) < 0 ? SRA_UNKNOWN + : base; + } + if (vd >= 0 && pv[vd] >= 0) + st[pv[vd]] = val; + else if (td >= 0 && ntemp > 0) + tv[td] = val, tstamp[td] = stamp; + } + for (int e = 0; e < bb->num_succs; e++) + { + int32_t *dst = entry + (size_t)bb->succs[e] * npv; + for (int k = 0; k < npv; k++) + { + int32_t m = dst[k] == SRA_NO_ADDR ? st[k] : st[k] == SRA_NO_ADDR || st[k] == dst[k] ? dst[k] : SRA_UNKNOWN; + if (m != dst[k]) + dst[k] = m, changed = 1; + } + } + } + } + tcc_free(st); + tcc_free(tv); + tcc_free(tstamp); + if (changed) /* did not settle: fall back */ + { + tcc_free(entry); + entry = NULL; + } + } + + int *bs = tcc_mallocz(sizeof(int) * (n + 1)); + ir_opt_mark_block_starts(ir, bs, 1, n); + int gen = 1; + for (int i = 0; i < n; i++) + { + IRQuadCompact *q = &ir->compact_instructions[i]; + if (entry && cfg->instr_to_block[i] >= 0 && cfg->blocks[cfg->instr_to_block[i]].start_idx == i) + { + /* Enter the block with the dataflow's values. */ + gen++; + const int32_t *e = entry + (size_t)cfg->instr_to_block[i] * npv; + for (int v = 0; v < nvar; v++) + if (pv[v] >= 0) + { + int32_t x = e[pv[v]] == SRA_NO_ADDR ? SRA_UNKNOWN : e[pv[v]]; + vcur[v] = x, vcurgen[v] = gen; + vcurroot[v] = x >= SRA_FIRST_ADDR ? sra_find_unit(u, nu, x) : -1; + } + } + else if (!entry && i && bs[i] == 1) + gen++; + if (q->op == TCCIR_OP_NOP) + continue; + int vd = sra_var_def(ir, q, nvar); + int td = ntemp > 0 ? sra_addr_def(ir, q, ntemp) : -1; + if (td >= 0 && defs[td] != 1) + td = -1; + /* What the instruction's destination holds. */ + int32_t val = SRA_NOT_ADDR, root = -1; + int src_var = -1; /* the VAR a followed copy reads */ + if ((vd >= 0 || td >= 0) && sra_is_copy_op(ir, q, nvar)) + { + IROperand a = tcc_ir_op_get_src1(ir, q); + int off, at, w; + int32_t base = SRA_NOT_ADDR, broot = -1; + if (!a.is_lval && !a.is_llocal && sra_concrete(a, &off) && q->op != TCCIR_OP_ADD && + q->op != TCCIR_OP_SUB && q->op != TCCIR_OP_LOAD) + base = off, broot = sra_find_unit(u, nu, off); + else if (!a.is_lval && (at = sra_temp_pos(a, ntemp)) >= 0 && toff[at] != SRA_NO_ADDR) + base = toff[at], broot = troot[at]; + else if (!a.is_lval && (at = sra_temp_pos(a, ntemp)) >= 0 && tmay[at]) + base = SRA_UNKNOWN; /* may hold an address the walk did not resolve */ + else if (!a.is_llocal && (w = sra_var_pos(a, nvar)) >= 0) + { + if (vcurgen[w] == gen) + base = vcur[w], broot = vcurroot[w]; + else if (!vmay[w]) + base = SRA_NOT_ADDR; + else if (vglob[w] != SRA_NO_ADDR && vglob[w] != SRA_VMIX) + base = vglob[w], broot = sra_find_unit(u, nu, vglob[w]); + else + base = SRA_UNKNOWN; + src_var = w; + } + if (base >= SRA_FIRST_ADDR && broot >= 0) + { + if (q->op == TCCIR_OP_ADD || q->op == TCCIR_OP_SUB) + base = sra_offset_by(ir, q, base); + if (base == SRA_VMIX) + val = SRA_UNKNOWN; /* a variable offset of an address: pinned below */ + else + val = base, root = broot; + } + else + val = base == SRA_UNKNOWN ? SRA_UNKNOWN : SRA_NOT_ADDR; + } + /* Reads of VARs that may hold an address. */ + for (int s2 = 0; s2 < 4; s2++) + { + IROperand o = sra_operand(ir, q, s2); + int v = sra_var_pos(o, nvar); + if (v < 0 || (s2 == 0 && ((q->op != TCCIR_OP_STORE && q->op != TCCIR_OP_STORE_INDEXED && !o.is_llocal) || + sra_var_def(ir, q, nvar) == v))) + continue; /* a definition, not a read */ + int32_t st = vcurgen[v] == gen ? vcur[v] + : !vmay[v] ? SRA_NOT_ADDR + : vglob[v] != SRA_NO_ADDR && vglob[v] != SRA_VMIX ? vglob[v] + : SRA_UNKNOWN; + if (st == SRA_NOT_ADDR) + continue; + if (st != SRA_UNKNOWN && s2 == 1 && v == src_var && val >= SRA_FIRST_ADDR) + continue; /* a followed copy or offset */ + if (!vbad[v]) + { + vbad[v] = st == SRA_UNKNOWN ? 1 : 2; + vbad_op[v] = (int16_t)q->op; + vbad_at[v] = i; + } + } + if (vd >= 0) + { + vcur[vd] = val, vcurroot[vd] = root, vcurgen[vd] = gen; + if (val >= SRA_FIRST_ADDR) + vdef[i] = val, vdefroot[i] = root; + } + else if (td >= 0 && toff[td] == SRA_NO_ADDR && val >= SRA_FIRST_ADDR) + toff[td] = val, troot[td] = root, tfrom[td] = -1; + } + tcc_free(bs); + tcc_free(tmay); + tcc_free(entry); + tcc_free(pv); + if (cfg) + tcc_ir_cfg_free(cfg); + /* A VAR pinned: every object it held keeps its memory. */ + for (int i = 0; i < n; i++) + { + int v = vdef[i] != SRA_NO_ADDR ? sra_var_def(ir, &ir->compact_instructions[i], nvar) : -1; + if (v >= 0 && vbad[v] && vdefroot[i] >= 0) + { + u[vdefroot[i]].bad = 1; + sra_dbg_at = vbad[v] == 3 ? -1 : vbad_at[v]; + sra_why(&u[vdefroot[i]], vbad[v] == 1 ? "ptrvar-unknown" : vbad[v] == 2 ? "ptrvar-use" : "ptrvar-addrtaken", + vbad_op[v]); + sra_dbg_at = -1; + } + } + } + + /* Calls by id: the instruction, arguments 0-2, and whether it is a copy + * helper -- for the calls that access an object whole (SraWhole). */ + const int nwcall = ir->next_call_id + 1; + int32_t *call_at = tcc_malloc(sizeof(int32_t) * nwcall); + int32_t(*cparam)[3] = tcc_malloc(sizeof(*cparam) * nwcall); + uint8_t *ckind = tcc_mallocz(nwcall); + for (int c = 0; c < nwcall; c++) + call_at[c] = cparam[c][0] = cparam[c][1] = cparam[c][2] = -1; + for (int i = 0; i < n; i++) + { + IRQuadCompact *q = &ir->compact_instructions[i]; + if (q->op == TCCIR_OP_FUNCPARAMVAL) + { + uint32_t e = (uint32_t)irop_get_imm64_ex(ir, tcc_ir_op_get_src2(ir, q)); + int c = TCCIR_DECODE_CALL_ID(e), p = TCCIR_DECODE_PARAM_IDX(e); + if (c >= 0 && c < nwcall && p >= 0 && p < 3) + cparam[c][p] = cparam[c][p] == -1 ? i : -2; + } + else if (q->op == TCCIR_OP_FUNCCALLVAL || q->op == TCCIR_OP_FUNCCALLVOID) + { + int c = TCCIR_DECODE_CALL_ID((uint32_t)irop_get_imm64_ex(ir, tcc_ir_op_get_src2(ir, q))); + if (c >= 0 && c < nwcall) + { + call_at[c] = call_at[c] == -1 ? i : INT32_MAX; /* twice: never matches */ + ckind[c] = (uint8_t)sra_call_kind(ir, q); + } + } + } + SraWhole *whole = NULL; + int nwhole = 0; + + /* A use other than as an access base or the source of a derived address + * lets the address escape. */ + for (int i = 0; i < n && ntemp > 0; i++) + { + IRQuadCompact *q = &ir->compact_instructions[i]; + if (q->op == TCCIR_OP_NOP) + continue; + int disp, dt = sra_addr_def(ir, q, ntemp); + IROperand acc; + int bt = sra_access_through(ir, q, ntemp, &disp, &acc); + int base_slot = q->op == TCCIR_OP_STORE || q->op == TCCIR_OP_STORE_INDEXED ? 0 : 1; + for (int s = 0; s < 4; s++) + { + int t = sra_temp_pos(sra_operand(ir, q, s), ntemp); + if (t < 0 || toff[t] == SRA_NO_ADDR) + continue; + if (s == 0 && dt == t) + continue; /* its own definition */ + if (s == base_slot && bt == t) + continue; + if (s == 1 && dt >= 0 && tfrom[dt] == t) + continue; + if (s == 1 && vdef && vdef[i] != SRA_NO_ADDR) + continue; /* derives a pointer VAR's address */ + int pw; + IROperand po = sra_operand(ir, q, s); + if (s == 1 && q->op == TCCIR_OP_FUNCPARAMVAL && struct_param && troot[t] >= 0 && + toff[t] == u[troot[t]].lo && po.is_lval && (pw = sra_struct_param_words(po)) > 0 && + !tcc_ir_access_is_volatile(ir, po)) + { + /* A by-value struct argument read through the object's address. */ + if (pw > u[troot[t]].param_words) + u[troot[t]].param_words = pw; + continue; + } + SraWhole w; + if (s == 1 && q->op == TCCIR_OP_FUNCPARAMVAL && troot[t] >= 0 && + sra_whole_use(ir, i, call_at, (const int32_t(*)[3])cparam, ckind, nwcall, &w)) + { + int ok = 1; + if (w.sret) + { + /* The buffer's address must still be computed for the call: from + * the slot itself, through TEMPs only. */ + int r = t; + while (tfrom[r] >= 0) + r = tfrom[r]; + IRQuadCompact *rq = def_idx[r] >= 0 ? &ir->compact_instructions[def_idx[r]] : NULL; + IROperand a = rq ? tcc_ir_op_get_src1(ir, rq) : IROP_NONE; + int aoff; + if (!rq || (rq->op != TCCIR_OP_LEA && rq->op != TCCIR_OP_ASSIGN) || a.is_lval || a.is_llocal || + !sra_concrete(a, &aoff)) + ok = 0; + else + w.slot = a; + } + else + { + w.other_t = sra_temp_pos(tcc_ir_op_get_src1(ir, &ir->compact_instructions[w.other]), ntemp); + if (w.other_t >= 0 && toff[w.other_t] != SRA_NO_ADDR && troot[w.other_t] == troot[t]) + ok = 0; /* within one object */ + } + if (ok) + { + w.unit = troot[t]; + w.off = toff[t]; + w.t = t; + whole = tcc_realloc(whole, sizeof(SraWhole) * (nwhole + 1)); + whole[nwhole++] = w; + continue; + } + } + if (!tbad[t] && tbad_op) + tbad_op[t] = (int16_t)(q->op * 4 + s), tbad_at[t] = i; /* TCC_SRA_DBG: op and operand slot */ + tbad[t] = 1; + } + } + for (int changed = 1; changed && ntemp > 0;) + { + changed = 0; + for (int t = 0; t < ntemp; t++) + if (tbad[t] && tfrom[t] >= 0 && !tbad[tfrom[t]]) + { + tbad[tfrom[t]] = 1, changed = 1; + if (tbad_op) + tbad_op[tfrom[t]] = tbad_op[t], tbad_at[tfrom[t]] = tbad_at[t]; + } + } + /* A root TEMP copied from a pointer VAR has no slot operand to carry the + * escape into the object's reference count: pin the object directly. */ + for (int t = 0; nvar && t < ntemp; t++) + if (tbad[t] && tfrom[t] < 0 && toff[t] != SRA_NO_ADDR && troot[t] >= 0 && def_idx[t] >= 0) + { + int off; + IROperand a = tcc_ir_op_get_src1(ir, &ir->compact_instructions[def_idx[t]]); + if (!sra_concrete(a, &off)) + { + u[troot[t]].bad = 1; + sra_dbg_at = tbad_at ? tbad_at[t] : -1; + sra_why(&u[troot[t]], "ptrvar-escape", tbad_op ? tbad_op[t] : 0); + sra_dbg_at = -1; + } + } + + /* Every reference a live instruction makes, the fourth operand slot of + * MLA / indexed accesses / SELECT included: an object keeps its memory + * unless all of them can be replaced. */ + for (int i = 0; i < n; i++) + { + IRQuadCompact *q = &ir->compact_instructions[i]; + if (q->op == TCCIR_OP_NOP) + continue; + int dt = ntemp > 0 ? sra_addr_def(ir, q, ntemp) : -1; + for (int s = 0; s < 4; s++) + { + IROperand o = sra_operand(ir, q, s); + int off, k; + if (!sra_concrete(o, &off) || (k = sra_find_unit(u, nu, off)) < 0) + continue; + u[k].refs++; + if (s == 1 && vdef && vdef[i] != SRA_NO_ADDR) + { + u[k].ok_refs++; /* a pointer VAR's root; sra's walk pinned it if it escapes */ + continue; + } + if (s == 1 && dt >= 0 && toff[dt] != SRA_NO_ADDR && tfrom[dt] < 0) + { + if (!tbad[dt]) + u[k].ok_refs++; /* the root of an address used only for field accesses */ + else + { + sra_dbg_at = tbad_at ? tbad_at[dt] : -1; + sra_why(&u[k], "addr-escape", tbad_op ? tbad_op[dt] : q->op); + sra_dbg_at = -1; + } + continue; + } + if (s < 3 && !no_wide && sra_wide_access(ir, q, s, o)) + { + sra_note_wide(&u[k], off); + u[k].ok_refs++; + continue; + } + int pw; + if (s == 1 && q->op == TCCIR_OP_FUNCPARAMVAL && struct_param && off == u[k].lo && + (pw = sra_struct_param_words(o)) > 0 && !tcc_ir_access_is_volatile(ir, o)) + { + if (pw > u[k].param_words) + u[k].param_words = pw; + u[k].ok_refs++; + continue; + } + SraWhole w; + if (s == 1 && q->op == TCCIR_OP_FUNCPARAMVAL && !o.is_lval && !o.is_llocal && + sra_whole_use(ir, i, call_at, (const int32_t(*)[3])cparam, ckind, nwcall, &w)) + { + if (!w.sret) + { + w.other_t = ntemp > 0 ? sra_temp_pos(tcc_ir_op_get_src1(ir, &ir->compact_instructions[w.other]), ntemp) : -1; + if (w.other_t >= 0 && toff[w.other_t] != SRA_NO_ADDR && troot[w.other_t] == k) + w.other_t = -2; /* within one object */ + } + if (w.other_t != -2) + { + w.unit = k; + w.off = off; + w.slot = o; + whole = tcc_realloc(whole, sizeof(SraWhole) * (nwhole + 1)); + whole[nwhole++] = w; + u[k].ok_refs++; + continue; + } + } + int bytes = s < 3 ? sra_slot_bytes(ir, q->op, s, o) : 0; + if (bytes && (irop_get_btype(o) != IROP_BTYPE_STRUCT || off == u[k].lo)) + { + if (bytes < 4 && s == 0 && q->op == TCCIR_OP_STORE && off - u[k].lo >= 0 && off - u[k].lo < SRA_MAX_BYTES) + { + if (!sra_narrow_store_fits(ir, q, bytes, def_idx, ntemp)) + u[k].nofit[off - u[k].lo] |= (uint8_t)bytes; + if (!irop_is_plain_imm(tcc_ir_op_get_src1(ir, q))) + { + u[k].stvar[off - u[k].lo] |= (uint8_t)bytes; + (bytes == 1 ? u[k].nvst1 : u[k].nvst2)[off - u[k].lo]++; + } + } + sra_note_field(&u[k], off, bytes, o.is_unsigned, s == 0); + u[k].ok_refs++; + } + else + { + sra_dbg_at = i; + sra_why(&u[k], s == 3 ? "slot3" : bytes ? "struct-offset" : sra_ref_kind(q->op, s, o), q->op); + sra_dbg_at = -1; + } + } + int disp; + IROperand acc; + int t = ntemp > 0 ? sra_access_through(ir, q, ntemp, &disp, &acc) : -1; + if (t >= 0 && toff[t] != SRA_NO_ADDR && !tbad[t] && troot[t] >= 0) + { + int off = toff[t] + disp, k = sra_find_unit(u, nu, off); + if (k != troot[t]) + { + u[troot[t]].bad = 1; /* strays outside the object */ + sra_why(&u[troot[t]], "stray", q->op); + } + else + { + u[k].refs++; + u[k].ok_refs++; + int abytes = sra_btype_bytes(irop_get_btype(acc)); + /* A narrow store must already fit its field (sra_narrow_store_fits). + * An indexed one by a constant is the same store: its value is src1 + * too, and the ASSIGN it becomes reads only dest and src1. */ + if (abytes < 4 && (q->op == TCCIR_OP_STORE || q->op == TCCIR_OP_STORE_INDEXED) && sra_narrow_idx_off() && + q->op == TCCIR_OP_STORE_INDEXED) + { + u[k].bad = 1; + sra_why(&u[k], "narrow-store", q->op); + } + else if (abytes < 4 && (q->op == TCCIR_OP_STORE || q->op == TCCIR_OP_STORE_INDEXED) && off - u[k].lo >= 0 && + off - u[k].lo < SRA_MAX_BYTES) + { + if (!sra_narrow_store_fits(ir, q, abytes, def_idx, ntemp)) + u[k].nofit[off - u[k].lo] |= (uint8_t)abytes; + if (!irop_is_plain_imm(tcc_ir_op_get_src1(ir, q))) + { + u[k].stvar[off - u[k].lo] |= (uint8_t)abytes; + (abytes == 1 ? u[k].nvst1 : u[k].nvst2)[off - u[k].lo]++; + } + } + sra_note_field(&u[k], off, abytes, acc.is_unsigned, + q->op == TCCIR_OP_STORE || q->op == TCCIR_OP_STORE_INDEXED); + } + } + } + { + uint8_t *has_whole = tcc_mallocz(nu); + for (int x = 0; x < nwhole; x++) + has_whole[whole[x].unit] = 1; + for (int k = 0; k < nu; k++) + if (!u[k].bad) + { + sra_resolve_fields(&u[k], has_whole[k]); + sra_partial_cost(&u[k], has_whole[k]); + } + tcc_free(has_whole); + } + for (int k = 0; k < nu; k++) + if (!u[k].bad) + sra_resolve_wide(&u[k]); + /* Ranges accessed whole: inside the object, with a field for each byte a + * call writes and a later whole read passes on, and no field across an + * end. */ + for (int x = 0; x < nwhole; x++) + { + SraUnit *uk = &u[whole[x].unit]; + int a0 = whole[x].off - uk->lo; + if (a0 < 0 || a0 + whole[x].n > uk->hi - uk->lo || a0 + whole[x].n > SRA_MAX_BYTES) + { + uk->bad = 1; + sra_why(uk, "whole-range", 0); + } + else if (whole[x].dir) + uk->whole_r = 1; + else + uk->whole_w = 1; + } + for (int x = 0; x < nwhole; x++) + { + SraUnit *uk = &u[whole[x].unit]; + if (!uk->bad && !whole[x].dir && (uk->whole_r || uk->param_words)) + sra_fill_gaps(uk, whole[x].off - uk->lo, whole[x].n); + } + for (int x = 0; x < nwhole; x++) + { + SraUnit *uk = &u[whole[x].unit]; + if (!uk->bad && !sra_range_clean(uk, whole[x].off - uk->lo, whole[x].n)) + { + uk->bad = 1; + sra_why(uk, "whole-straddle", 0); + } + } + /* Worth it only when the field copies (a load or store per field) cost no + * more than the block copies they replace (an LDM/STM pair per 16 bytes + * and the setup), plus the field accesses that stop going to memory. */ + if (nwhole) + { + int32_t *wcost = tcc_mallocz(sizeof(int32_t) * nu); + for (int x = 0; x < nwhole; x++) + { + SraUnit *uk = &u[whole[x].unit]; + if (uk->bad) + continue; + int a0 = whole[x].off - uk->lo, fields = 0; + for (int at = a0; at < a0 + whole[x].n; at++) + fields += uk->width[at] == 8 ? 2 : uk->width[at] ? 1 : 0; + wcost[whole[x].unit] += fields - (whole[x].sret ? 0 : 2 + 2 * ((whole[x].n + 15) / 16)); + } + for (int k = 0; k < nu; k++) + if (!u[k].bad && (u[k].whole_w || u[k].whole_r) && wcost[k] > u[k].field_refs) + { + u[k].bad = 1; + sra_why(&u[k], "whole-cost", 0); + } + tcc_free(wcost); + } + for (int k = 0; k < nu; k++) + { + if (!sra_fields_disjoint(&u[k])) + sra_why(&u[k], "overlap", 0); + else if (!sra_narrow_fields_unsigned(&u[k])) + sra_why(&u[k], "narrow-signed", 0); + if (u[k].ok_refs != u[k].refs || !sra_fields_disjoint(&u[k]) || !sra_narrow_fields_unsigned(&u[k])) + u[k].bad = 1; + /* Each word a call receives by value must be one field VAR: a 32-bit + * field, or one narrow unsigned field at its start (its VAR holds it + * zero-extended; the other bytes are padding or never written, so what + * the callee sees there is indeterminate either way). No 64-bit field. */ + for (int w = 0; w < u[k].param_words && !u[k].bad; w++) + { + int fields = 0; + for (int b = 4 * w; b < 4 * w + 4; b++) + { + if (u[k].has64[b] || u[k].split[b] || u[k].width[b] > 4) + fields += 2; + else if (u[k].width[b]) + fields += b == 4 * w ? 1 : 2; + } + if (fields > 1) + { + u[k].bad = 1; + sra_why(&u[k], "param-word", TCCIR_OP_FUNCPARAMVAL); + } + } + } + /* A copy between two objects that would both be replaced becomes copies + * between their field VARs when the two lay their fields out alike over + * the copied bytes (a field of one is never split across two of the + * other's); otherwise one of them keeps its memory. */ + for (int x = 0; x < nwhole; x++) + { + int ot = whole[x].other_t, k = whole[x].unit; + if (!whole[x].sret && ot >= 0 && toff[ot] != SRA_NO_ADDR && troot[ot] >= 0 && !u[troot[ot]].bad && !u[k].bad) + { + const int k2 = troot[ot], a1 = whole[x].off - u[k].lo, a2 = toff[ot] - u[k2].lo, n2 = whole[x].n; + int alike = !sra_pair_off() && a2 >= 0 && a2 + n2 <= u[k2].hi - u[k2].lo && a2 + n2 <= SRA_MAX_BYTES && + sra_range_clean(&u[k2], a2, n2); + for (int d = 0; alike && d < n2; d++) + alike = u[k].width[a1 + d] == u[k2].width[a2 + d]; + if (alike) + { + whole[x].pair = k2; + whole[x].pair_off = toff[ot]; + continue; + } + u[k].bad = 1; + sra_why(&u[k], "copy-pair", 0); + } + } + /* The function's field VARs, against the budget SSA promotion has. */ + { + int nvars = 0; + for (int k = 0; k < nu; k++) + for (int at = 0; !u[k].bad && at < SRA_MAX_BYTES; at++) + nvars += u[k].width[at] ? 1 : 0; + if (nvars > sra_var_budget()) + { + for (int k = 0; k < nu; k++) + if (!u[k].bad && u[k].partial) + { + u[k].bad = 1; + sra_why(&u[k], "budget", 0); + } + for (int x = 0; x < nwhole; x++) + if (whole[x].pair >= 0 && !u[whole[x].unit].bad && !u[whole[x].pair].bad) + { + u[whole[x].unit].bad = 1; + sra_why(&u[whole[x].unit], "budget", 0); + } + } + } + TCC_DBG_BLOCK(sra_dbg) + { + extern const char *funcname; + for (int k = 0; k < nu; k++) + { + const char *callee = "-"; + int pidx = -1; + int at = u[k].bad && u[k].why ? u[k].why_at : -1; + if (at >= 0 && at < n && ir->compact_instructions[at].op == TCCIR_OP_FUNCPARAMVAL) + { + uint32_t e = (uint32_t)irop_get_imm64_ex(ir, tcc_ir_op_get_src2(ir, &ir->compact_instructions[at])); + int c = TCCIR_DECODE_CALL_ID(e); + pidx = TCCIR_DECODE_PARAM_IDX(e); + for (int j = at + 1; j < n; j++) + { + IRQuadCompact *cq = &ir->compact_instructions[j]; + if ((cq->op == TCCIR_OP_FUNCCALLVAL || cq->op == TCCIR_OP_FUNCCALLVOID) && + TCCIR_DECODE_CALL_ID((uint32_t)irop_get_imm64_ex(ir, tcc_ir_op_get_src2(ir, cq))) == c) + { + Sym *cs = irop_get_sym_ex(ir, tcc_ir_op_get_src1(ir, cq)); + callee = cs ? get_tok_str(cs->v, NULL) : "(indirect)"; + break; + } + } + } + fprintf(stderr, "[SRA] %s size=%d refs=%d %s op=%d callee=%s param=%d lo=%d acc=", funcname, u[k].hi - u[k].lo, + u[k].refs, !u[k].bad ? "PROMOTED" : u[k].why ? u[k].why : "bad", u[k].why_op, callee, pidx, u[k].lo); + for (int at = 0; at < SRA_MAX_BYTES; at++) + if (u[k].acc[at] || u[k].has64[at]) + fprintf(stderr, "%d:L%d/S%d%s,", at, u[k].acc[at] & 7, u[k].acc[at] >> 3, u[k].has64[at] ? "/W" : ""); + fprintf(stderr, " whole=%d%d pw=%d pad=%llx\n", u[k].whole_w, u[k].whole_r, u[k].param_words, + (unsigned long long)u[k].pad); + } + } + + for (int k = 0; k < nu; k++) + if (!u[k].bad) + sra_index_fields(&u[k]); + + int changes = 0; + SraInsList ins = {0}; + + /* By-value struct arguments of replaced objects become one argument per + * word: the call's later arguments shift up and its argc grows. Collected + * first, so every index is final when an instruction is rewritten. */ + const int ncall = ir->next_call_id + 1; + int32_t *call_extra = NULL; /* per call: extra arguments in total */ + int32_t *split_first = NULL; /* per call: its first split argument in splits[], or -1 */ + int32_t(*splits)[3] = NULL; /* { argument index, extra words, next in this call } */ + int nsplit = 0; + if (struct_param) + { + for (int i = 0; i < n; i++) + { + IRQuadCompact *q = &ir->compact_instructions[i]; + IROperand o; + int off, k, pw; + if (q->op != TCCIR_OP_FUNCPARAMVAL || !sra_param_at(o = tcc_ir_op_get_src1(ir, q), ntemp, toff, &off) || + (k = sra_find_unit(u, nu, off)) < 0 || u[k].bad || off != u[k].lo || !(pw = sra_struct_param_words(o))) + continue; + int c = TCCIR_DECODE_CALL_ID((uint32_t)irop_get_imm64_ex(ir, tcc_ir_op_get_src2(ir, q))); + if (c < 0 || c >= ncall) + continue; + if (!call_extra) + { + call_extra = tcc_mallocz(sizeof(int32_t) * ncall); + split_first = tcc_malloc(sizeof(int32_t) * ncall); + for (int x = 0; x < ncall; x++) + split_first[x] = -1; + } + call_extra[c] += pw - 1; + splits = tcc_realloc(splits, sizeof(*splits) * (nsplit + 1)); + splits[nsplit][0] = TCCIR_DECODE_PARAM_IDX((uint32_t)irop_get_imm64_ex(ir, tcc_ir_op_get_src2(ir, q))); + splits[nsplit][1] = pw - 1; + splits[nsplit][2] = split_first[c]; + split_first[c] = nsplit++; + } + } + + /* Whole-range calls of replaced objects: a copy becomes its field copies, a + * result buffer keeps its address and is read back into the fields. */ + uint8_t *tkeep = ntemp > 0 ? tcc_mallocz(ntemp) : NULL; + for (int x = 0; x < nwhole; x++) + { + SraWhole *e = &whole[x]; + if (u[e->unit].bad) + continue; + if (e->sret) + { + for (int t = e->t; t >= 0; t = tfrom[t]) + tkeep[t] = 1; + sra_expand_whole(ir, &ins, &u[e->unit], e, IROP_NONE); + } + else if (e->pair >= 0 && !u[e->pair].bad) + { + /* Field VARs of the source to those of the destination; the entry of + * the side the call reads adds nothing. */ + if (e->dir == 0) + sra_expand_pair(ir, &ins, &u[e->unit], e->off - u[e->unit].lo, &u[e->pair], e->pair_off - u[e->pair].lo, e->n, + e->call); + for (int k = 0; k < 3; k++) + ir->compact_instructions[cparam[e->cid][k]].op = TCCIR_OP_NOP; + ir->compact_instructions[e->call].op = TCCIR_OP_NOP; + } + else + { + IROperand base = tcc_ir_op_get_src1(ir, &ir->compact_instructions[e->other]); + sra_expand_whole(ir, &ins, &u[e->unit], e, base); + for (int k = 0; k < 3; k++) + ir->compact_instructions[cparam[e->cid][k]].op = TCCIR_OP_NOP; + ir->compact_instructions[e->call].op = TCCIR_OP_NOP; + } + changes++; + } + + for (int i = 0; i < n; i++) + { + IRQuadCompact *q = &ir->compact_instructions[i]; + if (q->op == TCCIR_OP_NOP) + continue; + /* Address arithmetic of a replaced object: nothing reads it any more. */ + if (vdef && vdef[i] != SRA_NO_ADDR && vdefroot[i] >= 0 && !u[vdefroot[i]].bad) + { + q->op = TCCIR_OP_NOP; + changes++; + continue; + } + int dt = ntemp > 0 ? sra_addr_def(ir, q, ntemp) : -1; + if (dt >= 0 && toff[dt] != SRA_NO_ADDR && troot[dt] >= 0 && !tbad[dt] && !u[troot[dt]].bad && !tkeep[dt]) + { + q->op = TCCIR_OP_NOP; + changes++; + continue; + } + int disp; + IROperand acc; + int t = ntemp > 0 ? sra_access_through(ir, q, ntemp, &disp, &acc) : -1; + if (t >= 0 && toff[t] != SRA_NO_ADDR && !tbad[t] && troot[t] >= 0 && !u[troot[t]].bad) + { + int k = troot[t], at = toff[t] + disp - u[k].lo; + int is_store = q->op == TCCIR_OP_STORE || q->op == TCCIR_OP_STORE_INDEXED; + const int abytes = sra_btype_bytes(irop_get_btype(acc)); + const int fs = u[k].fstart[at]; + if (fs != 0xFF && (fs != at || (is_store && abytes < u[k].width[fs]) || + (!is_store && abytes == u[k].width[fs] && u[k].pmask[fs]))) + { + /* Part of a wider field (sra_resolve_fields). */ + if (is_store && sra_partial_store_padded(ir, i, &u[k], fs, at - fs, abytes, tcc_ir_op_get_src1(ir, q))) + ; + else if (is_store && sra_bfi() && !irop_is_plain_imm(tcc_ir_op_get_src1(ir, q))) + sra_partial_store_bfi(ir, i, &u[k], fs, at - fs, abytes, tcc_ir_op_get_src1(ir, q)); + else if (is_store) + sra_partial_store(ir, &ins, i, &u[k], fs, at - fs, abytes, tcc_ir_op_get_src1(ir, q), + !(u[k].nofit[at] & abytes)); + else + sra_partial_load(ir, &ins, i, &u[k], fs, at - fs, abytes, acc.is_unsigned, tcc_ir_op_get_dest(ir, q)); + changes++; + continue; + } + if (u[k].var[at] < 0) + u[k].var[at] = tcc_ir_vreg_alloc_var(ir); + IRLiveInterval *iv = tcc_ir_get_live_interval(ir, u[k].var[at]); + const int narrow = u[k].width[at] && u[k].width[at] < 4; + const int extract = !is_store && abytes && abytes < u[k].width[at] && u[k].width[at] <= 4; + const int rbt = narrow || extract ? IROP_BTYPE_INT32 : irop_get_btype(acc); + IROperand rep = irop_make_stackoff(u[k].var[at], iv ? iv->original_offset : 0, 1, 0, 0, rbt); + rep.is_unsigned = narrow || extract ? 1 : acc.is_unsigned; + rep.aux = IROP_AUX_NONVOLATILE; + if (extract) + { + sra_extract(ir, &ins, i, rep, abytes, acc.is_unsigned); + changes++; + continue; + } + if (!is_store && narrow && sra_covers_more(&u[k], at, abytes)) + { + IROperand v = sra_combine(ir, &ins, i, &u[k], at, abytes, acc.is_unsigned); + q = &ir->compact_instructions[i]; + sra_relay(ir, q, TCCIR_OP_ASSIGN, tcc_ir_op_get_dest(ir, q), v, IROP_NONE); + changes++; + continue; + } + if (is_store && narrow) + { + sra_store_as_def(ir, i, rep); + changes++; + continue; + } + q->op = is_store ? TCCIR_OP_STORE : TCCIR_OP_LOAD; + if (is_store) + tcc_ir_set_dest(ir, i, rep); + else + tcc_ir_set_src1(ir, i, rep); + changes++; + continue; + } + /* A call with split arguments takes the extra words in its argc. */ + if (call_extra && (q->op == TCCIR_OP_FUNCCALLVAL || q->op == TCCIR_OP_FUNCCALLVOID)) + { + IROperand enc = tcc_ir_op_get_src2(ir, q); + uint32_t e = (uint32_t)irop_get_imm64_ex(ir, enc); + int c = TCCIR_DECODE_CALL_ID(e); + if (c >= 0 && c < ncall && call_extra[c]) + { + enc.u.imm32 = (int32_t)TCCIR_ENCODE_CALL(c, TCCIR_DECODE_CALL_ARGC(e) + call_extra[c]); + tcc_ir_set_src2(ir, i, enc); + changes++; + } + } + /* Its arguments: a split one becomes its words, the later ones shift up. */ + if (call_extra && q->op == TCCIR_OP_FUNCPARAMVAL) + { + IROperand enc = tcc_ir_op_get_src2(ir, q); + uint32_t e = (uint32_t)irop_get_imm64_ex(ir, enc); + int c = TCCIR_DECODE_CALL_ID(e); + if (c >= 0 && c < ncall && call_extra[c]) + { + /* This argument's new index: every split argument ahead of it adds + * its extra words. The frontend numbers a call's arguments 0..argc-1, + * each once, so scanning for the lower ones is enough. */ + int idx = TCCIR_DECODE_PARAM_IDX(e), shift = 0; + for (int x = split_first[c]; x >= 0; x = splits[x][2]) + if (splits[x][0] < idx) + shift += splits[x][1]; + IROperand o = tcc_ir_op_get_src1(ir, q); + int off, k, pw; + if (sra_param_at(o, ntemp, toff, &off) && (k = sra_find_unit(u, nu, off)) >= 0 && !u[k].bad && + off == u[k].lo && (pw = sra_struct_param_words(o))) + { + /* Word w of the struct, from its field VAR (0 when no field). */ + for (int w = 0; w < pw; w++) + { + IROperand wv; + if (u[k].width[4 * w]) + { + if (u[k].var[4 * w] < 0) + u[k].var[4 * w] = tcc_ir_vreg_alloc_var(ir); + IRLiveInterval *iv = tcc_ir_get_live_interval(ir, u[k].var[4 * w]); + wv = irop_make_stackoff(u[k].var[4 * w], iv ? iv->original_offset : 0, 1, 0, 0, IROP_BTYPE_INT32); + wv.is_unsigned = 1; + wv.aux = IROP_AUX_NONVOLATILE; + } + else + wv = irop_make_imm32(-1, 0, IROP_BTYPE_INT32); + IROperand we = enc; + we.u.imm32 = (int32_t)TCCIR_ENCODE_PARAM(c, idx + shift + w); + if (w == 0) + { + tcc_ir_set_src1(ir, i, wv); + tcc_ir_set_src2(ir, i, we); + } + else + sra_ins_add(ir, &ins, i, 1, TCCIR_OP_FUNCPARAMVAL, wv, wv, we); + } + changes++; + continue; + } + if (shift) + { + enc.u.imm32 = (int32_t)TCCIR_ENCODE_PARAM(c, idx + shift); + tcc_ir_set_src2(ir, i, enc); + changes++; + } + } + } + for (int s = 0; s < 3; s++) + { + IROperand o = sra_operand(ir, q, s); + int off, k; + if (!sra_concrete(o, &off) || (k = sra_find_unit(u, nu, off)) < 0 || u[k].bad) + continue; + if (!o.is_lval) + continue; /* a result buffer's address, or a kept TEMP's root */ + int at = off - u[k].lo; + if (irop_get_btype(o) == IROP_BTYPE_INT64 && (u[k].width[at] == 8 || u[k].split[at])) + { + sra_rewrite_wide(ir, &ins, i, s, o, &u[k], at); + changes++; + continue; + } + const int obytes = sra_btype_bytes(irop_get_btype(o)); + const int fs = u[k].fstart[at]; + if (fs != 0xFF && (fs != at || (s == 0 && q->op == TCCIR_OP_STORE && obytes && obytes < u[k].width[fs]) || + (s == 1 && (q->op == TCCIR_OP_LOAD || q->op == TCCIR_OP_ASSIGN) && obytes && + obytes == u[k].width[fs] && u[k].pmask[fs]))) + { + /* Part of a wider field (sra_resolve_fields). */ + if (s == 0 && q->op == TCCIR_OP_STORE && + sra_partial_store_padded(ir, i, &u[k], fs, at - fs, obytes, tcc_ir_op_get_src1(ir, q))) + ; + else if (s == 0 && sra_bfi() && !irop_is_plain_imm(tcc_ir_op_get_src1(ir, q))) + sra_partial_store_bfi(ir, i, &u[k], fs, at - fs, obytes, tcc_ir_op_get_src1(ir, q)); + else if (s == 0) + sra_partial_store(ir, &ins, i, &u[k], fs, at - fs, obytes, tcc_ir_op_get_src1(ir, q), + !(u[k].nofit[at] & obytes)); + else + sra_partial_load(ir, &ins, i, &u[k], fs, at - fs, obytes ? obytes : 4, obytes ? o.is_unsigned : 1, + tcc_ir_op_get_dest(ir, q)); + changes++; + break; /* the instruction is rewritten whole */ + } + if (u[k].var[at] < 0) + u[k].var[at] = tcc_ir_vreg_alloc_var(ir); + int32_t v = u[k].var[at]; + const int narrow = u[k].width[at] && u[k].width[at] < 4; + const int extract = s == 1 && (q->op == TCCIR_OP_LOAD || q->op == TCCIR_OP_ASSIGN) && obytes && + obytes < u[k].width[at] && u[k].width[at] <= 4; + int bt = narrow || extract || irop_get_btype(o) == IROP_BTYPE_STRUCT ? IROP_BTYPE_INT32 : irop_get_btype(o); + IRLiveInterval *iv = tcc_ir_get_live_interval(ir, v); + IROperand rep = irop_make_stackoff(v, iv ? iv->original_offset : 0, 1, 0, 0, bt); + rep.is_unsigned = narrow || extract || irop_get_btype(o) == IROP_BTYPE_STRUCT ? 1 : o.is_unsigned; + rep.aux = IROP_AUX_NONVOLATILE; + if (extract) + { + sra_extract(ir, &ins, i, rep, obytes, o.is_unsigned); + changes++; + break; /* the instruction is rewritten whole */ + } + if (s == 1 && narrow && sra_covers_more(&u[k], at, obytes ? obytes : 4)) + { + /* A LOAD / ASSIGN becomes an ASSIGN of the value; a word struct + * argument or return value takes it as it is. */ + rep = sra_combine(ir, &ins, i, &u[k], at, obytes ? obytes : 4, + obytes ? o.is_unsigned : 1); + q = &ir->compact_instructions[i]; + if (q->op == TCCIR_OP_LOAD) + q->op = TCCIR_OP_ASSIGN; + } + if (s == 0 && narrow && q->op == TCCIR_OP_STORE) + { + sra_store_as_def(ir, i, rep); + changes++; + continue; + } + if (s == 0) + tcc_ir_set_dest(ir, i, rep); + else if (s == 1) + tcc_ir_set_src1(ir, i, rep); + else + tcc_ir_set_src2(ir, i, rep); + changes++; + } + } + + sra_ins_apply(ir, &ins); + tcc_free(ins.v); + tcc_free(tkeep); + tcc_free(whole); + tcc_free(call_at); + tcc_free(cparam); + tcc_free(ckind); + tcc_free(call_extra); + tcc_free(split_first); + tcc_free(splits); + + tcc_free(vglob); + tcc_free(vmay); + tcc_free(vbad); + tcc_free(vbad_op); + tcc_free(vbad_at); + tcc_free(vcur); + tcc_free(vcurroot); + tcc_free(vcurgen); + tcc_free(vdef); + tcc_free(vdefroot); + tcc_free(def_idx); + tcc_free(tbad); + tcc_free(tbad_op); + tcc_free(tbad_at); + tcc_free(troot); + tcc_free(tfrom); + tcc_free(toff); + tcc_free(defs); + tcc_free(u); + return changes; +} + +int tcc_ir_opt_sra(TCCIRState *ir) +{ + if (!ir || !tcc_state || tcc_state->optimize <= 0 || !sra_function_eligible(ir)) + return 0; + const int first_var = ir->next_local_variable; + int changes = sra_objects(ir); + /* The field VARs this run made, for ssa_var_promotable. */ + if (ir->next_local_variable > first_var) + ir->sra_var_lo = first_var, ir->sra_var_hi = ir->next_local_variable; + return changes + sra_forward_const_vars(ir, first_var); +} diff --git a/source/opt/flat/memory/sret_nrvo.c b/source/opt/flat/memory/sret_nrvo.c new file mode 100644 index 000000000..13bb86724 --- /dev/null +++ b/source/opt/flat/memory/sret_nrvo.c @@ -0,0 +1,849 @@ +/* + * TCC IR - build a returned struct in the caller's buffer (flat, pre-SSA) + * + * Copyright (c) 2026 Mateusz Stadnik + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation. + */ + +/* A function returning a struct through the hidden pointer P0 builds the value + * in a local and copies it out at the return: the Zig C backend's + * `t10 = t0; return t10;`, a 96-byte memmove at the end of every Wyhash.init. + * When the local -- and a local that is copied into it whole and then dies, + * t0 here -- is only ever reached through its own frame address, the object + * lives in *P0 instead: its address becomes P0 + k, a direct slot access a + * deref of a fresh `T <- P0 ADD #k`, and the copies out and between the two + * objects go away. + * + * The caller may pass a buffer the callee can see through another pointer + * (`*x = f(x)` passes x twice), so while the object is being built nothing + * else may touch memory: from the first reference to either object to the + * copy out there is no call except copies/fills between frame objects and + * .rodata, no deref of anything but the object's own addresses, and no global + * access. After the copy only the return follows. */ + +#define USING_GLOBALS + +#include "ir.h" +#include "opt.h" +#include "opt_utils.h" +#include "opt_alias.h" + +#define NRVO_MAX_OBJ 4 +#define NRVO_MAX_REFS 512 + +/* TCC_NRVO_DBG: the line of the check that refused a function. */ +TCC_DBG_ENV_FLAG(nrvo_dbg, "TCC_NRVO_DBG") +#define NRVO_FAIL() \ + do \ + { \ + if (nrvo_dbg()) \ + fprintf(stderr, "[NRVO] %s: refused at line %d\n", funcname, __LINE__); \ + return 0; \ + } while (0) + +static int nrvo_is_call(int op) +{ + return op == TCCIR_OP_FUNCCALLVOID || op == TCCIR_OP_FUNCCALLVAL; +} + +static const char *nrvo_callee(TCCIRState *ir, IRQuadCompact *q) +{ + Sym *s = irop_get_sym_ex(ir, tcc_ir_op_get_src1(ir, q)); + return s ? get_tok_str(s->v, NULL) : NULL; +} + +static int nrvo_is_memop_name(const char *nm) +{ + return nm && (ir_opt_is_memcpy_or_memmove_name(nm) || !strcmp(nm, "__aeabi_memset") || !strcmp(nm, "memset")); +} + +/* A frame address operand (not an access through the frame). */ +static int nrvo_frame_addr(IROperand o) +{ + return irop_get_tag(o) == IROP_TAG_STACKOFF && o.is_local && !o.is_llocal && !o.is_lval && irop_get_vreg(o) < 0; +} + +/* A direct access to a frame slot. */ +static int nrvo_frame_slot(IROperand o) +{ + return irop_get_tag(o) == IROP_TAG_STACKOFF && o.is_local && !o.is_llocal && o.is_lval && irop_get_vreg(o) < 0; +} + +/* The single def of TEMP `v` before `at`, or -1. */ +static int nrvo_temp_def(TCCIRState *ir, int32_t v, int at) +{ + if (v < 0 || TCCIR_DECODE_VREG_TYPE(v) != TCCIR_VREG_TYPE_TEMP) + return -1; + int found = -1; + for (int j = 0; j < ir->next_instruction_index; j++) + { + IRQuadCompact *q = &ir->compact_instructions[j]; + if (q->op == TCCIR_OP_NOP || !irop_config[q->op].has_dest) + continue; + IROperand d = tcc_ir_op_get_dest(ir, q); + if (d.is_lval || irop_get_vreg(d) != v) + continue; + if (found >= 0 || j >= at) + return -1; + found = j; + } + return found; +} + +/* What a call argument holds: a frame address (1, *off set), a symbol address + * (2), P0 or a copy of it (3), else 0. */ +static int nrvo_arg_kind(TCCIRState *ir, IROperand o, int at, int32_t p0, int *off) +{ + if (nrvo_frame_addr(o)) + { + *off = irop_get_stack_offset(o); + return 1; + } + if (irop_get_tag(o) == IROP_TAG_SYMREF && !o.is_lval) + return 2; + int32_t v = irop_get_vreg(o); + if (o.is_lval || v < 0) + return 0; + if (v == p0) + return 3; + int d = nrvo_temp_def(ir, v, at); + if (d < 0) + return 0; + IRQuadCompact *q = &ir->compact_instructions[d]; + if (q->op != TCCIR_OP_ASSIGN && q->op != TCCIR_OP_LEA) + return 0; + IROperand s = tcc_ir_op_get_src1(ir, q); + if (nrvo_frame_addr(s)) + { + *off = irop_get_stack_offset(s); + return 1; + } + if (irop_get_tag(s) == IROP_TAG_SYMREF && !s.is_lval) + return 2; + int32_t sv = irop_get_vreg(s); + if (!s.is_lval && sv == p0) + return 3; + /* P0's copy in a VAR, or the frontend's sret home slot */ + if (!s.is_lval && sv >= 0 && TCCIR_DECODE_VREG_TYPE(sv) == TCCIR_VREG_TYPE_VAR) + { + int defs = 0, ok = 1; + for (int j = 0; j < ir->next_instruction_index && ok; j++) + { + IRQuadCompact *vq = &ir->compact_instructions[j]; + if (vq->op == TCCIR_OP_NOP || !irop_config[vq->op].has_dest || !irop_config[vq->op].has_src1) + continue; + IROperand vd = tcc_ir_op_get_dest(ir, vq); + if (irop_get_vreg(vd) != sv) + continue; + defs++; + IROperand vs = tcc_ir_op_get_src1(ir, vq); + if (vs.is_lval || irop_get_vreg(vs) != p0) + ok = 0; + } + return ok && defs == 1 ? 3 : 0; + } + if (nrvo_frame_slot(s) && irop_get_stack_offset(s) == (int)func_vc) + return 3; + return 0; +} + +/* The CALL's arguments 0..2 and whether they are PARAM instructions we can find. */ +static int nrvo_call_args(TCCIRState *ir, int call, IROperand a[3]) +{ + for (int k = 0; k < 3; k++) + if (!ir_opt_get_call_param_operand(ir, call, k, &a[k])) + return 0; + return irop_get_tag(a[2]) == IROP_TAG_IMM32; +} + +typedef struct +{ + int lo, size; /* frame extent */ +} NrvoObj; + +static int nrvo_obj_of(const NrvoObj *o, int no, int off) +{ + for (int k = 0; k < no; k++) + if (off >= o[k].lo && off < o[k].lo + o[k].size) + return k; + return -1; +} + +/* Operand k of q (0 dest, 1 src1, 2 src2, 3 accum). */ +static int nrvo_operand(TCCIRState *ir, IRQuadCompact *q, int k, IROperand *op) +{ + switch (k) + { + case 0: + if (!irop_config[q->op].has_dest) + return 0; + *op = tcc_ir_op_get_dest(ir, q); + return 1; + case 1: + if (!irop_config[q->op].has_src1) + return 0; + *op = tcc_ir_op_get_src1(ir, q); + return 1; + case 2: + if (!irop_config[q->op].has_src2) + return 0; + *op = tcc_ir_op_get_src2(ir, q); + return 1; + default: + if (!tcc_ir_op_is_mac(q->op)) + return 0; + *op = tcc_ir_op_get_accum(ir, q); + return 1; + } +} + +static void nrvo_set_operand(TCCIRState *ir, IRQuadCompact *q, int k, IROperand op) +{ + if (k == 0) + tcc_ir_op_set_dest(ir, q, op); + else if (k == 1) + tcc_ir_op_set_src1(ir, q, op); + else if (k == 2) + tcc_ir_op_set_src2(ir, q, op); + else + tcc_ir_op_set_accum(ir, q, op); +} + +/* `t <- P0 ADD #k` (or `t <- P0` for k == 0), inserted before `at` so that + * every jump to `at` lands on it (tcc_ir_insert_instruction_before moves them + * past it). */ +static int32_t nrvo_insert_addr(TCCIRState *ir, int at, int32_t p0, int k) +{ + int32_t t = tcc_ir_get_vreg_temp(ir); + IRQuadCompact q = {0}; + q.op = k ? TCCIR_OP_ADD : TCCIR_OP_ASSIGN; + q.operand_base = tcc_ir_pool_add(ir, irop_make_vreg(t, IROP_BTYPE_INT32)); + tcc_ir_pool_add(ir, irop_make_vreg(p0, IROP_BTYPE_INT32)); + if (k) + tcc_ir_pool_add(ir, irop_make_imm32(-1, k, IROP_BTYPE_INT32)); + const int target = ir->compact_instructions[at].is_jump_target; + tcc_ir_insert_instruction_before(ir, at, &q); + if (target) + { + ir->compact_instructions[at].is_jump_target = 1; + ir->compact_instructions[at + 1].is_jump_target = 0; + for (int i = 0; i < ir->next_instruction_index; i++) + { + IRQuadCompact *jq = &ir->compact_instructions[i]; + if (jq->op != TCCIR_OP_JUMP && jq->op != TCCIR_OP_JUMPIF) + continue; + if ((int)irop_get_imm64_ex(ir, tcc_ir_op_get_dest(ir, jq)) == at + 1) + tcc_ir_op_set_dest(ir, jq, irop_make_imm32(-1, at, IROP_BTYPE_INT32)); + } + } + return t; +} + + +/* A .rodata/.data symbol argument at a word-aligned address (the check the + * backend's inline copy makes, arm-thumb-gen.c thumb_copy_arg_word_aligned). */ +static int nrvo_sym_word_aligned(TCCIRState *ir, IROperand op) +{ + IRPoolSymref *sr = irop_get_symref_ex(ir, op); + if (!sr || !sr->sym) + return 0; + ElfSym *esym = elfsym(sr->sym); + if (!esym || esym->st_shndx == SHN_UNDEF || esym->st_shndx >= SHN_LORESERVE || + esym->st_shndx >= (unsigned)tcc_state->nb_sections || ELFW(ST_BIND)(esym->st_info) != STB_LOCAL) + return 0; + Section *sec = tcc_state->sections[esym->st_shndx]; + return sec && sec->sh_addralign >= 4 && !((esym->st_value + (addr_t)sr->addend) & 3); +} + +/* The source of a TEMP argument (its ASSIGN/LEA def), else the argument. */ +static IROperand nrvo_arg_src(TCCIRState *ir, IROperand o, int at) +{ + int d = nrvo_temp_def(ir, irop_get_vreg(o), at); + if (d >= 0 && (ir->compact_instructions[d].op == TCCIR_OP_ASSIGN || ir->compact_instructions[d].op == TCCIR_OP_LEA)) + return tcc_ir_op_get_src1(ir, &ir->compact_instructions[d]); + return o; +} + +/* A zero fill of one of the objects, followed by a straight run of stores and + * copies into them: only the words the run leaves unwritten need the zeros. + * Those become direct `StackLoc[w] <-- #0` stores (rewritten into *P0 with + * the rest) and the call goes -- a 96-byte __aeabi_memset call in front of + * Wyhash.init's field stores otherwise, where block_copy_init had made the + * frame copy one image copy. Returns the number of instructions inserted + * before `site`. */ +static int nrvo_lower_fills(TCCIRState *ir, const NrvoObj *obj, int nobj, int32_t p0, int site) +{ + int shift = 0; + for (int i = site - 1; i >= 0; i--) + { + IRQuadCompact *q = &ir->compact_instructions[i]; + if (!nrvo_is_call(q->op)) + continue; + const char *nm = nrvo_callee(ir, q); + int aeabi = nm && !strcmp(nm, "__aeabi_memset"); + if (!nm || (!aeabi && strcmp(nm, "memset"))) + continue; + IROperand a[3]; + int off = 0; + for (int k = 0; k < 3; k++) + if (!ir_opt_get_call_param_operand(ir, i, k, &a[k])) + goto next; + { + IROperand sz = aeabi ? a[1] : a[2], val = aeabi ? a[2] : a[1]; + if (irop_get_tag(sz) != IROP_TAG_IMM32 || irop_get_tag(val) != IROP_TAG_IMM32 || irop_get_imm64_ex(ir, val) != 0) + continue; + if (nrvo_arg_kind(ir, a[0], i, p0, &off) != 1) + continue; + int ob = nrvo_obj_of(obj, nobj, off); + int n = (int)irop_get_imm64_ex(ir, sz); + if (ob < 0 || (off & 3) || (n & 3) || n <= 0 || n > 256 || off + n > obj[ob].lo + obj[ob].size) + continue; + uint8_t cov[256]; + memset(cov, 0, sizeof cov); + for (int j = i + 1; j < site; j++) + { + IRQuadCompact *jq = &ir->compact_instructions[j]; + if (jq->op == TCCIR_OP_NOP || jq->op == TCCIR_OP_FUNCPARAMVAL || jq->op == TCCIR_OP_FUNCPARAMVOID) + continue; + if (jq->op == TCCIR_OP_ASSIGN || jq->op == TCCIR_OP_LEA) + { + /* an address TEMP for a copy below */ + IROperand d = tcc_ir_op_get_dest(ir, jq), sr = tcc_ir_op_get_src1(ir, jq); + if (!d.is_lval && TCCIR_DECODE_VREG_TYPE(irop_get_vreg(d)) == TCCIR_VREG_TYPE_TEMP && + (nrvo_frame_addr(sr) || (irop_get_tag(sr) == IROP_TAG_SYMREF && !sr.is_lval))) + continue; + break; + } + if (jq->op == TCCIR_OP_STORE) + { + IROperand d = tcc_ir_op_get_dest(ir, jq), v = tcc_ir_op_get_src1(ir, jq); + if (!nrvo_frame_slot(d) || v.is_lval) + break; + int w = irop_is_64bit(d) ? 8 : ir_opt_store_btype_size_bytes(irop_get_btype(d)); + int so = irop_get_stack_offset(d); + for (int b = 0; b < w; b++) + if (so + b >= off && so + b < off + n) + cov[so + b - off] = 1; + continue; + } + if (nrvo_is_call(jq->op) && ir_opt_is_memcpy_or_memmove_name(nrvo_callee(ir, jq))) + { + IROperand c[3]; + int co = 0, so2 = 0; + if (!nrvo_call_args(ir, j, c) || nrvo_arg_kind(ir, c[0], j, p0, &co) != 1) + break; + int sk = nrvo_arg_kind(ir, c[1], j, p0, &so2); + if (sk != 2 && !(sk == 1 && nrvo_obj_of(obj, nobj, so2) < 0)) + break; /* reads the object, or some pointer */ + int cn = (int)irop_get_imm64_ex(ir, c[2]); + for (int b = 0; b < cn; b++) + if (co + b >= off && co + b < off + n) + cov[co + b - off] = 1; + continue; + } + break; + } + int ins = 0; + for (int w = n - 4; w >= 0; w -= 4) + { + if (cov[w] && cov[w + 1] && cov[w + 2] && cov[w + 3]) + continue; + IRQuadCompact st = {0}; + st.op = TCCIR_OP_STORE; + st.operand_base = tcc_ir_pool_add(ir, irop_make_stackoff(-1, off + w, 1, 0, 0, IROP_BTYPE_INT32)); + tcc_ir_pool_add(ir, irop_make_imm32(-1, 0, IROP_BTYPE_INT32)); + int pfirst = i; + for (int k = 0; k < 3; k++) + { + int pi = ir_opt_get_call_param_index(ir, i, k); + if (pi >= 0 && pi < pfirst) + pfirst = pi; + } + tcc_ir_insert_instruction_before(ir, pfirst, &st); + ins++; + } + int call = i + ins; + ir_opt_nop_call_params(ir, call); + ir->compact_instructions[call].op = TCCIR_OP_NOP; + shift += ins; + site += ins; + } + next:; + } + return shift; +} + +int tcc_ir_opt_sret_nrvo(TCCIRState *ir) +{ + int n = ir->next_instruction_index; + if (!func_vc || n == 0 || ir->inline_asm_count || ir->func_has_label_addr || tcc_ir_calls_returns_twice(ir) || + ir->num_switch_tables) + NRVO_FAIL(); + const int32_t p0 = TCCIR_ENCODE_VREG(TCCIR_VREG_TYPE_PARAM, 0); + + /* The one copy out: memmove/memcpy(sret, &obj, N), then only the return. */ + int site = -1; + NrvoObj obj[NRVO_MAX_OBJ]; + int nobj = 0; + for (int i = 0; i < n; i++) + { + IRQuadCompact *q = &ir->compact_instructions[i]; + if (q->op == TCCIR_OP_NOP) + continue; + /* P0 written, or read other than by copying it: not ours to reason about */ + int32_t dv = irop_config[q->op].has_dest ? irop_get_vreg(tcc_ir_op_get_dest(ir, q)) : -1; + if (dv == p0 && q->op != TCCIR_OP_FUNCPARAMVAL && q->op != TCCIR_OP_FUNCPARAMVOID) + NRVO_FAIL(); + if (!nrvo_is_call(q->op) || !ir_opt_is_memcpy_or_memmove_name(nrvo_callee(ir, q))) + continue; + IROperand a[3]; + int dst_off = 0, src_off = 0; + if (!nrvo_call_args(ir, i, a)) + continue; + if (nrvo_arg_kind(ir, a[0], i, p0, &dst_off) != 3) + continue; + if (site >= 0 || nrvo_arg_kind(ir, a[1], i, p0, &src_off) != 1) + NRVO_FAIL(); /* two copies out, or one from somewhere else */ + int size = (int)irop_get_imm64_ex(ir, a[2]); + if (size <= 0 || (size & 3) || (src_off & 3) || tcc_ir_frame_object_size_at(ir, src_off) != size) + NRVO_FAIL(); + site = i; + obj[nobj++] = (NrvoObj){.lo = src_off, .size = size}; + } + if (site < 0) + NRVO_FAIL(); + /* then only the return: NOPs, and one jump straight to the return or the end */ + { + int j = site + 1, jumped = 0; + while (j < n) + { + IRQuadCompact *q = &ir->compact_instructions[j]; + if (q->op == TCCIR_OP_NOP) + { + j++; + continue; + } + if (q->op == TCCIR_OP_JUMP && !jumped) + { + int t = (int)irop_get_imm64_ex(ir, tcc_ir_op_get_dest(ir, q)); + if (t <= site) + NRVO_FAIL(); + j = t; + jumped = 1; + continue; + } + if (q->op != TCCIR_OP_RETURNVOID) + NRVO_FAIL(); + break; + } + } + const int size = obj[0].size; + + /* Merge a same-sized object copied whole into one of ours, when that copy is + * the last reference to the source and the first to the destination. */ + for (int round = 0; round < NRVO_MAX_OBJ - 1 && nobj < NRVO_MAX_OBJ; round++) + { + int added = 0; + for (int i = 0; i < site && !added; i++) + { + IRQuadCompact *q = &ir->compact_instructions[i]; + if (!nrvo_is_call(q->op) || !ir_opt_is_memcpy_or_memmove_name(nrvo_callee(ir, q))) + continue; + IROperand a[3]; + int d = 0, s = 0; + if (!nrvo_call_args(ir, i, a) || nrvo_arg_kind(ir, a[0], i, p0, &d) != 1 || + nrvo_arg_kind(ir, a[1], i, p0, &s) != 1 || (int)irop_get_imm64_ex(ir, a[2]) != size) + continue; + int od = nrvo_obj_of(obj, nobj, d); + if (od < 0 || d != obj[od].lo || nrvo_obj_of(obj, nobj, s) >= 0 || (s & 3) || + tcc_ir_frame_object_size_at(ir, s) != size) + continue; + /* s referenced after i, or d before it (other than through this call's args)? */ + int bad = 0; + int pfirst = i, argdef[2] = {-1, -1}; + for (int k = 0; k < 3; k++) + { + int pi = ir_opt_get_call_param_index(ir, i, k); + if (pi >= 0 && pi < pfirst) + pfirst = pi; + } + for (int k = 0; k < 2; k++) + argdef[k] = nrvo_temp_def(ir, irop_get_vreg(a[k]), i); + for (int j = 0; j < n && !bad; j++) + { + IRQuadCompact *jq = &ir->compact_instructions[j]; + /* the copy itself: its PARAMs, CALL and the address TEMPs they pass */ + if (jq->op == TCCIR_OP_NOP || (j >= pfirst && j <= i) || j == argdef[0] || j == argdef[1]) + continue; + for (int k = 0; k < 4 && !bad; k++) + { + IROperand o; + if (!nrvo_operand(ir, jq, k, &o) || irop_get_tag(o) != IROP_TAG_STACKOFF || irop_get_vreg(o) >= 0) + continue; + int off = irop_get_stack_offset(o); + if (off >= s && off < s + size && j > i) + bad = 1; + if (off >= d && off < d + size && j < pfirst) + bad = 1; + } + } + if (bad) + continue; + obj[nobj++] = (NrvoObj){.lo = s, .size = size}; + added = 1; + } + if (!added) + break; + } + + /* Every reference to the objects, and the window they are built in. */ + int first = site; + int nrefs = 0; + for (int j = 0; j < n; j++) + { + IRQuadCompact *q = &ir->compact_instructions[j]; + if (q->op == TCCIR_OP_NOP) + continue; + for (int k = 0; k < 4; k++) + { + IROperand o; + if (!nrvo_operand(ir, q, k, &o) || irop_get_tag(o) != IROP_TAG_STACKOFF || !o.is_local || o.is_llocal || + irop_get_vreg(o) >= 0) + continue; + int ob = nrvo_obj_of(obj, nobj, irop_get_stack_offset(o)); + if (ob < 0) + continue; + if (j > site) + NRVO_FAIL(); + if (j < first) + first = j; + if ((o.is_lval && irop_get_btype(o) == IROP_BTYPE_STRUCT) || q->op == TCCIR_OP_BLOCK_COPY || o.is_complex || + tcc_ir_access_is_volatile(ir, o)) + { + if (nrvo_dbg()) + fprintf(stderr, "[NRVO] %s: instr %d op %s operand %d btype %d\n", funcname, j, tcc_ir_get_op_name(q->op), k, + irop_get_btype(o)); + NRVO_FAIL(); /* whole-object value uses, block copies: not rewritten here */ + } + if (!o.is_lval && q->op != TCCIR_OP_ASSIGN && q->op != TCCIR_OP_LEA && q->op != TCCIR_OP_ADD && + q->op != TCCIR_OP_FUNCPARAMVAL) + NRVO_FAIL(); + /* a write only by a STORE; a read anywhere (an ALU operand is loaded + * into a TEMP first) */ + if (o.is_lval && k == 0 && q->op != TCCIR_OP_STORE) + NRVO_FAIL(); + if (++nrefs > NRVO_MAX_REFS) + NRVO_FAIL(); + } + } + + /* TEMPs and VARs holding one of the objects' addresses: derefs through them + * are ours. Zig keeps field addresses in pointer locals (`t1 = &t0.state; + * t2 = &t1->array[0]; *t2 = x`), so a VAR counts when every def of it is + * such an address. */ + int ntemps = ir->next_temporary_variable + 1, nvars = ir->next_local_variable + 1; + /* own[] for TEMPs, then VARs. Greatest fixpoint: every vreg with a def is + * assumed owned, and loses it once any of its defs is not an address + * derivation from an owned address (a loop's `p = p + 8` keeps it). */ + uint8_t *own = tcc_malloc(ntemps + nvars), *hasdef = tcc_mallocz(ntemps + nvars); +#define NRVO_IDX(v) \ + ((v) < 0 ? -1 \ + : TCCIR_DECODE_VREG_TYPE(v) == TCCIR_VREG_TYPE_TEMP \ + ? (TCCIR_DECODE_VREG_POSITION(v) < ntemps ? TCCIR_DECODE_VREG_POSITION(v) : -1) \ + : TCCIR_DECODE_VREG_TYPE(v) == TCCIR_VREG_TYPE_VAR \ + ? (TCCIR_DECODE_VREG_POSITION(v) < nvars ? ntemps + TCCIR_DECODE_VREG_POSITION(v) : -1) \ + : -1) +#define NRVO_OWNED(v) (NRVO_IDX(v) >= 0 && own[NRVO_IDX(v)]) + for (int j = 0; j < n; j++) + { + IRQuadCompact *q = &ir->compact_instructions[j]; + if (q->op == TCCIR_OP_NOP || !irop_config[q->op].has_dest) + continue; + IROperand d = tcc_ir_op_get_dest(ir, q); + int x = (d.is_lval || irop_get_tag(d) != IROP_TAG_VREG) ? -1 : NRVO_IDX(irop_get_vreg(d)); + if (x >= 0) + hasdef[x] = 1; + } + memcpy(own, hasdef, ntemps + nvars); + int converged = 0; + for (int round = 0; round < 32 && !converged; round++) + { + converged = 1; + for (int j = 0; j < n; j++) + { + IRQuadCompact *q = &ir->compact_instructions[j]; + if (q->op == TCCIR_OP_NOP || !irop_config[q->op].has_dest) + continue; + IROperand d = tcc_ir_op_get_dest(ir, q); + int x = (d.is_lval || irop_get_tag(d) != IROP_TAG_VREG) ? -1 : NRVO_IDX(irop_get_vreg(d)); + if (x < 0 || !own[x]) + continue; + int from_own = 0; + if (q->op == TCCIR_OP_ASSIGN || q->op == TCCIR_OP_LEA || q->op == TCCIR_OP_ADD || q->op == TCCIR_OP_SUB || + q->op == TCCIR_OP_STORE) + for (int k = 1; k <= 2 && !from_own; k++) + { + IROperand s2; + if (!nrvo_operand(ir, q, k, &s2) || (k == 2 && q->op == TCCIR_OP_SUB)) + continue; + /* a VAR's value is read through its slot (is_lval); anything else + * with is_lval is a load, not an address */ + const int32_t sv2 = irop_get_vreg(s2); + const int var_value = sv2 >= 0 && TCCIR_DECODE_VREG_TYPE(sv2) == TCCIR_VREG_TYPE_VAR && !s2.is_llocal; + if (s2.is_lval && !var_value) + continue; + if ((nrvo_frame_addr(s2) && nrvo_obj_of(obj, nobj, irop_get_stack_offset(s2)) >= 0) || + NRVO_OWNED(irop_get_vreg(s2))) + from_own = 1; + } + if (!from_own) + { + own[x] = 0; + converged = 0; + if (nrvo_dbg() && getenv("TCC_NRVO_OWNDBG")) + fprintf(stderr, "[NRVO] lose %s%d at %d op %s\n", x < ntemps ? "T" : "V", x < ntemps ? x : x - ntemps, j, + tcc_ir_get_op_name(q->op)); + } + } + } + if (!converged) + memset(own, 0, ntemps + nvars); + + int ok = 1; + for (int j = first; j < site && ok; j++) + { + IRQuadCompact *q = &ir->compact_instructions[j]; + if (q->op == TCCIR_OP_NOP) + continue; + if (q->op == TCCIR_OP_INLINE_ASM || q->op == TCCIR_OP_ASM_INPUT || q->op == TCCIR_OP_ASM_OUTPUT) + ok = 0; + if (nrvo_is_call(q->op)) + { + /* a copy or fill among frame objects and .rodata only */ + IROperand a[3]; + int o0 = 0, o1 = 0; + if (!nrvo_is_memop_name(nrvo_callee(ir, q)) || !nrvo_call_args(ir, j, a)) + { + ok = 0; + break; + } + int k0 = nrvo_arg_kind(ir, a[0], j, p0, &o0), k1 = nrvo_arg_kind(ir, a[1], j, p0, &o1); + int fill = strcmp(nrvo_callee(ir, q), "__aeabi_memset") == 0 || strcmp(nrvo_callee(ir, q), "memset") == 0; + if (k0 != 1 || (!fill && k1 != 1 && k1 != 2)) + { + ok = 0; + if (nrvo_dbg()) + fprintf(stderr, "[NRVO] mem call at %d kinds %d %d\n", j, k0, k1); + } + continue; + } + for (int k = 0; k < 4 && ok; k++) + { + IROperand o; + if (!nrvo_operand(ir, q, k, &o)) + continue; + if (irop_get_tag(o) == IROP_TAG_SYMREF && o.is_lval) + { + ok = 0; /* a global's memory */ + if (nrvo_dbg()) + fprintf(stderr, "[NRVO] global at %d\n", j); + } + int32_t v = irop_get_vreg(o); + int deref = o.is_lval && irop_get_tag(o) == IROP_TAG_VREG && v >= 0 && TCCIR_DECODE_VREG_TYPE(v) != TCCIR_VREG_TYPE_VAR; + if (q->op == TCCIR_OP_LOAD_INDEXED || q->op == TCCIR_OP_STORE_INDEXED || q->op == TCCIR_OP_STORE_POSTINC || + q->op == TCCIR_OP_LOAD_POSTINC) + deref |= (k == (q->op == TCCIR_OP_LOAD_INDEXED || q->op == TCCIR_OP_LOAD_POSTINC ? 1 : 0)) && v >= 0; + if (deref && !NRVO_OWNED(v)) + { + ok = 0; /* memory through some other pointer */ + if (nrvo_dbg()) + fprintf(stderr, "[NRVO] foreign deref at %d operand %d\n", j, k); + } + } + } + if (!ok && nrvo_dbg()) + fprintf(stderr, "[NRVO] %s: window refused\n", funcname); +#undef NRVO_OWNED +#undef NRVO_IDX + tcc_free(own); + tcc_free(hasdef); + if (!ok) + NRVO_FAIL(); + + site += nrvo_lower_fills(ir, obj, nobj, p0, site); + n = ir->next_instruction_index; + first = site; + for (int j = 0; j < site; j++) + { + IRQuadCompact *q = &ir->compact_instructions[j]; + if (q->op == TCCIR_OP_NOP) + continue; + for (int k = 0; k < 4 && first > j; k++) + { + IROperand o; + if (nrvo_operand(ir, q, k, &o) && irop_get_tag(o) == IROP_TAG_STACKOFF && o.is_local && !o.is_llocal && + irop_get_vreg(o) < 0 && nrvo_obj_of(obj, nobj, irop_get_stack_offset(o)) >= 0) + first = j; + } + } + + /* Copies into the objects keep their inline expansion: the buffer has the + * return type's alignment, so a word-aligned destination and source make + * them __aeabi_memcpy4 (the backend expands only copies it can prove + * aligned, and P0 + k is no frame slot). */ + { + int align = 1; + type_size(&func_vt, &align); + for (int j = first; align >= 4 && j < site; j++) + { + IRQuadCompact *q = &ir->compact_instructions[j]; + const char *nm = nrvo_is_call(q->op) ? nrvo_callee(ir, q) : NULL; + if (!nm || (strcmp(nm, "__aeabi_memcpy") && strcmp(nm, "__aeabi_memmove") && strcmp(nm, "memcpy") && + strcmp(nm, "memmove"))) + continue; + IROperand a[3]; + int d = 0, so = 0; + if (!nrvo_call_args(ir, j, a) || nrvo_arg_kind(ir, a[0], j, p0, &d) != 1 || nrvo_obj_of(obj, nobj, d) < 0) + continue; + int sk = nrvo_arg_kind(ir, a[1], j, p0, &so), cn = (int)irop_get_imm64_ex(ir, a[2]); + int src_ok = (sk == 1 && !(so & 3)) || (sk == 2 && nrvo_sym_word_aligned(ir, nrvo_arg_src(ir, a[1], j))); + if (((d - obj[nrvo_obj_of(obj, nobj, d)].lo) & 3) || (cn & 3) || !src_ok) + continue; + if (!strcmp(nm, "memcpy") || !strcmp(nm, "memmove")) + continue; /* ISO order is the same, but keep to the AEABI names */ + change_callee_sym_keep_type(ir, j, !strcmp(nm, "__aeabi_memcpy") ? "__aeabi_memcpy4" : "__aeabi_memmove4"); + } + } + + /* Rewrite, last instruction first so the insertions do not move what is left. */ + int changes = 0; + int buf_align = 1; + type_size(&func_vt, &buf_align); + for (int j = site; j >= first; j--) + { + IRQuadCompact *q = &ir->compact_instructions[j]; + if (q->op == TCCIR_OP_NOP) + continue; + for (int k = 0; k < 4; k++) + { + IROperand o; + q = &ir->compact_instructions[j]; + if (!nrvo_operand(ir, q, k, &o) || irop_get_tag(o) != IROP_TAG_STACKOFF || !o.is_local || o.is_llocal || + irop_get_vreg(o) >= 0) + continue; + int off = irop_get_stack_offset(o); + int ob = nrvo_obj_of(obj, nobj, off); + if (ob < 0) + continue; + const int disp = off - obj[ob].lo; + if (!o.is_lval && (q->op == TCCIR_OP_ASSIGN || q->op == TCCIR_OP_LEA) && k == 1) + { + /* `T <- &obj + disp` is `T <- P0 + disp` */ + IROperand dd = tcc_ir_op_get_dest(ir, q); + IROperand np = irop_make_vreg(p0, IROP_BTYPE_INT32); + if (disp) + { + int base = tcc_ir_pool_add(ir, dd); + tcc_ir_pool_add(ir, np); + tcc_ir_pool_add(ir, irop_make_imm32(-1, disp, IROP_BTYPE_INT32)); + q->op = TCCIR_OP_ADD; + q->operand_base = base; + } + else + { + int base = tcc_ir_pool_add(ir, dd); + tcc_ir_pool_add(ir, np); + q->op = TCCIR_OP_ASSIGN; + q->operand_base = base; + } + changes++; + continue; + } + /* A 64-bit field: an indexed access off P0 directly. disp.c will not + * fuse `T <- P0 ADD #k; *T` for 64-bit (LDRD/STRD need word alignment it + * cannot see), which left an ADD and two STRs per u64 field; here the + * buffer has the return type's alignment. */ + if (o.is_lval && irop_is_64bit(o) && buf_align >= 4 && !(disp & 3) && disp <= 1020 && + ((k == 0 && q->op == TCCIR_OP_STORE) || (k == 1 && q->op == TCCIR_OP_LOAD))) + { + IROperand base = irop_make_vreg(p0, IROP_BTYPE_INT32); + irop_carry_access_marks(&base, o); + IROperand other = q->op == TCCIR_OP_STORE ? tcc_ir_op_get_src1(ir, q) : tcc_ir_op_get_dest(ir, q); + if (q->op == TCCIR_OP_LOAD && + (other.is_lval || TCCIR_DECODE_VREG_TYPE(irop_get_vreg(other)) != TCCIR_VREG_TYPE_TEMP)) + goto generic; + int nb = tcc_ir_pool_add(ir, q->op == TCCIR_OP_STORE ? base : other); + tcc_ir_pool_add(ir, q->op == TCCIR_OP_STORE ? other : base); + tcc_ir_pool_add(ir, irop_make_imm32(0, disp, IROP_BTYPE_INT32)); + tcc_ir_pool_add(ir, irop_make_imm32(0, 0, IROP_BTYPE_INT32)); + q->op = q->op == TCCIR_OP_STORE ? TCCIR_OP_STORE_INDEXED : TCCIR_OP_LOAD_INDEXED; + q->operand_base = nb; + changes++; + continue; + } + generic:; + /* A field read as an operand of some other op: load it first. */ + if (o.is_lval && k >= 1 && !(k == 1 && (q->op == TCCIR_OP_LOAD || q->op == TCCIR_OP_ASSIGN))) + { + int32_t ta = nrvo_insert_addr(ir, j, p0, disp); + int32_t tv = tcc_ir_get_vreg_temp(ir); + IROperand vd = irop_make_vreg(tv, irop_get_btype(o)); + vd.is_unsigned = o.is_unsigned; + IROperand vs = irop_make_vreg(ta, irop_get_btype(o)); + vs.is_unsigned = o.is_unsigned; + vs.is_lval = 1; + IRQuadCompact ld = {0}; + ld.op = TCCIR_OP_LOAD; + ld.operand_base = tcc_ir_pool_add(ir, vd); + tcc_ir_pool_add(ir, vs); + tcc_ir_insert_instruction_before(ir, j + 1, &ld); /* after the address, before q */ + site += 2; + j += 2; + q = &ir->compact_instructions[j]; + nrvo_set_operand(ir, q, k, vd); + changes++; + continue; + } + /* anything else takes the address in a fresh TEMP just before it; a + * call argument before the call's first PARAM */ + int at = j; + if (q->op == TCCIR_OP_FUNCPARAMVAL) + { + while (at > 0 && ir->compact_instructions[at - 1].op == TCCIR_OP_FUNCPARAMVAL) + at--; + } + int32_t t = nrvo_insert_addr(ir, at, p0, disp); + site++; + j++; /* q moved down by one */ + q = &ir->compact_instructions[j]; + IROperand no = irop_make_vreg(t, irop_get_btype(o)); + no.is_lval = o.is_lval; + no.is_unsigned = o.is_unsigned; + if (!o.is_lval) + no.btype = IROP_BTYPE_INT32; + nrvo_set_operand(ir, q, k, no); + changes++; + } + } + + /* The copies out and between the objects now copy *P0 onto itself. */ + n = ir->next_instruction_index; + for (int i = 0; i < n; i++) + { + IRQuadCompact *q = &ir->compact_instructions[i]; + if (!nrvo_is_call(q->op) || !ir_opt_is_memcpy_or_memmove_name(nrvo_callee(ir, q))) + continue; + IROperand a[3]; + int d = 0, s = 0; + if (!nrvo_call_args(ir, i, a) || nrvo_arg_kind(ir, a[0], i, p0, &d) != 3 || nrvo_arg_kind(ir, a[1], i, p0, &s) != 3) + continue; + ir_opt_nop_call_params(ir, i); + q->op = TCCIR_OP_NOP; + } + if (nrvo_dbg()) + fprintf(stderr, "[NRVO] %s: built in *P0, %d objects, %d rewrites\n", funcname, nobj, changes); + return changes; +} diff --git a/source/opt/flat/memory/struct_copy_roundtrip_elim.c b/source/opt/flat/memory/struct_copy_roundtrip_elim.c index 58dc327b8..eec9fb3f2 100644 --- a/source/opt/flat/memory/struct_copy_roundtrip_elim.c +++ b/source/opt/flat/memory/struct_copy_roundtrip_elim.c @@ -91,11 +91,12 @@ static int scre_resolve_slot_addr(TCCIRState *ir, IROperand op, int before_idx, IRQuadCompact *oq = &ir->compact_instructions[o]; if (oq->op == TCCIR_OP_NOP || !irop_config[oq->op].has_dest) continue; - if (oq->op == TCCIR_OP_STORE || oq->op == TCCIR_OP_STORE_INDEXED || - oq->op == TCCIR_OP_STORE_POSTINC) - continue; /* store dest is an address use, not a def */ + if (oq->op == TCCIR_OP_STORE_INDEXED) + continue; /* its destination is the base address */ + /* A store through a pointer is an address use, not a def; a VAR or + * PARAM stored as itself (a STACKOFF lvalue) is redefined. */ IROperand od = tcc_ir_op_get_dest(ir, oq); - if (irop_has_vreg(od) && irop_get_vreg(od) == vr && !od.is_lval) + if (irop_get_vreg(od) == vr && irop_dest_defines_vreg(od)) return 0; } if (dq->op != TCCIR_OP_LEA && dq->op != TCCIR_OP_ASSIGN) @@ -175,6 +176,58 @@ static int scre_touches_region(TCCIRState *ir, IRQuadCompact *q, int32_t lo, int return touched; } +static int scre_call_id(TCCIRState *ir, IRQuadCompact *q) +{ + return TCCIR_DECODE_CALL_ID((uint32_t)irop_get_imm64_ex(ir, tcc_ir_op_get_src2(ir, q))); +} + +/* Instruction `k` touches region B: is it only plumbing for the two copies? + * That is a FUNCPARAM of C1 or C2, or an address-of B (never a read of its + * contents) into a TEMP that nothing but those FUNCPARAMs reads. An address + * of B going anywhere else -- `p = &t40.field` read through later, `f(&t40)` + * -- is a use of B's contents that the pair's removal would leave unwritten. */ +static int scre_is_copy_plumbing(TCCIRState *ir, int k, int cid1, int cid2) +{ + IRQuadCompact *q = &ir->compact_instructions[k]; + if (q->op == TCCIR_OP_FUNCPARAMVAL || q->op == TCCIR_OP_FUNCPARAMVOID) + { + int cid = scre_call_id(ir, q); + return cid == cid1 || cid == cid2; + } + if (q->op != TCCIR_OP_LEA && q->op != TCCIR_OP_ASSIGN) + return 0; + IROperand s1 = tcc_ir_op_get_src1(ir, q); + if (irop_get_tag(s1) != IROP_TAG_STACKOFF || s1.is_lval) + return 0; + if (q->op == TCCIR_OP_ASSIGN && !irop_is_none(tcc_ir_op_get_src2(ir, q))) + return 0; + IROperand d = tcc_ir_op_get_dest(ir, q); + int32_t dv = irop_get_vreg(d); + if (dv < 0 || TCCIR_DECODE_VREG_TYPE(dv) != TCCIR_VREG_TYPE_TEMP || d.is_lval) + return 0; + for (int u = 0; u < ir->next_instruction_index; u++) + { + IRQuadCompact *uq = &ir->compact_instructions[u]; + if (u == k || uq->op == TCCIR_OP_NOP) + continue; + const IRRegistersConfig *cfg = &irop_config[uq->op]; + int reads = (cfg->has_src1 && irop_get_vreg(tcc_ir_op_get_src1(ir, uq)) == dv) || + (cfg->has_src2 && irop_get_vreg(tcc_ir_op_get_src2(ir, uq)) == dv) || + (uq->op == TCCIR_OP_MLA && irop_get_vreg(tcc_ir_op_get_accum(ir, uq)) == dv); + /* A store through it, or a write-back of it, is a use as well. */ + if (!reads && cfg->has_dest && irop_get_vreg(tcc_ir_op_get_dest(ir, uq)) == dv) + reads = 1; + if (!reads) + continue; + if (uq->op != TCCIR_OP_FUNCPARAMVAL && uq->op != TCCIR_OP_FUNCPARAMVOID) + return 0; + int cid = scre_call_id(ir, uq); + if (cid != cid1 && cid != cid2) + return 0; + } + return 1; +} + int tcc_ir_opt_struct_copy_roundtrip_elim(TCCIRState *ir) { int n = ir->next_instruction_index; @@ -282,6 +335,7 @@ int tcc_ir_opt_struct_copy_roundtrip_elim(TCCIRState *ir) * dst-addr and C2's src-addr (and their param/LEA setup), nowhere else. * Any other read/write/addr-of of B is disqualifying. */ int b_ok = 1; + const int cid1 = scre_call_id(ir, c1), cid2 = scre_call_id(ir, c2); for (int k = 0; k < n && b_ok; k++) { if (k == i1 || k == i2) @@ -298,8 +352,7 @@ int tcc_ir_opt_struct_copy_roundtrip_elim(TCCIRState *ir) int is_w; if (!scre_touches_region(ir, q, b_off, sz1, &is_w)) continue; - if (q->op == TCCIR_OP_LEA || q->op == TCCIR_OP_ASSIGN || - q->op == TCCIR_OP_FUNCPARAMVAL || q->op == TCCIR_OP_FUNCPARAMVOID) + if (scre_is_copy_plumbing(ir, k, cid1, cid2)) continue; /* address-of / param plumbing for the two copies */ b_ok = 0; } diff --git a/source/opt/flat/scalar/add_reassoc.c b/source/opt/flat/scalar/add_reassoc.c index 052a1f156..ed6268dfd 100644 --- a/source/opt/flat/scalar/add_reassoc.c +++ b/source/opt/flat/scalar/add_reassoc.c @@ -132,6 +132,12 @@ int tcc_ir_opt_add_reassoc(TCCIRState *ir) IROperand def_src2 = tcc_ir_op_get_src2(ir, def_q); if (!irop_is_immediate(def_src2)) continue; + /* A signed add is taken not to overflow (ssa:cmp_offset_fold folds + * `x + K < x` on it). Absorbing an unsigned, wrapping, inner add would + * extend that to the whole sum: pr55137's `(int)(x + 1U) + 1 < (int)x` + * became `x + 2 < x`, folded false, yet x = INT_MAX gives true. */ + if (tcc_ir_op_get_dest(ir, def_q).is_unsigned && !tcc_ir_op_get_dest(ir, q).is_unsigned) + continue; def_src1 = tcc_ir_op_get_src1(ir, def_q); int64_t c1 = irop_get_imm64_ex(ir, def_src2); eff_c1 = (def_q->op == TCCIR_OP_SUB) ? -c1 : c1; diff --git a/source/opt/flat/scalar/branch.c b/source/opt/flat/scalar/branch.c index 9dfdeac44..8bd10f440 100644 --- a/source/opt/flat/scalar/branch.c +++ b/source/opt/flat/scalar/branch.c @@ -380,10 +380,11 @@ OPT_GEN_FLAT(setif_branch_remat, TCCIR_OP_CMP) return 0; if (tcc_ir_access_is_volatile(ir, setif_dest)) return 0; - if (!tcc_ir_vreg_has_single_def(ir, sv)) - return 0; + /* That the SETIF is sv's only definition is settled by the use scan below, + * which turns down any other instruction writing sv. */ int nregs = 0; + int32_t reg_vr = -1; for (int k = 0; k < 2; k++) { IROperand o = k ? src2 : src1; @@ -400,8 +401,7 @@ OPT_GEN_FLAT(setif_branch_remat, TCCIR_OP_CMP) * would compare the NEW value: the f2 case in tests2 pins that.) */ if (TCCIR_DECODE_VREG_TYPE(vr) != TCCIR_VREG_TYPE_TEMP) return 0; - if (!tcc_ir_vreg_has_single_def(ir, vr)) - return 0; + reg_vr = vr; /* its single definition is counted in the use scan below */ if (++nregs > 1) return 0; } @@ -417,10 +417,19 @@ OPT_GEN_FLAT(setif_branch_remat, TCCIR_OP_CMP) int new_tok[REMAT_MAX_USES]; int nuses = 0; + /* One walk for three facts: the uses of sv, that nothing but the SETIF + * writes sv, and that the compared register has exactly one definition -- + * each was a walk of the whole function per CMP, and Zig's C backend emits + * a CMP + SETIF for nearly every condition. */ + int reg_defs = 0; for (int u = 0; u < n; u++) { IRQuadCompact *uq = &ir->compact_instructions[u]; - if (u == si || uq->op == TCCIR_OP_NOP) + if (uq->op == TCCIR_OP_NOP) + continue; + if (irop_config[uq->op].has_dest && irop_get_vreg(tcc_ir_op_get_dest(ir, uq)) == reg_vr && ++reg_defs > 1) + return 0; + if (u == si) continue; if (irop_config[uq->op].has_dest && irop_get_vreg(tcc_ir_op_get_dest(ir, uq)) == sv) return 0; /* redefined, or written through as an address */ @@ -483,7 +492,7 @@ OPT_GEN_FLAT(setif_branch_remat, TCCIR_OP_CMP) new_tok[nuses] = nt; nuses++; } - if (nuses == 0) + if (nuses == 0 || reg_defs != 1) return 0; /* TEST_ZERO owns ONE operand slot and CMP needs two, so the comparison @@ -650,6 +659,146 @@ const IROptGen branch_gens[] = { const int branch_gens_count = sizeof(branch_gens) / sizeof(branch_gens[0]); +/* Index of a VAR or TEMP in one array covering both, or -1. */ +static int paired_index(const TCCIRState *ir, int32_t vr) +{ + if (vr < 0) + return -1; + const int pos = TCCIR_DECODE_VREG_POSITION(vr); + if (TCCIR_DECODE_VREG_TYPE(vr) == TCCIR_VREG_TYPE_VAR) + return pos < ir->next_local_variable ? pos : -1; + if (TCCIR_DECODE_VREG_TYPE(vr) == TCCIR_VREG_TYPE_TEMP) + return pos < ir->next_temporary_variable ? ir->next_local_variable + pos : -1; + return -1; +} + +/* Previous non-NOP instruction, or -1 when there is none or a jump lands + * on or between them. */ +static int paired_prev(const TCCIRState *ir, int i) +{ + if (ir->compact_instructions[i].is_jump_target) + return -1; + for (int p = i - 1; p >= 0; p--) + { + const IRQuadCompact *q = &ir->compact_instructions[p]; + if (q->op != TCCIR_OP_NOP) + return p; + if (q->is_jump_target) + return -1; + } + return -1; +} + +/* Whether operand `k` of instruction `u` is a zero test of `vr` (TEST_ZERO, + * or CMP against #0) right after an instruction defining `vr`. */ +static int paired_zero_test(const TCCIRState *ir, int u, int k, IROperand op, int32_t vr) +{ + const IRQuadCompact *q = &ir->compact_instructions[u]; + if (k != 1 || irop_get_vreg(op) != vr || !irop_dest_defines_vreg(op) || irop_get_btype(op) == IROP_BTYPE_INT64) + return 0; + if (q->op == TCCIR_OP_CMP) + { + IROperand s2 = tcc_ir_op_get_src2(ir, q); + if (!irop_is_plain_imm(s2) || irop_get_imm64_ex(ir, s2) != 0) + return 0; + } + else if (q->op != TCCIR_OP_TEST_ZERO) + return 0; + const int p = paired_prev(ir, u); + if (p < 0 || !irop_config[ir->compact_instructions[p].op].has_dest) + return 0; + IROperand d = tcc_ir_op_get_dest(ir, &ir->compact_instructions[p]); + return irop_get_vreg(d) == vr && irop_dest_defines_vreg(d); +} + +/* One `bool t;` for every check of a function, as the Zig C backend writes + * them, is read by each check's own zero test right after the SETIF that sets + * it: `CMP a,b; t <- SETIF; TEST_ZERO t; JUMPIF`. setif_branch_fuse wants the + * SETIF result read once in the whole function. Enough is that every read of + * t is a zero test right after an instruction defining t (no jump landing in + * between): each definition then reaches its own test alone. Fusing a + * quartet removes one definition with its read, so what the table records + * holds to the end of the pass. */ +static int setif_paired_fuse(TCCIRState *ir) +{ + const int n = ir->next_instruction_index; + const int nv = ir->next_local_variable, nt = ir->next_temporary_variable; + uint8_t *unpaired = tcc_mallocz((size_t)(nv + nt + 1)); + for (int u = 0; u < n; u++) + { + const IRQuadCompact *q = &ir->compact_instructions[u]; + if (q->op == TCCIR_OP_NOP) + continue; + if (irop_config[q->op].has_dest) + { + IROperand d = tcc_ir_op_get_dest(ir, q); + const int di = paired_index(ir, irop_get_vreg(d)); + if (di >= 0 && !irop_dest_defines_vreg(d)) + unpaired[di] = 1; /* written through: its value is an address */ + } + for (int k = 1; k <= 3; k++) + { + if ((k == 1 && !irop_config[q->op].has_src1) || (k == 2 && !irop_config[q->op].has_src2) || + (k == 3 && q->op != TCCIR_OP_MLA)) + continue; + IROperand op = k == 1 ? tcc_ir_op_get_src1(ir, q) : k == 2 ? tcc_ir_op_get_src2(ir, q) + : tcc_ir_op_get_accum(ir, q); + const int32_t vr = irop_get_vreg(op); + const int idx = paired_index(ir, vr); + if (idx >= 0 && !paired_zero_test(ir, u, k, op, vr)) + unpaired[idx] = 1; + } + } + + int changes = 0; + for (int i = 0; i < n; i++) + { + if (ir->compact_instructions[i].op != TCCIR_OP_CMP) + continue; + const int si = ir_skip_nops_forward(ir, i + 1, n); + const int ti = si < n ? ir_skip_nops_forward(ir, si + 1, n) : n; + const int ji = ti < n ? ir_skip_nops_forward(ir, ti + 1, n) : n; + if (ji >= n) + continue; + IRQuadCompact *setif_q = &ir->compact_instructions[si]; + IRQuadCompact *test_q = &ir->compact_instructions[ti]; + IRQuadCompact *jump_q = &ir->compact_instructions[ji]; + if (setif_q->op != TCCIR_OP_SETIF || jump_q->op != TCCIR_OP_JUMPIF || setif_q->is_jump_target || + paired_prev(ir, ji) != ti) + continue; + IROperand sd = tcc_ir_op_get_dest(ir, setif_q); + const int32_t vr = irop_get_vreg(sd); + const int idx = paired_index(ir, vr); + if (idx < 0 || unpaired[idx] || !irop_dest_defines_vreg(sd)) + continue; + /* The test must be this SETIF's own. */ + if (!paired_zero_test(ir, ti, 1, tcc_ir_op_get_src1(ir, test_q), vr) || paired_prev(ir, ti) != si) + continue; + IRLiveInterval *li = tcc_ir_get_live_interval(ir, vr); + if (!li || li->addrtaken || li->is_volatile) + continue; + + IROperand jump_src1 = tcc_ir_op_get_src1(ir, jump_q); + const int setif_tok = (int)irop_get_imm64_ex(ir, tcc_ir_op_get_src1(ir, setif_q)); + const int jump_tok = (int)irop_get_imm64_ex(ir, jump_src1); + int new_tok; + if (jump_tok == 0x94) + new_tok = invert_cond_token(setif_tok); + else if (jump_tok == 0x95) + new_tok = setif_tok; + else + continue; + if (new_tok < 0) + continue; + tcc_ir_set_src1(ir, ji, irop_make_imm32(-1, new_tok, irop_get_btype(jump_src1))); + setif_q->op = TCCIR_OP_NOP; + test_q->op = TCCIR_OP_NOP; + changes++; + } + tcc_free(unpaired); + return changes; +} + int tcc_ir_opt_setif_branch_fuse(TCCIRState *ir) { if (ir->next_instruction_index < 4) @@ -658,6 +807,7 @@ int tcc_ir_opt_setif_branch_fuse(TCCIRState *ir) tcc_ir_opt_ctx_init(&ctx, ir); int changes = tcc_ir_opt_run_gens(&ctx, branch_gens, branch_gens_count); tcc_ir_opt_ctx_free(&ctx); + changes += setif_paired_fuse(ir); return changes; } diff --git a/source/opt/flat/scalar/branch_fold.c b/source/opt/flat/scalar/branch_fold.c deleted file mode 100644 index 5f84a0969..000000000 --- a/source/opt/flat/scalar/branch_fold.c +++ /dev/null @@ -1,309 +0,0 @@ -/* - * TCC IR - Float-branch folding (flat pass) - * - * Copyright (c) 2025 Mateusz Stadnik - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation. - */ -/* Relocated flat pass, Branch A [A*]; see docs/plan_legacy_flat_ir_ssa_retire.md */ - -#define USING_GLOBALS - -#include "ir.h" -#include "opt_utils.h" -#include "opt_du.h" -#include "opt_alias.h" -#include "opt_engine.h" -#include "opt/flat/branch.h" - -static int ir_opt_match_zero_test(TCCIRState *ir, int idx, IROperand *expr_out) -{ - IRQuadCompact *q; - IROperand src1; - IROperand src2; - - if (!ir || idx < 0 || idx >= ir->next_instruction_index || !expr_out) - return 0; - - q = &ir->compact_instructions[idx]; - if (q->op == TCCIR_OP_TEST_ZERO) - { - *expr_out = tcc_ir_op_get_src1(ir, q); - return 1; - } - - if (q->op != TCCIR_OP_CMP) - return 0; - - src1 = tcc_ir_op_get_src1(ir, q); - src2 = tcc_ir_op_get_src2(ir, q); - if (irop_is_immediate(src2) && irop_get_imm64_ex(ir, src2) == 0) - { - *expr_out = src1; - return 1; - } - if (irop_is_immediate(src1) && irop_get_imm64_ex(ir, src1) == 0) - { - *expr_out = src2; - return 1; - } - - return 0; -} - -int tcc_ir_opt_float_branch_fold(TCCIRState *ir) -{ - int n = ir->next_instruction_index; - int changes = 0; - uint8_t *is_merge; - - if (n < 4) - return 0; - - is_merge = ir_opt_build_merge_bitmap(ir, n); - - for (int i = 0; i < n; ++i) - { - IRQuadCompact *q = &ir->compact_instructions[i]; - - if (q->op == TCCIR_OP_FUNCCALLVOID || q->op == TCCIR_OP_FUNCCALLVAL) - { - Sym *callee; - const char *name; - int jump1_idx = ir_opt_next_non_nop(ir, i + 1); - int cmp2_idx; - int jump2_idx; - IRQuadCompact *jump1; - IRQuadCompact *cmp2; - IRQuadCompact *jump2; - IROperand arg0; - IROperand arg1; - IROperand cmp2_arg0; - IROperand cmp2_arg1; - int tok1; - int tok2; - int known_fact; - int effective_tok2 = -1; - int is_swapped = 0; - - callee = irop_get_sym_ex(ir, tcc_ir_op_get_src1(ir, q)); - if (!callee) - continue; - name = get_tok_str(callee->v, NULL); - if (!ir_opt_is_flag_cmp_helper_name(name)) - continue; - if (!ir_opt_get_call_param_operand(ir, i, 0, &arg0) || !ir_opt_get_call_param_operand(ir, i, 1, &arg1)) - continue; - - if (jump1_idx < 0) - continue; - jump1 = &ir->compact_instructions[jump1_idx]; - if (jump1->op != TCCIR_OP_JUMPIF) - continue; - - cmp2_idx = -1; - jump2_idx = -1; - for (int scan_idx = ir_opt_next_non_nop(ir, jump1_idx + 1); scan_idx >= 0 && scan_idx < n; - scan_idx = ir_opt_next_non_nop(ir, scan_idx + 1)) - { - IRQuadCompact *scan_q; - Sym *scan_callee; - const char *scan_name; - - if (is_merge[scan_idx / 8] & (1 << (scan_idx % 8))) - break; - - scan_q = &ir->compact_instructions[scan_idx]; - if (scan_q->op != TCCIR_OP_FUNCCALLVOID && scan_q->op != TCCIR_OP_FUNCCALLVAL) - { - if (!ir_opt_is_pure_fallthrough_instruction(ir, scan_idx)) - break; - continue; - } - - scan_callee = irop_get_sym_ex(ir, tcc_ir_op_get_src1(ir, scan_q)); - scan_name = scan_callee ? get_tok_str(scan_callee->v, NULL) : NULL; - if (!ir_opt_is_flag_cmp_helper_name(scan_name)) - { - if (!ir_opt_is_pure_fallthrough_instruction(ir, scan_idx)) - break; - continue; - } - - cmp2_idx = scan_idx; - jump2_idx = ir_opt_next_non_nop(ir, cmp2_idx + 1); - break; - } - - if (cmp2_idx < 0 || jump2_idx < 0) - continue; - - cmp2 = &ir->compact_instructions[cmp2_idx]; - jump2 = &ir->compact_instructions[jump2_idx]; - if (jump2->op != TCCIR_OP_JUMPIF) - continue; - - callee = irop_get_sym_ex(ir, tcc_ir_op_get_src1(ir, cmp2)); - if (!callee) - continue; - name = get_tok_str(callee->v, NULL); - if (!ir_opt_is_flag_cmp_helper_name(name)) - continue; - if (!ir_opt_get_call_param_operand(ir, cmp2_idx, 0, &cmp2_arg0) || - !ir_opt_get_call_param_operand(ir, cmp2_idx, 1, &cmp2_arg1)) - { - continue; - } - - tok1 = (int)irop_get_imm64_ex(ir, tcc_ir_op_get_src1(ir, jump1)); - tok2 = (int)irop_get_imm64_ex(ir, tcc_ir_op_get_src1(ir, jump2)); - known_fact = vrp_negate_cmp_tok(tok1); - if (known_fact < 0) - { - continue; - } - - int eq1 = ir_opt_pure_expr_equal(ir, arg0, i, cmp2_arg0, cmp2_idx, 0); - int eq2 = ir_opt_pure_expr_equal(ir, arg1, i, cmp2_arg1, cmp2_idx, 0); - if (eq1 && eq2) - effective_tok2 = tok2; - else if (ir_opt_pure_expr_equal(ir, arg0, i, cmp2_arg1, cmp2_idx, 0) && - ir_opt_pure_expr_equal(ir, arg1, i, cmp2_arg0, cmp2_idx, 0)) - { - is_swapped = 1; - effective_tok2 = vrp_swap_cmp_tok(tok2); - } - - if (effective_tok2 < 0) - { - continue; - } - - if (is_swapped) - { - IROperand jmp1_dest = tcc_ir_op_get_dest(ir, jump1); - IROperand jmp2_dest = tcc_ir_op_get_dest(ir, jump2); - if (jmp1_dest.u.imm32 != jmp2_dest.u.imm32) - { - switch (known_fact) - { - case TOK_LT: - case TOK_GT: - case TOK_ULT: - case TOK_UGT: - break; - default: - continue; - } - } - } - - if (fcmp_cmp_implies(known_fact, effective_tok2)) - { - IROperand jmp2_dest = tcc_ir_op_get_dest(ir, jump2); - cmp2->op = TCCIR_OP_NOP; - jump2->op = TCCIR_OP_JUMP; - tcc_ir_set_dest(ir, jump2_idx, jmp2_dest); - changes++; - } - else if (fcmp_cmp_implies(known_fact, vrp_negate_cmp_tok(effective_tok2))) - { - cmp2->op = TCCIR_OP_NOP; - jump2->op = TCCIR_OP_NOP; - changes++; - } - - continue; - } - - if (q->op == TCCIR_OP_TEST_ZERO || q->op == TCCIR_OP_CMP) - { - IRQuadCompact *jump1; - int jump1_idx = ir_opt_next_non_nop(ir, i + 1); - int known_zero = -1; - IROperand expr1; - - if (!ir_opt_match_zero_test(ir, i, &expr1)) - continue; - - if (jump1_idx < 0) - continue; - jump1 = &ir->compact_instructions[jump1_idx]; - if (jump1->op != TCCIR_OP_JUMPIF) - continue; - - switch ((int)irop_get_imm64_ex(ir, tcc_ir_op_get_src1(ir, jump1))) - { - case TOK_NE: - known_zero = 1; - break; - case TOK_EQ: - known_zero = 0; - break; - default: - break; - } - if (known_zero < 0) - continue; - - for (int test2_idx = ir_opt_next_non_nop(ir, jump1_idx + 1); test2_idx >= 0 && test2_idx + 1 < n; - test2_idx = ir_opt_next_non_nop(ir, test2_idx + 1)) - { - IRQuadCompact *test2; - IRQuadCompact *jump2; - int jump2_idx; - int tok2; - IROperand expr2; - int is_zero_test_candidate; - - if (is_merge[test2_idx / 8] & (1 << (test2_idx % 8))) - break; - - test2 = &ir->compact_instructions[test2_idx]; - is_zero_test_candidate = ir_opt_match_zero_test(ir, test2_idx, &expr2); - if (!is_zero_test_candidate) - { - if (!ir_opt_is_pure_fallthrough_instruction(ir, test2_idx)) - break; - continue; - } - - jump2_idx = ir_opt_next_non_nop(ir, test2_idx + 1); - if (jump2_idx < 0) - break; - - jump2 = &ir->compact_instructions[jump2_idx]; - if (jump2->op != TCCIR_OP_JUMPIF) - break; - - if (!ir_opt_pure_expr_equal(ir, expr1, i, expr2, test2_idx, 0)) - continue; - - tok2 = (int)irop_get_imm64_ex(ir, tcc_ir_op_get_src1(ir, jump2)); - if ((known_zero && tok2 == TOK_EQ) || (!known_zero && tok2 == TOK_NE)) - { - IROperand jmp2_dest = tcc_ir_op_get_dest(ir, jump2); - test2->op = TCCIR_OP_NOP; - jump2->op = TCCIR_OP_JUMP; - tcc_ir_set_dest(ir, jump2_idx, jmp2_dest); - changes++; - } - else if ((known_zero && tok2 == TOK_NE) || (!known_zero && tok2 == TOK_EQ)) - { - test2->op = TCCIR_OP_NOP; - jump2->op = TCCIR_OP_NOP; - changes++; - } - break; - } - } - } - - tcc_free(is_merge); - return changes; -} - - -int tcc_ir_opt_float_branch_fold_ex(IROptCtx *ctx) { return tcc_ir_opt_float_branch_fold(ctx->ir); } diff --git a/source/opt/flat/scalar/call_result.c b/source/opt/flat/scalar/call_result.c index 4c0f5b222..20452614b 100644 --- a/source/opt/flat/scalar/call_result.c +++ b/source/opt/flat/scalar/call_result.c @@ -184,6 +184,8 @@ OPT_GEN_FLAT(fold_call_result_store, TCCIR_OP_FUNCCALLVAL) irop_get_tag(call_dest) != IROP_TAG_STACKOFF) return 0; int64_t call_dest_off = irop_get_imm64_ex(ir, call_dest); + const int call_dest_64 = irop_is_64bit(call_dest); + const int call_sz = call_dest_64 ? 8 : 4; int load_idx = -1; int32_t load_dest_vr = -1; @@ -201,9 +203,15 @@ OPT_GEN_FLAT(fold_call_result_store, TCCIR_OP_FUNCCALLVAL) continue; if (irop_get_tag(ops[k]) != IROP_TAG_STACKOFF) continue; - if (irop_get_imm64_ex(ir, ops[k]) != call_dest_off) + if (irop_get_imm64_ex(ir, ops[k]) != call_dest_off) { + /* Another access overlapping what the call writes -- a word copy's + * read of a 64-bit result's high half -- is a second use. */ + int64_t o = irop_get_imm64_ex(ir, ops[k]); + if (o > call_dest_off - 8 && o < call_dest_off + call_sz) + multi_use = 1; continue; - if (p->op == TCCIR_OP_LOAD && k == 1 && load_idx < 0) { + } + if (p->op == TCCIR_OP_LOAD && k == 1 && load_idx < 0 && irop_is_64bit(ops[k]) == call_dest_64) { load_idx = j; load_dest_vr = irop_get_vreg(tcc_ir_op_get_dest(ir, p)); } else { diff --git a/source/opt/flat/scalar/cmp_expr_fold.c b/source/opt/flat/scalar/cmp_expr_fold.c index c8502cb4b..1bfb336ea 100644 --- a/source/opt/flat/scalar/cmp_expr_fold.c +++ b/source/opt/flat/scalar/cmp_expr_fold.c @@ -380,6 +380,16 @@ int tcc_ir_opt_cmp_expr_fold(TCCIRState *ir) } else { + /* Skip address-taken VARs, as the mixed case above does: a store + * through a pointer to one -- or a call handed its address -- gives it + * a new value without an IR def, so the single def the counts see is + * just the initialiser. `regoff_t so = 0, eo = 0; find(.., &so, &eo); + * if (so == eo)` folded to true here and toybox awk read its whole + * input as one record. */ + IRLiveInterval *iv1 = tcc_ir_get_live_interval(ir, vr1); + IRLiveInterval *iv2 = tcc_ir_get_live_interval(ir, vr2); + if ((iv1 && iv1->addrtaken) || (iv2 && iv2->addrtaken)) + continue; def1 = tcc_ir_find_defining_instruction(ir, vr1, i); def2 = tcc_ir_find_defining_instruction(ir, vr2, i); if (def1 < 0 || def2 < 0 || def1 == def2) diff --git a/source/opt/flat/scalar/complex_const_param_fold.c b/source/opt/flat/scalar/complex_const_param_fold.c deleted file mode 100644 index 98998145e..000000000 --- a/source/opt/flat/scalar/complex_const_param_fold.c +++ /dev/null @@ -1,161 +0,0 @@ -/* - * TCC IR - Complex-constant param folding (flat pass) - * - * Copyright (c) 2025 Mateusz Stadnik - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation. - */ -/* Relocated flat pass, Branch A [A*]; see docs/plan_legacy_flat_ir_ssa_retire.md */ - -#define USING_GLOBALS - -#include "ir.h" -#include "opt.h" -#include "opt_engine.h" -#include "opt_utils.h" -#include "opt_du.h" - -/* Fold two const-float component stores + complex FUNCPARAMVAL into a packed 64-bit immediate. */ -int tcc_ir_opt_complex_const_param_fold(TCCIRState *ir) -{ - if (!ir || ir->next_instruction_index == 0) - return 0; - - const int n = ir->next_instruction_index; - int changes = 0; - - for (int i = 0; i < n; i++) - { - IRQuadCompact *q = &ir->compact_instructions[i]; - if (q->op != TCCIR_OP_FUNCPARAMVAL) - continue; - - IROperand src1 = tcc_ir_op_get_src1(ir, q); - if (irop_get_tag(src1) != IROP_TAG_STACKOFF) - continue; - if (!src1.is_lval || !src1.is_complex) - continue; - /* Only handle _Complex float (8 bytes packed: real_u32 | imag_u32 << 32) */ - if (src1.btype != IROP_BTYPE_FLOAT32) - continue; - /* Skip stack offsets carrying a vreg (spill slots), not raw locals. */ - if (irop_get_vreg(src1) != -1) - continue; - /* Skip incoming-arg stack slots; they alias caller-allocated memory. */ - if (src1.is_param) - continue; - - int real_off = (int)irop_get_stack_offset(src1); - int imag_off = real_off + 4; - - int real_store_idx = -1; - int imag_store_idx = -1; - uint32_t real_bits = 0; - uint32_t imag_bits = 0; - int conflict = 0; - - for (int j = 0; j < n && !conflict; j++) - { - if (j == i) - continue; - IRQuadCompact *p = &ir->compact_instructions[j]; - if (p->op == TCCIR_OP_NOP) - continue; - - /* dest: detect a 4-byte constant STORE/ASSIGN to either component slot. */ - if (irop_config[p->op].has_dest) - { - IROperand dest = tcc_ir_op_get_dest(ir, p); - if (irop_get_tag(dest) == IROP_TAG_STACKOFF && irop_get_vreg(dest) == -1 && dest.is_lval && !dest.is_param) - { - int doff = (int)irop_get_stack_offset(dest); - if (doff == real_off || doff == imag_off) - { - if ((p->op != TCCIR_OP_STORE && p->op != TCCIR_OP_ASSIGN) || dest.is_complex || - dest.btype != IROP_BTYPE_FLOAT32) - { - conflict = 1; - break; - } - IROperand sv = tcc_ir_op_get_src1(ir, p); - int tag = irop_get_tag(sv); - if (sv.is_sym || sv.is_lval || sv.is_complex) - { - conflict = 1; - break; - } - if (tag != IROP_TAG_IMM32 && tag != IROP_TAG_F32) - { - conflict = 1; - break; - } - uint32_t bits = (uint32_t)irop_get_imm64_ex(ir, sv); - if (doff == real_off) - { - if (real_store_idx != -1 || j >= i) - { - conflict = 1; - break; - } - real_store_idx = j; - real_bits = bits; - } - else - { - if (imag_store_idx != -1 || j >= i) - { - conflict = 1; - break; - } - imag_store_idx = j; - imag_bits = bits; - } - continue; - } - } - } - - /* Any other reference to the 8-byte slot disqualifies the fold. */ - for (int s = 0; s < 2; s++) - { - if (s == 0 && !irop_config[p->op].has_src1) - continue; - if (s == 1 && !irop_config[p->op].has_src2) - continue; - IROperand src = s ? tcc_ir_op_get_src2(ir, p) : tcc_ir_op_get_src1(ir, p); - if (irop_get_tag(src) != IROP_TAG_STACKOFF) - continue; - if (irop_get_vreg(src) != -1) - continue; - int soff = (int)irop_get_stack_offset(src); - if (soff >= real_off && soff < imag_off + 4) - { - conflict = 1; - break; - } - } - } - - if (conflict || real_store_idx < 0 || imag_store_idx < 0) - continue; - - /* Pack {real, imag} into a 64-bit complex-float immediate. */ - uint64_t packed = (uint64_t)real_bits | ((uint64_t)imag_bits << 32); - uint32_t pool_idx = tcc_ir_pool_add_i64(ir, (int64_t)packed); - IROperand new_src1 = irop_make_i64(-1, pool_idx, IROP_BTYPE_FLOAT32); - new_src1.is_complex = 1; - new_src1.is_lval = 0; - - tcc_ir_set_src1(ir, i, new_src1); - ir->compact_instructions[real_store_idx].op = TCCIR_OP_NOP; - ir->compact_instructions[imag_store_idx].op = TCCIR_OP_NOP; - - LOG_IR_GEN("=== COMPLEX CONST PARAM FOLD: stack[%d..%d] folded into PARAM at i=%d ===", real_off, real_off + 7, i); - changes++; - } - - return changes; -} - diff --git a/source/opt/flat/scalar/float_narrow.c b/source/opt/flat/scalar/float_narrow.c deleted file mode 100644 index ac652eaa1..000000000 --- a/source/opt/flat/scalar/float_narrow.c +++ /dev/null @@ -1,276 +0,0 @@ -/* - * TCC IR - double->float library-call narrowing (flat, pre-SSA) - * - * Copyright (c) 2026 Mateusz Stadnik - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation. - */ - -#define USING_GLOBALS - -#include "ir.h" -#include "opt.h" -#include "opt_engine.h" -#include "opt_utils.h" -#include "opt_du.h" - -typedef struct -{ - const char *double_name; - const char *float_name; -} FloatNarrowEntry; - -static const FloatNarrowEntry float_narrow_table[] = { - {"floor", "floorf"}, {"ceil", "ceilf"}, {"trunc", "truncf"}, {"round", "roundf"}, - {"fabs", "fabsf"}, {"nearbyint", "nearbyintf"}, {"rint", "rintf"}, -}; -#define NUM_FLOAT_NARROW (sizeof(float_narrow_table) / sizeof(float_narrow_table[0])) - -typedef struct -{ - int param_idx; /* instruction index of the FUNCPARAMVAL */ - int call_idx; /* instruction index of the FUNCCALLVAL */ - int32_t src_vr; /* original source vreg (float for f2d, double for d2f) */ - int32_t dst_vr; -} ConvCallInfo; - -#define MAX_CONV_CALLS 32 - -static void capture_pending_param(TCCIRState *ir, IRQuadCompact *q, int i, int *pending_param_idx, - int32_t *pending_param_src_vr, int *pending_param_call_id) -{ - IROperand src1 = tcc_ir_op_get_src1(ir, q); - IROperand src2 = tcc_ir_op_get_src2(ir, q); - uint32_t encoded = (uint32_t)irop_get_imm64_ex(ir, src2); - int param_idx_val = TCCIR_DECODE_PARAM_IDX(encoded); - - if (param_idx_val == 0) - { - *pending_param_idx = i; - *pending_param_src_vr = irop_is_immediate(src1) ? -1 : irop_get_vreg(src1); - *pending_param_call_id = TCCIR_DECODE_CALL_ID(encoded); - } -} - -static void record_conv_call(ConvCallInfo *arr, int *count, int param_idx, int call_idx, int32_t src_vr, - int32_t dst_vr) -{ - if (*count < MAX_CONV_CALLS) - { - arr[*count].param_idx = param_idx; - arr[*count].call_idx = call_idx; - arr[*count].src_vr = src_vr; - arr[*count].dst_vr = dst_vr; - (*count)++; - } -} - -int tcc_ir_opt_float_narrowing(TCCIRState *ir) -{ - int n = ir->next_instruction_index; - int changes = 0; - - if (n < 4) - return 0; - - ConvCallInfo f2d_calls[MAX_CONV_CALLS]; - ConvCallInfo d2f_calls[MAX_CONV_CALLS]; - int num_f2d = 0, num_d2f = 0; - - - int pending_param_idx = -1; - int32_t pending_param_src_vr = -1; - int pending_param_call_id = -1; - - for (int i = 0; i < n; i++) - { - IRQuadCompact *q = &ir->compact_instructions[i]; - - if (q->op == TCCIR_OP_FUNCPARAMVAL) - { - capture_pending_param(ir, q, i, &pending_param_idx, &pending_param_src_vr, &pending_param_call_id); - continue; - } - - if (q->op == TCCIR_OP_FUNCCALLVAL && pending_param_idx >= 0) - { - IROperand src1 = tcc_ir_op_get_src1(ir, q); - IROperand src2 = tcc_ir_op_get_src2(ir, q); - Sym *callee = irop_get_sym_ex(ir, src1); - if (!callee) - { - pending_param_idx = -1; - continue; - } - - const char *name = get_tok_str(callee->v, NULL); - if (!name) - { - pending_param_idx = -1; - continue; - } - - uint32_t call_encoded = (uint32_t)irop_get_imm64_ex(ir, src2); - int this_call_id = TCCIR_DECODE_CALL_ID(call_encoded); - - IROperand dest = tcc_ir_op_get_dest(ir, q); - int32_t dst_vr = irop_get_vreg(dest); - - if (strcmp(name, "__aeabi_f2d") == 0 && this_call_id == pending_param_call_id) - { - record_conv_call(f2d_calls, &num_f2d, pending_param_idx, i, pending_param_src_vr, dst_vr); - } - else if (strcmp(name, "__aeabi_d2f") == 0 && this_call_id == pending_param_call_id) - { - record_conv_call(d2f_calls, &num_d2f, pending_param_idx, i, pending_param_src_vr, dst_vr); - } - - pending_param_idx = -1; - continue; - } - - if (q->op != TCCIR_OP_NOP) - pending_param_idx = -1; - } - - if (num_f2d == 0) - return 0; - - - pending_param_idx = -1; - pending_param_src_vr = -1; - pending_param_call_id = -1; - - for (int i = 0; i < n; i++) - { - IRQuadCompact *q = &ir->compact_instructions[i]; - - if (q->op == TCCIR_OP_FUNCPARAMVAL) - { - capture_pending_param(ir, q, i, &pending_param_idx, &pending_param_src_vr, &pending_param_call_id); - continue; - } - - if (q->op != TCCIR_OP_FUNCCALLVAL || pending_param_idx < 0) - { - if (q->op != TCCIR_OP_NOP && q->op != TCCIR_OP_FUNCPARAMVOID) - pending_param_idx = -1; - continue; - } - - IROperand src1 = tcc_ir_op_get_src1(ir, q); - Sym *callee = irop_get_sym_ex(ir, src1); - if (!callee) - { - pending_param_idx = -1; - continue; - } - - const char *name = get_tok_str(callee->v, NULL); - if (!name) - { - pending_param_idx = -1; - continue; - } - - const char *float_name = NULL; - for (size_t j = 0; j < NUM_FLOAT_NARROW; j++) - { - if (strcmp(name, float_narrow_table[j].double_name) == 0) - { - float_name = float_narrow_table[j].float_name; - break; - } - } - - if (!float_name) - { - pending_param_idx = -1; - continue; - } - - /* Never narrow into a self-call. Inside float_name's OWN definition — - * libm's `float ceilf(float x) { return (float)ceil((double)x); }` — the - * rewrite turns the wrapper into `return ceilf(x);`, which - * infinite_self_recursion then (rightly, by then) collapses into a `b .` - * self-loop: every ceilf/floorf/roundf/truncf call on the device hung. */ - if (funcname && strcmp(funcname, float_name) == 0) - { - pending_param_idx = -1; - continue; - } - - ConvCallInfo *f2d_info = NULL; - for (int k = 0; k < num_f2d; k++) - { - if (f2d_calls[k].dst_vr == pending_param_src_vr) - { - f2d_info = &f2d_calls[k]; - break; - } - } - - if (!f2d_info) - { - pending_param_idx = -1; - continue; - } - - IROperand func_dest = tcc_ir_op_get_dest(ir, q); - int32_t func_result_vr = irop_get_vreg(func_dest); - int func_call_idx = i; - int func_param_idx = pending_param_idx; - - ConvCallInfo *d2f_info = NULL; - for (int k = 0; k < num_d2f; k++) - { - if (d2f_calls[k].src_vr == func_result_vr) - { - d2f_info = &d2f_calls[k]; - break; - } - } - - if (d2f_info) - { - - IROperand orig_float_param = tcc_ir_op_get_src1(ir, &ir->compact_instructions[f2d_info->param_idx]); - tcc_ir_set_src1(ir, func_param_idx, orig_float_param); - - change_callee_sym(ir, func_call_idx, float_name, VT_FLOAT); - - IROperand d2f_dest = tcc_ir_op_get_dest(ir, &ir->compact_instructions[d2f_info->call_idx]); - tcc_ir_set_dest(ir, func_call_idx, d2f_dest); - - ir->compact_instructions[f2d_info->param_idx].op = TCCIR_OP_NOP; - ir->compact_instructions[f2d_info->call_idx].op = TCCIR_OP_NOP; - - ir->compact_instructions[d2f_info->param_idx].op = TCCIR_OP_NOP; - ir->compact_instructions[d2f_info->call_idx].op = TCCIR_OP_NOP; - - LOG_IR_GEN("FLOAT NARROW (Case 1): %s → %s at i=%d, NOP'd f2d@%d and d2f@%d", name, float_name, func_call_idx, - f2d_info->call_idx, d2f_info->call_idx); - changes++; - } - else - { - - change_callee_sym(ir, f2d_info->call_idx, float_name, VT_FLOAT); - - change_callee_sym(ir, func_call_idx, "__aeabi_f2d", VT_INT); - - LOG_IR_GEN("FLOAT NARROW (Case 2): swapped %s↔f2d at i=%d,%d", name, f2d_info->call_idx, func_call_idx); - changes++; - } - - /* Invalidate modified f2d entry to prevent double-processing */ - f2d_info->dst_vr = -1; - - pending_param_idx = -1; - } - - return changes; -} -int tcc_ir_opt_float_narrowing_ex(IROptCtx *ctx) { return tcc_ir_opt_float_narrowing(ctx->ir); } diff --git a/source/opt/flat/scalar/global_init_prop.c b/source/opt/flat/scalar/global_init_prop.c index 6d2c96b4d..cb9b015e9 100644 --- a/source/opt/flat/scalar/global_init_prop.c +++ b/source/opt/flat/scalar/global_init_prop.c @@ -44,8 +44,9 @@ int tcc_ir_opt_global_init_prop(TCCIRState *ir) continue; Sym *sym = ref->sym; - /* Linkage / attribute gates. */ - if (sym->a.weak || sym->a.dllimport) + /* Linkage / attribute gates. A tentative definition's bytes are not + * its value while its definition may still follow. */ + if (sym->a.weak || sym->a.dllimport || sym->a.tentative) continue; const int ttype = sym->type.t; diff --git a/source/opt/flat/scalar/inline_param_copy.c b/source/opt/flat/scalar/inline_param_copy.c index f483fdf72..69f96c8a3 100644 --- a/source/opt/flat/scalar/inline_param_copy.c +++ b/source/opt/flat/scalar/inline_param_copy.c @@ -147,42 +147,134 @@ static int ipc_var_ok(TCCIRState *ir, int32_t vr) * directly: control can only enter (store_idx, last_use] from outside by a jump * that lands strictly inside it. If no such edge exists, every path reaching a * use ran through the store. Unknowable targets (computed jumps, switch - * tables) anywhere in the function force a bail. */ -static int ipc_store_dominates_window(TCCIRState *ir, int n, int store_idx, int last_use) + * tables) anywhere in the function force a bail. + * + * The pass only ever turns STOREs into NOPs, so the function's jumps are + * gathered once, sorted by target: a window then looks only at the jumps that + * land inside it instead of walking the whole function. */ +typedef struct IpcJump +{ + int target, from; +} IpcJump; + +typedef struct IpcJumps +{ + IpcJump *j; + int n; + int unknown; /* an IJUMP or SWITCH_TABLE: targets unknowable */ +} IpcJumps; + +static int ipc_jump_cmp(const void *a, const void *b) { + const IpcJump *x = a, *y = b; + if (x->target != y->target) + return x->target < y->target ? -1 : 1; + return x->from < y->from ? -1 : x->from > y->from; +} + +static void ipc_collect_jumps(TCCIRState *ir, int n, IpcJumps *js) +{ + js->j = tcc_malloc(sizeof(IpcJump) * (n + 1)); + js->n = 0; + js->unknown = 0; for (int k = 0; k < n; k++) { IRQuadCompact *q = &ir->compact_instructions[k]; - if (q->op == TCCIR_OP_NOP) - continue; if (q->op == TCCIR_OP_IJUMP || q->op == TCCIR_OP_SWITCH_TABLE) - return 0; /* target set unknown */ + js->unknown = 1; if (q->op != TCCIR_OP_JUMP && q->op != TCCIR_OP_JUMPIF) continue; - if (k >= store_idx && k <= last_use) - continue; /* an edge from inside the window cannot bypass the store */ - IROperand jd = tcc_ir_op_get_dest(ir, q); - int jt = (int)jd.u.imm32; - if (jt > store_idx && jt <= last_use) - return 0; /* entry that skips the store */ + js->j[js->n].target = (int)tcc_ir_op_get_dest(ir, q).u.imm32; + js->j[js->n].from = k; + js->n++; } + qsort(js->j, js->n, sizeof(IpcJump), ipc_jump_cmp); +} + +static int ipc_store_dominates_window(const IpcJumps *js, int store_idx, int last_use) +{ + if (js->unknown) + return 0; /* target set unknown */ + int lo = 0, hi = js->n; /* first jump with target > store_idx */ + while (lo < hi) + { + int mid = (lo + hi) / 2; + if (js->j[mid].target <= store_idx) + lo = mid + 1; + else + hi = mid; + } + for (int k = lo; k < js->n && js->j[k].target <= last_use; k++) + if (js->j[k].from < store_idx || js->j[k].from > last_use) + return 0; /* entry that skips the store */ return 1; } -/* Last index at which `vr` is read, or -1. */ -static int ipc_last_use(TCCIRState *ir, int n, int32_t vr) +/* Where a VAR can be read: every src1/src2 naming it sits in [lo, hi]. The + * span only widens -- when a rewrite makes an instruction read the VAR -- so it + * stays a superset of the reads, and the scans for a VAR's reads need look + * nowhere else. Whole-function scans per candidate made this pass quadratic: + * Zig's zig.c spent 1.6 G iterations here. */ +typedef struct IpcSpan { - int last = -1; + int lo, hi; /* lo > hi: never read */ +} IpcSpan; + +static int ipc_var_pos(int32_t vr, int max_var) +{ + if (vr < 0 || TCCIR_DECODE_VREG_TYPE(vr) != TCCIR_VREG_TYPE_VAR) + return -1; + int p = TCCIR_DECODE_VREG_POSITION(vr); + if (vr != TCCIR_ENCODE_VREG(TCCIR_VREG_TYPE_VAR, p) || p > max_var) + return -1; + return p; +} + +static void ipc_span_note(IpcSpan *span, int max_var, int32_t vr, int idx) +{ + int p = ipc_var_pos(vr, max_var); + if (p < 0) + return; + if (idx < span[p].lo) + span[p].lo = idx; + if (idx > span[p].hi) + span[p].hi = idx; +} + +static IpcSpan *ipc_collect_spans(TCCIRState *ir, int n, int max_var) +{ + IpcSpan *span = tcc_malloc(sizeof(IpcSpan) * (max_var + 1)); + for (int p = 0; p <= max_var; p++) + { + span[p].lo = n; + span[p].hi = -1; + } for (int i = 0; i < n; i++) { IRQuadCompact *q = &ir->compact_instructions[i]; if (q->op == TCCIR_OP_NOP) continue; - if ((irop_config[q->op].has_src1 && irop_get_vreg(tcc_ir_op_get_src1(ir, q)) == vr) || - (irop_config[q->op].has_src2 && irop_get_vreg(tcc_ir_op_get_src2(ir, q)) == vr)) - last = i; + if (irop_config[q->op].has_src1) + ipc_span_note(span, max_var, irop_get_vreg(tcc_ir_op_get_src1(ir, q)), i); + if (irop_config[q->op].has_src2) + ipc_span_note(span, max_var, irop_get_vreg(tcc_ir_op_get_src2(ir, q)), i); } - return last; + return span; +} + +static int ipc_reads(TCCIRState *ir, IRQuadCompact *q, int32_t vr) +{ + return q->op != TCCIR_OP_NOP && ((irop_config[q->op].has_src1 && irop_get_vreg(tcc_ir_op_get_src1(ir, q)) == vr) || + (irop_config[q->op].has_src2 && irop_get_vreg(tcc_ir_op_get_src2(ir, q)) == vr)); +} + +/* Last index in [lo, hi] at which `vr` is read, or -1. */ +static int ipc_last_use(TCCIRState *ir, int lo, int hi, int32_t vr) +{ + for (int i = hi; i >= lo; i--) + if (ipc_reads(ir, &ir->compact_instructions[i], vr)) + return i; + return -1; } int tcc_ir_opt_inline_param_copy_elim(TCCIRState *ir) @@ -199,6 +291,9 @@ int tcc_ir_opt_inline_param_copy_elim(TCCIRState *ir) if (!ipc_collect_var_defs(ir, n, &var_def, &max_var)) return 0; ipc_collect_tmp_defs(ir, n, &tmp_def, &max_tmp); + IpcSpan *span = ipc_collect_spans(ir, n, max_var); + IpcJumps jumps; + ipc_collect_jumps(ir, n, &jumps); for (int i = 0; i < n; i++) { @@ -264,9 +359,11 @@ int tcc_ir_opt_inline_param_copy_elim(TCCIRState *ir) } } } + /* dst_vr's reads: dst_vr is a VAR defined here, so it has a span. */ + const IpcSpan dspan = span[dpos]; if (!matched_var) { - int lu = ipc_last_use(ir, n, dst_vr); + int lu = ipc_last_use(ir, dspan.lo, dspan.hi, dst_vr); if (lu < 0) continue; /* A use at a LOWER index than the store can still execute AFTER it: a @@ -276,18 +373,11 @@ int tcc_ir_opt_inline_param_copy_elim(TCCIRState *ir) * window is provably covered, so require the whole live range to sit in * it. */ int used_before = 0; - for (int b = 0; b < i && !used_before; b++) - { - IRQuadCompact *bq = &ir->compact_instructions[b]; - if (bq->op == TCCIR_OP_NOP) - continue; - if ((irop_config[bq->op].has_src1 && irop_get_vreg(tcc_ir_op_get_src1(ir, bq)) == dst_vr) || - (irop_config[bq->op].has_src2 && irop_get_vreg(tcc_ir_op_get_src2(ir, bq)) == dst_vr)) - used_before = 1; - } + for (int b = dspan.lo; b < i && !used_before; b++) + used_before = ipc_reads(ir, &ir->compact_instructions[b], dst_vr); if (used_before) continue; - if (!ipc_store_dominates_window(ir, n, i, lu)) + if (!ipc_store_dominates_window(&jumps, i, lu)) continue; lsrc = src; /* the stored temp */ src_vr = t_vr; @@ -306,7 +396,11 @@ int tcc_ir_opt_inline_param_copy_elim(TCCIRState *ir) * would read a slot nothing writes any more. */ int rewrote = 0; int left_behind = 0; - for (int j = i + 1; j < n; j++) + /* Past dspan.hi nothing reads dst_vr, and neither variable is written + * after i -- each has its one def at or before i, and the pass adds none -- + * so the rest of the function has nothing to rewrite or stop at. */ + const int j_end = dspan.hi < n - 1 ? dspan.hi + 1 : n; + for (int j = i + 1; j < j_end; j++) { IRQuadCompact *u = &ir->compact_instructions[j]; if (u->op == TCCIR_OP_NOP) @@ -348,6 +442,7 @@ int tcc_ir_opt_inline_param_copy_elim(TCCIRState *ir) !s.is_llocal) { tcc_ir_set_src1(ir, j, lsrc); + ipc_span_note(span, max_var, src_vr, j); rewrote++; } else @@ -362,6 +457,7 @@ int tcc_ir_opt_inline_param_copy_elim(TCCIRState *ir) !s.is_llocal) { tcc_ir_set_src2(ir, j, lsrc); + ipc_span_note(span, max_var, src_vr, j); rewrote++; } else if (irop_get_vreg(s) == dst_vr) @@ -380,6 +476,8 @@ int tcc_ir_opt_inline_param_copy_elim(TCCIRState *ir) tcc_free(var_def); tcc_free(tmp_def); + tcc_free(span); + tcc_free(jumps.j); return changes; } diff --git a/source/opt/flat/scalar/neg_chain_cse.c b/source/opt/flat/scalar/neg_chain_cse.c deleted file mode 100644 index 8d98e619a..000000000 --- a/source/opt/flat/scalar/neg_chain_cse.c +++ /dev/null @@ -1,236 +0,0 @@ -/* - * TCC IR - Negation-chain CSE - * - * Copyright (c) 2025 Mateusz Stadnik - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation. - */ - -#define USING_GLOBALS - -#include "tcc.h" -#include "tccir.h" -#include "tccir_operand.h" -#include "ir.h" -#include "opt.h" -#include "opt_utils.h" -#include "opt_engine.h" -#include "log.h" - -#ifndef LOG_NEG_CHAIN -#ifdef TCC_LOG_NEG_CHAIN -#define LOG_NEG_CHAIN(...) fprintf(stderr, "[NEG_CHAIN] " __VA_ARGS__), fprintf(stderr, "\n") -#else -#define LOG_NEG_CHAIN(...) ((void)0) -#endif -#endif - -typedef struct -{ - int32_t base_vr; /* base vreg this TEMP traces to (must be TEMP) */ - uint8_t sign; /* 0 = +base, 1 = -base */ - uint8_t valid; -} NegCanon; - -int tcc_ir_opt_neg_chain_cse(TCCIRState *ir) -{ - int n = ir->next_instruction_index; - if (n < 2) - return 0; - - int max_tmp = -1; - for (int i = 0; i < n; i++) - { - IRQuadCompact *q = &ir->compact_instructions[i]; - if (q->op == TCCIR_OP_NOP) - continue; - IROperand ops[3]; - ops[0] = tcc_ir_op_get_dest(ir, q); - ops[1] = tcc_ir_op_get_src1(ir, q); - ops[2] = tcc_ir_op_get_src2(ir, q); - for (int k = 0; k < 3; k++) - { - int32_t vr = irop_get_vreg(ops[k]); - if (vr >= 0 && TCCIR_DECODE_VREG_TYPE(vr) == TCCIR_VREG_TYPE_TEMP) - { - int pos = TCCIR_DECODE_VREG_POSITION(vr); - if (pos > max_tmp) - max_tmp = pos; - } - } - } - if (max_tmp < 0) - return 0; - - size_t canon_size = (size_t)(max_tmp + 1) * sizeof(NegCanon); - size_t first_size = (size_t)(max_tmp + 1) * sizeof(int32_t); - NegCanon *canon = (NegCanon *)tcc_mallocz(canon_size); - /* first_pos/first_neg[base_pos]: earliest TEMP vreg with canonical (base, +/-). */ - int32_t *first_pos = (int32_t *)tcc_malloc(first_size); - int32_t *first_neg = (int32_t *)tcc_malloc(first_size); - for (int i = 0; i <= max_tmp; i++) - { - first_pos[i] = -1; - first_neg[i] = -1; - } - - /* Merges clear only touched entries; whole-table clears were quadratic. */ - /* At most one canon and one first entry is recorded per instruction, so n bounds both lists. */ - int *touched_canon = (int *)tcc_malloc((size_t)n * sizeof(int)); - int *touched_first = (int *)tcc_malloc((size_t)n * sizeof(int)); - int n_touched_canon = 0; - int n_touched_first = 0; - - uint8_t *is_merge = ir_opt_build_merge_bitmap(ir, n); - - int changes = 0; - - for (int i = 0; i < n; i++) - { - /* Tracked TEMPs need not be live on every incoming edge: reset at merges. */ - if (is_merge[i / 8] & (1 << (i % 8))) - { - for (int k = 0; k < n_touched_canon; k++) - canon[touched_canon[k]].valid = 0; - n_touched_canon = 0; - for (int k = 0; k < n_touched_first; k++) - { - int bp = touched_first[k]; - first_pos[bp] = -1; - first_neg[bp] = -1; - } - n_touched_first = 0; - } - - IRQuadCompact *q = &ir->compact_instructions[i]; - if (q->op == TCCIR_OP_NOP) - continue; - - if (!irop_config[q->op].has_dest) - continue; - - IROperand dest = tcc_ir_op_get_dest(ir, q); - int32_t dest_vr = irop_get_vreg(dest); - if (dest_vr < 0 || TCCIR_DECODE_VREG_TYPE(dest_vr) != TCCIR_VREG_TYPE_TEMP) - continue; - if (dest.is_lval) - continue; - - int dest_pos = TCCIR_DECODE_VREG_POSITION(dest_vr); - - int32_t base_vr = dest_vr; - int sign = 0; - int did_replace = 0; - - if (q->op == TCCIR_OP_ASSIGN) - { - IROperand src1 = tcc_ir_op_get_src1(ir, q); - int32_t src_vr = irop_get_vreg(src1); - /* Width must match: a narrowing/widening ASSIGN is not value-preserving. */ - if (!src1.is_lval && src_vr >= 0 && TCCIR_DECODE_VREG_TYPE(src_vr) == TCCIR_VREG_TYPE_TEMP && - irop_get_btype(dest) == irop_get_btype(src1)) - { - int src_pos = TCCIR_DECODE_VREG_POSITION(src_vr); - if (src_pos <= max_tmp && canon[src_pos].valid) - { - base_vr = canon[src_pos].base_vr; - sign = canon[src_pos].sign; - } - else - { - base_vr = src_vr; - sign = 0; - } - } - } - else if (q->op == TCCIR_OP_SUB) - { - IROperand src1 = tcc_ir_op_get_src1(ir, q); - IROperand src2 = tcc_ir_op_get_src2(ir, q); - int dest_btype = irop_get_btype(dest); - int src_btype = irop_get_btype(src2); - /* T_b = #0 SUB T_a; widths must match or a later fold drops the truncation. */ - if (irop_is_immediate(src1) && irop_get_imm64_ex(ir, src1) == 0 && - dest_btype == src_btype) - { - int32_t src_vr = irop_get_vreg(src2); - if (!src2.is_lval && src_vr >= 0 && TCCIR_DECODE_VREG_TYPE(src_vr) == TCCIR_VREG_TYPE_TEMP) - { - int src_pos = TCCIR_DECODE_VREG_POSITION(src_vr); - if (src_pos <= max_tmp && canon[src_pos].valid) - { - base_vr = canon[src_pos].base_vr; - sign = canon[src_pos].sign ? 0 : 1; - } - else - { - base_vr = src_vr; - sign = 1; - } - - int base_pos = TCCIR_DECODE_VREG_POSITION(base_vr); - int32_t existing = (sign == 1) ? first_neg[base_pos] : first_pos[base_pos]; - if (existing >= 0 && existing != dest_vr) - { - IROperand new_src = irop_make_vreg(existing, dest_btype); - q->op = TCCIR_OP_ASSIGN; - tcc_ir_set_src1(ir, i, new_src); - tcc_ir_set_src2(ir, i, IROP_NONE); - LOG_NEG_CHAIN("@%d: T%d = -T%d folded to T%d = T%d (base=T%d sign=%d)", - i, dest_pos, TCCIR_DECODE_VREG_POSITION(src_vr), - dest_pos, TCCIR_DECODE_VREG_POSITION(existing), - base_pos, sign); - changes++; - did_replace = 1; - } - } - } - } - - canon[dest_pos].base_vr = base_vr; - canon[dest_pos].sign = (uint8_t)sign; - canon[dest_pos].valid = 1; - touched_canon[n_touched_canon++] = dest_pos; - - if (TCCIR_DECODE_VREG_TYPE(base_vr) == TCCIR_VREG_TYPE_TEMP) - { - int base_pos = TCCIR_DECODE_VREG_POSITION(base_vr); - if (base_pos >= 0 && base_pos <= max_tmp) - { - if (sign == 0) - { - if (first_pos[base_pos] < 0) - { - first_pos[base_pos] = dest_vr; - touched_first[n_touched_first++] = base_pos; - } - } - else - { - if (first_neg[base_pos] < 0) - { - first_neg[base_pos] = dest_vr; - touched_first[n_touched_first++] = base_pos; - } - } - } - } - (void)did_replace; - } - - tcc_free(canon); - tcc_free(first_pos); - tcc_free(first_neg); - tcc_free(touched_canon); - tcc_free(touched_first); - tcc_free(is_merge); - - return changes; -} - -int tcc_ir_opt_neg_chain_cse_ex(IROptCtx *ctx) -{ - return tcc_ir_opt_neg_chain_cse(ctx->ir); -} diff --git a/source/opt/flat/scalar/single_value_tmp.c b/source/opt/flat/scalar/single_value_tmp.c deleted file mode 100644 index 5bc4e08b2..000000000 --- a/source/opt/flat/scalar/single_value_tmp.c +++ /dev/null @@ -1,161 +0,0 @@ -/* - * TCC IR - Single-value temp propagation (flat pass) - * - * Copyright (c) 2025 Mateusz Stadnik - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation. - */ -/* Relocated flat pass, Branch A [A*]; see docs/plan_legacy_flat_ir_ssa_retire.md */ - -#define USING_GLOBALS - -#include "ir.h" -#include "opt.h" -#include "opt_engine.h" -#include "opt_utils.h" -#include "opt_du.h" - -/* If all defs of a TEMP are ASSIGN/LOAD of the same immediate, fold uses to it. */ -int tcc_ir_opt_single_value_tmp(TCCIRState *ir) -{ -#define SVT_MAX_TEMPS 128 - int n = ir->next_instruction_index; - if (n < 2) - return 0; - - int max_tmp = -1; - for (int i = 0; i < n; i++) { - IRQuadCompact *q = &ir->compact_instructions[i]; - if (q->op == TCCIR_OP_NOP || !irop_config[q->op].has_dest) - continue; - IROperand d = tcc_ir_op_get_dest(ir, q); - int32_t dvr = irop_get_vreg(d); - if (dvr >= 0 && TCCIR_DECODE_VREG_TYPE(dvr) == TCCIR_VREG_TYPE_TEMP) { - int pos = TCCIR_DECODE_VREG_POSITION(dvr); - if (pos > max_tmp) - max_tmp = pos; - } - } - if (max_tmp < 0 || max_tmp >= SVT_MAX_TEMPS) - return 0; - - int count = max_tmp + 1; - uint8_t state[SVT_MAX_TEMPS]; - int32_t vals[SVT_MAX_TEMPS]; - memset(state, 0, count); - - for (int i = 0; i < n; i++) { - IRQuadCompact *q = &ir->compact_instructions[i]; - if (q->op == TCCIR_OP_NOP || !irop_config[q->op].has_dest) - continue; - IROperand d = tcc_ir_op_get_dest(ir, q); - int32_t dvr = irop_get_vreg(d); - if (dvr < 0 || TCCIR_DECODE_VREG_TYPE(dvr) != TCCIR_VREG_TYPE_TEMP) - continue; - int pos = TCCIR_DECODE_VREG_POSITION(dvr); - if (pos >= count) continue; - - if ((q->op == TCCIR_OP_ASSIGN || q->op == TCCIR_OP_LOAD) && - state[pos] != 2) { - IROperand s = tcc_ir_op_get_src1(ir, q); - if (irop_is_immediate(s) && !s.is_lval && - irop_get_btype(s) == IROP_BTYPE_INT32) { - int32_t v = (int32_t)irop_get_imm64_ex(ir, s); - if (state[pos] == 0) { - state[pos] = 1; - vals[pos] = v; - } else if (vals[pos] != v) { - state[pos] = 2; - } - continue; - } - } - state[pos] = 2; - } - - int changes = 0; - for (int i = 0; i < n; i++) { - IRQuadCompact *q = &ir->compact_instructions[i]; - if (q->op == TCCIR_OP_NOP) - continue; - if (q->op != TCCIR_OP_RETURNVALUE) - continue; - for (int k = 0; k < 2; k++) { - IROperand op = k == 0 ? tcc_ir_op_get_src1(ir, q) - : tcc_ir_op_get_src2(ir, q); - int32_t vr = irop_get_vreg(op); - if (vr < 0 || op.is_lval) - continue; - if (TCCIR_DECODE_VREG_TYPE(vr) != TCCIR_VREG_TYPE_TEMP) - continue; - int pos = TCCIR_DECODE_VREG_POSITION(vr); - if (pos >= count || state[pos] != 1) - continue; - IROperand imm = irop_make_imm32(-1, vals[pos], IROP_BTYPE_INT32); - if (k == 0) - tcc_ir_set_src1(ir, i, imm); - else - tcc_ir_set_src2(ir, i, imm); - changes++; - } - } - - if (changes) { - /* Let DCE reclaim dead defs; a single-value temp may still have other uses. */ - changes += tcc_ir_opt_dce(ir); - } - - if (changes) { - n = ir->next_instruction_index; - int64_t ret_val = 0; - int ret_btype = 0; - int ret_idx = -1; - int all_same_ret = 1; - int has_side_effect = 0; - for (int i = 0; i < n && all_same_ret && !has_side_effect; i++) { - IRQuadCompact *q = &ir->compact_instructions[i]; - switch (q->op) { - case TCCIR_OP_RETURNVALUE: { - IROperand s = tcc_ir_op_get_src1(ir, q); - if (!irop_is_immediate(s)) { all_same_ret = 0; break; } - int64_t v = irop_get_imm64_ex(ir, s); - if (ret_idx < 0) { - ret_val = v; ret_btype = irop_get_btype(s); ret_idx = i; - } else if (v != ret_val) { all_same_ret = 0; } - break; - } - case TCCIR_OP_STORE: case TCCIR_OP_STORE_INDEXED: - case TCCIR_OP_FUNCCALLVAL: case TCCIR_OP_FUNCCALLVOID: - case TCCIR_OP_VLA_ALLOC: case TCCIR_OP_VLA_SP_SAVE: - case TCCIR_OP_VLA_SP_RESTORE: case TCCIR_OP_TRAP: - case TCCIR_OP_RETURNVOID: - has_side_effect = 1; break; - default: break; - } - } - if (all_same_ret && !has_side_effect && ret_idx >= 0) { - for (int i = 0; i < n; i++) { - IRQuadCompact *q = &ir->compact_instructions[i]; - if (q->op == TCCIR_OP_NOP || q->op == TCCIR_OP_RETURNVALUE) - continue; - q->op = TCCIR_OP_NOP; - changes++; - } - if (ret_idx > 0) { - ir->compact_instructions[0].op = TCCIR_OP_RETURNVALUE; - IROperand rv = irop_make_imm32(-1, (int32_t)ret_val, ret_btype); - tcc_ir_set_src1(ir, 0, rv); - ir->compact_instructions[ret_idx].op = TCCIR_OP_NOP; - changes++; - } - changes += tcc_ir_opt_dce(ir); - } - } - return changes; -#undef SVT_MAX_TEMPS -} - -int tcc_ir_opt_single_value_tmp_ex(IROptCtx *ctx) { return tcc_ir_opt_single_value_tmp(ctx->ir); } - diff --git a/source/opt/flat/scalar/stack_addr_fold.c b/source/opt/flat/scalar/stack_addr_fold.c index 1bddf7baa..b23e47830 100644 --- a/source/opt/flat/scalar/stack_addr_fold.c +++ b/source/opt/flat/scalar/stack_addr_fold.c @@ -113,8 +113,14 @@ static void sav_build_def_map(TCCIRState *ir) sav_def_present[k] = 1; /* A write THROUGH the vreg is a use, but so is `has_dest` with is_lval -- * it is not a def of the vreg, and counting it as one would let a second - * def slip past the uniqueness test. */ - sav_def_only[k] = d.is_lval ? -2 : (sav_def_only[k] == -1 ? j : -2); + * def slip past the uniqueness test. A parameter also arrives with its + * argument, a def no instruction shows (a stack-passed one has not even an + * entry copy), so any def written here is at least its second: toysh's + * "if (!ant) ant = &deck;" made every later "ant != &deck" fold to equal. */ + if (d.is_lval || type == TCCIR_VREG_TYPE_PARAM) + sav_def_only[k] = -2; + else + sav_def_only[k] = sav_def_only[k] == -1 ? j : -2; } } @@ -289,6 +295,7 @@ int tcc_ir_opt_stack_addr_simplify(TCCIRState *ir) { IROperand direct = irop_make_stackoff(-1, addr.off, 1, 0, addr.is_param, irop_get_btype(dest)); direct.is_unsigned = dest.is_unsigned; + irop_carry_access_marks(&direct, dest); /* same access: same volatility */ tcc_ir_set_dest(ir, i, direct); changes++; } @@ -307,6 +314,7 @@ int tcc_ir_opt_stack_addr_simplify(TCCIRState *ir) { IROperand direct = irop_make_stackoff(-1, addr.off, 1, 0, addr.is_param, irop_get_btype(src)); direct.is_unsigned = src.is_unsigned; + irop_carry_access_marks(&direct, src); /* same access: same volatility */ tcc_ir_set_src1(ir, i, direct); changes++; } diff --git a/source/opt/flat/scalar/symref_const_prop.c b/source/opt/flat/scalar/symref_const_prop.c index a9b6daa02..2c9122958 100644 --- a/source/opt/flat/scalar/symref_const_prop.c +++ b/source/opt/flat/scalar/symref_const_prop.c @@ -11,6 +11,8 @@ #define USING_GLOBALS +#include + #include "ir.h" #include "opt.h" #include "opt_engine.h" @@ -102,10 +104,28 @@ int tcc_ir_opt_symref_const_prop(TCCIRState *ir) if (pos > max_tmp_pos || map[pos].gen != current_gen) continue; - /* Replace with a fresh symref operand carrying use-site flags. */ + /* Replace with a fresh symref operand carrying use-site flags. A + * STRUCT operand (a by-value struct argument read through the pointer) + * keeps its type in u.s.ctype_idx and the symref index in u.s.aux_data + * (irop_get_symref_ex); written to u.pool_idx, the index overlaid the + * ctype and aux_data named whatever symbol was there before. */ + const int use_btype = irop_get_btype(opnd); + if (use_btype == IROP_BTYPE_STRUCT && map[pos].pool_idx > INT16_MAX) + continue; IROperand new_opnd = irop_make_symref(-1, map[pos].pool_idx, opnd.is_lval, map[pos].is_local, - map[pos].is_const, irop_get_btype(opnd)); + map[pos].is_const, use_btype); + if (use_btype == IROP_BTYPE_STRUCT) + { + new_opnd.u.s.ctype_idx = opnd.u.s.ctype_idx; + new_opnd.u.s.aux_data = (int16_t)map[pos].pool_idx; + } new_opnd.is_unsigned = opnd.is_unsigned; + /* Same access, its address spelled as the symbol: keep what is proven + * about it (not volatile, alignment). Without NONVOLATILE the load + * counts as possibly volatile in any function with a volatile or atomic + * access, and global_init will not fold a const table read through it. */ + if (opnd.is_lval) + new_opnd.aux = opnd.aux; if (slot == 0) tcc_ir_set_src1(ir, i, new_opnd); @@ -129,7 +149,8 @@ int tcc_ir_opt_symref_const_prop(TCCIRState *ir) if (src1.is_sym && !src1.is_lval && pos <= max_tmp_pos) { map[pos].gen = current_gen; - map[pos].pool_idx = (uint32_t)src1.u.pool_idx; + map[pos].pool_idx = irop_get_btype(src1) == IROP_BTYPE_STRUCT ? (uint32_t)(uint16_t)src1.u.s.aux_data + : (uint32_t)src1.u.pool_idx; map[pos].btype = irop_get_btype(src1); map[pos].is_local = src1.is_local; map[pos].is_const = src1.is_const; diff --git a/source/opt/include/licm.h b/source/opt/include/licm.h index 6ed29223a..c73671135 100644 --- a/source/opt/include/licm.h +++ b/source/opt/include/licm.h @@ -24,7 +24,9 @@ typedef struct IRLoop int start_idx; /* First instruction in loop */ int end_idx; /* Last instruction in loop */ int preheader_idx; /* Where to insert hoisted code (-1 if none) */ - int *body_instrs; /* Array of instruction indices in loop body */ + int *body_instrs; /* Instruction indices in the loop body: a contiguous range, so a + detected loop points into its IRLoops' shared ramp (never + freed per loop) */ int num_body_instrs; /* Number of instructions in loop body */ int body_instrs_capacity; /* Capacity of body_instrs array */ int depth; /* Nesting depth */ @@ -35,6 +37,9 @@ typedef struct IRLoops IRLoop *loops; /* Array of loops */ int num_loops; /* Number of loops found */ int capacity; /* Capacity of loops array */ + int *ramp; /* ramp[k] == k for every instruction; each loop's body_instrs is a + window of it. One array per detection instead of one copy of every + body -- nested loops in a big function summed to megabytes. */ } IRLoops; typedef enum TCCFuncPurity diff --git a/source/opt/include/opt.h b/source/opt/include/opt.h index b62a34a28..815508f41 100644 --- a/source/opt/include/opt.h +++ b/source/opt/include/opt.h @@ -149,6 +149,7 @@ int tcc_ir_opt_assign_fuse(struct TCCIRState *ir); /* `CMP T_u64, u64_imm_with_hi_0` -> 32-bit when T's hi half is provably zero (SHR>=32 or ZEXT). */ int tcc_ir_opt_cmp_narrow_64(struct TCCIRState *ir); +int tcc_ir_opt_cmp_hi_only(struct TCCIRState *ir); /* `CMP imm, reg` -> `CMP reg, imm` with the condition mirrored on every flag * reader, so the constant reaches `cmp`'s immediate slot instead of a `mov`. */ @@ -179,8 +180,6 @@ int tcc_ir_opt_shift64_extract_ubfx(struct TCCIRState *ir); int tcc_ir_opt_reroll(struct TCCIRState *ir); /* Collapses repeated `T = -T` chains by tracking each TEMP's canonical (base, sign) pair. */ -int tcc_ir_opt_neg_chain_cse(struct TCCIRState *ir); -int tcc_ir_opt_neg_chain_cse_ex(struct IROptCtx *ctx); /* `((V<> n` -> V when low < 2^n and V < 2^(32-n). */ int tcc_ir_opt_bitfield_insert_extract(struct TCCIRState *ir); @@ -200,6 +199,15 @@ int tcc_ir_opt_addrof_var_fwd(struct TCCIRState *ir); /* Rewrites derefs through single-def entry-block `P = &V` pointer VARs (and their TEMP copies) to direct V accesses. */ int tcc_ir_opt_ptr_local_fwd(struct TCCIRState *ir); +int tcc_ir_opt_param_copy_alias(struct TCCIRState *ir); +int tcc_ir_opt_param_copy_alias_ex(struct IROptCtx *ctx); +int tcc_ir_opt_slot_const_store_fold(struct TCCIRState *ir); +int tcc_ir_opt_slot_const_store_fold_ex(struct IROptCtx *ctx); +int tcc_ir_opt_sra(struct TCCIRState *ir); +/* Word reads of a by-value struct parameter's frame home become the parameter's vreg. */ +int tcc_ir_opt_param_home_fwd(struct TCCIRState *ir); +/* Build a struct returned through the hidden sret pointer in the caller's buffer. */ +int tcc_ir_opt_sret_nrvo(struct TCCIRState *ir); int tcc_ir_opt_ptr_local_fwd_ex(struct IROptCtx *ctx); /* Forwards a load of a copied local field (`x=G; ... x.field`) to the source global `G.field`. */ @@ -220,8 +228,6 @@ int tcc_ir_opt_deref_operand_cse(struct TCCIRState *ir); int tcc_ir_opt_bitfield_unit_narrow(struct TCCIRState *ir); /* Inserts one `T_v = T***DEREF***` after a singly-defined pointer TEMP and rewrites later derefs to T_v non-lval. */ -int tcc_ir_opt_invariant_temp_deref_hoist(struct TCCIRState *ir); -int tcc_ir_opt_invariant_temp_deref_hoist_ex(struct IROptCtx *ctx); /* Collapses the spill/addr/store/reload sequence of `f(int v){ helper(&v); return v; }` after inlining. */ int tcc_ir_opt_param_addrof_const_fold(struct TCCIRState *ir); @@ -239,7 +245,6 @@ int tcc_ir_opt_global_init_prop(struct TCCIRState *ir); int tcc_ir_opt_symref_const_prop(struct TCCIRState *ir); /* Packs a constant-initialized _Complex float local used as a single FUNCPARAMVAL into a 64-bit immediate. */ -int tcc_ir_opt_complex_const_param_fold(struct TCCIRState *ir); /* Sets sym->f.func_pure_via_sret when the only observable effect is writes through the sret-pointer parameter. */ void tcc_ir_analyze_pure_via_sret(struct TCCIRState *ir, struct Sym *func_sym); @@ -286,9 +291,7 @@ void tcc_ir_barrel_shift_fusion(struct TCCIRState *ir); int tcc_ir_opt_shift_pair_to_ubfx(struct TCCIRState *ir); int tcc_ir_opt_stack_addr_cse(struct TCCIRState *ir); int tcc_ir_opt_entry_store_prop(struct TCCIRState *ir); -int tcc_ir_opt_float_branch_fold(struct TCCIRState *ir); int tcc_ir_opt_redundant_loop_check(struct TCCIRState *ir); -int tcc_ir_opt_float_narrowing(struct TCCIRState *ir); int tcc_ir_opt_jump_threading(struct TCCIRState *ir); int tcc_ir_opt_bool_diamond_branch(struct TCCIRState *ir); int tcc_ir_opt_orphan_cmp_elim(struct TCCIRState *ir); @@ -298,6 +301,20 @@ int tcc_ir_opt_jumpif_invert(struct TCCIRState *ir, int allow_backward); /* Replaces the memset(0)+stores pattern with a BLOCK_COPY from rodata. */ int tcc_ir_opt_block_copy_init(struct TCCIRState *ir); +struct Section; +struct Sym; +/* A relocation in a constant image: the address of `sym` + addend at `off`. */ +typedef struct +{ + int off; + struct Sym *sym; + int32_t addend; +} BciReloc; +/* A symbol for `size` bytes (plus relocations) in `sec`, shared with any + * identical image emitted earlier in this compilation. */ +struct Sym *bci_image_sym(struct Section *sec, const uint8_t *bytes, int size, const BciReloc *rel, int nrel); +int tcc_ir_opt_const_local_table(struct TCCIRState *ir); +int tcc_ir_opt_const_local_table_ex(struct IROptCtx *ctx); /* memset(stack, N<=8, 0) -> one or two direct STORE #0, for when block_copy_init did not fire. */ int tcc_ir_opt_small_memset_to_store(struct TCCIRState *ir); @@ -320,7 +337,6 @@ int tcc_ir_opt_backedge_phi_hoist(struct TCCIRState *ir); int tcc_ir_opt_abort_tail_merge(struct TCCIRState *ir); int tcc_ir_opt_const_prop_tmp_ex(struct IROptCtx *ctx); -int tcc_ir_opt_single_value_tmp_ex(struct IROptCtx *ctx); int tcc_ir_opt_known_bits_ex(struct IROptCtx *ctx); int tcc_ir_opt_dead_lea_store_elim_ex(struct IROptCtx *ctx); int tcc_ir_opt_const_aggregate_fold_ex(struct IROptCtx *ctx); @@ -338,11 +354,11 @@ int tcc_ir_opt_mem_inline(struct TCCIRState *ir); int tcc_ir_opt_mem_inline_ex(struct IROptCtx *ctx); int tcc_ir_opt_setif_branch_fuse_ex(struct IROptCtx *ctx); int tcc_ir_opt_stack_bool_diamond_ex(struct IROptCtx *ctx); -int tcc_ir_opt_float_narrowing_ex(struct IROptCtx *ctx); int tcc_ir_opt_pack64_ex(struct IROptCtx *ctx); int tcc_ir_opt_pack64_tautology_ex(struct IROptCtx *ctx); int tcc_ir_opt_pack64_from_stack_stores_ex(struct IROptCtx *ctx); int tcc_ir_opt_cmp_narrow_64_ex(struct IROptCtx *ctx); +int tcc_ir_opt_cmp_hi_only_ex(struct IROptCtx *ctx); int tcc_ir_opt_cmp_imm_swap_ex(struct IROptCtx *ctx); int tcc_ir_opt_and64_narrow_ex(struct IROptCtx *ctx); int tcc_ir_opt_sl_forward_ex(struct IROptCtx *ctx); @@ -354,8 +370,10 @@ int tcc_ir_opt_switch_to_data(struct TCCIRState *ir); int tcc_ir_opt_switch_collapse_ex(struct IROptCtx *ctx); int tcc_ir_opt_switch_collapse(struct TCCIRState *ir); int tcc_ir_opt_redundant_loop_check_ex(struct IROptCtx *ctx); -int tcc_ir_opt_float_branch_fold_ex(struct IROptCtx *ctx); int tcc_ir_opt_eliminate_fallthrough_ex(struct IROptCtx *ctx); +/* Branches whose one side can only run into __builtin_unreachable fold to the other side. */ +int tcc_ir_opt_unreachable_fold(struct TCCIRState *ir); +int tcc_ir_opt_unreachable_fold_ex(struct IROptCtx *ctx); int tcc_ir_opt_uninit_local_ub(struct TCCIRState *ir); int tcc_ir_opt_uninit_local_ub_ex(struct IROptCtx *ctx); int tcc_ir_opt_uninit_dominates_return(struct TCCIRState *ir); @@ -377,8 +395,6 @@ int tcc_ir_opt_byte_store_merge(struct TCCIRState *ir); int tcc_ir_opt_byte_store_merge_ex(struct IROptCtx *ctx); int tcc_ir_opt_const_memcpy_to_dest(struct TCCIRState *ir); int tcc_ir_opt_const_memcpy_to_dest_ex(struct IROptCtx *ctx); -int tcc_ir_opt_local_copy_prop(struct TCCIRState *ir); -int tcc_ir_opt_local_copy_prop_ex(struct IROptCtx *ctx); /* Drops a `memmove(B,A,N); memmove(A,B,N)` pair where B is a dead round-trip temp and A is unmodified between. */ int tcc_ir_opt_struct_copy_roundtrip_elim(struct TCCIRState *ir); /* `memmove(local, &global, N)` into a private read-only slot: reads the global directly and drops the copy. */ @@ -386,20 +402,6 @@ int tcc_ir_opt_memmove_global_load_fwd(struct TCCIRState *ir); int tcc_ir_opt_addrof_var_fwd_ex(struct IROptCtx *ctx); int tcc_ir_opt_global_sl_fwd_ex(struct IROptCtx *ctx); -typedef struct TCCOptStats -{ - int dce_removed; - int dse_removed; - int const_folded; - int copies_propagated; - int cse_eliminated; - int stores_forwarded; -} TCCOptStats; - -void tcc_ir_opt_stats_get(TCCOptStats *stats); - -void tcc_ir_opt_stats_reset(void); - void tcc_ir_opt_fp_cache_init(struct TCCIRState *ir); void tcc_ir_opt_fp_cache_clear(struct TCCIRState *ir); @@ -417,6 +419,12 @@ int tcc_ir_find_defining_instruction(struct TCCIRState *ir, int32_t vreg, int be int tcc_ir_vreg_has_single_use(struct TCCIRState *ir, int32_t vreg, int exclude_idx); +/* Post-RA: 1 when graph coalescing merged `vreg` into a class. The copies + * between the members were erased, so the members' register is also read + * and written under the other members' names: no scan of this vreg's own + * uses or defs sees all of them. */ +int tcc_ir_vreg_coalesced(struct TCCIRState *ir, int32_t vreg); + /* Inserts a quad before an index, renumbering jump + switch-table targets. */ int tcc_ir_insert_instruction_before(struct TCCIRState *ir, int before_idx, struct IRQuadCompact *new_q); diff --git a/source/opt/include/opt_loop_utils.h b/source/opt/include/opt_loop_utils.h index 7be8ecb42..2b67ef7b9 100644 --- a/source/opt/include/opt_loop_utils.h +++ b/source/opt/include/opt_loop_utils.h @@ -18,6 +18,10 @@ #define UNROLL_MAX_TRIP_COUNT 16 #define UNROLL_MAX_BODY_INSNS 32 #define UNROLL_MAX_TOTAL_INSNS 128 +/* A body that loads or stores is capped by what survives constant folding of + * the unrolled copies (unroll_residual_insns), and by a raw ceiling. */ +#define UNROLL_MAX_TOTAL_MEM_INSNS 96 +#define UNROLL_MAX_RAW_MEM_INSNS 256 typedef struct InductionVar { @@ -37,6 +41,16 @@ typedef struct DerivedIV int use_idx; int shl_idx; int share_with; + /* INDEXED-DIV whose index is `iv + k` / `iv - k`: the access sits `offset` + * bytes from base + iv*stride, and off_idx is the ADD/SUB that computes the + * index (single-use, NOPed once the access reads through the pointer). + * 0 / -1 for a plain `base[iv]`. */ + int offset; + int off_idx; + /* Set by the driver on a group primary: the byte offset the shared pointer + * is aimed at (the group's smallest `offset`), so every member's access is + * a non-negative immediate off one pointer. 0 when nothing is shared. */ + int origin; } DerivedIV; /* IV analysis */ @@ -47,6 +61,17 @@ int find_derived_ivs(struct TCCIRState *ir, struct IRLoop *loop, InductionVar *ivs, int num_ivs, DerivedIV *divs, int max_divs); +/* 1 when a read of `iv_vr` outside the loop can observe the value the loop + * leaves in it: reachable from a loop exit without passing a redefinition. + * Conservative (1) on indirect jumps, switch tables, an address-taken IV, or + * a path that re-enters the loop. */ +int iv_read_reachable_outside(struct TCCIRState *ir, struct IRLoop *loop, int32_t iv_vr); + +/* Same base object, looking through a single-def ASSIGN of an address (a + * `T <- Addr[StackLoc[k]]` copy the loads may use directly and the stores + * through the temp, or vice versa). */ +int iv_same_base(struct TCCIRState *ir, IROperand a, IROperand b); + int transform_derived_iv(struct TCCIRState *ir, struct IRLoop *loop, InductionVar *iv, DerivedIV *div, int *out_ptr_vreg, int *out_idx_shift, @@ -77,9 +102,10 @@ int compute_trip_count(int init_val, int limit, int step, int cond_token); int collect_body_instructions(struct TCCIRState *ir, struct IRLoop *loop, int iv_vreg, int cmp_idx, int jmpif_idx, - int iv_def_idx, int *body_indices, int max_body); + int iv_def_idx, int *body_indices, int max_body, int *out_has_memory); /* NOP-slot writers */ +void tcc_ir_copy_orig_annotations(struct TCCIRState *ir, int from, int to); void write_instr_at_nop(struct TCCIRState *ir, int pos, TccIrOp op, IROperand dest, IROperand src1, IROperand src2); diff --git a/source/opt/include/opt_utils.h b/source/opt/include/opt_utils.h index e0f0e5c05..4a5af303f 100644 --- a/source/opt/include/opt_utils.h +++ b/source/opt/include/opt_utils.h @@ -60,6 +60,7 @@ int32_t ir_opt_mla_accum_vreg(const struct TCCIRState *ir, const struct IRQuadCo * Answers 0 for every instruction of a function that never touches volatile * memory — see TCCIRState.func_has_volatile_access. */ int tcc_ir_instr_access_is_volatile(const struct TCCIRState *ir, const struct IRQuadCompact *q); +int tcc_ir_operand_names_volatile_var(const struct TCCIRState *ir, IROperand op); int is_power_of_2(int64_t n); diff --git a/source/opt/ra/umaal_fusion.c b/source/opt/ra/umaal_fusion.c new file mode 100644 index 000000000..f1bb47c11 --- /dev/null +++ b/source/opt/ra/umaal_fusion.c @@ -0,0 +1,366 @@ +/* + * TCC IR - UMAAL fusion + * + * Copyright (c) 2026 Mateusz Stadnik + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + */ + +#define USING_GLOBALS +#include "ir.h" +#include "regalloc.h" + +/* A 32x32->64 product plus one or two zero-extended words, + * + * T = a UMULL b; Z1 = ZEXT c; S = T ADD Z1 (one addend) + * T = a UMULL b; Z1 = ZEXT c; S = T ADD Z1; Z2 = ZEXT d; R = S ADD Z2 + * + * is ARM's UMAAL, {hi:lo} = a * b + c + d, which cannot overflow 64 bits. It is + * exactly the step of a multi-word multiply (zig.h's u128 product, soft-float + * mantissa products), and as separate instructions each step keeps two 64-bit + * temporaries alive -- the product and the widened addend -- which is where a + * register-hungry caller starts spilling. + * + * Runs after phi resolution and the late address hoists, right before live + * intervals are built, so only the allocator, the post-RA passes and codegen + * ever see a TCCIR_OP_UMAAL (they read its accumulator through + * tcc_ir_op_is_mac). The accumulator operand is a 64-bit value whose two + * words are the addends: the ZEXT itself for one addend, a PACK64 of both + * for two (built in the slot of one of the instructions the fusion frees). + * + * The UMAAL sits where the final ADD was, so it reads a, b (and the PACK64 + * reads c, d) later than the original instructions did: none of them may be + * redefined in between, and the whole chain must be one straight-line run. */ + +typedef struct UmaalCtx +{ + TCCIRState *ir; + int n; + int ntemp; + int *mentions; /* per TEMP position: operand slots naming it (defs and uses) */ + int *def_idx; /* per TEMP position: the (last) instruction defining it */ +} UmaalCtx; + +static int umaal_nops(const IRQuadCompact *q) +{ + int n = irop_config[q->op].has_dest + irop_config[q->op].has_src1 + irop_config[q->op].has_src2; + if (tcc_ir_op_is_mac(q->op) || q->op == TCCIR_OP_LOAD_INDEXED || q->op == TCCIR_OP_STORE_INDEXED || + q->op == TCCIR_OP_SELECT) + n++; + return n; +} + +static int umaal_temp_pos(IROperand op) +{ + if (op.is_lval || !irop_has_vreg(op)) + return -1; + int32_t vr = irop_get_vreg(op); + if (vr < 0 || TCCIR_DECODE_VREG_TYPE(vr) != TCCIR_VREG_TYPE_TEMP) + return -1; + return TCCIR_DECODE_VREG_POSITION(vr); +} + +/* The instruction defining `op` when `op` is a TEMP with one def and one use, + * both before `use`; else -1. */ +static int umaal_single_def(UmaalCtx *c, IROperand op, int use) +{ + int p = umaal_temp_pos(op); + if (p < 0 || p >= c->ntemp || c->mentions[p] != 2) + return -1; + int d = c->def_idx[p]; + if (d < 0 || d >= use) + return -1; + return d; +} + +/* No label, branch or redefinition of `v` in (from, to]. A register value + * (not an lvalue) only changes when an instruction names it as dest. */ +static int umaal_value_stable(TCCIRState *ir, IROperand v, int from, int to) +{ + if (v.is_lval) + return 0; + int32_t vr = irop_has_vreg(v) ? irop_get_vreg(v) : -1; + for (int k = from + 1; k <= to; k++) + { + IRQuadCompact *q = &ir->compact_instructions[k]; + if (q->is_jump_target && k > from) + return 0; + if (q->op == TCCIR_OP_NOP) + continue; + if (vr >= 0 && k < to && irop_config[q->op].has_dest) + { + IROperand d = tcc_ir_op_get_dest(ir, q); + if (!d.is_lval && irop_has_vreg(d) && irop_get_vreg(d) == vr) + return 0; + } + } + return 1; +} + +/* No control flow in [from, to): a jump ends the straight-line run. */ +static int umaal_straight(TCCIRState *ir, int from, int to) +{ + for (int k = from; k < to; k++) + { + IRQuadCompact *q = &ir->compact_instructions[k]; + if (k > from && q->is_jump_target) + return 0; + switch (q->op) + { + case TCCIR_OP_JUMP: + case TCCIR_OP_JUMPIF: + case TCCIR_OP_IJUMP: + case TCCIR_OP_SWITCH_TABLE: + case TCCIR_OP_RETURNVALUE: + case TCCIR_OP_RETURNVOID: + case TCCIR_OP_SETJMP: + case TCCIR_OP_NL_SETJMP: + case TCCIR_OP_LONGJMP: + case TCCIR_OP_NL_LONGJMP: + return 0; + default: + break; + } + } + return !ir->compact_instructions[to].is_jump_target || to == from; +} + +static int umaal_is_add64(TCCIRState *ir, int i) +{ + IRQuadCompact *q = &ir->compact_instructions[i]; + if (q->op != TCCIR_OP_ADD || tcc_ir_barrel_shift_at(ir, q)) + return 0; + IROperand d = tcc_ir_op_get_dest(ir, q); + IROperand s1 = tcc_ir_op_get_src1(ir, q); + IROperand s2 = tcc_ir_op_get_src2(ir, q); + return !d.is_lval && irop_get_btype(d) == IROP_BTYPE_INT64 && irop_get_btype(s1) == IROP_BTYPE_INT64 && + irop_get_btype(s2) == IROP_BTYPE_INT64; +} + +/* `idx` is `ZEXT c` from a 32-bit c into a 64-bit TEMP. */ +static int umaal_is_zext32(TCCIRState *ir, int idx) +{ + IRQuadCompact *q = &ir->compact_instructions[idx]; + if (q->op != TCCIR_OP_ZEXT) + return 0; + IROperand d = tcc_ir_op_get_dest(ir, q); + IROperand s = tcc_ir_op_get_src1(ir, q); + /* Exactly a word: a narrower source's register is not known to be clean. */ + return irop_get_btype(d) == IROP_BTYPE_INT64 && irop_get_btype(s) == IROP_BTYPE_INT32 && !s.is_lval; +} + +static int umaal_is_umull(TCCIRState *ir, int idx) +{ + IRQuadCompact *q = &ir->compact_instructions[idx]; + if (q->op != TCCIR_OP_UMULL || tcc_ir_barrel_shift_at(ir, q)) + return 0; + IROperand s1 = tcc_ir_op_get_src1(ir, q); + IROperand s2 = tcc_ir_op_get_src2(ir, q); + return !s1.is_lval && !s2.is_lval; +} + +/* Split ADD i into (umull, zext): the index of each, or -1. */ +static void umaal_split(UmaalCtx *c, int i, int *mul, int *zext, int *other) +{ + TCCIRState *ir = c->ir; + IRQuadCompact *q = &ir->compact_instructions[i]; + int d1 = umaal_single_def(c, tcc_ir_op_get_src1(ir, q), i); + int d2 = umaal_single_def(c, tcc_ir_op_get_src2(ir, q), i); + *mul = *zext = *other = -1; + if (d1 < 0 || d2 < 0) + return; + int ds[2] = {d1, d2}; + for (int k = 0; k < 2; k++) + { + if (*mul < 0 && umaal_is_umull(ir, ds[k])) + *mul = ds[k]; + else if (*zext < 0 && umaal_is_zext32(ir, ds[k])) + *zext = ds[k]; + else + *other = ds[k]; + } +} + +/* `op` is `(uint32)Y`, a truncating ASSIGN of a 64-bit Y used only here: + * return a word-typed view of Y (which reads its low register) and the + * ASSIGN's index, so the copy and its temporary go away. Y must still hold + * the same value at `read_at`. */ +static IROperand umaal_through_trunc(UmaalCtx *c, IROperand op, int use, int read_at, int *assign_idx) +{ + TCCIRState *ir = c->ir; + *assign_idx = -1; + int d = umaal_single_def(c, op, use); + if (d < 0) + return op; + IRQuadCompact *q = &ir->compact_instructions[d]; + if (q->op != TCCIR_OP_ASSIGN || tcc_ir_barrel_shift_at(ir, q)) + return op; + IROperand dst = tcc_ir_op_get_dest(ir, q); + IROperand src = tcc_ir_op_get_src1(ir, q); + if (irop_get_btype(dst) != IROP_BTYPE_INT32 || irop_get_btype(src) != IROP_BTYPE_INT64 || src.is_lval || + !irop_has_vreg(src) || irop_get_vreg(src) < 0 || src.is_complex) + return op; + if (!umaal_value_stable(ir, src, d, read_at)) + return op; + *assign_idx = d; + src.btype = IROP_BTYPE_INT32; + return src; +} + +static void umaal_set_ops(TCCIRState *ir, IRQuadCompact *q, const IROperand *ops, int n) +{ + int base = ir->iroperand_pool_count; + tcc_ir_pool_ensure(ir, n); + for (int k = 0; k < n; k++) + tcc_ir_pool_add(ir, ops[k]); + q->operand_base = base; +} + +static int umaal_try(UmaalCtx *c, int i) +{ + TCCIRState *ir = c->ir; + if (!umaal_is_add64(ir, i)) + return 0; + + int mul, z1, other, z2 = -1, inner = -1; + umaal_split(c, i, &mul, &z1, &other); + if (mul < 0 && z1 >= 0 && other >= 0 && umaal_is_add64(ir, other)) + { + /* R = (T ADD Z1) ADD Z2: the inner ADD must itself be product + addend. */ + int im, iz, io; + umaal_split(c, other, &im, &iz, &io); + if (im < 0 || iz < 0 || io >= 0) + return 0; + inner = other; + z2 = z1; + z1 = iz; + mul = im; + } + else if (mul < 0 || z1 < 0 || other >= 0) + return 0; + + IRQuadCompact *mq = &ir->compact_instructions[mul]; + IROperand a = tcc_ir_op_get_src1(ir, mq); + IROperand b = tcc_ir_op_get_src2(ir, mq); + IROperand cz = tcc_ir_op_get_src1(ir, &ir->compact_instructions[z1]); + int first = mul < z1 ? mul : z1; + if (inner >= 0 && inner < first) + first = inner; + if (z2 >= 0 && z2 < first) + first = z2; + if (!umaal_straight(ir, first, i)) + return 0; + if (!umaal_value_stable(ir, a, mul, i) || !umaal_value_stable(ir, b, mul, i)) + return 0; + + IROperand accum; + if (z2 < 0) + { + /* One addend: ZEXT c already is {c, 0}. */ + IRQuadCompact *zq = &ir->compact_instructions[z1]; + accum = tcc_ir_op_get_dest(ir, zq); + int ai; + IROperand w = umaal_through_trunc(c, cz, z1, z1, &ai); + if (ai >= 0) + { + IROperand zo[2] = {accum, w}; + umaal_set_ops(ir, zq, zo, 2); + ir->compact_instructions[ai].op = TCCIR_OP_NOP; + } + } + else + { + /* Two: PACK64 {c, d} in the latest freed slot (after both ZEXTs, so both + * words are defined there). */ + IROperand dz = tcc_ir_op_get_src1(ir, &ir->compact_instructions[z2]); + int slot = z1 > z2 ? z1 : z2; + if (inner > slot) + slot = inner; + if (!umaal_value_stable(ir, cz, z1, slot) || !umaal_value_stable(ir, dz, z2, slot)) + return 0; + IRQuadCompact *sq = &ir->compact_instructions[slot]; + accum = tcc_ir_op_get_dest(ir, sq); + int a1, a2; + IROperand lo = umaal_through_trunc(c, cz, z1, slot, &a1); + IROperand hi = umaal_through_trunc(c, dz, z2, slot, &a2); + if (a1 >= 0) + ir->compact_instructions[a1].op = TCCIR_OP_NOP; + if (a2 >= 0) + ir->compact_instructions[a2].op = TCCIR_OP_NOP; + lo.btype = IROP_BTYPE_INT32; + hi.btype = IROP_BTYPE_INT32; + IROperand pk[3] = {accum, lo, hi}; + sq->op = TCCIR_OP_PACK64; + umaal_set_ops(ir, sq, pk, 3); + if (z1 != slot) + ir->compact_instructions[z1].op = TCCIR_OP_NOP; + if (z2 != slot) + ir->compact_instructions[z2].op = TCCIR_OP_NOP; + if (inner != slot) + ir->compact_instructions[inner].op = TCCIR_OP_NOP; + } + + IRQuadCompact *q = &ir->compact_instructions[i]; + IROperand ops[4] = {tcc_ir_op_get_dest(ir, q), a, b, accum}; + ops[1].btype = IROP_BTYPE_INT32; + ops[2].btype = IROP_BTYPE_INT32; + ops[0].is_unsigned = 1; + q->op = TCCIR_OP_UMAAL; + umaal_set_ops(ir, q, ops, 4); + mq->op = TCCIR_OP_NOP; + return 1; +} + +int ra_fuse_umaal(TCCIRState *ir) +{ + if (!tcc_machine_has_umaal() || ir->next_instruction_index < 3) + return 0; + UmaalCtx c = {.ir = ir, .n = ir->next_instruction_index}; + c.ntemp = ir->next_temporary_variable > 0 ? ir->next_temporary_variable : 1; + + /* Cheap prefilter: a UMULL feeding anything at all. */ + int any = 0; + for (int i = 0; i < c.n && !any; i++) + any = ir->compact_instructions[i].op == TCCIR_OP_UMULL; + if (!any) + return 0; + + c.mentions = tcc_mallocz(sizeof(int) * (size_t)c.ntemp); + c.def_idx = tcc_malloc(sizeof(int) * (size_t)c.ntemp); + for (int p = 0; p < c.ntemp; p++) + c.def_idx[p] = -1; + for (int i = 0; i < c.n; i++) + { + IRQuadCompact *q = &ir->compact_instructions[i]; + if (q->op == TCCIR_OP_NOP) + continue; + int nops = umaal_nops(q); + for (int k = 0; k < nops; k++) + { + int32_t vr = irop_get_vreg(ir->iroperand_pool[q->operand_base + k]); + if (vr >= 0 && TCCIR_DECODE_VREG_TYPE(vr) == TCCIR_VREG_TYPE_TEMP && + TCCIR_DECODE_VREG_POSITION(vr) < c.ntemp) + c.mentions[TCCIR_DECODE_VREG_POSITION(vr)]++; + } + if (irop_config[q->op].has_dest) + { + IROperand d = tcc_ir_op_get_dest(ir, q); + int p = umaal_temp_pos(d); + if (p >= 0 && p < c.ntemp) + c.def_idx[p] = i; + } + } + + /* Backwards, so a two-addend chain is met at its outer ADD: forwards, its + * inner `T ADD Z1` would fuse alone first and hide the second addend. */ + int changes = 0; + for (int i = c.n - 1; i >= 0; i--) + changes += umaal_try(&c, i); + + tcc_free(c.mentions); + tcc_free(c.def_idx); + return changes; +} diff --git a/source/opt/ssa/Makefile b/source/opt/ssa/Makefile index ac3e48814..e23eec399 100644 --- a/source/opt/ssa/Makefile +++ b/source/opt/ssa/Makefile @@ -64,6 +64,7 @@ SSA_OPT_SRC = \ source/opt/ssa/loop/loop_unroll.c \ source/opt/ssa/loop/iv_strength_reduction.c \ source/opt/ssa/loop/decrement_to_zero.c \ + source/opt/ssa/loop/loop_bottom_test.c \ source/opt/ssa/loop/dead_loop.c \ source/opt/ssa/scalar/sccp.c \ source/opt/ssa/memory/global_store_dse.c \ diff --git a/source/opt/ssa/cfg/vrp.c b/source/opt/ssa/cfg/vrp.c index 8cc6313ab..c7ae03d7d 100644 --- a/source/opt/ssa/cfg/vrp.c +++ b/source/opt/ssa/cfg/vrp.c @@ -775,6 +775,19 @@ static int sv_cmp_verdict(SVState *s, int cmp_idx, int tok, SVCmp *cmp, int32_t if (tok == VRP_TOK_UGE) verdict = 1; else if (tok == VRP_TOK_ULT) verdict = 0; } + /* ... and vs the other ends of the 32-bit compare: nothing is above + * 0xFFFFFFFF unsigned, below INT32_MIN or above INT32_MAX signed. zig.h + * checks every __builtin_*_overflow result against zig_minInt/zig_maxInt + * of its width, which inlining makes exactly these at full width. */ + if (verdict < 0 && cmp->c_ok && cmp->x.btype != IROP_BTYPE_INT64) + { + if (cmp->c == -1 && (tok == VRP_TOK_UGT || tok == VRP_TOK_ULE)) + verdict = tok == VRP_TOK_ULE; + else if (cmp->c == INT32_MIN && (tok == VRP_TOK_LT || tok == VRP_TOK_GE)) + verdict = tok == VRP_TOK_GE; + else if (cmp->c == INT32_MAX && (tok == VRP_TOK_GT || tok == VRP_TOK_LE)) + verdict = tok == VRP_TOK_LE; + } if (verdict < 0 && cmp->slot >= 0 && s->ranges[cmp->slot].valid && cmp->c_ok) verdict = sv_range_verdict(s->ranges[cmp->slot].lo, s->ranges[cmp->slot].hi, cmp->c, tok); /* A phi whose operands all exclude c decides `x == c` even when the interval hull cannot. */ diff --git a/source/opt/ssa/dce/dce_common.c b/source/opt/ssa/dce/dce_common.c index c88569e6a..dd5a7aca5 100644 --- a/source/opt/ssa/dce/dce_common.c +++ b/source/opt/ssa/dce/dce_common.c @@ -13,6 +13,18 @@ #include "ssa_opt.h" #include "dce_common.h" +#include "opt_utils.h" + +int dce_instr_reads_volatile(TCCIRState *ir, IRQuadCompact *q) +{ + if (tcc_ir_instr_access_is_volatile(ir, q)) + return 1; + if (irop_config[q->op].has_src1 && tcc_ir_operand_names_volatile_var(ir, tcc_ir_op_get_src1(ir, q))) + return 1; + if (irop_config[q->op].has_src2 && tcc_ir_operand_names_volatile_var(ir, tcc_ir_op_get_src2(ir, q))) + return 1; + return 0; +} int sl_temp_has_live_uses(IRSSAOptCtx *ctx, int32_t vreg) { diff --git a/source/opt/ssa/dce/dce_common.h b/source/opt/ssa/dce/dce_common.h index 4d135abe9..e60ea9538 100644 --- a/source/opt/ssa/dce/dce_common.h +++ b/source/opt/ssa/dce/dce_common.h @@ -14,6 +14,14 @@ struct IRSSAOptCtx; +/* Every dead-store killer: does deleting this instruction delete a volatile + * access? A memory operand marked volatile, or a source naming a volatile + * variable -- a fused `STORE dead <- lval(V)` embodies the read of V, and C + * requires that read even when the stored-to slot is never read. */ +struct TCCIRState; +struct IRQuadCompact; +int dce_instr_reads_volatile(struct TCCIRState *ir, struct IRQuadCompact *q); + /* dead_var_stores + stackloc_stores */ int sl_temp_has_live_uses(struct IRSSAOptCtx *ctx, int32_t vreg); diff --git a/source/opt/ssa/dce/dead_global_stores.c b/source/opt/ssa/dce/dead_global_stores.c index 394978db5..92fa7065f 100644 --- a/source/opt/ssa/dce/dead_global_stores.c +++ b/source/opt/ssa/dce/dead_global_stores.c @@ -384,8 +384,11 @@ int dce_dead_global_stores(IRSSAOptCtx *ctx) pending[p].ref.off == r.off && pending[p].ref.has_addr == r.has_addr && (r.has_addr ? pending[p].ref.shndx == r.shndx : pending[p].ref.sym == r.sym)) { - ssa_opt_nop_instr(ctx, pending[p].idx); - changes++; + /* The overwritten store's value may be a volatile read. */ + if (!dce_instr_reads_volatile(ir, &ir->compact_instructions[pending[p].idx])) { + ssa_opt_nop_instr(ctx, pending[p].idx); + changes++; + } pending[p] = pending[--npending]; } else if (gs_overlaps(&pending[p].ref, &r)) { pending[p] = pending[--npending]; diff --git a/source/opt/ssa/dce/dead_overwrite_stores.c b/source/opt/ssa/dce/dead_overwrite_stores.c index a19380220..f24146feb 100644 --- a/source/opt/ssa/dce/dead_overwrite_stores.c +++ b/source/opt/ssa/dce/dead_overwrite_stores.c @@ -262,8 +262,11 @@ int dce_dead_overwrite_stores(IRSSAOptCtx *ctx) } int po = pending[k].off, pw = pending[k].width; if (po == so && pw == sw) { - ssa_opt_nop_instr(ctx, pending[k].idx); - changes++; + /* The overwritten store's value may be a volatile read. */ + if (!dce_instr_reads_volatile(ir, &ir->compact_instructions[pending[k].idx])) { + ssa_opt_nop_instr(ctx, pending[k].idx); + changes++; + } pending[k] = pending[--npending]; } else if (so < po + pw && so + sw > po) { pending[k] = pending[--npending]; diff --git a/source/opt/ssa/dce/dead_var_stores.c b/source/opt/ssa/dce/dead_var_stores.c index 5620a86f4..64430ce40 100644 --- a/source/opt/ssa/dce/dead_var_stores.c +++ b/source/opt/ssa/dce/dead_var_stores.c @@ -181,6 +181,8 @@ int dce_dead_var_stores(IRSSAOptCtx *ctx) continue; if (var_used[pos / 8] & (1 << (pos % 8))) continue; + if (dce_instr_reads_volatile(ir, q)) + continue; ssa_opt_nop_instr(ctx, i); changes++; @@ -216,6 +218,13 @@ int dce_dead_var_stores(IRSSAOptCtx *ctx) IRSSAVregInfo *vi = ssa_opt_vinfo(ctx, dvr); if (!vi) continue; + int keeps_volatile = 0; + for (int u = 0; u < vi->use_count; u++) + if (vi->uses[u].kind == SSA_USE_INSTR && + dce_instr_reads_volatile(ir, &ir->compact_instructions[vi->uses[u].idx])) + keeps_volatile = 1; + if (keeps_volatile) + continue; for (int u = vi->use_count - 1; u >= 0; u--) { if (vi->uses[u].kind != SSA_USE_INSTR) continue; diff --git a/source/opt/ssa/dce/ret_path_frame_store.c b/source/opt/ssa/dce/ret_path_frame_store.c index d1ac68124..1861b0d76 100644 --- a/source/opt/ssa/dce/ret_path_frame_store.c +++ b/source/opt/ssa/dce/ret_path_frame_store.c @@ -12,6 +12,7 @@ #include "ir.h" #include "ssa_opt.h" #include "dce_passes.h" +#include "dce_common.h" #include @@ -86,7 +87,7 @@ int dce_ret_path_frame_store(IRSSAOptCtx *ctx) if (jq->op == TCCIR_OP_MLA && !dce_ret_op_is_pure_read(ir, tcc_ir_op_get_accum(ir, jq))) break; } - if (dead) { + if (dead && !dce_instr_reads_volatile(ir, q)) { ssa_opt_nop_instr(ctx, i); changes++; } diff --git a/source/opt/ssa/dce/stackloc_stores.c b/source/opt/ssa/dce/stackloc_stores.c index 4e95121d3..369f88ca9 100644 --- a/source/opt/ssa/dce/stackloc_stores.c +++ b/source/opt/ssa/dce/stackloc_stores.c @@ -199,7 +199,7 @@ int dce_dead_stackloc_stores(IRSSAOptCtx *ctx) break; } } - if (!any_read) { + if (!any_read && !dce_instr_reads_volatile(ir, q)) { ssa_opt_nop_instr(ctx, i); changes++; } diff --git a/source/opt/ssa/dce/temp_worklist.c b/source/opt/ssa/dce/temp_worklist.c index 82e2aebd9..37bbca6f8 100644 --- a/source/opt/ssa/dce/temp_worklist.c +++ b/source/opt/ssa/dce/temp_worklist.c @@ -39,6 +39,11 @@ int dce_temp_worklist(IRSSAOptCtx *ctx) IRQuadCompact *q = &ctx->ir->compact_instructions[def]; if (q->op == TCCIR_OP_NOP) continue; + /* Only ever delete this vreg's own definition: a stale def_instr must not + * take an unrelated instruction with it. */ + if (!irop_config[q->op].has_dest || + irop_get_vreg(tcc_ir_op_get_dest(ctx->ir, q)) != TCCIR_ENCODE_VREG(TCCIR_VREG_TYPE_TEMP, pos)) + continue; /* A write to a volatile slot is a mandated side effect even if never read. */ if (irop_config[q->op].has_dest) { int32_t dvr = irop_get_vreg(tcc_ir_op_get_dest(ctx->ir, q)); diff --git a/source/opt/ssa/engine/driver.c b/source/opt/ssa/engine/driver.c index 4817c8a77..9facf9a25 100644 --- a/source/opt/ssa/engine/driver.c +++ b/source/opt/ssa/engine/driver.c @@ -124,6 +124,8 @@ int tcc_ir_ssa_opt_run(IRSSAOptCtx *ctx) * belongs to the heavy tier alongside unrolling/LICM/IV-strength-red. */ SSA_RUN("ssa:dead_loop", (tcc_state && tcc_state->optimize >= 2) ? ssa_opt_dead_loop(ctx) : 0); + SSA_RUN("ssa:stack_deref_fold", + (tcc_state && tcc_state->optimize > 0) ? ssa_opt_stack_deref_fold(ctx) : 0); SSA_RUN("ssa:dce", ssa_opt_dce(ctx)); if (target_gens && target_gen_count > 0) { diff --git a/source/opt/ssa/engine/stack_resolve.c b/source/opt/ssa/engine/stack_resolve.c index 0cbe3d6c4..5e072ae4c 100644 --- a/source/opt/ssa/engine/stack_resolve.c +++ b/source/opt/ssa/engine/stack_resolve.c @@ -228,3 +228,128 @@ int ssa_opt_indirect_stack_offset_ex(IRSSAOptCtx *ctx, const IRQuadCompact *q, i return INT_MIN; return base_off + irop_get_imm32(idx); } + +/* ssa:stack_deref_fold -- a load or store through a TEMP pointer whose value is + * a constant frame address becomes a direct StackLoc access. + * + * The Zig C backend names every sub-object through a pointer temporary + * (`t3 = &t2->major; (*t3) = 5;`), assigned in each arm of a switch, so before + * SSA the pointer is a multi-definition VAR and stack_addr_simplify cannot + * follow it; it also stays away from any function with a loop. After renaming + * each arm's pointer is a single-definition TEMP, and resolving it here turns + * `add rX, sp, #off; str rY, [rX]` into `str rY, [sp, #off]`; DCE then drops + * the address arithmetic. Every hop must be the value's only definition, phi + * included: the rewrite replaces the use outright. */ +static int sdf_resolve(IRSSAOptCtx *ctx, int32_t vr, int *out_off, int32_t *out_slot_vr) +{ + TCCIRState *ir = ctx->ir; + int acc = 0; + for (int hop = 0; hop < 16; hop++) { + if (vr < 0 || TCCIR_DECODE_VREG_TYPE(vr) != TCCIR_VREG_TYPE_TEMP) + return 0; + IRSSAVregInfo *vi = ssa_opt_vinfo(ctx, vr); + if (!vi || vi->def_instr < 0 || ssa_opt_def_total(vi) != 1) + return 0; + IRQuadCompact *dq = &ir->compact_instructions[vi->def_instr]; + IROperand src = tcc_ir_op_get_src1(ir, dq); + int direct = irop_get_tag(src) == IROP_TAG_STACKOFF && irop_get_vreg(src) < 0 && src.is_local && + !src.is_lval && !src.is_llocal && !src.is_param; + int copy = irop_get_tag(src) == IROP_TAG_VREG && !src.is_lval && !src.is_local && !src.is_llocal; + if (dq->op == TCCIR_OP_ASSIGN || dq->op == TCCIR_OP_LEA) { + if (direct) { + *out_off = irop_get_stack_offset(src) + acc; + *out_slot_vr = irop_get_vreg(src); + return 1; + } + if (dq->op == TCCIR_OP_ASSIGN && copy) { + vr = irop_get_vreg(src); + continue; + } + return 0; + } + if (dq->op == TCCIR_OP_ADD || dq->op == TCCIR_OP_SUB) { + IROperand s2 = tcc_ir_op_get_src2(ir, dq); + if (irop_get_tag(s2) != IROP_TAG_IMM32 || s2.is_lval) + return 0; + int d = irop_get_imm32(s2); + acc += dq->op == TCCIR_OP_ADD ? d : -d; + if (direct) { + *out_off = irop_get_stack_offset(src) + acc; + *out_slot_vr = irop_get_vreg(src); + return 1; + } + if (copy) { + vr = irop_get_vreg(src); + continue; + } + return 0; + } + return 0; + } + return 0; +} + +int ssa_opt_stack_deref_fold(IRSSAOptCtx *ctx) +{ + TCCIRState *ir = ctx->ir; + int changes = 0; + for (int i = 0; i < ir->next_instruction_index; i++) { + IRQuadCompact *q = &ir->compact_instructions[i]; + int op = q->op; + if (op != TCCIR_OP_STORE && op != TCCIR_OP_LOAD && op != TCCIR_OP_STORE_INDEXED && op != TCCIR_OP_LOAD_INDEXED) + continue; + int is_store = op == TCCIR_OP_STORE || op == TCCIR_OP_STORE_INDEXED; + int indexed = op == TCCIR_OP_STORE_INDEXED || op == TCCIR_OP_LOAD_INDEXED; + IROperand base = is_store ? tcc_ir_op_get_dest(ir, q) : tcc_ir_op_get_src1(ir, q); + if (irop_get_tag(base) != IROP_TAG_VREG || base.is_local || base.is_llocal || base.is_lval == indexed) + continue; + int32_t bvr = irop_get_vreg(base), slot_vr; + int off; + if (!sdf_resolve(ctx, bvr, &off, &slot_vr)) + continue; + + /* The access's width and sign: the deref operand for a plain access, the + * value (store) or result (load) for an indexed one, whose marks were + * moved onto the base when it was fused. */ + IROperand acc = base; + if (indexed) { + IROperand idx = tcc_ir_op_get_src2(ir, q); + IROperand scale = tcc_ir_op_get_scale(ir, q); + if (irop_get_tag(idx) != IROP_TAG_IMM32 || idx.is_lval || + (!irop_is_none(scale) && (irop_get_tag(scale) != IROP_TAG_IMM32 || irop_get_imm32(scale) != 0))) + continue; + off += irop_get_imm32(idx); + acc = is_store ? tcc_ir_op_get_src1(ir, q) : tcc_ir_op_get_dest(ir, q); + acc.aux = base.aux; + if (irop_is_64bit(acc)) + continue; + } + if (irop_get_btype(acc) == IROP_BTYPE_STRUCT || acc.is_complex || tcc_ir_access_is_volatile(ir, acc)) + continue; + + /* Keep the slot's own identity: a frontend temporary slot carries a + * negative vreg (VR_TEMP_LOCAL), which passes match its accesses by. */ + IROperand slot = irop_make_stackoff(slot_vr, off, 1, 0, 0, irop_get_btype(acc)); + slot.is_unsigned = acc.is_unsigned; + slot.aux = acc.aux; + if (indexed) + q->op = is_store ? TCCIR_OP_STORE : TCCIR_OP_LOAD; + if (is_store) + tcc_ir_set_dest(ir, i, slot); + else + tcc_ir_set_src1(ir, i, slot); + + /* The pointer is no longer used here unless another operand names it. */ + int still = 0; + if (irop_config[q->op].has_dest) + still |= irop_get_vreg(tcc_ir_op_get_dest(ir, q)) == bvr; + if (irop_config[q->op].has_src1) + still |= irop_get_vreg(tcc_ir_op_get_src1(ir, q)) == bvr; + if (irop_config[q->op].has_src2) + still |= irop_get_vreg(tcc_ir_op_get_src2(ir, q)) == bvr; + if (!still) + ssa_opt_remove_use_instr(ssa_opt_vinfo(ctx, bvr), i); + changes++; + } + return changes; +} diff --git a/source/opt/ssa/engine/use_def.c b/source/opt/ssa/engine/use_def.c index 4f97b4c6f..5ce54bfe0 100644 --- a/source/opt/ssa/engine/use_def.c +++ b/source/opt/ssa/engine/use_def.c @@ -173,6 +173,25 @@ void tcc_ir_ssa_opt_init(IRSSAOptCtx *ctx, TCCIRState *ir, ssa_opt_build_chains(ctx); } +/* Grow vinfo to at least `new_cap` entries. A new entry must read as "no + * definition": zero-filled it claimed def_instr 0, and dce_temp_worklist, + * seeing a def with no uses, deleted instruction 0 -- whatever it defined. + * dead_loop grew vinfo with 16 spare entries that way, and the entry block's + * `T31 <- #-5`, a loop counter's initial value, went (test 451). */ +void ssa_opt_vinfo_grow(IRSSAOptCtx *ctx, int new_cap) +{ + if (new_cap <= ctx->vinfo_cap) + return; + ctx->vinfo = tcc_realloc(ctx->vinfo, new_cap * sizeof(IRSSAVregInfo)); + memset(&ctx->vinfo[ctx->vinfo_cap], 0, (new_cap - ctx->vinfo_cap) * sizeof(IRSSAVregInfo)); + for (int i = ctx->vinfo_cap; i < new_cap; i++) + { + ctx->vinfo[i].def_instr = -1; + ctx->vinfo[i].def_phi_block = -1; + } + ctx->vinfo_cap = new_cap; +} + void tcc_ir_ssa_opt_rebuild(IRSSAOptCtx *ctx) { for (int i = 0; i < ctx->vinfo_cap; i++) { @@ -182,13 +201,7 @@ void tcc_ir_ssa_opt_rebuild(IRSSAOptCtx *ctx) ctx->vinfo[i].use_cap = 0; } - int new_cap = ctx->ir->next_temporary_variable; - if (new_cap > ctx->vinfo_cap) { - ctx->vinfo = tcc_realloc(ctx->vinfo, new_cap * sizeof(IRSSAVregInfo)); - memset(&ctx->vinfo[ctx->vinfo_cap], 0, - (new_cap - ctx->vinfo_cap) * sizeof(IRSSAVregInfo)); - ctx->vinfo_cap = new_cap; - } + ssa_opt_vinfo_grow(ctx, ctx->ir->next_temporary_variable); ssa_opt_build_chains(ctx); } diff --git a/source/opt/ssa/include/opt/ssa/ssa_opt_helpers.h b/source/opt/ssa/include/opt/ssa/ssa_opt_helpers.h index 178d14fae..da182e828 100644 --- a/source/opt/ssa/include/opt/ssa/ssa_opt_helpers.h +++ b/source/opt/ssa/include/opt/ssa/ssa_opt_helpers.h @@ -85,7 +85,7 @@ static inline int is_value_dest(IROperand op) /* v == 2^s: returns 1 and writes *shift on success, 0 otherwise (v==0 or * non-power-of-two). */ -static inline int is_power_of_2(uint32_t v, int *shift) +static inline int is_power_of_2_shift(uint32_t v, int *shift) { if (v == 0 || (v & (v - 1)) != 0) return 0; diff --git a/source/opt/ssa/include/ssa_opt.h b/source/opt/ssa/include/ssa_opt.h index 18d7ae6e2..1202c7326 100644 --- a/source/opt/ssa/include/ssa_opt.h +++ b/source/opt/ssa/include/ssa_opt.h @@ -127,6 +127,7 @@ int ssa_opt_run_gens(IRSSAOptCtx *ctx, const IRSSAOptGen *gens, int count); * ============================================================================ */ IRSSAVregInfo *ssa_opt_vinfo(IRSSAOptCtx *ctx, int32_t vreg); +void ssa_opt_vinfo_grow(IRSSAOptCtx *ctx, int new_cap); void ssa_opt_add_use_instr(IRSSAVregInfo *vi, int instr_idx); void ssa_opt_add_use_phi(IRSSAVregInfo *vi, int block, int slot); /* Append use-list entries for every vreg `q` (at index i) reads — same rules @@ -220,6 +221,21 @@ int ssa_opt_loop_unroll(struct TCCIRState *ir); * shape provider). See docs/plan_legacy_loop_decrement_to_zero_ssa.md. */ int ssa_opt_decrement_to_zero(struct TCCIRState *ir); +/* Bottom-testing for the `while`-shaped loops rotation cannot match + * (ssa:loop_bottom_test). ssa:loop_rotate hard-requires the frontend header + * `CMP; JUMPIF exit; JUMP body`; a loop whose body already falls through from + * the header -- the shape ssa:cfg_cleanup leaves, and the one + * ssa:iv_strength_reduction produces when it eliminates a counter against an + * end pointer -- keeps its top test and an unconditional back-edge. This pass + * copies the header CMP in front of the back-edge and turns the back-edge into + * the inverted conditional branch, leaving the header pair as the zero-trip + * guard: one taken branch less per iteration, one instruction more of code. + * Nothing moves, so none of rotation's relocation hazards apply. Flat IR, + * right after ssa_opt_iv_strength_reduction (its shape provider) and before + * ssa_opt_decrement_to_zero. Gated -O1+; level knob TCC_BOTTOM_TEST. + * See source/opt/ssa/loop/loop_bottom_test.c. */ +int ssa_opt_loop_bottom_test(struct TCCIRState *ir); + /* Induction-variable strength reduction (ssa:iv_strength_reduction). Transforms * array-indexing recurrences base + i*stride into a maintained stride pointer * and optionally eliminates the counter IV against a hoisted end pointer. Per @@ -289,6 +305,9 @@ int ssa_opt_resolve_temp_to_base_off(IRSSAOptCtx *ctx, int32_t vr, * dest is not TEMP-DEREF or the LEA chain does not resolve, or the index * is not a constant with scale 0. */ int ssa_opt_indirect_stack_offset(IRSSAOptCtx *ctx, const IRQuadCompact *q, int side); +/* Loads and stores through a TEMP holding a constant frame address become + * direct StackLoc accesses (stack_resolve.c). */ +int ssa_opt_stack_deref_fold(IRSSAOptCtx *ctx); /* Variant that also reports the resolved address identity via *out_base_var * (see ssa_opt_resolve_lea_stackloc_ex for the -1 / >=0 contract). */ int ssa_opt_indirect_stack_offset_ex(IRSSAOptCtx *ctx, const IRQuadCompact *q, int side, diff --git a/source/opt/ssa/loop/dead_loop.c b/source/opt/ssa/loop/dead_loop.c index 048f0551c..bf2eff321 100644 --- a/source/opt/ssa/loop/dead_loop.c +++ b/source/opt/ssa/loop/dead_loop.c @@ -667,13 +667,8 @@ static int rewrite_loop_exit_phis_guarded(IRSSAOptCtx *ctx, IRLoop *loop, LoopEn int p = TCCIR_DECODE_VREG_POSITION(cands[i].new_vr); if (p > max_pos) max_pos = p; } - if (max_pos >= ctx->vinfo_cap) { - int new_cap = max_pos + 16; - ctx->vinfo = tcc_realloc(ctx->vinfo, new_cap * sizeof(IRSSAVregInfo)); - memset(&ctx->vinfo[ctx->vinfo_cap], 0, - (new_cap - ctx->vinfo_cap) * sizeof(IRSSAVregInfo)); - ctx->vinfo_cap = new_cap; - } + if (max_pos >= ctx->vinfo_cap) + ssa_opt_vinfo_grow(ctx, max_pos + 16); /* Capture the exit target before clobbering the JUMPIF. */ IRQuadCompact *jpf_q = &ir->compact_instructions[info->jpf_idx]; diff --git a/source/opt/ssa/loop/loop_bottom_test.c b/source/opt/ssa/loop/loop_bottom_test.c new file mode 100644 index 000000000..9938fe283 --- /dev/null +++ b/source/opt/ssa/loop/loop_bottom_test.c @@ -0,0 +1,577 @@ +/* + * TCC IR - SSA loop: bottom-test a counter-eliminated pointer walk + * + * Copyright (c) 2025 Mateusz Stadnik + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation. + */ + +/* ssa:loop_bottom_test — the second half of loop rotation, for the one shape + * ssa:loop_rotate structurally cannot see. + * + * ssa:loop_rotate matches the frontend's canonical un-rotated header + * `CMP; JUMPIF exit; JUMP body` and relocates the body/latch up into the + * header's slots. Two things leave loops outside that matcher: + * + * - ssa:cfg_cleanup collapses the header's third slot (the `JUMP body`) into + * a fall-through whenever the body already follows the header, so the loop + * arrives at rotation as a plain `while` — `CMP; JUMPIF exit; ; + * JUMP header` — and is refused before any gate runs; + * - ssa:iv_strength_reduction runs AFTER rotation and produces exactly that + * shape when it eliminates a counter: the walk's `p != end` test sits at + * the top and an unconditional back-edge at the bottom. + * + * So tcc's pointer walks retire `cmp; b exit; ; b header` — four + * instructions for a one-store loop — where gcc emits `; cmp; b + * body`. The project's own per-element measurements on RP2350 silicon + * (iv_analysis.c) put the bottom-tested post-indexed walk at 6.012 cycles + * against 7.012 for the split-bump form and 8.012 for the scaled index, i.e. + * the bottom test is worth one branch per iteration on the fastest shape we + * can already produce. + * + * The rewrite here is deliberately NOT a rotation: the body already sits + * between the header and the back-edge, so nothing moves. A copy of the + * header CMP is inserted in front of the back-edge and the back-edge JUMP + * becomes a JUMPIF on the inverted condition targeting the body. The header + * CMP/JUMPIF stay put as the zero-trip guard. Cost is +1 instruction of code; + * the saving is one taken branch per iteration. + * + * Correctness argument (why this needs none of loop_rotate's machinery): + * before: ... body; JUMP hi; hi: CMP a,b; JUMPIF cond -> exit; fall -> body + * after: ... body; CMP a,b; JUMPIF !cond -> body; fall -> exit + * The two CMPs read the same operands and nothing runs between the end of the + * body and the header, so the branch decision is identical. Entry still goes + * through the header guard. No instruction changes index relative to another + * (insert_instr_at renumbers every jump target and switch arm), so the + * orig_index side tables that made rotation fragile are not disturbed — + * the only synthesized instruction is the tail CMP, and it is created before + * tcc_ir_barrel_shift_fusion (post-RA) has annotated anything. + * + * One instruction fewer per iteration is NOT automatically faster on this + * silicon, and the reason is worth keeping: the first rig A/B had + * bench_memcpy's checksum walk go from five instructions to four and get + * 13.8% SLOWER (1,815,226 -> 2,066,226 cycles), while stringsearch's fill loop + * -- the same transform, the same shape -- won 3.6%. The difference was the + * BRANCH TARGET's alignment: stringsearch's head landed on a word boundary, + * bench_memcpy's on a halfword-odd one, so every refill after the back edge + * paid an extra fetch. Padding that one head to a word boundary handed back + * all 251,000 cycles and then some (1,812,226). Hence + * IRQuadCompact.align_target, which this pass sets on every loop head it + * bottom-tests; without it the pass is a net rig LOSS. The effect is general + * and has nothing to do with the head's width -- codegen now aligns every + * backward-branch target, worth -0.944% on the rig set by itself. + * + * Every level requires the same structural core: a straight-line, call-free + * body, at least one memory access in it, and exactly one constant + * self-increment of the compared value. The levels differ only in how close + * the loop has to be to the measured-fastest shape — TCC_BOTTOM_TEST: + * 0 off + * 1 (default) the walk must end post-indexed (the 6.012 form) and the body + * must carry no register-scaled access (the 8.012 form) + * 2 any constant-stride walk without a register-scaled access + * 3 any constant-stride walk + * + * Levels 1 and 3 measured IDENTICAL on the RP2350 benchmark set (level 3 only + * moves the QEMU corpus's own mibench_crc32, by -65,536 instructions); the + * default stays at the tightest gate that produces the whole measured win. + * + * Disableable via TCC_DISABLE_PASS=ssa:loop_bottom_test. + */ + +#include "ir.h" +#include "ssa_opt.h" +#include "opt_loop_utils.h" +#include "opt_utils.h" +#include "tccdbgenv.h" + +TCC_DBG_ENV_INT(lbt_level_env, "TCC_BOTTOM_TEST", 1) + +#define SSA_LBT_MAX_REWRITES 64 + +/* First non-NOP at or after `i`, or n. */ +static int lbt_skip_nops(const TCCIRState *ir, int i) +{ + int n = ir->next_instruction_index; + while (i < n && ir->compact_instructions[i].op == TCCIR_OP_NOP) + i++; + return i; +} + +/* A CMP operand may be duplicated into the tail test only when re-evaluating it + * is free and side-effect-free: a register value or a literal. An lval operand + * dereferences memory, and a second read of a volatile location would be a + * visible extra access. */ +static int lbt_operand_reusable(const TCCIRState *ir, IROperand op) +{ + if (irop_is_immediate(op)) + return 1; + if (!irop_has_vreg(op)) + return 0; + if (!op.is_lval) + return 1; + /* A direct local VAR is a register or a frame slot, not a pointer + * dereference: re-reading it in the tail test is what the header test would + * have read, at worst one more frame load. Anything else -- a deref through + * a pointer, a global, a `*p` -- must not be evaluated a second time. Same + * convention as loop_rotate's ROT_VAR_DIRECT. */ + if (!(TCCIR_DECODE_VREG_TYPE(irop_get_vreg(op)) == TCCIR_VREG_TYPE_VAR && op.is_local)) + return 0; + /* ... and not when the function has volatile memory and this access is not + * proven clean: a duplicated read of `volatile int i` is a visible extra + * access. */ + return !tcc_ir_access_is_volatile(ir, op); +} + +/* Does `q` write `v`? A non-lval dest is an ordinary definition; a direct local + * VAR lval is one too (loop_rotate's ROT_VAR_DIRECT convention). A dest that + * dereferences a pointer stores THROUGH v and leaves v alone. The post-indexed + * forms also write their base, but they are created after this pass runs. */ +static int lbt_defines_vreg(const TCCIRState *ir, const IRQuadCompact *q, int32_t v) +{ + int op = q->op; + if (v < 0) + return 0; + if (op == TCCIR_OP_LOAD_POSTINC || op == TCCIR_OP_STORE_POSTINC) + { + IROperand b = (op == TCCIR_OP_LOAD_POSTINC) ? tcc_ir_op_get_src1(ir, q) : tcc_ir_op_get_dest(ir, q); + if (irop_get_vreg(b) == v) + return 1; + } + if (!irop_config[op].has_dest) + return 0; + IROperand d = tcc_ir_op_get_dest(ir, q); + if (irop_get_vreg(d) != v) + return 0; + if (!d.is_lval) + return 1; + return TCCIR_DECODE_VREG_TYPE(v) == TCCIR_VREG_TYPE_VAR && d.is_local; +} + +static int lbt_is_memory_op(int op) +{ + return op == TCCIR_OP_LOAD || op == TCCIR_OP_STORE || op == TCCIR_OP_LOAD_INDEXED || + op == TCCIR_OP_STORE_INDEXED || op == TCCIR_OP_LOAD_POSTINC || op == TCCIR_OP_STORE_POSTINC || + op == TCCIR_OP_BLOCK_COPY; +} + +/* Straight-line-ness: the body must be one basic block with no way out but the + * back-edge, so the tail test is reached exactly when the back-edge was. */ +static int lbt_body_is_straight_line(const TCCIRState *ir, int from, int to) +{ + for (int i = from; i <= to; i++) + { + int op = ir->compact_instructions[i].op; + switch (op) + { + case TCCIR_OP_JUMP: + case TCCIR_OP_JUMPIF: + case TCCIR_OP_IJUMP: + case TCCIR_OP_SWITCH_TABLE: + case TCCIR_OP_RETURNVOID: + case TCCIR_OP_RETURNVALUE: + case TCCIR_OP_TRAP: + case TCCIR_OP_FUNCCALLVAL: + case TCCIR_OP_FUNCCALLVOID: + return 0; + default: + break; + } + } + return 1; +} + +/* The profitability signature: `X <- X ADD #k` for one of the compared values, + * plus real memory traffic in the body. Without the self-increment this is not + * a walk at all (and may not even terminate through this test); without the + * memory access there is nothing for the saved branch to be measured against. */ +static int lbt_body_profitable(const TCCIRState *ir, int from, int to, int32_t vr1, int32_t vr2, int level, + int32_t *out_walk_vreg, int *out_bump_idx) +{ + int bumps = 0, mem = 0; + for (int i = from; i <= to; i++) + { + const IRQuadCompact *q = &ir->compact_instructions[i]; + int op = q->op; + if (lbt_is_memory_op(op)) + mem = 1; + if (op != TCCIR_OP_ADD && op != TCCIR_OP_SUB) + continue; + IROperand d = tcc_ir_op_get_dest(ir, q); + IROperand s1 = tcc_ir_op_get_src1(ir, q); + IROperand s2 = tcc_ir_op_get_src2(ir, q); + if (d.is_lval || s1.is_lval || !irop_has_vreg(d) || !irop_has_vreg(s1)) + continue; + if (!irop_is_immediate(s2)) + continue; + int32_t dv = irop_get_vreg(d); + if (dv != irop_get_vreg(s1)) + continue; + if (dv == vr1 || dv == vr2) + { + bumps++; + *out_walk_vreg = dv; + *out_bump_idx = i; + } + } + if (bumps != 1 || !mem) + return 0; + /* The bump must be the walked value's ONLY definition in the body. A second + * one means the value is not a constant-stride walk at all -- 20000801-1's + * `*bp++ = c; ...; bp += 2` writes it three times -- and the copy forwarding + * below reasons from `P is written once, at bump_idx`. */ + for (int i = from; i <= to; i++) + if (i != *out_bump_idx && lbt_defines_vreg(ir, &ir->compact_instructions[i], *out_walk_vreg)) + return 0; + return 1; +} + +/* A register-scaled access -- `ldr rd,[rn,rm,lsl #k]`, whether already fused + * into an indexed op or still open as `t <- i SHL #k; a <- base ADD t; *a`. + * The M33 charges an extra cycle for it that QEMU's icount does not + * (tcc-m33-addressing-mode-cycle-costs), and it is the form every previous + * rotation experiment lost on. An indexed op with an IMMEDIATE index is just + * `[rn,#imm]` and carries no such penalty. */ +static int lbt_body_has_scaled_index(const TCCIRState *ir, int from, int to) +{ + for (int i = from; i <= to; i++) + { + const IRQuadCompact *q = &ir->compact_instructions[i]; + int op = q->op; + if (op == TCCIR_OP_LOAD_INDEXED || op == TCCIR_OP_STORE_INDEXED) + { + if (!irop_is_immediate(tcc_ir_op_get_src2(ir, q))) + return 1; + continue; + } + if (op == TCCIR_OP_SHL || op == TCCIR_OP_MUL) + return 1; + } + return 0; +} + +/* Will the walk end post-indexed? That is the 6.012 cycles/element form in + * iv_analysis.c's table, and the only one the rig data actually endorses; the + * split-bump form is 7.012 and the scaled index 8.012. ra:load_postinc / + * ra:store_postinc fuse `*P; P <- P + #k`, so the body must dereference the + * walked register itself -- or a copy of it that lbt_forward_body_copies is + * about to fold away. */ +static int lbt_walk_is_postinc_shaped(const TCCIRState *ir, int from, int bump_idx, int32_t walk_vreg) +{ + int32_t alias[8]; + int nalias = 0; + for (int i = from; i < bump_idx; i++) + { + const IRQuadCompact *q = &ir->compact_instructions[i]; + int op = q->op; + if (op == TCCIR_OP_ASSIGN) + { + IROperand d = tcc_ir_op_get_dest(ir, q); + IROperand s1 = tcc_ir_op_get_src1(ir, q); + if (!d.is_lval && !s1.is_lval && irop_has_vreg(d) && irop_get_vreg(s1) == walk_vreg && + nalias < (int)(sizeof(alias) / sizeof(alias[0]))) + alias[nalias++] = irop_get_vreg(d); + continue; + } + if (op != TCCIR_OP_LOAD && op != TCCIR_OP_STORE) + continue; + IROperand ptr = (op == TCCIR_OP_LOAD) ? tcc_ir_op_get_src1(ir, q) : tcc_ir_op_get_dest(ir, q); + if (!ptr.is_lval || ptr.is_local || ptr.is_llocal || ptr.is_sym || !irop_has_vreg(ptr)) + continue; + int32_t pv = irop_get_vreg(ptr); + if (pv == walk_vreg) + return 1; + for (int a = 0; a < nalias; a++) + if (alias[a] == pv) + return 1; + } + return 0; +} + +/* Every vreg slot of an instruction that a rewrite has to reach. The 4th pool + * slot is an immediate scale on the indexed/postinc forms and a condition on + * SELECT; only MLA carries a vreg there. */ +static IROperand *lbt_operand_slot(TCCIRState *ir, IRQuadCompact *q, int which) +{ + int op = q->op; + int base = (int)q->operand_base; + switch (which) + { + case 0: + return irop_config[op].has_dest ? &ir->iroperand_pool[base] : NULL; + case 1: + return irop_config[op].has_src1 ? &ir->iroperand_pool[base + irop_config[op].has_dest] : NULL; + case 2: + return irop_config[op].has_src2 + ? &ir->iroperand_pool[base + irop_config[op].has_dest + irop_config[op].has_src1] + : NULL; + default: + return op == TCCIR_OP_MLA ? &ir->iroperand_pool[base + 3] : NULL; + } +} + +/* Forward `T <- P` copies inside the body onto P itself. + * + * ssa:iv_strength_reduction leaves the walked pointer's every access reading a + * COPY of the walk register (`T <- P; *T <- v; P <- P + k`). In the un-rotated + * shape SSA renaming versions P at the loop header, and ssa:cprop then folds + * the copy away. Bottom-testing makes the body its own predecessor, and + * ra_promote_multidef_temps_to_vars deliberately declines a TEMP whose every + * use sits in one of its own def-blocks -- which is now true of P -- so no phi + * is built, P keeps two defs, and the copy survives. It costs no instruction + * (the coalescer gives T and P one register and the move disappears), but it + * hides the walk from ra:store_postinc/ra:load_postinc, whose matcher wants the + * dereferenced vreg and the bumped vreg to be the SAME -- and the post-indexed + * access is the entire point of bottom-testing this shape. + * + * Safe here for reasons the body checks already established: the body is one + * straight-line block, P is written exactly once in it (the bump), and a use of + * T before the bump therefore sees exactly P. What is left to prove per copy is + * that T is confined to the body and not redefined inside it. */ +static int lbt_forward_body_copies(TCCIRState *ir, int body_start, int body_end, int32_t walk_vreg, int bump_idx) +{ + int n = ir->next_instruction_index; + int forwarded = 0; + for (int c = body_start; c <= body_end; c++) + { + IRQuadCompact *cq = &ir->compact_instructions[c]; + if (cq->op != TCCIR_OP_ASSIGN) + continue; + IROperand d = tcc_ir_op_get_dest(ir, cq); + IROperand s = tcc_ir_op_get_src1(ir, cq); + if (d.is_lval || s.is_lval || !irop_has_vreg(d) || !irop_has_vreg(s)) + continue; + if (irop_get_vreg(s) != walk_vreg) + continue; + int32_t tv = irop_get_vreg(d); + if (tv == walk_vreg) + continue; + if (TCCIR_DECODE_VREG_TYPE(tv) != TCCIR_VREG_TYPE_TEMP) + continue; + if (irop_get_btype(d) != irop_get_btype(s) || d.is_unsigned != s.is_unsigned) + continue; + /* The copy must dominate the uses it feeds, and P must still hold the + * copied value there: only uses strictly before the bump qualify. P having + * no other definition in the body is what lbt_body_profitable proved; assert + * it again rather than inherit it, because getting this wrong is silent + * wrong code (20000801-1 stored through the already-incremented pointer). */ + if (c >= bump_idx) + continue; + { + int other_def = 0; + for (int i = body_start; i <= body_end && !other_def; i++) + if (i != bump_idx && lbt_defines_vreg(ir, &ir->compact_instructions[i], walk_vreg)) + other_def = 1; + if (other_def) + continue; + } + + /* T must live only inside this body, and only from this one def. */ + int confined = 1; + for (int i = 0; i < n && confined; i++) + { + if (i >= body_start && i <= body_end) + continue; + IRQuadCompact *q = &ir->compact_instructions[i]; + for (int w = 0; w < 4; w++) + { + IROperand *o = lbt_operand_slot(ir, q, w); + if (o && irop_get_vreg(*o) == tv) + { + confined = 0; + break; + } + } + } + if (!confined) + continue; + int redefined = 0, used_after_bump = 0; + for (int i = c + 1; i <= body_end && !redefined; i++) + { + IRQuadCompact *q = &ir->compact_instructions[i]; + for (int w = 0; w < 4; w++) + { + IROperand *o = lbt_operand_slot(ir, q, w); + if (!o || irop_get_vreg(*o) != tv) + continue; + if (w == 0 && irop_config[q->op].has_dest && !o->is_lval) + { + redefined = 1; + break; + } + if (i > bump_idx) + used_after_bump = 1; + } + } + if (redefined || used_after_bump) + continue; + + for (int i = c + 1; i <= body_end; i++) + { + IRQuadCompact *q = &ir->compact_instructions[i]; + for (int w = 0; w < 4; w++) + { + IROperand *o = lbt_operand_slot(ir, q, w); + if (o && irop_get_vreg(*o) == tv) + irop_set_vreg(o, walk_vreg); + } + } + cq->op = TCCIR_OP_NOP; + forwarded++; + } + return forwarded; +} + +/* Match and rewrite the loop whose back-edge JUMP sits at `be`. Returns 1 on a + * rewrite (indices after `be` have moved), 0 otherwise. */ +static int lbt_try_backedge(TCCIRState *ir, int be, int level) +{ + int n = ir->next_instruction_index; + IRQuadCompact *jmp_q = &ir->compact_instructions[be]; + if (jmp_q->op != TCCIR_OP_JUMP) + return 0; + /* A jump landing on the back-edge itself would skip the tail CMP once the + * slot in front of it is claimed. */ + if (jmp_q->is_jump_target) + return 0; + + int hi = (int)irop_get_imm64_ex(ir, tcc_ir_op_get_dest(ir, jmp_q)); + if (hi < 0 || hi >= be) + return 0; + + IRQuadCompact *cmp_q = &ir->compact_instructions[hi]; + if (cmp_q->op != TCCIR_OP_CMP) + return 0; + int jif = lbt_skip_nops(ir, hi + 1); + if (jif >= be) + return 0; + IRQuadCompact *jif_q = &ir->compact_instructions[jif]; + if (jif_q->op != TCCIR_OP_JUMPIF) + return 0; + /* Nothing may reach the exit test without the CMP that feeds it. */ + if (jif_q->is_jump_target) + return 0; + + int cond = (int)irop_get_imm64_ex(ir, tcc_ir_op_get_src1(ir, jif_q)); + int inv_cond = invert_condition(cond); + if (inv_cond < 0) + return 0; + int exit_target = (int)irop_get_imm64_ex(ir, tcc_ir_op_get_dest(ir, jif_q)); + /* The guard's target must leave the loop forwards; a target inside would make + * the fall-through after the tail test wrong. */ + if (exit_target <= be || exit_target > n) + return 0; + + int body_start = lbt_skip_nops(ir, jif + 1); + if (body_start >= be) + return 0; /* empty body: nothing to bottom-test */ + + IROperand cmp_src1 = tcc_ir_op_get_src1(ir, cmp_q); + IROperand cmp_src2 = tcc_ir_op_get_src2(ir, cmp_q); + if (!lbt_operand_reusable(ir, cmp_src1) || !lbt_operand_reusable(ir, cmp_src2)) + return 0; + /* A CMP fused with a barrel shift or any other orig_index-keyed annotation + * cannot be duplicated: the copy gets a fresh index and would compare the + * unshifted value (the pr71083 family). Nothing sets these before regalloc + * today, but the check costs nothing and pins the assumption. */ + if (tcc_ir_barrel_shift_at(ir, cmp_q)) + return 0; + + if (!lbt_body_is_straight_line(ir, body_start, be - 1)) + return 0; + + int32_t vr1 = irop_has_vreg(cmp_src1) ? irop_get_vreg(cmp_src1) : -1; + int32_t vr2 = irop_has_vreg(cmp_src2) ? irop_get_vreg(cmp_src2) : -1; + int32_t walk_vreg = -1; + int bump_idx = -1; + if (!lbt_body_profitable(ir, body_start, be - 1, vr1, vr2, level, &walk_vreg, &bump_idx)) + return 0; + if (level < 3 && lbt_body_has_scaled_index(ir, body_start, be - 1)) + return 0; + if (level < 2 && !lbt_walk_is_postinc_shaped(ir, body_start, bump_idx, walk_vreg)) + return 0; + + lbt_forward_body_copies(ir, body_start, be - 1, walk_vreg, bump_idx); + + /* Does the tail test fall straight into the exit, or does it need a jump? */ + int need_exit_jump = lbt_skip_nops(ir, be + 1) != exit_target; + + /* Claim the slots BEFORE writing anything, so a failed allocation leaves the + * loop exactly as it was (a bare NOP at worst) rather than a half-built tail + * test in front of a still-unconditional back-edge. insert_instr_at + * renumbers every jump target and switch arm past the insertion point, and + * exit_target is past it by the check above. */ + IROperand none_op = irop_make_none(); + if (insert_instr_at(ir, be, TCCIR_OP_NOP, none_op, none_op, none_op) < 0) + return 0; + exit_target++; + int cmp_pos = be; + int jmp_pos = be + 1; + if (need_exit_jump) + { + if (insert_instr_at(ir, jmp_pos + 1, TCCIR_OP_NOP, none_op, none_op, none_op) < 0) + return 0; + exit_target++; + } + + write_instr_at_nop(ir, cmp_pos, TCCIR_OP_CMP, none_op, cmp_src1, cmp_src2); + ir->compact_instructions[cmp_pos].line_num = ir->compact_instructions[hi].line_num; + + if (need_exit_jump) + { + write_instr_at_nop(ir, jmp_pos + 1, TCCIR_OP_JUMP, + irop_make_imm32(-1, exit_target, IROP_BTYPE_INT32), none_op, none_op); + if (exit_target < ir->next_instruction_index) + ir->compact_instructions[exit_target].is_jump_target = 1; + } + + ir->compact_instructions[jmp_pos].op = TCCIR_OP_NOP; + write_instr_at_nop(ir, jmp_pos, TCCIR_OP_JUMPIF, irop_make_imm32(-1, body_start, IROP_BTYPE_INT32), + irop_make_imm32(-1, inv_cond, IROP_BTYPE_INT32), none_op); + ir->compact_instructions[jmp_pos].line_num = ir->compact_instructions[hi].line_num; + ir->compact_instructions[body_start].is_jump_target = 1; + /* Ask codegen to word-align the head. The flag goes on the first REAL + * instruction, not on body_start: the copy forwarding above may have left a + * NOP there, and tcc_ir_opt_compact_nops (which runs a few lines later in + * the pipeline) would drop it and the flag with it. A back-edge branch onto + * a halfword-odd 32-bit instruction costs about a cycle per iteration on the + * RP2350 -- see IRQuadCompact.align_target. */ + ir->compact_instructions[lbt_skip_nops(ir, body_start)].align_target = 1; + + LOG_IR_GEN("[LOOP-BOTTOM-TEST] header=%d body=[%d..%d] -> tail CMP@%d JUMPIF@%d exit=%d", hi, body_start, + cmp_pos - 1, cmp_pos, jmp_pos, exit_target); + return 1; +} + +int ssa_opt_loop_bottom_test(TCCIRState *ir) +{ + if (!ir || ir->next_instruction_index == 0) + return 0; + int level = lbt_level_env(); + if (level <= 0) + return 0; + if (!tcc_ir_cfg_flat_has_backedge(ir)) + return 0; + + int total = 0; + int scan = 0; + while (total < SSA_LBT_MAX_REWRITES) + { + int hit = 0; + for (int i = scan; i < ir->next_instruction_index; i++) + { + if (lbt_try_backedge(ir, i, level)) + { + total++; + /* the rewritten back-edge is now a JUMPIF; resume past it */ + scan = i + 2; + hit = 1; + break; + } + } + if (!hit) + break; + } + return total; +} diff --git a/source/opt/ssa/loop/loop_unroll.c b/source/opt/ssa/loop/loop_unroll.c index 6e6a16fa5..56c5d9a86 100644 --- a/source/opt/ssa/loop/loop_unroll.c +++ b/source/opt/ssa/loop/loop_unroll.c @@ -16,6 +16,286 @@ #define SSA_LOOP_UNROLL_MAX_PASSES 4 #define SSA_UNROLL_MAX_SPAN 256 +/* Whether instruction Q names vreg VR in any operand slot. */ +static int insn_mentions_vreg(TCCIRState *ir, IRQuadCompact *q, int32_t vr) +{ + if (q->op == TCCIR_OP_NOP) + return 0; + if (irop_config[q->op].has_dest && irop_get_vreg(tcc_ir_op_get_dest(ir, q)) == vr) + return 1; + if (irop_config[q->op].has_src1 && irop_get_vreg(tcc_ir_op_get_src1(ir, q)) == vr) + return 1; + if (irop_config[q->op].has_src2 && irop_get_vreg(tcc_ir_op_get_src2(ir, q)) == vr) + return 1; + if ((tcc_ir_op_is_mac(q->op) || q->op == TCCIR_OP_LOAD_INDEXED || q->op == TCCIR_OP_STORE_INDEXED) && + irop_get_vreg(ir->iroperand_pool[q->operand_base + 3]) == vr) + return 1; + return 0; +} + +/* Whether source operand S reads a vreg's own value: a register vreg, or a + * variable in its stack slot (a STACKOFF lvalue naming it) -- not a load + * through the pointer it holds. */ +static int src_reads_vreg_value(IROperand s) +{ + if (irop_get_vreg(s) < 0 || s.is_llocal) + return 0; + int tag = irop_get_tag(s); + return tag == IROP_TAG_VREG ? !s.is_lval : tag == IROP_TAG_STACKOFF; +} + +/* `DST <- SRC` between two plain 32-bit VARs (a VAR read is a LOAD or an ASSIGN). */ +static int is_var_copy(TCCIRState *ir, IRQuadCompact *q, int32_t *dst, int32_t *src) +{ + if (q->op != TCCIR_OP_ASSIGN && q->op != TCCIR_OP_LOAD) + return 0; + IROperand d = tcc_ir_op_get_dest(ir, q), s = tcc_ir_op_get_src1(ir, q); + int32_t dv = irop_get_vreg(d), sv = irop_get_vreg(s); + if (dv < 0 || sv < 0 || TCCIR_DECODE_VREG_TYPE(dv) != TCCIR_VREG_TYPE_VAR || + TCCIR_DECODE_VREG_TYPE(sv) != TCCIR_VREG_TYPE_VAR || !irop_dest_defines_vreg(d) || !src_reads_vreg_value(s) || + irop_get_btype(d) != IROP_BTYPE_INT32 || irop_get_btype(s) != IROP_BTYPE_INT32) + return 0; + *dst = dv; + *src = sv; + return 1; +} + +static int next_real(TCCIRState *ir, int i, int end) +{ + while (i <= end && ir->compact_instructions[i].op == TCCIR_OP_NOP) + i++; + return i; +} + +static int prev_real(TCCIRState *ir, int i, int start) +{ + while (i >= start && ir->compact_instructions[i].op == TCCIR_OP_NOP) + i--; + return i; +} + +/* Q computes a value equal to one of SAME[0..N): a copy of it, or it ANDed + * with all ones (an immediate or a TEMP whose only def is `#-1`). */ +static int cz_value_step(TCCIRState *ir, IRQuadCompact *q, const int32_t *same, int n) +{ + if (q->op != TCCIR_OP_ASSIGN && q->op != TCCIR_OP_LOAD && q->op != TCCIR_OP_STORE && q->op != TCCIR_OP_AND) + return 0; + IROperand d = tcc_ir_op_get_dest(ir, q), s = tcc_ir_op_get_src1(ir, q); + int32_t dv = irop_get_vreg(d); + if (dv < 0 || !irop_dest_defines_vreg(d) || !src_reads_vreg_value(s) || + irop_get_btype(d) != IROP_BTYPE_INT32) + return 0; + int in = 0; + for (int k = 0; k < n; k++) + in |= irop_get_vreg(s) == same[k]; + if (!in) + return 0; + if (q->op != TCCIR_OP_AND) + return 1; + IROperand m = tcc_ir_op_get_src2(ir, q); + if (irop_is_immediate(m)) + return (uint32_t)irop_get_imm64_ex(ir, m) == 0xffffffffu; + int32_t mv = irop_get_vreg(m); + if (mv < 0 || TCCIR_DECODE_VREG_TYPE(mv) != TCCIR_VREG_TYPE_TEMP || !src_reads_vreg_value(m)) + return 0; + int defs = 0, ones = 0; + for (int i = 0; i < ir->next_instruction_index; i++) + { + IRQuadCompact *dq = &ir->compact_instructions[i]; + if (dq->op == TCCIR_OP_NOP || !irop_config[dq->op].has_dest) + continue; + IROperand dd = tcc_ir_op_get_dest(ir, dq); + if (irop_get_vreg(dd) != mv || !irop_dest_defines_vreg(dd)) + continue; + defs++; + IROperand ds = tcc_ir_op_get_src1(ir, dq); + ones = dq->op == TCCIR_OP_ASSIGN && irop_is_immediate(ds) && (uint32_t)irop_get_imm64_ex(ir, ds) == 0xffffffffu; + } + return defs == 1 && ones; +} +#define CZ_MAX_SLOTS 12 + +/* The Zig C backend spells every loop counter with two locals: + * + * B = init; + * loop: + * A = B; C = A; ... if (!(C < n)) goto out; + * ... A ... + * A = A + step; B = A; goto loop; + * + * (C and the steps before the test are value copies: an inlined + * zig_u32_bitCast_u32 binds its parameter and masks it with all ones.) So A + * is defined twice in the loop and B never incremented: no induction + * variable. When B is mentioned nowhere else and A, and its copies, are not + * read after the loop, B only carries A round the back edge, and the loop is + * the classic `A = init; loop: if (!(A < n)) goto out; ...; A += step;`. + * Rewrite it so: the init defines A, both copies of the carrier go, the exit + * test reads A and comes first in the loop, where find_loop_exit_condition + * looks for it. Returns 1 if rewritten. Done only on a loop the unroller is + * about to consider: the parked loop_var_coalesce pass merged such counters + * in every loop of every function and shifted inline decisions. */ +static int canonicalize_zig_counter(TCCIRState *ir, int start, int end) +{ + const int n = ir->next_instruction_index; + int h = next_real(ir, start, end); + int32_t a, b; + if (h > end || !is_var_copy(ir, &ir->compact_instructions[h], &a, &b) || a == b) + return 0; + + /* Between the copy and the exit test, only steps that pass A's value on + * unchanged: copies (an inlined zig_u32_bitCast_u32 parameter binds it by + * `V <- A [STORE]`) and an AND with all ones (its mask). They stay; the + * test reads A itself. */ +#define CZ_MAX_CHAIN 8 + int32_t same[CZ_MAX_CHAIN + 1]; + int nsame = 0; + same[nsame++] = a; + int cmp = next_real(ir, h + 1, end); + while (cmp <= end && ir->compact_instructions[cmp].op != TCCIR_OP_CMP) + { + IRQuadCompact *q = &ir->compact_instructions[cmp]; + if (nsame > CZ_MAX_CHAIN || !cz_value_step(ir, q, same, nsame)) + return 0; + int32_t d = irop_get_vreg(tcc_ir_op_get_dest(ir, q)); + if (d == a || d == b) + return 0; + same[nsame++] = d; + cmp = next_real(ir, cmp + 1, end); + } + if (cmp > end) + return 0; + IRQuadCompact *cq = &ir->compact_instructions[cmp]; + IROperand cmp_src = tcc_ir_op_get_src1(ir, cq); + int cmp_ok = 0; + for (int k = 0; k < nsame; k++) + cmp_ok |= irop_get_vreg(cmp_src) == same[k]; + if (!cmp_ok || !src_reads_vreg_value(cmp_src) || !irop_is_immediate(tcc_ir_op_get_src2(ir, cq))) + return 0; +#undef CZ_MAX_CHAIN + int jif = next_real(ir, cmp + 1, end); + if (jif > end || ir->compact_instructions[jif].op != TCCIR_OP_JUMPIF || + (int)irop_get_imm64_ex(ir, tcc_ir_op_get_dest(ir, &ir->compact_instructions[jif])) <= end) + return 0; + + /* Latch: A <- A + #step; B <- A; JUMP header. */ + int back = prev_real(ir, end, start); + if (back < start || ir->compact_instructions[back].op != TCCIR_OP_JUMP) + return 0; + int l1 = prev_real(ir, back - 1, start); + int32_t l1d, l1s; + if (l1 <= jif || !is_var_copy(ir, &ir->compact_instructions[l1], &l1d, &l1s) || l1d != b || l1s != a) + return 0; + int l0 = prev_real(ir, l1 - 1, start); + if (l0 <= jif) + return 0; + IRQuadCompact *iq = &ir->compact_instructions[l0]; + if (iq->op != TCCIR_OP_ADD || irop_get_vreg(tcc_ir_op_get_dest(ir, iq)) != a || + irop_get_vreg(tcc_ir_op_get_src1(ir, iq)) != a || !irop_is_immediate(tcc_ir_op_get_src2(ir, iq))) + return 0; + + /* Init `B <- #imm` just before the loop (constants may be set up after it). */ + int init = -1; + for (int i = prev_real(ir, start - 1, 0), k = 0; i >= 0 && k < 6; i = prev_real(ir, i - 1, 0), k++) + { + IRQuadCompact *q = &ir->compact_instructions[i]; + if (q->op == TCCIR_OP_JUMP || q->op == TCCIR_OP_JUMPIF || ir->compact_instructions[i].is_jump_target) + break; + if (irop_config[q->op].has_dest && irop_get_vreg(tcc_ir_op_get_dest(ir, q)) == b) + { + init = i; + break; + } + } + if (init < 0 || ir->compact_instructions[init].op != TCCIR_OP_ASSIGN) + return 0; + IRQuadCompact *initq = &ir->compact_instructions[init]; + IROperand init_d = tcc_ir_op_get_dest(ir, initq); + if (!irop_dest_defines_vreg(init_d) || !irop_is_immediate(tcc_ir_op_get_src1(ir, initq))) + return 0; + + /* B: only the init, the header copy and the latch copy. A: defined in the + * loop only by the header copy and the increment, never read after it. */ + for (int i = 0; i < n; i++) + { + IRQuadCompact *q = &ir->compact_instructions[i]; + if (q->op == TCCIR_OP_NOP) + continue; + if (i != init && i != h && i != l1 && insn_mentions_vreg(ir, q, b)) + return 0; + /* A and the values copied from it keep their exit value, which + * unrolling does not reproduce, so none may be read after the loop. */ + for (int k = 0; k < nsame; k++) + if (i > end && insn_mentions_vreg(ir, q, same[k])) + return 0; + if (i >= start && i <= end && i != h && i != l0 && irop_config[q->op].has_dest && + irop_get_vreg(tcc_ir_op_get_dest(ir, q)) == a) + return 0; + } + /* A jump into the loop other than to its header would skip `A = B`, and + * one to a step between the copy and the exit test would land in the + * reordered run below. */ + for (int i = 0; i < n; i++) + { + IRQuadCompact *q = &ir->compact_instructions[i]; + if (q->op != TCCIR_OP_JUMP && q->op != TCCIR_OP_JUMPIF) + continue; + int t = next_real(ir, (int)irop_get_imm64_ex(ir, tcc_ir_op_get_dest(ir, q)), end); + if ((i < start || i > end) && t > h && t <= end) + return 0; + if (t > h && t <= jif) + return 0; + } + + IROperand nd = init_d; + irop_set_vreg(&nd, a); + tcc_ir_op_set_dest(ir, initq, nd); + IROperand ns = cmp_src; + irop_set_vreg(&ns, a); + tcc_ir_op_set_src1(ir, cq, ns); + ir->compact_instructions[h].op = TCCIR_OP_NOP; + ir->compact_instructions[l1].op = TCCIR_OP_NOP; + /* A value step spelled `V <- A [STORE]` is a copy between variables: say + * so, since a STORE-bound VAR reads as a load bound to a parameter, which + * the unroller must keep as a loop. */ + for (int i = h + 1; i < cmp; i++) + if (ir->compact_instructions[i].op == TCCIR_OP_STORE) + ir->compact_instructions[i].op = TCCIR_OP_ASSIGN; + /* The exit test goes first, the value steps after it (they are dead on the + * exit path, nothing reads them after the loop): the loop is then the + * classic top-tested shape whose test find_loop_exit_condition looks for at + * the header. Slots keep their jump-target flags; the quads move. */ + { + int slots[CZ_MAX_SLOTS], ns = 0; + IRQuadCompact moved[CZ_MAX_SLOTS]; + for (int i = h; i <= jif && ns < CZ_MAX_SLOTS; i++) + if (ir->compact_instructions[i].op != TCCIR_OP_NOP || i == h) + slots[ns++] = i; + if (ns < CZ_MAX_SLOTS) + { + int m = 0; + moved[m++] = ir->compact_instructions[cmp]; + moved[m++] = ir->compact_instructions[jif]; + for (int k = 1; k < ns; k++) + if (slots[k] != cmp && slots[k] != jif) + moved[m++] = ir->compact_instructions[slots[k]]; + for (; m < ns; m++) + { + moved[m] = ir->compact_instructions[h]; + moved[m].op = TCCIR_OP_NOP; + } + for (int k = 0; k < ns; k++) + { + IRQuadCompact *q = &ir->compact_instructions[slots[k]]; + int jt = q->is_jump_target; + *q = moved[k]; + q->is_jump_target = jt; + } + } + } + LOG_LOOP_OPT("canonicalize_zig_counter: loop [%d..%d] counter VAR%d", start, end, TCCIR_DECODE_VREG_POSITION(a)); + return 1; +} + static int unroll_try_candidate(TCCIRState *ir, IRCFG *cfg, int header_b, uint8_t *member, uint8_t *scratch) { @@ -66,6 +346,8 @@ static int unroll_try_candidate(TCCIRState *ir, IRCFG *cfg, int header_b, loop.preheader_idx = preheader; loop.depth = 1; + canonicalize_zig_counter(ir, eff_start, eff_end); + if (try_eliminate_loop(ir, &loop)) return 1; if (try_eliminate_loop_symbolic(ir, &loop)) diff --git a/source/opt/ssa/memory/load_cse.c b/source/opt/ssa/memory/load_cse.c index 0a1c2f450..c33dac185 100644 --- a/source/opt/ssa/memory/load_cse.c +++ b/source/opt/ssa/memory/load_cse.c @@ -1165,7 +1165,11 @@ static int gload_process_block(IRSSAOptCtx *ctx, const LcseDefCounts *dc, int store_btype = irop_get_btype(dest); /* dest vreg -1 = real slot (offset is identity); else VAR vreg = named local (placeholder offset) */ int32_t dest_base = irop_get_vreg(dest); - if (svr >= 0 && TCCIR_DECODE_VREG_TYPE(svr) == TCCIR_VREG_TYPE_TEMP) + /* `StackLoc <- *T` (a fused memory copy) stores what T points at, not + * T: only a plain value source is the slot's new contents -- as the + * vslot/tvstore/gstore trackers already require. */ + if (svr >= 0 && src.tag == IROP_TAG_VREG && !src.is_lval && + TCCIR_DECODE_VREG_TYPE(svr) == TCCIR_VREG_TYPE_TEMP) sstore_track_vr(st, irop_get_stack_offset(dest), store_btype, svr, dest_base); else if (irop_is_immediate(src)) sstore_track_imm(st, irop_get_stack_offset(dest), store_btype, src, dest_base); @@ -1210,7 +1214,9 @@ static int gload_process_block(IRSSAOptCtx *ctx, const LcseDefCounts *dc, IROperand src = tcc_ir_op_get_src1(ir, q); int store_btype = irop_get_btype(dest); int32_t svr = irop_get_vreg(src); - if (svr >= 0 && TCCIR_DECODE_VREG_TYPE(svr) == TCCIR_VREG_TYPE_TEMP) + /* Plain value sources only, as above: `*T <- *U` stores *U. */ + if (svr >= 0 && src.tag == IROP_TAG_VREG && !src.is_lval && + TCCIR_DECODE_VREG_TYPE(svr) == TCCIR_VREG_TYPE_TEMP) sstore_track_vr(st, eff_off, store_btype, svr, store_base); else if (irop_is_immediate(src)) sstore_track_imm(st, eff_off, store_btype, src, store_base); diff --git a/source/opt/ssa/scalar/cmp_offset_fold.c b/source/opt/ssa/scalar/cmp_offset_fold.c index 0dd7ecb15..61cd866da 100644 --- a/source/opt/ssa/scalar/cmp_offset_fold.c +++ b/source/opt/ssa/scalar/cmp_offset_fold.c @@ -73,8 +73,9 @@ static int co_vreg_ever_written(TCCIRState *ir, int32_t vr) IRQuadCompact *q = &ir->compact_instructions[i]; if (q->op == TCCIR_OP_NOP || !irop_config[q->op].has_dest) continue; + /* A PARAM written as itself is a STACKOFF lvalue: a write all the same. */ IROperand d = tcc_ir_op_get_dest(ir, q); - if (!d.is_lval && irop_get_vreg(d) == vr) + if (irop_get_vreg(d) == vr && irop_dest_defines_vreg(d)) return 1; } return 0; diff --git a/source/opt/ssa/scalar/cprop.c b/source/opt/ssa/scalar/cprop.c index 42f22fde2..2e2efb751 100644 --- a/source/opt/ssa/scalar/cprop.c +++ b/source/opt/ssa/scalar/cprop.c @@ -281,6 +281,11 @@ OPT_GEN_SSA(cprop_load_redundant, TCCIR_OP_LOAD) { continue; if (irop_get_vreg(ps) != src_vr) continue; + /* A vreg-backed slot names memory by vreg AND offset: two fields of one + * stack-resident struct parameter share the vreg. */ + if (ps.tag == IROP_TAG_STACKOFF && + (irop_get_stack_offset(ps) != irop_get_stack_offset(src) || ps.btype != src.btype)) + continue; IROperand pd = tcc_ir_op_get_dest(ir, pq); int32_t pd_vr = irop_get_vreg(pd); if (pd_vr < 0 || TCCIR_DECODE_VREG_TYPE(pd_vr) != TCCIR_VREG_TYPE_TEMP) @@ -292,11 +297,19 @@ OPT_GEN_SSA(cprop_load_redundant, TCCIR_OP_LOAD) { if (prior_dest_vr < 0) return 0; + /* A plain register read of the TEMP: a STACKOFF source's tag and offset must + * not survive, or the offset (V's frontend watermark) is taken for a spot in + * the TEMP's own spill slot and sizes the frame. */ IROperand new_src = src; irop_set_vreg(&new_src, prior_dest_vr); + new_src.tag = IROP_TAG_VREG; new_src.is_lval = 0; new_src.is_local = 0; new_src.is_llocal = 0; + if (new_src.btype == IROP_BTYPE_STRUCT) + new_src.u.s.aux_data = 0; + else + new_src.u.imm32 = 0; /* Move the use edge from src_vr to prior_dest_vr. */ IRSSAVregInfo *svi = ssa_opt_vinfo(ctx, src_vr); @@ -670,8 +683,39 @@ static int ssa_gen_cprop_load_any(IRSSAOptCtx *ctx, int idx) return opt_dsl_dispatch_cprop_load_redundant(ctx, idx); } +/* STORE dest=src into a TEMP register (not through it), both TEMPs of one + * width: ssa_rename spells a renamed VAR assignment `V3 <- V2` this way, which + * is how an inlined helper's result reaches its caller. With one def of dest + * it is a copy like the ASSIGN above; forward src into dest's uses, else the + * copy reaches the allocator and becomes a register move. */ +static int ssa_gen_cprop_store_copy(IRSSAOptCtx *ctx, int idx) +{ + TCCIRState *ir = ctx->ir; + IRQuadCompact *q = &ir->compact_instructions[idx]; + IROperand dest = tcc_ir_op_get_dest(ir, q); + IROperand src = tcc_ir_op_get_src1(ir, q); + if (dest.tag != IROP_TAG_VREG || src.tag != IROP_TAG_VREG) + return 0; + if (dest.is_lval || dest.is_llocal || dest.is_local || src.is_lval || src.is_llocal || src.is_local) + return 0; + if (irop_get_btype(dest) != irop_get_btype(src) || irop_is_64bit(dest) != irop_is_64bit(src)) + return 0; + int32_t dest_vr = irop_get_vreg(dest), src_vr = irop_get_vreg(src); + if (dest_vr < 0 || src_vr < 0 || dest_vr == src_vr || TCCIR_DECODE_VREG_TYPE(dest_vr) != TCCIR_VREG_TYPE_TEMP || + TCCIR_DECODE_VREG_TYPE(src_vr) != TCCIR_VREG_TYPE_TEMP) + return 0; + IRSSAVregInfo *vi = ssa_opt_vinfo(ctx, dest_vr), *svi = ssa_opt_vinfo(ctx, src_vr); + if (!vi || ssa_opt_def_total(vi) > 1 || ssa_opt_def_total(svi) > 1) + return 0; + for (int u = 0; u < vi->use_count; u++) + if (vi->uses[u].kind == SSA_USE_PHI) + return 0; + return ssa_opt_replace_all_uses(ctx, dest_vr, src_vr) > 0 ? 1 : 0; +} + static const IRSSAOptGen cprop_gens[] = { { TCCIR_OP_ASSIGN, ssa_gen_cprop_assign_any, "cprop_assign_any" }, + { TCCIR_OP_STORE, ssa_gen_cprop_store_copy, "cprop_store_copy" }, { TCCIR_OP_LOAD, ssa_gen_cprop_load_any, "cprop_load_any" }, }; diff --git a/source/opt/ssa/scalar/fold.c b/source/opt/ssa/scalar/fold.c index 15143b90c..18271665c 100644 --- a/source/opt/ssa/scalar/fold.c +++ b/source/opt/ssa/scalar/fold.c @@ -170,11 +170,57 @@ static uint32_t fold_maybe_set_bits(IRSSAOptCtx *ctx, IROperand op, int at_idx, case TCCIR_OP_SETIF: /* Materialises its condition as 0 or 1. */ return 1u; + case TCCIR_OP_LOAD: + /* The emitter picks LDRB/LDRH from the source's type and signedness: an + * unsigned narrow load zero-extends. */ + if (s1.is_lval && s1.is_unsigned) { + if (irop_get_btype(s1) == IROP_BTYPE_INT8) + return 0xFFu; + if (irop_get_btype(s1) == IROP_BTYPE_INT16) + return 0xFFFFu; + } + return 0xFFFFFFFFu; + case TCCIR_OP_UBFX: { + int32_t f; + if (bs || !fold_read_imm32(ir, s2, &f) || (f & UBFX_HI_HALF)) + return 0xFFFFFFFFu; + int width = (f >> 5) & 0x1F; + return (width > 0 && width < 32) ? (1u << width) - 1u : 0xFFFFFFFFu; + } + case TCCIR_OP_STORE: + /* A non-lval STORE into a TEMP is a register copy of the source (MOV, or + * a word store of the extended value when spilled): the same bits. */ + if (TCCIR_DECODE_VREG_TYPE(irop_get_vreg(dd)) == TCCIR_VREG_TYPE_TEMP && !dd.is_local && + !s1.is_lval && !irop_is_64bit(s1)) + return fold_maybe_set_bits(ctx, s1, d_idx, depth - 1, budget); + return 0xFFFFFFFFu; default: return 0xFFFFFFFFu; } } +/* `X AND M` / `UBFX X,#0,#w` is a copy of X when X cannot have a bit outside + * the mask. A TEMP read at the dest's width: its register (or spill slot) is + * always a whole word, so a narrow-typed TEMP read as a word is the same + * value. A VAR may live in a narrow home slot, so it keeps its own width. */ +static int fold_mask_is_copy(IRSSAOptCtx *ctx, IROperand dest, IROperand src, int i, uint32_t mask, + IROperand *out) +{ + if (src.tag != IROP_TAG_VREG || src.is_lval || irop_is_64bit(src) || irop_is_64bit(dest)) + return 0; + int budget = 4096; + if (fold_maybe_set_bits(ctx, src, i, 24, &budget) & ~mask) + return 0; + *out = src; + if (irop_get_btype(src) == irop_get_btype(dest)) + return 1; + if (TCCIR_DECODE_VREG_TYPE(irop_get_vreg(src)) != TCCIR_VREG_TYPE_TEMP || + irop_get_btype(dest) != IROP_BTYPE_INT32) + return 0; + out->btype = IROP_BTYPE_INT32; + return 1; +} + /* Barrel-shift-annotated op: fold only the all-constant case, peek-only — * materialising into an annotated src2 would drop the shift (the * var_to_param_forward LSL-drop bug class). */ @@ -312,6 +358,9 @@ OPT_GEN_SSA(fold_const_eval, -1) { case TCCIR_OP_ADD: result = (int64_t)((uint64_t)v1 + (uint64_t)v2); break; case TCCIR_OP_SUB: result = (int64_t)((uint64_t)v1 - (uint64_t)v2); break; case TCCIR_OP_MUL: result = (int64_t)((uint64_t)v1 * (uint64_t)v2); break; + /* Widening multiplies read only the low word of each operand. */ + case TCCIR_OP_UMULL: result = (int64_t)((uint64_t)(uint32_t)val1 * (uint32_t)val2); break; + case TCCIR_OP_SMULL: result = (int64_t)val1 * val2; break; case TCCIR_OP_AND: result = v1 & v2; break; case TCCIR_OP_OR: result = v1 | v2; break; case TCCIR_OP_XOR: result = v1 ^ v2; break; @@ -708,6 +757,10 @@ OPT_GEN_SSA(fold_identity_src2, -1) { case TCCIR_OP_ROR: is_identity = (v2 == 0); break; case TCCIR_OP_OR: is_identity = (v2 == 0); is_absorb = (v2 == -1); absorb_val = -1; break; case TCCIR_OP_MUL: is_identity = (v2 == 1); is_absorb = (v2 == 0); break; + /* A widening multiply by 0 is a 64-bit 0 (a u64 widened to u128 has a + * constant-zero high half, so zig.h's cross terms are these). */ + case TCCIR_OP_UMULL: + case TCCIR_OP_SMULL: is_absorb = (v2 == 0); break; case TCCIR_OP_AND: is_identity = (v2 == -1); is_absorb = (v2 == 0); break; case TCCIR_OP_DIV: case TCCIR_OP_UDIV: is_identity = (v2 == 1); break; @@ -773,6 +826,8 @@ OPT_GEN_SSA(fold_identity_src1, -1) { case TCCIR_OP_XOR: is_identity = (v1 == 0); break; case TCCIR_OP_OR: is_identity = (v1 == 0); is_absorb = (v1 == -1); absorb_val = -1; break; case TCCIR_OP_MUL: is_identity = (v1 == 1); is_absorb = (v1 == 0); break; + case TCCIR_OP_UMULL: + case TCCIR_OP_SMULL: is_absorb = (v1 == 0); break; case TCCIR_OP_AND: is_identity = (v1 == -1); is_absorb = (v1 == 0); break; case TCCIR_OP_SHL: case TCCIR_OP_SHR: @@ -816,6 +871,9 @@ OPT_GEN_SSA(fold_and_masked, TCCIR_OP_AND) { if ((maybe & ~(uint32_t)M) == 0 && irop_get_btype(src1) == irop_get_btype(dest)) REWRITE(.new_op = TCCIR_OP_ASSIGN); + IROperand w; + if (fold_mask_is_copy(ctx, dest, src1, i, (uint32_t)M, &w)) + REWRITE(.new_op = TCCIR_OP_ASSIGN, .src1 = w); } IRQuadCompact *dq = fold_single_dom_def(ctx, src1, i); if (!dq || tcc_ir_barrel_shift_at(ir, dq)) @@ -966,6 +1024,15 @@ OPT_GEN_SSA(fold_ubfx, TCCIR_OP_UBFX) { /* fold_bfx_value extracts from the source's LOW word; UBFX_HI_HALF does not * name that word. See source/opt/flat/fusion/shift64_extract_ubfx.c. */ GUARD(when(!(is_imm32(src2) && ((int32_t)imm(src2) & UBFX_HI_HALF)))); + /* A field starting at bit 0 that covers every bit the source can hold. */ + { + int32_t f = (int32_t)imm(src2); + int width = (f >> 5) & 0x1F; + IROperand w; + if ((f & 0x1F) == 0 && width > 0 && width < 32 && !tcc_ir_barrel_shift_at(ir, q) && + fold_mask_is_copy(ctx, dest, src1, i, (1u << width) - 1u, &w)) + REWRITE(.new_op = TCCIR_OP_ASSIGN, .src1 = w); + } GUARD(when(fold_bfx_value(ctx, i, 0, &result))); REWRITE(.new_op = TCCIR_OP_ASSIGN, .src1 = mk_imm_bt(result, irop_get_btype(dest))); @@ -1014,6 +1081,141 @@ OPT_GEN_SSA(fold_bitop1, -1) { .src1 = mk_imm_bt(result, irop_get_btype(dest))); } +/* ---- A 64-bit value taken apart and put back together ---------------------- + * + * A 64-bit value the frontend or SRA keeps as two words is joined by PACK64 + * (dest_lo = src1, dest_hi = src2) and taken apart by an ASSIGN to a word -- + * which keeps the low word, like the backend's copy -- and by `SHR #32`. The + * Zig C backend's zig_u128 is a struct of two uint64_t, so every one of its + * helpers repacks, and SRA splits each such doubleword into word VARs + * (source/opt/flat/memory/sra.c): the chains below are all that is left once + * SSA has renamed them, and they cancel. */ + +/* The single dominating def of `op` at instruction `at` if it is `op_want`. */ +static IRQuadCompact *fold_def_if(IRSSAOptCtx *ctx, IROperand op, int at, int op_want) +{ + IRQuadCompact *dq = fold_single_dom_def(ctx, op, at); + if (!dq || dq->op != op_want || tcc_ir_barrel_shift_at(ctx->ir, dq)) + return NULL; + return dq; +} + +static int fold_def_index(IRSSAOptCtx *ctx, IRQuadCompact *dq) +{ + return (int)(dq - ctx->ir->compact_instructions); +} + +/* A word operand that means the same thing wherever the PACK64 it came from + * is visible: a single-def TEMP or an inline constant. */ +static int fold_word_stable(IRSSAOptCtx *ctx, IROperand w) +{ + if (w.is_lval || w.is_local || w.is_llocal || w.is_sym || irop_is_64bit(w)) + return 0; + return w.tag == IROP_TAG_IMM32 || fold_single_def_temp(ctx, w); +} + +/* `x SHR #32` of a 64-bit x: the immediate test the rules below share. */ +static int fold_is_hi_shift(TCCIRState *ir, IRQuadCompact *q) +{ + if (q->op != TCCIR_OP_SHR) + return 0; + IROperand d = tcc_ir_op_get_dest(ir, q), c = tcc_ir_op_get_src2(ir, q); + return irop_get_btype(d) == IROP_BTYPE_INT64 && is_imm32(c) && !c.is_lval && c.u.imm32 == 32; +} + +/* Replace src1 of instruction i by `w`, keeping the use lists right. */ +static void fold_retarget_src1(IRSSAOptCtx *ctx, int i, IROperand old, IROperand w) +{ + opt_dsl_drop_use(ctx, old, i); + tcc_ir_set_src1(ctx->ir, i, w); + opt_dsl_add_use(ctx, w, i); +} + +/* `d <- (PACK64 a b) [ASSIGN]` into a word is `d <- a`. */ +static int fold_assign_lo_word(IRSSAOptCtx *ctx, int i) +{ + TCCIRState *ir = ctx->ir; + IRQuadCompact *q = &ir->compact_instructions[i]; + IROperand dest = tcc_ir_op_get_dest(ir, q); + IROperand src = tcc_ir_op_get_src1(ir, q); + if (!is_value_dest(dest) || irop_get_btype(dest) != IROP_BTYPE_INT32 || src.is_lval || + irop_get_btype(src) != IROP_BTYPE_INT64) + return 0; + IRQuadCompact *pq = fold_def_if(ctx, src, i, TCCIR_OP_PACK64); + if (!pq) + return 0; + IROperand lo = tcc_ir_op_get_src1(ir, pq); + if (!fold_word_stable(ctx, lo)) + return 0; + fold_retarget_src1(ctx, i, src, lo); + return 1; +} + +/* `(PACK64 a b) SHR #32` is `PACK64 b #0`. */ +static int fold_shr_pack64(IRSSAOptCtx *ctx, int i) +{ + TCCIRState *ir = ctx->ir; + IRQuadCompact *q = &ir->compact_instructions[i]; + if (!fold_is_hi_shift(ir, q)) + return 0; + IROperand src = tcc_ir_op_get_src1(ir, q); + if (src.is_lval) + return 0; + IRQuadCompact *pq = fold_def_if(ctx, src, i, TCCIR_OP_PACK64); + if (!pq) + return 0; + IROperand hi = tcc_ir_op_get_src2(ir, pq); + if (!fold_word_stable(ctx, hi)) + return 0; + q->op = TCCIR_OP_PACK64; + fold_retarget_src1(ctx, i, src, hi); + tcc_ir_set_src2(ir, i, irop_make_imm32(-1, 0, IROP_BTYPE_INT32)); + return 1; +} + +/* `PACK64 (x [ASSIGN] to a word) ((x SHR #32) [ASSIGN] to a word)` is x, and + * a PACK64 of two constants -- which cprop can leave behind after ssa:sccp has + * run -- is their doubleword. */ +static int fold_pack64_of_halves(IRSSAOptCtx *ctx, int i) +{ + TCCIRState *ir = ctx->ir; + IRQuadCompact *q = &ir->compact_instructions[i]; + IROperand dest = tcc_ir_op_get_dest(ir, q); + IROperand a = tcc_ir_op_get_src1(ir, q), b = tcc_ir_op_get_src2(ir, q); + if (!is_value_dest(dest) || irop_get_btype(dest) != IROP_BTYPE_INT64 || a.is_lval || b.is_lval) + return 0; + int32_t ca, cb; + if (fold_read_imm32(ir, a, &ca) && fold_read_imm32(ir, b, &cb)) + { + int64_t v = (int64_t)(((uint64_t)(uint32_t)cb << 32) | (uint32_t)ca); + IROperand imm = v == (int64_t)(int32_t)v ? irop_make_imm32(-1, (int32_t)v, IROP_BTYPE_INT64) + : irop_make_i64(-1, tcc_ir_pool_add_i64(ir, v), IROP_BTYPE_INT64); + imm.is_unsigned = dest.is_unsigned; + q->op = TCCIR_OP_ASSIGN; + tcc_ir_set_src1(ir, i, imm); + tcc_ir_set_src2(ir, i, IROP_NONE); + return 1; + } + IRQuadCompact *da = fold_def_if(ctx, a, i, TCCIR_OP_ASSIGN); + IRQuadCompact *db = fold_def_if(ctx, b, i, TCCIR_OP_ASSIGN); + if (!da || !db || irop_get_btype(tcc_ir_op_get_dest(ir, da)) != IROP_BTYPE_INT32 || + irop_get_btype(tcc_ir_op_get_dest(ir, db)) != IROP_BTYPE_INT32) + return 0; + IROperand x = tcc_ir_op_get_src1(ir, da), y = tcc_ir_op_get_src1(ir, db); + if (x.is_lval || y.is_lval || irop_get_btype(x) != IROP_BTYPE_INT64 || !fold_single_def_temp(ctx, x)) + return 0; + IRQuadCompact *sq = fold_def_if(ctx, y, fold_def_index(ctx, db), TCCIR_OP_SHR); + if (!sq || !fold_is_hi_shift(ir, sq)) + return 0; + IROperand sx = tcc_ir_op_get_src1(ir, sq); + if (sx.is_lval || irop_get_vreg(sx) != irop_get_vreg(x)) + return 0; + opt_dsl_drop_use(ctx, b, i); + q->op = TCCIR_OP_ASSIGN; + fold_retarget_src1(ctx, i, a, x); + return 1; +} + /* Rule order matches the original monolith. */ static const OptDslSSARule fold_binary_rules[] = { opt_dsl_dispatch_fold_symref_addend, @@ -1032,6 +1234,7 @@ static const OptDslSSARule fold_binary_rules[] = { opt_dsl_dispatch_fold_identity_src2, opt_dsl_dispatch_fold_double_neg, opt_dsl_dispatch_fold_identity_src1, + fold_shr_pack64, }; /* A barrel-annotated op folds only all-constant (or nothing at all). */ @@ -1073,9 +1276,13 @@ static const IRSSAOptGen fold_gens[] = { OPT_GEN_ENTRY(fold_ubfx, TCCIR_OP_UBFX), OPT_GEN_ENTRY(fold_sbfx, TCCIR_OP_SBFX), { TCCIR_OP_SELECT, fold_select_entry, "fold_select" }, + { TCCIR_OP_ASSIGN, fold_assign_lo_word, "fold_assign_lo_word" }, + { TCCIR_OP_PACK64, fold_pack64_of_halves, "fold_pack64_halves" }, { TCCIR_OP_ADD, fold_binary, "fold_add" }, { TCCIR_OP_SUB, fold_binary, "fold_sub" }, { TCCIR_OP_MUL, fold_binary, "fold_mul" }, + { TCCIR_OP_UMULL, fold_binary, "fold_umull" }, + { TCCIR_OP_SMULL, fold_binary, "fold_smull" }, { TCCIR_OP_DIV, fold_binary, "fold_div" }, { TCCIR_OP_UDIV, fold_binary, "fold_udiv" }, { TCCIR_OP_IMOD, fold_binary, "fold_mod" }, diff --git a/source/opt/ssa/scalar/narrow.c b/source/opt/ssa/scalar/narrow.c index daab86c5e..e21cc7f10 100644 --- a/source/opt/ssa/scalar/narrow.c +++ b/source/opt/ssa/scalar/narrow.c @@ -15,18 +15,9 @@ #include "opt_dsl_ssa.h" #include "opt/ssa/narrow.h" #include "opt/ssa/ssa_opt_helpers.h" +#include "opt_utils.h" #include -/* Call-param helpers from ir/opt_utils.c, declared locally: opt_utils.h's - * is_power_of_2() prototype conflicts with the static inline in - * ssa_opt_helpers.h, so the header cannot be included here. */ -int ir_opt_get_call_param_operand(struct TCCIRState *ir, int call_idx, - int param_idx, IROperand *out); -int ir_opt_get_call_param_index(struct TCCIRState *ir, int call_idx, - int param_idx); -void ir_opt_nop_call_params(struct TCCIRState *ir, int call_idx); -int change_callee_sym(struct TCCIRState *ir, int instr_idx, const char *new_name, int ret_btype); - static IROperand narrow_copy_src(IROperand dest, int32_t vr) { IROperand s = dest; @@ -154,6 +145,54 @@ OPT_GEN_SSA(narrow_and, TCCIR_OP_AND) { return 0; } +/* UBFX(x, #0, #w) of an x that already fits in w bits is a copy: a narrower + * or equal UBFX, an AND with a mask below 1<> 5) & 63; + GUARD( + when(!(param & UBFX_HI_HALF)); + and(lsb == 0 && width > 0 && width < 32); + and_not(irop_is_64bit(dest) || irop_is_64bit(src1) || irop_is_64bit(pdest))); + + int fits = 0; + if (pop == TCCIR_OP_UBFX && is_imm32(psrc2)) { + int32_t pparam = (int32_t)imm(psrc2); + int pwidth = (pparam >> 5) & 63; + fits = !(pparam & UBFX_HI_HALF) && pwidth > 0 && pwidth <= width; + } else if (pop == TCCIR_OP_AND && is_imm32(psrc2) && !irop_is_64bit(psrc1)) { + fits = (uint32_t)imm(psrc2) < (1u << width); + } else if ((pop == TCCIR_OP_LOAD || pop == TCCIR_OP_ASSIGN) && psrc1.is_lval && !psrc1.is_llocal && + psrc1.is_unsigned) { + int bt = irop_get_btype(psrc1); + fits = (bt == IROP_BTYPE_INT8 && width >= 8) || (bt == IROP_BTYPE_INT16 && width >= 16); + } else if (pop == TCCIR_OP_SETIF || pop == TCCIR_OP_BOOL_OR || pop == TCCIR_OP_BOOL_AND) { + fits = 1; + } else if (pop == TCCIR_OP_FUNCCALLVAL && irop_get_tag(psrc1) == IROP_TAG_SYMREF) { + /* The AAPCS has the callee extend a narrow result to a word -- tcc's + * return path does, and gcc and LLVM callers rely on it -- so a direct + * call declared to return bool or an unsigned byte or halfword already + * fits. (A narrow local that receives the result re-extends it on every + * read otherwise.) */ + Sym *fs = irop_get_sym_ex(ir, psrc1); + if (fs && (fs->type.t & VT_BTYPE) == VT_FUNC && fs->type.ref) { + int rt = fs->type.ref->type.t, rbt = rt & VT_BTYPE; + int rw = rbt == VT_BOOL ? 1 : rbt == VT_BYTE ? 8 : rbt == VT_SHORT ? 16 : 0; + fits = rw && (rbt == VT_BOOL || (rt & VT_UNSIGNED)) && rw <= width; + } + } + GUARD(when(fits)); + + REWRITE(.new_op = TCCIR_OP_ASSIGN, .src1 = narrow_copy_src(dest, vreg(src1))); +} + /* (x SHR #n) fed into UBFX(#lsb,#w) -> UBFX(x, #(lsb+n), #w) when the SHR dies. */ OPT_GEN_SSA(narrow_ubfx, TCCIR_OP_UBFX) { PATTERN(.constraints = { .src2 = IR_CONSTRAINT_IMM }); @@ -181,6 +220,7 @@ OPT_GEN_SSA(narrow_ubfx, TCCIR_OP_UBFX) { static const IRSSAOptGen narrow_gens[] = { OPT_GEN_ENTRY(narrow_shr, TCCIR_OP_SHR), OPT_GEN_ENTRY(narrow_and, TCCIR_OP_AND), + OPT_GEN_ENTRY(narrow_ubfx_noop, TCCIR_OP_UBFX), OPT_GEN_ENTRY(narrow_ubfx, TCCIR_OP_UBFX), }; diff --git a/source/opt/ssa/scalar/reassoc.c b/source/opt/ssa/scalar/reassoc.c index 8f6a9f4fb..4087a1723 100644 --- a/source/opt/ssa/scalar/reassoc.c +++ b/source/opt/ssa/scalar/reassoc.c @@ -104,6 +104,9 @@ OPT_GEN_SSA(reassoc_bin, 0) { and_not(psrc1.is_local); and_not(psrc1.is_llocal); and_not(tcc_ir_barrel_shift_at(ir, &ir->compact_instructions[pidx])); + /* A signed ADD/SUB is taken not to overflow; one that absorbs an unsigned, + * wrapping, inner ADD/SUB is not (pr55137, see flat add_reassoc). */ + and_not((q->op == TCCIR_OP_ADD || q->op == TCCIR_OP_SUB) && pdest.is_unsigned && !dest.is_unsigned); and(reassoc_combine(q->op, pop, (int32_t)psrc2.u.imm32, (int32_t)src2.u.imm32, &new_op, &combined))); RETIRE_PAIR(psrc1, 0); diff --git a/source/opt/ssa/scalar/sccp.c b/source/opt/ssa/scalar/sccp.c index 947ab7f67..8210d0791 100644 --- a/source/opt/ssa/scalar/sccp.c +++ b/source/opt/ssa/scalar/sccp.c @@ -145,6 +145,25 @@ static int sccp_get_operand_value(SCCPState *s, IROperand op, int64_t *out) static int sccp_resolve_stack_load(SCCPState *s, int soff, int load_btype, int instr_idx, int64_t *out, int *dep_pos); +/* The frame slot a TEMP address names, or INT_MIN. `&VAR` resolves to the + * VAR's spill placeholder, an offset it shares with unrelated anonymous + * StackLocs (ssa_opt.h): matched as a slot, a memset through `&t0` read as a + * store of 0 to the by-value struct parameter spilled at the same offset, and + * Zig's InternPool.AnalUnit.wrap lost its payload. Here it is no slot. */ +static int sccp_lea_slot(IRSSAOptCtx *ctx, int32_t vr) +{ + int32_t base_var; + int off = ssa_opt_resolve_lea_stackloc_ex(ctx, vr, &base_var); + return base_var >= 0 ? INT_MIN : off; +} + +static int sccp_indirect_slot(IRSSAOptCtx *ctx, const IRQuadCompact *q, int side) +{ + int32_t base_var; + int off = ssa_opt_indirect_stack_offset_ex(ctx, q, side, &base_var); + return base_var >= 0 ? INT_MIN : off; +} + static int sccp_get_store_src_value(SCCPState *s, IROperand src, int64_t *out, int *src_pos_out) { @@ -186,7 +205,7 @@ static int sccp_get_store_src_value_ex(SCCPState *s, IROperand src, !src.is_local) { int32_t svr = irop_get_vreg(src); if (svr >= 0 && TCCIR_DECODE_VREG_TYPE(svr) == TCCIR_VREG_TYPE_TEMP) - load_off = ssa_opt_resolve_lea_stackloc(s->ctx, svr); + load_off = sccp_lea_slot(s->ctx, svr); } if (load_off != INT_MIN) { @@ -225,16 +244,25 @@ static int sccp_store_target_off(IRSSAOptCtx *ctx, IRQuadCompact *sq, IROperand sd = tcc_ir_op_get_dest(ir, sq); if (sq->op == TCCIR_OP_STORE) { if (sd.tag == IROP_TAG_STACKOFF && sd.is_lval && sd.is_local) { + /* A store to a VAR carries the VAR's spill placeholder, not a frame + * slot: every load site already refuses to match a VAR operand by its + * offset, and the store side must too. Zig's CaptureValue.wrap zeroes + * its result local (`V8 <-- #0`) and the placeholder equalled the slot + * the by-value parameter is spilled to, so the load of a0.payload in + * case 0 folded to 0. */ + int32_t dvr = irop_get_vreg(sd); + if (dvr >= 0 && TCCIR_DECODE_VREG_TYPE(dvr) == TCCIR_VREG_TYPE_VAR) + return INT_MIN; if (out_btype) *out_btype = irop_get_btype(sd); return irop_get_stack_offset(sd); } - int off = ssa_opt_indirect_stack_offset(ctx, sq, SSA_OPT_INDIRECT_DEST); + int off = sccp_indirect_slot(ctx, sq, SSA_OPT_INDIRECT_DEST); if (off != INT_MIN && out_btype) *out_btype = irop_get_btype(sd); return off; } if (sq->op == TCCIR_OP_STORE_INDEXED) { - int off = ssa_opt_indirect_stack_offset(ctx, sq, SSA_OPT_INDIRECT_DEST); + int off = sccp_indirect_slot(ctx, sq, SSA_OPT_INDIRECT_DEST); if (off != INT_MIN && out_btype) *out_btype = irop_get_btype(sd); return off; @@ -251,7 +279,7 @@ static int sccp_store_may_escape(IRSSAOptCtx *ctx, IRQuadCompact *sq) if (sd.tag == IROP_TAG_STACKOFF && sd.is_lval && sd.is_local) return 0; if (sd.tag == IROP_TAG_VREG && sd.is_lval && !sd.is_local) { - int off = ssa_opt_indirect_stack_offset(ctx, sq, SSA_OPT_INDIRECT_DEST); + int off = sccp_indirect_slot(ctx, sq, SSA_OPT_INDIRECT_DEST); if (off != INT_MIN) return 0; } @@ -556,7 +584,7 @@ static int sccp_store_indexed_base_off(IRSSAOptCtx *ctx, IRQuadCompact *q) int32_t bvr = irop_get_vreg(base); if (bvr < 0 || TCCIR_DECODE_VREG_TYPE(bvr) != TCCIR_VREG_TYPE_TEMP) return INT_MIN; - return ssa_opt_resolve_lea_stackloc(ctx, bvr); + return sccp_lea_slot(ctx, bvr); } /* Any store-to-load path is a subset of the linear IR range, so scanning it is sound. */ @@ -623,12 +651,15 @@ static int sccp_indexed_store_base_off(IRSSAOptCtx *ctx, IRQuadCompact *q) if (base.tag == IROP_TAG_VREG && !base.is_local) { int32_t bvr = irop_get_vreg(base); if (bvr >= 0 && TCCIR_DECODE_VREG_TYPE(bvr) == TCCIR_VREG_TYPE_TEMP) - return ssa_opt_resolve_lea_stackloc(ctx, bvr); + return sccp_lea_slot(ctx, bvr); } return INT_MIN; } -/* Narrower than sccp_no_aliasing_between: calls are not barriers here. */ +/* Narrower than sccp_no_aliasing_between: an unresolved pointer store only + * clobbers when it may escape, and a call only when the slot's address + * escaped. Without the call check, `struct d x = {0}; f(&x); if (!x.p)` + * kept the initializer's 0 across f (ctags lregex choose_backend). */ static int sccp_resolved_stack_write_between(SCCPState *s, int store_idx, int load_idx, int soff, int load_btype) { @@ -640,6 +671,9 @@ static int sccp_resolved_stack_write_between(SCCPState *s, int store_idx, int lo IRQuadCompact *q = &ir->compact_instructions[i]; if (sccp_var_def_clobbers_slot(s, q, load_lo, load_hi)) return 1; + if ((q->op == TCCIR_OP_FUNCCALLVOID || q->op == TCCIR_OP_FUNCCALLVAL) && + sccp_slot_addr_escapes(s, load_lo, load_hi)) + return 1; if (q->op != TCCIR_OP_STORE && q->op != TCCIR_OP_STORE_INDEXED && q->op != TCCIR_OP_STORE_POSTINC) continue; @@ -769,7 +803,7 @@ static int sccp_loop_writes_slot_between(SCCPState *s, int from_idx, int to_idx, else { int32_t pvr = irop_get_vreg(p); if (pvr >= 0) - aoff = ssa_opt_resolve_lea_stackloc(s->ctx, pvr); + aoff = sccp_lea_slot(s->ctx, pvr); } const int SCCP_OBJ_BOUND = 4096; if (aoff != INT_MIN && aoff <= load_lo && load_lo - aoff < SCCP_OBJ_BOUND) @@ -1014,7 +1048,7 @@ static int sccp_get_operand_value_ex(SCCPState *s, IROperand op, if (op.tag == IROP_TAG_VREG && op.is_lval && !op.is_local) { int32_t tvr = irop_get_vreg(op); if (tvr >= 0 && TCCIR_DECODE_VREG_TYPE(tvr) == TCCIR_VREG_TYPE_TEMP) { - int load_off = ssa_opt_resolve_lea_stackloc(s->ctx, tvr); + int load_off = sccp_lea_slot(s->ctx, tvr); if (load_off != INT_MIN) { int dep_pos = -1; int st = sccp_resolve_stack_load(s, load_off, irop_get_btype(op), @@ -1072,6 +1106,11 @@ static int sccp_eval_binary(int op, int64_t v1, int64_t v2, int64_t *result, case TCCIR_OP_AND: *result = v1 & v2; break; case TCCIR_OP_OR: *result = v1 | v2; break; case TCCIR_OP_XOR: *result = v1 ^ v2; break; + case TCCIR_OP_PACK64: /* dest_lo = src1, dest_hi = src2 */ + if (!is_64) + return 0; + *result = (int64_t)(((uint64_t)(uint32_t)v2 << 32) | (uint32_t)v1); + break; case TCCIR_OP_SHL: { int mask = is_64 ? 63 : 31; *result = (int64_t)((uint64_t)v1 << (v2 & mask)); @@ -1307,6 +1346,10 @@ static void sccp_visit_instr(SCCPState *s, int idx) int64_t val; int st = sccp_get_operand_value(s, src, &val); int changed = 0; + /* A word dest keeps the low word of a 64-bit value, as the backend's + * copy does: the lattice holds 32-bit values sign-extended. */ + if (st == SCCP_CONST && !is_64) + val = (int64_t)(int32_t)(uint32_t)val; if (st == SCCP_CONST) changed = sccp_meet(dest_cell, val); else if (st == SCCP_BOTTOM) @@ -1387,7 +1430,7 @@ static void sccp_visit_instr(SCCPState *s, int idx) } if (src.tag == IROP_TAG_VREG && src.is_lval && !src.is_local) { - int eff_off = ssa_opt_indirect_stack_offset(s->ctx, q, SSA_OPT_INDIRECT_SRC1); + int eff_off = sccp_indirect_slot(s->ctx, q, SSA_OPT_INDIRECT_SRC1); if (eff_off != INT_MIN) { int64_t sval = 0; int dep_pos = -1; @@ -1804,7 +1847,7 @@ static int sccp_apply(SCCPState *s) if (src.tag == IROP_TAG_VREG && src.is_lval && !src.is_local) { int32_t tvr = irop_get_vreg(src); if (tvr >= 0 && TCCIR_DECODE_VREG_TYPE(tvr) == TCCIR_VREG_TYPE_TEMP) { - int load_off = ssa_opt_resolve_lea_stackloc(s->ctx, tvr); + int load_off = sccp_lea_slot(s->ctx, tvr); if (load_off != INT_MIN) { int dep_pos = -1; int st = sccp_resolve_stack_load(s, load_off, irop_get_btype(src), diff --git a/source/opt/ssa/scalar/setif_or_taut.c b/source/opt/ssa/scalar/setif_or_taut.c index fd5837369..3ebd40965 100644 --- a/source/opt/ssa/scalar/setif_or_taut.c +++ b/source/opt/ssa/scalar/setif_or_taut.c @@ -72,8 +72,9 @@ static int so_vreg_ever_written(TCCIRState *ir, int32_t vr) IRQuadCompact *q = &ir->compact_instructions[i]; if (q->op == TCCIR_OP_NOP || !irop_config[q->op].has_dest) continue; + /* A PARAM written as itself is a STACKOFF lvalue: a write all the same. */ IROperand d = tcc_ir_op_get_dest(ir, q); - if (!d.is_lval && irop_get_vreg(d) == vr) + if (irop_get_vreg(d) == vr && irop_dest_defines_vreg(d)) return 1; } return 0; diff --git a/source/opt/ssa/scalar/strength.c b/source/opt/ssa/scalar/strength.c index 042b02ca2..a82cce3dd 100644 --- a/source/opt/ssa/scalar/strength.c +++ b/source/opt/ssa/scalar/strength.c @@ -20,7 +20,7 @@ * UDIV x, 2^n -> SHR x, n * UMOD x, 2^n -> AND x, 2^n - 1 * Multi-instruction patterns (2^n+-1) stay in the pre-SSA pass. - * is_imm32 / is_power_of_2 come from ssa_opt_helpers.h. */ + * is_imm32 / is_power_of_2_shift come from ssa_opt_helpers.h. */ static int mul_pick_imm(IROperand src1, IROperand src2, IROperand *val, IROperand *imm) @@ -36,7 +36,7 @@ OPT_GEN_SSA(sr_mul, TCCIR_OP_MUL) { PATTERN(.constraints = { .dest = IR_CONSTRAINT_ANY }); GUARD( when(mul_pick_imm(src1, src2, &val_op, &imm_op)); - and(is_power_of_2((uint32_t)imm_op.u.imm32, &shift))); + and(is_power_of_2_shift((uint32_t)imm_op.u.imm32, &shift))); REWRITE( .new_op = TCCIR_OP_SHL, .src1 = val_op, @@ -48,7 +48,7 @@ OPT_GEN_SSA(sr_udiv, TCCIR_OP_UDIV) { PATTERN(.constraints = { .src2 = IR_CONSTRAINT_IMM }); GUARD( when(is_imm32(src2)); - and(is_power_of_2((uint32_t)src2.u.imm32, &shift))); + and(is_power_of_2_shift((uint32_t)src2.u.imm32, &shift))); REWRITE( .new_op = TCCIR_OP_SHR, .src2 = mk_imm(shift)); @@ -59,7 +59,7 @@ OPT_GEN_SSA(sr_umod, TCCIR_OP_UMOD) { PATTERN(.constraints = { .src2 = IR_CONSTRAINT_IMM }); GUARD( when(is_imm32(src2)); - and(is_power_of_2((uint32_t)src2.u.imm32, &shift))); + and(is_power_of_2_shift((uint32_t)src2.u.imm32, &shift))); REWRITE( .new_op = TCCIR_OP_AND, .src2 = mk_imm((uint32_t)src2.u.imm32 - 1u)); diff --git a/source/opt/util/const_eval.c b/source/opt/util/const_eval.c index cb561deb9..f4cb91cbe 100644 --- a/source/opt/util/const_eval.c +++ b/source/opt/util/const_eval.c @@ -47,6 +47,10 @@ int ir_opt_eval_const_u64(TCCIRState *ir, IROperand op, int use_idx, uint64_t *o if (vr < 0) return 0; + /* A volatile variable's initialiser is not the value a later read sees. */ + if (tcc_ir_operand_names_volatile_var(ir, op)) + return 0; + if (ir_opt_vreg_address_taken_between(ir, vr, 0, use_idx)) return 0; @@ -212,6 +216,10 @@ int ir_opt_eval_const_string(TCCIRState *ir, IROperand op, int use_idx, const ch if (vr < 0) return 0; + /* A volatile variable's initialiser is not the value a later read sees. */ + if (tcc_ir_operand_names_volatile_var(ir, op)) + return 0; + if (ir_opt_vreg_address_taken_between(ir, vr, 0, use_idx)) return 0; @@ -271,8 +279,8 @@ const char *ir_opt_get_constant_string_from_symref(TCCIRState *ir, IROperand op) return NULL; sym = symref->sym; - if (!sym) - return NULL; + if (!sym || sym->a.tentative) + return NULL; /* a tentative definition's bytes are not its value yet */ esym = elfsym(sym); if (!esym) diff --git a/source/opt/util/expr_equal.c b/source/opt/util/expr_equal.c index ea061ec70..a050ff2c1 100644 --- a/source/opt/util/expr_equal.c +++ b/source/opt/util/expr_equal.c @@ -60,6 +60,8 @@ int ir_opt_nonvreg_expr_equal(TCCIRState *ir, IROperand a, IROperand b) static int ir_opt_setif_cmp_operand_equal(TCCIRState *ir, IROperand a, IROperand b, int a_use_idx, int b_use_idx, int depth) { + if (tcc_ir_operand_names_volatile_var(ir, a) || tcc_ir_operand_names_volatile_var(ir, b)) + return 0; if (ir_opt_pure_expr_equal_impl(ir, a, a_use_idx, b, b_use_idx, depth + 1)) return 1; @@ -177,6 +179,10 @@ int ir_opt_pure_def_equal(TCCIRState *ir, int a_def_idx, int b_def_idx, int dept if (qa->op != qb->op) return 0; + /* A volatile access is not a pure definition: each one is its own read. */ + if (tcc_ir_instr_access_is_volatile(ir, qa) || tcc_ir_instr_access_is_volatile(ir, qb)) + return 0; + /* Without this gate a STORE or call between two structurally-identical `*p` loads would fold a stale read. */ if (a_def_idx != b_def_idx && (ir_opt_pure_def_has_memory_read(ir, qa) || @@ -371,6 +377,12 @@ static int ir_opt_pure_expr_equal_impl(TCCIRState *ir, IROperand a, int a_use_id return irop_get_imm64_ex(ir, a) == irop_get_imm64_ex(ir, b); } + /* Two reads of a volatile variable -- even the same one -- may differ, and + * following its definition would equate it with its initialiser: `volatile + * int a = K, b = K; (a < 0) != (b < 0)` folded to 0 and dropped b's read. */ + if (tcc_ir_operand_names_volatile_var(ir, a) || tcc_ir_operand_names_volatile_var(ir, b)) + return 0; + a_tag = irop_get_tag(a); b_tag = irop_get_tag(b); diff --git a/source/opt/util/volatile_access.c b/source/opt/util/volatile_access.c index 360e84ad5..ab0f57ee6 100644 --- a/source/opt/util/volatile_access.c +++ b/source/opt/util/volatile_access.c @@ -19,6 +19,18 @@ * An ordinary ALU op can also embed a deref operand, so every source is * checked — but only lvalue ones, so a function that touches volatile memory * once does not turn its whole body unremovable. */ +/* Does `op` name a volatile variable (a VAR/PARAM whose live interval carries + * is_volatile)? Its value is whatever the next read returns: two reads are + * never provably equal and its initialiser is not its value. */ +int tcc_ir_operand_names_volatile_var(const TCCIRState *ir, IROperand op) +{ + int32_t v = irop_get_vreg(op); + if (v < 0 || !tcc_ir_vreg_is_valid((TCCIRState *)ir, v)) + return 0; + IRLiveInterval *iv = tcc_ir_get_live_interval((TCCIRState *)ir, v); + return iv && iv->is_volatile; +} + int tcc_ir_instr_access_is_volatile(const TCCIRState *ir, const IRQuadCompact *q) { if (!ir->func_has_volatile_access) diff --git a/source/opt/util/vreg_def_use.c b/source/opt/util/vreg_def_use.c index 8379ba2a1..d2fc5e6ef 100644 --- a/source/opt/util/vreg_def_use.c +++ b/source/opt/util/vreg_def_use.c @@ -30,6 +30,16 @@ int tcc_ir_find_defining_instruction(TCCIRState *ir, int32_t vreg, int before_id return -1; } +int tcc_ir_vreg_coalesced(TCCIRState *ir, int32_t vreg) +{ + if (!ir || vreg < 0) + return 0; + for (int k = 0; k < ir->ls.next_interval_index; k++) + if (ir->ls.intervals[k].vreg == (uint32_t)vreg) + return ir->ls.intervals[k].co_member != 0; + return 0; +} + int tcc_ir_vreg_has_single_use(TCCIRState *ir, int32_t vreg, int exclude_idx) { if (!ir || vreg < 0) @@ -53,7 +63,7 @@ int tcc_ir_vreg_has_single_use(TCCIRState *ir, int32_t vreg, int exclude_idx) * counting it: counting would newly admit the accumulator-only case, which * today bails (use_count 0). Same blind spot as the ssa:sccp phi * materialization fixed for fuzz seeds volatile:82433 / bitfield:88932. */ - if (q->op == TCCIR_OP_MLA && + if (tcc_ir_op_is_mac(q->op) && irop_get_vreg(tcc_ir_op_get_accum(ir, q)) == vreg) return 0; diff --git a/source/opt/util/vreg_query.c b/source/opt/util/vreg_query.c index 4c36ec295..eaa16f423 100644 --- a/source/opt/util/vreg_query.c +++ b/source/opt/util/vreg_query.c @@ -78,7 +78,7 @@ int tcc_ir_vreg_has_multi_def(TCCIRState *ir, int32_t vreg) /* MLA's 4th (accumulator) pool operand is a real vreg USE invisible to has_src1/2; use-scans must count it or the def is wrongly killed. Returns -1 if none. */ int32_t ir_opt_mla_accum_vreg(const TCCIRState *ir, const IRQuadCompact *q) { - if (q->op != TCCIR_OP_MLA) + if (!tcc_ir_op_is_mac(q->op)) return -1; return irop_get_vreg(tcc_ir_op_get_accum(ir, q)); } diff --git a/source/support/Makefile b/source/support/Makefile index 2b3b1afdb..fb7e25336 100644 --- a/source/support/Makefile +++ b/source/support/Makefile @@ -16,7 +16,7 @@ SUPPORT_HDRS = \ source/support/tccdbgenv.h \ source/support/tccdebug.h -SUPPORT_OBJ = $(patsubst %.c,$(X)%.o,$(SUPPORT_SRC)) +SUPPORT_OBJ = $(call lib-objs,support,$(SUPPORT_SRC)) SUPPORT_LIB = $(X)source/support/libsupport.a $(X)source/support/%.o: source/support/%.c $(SUPPORT_HDRS) $(TCC_CORE_DEPS) diff --git a/source/unity/arm_01.c b/source/unity/arm_01.c new file mode 100644 index 000000000..6f3819851 --- /dev/null +++ b/source/unity/arm_01.c @@ -0,0 +1,5 @@ +/* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */ +#include "../backend/arch/arm/arm.c" +#undef USING_GLOBALS +#include "../backend/arch/arm/ssa_opt_arm.c" +#undef USING_GLOBALS diff --git a/source/unity/arm_02.c b/source/unity/arm_02.c new file mode 100644 index 000000000..2b536b769 --- /dev/null +++ b/source/unity/arm_02.c @@ -0,0 +1,4 @@ +/* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */ +#include "../backend/arch/arm/arm_aapcs.c" +#include "../backend/arch/arm/arm_regalloc.c" +#include "../backend/arch/arm/arm-link.c" diff --git a/source/unity/frontend_01.c b/source/unity/frontend_01.c new file mode 100644 index 000000000..59f76fa64 --- /dev/null +++ b/source/unity/frontend_01.c @@ -0,0 +1,11 @@ +/* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */ +#include "../frontend/tccasm.c" +#undef ASM_MACRO_MAX_ARGS +#undef USING_GLOBALS +#include "../frontend/gen/builtin/call.c" +#undef CONST_LOOP_INLINE_BUDGET +#include "../frontend/gen/builtin/chk.c" +#undef FIND_LOCAL_OBJSIZE +#include "../frontend/gen/builtin/fp2.c" +#include "../frontend/gen/core/predicates.c" +#undef TCC_IS_NATIVE_387 diff --git a/source/unity/frontend_02.c b/source/unity/frontend_02.c new file mode 100644 index 000000000..9f542ec2d --- /dev/null +++ b/source/unity/frontend_02.c @@ -0,0 +1,25 @@ +/* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */ +#include "../frontend/gen/builtin/fp.c" +#include "../frontend/gen/builtin/misc.c" +#undef GCC_TYPE_CLASS_ARRAY +#undef GCC_TYPE_CLASS_COMPLEX +#undef GCC_TYPE_CLASS_FUNCTION +#undef GCC_TYPE_CLASS_INTEGER +#undef GCC_TYPE_CLASS_POINTER +#undef GCC_TYPE_CLASS_REAL +#undef GCC_TYPE_CLASS_STRUCT +#undef GCC_TYPE_CLASS_UNION +#undef GCC_TYPE_CLASS_VECTOR +#undef GCC_TYPE_CLASS_VOID +#include "../frontend/gen/builtin/overflow.c" +#include "../frontend/gen/builtin/simd.c" +#include "../frontend/gen/builtin/string.c" +#include "../frontend/gen/core/state.c" +#include "../frontend/gen/core/suppress.c" +#include "../frontend/gen/decl/attribute.c" +#include "../frontend/gen/decl/btype.c" +#include "../frontend/gen/decl/declarator.c" +#include "../frontend/gen/decl/predef_protos.c" +#include "../frontend/gen/expr/atomic.c" +#undef __ATOMIC_SEQ_CST_VALUE +#include "../frontend/gen/value/strlit_pool.c" diff --git a/source/unity/frontend_03.c b/source/unity/frontend_03.c new file mode 100644 index 000000000..a40e72a0a --- /dev/null +++ b/source/unity/frontend_03.c @@ -0,0 +1,12 @@ +/* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */ +#include "../frontend/gen/decl/struct.c" +#include "../frontend/gen/expr/cond.c" +#include "../frontend/gen/expr/indir.c" +#include "../frontend/gen/expr/infix.c" +#undef precedence +#include "../frontend/gen/expr/primary.c" +#include "../frontend/gen/expr/unary.c" +#include "../frontend/gen/init/alloc.c" +#include "../frontend/gen/op/float.c" +#undef gen_negf +#include "../frontend/gen/sym/elfsym.c" diff --git a/source/unity/frontend_04.c b/source/unity/frontend_04.c new file mode 100644 index 000000000..2e3b74159 --- /dev/null +++ b/source/unity/frontend_04.c @@ -0,0 +1,15 @@ +/* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */ +#include "../frontend/gen/inline/analysis.c" +#undef AUTO_INLINE_NONSTATIC_STRUCT_MAX_TOKENS +#include "../frontend/gen/inline/const_eval.c" +#include "../frontend/gen/inline/emit.c" +#undef CALLED_ONCE_LARGE_BODY +#undef GC_MARK +#include "../frontend/gen/nested/nested.c" +#include "../frontend/gen/op/complex.c" +#undef PUSH_COMP +#undef PUSH_FCOMP +#undef STORE_COMP +#undef STORE_FCOMP +#include "../frontend/gen/op/fold_math.c" +#undef NUM_FOLDABLE_MATH_FUNCS diff --git a/source/unity/frontend_05.c b/source/unity/frontend_05.c new file mode 100644 index 000000000..67c98165d --- /dev/null +++ b/source/unity/frontend_05.c @@ -0,0 +1,16 @@ +/* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */ +#include "../frontend/gen/op/int.c" +#include "../frontend/gen/op/op.c" +#include "../frontend/gen/op/vector.c" +#undef VEC_EMIT_RANGE_MAX +#undef VEC_RECIPE_MAX +#include "../frontend/gen/stmt/block.c" +#include "../frontend/gen/stmt/cleanup.c" +#include "../frontend/gen/stmt/ret.c" +#include "../frontend/gen/stmt/switch.c" +#include "../frontend/gen/store/struct_copy.c" +#include "../frontend/gen/store/vstore.c" +#undef IS_GLOBAL_LVAL +#undef IS_LOCAL_LVAL +#undef IS_REG_DEREF_LVAL +#include "../frontend/gen/sym/attr_merge.c" diff --git a/source/unity/frontend_06.c b/source/unity/frontend_06.c new file mode 100644 index 000000000..936d29269 --- /dev/null +++ b/source/unity/frontend_06.c @@ -0,0 +1,10 @@ +/* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */ +#include "../frontend/gen/sym/symtab.c" +#include "../frontend/gen/type/assign_check.c" +#include "../frontend/gen/type/cast.c" +#undef ALLOW_SUBTYPE_ACCESS +#undef gen_cvt_itof1 +#include "../frontend/gen/type/size.c" +#include "../frontend/gen/value/load.c" +#include "../frontend/gen/value/longlong.c" +#include "../frontend/gen/value/vstack.c" diff --git a/source/unity/ir_01.c b/source/unity/ir_01.c new file mode 100644 index 000000000..7b5780ee8 --- /dev/null +++ b/source/unity/ir_01.c @@ -0,0 +1,34 @@ +/* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */ +#include "../ir/type.c" +#undef USING_GLOBALS +#include "../ir/pool.c" +#undef USING_GLOBALS +#include "../ir/vreg.c" +#undef IGNORED_VREG_BITS_PER_ENTRY +#undef IGNORED_VREG_LOCAL_VAR_BIT +#undef IGNORED_VREG_PARAM_BIT +#undef IGNORED_VREG_TEMP_BIT +#undef USING_GLOBALS +#include "../ir/frame.c" +#undef FRAME_LIVE_MAX_WORDS +#undef FRAME_OBJ_ARG_COPY +#undef FRAME_OBJ_RET_TEMP +#undef TAINT_MULTI +#undef TAINT_NONE +#undef USING_GLOBALS +#include "../ir/cross_jump.c" +#undef CJ_MAX_TAIL +#undef RM_BUCKET_TRIES +#undef RM_MAX_LEN +#undef RM_MAX_PARAMS +#undef RM_MIN_LEN +#undef RM_MULTI +#undef RM_NOCASE +#include "../ir/self_store.c" +#undef SC_MAX_DIST +#include "../ir/dead_def.c" +#undef DD_MAX_DIST +#include "../ir/dump.c" +#undef SPILL_MARK_BEGIN +#undef SPILL_MARK_END +#undef USING_GLOBALS diff --git a/source/unity/ir_02.c b/source/unity/ir_02.c new file mode 100644 index 000000000..d072db47d --- /dev/null +++ b/source/unity/ir_02.c @@ -0,0 +1,20 @@ +/* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */ +#include "../ir/codegen.c" +#undef DECODE +#undef DEMOTE_PENDING_SLOT +#undef DEMOTE_REF_NONE +#undef DEMOTE_REF_SLOTS +#undef DYN_ARGS_MAX +#undef MACH_BLOCK_COPY_MAX_WORDS +#undef MOP_CACHE_CHUNK +#undef MOP_CACHE_HDR +#undef SCAN_INTERVALS +#undef SCRATCH_WRAP +#undef USING_GLOBALS +#undef tcc_ir_debug_codegen_generate_entry +#include "../ir/gen/control.c" +#undef USING_GLOBALS +#include "../ir/gen/vla.c" +#undef USING_GLOBALS +#include "../ir/gen/util.c" +#undef USING_GLOBALS diff --git a/source/unity/ir_03.c b/source/unity/ir_03.c new file mode 100644 index 000000000..2b9f2a297 --- /dev/null +++ b/source/unity/ir_03.c @@ -0,0 +1,28 @@ +/* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */ +#include "../ir/cfg.c" +#undef USING_GLOBALS +#include "../ir/ssa.c" +#undef DD_VAR +#undef USING_GLOBALS +#include "../ir/regalloc.c" +#undef ASSIGN_VREG_IDX +#undef BFI_VREG_IDX +#undef LOAD_VREG_IDX +#undef OPH_VREG_IDX +#undef PHI_VREG_IDX +#undef RA_EXTEND_BACKEDGE +#undef RA_VREG_HAS_ASSIGN_IN_RANGE +#undef VREG_IDX +#include "../ir/regalloc_scan.c" +#undef AVIDX +#undef AV_FOR_PARAM_USES +#undef HINT_IDX +#undef RA_EXITPHI_PUSH +#include "../ir/machine_op.c" +#undef USING_GLOBALS +#include "../ir/gen/state.c" +#undef IR_PARAM_INTERVAL_INIT_SIZE +#undef IR_TEMP_INTERVAL_INIT_SIZE +#undef IR_VAR_INTERVAL_INIT_SIZE +#undef QUADRUPLE_INIT_SIZE +#undef USING_GLOBALS diff --git a/source/unity/ir_04.c b/source/unity/ir_04.c new file mode 100644 index 000000000..ae5339965 --- /dev/null +++ b/source/unity/ir_04.c @@ -0,0 +1,23 @@ +/* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */ +#include "../ir/regalloc_phi.c" +#undef ADD_CAND +#undef DVIDX +#undef RA_CO_ADD_USE +#undef RA_CO_FOR_CALL_PARAM_USES +#undef RA_CO_LIVE_OUT +#undef RA_CO_NOTE_USE +#undef RA_DEADCOPY_IDX +#undef RA_DEADCOPY_MARK +#undef UF_FIND +#undef VIDX +#include "../ir/regalloc_entry.c" +#undef REMAP +#undef RUN_SSA +#undef SPLIT_CAND +#undef SPLIT_DEF_BETWEEN +#undef SPLIT_IDX +#undef SPLIT_MEMBER +#undef SPLIT_PUSH +#undef SPLIT_REGION_OK +#undef SPLIT_STRAIGHT +#include "../ir/regalloc_post.c" diff --git a/source/unity/ir_05.c b/source/unity/ir_05.c new file mode 100644 index 000000000..8aa8376f9 --- /dev/null +++ b/source/unity/ir_05.c @@ -0,0 +1,19 @@ +/* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */ +#include "../ir/gen/put.c" +#undef USING_GLOBALS +#include "../ir/gen/params.c" +#undef USING_GLOBALS +#include "../ir/gen/arith.c" +#undef USING_GLOBALS +#include "../ir/gen/float.c" +#undef USING_GLOBALS +#include "../ir/gen/jump.c" +#undef USING_GLOBALS +#include "../ir/gen/asm.c" +#undef USING_GLOBALS +#include "../ir/gen/softfloat.c" +#undef USING_GLOBALS +#include "../ir/gen/config.c" +#undef USING_GLOBALS +#include "../ir/gen/live.c" +#undef USING_GLOBALS diff --git a/source/unity/machine_01.c b/source/unity/machine_01.c new file mode 100644 index 000000000..25a1afe16 --- /dev/null +++ b/source/unity/machine_01.c @@ -0,0 +1,7 @@ +/* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */ +#include "../machine/tccls.c" +#undef LS_DBG +#undef LS_LIVE_INTERVAL_INIT_SIZE +#undef SLOT_HASH +#undef STACK_ALLOC_LOG +#include "../machine/tccmachine.c" diff --git a/source/unity/memory_01.c b/source/unity/memory_01.c new file mode 100644 index 000000000..1dde36f91 --- /dev/null +++ b/source/unity/memory_01.c @@ -0,0 +1,3 @@ +/* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */ +#include "../memory/unique_ptr.c" +#include "../memory/vector.c" diff --git a/source/unity/obj_01.c b/source/unity/obj_01.c new file mode 100644 index 000000000..08f0af7dc --- /dev/null +++ b/source/unity/obj_01.c @@ -0,0 +1,7 @@ +/* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */ +#include "../obj/tccelf.c" +#include "../obj/tccelf_sym.c" +#undef ELFCLASS_BITS +#include "../obj/tccelf_reloc.c" +#include "../obj/tccelf_layout.c" +#include "../obj/tccelf_output.c" diff --git a/source/unity/obj_02.c b/source/unity/obj_02.c new file mode 100644 index 000000000..69507fa11 --- /dev/null +++ b/source/unity/obj_02.c @@ -0,0 +1,11 @@ +/* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */ +#include "../obj/tccelf_load.c" +#include "../obj/tccelf_ld.c" +#undef DEBUG_VERSION +#undef LD_TOK_EOF +#undef LD_TOK_NAME +#include "../obj/tccld.c" +#undef LDTOK_EOF +#undef LDTOK_NAME +#undef LDTOK_NUM +#undef LDTOK_STRING diff --git a/source/unity/obj_03.c b/source/unity/obj_03.c new file mode 100644 index 000000000..8688ebf2c --- /dev/null +++ b/source/unity/obj_03.c @@ -0,0 +1,63 @@ +/* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */ +#include "../obj/tccyaff.c" +#undef SHF_DYNSYM +#undef TCC_YAFF_MAX_SYMBOL_ENTRY_SIZE +#include "../obj/tccdbg.c" +#undef DWARF_ABBREV_ARRAY_TYPE +#undef DWARF_ABBREV_BASE_TYPE +#undef DWARF_ABBREV_COMPILE_UNIT +#undef DWARF_ABBREV_COMPILE_UNIT_RANGES +#undef DWARF_ABBREV_ENUMERATION_TYPE +#undef DWARF_ABBREV_ENUMERATOR_SIGNED +#undef DWARF_ABBREV_ENUMERATOR_UNSIGNED +#undef DWARF_ABBREV_FORMAL_PARAMETER +#undef DWARF_ABBREV_FORMAL_PARAMETER2 +#undef DWARF_ABBREV_LEXICAL_BLOCK +#undef DWARF_ABBREV_LEXICAL_EMPTY_BLOCK +#undef DWARF_ABBREV_MEMBER +#undef DWARF_ABBREV_MEMBER_BF +#undef DWARF_ABBREV_POINTER +#undef DWARF_ABBREV_STRUCTURE_EMPTY_TYPE +#undef DWARF_ABBREV_STRUCTURE_TYPE +#undef DWARF_ABBREV_SUBPROGRAM_EXTERNAL +#undef DWARF_ABBREV_SUBPROGRAM_STATIC +#undef DWARF_ABBREV_SUBRANGE_TYPE +#undef DWARF_ABBREV_SUBROUTINE_EMPTY_TYPE +#undef DWARF_ABBREV_SUBROUTINE_TYPE +#undef DWARF_ABBREV_TYPEDEF +#undef DWARF_ABBREV_UNION_EMPTY_TYPE +#undef DWARF_ABBREV_UNION_TYPE +#undef DWARF_ABBREV_VARIABLE_EXTERNAL +#undef DWARF_ABBREV_VARIABLE_LOCAL +#undef DWARF_ABBREV_VARIABLE_STATIC +#undef DWARF_LINE_BASE +#undef DWARF_LINE_RANGE +#undef DWARF_MIN_INSTR_LEN +#undef DWARF_OPCODE_BASE +#undef FDE_ENCODING +#undef N_DEFAULT_DEBUG +#undef STRUCT_NODEBUG +#undef debug_anon_hash +#undef debug_hash +#undef debug_info +#undef debug_info_root +#undef debug_next_type +#undef dwarf_data1 +#undef dwarf_data2 +#undef dwarf_data4 +#undef dwarf_data8 +#undef dwarf_info +#undef dwarf_line +#undef dwarf_line_relocs +#undef dwarf_sym +#undef dwarf_text_sections +#undef last_line_num +#undef max_dwarf_line_relocs +#undef max_dwarf_text_sections +#undef n_debug_anon_hash +#undef n_debug_hash +#undef n_dwarf_line_relocs +#undef n_dwarf_text_sections +#undef new_file +#undef section_sym +#undef tcov_data diff --git a/source/unity/opt_01.c b/source/unity/opt_01.c new file mode 100644 index 000000000..9f8145d92 --- /dev/null +++ b/source/unity/opt_01.c @@ -0,0 +1,74 @@ +/* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */ +#include "../opt/util/pass_disable.c" +#undef USING_GLOBALS +#include "../opt/util/volatile_access.c" +#undef USING_GLOBALS +#include "../opt/util/const_eval.c" +#undef USING_GLOBALS +#include "../opt/util/const_string_eval.c" +#undef SB_A0 +#undef SB_A1 +#undef USING_GLOBALS +#include "../opt/util/cond_util.c" +#undef USING_GLOBALS +#include "../opt/util/block_scan.c" +#undef USING_GLOBALS +#include "../opt/util/purity.c" +#undef USING_GLOBALS +#include "../opt/util/expr_equal.c" +#undef USING_GLOBALS +#include "../opt/util/call_params.c" +#undef USING_GLOBALS +#include "../opt/util/vreg_query.c" +#undef USING_GLOBALS +#include "../opt/util/vreg_def_use.c" +#undef USING_GLOBALS +#include "../opt/util/xform.c" +#undef USING_GLOBALS +#include "../opt/util/ir_insert.c" +#undef USING_GLOBALS +#include "../opt/analysis/du_chains.c" +#undef USING_GLOBALS +#include "../opt/analysis/alias.c" +#undef USING_GLOBALS +#include "../opt/analysis/memref.c" +#undef USING_GLOBALS +#include "../opt/analysis/mem_ssa.c" +#undef USING_GLOBALS +#include "../opt/engine/ctx.c" +#undef USING_GLOBALS +#include "../opt/engine/run_gens.c" +#undef USING_GLOBALS +#include "../opt/engine/pipeline_run.c" +#undef PIPELINE_MAX_TRACKED +#undef USING_GLOBALS +#include "../opt/engine/pipeline_table.c" +#undef CASCADE_END +#undef COUNTOF +#undef FLAG +#undef PASS +#undef PASS_GATED +#undef USING_GLOBALS +#include "../opt/engine/gen_adapters.c" +#undef USING_GLOBALS +#include "../opt/engine/pass_timing.c" +#undef TCC_PASS_TIMING_MAX +#undef TCC_PT_STACK_MAX +#undef USING_GLOBALS +#include "../opt/engine/fp_cache_shim.c" +#undef USING_GLOBALS +#include "../opt/engine/fp_mat_cache.c" +#undef FP_MAT_CACHE_SIZE +#undef USING_GLOBALS +#include "../opt/engine/pass_registry.c" +#undef USING_GLOBALS +#include "../opt/ra/const_branch_fold.c" +#undef USING_GLOBALS +#include "../opt/ra/incomplete_calls.c" +#undef USING_GLOBALS +#include "../opt/ra/phi_const_chain.c" +#undef USING_GLOBALS +#include "../opt/ssa/string/const_string_fold.c" +#include "../opt/ssa/string/str_strlen.c" +#include "../opt/flat/memory/mem_init.c" +#undef USING_GLOBALS diff --git a/source/unity/opt_02.c b/source/unity/opt_02.c new file mode 100644 index 000000000..4d626c96a --- /dev/null +++ b/source/unity/opt_02.c @@ -0,0 +1,56 @@ +/* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */ +#include "../opt/ra/umaal_fusion.c" +#undef USING_GLOBALS +#include "../opt/ssa/string/str_strcmp.c" +#include "../opt/ssa/string/str_strcpy.c" +#include "../opt/ssa/string/str_strspn.c" +#include "../opt/ssa/string/str_strchr.c" +#include "../opt/ssa/string/str_strstr.c" +#include "../opt/ssa/string/str_memchr.c" +#include "../opt/ssa/memory/global_addr_hoist.c" +#undef GAH_CONST_MIN_MAG +#undef GAH_CONST_TIGHT_BUDGET +#undef GAH_CONST_TIGHT_RELOADS +#undef GAH_LARGE_2ND_RELOADS +#undef GAH_LARGE_FN_INSTRS +#undef GAH_MAX_CALLS_BEFORE +#undef GAH_MAX_HOISTS +#undef GAH_MIN_RELOADS +#undef LAC_MAX_CLASSES +#undef LAC_MAX_FN_INSTRS +#undef LAC_MAX_TEMPS +#undef LAH_CALL_MAX_HOISTS +#undef LAH_CALL_MIN_BUDGET +#undef LAH_MAX_HOISTS +#undef LAH_MAX_LOOPS +#undef LAH_MIN_BUDGET +#include "../opt/ssa/memory/load_cse.c" +#undef GLOAD_MAX +#undef GSTORE_MAX +#undef ILOAD_MAX +#undef SSTORE_MAX +#undef TVSTORE_MAX +#undef USING_GLOBALS +#undef VSLOT_MAX +#undef resolve_lea_stackloc +#include "../opt/ssa/memory/diamond_store_fwd.c" +#undef USING_GLOBALS +#include "../opt/ssa/scalar/bitop_const_fold.c" +#include "../opt/ssa/scalar/strength.c" +#undef USING_GLOBALS +#include "../opt/ssa/scalar/reassoc.c" +#undef USING_GLOBALS +#include "../opt/ssa/scalar/narrow.c" +#undef NARROW_FLOAT_TABLE_COUNT +#undef USING_GLOBALS +#include "../opt/ssa/scalar/var_imm_prop.c" +#undef USING_GLOBALS +#include "../opt/ssa/scalar/bool_norm.c" +#undef K01_MAX_DEPTH +#undef K01_MAX_INSTRS +#undef USING_GLOBALS +#include "../opt/ssa/cfg/switch_fold.c" +#undef USING_GLOBALS +#include "../opt/ssa/dce/unreachable.c" +#undef DCE_MARK +#undef USING_GLOBALS diff --git a/source/unity/opt_03.c b/source/unity/opt_03.c new file mode 100644 index 000000000..465e2dc22 --- /dev/null +++ b/source/unity/opt_03.c @@ -0,0 +1,31 @@ +/* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */ +#include "../opt/ssa/scalar/fold.c" +#undef USING_GLOBALS +#include "../opt/ssa/scalar/gvn.c" +#undef GVN_HASH_SIZE +#undef GVN_KILL_SRC +#undef GVN_LOCAL_MAX +#undef GVN_PURE_CALL_MAX_ARGS +#undef GVN_SRC_OK +#undef USING_GLOBALS +#include "../opt/ssa/scalar/cprop.c" +#undef USING_GLOBALS +#undef VARF_BIT +#undef VARF_SET +#include "../opt/ssa/scalar/setif_or_taut.c" +#undef USING_GLOBALS +#include "../opt/ssa/scalar/cmp_offset_fold.c" +#undef USING_GLOBALS +#include "../opt/ssa/scalar/tmp_block_const.c" +#undef USING_GLOBALS +#include "../opt/ssa/cfg/phi.c" +#undef USING_GLOBALS +#include "../opt/ssa/cfg/or_bool_diamond.c" +#undef ORBD_REFS_SLOT +#undef USING_GLOBALS +#include "../opt/ssa/dce/temp_worklist.c" +#undef USING_GLOBALS +#include "../opt/ssa/dce/dead_var_stores.c" +#undef USING_GLOBALS +#include "../opt/ssa/dce/dce_common.c" +#undef USING_GLOBALS diff --git a/source/unity/opt_04.c b/source/unity/opt_04.c new file mode 100644 index 000000000..0cbe620e0 --- /dev/null +++ b/source/unity/opt_04.c @@ -0,0 +1,62 @@ +/* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */ +#include "../opt/ssa/cfg/branch.c" +#undef MARK +#undef SSA_SA_SLOT +#undef USING_GLOBALS +#include "../opt/ssa/cfg/vrp.c" +#undef USING_GLOBALS +#undef VRP_TOK_EQ +#undef VRP_TOK_GE +#undef VRP_TOK_GT +#undef VRP_TOK_LE +#undef VRP_TOK_LT +#undef VRP_TOK_NE +#undef VRP_TOK_UGE +#undef VRP_TOK_UGT +#undef VRP_TOK_ULE +#undef VRP_TOK_ULT +#include "../opt/ssa/dce/stackloc_stores.c" +#undef SL_HASH +#undef SL_HASH_SIZE +#undef SL_SET +#undef SL_TEST +#undef USING_GLOBALS +#include "../opt/ssa/dce/orphan_params.c" +#undef USING_GLOBALS +#include "../opt/ssa/dce/ret_path_frame_store.c" +#undef USING_GLOBALS +#include "../opt/ssa/dce/dead_phi_cycles.c" +#undef BM_SET +#undef BM_TEST +#undef MARK_TEMP_LIVE +#undef USING_GLOBALS +#include "../opt/ssa/dce/dead_overwrite_stores.c" +#undef DOS_PEND_MAX +#undef USING_GLOBALS +#include "../opt/ssa/dce/dead_global_stores.c" +#undef GS_PEND_MAX +#undef USING_GLOBALS +#include "../opt/ssa/dce/var_liveness.c" +#undef USING_GLOBALS +#undef VL_BIT +#undef VL_CLR +#undef VL_SET +#include "../opt/ssa/dce/dce.c" +#undef USING_GLOBALS +#include "../opt/ssa/loop/loop_rotate.c" +#undef SSA_LOOP_ROTATE_MAX_PASSES +#include "../opt/ssa/loop/first_iter_exit.c" +#undef FIE_EXIT_TEST_LOOKAHEAD +#undef LD_MAX_TMPS +#undef LD_MAX_VARS +#undef LD_TOK_EQ +#undef LD_TOK_NE +#undef SSA_FIRST_ITER_EXIT_MAX_PASSES +#include "../opt/ssa/loop/loop_cand.c" +#include "../opt/ssa/loop/loop_const_sim.c" +#undef SSA_LCS_MAX_SPAN +#undef SSA_LOOP_CONST_SIM_MAX_PASSES +#include "../opt/ssa/loop/iv_strength_reduction.c" +#undef SSA_IVSR_MAX_PASSES +#include "../opt/ssa/loop/decrement_to_zero.c" +#undef SSA_DTZ_MAX_PASSES diff --git a/source/unity/opt_05.c b/source/unity/opt_05.c new file mode 100644 index 000000000..c70c2fec7 --- /dev/null +++ b/source/unity/opt_05.c @@ -0,0 +1,33 @@ +/* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */ +#include "../opt/ssa/loop/ptr_iv_exit_subst.c" +#undef PTRIV_MAX +#undef SSA_PTR_IV_EXIT_SUBST_MAX_PASSES +#include "../opt/ssa/loop/loop_unroll.c" +#undef CZ_MAX_CHAIN +#undef CZ_MAX_SLOTS +#undef SSA_LOOP_UNROLL_MAX_PASSES +#undef SSA_UNROLL_MAX_SPAN +#include "../opt/ssa/loop/loop_bottom_test.c" +#undef SSA_LBT_MAX_REWRITES +#include "../opt/ssa/loop/dead_loop.c" +#undef DL_MAX_ESCAPE_CANDS +#undef USING_GLOBALS +#include "../opt/ssa/scalar/sccp.c" +#undef USING_GLOBALS +#include "../opt/ssa/memory/global_store_dse.c" +#undef USING_GLOBALS +#include "../opt/ssa/memory/ptr_store_dse.c" +#undef USING_GLOBALS +#include "../opt/ssa/cfg/guard_collapse.c" +#undef USING_GLOBALS +#include "../opt/ssa/engine/use_def.c" +#undef USING_GLOBALS +#include "../opt/ssa/engine/rewrite.c" +#undef USING_GLOBALS +#include "../opt/ssa/engine/driver.c" +#undef SSA_RUN +#undef USING_GLOBALS +#include "../opt/flat/scalar/bool.c" +#undef USING_GLOBALS +#include "../opt/flat/scalar/narrow_store.c" +#undef USING_GLOBALS diff --git a/source/unity/opt_06.c b/source/unity/opt_06.c new file mode 100644 index 000000000..a77b022af --- /dev/null +++ b/source/unity/opt_06.c @@ -0,0 +1,39 @@ +/* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */ +#include "../opt/ssa/engine/stack_resolve.c" +#undef USING_GLOBALS +#include "../opt/flat/scalar/branch.c" +#undef REMAT_MAX_USES +#undef USING_GLOBALS +#include "../opt/flat/scalar/stack_bool_diamond.c" +#undef USING_GLOBALS +#include "../opt/flat/scalar/const_string_calls.c" +#undef USING_GLOBALS +#include "../opt/flat/scalar/const_call_fold.c" +#undef SWITCH_FUNC_MAX_OPS +#undef SWITCH_FUNC_SIM_MAX_REPLAY +#undef SWITCH_FUNC_SIM_MAX_VREGS +#undef USING_GLOBALS +#include "../opt/flat/scalar/addrof_const_fold.c" +#undef USING_GLOBALS +#include "../opt/flat/scalar/global_init_prop.c" +#undef USING_GLOBALS +#include "../opt/flat/scalar/symref_const_prop.c" +#undef USING_GLOBALS +#include "../opt/flat/scalar/stack_addr_fold.c" +#undef USING_GLOBALS +#include "../opt/flat/scalar/call_result.c" +#undef USING_GLOBALS +#include "../opt/flat/scalar/self_arith.c" +#undef USING_GLOBALS +#include "../opt/flat/scalar/self_copy.c" +#undef USING_GLOBALS +#include "../opt/flat/scalar/var_tmp_fwd.c" +#undef USING_GLOBALS +#include "../opt/flat/scalar/inline_param_copy.c" +#undef USING_GLOBALS +#include "../opt/flat/scalar/switch_collapse.c" +#undef USING_GLOBALS +#include "../opt/flat/fusion/indexed_chain.c" +#undef USING_GLOBALS +#include "../opt/flat/dce/trap_only_body_suppress.c" +#undef USING_GLOBALS diff --git a/source/unity/opt_07.c b/source/unity/opt_07.c new file mode 100644 index 000000000..becb952e8 --- /dev/null +++ b/source/unity/opt_07.c @@ -0,0 +1,33 @@ +/* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */ +#include "../opt/flat/scalar/known_bits.c" +#undef KB_MAX_STACK_SLOTS +#undef USING_GLOBALS +#include "../opt/flat/scalar/const_prop_tmp.c" +#undef USING_GLOBALS +#include "../opt/flat/scalar/value_tracking.c" +#undef USING_GLOBALS +#undef VT_BINOP +#undef VT_CLEAR_DEF +#undef VT_HAS_DEF +#undef VT_INVALIDATE +#undef VT_IS_CONST +#undef VT_MAX_ADDRTAKEN +#undef VT_SET_CONST +#undef VT_SET_CONST_DEF +#include "../opt/flat/scalar/const_var_prop.c" +#undef NEG_VREG_MAX +#undef USING_GLOBALS +#include "../opt/flat/scalar/const_aggregate.c" +#undef CAF_ESCAPE +#undef CAF_MAX_SLOTS +#undef CAF_RESOLVE_ADDR_ROOT +#undef CAF_RESOLVE_LVAL +#undef CAF_ROOT_CONFLICT +#undef CAF_ROOT_ESCAPED +#undef CAF_ROOT_NONE +#undef CAF_SLOT_IDX +#undef USING_GLOBALS +#include "../opt/flat/scalar/add_reassoc.c" +#undef USING_GLOBALS +#include "../opt/flat/fusion/pair_reorder.c" +#undef USING_GLOBALS diff --git a/source/unity/opt_08.c b/source/unity/opt_08.c new file mode 100644 index 000000000..2e9cf595f --- /dev/null +++ b/source/unity/opt_08.c @@ -0,0 +1,49 @@ +/* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */ +#include "../opt/flat/scalar/cmp_field_fuse.c" +#undef LOG_CMPFUSE +#undef USING_GLOBALS +#include "../opt/flat/scalar/cmp_expr_fold.c" +#undef USING_GLOBALS +#include "../opt/flat/scalar/bitfield.c" +#undef LOG_BITFIELD +#undef USING_GLOBALS +#include "../opt/flat/memory/symaddr_cse.c" +#undef SYMADDR_CSE_MAX_HOISTS +#undef USING_GLOBALS +#include "../opt/flat/memory/dse.c" +#undef DSE_CASCADE_PURE_CALL_PARAMS +#undef DSE_DEC_SOURCES +#undef DSE_DEC_TEMP +#undef DSE_ENSURE_CAP +#undef DSE_INC_USE +#undef DSE_IS_DEAD_ELIGIBLE +#undef DSE_IS_PURE_CALL +#undef DSE_SOURCE_IS_PURE +#undef GET_ORIGIN +#undef MARK_ORIGIN_READ +#undef MARK_STACKLOC_OP +#undef MARK_VAR_USED +#undef STACKLOC_HASH +#undef STACKLOC_HASH_SIZE +#undef STACKLOC_SET +#undef STACKLOC_TEST +#undef USING_GLOBALS +#include "../opt/flat/memory/var_store_elim.c" +#undef USING_GLOBALS +#include "../opt/flat/memory/entry_store_prop.c" +#undef ESTORE_COMPACT +#undef LEA_SET +#undef MAX_ADDRTAKEN_BASES +#undef MAX_BC_RANGES +#undef MAX_ENTRY_STORES +#undef PHF_MAX +#undef USING_GLOBALS +#include "../opt/flat/memory/sret_nrvo.c" +#undef NRVO_FAIL +#undef NRVO_IDX +#undef NRVO_MAX_OBJ +#undef NRVO_MAX_REFS +#undef NRVO_OWNED +#undef USING_GLOBALS +#include "../opt/flat/memory/dead_temp_local.c" +#undef USING_GLOBALS diff --git a/source/unity/opt_09.c b/source/unity/opt_09.c new file mode 100644 index 000000000..c6d366b04 --- /dev/null +++ b/source/unity/opt_09.c @@ -0,0 +1,25 @@ +/* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */ +#include "../opt/flat/memory/sl_forward.c" +#undef CHECK_ADDR_ALIAS +#undef CHECK_WIDTH_OVERLAP +#undef GSLFWD_MAX_ENTRIES +#undef SL_FWD_MAX_DEAD_STORES +#undef USING_GLOBALS +#include "../opt/flat/memory/copy_source_load_fwd.c" +#undef CSF_ALL64 +#undef CSF_DBG +#undef CSF_MAX_BLOCK_COPY +#undef CSF_MAX_FIELDS +#undef USING_GLOBALS +#include "../opt/flat/memory/dead_local_slot.c" +#undef DLS_LIVE_ADD +#undef DLS_NONTAME_RANGE_OVERLAPS +#undef TAME_FIND +#undef TAME_FIND_OR_ADD +#undef USING_GLOBALS +#undef VR_FLAT +#undef VR_SLOT_AMBIG +#include "../opt/flat/memory/dead_static_store.c" +#undef DSS_VREG_MAP_MAX +#undef DSS_VREG_MAP_OVERFLOW +#undef USING_GLOBALS diff --git a/source/unity/opt_10.c b/source/unity/opt_10.c new file mode 100644 index 000000000..a51d7eec2 --- /dev/null +++ b/source/unity/opt_10.c @@ -0,0 +1,49 @@ +/* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */ +#include "../opt/flat/memory/global_base_share.c" +#undef GBS_DELTA_MAX +#undef GBS_DELTA_MIN +#undef GBS_MAX_CLUSTER +#undef USING_GLOBALS +#include "../opt/flat/memory/byte_store_merge.c" +#undef CMD_MAX_BYTES +#undef CMD_OVL +#undef RSE_DEF_MULTI +#undef RSE_MAX_COPY_SRC +#undef USING_GLOBALS +#include "../opt/flat/memory/mem_inline.c" +#undef MI_MAX_PIECES +#undef MI_MEMCPY_MAX +#undef MI_MEMCPY_PIECES +#undef MI_MEMSET_MAX +#undef MI_MEMSET_PIECES +#undef USING_GLOBALS +#include "../opt/flat/fusion/disp.c" +#undef USING_GLOBALS +#include "../opt/flat/fusion/gens_fusion.c" +#undef USING_GLOBALS +#include "../opt/flat/fusion/add_deref_fold.c" +#undef USING_GLOBALS +#include "../opt/flat/fusion/call_chain_rename.c" +#undef USING_GLOBALS +#include "../opt/flat/fusion/stackoff_addr_cse.c" +#undef SAC_MAX_OFFSETS +#undef SIB_MAX_OFFSETS +#undef USING_GLOBALS +#include "../opt/flat/fusion/lea_fold.c" +#undef USING_GLOBALS +#include "../opt/flat/fusion/lea_rmw_fold.c" +#undef LEA_RMW_MAX_SITES +#undef USING_GLOBALS +#include "../opt/flat/fusion/assign_fuse.c" +#undef USING_GLOBALS +#include "../opt/flat/fusion/pack64.c" +#undef USING_GLOBALS +#include "../opt/flat/fusion/pack64_tautology.c" +#undef USING_GLOBALS +#include "../opt/flat/fusion/cmp_narrow_64.c" +#undef USING_GLOBALS +#include "../opt/flat/fusion/cmp_hi_only.c" +#undef USING_GLOBALS +#undef ZH64_CMP_HI +#include "../opt/flat/fusion/cmp_imm_swap.c" +#undef USING_GLOBALS diff --git a/source/unity/opt_11.c b/source/unity/opt_11.c new file mode 100644 index 000000000..107b59471 --- /dev/null +++ b/source/unity/opt_11.c @@ -0,0 +1,51 @@ +/* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */ +#include "../opt/flat/fusion/and64_narrow.c" +#undef USING_GLOBALS +#include "../opt/flat/fusion/shl32_or_chain.c" +#undef USING_GLOBALS +#include "../opt/flat/scalar/sdiv_pow2.c" +#undef USING_GLOBALS +#include "../opt/flat/memory/dead_lea_store.c" +#undef ADD_READ +#undef RESOLVE_LVAL_SLOT +#undef USING_GLOBALS +#include "../opt/flat/memory/store_inplace_arith.c" +#undef USING_GLOBALS +#include "../opt/flat/scalar/redundant_loop_check.c" +#undef USING_GLOBALS +#include "../opt/flat/cfg/if_convert.c" +#undef JT_HAS_OTHER +#undef JT_NOP_JUMP +#undef USING_GLOBALS +#include "../opt/flat/dce/body_essential.c" +#undef GETBIT +#undef IS_LOCAL_PTR_OP +#undef LOCAL_ONLY_MAX_MEMMOVE_CALLS +#undef LOCAL_ONLY_MAX_TEMPS +#undef LP_GET +#undef LP_SET +#undef MARKENTRY +#undef PUSH +#undef SETBIT +#undef USING_GLOBALS +#include "../opt/flat/dce/noreturn_collapse.c" +#undef USING_GLOBALS +#include "../opt/flat/dce/infinite_loop_simplify.c" +#undef MAX_HOIST +#undef USING_GLOBALS +#include "../opt/flat/dce/infinite_self_recursion.c" +#undef USING_GLOBALS +#include "../opt/flat/dce/noreturn_call_epilogue.c" +#undef USING_GLOBALS +#include "../opt/flat/dce/dead_addrvar_elim.c" +#undef USING_GLOBALS +#include "../opt/flat/dce/dead_trailing_addrvar_store.c" +#undef USING_GLOBALS +#include "../opt/flat/dce/uninit_ub.c" +#undef UDR_MAX_VAR_POS +#undef UDR_NOPSKIP +#undef UDR_RR_GET +#undef UNINIT_MAX_VAR_POS +#undef USING_GLOBALS +#include "../opt/flat/cfg/returnvalue_merge.c" +#undef USING_GLOBALS diff --git a/source/unity/opt_12.c b/source/unity/opt_12.c new file mode 100644 index 000000000..73aede6d9 --- /dev/null +++ b/source/unity/opt_12.c @@ -0,0 +1,44 @@ +/* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */ +#include "../opt/flat/dce/ub_only_body_elide.c" +#undef SRC_IS_UNINIT_READ +#undef TEMP_IS_TAINTED +#undef UB_ELIDE_MAX_STACK_OFFS +#undef UB_ELIDE_MAX_TEMPS +#undef UB_ELIDE_MAX_VAR_POS +#undef USING_GLOBALS +#include "../opt/flat/dce/const_return_uninit_elide.c" +#undef CRUE_MAX_STORE_OFFSETS +#undef CRUE_MAX_VAR_POS +#undef USING_GLOBALS +#include "../opt/flat/dce/null_store_dom_return.c" +#undef NSDR_MAX_TEMP +#undef NSDR_MAX_VAR +#undef USING_GLOBALS +#undef VREG_IS_KNOWN_ZERO +#include "../opt/flat/dce/dce.c" +#undef MARK_REACHABLE +#undef USING_GLOBALS +#include "../opt/flat/dce/compact_nops.c" +#undef USING_GLOBALS +#include "../opt/flat/dce/orphan_cmp_elim.c" +#undef USING_GLOBALS +#include "../opt/flat/dce/redundant_cmp.c" +#undef USING_GLOBALS +#include "../opt/flat/dce/zero_vla_elim.c" +#undef SLOT_USED_BY +#undef USING_GLOBALS +#include "../opt/flat/loop/loop_ir_mutate.c" +#undef USING_GLOBALS +#include "../opt/flat/loop/iv_analysis.c" +#undef IV_AFFINE_MAX +#undef USING_GLOBALS +#include "../opt/flat/loop/iv_strength_reduction.c" +#undef APPLY_SHIFT +#undef SR_TAINT_MAX +#undef USING_GLOBALS +#include "../opt/flat/loop/loop_exit_analysis.c" +#undef USING_GLOBALS +#include "../opt/flat/loop/loop_eliminate.c" +#undef USING_GLOBALS +#include "../opt/flat/loop/decrement_to_zero.c" +#undef USING_GLOBALS diff --git a/source/unity/opt_13.c b/source/unity/opt_13.c new file mode 100644 index 000000000..2fc874273 --- /dev/null +++ b/source/unity/opt_13.c @@ -0,0 +1,47 @@ +/* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */ +#include "../opt/flat/loop/loop_unroll.c" +#undef UNROLL_MAX_FOLD +#undef UNROLL_MAX_RENAME +#undef USING_GLOBALS +#include "../opt/flat/loop/loop_rotate.c" +#undef ROT_LVAL_IS_INDIRECT +#undef ROT_NOTE_DEF +#undef ROT_NOTE_READ +#undef ROT_VAR_DIRECT +#undef USING_GLOBALS +#undef _ROFF +#include "../opt/flat/loop/loop_relayout.c" +#undef RJ +#undef USING_GLOBALS +#include "../opt/flat/loop/seq_guard_elim.c" +#undef SEQ_MAX_INSTRS +#undef SEQ_MAX_TRACK +#undef SEQ_ROTATED +#undef SEQ_TOP_TESTED +#undef USING_GLOBALS +#include "../opt/flat/loop/const_sim.c" +#undef LCS_MAX_CALLS +#undef LCS_MAX_ITER_STEPS +#undef LCS_MAX_MEM_SLOTS +#undef LCS_MAX_PARAMS +#undef LCS_MAX_TMP_POS +#undef LCS_MAX_TRACKED_TMPS +#undef LCS_MAX_TRACKED_VARS +#undef LCS_MAX_TRIP_COUNT +#undef LCS_MAX_VAR_POS +#undef LCS_RECORD_EXIT +#undef USING_GLOBALS +#include "../opt/flat/loop/reroll.c" +#undef LOG_REROLL +#undef REROLL_MAX_PERIOD +#undef REROLL_MIN_PERIOD +#undef REROLL_MIN_REPEATS +#undef RS_HASH +#undef USING_GLOBALS +#include "../opt/flat/cfg/switch_to_data.c" +#undef SWITCHDATA_MAX_BODY_OPS +#undef USING_GLOBALS +#include "../opt/flat/cfg/jump_thread.c" +#undef USING_GLOBALS +#include "../opt/flat/cfg/bool_diamond_branch.c" +#undef USING_GLOBALS diff --git a/source/unity/opt_14.c b/source/unity/opt_14.c new file mode 100644 index 000000000..7112b6fc2 --- /dev/null +++ b/source/unity/opt_14.c @@ -0,0 +1,34 @@ +/* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */ +#include "../opt/flat/dce/dead_vla.c" +#undef CLASSIFY +#undef USING_GLOBALS +#include "../opt/flat/cfg/jumpif_invert.c" +#undef USING_GLOBALS +#include "../opt/flat/scalar/var_to_tmp.c" +#undef USING_GLOBALS +#include "../opt/flat/loop/backedge_phi_hoist.c" +#undef USING_GLOBALS +#include "../opt/flat/cfg/abort_tail_merge.c" +#undef USING_GLOBALS +#include "../opt/flat/loop/loop_detect.c" +#undef BUDGET_MAX_VREGS +#undef WINDOW_SIZE +#include "../opt/flat/loop/func_purity.c" +#undef NUM_PURE_FUNCS +#include "../opt/flat/loop/licm.c" +#undef GLH_MAX_PER_LOOP +#undef GLH_MAX_WRITTEN +#undef LICM_MAX_ADDR_TEMPS +#undef LICM_MAX_MEM_PARAMS +#undef MAX_HOISTABLE_CALLS +#include "../opt/flat/loop/stack_param_promote.c" +#include "../opt/flat/memory/invariant_global_load_hoist.c" +#undef IGLH_MAX_TRACKED +#undef IGLH_MAX_WRITTEN +#undef USING_GLOBALS +#include "../opt/flat/memory/global_deref_cse.c" +#undef GDCSE_MAX_ENTRIES +#undef GDCSE_MAX_USES +#undef USING_GLOBALS +#include "../opt/flat/ipa/tu_noreturn.c" +#undef USING_GLOBALS diff --git a/source/unity/opt_15.c b/source/unity/opt_15.c new file mode 100644 index 000000000..b090ce3ee --- /dev/null +++ b/source/unity/opt_15.c @@ -0,0 +1,55 @@ +/* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */ +#include "../opt/flat/memory/deref_operand_cse.c" +#undef DOCSE_MAX_ENTRIES +#undef DOCSE_MAX_USES +#undef USING_GLOBALS +#include "../opt/flat/memory/bitfield_unit_narrow.c" +#undef BFUN_MAX_EXTRACTS +#undef USING_GLOBALS +#include "../opt/flat/memory/rmw_byte_clear.c" +#undef USING_GLOBALS +#include "../opt/flat/memory/struct_copy_roundtrip_elim.c" +#undef MGLF_MAX +#undef USING_GLOBALS +#include "../opt/flat/fusion/barrel_shift.c" +#undef USING_GLOBALS +#include "../opt/flat/fusion/shift_pair_ubfx.c" +#undef USING_GLOBALS +#include "../opt/flat/fusion/shift64_dead_half.c" +#undef USING_GLOBALS +#include "../opt/flat/fusion/zero_half64.c" +#undef USING_GLOBALS +#undef ZH64_D_HI +#undef ZH64_D_LO +#undef ZH64_S1_HI +#undef ZH64_S1_LO +#undef ZH64_S2_HI +#undef ZH64_S2_LO +#undef ZK_HI +#undef ZK_LO +#include "../opt/flat/fusion/shift64_extract_ubfx.c" +#undef USING_GLOBALS +#include "../opt/flat/ipa/pure_via_sret.c" +#undef SRET_MARK +#undef SRET_TEST +#undef SRET_VR_TO_BIT +#undef USING_GLOBALS +#include "../opt/flat/ipa/func_write_summary.c" +#undef USING_GLOBALS +#undef VR_BIT +#include "../opt/flat/ipa/tu_func_summary.c" +#undef TU_MODREF_MAX_VISIT +#undef TU_VREG_MAP_MAX +#undef TU_VREG_MAP_OVERFLOW +#undef USING_GLOBALS +#include "../opt/flat/ipa/tu_dead_statics.c" +#undef USING_GLOBALS +#include "../opt/flat/memory/dead_init_via_call.c" +#undef USING_GLOBALS +#include "../opt/flat/memory/stack_addr_cse.c" +#undef STACK_CSE_MAX_ENTRIES +#undef USING_GLOBALS +#include "../opt/flat/memory/block_copy_init.c" +#undef USING_GLOBALS +#include "../opt/flat/memory/small_memset_to_store.c" +#undef USING_GLOBALS diff --git a/source/unity/opt_16.c b/source/unity/opt_16.c new file mode 100644 index 000000000..6bd97b35e --- /dev/null +++ b/source/unity/opt_16.c @@ -0,0 +1,31 @@ +/* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */ +#include "../opt/flat/memory/const_local_table.c" +#undef CLT_MAX_OBJ +#undef CLT_MAX_SIZE +#undef USING_GLOBALS +#include "../opt/flat/memory/small_global_memset_to_store.c" +#undef USING_GLOBALS +#include "../opt/flat/memory/addrof_var_fwd.c" +#undef ADDROF_VAR_MAX_ALIASES +#undef ADDROF_VAR_MAX_REWRITES +#undef USING_GLOBALS +#include "../opt/flat/memory/ptr_local_fwd.c" +#undef PLF_CONFLICT +#undef PLF_UNKNOWN +#undef USING_GLOBALS +#include "../opt/flat/memory/param_copy_alias.c" +#undef PCA_MAX_WORDS +#undef USING_GLOBALS +#include "../opt/flat/memory/slot_const_store_fold.c" +#undef SCSF_MAX +#undef USING_GLOBALS +#include "../opt/flat/memory/sra.c" +#undef SRA_DF_MAX +#undef SRA_FIRST_ADDR +#undef SRA_MAX_BYTES +#undef SRA_NOT_ADDR +#undef SRA_NO_ADDR +#undef SRA_UNKNOWN +#undef SRA_VMIX +#include "../opt/flat/memory/memmove_to_indexed_stores.c" +#undef USING_GLOBALS diff --git a/source/unity/thumb_01.c b/source/unity/thumb_01.c new file mode 100644 index 000000000..80400d61e --- /dev/null +++ b/source/unity/thumb_01.c @@ -0,0 +1,23 @@ +/* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */ +#include "../backend/arch/arm/thumb/thumb.c" +#undef TH_PACK_CONST_CACHE_SIZE +#undef USING_GLOBALS +#include "../backend/arch/arm/thumb/thop_mem_reg.c" +#undef THOP_MEM_REG_FN +#undef USING_GLOBALS +#undef V_MEM_REG_T1 +#undef V_MEM_REG_T32 +#include "../backend/arch/arm/thumb/thop_mem_imm.c" +#undef USING_GLOBALS +#include "../backend/arch/arm/thumb/thop_bitfield.c" +#undef USING_GLOBALS +#undef V_BFX +#undef V_SSAT_ASR +#undef V_SSAT_LSL +#undef V_USAT_ASR +#undef V_USAT_LSL +#include "../backend/arch/arm/thumb/thop_mov.c" +#undef USING_GLOBALS +#include "../backend/arch/arm/thumb/arm-thumb-gen.c" +#include "../backend/arch/arm/thumb/arm-thumb-callsite.c" +#undef USING_GLOBALS diff --git a/source/unity/thumb_02.c b/source/unity/thumb_02.c new file mode 100644 index 000000000..a2e305d80 --- /dev/null +++ b/source/unity/thumb_02.c @@ -0,0 +1,63 @@ +/* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */ +#include "../backend/arch/arm/thumb/thop_alu_imm.c" +#undef THOP_ALU_IMM_FN +#undef THOP_ALU_WIDE_FN +#undef V_IMM12 +#undef V_IMM3 +#undef V_IMM8 +#undef V_MOD_IMM +#include "../backend/arch/arm/thumb/thop_alu_reg.c" +#undef THOP_ALU_REG_FN +#undef V_ADD_SP_REG +#undef V_ADD_T2 +#undef V_REG3 +#undef V_REGS +#undef V_REG_RDN_RM +#include "../backend/arch/arm/thumb/thop_cmp.c" +#undef THOP_CMP_IMM_FN +#undef THOP_CMP_REG_FN +#undef V_CMP_IMM +#undef V_CMP_IMM8 +#undef V_CMP_REG +#undef V_CMP_REGS +#undef V_CMP_REG_T2 +#include "../backend/arch/arm/thumb/thop_shift_imm.c" +#undef THOP_SHIFT_IMM_FN +#undef V_SHIFT_IMM16 +#undef V_SHIFT_IMM32 +#include "../backend/arch/arm/thumb/thop_shift_reg.c" +#undef THOP_SHIFT_REG_FN +#undef V_ASR_REG_T1 +#undef V_LSL_REG_T1 +#undef V_LSR_REG_T1 +#undef V_SHIFT_REG32 +#include "../backend/arch/arm/thumb/thop_mem_unpriv.c" +#undef THOP_MEM_UNPRIV_FN +#undef V_MEM_UNPRIV +#include "../backend/arch/arm/thumb/thop_mem_exclusive.c" +#undef THOP_EXCLUSIVE_FN +#undef V_EXCLUSIVE +#include "../backend/arch/arm/thumb/thop_extend.c" +#undef THOP_EXTEND_FN +#undef V_EXTEND_T16 +#undef V_EXTEND_T32 +#include "../backend/arch/arm/thumb/thop_rev.c" +#undef V_RBIT_T32 +#undef V_REV_T16 +#undef V_REV_T32 +#include "../backend/arch/arm/thumb/thop_vfp.c" +#include "../backend/arch/arm/thumb/thop_coproc.c" +#include "../backend/arch/arm/thumb/thop_system.c" +#include "../backend/arch/arm/thumb/thop_branch.c" +#include "../backend/arch/arm/thumb/thop_block.c" +#include "../backend/arch/arm/thumb/thop_mul.c" +#include "../backend/arch/arm/thumb/thop_dsp.c" +#include "../backend/arch/arm/thumb/thop_tbb.c" +#include "../backend/arch/arm/thumb/thop_ldrd.c" +#include "../backend/arch/arm/thumb/thop_ldrex.c" +#include "../backend/arch/arm/thumb/thop_ldaex.c" +#include "../backend/arch/arm/thumb/thop_mrs.c" +#include "../backend/arch/arm/thumb/thop_pld.c" +#include "../backend/arch/arm/thumb/thop_mvn.c" +#include "../backend/arch/arm/thumb/thop_adr.c" +#include "../backend/arch/arm/thumb/thop_ldr_literal.c" diff --git a/source/unity/thumb_03.c b/source/unity/thumb_03.c new file mode 100644 index 000000000..1922f8136 --- /dev/null +++ b/source/unity/thumb_03.c @@ -0,0 +1,3 @@ +/* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */ +#include "../backend/arch/arm/thumb/arm-thumb-emit.c" +#include "../backend/arch/arm/thumb/arm-thumb-alu.c" diff --git a/source/unity/thumb_04.c b/source/unity/thumb_04.c new file mode 100644 index 000000000..1c48ca4a9 --- /dev/null +++ b/source/unity/thumb_04.c @@ -0,0 +1,11 @@ +/* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */ +#include "../backend/arch/arm/thumb/arm-thumb-mem.c" +#include "../backend/arch/arm/thumb/arm-thumb-fp.c" +#include "../backend/arch/arm/thumb/arm-thumb-frame.c" +#include "../backend/arch/arm/thumb/thumb-outline.c" +#undef OUTLINE_MAX_OPS +#undef OUTLINE_MIN_BYTES +#undef R_IS_SP_LR_PC +#undef USING_GLOBALS +#undef WINDOW_EXTEND +#undef WINDOW_SIZE diff --git a/source/unity/unity.mk b/source/unity/unity.mk new file mode 100644 index 000000000..47b4a07c2 --- /dev/null +++ b/source/unity/unity.mk @@ -0,0 +1,422 @@ +# Generated by scripts/gen_unity.py -- do not edit; rerun it instead. +# Unity-build groups: see the "Unity build" block in the Makefile. +# Budget 6000 source lines per group. +# own TU: source/backend/generators/regalloc.c (expands differently inside a group) +# own TU: source/driver/libtcc.c (redefines header macros WARN_ON _tcc_error free gnu_ext realloc tcc_malloc tcc_mallocz tcc_realloc) +# own TU: source/frontend/gen/decl/decl.c (redefines header macros gjmp gjmp_addr) +# own TU: source/frontend/gen/init/initializer.c (redefines header macros DIF_CLEAR DIF_FIRST DIF_HAVE_ELEM DIF_SIZE_ONLY EXPR_ANY EXPR_CONST) +# own TU: source/frontend/gen/type/compare.c (redefines header macros CMP_OP SHIFT_OP) +# own TU: source/frontend/tccpp.c (redefines header macros DEF) +# own TU: source/ir/stack.c (redefines header macros INT32_MIN) +# own TU: source/ir/tccir_operand.c (redefines header macros INT32_MAX INT32_MIN UINT32_MAX) +# own TU: source/opt/ssa/cfg/cmp_eq.c (redefines header macros TOK_EQ TOK_NE) + +UNITY_GROUPS_frontend = frontend_01 frontend_02 frontend_03 frontend_04 frontend_05 frontend_06 +UNITY_MEMBERS_frontend_01 = \ + source/frontend/tccasm.c \ + source/frontend/gen/builtin/call.c \ + source/frontend/gen/builtin/chk.c \ + source/frontend/gen/builtin/fp2.c \ + source/frontend/gen/core/predicates.c +UNITY_MEMBERS_frontend_02 = \ + source/frontend/gen/builtin/fp.c \ + source/frontend/gen/builtin/misc.c \ + source/frontend/gen/builtin/overflow.c \ + source/frontend/gen/builtin/simd.c \ + source/frontend/gen/builtin/string.c \ + source/frontend/gen/core/state.c \ + source/frontend/gen/core/suppress.c \ + source/frontend/gen/decl/attribute.c \ + source/frontend/gen/decl/btype.c \ + source/frontend/gen/decl/declarator.c \ + source/frontend/gen/decl/predef_protos.c \ + source/frontend/gen/expr/atomic.c \ + source/frontend/gen/value/strlit_pool.c +UNITY_MEMBERS_frontend_03 = \ + source/frontend/gen/decl/struct.c \ + source/frontend/gen/expr/cond.c \ + source/frontend/gen/expr/indir.c \ + source/frontend/gen/expr/infix.c \ + source/frontend/gen/expr/primary.c \ + source/frontend/gen/expr/unary.c \ + source/frontend/gen/init/alloc.c \ + source/frontend/gen/op/float.c \ + source/frontend/gen/sym/elfsym.c +UNITY_MEMBERS_frontend_04 = \ + source/frontend/gen/inline/analysis.c \ + source/frontend/gen/inline/const_eval.c \ + source/frontend/gen/inline/emit.c \ + source/frontend/gen/nested/nested.c \ + source/frontend/gen/op/complex.c \ + source/frontend/gen/op/fold_math.c +UNITY_MEMBERS_frontend_05 = \ + source/frontend/gen/op/int.c \ + source/frontend/gen/op/op.c \ + source/frontend/gen/op/vector.c \ + source/frontend/gen/stmt/block.c \ + source/frontend/gen/stmt/cleanup.c \ + source/frontend/gen/stmt/ret.c \ + source/frontend/gen/stmt/switch.c \ + source/frontend/gen/store/struct_copy.c \ + source/frontend/gen/store/vstore.c \ + source/frontend/gen/sym/attr_merge.c +UNITY_MEMBERS_frontend_06 = \ + source/frontend/gen/sym/symtab.c \ + source/frontend/gen/type/assign_check.c \ + source/frontend/gen/type/cast.c \ + source/frontend/gen/type/size.c \ + source/frontend/gen/value/load.c \ + source/frontend/gen/value/longlong.c \ + source/frontend/gen/value/vstack.c + +UNITY_GROUPS_ir = ir_01 ir_02 ir_03 ir_04 ir_05 +UNITY_MEMBERS_ir_01 = \ + source/ir/type.c \ + source/ir/pool.c \ + source/ir/vreg.c \ + source/ir/frame.c \ + source/ir/cross_jump.c \ + source/ir/self_store.c \ + source/ir/dead_def.c \ + source/ir/dump.c +UNITY_MEMBERS_ir_02 = \ + source/ir/codegen.c \ + source/ir/gen/control.c \ + source/ir/gen/vla.c \ + source/ir/gen/util.c +UNITY_MEMBERS_ir_03 = \ + source/ir/cfg.c \ + source/ir/ssa.c \ + source/ir/regalloc.c \ + source/ir/regalloc_scan.c \ + source/ir/machine_op.c \ + source/ir/gen/state.c +UNITY_MEMBERS_ir_04 = \ + source/ir/regalloc_phi.c \ + source/ir/regalloc_entry.c \ + source/ir/regalloc_post.c +UNITY_MEMBERS_ir_05 = \ + source/ir/gen/put.c \ + source/ir/gen/params.c \ + source/ir/gen/arith.c \ + source/ir/gen/float.c \ + source/ir/gen/jump.c \ + source/ir/gen/asm.c \ + source/ir/gen/softfloat.c \ + source/ir/gen/config.c \ + source/ir/gen/live.c + +UNITY_GROUPS_machine = machine_01 +UNITY_MEMBERS_machine_01 = \ + source/machine/tccls.c \ + source/machine/tccmachine.c + +UNITY_GROUPS_memory = memory_01 +UNITY_MEMBERS_memory_01 = \ + source/memory/unique_ptr.c \ + source/memory/vector.c + +UNITY_GROUPS_obj = obj_01 obj_02 obj_03 +UNITY_MEMBERS_obj_01 = \ + source/obj/tccelf.c \ + source/obj/tccelf_sym.c \ + source/obj/tccelf_reloc.c \ + source/obj/tccelf_layout.c \ + source/obj/tccelf_output.c +UNITY_MEMBERS_obj_02 = \ + source/obj/tccelf_load.c \ + source/obj/tccelf_ld.c \ + source/obj/tccld.c +UNITY_MEMBERS_obj_03 = \ + source/obj/tccyaff.c \ + source/obj/tccdbg.c + +UNITY_GROUPS_opt = opt_01 opt_02 opt_03 opt_04 opt_05 opt_06 opt_07 opt_08 opt_09 opt_10 opt_11 opt_12 opt_13 opt_14 opt_15 opt_16 +UNITY_MEMBERS_opt_01 = \ + source/opt/util/pass_disable.c \ + source/opt/util/volatile_access.c \ + source/opt/util/const_eval.c \ + source/opt/util/const_string_eval.c \ + source/opt/util/cond_util.c \ + source/opt/util/block_scan.c \ + source/opt/util/purity.c \ + source/opt/util/expr_equal.c \ + source/opt/util/call_params.c \ + source/opt/util/vreg_query.c \ + source/opt/util/vreg_def_use.c \ + source/opt/util/xform.c \ + source/opt/util/ir_insert.c \ + source/opt/analysis/du_chains.c \ + source/opt/analysis/alias.c \ + source/opt/analysis/memref.c \ + source/opt/analysis/mem_ssa.c \ + source/opt/engine/ctx.c \ + source/opt/engine/run_gens.c \ + source/opt/engine/pipeline_run.c \ + source/opt/engine/pipeline_table.c \ + source/opt/engine/gen_adapters.c \ + source/opt/engine/pass_timing.c \ + source/opt/engine/fp_cache_shim.c \ + source/opt/engine/fp_mat_cache.c \ + source/opt/engine/pass_registry.c \ + source/opt/ra/const_branch_fold.c \ + source/opt/ra/incomplete_calls.c \ + source/opt/ra/phi_const_chain.c \ + source/opt/ssa/string/const_string_fold.c \ + source/opt/ssa/string/str_strlen.c \ + source/opt/flat/memory/mem_init.c +UNITY_MEMBERS_opt_02 = \ + source/opt/ra/umaal_fusion.c \ + source/opt/ssa/string/str_strcmp.c \ + source/opt/ssa/string/str_strcpy.c \ + source/opt/ssa/string/str_strspn.c \ + source/opt/ssa/string/str_strchr.c \ + source/opt/ssa/string/str_strstr.c \ + source/opt/ssa/string/str_memchr.c \ + source/opt/ssa/memory/global_addr_hoist.c \ + source/opt/ssa/memory/load_cse.c \ + source/opt/ssa/memory/diamond_store_fwd.c \ + source/opt/ssa/scalar/bitop_const_fold.c \ + source/opt/ssa/scalar/strength.c \ + source/opt/ssa/scalar/reassoc.c \ + source/opt/ssa/scalar/narrow.c \ + source/opt/ssa/scalar/var_imm_prop.c \ + source/opt/ssa/scalar/bool_norm.c \ + source/opt/ssa/cfg/switch_fold.c \ + source/opt/ssa/dce/unreachable.c +UNITY_MEMBERS_opt_03 = \ + source/opt/ssa/scalar/fold.c \ + source/opt/ssa/scalar/gvn.c \ + source/opt/ssa/scalar/cprop.c \ + source/opt/ssa/scalar/setif_or_taut.c \ + source/opt/ssa/scalar/cmp_offset_fold.c \ + source/opt/ssa/scalar/tmp_block_const.c \ + source/opt/ssa/cfg/phi.c \ + source/opt/ssa/cfg/or_bool_diamond.c \ + source/opt/ssa/dce/temp_worklist.c \ + source/opt/ssa/dce/dead_var_stores.c \ + source/opt/ssa/dce/dce_common.c +UNITY_MEMBERS_opt_04 = \ + source/opt/ssa/cfg/branch.c \ + source/opt/ssa/cfg/vrp.c \ + source/opt/ssa/dce/stackloc_stores.c \ + source/opt/ssa/dce/orphan_params.c \ + source/opt/ssa/dce/ret_path_frame_store.c \ + source/opt/ssa/dce/dead_phi_cycles.c \ + source/opt/ssa/dce/dead_overwrite_stores.c \ + source/opt/ssa/dce/dead_global_stores.c \ + source/opt/ssa/dce/var_liveness.c \ + source/opt/ssa/dce/dce.c \ + source/opt/ssa/loop/loop_rotate.c \ + source/opt/ssa/loop/first_iter_exit.c \ + source/opt/ssa/loop/loop_cand.c \ + source/opt/ssa/loop/loop_const_sim.c \ + source/opt/ssa/loop/iv_strength_reduction.c \ + source/opt/ssa/loop/decrement_to_zero.c +UNITY_MEMBERS_opt_05 = \ + source/opt/ssa/loop/ptr_iv_exit_subst.c \ + source/opt/ssa/loop/loop_unroll.c \ + source/opt/ssa/loop/loop_bottom_test.c \ + source/opt/ssa/loop/dead_loop.c \ + source/opt/ssa/scalar/sccp.c \ + source/opt/ssa/memory/global_store_dse.c \ + source/opt/ssa/memory/ptr_store_dse.c \ + source/opt/ssa/cfg/guard_collapse.c \ + source/opt/ssa/engine/use_def.c \ + source/opt/ssa/engine/rewrite.c \ + source/opt/ssa/engine/driver.c \ + source/opt/flat/scalar/bool.c \ + source/opt/flat/scalar/narrow_store.c +UNITY_MEMBERS_opt_06 = \ + source/opt/ssa/engine/stack_resolve.c \ + source/opt/flat/scalar/branch.c \ + source/opt/flat/scalar/stack_bool_diamond.c \ + source/opt/flat/scalar/const_string_calls.c \ + source/opt/flat/scalar/const_call_fold.c \ + source/opt/flat/scalar/addrof_const_fold.c \ + source/opt/flat/scalar/global_init_prop.c \ + source/opt/flat/scalar/symref_const_prop.c \ + source/opt/flat/scalar/stack_addr_fold.c \ + source/opt/flat/scalar/call_result.c \ + source/opt/flat/scalar/self_arith.c \ + source/opt/flat/scalar/self_copy.c \ + source/opt/flat/scalar/var_tmp_fwd.c \ + source/opt/flat/scalar/inline_param_copy.c \ + source/opt/flat/scalar/switch_collapse.c \ + source/opt/flat/fusion/indexed_chain.c \ + source/opt/flat/dce/trap_only_body_suppress.c +UNITY_MEMBERS_opt_07 = \ + source/opt/flat/scalar/known_bits.c \ + source/opt/flat/scalar/const_prop_tmp.c \ + source/opt/flat/scalar/value_tracking.c \ + source/opt/flat/scalar/const_var_prop.c \ + source/opt/flat/scalar/const_aggregate.c \ + source/opt/flat/scalar/add_reassoc.c \ + source/opt/flat/fusion/pair_reorder.c +UNITY_MEMBERS_opt_08 = \ + source/opt/flat/scalar/cmp_field_fuse.c \ + source/opt/flat/scalar/cmp_expr_fold.c \ + source/opt/flat/scalar/bitfield.c \ + source/opt/flat/memory/symaddr_cse.c \ + source/opt/flat/memory/dse.c \ + source/opt/flat/memory/var_store_elim.c \ + source/opt/flat/memory/entry_store_prop.c \ + source/opt/flat/memory/sret_nrvo.c \ + source/opt/flat/memory/dead_temp_local.c +UNITY_MEMBERS_opt_09 = \ + source/opt/flat/memory/sl_forward.c \ + source/opt/flat/memory/copy_source_load_fwd.c \ + source/opt/flat/memory/dead_local_slot.c \ + source/opt/flat/memory/dead_static_store.c +UNITY_MEMBERS_opt_10 = \ + source/opt/flat/memory/global_base_share.c \ + source/opt/flat/memory/byte_store_merge.c \ + source/opt/flat/memory/mem_inline.c \ + source/opt/flat/fusion/disp.c \ + source/opt/flat/fusion/gens_fusion.c \ + source/opt/flat/fusion/add_deref_fold.c \ + source/opt/flat/fusion/call_chain_rename.c \ + source/opt/flat/fusion/stackoff_addr_cse.c \ + source/opt/flat/fusion/lea_fold.c \ + source/opt/flat/fusion/lea_rmw_fold.c \ + source/opt/flat/fusion/assign_fuse.c \ + source/opt/flat/fusion/pack64.c \ + source/opt/flat/fusion/pack64_tautology.c \ + source/opt/flat/fusion/cmp_narrow_64.c \ + source/opt/flat/fusion/cmp_hi_only.c \ + source/opt/flat/fusion/cmp_imm_swap.c +UNITY_MEMBERS_opt_11 = \ + source/opt/flat/fusion/and64_narrow.c \ + source/opt/flat/fusion/shl32_or_chain.c \ + source/opt/flat/scalar/sdiv_pow2.c \ + source/opt/flat/memory/dead_lea_store.c \ + source/opt/flat/memory/store_inplace_arith.c \ + source/opt/flat/scalar/redundant_loop_check.c \ + source/opt/flat/cfg/if_convert.c \ + source/opt/flat/dce/body_essential.c \ + source/opt/flat/dce/noreturn_collapse.c \ + source/opt/flat/dce/infinite_loop_simplify.c \ + source/opt/flat/dce/infinite_self_recursion.c \ + source/opt/flat/dce/noreturn_call_epilogue.c \ + source/opt/flat/dce/dead_addrvar_elim.c \ + source/opt/flat/dce/dead_trailing_addrvar_store.c \ + source/opt/flat/dce/uninit_ub.c \ + source/opt/flat/cfg/returnvalue_merge.c +UNITY_MEMBERS_opt_12 = \ + source/opt/flat/dce/ub_only_body_elide.c \ + source/opt/flat/dce/const_return_uninit_elide.c \ + source/opt/flat/dce/null_store_dom_return.c \ + source/opt/flat/dce/dce.c \ + source/opt/flat/dce/compact_nops.c \ + source/opt/flat/dce/orphan_cmp_elim.c \ + source/opt/flat/dce/redundant_cmp.c \ + source/opt/flat/dce/zero_vla_elim.c \ + source/opt/flat/loop/loop_ir_mutate.c \ + source/opt/flat/loop/iv_analysis.c \ + source/opt/flat/loop/iv_strength_reduction.c \ + source/opt/flat/loop/loop_exit_analysis.c \ + source/opt/flat/loop/loop_eliminate.c \ + source/opt/flat/loop/decrement_to_zero.c +UNITY_MEMBERS_opt_13 = \ + source/opt/flat/loop/loop_unroll.c \ + source/opt/flat/loop/loop_rotate.c \ + source/opt/flat/loop/loop_relayout.c \ + source/opt/flat/loop/seq_guard_elim.c \ + source/opt/flat/loop/const_sim.c \ + source/opt/flat/loop/reroll.c \ + source/opt/flat/cfg/switch_to_data.c \ + source/opt/flat/cfg/jump_thread.c \ + source/opt/flat/cfg/bool_diamond_branch.c +UNITY_MEMBERS_opt_14 = \ + source/opt/flat/dce/dead_vla.c \ + source/opt/flat/cfg/jumpif_invert.c \ + source/opt/flat/scalar/var_to_tmp.c \ + source/opt/flat/loop/backedge_phi_hoist.c \ + source/opt/flat/cfg/abort_tail_merge.c \ + source/opt/flat/loop/loop_detect.c \ + source/opt/flat/loop/func_purity.c \ + source/opt/flat/loop/licm.c \ + source/opt/flat/loop/stack_param_promote.c \ + source/opt/flat/memory/invariant_global_load_hoist.c \ + source/opt/flat/memory/global_deref_cse.c \ + source/opt/flat/ipa/tu_noreturn.c +UNITY_MEMBERS_opt_15 = \ + source/opt/flat/memory/deref_operand_cse.c \ + source/opt/flat/memory/bitfield_unit_narrow.c \ + source/opt/flat/memory/rmw_byte_clear.c \ + source/opt/flat/memory/struct_copy_roundtrip_elim.c \ + source/opt/flat/fusion/barrel_shift.c \ + source/opt/flat/fusion/shift_pair_ubfx.c \ + source/opt/flat/fusion/shift64_dead_half.c \ + source/opt/flat/fusion/zero_half64.c \ + source/opt/flat/fusion/shift64_extract_ubfx.c \ + source/opt/flat/ipa/pure_via_sret.c \ + source/opt/flat/ipa/func_write_summary.c \ + source/opt/flat/ipa/tu_func_summary.c \ + source/opt/flat/ipa/tu_dead_statics.c \ + source/opt/flat/memory/dead_init_via_call.c \ + source/opt/flat/memory/stack_addr_cse.c \ + source/opt/flat/memory/block_copy_init.c \ + source/opt/flat/memory/small_memset_to_store.c +UNITY_MEMBERS_opt_16 = \ + source/opt/flat/memory/const_local_table.c \ + source/opt/flat/memory/small_global_memset_to_store.c \ + source/opt/flat/memory/addrof_var_fwd.c \ + source/opt/flat/memory/ptr_local_fwd.c \ + source/opt/flat/memory/param_copy_alias.c \ + source/opt/flat/memory/slot_const_store_fold.c \ + source/opt/flat/memory/sra.c \ + source/opt/flat/memory/memmove_to_indexed_stores.c + +UNITY_GROUPS_arm = arm_01 arm_02 +UNITY_MEMBERS_arm_01 = \ + source/backend/arch/arm/arm.c \ + source/backend/arch/arm/ssa_opt_arm.c +UNITY_MEMBERS_arm_02 = \ + source/backend/arch/arm/arm_aapcs.c \ + source/backend/arch/arm/arm_regalloc.c \ + source/backend/arch/arm/arm-link.c + +UNITY_GROUPS_thumb = thumb_01 thumb_02 thumb_03 thumb_04 +UNITY_MEMBERS_thumb_01 = \ + source/backend/arch/arm/thumb/thumb.c \ + source/backend/arch/arm/thumb/thop_mem_reg.c \ + source/backend/arch/arm/thumb/thop_mem_imm.c \ + source/backend/arch/arm/thumb/thop_bitfield.c \ + source/backend/arch/arm/thumb/thop_mov.c \ + source/backend/arch/arm/thumb/arm-thumb-gen.c \ + source/backend/arch/arm/thumb/arm-thumb-callsite.c +UNITY_MEMBERS_thumb_02 = \ + source/backend/arch/arm/thumb/thop_alu_imm.c \ + source/backend/arch/arm/thumb/thop_alu_reg.c \ + source/backend/arch/arm/thumb/thop_cmp.c \ + source/backend/arch/arm/thumb/thop_shift_imm.c \ + source/backend/arch/arm/thumb/thop_shift_reg.c \ + source/backend/arch/arm/thumb/thop_mem_unpriv.c \ + source/backend/arch/arm/thumb/thop_mem_exclusive.c \ + source/backend/arch/arm/thumb/thop_extend.c \ + source/backend/arch/arm/thumb/thop_rev.c \ + source/backend/arch/arm/thumb/thop_vfp.c \ + source/backend/arch/arm/thumb/thop_coproc.c \ + source/backend/arch/arm/thumb/thop_system.c \ + source/backend/arch/arm/thumb/thop_branch.c \ + source/backend/arch/arm/thumb/thop_block.c \ + source/backend/arch/arm/thumb/thop_mul.c \ + source/backend/arch/arm/thumb/thop_dsp.c \ + source/backend/arch/arm/thumb/thop_tbb.c \ + source/backend/arch/arm/thumb/thop_ldrd.c \ + source/backend/arch/arm/thumb/thop_ldrex.c \ + source/backend/arch/arm/thumb/thop_ldaex.c \ + source/backend/arch/arm/thumb/thop_mrs.c \ + source/backend/arch/arm/thumb/thop_pld.c \ + source/backend/arch/arm/thumb/thop_mvn.c \ + source/backend/arch/arm/thumb/thop_adr.c \ + source/backend/arch/arm/thumb/thop_ldr_literal.c +UNITY_MEMBERS_thumb_03 = \ + source/backend/arch/arm/thumb/arm-thumb-emit.c \ + source/backend/arch/arm/thumb/arm-thumb-alu.c +UNITY_MEMBERS_thumb_04 = \ + source/backend/arch/arm/thumb/arm-thumb-mem.c \ + source/backend/arch/arm/thumb/arm-thumb-fp.c \ + source/backend/arch/arm/thumb/arm-thumb-frame.c \ + source/backend/arch/arm/thumb/thumb-outline.c diff --git a/tests/fuzz/batch_sweep.py b/tests/fuzz/batch_sweep.py index f156bc6eb..ab714a118 100644 --- a/tests/fuzz/batch_sweep.py +++ b/tests/fuzz/batch_sweep.py @@ -769,6 +769,13 @@ def main(argv=None) -> int: # len(gvals) == 0: gcc built for no level -> nothing to compare, skip. divergent = sorted(set(olevels_bad) | set(vsgcc_bad)) + # Each divergent seed's per-level result, so triage starts from "which level + # is the odd one out" instead of a second sweep. Progress channel, so the + # stdout contract callers parse is untouched. + for s in divergent + gcc_inconsistent: + progress(" seed %d: %s" % (s, " ".join( + "%s=%s" % (o, signatures[s].get(o, "?")) for o in olevels))) + if gcc_levels: progress(f"\nswept {len(seeds)} seeds — {len(divergent)} divergent " f"(olevels={len(olevels_bad)}, vsgcc-only={len(vsgcc_bad)}, " diff --git a/tests/ir_tests/112_builtin_puts.c b/tests/ir_tests/112_builtin_puts.c index 796fbaa73..93cbe8d0e 100644 --- a/tests/ir_tests/112_builtin_puts.c +++ b/tests/ir_tests/112_builtin_puts.c @@ -4,7 +4,9 @@ int main(void) // Test __builtin_puts with simple string ret = __builtin_puts("Hello from __builtin_puts!"); - __builtin_printf("Return value: %d\n", ret); + /* The value itself is the libc's: newlib's puts returns '\n' (10), YasOS + * libc's the number of characters written (27). */ + __builtin_printf("Return value non-negative: %d\n", ret >= 0); // C only guarantees a non-negative return value on success. if (ret >= 0) { diff --git a/tests/ir_tests/112_builtin_puts.expect b/tests/ir_tests/112_builtin_puts.expect index 72b1c9d23..62222a887 100644 --- a/tests/ir_tests/112_builtin_puts.expect +++ b/tests/ir_tests/112_builtin_puts.expect @@ -1,3 +1,3 @@ Hello from __builtin_puts! -Return value: +Return value non-negative: 1 SUCCESS diff --git a/tests/ir_tests/466_alive_share_boundary_coholder.c b/tests/ir_tests/466_alive_share_boundary_coholder.c new file mode 100644 index 000000000..ca4822deb --- /dev/null +++ b/tests/ir_tests/466_alive_share_boundary_coholder.c @@ -0,0 +1,87 @@ +/* 466_alive_share_boundary_coholder.c — ra:alive_share leaves a SECOND live + * holder on a register, and the phi-hint BOUNDARY REUSE handed that register + * on without asking whether anyone else was still living there. + * + * When a phi-coalescing hint's partner expires exactly at cur->start, the + * allocator lets cur take partner's register (sources are read before the dest + * is written) and forces partner out of `active`, marking the register free. + * That is only sound when partner was the register's ONLY holder: alive_share + * deliberately parks a borrower on a register the allocator still shows as + * owned, and here the borrower (u6's pre-branch value, live across the whole + * if/else and read again at the join) outlived the partner. Its register was + * handed to cur and then marked free, so the join read a clobbered value — + * after the else arm ran, u6 read back as the then arm's value while the then + * arm's other assignments were untouched. + * + * This is the FIFTH wrong-code variant of this pass; the expire path guards it + * with survivor_int and eviction learned to in 030e6fa7. The same co-holder + * rule now covers the two other unguarded sites (the single-register boundary + * and the return/call pair boundary eviction). + * + * Verbatim fuzz repro: tests/fuzz combo_num seed 2412, reduced. `float` seed + * 3906 was the same bug. Pre-fix: -O1/-O2/-Os give 79ec3067; gcc -O0/-O2 and + * tcc -O0 all give da58982b. + */ +#include +#include +static unsigned csmix(unsigned h, unsigned v) +{ + h ^= v + 0x9e3779b9u + (h << 6) + (h >> 2); + h = (h << 13) | (h >> 19); + return h * 2654435761u; +} +static unsigned fbits_d(double d){ unsigned u[2]; memcpy(u, &d, sizeof u); return csmix(u[0], u[1]); } +static unsigned fbits_f(float f){ unsigned u; memcpy(&u, &f, sizeof u); return u; } +static unsigned helper1(unsigned pa, unsigned pb) +{ + unsigned lr = pa ^ (pb * 3u); + lr = (unsigned)(((unsigned)(((unsigned)(((unsigned)(pb) & (unsigned)(1081041013u))) < ((unsigned)(2680042698u) ^ lr))) + (unsigned)(((unsigned)(pa) | (unsigned)(1315446814u))))); + if ((unsigned)(pa) & 1u) lr += (unsigned)(((unsigned)(((unsigned)(3293082258u) >> ((unsigned)(lr) & 31u))) ^ (unsigned)(((unsigned)(1004440627u) / ((unsigned)(pb) | 1u))))); + return (unsigned)(pa) ^ lr; +} +static unsigned helper2(unsigned pa, unsigned pb) +{ + unsigned lr = pa ^ (pb * 3u); + return (unsigned)(((unsigned)(((unsigned)(((unsigned)(lr) << ((unsigned)(pa) & 31u))) - (unsigned)((((unsigned)(533731999u) & 1u) ? (unsigned)(3650960205u) : (unsigned)(pa))))) >= ((unsigned)(1337666855u) ^ lr))) ^ lr; +}; +int main(void) +{ + unsigned cs = 0x12345678u; + int s3 = (int)(1373052877u & 0xffffffff); + long s4 = (long)(1885788017u & 0xffffffff); + unsigned u5 = 3346008089u; + unsigned u6 = 2687936074u; + unsigned long long q7 = (((unsigned long long)(u6)) << 32) | (unsigned long long)(u5); + unsigned long long q8 = (((unsigned long long)(u6)) << 32) | (unsigned long long)(u5); + unsigned long long q9 = (((unsigned long long)(u5)) << 32) | (unsigned long long)(u6); + int si10 = -20415; + int si11 = 14839; + int si12 = -6927; + double f13 = -0x1.9c93100000000p+40; + float f14 = -0x1.ce8f3e0000000p+31f; + double f15 = 0x1.ef6c4a0000000p+23; + double f16 = 0x1.c5a18c0000000p+32; + u5 = (unsigned)((((unsigned)(((unsigned)(((unsigned)((((unsigned)(1335158747u) & 1u) ? (unsigned)(3818024347u) : (unsigned)(1922863576u))) >> ((unsigned)(((unsigned)(u6) - (unsigned)(((unsigned)(u6) ^ cs)))) & 31u))) + (unsigned)(((unsigned)(((unsigned)(2041510752u) % ((unsigned)(1706383214u) | 1u))) % ((unsigned)(((unsigned)(u5) * (unsigned)(1243632820u))) | 1u))))) & 1u) ? (unsigned)((~((unsigned)((unsigned)(s3)) | 0u))) : (unsigned)(helper1(((unsigned)(137600503u) + (unsigned)(((unsigned)(u5) - (unsigned)(((unsigned)(u5) ^ cs))))), ((unsigned)((~((unsigned)(u6) | 0u))) + (unsigned)(helper2(u6, (unsigned)(s3)))))))) & 0xffffffffu; + cs = csmix(cs, ((q9) >= (q8)) ? 1u : 0u); + if ((unsigned)(1758928585u) & 1u) { + cs = csmix(cs, ((q8) == (q7)) ? 1u : 0u); + cs = csmix(cs, (unsigned)(((si10) != (si12)) ? 1 : 0)); + q9 = (q8) & (9779368045435514120ull); + u6 = (unsigned)(((unsigned)(u5) * (unsigned)(((unsigned)(((unsigned)(((unsigned)(614381499u) + (unsigned)(4012616354u))) | (unsigned)(((unsigned)(u6) * (unsigned)(845630637u))))) * (unsigned)(((unsigned)(u5) <= ((unsigned)(u6) ^ cs))))))) & 0xffffffffu; + if ((unsigned)(((unsigned)((~((unsigned)(((unsigned)(u5) * (unsigned)(((unsigned)(u5) << ((unsigned)(u6) & 31u))))) | 0u))) % ((unsigned)((~((unsigned)((((unsigned)((((unsigned)((unsigned)(s4)) & 1u) ? (unsigned)(1153068592u) : (unsigned)(u6))) & 1u) ? (unsigned)(u6) : (unsigned)(((unsigned)(2360612077u) + (unsigned)(614655603u))))) | 0u))) | 1u))) & 1u) { + f13 = ((double)((unsigned)(u6))) / ((((double)(f16)) == (double)0) ? (double)1 : ((double)(f16))); + u6 = (unsigned)((((unsigned)(((unsigned)(u5) / ((unsigned)(((unsigned)(((unsigned)(u6) >= ((unsigned)(2513126341u) ^ cs))) % ((unsigned)((((unsigned)(1267472430u) & 1u) ? (unsigned)(3836295788u) : (unsigned)(502572453u))) | 1u))) | 1u))) & 1u) ? (unsigned)(((unsigned)(((unsigned)(u6) ^ (unsigned)(((unsigned)(2639922324u) & (unsigned)(u6))))) - (unsigned)(helper2(((unsigned)(3125754381u) ^ (unsigned)(3907707548u)), (unsigned)(s3))))) : (unsigned)(((unsigned)(helper1((((unsigned)((unsigned)(s4)) & 1u) ? (unsigned)(3978502467u) : (unsigned)(u5)), (((unsigned)((unsigned)(s4)) & 1u) ? (unsigned)(u5) : (unsigned)(u6)))) & (unsigned)(u5))))) & 0xffffffffu; + f14 = (0x1.e15bf20000000p+34f) + ((float)((unsigned)(u5))); + f14 = (f14 < -0x1p40f || f14 > 0x1p40f) ? (float)1 : f14; + } else { + cs = csmix(cs, (unsigned)(((((int)(short)(u5))) <= (9167)) ? 1 : 0)); + } + } + cs = csmix(cs, u6); + cs = csmix(cs, (unsigned)(q7) ^ (unsigned)(q7 >> 32)); + cs = csmix(cs, (unsigned)(q8) ^ (unsigned)(q8 >> 32)); + cs = csmix(cs, (unsigned)(q9) ^ (unsigned)(q9 >> 32)); + cs = csmix(cs, fbits_d(f13)); + cs = csmix(cs, fbits_f(f14)); + printf("checksum=%08x\n", cs); +} diff --git a/tests/ir_tests/466_alive_share_boundary_coholder.expect b/tests/ir_tests/466_alive_share_boundary_coholder.expect new file mode 100644 index 000000000..9d60fd2a0 --- /dev/null +++ b/tests/ir_tests/466_alive_share_boundary_coholder.expect @@ -0,0 +1 @@ +checksum=da58982b diff --git a/tests/ir_tests/472_pragma_pack_saved_bodies.c b/tests/ir_tests/472_pragma_pack_saved_bodies.c new file mode 100644 index 000000000..868c3513d --- /dev/null +++ b/tests/ir_tests/472_pragma_pack_saved_bodies.c @@ -0,0 +1,75 @@ +/* #pragma pack around function bodies that are saved as tokens and generated + away from their definition: static inline bodies replayed at call sites, and + -O1 deferred bodies (-finline-functions-called-once) generated at the end of + the translation unit or replayed at their only call. Each body must lay out + its structs under the pack state at its definition, and every directive must + take effect at its position in the file. */ +#include +#include + +/* A directive right after a body: the token after the closing brace was read + while the body was still being saved, and the pack(1) went into the body. */ +int after_body(void) { return 1; } +#pragma pack(1) +struct S1 { char c; int i; }; +#pragma pack() + +/* Defined under pack(1), generated at the end of the TU under pack(). */ +#pragma pack(1) +int local_under_pack(void) +{ + struct L1 { char c; int i; }; + return sizeof(struct L1); +} +#pragma pack() + +/* Defined under pack(2), replayed at a call site under pack(1). */ +#pragma pack(2) +static inline int inline_under_pack(int x) +{ + struct L2 { char c; int i; } v; + v.i = x; + return (int)sizeof v * 100 + (int)offsetof(struct L2, i) * 10 + v.i; +} +#pragma pack() + +/* Called once: its body is replayed at the call, under the caller's pack(1). */ +#pragma pack(push, 4) +static int once_under_pack(int x) +{ + struct L3 { char c; short s; int i; } v; + v.i = x; + return (int)sizeof v * 100 + (int)offsetof(struct L3, i) * 10 + v.i; +} +#pragma pack(pop) + +/* A directive inside a body keeps applying after it, in file order. */ +int body_with_pragma(void) +{ +#pragma pack(1) + struct L4 { char c; int i; }; + return sizeof(struct L4); +} +struct S5 { char c; int i; }; +#pragma pack() +struct S6 { char c; int i; }; + +int local_default(void) +{ + struct L5 { char c; int i; }; + return sizeof(struct L5); +} + +int main(void) +{ +#pragma pack(1) + struct M { char c; int i; }; + int a = inline_under_pack(3); + int b = once_under_pack(4); +#pragma pack() + printf("S1=%d L1=%d L4=%d S5=%d S6=%d L5=%d M=%d\n", (int)sizeof(struct S1), local_under_pack(), + body_with_pragma(), (int)sizeof(struct S5), (int)sizeof(struct S6), local_default(), + (int)sizeof(struct M)); + printf("inline=%d once=%d after=%d\n", a, b, after_body()); + return 0; +} diff --git a/tests/ir_tests/472_pragma_pack_saved_bodies.expect b/tests/ir_tests/472_pragma_pack_saved_bodies.expect new file mode 100644 index 000000000..6186a7236 --- /dev/null +++ b/tests/ir_tests/472_pragma_pack_saved_bodies.expect @@ -0,0 +1,2 @@ +S1=5 L1=5 L4=5 S5=5 S6=8 L5=8 M=5 +inline=623 once=844 after=1 diff --git a/tests/ir_tests/473_reassoc_unsigned_into_signed.c b/tests/ir_tests/473_reassoc_unsigned_into_signed.c new file mode 100644 index 000000000..27a95ba3c --- /dev/null +++ b/tests/ir_tests/473_reassoc_unsigned_into_signed.c @@ -0,0 +1,29 @@ +/* An unsigned add folded into a signed add must not inherit the signed add's + no-overflow assumption. add_reassoc merged `(int)(x + 1U) + 1` into + `x + 2`, then ssa:cmp_offset_fold folded `x + 2 < x` to false, but + x = INT_MAX gives INT_MIN + 1 < INT_MAX: true (gcc pr55137, reached once + main stopped inlining foo). */ +#include + +__attribute__((noinline)) int add_then_add(unsigned int x) +{ + return ((int)(x + 1U) + 1) < (int)x; +} + +__attribute__((noinline)) int sub_then_sub(unsigned int x) +{ + return ((int)(x - 1U) - 1) > (int)x; +} + +__attribute__((noinline)) int add_then_sub(unsigned int x) +{ + return ((int)(x + 3U) - 1) < (int)x; +} + +int main(void) +{ + printf("%d %d %d\n", add_then_add(__INT_MAX__), sub_then_sub((unsigned int)__INT_MAX__ + 1U), + add_then_sub(__INT_MAX__)); + printf("%d %d %d\n", add_then_add(5), sub_then_sub(5), add_then_sub(5)); + return 0; +} diff --git a/tests/ir_tests/473_reassoc_unsigned_into_signed.expect b/tests/ir_tests/473_reassoc_unsigned_into_signed.expect new file mode 100644 index 000000000..29c0cd5b5 --- /dev/null +++ b/tests/ir_tests/473_reassoc_unsigned_into_signed.expect @@ -0,0 +1,2 @@ +1 1 1 +0 0 0 diff --git a/tests/ir_tests/474_late_reopt_owed_inline_bodies.c b/tests/ir_tests/474_late_reopt_owed_inline_bodies.c new file mode 100644 index 000000000..712aa4d4f --- /dev/null +++ b/tests/ir_tests/474_late_reopt_owed_inline_bodies.c @@ -0,0 +1,28 @@ +/* The end-of-TU analyses must see every body the TU emits. A `static inline` + function whose address is taken is not expanded at a call site; its body was + parsed only when gen_inline_functions emitted it, after the late_reopt + global_init fold and the dead-static-store analysis had run: + - a static written only in that body folded to its zero initializer; + - a store to a static read only in that body was dropped as dead. */ +#include + +static int written_in_inline; +static inline void set_written(int v) { written_in_inline = v; } +__attribute__((noinline)) int get_written(void) { return written_in_inline; } + +static int read_in_inline; +static inline int get_read(void) { return read_in_inline; } +__attribute__((noinline)) void set_read(int v) { read_in_inline = v; } + +void (*volatile set_hook)(int); +int (*volatile get_hook)(void); + +int main(void) +{ + set_hook = set_written; + set_hook(5); + get_hook = get_read; + set_read(7); + printf("%d %d\n", get_written(), get_hook()); + return 0; +} diff --git a/tests/ir_tests/474_late_reopt_owed_inline_bodies.expect b/tests/ir_tests/474_late_reopt_owed_inline_bodies.expect new file mode 100644 index 000000000..149a6b321 --- /dev/null +++ b/tests/ir_tests/474_late_reopt_owed_inline_bodies.expect @@ -0,0 +1 @@ +5 7 diff --git a/tests/ir_tests/475_drop_unused_statics.c b/tests/ir_tests/475_drop_unused_statics.c new file mode 100644 index 000000000..6506654c2 --- /dev/null +++ b/tests/ir_tests/475_drop_unused_statics.c @@ -0,0 +1,64 @@ +/* -fdrop-unused-statics defers static functions and initialized static objects + to the end of the TU and emits only those something live refers to. Each + shape below is one the deferral has to get right: what is used must still + be defined, with its bytes, where the code that reads it can see them. */ +#include +#include + +/* Forward-declared const object defined later (zig's C output does this for + every constant): the tentative must wait in COMMON, not take .rodata bytes + the definition then fills. */ +struct pair { int a, b; }; +static const struct pair fwd_const; +static int read_fwd_const(void) { return fwd_const.a * 10 + fwd_const.b; } +static const struct pair fwd_const = {4, 2}; + +/* Unsized arrays: the saved initializer sizes them at the declaration, so a + sizeof further down (even at file scope) sees the complete type. */ +static const char greeting[] = "deferred"; +static const int greeting_len = sizeof(greeting); +static const int squares[] = {0, 1, 4, 9, 16}; + +/* Liveness through data: a table of function pointers reached only from a + function that is itself reached only through another table. */ +static int twice(int x) { return 2 * x; } +static int thrice(int x) { return 3 * x; } +static int (*const ops[])(int) = {twice, thrice}; +static int apply_ops(int x) { return ops[0](x) + ops[1](x); } +struct vt { int (*run)(int); }; +static const struct vt vtable = {apply_ops}; +static const struct vt *const vtable_ptr = &vtable; + +/* A block-scope prototype is a declaration, not a call to an unprototyped + function: the body stays deferred. */ +static int via_local_decl(int x) +{ + extern size_t strlen(const char *); + return x + (int)strlen(greeting); +} + +/* Kept whatever refers to it. */ +__attribute__((used)) static int used_counter = 11; +static int alias_target(void) { return 99; } +int aliased(void) __attribute__((alias("alias_target"))); + +/* Unused: dropped (the object-level check is test_drop_unused_statics.py). */ +static int never_called(int x) { return x + fwd_const.b; } +static const int never_read[4] = {1, 2, 3, 4}; + +/* A static written in one place and read in another, both deferred. */ +static int counter; +static void bump(void) { counter += 3; } + +int main(void) +{ + bump(); + bump(); + printf("fwd_const=%d\n", read_fwd_const()); + printf("greeting=%s len=%d sq=%d n=%d\n", greeting, greeting_len, squares[4], + (int)(sizeof(squares) / sizeof(squares[0]))); + printf("vtable=%d\n", vtable_ptr->run(5)); + printf("local_decl=%d\n", via_local_decl(1)); + printf("used=%d aliased=%d counter=%d\n", used_counter, aliased(), counter); + return 0; +} diff --git a/tests/ir_tests/475_drop_unused_statics.expect b/tests/ir_tests/475_drop_unused_statics.expect new file mode 100644 index 000000000..2b97bc868 --- /dev/null +++ b/tests/ir_tests/475_drop_unused_statics.expect @@ -0,0 +1,5 @@ +fwd_const=42 +greeting=deferred len=9 sq=16 n=5 +vtable=25 +local_decl=9 +used=11 aliased=99 counter=6 diff --git a/tests/ir_tests/476_write_only_statics.c b/tests/ir_tests/476_write_only_statics.c new file mode 100644 index 000000000..e8c5c1227 --- /dev/null +++ b/tests/ir_tests/476_write_only_statics.c @@ -0,0 +1,60 @@ +/* A static nothing reads has its stores removed (dead_static_store) and is + then collected from the object (gc_unreferenced_statics). "Nothing reads" + no longer rests on the frontend's addrtaken -- set by every `&x`, which + zig's C output writes for every field store -- but on the function + summaries and on the relocations of emitted data. Every static below IS + read, by a path those have to see; each must keep its stores. */ +#include +#include + +struct big { int v[12]; }; + +/* Read through a pointer handed to a function. */ +static int via_call; +__attribute__((noinline)) int read_through(const int *p) { return *p; } + +/* Read through a pointer stored in data. */ +static int via_data; +static int *const via_data_ptr = &via_data; + +/* Read by a function reached only through a table. */ +static int via_table; +static int reads_via_table(void) { return via_table; } +static int (*const table[])(void) = {reads_via_table}; + +/* Written by a struct copy (a memmove call), read back by another. */ +static struct big copied; +__attribute__((noinline)) struct big make(int k) +{ + struct big b; + for (int i = 0; i < 12; i++) + b.v[i] = k + i; + return b; +} + +/* Written through zig's field-store shape, read the same way. */ +struct holder { int a; struct big b; }; +static struct holder zig_shape; + +/* Written, never read: its stores and the object go (checked by + test_drop_unused_statics.py); the program cannot tell. */ +static struct holder write_only; + +int main(void) +{ + via_call = 11; + via_data = 22; + via_table = 33; + copied = make(40); + (*(&((struct holder *)&zig_shape)->a)) = 55; + (*(&((struct holder *)&zig_shape)->b)) = make(60); + (*(&((struct holder *)&write_only)->b)) = make(70); + write_only.a = 77; + + struct big out; + memcpy(&out, &copied, sizeof out); + printf("via_call=%d via_data=%d via_table=%d\n", read_through(&via_call), *via_data_ptr, table[0]()); + printf("copied=%d,%d zig_shape=%d,%d\n", out.v[0], out.v[11], (*(&((struct holder *)&zig_shape)->a)), + zig_shape.b.v[5]); + return 0; +} diff --git a/tests/ir_tests/476_write_only_statics.expect b/tests/ir_tests/476_write_only_statics.expect new file mode 100644 index 000000000..9832e0faa --- /dev/null +++ b/tests/ir_tests/476_write_only_statics.expect @@ -0,0 +1,2 @@ +via_call=11 via_data=22 via_table=33 +copied=40,51 zig_shape=55,65 diff --git a/tests/ir_tests/477_frame_copy_coalescing.c b/tests/ir_tests/477_frame_copy_coalescing.c new file mode 100644 index 000000000..35d09a52f --- /dev/null +++ b/tests/ir_tests/477_frame_copy_coalescing.c @@ -0,0 +1,91 @@ +/* Frame copy coalescing (frame.c, frame_find_copies + frame_colour): a + whole-object block copy whose source dies at the copy and whose destination + is born there gives both objects the same bytes, and the copy goes. The + shapes are zig's C, where a value moves through a chain of temporaries and + is then read through a pointer (Io_File_Reader_getSize): + + t17 = t16.payload; t18 = t17; t19 = &t18; ... t19->kind ... + + Each function prints what a correct copy leaves; the ones marked "must not" + keep a side in use across the copy, so coalescing there would be wrong. */ +#include + +struct stat_like { int kind; int size[10]; }; +struct errunion { struct stat_like payload; unsigned short error; }; /* zig: payload first */ + +__attribute__((noinline)) struct errunion get(int k) +{ + struct errunion e; + e.error = (unsigned short)(k < 0); + e.payload.kind = k; + for (int i = 0; i < 10; i++) + e.payload.size[i] = k * 10 + i; + return e; +} + +__attribute__((noinline)) int consume(const struct stat_like *p) { return p->kind * 1000 + p->size[9]; } + +/* The zig shape: coalesced. */ +__attribute__((noinline)) int zig_shape(int k) +{ + struct errunion t16 = get(k); + struct stat_like t17, t18; + const struct stat_like *t19; + if (t16.error) + return -1; + t17 = t16.payload; + t18 = t17; + t19 = (const struct stat_like *)&t18; + if (t19->kind == 5) + return consume(t19) + 1; + return consume(t19); +} + +/* Must not: the source is read after the copy. */ +__attribute__((noinline)) int source_read_after(int k) +{ + struct errunion t16 = get(k); + struct stat_like t17, t18; + t17 = t16.payload; + t18 = t17; + t18.kind = 77; + const struct stat_like *p = &t18; + return consume(p) * 100 + t17.kind; +} + +/* Must not: the destination holds a value before the copy that is read + after it. */ +__attribute__((noinline)) int dest_live_before(int k) +{ + struct errunion t16 = get(k), t20 = get(k + 1); + struct stat_like t17, t18; + t18 = t20.payload; + int before = t18.size[2]; + t17 = t16.payload; + t18 = t17; + const struct stat_like *p = &t18; + return before * 100000 + consume(p); +} + +/* In a loop, the chain rebuilt every iteration. */ +__attribute__((noinline)) int in_loop(int n) +{ + int total = 0; + for (int i = 0; i < n; i++) + { + struct errunion t16 = get(i); + struct stat_like t17, t18; + t17 = t16.payload; + t18 = t17; + const struct stat_like *p = &t18; + total += consume(p); + } + return total; +} + +int main(void) +{ + printf("zig_shape=%d,%d source_read_after=%d\n", zig_shape(5), zig_shape(3), source_read_after(4)); + printf("dest_live_before=%d in_loop=%d\n", dest_live_before(2), in_loop(4)); + return 0; +} diff --git a/tests/ir_tests/477_frame_copy_coalescing.expect b/tests/ir_tests/477_frame_copy_coalescing.expect new file mode 100644 index 000000000..d5c5a7590 --- /dev/null +++ b/tests/ir_tests/477_frame_copy_coalescing.expect @@ -0,0 +1,2 @@ +zig_shape=5060,3039 source_read_after=7704904 +dest_live_before=3202029 in_loop=6096 diff --git a/tests/ir_tests/478_inline_atomic_load_store.c b/tests/ir_tests/478_inline_atomic_load_store.c new file mode 100644 index 000000000..022f8c340 --- /dev/null +++ b/tests/ir_tests/478_inline_atomic_load_store.c @@ -0,0 +1,57 @@ +/* __atomic_load / __atomic_store of 1-, 2- and 4-byte objects are inline + (parse_atomic): an LDR/STR, with a DMB for the stronger orders -- no call, + so no runtime helper is needed (the ARMv8-M libtcc1 has none). Every + order, a run-time order, the generic pointer forms, and a value that must + come through the access unchanged at each width. */ +#include +#include + +#define RELAXED 0 +#define ACQUIRE 2 +#define RELEASE 3 +#define SEQ_CST 5 + +static uint8_t b8; +static uint16_t h16; +static uint32_t w32; +static int32_t s32 = -5; + +__attribute__((noinline)) uint32_t load_with(uint32_t *p, int order) +{ + uint32_t r; + __atomic_load(p, &r, order); + return r; +} + +__attribute__((noinline)) void store_with(uint32_t *p, uint32_t v, int order) +{ + __atomic_store(p, &v, order); +} + +int main(void) +{ + uint8_t v8 = 0xA5; + uint16_t v16 = 0xBEEF; + uint32_t v32 = 0xDEADBEEF, r32; + uint8_t r8; + uint16_t r16; + int32_t rs; + + __atomic_store(&b8, &v8, RELAXED); + __atomic_store(&h16, &v16, RELEASE); + __atomic_store(&w32, &v32, SEQ_CST); + __atomic_load(&b8, &r8, ACQUIRE); + __atomic_load(&h16, &r16, SEQ_CST); + __atomic_load(&w32, &r32, RELAXED); + __atomic_load(&s32, &rs, ACQUIRE); + printf("b8=%x h16=%x w32=%x s32=%d\n", r8, r16, (unsigned)r32, (int)rs); + + store_with(&w32, 1234, RELAXED); + unsigned a = load_with(&w32, ACQUIRE); + store_with(&w32, 5678, SEQ_CST); + unsigned b = load_with(&w32, SEQ_CST); + store_with(&w32, 42, 1 /* consume: not valid for a store, taken as seq_cst */); + unsigned c = load_with(&w32, 4 /* acq_rel: not valid for a load */); + printf("runtime orders: a=%u b=%u c=%u\n", a, b, c); + return 0; +} diff --git a/tests/ir_tests/478_inline_atomic_load_store.expect b/tests/ir_tests/478_inline_atomic_load_store.expect new file mode 100644 index 000000000..e5871188d --- /dev/null +++ b/tests/ir_tests/478_inline_atomic_load_store.expect @@ -0,0 +1,2 @@ +b8=a5 h16=beef w32=deadbeef s32=-5 +runtime orders: a=1234 b=5678 c=42 diff --git a/tests/ir_tests/479_dse_ambiguous_addr_copy.c b/tests/ir_tests/479_dse_ambiguous_addr_copy.c new file mode 100644 index 000000000..80df263ef --- /dev/null +++ b/tests/ir_tests/479_dse_ambiguous_addr_copy.c @@ -0,0 +1,87 @@ +/* DSE's write-only address tracking (dse.c, prop_tmp/prop_var): a temp that + holds one of two stack addresses is "ambiguous" (-2), and a copy of it must + stay ambiguous. The copy used to drop it, so the loads through the copy + marked nothing read, both addresses looked write-only, and DSE deleted the + stores that filled the objects. + + The shape is tinycc's own arm-thumb-gen.c: thumb_emit_data_processing_mop64 + takes two handler structs by value and, once inlined into + data_processing_mop_impl, picks one per half: + + const H *h = (half == 0) ? ®ular : &carry_h; ... h->reg(...) ... + + The self-hosted tcc then called through a stack address on every 64-bit + add, sub or compare. */ +#include + +typedef struct +{ + int (*imm)(int); + int (*reg)(int); +} H; + +__attribute__((noinline)) int fa(int x) { return x + 100; } +__attribute__((noinline)) int fb(int x) { return x + 200; } +__attribute__((noinline)) int fc(int x) { return x + 300; } +__attribute__((noinline)) int fd(int x) { return x + 400; } + +static void two_halves(int *out, int n, H regular, H carry_h, int uses) +{ + for (int half = 0; half < 2; half++) + { + const H *h = (half == 0) ? ®ular : &carry_h; + if (uses && n > 3) + out[half] = h->imm(n + half); + else + out[half] = h->reg(n - half); + } +} + +__attribute__((noinline)) void pick(int op, int *out, int n, int wide) +{ + H handler, carry; + int uses = 0; + switch (op) + { + case 0: + handler.imm = fa; + handler.reg = fb; + carry.imm = fc; + carry.reg = fd; + uses = 1; + break; + case 1: + handler.imm = fc; + handler.reg = fd; + carry = handler; + break; + case 2: + handler.imm = fb; + handler.reg = fa; + carry = handler; + break; + default: + return; + } + if (wide) + { + two_halves(out, n, handler, carry, uses); + return; + } + out[0] = handler.reg(n); + out[1] = -1; +} + +int main(void) +{ + int out[2]; + for (int op = 0; op < 3; op++) + for (int n = 2; n <= 5; n += 3) + { + pick(op, out, n, 1); + printf("wide op=%d n=%d: %d %d\n", op, n, out[0], out[1]); + } + pick(0, out, 7, 0); + printf("narrow op=0 n=7: %d %d\n", out[0], out[1]); + return 0; +} diff --git a/tests/ir_tests/479_dse_ambiguous_addr_copy.expect b/tests/ir_tests/479_dse_ambiguous_addr_copy.expect new file mode 100644 index 000000000..84cdd9181 --- /dev/null +++ b/tests/ir_tests/479_dse_ambiguous_addr_copy.expect @@ -0,0 +1,7 @@ +wide op=0 n=2: 202 401 +wide op=0 n=5: 105 306 +wide op=1 n=2: 402 401 +wide op=1 n=5: 405 404 +wide op=2 n=2: 102 101 +wide op=2 n=5: 105 104 +narrow op=0 n=7: 207 -1 diff --git a/tests/ir_tests/480_fputs_printf_result_used.c b/tests/ir_tests/480_fputs_printf_result_used.c new file mode 100644 index 000000000..713b08e1d --- /dev/null +++ b/tests/ir_tests/480_fputs_printf_result_used.c @@ -0,0 +1,49 @@ +/* The frontend lowers fputs(s, f) to fwrite(s, 1, strlen(s), f), and + printf("%s\n", s) to puts(s), when nothing reads the call's value. It + decided that by the `;` after the call, which `x = fputs(...);` and + `int r = printf(...);` have too, so their value became a constant 0. + + YasOS libc's puts is `ret = fputs(s, stdout); ... return ret + 1;`, which + then returned 1 for every string (ir_tests/112_builtin_puts on the device). + + Checked through printf, whose value (the number of characters written) is + the same in every libc; fputs only promises "non-negative", and newlib's + is 0, so a wrong 0 from fputs would not show here. */ +#include + +int g; + +__attribute__((noinline)) int printf_s_nl(const char *s) +{ + int n = printf("%s\n", s); + return n; +} + +__attribute__((noinline)) int printf_assign_global(const char *s) +{ + g = printf("%s\n", s); + return g; +} + +__attribute__((noinline)) int printf_in_expression(const char *s) +{ + return printf("%s\n", s) * 10; +} + +/* The statement forms keep their lowering: nothing reads the value. */ +__attribute__((noinline)) void statements(const char *s) +{ + printf("%s\n", s); + fputs(s, stdout); + fputs("\n", stdout); +} + +int main(void) +{ + int a = printf_s_nl("four"); + int b = printf_assign_global("seven!!"); + int c = printf_in_expression("xy"); + statements("stmt"); + printf("printf_s_nl=%d printf_assign_global=%d printf_in_expression=%d\n", a, b, c); + return (a == 5 && b == 8 && c == 30) ? 0 : 1; +} diff --git a/tests/ir_tests/480_fputs_printf_result_used.expect b/tests/ir_tests/480_fputs_printf_result_used.expect new file mode 100644 index 000000000..27d33ced5 --- /dev/null +++ b/tests/ir_tests/480_fputs_printf_result_used.expect @@ -0,0 +1,6 @@ +four +seven!! +xy +stmt +stmt +printf_s_nl=5 printf_assign_global=8 printf_in_expression=30 diff --git a/tests/ir_tests/481_narrow_local_arg_width.c b/tests/ir_tests/481_narrow_local_arg_width.c new file mode 100644 index 000000000..c5949c34d --- /dev/null +++ b/tests/ir_tests/481_narrow_local_arg_width.c @@ -0,0 +1,60 @@ +/* A char or short local whose address is taken lives in its stack home, and + a store through the pointer writes only its byte or halfword. Passed as + a variadic argument (no promotion in the IR) or to a char/short parameter, + the value went to the argument register with a word load, so the bytes + above it came from whatever the slot held before: on the device + ir_tests/478 printed b8=aaaaaaa5 h16=aaaabeef. + + scribble() leaves 0xA5 in the stack the callees below reuse, so a word + load shows up on any target, not just on one whose stack starts dirty. */ +#include +#include + +__attribute__((noinline)) void scribble(void) +{ + volatile uint8_t junk[128]; + for (int i = 0; i < 128; i++) + junk[i] = 0xA5; +} + +__attribute__((noinline)) void put8(uint8_t *p, uint8_t v) { *p = v; } +__attribute__((noinline)) void put16(uint16_t *p, uint16_t v) { *p = v; } + +/* Compares the whole register: a caller that loaded a word hands in bits + above the byte, which `v == 0x5a` would not look at after its own cast. */ +__attribute__((noinline)) int arg_is(uint32_t reg, uint32_t want) { return reg == want; } +__attribute__((noinline)) uint32_t raw_u8(uint8_t v) +{ + uint32_t r; + __asm__("mov %0, %1" : "=r"(r) : "r"(v)); + return r; +} + +__attribute__((noinline)) void variadic(void) +{ + uint8_t u8; + int8_t s8; + uint16_t u16; + int16_t s16; + put8(&u8, 0x5a); + put8((uint8_t *)&s8, 0xfb); + put16(&u16, 0xbeef); + put16((uint16_t *)&s16, 0xfffe); + printf("variadic: u8=%x s8=%d u16=%x s16=%d\n", u8, s8, u16, s16); +} + +__attribute__((noinline)) void narrow_param(void) +{ + uint8_t u8; + put8(&u8, 0x5a); + printf("narrow param: %s\n", arg_is(raw_u8(u8), 0x5a) ? "zero-extended" : "garbage above the byte"); +} + +int main(void) +{ + scribble(); + variadic(); + scribble(); + narrow_param(); + return 0; +} diff --git a/tests/ir_tests/481_narrow_local_arg_width.expect b/tests/ir_tests/481_narrow_local_arg_width.expect new file mode 100644 index 000000000..9de7d56d1 --- /dev/null +++ b/tests/ir_tests/481_narrow_local_arg_width.expect @@ -0,0 +1,2 @@ +variadic: u8=5a s8=-5 u16=beef s16=-2 +narrow param: zero-extended diff --git a/tests/ir_tests/482_sret_into_struct_param.c b/tests/ir_tests/482_sret_into_struct_param.c new file mode 100644 index 000000000..49a8b82fd --- /dev/null +++ b/tests/ir_tests/482_sret_into_struct_param.c @@ -0,0 +1,54 @@ +/* `s = f(s)` where `s` is a by-value struct parameter that arrived (at least + partly) in the caller's argument area. The call's sret pointer is `&s`, and + the argument move for it read the slot instead of computing its address, so + f wrote its result through the struct's first word -- gcc_execute/20040703-1 + exited 255 on the device at every -O level. + + Three placements: split across r1-r3 and the stack behind an sret pointer + (the torture test's num_rshift/num_lshift), split behind an int, and wholly + on the stack behind four ints. */ +#include + +struct S +{ + unsigned a, b; + int c, d; +}; + +__attribute__((noinline)) static struct S bump(struct S s, unsigned k) +{ + s.a += k; + s.b += 2 * k; + s.c += 3 * (int)k; + s.d += 4 * (int)k; + return s; +} + +__attribute__((noinline)) static struct S behind_sret(struct S s, unsigned k) +{ + s = bump(s, k); + s.d = -s.d; + return s; +} + +__attribute__((noinline)) static int behind_int(int x, struct S s) +{ + s = bump(s, (unsigned)x); + return (int)(s.a + s.b) + s.c + s.d; +} + +__attribute__((noinline)) static int on_stack(int p, int q, int r, int t, struct S s) +{ + s = bump(s, (unsigned)(p + q + r + t)); + return (int)(s.a * 1000 + s.b * 100) + s.c * 10 + s.d; +} + +int main(void) +{ + struct S s = {1, 2, 3, 4}; + struct S r = behind_sret(s, 10); + printf("%u %u %d %d\n", r.a, r.b, r.c, r.d); + printf("%d\n", behind_int(5, s)); + printf("%d\n", on_stack(1, 1, 1, 1, s)); + return 0; +} diff --git a/tests/ir_tests/482_sret_into_struct_param.expect b/tests/ir_tests/482_sret_into_struct_param.expect new file mode 100644 index 000000000..ab3b7b33d --- /dev/null +++ b/tests/ir_tests/482_sret_into_struct_param.expect @@ -0,0 +1,3 @@ +11 22 33 -44 +60 +6170 diff --git a/tests/ir_tests/483_narrow_temp_spill_word.c b/tests/ir_tests/483_narrow_temp_spill_word.c new file mode 100644 index 000000000..18d3ebc10 --- /dev/null +++ b/tests/ir_tests/483_narrow_temp_spill_word.c @@ -0,0 +1,90 @@ +/* A byte or halfword temporary that the register allocator spills is stored + to its slot as a word, because every reload of that slot is a word load. + The STORE handler wrote it with its own width, so the reload picked up the + slot's three stale bytes: gcc_execute/pr82524 computed `foo(y->c.b, w)` + from two such operands and got 0x55 instead of 0xff for the blue channel. + + bar() is pr82524's; enough values are live to spill. scribble() leaves + 0xA5 in the stack bar reuses, so the stale bytes are not zero on any + target. */ +#include + +struct S +{ + unsigned char b, g, r, a; +}; +union U +{ + struct S c; + unsigned v; +}; + +__attribute__((noinline)) void scribble(void) +{ + volatile unsigned char junk[256]; + for (int i = 0; i < 256; i++) + junk[i] = 0xA5; +} + +static inline unsigned char foo(unsigned char a, unsigned char b) +{ + return ((a + 1) * b) >> 8; +} + +__attribute__((noinline)) unsigned bar(union U *x, union U *y) +{ + union U z; + unsigned char v = x->c.a; + unsigned char w = foo(y->c.a, 255 - v); + z.c.r = foo(x->c.r, v) + foo(y->c.r, w); + z.c.g = foo(x->c.g, v) + foo(y->c.g, w); + z.c.b = foo(x->c.b, v) + foo(y->c.b, w); + z.c.a = 0; + return z.v; +} + +/* The same shape with halfwords and signed bytes. */ +static inline short hfoo(unsigned short a, unsigned short b) +{ + return (short)(((a + 1) * b) >> 16); +} + +__attribute__((noinline)) int hbar(unsigned short *x, unsigned short *y) +{ + unsigned short v = x[3]; + unsigned short w = (unsigned short)hfoo(y[3], 65535 - v); + short r = (short)(hfoo(x[2], v) + hfoo(y[2], w)); + short g = (short)(hfoo(x[1], v) + hfoo(y[1], w)); + short b = (short)(hfoo(x[0], v) + hfoo(y[0], w)); + return r + g * 3 + b * 7; +} + +__attribute__((noinline)) int sbar(signed char *x, signed char *y) +{ + signed char v = x[3]; + signed char w = (signed char)(y[3] - v); + int r = (x[2] + 1) * v + (y[2] + 1) * w; + int g = (x[1] + 1) * v + (y[1] + 1) * w; + int b = (x[0] + 1) * v + (y[0] + 1) * w; + return r + g * 3 + b * 7; +} + +int main(void) +{ + union U a, b; + a.c = (struct S){255, 255, 255, 0}; + /* Not 255: (255 + 1) * w >> 8 is w's own low byte, whatever its stale + upper bytes hold, which is how pr82524's inputs hid the bug on a host. */ + b.c = (struct S){200, 150, 100, 255}; + scribble(); + printf("%08x\n", bar(&a, &b)); + + unsigned short ha[4] = {65535, 65535, 65535, 0}, hb[4] = {65535, 65535, 65535, 65535}; + scribble(); + printf("%d\n", hbar(ha, hb)); + + signed char sa[4] = {-3, 5, -7, 2}, sb[4] = {4, -6, 8, -9}; + scribble(); + printf("%d\n", sbar(sa, sb)); + return 0; +} diff --git a/tests/ir_tests/483_narrow_temp_spill_word.expect b/tests/ir_tests/483_narrow_temp_spill_word.expect new file mode 100644 index 000000000..64a3e0ea8 --- /dev/null +++ b/tests/ir_tests/483_narrow_temp_spill_word.expect @@ -0,0 +1,3 @@ +006496c8 +-11 +-323 diff --git a/tests/ir_tests/484_struct_ternary_arg_copy.c b/tests/ir_tests/484_struct_ternary_arg_copy.c new file mode 100644 index 000000000..decd68533 --- /dev/null +++ b/tests/ir_tests/484_struct_ternary_arg_copy.c @@ -0,0 +1,85 @@ +/* `f(c ? g() : h())` where g and h return an 8-byte struct through sret: the + selected result is copied into the argument slot a word at a time, and + dead_temp_local took the by-value struct argument that reads the slot as a + 4-byte read, so it dropped the store of the second word. f got the first + word and whatever the stack held after it. The self-hosted tcc builds its + Thumb opcodes this way (`ot_check(u ? th_uxtb(..) : th_sxtb(..))`, an + 8-byte thumb_opcode) and emitted 0x0000 for them. + + scribble() fills the stack first, so an uncopied word is 0xA5A5A5A5 on any + target rather than a lucky zero. */ +#include +#include + +typedef struct op +{ + uint8_t size; + uint32_t opcode; +} op; + +typedef struct mo +{ + int kind; + unsigned is_unsigned : 1; + unsigned needs_deref : 1; + int btype; +} mo; + +typedef struct big +{ + int a, b, c, d, e; +} big; + +__attribute__((noinline)) void scribble(void) +{ + volatile unsigned char junk[256]; + for (int i = 0; i < 256; i++) + junk[i] = 0xA5; +} + +__attribute__((noinline)) op mk_u(int r) +{ + op o = {2, 0xb2c0u | (unsigned)r}; + return o; +} + +__attribute__((noinline)) op mk_s(int r) +{ + op o = {2, 0xb240u | (unsigned)r}; + return o; +} + +__attribute__((noinline)) big mk_big(int k) +{ + big b = {k, k + 1, k + 2, k + 3, k + 4}; + return b; +} + +__attribute__((noinline)) void check(op o) { printf("%u %04x\n", o.size, (unsigned)o.opcode); } + +__attribute__((noinline)) void check_big(big b) { printf("%d %d %d %d %d\n", b.a, b.b, b.c, b.d, b.e); } + +__attribute__((noinline)) void via_bitfield(mo d, int r) { check(d.is_unsigned ? mk_u(r) : mk_s(r)); } + +__attribute__((noinline)) void via_int(int u, int r) { check(u ? mk_u(r) : mk_s(r)); } + +__attribute__((noinline)) void via_big(int u) { check_big(u ? mk_big(10) : mk_big(20)); } + +int main(void) +{ + mo d = {3, 1, 0, 1}; + scribble(); + via_bitfield(d, 1); + d.is_unsigned = 0; + scribble(); + via_bitfield(d, 2); + scribble(); + via_int(1, 3); + scribble(); + via_int(0, 4); + scribble(); + via_big(1); + scribble(); + via_big(0); + return 0; +} diff --git a/tests/ir_tests/484_struct_ternary_arg_copy.expect b/tests/ir_tests/484_struct_ternary_arg_copy.expect new file mode 100644 index 000000000..c9ba35bef --- /dev/null +++ b/tests/ir_tests/484_struct_ternary_arg_copy.expect @@ -0,0 +1,6 @@ +2 b2c1 +2 b242 +2 b2c3 +2 b244 +10 11 12 13 14 +20 21 22 23 24 diff --git a/tests/ir_tests/485_frame_colour_struct_far.c b/tests/ir_tests/485_frame_colour_struct_far.c new file mode 100644 index 000000000..3d902777d --- /dev/null +++ b/tests/ir_tests/485_frame_colour_struct_far.c @@ -0,0 +1,70 @@ +/* A frame just over 32 KiB: small struct locals declared first, so the + frontend gives them offsets near the frame base, then a 32 KiB buffer whose + address escapes. At -O1 the frame relayout colours the hot struct copies + down to the frame bottom, below the buffer, and a struct-typed stack + operand keeps its offset in a 16-bit field: StackLoc[-32776] wrapped to + StackLoc[32760], so the by-value argument was read from 32 KiB past the + copy. Zig's std.AutoHashMapUnmanaged with a FixedBufferAllocator over a + 64 KiB stack buffer reported count=0 after 500 puts. + + scribble() fills the stack first, so a word read from the wrong slot is + 0xA5A5A5A5 rather than a lucky copy of the right one. */ +#include + +struct map +{ + unsigned *meta; + unsigned size, avail; +}; + +struct fba +{ + unsigned char *buf; + unsigned len, end; +}; + +__attribute__((noinline)) void scribble(void) +{ + volatile unsigned char junk[33024]; + for (unsigned i = 0; i < sizeof junk; i++) + junk[i] = 0xA5; +} + +__attribute__((noinline)) void put(struct map *m, struct fba *f, unsigned k) +{ + m->size++; + f->buf[f->end] = (unsigned char)k; + f->end += k; +} + +__attribute__((noinline)) unsigned count(struct map m) { return m.size; } +__attribute__((noinline)) unsigned avail(struct map m) { return m.avail; } + +__attribute__((noinline)) void run(unsigned n) +{ + struct map m2; + struct map m; + struct fba f1; + unsigned i; + unsigned char buf[32800]; + f1.buf = buf; + f1.len = sizeof buf; + f1.end = 0; + m = (struct map){0, 0, 7}; + for (i = 0; i < n; i++) + put(&m, &f1, i); + m2 = m; + unsigned c = count(m2); + m2 = m; + unsigned a = avail(m2); + printf("count=%u avail=%u end=%u byte=%u\n", c, a, f1.end, buf[f1.end - n + 1]); +} + +int main(void) +{ + scribble(); + run(10); + scribble(); + run(20); + return 0; +} diff --git a/tests/ir_tests/485_frame_colour_struct_far.expect b/tests/ir_tests/485_frame_colour_struct_far.expect new file mode 100644 index 000000000..8f3dc0b10 --- /dev/null +++ b/tests/ir_tests/485_frame_colour_struct_far.expect @@ -0,0 +1,2 @@ +count=10 avail=7 end=45 byte=9 +count=20 avail=7 end=190 byte=19 diff --git a/tests/ir_tests/486_addrtaken_escape_slot.c b/tests/ir_tests/486_addrtaken_escape_slot.c new file mode 100644 index 000000000..b35b9f4c3 --- /dev/null +++ b/tests/ir_tests/486_addrtaken_escape_slot.c @@ -0,0 +1,79 @@ +/* An address-taken local whose address leaves through a call's result (or + a store to memory) keeps its stack slot until the function ends. The + allocator followed the address only through copies and pointer arithmetic, + so `t = as_bytes(&b)` ended b's lifetime at the `&b`, and a later + address-taken local got the same slot while t still pointed into it: the + memcpy through t copied the low byte of the pointer stored there. Zig's + autoHash of a u8 is this shape (mem.asBytes, then Wyhash.update), and the + -O0-built Zig compiler hashed garbage and panicked `std.lang is corrupt`. */ +#include +#include +#include + +struct W +{ + uint32_t total_len, buf_len; + uint8_t buf[48]; +}; + +struct slice +{ + const uint8_t *ptr; + uintptr_t len; +}; + +__attribute__((noinline)) const uint8_t *as_bytes(const uint8_t *p) { return p; } + +__attribute__((noinline)) void update(struct W *const a0, uint8_t const a1) +{ + struct W *t5, *t7; + struct W *const *t8; + const uint8_t *t3; + struct slice t6, t9; + struct slice const *t10; + uint8_t t1; + t1 = a1; + t3 = as_bytes(&t1); + t6.ptr = t3; + t6.len = 1; + t7 = a0; + t8 = (struct W *const *)&t7; + t9 = t6; + t10 = &t9; + t5 = *t8; + t5->total_len += t10->len; + if (t5->buf_len + t10->len <= 48) + { + memcpy(t5->buf + t5->buf_len, t6.ptr, t10->len); + t5->buf_len += t10->len; + } +} + +static const uint8_t *volatile stash; + +/* The same through a store: the address is parked in memory and read back. */ +__attribute__((noinline)) unsigned via_store(unsigned k) +{ + uint8_t b; + unsigned r; + b = (uint8_t)k; + stash = &b; + unsigned *q; + unsigned w = 0xdeadbeefu; + q = &w; + *q ^= k; + r = *stash; + return r + (w == (0xdeadbeefu ^ k) ? 0 : 1000); +} + +int main(void) +{ + static struct W w; + static const uint8_t in[] = {0x5a, 0x3c, 0x01, 0xff}; + for (unsigned i = 0; i < sizeof in; i++) + update(&w, in[i]); + printf("%02x %02x %02x %02x len=%u total=%u\n", w.buf[0], w.buf[1], w.buf[2], w.buf[3], (unsigned)w.buf_len, + (unsigned)w.total_len); + printf("%u %u\n", via_store(0x21), via_store(0x7e)); + return 0; +} diff --git a/tests/ir_tests/486_addrtaken_escape_slot.expect b/tests/ir_tests/486_addrtaken_escape_slot.expect new file mode 100644 index 000000000..a2cd9207a --- /dev/null +++ b/tests/ir_tests/486_addrtaken_escape_slot.expect @@ -0,0 +1,2 @@ +5a 3c 01 ff len=4 total=4 +33 126 diff --git a/tests/ir_tests/487_tail_call_early_return.c b/tests/ir_tests/487_tail_call_early_return.c new file mode 100644 index 000000000..e4e933d88 --- /dev/null +++ b/tests/ir_tests/487_tail_call_early_return.c @@ -0,0 +1,48 @@ +/* A function whose one call is a tail call but which also returns earlier: + the tail call becomes a branch and no epilogue was emitted, while the + early returns still jumped to "the epilogue" -- the first instruction of + whatever function the linker put next. Zig's InternPool.Alignment.max + (`if (a == .none) return b; if (b == .none) return a; return maxStrict(a, + b);`) ran into the next function at -O1/-O2, and the tcc-built Zig + compiler crashed at startup. after() follows each function, so a missing + epilogue shows up as its result. */ +#include +#include + +typedef uint8_t al; + +__attribute__((noinline)) al max_strict(al a, al b) { return (a & 63) > (b & 63) ? a : b; } + +__attribute__((noinline)) al amax(al const a0, al const a1) +{ + if (a0 == 63) + return a1; + if (a1 == 63) + return a0; + return max_strict(a0, a1); +} + +__attribute__((noinline)) int after1(int x) { return x * 1000 + 7; } + +static int sink; +__attribute__((noinline)) void note(int v) { sink += v; } + +/* void flavour: an early `return;` before the tail call. */ +__attribute__((noinline)) void maybe_note(int v) +{ + if (v < 0) + return; + note(v); +} + +__attribute__((noinline)) int after2(int x) { return x * 2000 + 9; } + +int main(void) +{ + printf("%u %u %u %u\n", amax(63, 5), amax(4, 63), amax(3, 9), amax(12, 2)); + maybe_note(-1); + maybe_note(5); + maybe_note(-7); + printf("%d %d %d\n", sink, after1(1), after2(1)); + return 0; +} diff --git a/tests/ir_tests/487_tail_call_early_return.expect b/tests/ir_tests/487_tail_call_early_return.expect new file mode 100644 index 000000000..3c26fa123 --- /dev/null +++ b/tests/ir_tests/487_tail_call_early_return.expect @@ -0,0 +1,2 @@ +5 4 9 12 +5 1007 2009 diff --git a/tests/ir_tests/488_sccp_var_addr_not_slot.c b/tests/ir_tests/488_sccp_var_addr_not_slot.c new file mode 100644 index 000000000..f9f3f62c1 --- /dev/null +++ b/tests/ir_tests/488_sccp_var_addr_not_slot.c @@ -0,0 +1,261 @@ +/* SCCP matched stores and loads to frame slots by offset, and `&t0` (t0 an + address-taken local, zeroed by a memset through its address) resolves to + the local's spill placeholder -- an offset shared with unrelated anonymous + slots. Here it equalled the slot the by-value struct parameter is spilled + to, so in `case 0` the load of a0.payload became the memset's 0. This is + Zig's InternPool.AnalUnit.wrap as the Zig C backend emits it (zig.h helpers + replaced by casts; the collision depends on the exact local declaration + order, so the body is kept verbatim); under the tcc -O1/-O2 Zig compiler + every comptime unit packed a zero payload. */ +#include +#include +#include +typedef uint32_t enum__InternPool_ComptimeUnit_Id_80413; +typedef uint32_t enum__InternPool_Nav_Index_78147; +typedef uint32_t enum__InternPool_Index_21851; +typedef uint32_t enum__InternPool_MemoizedStateStage_80415; +typedef uint32_t enum__InternPool_AnalUnit_Kind_78055; +typedef uint64_t bitpack__InternPool_AnalUnit_78050; +struct InternPool_AnalUnit_Unwrapped_80411 { + union { + enum__InternPool_ComptimeUnit_Id_80413 comptime; + enum__InternPool_Nav_Index_78147 nav_val; + enum__InternPool_Nav_Index_78147 nav_ty; + enum__InternPool_Index_21851 type_layout; + enum__InternPool_Index_21851 struct_defaults; + enum__InternPool_Index_21851 func; + enum__InternPool_MemoizedStateStage_80415 memoized_state; + } payload; + enum__InternPool_AnalUnit_Kind_78055 tag; +}; +__attribute__((noinline)) +bitpack__InternPool_AnalUnit_78050 InternPool_AnalUnit_wrap__23013(struct InternPool_AnalUnit_Unwrapped_80411 const a0) { + uint64_t t5; + uint64_t t7; + bitpack__InternPool_AnalUnit_78050 t0; + enum__InternPool_AnalUnit_Kind_78055 t1; + enum__InternPool_ComptimeUnit_Id_80413 t2; + uint64_t *t3; + uint64_t *t4; + uint32_t t6; + uint64_t *t8; + enum__InternPool_Nav_Index_78147 t9; + enum__InternPool_Index_21851 t10; + enum__InternPool_MemoizedStateStage_80415 t11; + memset(&t0, 0x00, sizeof(bitpack__InternPool_AnalUnit_78050)); + t1 = a0.tag; + switch (t1) { + case UINT32_C(0): { + t2 = a0.payload.comptime; + t3 = (uint64_t *)&t0; + t4 = (uint64_t *)t3; + t5 = (*t4); + t5 = t5 & UINT64_C(18446744069414584320); + t6 = (uint32_t)(UINT32_C(0)); + t7 = (uint64_t)(t6); + t7 = t7 << UINT8_C(0); + t7 = t5 | t7; + (*t4) = t7; + goto zig_block_1; + +zig_block_1:; + t8 = (uint64_t *)&t0; + t6 = (uint32_t)(t2); + t4 = (uint64_t *)t8; + t7 = (*t4); + t7 = t7 & UINT64_C(4294967295); + t5 = (uint64_t)(t6); + t5 = t5 << UINT8_C(32); + t5 = t7 | t5; + (*t4) = t5; + goto zig_block_2; + +zig_block_2:; + goto zig_block_0; + } + case UINT32_C(1): { + t9 = a0.payload.nav_val; + t3 = (uint64_t *)&t0; + t4 = (uint64_t *)t3; + t7 = (*t4); + t7 = t7 & UINT64_C(18446744069414584320); + t6 = (uint32_t)(UINT32_C(1)); + t5 = (uint64_t)(t6); + t5 = t5 << UINT8_C(0); + t5 = t7 | t5; + (*t4) = t5; + goto zig_block_3; + +zig_block_3:; + t8 = (uint64_t *)&t0; + t6 = (uint32_t)(t9); + t4 = (uint64_t *)t8; + t7 = (*t4); + t7 = t7 & UINT64_C(4294967295); + t5 = (uint64_t)(t6); + t5 = t5 << UINT8_C(32); + t5 = t7 | t5; + (*t4) = t5; + goto zig_block_4; + +zig_block_4:; + goto zig_block_0; + } + case UINT32_C(2): { + t9 = a0.payload.nav_ty; + t3 = (uint64_t *)&t0; + t4 = (uint64_t *)t3; + t7 = (*t4); + t7 = t7 & UINT64_C(18446744069414584320); + t6 = (uint32_t)(UINT32_C(2)); + t5 = (uint64_t)(t6); + t5 = t5 << UINT8_C(0); + t5 = t7 | t5; + (*t4) = t5; + goto zig_block_5; + +zig_block_5:; + t8 = (uint64_t *)&t0; + t6 = (uint32_t)(t9); + t4 = (uint64_t *)t8; + t7 = (*t4); + t7 = t7 & UINT64_C(4294967295); + t5 = (uint64_t)(t6); + t5 = t5 << UINT8_C(32); + t5 = t7 | t5; + (*t4) = t5; + goto zig_block_6; + +zig_block_6:; + goto zig_block_0; + } + case UINT32_C(3): { + t10 = a0.payload.type_layout; + t3 = (uint64_t *)&t0; + t4 = (uint64_t *)t3; + t7 = (*t4); + t7 = t7 & UINT64_C(18446744069414584320); + t6 = (uint32_t)(UINT32_C(3)); + t5 = (uint64_t)(t6); + t5 = t5 << UINT8_C(0); + t5 = t7 | t5; + (*t4) = t5; + goto zig_block_7; + +zig_block_7:; + t8 = (uint64_t *)&t0; + t6 = (uint32_t)(t10); + t4 = (uint64_t *)t8; + t7 = (*t4); + t7 = t7 & UINT64_C(4294967295); + t5 = (uint64_t)(t6); + t5 = t5 << UINT8_C(32); + t5 = t7 | t5; + (*t4) = t5; + goto zig_block_8; + +zig_block_8:; + goto zig_block_0; + } + case UINT32_C(4): { + t10 = a0.payload.struct_defaults; + t3 = (uint64_t *)&t0; + t4 = (uint64_t *)t3; + t7 = (*t4); + t7 = t7 & UINT64_C(18446744069414584320); + t6 = (uint32_t)(UINT32_C(4)); + t5 = (uint64_t)(t6); + t5 = t5 << UINT8_C(0); + t5 = t7 | t5; + (*t4) = t5; + goto zig_block_9; + +zig_block_9:; + t8 = (uint64_t *)&t0; + t6 = (uint32_t)(t10); + t4 = (uint64_t *)t8; + t7 = (*t4); + t7 = t7 & UINT64_C(4294967295); + t5 = (uint64_t)(t6); + t5 = t5 << UINT8_C(32); + t5 = t7 | t5; + (*t4) = t5; + goto zig_block_10; + +zig_block_10:; + goto zig_block_0; + } + case UINT32_C(5): { + t10 = a0.payload.func; + t3 = (uint64_t *)&t0; + t4 = (uint64_t *)t3; + t7 = (*t4); + t7 = t7 & UINT64_C(18446744069414584320); + t6 = (uint32_t)(UINT32_C(5)); + t5 = (uint64_t)(t6); + t5 = t5 << UINT8_C(0); + t5 = t7 | t5; + (*t4) = t5; + goto zig_block_11; + +zig_block_11:; + t8 = (uint64_t *)&t0; + t6 = (uint32_t)(t10); + t4 = (uint64_t *)t8; + t7 = (*t4); + t7 = t7 & UINT64_C(4294967295); + t5 = (uint64_t)(t6); + t5 = t5 << UINT8_C(32); + t5 = t7 | t5; + (*t4) = t5; + goto zig_block_12; + +zig_block_12:; + goto zig_block_0; + } + case UINT32_C(6): { + t11 = a0.payload.memoized_state; + t3 = (uint64_t *)&t0; + t4 = (uint64_t *)t3; + t7 = (*t4); + t7 = t7 & UINT64_C(18446744069414584320); + t6 = (uint32_t)(UINT32_C(6)); + t5 = (uint64_t)(t6); + t5 = t5 << UINT8_C(0); + t5 = t7 | t5; + (*t4) = t5; + goto zig_block_13; + +zig_block_13:; + t8 = (uint64_t *)&t0; + t6 = (uint32_t)(t11); + t4 = (uint64_t *)t8; + t7 = (*t4); + t7 = t7 & UINT64_C(4294967295); + t5 = (uint64_t)(t6); + t5 = t5 << UINT8_C(32); + t5 = t7 | t5; + (*t4) = t5; + goto zig_block_14; + +zig_block_14:; + goto zig_block_0; + } + default: { + __builtin_unreachable(); + } + } + +zig_block_0:; + return t0; +} +int main(void) +{ + for (uint32_t k = 0; k < 7; k++) { + struct InternPool_AnalUnit_Unwrapped_80411 u; + u.payload.comptime = 0x1234 + k; + u.tag = k; + printf("%llx\n", (unsigned long long)InternPool_AnalUnit_wrap__23013(u)); + } + return 0; +} diff --git a/tests/ir_tests/488_sccp_var_addr_not_slot.expect b/tests/ir_tests/488_sccp_var_addr_not_slot.expect new file mode 100644 index 000000000..d2d71fa57 --- /dev/null +++ b/tests/ir_tests/488_sccp_var_addr_not_slot.expect @@ -0,0 +1,7 @@ +123400000000 +123500000001 +123600000002 +123700000003 +123800000004 +123900000005 +123a00000006 diff --git a/tests/ir_tests/489_reload_elim_pins_regs.c b/tests/ir_tests/489_reload_elim_pins_regs.c new file mode 100644 index 000000000..d8dadaf3e --- /dev/null +++ b/tests/ir_tests/489_reload_elim_pins_regs.c @@ -0,0 +1,117 @@ +/* Post-allocation reload elimination drops `T <- StackLoc[s] [LOAD]` when the + store that filled the slot read the very register T was allocated. With the + load gone T has no definition: it holds the value only because it shares + that register with the stored value (here the slice length, parameter P1 in + r1). Codegen's scratch-conflict fixup then moved T to a free callee-saved + register that nothing ever wrote, so the loop bound was r6's garbage. This + is Zig's std.mem.containsAtLeastScalar as the Zig C backend emits it (zig.h + helpers replaced by casts; debug_assert and isPowerOfTwo must stay + inlinable so the branch folds away); under the tcc -O1 Zig compiler + `zig build-obj` failed at startup with "unable to open zig lib directory: + BadPathName". */ +#include +#include +#include +void debug_assert__163(bool const a0) { + bool t0; + t0 = !a0; + if (t0) { + __builtin_unreachable(); + } + goto zig_block_0; + +zig_block_0:; + return; +} +struct slice_u8_49 { uint8_t const *ptr; uintptr_t len; }; +bool math_isPowerOfTwo__anon_6516__6850(void) { + debug_assert__163(true); + return true; +} +__attribute__((noinline)) +bool mem_containsAtLeastScalar__anon_10167__7865(struct slice_u8_49 const a0, uint8_t const a1, uintptr_t const a2) { + struct slice_u8_49 const *t1; + uintptr_t const *t2; + uintptr_t t3; + uintptr_t t8; + uintptr_t t14; + uintptr_t t16; + uintptr_t t4; + uintptr_t t5; + uintptr_t t7; + struct slice_u8_49 t9; + struct slice_u8_49 t0; + uint8_t const *t10; + uint32_t t11; + uint32_t t12; + bool t6; + uint8_t t13; + uint8_t t15; + t0 = a0; + t1 = (struct slice_u8_49 const *)&t0; + t2 = &t1->len; + t3 = (*t2); + t4 = (uintptr_t)0ul; + t5 = (uintptr_t)0ul; + t6 = math_isPowerOfTwo__anon_6516__6850(); + if (t6) { + goto zig_block_0; + } + goto zig_block_0; + +zig_block_0:; + t7 = (uintptr_t)0ul; + t8 = t4; + t9 = (*t1); + t10 = t9.ptr; + t10 = (uint8_t const *)(((uintptr_t)t10) + (t8*sizeof(uint8_t))); + t8 = t3 - t8; + t9.ptr = t10; + t9.len = t8; + t8 = t9.len; + zig_loop_31: + t3 = t7; + t11 = (uint32_t)(t3); + t12 = (uint32_t)(t8); + t6 = t11 < t12; + if (t6) { + t13 = t9.ptr[t3]; + t14 = t5; + t6 = t13 == a1; + t15 = (uint8_t)t6; + t16 = (uint32_t)(t15); + t16 = t14 + t16; + t5 = t16; + t16 = t5; + t12 = (uint32_t)(t16); + t11 = (uint32_t)(a2); + t6 = t12 >= t11; + if (t6) { + return true; + } + goto zig_block_3; + +zig_block_3:; + goto zig_block_2; + } + goto zig_block_1; + +zig_block_2:; + t3 = t3 + (uintptr_t)1ul; + t7 = t3; + goto zig_loop_31; + +zig_block_1:; + return false; +} +int main(void) +{ + static const uint8_t p1[] = "/home/x/lib"; + static const uint8_t p2[] = {'a', 0, 'b'}; + struct slice_u8_49 s1 = {p1, sizeof p1 - 1}, s2 = {p2, 3}; + printf("%d\n", mem_containsAtLeastScalar__anon_10167__7865(s1, 0, 1)); + printf("%d\n", mem_containsAtLeastScalar__anon_10167__7865(s2, 0, 1)); + printf("%d\n", mem_containsAtLeastScalar__anon_10167__7865(s1, '/', 3)); + printf("%d\n", mem_containsAtLeastScalar__anon_10167__7865(s1, '/', 4)); + return 0; +} diff --git a/tests/ir_tests/489_reload_elim_pins_regs.expect b/tests/ir_tests/489_reload_elim_pins_regs.expect new file mode 100644 index 000000000..f2599a746 --- /dev/null +++ b/tests/ir_tests/489_reload_elim_pins_regs.expect @@ -0,0 +1,4 @@ +0 +1 +1 +0 diff --git a/tests/ir_tests/490_dse_addr_through_var_add.c b/tests/ir_tests/490_dse_addr_through_var_add.c new file mode 100644 index 000000000..a73bb95e1 --- /dev/null +++ b/tests/ir_tests/490_dse_addr_through_var_add.c @@ -0,0 +1,195 @@ +/* dse's dead-stackloc scan follows an address taken of a frame object through + copies and pointer arithmetic, and counts a deref through any of them as a + read of the object. It followed ADD/SUB only from a TEMP into a TEMP, yet + treats every ADD of an address as safe arithmetic -- so `V <-- T ADD #8` + (the address into a VAR) was neither followed nor counted, and the stores + filling the object died although it was read through V. This is Zig's + Io.Threaded.dirCreateDirPath as the Zig C backend emits it: `t11 = + &t4.path; t12 = *t11` read a path whose stores were gone, and the tcc -O1 + Zig compiler failed "failed to create path 'h' in local cache directory". + The path iterator and the filesystem are simple stand-ins. */ +#include +#include +#include +#include +enum { zig_error_PathAlreadyExists = 39u, zig_error_FileNotFound = 42u, zig_error_NotDir = 43u, zig_error_BadPathName = 47u }; +typedef uint32_t enum__Io_File_Permissions__enum_3216_3216; +typedef uint8_t enum__Io_File_Kind_3902; +typedef uint8_t enum__Io_Dir_CreatePathStatus_3243; +struct slice_u8_49 { uint8_t const *ptr; uintptr_t len; }; +struct Io_Threaded_1733; +struct errunion_Io_Dir_CreatePathStatus_3243 { enum__Io_Dir_CreatePathStatus_3243 payload; uint16_t error; }; +struct Io_Dir_3194 { int32_t handle; }; +struct fs_path_ComponentIterator_28_posix_2cu8_29_12730 { struct slice_u8_49 path; uintptr_t root_len; uintptr_t root_end_index; uintptr_t start_index; uintptr_t end_index; }; +struct fs_path_ComponentIterator_28_posix_2cu8_29_Component_12739 { struct slice_u8_49 name; struct slice_u8_49 path; }; +struct opt_fs_path_ComponentIterator_28_posix_2cu8_29_Component_12741 { struct fs_path_ComponentIterator_28_posix_2cu8_29_Component_12739 payload; bool is_null; }; +struct errunion_void_34 { uint16_t error; }; +struct errunion_Io_File_Kind_3902 { enum__Io_File_Kind_3902 payload; uint16_t error; }; +struct fs_path_ComponentIterator_28_posix_2cu8_29_12730 fs_path_componentIterator__7103(struct slice_u8_49 a0); +struct opt_fs_path_ComponentIterator_28_posix_2cu8_29_Component_12741 fs_path_ComponentIterator_28_posix_2cu8_29_last__8209(struct fs_path_ComponentIterator_28_posix_2cu8_29_12730 *a0); +struct opt_fs_path_ComponentIterator_28_posix_2cu8_29_Component_12741 fs_path_ComponentIterator_28_posix_2cu8_29_previous__8212(struct fs_path_ComponentIterator_28_posix_2cu8_29_12730 *a0); +struct opt_fs_path_ComponentIterator_28_posix_2cu8_29_Component_12741 fs_path_ComponentIterator_28_posix_2cu8_29_next__8210(struct fs_path_ComponentIterator_28_posix_2cu8_29_12730 *a0); +struct errunion_void_34 Io_Threaded_dirCreateDirPosix__2914(void *a0, struct Io_Dir_3194 a1, struct slice_u8_49 a2, enum__Io_File_Permissions__enum_3216_3216 a3); +struct errunion_Io_File_Kind_3902 Io_Threaded_filePathKind__2929(struct Io_Threaded_1733 *a0, struct Io_Dir_3194 a1, struct slice_u8_49 a2); +struct errunion_Io_Dir_CreatePathStatus_3243 Io_Threaded_dirCreateDirPath__2917(void *const a0, struct Io_Dir_3194 const a1, struct slice_u8_49 const a2, enum__Io_File_Permissions__enum_3216_3216 const a3); +struct errunion_Io_Dir_CreatePathStatus_3243 Io_Threaded_dirCreateDirPath__2917(void *const a0, struct Io_Dir_3194 const a1, struct slice_u8_49 const a2, enum__Io_File_Permissions__enum_3216_3216 const a3) { + struct Io_Threaded_1733 *t0; + struct fs_path_ComponentIterator_28_posix_2cu8_29_12730 t2; + struct fs_path_ComponentIterator_28_posix_2cu8_29_12730 t1; + struct fs_path_ComponentIterator_28_posix_2cu8_29_Component_12739 t5; + struct fs_path_ComponentIterator_28_posix_2cu8_29_Component_12739 t8; + struct fs_path_ComponentIterator_28_posix_2cu8_29_Component_12739 t4; + struct opt_fs_path_ComponentIterator_28_posix_2cu8_29_Component_12741 t6; + void *t9; + void *t10; + struct slice_u8_49 *t11; + struct slice_u8_49 t12; + struct errunion_void_34 t13; + uint16_t t14; + struct errunion_Io_File_Kind_3902 t15; + uint16_t t16; + uint16_t t17; + struct errunion_Io_Dir_CreatePathStatus_3243 t18; + uint16_t t20; + bool t7; + enum__Io_File_Kind_3902 t19; + enum__Io_Dir_CreatePathStatus_3243 t21; + enum__Io_Dir_CreatePathStatus_3243 t3; + t0 = (struct Io_Threaded_1733 *)a0; + t2 = fs_path_componentIterator__7103(a2); + t1 = t2; + t3 = UINT8_C(0); + t6 = fs_path_ComponentIterator_28_posix_2cu8_29_last__8209(&t1); + t7 = !t6.is_null; + if (t7) { + t8 = t6.payload; + t5 = t8; + goto zig_block_0; + } + return (struct errunion_Io_Dir_CreatePathStatus_3243){ .error = zig_error_BadPathName, .payload = UINT8_C(0x0) }; + +zig_block_0:; + t4 = t5; + zig_loop_22: + t9 = (void *)t0; + t10 = t9; + t11 = (struct slice_u8_49 *)&t4.path; + t12 = (*t11); + t13 = Io_Threaded_dirCreateDirPosix__2914(t10, a1, t12, a3); + t7 = t13.error == UINT16_C(0); + if (t7) { + t3 = UINT8_C(1); + goto zig_block_2; + } + t14 = t13.error; + switch (t14) { + case zig_error_PathAlreadyExists: { + t11 = (struct slice_u8_49 *)&t4.path; + t12 = (*t11); + t15 = Io_Threaded_filePathKind__2929(t0, a1, t12); + if (t15.error) { + t16 = t15.error; + t17 = t16; + t18.payload = UINT8_C(0x0); + t18.error = t17; + return t18; + } + t19 = t15.payload; + t7 = t19 != UINT8_C(2); + if (t7) { + return (struct errunion_Io_Dir_CreatePathStatus_3243){ .error = zig_error_NotDir, .payload = UINT8_C(0x0) }; + } + goto zig_block_3; + +zig_block_3:; + goto zig_block_2; + } + case zig_error_FileNotFound: { + t6 = fs_path_ComponentIterator_28_posix_2cu8_29_previous__8212(&t1); + t7 = !t6.is_null; + if (t7) { + t8 = t6.payload; + t5 = t8; + goto zig_block_4; + } + return (struct errunion_Io_Dir_CreatePathStatus_3243){ .error = zig_error_FileNotFound, .payload = UINT8_C(0x0) }; + +zig_block_4:; + t4 = t5; + goto zig_block_1; + } + default: { + t20 = t14; + t17 = t20; + t18.payload = UINT8_C(0x0); + t18.error = t17; + return t18; + } + } + +zig_block_2:; + t6 = fs_path_ComponentIterator_28_posix_2cu8_29_next__8210(&t1); + t7 = !t6.is_null; + if (t7) { + t8 = t6.payload; + t5 = t8; + goto zig_block_5; + } + t21 = t3; + t18.payload = t21; + t18.error = UINT16_C(0); + return t18; + +zig_block_5:; + t4 = t5; + goto zig_block_1; + +zig_block_1:; + goto zig_loop_22; +} +typedef struct fs_path_ComponentIterator_28_posix_2cu8_29_12730 It; +typedef struct opt_fs_path_ComponentIterator_28_posix_2cu8_29_Component_12741 Opt; +/* end_index = end of the current component; start_index = its start */ +It fs_path_componentIterator__7103(struct slice_u8_49 a0) { It it = {a0, 0, 0, 0, 0}; return it; } +static Opt cur(It *it) { + Opt o; o.is_null = false; + o.payload.name.ptr = it->path.ptr + it->start_index; o.payload.name.len = it->end_index - it->start_index; + o.payload.path.ptr = it->path.ptr; o.payload.path.len = it->end_index; return o; +} +Opt fs_path_ComponentIterator_28_posix_2cu8_29_last__8209(It *it) { + if (!it->path.len) { Opt o; memset(&o, 0, sizeof o); o.is_null = true; return o; } + it->end_index = it->path.len; uintptr_t s = it->end_index; while (s && it->path.ptr[s - 1] != '/') s--; it->start_index = s; return cur(it); +} +Opt fs_path_ComponentIterator_28_posix_2cu8_29_previous__8212(It *it) { + if (it->start_index == 0) { Opt o; memset(&o, 0, sizeof o); o.is_null = true; return o; } + it->end_index = it->start_index - 1; uintptr_t s = it->end_index; while (s && it->path.ptr[s - 1] != '/') s--; it->start_index = s; return cur(it); +} +Opt fs_path_ComponentIterator_28_posix_2cu8_29_next__8210(It *it) { + if (it->end_index >= it->path.len) { Opt o; memset(&o, 0, sizeof o); o.is_null = true; return o; } + it->start_index = it->end_index + 1; uintptr_t e = it->start_index; while (e < it->path.len && it->path.ptr[e] != '/') e++; it->end_index = e; return cur(it); +} +static char dirs[16][32]; static int ndirs; +static int exists(struct slice_u8_49 p) { for (int i = 0; i < ndirs; i++) if (strlen(dirs[i]) == p.len && !memcmp(dirs[i], p.ptr, p.len)) return 1; return 0; } +struct errunion_void_34 Io_Threaded_dirCreateDirPosix__2914(void *a0, struct Io_Dir_3194 a1, struct slice_u8_49 a2, enum__Io_File_Permissions__enum_3216_3216 a3) { + struct errunion_void_34 r = {0}; + printf("mkdir %.*s (h=%d p=%o)\n", (int)a2.len, (const char *)a2.ptr, (int)a1.handle, (unsigned)a3); + if (exists(a2)) { r.error = zig_error_PathAlreadyExists; return r; } + uintptr_t s = a2.len; while (s && a2.ptr[s - 1] != '/') s--; + if (s) { struct slice_u8_49 par = {a2.ptr, s - 1}; if (!exists(par)) { r.error = zig_error_FileNotFound; return r; } } + memcpy(dirs[ndirs], a2.ptr, a2.len); dirs[ndirs++][a2.len] = 0; return r; +} +struct errunion_Io_File_Kind_3902 Io_Threaded_filePathKind__2929(struct Io_Threaded_1733 *a0, struct Io_Dir_3194 a1, struct slice_u8_49 a2) { + struct errunion_Io_File_Kind_3902 r = {2, 0}; return r; +} +int main(void) { + strcpy(dirs[ndirs++], "a"); + static const char p[] = "a/b/c"; + struct slice_u8_49 s = {(const uint8_t *)p, 5}; + struct errunion_Io_Dir_CreatePathStatus_3243 r = Io_Threaded_dirCreateDirPath__2917((void *)0x1234, (struct Io_Dir_3194){-100}, s, 0755); + printf("-> payload=%d error=%d\n", r.payload, r.error); + static const char q[] = "h"; + s.ptr = (const uint8_t *)q; s.len = 1; + r = Io_Threaded_dirCreateDirPath__2917((void *)0x1234, (struct Io_Dir_3194){-100}, s, 0755); + printf("-> payload=%d error=%d\n", r.payload, r.error); + return 0; +} diff --git a/tests/ir_tests/490_dse_addr_through_var_add.expect b/tests/ir_tests/490_dse_addr_through_var_add.expect new file mode 100644 index 000000000..2fdeebf2b --- /dev/null +++ b/tests/ir_tests/490_dse_addr_through_var_add.expect @@ -0,0 +1,6 @@ +mkdir a/b/c (h=-100 p=755) +mkdir a/b (h=-100 p=755) +mkdir a/b/c (h=-100 p=755) +-> payload=1 error=0 +mkdir h (h=-100 p=755) +-> payload=1 error=0 diff --git a/tests/ir_tests/491_block_copy_last_word_live.c b/tests/ir_tests/491_block_copy_last_word_live.c new file mode 100644 index 000000000..42bb6f311 --- /dev/null +++ b/tests/ir_tests/491_block_copy_last_word_live.c @@ -0,0 +1,52 @@ +/* Codegen turns a run of frame word copies (LOAD a slot, STORE it to another, + at consecutive offsets) into LDM/STM through scratch registers. Each loaded + value must die at its store; a run whose loads all use one register was + taken as proof of that, since each load overwrites the last -- but nothing + overwrites the run's last load. Here load_cse forwards the copy's last two + words (t37.body) to the by-value argument, so the last word's register is + read after the copy, and the block copy left it holding something else. + This is the shape of Zig's Air.Liveness.analyzeInstSwitchBr as the Zig C + backend emits it; it and seven other functions of the tcc -O1 Zig compiler + crashed. */ +#include +#include +#include +struct slice { uint32_t const *ptr; uintptr_t len; }; +struct Case { uint32_t items[5]; struct slice body; }; /* 28 bytes: body at 20 */ +struct OptCase { struct Case payload; bool is_null; }; +struct It { uint32_t n; uint32_t data[2][7]; }; +__attribute__((noinline)) struct OptCase next(struct It *it) { + struct OptCase o = {0}; + if (it->n >= 2) { o.is_null = true; return o; } + for (int i = 0; i < 5; i++) o.payload.items[i] = it->data[it->n][i]; + o.payload.body.ptr = &it->data[it->n][5]; o.payload.body.len = 2; + it->n++; + return o; +} +__attribute__((noinline)) uint32_t analyze(void *a, void *b, struct slice s) { + uint32_t h = 0; for (uintptr_t i = 0; i < s.len; i++) h = h * 31 + s.ptr[i]; return h + (a == b); +} +__attribute__((noinline)) uint32_t walk(void *a0, void *a1, struct It *it) { + struct OptCase t34; + struct Case t36, t37; + struct Case const *t38; + struct slice const *t39; + struct slice t40; + uint32_t h = 0; + for (;;) { + t34 = next(it); + if (t34.is_null) break; + t36 = t34.payload; + t37 = t36; + t38 = (struct Case const *)&t37; + t39 = (struct slice const *)&t38->body; + t40 = (*t39); + h = h * 7 + analyze(a0, a1, t40); + } + return h; +} +int main(void) { + struct It it = {0, {{1, 2, 3, 4, 5, 6, 7}, {8, 9, 10, 11, 12, 13, 14}}}; + printf("%u\n", (unsigned)walk(&it, &it.n, &it)); + return 0; +} diff --git a/tests/ir_tests/491_block_copy_last_word_live.expect b/tests/ir_tests/491_block_copy_last_word_live.expect new file mode 100644 index 000000000..33e82b722 --- /dev/null +++ b/tests/ir_tests/491_block_copy_last_word_live.expect @@ -0,0 +1 @@ +1776 diff --git a/tests/ir_tests/492_struct_roundtrip_addr_escape.c b/tests/ir_tests/492_struct_roundtrip_addr_escape.c new file mode 100644 index 000000000..c35d5c420 --- /dev/null +++ b/tests/ir_tests/492_struct_roundtrip_addr_escape.c @@ -0,0 +1,48 @@ +/* struct_copy_roundtrip_elim removes `memmove(B, A); memmove(A, B)` -- A is + unchanged -- when B is private to the pair. Its privacy check waved + through every address-of B and every call parameter touching B anywhere in + the function as the pair's own plumbing, so `p = &t40.ok` read later, and + `fill(&t40)` on another path, did not count: the copy into t40 went away + and the switch read t40's slot unwritten. This is the shape of Zig's + Sema.coerceExtra as the Zig C backend emits it (`t43 = t41.payload; + t40 = t43; t43 = t40;` then a switch through `&t40.payload.ok`), which + crashed the tcc -O1 Zig compiler. */ +#include +#include +struct Res { _Alignas(8) union { uint8_t ok; uint32_t w[16]; uint64_t d; } payload; uint8_t tag; }; /* 72 bytes */ +struct ErrRes { struct Res payload; uint16_t error; }; +__attribute__((noinline)) struct ErrRes check(int k) { + struct ErrRes r = {0}; + for (int i = 0; i < 16; i++) r.payload.payload.w[i] = 0x01010101u * (unsigned)(k + i); + r.payload.payload.ok = (uint8_t)(k % 5); + r.payload.tag = (uint8_t)(k & 1); + return r; +} +__attribute__((noinline)) int attrs(struct Res *r) { return r->payload.w[3] & 0xff; } +__attribute__((noinline)) int fill(struct Res *r, int k) { for (int i = 0; i < 16; i++) r->payload.w[i] = (uint32_t)(k * 3 + i); r->payload.ok = (uint8_t)k; r->tag = 1; return k; } +__attribute__((noinline)) int coerce(int k) { + struct ErrRes t41; + struct Res t43, t40; + uint8_t const *t49; + if (k > 100) { + fill(&t40, k); + t43 = t40; + return t43.payload.ok + attrs(&t40); + } + t41 = check(k); + if (t41.error) return -1; + t43 = t41.payload; + t40 = t43; + t43 = t40; + if (t43.tag == 0) return 100 + attrs(&t40); + t49 = (uint8_t const *)&t40.payload.ok; + switch (*t49) { + case 0: return 10; + case 1: return 11; + case 2: return 12; + case 3: return 13; + case 4: return 14; + default: return 99; + } +} +int main(void) { printf("%d %d %d %d %d\n", coerce(1), coerce(2), coerce(3), coerce(8), coerce(103)); return 0; } diff --git a/tests/ir_tests/492_struct_roundtrip_addr_escape.expect b/tests/ir_tests/492_struct_roundtrip_addr_escape.expect new file mode 100644 index 000000000..df5a440fa --- /dev/null +++ b/tests/ir_tests/492_struct_roundtrip_addr_escape.expect @@ -0,0 +1 @@ +11 105 13 111 159 diff --git a/tests/ir_tests/493_sccp_var_store_not_slot.c b/tests/ir_tests/493_sccp_var_store_not_slot.c new file mode 100644 index 000000000..01d700ab3 --- /dev/null +++ b/tests/ir_tests/493_sccp_var_store_not_slot.c @@ -0,0 +1,177 @@ +/* SCCP matches stores and loads to frame slots by offset. A store to a VAR + (`V8 <-- #0`, the zeroing memset of an address-taken local) carries the + VAR's spill placeholder, not a slot -- every load site refuses to match a + VAR operand by its offset, but the store side took it as one. Here the + placeholder equalled the slot the by-value parameter a0 is spilled to, so + in case 0 the load of a0.payload folded to the memset's 0. This is Zig's + InternPool.CaptureValue.wrap as the Zig C backend emits it (zig.h helpers + replaced by equivalents; the collision depends on the exact local order, + so the body is kept verbatim). Same bug class as test 488. */ +#include +#include +#include +typedef uint32_t enum__InternPool_Index_21851; +typedef uint32_t enum__InternPool_Nav_Index_78147; +typedef uint32_t bitpack__InternPool_CaptureValue_80627; +typedef uint8_t enum___40typeInfo_28InternPool_CaptureValue_Unwrapped_29__40_22union_22_tag_type__3f_197373; +struct InternPool_CaptureValue_Unwrapped_197371 { + union { + enum__InternPool_Index_21851 comptime; + enum__InternPool_Index_21851 runtime; + enum__InternPool_Nav_Index_78147 nav_val; + enum__InternPool_Nav_Index_78147 nav_ref; + } payload; + enum___40typeInfo_28InternPool_CaptureValue_Unwrapped_29__40_22union_22_tag_type__3f_197373 tag; +}; +static inline uint8_t zig_u8_bitCast_u8(uint8_t v, uint8_t b) { (void)b; return v; } +static inline uint32_t zig_u32_bitCast_u32(uint32_t v, uint8_t b) { (void)b; return v; } +static inline uint32_t zig_u32_intCast_u8(uint8_t v) { return v; } +static inline uint32_t zig_u32_intCast_u32(uint32_t v) { return v; } +__attribute__((noinline)) +bitpack__InternPool_CaptureValue_80627 InternPool_CaptureValue_wrap__30813(struct InternPool_CaptureValue_Unwrapped_197371 const a0) { + enum__InternPool_Index_21851 t2; + uint32_t *t3; + uint32_t *t4; + uint32_t t5; + uint32_t t7; + uint32_t *t8; + uint32_t t9; + enum__InternPool_Nav_Index_78147 t10; + bitpack__InternPool_CaptureValue_80627 t0; + enum___40typeInfo_28InternPool_CaptureValue_Unwrapped_29__40_22union_22_tag_type__3f_197373 t1; + uint8_t t6; + memset(&t0, 0x00, sizeof(bitpack__InternPool_CaptureValue_80627)); + t1 = a0.tag; + switch (t1) { + case UINT8_C(0): { + t2 = a0.payload.comptime; + t3 = (uint32_t *)&t0; + t4 = (uint32_t *)t3; + t5 = (*t4); + t5 = t5 & UINT32_C(4294967292); + t6 = zig_u8_bitCast_u8(UINT8_C(0), UINT8_C(2)); + t7 = zig_u32_intCast_u8(t6); + t7 = t7 << UINT8_C(0); + t7 = t5 | t7; + (*t4) = t7; + goto zig_block_1; + +zig_block_1:; + t8 = (uint32_t *)&t0; + t7 = zig_u32_bitCast_u32(t2, UINT8_C(32)); + t9 = zig_u32_intCast_u32(t7); + t4 = (uint32_t *)t8; + t7 = (*t4); + t7 = t7 & UINT32_C(3); + t5 = zig_u32_intCast_u32(t9); + t5 = t5 << UINT8_C(2); + t5 = t7 | t5; + (*t4) = t5; + goto zig_block_2; + +zig_block_2:; + goto zig_block_0; + } + case UINT8_C(1): { + t2 = a0.payload.runtime; + t3 = (uint32_t *)&t0; + t4 = (uint32_t *)t3; + t7 = (*t4); + t7 = t7 & UINT32_C(4294967292); + t6 = zig_u8_bitCast_u8(UINT8_C(1), UINT8_C(2)); + t5 = zig_u32_intCast_u8(t6); + t5 = t5 << UINT8_C(0); + t5 = t7 | t5; + (*t4) = t5; + goto zig_block_3; + +zig_block_3:; + t8 = (uint32_t *)&t0; + t7 = zig_u32_bitCast_u32(t2, UINT8_C(32)); + t9 = zig_u32_intCast_u32(t7); + t4 = (uint32_t *)t8; + t7 = (*t4); + t7 = t7 & UINT32_C(3); + t5 = zig_u32_intCast_u32(t9); + t5 = t5 << UINT8_C(2); + t5 = t7 | t5; + (*t4) = t5; + goto zig_block_4; + +zig_block_4:; + goto zig_block_0; + } + case UINT8_C(2): { + t10 = a0.payload.nav_val; + t3 = (uint32_t *)&t0; + t4 = (uint32_t *)t3; + t7 = (*t4); + t7 = t7 & UINT32_C(4294967292); + t6 = zig_u8_bitCast_u8(UINT8_C(2), UINT8_C(2)); + t5 = zig_u32_intCast_u8(t6); + t5 = t5 << UINT8_C(0); + t5 = t7 | t5; + (*t4) = t5; + goto zig_block_5; + +zig_block_5:; + t8 = (uint32_t *)&t0; + t7 = zig_u32_bitCast_u32(t10, UINT8_C(32)); + t9 = zig_u32_intCast_u32(t7); + t4 = (uint32_t *)t8; + t7 = (*t4); + t7 = t7 & UINT32_C(3); + t5 = zig_u32_intCast_u32(t9); + t5 = t5 << UINT8_C(2); + t5 = t7 | t5; + (*t4) = t5; + goto zig_block_6; + +zig_block_6:; + goto zig_block_0; + } + case UINT8_C(3): { + t10 = a0.payload.nav_ref; + t3 = (uint32_t *)&t0; + t4 = (uint32_t *)t3; + t7 = (*t4); + t7 = t7 & UINT32_C(4294967292); + t6 = zig_u8_bitCast_u8(UINT8_C(3), UINT8_C(2)); + t5 = zig_u32_intCast_u8(t6); + t5 = t5 << UINT8_C(0); + t5 = t7 | t5; + (*t4) = t5; + goto zig_block_7; + +zig_block_7:; + t8 = (uint32_t *)&t0; + t7 = zig_u32_bitCast_u32(t10, UINT8_C(32)); + t9 = zig_u32_intCast_u32(t7); + t4 = (uint32_t *)t8; + t7 = (*t4); + t7 = t7 & UINT32_C(3); + t5 = zig_u32_intCast_u32(t9); + t5 = t5 << UINT8_C(2); + t5 = t7 | t5; + (*t4) = t5; + goto zig_block_8; + +zig_block_8:; + goto zig_block_0; + } + default: __builtin_unreachable(); + + } + +zig_block_0:; + return t0; +} +int main(void) { + for (int tag = 0; tag < 4; tag++) { + struct InternPool_CaptureValue_Unwrapped_197371 u; + u.payload.comptime = 0x1234 + tag; + u.tag = (uint8_t)tag; + printf("%d: %08x\n", tag, (unsigned)InternPool_CaptureValue_wrap__30813(u)); + } + return 0; +} diff --git a/tests/ir_tests/493_sccp_var_store_not_slot.expect b/tests/ir_tests/493_sccp_var_store_not_slot.expect new file mode 100644 index 000000000..1dcab5de2 --- /dev/null +++ b/tests/ir_tests/493_sccp_var_store_not_slot.expect @@ -0,0 +1,4 @@ +0: 000048d0 +1: 000048d5 +2: 000048da +3: 000048df diff --git a/tests/ir_tests/494_dead_addrvar_store_indexed.c b/tests/ir_tests/494_dead_addrvar_store_indexed.c new file mode 100644 index 000000000..772004770 --- /dev/null +++ b/tests/ir_tests/494_dead_addrvar_store_indexed.c @@ -0,0 +1,45 @@ +/* dead_addrvar deletes the LEA of an address-taken local that is never read, + and every STORE through that LEA. A memset lowered by mem_inline writes + through STORE_INDEXED, which carries its base in a plain dest: the pass + neither counted it as a use nor deleted it, so the stores stayed behind and + wrote through the TEMP whose LEA was gone -- through whatever register the + allocator gave it. This is Zig's meta.eql for a u64 key as the Zig C backend + emits it: at -O1 the two zeroing memsets stored eight zeros through the + caller's r4 and r5, here the canary pointers of probe(). */ +#include +#include +#include +#include +typedef uint64_t bitpack__InternPool_AnalUnit_78050; +__attribute__((noinline)) +bool meta_eql__anon_81003__30906(bitpack__InternPool_AnalUnit_78050 const a0, bitpack__InternPool_AnalUnit_78050 const a1) { + bitpack__InternPool_AnalUnit_78050 t0; + bitpack__InternPool_AnalUnit_78050 t1; + bool t2; + memset(&t0, 0x00, sizeof(bitpack__InternPool_AnalUnit_78050)); + t0 = a0; + memset(&t1, 0x00, sizeof(bitpack__InternPool_AnalUnit_78050)); + t1 = a1; + t2 = a0 == a1; + return t2; +} +/* Four pointers live across the call land in the callee-saved registers, so + the stray stores zero one of the canaries -- a visible difference on bare + metal too, where a store through a small integer would go unnoticed. */ +__attribute__((noinline)) +int probe(uint64_t *c0, uint64_t *c1, uint64_t *c2, uint64_t *c3, uint64_t x, uint64_t y) { + int eq = meta_eql__anon_81003__30906(x, y); + *c0 += 1; *c1 += 2; *c2 += 3; *c3 += 4; + return eq; +} +int main(void) { + uint64_t c[4] = { 0x1111111111111111ull, 0x2222222222222222ull, + 0x3333333333333333ull, 0x4444444444444444ull }; + int hits = 0; + for (int i = 0; i < 4; i++) + hits += probe(&c[0], &c[1], &c[2], &c[3], 0x100000000ull * i + i, (i & 1) ? 0x100000000ull * i + i : 5); + for (int i = 0; i < 4; i++) + printf("c%d=%016llx\n", i, (unsigned long long)c[i]); + printf("hits=%d\n", hits); + return 0; +} diff --git a/tests/ir_tests/494_dead_addrvar_store_indexed.expect b/tests/ir_tests/494_dead_addrvar_store_indexed.expect new file mode 100644 index 000000000..cad03254b --- /dev/null +++ b/tests/ir_tests/494_dead_addrvar_store_indexed.expect @@ -0,0 +1,5 @@ +c0=1111111111111115 +c1=222222222222222a +c2=333333333333333f +c3=4444444444444454 +hits=2 diff --git a/tests/ir_tests/495_stack_struct_arg_base_in_lr.c b/tests/ir_tests/495_stack_struct_arg_base_in_lr.c new file mode 100644 index 000000000..423f1aced --- /dev/null +++ b/tests/ir_tests/495_stack_struct_arg_base_in_lr.c @@ -0,0 +1,43 @@ +/* A by-value struct argument copied to the outgoing stack area needs a base + register when its source is out of LDR/LDRD immediate reach -- here rs[60], + 1200 bytes up the frame. With every register holding a value (v1-v8 across + the call, the scalar arguments not yet stored), the scratch for that base + was LR, and the copy loaded its first pair into LR too: the base was + overwritten and the last word read through the loaded value. With the base + out of LR, the second data register was find_call_scratch's last resort IP, + which here still holds the argument x3. Zig's AstGen.containerDecl passing + a 20-byte ResultInfo split across r2-r3 and the stack segfaulted on the + first. */ +#include + +struct RI +{ + int a, b, c, d, e; +}; + +__attribute__((noinline)) int callee(int p, int q, struct RI r, int x1, int x2, int x3, int x4, int x5, int x6) +{ + return p + q * 2 + r.a * 3 + r.b * 5 + r.c * 7 + r.d * 11 + r.e * 13 + x1 * 17 + x2 * 19 + x3 * 23 + x4 * 29 + + x5 * 31 + x6 * 37; +} + +__attribute__((noinline)) void fill(struct RI *rs, int n) +{ + for (int i = 0; i < n; i++) + rs[i] = (struct RI){i, i + 1, i + 2, i + 3, i + 4}; +} + +__attribute__((noinline)) int f(int n) +{ + struct RI rs[64]; + int v1 = n * 3, v2 = n + 7, v3 = n ^ 0x55, v4 = n * n, v5 = n - 9, v6 = n << 3, v7 = n | 0x100, v8 = n * 13; + fill(rs, 64); + int t = callee(n * 41, n * 43, rs[60], n * 47, n * 53, n * 59, n * 61, n * 67, n * 71); + return t + v1 + v2 + v3 + v4 + v5 + v6 + v7 + v8; +} + +int main(void) +{ + printf("%d\n", f(3)); + return 0; +} diff --git a/tests/ir_tests/495_stack_struct_arg_base_in_lr.expect b/tests/ir_tests/495_stack_struct_arg_base_in_lr.expect new file mode 100644 index 000000000..64575435e --- /dev/null +++ b/tests/ir_tests/495_stack_struct_arg_base_in_lr.expect @@ -0,0 +1 @@ +32163 diff --git a/tests/ir_tests/496_frame_colour_ptr_via_var_load.c b/tests/ir_tests/496_frame_colour_ptr_via_var_load.c new file mode 100644 index 000000000..9293ba9e3 --- /dev/null +++ b/tests/ir_tests/496_frame_colour_ptr_via_var_load.c @@ -0,0 +1,87 @@ +/* frame_colour gives frame objects whose lifetimes do not overlap the same + bytes, and an object whose address goes somewhere it cannot follow -- into + memory -- escapes and lives to the end. The address of t0 (a by-value + argument's copy) sits in t1, a VAR read in two blocks, so SSA leaves it a + VAR, and each read is `T <-- V [LOAD]`: a move of the VAR's value. The + pointer flow followed ASSIGN, ADD and the like but not that LOAD, so the + pointer stored into t13.tree carried no object and escaped nothing; t0's + lifetime ended at its last direct use, and mk()'s struct results were + placed over it while t13.tree still pointed there. This is Zig's + AstGen.generate storing `&tree` into its AstGen state; the tcc -O1 Zig + compiler segfaulted reporting a syntax error. */ +#include + +struct Alloc +{ + void *ptr; + const void *vt; +}; +struct Ast +{ + int a[16]; +}; +struct Gen +{ + struct Alloc gpa; + const struct Ast *tree; + int x; +}; +struct Big +{ + int v[16]; +}; + +__attribute__((noinline)) struct Big mk(int i) +{ + struct Big b; + for (int k = 0; k < 16; k++) + b.v[k] = i * 100 + k; + return b; +} + +__attribute__((noinline)) int annotate(struct Alloc a, struct Ast t) { return t.a[4] + (a.ptr != 0); } + +__attribute__((noinline)) int use(struct Gen *g) { return g->x + g->tree->a[1]; } + +__attribute__((noinline)) int generate(struct Alloc const a0, struct Ast const a1) +{ + struct Gen t13; + struct Ast t0; + const struct Ast *t1; + const struct Ast **t15; + struct Big t2, t3, t4; + int s, t9; + t0 = a1; + t1 = (const struct Ast *)&t0; + s = t1->a[15]; + t9 = annotate(a0, a1); + t13.gpa = a0; + if (t9 > 0) + { + t15 = (const struct Ast **)&t13.tree; + (*t15) = t1; + } + else + { + t15 = (const struct Ast **)&t13.tree; + (*t15) = t1 + 0; + s += t1->a[3]; + } + t13.x = s + t9; + s = use(&t13); + t2 = mk(1); + t3 = mk(2); + t4 = mk(3); + s += t2.v[3] + t3.v[5] + t4.v[7]; + return s + t13.tree->a[13] + t13.tree->a[2]; +} + +int main(void) +{ + struct Ast a; + struct Alloc al = {&a, 0}; + for (int k = 0; k < 16; k++) + a.a[k] = 1000 * k; + printf("%d\n", generate(al, a)); + return 0; +} diff --git a/tests/ir_tests/496_frame_colour_ptr_via_var_load.expect b/tests/ir_tests/496_frame_colour_ptr_via_var_load.expect new file mode 100644 index 000000000..a6ac541f0 --- /dev/null +++ b/tests/ir_tests/496_frame_colour_ptr_via_var_load.expect @@ -0,0 +1 @@ +35616 diff --git a/tests/ir_tests/497_late_reopt_keeps_icf_alias.c b/tests/ir_tests/497_late_reopt_keeps_icf_alias.c new file mode 100644 index 000000000..cc5345ced --- /dev/null +++ b/tests/ir_tests/497_late_reopt_keeps_icf_alias.c @@ -0,0 +1,41 @@ +/* f_a and f_b compile to the same code, and f_b is folded onto f_a + (identical code folding: its symbol points at f_a's body). Both read + `mode`, a static never written, which only the end of the TU can prove, so + late reopt compiles f_a again: it erases f_a's first body, shifting + everything after it down, and emits it anew at the end. f_b's symbol + stayed at the erased address -- on the range's filler and then on the code + shifted there -- and f_b returned garbage. This is Zig's + link.File.startProgress; the tcc -O2 Zig compiler segfaulted. */ +#include +#include + +static int mode = 3; + +struct File +{ + char pad[64]; + unsigned char tag; +}; + +__attribute__((noinline)) static int f_a(struct File *f) +{ + if (f->tag == 2) + abort(); + return f->tag + mode; +} + +__attribute__((noinline)) static int f_b(struct File *f) +{ + if (f->tag == 2) + abort(); + return f->tag + mode; +} + +__attribute__((noinline)) static int after(int x) { return x * 7 + 1; } + +int main(void) +{ + struct File f = {{0}, 5}; + printf("%d %d %d\n", f_a(&f), f_b(&f), after(2)); + return 0; +} diff --git a/tests/ir_tests/497_late_reopt_keeps_icf_alias.expect b/tests/ir_tests/497_late_reopt_keeps_icf_alias.expect new file mode 100644 index 000000000..e14b4c260 --- /dev/null +++ b/tests/ir_tests/497_late_reopt_keeps_icf_alias.expect @@ -0,0 +1 @@ +8 8 15 diff --git a/tests/ir_tests/498_aeabi_mem_div_runtime.c b/tests/ir_tests/498_aeabi_mem_div_runtime.c new file mode 100644 index 000000000..1d9683801 --- /dev/null +++ b/tests/ir_tests/498_aeabi_mem_div_runtime.c @@ -0,0 +1,101 @@ +/* The ARMv8-M runtime helpers: __aeabi_memcpy/memmove/memset from + lib/arm_mem.S (LDM/STM blocks, an aligned-destination loop with unaligned + source loads, word/halfword/byte tails picked from the count's bits, and a + top-down memmove when the destination starts inside the source), and the + 64-bit division from lib/armeabi.c + armeabi_divmod.S (a 32-bit hardware + divide when both high words are zero, Hacker's Delight divlu/divdu + otherwise). Every alignment, every tail length and every overlap shift is + checked against byte loops, and the divisions against a bit-at-a-time + reference that uses no library call. */ +#include +#include +#include + +void *__aeabi_memcpy(void *, const void *, size_t); +void *__aeabi_memmove(void *, const void *, size_t); +void *__aeabi_memmove4(void *, const void *, size_t); +void *__aeabi_memset(void *, size_t, int); +void *__aeabi_memclr(void *, size_t); + +static unsigned char buf[400], ref[400]; +static uint32_t seed = 12345; +static uint32_t rnd(void) { seed = seed * 1103515245u + 12345u; return seed >> 4; } +static void fill(void) { + for (int i = 0; i < (int)sizeof buf; i++) buf[i] = ref[i] = (unsigned char)rnd(); +} +static int same(void) { + for (int i = 0; i < (int)sizeof buf; i++) if (buf[i] != ref[i]) return 0; + return 1; +} +static void refmove(unsigned char *d, const unsigned char *s, int n) { + unsigned char t[128]; + for (int i = 0; i < n; i++) t[i] = s[i]; + for (int i = 0; i < n; i++) d[i] = t[i]; +} + +/* restoring division, no helper calls: 32-bit operations on the halves */ +static void refdiv(uint32_t nl, uint32_t nh, uint32_t dl, uint32_t dh, + uint32_t *ql, uint32_t *qh, uint32_t *rl, uint32_t *rh) { + uint32_t a = 0, b = 0, x = 0, y = 0; + for (int i = 63; i >= 0; i--) { + b = (b << 1) | (a >> 31); a <<= 1; + a |= (i >= 32 ? nh >> (i - 32) : nl >> i) & 1; + if (b > dh || (b == dh && a >= dl)) { + b = b - dh - (a < dl); a -= dl; + if (i >= 32) y |= 1u << (i - 32); else x |= 1u << i; + } + } + *ql = x; *qh = y; *rl = a; *rh = b; +} + +int main(void) { + int bad = 0, n, d, s, sh; + + for (n = 0; n <= 70; n++) + for (d = 0; d < 4; d++) + for (s = 0; s < 4; s++) { + fill(); refmove(ref + 20 + d, ref + 200 + s, n); + if (__aeabi_memcpy(buf + 20 + d, buf + 200 + s, n) != buf + 20 + d || !same()) bad++; + for (sh = -20; sh <= 20; sh++) { + int dd = 100 + d, ss = 100 + d + sh + s; + fill(); refmove(ref + dd, ref + ss, n); + if (__aeabi_memmove(buf + dd, buf + ss, n) != buf + dd || !same()) bad++; + if (((dd | ss | n) & 3) == 0) { + fill(); refmove(ref + dd, ref + ss, n); + if (__aeabi_memmove4(buf + dd, buf + ss, n) != buf + dd || !same()) bad++; + } + } + if (s == 0) { + int c = 0x1a5 + n; /* bits above the byte are ignored */ + fill(); for (int i = 0; i < n; i++) ref[20 + d + i] = (unsigned char)c; + if (__aeabi_memset(buf + 20 + d, n, c) != buf + 20 + d || !same()) bad++; + fill(); for (int i = 0; i < n; i++) ref[20 + d + i] = 0; + if (__aeabi_memclr(buf + 20 + d, n) != buf + 20 + d || !same()) bad++; + } + } + printf("mem bad=%d\n", bad); + + bad = 0; + uint32_t h = 0; + for (int i = 0; i < 3000; i++) { + int wn = rnd() % 65, wd = 1 + rnd() % 64; + uint64_t nv = ((uint64_t)rnd() << 40) ^ ((uint64_t)rnd() << 20) ^ rnd() ^ ((uint64_t)rnd() << 52); + uint64_t dv = ((uint64_t)rnd() << 40) ^ ((uint64_t)rnd() << 20) ^ rnd() ^ ((uint64_t)rnd() << 52); + if (wn < 64) nv &= ((uint64_t)1 << wn) - 1; + if (wd < 64) dv &= ((uint64_t)1 << wd) - 1; + if (dv == 0) dv = 7; + volatile uint64_t vn = nv, vd = dv; + uint64_t q = vn / vd, r = vn % vd; + uint32_t ql, qh, rl, rh; + refdiv((uint32_t)nv, (uint32_t)(nv >> 32), (uint32_t)dv, (uint32_t)(dv >> 32), &ql, &qh, &rl, &rh); + if ((uint32_t)q != ql || (uint32_t)(q >> 32) != qh || (uint32_t)r != rl || (uint32_t)(r >> 32) != rh) bad++; + /* signed: negate either operand; the quotient truncates toward zero */ + volatile int64_t sn = (i & 1) ? -(int64_t)(nv >> 1) : (int64_t)(nv >> 1); + volatile int64_t sd = (i & 2) ? -(int64_t)(dv >> 1 | 1) : (int64_t)(dv >> 1 | 1); + int64_t sq = sn / sd, sr = sn % sd; + if (sq * sd + sr != sn || (sr != 0 && (sr < 0) != (sn < 0)) || (sr < 0 ? -sr : sr) >= (sd < 0 ? -sd : sd)) bad++; + h = h * 31 + (uint32_t)q + (uint32_t)(r >> 3) + (uint32_t)sq; + } + printf("div bad=%d hash=%08x\n", bad, (unsigned)h); + return 0; +} diff --git a/tests/ir_tests/498_aeabi_mem_div_runtime.expect b/tests/ir_tests/498_aeabi_mem_div_runtime.expect new file mode 100644 index 000000000..156cf6692 --- /dev/null +++ b/tests/ir_tests/498_aeabi_mem_div_runtime.expect @@ -0,0 +1,2 @@ +mem bad=0 +div bad=0 hash=267b1c26 diff --git a/tests/ir_tests/499_redundant_cmp_cbz_i64_flags.c b/tests/ir_tests/499_redundant_cmp_cbz_i64_flags.c new file mode 100644 index 000000000..ef6b9b21e --- /dev/null +++ b/tests/ir_tests/499_redundant_cmp_cbz_i64_flags.c @@ -0,0 +1,72 @@ +/* ra:redundant_cmp deletes a CMP identical to one just before a branch and + lets its reader use the flags the first left. Two ways that first compare + leaves no such flags: + - a 32-bit `CMP r, #0` branched on EQ/NE becomes CBZ/CBNZ in codegen, and + CBZ sets no flags: `if (x != 0) { if (x < 0) ...` compiled to + `cbz r0; bge`, the bge reading whatever an earlier compare left; + - a 64-bit CMP is lowered by its reader: `!= 0` is `ORRS t, lo, hi`, whose N + is the sign of lo | hi, not of the value, and `< 0` then read it. + sign_cbz is entered with N = Z = C = V = 0 ("greater or equal") set by the + caller, which is what its bge read; the others have an earlier compare + leave the same. */ +#include +#include + +__attribute__((noinline)) int sign_cbz(int x, int y) { + if (x != 0) { + if (x < 0) + return y + 1; + return y * 3; + } + return 7; +} + +/* Call sign_cbz with known flags: APSR.NZCV all clear. */ +int call_with_ge_flags(int x, int y); +__asm__(".text\n" + ".global call_with_ge_flags\n" + ".thumb_func\n" + "call_with_ge_flags:\n" + " push {r4, lr}\n" + " movs r2, #0\n" + " msr APSR_nzcvq, r2\n" + " bl sign_cbz\n" + " pop {r4, pc}\n"); + +__attribute__((noinline)) int sign3(int x, int y, int z) { + int r = 0; + if (y > z) + r = 10; + if (x != 0) { + if (x < 0) + return r + 1; + return r + 3; + } + return r + 7; +} + +__attribute__((noinline)) int sign3_eq(int x, unsigned y, unsigned z) { + int r = y >= z ? 20 : 0; + if (x == 0) + return r + 7; + if (x < 0) + return r + 1; + return r + 3; +} + +__attribute__((noinline)) int sign64(int64_t v, int64_t w) { + if (v != 0 && (v < 0) != (w < 0)) + return 1; + return 0; +} + +int main(void) { + volatile int a = -4, b = 9, c = 0, big = 100, small = 1; + volatile int64_t p = 1243716170416LL, n = -176628268885896625LL, lowbit = 0x80000000LL; + printf("%d %d %d\n", call_with_ge_flags(a, 1), call_with_ge_flags(b, 1), call_with_ge_flags(c, 1)); + printf("%d %d %d\n", sign3(a, big, small), sign3(b, big, small), sign3(c, big, small)); + printf("%d %d %d\n", sign3(a, small, big), sign3(b, small, big), sign3(c, small, big)); + printf("%d %d %d\n", sign3_eq(a, big, small), sign3_eq(b, big, small), sign3_eq(c, big, small)); + printf("%d %d %d %d\n", sign64(p, p), sign64(lowbit, p), sign64(n, p), sign64(p, n)); + return 0; +} diff --git a/tests/ir_tests/499_redundant_cmp_cbz_i64_flags.expect b/tests/ir_tests/499_redundant_cmp_cbz_i64_flags.expect new file mode 100644 index 000000000..450dde68d --- /dev/null +++ b/tests/ir_tests/499_redundant_cmp_cbz_i64_flags.expect @@ -0,0 +1,5 @@ +2 3 7 +11 13 17 +1 3 7 +21 23 27 +0 0 1 1 diff --git a/tests/ir_tests/500_asm_lr_clobber_is_a_call.c b/tests/ir_tests/500_asm_lr_clobber_is_a_call.c new file mode 100644 index 000000000..39757f458 --- /dev/null +++ b/tests/ir_tests/500_asm_lr_clobber_is_a_call.c @@ -0,0 +1,34 @@ +/* An asm statement that clobbers LR -- a `bl` in its body -- needs LR saved + like a call does. asm_gen_code saves only r4-r11 around the statement, + and a function whose only call is inside asm was a leaf, so its prologue + did not save LR either: call_bump returned through the LR its own `bl` + left, back into itself, forever. */ +#include + +int counter; + +void bump(void) { counter += 3; } + +__attribute__((noinline)) void call_bump(void) +{ + __asm__ volatile("bl bump" : : : "r0", "r1", "r2", "r3", "ip", "lr", "memory", "cc"); +} + +/* the same with work around it, and a value live across the statement */ +__attribute__((noinline)) int call_bump_twice(int x) +{ + int y = x * 5; + __asm__ volatile("bl bump" : : : "r0", "r1", "r2", "r3", "ip", "lr", "memory", "cc"); + __asm__ volatile("bl bump" : : : "r0", "r1", "r2", "r3", "ip", "lr", "memory", "cc"); + return y + counter; +} + +int main(void) +{ + call_bump(); + call_bump(); + printf("counter=%d\n", counter); + int t = call_bump_twice(4); + printf("twice=%d counter=%d\n", t, counter); + return 0; +} diff --git a/tests/ir_tests/500_asm_lr_clobber_is_a_call.expect b/tests/ir_tests/500_asm_lr_clobber_is_a_call.expect new file mode 100644 index 000000000..d911c66d5 --- /dev/null +++ b/tests/ir_tests/500_asm_lr_clobber_is_a_call.expect @@ -0,0 +1,2 @@ +counter=6 +twice=32 counter=12 diff --git a/tests/ir_tests/501_struct_copy_through_pointer_local.c b/tests/ir_tests/501_struct_copy_through_pointer_local.c new file mode 100644 index 000000000..e10ec062f --- /dev/null +++ b/tests/ir_tests/501_struct_copy_through_pointer_local.c @@ -0,0 +1,97 @@ +/* A small aggregate copied from `*p`, where p is a pointer held in a local, + is now copied inline like one through a parameter: the frontend had kept it + a memmove because that deref still carried the local's frame offset in c.i, + which is no displacement. And a struct returned into a local through the + hidden result pointer is no longer copied onto itself. Copies from every + byte offset, of byte arrays, words and a struct with a narrow member and + padding, through pointer locals and into pointer locals. */ +#include +#include + +struct A2 { unsigned char b[2]; }; +struct A4 { unsigned char b[4]; }; +struct A7 { unsigned char b[7]; }; +struct W3 { unsigned w[3]; }; +struct M { unsigned short h; unsigned char c; unsigned w; }; +struct Big { unsigned long long q[12]; }; + +static unsigned char src[64] __attribute__((aligned(8))), dst[64] __attribute__((aligned(8))); + +__attribute__((noinline)) unsigned sum4(int off) +{ + const unsigned char *p = src + off; + const struct A4 *t20; + struct A4 t21; + t20 = (const struct A4 *)p; + t21 = *t20; + return t21.b[0] | t21.b[1] << 8 | t21.b[2] << 16 | (unsigned)t21.b[3] << 24; +} + +__attribute__((noinline)) unsigned sum2_7(int off) +{ + const unsigned char *p = src + off; + const struct A2 *a = (const struct A2 *)p; + const struct A7 *b = (const struct A7 *)(p + 2); + struct A2 x = *a; + struct A7 y = *b; + unsigned s = x.b[0] + x.b[1] * 3; + for (int i = 0; i < 7; i++) + s = s * 31 + y.b[i]; + return s; +} + +__attribute__((noinline)) void put_w3(int off, struct W3 v) +{ + struct W3 *d = (struct W3 *)(dst + off); + *d = v; +} + +__attribute__((noinline)) struct M get_m(const struct M *pm) +{ + const struct M *q = pm; + struct M m; + m = *q; + m.c++; + return m; +} + +__attribute__((noinline)) struct Big make_big(unsigned long long s) +{ + struct Big b; + for (int i = 0; i < 12; i++) + b.q[i] = s * (i + 1); + return b; +} + +__attribute__((noinline)) unsigned long long use_big(unsigned long long s) +{ + struct Big t3, t2; + t3 = make_big(s); + t2 = t3; + t2.q[0] ^= 5; + return t2.q[0] + t2.q[11] + t3.q[0]; +} + +int main(void) +{ + for (int i = 0; i < 64; i++) + src[i] = (unsigned char)(i * 37 + 11); + unsigned h = 0; + for (int off = 0; off < 8; off++) + h = h * 131 + sum4(off) + sum2_7(off); + printf("reads %08x\n", h); + + struct W3 v = {{0x11223344u, 0x55667788u, 0x99aabbccu}}; + memset(dst, 0xee, sizeof dst); + put_w3(8, v); + h = 0; + for (int i = 0; i < 24; i++) + h = h * 33 + dst[i]; + printf("writes %08x\n", h); + + struct M m0 = {0xbeef, 7, 0xcafef00du}; + struct M m1 = get_m(&m0); + printf("m %x %u %x\n", m1.h, m1.c, m1.w); + printf("big %llx\n", use_big(0x123456789ull)); + return 0; +} diff --git a/tests/ir_tests/501_struct_copy_through_pointer_local.expect b/tests/ir_tests/501_struct_copy_through_pointer_local.expect new file mode 100644 index 000000000..3b7b907fb --- /dev/null +++ b/tests/ir_tests/501_struct_copy_through_pointer_local.expect @@ -0,0 +1,4 @@ +reads 767c8bd8 +writes d4fe0e56 +m beef 8 cafef00d +big fedcba981 diff --git a/tests/ir_tests/502_dead_loop_vinfo_grow_def_instr.c b/tests/ir_tests/502_dead_loop_vinfo_grow_def_instr.c new file mode 100644 index 000000000..8f8d7c9a8 --- /dev/null +++ b/tests/ir_tests/502_dead_loop_vinfo_grow_def_instr.c @@ -0,0 +1,46 @@ +/* ssa:dead_loop grew the SSA vreg table for the TEMPs it creates with 16 + spare entries, zero-filled -- and a zero entry reads as "defined by + instruction 0, never used". The next ssa:dce deleted instruction 0 on the + spare entry's behalf: here the entry block's `i = -5`, so main's first loop + ran from whatever r4 held and printed nothing. It takes a loop dead_loop + rewrites (in_loop(0), inlined, its switch folded to a constant) after one + whose counter is set in the entry block. */ +#include +static int variable(int i) +{ + switch (i) { + case 0: return 70; + case 1: return 71; + case 2: return 72; + case 3: return 73; + case 4: return 74; + default: return -70; + } +} +static int in_loop(int iterations) +{ + int r = 0; + for (int n = 0; n < iterations; n++) { + int i = 7; + r = 1000; + switch (i) { + case 0: r += i + 1; break; + case 1: r -= i; break; + case 2: r += 1; break; + case 3: r = r / 2 + 1; break; + case 4: r ^= i; break; + case 5: r &= (0xffff + i); break; + case 6: r |= (i & 0x0f); break; + case 7: r = (r ^ 0xff) ^ 0xff; break; + default: r = 0; break; + } + } + return r; +} +int main(void) +{ + for (int i = -5; i <= 9; i++) + printf("%d\n", variable(i)); + printf("%d\n", in_loop(0)); + return 0; +} diff --git a/tests/ir_tests/502_dead_loop_vinfo_grow_def_instr.expect b/tests/ir_tests/502_dead_loop_vinfo_grow_def_instr.expect new file mode 100644 index 000000000..48ce164e8 --- /dev/null +++ b/tests/ir_tests/502_dead_loop_vinfo_grow_def_instr.expect @@ -0,0 +1,16 @@ +-70 +-70 +-70 +-70 +-70 +70 +71 +72 +73 +74 +-70 +-70 +-70 +-70 +-70 +0 diff --git a/tests/ir_tests/503_coalesce_swap_stale_allocation.c b/tests/ir_tests/503_coalesce_swap_stale_allocation.c new file mode 100644 index 000000000..ddd09a192 --- /dev/null +++ b/tests/ir_tests/503_coalesce_swap_stale_allocation.c @@ -0,0 +1,105 @@ +/* run_register_coalescing swaps two intervals' registers after allocation so + a variable can sit where its value arrives -- here from_type's result wants + R0, which the reload of the by-value struct argument had. The swap updated + only the linear-scan intervals; the per-vreg allocation that ra:reload_elim + reads kept the old registers until compute_stack_layout. reload_elim saw + `slot <- R0(a)` then `R0(t) <- slot`, deleted the reload as a no-op, and t's + new register (R2) was never written: from_type compared and returned + whatever R2 held. Zig's compiler (Air.Inst.Ref.fromType, this shape in + zig.c) segfaulted in its compile-to-C job at -finline-limit=200. */ +#include +#include +#include + +static inline uint32_t shr_u32(uint32_t lhs, uint8_t rhs) { return lhs >> rhs; } +static inline uint32_t intcast_u32(uint32_t arg) { return arg; } +static inline uint32_t truncate_u32(uint32_t arg, uint8_t bits) { return arg & shr_u32(0xffffffffu, 32 - bits); } +static inline uint32_t bitcast_u32(uint32_t arg, uint8_t bits) { return truncate_u32(arg, bits); } + +struct Type { uint32_t ip_index; }; + +static void debug_assert(bool const a0) { + bool t0; + t0 = !a0; + if (t0) { + __builtin_unreachable(); + } + goto block_0; +block_0:; + return; +} + +static uint32_t to_intern(struct Type const a0) { + struct Type const *t1; + uint32_t const *t2; + uint32_t t3; + struct Type t0; + bool t4; + t0 = a0; + t1 = (struct Type const *)&t0; + t2 = (uint32_t const *)&t1->ip_index; + t3 = (*t2); + t4 = t3 != UINT32_MAX; + debug_assert(t4); + t2 = (uint32_t const *)&t1->ip_index; + t3 = (*t2); + return t3; +} + +static uint32_t from_intern(uint32_t const a0) { + uint32_t t0; + uint32_t t2; + uint32_t t3; + bool t1; + switch (a0) { + case UINT32_MAX: { + goto block_0; + } + default: { + t0 = bitcast_u32(a0, UINT8_C(32)); + t0 = shr_u32(t0, UINT8_C(31)); + t1 = t0 == UINT32_C(0); + debug_assert(t1); + t0 = bitcast_u32(a0, UINT8_C(32)); + t2 = intcast_u32(t0); + t0 = intcast_u32(t2); + t3 = bitcast_u32(t0, UINT8_C(32)); + return t3; + } + } + +block_0:; + return UINT32_MAX; +} + +__attribute__((noinline)) uint32_t from_type(struct Type const a0) { + struct Type const *t1; + struct Type t2; + struct Type t0; + uint32_t t3; + uint32_t t4; + t0 = a0; + t1 = (struct Type const *)&t0; + t2 = (*t1); + t3 = to_intern(t2); + t4 = from_intern(t3); + return t4; +} + +/* Arrives with something other than the argument in R2. */ +__attribute__((noinline)) uint32_t call(uint32_t junk0, uint32_t junk1, uint32_t junk2, uint32_t v) +{ + struct Type t = { v }; + (void)junk0; + (void)junk1; + (void)junk2; + return from_type(t); +} + +int main(void) +{ + printf("%u\n", (unsigned)call(0, 0, 0xdead, 7)); + printf("%u\n", (unsigned)call(1, 2, 0xbeef, 12345)); + printf("%u\n", (unsigned)call(3, 4, 5, 0x7fffffff)); + return 0; +} diff --git a/tests/ir_tests/503_coalesce_swap_stale_allocation.expect b/tests/ir_tests/503_coalesce_swap_stale_allocation.expect new file mode 100644 index 000000000..3ac74bcf9 --- /dev/null +++ b/tests/ir_tests/503_coalesce_swap_stale_allocation.expect @@ -0,0 +1,3 @@ +7 +12345 +2147483647 diff --git a/tests/ir_tests/504_sra_wide_doubleword_fields.c b/tests/ir_tests/504_sra_wide_doubleword_fields.c new file mode 100644 index 000000000..c62710d11 --- /dev/null +++ b/tests/ir_tests/504_sra_wide_doubleword_fields.c @@ -0,0 +1,124 @@ +/* sra: a 64-bit field of a frame object becomes a VAR, and one also touched a + word at a time -- the Zig C backend's zig_u128, zero-filled and copied a word + at a time, its halves read and written whole -- becomes two word VARs whose + 64-bit reads are PACK64s and whose 64-bit writes are split. ssa:sccp folds a + PACK64 of constants, and ssa:fold cancels a split taken apart again. Every + shape here is checked against values computed with plain uint64_t. */ +#include +#include + +typedef struct { uint64_t lo; uint64_t hi; } u128; + +#define MK(h, l) ((u128){ .hi = (h), .lo = (l) }) + +static inline u128 u128_from64(uint64_t x) { return MK(0, x); } +static inline u128 u128_and(u128 a, u128 b) { return MK(a.hi & b.hi, a.lo & b.lo); } +static inline u128 u128_xor(u128 a, u128 b) { return MK(a.hi ^ b.hi, a.lo ^ b.lo); } +static inline u128 u128_shr(u128 a, unsigned n) +{ + if (n == 0) return a; + if (n >= 64) return MK(0, a.hi >> (n - 64)); + return MK(a.hi >> n, a.hi << (64 - n) | a.lo >> n); +} +static inline u128 u128_shl(u128 a, unsigned n) +{ + if (n == 0) return a; + if (n >= 64) return MK(a.lo << (n - 64), 0); + return MK(a.hi << n | a.lo >> (64 - n), a.lo << n); +} +static inline u128 u128_max(unsigned bits) { return u128_shr(MK(~0ull, ~0ull), 128 - bits); } +static inline u128 u128_trunc(u128 a, unsigned bits) { return u128_and(a, u128_max(bits)); } +static inline uint64_t u64_trunc(u128 a) { return a.lo; } + +/* By value and out of line, like __multi3: its arguments stay in memory. */ +__attribute__((noinline)) static u128 mul_full(u128 a, u128 b) +{ + uint64_t al = (uint32_t)a.lo, ah = a.lo >> 32, bl = (uint32_t)b.lo, bh = b.lo >> 32; + uint64_t ll = al * bl, lh = al * bh, hl = ah * bl, hh = ah * bh; + uint64_t mid = (ll >> 32) + (uint32_t)lh + (uint32_t)hl; + u128 r = MK(hh + (lh >> 32) + (hl >> 32) + (mid >> 32), (mid << 32) | (uint32_t)ll); + r.hi += a.lo * b.hi + a.hi * b.lo; + return r; +} + +/* wyhash's mum: the shape the Zig compiler runs 3 million times. */ +static inline void mum(uint64_t *a, uint64_t *b) +{ + u128 x = u128_from64(*a), y = u128_from64(*b); + u128 p = u128_trunc(mul_full(x, y), 128); + *a = u64_trunc(p); + p = u128_shr(p, 64); + *b = u64_trunc(p); +} + +/* A doubleword read a word at a time, then whole, across a branch. */ +__attribute__((noinline)) static uint64_t words_then_whole(uint64_t v, int c) +{ + u128 s = MK(v * 3, v ^ 0x0123456789abcdefull); + uint32_t w0, w1; + __builtin_memcpy(&w0, (char *)&s.lo, 4); + __builtin_memcpy(&w1, (char *)&s.lo + 4, 4); + if (c) + s.lo = ((uint64_t)w0 << 32) | w1; + else + s.hi = 0xfedcba9876543210ull; + return s.lo + s.hi; +} + +/* A loop-carried u128 accumulator: phis over both split words. */ +__attribute__((noinline)) static u128 accumulate(uint64_t seed, int n) +{ + u128 acc = MK(seed, ~seed); + for (int i = 0; i < n; i++) { + acc = u128_xor(acc, u128_shl(u128_from64(seed + i), (unsigned)(i * 7) & 127)); + if (acc.lo & 1) + acc = u128_shr(acc, 3); + } + return acc; +} + +/* 64-bit fields only: one 64-bit VAR each; compare, pass, return them. */ +typedef struct { uint64_t a; uint64_t b; } pair64; +__attribute__((noinline)) static uint64_t take64(uint64_t x) { return x * 5 + 1; } +__attribute__((noinline)) static uint64_t pure64(uint64_t x, uint64_t y) +{ + pair64 p; + p.a = x; + p.b = 0x8000000000000001ull; + if (p.a > y) + p.b = take64(p.a) + y; + else + p.a = p.b ^ y; + return p.a < p.b ? p.b - p.a : take64(p.a - p.b); +} + +/* A double shares the object: that doubleword stays in memory. */ +typedef struct { uint64_t u; double d; } mixed; +__attribute__((noinline)) static uint64_t with_double(uint64_t x) +{ + mixed m; + m.u = x; + m.d = (double)(x & 0xffff) * 0.5; + m.u += (uint64_t)m.d; + return m.u; +} + +int main(void) +{ + uint64_t s = 0x9E3779B97F4A7C15ull, sum = 0; + for (int i = 0; i < 64; i++) { + s ^= s << 13; s ^= s >> 7; s ^= s << 17; + uint64_t a = s, b = (i & 1) ? s >> 17 : ~s * 3; + mum(&a, &b); + sum += a ^ (b * 7); + sum += words_then_whole(s, i & 2); + u128 acc = accumulate(s, i & 15); + sum ^= acc.lo + acc.hi * 11; + sum += pure64(s, s >> (i & 7)); + sum += with_double(s); + } + u128 t = u128_trunc(MK(0x1111222233334444ull, 0x5555666677778888ull), 72); + printf("%016llx %016llx\n", (unsigned long long)t.hi, (unsigned long long)t.lo); + printf("%016llx\n", (unsigned long long)sum); + return 0; +} diff --git a/tests/ir_tests/504_sra_wide_doubleword_fields.expect b/tests/ir_tests/504_sra_wide_doubleword_fields.expect new file mode 100644 index 000000000..50b96044f --- /dev/null +++ b/tests/ir_tests/504_sra_wide_doubleword_fields.expect @@ -0,0 +1,2 @@ +0000000000000044 5555666677778888 +1f3cd470f6e55090 diff --git a/tests/ir_tests/505_const_store_var_forward.c b/tests/ir_tests/505_const_store_var_forward.c new file mode 100644 index 000000000..ede285159 --- /dev/null +++ b/tests/ir_tests/505_const_store_var_forward.c @@ -0,0 +1,100 @@ +/* A 64-bit or narrow slot STORE that covers every read of a local is a fresh + SSA definition (a narrow one only when it stores a constant, written as the + local holds it), and a local only ever stored one constant reads as that + constant everywhere (sra.c, sra_forward_const_vars). The inliner binds a + parameter with a slot STORE: the Zig C backend's zig.h passes `bits` (a + uint8_t) and 64-bit halves to every u128 helper this way. */ +#include +#include + +static inline uint64_t sign_trunc(uint64_t v, uint8_t bits) +{ + if (bits >= 64) + return (int64_t)v < 0 ? v | 0x8000000000000000ull : v & 0x7fffffffffffffffull; + uint64_t m = (1ull << bits) - 1; + return (v & (1ull << (bits - 1))) ? v | ~m : v & m; +} + +static inline uint64_t shr_by(uint64_t hi, uint64_t lo, uint8_t n) +{ + if (n == 0) return lo; + if (n >= 64) return hi >> (n - 64); + return hi << (64 - n) | lo >> n; +} + +__attribute__((noinline)) static uint64_t helpers(uint64_t a, int c) +{ + uint64_t r = sign_trunc(a, 64) ^ sign_trunc(a, 12); + if (c) + r += shr_by(0, a, 64); + else + r += shr_by(a, a ^ 1, 3); + return r + shr_by(a, 7, 0); +} + +/* Narrow constants: the store truncates, the value read must too. */ +__attribute__((noinline)) static int narrow(int c) +{ + unsigned char u; + signed char s; + unsigned short w; + u = 300; + s = 200; + w = 70000; + int r = 0; + if (c > 2) + r = u; + else if (c > 1) + r = s; + else + r = w; + return r + (c & 1 ? u : s); +} + +/* Written twice with the same constant: still that constant. */ +__attribute__((noinline)) static int same_twice(int c) +{ + int k; + k = 41; + if (c) + k = 41; + return k + c; +} + +/* Two different constants: never folded. */ +__attribute__((noinline)) static int two_values(int c) +{ + int k; + k = 5; + if (c & 1) + k = 9; + return k * 3 + c; +} + +/* Read before the write in a loop: every read after the first iteration sees + the constant; the first reads a value the program sets itself. */ +__attribute__((noinline)) static int loop_read_first(int n) +{ + int acc = 0, k = 0; + for (int i = 0; i < n; i++) { + acc += k; + k = 17; + } + return acc; +} + +int main(void) +{ + uint64_t sum = 0; + for (int i = 0; i < 8; i++) { + uint64_t a = 0x0123456789abcdefull * (uint64_t)(i + 1) ^ ((uint64_t)i << 60); + sum = sum * 31 + helpers(a, i & 1); + } + printf("%016llx\n", (unsigned long long)sum); + for (int c = 0; c < 4; c++) + printf("narrow %d %d\n", c, narrow(c)); + printf("same %d %d\n", same_twice(0), same_twice(3)); + printf("two %d %d\n", two_values(0), two_values(1)); + printf("loop %d %d\n", loop_read_first(1), loop_read_first(5)); + return 0; +} diff --git a/tests/ir_tests/505_const_store_var_forward.expect b/tests/ir_tests/505_const_store_var_forward.expect new file mode 100644 index 000000000..155d2cdb9 --- /dev/null +++ b/tests/ir_tests/505_const_store_var_forward.expect @@ -0,0 +1,8 @@ +513e85f623aed2a4 +narrow 0 4408 +narrow 1 4508 +narrow 2 -112 +narrow 3 88 +same 41 44 +two 15 28 +loop 0 68 diff --git a/tests/ir_tests/506_inline_aligned_struct_copy.c b/tests/ir_tests/506_inline_aligned_struct_copy.c new file mode 100644 index 000000000..f103794de --- /dev/null +++ b/tests/ir_tests/506_inline_aligned_struct_copy.c @@ -0,0 +1,106 @@ +/* A struct assignment the frontend lowers to __aeabi_memmove4/8 with a constant + size of up to 128 bytes is emitted as the copy itself (ldmia/stmia through + r0/r1 with r2, r3, r12 and lr) instead of a call. A copy that reaches either + side through a packed member names plain __aeabi_memmove: its address need + not be word aligned, and LDM/STM fault on one that is not. */ +#include +#include +#include + +#define DEFS(N) typedef struct { uint32_t w[N]; } s##N; +DEFS(1) DEFS(2) DEFS(3) DEFS(5) DEFS(8) DEFS(13) DEFS(24) DEFS(32) DEFS(33) +typedef struct { uint64_t d[5]; double f; } s48d; /* 8-aligned: memmove8 */ + +static uint32_t mix; +#define CHECK(N) \ + __attribute__((noinline)) static uint32_t copy##N(const s##N *p, s##N *q) \ + { \ + s##N t = *p; \ + t.w[N - 1] ^= 0x5a5a5a5au; \ + *q = t; \ + uint32_t h = 0; \ + for (int i = 0; i < N; i++) \ + h = h * 33 + q->w[i]; \ + return h; \ + } +CHECK(1) CHECK(2) CHECK(3) CHECK(5) CHECK(8) CHECK(13) CHECK(24) CHECK(32) CHECK(33) + +__attribute__((noinline)) static double copy48d(const s48d *p, s48d *q) +{ + s48d t = *p; + t.f += 0.5; + *q = t; + return q->f + (double)(q->d[4] & 0xffff); +} + +/* Exact self-assignment through two pointers to the same object. */ +__attribute__((noinline)) static void self_copy(s13 *a, s13 *b) { *a = *b; } + +/* Packed members at odd offsets: the copies must not assume alignment. */ +struct __attribute__((packed)) pk { + char c; + s5 inner; + char d; + s8 inner2; +}; +__attribute__((noinline)) static uint32_t packed_copy(struct pk *x, const s5 *v, const s8 *v2) +{ + x->inner = *v; + x->inner2 = *v2; + s5 back = x->inner; + s8 back2 = x->inner2; + return back.w[0] + back.w[4] + back2.w[7] + (uint32_t)x->c + (uint32_t)x->d; +} + +/* An array of structs walked by index: the copies are in a loop. */ +__attribute__((noinline)) static uint32_t rotate(s24 *arr, int n) +{ + s24 first = arr[0]; + for (int i = 0; i + 1 < n; i++) + arr[i] = arr[i + 1]; + arr[n - 1] = first; + uint32_t h = 0; + for (int i = 0; i < n; i++) + h = h * 7 + arr[i].w[i % 24]; + return h; +} + +#define RUN(N) \ + { \ + s##N a, b; \ + for (int i = 0; i < N; i++) \ + a.w[i] = 0x01010101u * (uint32_t)(i + N); \ + memset(&b, 0, sizeof b); \ + uint32_t h = copy##N(&a, &b); \ + printf("s%d %08x %08x\n", N, h, b.w[N - 1]); \ + } + +int main(void) +{ + RUN(1) RUN(2) RUN(3) RUN(5) RUN(8) RUN(13) RUN(24) RUN(32) RUN(33) + s48d x, y; + for (int i = 0; i < 5; i++) + x.d[i] = 0x0102030405060708ull * (uint64_t)(i + 1); + x.f = 3.25; + double r48 = copy48d(&x, &y); + printf("s48d %.2f %016llx\n", r48, (unsigned long long)y.d[2]); + s13 z; + for (int i = 0; i < 13; i++) + z.w[i] = (uint32_t)i * 3u; + self_copy(&z, &z); + printf("self %u %u\n", z.w[0], z.w[12]); + struct pk k; + memset(&k, 0, sizeof k); + k.c = 7; + k.d = 9; + s5 v = {{10, 20, 30, 40, 50}}; + s8 v2 = {{1, 2, 3, 4, 5, 6, 7, 8}}; + printf("packed %u\n", packed_copy(&k, &v, &v2)); + s24 arr[5]; + for (int i = 0; i < 5; i++) + for (int j = 0; j < 24; j++) + arr[i].w[j] = (uint32_t)(i * 100 + j); + uint32_t rh = rotate(arr, 5); + printf("rotate %08x %u\n", rh, arr[4].w[0]); + return 0; +} diff --git a/tests/ir_tests/506_inline_aligned_struct_copy.expect b/tests/ir_tests/506_inline_aligned_struct_copy.expect new file mode 100644 index 000000000..ff017a4d8 --- /dev/null +++ b/tests/ir_tests/506_inline_aligned_struct_copy.expect @@ -0,0 +1,13 @@ +s1 5b5b5b5b 5b5b5b5b +s2 9b9b9b9b 59595959 +s3 b3b3b3a6 5f5f5f5f +s5 2f2ed0ed 53535353 +s8 08d9dda2 55555555 +s13 6f6988a1 43434343 +s24 7cdbde9a 75757575 +s32 7de42816 65656565 +s33 effcfd2b 1b1b1b1b +s48d 9003.75 0306090c0f121518 +self 0 36 +packed 84 +rotate 0004fc0a 0 diff --git a/tests/ir_tests/507_widening_mul_fold.c b/tests/ir_tests/507_widening_mul_fold.c new file mode 100644 index 000000000..b1b4256cf --- /dev/null +++ b/tests/ir_tests/507_widening_mul_fold.c @@ -0,0 +1,117 @@ +/* A widening multiply (UMULL/SMULL: 32x32 -> 64) with a constant operand + folds like MUL does: by 0 to a 64-bit 0, and with both operands constant to + the product -- reading each operand as its low word, zero-extended for + UMULL and sign-extended for SMULL. zig.h builds a u128 multiply from these + products, and a u64 widened to u128 has a constant-zero high half, so its + cross terms are exactly `x UMULL #0`. */ +#include +#include + +typedef struct { uint64_t lo, hi; } u128; + +static inline uint64_t umull(uint32_t a, uint32_t b) { return (uint64_t)a * b; } +static inline int64_t smull(int32_t a, int32_t b) { return (int64_t)a * b; } + +/* zig.h's 64x64->128: every step is a*b + c + d with c, d < 2^32. */ +static inline u128 mul_wide(uint64_t l, uint64_t r) +{ + uint32_t l0 = (uint32_t)l, l1 = (uint32_t)(l >> 32); + uint32_t r0 = (uint32_t)r, r1 = (uint32_t)(r >> 32); + uint64_t p0 = (uint64_t)l0 * r0; + uint64_t p1 = (uint64_t)l1 * r0 + (uint32_t)(p0 >> 32); + uint64_t p2 = (uint64_t)l0 * r1 + (uint32_t)p1; + uint64_t p3 = (uint64_t)l1 * r1 + (uint32_t)(p1 >> 32) + (uint32_t)(p2 >> 32); + u128 res = { ((uint64_t)(uint32_t)p2 << 32) | (uint32_t)p0, p3 }; + return res; +} + +/* The low 128 bits of a 128x128 product; hi halves of 0 leave x UMULL #0. */ +static inline u128 mul128(u128 a, u128 b) +{ + u128 res = mul_wide(a.lo, b.lo); + res.hi += a.lo * b.hi + a.hi * b.lo; + return res; +} + +__attribute__((noinline)) static uint64_t mum(uint64_t a, uint64_t b, uint64_t *hi) +{ + u128 x = { a, 0 }, y = { b, 0 }; + u128 r = mul128(x, y); + *hi = r.hi; + return r.lo; +} + +/* Same with only one side widened: one cross term is still live. */ +__attribute__((noinline)) static uint64_t half(uint64_t a, uint64_t bhi, uint64_t blo, uint64_t *hi) +{ + u128 x = { a, 0 }, y = { blo, bhi }; + u128 r = mul128(x, y); + *hi = r.hi; + return r.lo; +} + +__attribute__((noinline)) static uint64_t zeros(uint32_t x, int32_t s) +{ + return umull(x, 0) + umull(0, x) + (uint64_t)smull(s, 0) + (uint64_t)smull(0, s) + 5; +} + +__attribute__((noinline)) static void consts(void) +{ + printf("u %016llx %016llx %016llx\n", (unsigned long long)umull(0xffffffffu, 0xffffffffu), + (unsigned long long)umull(0x80000000u, 2), (unsigned long long)umull(7, 9)); + printf("s %016llx %016llx %016llx\n", (unsigned long long)smull(-1, -1), + (unsigned long long)smull(INT32_MIN, INT32_MIN), (unsigned long long)smull(-3, 5)); +} + +/* Reference: 64x64 -> 128 by 16-bit digits, no widening multiply shapes. */ +static void ref_mul(uint64_t a, uint64_t b, uint64_t *lo, uint64_t *hi) +{ + uint32_t r[8] = {0}; + for (int i = 0; i < 4; i++) { + uint32_t carry = 0; + uint32_t ai = (uint32_t)(a >> (16 * i)) & 0xffff; + for (int j = 0; j < 4; j++) { + uint32_t bj = (uint32_t)(b >> (16 * j)) & 0xffff; + uint32_t t = ai * bj + r[i + j] + carry; + r[i + j] = t & 0xffff; + carry = t >> 16; + } + r[i + 4] = carry; + } + *lo = (uint64_t)r[0] | (uint64_t)r[1] << 16 | (uint64_t)r[2] << 32 | (uint64_t)r[3] << 48; + *hi = (uint64_t)r[4] | (uint64_t)r[5] << 16 | (uint64_t)r[6] << 32 | (uint64_t)r[7] << 48; +} + +int main(void) +{ + static const uint64_t v[] = { + 0, 1, 2, 0xffffffffull, 0x100000000ull, 0xffffffffffffffffull, + 0x9e3779b97f4a7c15ull, 0xa0761d6478bd642full, 0xe7037ed1a0b428dbull, + 0x8000000000000000ull, 0x00000000ffffffffull << 16, + }; + int n = sizeof v / sizeof v[0], bad = 0; + uint64_t acc = 0; + for (int i = 0; i < n; i++) + for (int j = 0; j < n; j++) { + uint64_t hi, lo = mum(v[i], v[j], &hi), rlo, rhi; + ref_mul(v[i], v[j], &rlo, &rhi); + if (lo != rlo || hi != rhi) { + printf("mum %d %d: %016llx %016llx want %016llx %016llx\n", i, j, + (unsigned long long)hi, (unsigned long long)lo, + (unsigned long long)rhi, (unsigned long long)rlo); + bad++; + } + uint64_t h2, l2 = half(v[i], v[(i + j) % n], v[j], &h2); + /* (a * (bhi:blo)) mod 2^128 = a*blo + (a*bhi << 64) */ + if (l2 != rlo || h2 != rhi + v[i] * v[(i + j) % n]) { + printf("half %d %d\n", i, j); + bad++; + } + acc ^= lo * 3 + hi; + } + printf("mul %s %016llx\n", bad ? "BAD" : "ok", (unsigned long long)acc); + printf("zeros %llu %llu\n", (unsigned long long)zeros(0xdeadbeefu, -7), + (unsigned long long)zeros(0, 0)); + consts(); + return 0; +} diff --git a/tests/ir_tests/507_widening_mul_fold.expect b/tests/ir_tests/507_widening_mul_fold.expect new file mode 100644 index 000000000..572285e7b --- /dev/null +++ b/tests/ir_tests/507_widening_mul_fold.expect @@ -0,0 +1,4 @@ +mul ok 90a35ae903f43b46 +zeros 5 5 +u fffffffe00000001 0000000100000000 000000000000003f +s 0000000000000001 4000000000000000 fffffffffffffff1 diff --git a/tests/ir_tests/508_umaal_fusion.c b/tests/ir_tests/508_umaal_fusion.c new file mode 100644 index 000000000..6b00c6b48 --- /dev/null +++ b/tests/ir_tests/508_umaal_fusion.c @@ -0,0 +1,184 @@ +/* A 32x32->64 product plus one or two zero-extended words is fused into + UMAAL right before register allocation (source/opt/ra/umaal_fusion.c): + {hi:lo} = a * b + c + d cannot overflow. The destination inherits the + accumulator's register pair, so the emitter must never let the addend + moves clobber a multiplicand, and every pass after the fusion must see the + accumulator (the fourth operand) as a read of both of its words. Shapes: + one addend, two addends in either order, addends taken from a pair's low + and high words, loads, constants, a loop-carried bignum row, and enough + live 128-bit products at once to force the operands onto the stack. */ +#include +#include + +typedef struct { uint64_t lo, hi; } u128; + +static uint32_t rs = 0x12345678u; +static uint32_t rnd(void) { rs ^= rs << 13; rs ^= rs >> 17; rs ^= rs << 5; return rs; } + +/* Reference: schoolbook by 16-bit digits, nothing a UMULL can match. */ +static void ref(uint64_t a, uint64_t b, uint64_t *lo, uint64_t *hi) +{ + uint32_t r[8] = {0}; + for (int i = 0; i < 4; i++) { + uint32_t carry = 0, ai = (uint32_t)(a >> (16 * i)) & 0xffff; + for (int j = 0; j < 4; j++) { + uint32_t t = ai * ((uint32_t)(b >> (16 * j)) & 0xffff) + r[i + j] + carry; + r[i + j] = t & 0xffff; + carry = t >> 16; + } + r[i + 4] = carry; + } + *lo = r[0] | (uint64_t)r[1] << 16 | (uint64_t)r[2] << 32 | (uint64_t)r[3] << 48; + *hi = r[4] | (uint64_t)r[5] << 16 | (uint64_t)r[6] << 32 | (uint64_t)r[7] << 48; +} + +static inline u128 wide(uint64_t l, uint64_t r) +{ + uint32_t l0 = (uint32_t)l, l1 = (uint32_t)(l >> 32), r0 = (uint32_t)r, r1 = (uint32_t)(r >> 32); + uint64_t p0 = (uint64_t)l0 * r0; + uint64_t p1 = (uint64_t)l1 * r0 + (uint32_t)(p0 >> 32); + uint64_t p2 = (uint64_t)l0 * r1 + (uint32_t)p1; + uint64_t p3 = (uint64_t)l1 * r1 + (uint32_t)(p1 >> 32) + (uint32_t)(p2 >> 32); + u128 x = { ((uint64_t)(uint32_t)p2 << 32) | (uint32_t)p0, p3 }; + return x; +} + +/* The addends in the other order, and added the other way round. */ +static inline u128 wide_swapped(uint64_t l, uint64_t r) +{ + uint32_t l0 = (uint32_t)l, l1 = (uint32_t)(l >> 32), r0 = (uint32_t)r, r1 = (uint32_t)(r >> 32); + uint64_t p0 = (uint64_t)r0 * l0; + uint64_t p1 = (uint32_t)(p0 >> 32) + (uint64_t)r0 * l1; + uint64_t p2 = (uint32_t)p1 + (uint64_t)r1 * l0; + uint64_t p3 = (uint32_t)(p2 >> 32) + ((uint32_t)(p1 >> 32) + (uint64_t)l1 * r1); + u128 x = { ((uint64_t)(uint32_t)p2 << 32) | (uint32_t)p0, p3 }; + return x; +} + +__attribute__((noinline)) static uint64_t mum(uint64_t a, uint64_t b) +{ + u128 x = wide(a, b); + return x.lo ^ x.hi; +} + +__attribute__((noinline)) static void full(uint64_t a, uint64_t b, u128 *o, u128 *o2) +{ + *o = wide(a, b); + *o2 = wide_swapped(a, b); +} + +/* Multiplicand reused as an addend, and a constant addend. */ +__attribute__((noinline)) static uint64_t self_add(uint32_t a, uint32_t b) +{ + return (uint64_t)a * b + a + (uint32_t)0xffffffffu; +} + +/* Addends straight from memory. */ +__attribute__((noinline)) static uint64_t mem_add(const uint32_t *p, uint32_t a, uint32_t b) +{ + return (uint64_t)a * b + p[0] + p[1]; +} + +/* r[0..n] += a[0..n) * b: the bignum row, carry loop-carried. */ +__attribute__((noinline)) static uint32_t row(uint32_t *r, const uint32_t *a, int n, uint32_t b) +{ + uint32_t carry = 0; + for (int i = 0; i < n; i++) { + uint64_t t = (uint64_t)a[i] * b + carry + r[i]; + r[i] = (uint32_t)t; + carry = (uint32_t)(t >> 32); + } + return carry; +} + +/* Eight products alive at once: more pairs than there are registers. */ +__attribute__((noinline)) static uint64_t pressure(const uint64_t *v) +{ + u128 w[8]; + for (int k = 0; k < 8; k++) + w[k] = wide(v[k], v[(k + 3) & 7]); + uint64_t acc = 0; + for (int k = 0; k < 8; k++) + acc = acc * 31 + (w[k].lo ^ (w[7 - k].hi >> 1)); + return acc; +} + +static uint64_t pressure_ref(const uint64_t *v) +{ + uint64_t lo[8], hi[8]; + for (int k = 0; k < 8; k++) + ref(v[k], v[(k + 3) & 7], &lo[k], &hi[k]); + uint64_t acc = 0; + for (int k = 0; k < 8; k++) + acc = acc * 31 + (lo[k] ^ (hi[7 - k] >> 1)); + return acc; +} + +int main(void) +{ + static const uint64_t edge[] = {0, 1, 0xffffffffull, 0x100000000ull, 0xffffffffffffffffull, + 0x9e3779b97f4a7c15ull, 0x8000000000000000ull}; + int bad = 0, n = 0; + uint64_t v[64]; + for (int i = 0; i < 64; i++) + v[i] = i < 7 ? edge[i] : ((uint64_t)rnd() << 32 | rnd()); + for (int i = 0; i < 64; i++) + for (int j = 0; j < 64; j++, n++) { + uint64_t lo, hi; + u128 x, y; + ref(v[i], v[j], &lo, &hi); + full(v[i], v[j], &x, &y); + if (x.lo != lo || x.hi != hi || y.lo != lo || y.hi != hi || mum(v[i], v[j]) != (lo ^ hi)) + bad++; + uint32_t a = (uint32_t)v[i], b = (uint32_t)v[j]; + if (self_add(a, b) != (uint64_t)a * b + a + 0xffffffffull) + bad++; + uint32_t m[2] = {(uint32_t)(v[i] >> 32), (uint32_t)(v[j] >> 32)}; + if (mem_add(m, a, b) != (uint64_t)a * b + m[0] + m[1]) + bad++; + } + printf("products %d %s\n", n, bad ? "BAD" : "ok"); + + /* bignum: (sum a[i] 2^32i) * b, checked against 16-bit digits */ + uint32_t A[9], R[10], R2[10]; + for (int i = 0; i < 9; i++) + A[i] = i < 2 ? 0xffffffffu : rnd(); + for (int t = 0; t < 20; t++) { + uint32_t b = t ? rnd() : 0xffffffffu; + for (int i = 0; i < 10; i++) + R[i] = R2[i] = t & 1 ? rnd() : 0xffffffffu; + R[9] = R2[9] = 0; + uint32_t c = row(R, A, 9, b); + R[9] += c; + /* reference: R2 += A * b, 16-bit digits */ + uint32_t carry = 0; + for (int i = 0; i < 18; i++) { + uint32_t ad = (A[i / 2] >> (16 * (i & 1))) & 0xffff; + uint32_t rd = (R2[i / 2] >> (16 * (i & 1))) & 0xffff; + uint32_t s = ad * (b & 0xffff) + rd + carry; + carry = s >> 16; + R2[i / 2] = (R2[i / 2] & ~(0xffffu << (16 * (i & 1)))) | (s & 0xffff) << (16 * (i & 1)); + /* the b-high half lands one digit up: add it in a second sweep */ + } + R2[9] += carry; + carry = 0; + for (int i = 1; i < 20; i++) { + uint32_t ad = i - 1 < 18 ? (A[(i - 1) / 2] >> (16 * ((i - 1) & 1))) & 0xffff : 0; + uint32_t rd = (R2[i / 2] >> (16 * (i & 1))) & 0xffff; + uint32_t s = ad * (b >> 16) + rd + carry; + carry = s >> 16; + R2[i / 2] = (R2[i / 2] & ~(0xffffu << (16 * (i & 1)))) | (s & 0xffff) << (16 * (i & 1)); + } + for (int i = 0; i < 10; i++) + if (R[i] != R2[i]) + bad++; + } + printf("row %s\n", bad ? "BAD" : "ok"); + + uint64_t pv[8]; + for (int i = 0; i < 8; i++) + pv[i] = v[5 + i * 7]; + printf("pressure %s %016llx\n", pressure(pv) == pressure_ref(pv) ? "ok" : "BAD", + (unsigned long long)pressure(pv)); + return 0; +} diff --git a/tests/ir_tests/508_umaal_fusion.expect b/tests/ir_tests/508_umaal_fusion.expect new file mode 100644 index 000000000..23c619383 --- /dev/null +++ b/tests/ir_tests/508_umaal_fusion.expect @@ -0,0 +1,3 @@ +products 4096 ok +row ok +pressure ok f3be53a422ac848c diff --git a/tests/ir_tests/509_narrow_temp_store_def.c b/tests/ir_tests/509_narrow_temp_store_def.c new file mode 100644 index 000000000..afb37a3d9 --- /dev/null +++ b/tests/ir_tests/509_narrow_temp_store_def.c @@ -0,0 +1,96 @@ +/* A STORE of a byte or halfword into a register temporary is a whole + definition: codegen writes the register with a MOV, or the temp's spill + slot with a word store of the extended value, so nothing of the old value + survives. The allocator used to call it a partial def, which made every + byte temp an inlined helper leaves behind live from function entry: across + every call, each pinned a callee-saved register, and std.HashMap's probe + loop (the Zig C backend's shape, below) had its index, count and metadata + pointer pushed to the stack. The shapes here: byte temps in a probe loop + around a call, halfword temps, and more byte temps live across a call than + there are callee-saved registers, so some of them spill. */ +#include +#include +#include +#include + +typedef uint8_t meta_t; +struct map { meta_t *metadata; uint32_t *keys; uint32_t cap; }; + +static bool is_used(meta_t const a0) +{ + meta_t t0; + memset(&t0, 0, sizeof t0); + t0 = a0; + return (*(uint8_t const *)&t0 >> 7) == 1; +} +static bool is_free(meta_t const a0) { return a0 == 0; } + +__attribute__((noinline)) static bool key_eql(uint32_t a, uint32_t b) { return a == b; } + +__attribute__((noinline)) static int get_index(struct map const a0, uint32_t key) +{ + struct map t0 = a0; + struct map const *t1 = &t0; + uint32_t h = key * 2654435761u; + uint8_t fp = (uint8_t)(h >> 25); + uint32_t mask = t1->cap - 1, limit = t1->cap, idx = h & mask; + meta_t *m = t1->metadata + idx; + for (;;) { + meta_t b = *m; + if (is_free(b) || limit == 0) + return -1; + if (is_used(*m) && (*m & 0x7f) == fp && key_eql(t1->keys[idx], key)) + return (int)idx; + limit--; + idx = (idx + 1) & mask; + m = t1->metadata + idx; + } +} + +__attribute__((noinline)) static void put(struct map *mp, uint32_t key) +{ + uint32_t h = key * 2654435761u, mask = mp->cap - 1, idx = h & mask; + while (mp->metadata[idx] != 0) + idx = (idx + 1) & mask; + mp->metadata[idx] = (uint8_t)(0x80 | (h >> 25)); + mp->keys[idx] = key; +} + +__attribute__((noinline)) static uint32_t sink(uint32_t x) { return x * 3 + 1; } + +/* Twelve byte temps and four halfword temps, all live across calls. */ +__attribute__((noinline)) static uint32_t many(const uint8_t *p, const uint16_t *q) +{ + uint8_t a = p[0], b = p[1], c = p[2], d = p[3], e = p[4], f = p[5]; + uint8_t g = p[6], h = p[7], i = p[8], j = p[9], k = p[10], l = p[11]; + uint16_t w = q[0], x = q[1], y = q[2], z = q[3]; + uint32_t acc = sink(a + b); + acc += sink(c ^ d) + sink((uint32_t)e * f); + acc ^= sink(g) + h + sink(i + j); + acc += sink(w) ^ x; + acc += (uint32_t)k * l + sink(y + z); + return acc + a + b + c + d + e + f + g + h + i + j + k + l + w + x + y + z; +} + +int main(void) +{ + static uint8_t meta[64]; + static uint32_t keys[64]; + struct map m = { meta, keys, 64 }; + for (uint32_t k = 1; k <= 40; k++) + put(&m, k * 7919u); + int found = 0, sum = 0, missing = 0; + for (uint32_t k = 1; k <= 60; k++) { + int i = get_index(m, k * 7919u); + if (i >= 0) { found++; sum += i; } + else missing++; + } + printf("probe found=%d missing=%d sum=%d\n", found, missing, sum); + + uint8_t bytes[12]; + uint16_t halves[4]; + for (int i = 0; i < 12; i++) bytes[i] = (uint8_t)(0xf0 + i * 7); + for (int i = 0; i < 4; i++) halves[i] = (uint16_t)(0xfff0 - i * 911); + printf("many %u\n", many(bytes, halves)); + return 0; +} diff --git a/tests/ir_tests/509_narrow_temp_store_def.expect b/tests/ir_tests/509_narrow_temp_store_def.expect new file mode 100644 index 000000000..8275c9286 --- /dev/null +++ b/tests/ir_tests/509_narrow_temp_store_def.expect @@ -0,0 +1,2 @@ +probe found=40 missing=20 sum=1740 +many 775013 diff --git a/tests/ir_tests/510_known_zero_extend_fold.c b/tests/ir_tests/510_known_zero_extend_fold.c new file mode 100644 index 000000000..ef7da56e4 --- /dev/null +++ b/tests/ir_tests/510_known_zero_extend_fold.c @@ -0,0 +1,90 @@ +/* `x & M` and `UBFX x,#0,#w` are copies of x when x cannot have a bit outside + the mask (ssa:fold, fold_mask_is_copy). The maybe-set-bits walk now knows + that an unsigned byte/halfword load zero-extends, that a UBFX yields only + its field, and that a STORE into a register temp copies its source -- so the + `uxtb` a byte load, or a byte handed through an inlined Zig helper, used to + carry is gone. A signed narrow load sign-extends and keeps its mask, and so + does a mask narrower than the value. */ +#include +#include + +typedef uint8_t meta_t; + +static inline uint32_t fp_of(meta_t m) { return m & 0x7fu; } +static inline int used_of(meta_t m) { return (m >> 7) & 1; } + +__attribute__((noinline)) static uint32_t byte_masks(const uint8_t *p, int n) +{ + uint32_t acc = 0; + for (int i = 0; i < n; i++) { + uint32_t b = p[i]; + acc += (b & 0xffu) + ((uint32_t)p[i] & 0x1ffu) + fp_of(p[i]) * 3u + (uint32_t)used_of(p[i]); + } + return acc; +} + +__attribute__((noinline)) static uint32_t half_masks(const uint16_t *p, int n) +{ + uint32_t acc = 0; + for (int i = 0; i < n; i++) + acc += (p[i] & 0xffffu) + (p[i] & 0xffu) * 7u + ((uint32_t)p[i] >> 3 & 0x1fffu); + return acc; +} + +/* Signed loads sign-extend: the mask is real. */ +__attribute__((noinline)) static uint32_t signed_masks(const int8_t *p, const int16_t *q, int n) +{ + uint32_t acc = 0; + for (int i = 0; i < n; i++) + acc += ((uint32_t)p[i] & 0xffu) + ((uint32_t)q[i] & 0xffffu) + ((uint32_t)p[i] & 0x7fu); + return acc; +} + +/* A byte field extracted with UBFX, then masked again. */ +__attribute__((noinline)) static uint32_t fields(const uint32_t *w, int n) +{ + uint32_t acc = 0; + for (int i = 0; i < n; i++) { + uint32_t f = (w[i] >> 8) & 0xffu; + acc += (f & 0xffu) + (f & 0x3fu) + ((uint8_t)f & 0xffu); + } + return acc; +} + +/* A narrow local kept in a byte-typed variable across a call. */ +__attribute__((noinline)) static uint32_t id(uint32_t x) { return x; } +__attribute__((noinline)) static uint32_t narrow_var(const uint8_t *p, int n) +{ + uint32_t acc = 0; + for (int i = 0; i < n; i++) { + volatile uint8_t keep = p[i]; + uint8_t b = keep; + acc += id(b) + (b & 0xffu) + (uint8_t)(b + 200u); + } + return acc; +} + +int main(void) +{ + uint8_t bytes[64]; + uint16_t halves[64]; + int8_t sb[64]; + int16_t sh[64]; + uint32_t words[64]; + uint32_t s = 0x2545f491u; + for (int i = 0; i < 64; i++) { + s ^= s << 13; s ^= s >> 17; s ^= s << 5; + bytes[i] = (uint8_t)s; + halves[i] = (uint16_t)(s >> 8); + sb[i] = (int8_t)(s >> 16); + sh[i] = (int16_t)(s >> 3); + words[i] = s; + } + bytes[0] = 0xff; bytes[1] = 0x80; bytes[2] = 0; sb[0] = -1; sb[1] = -128; sh[0] = -32768; + printf("byte %u\n", byte_masks(bytes, 64)); + printf("half %u\n", half_masks(halves, 64)); + printf("signed %u\n", signed_masks(sb, sh, 64)); + printf("fields %u\n", fields(words, 64)); + printf("narrow %u\n", narrow_var(bytes, 64)); + return 0; +} diff --git a/tests/ir_tests/510_known_zero_extend_fold.expect b/tests/ir_tests/510_known_zero_extend_fold.expect new file mode 100644 index 000000000..2c8198753 --- /dev/null +++ b/tests/ir_tests/510_known_zero_extend_fold.expect @@ -0,0 +1,5 @@ +byte 28396 +half 2512817 +signed 2065453 +fields 18055 +narrow 24365 diff --git a/tests/ir_tests/511_late_setif_branch_fuse.c b/tests/ir_tests/511_late_setif_branch_fuse.c new file mode 100644 index 000000000..c9ca1eb91 --- /dev/null +++ b/tests/ir_tests/511_late_setif_branch_fuse.c @@ -0,0 +1,67 @@ +/* The Zig C backend declares one `bool t` per function and reuses it for + every check, copying it around between them. The flat setif_fuse runs + before SSA, when `t` is still one variable with several non-test reads, so + it keeps every SETIF. SSA renaming then gives each check its own + single-use temp, and `CMP; t <- SETIF; TEST_ZERO t; JUMPIF` fuses into + `CMP; JUMPIF` once more after phi resolution (ra:setif_fuse). Every + relation, signed and unsigned, tested both ways round (`if (t)` and + `if (!t)`), so each fused condition -- including the inverted ones -- is + checked against the value the SETIF would have produced. */ +#include +#include +#include + +__attribute__((noinline)) static unsigned classify(int32_t a, int32_t b, uint32_t ua, uint32_t ub) +{ + bool t4, t19; + unsigned r = 0; + t4 = a < b; if (t4) r |= 1u << 0; + t4 = a <= b; if (!t4) r |= 1u << 1; + t4 = a > b; t19 = t4; if (t19) r |= 1u << 2; + t4 = a >= b; if (t4) goto l3; r |= 1u << 3; +l3: + t4 = a == b; if (!t4) r |= 1u << 4; + t4 = a != b; if (t4) r |= 1u << 5; + t4 = ua < ub; if (t4) r |= 1u << 6; + t4 = ua <= ub; t19 = t4; if (!t19) r |= 1u << 7; + t4 = ua > ub; if (t4) r |= 1u << 8; + t4 = ua >= ub; if (!t4) r |= 1u << 9; + t4 = (ua & 0x80u) != 0; if (t4) r |= 1u << 10; + t4 = (uint8_t)ua == 0; if (!t4) r |= 1u << 11; + return r; +} + +/* The probe-loop shape: `t19` joins a SETIF and a constant false. */ +__attribute__((noinline)) static int probe(const uint8_t *meta, uint32_t n, uint32_t limit) +{ + bool t4, t19; + uint32_t i = 0; + for (;;) { + t4 = !(meta[i] == 0); + if (t4) { + t4 = limit != 0; + t19 = t4; + goto b3; + } + t19 = false; +b3: + if (!t19) + return (int)i; + limit--; + i = (i + 1) % n; + } +} + +int main(void) +{ + static const int32_t v[] = {-5, -1, 0, 1, 7, INT32_MIN, INT32_MAX}; + unsigned acc = 0; + for (int i = 0; i < 7; i++) + for (int j = 0; j < 7; j++) + acc = acc * 31 + classify(v[i], v[j], (uint32_t)v[i], (uint32_t)v[j]); + printf("classify %08x\n", acc); + printf("one %03x %03x %03x\n", classify(1, 2, 1, 2), classify(2, 1, 0x80, 1), classify(3, 3, 0x100, 0x100)); + static const uint8_t meta[8] = {3, 5, 0x81, 9, 0, 7, 1, 2}; + printf("probe %d %d %d\n", probe(meta, 8, 100), probe(meta, 8, 2), probe(meta + 5, 3, 100)); + return 0; +} diff --git a/tests/ir_tests/511_late_setif_branch_fuse.expect b/tests/ir_tests/511_late_setif_branch_fuse.expect new file mode 100644 index 000000000..b5ffebdb4 --- /dev/null +++ b/tests/ir_tests/511_late_setif_branch_fuse.expect @@ -0,0 +1,3 @@ +classify be316bb3 +one a79 db6 000 +probe 4 2 1 diff --git a/tests/ir_tests/512_nested_inline_ptr_return.c b/tests/ir_tests/512_nested_inline_ptr_return.c new file mode 100644 index 000000000..8d4173313 --- /dev/null +++ b/tests/ir_tests/512_nested_inline_ptr_return.c @@ -0,0 +1,66 @@ +/* A pointer-returning helper called from inside another inline expansion is + now inlined too. It used to be refused below depth 1 as a guard against + gcc.c-torture 930725-1 (a ternary between two pointer returns); the Zig C + backend's accessors -- `header()` is `bytes - 4`, `acquire()` a load -- + are exactly these helpers, called from `view()`/`itemPtr()`. 930725-1's + shape is repeated here one and two expansions deep, beside the accessor + shape, a pointer into a caller's local, and a pointer that feeds a store. */ +#include +#include +#include + +int v; + +static char *g(void) { return ""; } +static char *f(void) { return (v == 0 ? g() : "abc"); } +static inline char *f_deeper(void) { return f(); } +static inline const char *pick(int k) { return k ? f_deeper() : f(); } + +struct list { uint8_t *bytes; }; +struct header { uint32_t len, cap; }; + +static struct header *header(struct list const a0) +{ + struct list t0 = a0; + struct list const *t1 = &t0; + uint8_t *t3 = t1->bytes; + t3 = t3 - 8; + return (struct header *)t3; +} + +static uint32_t *items(struct list l) { return (uint32_t *)l.bytes; } + +static uint32_t view_sum(struct list l) +{ + struct header *h = header(l); + uint32_t *it = items(l), s = 0; + for (uint32_t i = 0; i < h->len; i++) + s += it[i]; + return s + h->cap; +} + +static int *slot(int *base, int i) { return base + (i & 3); } +static int bump(int *base, int i) { int *p = slot(base, i); *p += i; return *p; } + +__attribute__((noinline)) static int locals(int x) +{ + int a[4] = {1, 2, 3, 4}; + int r = 0; + for (int i = 0; i < 6; i++) + r += bump(a, i + x); + return r + a[0] + a[1] + a[2] + a[3]; +} + +int main(void) +{ + v = 1; + printf("930725 %d %d\n", !strcmp(pick(0), "abc"), !strcmp(pick(1), "abc")); + v = 0; + printf("930725 %d %d\n", !strcmp(pick(0), ""), !strcmp(pick(1), "")); + + static uint32_t store[2 + 5] = {5, 99, 10, 20, 30, 40, 50}; + struct list l = { (uint8_t *)&store[2] }; + printf("view %u\n", view_sum(l)); + printf("locals %d %d\n", locals(0), locals(3)); + return 0; +} diff --git a/tests/ir_tests/512_nested_inline_ptr_return.expect b/tests/ir_tests/512_nested_inline_ptr_return.expect new file mode 100644 index 000000000..dd4a39e0a --- /dev/null +++ b/tests/ir_tests/512_nested_inline_ptr_return.expect @@ -0,0 +1,4 @@ +930725 1 1 +930725 1 1 +view 249 +locals 54 98 diff --git a/tests/ir_tests/513_param_across_block_copy.c b/tests/ir_tests/513_param_across_block_copy.c new file mode 100644 index 000000000..2bd802fa4 --- /dev/null +++ b/tests/ir_tests/513_param_across_block_copy.c @@ -0,0 +1,42 @@ +/* A leaf whose body opens with a 64-byte-or-larger BLOCK_COPY -- a local + array initialised from a .rodata template -- runs that copy in r0-r3/IP/LR, + as it would a memcpy call. The allocator's call-crossing test counted only + calls strictly after an interval's start, and a parameter's interval starts + at instruction 0, where the copy sits: `i` stayed precolored in r0 and was + overwritten by the copy's destination address. One, two and four register + parameters, and a 64-bit one. */ +#include +#include + +__attribute__((noinline)) static uint32_t one(int i) +{ + const uint32_t t[24] = {3, 1, 4, 1, 5, 9, 2, 6, 5, 3, 5, 8, 9, 7, 9, 3, 2, 3, 8, 4, 6, 2, 6, 4}; + return t[i % 24] * 100 + t[(i + 7) % 24]; +} + +__attribute__((noinline)) static uint32_t two(uint32_t a, uint32_t b) +{ + const uint32_t t[16] = {11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26}; + return t[a & 15] * b + t[b & 15]; +} + +__attribute__((noinline)) static uint32_t four(uint32_t a, uint32_t b, uint32_t c, uint32_t d) +{ + const uint32_t t[20] = {9, 8, 7, 6, 5, 4, 3, 2, 1, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 110}; + return t[a % 20] + t[b % 20] * 3 + t[c % 20] * 5 + t[d % 20] * 7 + a + b + c + d; +} + +__attribute__((noinline)) static uint64_t wide(uint64_t x) +{ + const uint64_t t[8] = {1, 3, 5, 7, 11, 13, 17, 19}; + return t[x & 7] * x + t[(x >> 32) & 7]; +} + +int main(void) +{ + printf("one %u %u %u\n", one(0), one(11), one(23)); + printf("two %u %u\n", two(3, 5), two(17, 2)); + printf("four %u\n", four(1, 22, 45, 7)); + printf("wide %llu\n", (unsigned long long)wide(0x500000003ull)); + return 0; +} diff --git a/tests/ir_tests/513_param_across_block_copy.expect b/tests/ir_tests/513_param_across_block_copy.expect new file mode 100644 index 000000000..bbe66ce5c --- /dev/null +++ b/tests/ir_tests/513_param_across_block_copy.expect @@ -0,0 +1,4 @@ +one 306 808 402 +two 86 37 +four 138 +wide 150323855394 diff --git a/tests/ir_tests/514_inline_large_copies.c b/tests/ir_tests/514_inline_large_copies.c new file mode 100644 index 000000000..083aa2319 --- /dev/null +++ b/tests/ir_tests/514_inline_large_copies.c @@ -0,0 +1,115 @@ +/* Constant-size copies of 32 bytes and up are expanded inline instead of + calling memcpy: a by-value struct argument of eight words or more placed on + the stack (unrolled LDM/STM to 16 words, a counted loop above), a BLOCK_COPY + of 64 bytes or more from .rodata (a strcpy of a long literal, byte tail + included), an aligned struct assignment over 128 bytes, a local array + initialised from a .rodata template, and a string literal filling a char + array field of a word-aligned local (Zig's `undefined` fill) -- next to one + at an odd offset, which keeps its unaligned memcpy. Every stack-struct + size from 8 to 26 words -- each remainder of both the unrolled and the + looped form -- plus one of 75 words, with register arguments live around + the copy. */ +#include +#include +#include + +#define S(n) struct s##n { uint32_t w[n]; }; +S(8) S(9) S(10) S(11) S(12) S(13) S(15) S(16) S(17) S(18) S(19) S(20) S(26) S(75) + +static uint32_t mix(const uint32_t *w, int n, uint32_t a, uint32_t b, uint32_t c) +{ + uint32_t h = a * 3 + b * 5 + c * 7; + for (int i = 0; i < n; i++) + h = h * 31 + w[i]; + return h; +} + +/* Three register args first, so the struct is split: r3 plus the stack. */ +#define F(n) __attribute__((noinline)) static uint32_t f##n(uint32_t a, uint32_t b, uint32_t c, struct s##n s) \ + { return mix(s.w, n, a, b, c); } +F(8) F(9) F(10) F(11) F(12) F(13) F(15) F(16) F(17) F(18) F(19) F(20) F(26) F(75) + +/* Four register args first: the struct is all on the stack. */ +__attribute__((noinline)) static uint32_t g17(uint32_t a, uint32_t b, uint32_t c, uint32_t d, struct s17 s) +{ + return mix(s.w, 17, a, b, c + d); +} + +/* A struct with a byte tail passed by value. */ +struct odd { uint32_t w[9]; uint8_t t[3]; }; +__attribute__((noinline)) static uint32_t fodd(uint32_t a, struct odd o) +{ + return mix(o.w, 9, a, o.t[0] | o.t[1] << 8, o.t[2]); +} + +/* Leaf functions: the copy clobbers LR, which they did not save. */ +__attribute__((noinline)) static uint32_t leaf_strcpy(int k) +{ + char buf[80]; + strcpy(buf, "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ-+*/!?"); + uint32_t h = 0; + for (int i = 0; buf[i]; i++) + h = h * 33 + (unsigned char)buf[i] + (uint32_t)k; + return h + (uint32_t)strlen(buf); +} + +__attribute__((noinline)) static uint32_t leaf_table(int i) +{ + const uint32_t t[24] = {3, 1, 4, 1, 5, 9, 2, 6, 5, 3, 5, 8, 9, 7, 9, 3, 2, 3, 8, 4, 6, 2, 6, 4}; + return t[i % 24] * 100 + t[(i + 7) % 24]; +} + +struct big { uint32_t w[50]; }; +__attribute__((noinline)) static void assign(struct big *d, const struct big *s) { *d = *s; } + +#define AA "\252\252\252\252\252\252\252\252\252\252\252\252" +struct wy { uint64_t seed; uint32_t a, b; char buf[48]; }; +__attribute__((noinline)) static uint32_t wy_fill(uint32_t k) +{ + struct wy w = {k, k * 3, k ^ 5, {AA AA AA AA}}; + w.buf[k % 48] = 1; + return mix((const uint32_t *)w.buf, 12, (uint32_t)w.seed, w.a, w.b); +} + +struct odd_off { char c; char buf[41]; }; +__attribute__((noinline)) static uint32_t odd_fill(uint32_t k) +{ + struct odd_off o = {(char)k, {"0123456789abcdefghijklmnopqrstuvwxyzABCD"}}; + uint32_t h = (unsigned char)o.c; + for (int i = 0; i < 41; i++) + h = h * 31 + (unsigned char)o.buf[i]; + return h; +} + +int main(void) +{ + uint32_t acc = 0, seed = 0x12345678; +#define R() (seed = seed * 1103515245u + 12345u) +#define CALL(n) { struct s##n s; for (int i = 0; i < n; i++) s.w[i] = R(); \ + uint32_t a = R(), b = R(), c = R(); acc = acc * 7 + f##n(a, b, c, s); \ + acc ^= mix(s.w, n, a, b, c) == f##n(a, b, c, s) ? 0 : 0xdead0000u + n; } + CALL(8) CALL(9) CALL(10) CALL(11) CALL(12) CALL(13) CALL(15) CALL(16) + CALL(17) CALL(18) CALL(19) CALL(20) CALL(26) CALL(75) + printf("byval %08x\n", acc); + + struct s17 s; + for (int i = 0; i < 17; i++) s.w[i] = R(); + printf("stack-only %d\n", g17(1, 2, 3, 4, s) == mix(s.w, 17, 1, 2, 7)); + + struct odd o; + for (int i = 0; i < 9; i++) o.w[i] = R(); + o.t[0] = 0x11; o.t[1] = 0x22; o.t[2] = 0x33; + printf("odd %d\n", fodd(5, o) == mix(o.w, 9, 5, 0x2211, 0x33)); + + printf("strcpy %08x\n", leaf_strcpy(3)); + printf("table %u %u %u\n", leaf_table(0), leaf_table(11), leaf_table(23)); + + static struct big a, b; + for (int i = 0; i < 50; i++) { a.w[i] = R(); b.w[i] = 0; } + assign(&b, &a); + assign(&a, &a); + printf("assign %d\n", !memcmp(&a, &b, sizeof a)); + printf("wy %08x %08x\n", wy_fill(7), wy_fill(40)); + printf("oddoff %08x\n", odd_fill(9)); + return 0; +} diff --git a/tests/ir_tests/514_inline_large_copies.expect b/tests/ir_tests/514_inline_large_copies.expect new file mode 100644 index 000000000..165e8287d --- /dev/null +++ b/tests/ir_tests/514_inline_large_copies.expect @@ -0,0 +1,8 @@ +byval cd10b864 +stack-only 1 +odd 1 +strcpy e74894ec +table 306 808 402 +assign 1 +wy 8432de0c c9468694 +oddoff 9b2631e3 diff --git a/tests/ir_tests/515_const_local_table_rodata.c b/tests/ir_tests/515_const_local_table_rodata.c new file mode 100644 index 000000000..a5040342e --- /dev/null +++ b/tests/ir_tests/515_const_local_table_rodata.c @@ -0,0 +1,138 @@ +/* A local array or struct written only by one run of constant stores covering + all of it, and only ever read through its address, is read from an image + in .rodata (const_local_table) -- the Zig C backend's + `(struct arr_8_usize){{4,4,...,2}}.array[i]` in every MultiArrayList.slice + no longer rebuilds the table on each loop turn. Beside the shapes it + rewrites (word, byte, halfword and 64-bit tables; a table built once + outside the loop), the ones it must leave alone: an element written by a + later store or through a pointer, the address passed to a call or + compared, a partly initialised object, and two tables sharing a slot in + sibling scopes. */ +#include +#include + +struct arr8 { uintptr_t array[8]; }; +struct parr8 { uint8_t *array[8]; }; +struct mal { uint8_t *bytes; uintptr_t len, capacity; }; +struct slice { struct parr8 ptrs; uintptr_t len, capacity; }; + +/* The Zig C backend's MultiArrayList.slice, as emitted. */ +__attribute__((noinline)) static struct slice slice_fn(struct mal const a0) +{ + struct mal const *t1; + struct parr8 *t3; + uintptr_t *t4, t6, t13, t14, t10; + uintptr_t const *t5; + uint8_t *const *t8; + uint8_t *t9, *t7, **t15; + uint32_t t11; + struct slice t16, t2; + struct mal t0; + _Bool t12; + t0 = a0; t1 = &t0; + t3 = &t2.ptrs; t4 = &t2.len; t5 = &t1->len; t6 = *t5; *t4 = t6; + t4 = &t2.capacity; t5 = &t1->capacity; t6 = *t5; *t4 = t6; + t8 = &t1->bytes; t9 = *t8; t7 = t9; t10 = 0; +zig_loop_22: + t6 = t10; t11 = (uint32_t)t6; t12 = t11 < 8u; + if (t12) { + t13 = (struct arr8){{4, 4, 4, 4, 4, 4, 4, 2}}.array[t6]; + t14 = (struct arr8){{7, 6, 5, 4, 3, 2, 1, 0}}.array[t6]; + t3 = &t2.ptrs; t15 = &t3->array[t14]; t9 = t7; *t15 = t9; t9 = t7; + t5 = &t1->capacity; t14 = *t5; t14 = t13 * t14; t9 = t9 + t14; t7 = t9; + goto zig_block_1; + } + goto zig_block_0; +zig_block_1:; + t6 = t6 + 1; t10 = t6; goto zig_loop_22; +zig_block_0:; + t16 = t2; + return t16; +} + +__attribute__((noinline)) static uint32_t bytes_halves(unsigned i) +{ + uint32_t s = 0; + for (unsigned k = 0; k < 12; k++) { + const uint8_t b[12] = {9, 250, 3, 7, 1, 0, 128, 64, 33, 17, 5, 2}; + const int16_t h[6] = {-3, 1000, -32768, 7, 0, 12345}; + s = s * 7 + b[(k + i) % 12] + (uint32_t)h[(k * i) % 6]; + } + return s; +} + +__attribute__((noinline)) static uint64_t wide(unsigned i) +{ + const uint64_t t[4] = {0x123456789abcdef0ull, 1, 0xffffffff00000000ull, 42}; + uint64_t s = 0; + for (unsigned k = 0; k < 9; k++) + s += t[(k + i) & 3] >> (k & 7); + return s; +} + +/* Must stay on the stack: written again after its initialiser. */ +__attribute__((noinline)) static int rewritten(unsigned i, unsigned j) +{ + int t[4] = {1, 2, 3, 4}; + t[2] = 30; + int *p = &t[0]; + p[i & 3] += 100; + return t[j & 3] * 1000 + t[(j + 1) & 3]; +} + +__attribute__((noinline)) static int sum_of(const int *p, int n) +{ + int s = 0; + for (int k = 0; k < n; k++) + s += p[k]; + return s; +} + +/* Must stay on the stack: its address leaves, and is compared. */ +__attribute__((noinline)) static int escapes(unsigned i) +{ + const int t[5] = {5, 10, 15, 20, 25}; + const int *p = &t[i % 5]; + return sum_of(t, 5) + *p + (p == &t[2]); +} + +/* Partly initialised: the rest is zero. */ +struct part { uint32_t a; uint8_t c; uint32_t w[3]; }; +__attribute__((noinline)) static uint32_t partial(unsigned i) +{ + struct part s = {7, 1}; + const uint32_t *w = s.w; + return s.a + s.c + w[i % 3]; +} + +/* Two tables that may share a slot, in sibling scopes. */ +__attribute__((noinline)) static int siblings(unsigned i, int which) +{ + int r = 0; + if (which) { + const int a[4] = {11, 12, 13, 14}; + r = a[i & 3]; + } else { + const int b[4] = {21, 22, 23, 24}; + r = b[i & 3]; + } + return r; +} + +int main(void) +{ + static uint8_t mem[256]; + struct mal m = {mem, 3, 5}; + struct slice sl = slice_fn(m); + unsigned acc = 0; + for (int k = 0; k < 8; k++) + acc = acc * 31 + (unsigned)(sl.ptrs.array[k] - mem); + printf("slice %u %u %u\n", acc, (unsigned)sl.len, (unsigned)sl.capacity); + printf("bytes %u %u\n", bytes_halves(0), bytes_halves(5)); + printf("wide %llu %llu\n", (unsigned long long)wide(0), (unsigned long long)wide(3)); + printf("rewritten %d %d\n", rewritten(1, 1), rewritten(2, 3)); + printf("escapes %d %d\n", escapes(2), escapes(4)); + printf("partial %u %u\n", partial(0), partial(2)); + printf("siblings %d %d %d\n", siblings(1, 1), siblings(2, 0), siblings(7, 1)); + return 0; +} diff --git a/tests/ir_tests/515_const_local_table_rodata.expect b/tests/ir_tests/515_const_local_table_rodata.expect new file mode 100644 index 000000000..55c2b9b03 --- /dev/null +++ b/tests/ir_tests/515_const_local_table_rodata.expect @@ -0,0 +1,7 @@ +slice 1245193648 3 5 +bytes 54285833 1562495301 +wide 7605438857582316500 3146835195059263173 +rewritten 102030 4001 +escapes 91 100 +partial 8 8 +siblings 12 23 14 diff --git a/tests/ir_tests/516_symref_struct_arg_from_table.c b/tests/ir_tests/516_symref_struct_arg_from_table.c new file mode 100644 index 000000000..6f185ec6c --- /dev/null +++ b/tests/ir_tests/516_symref_struct_arg_from_table.c @@ -0,0 +1,35 @@ +/* A struct passed by value out of a constant local spec -- the opt DSL's + `const IROptRewriteSpec _opt_dsl_rw = {...}` handing `_opt_dsl_rw.src1` to a + setter. const_local_table reads the spec from .rodata, so the argument is + `*(image + 4)`, a STRUCT-typed symref operand. symref_prop rebuilt such + operands with the index in u.pool_idx, but a STRUCT operand keeps its + ctype in u.s.ctype_idx and the symref index in u.s.aux_data: the argument + came from whatever symbol aux_data last named (the self-hosted tcc read + tcc_ir_op_get_dest's GOT slot and crashed). */ +#include +#include + +struct op { uint32_t a, b, c; }; +struct spec { uint32_t tag; struct op d, s1; uint8_t flags; }; + +__attribute__((noinline)) static uint32_t take(struct op o) { return o.a * 100 + o.b * 10 + o.c; } +__attribute__((noinline)) static uint32_t take2(uint32_t k, struct op o) { return k + take(o); } + +__attribute__((noinline)) static uint32_t apply(int which, uint32_t k) +{ + const struct spec sp = {7, {1, 2, 3}, {4, 5, 6}, 9}; + uint32_t r = sp.tag; + if (which & 1) + r += take(sp.d); + if (which & 2) + r += take2(k, sp.s1); + if (sp.flags == 9) + r += take(sp.s1); + return r; +} + +int main(void) +{ + printf("apply %u %u %u %u\n", apply(0, 5), apply(1, 5), apply(2, 5), apply(3, 1000)); + return 0; +} diff --git a/tests/ir_tests/516_symref_struct_arg_from_table.expect b/tests/ir_tests/516_symref_struct_arg_from_table.expect new file mode 100644 index 000000000..33a6d8c7b --- /dev/null +++ b/tests/ir_tests/516_symref_struct_arg_from_table.expect @@ -0,0 +1 @@ +apply 463 586 924 2042 diff --git a/tests/ir_tests/517_sra_pointer_var_roots.c b/tests/ir_tests/517_sra_pointer_var_roots.c new file mode 100644 index 000000000..3c2b632d6 --- /dev/null +++ b/tests/ir_tests/517_sra_pointer_var_roots.c @@ -0,0 +1,160 @@ +/* Scalar replacement of struct locals reached through pointer VARs -- the + Zig C backend's `t1 = &t0; t5 = &t1->len; x = *t5;`. A pointer local is + followed within a block (and everywhere when its only definition is one + `&obj`); its fields become VARs. Shapes: a pointer reused for different + fields, a pointer copied to another, one defined before a loop and read + in it, writes through it read back directly, and the ones that must keep + memory -- a pointer set on two paths and read after the join, compared, + passed to a call, stored, or indexed at run time. */ +#include +#include + +struct mal { uint8_t *bytes; uint32_t len, cap; }; +struct pair { uint32_t a, b; }; + +__attribute__((noinline)) static uint32_t reuse(struct mal const a0) +{ + struct mal t0; + struct mal const *t1; + uint32_t const *t5; + uint32_t x, y; + t0 = a0; + t1 = &t0; + t5 = &t1->len; + x = *t5; + t5 = &t1->cap; + y = *t5; + return x * 1000 + y + (uint32_t)(t1->bytes != 0); +} + +__attribute__((noinline)) static uint32_t copy_and_loop(struct pair const a0, int n) +{ + struct pair t0 = a0; + struct pair *t1 = &t0, *t2; + uint32_t acc = 0; + t2 = t1; + for (int i = 0; i < n; i++) { + t2->a += (uint32_t)i; + acc += t1->b * t2->a; + } + return acc + t0.a; +} + +__attribute__((noinline)) static uint32_t write_through(uint32_t v) +{ + struct pair t0 = {1, 2}; + uint32_t *p = &t0.b; + *p = v; + p = &t0.a; + *p += 5; + return t0.a * 100 + t0.b; +} + +__attribute__((noinline)) static uint32_t join(int c, uint32_t v) +{ + struct pair x = {10, 20}, y = {30, 40}; + uint32_t *p; + if (c) + p = &x.a; + else + p = &y.b; + *p = v; + return x.a + x.b * 3 + y.a * 5 + y.b * 7; +} + +__attribute__((noinline)) static int is_first(const struct pair *p, const struct pair *q) { return p == q; } + +__attribute__((noinline)) static uint32_t escapes(uint32_t v) +{ + struct pair t0 = {v, v + 1}; + struct pair *t1 = &t0; + uint32_t r = t1->a; + if (t1 != 0 && is_first(t1, &t0)) + r += 100; + static struct pair *keep; + keep = t1; + keep->b += 7; + return r + t0.b; +} + +__attribute__((noinline)) static uint32_t indexed(uint32_t i) +{ + struct pair t0 = {3, 4}; + struct pair *t1 = &t0; + uint32_t *w = &t1->a; + return w[i & 1] * 10 + t0.a; +} + +/* The address reaches a pointer through a TEMP (`T <- t1; t5 <- T + 4`) and + is used in another block: t5 must count as a pointer that may hold it. */ +__attribute__((noinline)) static uint32_t other_block(struct pair const a0, int c, uint32_t v) +{ + struct pair t0; + struct pair *t1; + uint32_t *t5; + t0 = a0; + t1 = &t0; + t5 = &t1->b; + if (c) + *t5 = v; + return t0.b + *t5; +} + +/* One local reused for an unrelated pointer (Zig reuses locals by type): its + loaded value is no frame address, and t0 stays promotable. */ +__attribute__((noinline)) static uint32_t reused_local(struct pair const a0, uint32_t *const *pp) +{ + struct pair t0; + uint32_t *t5; + uint32_t x; + t0 = a0; + t5 = &t0.a; + x = *t5; + t5 = *pp; + x += *t5; + return x + t0.b; +} + +/* Across blocks by dataflow: the same address on both paths into a join + (promotable), different ones (kept), and a pointer moved inside a loop. */ +__attribute__((noinline)) static uint32_t df_agree(struct pair const a0, int c) +{ + struct pair t0 = a0; + uint32_t *t5 = &t0.b, acc = 1; + if (c) + acc += 5; + else + acc *= 3; + return acc * 100 + *t5; +} + +__attribute__((noinline)) static uint32_t df_loop(struct pair const a0, int n) +{ + struct pair t0 = a0; + uint32_t *t5 = &t0.a, acc = 0; + for (int i = 0; i < n; i++) { + acc = acc * 7 + *t5; + if (i & 1) + t5 = &t0.b; + *t5 += 1; + } + return acc + t0.a * 11 + t0.b; +} + +int main(void) +{ + static uint8_t mem[4]; + struct mal m = {mem, 7, 9}; + printf("reuse %u\n", reuse(m)); + struct pair pr = {3, 5}; + printf("loop %u %u\n", copy_and_loop(pr, 0), copy_and_loop(pr, 6)); + printf("write %u\n", write_through(42)); + printf("join %u %u\n", join(1, 99), join(0, 99)); + printf("escapes %u\n", escapes(11)); + printf("indexed %u %u\n", indexed(0), indexed(1)); + printf("other_block %u %u\n", other_block(pr, 1, 77), other_block(pr, 0, 77)); + uint32_t cell = 1000, *cp = &cell; + printf("reused %u\n", reused_local(pr, &cp)); + printf("df %u %u %u %u\n", df_agree(pr, 1), df_agree(pr, 0), df_loop(pr, 0), df_loop(pr, 7)); + return 0; +} diff --git a/tests/ir_tests/517_sra_pointer_var_roots.expect b/tests/ir_tests/517_sra_pointer_var_roots.expect new file mode 100644 index 000000000..def21f513 --- /dev/null +++ b/tests/ir_tests/517_sra_pointer_var_roots.expect @@ -0,0 +1,9 @@ +reuse 7010 +loop 3 283 +write 642 +join 589 913 +escapes 130 +indexed 33 43 +other_block 154 10 +reused 1008 +df 605 305 38 437502 diff --git a/tests/ir_tests/518_ssa_promote_entry_and_sra_fields.c b/tests/ir_tests/518_ssa_promote_entry_and_sra_fields.c new file mode 100644 index 000000000..cc0f6e675 --- /dev/null +++ b/tests/ir_tests/518_ssa_promote_entry_and_sra_fields.c @@ -0,0 +1,166 @@ +/* SSA promotion of VARs the old rules left in memory, and the gates that keep + the widening sound. Promoted now: a VAR defined only in the entry block + (which nothing jumps back to), read in later blocks and in a loop; and the + field VARs SRA makes for a small struct, whose defs are all slot stores. + Kept as before: a VAR whose only def is NOT in the entry block (an empty + dominance frontier is not trusted: the un-rotated loop shape under-reports + it, gcc-torture pr78675), and a VAR read at two offsets (renaming clears the + offset: a _Complex double's two halves, ieee/cdivchk). */ +#include +#include + +struct pair { uint32_t a, b; }; +struct eu { uint32_t payload; uint16_t error; }; + +__attribute__((noinline)) static int entry_only(int a, int n) +{ + int k = a * 3; + int base = a - 1; + int s = 0; + for (int i = 0; i < n; i++) + s += k + i; + if (s > 100) + s -= base; + return s + k; +} + +__attribute__((noinline)) static int def_in_loop(int n) +{ + int last; + int s = 0; + for (int i = 0; i < n; i++) + { + if (i == n - 1) + last = i * 7; + s += i; + } + return n > 0 ? s + last : s; +} + +__attribute__((noinline)) static double halves(double _Complex z) +{ + double re = __real__ z; + double im = __imag__ z; + return re * 10 + im; +} + +__attribute__((noinline)) static uint32_t sra_fields(uint32_t x, uint32_t y, int n) +{ + struct pair t; + t.a = x; + t.b = y; + for (int i = 0; i < n; i++) + { + uint32_t na = t.b + (uint32_t)i; + t.b = t.a ^ na; + t.a = na; + } + return t.a * 3 + t.b; +} + +__attribute__((noinline)) static struct eu make_eu(uint32_t v, int fail) +{ + struct eu r; + if (fail) + { + r.payload = 0xaaaaaaaa; + r.error = 5; + } + else + { + r.payload = v * 2; + r.error = 0; + } + return r; +} + +__attribute__((noinline)) static uint32_t use_eu(uint32_t v) +{ + struct eu t = make_eu(v, v > 50); + struct eu u = t; + if (u.error != 0) + return 1000u + u.error; + return u.payload; +} + +/* The Zig C backend's shapes (from zig.c): a parameter copied into a local and + read back through a pointer local, the same field loaded twice through it + (GenZir.isEmpty) -- promoted, the second load is the first one's value. */ +struct list { uint32_t *items; uintptr_t len; }; +struct genzir { uint32_t pad[4]; struct list *instructions; uintptr_t instructions_top; }; + +__attribute__((noinline)) static _Bool zig_is_empty(struct genzir const *const a0) +{ + struct genzir const *const *t1; + struct genzir const *t2; + struct genzir const *t0; + uintptr_t const *t3; + uintptr_t t4, t12; + struct list *const *t8; + struct list *t9; + uintptr_t *t11; + _Bool t6, t7; + t0 = a0; + t1 = (struct genzir const *const *)&t0; + t2 = (*t1); + t3 = (uintptr_t const *)&t2->instructions_top; + t4 = (*t3); + t6 = t4 == (uintptr_t)-1; + if (t6) { + t7 = 1; + goto zig_block_0; + } + t2 = (*t1); + t8 = (struct list *const *)&t2->instructions; + t9 = (*t8); + t11 = &t9->len; + t4 = (*t11); + t2 = (*t1); + t3 = (uintptr_t const *)&t2->instructions_top; + t12 = (*t3); + t6 = t4 == t12; + t7 = t6; + goto zig_block_0; +zig_block_0:; + return t7; +} + +/* An optional built in one local and copied into the returned one + (Ast.Node.OptionalIndex.unwrap): SRA's field VARs, promoted. */ +struct opt_index { uint32_t payload; _Bool is_null; }; + +__attribute__((noinline)) static struct opt_index zig_unwrap(uint32_t const a0) +{ + struct opt_index t0; + struct opt_index t4; + uint32_t t2; + _Bool t1; + t1 = a0 == UINT32_MAX; + if (t1) { + t0 = (struct opt_index){ .is_null = 1, .payload = UINT32_C(0xaaaaaaaa) }; + goto zig_block_0; + } + t2 = a0 + 1; + t4.is_null = 0; + t4.payload = t2; + t0 = t4; + goto zig_block_0; +zig_block_0:; + return t0; +} + +int main(void) +{ + uint32_t items[3] = {1, 2, 3}; + struct list l = {items, 3}; + struct genzir g1 = {{0}, &l, 3}, g2 = {{0}, &l, 1}, g3 = {{0}, &l, (uintptr_t)-1}; + printf("empty %d %d %d\n", zig_is_empty(&g1), zig_is_empty(&g2), zig_is_empty(&g3)); + struct opt_index o1 = zig_unwrap(41), o2 = zig_unwrap(UINT32_MAX); + printf("unwrap %u %d %d\n", (unsigned)o1.payload, o1.is_null, o2.is_null); + printf("entry %d %d %d\n", entry_only(4, 3), entry_only(9, 20), entry_only(-2, 0)); + printf("loop %d %d\n", def_in_loop(5), def_in_loop(0)); + printf("halves %.1f %.1f\n", halves(2.0 + 3.0 * 1.0i), halves(-1.5 + 0.5 * 1.0i)); + printf("sra %u %u\n", (unsigned)sra_fields(1, 2, 5), (unsigned)sra_fields(7, 3, 0)); + printf("eu %u %u\n", (unsigned)use_eu(21), (unsigned)use_eu(60)); + return 0; +} diff --git a/tests/ir_tests/518_ssa_promote_entry_and_sra_fields.expect b/tests/ir_tests/518_ssa_promote_entry_and_sra_fields.expect new file mode 100644 index 000000000..d6574bf37 --- /dev/null +++ b/tests/ir_tests/518_ssa_promote_entry_and_sra_fields.expect @@ -0,0 +1,7 @@ +empty 1 0 1 +unwrap 42 0 1 +entry 51 749 -6 +loop 38 0 +halves 23.0 -14.5 +sra 37 24 +eu 42 1005 diff --git a/tests/ir_tests/519_sra_struct_arg_words.c b/tests/ir_tests/519_sra_struct_arg_words.c new file mode 100644 index 000000000..3908127ac --- /dev/null +++ b/tests/ir_tests/519_sra_struct_arg_words.c @@ -0,0 +1,96 @@ +/* A small struct passed BY VALUE from a local whose fields SRA replaced is + passed as one argument per word, sourced from the field VARs. For a + 4-aligned, float-free 8/12/16-byte struct AAPCS places the words exactly + where N word arguments go: from the NCRN, straddling r3 onto the stack. + Shapes: the Zig slice (2 words, r0-r1), two split structs in one call with + a scalar after them (later arguments renumbered), 3 words straddling r3, + 4 words wholly on the stack, a narrow field at a word's start, a word no + field covers, and an 8-aligned struct that keeps its memory (it would start + at an even register). */ +#include +#include + +struct slice { uint8_t const *ptr; uint32_t len; }; +struct alloc { void *ctx; uint32_t id; }; +struct three { uint32_t a, b, c; }; +struct four { uint32_t a, b, c, d; }; +struct eu { uint32_t payload; uint16_t error; }; +struct gap { uint32_t a, unused, c; }; +struct wide { uint64_t v; }; + +__attribute__((noinline)) static uint32_t sum(struct slice s) +{ + uint32_t r = 0; + for (uint32_t i = 0; i < s.len; i++) + r = r * 31 + s.ptr[i]; + return r; +} + +__attribute__((noinline)) static uint32_t take(struct alloc a, struct slice s, uint32_t n) +{ + return a.id * 1000 + s.len * 10 + n + (s.ptr[0] == 'e'); +} + +__attribute__((noinline)) static uint32_t straddle(uint32_t x, uint32_t y, uint32_t z, struct three t) +{ + return x + y * 10 + z * 100 + t.a * 1000 + t.b * 10000 + t.c * 100000; +} + +__attribute__((noinline)) static uint32_t on_stack(uint32_t w, uint32_t x, uint32_t y, uint32_t z, struct four f) +{ + return w + x + y + z + f.a * 10 + f.b * 100 + f.c * 1000 + f.d * 10000; +} + +__attribute__((noinline)) static uint32_t check_eu(struct eu e) +{ + return e.error ? 100000u + e.error : e.payload; +} + +__attribute__((noinline)) static uint32_t check_gap(struct gap g) +{ + return g.a * 7 + g.c; +} + +__attribute__((noinline)) static uint64_t check_wide(uint32_t x, struct wide w) +{ + return w.v + x; +} + +__attribute__((noinline)) static uint32_t caller(uint8_t const *p, uint32_t n, void *ctx, uint32_t k) +{ + struct slice t0; + struct alloc t1; + struct three t2; + struct four t3; + struct eu t4; + struct gap t5; + struct wide t6; + t0.ptr = p + 1; + t0.len = n - 1; + t1.ctx = ctx; + t1.id = k; + uint32_t r = sum(t0); + r += take(t1, t0, k + 3); + t2.a = 4; t2.b = 5; t2.c = k; + r += straddle(1, 2, 3, t2); + t3.a = 1; t3.b = 2; t3.c = 3; t3.d = k; + r += on_stack(9, 8, 7, 6, t3); + t4.payload = n * 3; + t4.error = (uint16_t)(k > 5 ? k : 0); + r += check_eu(t4); + t5.a = k; + t5.c = n; + r += check_gap(t5); + t6.v = ((uint64_t)k << 32) | n; + r += (uint32_t)(check_wide(2, t6) >> 32); + return r; +} + +int main(void) +{ + static const uint8_t text[] = "hello"; + int dummy; + printf("%u\n", (unsigned)caller(text, 5, &dummy, 2)); + printf("%u\n", (unsigned)caller(text, 3, &dummy, 9)); + return 0; +} diff --git a/tests/ir_tests/519_sra_struct_arg_words.expect b/tests/ir_tests/519_sra_struct_arg_words.expect new file mode 100644 index 000000000..6a61a706a --- /dev/null +++ b/tests/ir_tests/519_sra_struct_arg_words.expect @@ -0,0 +1,2 @@ +3395781 +1159917 diff --git a/tests/ir_tests/520_sra_copy_and_result_buffer.c b/tests/ir_tests/520_sra_copy_and_result_buffer.c new file mode 100644 index 000000000..2a9e991be --- /dev/null +++ b/tests/ir_tests/520_sra_copy_and_result_buffer.c @@ -0,0 +1,165 @@ +/* Scalar replacement of struct locals a call accesses whole: an aggregate + over 16 bytes copied with __aeabi_memmove4/8 into or out of the object, and + a struct-returning call writing its result buffer. The copy becomes one + load or store per field and goes; the buffer keeps its slot and the fields + are read back after the call. Bytes the object only passes on (copied in, + then copied out or passed by value) need fields too, or they are lost. */ +#include +#include + +struct six { uint32_t a, b, c, d, e, f; }; +struct wide { uint64_t a; uint32_t b, c; uint64_t d; }; +struct narrow { uint8_t a; uint8_t pad; uint16_t b; uint32_t c, d, e, f, g; }; +struct outer { uint32_t tag; struct six in; }; + +__attribute__((noinline)) static struct six make(uint32_t x) +{ + struct six r = {x, x + 1, x + 2, x + 3, x + 4, x + 5}; + return r; +} + +/* Copied in from a pointer, every field read. */ +__attribute__((noinline)) static uint32_t copy_in(struct six const *p, int n) +{ + struct six t0; + uint32_t acc = 0, a, b, c, d, e, f; + t0 = *p; + a = t0.a; + b = t0.b; + c = t0.c; + d = t0.d; + e = t0.e; + f = t0.f; + for (int i = 0; i < n; i++) + acc = acc * 3 + a + b * 2 + c * 3 + d * 5 + e * 7 + f * 11; + return acc; +} + +/* Built field by field, copied out. */ +__attribute__((noinline)) static void copy_out(struct six *out, uint32_t v) +{ + struct six t0; + t0.a = v; + t0.b = v * 2; + t0.c = v * 3; + t0.d = v * 4; + t0.e = v * 5; + t0.f = v * 6; + *out = t0; +} + +/* Copied in, one field changed, copied out: the others pass through. */ +__attribute__((noinline)) static void through(struct six const *p, struct six *out, uint32_t v) +{ + struct six t0; + uint32_t x; + t0 = *p; + x = t0.c; + t0.c = x + v; + *out = t0; +} + +/* A result buffer, its fields read. */ +__attribute__((noinline)) static uint32_t result(uint32_t x) +{ + struct six t0; + uint32_t a, f; + t0 = make(x); + a = t0.a; + f = t0.f; + return a * 100 + f; +} + +/* A result buffer copied out after a field changed: the bytes nothing reads + one at a time pass through too. */ +__attribute__((noinline)) static void result_through(struct six *out, uint32_t x) +{ + struct six t0; + uint32_t a, b, c, d; + t0 = make(x); + a = t0.a; + b = t0.b; + c = t0.c; + d = t0.d; + t0.b = a + b * 10 + c + d; + t0.d = d * 2; + *out = t0; +} + +/* A result written into a member of a bigger object. */ +__attribute__((noinline)) static uint32_t result_member(uint32_t x) +{ + struct outer o; + uint32_t t, a, e; + o.tag = 7; + o.in = make(x); + t = o.tag; + a = o.in.a; + e = o.in.e; + return t * 1000 + a + e; +} + +/* 64-bit fields through a copy. */ +__attribute__((noinline)) static void wide_through(struct wide const *p, struct wide *out) +{ + struct wide t0; + uint64_t a; + t0 = *p; + a = t0.a; + t0.a = a + 0x100000001ull; + *out = t0; +} + +/* Narrow fields through a copy. */ +__attribute__((noinline)) static uint32_t narrow_through(struct narrow const *p, struct narrow *out) +{ + struct narrow t0; + uint32_t s; + t0 = *p; + s = t0.a + t0.b; + t0.a = (uint8_t)(t0.a + 1); + *out = t0; + return s; +} + +/* A copy between two objects both could replace. */ +__attribute__((noinline)) static uint32_t pair(struct six const *p, int k) +{ + struct six t0, t1; + struct six *q = &t1; + uint32_t a, f; + t0 = *p; + a = t0.a; + t0.a = a + (uint32_t)k; + *q = t0; + a = t1.a; + f = t1.f; + return a + f; +} + +static void show(const char *n, struct six const *s) +{ + printf("%s %u %u %u %u %u %u\n", n, s->a, s->b, s->c, s->d, s->e, s->f); +} + +int main(void) +{ + struct six s = {1, 2, 3, 4, 5, 6}, o; + printf("copy_in %u %u\n", copy_in(&s, 0), copy_in(&s, 3)); + copy_out(&o, 9); + show("copy_out", &o); + through(&s, &o, 40); + show("through", &o); + printf("result %u\n", result(20)); + result_through(&o, 30); + show("result_through", &o); + printf("result_member %u\n", result_member(50)); + struct wide w = {0x1111111122222222ull, 3, 4, 0x3333333344444444ull}, wo; + wide_through(&w, &wo); + printf("wide %llx %u %u %llx\n", (unsigned long long)wo.a, wo.b, wo.c, (unsigned long long)wo.d); + struct narrow nr = {200, 0, 60000, 1, 2, 3, 4, 5}, no; + uint32_t ns = narrow_through(&nr, &no); + printf("narrow %u %u %u %u %u %u %u %u\n", ns, no.a, no.b, no.c, no.d, no.e, no.f, no.g); + printf("pair %u\n", pair(&s, 100)); + return 0; +} diff --git a/tests/ir_tests/520_sra_copy_and_result_buffer.expect b/tests/ir_tests/520_sra_copy_and_result_buffer.expect new file mode 100644 index 000000000..2c4517d68 --- /dev/null +++ b/tests/ir_tests/520_sra_copy_and_result_buffer.expect @@ -0,0 +1,9 @@ +copy_in 0 1755 +copy_out 9 18 27 36 45 54 +through 1 2 43 4 5 6 +result 2025 +result_through 30 405 32 66 34 35 +result_member 7104 +wide 1111111222222223 3 4 3333333344444444 +narrow 60200 201 60000 1 2 3 4 5 +pair 107 diff --git a/tests/ir_tests/521_sra_struct_arg_words_large.c b/tests/ir_tests/521_sra_struct_arg_words_large.c new file mode 100644 index 000000000..fc9cf19fe --- /dev/null +++ b/tests/ir_tests/521_sra_struct_arg_words_large.c @@ -0,0 +1,102 @@ +/* Larger by-value structs from SRA-replaced locals, passed one argument per + word (up to SRA's 32 bytes). Shapes: Zig's std.fmt.Options (24 bytes, + optional usize = word + bool, the bools written through an indexed narrow + store) after a pointer, straddling r1-r3 onto the stack, with a scalar after + it; the same object passed to two calls; 28 bytes wholly on the stack with + words no field covers; exactly 32 bytes; 36 bytes (over SRA's limit: keeps + its memory); and two narrow fields in one word (keeps its memory: the word + is no single field). */ +#include +#include + +struct opt_usize { uint32_t v; uint8_t is_null; }; +struct options { struct opt_usize precision, width; uint8_t alignment; uint32_t fill; }; +struct seven { uint32_t w[7]; }; +struct eight { uint32_t w[8]; }; +struct nine { uint32_t w[9]; }; +struct packed2 { uint8_t lo, hi; uint32_t rest[5]; }; + +__attribute__((noinline)) static uint32_t print_value(void *w, struct options o, uint32_t depth) +{ + uint32_t r = w != 0; + r += o.precision.is_null ? 7 : o.precision.v * 3; + r += o.width.is_null ? 11 : o.width.v * 5; + return r * 1000 + o.alignment * 100 + o.fill + depth * 13; +} + +__attribute__((noinline)) static uint32_t align_buffer(struct options o) +{ + return (o.width.is_null ? 0 : o.width.v) + o.fill; +} + +__attribute__((noinline)) static uint32_t sum7(uint32_t a, uint32_t b, uint32_t c, uint32_t d, struct seven f) +{ + uint32_t r = a + b * 3 + c * 5 + d * 7; + for (int i = 0; i < 7; i++) + if (i != 3) + r = r * 17 + f.w[i]; + return r; +} + +__attribute__((noinline)) static uint32_t sum8(uint32_t a, struct eight f, uint32_t z) +{ + uint32_t r = a; + for (int i = 0; i < 8; i++) + r = r * 31 + f.w[i]; + return r ^ z; +} + +__attribute__((noinline)) static uint32_t sum9(struct nine f) +{ + uint32_t r = 0; + for (int i = 0; i < 9; i++) + r = r * 29 + f.w[i]; + return r; +} + +__attribute__((noinline)) static uint32_t sum_packed(uint32_t a, struct packed2 p) +{ + return a + p.lo * 3 + p.hi * 5 + p.rest[0] + p.rest[4] * 9; +} + +__attribute__((noinline)) static uint32_t caller(void *w, uint32_t n, uint32_t k) +{ + struct options o; + struct seven f; + struct eight s; + struct nine e; + struct packed2 p; + o.precision.v = 0; + o.precision.is_null = 1; + o.width.v = n; + o.width.is_null = k > 4; + o.alignment = 2; + o.fill = ' '; + uint32_t r = print_value(w, o, k + 1); + o.fill = '0' + k; + r += align_buffer(o); + r += print_value(0, o, 0); + f.w[0] = k; f.w[1] = n + k; f.w[2] = n * 2 + k; + f.w[4] = n * 4 + k; f.w[5] = n * 5 + k; f.w[6] = n * 6 + k; + r += sum7(1, 2, 3, 4, f); + s.w[0] = k; s.w[1] = 1 ^ k; s.w[2] = 2 ^ k; s.w[3] = 3 ^ k; + s.w[4] = 4 ^ k; s.w[5] = 5 ^ k; s.w[6] = 6 ^ k; s.w[7] = 7 ^ k; + r += sum8(n, s, 0x55); + e.w[0] = n; e.w[1] = 1 + n; e.w[2] = 2 + n; e.w[3] = 3 + n; e.w[4] = 4 + n; + e.w[5] = 5 + n; e.w[6] = 6 + n; e.w[7] = 7 + n; e.w[8] = 8 + n; + r += sum9(e); + p.lo = (uint8_t)n; + p.hi = (uint8_t)k; + p.rest[0] = n + k; + p.rest[4] = n * k; + r += sum_packed(3, p); + return r; +} + +int main(void) +{ + int dummy; + printf("%u\n", (unsigned)caller(&dummy, 5, 2)); + printf("%u\n", (unsigned)caller(&dummy, 9, 7)); + return 0; +} diff --git a/tests/ir_tests/521_sra_struct_arg_words_large.expect b/tests/ir_tests/521_sra_struct_arg_words_large.expect new file mode 100644 index 000000000..1ef51c6b2 --- /dev/null +++ b/tests/ir_tests/521_sra_struct_arg_words_large.expect @@ -0,0 +1,2 @@ +343048378 +2623730393 diff --git a/tests/ir_tests/522_sra_narrow_field_read_wider.c b/tests/ir_tests/522_sra_narrow_field_read_wider.c new file mode 100644 index 000000000..d77924e3b --- /dev/null +++ b/tests/ir_tests/522_sra_narrow_field_read_wider.c @@ -0,0 +1,114 @@ +/* A field stored at one width and loaded at others. + - Loaded wider: a narrow field stored only at its own width, then read as a + halfword or a word whose other bytes nothing writes. SRA makes it a field + VAR and the wider loads read that VAR (the bytes beyond it hold nothing, + so its zero bits stand for them). Shapes: Zig's error union + `{ uint16_t error; }` returned by value (loaded as a word); a 4-aligned + one-byte struct returned the same way; an optional's flag byte read as a + u16 and at its own width. The callers mask off the never-written bytes. + - Loaded wider across several narrow fields: two bytes stored one at a time + and read back as one halfword (their VARs shifted and ORed), unsigned and + signed (the high byte's top bit then sign-extends). + - Loaded narrower: a word stored, then read as its low byte and halfword, + signed and unsigned (an AND, or a shift pair), across a join. + + Kept in memory, with outputs that depend on every byte: a word written and + then a byte of it (a partial update of a wider field); a signed narrow + field read wider. */ +#include +#include + +struct err2 { uint16_t error; }; +union flag { uint8_t b; uint16_t h; uint32_t w; }; +struct tag4 { uint8_t tag; } __attribute__((aligned(4))); +union bytes2 { uint8_t b[2]; uint16_t h; int16_t sh; }; +union sflag { int8_t b; uint16_t h; }; +union w8 { uint32_t w; uint8_t b; int8_t sb; uint16_t h; int16_t sh; }; + +__attribute__((noinline)) static struct err2 make_err(uint32_t k) +{ + struct err2 t; + t.error = (uint16_t)(k * 3u + 1u); + return t; +} + +__attribute__((noinline)) static uint32_t flag_as_half(uint32_t k) +{ + union flag f; + f.b = k > 4; + return (f.h & 0xff) * 10 + f.b; +} + +__attribute__((noinline)) static struct tag4 make_tag(uint32_t n) +{ + struct tag4 t; + t.tag = (uint8_t)(n * 7u); + return t; +} + +__attribute__((noinline)) static uint32_t word_parts(uint32_t n, uint32_t k) +{ + union w8 f; + f.w = n * 5u; + if (k & 1) + f.w = n + 0xf0f3u; + return f.b + f.sb * 3 + f.h * 7 + f.sh * 11; +} + +__attribute__((noinline)) static uint32_t partial_update(uint32_t n) +{ + union flag f; + f.w = n; + f.b = 7; + return f.w; +} + +__attribute__((noinline)) static uint32_t bytes_as_half(uint32_t n) +{ + union bytes2 x; + x.b[0] = (uint8_t)n; + x.b[1] = (uint8_t)(n >> 8); + return x.h; +} + +__attribute__((noinline)) static uint32_t bytes_as_signed_half(uint32_t n) +{ + union bytes2 x; + x.b[0] = (uint8_t)n; + x.b[1] = (uint8_t)(n >> 8); + return (uint32_t)(int32_t)x.sh; +} + +__attribute__((noinline)) static uint32_t word_as_byte(uint32_t n) +{ + union flag f; + f.w = n * 5u; + return f.b; +} + +__attribute__((noinline)) static uint32_t signed_as_half(int32_t n) +{ + union sflag x; + x.b = (int8_t)(n & 0x7f); + return (x.h & 0xff) + (uint32_t)(int32_t)x.b; +} + +int main(void) +{ + uint32_t r = 0; + for (uint32_t k = 0; k < 9; k++) + { + struct err2 e = make_err(k * 1000u); + r = r * 31 + e.error; + r = r * 31 + flag_as_half(k); + r = r * 31 + make_tag(k * 12345u).tag; + r = r * 31 + word_parts(k * 0x10203fu + 0x7f80u, k); + r = r * 31 + partial_update(k * 0x01020304u); + r = r * 31 + bytes_as_half(k * 4099u); + r = r * 31 + bytes_as_signed_half(k * 0x3907u); + r = r * 31 + word_as_byte(k * 77u); + r = r * 31 + signed_as_half((int32_t)k * 45 - 100); + } + printf("%u\n", (unsigned)r); + return 0; +} diff --git a/tests/ir_tests/522_sra_narrow_field_read_wider.expect b/tests/ir_tests/522_sra_narrow_field_read_wider.expect new file mode 100644 index 000000000..7f51de4c5 --- /dev/null +++ b/tests/ir_tests/522_sra_narrow_field_read_wider.expect @@ -0,0 +1 @@ +2339444304 diff --git a/tests/ir_tests/522_unroll_zig_loop_tables.c b/tests/ir_tests/522_unroll_zig_loop_tables.c new file mode 100644 index 000000000..4e1f3c852 --- /dev/null +++ b/tests/ir_tests/522_unroll_zig_loop_tables.c @@ -0,0 +1,137 @@ +/* The Zig C backend writes a loop counter through two locals -- `t6 = t10; + ... t6 = t6 + 1; t10 = t6;` -- which hid every Zig loop's induction + variable (ssa:loop_unroll now reads the pair as one counter). A + constant-trip loop whose body loads and stores now unrolls, and a table it + indexes by the counter folds to the table's values once the index is + constant (ssa:unroll_cascade): + MultiArrayList.slice becomes straight-line stores. Beside those: an + indexed access whose shift was fused (barrel_shifts, carried into every + copy), a body binding a load to an inlined parameter (kept as a loop), a + packed bitfield RMW, 64-bit and 128-bit values, a stack array written at a + runtime index before the unrolled loop reads it, and a counter copy read + after the loop. */ +#include +#include + +struct arr8 { uintptr_t array[8]; }; +struct parr8 { uint8_t *array[8]; }; +struct mal { uint8_t *bytes; uintptr_t len, capacity; }; +struct slice { struct parr8 ptrs; uintptr_t len, capacity; }; + +__attribute__((noinline)) static struct slice slice_fn(struct mal const a0) +{ + struct mal const *t1; + uintptr_t *t4, t6, t13, t14, t10; + uintptr_t const *t5; + uint8_t *t9, *t7, **t15; + uint32_t t11; + struct slice t16, t2; + struct mal t0; + _Bool t12; + t0 = a0; t1 = &t0; + t4 = &t2.len; t5 = &t1->len; t6 = *t5; *t4 = t6; + t4 = &t2.capacity; t5 = &t1->capacity; t6 = *t5; *t4 = t6; + t9 = t1->bytes; t7 = t9; t10 = 0; +zig_loop_22: + t6 = t10; t11 = (uint32_t)t6; t12 = t11 < 8u; + if (t12) { + t13 = (struct arr8){{4, 4, 4, 4, 4, 4, 4, 2}}.array[t6]; + t14 = (struct arr8){{7, 6, 5, 4, 3, 2, 1, 0}}.array[t6]; + t15 = &t2.ptrs.array[t14]; t9 = t7; *t15 = t9; t9 = t7; + t5 = &t1->capacity; t14 = *t5; t14 = t13 * t14; t9 = t9 + t14; t7 = t9; + goto zig_block_1; + } + goto zig_block_0; +zig_block_1:; + t6 = t6 + 1; t10 = t6; goto zig_loop_22; +zig_block_0:; + t16 = t2; + return t16; +} + +/* Zig's counter shape, counter read after the loop, sum of a const table. */ +static const uint16_t weights[6] = {3, 1000, 7, 65535, 12, 9}; +__attribute__((noinline)) static uint32_t zig_sum(uint32_t scale, uint32_t *last) +{ + uintptr_t t6, t10 = 0; + uint32_t acc = 0; +loop: + t6 = t10; + if ((uint32_t)t6 < 6u) { + acc += weights[t6] * scale + (uint32_t)t6; + t6 = t6 + 1; t10 = t6; goto loop; + } + *last = (uint32_t)t10; + return acc; +} + +typedef struct { uint64_t lo, hi; } u128; +static inline u128 mul64(uint64_t a, uint64_t b) +{ + uint64_t p0 = (uint64_t)(uint32_t)a * (uint32_t)b, p1 = (a >> 32) * (uint32_t)b; + uint64_t p2 = (uint64_t)(uint32_t)a * (b >> 32), p3 = (a >> 32) * (b >> 32); + uint64_t mid = (p0 >> 32) + (uint32_t)p1 + (uint32_t)p2; + u128 r = {(mid << 32) | (uint32_t)p0, p3 + (p1 >> 32) + (p2 >> 32) + (mid >> 32)}; + return r; +} + +/* 128-bit values stored by index, read back reversed with a fused shift. */ +__attribute__((noinline)) static uint64_t wide(const uint64_t *v) +{ + u128 w[8]; + for (int k = 0; k < 8; k++) + w[k] = mul64(v[k], v[(k + 3) & 7]); + uint64_t acc = 0; + for (int k = 0; k < 8; k++) + acc = acc * 31 + (w[k].lo ^ (w[7 - k].hi >> 1)); + return acc; +} + +static uint32_t mix(uint32_t h, uint32_t v) { return (h ^ (v + 0x9e3779b9u + (h << 6) + (h >> 2))) * 2654435761u; } + +/* A runtime-index store into a stack array, then an unrolled read loop, + with the element bound to an inlined parameter. */ +__attribute__((noinline)) static uint32_t alias(uint32_t idx, uint32_t val) +{ + uint32_t a[8] = {10, 20, 30, 40, 50, 60, 70, 80}; + a[idx & 7] = val; + uint32_t h = 1; + for (uint32_t k = 0; k < 8; k++) + h = mix(h, a[k]); + return h; +} + +struct bf { unsigned b0 : 1, b1 : 5, b2 : 6, b3 : 11, b4 : 5; } __attribute__((packed)); +__attribute__((noinline)) static uint32_t bits(uint32_t s) +{ + struct bf x = {0, 0, 0, 0, 0}; + x.b1 = 19; + for (unsigned k = 0; k < 7; k++) + x.b2 = (s + k) & 63; + return x.b1 * 1000 + x.b2; +} + +int main(void) +{ + static uint8_t mem[512]; + struct slice sl = slice_fn((struct mal){mem, 3, 5}); + uint32_t acc = 0; + for (int k = 0; k < 8; k++) + acc = acc * 31 + (uint32_t)(sl.ptrs.array[k] - mem); + printf("slice %u %u %u\n", acc, (unsigned)sl.len, (unsigned)sl.capacity); + + uint32_t last = 0; + uint32_t s = zig_sum(3, &last); + printf("zig_sum %u %u\n", s, last); + + uint64_t v[8]; + uint32_t r = 0x2545f491u; + for (int i = 0; i < 8; i++) { + r ^= r << 13; r ^= r >> 17; r ^= r << 5; + v[i] = (uint64_t)r << 32 | (r * 2654435761u); + } + printf("wide %016llx\n", (unsigned long long)wide(v)); + printf("alias %08x %08x\n", alias(3, 7), alias(12, 99)); + printf("bits %u %u\n", bits(5), bits(60)); + return 0; +} diff --git a/tests/ir_tests/522_unroll_zig_loop_tables.expect b/tests/ir_tests/522_unroll_zig_loop_tables.expect new file mode 100644 index 000000000..227804064 --- /dev/null +++ b/tests/ir_tests/522_unroll_zig_loop_tables.expect @@ -0,0 +1,5 @@ +slice 1245193648 3 5 +zig_sum 199713 6 +wide b777d14e69876d8d +alias 17d706ba c1fe527f +bits 19011 19002 diff --git a/tests/ir_tests/523_zig_accessor_chain_collapse.c b/tests/ir_tests/523_zig_accessor_chain_collapse.c new file mode 100644 index 000000000..dd2a7fe65 --- /dev/null +++ b/tests/ir_tests/523_zig_accessor_chain_collapse.c @@ -0,0 +1,295 @@ +/* The Zig compiler's InternPool accessor chain, as its C backend writes it. + MultiArrayList.slice is a loop over the element's fields whose counter the + backend spells with two locals and passes through an inlined + zig_u32_bitCast_u32 before the exit test; once ssa:loop_unroll sees the + counter through that, the loop unrolls, its per-field tables fold, and the + body -- over 512 tokens of source, a few ops once optimized -- inlines as a + small leaf into its callers, including one whose table addresses were + hoisted into registers first and one that does an acquire load (which must + not make the table reads look volatile). Also: __atomic_load_n, and ordered + and relaxed loads still re-read memory that changed in between; and a + post-increment index read by a store in an unrolled body, and an unrolled + store whose index went through a fused barrel shift. */ +#include +#include +#include +/* What tcc's spells atomic_load_explicit as (the IR test + environment puts newlib's header first, and a device has no relative + path to tcc's). */ +#ifndef __ATOMIC_ACQUIRE +#define __ATOMIC_RELAXED 0 +#define __ATOMIC_ACQUIRE 2 +#define __ATOMIC_SEQ_CST 5 +#endif +#define memory_order_acquire __ATOMIC_ACQUIRE +#define atomic_load_explicit(object, order) __atomic_load_n((object), (order)) + +static inline uint32_t zig_u32_truncate_u32(uint32_t arg, uint8_t bits) +{ + return arg & (bits < UINT8_C(32) ? ((UINT32_C(1) << bits) - UINT32_C(1)) : UINT32_C(0xffffffff)); +} +static inline uint32_t zig_u32_bitCast_u32(uint32_t arg, uint8_t bits) { return zig_u32_truncate_u32(arg, bits); } + +struct arr_2_ptr_u8 { uint8_t *array[2]; }; +struct arr_1_ptr_u8 { uint8_t *array[1]; }; +struct arr_2_usize { uintptr_t array[2]; }; +struct arr_1_usize { uintptr_t array[1]; }; +struct MAL { uint8_t *bytes; uintptr_t len; uintptr_t capacity; }; +struct Slice2 { struct arr_2_ptr_u8 ptrs; uintptr_t len; uintptr_t capacity; }; +struct Slice1 { struct arr_1_ptr_u8 ptrs; uintptr_t len; uintptr_t capacity; }; +struct slice_u32 { uint32_t *ptr; uintptr_t len; }; +struct slice_u8 { uint8_t *ptr; uintptr_t len; }; + +/* Two fields: a u8 tag array after a u32 data array, larger field first. */ +static struct Slice2 slice2(struct MAL const a0) +{ + struct MAL const *t1; + struct arr_2_ptr_u8 *t3; + uintptr_t *t4; + uintptr_t const *t5; + uintptr_t t6; + uintptr_t t13; + uintptr_t t14; + uintptr_t t10; + uint8_t *const *t8; + uint8_t *t9; + uint8_t *t7; + uint32_t t11; + uint8_t **t15; + struct Slice2 t16; + struct Slice2 t2; + struct MAL t0; + bool t12; + t0 = a0; + t1 = (struct MAL const *)&t0; + t3 = (struct arr_2_ptr_u8 *)&t2.ptrs; + t4 = (uintptr_t *)&t2.len; + t5 = (uintptr_t const *)&t1->len; + t6 = (*t5); + (*t4) = t6; + t4 = (uintptr_t *)&t2.capacity; + t5 = (uintptr_t const *)&t1->capacity; + t6 = (*t5); + (*t4) = t6; + t8 = (uint8_t *const *)&t1->bytes; + t9 = (*t8); + t7 = t9; + t10 = (uintptr_t)0ul; +zig_loop_22: + t6 = t10; + t11 = zig_u32_bitCast_u32(t6, UINT8_C(32)); + t12 = t11 < UINT32_C(2); + if (t12) { + t13 = (struct arr_2_usize){{(uintptr_t)4ul, (uintptr_t)1ul}}.array[t6]; + t14 = (struct arr_2_usize){{(uintptr_t)1ul, (uintptr_t)0ul}}.array[t6]; + t3 = (struct arr_2_ptr_u8 *)&t2.ptrs; + t15 = &t3->array[t14]; + t9 = t7; + (*t15) = t9; + t9 = t7; + t5 = (uintptr_t const *)&t1->capacity; + t14 = (*t5); + t14 = t13 * t14; + t9 = (uint8_t *)(((uintptr_t)t9) + (t14 * sizeof(uint8_t))); + t7 = t9; + goto zig_block_1; + } + goto zig_block_0; + +zig_block_1:; + t6 = t6 + (uintptr_t)1ul; + t10 = t6; + goto zig_loop_22; + +zig_block_0:; + t16 = t2; + return t16; +} + +/* One field: a one-trip loop. */ +static struct Slice1 slice1(struct MAL const a0) +{ + struct MAL const *t1; + uintptr_t t6, t10, t13, t14; + uint8_t *t7, *t9; + struct Slice1 t2; + struct MAL t0; + t0 = a0; + t1 = (struct MAL const *)&t0; + t2.len = t1->len; + t2.capacity = t1->capacity; + t9 = t1->bytes; + t7 = t9; + t10 = (uintptr_t)0ul; +zig_loop_7: + t6 = t10; + if (zig_u32_bitCast_u32(t6, UINT8_C(32)) < UINT32_C(1)) { + t13 = (struct arr_1_usize){{(uintptr_t)4ul}}.array[t6]; + t14 = (struct arr_1_usize){{(uintptr_t)0ul}}.array[t6]; + t2.ptrs.array[t14] = t7; + t9 = t7; + t9 = t9 + t13 * t1->capacity; + t7 = t9; + t6 = t6 + (uintptr_t)1ul; + t10 = t6; + goto zig_loop_7; + } + return t2; +} + +/* The same counter read after the loop: stays a loop, and must still count. */ +__attribute__((noinline)) static uintptr_t fields_seen(struct MAL const a0, uintptr_t *sum) +{ + uintptr_t t6, t10 = 0, acc = 0; +loop: + t6 = t10; + if (zig_u32_bitCast_u32(t6, UINT8_C(32)) < UINT32_C(3)) { + acc += (struct arr_2_usize){{(uintptr_t)7ul, (uintptr_t)9ul}}.array[t6 & 1] * a0.capacity; + t6 = t6 + 1; + t10 = t6; + goto loop; + } + *sum = acc; + return t6; +} + +static struct slice_u8 items_tag(struct MAL m) +{ + struct Slice2 s = slice2(m); + if (s.capacity == 0) + return (struct slice_u8){(uint8_t *)0, 0}; + return (struct slice_u8){s.ptrs.array[0], s.len}; +} +static struct slice_u32 items_data(struct MAL m) +{ + struct Slice2 s = slice2(m); + if (s.capacity == 0) + return (struct slice_u32){(uint32_t *)0, 0}; + return (struct slice_u32){(uint32_t *)s.ptrs.array[1], s.len}; +} + +struct List { uint8_t *bytes; }; +struct Local { struct List items; }; + +/* itemPtr: an acquire load of the list's bytes, then two slice lookups. */ +struct ItemPtr { uint8_t *tag_ptr; uint32_t *data_ptr; }; +__attribute__((noinline)) static struct ItemPtr item_ptr(struct Local *const *local, uint32_t index) +{ + uint8_t *bytes = atomic_load_explicit((uint8_t **)&(*local)->items.bytes, memory_order_acquire); + struct MAL m = {bytes, ((uintptr_t *)bytes)[-1], ((uintptr_t *)bytes)[-1]}; + struct ItemPtr r; + r.tag_ptr = &items_tag(m).ptr[index]; + r.data_ptr = &items_data(m).ptr[index]; + return r; +} + +/* A caller with many inlined slices across a switch: the table addresses are + shared across the arms. */ +__attribute__((noinline)) static uint32_t key_of(struct Local *local, uint32_t index, uint32_t kind) +{ + uint8_t *bytes = __atomic_load_n(&local->items.bytes, __ATOMIC_ACQUIRE); + struct MAL m = {bytes, ((uintptr_t *)bytes)[-1], ((uintptr_t *)bytes)[-1]}; + switch (kind) { + case 0: return items_tag(m).ptr[index]; + case 1: return items_data(m).ptr[index]; + case 2: return items_tag(m).ptr[index] * 1000u + items_data(m).ptr[index]; + case 3: { + struct Slice1 s1 = slice1(m); + return (uint32_t)(s1.ptrs.array[0] - bytes) + (uint32_t)s1.len; + } + case 4: return (uint32_t)items_tag(m).len + (uint32_t)items_data(m).len * 7u; + default: { + struct Slice2 s = slice2(m); + return (uint32_t)(s.ptrs.array[0] - s.ptrs.array[1]); + } + } +} + +/* A post-increment index in a memory body: the saved old counter is read by + the store, so each unrolled copy keeps it (990525-1). */ +struct pair { int m1, m2; }; +__attribute__((noinline)) static int post_inc_store(struct pair arg) +{ + int i; + struct pair buf[3]; + for (i = 0; i < 3; buf[i++] = arg) + arg.m1 += i; + return buf[0].m1 * 100 + buf[1].m1 * 10 + buf[2].m1 + buf[2].m2; +} + +/* An unrolled store through `base + (j << 4)`, whose shift was fused into the + ADD: every copy must scale its constant index (pr93434 stored the imaginary + parts at t + 17, 18, ... -- unnoticed wherever the stack starts zeroed). */ +struct cplx { double re, im; }; +__attribute__((noinline)) static int clear_pairs(int fill) +{ + struct cplx t[16]; + unsigned char *p = (unsigned char *)t; + for (unsigned k = 0; k < sizeof t; k++) + p[k] = (unsigned char)(fill + k * 7); + for (int j = 0; j < 16; ++j) { + t[j].re = 0; + t[j].im = 0; + } + int bad = 0; + for (unsigned k = 0; k < sizeof t; k++) + bad += p[k] != 0; + return bad; +} + +static uint32_t loads(uint32_t *p, uint32_t *alias) +{ + uint32_t a = __atomic_load_n(p, __ATOMIC_ACQUIRE); + *alias = a + 5; + uint32_t b = __atomic_load_n(p, __ATOMIC_SEQ_CST); + *alias = b * 3; + uint32_t c = __atomic_load_n(p, __ATOMIC_RELAXED); + *alias = c + 1; + uint32_t d = atomic_load_explicit(p, memory_order_acquire); + uint16_t h = 0x1234; + uint16_t e = __atomic_load_n(&h, __ATOMIC_ACQUIRE); + return a + b * 10 + c * 100 + d * 1000 + e; +} + +int main(void) +{ + enum { CAP = 5 }; + static uintptr_t storage[1 + CAP * 2]; + uint8_t *bytes = (uint8_t *)&storage[1]; + storage[0] = CAP; + uint32_t *data = (uint32_t *)bytes; + uint8_t *tags = bytes + 4 * CAP; + for (int i = 0; i < CAP; i++) { + data[i] = 100 + i * 11; + tags[i] = (uint8_t)(3 + i); + } + struct Local local = {{bytes}}; + struct Local *lp = &local; + + struct MAL m = {bytes, 3, CAP}; + struct Slice2 s = slice2(m); + printf("slice2 %d %d %u %u\n", (int)(s.ptrs.array[0] - bytes), (int)(s.ptrs.array[1] - bytes), + (unsigned)s.len, (unsigned)s.capacity); + struct Slice1 s1 = slice1(m); + printf("slice1 %d %u %u\n", (int)(s1.ptrs.array[0] - bytes), (unsigned)s1.len, (unsigned)s1.capacity); + uintptr_t sum = 0; + uintptr_t seen = fields_seen(m, &sum); + printf("fields_seen %u %u\n", (unsigned)seen, (unsigned)sum); + + for (uint32_t i = 0; i < CAP; i += 2) { + struct ItemPtr ip = item_ptr(&lp, i); + printf("item %u %u %u\n", i, *ip.tag_ptr, *ip.data_ptr); + } + uint32_t k = 0; + for (uint32_t kind = 0; kind < 6; kind++) + for (uint32_t i = 0; i < CAP; i++) + k = k * 31 + key_of(&local, i, kind); + printf("keys %u\n", k); + + printf("post_inc_store %d\n", post_inc_store((struct pair){1, 2})); + printf("clear_pairs %d %d\n", clear_pairs(1), clear_pairs(0x5a)); + uint32_t x = 7; + uint32_t l = loads(&x, &x); + printf("loads %u %u\n", l, x); + return 0; +} diff --git a/tests/ir_tests/523_zig_accessor_chain_collapse.expect b/tests/ir_tests/523_zig_accessor_chain_collapse.expect new file mode 100644 index 000000000..e18e87225 --- /dev/null +++ b/tests/ir_tests/523_zig_accessor_chain_collapse.expect @@ -0,0 +1,10 @@ +slice2 20 0 3 5 +slice1 0 3 5 +fields_seen 3 115 +item 0 3 100 +item 2 5 122 +item 4 7 144 +keys 3413819844 +post_inc_store 126 +clear_pairs 0 0 +loads 45387 37 diff --git a/tests/ir_tests/524_sra_far_struct_arg.c b/tests/ir_tests/524_sra_far_struct_arg.c new file mode 100644 index 000000000..50e9967a6 --- /dev/null +++ b/tests/ir_tests/524_sra_far_struct_arg.c @@ -0,0 +1,100 @@ +/* By-value struct arguments of locals more than 32 KiB down the frame. A + struct stack operand keeps its offset in 16 bits, so tcc_ir_put names such + an argument through its address (`T <- Addr[slot]; PARAM T***DEREF***`). + SRA takes that form as it takes the slot itself: the argument becomes the + field VARs, one per word. The Zig C backend's slices and format options + reach calls this way from its large frames. + + scribble() fills the stack first, so a word read from a slot nobody wrote + is 0xA5A5A5A5 rather than a lucky leftover. */ +#include + +struct slice +{ + const unsigned char *ptr; + unsigned len; +}; + +struct opts +{ + unsigned char width; + unsigned char fill; + unsigned short prec; + unsigned flags; +}; + +struct quad +{ + unsigned a, b, c, d; +}; + +__attribute__((noinline)) void scribble(void) +{ + volatile unsigned char junk[36000]; + for (unsigned i = 0; i < sizeof junk; i++) + junk[i] = 0xA5; +} + +__attribute__((noinline)) unsigned hash(struct slice s) +{ + unsigned h = s.len; + for (unsigned i = 0; i < s.len; i++) + h = h * 31 + s.ptr[i]; + return h; +} + +__attribute__((noinline)) unsigned fmt(struct slice s, struct opts o) +{ + return hash(s) ^ o.width ^ (o.fill << 8) ^ ((unsigned)o.prec << 16) ^ o.flags; +} + +__attribute__((noinline)) unsigned mix(unsigned x, struct quad q) +{ + return x * 3 + q.a + q.b * 5 + q.c * 7 + q.d * 11; +} + +__attribute__((noinline)) unsigned run(const unsigned char *p, unsigned n, unsigned k) +{ + unsigned char buf[33000]; + struct slice s; + struct opts o; + struct quad q; + for (unsigned i = 0; i < n; i++) + buf[i] = p[i] + k; + s.ptr = buf; + s.len = n; + o.width = k; + o.fill = ' '; + o.prec = k * 300; + o.flags = n ^ 0x5a5a; + unsigned r = hash(s); + s.len = n - 1; + r += fmt(s, o); + q.a = r; + q.b = k; + q.c = n; + q.d = r >> 3; + r = mix(r, q); + if (k & 1) + { + q.b = 99; + s.ptr = p; + } + r += mix(r, q) + hash(s); + return r + buf[n / 2]; +} + +int main(void) +{ + static const unsigned char text[] = "far struct arguments"; + unsigned total = 0; + for (unsigned k = 0; k < 4; k++) + { + scribble(); + unsigned r = run(text, sizeof text - 1, k); + printf("k=%u r=%u\n", k, r); + total ^= r; + } + printf("total=%u\n", total); + return 0; +} diff --git a/tests/ir_tests/524_sra_far_struct_arg.expect b/tests/ir_tests/524_sra_far_struct_arg.expect new file mode 100644 index 000000000..1594f5f1b --- /dev/null +++ b/tests/ir_tests/524_sra_far_struct_arg.expect @@ -0,0 +1,5 @@ +k=0 r=1027175093 +k=1 r=2000559285 +k=2 r=729693747 +k=3 r=2179408231 +total=3768480084 diff --git a/tests/ir_tests/525_cmp_hi_only.c b/tests/ir_tests/525_cmp_hi_only.c new file mode 100644 index 000000000..8104d67cc --- /dev/null +++ b/tests/ir_tests/525_cmp_hi_only.c @@ -0,0 +1,124 @@ +/* A 64-bit compare against a constant whose LOW half is zero + * (source/opt/flat/fusion/cmp_hi_only.c). + * + * The pass annotates such a compare as high-words-only, and codegen then drops + * the `cmp rlo, #0` and turns the borrow-folding `sbcs rt, rhi, #imm_hi` into a + * plain flag-setting `cmp rhi, #imm_hi`. A mistake is a branch that goes the + * wrong way, so every case below is a printed answer with host gcc as the + * oracle -- there is nothing to crash on. + * + * The values that matter are the TIES: a v whose high word EQUALS the + * constant's, with a non-zero low word. That is the only place the low + * comparison could have changed the answer, and the proof is that against a + * zero low half it cannot: `v < C` is already settled false by the high words + * being equal, because v's low word cannot be below zero. `hi_tie_*` and the + * `0x0000000100000001` / `0x00000000FFFFFFFF` seeds are those cases. + * + * The refusals are the other half of the test. `>` and `<=` are NOT covered by + * the proof -- `v > C` is true when the high words are equal and v's low word + * is merely non-zero, which a high-word compare cannot see -- and neither are + * `==`/`!=`, a constant with a non-zero low half, or a constant on the left, + * where the mirrored condition is the non-strict one. Each has a case here and + * each must still give gcc's answer. */ + +extern int printf(const char *, ...); + +typedef unsigned long long u64; +typedef long long s64; + +volatile u64 sink; + +#define NORM ((u64)1 << 55) /* what sfp_round_pack_double actually guards on */ + +int main(void) +{ + static const u64 seeds[] = { + 0ULL, + 1ULL, + 0x00000000FFFFFFFFULL, /* high word 0, low word saturated */ + 0x0000000100000000ULL, + 0x0000000100000001ULL, /* tie against 1<<32, low word non-zero */ + 0x007FFFFFFFFFFFFFULL, + 0x0080000000000000ULL, /* exactly NORM */ + 0x0080000000000001ULL, /* tie against NORM, low word non-zero */ + 0x00FFFFFFFFFFFFFFULL, + 0x0100000000000000ULL, /* exactly NORM << 1 */ + 0x0100000000000001ULL, /* tie against NORM << 1 */ + 0x8000000000000000ULL, /* negative as s64 */ + 0x8000000000000001ULL, + 0xFFFFFFFFFFFFFFFFULL, + }; + unsigned i; + for (i = 0; i < sizeof(seeds) / sizeof(seeds[0]); i++) + { + u64 v = seeds[i]; + s64 s = (s64)v; + + /* The two shapes the pass exists for: strict <, and >=, against a constant + * whose low half is zero. These are round_pack's normalisation guards. */ + int a1 = v < NORM; + int a2 = v >= NORM; + int a3 = v < (NORM << 1); + int a4 = v >= (NORM << 1); + + /* Ties: the high words are equal and the low word is not zero. */ + int b1 = v < 0x0000000100000000ULL; + int b2 = v >= 0x0000000100000000ULL; + int b3 = v < 0x0080000000000000ULL; + int b4 = v >= 0x0080000000000000ULL; + + /* Signed order. A 64-bit compare orders the high words signed and the low + * words unsigned, so the same proof holds and the high-word CMP feeds the + * same flags to a signed branch. */ + int c1 = s < (s64)0x0080000000000000LL; + int c2 = s >= (s64)0x0080000000000000LL; + int c3 = s < (s64)0xFF00000000000000LL; /* negative constant */ + int c4 = s >= (s64)0xFF00000000000000LL; + + /* REFUSALS. `>` and `<=` cannot narrow to the high words: with equal high + * words the answer turns on the low word being non-zero. */ + int d1 = v > 0x0000000100000000ULL; + int d2 = v <= 0x0000000100000000ULL; + int d3 = v > 0x0080000000000000ULL; + int d4 = v <= 0x0080000000000000ULL; + + /* Equality needs both halves. */ + int e1 = v == 0x0080000000000000ULL; + int e2 = v != 0x0000000100000000ULL; + + /* A constant with a non-zero LOW half must keep its low comparison. */ + int f1 = v < 0x0080000000000001ULL; + int f2 = v >= 0x00000001FFFFFFFFULL; + + /* Constant on the LEFT: the mirrored conditions are the non-strict ones, + * which the pass must refuse. */ + int g1 = 0x0080000000000000ULL < v; + int g2 = 0x0080000000000000ULL >= v; + + printf("%u %d%d%d%d %d%d%d%d %d%d%d%d %d%d%d%d %d%d %d%d %d%d\n", i, a1, a2, + a3, a4, b1, b2, b3, b4, c1, c2, c3, c4, d1, d2, d3, d4, e1, e2, f1, + f2, g1, g2); + sink = v; + } + + /* The loop shape the pass was written for: the compare is the latch of a + * `while`, so it is annotated once and executed many times. Both directions, + * exactly as sfp_round_pack_double normalises a mantissa. */ + { + u64 m = 0x0123456789ABCDEFULL; + int e = 0; + while (m >= (NORM << 1)) + { + m = (m >> 1) | (m & 1); + e++; + } + while (m < NORM) + { + m <<= 1; + e--; + } + printf("norm %llu %d\n", (unsigned long long)m, e); + sink = m; + } + return 0; +} diff --git a/tests/ir_tests/525_cmp_hi_only.expect b/tests/ir_tests/525_cmp_hi_only.expect new file mode 100644 index 000000000..2ab1ef7c8 --- /dev/null +++ b/tests/ir_tests/525_cmp_hi_only.expect @@ -0,0 +1,15 @@ +0 1010 1010 1001 0101 01 10 01 +1 1010 1010 1001 0101 01 10 01 +2 1010 1010 1001 0101 01 10 01 +3 1010 0110 1001 0101 00 10 01 +4 1010 0110 1001 1001 01 10 01 +5 1010 0110 1001 1001 01 11 01 +6 0110 0101 0101 1001 11 11 01 +7 0110 0101 0101 1010 01 01 10 +8 0110 0101 0101 1010 01 01 10 +9 0101 0101 0101 1010 01 01 10 +10 0101 0101 0101 1010 01 01 10 +11 0101 0101 1010 1010 01 01 10 +12 0101 0101 1010 1010 01 01 10 +13 0101 0101 1001 1010 01 01 10 +norm 40992764608243447 1 diff --git a/tests/ir_tests/526_bottom_test_ptr_walk.c b/tests/ir_tests/526_bottom_test_ptr_walk.c new file mode 100644 index 000000000..4f3f1b8c0 --- /dev/null +++ b/tests/ir_tests/526_bottom_test_ptr_walk.c @@ -0,0 +1,115 @@ +/* ssa:loop_bottom_test — the `while`-shaped loops ssa:loop_rotate cannot match. + * + * A loop whose body falls through from its header (the shape ssa:cfg_cleanup + * leaves, and the one ssa:iv_strength_reduction produces once it eliminates a + * counter against an end pointer) keeps its top test and an unconditional + * back-edge. ssa:loop_bottom_test copies the header CMP in front of the + * back-edge and makes the back-edge the inverted conditional branch. + * + * Nothing here checks the SHAPE — these are correctness cases. Each one is a + * way the rewrite, or the copy forwarding that goes with it, can silently + * produce a wrong value: + * + * fill — the canonical admitted walk (`str r,[p],#4` bottom-tested). + * A dropped or duplicated final iteration shows up in the sum. + * fill_bytes — byte stride, so the guard and the tail test disagree about + * the last element if the inverted condition is wrong. + * swap3 — 20000801-1's shape: the walked pointer is written THREE + * times in one body (`*bp++`, `*bp++`, `bp += 2`), so the + * `T <- P` copies each name a DIFFERENT value. Forwarding + * them onto P stored through the already-incremented pointer + * and byte-swapped the wrong pair. + * empty_trip — zero-trip: the header guard must still be the thing that + * decides, or the body runs once. + * one_trip — exactly one iteration, the boundary the tail test owns. + * sum_offsets — reads at fixed offsets off the walked pointer (the sha + * W-expansion shape) alongside the bump. + * after_loop — the walked pointer is read AFTER the loop, so the exit + * value has to be the end pointer, not one step short. + */ +#include + +static int g[64]; +static unsigned char b[32]; + +static void fill(int v) +{ + for (int i = 0; i < 64; i++) + g[i] = v + i; +} + +static int fill_bytes(unsigned char v) +{ + int s = 0; + for (int i = 0; i < 32; i++) + b[i] = (unsigned char)(v + i); + for (int i = 0; i < 32; i++) + s += b[i]; + return s; +} + +/* gcc.c-torture 20000801-1's foo(): three writes of bp per iteration. */ +static void swap3(char *bp, unsigned n) +{ + char c; + char *ep = bp + n; + char *sp; + + while (bp < ep) + { + sp = bp + 3; + c = *sp; + *sp = *bp; + *bp++ = c; + sp = bp + 1; + c = *sp; + *sp = *bp; + *bp++ = c; + bp += 2; + } +} + +static int empty_trip(int n) +{ + int s = 0; + for (int i = 0; i < n; i++) + s += g[i]; + return s; +} + +static int sum_offsets(void) +{ + int s = 0; + for (int i = 8; i < 64; i++) + s += g[i - 8] + g[i - 3] + g[i]; + return s; +} + +static int after_loop(void) +{ + int *p = g; + int *end = g + 64; + int s = 0; + while (p < end) + s += *p++; + return s + (int)(p - g); +} + +int main(void) +{ + int s = 0; + unsigned int word = 0x11223344u; + + fill(1); + for (int i = 0; i < 64; i++) + s += g[i]; + printf("fill=%d bytes=%d\n", s, fill_bytes(2)); + + swap3((char *)&word, sizeof(word)); + printf("swap3=%08x\n", word); + swap3((char *)&word, sizeof(word)); + printf("swap3back=%08x\n", word); + + printf("empty=%d one=%d offsets=%d after=%d\n", empty_trip(0), empty_trip(1), sum_offsets(), after_loop()); + return 0; +} diff --git a/tests/ir_tests/526_bottom_test_ptr_walk.expect b/tests/ir_tests/526_bottom_test_ptr_walk.expect new file mode 100644 index 000000000..4f9392f40 --- /dev/null +++ b/tests/ir_tests/526_bottom_test_ptr_walk.expect @@ -0,0 +1,4 @@ +fill=2080 bytes=560 +swap3=44332211 +swap3back=11223344 +empty=0 one=1 offsets=5516 after=2144 diff --git a/tests/ir_tests/527_landor_const_last_operand.c b/tests/ir_tests/527_landor_const_last_operand.c new file mode 100644 index 000000000..2867c6d10 --- /dev/null +++ b/tests/ir_tests/527_landor_const_last_operand.c @@ -0,0 +1,102 @@ +/* && / || whose LAST operand is a compile-time constant, after runtime + * operands. The constant is the operator's neutral element, so the result is + * whatever the jump chain says -- but && used to read it off a synthetic + * compare with no CMP behind it, taking the flags the previous operand's + * test left. After `!x` (a jump on NE) those flags say EQ on the + * fallthrough, and `a && !x && 1` came out 0. The self-hosted tcc hit it in + * frame_record's `guard = ir && optimize > 0 && !do_debug && + * !tcc_bounds_checking(s)`, with tcc_bounds_checking the constant 0. + * + * Each case is used both as a value and as a branch condition, at every + * combination of its runtime operands. */ +#include + +#define BCHECK(s) 0 + +struct st +{ + void *ir; + signed char optimize; + unsigned char do_debug; +}; + +__attribute__((noinline)) int guard_of(struct st *s, int size, int mask) +{ + void *ir = s ? s->ir : NULL; + int guard = ir && s->optimize > 0 && !s->do_debug && !BCHECK(s); + if (guard && size >= 4) + mask &= -4; + return mask + guard * 100; +} + +__attribute__((noinline)) int and_not_one(int a, int x) +{ + return a && !x && 1; +} + +__attribute__((noinline)) int and_cmp_one(int a, int x) +{ + return a > 0 && x != 3 && 1; +} + +__attribute__((noinline)) int and_branch(int a, int x) +{ + if (a && !x && 1) + return 7; + return 9; +} + +__attribute__((noinline)) int or_not_zero(int a, int x) +{ + return a || !x || 0; +} + +__attribute__((noinline)) int or_branch(int a, int x) +{ + if (a || !x || 0) + return 7; + return 9; +} + +__attribute__((noinline)) int and_value_store(volatile int *out, int a, int x) +{ + int v = !a && !x && 1; + *out = v; + return v ? 11 : 13; +} + +int main(void) +{ + static struct st st = {(void *)&st, 2, 0}; + int fails = 0; + + int r = guard_of(&st, 8, -1); + printf("guard %d\n", r); + fails += r != 96; + st.do_debug = 1; + fails += guard_of(&st, 8, -1) != -1; + st.do_debug = 0; + st.optimize = 0; + fails += guard_of(&st, 8, -1) != -1; + fails += guard_of(NULL, 8, -1) != -1; + + for (int a = 0; a < 2; a++) + for (int x = 0; x < 2; x++) + { + volatile int out = -1; + int e_and = a && !x, e_or = a || !x, e_nn = !a && !x; + fails += and_not_one(a, x) != e_and; + fails += and_branch(a, x) != (e_and ? 7 : 9); + fails += or_not_zero(a, x) != e_or; + fails += or_branch(a, x) != (e_or ? 7 : 9); + fails += and_value_store(&out, a, x) != (e_nn ? 11 : 13); + fails += out != e_nn; + printf("a=%d x=%d and=%d or=%d\n", a, x, and_not_one(a, x), or_not_zero(a, x)); + } + for (int a = -1; a < 2; a++) + for (int x = 2; x < 5; x++) + fails += and_cmp_one(a, x) != (a > 0 && x != 3); + + printf("fails %d\n", fails); + return fails; +} diff --git a/tests/ir_tests/527_landor_const_last_operand.expect b/tests/ir_tests/527_landor_const_last_operand.expect new file mode 100644 index 000000000..53ba2ebbf --- /dev/null +++ b/tests/ir_tests/527_landor_const_last_operand.expect @@ -0,0 +1,6 @@ +guard 96 +a=0 x=0 and=0 or=1 +a=0 x=1 and=0 or=0 +a=1 x=0 and=1 or=1 +a=1 x=1 and=0 or=1 +fails 0 diff --git a/tests/ir_tests/528_sra_partial_fields.c b/tests/ir_tests/528_sra_partial_fields.c new file mode 100644 index 000000000..1ee38c821 --- /dev/null +++ b/tests/ir_tests/528_sra_partial_fields.c @@ -0,0 +1,137 @@ +/* Fields of a small frame object accessed in part: SRA makes the widest + access at an offset the field and rewrites the others as extracts and + inserts of its VAR. The Zig C backend writes an error union's `u16 error` + by itself and copies the union a word at a time; an optional of bytes is + built one byte at a time and returned as a word; a result buffer is read + at two widths after the call. */ +#include + +struct eu +{ + unsigned payload; + unsigned short err; + unsigned short pad; +}; + +struct opt +{ + unsigned char val; + unsigned char is_null; + signed char delta; + unsigned char tag; +}; + +struct pair +{ + unsigned lo, hi; +}; + +__attribute__((noinline)) struct eu make_eu(unsigned p, unsigned e) +{ + struct eu r; + r.payload = p; + r.err = (unsigned short)e; + r.pad = 0; + return r; +} + +__attribute__((noinline)) struct pair make_pair(unsigned a, unsigned b) +{ + struct pair r; + r.lo = a; + r.hi = b; + return r; +} + +__attribute__((noinline)) unsigned word_of(struct opt o) +{ + return o.val | (o.is_null << 8) | ((unsigned)(unsigned char)o.delta << 16) | ((unsigned)o.tag << 24); +} + +__attribute__((noinline)) unsigned two_widths(unsigned p, unsigned e, unsigned k) +{ + struct eu t = make_eu(p, e); /* the whole union copied in, a word at a time */ + unsigned r = t.payload; + if (k & 1) + t.err = (unsigned short)(k * 7919); /* the u16 by itself: a partial store */ + if (k & 2) + t.err = k; /* a value wider than the field: masked */ + r += t.err; /* the u16 read back */ + r ^= *(unsigned *)&t.err; /* the word holding it read whole */ + return r; +} + +__attribute__((noinline)) unsigned bytes_in_word(unsigned k) +{ + struct opt o; + *(unsigned *)&o = 0x11223344u; /* the word written whole */ + o.val = (unsigned char)k; /* then its bytes */ + o.is_null = (k & 1); + o.delta = (signed char)(k * 37); + if (k & 4) + o.tag = 0xEE; + int d = o.delta; /* a signed byte read out of the word */ + unsigned r = word_of(o) + (unsigned)d; + r += o.is_null; + r += o.tag; + return r; +} + +__attribute__((noinline)) unsigned buffer_two_widths(unsigned a, unsigned b) +{ + struct pair p = make_pair(a, b); /* a result buffer, then read at two widths */ + unsigned r = p.lo; + r += *(unsigned short *)&p.hi; + r += *((unsigned char *)&p.hi + 1); + r += p.hi; + return r; +} + +struct ev +{ + unsigned short err; +}; + +__attribute__((noinline)) void dirty(void) +{ + volatile unsigned junk[64]; + for (unsigned i = 0; i < 64; i++) + junk[i] = 0xDEADBEEFu + i; +} + +/* Zig's `!void`: a two-byte struct zero-initialized a byte at a time, set + whole on the error path, read whole and tested. The field's VAR is only + ever written in part on the success path, so its upper bits are whatever + the register held: the read must mask them. */ +__attribute__((noinline)) unsigned byte_init_then_test(unsigned k) +{ + struct ev e; + dirty(); + if (k & 1) + e.err = (unsigned short)(k * 3); + else + { + ((unsigned char *)&e)[0] = 0; + ((unsigned char *)&e)[1] = 0; + } + struct ev f = e; + if (f.err) + return 1000 + f.err; + return 7; +} + +int main(void) +{ + unsigned total = 0; + for (unsigned k = 0; k < 8; k++) + { + unsigned a = two_widths(k * 1000003u, k * 5 + 1, k); + unsigned b = bytes_in_word(k); + unsigned c = buffer_two_widths(k * 77777u, k * 0x01010101u + 0x0badu); + unsigned d = byte_init_then_test(k); + printf("k=%u a=%u b=%u c=%u d=%u\n", k, a, b, c, d); + total += a ^ b ^ c ^ d; + } + printf("total=%u\n", total); + return 0; +} diff --git a/tests/ir_tests/528_sra_partial_fields.expect b/tests/ir_tests/528_sra_partial_fields.expect new file mode 100644 index 000000000..fcdde1516 --- /dev/null +++ b/tests/ir_tests/528_sra_partial_fields.expect @@ -0,0 +1,9 @@ +k=0 a=0 b=285212689 c=5989 d=7 +k=1 a=1015773 b=287637816 c=16927033 d=1003 +k=2 a=2000010 b=290062429 c=33848077 d=7 +k=3 a=3000015 b=292487556 c=50769121 d=1009 +k=4 a=4000052 b=4002676870 c=67690165 d=7 +k=5 a=5013329 b=4005101997 c=84611209 d=1015 +k=6 a=6000030 b=4007526610 c=101532253 d=7 +k=7 a=7000027 b=3993174521 c=118453297 d=1021 +total=4014834024 diff --git a/tests/ir_tests/529_sra_pair_copy.c b/tests/ir_tests/529_sra_pair_copy.c new file mode 100644 index 000000000..8a0baa044 --- /dev/null +++ b/tests/ir_tests/529_sra_pair_copy.c @@ -0,0 +1,77 @@ +/* Two more whole-object shapes SRA takes apart. + A copy between two small locals over 16 bytes reaches the IR as an + __aeabi_memmove4 call; when both objects are promotable and lay their + fields out alike, the copy becomes copies between their field VARs (before, + one of them kept its memory). + A local initialized with constants -- `struct s t = {...}`, a memset and a + run of constant stores that block_copy_init turns into one BLOCK_COPY from + an image in .rodata -- gets its fields defined as those constants, a word + holding an address as that address. */ +#include + +struct rec +{ + unsigned a, b, c, d, e; +}; + +struct tab +{ + const char *name; + unsigned lo, hi; + unsigned char kind, flag; + unsigned short id; +}; + +static const char greeting[] = "hello"; + +__attribute__((noinline)) unsigned sink(unsigned x) +{ + return x * 2654435761u; +} + +__attribute__((noinline)) unsigned pair_copy(unsigned k) +{ + struct rec s, t; + s.a = k; + s.b = k + 1; + s.c = k * 3; + s.d = k ^ 0x55; + s.e = sink(k); + t = s; /* 20 bytes: a memmove4 between two promotable objects */ + s.a = 99; /* the source lives on */ + return t.a + t.b * 3 + t.c * 5 + t.d * 7 + t.e + s.a; +} + +__attribute__((noinline)) unsigned image_init(unsigned k) +{ + struct tab t = {greeting, 10, 20, 3, 0, 0x1234}; + if (k & 1) + t.lo = k; + if (k & 2) + t.flag = 1; + unsigned r = t.name[k % 5] + t.lo + t.hi * 2 + t.kind + t.flag * 100 + t.id; + return r; +} + +__attribute__((noinline)) unsigned image_then_copy(unsigned k) +{ + struct rec s = {1, 2, 3, 4, 5}; + struct rec t; + s.c += k; + t = s; + t.e = sink(t.a); + return t.a + t.b + t.c + t.d + t.e; +} + +int main(void) +{ + unsigned total = 0; + for (unsigned k = 0; k < 6; k++) + { + unsigned a = pair_copy(k), b = image_init(k), c = image_then_copy(k); + printf("k=%u a=%u b=%u c=%u\n", k, a, b, c); + total += a ^ b ^ c; + } + printf("total=%u\n", total); + return 0; +} diff --git a/tests/ir_tests/529_sra_pair_copy.expect b/tests/ir_tests/529_sra_pair_copy.expect new file mode 100644 index 000000000..7d4b5a327 --- /dev/null +++ b/tests/ir_tests/529_sra_pair_copy.expect @@ -0,0 +1,7 @@ +k=0 a=697 b=4817 c=2654435771 +k=1 a=2654436470 b=4805 c=2654435772 +k=2 a=1013904975 b=4921 c=2654435773 +k=3 a=3668340748 b=4914 c=2654435774 +k=4 a=2027809197 b=4824 c=2654435775 +k=5 a=387277674 b=4812 c=2654435776 +total=4113482589 diff --git a/tests/ir_tests/530_cbz_over_aligned_branch_target.c b/tests/ir_tests/530_cbz_over_aligned_branch_target.c new file mode 100644 index 000000000..e70c14f8b --- /dev/null +++ b/tests/ir_tests/530_cbz_over_aligned_branch_target.c @@ -0,0 +1,66 @@ +/* A CBZ that spans a word-aligned branch target. + The `if (!x) goto end` below fuses into CBZ, whose reach is 126 bytes and + which cannot be widened once emitted. The distance is taken from the + rehearsal pass, which emits every branch wide; the real pass is smaller, + except for the 2-byte pad that word-aligns a backward-branch target + (`again`). Narrowing the `y == 7` branch ahead of the CBZ flips the + parity: the rehearsal found `again` aligned, the real pass needs the pad, + and a CBZ the rehearsal measured at exactly 126 bytes landed at 128 -- + "CBZ/CBNZ target out of range: offset=128" at -O1, -O2 and -Os. (Found in + the Zig compiler's InternPool.getOrPutTrailingString.) The filler below + sizes the skipped block to that edge; keep it exactly as it is. */ +#include + +__attribute__((noinline)) int f(int *p, int n, int x, int y) +{ + int s = 0; + if (y == 7) + p[40] = n * 9 + x; + p[30] = y; + if (!x) + goto end; + p[0] = p[1] + y; + p[1] = p[2] + y; + p[2] = p[3] + y; + p[3] = p[4] + y; + p[4] = p[5] + y; + p[5] = p[6] + y; + p[6] = p[7] + y; + p[7] = p[8] + y; + p[8] = p[9] + y; + p[9] = p[10] + y; + p[10] = p[11] + y; + p[11] = p[12] + y; + p[12] = p[13] + y; + p[13] = p[14] + y; + p[14] = p[15] + y; + p[15] = p[16] + y; + p[16] = p[17] + y; + p[17] = p[18] + y; + p[18] = p[19] + y; + s += y; +again: + s += p[s & 7]; +end: + s += 1; + if (s < n) + goto again; + return s; +} + +int main(void) +{ + static const int cases[][3] = {{0, 50, 7}, {1, 50, 7}, {0, 200, 3}, {5, 300, 2}, {1, 1, 9}}; + for (unsigned c = 0; c < sizeof cases / sizeof cases[0]; c++) + { + int p[48]; + for (int i = 0; i < 48; i++) + p[i] = i * 3 + 1; + int r = f(p, cases[c][1], cases[c][0], cases[c][2]); + unsigned sum = 0; + for (int i = 0; i < 48; i++) + sum = sum * 31u + (unsigned)p[i]; + printf("x=%d n=%d y=%d -> %d sum=%u\n", cases[c][0], cases[c][1], cases[c][2], r, sum); + } + return 0; +} diff --git a/tests/ir_tests/530_cbz_over_aligned_branch_target.expect b/tests/ir_tests/530_cbz_over_aligned_branch_target.expect new file mode 100644 index 000000000..8ee477a30 --- /dev/null +++ b/tests/ir_tests/530_cbz_over_aligned_branch_target.expect @@ -0,0 +1,5 @@ +x=0 n=50 y=7 -> 50 sum=2871708595 +x=1 n=50 y=7 -> 52 sum=1744848904 +x=0 n=200 y=3 -> 202 sum=411080864 +x=5 n=300 y=2 -> 315 sum=2120801852 +x=1 n=1 y=9 -> 26 sum=433970766 diff --git a/tests/ir_tests/531_dyn_arg_window.c b/tests/ir_tests/531_dyn_arg_window.c new file mode 100644 index 000000000..39220be80 --- /dev/null +++ b/tests/ir_tests/531_dyn_arg_window.c @@ -0,0 +1,73 @@ +/* A call passing more stack-argument bytes than the frame reserves drops SP + for itself alone (`sub sp` ... `add sp`); the other calls use the reserved + area. Everything addressed off SP in between -- locals feeding the + arguments, a struct copied by value, spilled values, the return value's + home -- has to see the moved SP, and the arguments land at the new one. + scribble() fills the stack so a word read from the wrong slot shows. */ +#include +#include + +struct big +{ + unsigned w[40]; /* 160 bytes by value: past the reserved area */ +}; + +struct small +{ + unsigned a, b; +}; + +__attribute__((noinline)) void scribble(void) +{ + volatile unsigned char junk[4096]; + for (unsigned i = 0; i < sizeof junk; i++) + junk[i] = (unsigned char)(0xA5 ^ i); +} + +__attribute__((noinline)) unsigned many(unsigned a, unsigned b, unsigned c, unsigned d, unsigned e, unsigned f, + unsigned g, unsigned h, unsigned i, unsigned j) +{ + return a + 2 * b + 3 * c + 5 * d + 7 * e + 11 * f + 13 * g + 17 * h + 19 * i + 23 * j; +} + +__attribute__((noinline)) unsigned take_big(unsigned x, struct big b, unsigned y, struct small s, unsigned z) +{ + unsigned h = x ^ y ^ z ^ s.a ^ (s.b << 1); + for (unsigned k = 0; k < 40; k++) + h = h * 31 + b.w[k]; + return h; +} + +__attribute__((noinline)) unsigned host(unsigned k) +{ + struct big b; + struct small s = {k * 3, k + 7}; + unsigned loc[24]; + for (unsigned i = 0; i < 24; i++) + loc[i] = i * k + 1; + for (unsigned i = 0; i < 40; i++) + b.w[i] = i ^ (k << 3); + /* a small stack-argument call: uses the reserved area */ + unsigned r = many(loc[0], loc[1], loc[2], loc[3], loc[4], loc[5], loc[6], loc[7], loc[8], loc[9]); + /* the big one: its own window; its arguments read locals through SP */ + r += take_big(loc[10] + r, b, loc[11], s, loc[12] ^ r); + /* the frame is intact afterwards */ + for (unsigned i = 0; i < 24; i++) + r = r * 7 + loc[i]; + r += many(r, loc[13], loc[14], loc[15], loc[16], loc[17], loc[18], loc[19], loc[20], s.a); + return r ^ s.b; +} + +int main(void) +{ + unsigned total = 0; + for (unsigned k = 0; k < 5; k++) + { + scribble(); + unsigned r = host(k); + printf("k=%u r=%u\n", k, r); + total += r; + } + printf("total=%u\n", total); + return 0; +} diff --git a/tests/ir_tests/531_dyn_arg_window.expect b/tests/ir_tests/531_dyn_arg_window.expect new file mode 100644 index 000000000..2597710fc --- /dev/null +++ b/tests/ir_tests/531_dyn_arg_window.expect @@ -0,0 +1,6 @@ +k=0 r=264931614 +k=1 r=4165716161 +k=2 r=3933632312 +k=3 r=2083393971 +k=4 r=1481280866 +total=3339020332 diff --git a/tests/ir_tests/532_unreachable_branches.c b/tests/ir_tests/532_unreachable_branches.c new file mode 100644 index 000000000..35bb661aa --- /dev/null +++ b/tests/ir_tests/532_unreachable_branches.c @@ -0,0 +1,94 @@ +/* __builtin_unreachable() marks where control never goes, and the branches + that could only lead there are folded: an exhaustive switch loses its + range check and last compare, a loop whose exit is unreachable its bound. + Every reachable path must still compute what it did. The Zig C backend + ends every exhaustive switch with `default: zig_unreachable();`. */ +#include + +enum tag { T_A, T_B, T_C, T_D, T_E, T_F }; + +__attribute__((noinline)) const char *tag_name(enum tag t) +{ + switch (t) + { + case T_A: return "alpha"; + case T_B: return "beta"; + case T_C: return "gamma"; + case T_D: return "delta"; + case T_E: return "epsilon"; + case T_F: return "zeta"; + } + __builtin_unreachable(); +} + +__attribute__((noinline)) int pick(int t, int x) +{ + switch (t) + { + case 0: return x + 1; + case 1: return x * 3; + case 2: return x - 7; + default: __builtin_unreachable(); + } +} + +/* ULEB128 of a u16: at most three bytes, the third iteration's exit is unreachable. */ +__attribute__((noinline)) int uleb(unsigned v, unsigned char *out) +{ + for (unsigned i = 0;; i++) + { + if (i < 3) + { + unsigned char b = v & 127; + v >>= 7; + if (v == 0) + { + out[i] = b; + return (int)i + 1; + } + out[i] = b | 128; + continue; + } + __builtin_unreachable(); + } +} + +/* A value computed on the doomed side must not leak into the live one. */ +__attribute__((noinline)) int side(int k) +{ + int r = k * 5; + if (k > 100) + { + r = 99; + __builtin_unreachable(); + } + return r + 1; +} + +int main(void) +{ + unsigned total = 0; + for (int t = 0; t < 6; t++) + { + printf("%d %s\n", t, tag_name((enum tag)t)); + total += (unsigned)tag_name((enum tag)t)[1]; + } + for (int t = 0; t < 3; t++) + { + int r = pick(t, 40 + t); + printf("pick %d %d\n", t, r); + total += (unsigned)r; + } + unsigned vals[] = {0, 1, 127, 128, 300, 16383, 16384, 65535}; + for (unsigned i = 0; i < sizeof vals / sizeof vals[0]; i++) + { + unsigned char buf[4] = {0}; + int n = uleb(vals[i], buf); + printf("uleb %u -> %d: %02x %02x %02x\n", vals[i], n, buf[0], buf[1], buf[2]); + total += (unsigned)n * 1000u + buf[0] + buf[1] * 7u + buf[2] * 13u; + } + for (int k = 0; k < 5; k++) + total += (unsigned)side(k); + printf("total=%u\n", total); + return 0; +} diff --git a/tests/ir_tests/532_unreachable_branches.expect b/tests/ir_tests/532_unreachable_branches.expect new file mode 100644 index 000000000..cbca83edd --- /dev/null +++ b/tests/ir_tests/532_unreachable_branches.expect @@ -0,0 +1,18 @@ +0 alpha +1 beta +2 gamma +3 delta +4 epsilon +5 zeta +pick 0 41 +pick 1 123 +pick 2 35 +uleb 0 -> 1: 00 00 00 +uleb 1 -> 1: 01 00 00 +uleb 127 -> 1: 7f 00 00 +uleb 128 -> 2: 80 01 00 +uleb 300 -> 2: ac 02 00 +uleb 16383 -> 2: ff 7f 00 +uleb 16384 -> 3: 80 80 01 +uleb 65535 -> 3: ff ff 03 +total=20583 diff --git a/tests/ir_tests/533_inline_scope_caller_object.c b/tests/ir_tests/533_inline_scope_caller_object.c new file mode 100644 index 000000000..04ff91a8d --- /dev/null +++ b/tests/ir_tests/533_inline_scope_caller_object.c @@ -0,0 +1,52 @@ +#include +#include + +/* A caller's object whose address escapes must stay live across an + * always_inline expansion: the expansion used to end the lifetime of every + * object of the function, and a later block-local was laid over it. */ +struct table { long v[11]; }; + +__attribute__((noinline)) void fill(struct table *t, long x) +{ + for (int i = 0; i < 11; i++) + t->v[i] = x + i; +} + +__attribute__((noinline)) long sum(const struct table *t) +{ + long s = 0; + for (int i = 0; i < 11; i++) + s += t->v[i]; + return s; +} + +__attribute__((noinline)) void scribble(char *b) +{ + memset(b, 0x55, 44); +} + +static inline __attribute__((always_inline)) int odd(int a) +{ + return a & 1; +} + +int main(void) +{ + struct table t; + int n = odd(3); + fill(&t, 100); + n += odd(n + 4); + { + char buf[44]; + scribble(buf); + n += buf[3] == 0x55; + } + { + char other[44]; + scribble(other); + n += other[40] == 0x55; + } + printf("sum=%ld n=%d\n", sum(&t), n); + printf("first=%ld last=%ld\n", t.v[0], t.v[10]); + return 0; +} diff --git a/tests/ir_tests/533_inline_scope_caller_object.expect b/tests/ir_tests/533_inline_scope_caller_object.expect new file mode 100644 index 000000000..8a1d0eb5d --- /dev/null +++ b/tests/ir_tests/533_inline_scope_caller_object.expect @@ -0,0 +1,2 @@ +sum=1155 n=4 +first=100 last=110 diff --git a/tests/ir_tests/534_sccp_entry_store_escaped_call.c b/tests/ir_tests/534_sccp_entry_store_escaped_call.c new file mode 100644 index 000000000..1204e0939 --- /dev/null +++ b/tests/ir_tests/534_sccp_entry_store_escaped_call.c @@ -0,0 +1,43 @@ +#include + +/* A local initialised in the entry block, whose address then goes to a call + * that writes it, must be re-read after the call. SCCP kept the initializer + * for an entry-block store across the call (the entry-block alias scan did + * not treat calls as writers), so `d.backend == 0` folded to true and the + * returned fields were the zeros -- ctags' lregex choose_backend returned a + * NULL backend and faulted on `desc.backend->fdefs`. */ +struct desc +{ + const char *backend; + unsigned flags; + int type; +}; + +__attribute__((noinline)) void eval(const char *f, struct desc *d) +{ + if (f[0] == 'x') + return; + d->backend = f; + d->flags |= 0x10; +} + +__attribute__((noinline)) struct desc choose(const char *flags, int type) +{ + struct desc d = { .backend = 0, .flags = 0, .type = type }; + if (flags) + eval(flags, &d); + if (d.backend == 0) + eval("default", &d); + return d; +} + +int main(void) +{ + struct desc a = choose("pcre2", 1); + struct desc b = choose("x", 2); + struct desc c = choose(0, 3); + printf("%s %u %d\n", a.backend, a.flags, a.type); + printf("%s %u %d\n", b.backend, b.flags, b.type); + printf("%s %u %d\n", c.backend, c.flags, c.type); + return 0; +} diff --git a/tests/ir_tests/534_sccp_entry_store_escaped_call.expect b/tests/ir_tests/534_sccp_entry_store_escaped_call.expect new file mode 100644 index 000000000..252943dc7 --- /dev/null +++ b/tests/ir_tests/534_sccp_entry_store_escaped_call.expect @@ -0,0 +1,3 @@ +pcre2 16 1 +default 16 2 +default 16 3 diff --git a/tests/ir_tests/535_inline_big_struct_param.c b/tests/ir_tests/535_inline_big_struct_param.c new file mode 100644 index 000000000..209856774 --- /dev/null +++ b/tests/ir_tests/535_inline_big_struct_param.c @@ -0,0 +1,65 @@ +/* A static function taking a struct larger than 16 bytes by value is an + inline candidate. The expansion must still bind a COPY: the callee may + write its parameter, and the caller's object may change through a pointer + the callee also holds, and neither may show on the other side. The + accessor shape Zig emits -- a whole struct passed to read one field -- has + to come out as that field's load. */ +#include + +struct block +{ + unsigned a, b, c, d, e; + unsigned char tag; + unsigned short w[5]; +}; + +static unsigned block_src(const struct block blk) { return blk.c; } + +static unsigned bump(struct block blk, unsigned k) +{ + blk.c += k; /* writes the copy only */ + return blk.c * 3 + blk.tag; +} + +static unsigned alias(struct block blk, struct block *orig) +{ + orig->a = 1000; /* the caller's object changes ... */ + return blk.a + orig->a; /* ... but the parameter keeps the value passed */ +} + +static struct block make(unsigned seed) +{ + struct block r = {seed, seed + 1, seed + 2, seed + 3, seed + 4, (unsigned char)(seed * 7), {1, 2, 3, 4, 5}}; + return r; +} + +static unsigned sum_w(struct block blk) +{ + unsigned s = 0; + for (int i = 0; i < 5; i++) + s += blk.w[i]; + return s; +} + +struct holder +{ + int pad; + struct block blk; +}; + +static unsigned via_ptr(const struct holder *h) { return block_src(h->blk); } + +int main(void) +{ + struct block b = make(10); + struct holder h = {7, make(20)}; + printf("src %u\n", block_src(b)); + printf("bump %u c %u\n", bump(b, 5), b.c); + unsigned r = alias(b, &b); + printf("alias %u a %u\n", r, b.a); + printf("w %u\n", sum_w(b)); + printf("ptr %u\n", via_ptr(&h)); + h.blk.c = 99; + printf("ptr %u\n", via_ptr(&h)); + return 0; +} diff --git a/tests/ir_tests/535_inline_big_struct_param.expect b/tests/ir_tests/535_inline_big_struct_param.expect new file mode 100644 index 000000000..c07f23ed5 --- /dev/null +++ b/tests/ir_tests/535_inline_big_struct_param.expect @@ -0,0 +1,6 @@ +src 12 +bump 121 c 12 +alias 1010 a 1000 +w 15 +ptr 22 +ptr 99 diff --git a/tests/ir_tests/536_frame_relayout_retry_keeps_copies.c b/tests/ir_tests/536_frame_relayout_retry_keeps_copies.c new file mode 100644 index 000000000..b32b64e66 --- /dev/null +++ b/tests/ir_tests/536_frame_relayout_retry_keeps_copies.c @@ -0,0 +1,85 @@ +/* Frame relayout colours the frame, and when a STRUCT operand then sits below + -32768 (past its 16-bit offset) it throws that layout away and colours + again with those objects pinned. Colouring deleted the copies it merged as + it went -- `r = t` inside an inlined `return t` -- so the second layout no + longer saw the copy, placed its two objects apart, and the struct the call + returned into was not the one the next copy read. Zig's Dir.deleteTree: + Iterator.init returned into one slot, the stack item was filled from + another, and the compiler crashed or hung on exit walking a garbage + directory reader. Needs a frame over 32 KB and a by-value struct + argument. */ +#include +#include + +struct iter +{ + int fd; + unsigned index, end; + unsigned char state; + unsigned char buf[2048]; +}; + +struct item +{ + const char *name; + int parent; + struct iter iter; +}; + +__attribute__((noinline)) struct iter iter_init(int fd, unsigned char state) +{ + struct iter t; + t.fd = fd; + t.index = 0; + t.end = 0; + t.state = state; + return t; +} + +static struct iter iterate_first(int fd) +{ + struct iter t0; + t0 = iter_init(fd, 1); + return t0; +} + +static struct item stack_items[4]; +static unsigned stack_len; + +__attribute__((noinline)) void append(struct item it) +{ + stack_items[stack_len++] = it; +} + +__attribute__((noinline)) void scribble(unsigned char *p, unsigned n) +{ + memset(p, 0x5a, n); +} + +__attribute__((noinline)) unsigned walk(const char *name, int fd) +{ + struct iter t15; + struct item t16; + unsigned char big[40000]; + scribble(big, sizeof big); + t15 = iterate_first(fd); + t16.name = name; + t16.parent = fd - 1; + t16.iter = t15; + append(t16); + t15 = iterate_first(fd + 1); + t16.parent = fd; + t16.iter = t15; + append(t16); + return big[123] + big[39999]; +} + +int main(void) +{ + unsigned junk = walk("tmp", 7); + for (unsigned i = 0; i < stack_len; i++) + printf("item %u: parent=%d fd=%d index=%u end=%u state=%u\n", i, stack_items[i].parent, stack_items[i].iter.fd, + stack_items[i].iter.index, stack_items[i].iter.end, stack_items[i].iter.state); + printf("junk=%u\n", junk); + return 0; +} diff --git a/tests/ir_tests/536_frame_relayout_retry_keeps_copies.expect b/tests/ir_tests/536_frame_relayout_retry_keeps_copies.expect new file mode 100644 index 000000000..f0fab3e36 --- /dev/null +++ b/tests/ir_tests/536_frame_relayout_retry_keeps_copies.expect @@ -0,0 +1,3 @@ +item 0: parent=6 fd=7 index=0 end=0 state=1 +item 1: parent=7 fd=8 index=0 end=0 state=1 +junk=180 diff --git a/tests/ir_tests/537_indexed_chain_deref_base.c b/tests/ir_tests/537_indexed_chain_deref_base.c new file mode 100644 index 000000000..03bbeb6ed --- /dev/null +++ b/tests/ir_tests/537_indexed_chain_deref_base.c @@ -0,0 +1,43 @@ +/* indexed_chain folds `T <- B ADD #k1; LOAD_INDEXED [T, #k2]` into + `LOAD_INDEXED [B, #(k1+k2)]`. When the ADD's operand is a memory read -- + `T <- P***DEREF*** ADD #k1`, the value stored at P plus k1 -- the fold + dropped that read and addressed P + k1 + k2 instead. Here P is &l, so + `l.ptr[1].b` read the frame slot 12 bytes past l rather than the array + element: a local struct holding a pointer, kept in memory because its + address escapes, indexed with a constant and used as an arithmetic + operand. */ +#include + +struct elem +{ + unsigned a, b, c; +}; + +struct list +{ + struct elem *ptr; + unsigned len; +}; + +static struct elem items[3] = {{1, 2, 3}, {40, 50, 60}, {700, 800, 900}}; + +__attribute__((noinline)) void touch(struct list *l) +{ + l->len = 3; +} + +__attribute__((noinline)) unsigned pick(unsigned k) +{ + struct list l; + l.ptr = items; + l.len = 0; + touch(&l); + return l.ptr[1].b * 10u + l.ptr[2].c * k + l.len; +} + +int main(void) +{ + printf("pick=%u\n", pick(2)); + printf("pick=%u\n", pick(5)); + return 0; +} diff --git a/tests/ir_tests/537_indexed_chain_deref_base.expect b/tests/ir_tests/537_indexed_chain_deref_base.expect new file mode 100644 index 000000000..8c8f1f892 --- /dev/null +++ b/tests/ir_tests/537_indexed_chain_deref_base.expect @@ -0,0 +1,2 @@ +pick=2303 +pick=5003 diff --git a/tests/ir_tests/538_const_local_table_zero_fill.c b/tests/ir_tests/538_const_local_table_zero_fill.c new file mode 100644 index 000000000..12b79571f --- /dev/null +++ b/tests/ir_tests/538_const_local_table_zero_fill.c @@ -0,0 +1,75 @@ +/* A big constant initializer is lowered to memset(obj, 0, size) followed by + stores of its non-zero words only. const_local_table takes the zero fill + as the start of the table's run, so a compound literal indexed at run time + -- Zig's fmt.float `Backend64_TablesFull.computePow5`, a 5 KB table with a + zero low word in its first 27 entries -- is read from .rodata instead of + being rebuilt on the stack on every call. Also here: tables past the old + 1 KB limit, and the shapes that must stay on the stack: an element written + after the initializer, the address passed to a call, and a read of the + object between the fill and the stores. */ +#include +#include +#include + +struct pair { uint64_t a[2]; }; +struct tbl { struct pair array[40]; }; + +__attribute__((noinline)) static struct pair get(uint32_t i) +{ + struct pair t1; + t1 = (struct tbl){{{{0, 1}}, {{0, 2}}, {{5, 3}}, {{0, 4}}, {{0, 5}}, {{7, 6}}, {{0, 7}}, {{0, 8}}, + {{0, 9}}, {{0, 10}}, {{0, 11}}, {{0, 12}}, {{5, 13}}, {{0, 14}}, {{0, 15}}, + {{7, 16}}, {{0, 17}}, {{0, 18}}, {{0, 19}}, {{0, 20}}, {{0, 21}}, {{0, 22}}, + {{5, 23}}, {{0, 24}}, {{0, 25}}, {{7, 26}}, {{0, 27}}, {{0, 28}}, {{0, 29}}, + {{0, 30}}, {{0, 31}}, {{0, 32}}, {{5, 33}}, {{0, 34}}, {{0, 35}}, + {{7, 36}}, {{0, 37}}, {{0, 38}}, {{0, 39}}, {{0x123456789abcdefull, 40}}}} + .array[i]; + return t1; +} + +/* 2 KB, mostly zeros: past the old 1 KB cap. */ +struct big { uint32_t array[512]; }; +__attribute__((noinline)) static uint32_t big_at(uint32_t i) +{ + return (struct big){{[3] = 33, [100] = 1000, [511] = 0xdeadbeef}}.array[i]; +} + +/* Written after the initializer: the stack copy is the table. */ +__attribute__((noinline)) static uint32_t rewritten(uint32_t i, uint32_t v) +{ + struct big t = {{[3] = 33, [100] = 1000, [511] = 7}}; + t.array[i] = v; + return t.array[3] + t.array[100] + t.array[511]; +} + +__attribute__((noinline)) static uint32_t sum_words(const uint32_t *p, int n) +{ + uint32_t s = 0; + for (int k = 0; k < n; k++) + s += p[k]; + return s; +} + +/* The address reaches a call: kept. */ +__attribute__((noinline)) static uint32_t escapes(uint32_t i) +{ + struct big t = {{[3] = 33, [100] = 1000, [511] = 9}}; + return sum_words(t.array, 512) + t.array[i]; +} + +int main(void) +{ + uint64_t lo = 0, hi = 0; + for (uint32_t i = 0; i < 40; i++) + { + struct pair p = get(i); + lo = lo * 31 + p.a[0]; + hi = hi * 31 + p.a[1]; + } + printf("pairs %llu %llu\n", (unsigned long long)lo, (unsigned long long)hi); + printf("big %u %u %u %u\n", (unsigned)big_at(0), (unsigned)big_at(3), (unsigned)big_at(100), + (unsigned)big_at(511)); + printf("rewritten %u %u\n", (unsigned)rewritten(3, 5), (unsigned)rewritten(200, 5)); + printf("escapes %u %u\n", (unsigned)escapes(3), (unsigned)escapes(4)); + return 0; +} diff --git a/tests/ir_tests/538_const_local_table_zero_fill.expect b/tests/ir_tests/538_const_local_table_zero_fill.expect new file mode 100644 index 000000000..4b73b73b3 --- /dev/null +++ b/tests/ir_tests/538_const_local_table_zero_fill.expect @@ -0,0 +1,4 @@ +pairs 10713737280234664567 5979871330815707796 +big 0 33 1000 3735928559 +rewritten 1012 1040 +escapes 1075 1042 diff --git a/tests/ir_tests/539_cprop_store_copy_inline_helper.c b/tests/ir_tests/539_cprop_store_copy_inline_helper.c new file mode 100644 index 000000000..92c8ecf0d --- /dev/null +++ b/tests/ir_tests/539_cprop_store_copy_inline_helper.c @@ -0,0 +1,74 @@ +/* An inlined helper's result reaches its caller through `V3 <- V2`, which + ssa_rename spells `T14 <-- T16 [STORE]`: a copy between two TEMPs. cprop + forwards it like an ASSIGN copy (it was left for the allocator, as a + register move in every loop turn of Zig's readInt). The copy must still + keep a truncation when the two sides differ in width, and a TEMP stored + twice is not a copy. */ +#include +#include + +static inline uint32_t cast8(uint8_t v) { return v; } +static inline uint8_t low8(uint32_t v) { return (uint8_t)v; } +static inline int16_t low16s(uint32_t v) { return (int16_t)v; } + +struct arr_4_u8 { uint8_t array[4]; }; + +/* Zig's std.mem.readInt(u32, .little), as the C backend emits it. */ +__attribute__((noinline)) static uint32_t read_le32(const uint8_t *p) +{ + struct arr_4_u8 t21; + uintptr_t t17, t23; + uint32_t t24, t26, t27, t22; + uint8_t t25; + t21 = *(const struct arr_4_u8 *)p; + t23 = 3; + t24 = 0; +zig_loop: + t17 = t23; + t25 = t21.array[t17]; + t26 = cast8(t25); + t27 = t24; + t27 = t27 << 8; + t26 = t27 | t26; + if (t17 == 0) + { + t22 = t26; + goto out; + } + t24 = t26; + t17 = t17 - 1; + t23 = t17; + goto zig_loop; +out: + return t22; +} + +__attribute__((noinline)) static uint32_t narrow_sum(const uint32_t *p, int n) +{ + uint32_t s = 0; + for (int i = 0; i < n; i++) + { + uint8_t b = low8(p[i]); + int16_t h = low16s(p[i]); + s = s * 3 + b + (uint32_t)(int32_t)h; + } + return s; +} + +__attribute__((noinline)) static uint32_t twice(uint32_t a, uint32_t b) +{ + uint32_t x = cast8((uint8_t)a), y; + y = x; + x = cast8((uint8_t)b); + return y * 1000 + x; +} + +int main(void) +{ + static const uint8_t bytes[8] = {0x78, 0x56, 0x34, 0x12, 0xef, 0xcd, 0xab, 0x90}; + static const uint32_t words[4] = {0x12348081u, 0xffff7fffu, 0x000000ffu, 0xabcdef01u}; + printf("le32 %08x %08x\n", (unsigned)read_le32(bytes), (unsigned)read_le32(bytes + 4)); + printf("narrow %u\n", (unsigned)narrow_sum(words, 4)); + printf("twice %u\n", (unsigned)twice(0x1ff, 0x2ab)); + return 0; +} diff --git a/tests/ir_tests/539_cprop_store_copy_inline_helper.expect b/tests/ir_tests/539_cprop_store_copy_inline_helper.expect new file mode 100644 index 000000000..f37a02a95 --- /dev/null +++ b/tests/ir_tests/539_cprop_store_copy_inline_helper.expect @@ -0,0 +1,3 @@ +le32 12345678 90abcdef +narrow 4294383904 +twice 255171 diff --git a/tests/ir_tests/540_coalesce_high_pressure_latch.c b/tests/ir_tests/540_coalesce_high_pressure_latch.c new file mode 100644 index 000000000..dde0529cd --- /dev/null +++ b/tests/ir_tests/540_coalesce_high_pressure_latch.c @@ -0,0 +1,60 @@ +/* The graph coalescer used to give up on a whole function once any point in + it had as many live values as registers. It now still merges a copy class + whose members' ranges leave no gap -- a loop's `i - 1` and `acc << 8 | b` + latch copies lie inside the phi's range, so the merge adds pressure + nowhere. Zig's readInt inside Wyhash.hash, next to a u128 multiply, is the + shape; this checks the loops still compute the same values next to 64-bit + products that fill the registers, with narrow values spilled around them. */ +#include +#include + +typedef uint8_t u8; +typedef uint32_t u32; +typedef uint64_t u64; + +__attribute__((noinline)) static u32 read_then_mix(const u8 *p, u32 n, u64 s0, u64 s1, u64 s2, u64 s3) +{ + u32 acc = 0; + u32 i = n; + for (;;) + { + acc = acc << 8 | p[i]; + if (i == 0) + break; + i = i - 1; + } + u64 a = s0 * s1, b = s1 * s2, c = s2 * s3, d = s3 * s0; + u64 e = a ^ c, f = b ^ d, g = a + d, h = b + c; + return (u32)(a ^ b ^ c ^ d ^ (e * f) ^ (g * h) ^ (e + h) ^ (f + g)) + acc; +} + +__attribute__((noinline)) static u32 bytes_then_select(const u8 *p, u32 n, u8 mode, u64 s0, u64 s1) +{ + u32 acc = 0, sum = 0; + for (u32 i = 0; i < n; i++) + { + u32 b = p[i]; + acc = acc * 31 + b; + sum = sum + (b ^ mode); + } + u64 a = s0 * s1, c = s0 + s1, d = s0 ^ (s1 << 7); + u64 x = a * c ^ d * a ^ c * d; + switch (mode) + { + case 0: return (u32)x + acc; + case 1: return (u32)(x >> 32) ^ sum; + default: return acc - sum + (u32)x; + } +} + +int main(void) +{ + static const u8 bytes[16] = {0x78, 0x56, 0x34, 0x12, 0xef, 0xcd, 0xab, 0x90, + 0x01, 0x80, 0x7f, 0xff, 0x10, 0x20, 0x30, 0x40}; + printf("%08x\n", (unsigned)read_then_mix(bytes, 3, 0x123456789ull, 0xfedcba987ull, 7, 0x1111111111ull)); + printf("%08x\n", (unsigned)read_then_mix(bytes + 4, 3, 1, 2, 3, 4)); + printf("%08x\n", (unsigned)read_then_mix(bytes + 8, 0, 5, 6, 7, 8)); + for (u8 m = 0; m < 3; m++) + printf("%u %08x\n", m, (unsigned)bytes_then_select(bytes, 16, m, 0xdeadbeefcafeull, 0x1234567ull)); + return 0; +} diff --git a/tests/ir_tests/540_coalesce_high_pressure_latch.expect b/tests/ir_tests/540_coalesce_high_pressure_latch.expect new file mode 100644 index 000000000..21b3ee7c0 --- /dev/null +++ b/tests/ir_tests/540_coalesce_high_pressure_latch.expect @@ -0,0 +1,6 @@ +eb27aa88 +90abce43 +00001a65 +0 d09b3bae +1 83d0ff6c +2 d09b34fc diff --git a/tests/ir_tests/541_param_home_fwd.c b/tests/ir_tests/541_param_home_fwd.c new file mode 100644 index 000000000..c43a6ca12 --- /dev/null +++ b/tests/ir_tests/541_param_home_fwd.c @@ -0,0 +1,65 @@ +/* A by-value struct parameter is homed in the frame by entry-block word + stores, and field reads that lea folding turned into direct slot loads + reloaded it on every loop turn: Zig's StaticStringMap.defaultEql compared + two slices with both kept in memory. In a leaf function the reads now take + the parameter's register. Beside that shape, the ones that must keep the + home: a field written after entry, a read through a pointer to the + parameter, and a narrow read of a word. */ +#include +#include + +struct slice { const uint8_t *ptr; uintptr_t len; }; + +__attribute__((noinline)) static int eql(struct slice a, struct slice b) +{ + if (a.ptr == b.ptr) + return 1; + for (uintptr_t i = 0; i < a.len; i++) + if (a.ptr[i] != b.ptr[i]) + return 0; + return 1; +} + +/* a.len is rewritten: the home is no longer the entry value */ +__attribute__((noinline)) static uintptr_t count_upto(struct slice a, uintptr_t cap) +{ + if (a.len > cap) + a.len = cap; + uintptr_t n = 0; + for (uintptr_t i = 0; i < a.len; i++) + n += a.ptr[i] & 1; + return n; +} + +/* read through a pointer to the parameter */ +__attribute__((noinline)) static uintptr_t via_ptr(struct slice a, int k) +{ + struct slice *p = &a; + if (k) + p->len = p->len / 2; + uintptr_t s = 0; + for (uintptr_t i = 0; i < p->len; i++) + s += p->ptr[i]; + return s + a.len; +} + +/* a narrow read of the word */ +__attribute__((noinline)) static unsigned low_byte_sum(struct slice a) +{ + unsigned s = 0; + for (uintptr_t i = 0; i < a.len; i++) + s += (uint8_t)a.len + a.ptr[i]; + return s; +} + +int main(void) +{ + static const uint8_t x[] = "hello, world"; + static const uint8_t y[] = "hello, there"; + struct slice a = {x, 12}, b = {y, 12}, c = {x, 5}, d = {y, 5}; + printf("eql %d %d %d %d\n", eql(a, b), eql(c, d), eql(a, a), eql((struct slice){x + 1, 3}, (struct slice){y + 1, 3})); + printf("count %u %u\n", (unsigned)count_upto(a, 4), (unsigned)count_upto(a, 100)); + printf("ptr %u %u\n", (unsigned)via_ptr(a, 0), (unsigned)via_ptr(a, 1)); + printf("low %u\n", low_byte_sum((struct slice){x, 260 - 250})); + return 0; +} diff --git a/tests/ir_tests/541_param_home_fwd.expect b/tests/ir_tests/541_param_home_fwd.expect new file mode 100644 index 000000000..fbb8150e7 --- /dev/null +++ b/tests/ir_tests/541_param_home_fwd.expect @@ -0,0 +1,4 @@ +eql 0 1 1 1 +count 1 4 +ptr 1172 582 +low 1052 diff --git a/tests/ir_tests/542_sret_nrvo.c b/tests/ir_tests/542_sret_nrvo.c new file mode 100644 index 000000000..b01d3a75d --- /dev/null +++ b/tests/ir_tests/542_sret_nrvo.c @@ -0,0 +1,86 @@ +/* A struct returned through the hidden pointer is built in the caller's buffer + when the local holding it -- and a local copied into it whole, which then + dies -- is reached only through its own address, and nothing else touches + memory while it is built: Zig's `t0 = {...}; ...; t10 = t0; return t10;` + in every Wyhash.init. The caller may pass a buffer the callee sees through + another pointer (`*x = f(x)`), so a read or write through a pointer, or a + call, while the object is built keeps the local. Here: the Wyhash.init + shape (zero fill, 64-bit fields, a copy from .rodata, a whole-object copy + between two locals), and callees that read their argument after writing + part of the result, with the caller passing that argument as the buffer. */ +#include +#include +#include + +struct wy { uint64_t a, b, state[3]; uintptr_t total, blen; uint8_t buf[48]; }; + +__attribute__((noinline)) static struct wy wy_init(uint64_t seed) +{ + struct wy t0 = {0x1111, 0x2222, {3, 4, 5}, 0, 0, "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUV"}; + struct wy t10; + uint64_t *p = &t0.state[0]; + *p = seed ^ 0x9e3779b97f4a7c15ull; + t0.state[1] = t0.state[0] * 3; + t0.state[2] = t0.state[1] + t0.a; + t10 = t0; + return t10; +} + +struct pair { unsigned x, y, z, w, pad[4]; }; + +/* reads *in after writing r.x: x aliases the result when called as *p = swap(p) */ +__attribute__((noinline)) static struct pair swap(const struct pair *in) +{ + struct pair r; + r.x = in->y; + r.y = in->x; + r.z = in->w + in->x; + r.w = in->z; + r.pad[0] = r.pad[1] = r.pad[2] = r.pad[3] = in->pad[0] + 1; + return r; +} + +static struct pair g; + +/* a global read while building */ +__attribute__((noinline)) static struct pair from_global(unsigned k) +{ + struct pair r; + r.x = k; + r.y = g.x + k; + r.z = g.y; + r.w = 7; + r.pad[0] = r.pad[1] = r.pad[2] = r.pad[3] = g.pad[0] + 1; + return r; +} + +/* a call while building */ +__attribute__((noinline)) static unsigned peek(const struct pair *p) { return p->x * 10 + p->y; } +__attribute__((noinline)) static struct pair with_call(const struct pair *in) +{ + struct pair r; + r.x = 1; + r.y = 2; + r.z = peek(in); + r.w = in->x; + r.pad[0] = r.pad[1] = r.pad[2] = r.pad[3] = 0; + return r; +} + +int main(void) +{ + struct wy w = wy_init(42); + printf("wy %llx %llx %llx %llx %llx %u %u %.5s %.5s\n", (unsigned long long)w.a, (unsigned long long)w.b, + (unsigned long long)w.state[0], (unsigned long long)w.state[1], (unsigned long long)w.state[2], + (unsigned)w.total, (unsigned)w.blen, (const char *)w.buf, (const char *)w.buf + 43); + struct pair p = {1, 2, 3, 4, {5}}; + p = swap(&p); + printf("swap %u %u %u %u %u\n", p.x, p.y, p.z, p.w, p.pad[3]); + g = (struct pair){5, 6, 7, 8, {9}}; + g = from_global(3); + printf("global %u %u %u %u %u\n", g.x, g.y, g.z, g.w, g.pad[3]); + struct pair q = {9, 8, 7, 6, {1}}; + q = with_call(&q); + printf("call %u %u %u %u\n", q.x, q.y, q.z, q.w); + return 0; +} diff --git a/tests/ir_tests/542_sret_nrvo.expect b/tests/ir_tests/542_sret_nrvo.expect new file mode 100644 index 000000000..1084d2f36 --- /dev/null +++ b/tests/ir_tests/542_sret_nrvo.expect @@ -0,0 +1,4 @@ +wy 1111 2222 9e3779b97f4a7c3f daa66d2c7ddf74bd daa66d2c7ddf85ce 0 0 abcde RSTUV +swap 2 1 5 3 6 +global 3 8 6 7 10 +call 1 2 98 9 diff --git a/tests/ir_tests/543_dead_def_cbz_narrow_ext.c b/tests/ir_tests/543_dead_def_cbz_narrow_ext.c new file mode 100644 index 000000000..dab77d67b --- /dev/null +++ b/tests/ir_tests/543_dead_def_cbz_narrow_ext.c @@ -0,0 +1,109 @@ +/* Size peepholes that delete or merge instructions, and the shapes each one + must leave alone. + + dead_def (ir/dead_def.c): a compound literal zero-fills a struct and then + stores its fields; once SRA splits it into field TEMPs, the zero is + overwritten before anything reads it (`movs r1,#0; ldr r1,[pc,#k]`). + + Narrow loads and their casts: `(signed char)u8` after LDRB, or + `(unsigned short)s16` after LDRSH, still have to extend -- a rewrite + that treated a narrow load's extension as settled by its operand types + was tried and dropped (the code generator loaded an i8 with LDRB). + + CBZ bound (arm-thumb-gen.c rehearsal_max_shrink_between): the Zig error + return `if (e.error) return e;` became CBZ once the fused branch stopped + being counted twice. Each chain must still take the right exit. */ +#include + +typedef unsigned char u8; +typedef signed char s8; +typedef unsigned short u16; +typedef short s16; + +struct slice +{ + const u8 *ptr; + unsigned len; +}; + +static const u8 hello[5] = {'h', 'e', 'l', 'l', 'o'}; +static const u8 world[6] = {'w', 'o', 'r', 'l', 'd', '!'}; + +__attribute__((noinline)) unsigned hash(struct slice s) +{ + unsigned t = 7; + for (unsigned i = 0; i < s.len; i++) + t = t * 31u + s.ptr[i]; + return t; +} + +__attribute__((noinline)) unsigned two(unsigned k) +{ + unsigned a = hash((struct slice){hello, 5}); + if (k) + a ^= hash((struct slice){world, k}); + return a; +} + +__attribute__((noinline)) int ext_sb(const u8 *p, int i) { return (s8)p[i]; } +__attribute__((noinline)) int ext_sh(const u8 *p, int i) { return (s16)p[i]; } +__attribute__((noinline)) int ext_ub(const u8 *p, int i) { return (u8)p[i]; } +__attribute__((noinline)) int ext_uh_of_s16(const s16 *q, int i) { return (u16)q[i]; } +__attribute__((noinline)) int ext_sb_of_s16(const s16 *q, int i) { return (s8)q[i]; } +__attribute__((noinline)) int ext_sh_of_s16(const s16 *q, int i) { return (s16)q[i]; } +__attribute__((noinline)) int ext_ub_of_u16(const u16 *q, int i) { return (u8)q[i]; } + +struct eu +{ + u16 error; +}; + +__attribute__((noinline)) struct eu step(int x) +{ + struct eu r = {(u16)(x == 3 ? 7 : x == 5 ? 9 : 0)}; + return r; +} + +__attribute__((noinline)) void note(int *n) { ++*n; } + +__attribute__((noinline)) struct eu chain(int base, int *calls) +{ + struct eu t = step(base); + if (t.error) + return t; + t = step(base + 1); + if (t.error) + { + note(calls); + return t; + } + t = step(base + 2); + if (t.error) + return (struct eu){(u16)(t.error + 100)}; + return (struct eu){0}; +} + +int main(void) +{ + printf("two %u %u %u\n", two(0), two(3), two(6)); + + const u8 bytes[4] = {0x7f, 0x80, 0xff, 0x01}; + for (int i = 0; i < 4; i++) + printf("u8[%d] sb=%d sh=%d ub=%d\n", i, ext_sb(bytes, i), ext_sh(bytes, i), ext_ub(bytes, i)); + + const s16 halves[3] = {-2, 0x1234, -32768}; + for (int i = 0; i < 3; i++) + printf("s16[%d] uh=%d sb=%d sh=%d\n", i, ext_uh_of_s16(halves, i), ext_sb_of_s16(halves, i), + ext_sh_of_s16(halves, i)); + const u16 uh[2] = {0xff80, 0x017f}; + for (int i = 0; i < 2; i++) + printf("u16[%d] ub=%d\n", i, ext_ub_of_u16(uh, i)); + + for (int b = 0; b < 6; b++) + { + int calls = 0; + struct eu e = chain(b, &calls); + printf("chain(%d) = %u calls=%d\n", b, e.error, calls); + } + return 0; +} diff --git a/tests/ir_tests/543_dead_def_cbz_narrow_ext.expect b/tests/ir_tests/543_dead_def_cbz_narrow_ext.expect new file mode 100644 index 000000000..c9b9494e8 --- /dev/null +++ b/tests/ir_tests/543_dead_def_cbz_narrow_ext.expect @@ -0,0 +1,16 @@ +two 299566379 299892248 1383526589 +u8[0] sb=127 sh=127 ub=127 +u8[1] sb=-128 sh=128 ub=128 +u8[2] sb=-1 sh=255 ub=255 +u8[3] sb=1 sh=1 ub=1 +s16[0] uh=65534 sb=-2 sh=-2 +s16[1] uh=4660 sb=52 sh=4660 +s16[2] uh=32768 sb=0 sh=-32768 +u16[0] ub=128 +u16[1] ub=127 +chain(0) = 0 calls=0 +chain(1) = 107 calls=0 +chain(2) = 7 calls=1 +chain(3) = 7 calls=0 +chain(4) = 9 calls=1 +chain(5) = 9 calls=0 diff --git a/tests/ir_tests/544_frame_word_pair_slot_copy.c b/tests/ir_tests/544_frame_word_pair_slot_copy.c new file mode 100644 index 000000000..56679aef5 --- /dev/null +++ b/tests/ir_tests/544_frame_word_pair_slot_copy.c @@ -0,0 +1,70 @@ +/* Frame-word pairing and same-slot copies (arm-thumb-gen.c). + + frame_word_pair_rewind merges `ldr rA,[sp,#k]; ldr rB,[sp,#k+4]` (and the + STR pair) into one LDRD/STRD after the fact, by rewinding over the first + access. Values kept alive across calls in more registers than the + callee-saved set spill and reload in neighbouring words; the reloads must + still deliver each value to its own register, in either order, and a pair + must never swallow a load into the same register. + + tcc_gen_machine_assign_mop_ex skips a copy between two values spilled to + the same slot: a phi copy of a rotating set of spilled values. */ +#include + +__attribute__((noinline)) unsigned id(unsigned x) { return x; } + +__attribute__((noinline)) unsigned pressure(unsigned a) +{ + unsigned v0 = id(a), v1 = id(a + 1), v2 = id(a + 2), v3 = id(a + 3), v4 = id(a + 4), v5 = id(a + 5); + unsigned v6 = id(a + 6), v7 = id(a + 7), v8 = id(a + 8), v9 = id(a + 9), v10 = id(a + 10), v11 = id(a + 11); + unsigned s = 0; + for (int k = 0; k < 5; k++) + { + unsigned t = v0; + v0 = v1; v1 = v2; v2 = v3; v3 = v4; v4 = v5; v5 = v6; + v6 = v7; v7 = v8; v8 = v9; v9 = v10; v10 = v11; v11 = t; + s += id(v0) ^ (v5 * 3u) ^ (v11 << 2); + } + return s + v0 * 1u + v1 * 2u + v2 * 3u + v3 * 5u + v4 * 7u + v5 * 11u + v6 * 13u + v7 * 17u + v8 * 19u + + v9 * 23u + v10 * 29u + v11 * 31u; +} + +struct quad +{ + unsigned a, b, c, d; +}; + +__attribute__((noinline)) struct quad mix(struct quad q, unsigned k) +{ + struct quad r; + r.a = id(q.b + k); + r.b = id(q.a ^ k); + r.c = id(q.d - k); + r.d = id(q.c * k); + return r; +} + +__attribute__((noinline)) unsigned swapper(unsigned x, unsigned y, int n) +{ + unsigned p = id(x), q = id(y), r = id(x ^ y), s = id(x + y); + for (int i = 0; i < n; i++) + { + unsigned t = p; + p = q; + q = id(t + r); + r = s; + s = id(r ^ t); + } + return p * 3u + q * 5u + r * 7u + s * 11u; +} + +int main(void) +{ + printf("pressure %u %u\n", pressure(1), pressure(1000)); + struct quad q = {1, 2, 3, 4}; + for (unsigned k = 1; k < 4; k++) + q = mix(q, k); + printf("mix %u %u %u %u\n", q.a, q.b, q.c, q.d); + printf("swapper %u %u\n", swapper(3, 5, 7), swapper(100, 7, 12)); + return 0; +} diff --git a/tests/ir_tests/544_frame_word_pair_slot_copy.expect b/tests/ir_tests/544_frame_word_pair_slot_copy.expect new file mode 100644 index 000000000..5a987b709 --- /dev/null +++ b/tests/ir_tests/544_frame_word_pair_slot_copy.expect @@ -0,0 +1,3 @@ +pressure 950 171325 +mix 4 1 3 3 +swapper 761 17464 diff --git a/tests/ir_tests/545_cross_jump_alloc.c b/tests/ir_tests/545_cross_jump_alloc.c new file mode 100644 index 000000000..71ed5b9ef --- /dev/null +++ b/tests/ir_tests/545_cross_jump_alloc.c @@ -0,0 +1,147 @@ +/* Tail merging by final location (ir/cross_jump.c tcc_ir_cross_jump_alloc). + + Block tails that differ only in which vreg brings a value in are merged + once allocation is final, when both vregs sit in the same place; tails + ending in a return merge too, and so do tails holding whole calls. Each + shape here is repeated so that a merge happens, and each path must still + deliver its own value: the error code of the call that failed, the right + argument to a shared call, the return value of its own path. + + Must not merge (or must merge correctly): tails whose incoming values sit + in different places, and a shared call whose first arguments are set up + before the common part. */ +#include + +typedef unsigned short u16; + +struct eu +{ + u16 error; + unsigned payload; +}; + +static int log_count; + +__attribute__((noinline)) u16 step(int x, int fail_at) +{ + return (u16)(x == fail_at ? 40 + x : 0); +} + +__attribute__((noinline)) void note(unsigned v) +{ + log_count += (int)v; +} + +/* Zig's `try` chain: every failure stores its own error through the sret + pointer and leaves by the same exit. */ +__attribute__((noinline)) struct eu chain(int fail_at) +{ + struct eu r; + u16 e = step(1, fail_at); + if (e) + { + r.error = e; + r.payload = 0xaaaaaaaau; + return r; + } + e = step(2, fail_at); + if (e) + { + r.error = e; + r.payload = 0xaaaaaaaau; + return r; + } + e = step(3, fail_at); + if (e) + { + r.error = e; + r.payload = 0xaaaaaaaau; + return r; + } + r.error = 0; + r.payload = 7; + return r; +} + +/* Tails with a whole call: note(e) then return e. */ +__attribute__((noinline)) unsigned calls(int fail_at) +{ + unsigned e = step(1, fail_at); + if (e) + { + note(e); + return e + 1000; + } + e = step(2, fail_at); + if (e) + { + note(e); + return e + 1000; + } + e = step(3, fail_at); + if (e) + { + note(e); + return e + 1000; + } + return 0; +} + +/* The shared part starts after the first argument is set: the calls differ + in it, so they must not collapse into one call with one argument. */ +__attribute__((noinline)) unsigned two_arg(unsigned a, unsigned b) +{ + return a * 100u + b; +} + +__attribute__((noinline)) unsigned split_args(int k, unsigned b) +{ + if (k == 1) + return two_arg(11, b) + 5; + if (k == 2) + return two_arg(22, b) + 5; + return two_arg(33, b) + 5; +} + +/* Different incoming values in different places. */ +__attribute__((noinline)) unsigned mixed(unsigned x, unsigned y, int k) +{ + unsigned r; + if (k == 1) + { + r = x * 3u; + note(r); + return r + 1; + } + if (k == 2) + { + r = y * 5u; + note(r); + return r + 1; + } + r = x + y; + note(r); + return r + 1; +} + +int main(void) +{ + for (int f = 0; f <= 4; f++) + { + struct eu r = chain(f); + printf("chain(%d) error=%u payload=%x\n", f, r.error, r.payload); + } + for (int f = 0; f <= 4; f++) + { + unsigned v = calls(f); + printf("calls(%d)=%u log=%d\n", f, v, log_count); + } + for (int k = 1; k <= 3; k++) + printf("split_args(%d)=%u\n", k, split_args(k, 9)); + for (int k = 1; k <= 3; k++) + { + unsigned v = mixed(7, 11, k); + printf("mixed(%d)=%u log=%d\n", k, v, log_count); + } + return 0; +} diff --git a/tests/ir_tests/545_cross_jump_alloc.expect b/tests/ir_tests/545_cross_jump_alloc.expect new file mode 100644 index 000000000..5be533be2 --- /dev/null +++ b/tests/ir_tests/545_cross_jump_alloc.expect @@ -0,0 +1,16 @@ +chain(0) error=0 payload=7 +chain(1) error=41 payload=aaaaaaaa +chain(2) error=42 payload=aaaaaaaa +chain(3) error=43 payload=aaaaaaaa +chain(4) error=0 payload=7 +calls(0)=0 log=0 +calls(1)=1041 log=41 +calls(2)=1042 log=83 +calls(3)=1043 log=126 +calls(4)=0 log=126 +split_args(1)=1114 +split_args(2)=2214 +split_args(3)=3314 +mixed(1)=22 log=147 +mixed(2)=56 log=202 +mixed(3)=19 log=220 diff --git a/tests/ir_tests/546_ssa_symref_store_def.c b/tests/ir_tests/546_ssa_symref_store_def.c new file mode 100644 index 000000000..a315d0d60 --- /dev/null +++ b/tests/ir_tests/546_ssa_symref_store_def.c @@ -0,0 +1,45 @@ +/* A var-slot STORE of a symbol's address (a SYMREF without is_lval) or of a + 64-bit immediate defines the var like an immediate store does, so SSA + renaming gives it a fresh name. Kept a memory write, it stayed a second + def under the previous store's TEMP name, and the zero a compound literal's + fill had stored first reached the code before every Zig panic message + slice (`movs r0, #0; ldr r0, =msg`). Checks the values arrive -- by value + in registers, through a join, and for a 64-bit field. */ +#include +#include + +struct sl { const char *p; unsigned n; }; +struct opt { unsigned char is_null; unsigned payload; }; +struct w64 { uint64_t v; const char *tag; }; + +static const char msg_a[] = "first message"; +static const char msg_b[] = "second"; + +__attribute__((noinline)) static unsigned show(struct sl s, struct opt o) +{ + printf("%.*s %u %u\n", (int)s.n, s.p, o.is_null, o.payload); + return s.n + o.payload; +} + +__attribute__((noinline)) static unsigned show64(struct w64 w) +{ + printf("%llx %s\n", (unsigned long long)w.v, w.tag); + return (unsigned)w.v; +} + +__attribute__((noinline)) static unsigned pick(int k) +{ + struct sl s = (struct sl){msg_a, 5}; + if (k) + s = (struct sl){msg_b, 6}; + return show(s, (struct opt){0, 7}); +} + +int main(void) +{ + unsigned r = show((struct sl){msg_a, 13}, (struct opt){1, 0xaaaaaaaa}); + r += pick(0) + pick(1); + r += show64((struct w64){0x123456789abcdef0ull, msg_b}); + printf("%u\n", r); + return 0; +} diff --git a/tests/ir_tests/546_ssa_symref_store_def.expect b/tests/ir_tests/546_ssa_symref_store_def.expect new file mode 100644 index 000000000..ed83b1966 --- /dev/null +++ b/tests/ir_tests/546_ssa_symref_store_def.expect @@ -0,0 +1,5 @@ +first message 1 2863311530 +first 0 7 +second 0 7 +123456789abcdef0 second +1164413376 diff --git a/tests/ir_tests/548_store_imm_scratch_not_base.c b/tests/ir_tests/548_store_imm_scratch_not_base.c new file mode 100644 index 000000000..567366fa7 --- /dev/null +++ b/tests/ir_tests/548_store_imm_scratch_not_base.c @@ -0,0 +1,49 @@ +/* A constant stored through a register while every register holds a live + value. + + sret_nrvo builds make()'s result straight in *P0, so the zero fill is + `*(P0 + 64) = 0; *(P0 + 0) = 0; ...` with P0 in r0 and r1-r3 holding the + other parameters: no register is free. The constant-displacement store + took a scratch for the 0 without excluding the base, got r0 (saved and + restored around the use), and emitted + str r0,[sp]; movs r0,#0; str r0,[r0,#64] + -- a store to address 64, so the result's tail word kept the caller's + 0x55 (gcc torture pr108498-1, device only). */ +#include +#include + +struct R +{ + unsigned char a, b; + signed char c; + unsigned char d; + int w[15]; + int tail; +}; + +__attribute__((noinline)) static struct R make(unsigned char a, unsigned b, int c, int e, int f) +{ + struct R x = {.a = b, .c = c, .d = a, .w = {[2] = e, [9] = f}}; + return x; +} + +/* Leave 0x55 in the stack the result's temporary will occupy. */ +__attribute__((noinline)) void dirty(struct R *x) +{ + char buf[1024]; + memset(buf, 0x55, sizeof(buf)); + __asm__ volatile("" : : "r"(buf) : "memory"); + memset(x, 0x55, sizeof(*x)); +} + +int main(void) +{ + struct R x; + dirty(&x); + x = make(1, 2, -1, 30, 90); + int sum = 0; + for (int i = 0; i < 15; i++) + sum += x.w[i]; + printf("a=%d b=%d c=%d d=%d sum=%d tail=%d\n", x.a, x.b, x.c, x.d, sum, x.tail); + return 0; +} diff --git a/tests/ir_tests/548_store_imm_scratch_not_base.expect b/tests/ir_tests/548_store_imm_scratch_not_base.expect new file mode 100644 index 000000000..7e3435838 --- /dev/null +++ b/tests/ir_tests/548_store_imm_scratch_not_base.expect @@ -0,0 +1 @@ +a=2 b=0 c=-1 d=1 sum=120 tail=0 diff --git a/tests/ir_tests/549_copy_stub_calls.c b/tests/ir_tests/549_copy_stub_calls.c new file mode 100644 index 000000000..78f51990b --- /dev/null +++ b/tests/ir_tests/549_copy_stub_calls.c @@ -0,0 +1,134 @@ +/* Word-aligned block copies of 3..32 words call libtcc1's __tcc_wcopy_N + (lib/arm_copystub.S) instead of expanding inline: struct assignment + (__aeabi_memmove4/8), a plain memcpy the compiler proves aligned, a struct + argument copied into the outgoing area (also for a tail call, whose copy + must still return), and an initialiser image copied into a leaf function's + local (the call clobbers LR, which the leaf saves around it). Every size + from 3 to 33 words, so each entry into the three chains is run, and the + words around the destination stay untouched. */ +#include +#include +#include + +#define GUARD 0xdeadbeefu +#define SIZES(X) \ + X(3) X(4) X(5) X(6) X(7) X(8) X(9) X(10) X(11) X(12) X(13) X(14) X(15) \ + X(16) X(17) X(18) X(19) X(20) X(21) X(22) X(23) X(24) X(25) X(26) X(27) \ + X(28) X(29) X(30) X(31) X(32) X(33) + +#define DEFS(N) \ + typedef struct { uint32_t w[N]; } s##N; \ + typedef struct { uint32_t lo; s##N v; uint32_t hi; } g##N; +SIZES(DEFS) +typedef struct { uint64_t d[7]; } d7; /* 8-aligned: __aeabi_memmove8 */ + +static uint32_t hash(const uint32_t *w, int n) +{ + uint32_t h = 0; + for (int i = 0; i < n; i++) + h = h * 33 + w[i]; + return h; +} + +static uint32_t fail; +static void check_guard(const char *what, int n, uint32_t lo, uint32_t hi) +{ + if (lo != GUARD || hi != GUARD) + { + printf("%s %d: guard %08x %08x\n", what, n, (unsigned)lo, (unsigned)hi); + fail++; + } +} + +/* Struct assignment into the middle of a guarded object. */ +#define ASSIGN(N) \ + __attribute__((noinline)) static uint32_t assign##N(const s##N *p) \ + { \ + g##N g; \ + g.lo = g.hi = GUARD; \ + g.v = *p; \ + check_guard("assign", N, g.lo, g.hi); \ + return hash(g.v.w, N); \ + } +SIZES(ASSIGN) + +/* A plain memcpy between word-aligned locals. */ +#define MEMCPY(N) \ + __attribute__((noinline)) static uint32_t mcpy##N(const s##N *p) \ + { \ + s##N a = *p, b; \ + a.w[0] ^= 1; \ + memcpy(&b, &a, sizeof b); \ + return hash(b.w, N); \ + } +SIZES(MEMCPY) + +/* By value: past r0-r3 the struct is copied into the outgoing area. */ +#define BYVAL(N) \ + __attribute__((noinline)) static uint32_t take##N(int a, int b, s##N v) \ + { \ + return hash(v.w, N) + (uint32_t)(a * 7 + b); \ + } \ + __attribute__((noinline)) static uint32_t pass##N(const s##N *p) \ + { \ + s##N v = *p; \ + v.w[N - 1] += 3; \ + uint32_t r = take##N(1, 2, v); \ + return r + take##N(3, 4, v); \ + } \ + __attribute__((noinline)) static uint32_t tail##N(const s##N *p) \ + { \ + return take##N(5, 6, *p); \ + } +SIZES(BYVAL) + +/* An initialiser image copied into a local of a leaf function. */ +#define INIT(N) \ + __attribute__((noinline)) static uint32_t init##N(uint32_t k) \ + { \ + s##N v = {{[0] = 0x11, [1] = 0x2200, [2] = 0x330000, [N / 2] = 0x44, \ + [N - 1] = 0x55000000}}; \ + v.w[k % N] += k; \ + uint32_t h = 0; \ + for (int i = 0; i < N; i++) \ + h = h * 33 + v.w[i]; \ + return h; \ + } +SIZES(INIT) + +__attribute__((noinline)) static uint64_t copy_d7(const d7 *p) +{ + d7 t = *p; + t.d[6] ^= 0x0123456789abcdefull; + d7 u = t; + uint64_t h = 0; + for (int i = 0; i < 7; i++) + h = h * 31 + u.d[i]; + return h; +} + +int main(void) +{ + uint32_t src[33]; + for (int i = 0; i < 33; i++) + src[i] = 0x10203040u * (uint32_t)(i + 1) + (uint32_t)i; + uint32_t total = 0; +#define RUN(N) \ + { \ + s##N s; \ + memcpy(&s, src, sizeof s); \ + uint32_t a = assign##N(&s), m = mcpy##N(&s), b = pass##N(&s), \ + t = tail##N(&s), c = init##N(N + 5); \ + printf("%2d %08x %08x %08x %08x %08x\n", N, (unsigned)a, (unsigned)m, \ + (unsigned)b, (unsigned)t, (unsigned)c); \ + total += a ^ m ^ b ^ t ^ c; \ + } + SIZES(RUN) + d7 d; + for (int i = 0; i < 7; i++) + d.d[i] = 0x1111111111111111ull * (uint64_t)(i + 1); + uint64_t h = copy_d7(&d); + printf("d7 %08x%08x\n", (unsigned)(h >> 32), (unsigned)h); + printf("total %08x fail %u\n", (unsigned)total, (unsigned)fail); + return 0; +} diff --git a/tests/ir_tests/549_copy_stub_calls.expect b/tests/ir_tests/549_copy_stub_calls.expect new file mode 100644 index 000000000..4a3ccc250 --- /dev/null +++ b/tests/ir_tests/549_copy_stub_calls.expect @@ -0,0 +1,33 @@ + 3 f19a41a3 f19a45e4 e334836e f19a41cc 5500511d + 4 65633706 6563c367 cac66e34 6563372f 559a237e + 5 626b090a 627d218b c4d6123c 626b0933 698e98df + 6 108d4bcf 12e27470 211a97c6 108d4bf8 db3cf600 + 7 93181775 e0125436 26302f12 9318179e a2dbb621 + 8 771c881c 635e5cfd ee391060 771c8845 59a94602 + 9 ebcf3de4 604baee5 d79e7bf0 ebcf3e0d eed292a3 +10 06f8dced 0b036e0e 0df1ba02 06f8dd16 8f66b704 +11 97768d57 1cd34298 2eed1ad6 97768d80 de92c025 +12 47ca7b42 78bdd8a3 8f94f6ac 47ca7b6b 8a0ba906 +13 12bc56ce 621b604f 2578adc4 12bc56f7 17c29ea7 +14 4c09d41b 874a0dbc 9813a85e 4c09d444 958e7c08 +15 bf272b49 626e990a 7e4e56ba bf272b72 ab688e29 +16 a60f9878 b244be59 4c1f3118 a60f98a1 ce45900a +17 7a25dbc8 0cffbdc9 f44bb7b8 7a25dbf1 27eaeeab +18 e124b959 cf3adb7a c24972da e124b982 b2e7450c +19 381f794b e8f9df8c 703ef2be 381f7974 ab10202d +20 7e9267be 4ab9961f fd24cfa4 7e9267e7 b689fb0e +21 a38354d2 f4904f53 4706a9cc a38354fb f3fe82af +22 76b214a7 e95e5f48 ed642976 76b214d0 d2a41210 +23 bfd8ff5d 880e9e1e 7fb1fee2 bfd8ff86 753c7631 +24 3dfd7114 0ce6e7f5 7bfae250 3dfd713d 190bea12 +25 90d049ec 3ce89ced 21a09400 90d04a15 66b05ab3 +26 4e1e6e05 7d412126 9c3cdc32 4e1e6e2e 45f7e314 +27 c551457f d8ca5cc0 8aa28b26 c55145a8 d7a09035 +28 32ff3c7a b59b3bdb 65fe791c 32ff3ca3 51fe5d16 +29 668c4316 3ca82e97 cd188654 668c433f c0b376b7 +30 1bda4d73 b573a914 37b49b0e 1bda4d9c 0415b818 +31 8b09d3b1 57cea372 1613a78a 8b09d3da 15ef6e39 +32 f04a51f0 55a919d1 e094a408 f04a5219 8394541a +33 0dbac850 1ef28c51 1b7590c8 0dbac879 d8bad6bb +d7 dcba9877617f1f65 +total 6641c241 fail 0 diff --git a/tests/ir_tests/600_sra_padded_partial_store.c b/tests/ir_tests/600_sra_padded_partial_store.c new file mode 100644 index 000000000..af2d8c092 --- /dev/null +++ b/tests/ir_tests/600_sra_padded_partial_store.c @@ -0,0 +1,99 @@ +/* SRA: a u16 stored into the low half of a word field whose other half is + * struct padding becomes a plain move of the value -- the padding may hold + * anything after a member store (C11 6.2.6.1p6). Zig's error unions have + * exactly this shape: `{ payload; u16 error; }` returned through a result + * pointer, the error written by itself and the struct copied whole. The + * values stored carry high garbage bits, so a read that forgot to mask the + * field would show them. */ +#include +#include + +typedef struct +{ + uint32_t payload; + uint16_t err; /* bytes 6-7: padding */ +} EU; + +/* The u16 at byte 6 over an unnamed bit-field, which counts as a member: + * its store stays an insert that keeps bytes 4-5. */ +typedef struct +{ + uint32_t payload; + unsigned : 16; + uint16_t hi; +} EH; + +static volatile uint32_t seed = 0x12345678u; + +__attribute__((noinline)) EU make(uint32_t v, uint32_t e) +{ + EU r; + r.payload = v * 0x9E3779B9u; + r.err = (uint16_t)(e ^ (v >> 7)); + return r; +} + +/* Zig's `if (t) |err| return err; ... r.err = ...`: the u16 of a call's + * result updated in place, the struct returned whole. */ +__attribute__((noinline)) EU check(uint32_t v, uint32_t e) +{ + EU t = make(v, e); + EU r; + if (t.err & 1) + { + r.err = t.err; + r.payload = 0xAAAAAAAAu; + return r; + } + r = make(e, v); + r.err = (uint16_t)(r.err + v + 0x7FFF0000u); + return r; +} + +__attribute__((noinline)) void update(EU *out, uint32_t v, uint32_t e) +{ + EU t = make(v, e); + if (t.err & 2) + t.err = (uint16_t)((e >> 1) | 0xFFFF0000u); + else + t.err = (uint16_t)(t.payload + e); + *out = t; +} + +__attribute__((noinline)) EH make_hi(uint32_t v, uint32_t h) +{ + EH r; + r.payload = v; + r.hi = (uint16_t)h; + return r; +} + +__attribute__((noinline)) EH pass_hi(uint32_t v, uint32_t k) +{ + EH r = make_hi(v, v * 3 + k); + if (k & 1) + r.hi = (uint16_t)(r.hi ^ (k | 0xFFFF0000u)); + return r; +} + +int main(void) +{ + uint32_t sum = 0; + for (uint32_t v = 0; v < 6; v++) + { + EU a = check(seed * (v + 1), v + 0x10000u); + EU b; + update(&b, seed + v, v * 0x01010101u); + printf("%u: %08x %04x %08x %04x\n", (unsigned)v, (unsigned)a.payload, (unsigned)a.err, (unsigned)b.payload, + (unsigned)b.err); + sum += a.payload ^ a.err ^ b.payload ^ b.err; + } + for (uint32_t k = 0; k < 4; k++) + { + EH h = pass_hi(seed + k, k + 0xABCD0000u); + printf("hi %u: %08x %04x\n", (unsigned)k, (unsigned)h.payload, (unsigned)h.hi); + sum += h.payload ^ h.hi; + } + printf("sum %08x\n", (unsigned)sum); + return 0; +} diff --git a/tests/ir_tests/600_sra_padded_partial_store.expect b/tests/ir_tests/600_sra_padded_partial_store.expect new file mode 100644 index 000000000..b0e2063e8 --- /dev/null +++ b/tests/ir_tests/600_sra_padded_partial_store.expect @@ -0,0 +1,11 @@ +0: 79b90000 aaf0 887934b8 34b8 +1: 17f079b9 5be0 26b0ae71 af72 +2: b627f372 04d0 c4e8282a 0101 +3: 545f6d2b b5c0 631fa1e3 8181 +4: f296e6e4 62b0 01571b9c 1fa0 +5: 90ce609d 0ba0 9f8e9555 9a5a +hi 0: 12345678 0368 +hi 1: 12345679 036d +hi 2: 1234567a 0370 +hi 3: 1234567b 0377 +sum 18e4de30 diff --git a/tests/ir_tests/601_overflow_builtins_narrow.c b/tests/ir_tests/601_overflow_builtins_narrow.c new file mode 100644 index 000000000..20c30a3cb --- /dev/null +++ b/tests/ir_tests/601_overflow_builtins_narrow.c @@ -0,0 +1,272 @@ +/* __builtin_{add,sub,mul}_overflow on 8/16/32-bit results against 64-bit + * reference arithmetic. 32-bit add/sub with operands of the result's width + * and signedness take the r < a / sign-of-xor checks at 32 bits; 8/16-bit + * results are computed in a 32-bit int instead of long long (unsigned for + * u16 * u16); mixed-type calls keep the general path. */ +#include +#include + +static const int64_t vals[] = {0, 1, 2, 3, 7, 0x7f, 0x80, 0xff, 0x100, 0x7fff, 0x8000, 0xffff, 0x10000, + 0x7fffffff, 0x80000000LL, 0xffffffffLL, -1, -2, -0x80, -0x81, -0x8000, + -0x8001, -0x7fffffffLL, -0x80000000LL, 12345, -12345, 0x1234567, 250}; +#define NV (int)(sizeof vals / sizeof vals[0]) + +static uint32_t h = 2166136261u; +static void mix(uint32_t x) { h = (h ^ x) * 16777619u; } +static unsigned bad; + +__attribute__((noinline)) static int add_u8(uint8_t a, uint8_t b, uint8_t *r) { return __builtin_add_overflow(a, b, r); } +__attribute__((noinline)) static int sub_u8(uint8_t a, uint8_t b, uint8_t *r) { return __builtin_sub_overflow(a, b, r); } +__attribute__((noinline)) static int mul_u8(uint8_t a, uint8_t b, uint8_t *r) { return __builtin_mul_overflow(a, b, r); } +__attribute__((noinline)) static int add_i8(int8_t a, int8_t b, int8_t *r) { return __builtin_add_overflow(a, b, r); } +__attribute__((noinline)) static int sub_i8(int8_t a, int8_t b, int8_t *r) { return __builtin_sub_overflow(a, b, r); } +__attribute__((noinline)) static int mul_i8(int8_t a, int8_t b, int8_t *r) { return __builtin_mul_overflow(a, b, r); } +__attribute__((noinline)) static int add_u16(uint16_t a, uint16_t b, uint16_t *r) { return __builtin_add_overflow(a, b, r); } +__attribute__((noinline)) static int sub_u16(uint16_t a, uint16_t b, uint16_t *r) { return __builtin_sub_overflow(a, b, r); } +__attribute__((noinline)) static int mul_u16(uint16_t a, uint16_t b, uint16_t *r) { return __builtin_mul_overflow(a, b, r); } +__attribute__((noinline)) static int add_i16(int16_t a, int16_t b, int16_t *r) { return __builtin_add_overflow(a, b, r); } +__attribute__((noinline)) static int sub_i16(int16_t a, int16_t b, int16_t *r) { return __builtin_sub_overflow(a, b, r); } +__attribute__((noinline)) static int mul_i16(int16_t a, int16_t b, int16_t *r) { return __builtin_mul_overflow(a, b, r); } +__attribute__((noinline)) static int add_u32(uint32_t a, uint32_t b, uint32_t *r) { return __builtin_add_overflow(a, b, r); } +__attribute__((noinline)) static int sub_u32(uint32_t a, uint32_t b, uint32_t *r) { return __builtin_sub_overflow(a, b, r); } +__attribute__((noinline)) static int mul_u32(uint32_t a, uint32_t b, uint32_t *r) { return __builtin_mul_overflow(a, b, r); } +__attribute__((noinline)) static int add_i32(int32_t a, int32_t b, int32_t *r) { return __builtin_add_overflow(a, b, r); } +__attribute__((noinline)) static int sub_i32(int32_t a, int32_t b, int32_t *r) { return __builtin_sub_overflow(a, b, r); } +__attribute__((noinline)) static int mul_i32(int32_t a, int32_t b, int32_t *r) { return __builtin_mul_overflow(a, b, r); } +__attribute__((noinline)) static int add_mix_u32(int32_t a, int32_t b, uint32_t *r) { return __builtin_add_overflow(a, b, r); } +__attribute__((noinline)) static int sub_mix_i32(uint16_t a, uint16_t b, int32_t *r) { return __builtin_sub_overflow(a, b, r); } + +int main(void) +{ + for (int i = 0; i < NV; i++) + for (int j = 0; j < NV; j++) + { + uint8_t a = (uint8_t)vals[i], b = (uint8_t)vals[j], r; + int o = add_u8(a, b, &r); + int64_t full = (int64_t)a + (int64_t)b; + uint8_t want = (uint8_t)full; + int want_o = full < 0LL || full > 255LL; + if (o != want_o || r != want) bad++; + mix((uint32_t)r); mix((uint32_t)o); + } + for (int i = 0; i < NV; i++) + for (int j = 0; j < NV; j++) + { + uint8_t a = (uint8_t)vals[i], b = (uint8_t)vals[j], r; + int o = sub_u8(a, b, &r); + int64_t full = (int64_t)a - (int64_t)b; + uint8_t want = (uint8_t)full; + int want_o = full < 0LL || full > 255LL; + if (o != want_o || r != want) bad++; + mix((uint32_t)r); mix((uint32_t)o); + } + for (int i = 0; i < NV; i++) + for (int j = 0; j < NV; j++) + { + uint8_t a = (uint8_t)vals[i], b = (uint8_t)vals[j], r; + int o = mul_u8(a, b, &r); + int64_t full = (int64_t)a * (int64_t)b; + uint8_t want = (uint8_t)full; + int want_o = full < 0LL || full > 255LL; + if (o != want_o || r != want) bad++; + mix((uint32_t)r); mix((uint32_t)o); + } + for (int i = 0; i < NV; i++) + for (int j = 0; j < NV; j++) + { + int8_t a = (int8_t)vals[i], b = (int8_t)vals[j], r; + int o = add_i8(a, b, &r); + int64_t full = (int64_t)a + (int64_t)b; + int8_t want = (int8_t)full; + int want_o = full < -128LL || full > 127LL; + if (o != want_o || r != want) bad++; + mix((uint32_t)r); mix((uint32_t)o); + } + for (int i = 0; i < NV; i++) + for (int j = 0; j < NV; j++) + { + int8_t a = (int8_t)vals[i], b = (int8_t)vals[j], r; + int o = sub_i8(a, b, &r); + int64_t full = (int64_t)a - (int64_t)b; + int8_t want = (int8_t)full; + int want_o = full < -128LL || full > 127LL; + if (o != want_o || r != want) bad++; + mix((uint32_t)r); mix((uint32_t)o); + } + for (int i = 0; i < NV; i++) + for (int j = 0; j < NV; j++) + { + int8_t a = (int8_t)vals[i], b = (int8_t)vals[j], r; + int o = mul_i8(a, b, &r); + int64_t full = (int64_t)a * (int64_t)b; + int8_t want = (int8_t)full; + int want_o = full < -128LL || full > 127LL; + if (o != want_o || r != want) bad++; + mix((uint32_t)r); mix((uint32_t)o); + } + for (int i = 0; i < NV; i++) + for (int j = 0; j < NV; j++) + { + uint16_t a = (uint16_t)vals[i], b = (uint16_t)vals[j], r; + int o = add_u16(a, b, &r); + int64_t full = (int64_t)a + (int64_t)b; + uint16_t want = (uint16_t)full; + int want_o = full < 0LL || full > 65535LL; + if (o != want_o || r != want) bad++; + mix((uint32_t)r); mix((uint32_t)o); + } + for (int i = 0; i < NV; i++) + for (int j = 0; j < NV; j++) + { + uint16_t a = (uint16_t)vals[i], b = (uint16_t)vals[j], r; + int o = sub_u16(a, b, &r); + int64_t full = (int64_t)a - (int64_t)b; + uint16_t want = (uint16_t)full; + int want_o = full < 0LL || full > 65535LL; + if (o != want_o || r != want) bad++; + mix((uint32_t)r); mix((uint32_t)o); + } + for (int i = 0; i < NV; i++) + for (int j = 0; j < NV; j++) + { + uint16_t a = (uint16_t)vals[i], b = (uint16_t)vals[j], r; + int o = mul_u16(a, b, &r); + int64_t full = (int64_t)a * (int64_t)b; + uint16_t want = (uint16_t)full; + int want_o = full < 0LL || full > 65535LL; + if (o != want_o || r != want) bad++; + mix((uint32_t)r); mix((uint32_t)o); + } + for (int i = 0; i < NV; i++) + for (int j = 0; j < NV; j++) + { + int16_t a = (int16_t)vals[i], b = (int16_t)vals[j], r; + int o = add_i16(a, b, &r); + int64_t full = (int64_t)a + (int64_t)b; + int16_t want = (int16_t)full; + int want_o = full < -32768LL || full > 32767LL; + if (o != want_o || r != want) bad++; + mix((uint32_t)r); mix((uint32_t)o); + } + for (int i = 0; i < NV; i++) + for (int j = 0; j < NV; j++) + { + int16_t a = (int16_t)vals[i], b = (int16_t)vals[j], r; + int o = sub_i16(a, b, &r); + int64_t full = (int64_t)a - (int64_t)b; + int16_t want = (int16_t)full; + int want_o = full < -32768LL || full > 32767LL; + if (o != want_o || r != want) bad++; + mix((uint32_t)r); mix((uint32_t)o); + } + for (int i = 0; i < NV; i++) + for (int j = 0; j < NV; j++) + { + int16_t a = (int16_t)vals[i], b = (int16_t)vals[j], r; + int o = mul_i16(a, b, &r); + int64_t full = (int64_t)a * (int64_t)b; + int16_t want = (int16_t)full; + int want_o = full < -32768LL || full > 32767LL; + if (o != want_o || r != want) bad++; + mix((uint32_t)r); mix((uint32_t)o); + } + for (int i = 0; i < NV; i++) + for (int j = 0; j < NV; j++) + { + uint32_t a = (uint32_t)vals[i], b = (uint32_t)vals[j], r; + int o = add_u32(a, b, &r); + int64_t full = (int64_t)a + (int64_t)b; + uint32_t want = (uint32_t)full; + int want_o = full < 0LL || full > 4294967295LL; + if (o != want_o || r != want) bad++; + mix((uint32_t)r); mix((uint32_t)o); + } + for (int i = 0; i < NV; i++) + for (int j = 0; j < NV; j++) + { + uint32_t a = (uint32_t)vals[i], b = (uint32_t)vals[j], r; + int o = sub_u32(a, b, &r); + int64_t full = (int64_t)a - (int64_t)b; + uint32_t want = (uint32_t)full; + int want_o = full < 0LL || full > 4294967295LL; + if (o != want_o || r != want) bad++; + mix((uint32_t)r); mix((uint32_t)o); + } + for (int i = 0; i < NV; i++) + for (int j = 0; j < NV; j++) + { + uint32_t a = (uint32_t)vals[i], b = (uint32_t)vals[j], r; + int o = mul_u32(a, b, &r); + uint64_t full = (uint64_t)a * (uint64_t)b; /* up to ~2^64: not an int64_t */ + uint32_t want = (uint32_t)full; + int want_o = full > 4294967295ULL; + if (o != want_o || r != want) bad++; + mix((uint32_t)r); mix((uint32_t)o); + } + for (int i = 0; i < NV; i++) + for (int j = 0; j < NV; j++) + { + int32_t a = (int32_t)vals[i], b = (int32_t)vals[j], r; + int o = add_i32(a, b, &r); + int64_t full = (int64_t)a + (int64_t)b; + int32_t want = (int32_t)full; + int want_o = full < -2147483648LL || full > 2147483647LL; + if (o != want_o || r != want) bad++; + mix((uint32_t)r); mix((uint32_t)o); + } + for (int i = 0; i < NV; i++) + for (int j = 0; j < NV; j++) + { + int32_t a = (int32_t)vals[i], b = (int32_t)vals[j], r; + int o = sub_i32(a, b, &r); + int64_t full = (int64_t)a - (int64_t)b; + int32_t want = (int32_t)full; + int want_o = full < -2147483648LL || full > 2147483647LL; + if (o != want_o || r != want) bad++; + mix((uint32_t)r); mix((uint32_t)o); + } + for (int i = 0; i < NV; i++) + for (int j = 0; j < NV; j++) + { + int32_t a = (int32_t)vals[i], b = (int32_t)vals[j], r; + int o = mul_i32(a, b, &r); + int64_t full = (int64_t)a * (int64_t)b; + int32_t want = (int32_t)full; + int want_o = full < -2147483648LL || full > 2147483647LL; + if (o != want_o || r != want) bad++; + mix((uint32_t)r); mix((uint32_t)o); + } + for (int i = 0; i < NV; i++) + for (int j = 0; j < NV; j++) + { + int32_t a = (int32_t)vals[i], b = (int32_t)vals[j]; + uint32_t r; + int o = add_mix_u32(a, b, &r); + int64_t full = (int64_t)a + b; + if (o != (full < 0 || full > 0xffffffffLL) || r != (uint32_t)full) bad++; + uint16_t c = (uint16_t)vals[i], d = (uint16_t)vals[j]; + int32_t s; + int p = sub_mix_i32(c, d, &s); + if (p != 0 || s != (int32_t)c - (int32_t)d) bad++; + mix(r); mix((uint32_t)s); + } + /* The result pointer aliases an operand: the check must use the operand's + * value from before the store (gcc PR85095, PR108789). */ + for (int i = 0; i < NV; i++) + for (int j = 0; j < NV; j++) + { + uint32_t x = (uint32_t)vals[i]; + int o = add_u32(x, (uint32_t)vals[j], &x); + if (o != ((uint64_t)(uint32_t)vals[i] + (uint32_t)vals[j] > 0xffffffffULL)) bad++; + int32_t y = (int32_t)vals[i]; + int p = sub_i32(y, (int32_t)vals[j], &y); + int64_t fy = (int64_t)(int32_t)vals[i] - (int32_t)vals[j]; + if (p != (fy < -0x80000000LL || fy > 0x7fffffffLL)) bad++; + uint16_t z = (uint16_t)vals[i]; + int q = mul_u16(z, (uint16_t)vals[j], &z); + if (q != ((uint32_t)(uint16_t)vals[i] * (uint16_t)vals[j] > 0xffff)) bad++; + mix(x); mix((uint32_t)y); mix(z); + } + printf("bad %u hash %08x\n", bad, (unsigned)h); + return 0; +} diff --git a/tests/ir_tests/601_overflow_builtins_narrow.expect b/tests/ir_tests/601_overflow_builtins_narrow.expect new file mode 100644 index 000000000..890b25205 --- /dev/null +++ b/tests/ir_tests/601_overflow_builtins_narrow.expect @@ -0,0 +1 @@ +bad 0 hash 9d779ffe diff --git a/tests/ir_tests/602_outline_shared_windows.c b/tests/ir_tests/602_outline_shared_windows.c new file mode 100644 index 000000000..8b7989dd7 --- /dev/null +++ b/tests/ir_tests/602_outline_shared_windows.c @@ -0,0 +1,87 @@ +/* -Os machine outliner: runs of IR ops that compile to the same code in + * several functions are emitted once and called with BL. The functions below + * are instances of one template (as Zig's generics are), so their windows + * repeat: a list append that stores two words and a halfword into a struct + * through a pointer, a field update on the loop head (a window starting at a + * jump target), a compare feeding a branch after the window (no CBZ fusion + * inside one), and a conditional select (an IT block inside the window). + * Non-static, so identical code folding does not merge them; each calls out, + * so LR is saved and a BL may clobber it. */ +#include +#include + +typedef struct +{ + uint32_t *items; + uint32_t len; + uint32_t cap; + uint16_t err; + uint16_t flags; + uint32_t sum; +} List; + +static uint32_t pool[8][32]; +static volatile uint32_t calls; + +__attribute__((noinline)) uint16_t grow(List *l, uint32_t want) +{ + calls++; + if (want > 32) + return 7; + l->cap = want; + return 0; +} + +#define TEMPLATE(NAME, K) \ + __attribute__((noinline)) uint16_t NAME(List *l, uint32_t v, uint32_t n) \ + { \ + for (uint32_t i = 0; i < n; i++) \ + { \ + l->sum = l->sum * 31u + (v ^ i) + l->len; \ + l->flags = (uint16_t)(l->flags | (1u << (i & 7))); \ + if (l->len == l->cap) \ + { \ + uint16_t e = grow(l, l->cap * 2 + 1); \ + if (e != 0) \ + { \ + l->err = e; \ + return e; \ + } \ + } \ + uint32_t w = (v + i * (K)) > l->sum ? v + i : l->sum - v; \ + l->items[l->len] = w; \ + l->len = l->len + 1; \ + } \ + l->err = 0; \ + return 0; \ + } + +TEMPLATE(append_a, 3) +TEMPLATE(append_b, 5) +TEMPLATE(append_c, 7) +TEMPLATE(append_d, 9) +TEMPLATE(append_e, 11) +TEMPLATE(append_f, 13) +TEMPLATE(append_g, 15) +TEMPLATE(append_h, 17) + +typedef uint16_t (*AppendFn)(List *, uint32_t, uint32_t); + +int main(void) +{ + static const AppendFn fns[8] = {append_a, append_b, append_c, append_d, append_e, append_f, append_g, append_h}; + uint32_t total = 0; + for (int f = 0; f < 8; f++) + { + List l = {pool[f], 0, 1, 0, 0, (uint32_t)f}; + uint16_t e = fns[f](&l, 0x1234u * (uint32_t)(f + 1), 20u + (uint32_t)f * 3u); + uint32_t x = 0; + for (uint32_t i = 0; i < l.len; i++) + x = x * 33u + l.items[i]; + printf("%d: err %u len %u cap %u flags %04x sum %08x items %08x\n", f, (unsigned)e, (unsigned)l.len, + (unsigned)l.cap, (unsigned)l.flags, (unsigned)l.sum, (unsigned)x); + total ^= x + l.sum; + } + printf("calls %u total %08x\n", (unsigned)calls, (unsigned)total); + return 0; +} diff --git a/tests/ir_tests/602_outline_shared_windows.expect b/tests/ir_tests/602_outline_shared_windows.expect new file mode 100644 index 000000000..9dd91aeee --- /dev/null +++ b/tests/ir_tests/602_outline_shared_windows.expect @@ -0,0 +1,9 @@ +0: err 0 len 20 cap 31 flags 00ff sum 290e7d14 items 061f0b60 +1: err 0 len 23 cap 31 flags 00ff sum b715971d items 27e15c6f +2: err 0 len 26 cap 31 flags 00ff sum c8f42b1c items be687946 +3: err 0 len 29 cap 31 flags 00ff sum 750a2329 items b503b941 +4: err 7 len 31 cap 31 flags 00ff sum d216b024 items de31e060 +5: err 7 len 31 cap 31 flags 00ff sum 9454ac25 items 677bbbb3 +6: err 7 len 31 cap 31 flags 00ff sum 5e9ee826 items 323d3906 +7: err 7 len 31 cap 31 flags 00ff sum 5d152427 items 2db55659 +calls 36 total 0d04af00 diff --git a/tests/ir_tests/603_inline_outparam_record_split.c b/tests/ir_tests/603_inline_outparam_record_split.c new file mode 100644 index 000000000..be6b10db9 --- /dev/null +++ b/tests/ir_tests/603_inline_outparam_record_split.c @@ -0,0 +1,113 @@ +/* awk's record reader (toybox getr/rx_find_rs), reduced: a static helper + * writes the separator's offsets through pointers to the caller's locals and + * returns 0; at -O1 and above the caller saw "not found" and returned the + * whole buffer as one record. */ +#include +#include +#include + +typedef long regoff_t; +typedef struct { int dummy; } fake_rx; +typedef struct { regoff_t rm_so, rm_eo; } fake_match; +#define REG_NOMATCH 1 + +static int fake_regexec(fake_rx *rx, const char *s, fake_match *m) +{ + (void)rx; + const char *p = strchr(s, '\n'); + if (!p) return REG_NOMATCH; + m->rm_so = p - s; + m->rm_eo = m->rm_so + 1; + return 0; +} + +struct zfile { const char *src; long srclen, srcpos; char *buf; long buflen, ro, lim; int eof, is_tty; }; +static char *recptr; + +static long zread(struct zfile *z, char *dst, long m) +{ + long n = z->srclen - z->srcpos; + if (n > m) n = m; + memcpy(dst, z->src + z->srcpos, n); + z->srcpos += n; + return n; +} + +static int rx_find_rs(fake_rx *rx, char *s, long len, regoff_t *start, regoff_t *end, int one_byte_rs) +{ + fake_match matches[1]; + if (one_byte_rs) { + char *p = memchr(s, one_byte_rs, len); + if (!p) return REG_NOMATCH; + *start = p - s; + *end = *start + 1; + } else { + int r = fake_regexec(rx, s, matches); + if (r == REG_NOMATCH) return r; + if (r) exit(9); + *start = matches[0].rm_so; + *end = matches[0].rm_eo; + } + return 0; +} + +static long getr(struct zfile *zfp, int rs_mode, const char *rs) +{ + fake_rx rsrx; + long ret = -1; + int r = -REG_NOMATCH; + regoff_t so = 0, eo = 0; + long m = 0, n = 0; + + rs_mode = strlen(rs) == 1 ? rs[0] : 0; + for (;;) { + if (zfp->ro == zfp->lim && zfp->eof) break; + if (zfp->ro == 0 && zfp->lim == zfp->buflen) + zfp->buf = realloc(zfp->buf, (zfp->buflen = zfp->buflen * 2 > 512 ? zfp->buflen * 2 : 512) + 1); + if ((m = zfp->buflen - zfp->lim) && !zfp->eof) { + if (zfp->is_tty) m = 1; + n = zread(zfp, zfp->buf + zfp->lim, m); + if (n < m) { + zfp->eof = 1; + if (!n && r == -REG_NOMATCH) break; + } + zfp->lim += n; + zfp->buf[zfp->lim] = 0; + } + recptr = zfp->buf + zfp->ro; + r = rx_find_rs(&rsrx, recptr, zfp->lim - zfp->ro, &so, &eo, rs_mode); + if (!r && so == eo) r = 1; + if (!zfp->eof && (r || (zfp->lim - (zfp->ro + eo)) < zfp->buflen / 4) && !zfp->is_tty) { + memmove(zfp->buf, recptr, zfp->lim - zfp->ro); + zfp->lim -= zfp->ro; + zfp->ro = 0; + continue; + } + ret = so; + if (zfp->eof) { + if (r) { + ret = zfp->lim - zfp->ro; + zfp->ro = zfp->lim; + } else zfp->ro += eo; + } else zfp->ro += eo; + if (!r || !zfp->is_tty) { + if (zfp->is_tty) zfp->ro = zfp->lim = 0; + break; + } + } + return ret; +} + +int main(void) +{ + static const char text[] = "1 a\n2 b\n3 c\n"; + struct zfile z = {0}; + long k; + int nr = 0; + z.src = text; + z.srclen = sizeof(text) - 1; + while ((k = getr(&z, 0, "\n")) >= 0) + printf("rec %d len %ld [%.*s]\n", ++nr, k, (int)k, recptr); + printf("NR %d\n", nr); + return 0; +} diff --git a/tests/ir_tests/603_inline_outparam_record_split.expect b/tests/ir_tests/603_inline_outparam_record_split.expect new file mode 100644 index 000000000..aa9043469 --- /dev/null +++ b/tests/ir_tests/603_inline_outparam_record_split.expect @@ -0,0 +1,4 @@ +rec 1 len 3 [1 a] +rec 2 len 3 [2 b] +rec 3 len 3 [3 c] +NR 3 diff --git a/tests/ir_tests/604_toysh_wildcard_literal.c b/tests/ir_tests/604_toysh_wildcard_literal.c new file mode 100644 index 000000000..c9e088b66 --- /dev/null +++ b/tests/ir_tests/604_toysh_wildcard_literal.c @@ -0,0 +1,163 @@ +// toysh's wildcard_matchlen (toybox toys/pending/sh.c), matching a case +// pattern against a word. At -O1 and above the device compiler returned +// failure for any literal character ("e" against "e", "a*" against "ab"). +#include +#include +#include +#include + +struct sh_arg { + char **v; + int c; +}; + +#define WILD_SHORT 1 +#define WILD_CASE 2 +#define WILD_ANY 4 + +static void arg_add(struct sh_arg *arg, char *data) +{ + if (!(arg->c&31)) arg->v = realloc(arg->v, sizeof(char *)*(arg->c+33)); + arg->v[arg->c++] = data; + arg->v[arg->c] = 0; +} + +__attribute__((noinline)) static int utf8towc(unsigned *wc, char *str, unsigned len) +{ + if (!len || !*str) return 0; + *wc = (unsigned char)*str; + return 1; +} + +static int getutf8(char *s, int len, int *cc) +{ + unsigned wc; + + if (len<0) wc = len = 0; + else if (1>(len = utf8towc(&wc, s, len))) wc = *s, len = 1; + if (cc) *cc = wc; + + return len; +} + +static int wildcard_matchlen(char *str, int len, char *pattern, int plen, + struct sh_arg *deck, int flags) +{ + struct sh_arg ant = {0}; // stack: of str offsets + long ss, pp, dd, best = -1; + int i, j, k, c, not, hit; + + // Loop through wildcards in pattern. + for (ss = pp = dd = 0; ;) { + if ((flags&WILD_ANY) && best!=-1) break; + + // did we consume pattern? + if (pp==plen) { + if (ss>best) best = ss; + if (ss==len || (flags&WILD_SHORT)) break; + // attempt literal match? + } else if (dd>=deck->c || pp!=(long)deck->v[dd]) { + if (ssv[dd-1] || pattern[pp-2]!='*') { + arg_add(&ant, (void *)ss); + arg_add(&ant, 0); + } + + continue; + } else if (c == '[') { + // next char of str in the set (or not in it, after [! or [^) + pp += (not = !!strchr("!^", pattern[pp])); + ss += (k = getutf8(str+ss, len-ss, &c)); + if (flags&WILD_CASE) c = towupper(c); + for (hit = 0; pp<(long)deck->v[dd];) { + pp += getutf8(pattern+pp, plen-pp, &i); + j = i; + if (pattern[pp]=='-' && pp+1<(long)deck->v[dd]) + pp += 1+getutf8(pattern+pp+1, plen-pp-1, &j); + if (flags&WILD_CASE) i = towupper(i), j = towupper(j); + hit |= i<=c && c<=j; + } + if (k && hit!=not) { + pp = 1+(long)deck->v[dd++]; + + continue; + } + + // ( preceded by +@!*? + + } else { // TODO ( ) | + dd++; + continue; + } + } + + // match failure + if (flags&WILD_ANY) { + ss = 0; + if (plen==pp) break; + continue; + } + + // pop retry stack or return failure (TODO: seek to next | in paren) + while (ant.c) { + if ((c = pattern[(long)deck->v[--dd]])=='*') { + if (len<(ss = (long)ant.v[ant.c-2]+(long)++ant.v[ant.c-1])) ant.c -= 2; + else { + pp = (long)deck->v[dd++]+1; + break; + } + } else if (c == '(') dprintf(2, "TODO: ("); + } + + if (!ant.c) break; + } + free (ant.v); + + return best; +} + +static int wildcard_match(char *s, char *p, struct sh_arg *deck, int flags) +{ + return wildcard_matchlen(s, strlen(s), p, strlen(p), deck, flags); +} + +static void check(char *s, char *p, long *wild, int nwild) +{ + struct sh_arg deck = {(char **)wild, nwild}; + + printf("%s ~ %s: %d\n", s, p, wildcard_match(s, p, &deck, 0)); +} + +int main(void) +{ + long star[] = {1}, q[] = {0}, br[] = {0, 2}, st2[] = {0, 3}; + + check("e", "e", 0, 0); + check("ab", "a*", star, 1); + check("e", "?", q, 1); + check("e", "[e]", br, 2); + check("b", "[!a]", st2, 2); + check("a", "[!a]", st2, 2); + check("abc", "abd", 0, 0); + + return 0; +} diff --git a/tests/ir_tests/604_toysh_wildcard_literal.expect b/tests/ir_tests/604_toysh_wildcard_literal.expect new file mode 100644 index 000000000..31db0e920 --- /dev/null +++ b/tests/ir_tests/604_toysh_wildcard_literal.expect @@ -0,0 +1,7 @@ +e ~ e: 1 +ab ~ a*: 2 +e ~ ?: 1 +e ~ [e]: 1 +b ~ [!a]: 1 +a ~ [!a]: -1 +abc ~ abd: -1 diff --git a/tests/ir_tests/605_strchr_nul_terminator.c b/tests/ir_tests/605_strchr_nul_terminator.c new file mode 100644 index 000000000..a93bd94bb --- /dev/null +++ b/tests/ir_tests/605_strchr_nul_terminator.c @@ -0,0 +1,11 @@ +#include +#include +__attribute__((noinline)) static int hit(int c) { return !!strchr("|)", c); } +__attribute__((noinline)) static int hit2(char *s) { return !!strchr("{(", *s); } +int main(void) +{ + volatile char z = 0, a = 'a', p = ')'; + char buf[2] = {0, 0}; + printf("%d %d %d %d %d\n", hit(z), hit(a), hit(p), hit2(buf), !!strchr("xy", z)); + return 0; +} diff --git a/tests/ir_tests/605_strchr_nul_terminator.expect b/tests/ir_tests/605_strchr_nul_terminator.expect new file mode 100644 index 000000000..dbf33d449 --- /dev/null +++ b/tests/ir_tests/605_strchr_nul_terminator.expect @@ -0,0 +1 @@ +1 0 1 1 1 diff --git a/tests/ir_tests/606_param_ne_local_address.c b/tests/ir_tests/606_param_ne_local_address.c new file mode 100644 index 000000000..631f751a4 --- /dev/null +++ b/tests/ir_tests/606_param_ne_local_address.c @@ -0,0 +1,56 @@ +// toysh's expand_arg_nobrace() takes an optional output deck as its fifth +// (stack-passed) parameter and falls back to a local one: "if (!ant) ant = +// &deck;" ... "if (ant != &deck && ant->v) finish(ant);". With a caller's +// deck passed in, the comparison must be true. +#include +#include + +struct deck { + long *v; + int c; +}; + +__attribute__((noinline)) static void add(struct deck *d, long x) +{ + d->v = realloc(d->v, sizeof(long)*(d->c+1)); + d->v[d->c++] = x; +} + +__attribute__((noinline)) static void finish(struct deck *d) +{ + d->c--; +} + +__attribute__((noinline)) static int expand(struct deck *out, const char *s, + unsigned flags, void **del, struct deck *ant, long *measure) +{ + struct deck deck = {0}; + int n = 0; + + if (!ant) ant = &deck; + for (; *s; s++) { + if (*s == '*' && !(flags&2)) add(ant, n); + n++; + } + add(ant, 99); + if (out) out->c = n; + if (del) *del = 0; + free(deck.v); + if (ant != &deck && ant->v) finish(ant); + if (measure) *measure = n; + + return n; +} + +int main(void) +{ + struct deck d = {0}, o = {0}; + void *del = &o; + long m = 0; + int n = expand(&o, "a*b", 0, &del, &d, &m); + + printf("n=%d c=%d m=%ld o=%d\n", n, d.c, m, o.c); + printf("local=%d\n", expand(0, "**", 0, 0, 0, 0)); + + return 0; +} diff --git a/tests/ir_tests/606_param_ne_local_address.expect b/tests/ir_tests/606_param_ne_local_address.expect new file mode 100644 index 000000000..0f9a6ec28 --- /dev/null +++ b/tests/ir_tests/606_param_ne_local_address.expect @@ -0,0 +1,2 @@ +n=3 c=1 m=3 o=3 +local=2 diff --git a/tests/ir_tests/607_assign_not_of_call_writing_lhs.c b/tests/ir_tests/607_assign_not_of_call_writing_lhs.c new file mode 100644 index 000000000..d8ec03791 --- /dev/null +++ b/tests/ir_tests/607_assign_not_of_call_writing_lhs.c @@ -0,0 +1,55 @@ +// toybox awk's logical NOT is "STKP->num = ! get_set_logical();", where the +// callee also rewrites STKP->num (and its flags). The store of the negated +// result has to land last. On the device `!0` came out 0 and every awk loop +// body was skipped. +#include + +struct zvalue { + unsigned flags; + double num; + void *p; +}; + +static struct zvalue stack[4]; +static struct zvalue *stackp = stack; + +__attribute__((noinline)) static int get_set_logical(void) +{ + struct zvalue *v = stackp; + int r = !!v->num; + + v->num = r; + v->flags = 1; + return r; +} + +static int get_set_logical_inl(void) +{ + struct zvalue *v = stackp; + int r = !!v->num; + + v->num = r; + v->flags = 1; + return r; +} + +int main(void) +{ + stackp->num = 0; + stackp->num = ! get_set_logical(); + printf("not0=%d\n", (int)stackp->num); + stackp->num = 5; + stackp->num = ! get_set_logical(); + printf("not5=%d\n", (int)stackp->num); + stackp->num = 0; + stackp->num = ! get_set_logical_inl(); + printf("inl not0=%d\n", (int)stackp->num); + stackp->num = 2; + stackp->num = ! get_set_logical_inl(); + printf("inl not2=%d\n", (int)stackp->num); + stackp->num = 0; + (stackp)->num = ! get_set_logical_inl(); + int t = get_set_logical_inl(); + printf("while-shape t=%d\n", t); + return 0; +} diff --git a/tests/ir_tests/607_assign_not_of_call_writing_lhs.expect b/tests/ir_tests/607_assign_not_of_call_writing_lhs.expect new file mode 100644 index 000000000..4d01238d7 --- /dev/null +++ b/tests/ir_tests/607_assign_not_of_call_writing_lhs.expect @@ -0,0 +1,5 @@ +not0=1 +not5=0 +inl not0=1 +inl not2=0 +while-shape t=1 diff --git a/tests/ir_tests/abi_mix.h b/tests/ir_tests/abi_mix.h new file mode 100644 index 000000000..ffcc9be1d --- /dev/null +++ b/tests/ir_tests/abi_mix.h @@ -0,0 +1,140 @@ +/* AAPCS32 struct argument passing between tcc and gcc code. abi_mix_a.c and + * abi_mix_b.c each define the callees below and a driver calling the other + * file's callees; test_abi_gcc_interop builds one with tcc and the other with + * arm-none-eabi-gcc, so every call crosses compilers in both directions. + * Composites of any size go by value: in the free core registers r0-r3, then + * the stack, split between the two when they straddle r3 (AAPCS32 C.5); an + * 8-aligned one starts at an even register. No padding anywhere, so the + * byte checksums are well defined. */ +#include +#include +#include + +typedef struct { unsigned char c[17]; } S17; +typedef struct { int32_t a[5]; } S20; +typedef struct { int32_t a[6]; } S24; +typedef struct { int64_t a, b, c; } S24L; /* 8-aligned */ +typedef struct { int32_t a[8]; } S32; +typedef struct { int32_t a[9]; } S36; +typedef struct { int16_t s; unsigned char c[15]; int8_t t[3]; } S20H; /* 2-aligned */ +typedef struct { int32_t a[40]; } S160; /* copied by memcpy on the tcc side */ +/* 8-aligned only through a member's own alignment: gcc's doubleword test + * reads each field's DECL_ALIGN, which counts _Alignas on the member -- also on + * a packed member, whose type alone has alignment 1 (Zig's C backend emits + * `zig_align(8) zig_packed(union {...}) payload`). */ +typedef struct { _Alignas(8) int32_t a; int32_t b, c, d; } SA8; +typedef struct { _Alignas(8) union { uint64_t u; int32_t w[4]; } __attribute__((packed)) p; } SP8; + +static uint32_t mix_ck(const void *p, unsigned n) +{ + const unsigned char *b = (const unsigned char *)p; + uint32_t h = 2166136261u; + for (unsigned i = 0; i < n; i++) + h = (h ^ b[i]) * 16777619u; + return h; +} +#define CK(x) mix_ck(&(x), sizeof(x)) + +static void mix_fill(void *p, unsigned n, unsigned seed) +{ + unsigned char *b = (unsigned char *)p; + for (unsigned i = 0; i < n; i++) + b[i] = (unsigned char)(seed * 37u + i * 11u + 5u); +} + +#define DECLARE_CALLEES(P) \ + uint32_t P##_f1(S17 s); \ + uint32_t P##_f2(int x, S20 s); \ + uint32_t P##_f3(int x, int y, int z, S24L s); \ + uint32_t P##_f4(int a, int b, int c, int d, S20 s, int e); \ + uint32_t P##_f5(S24 s, S17 t, int u); \ + uint32_t P##_f6(int x, S24L s, S36 t); \ + uint32_t P##_f7(S32 s, S32 t); \ + uint32_t P##_f8(S20H s, int x); \ + uint32_t P##_f9(int x, int y, S24L s, int z); \ + uint32_t P##_f10(int x, S160 s, int y); \ + uint32_t P##_f11(int x, SA8 s, int y); \ + uint32_t P##_f12(int x, SP8 s, int y); \ + S24 P##_r1(int k, S20 s); \ + uint32_t P##_v1(int n, ...); \ + uint32_t P##_m1(S32 s); + +#define DEFINE_CALLEES(P) \ + uint32_t P##_f1(S17 s) { return CK(s); } \ + uint32_t P##_f2(int x, S20 s) { return (uint32_t)x * 31u + CK(s); } \ + uint32_t P##_f3(int x, int y, int z, S24L s) { return (uint32_t)(x + y * 7 + z * 13) + CK(s); } \ + uint32_t P##_f4(int a, int b, int c, int d, S20 s, int e) { return (uint32_t)(a + b * 3 + c * 5 + d * 7 + e * 9) + CK(s); } \ + uint32_t P##_f5(S24 s, S17 t, int u) { return CK(s) ^ (CK(t) * 3u) ^ (uint32_t)u; } \ + uint32_t P##_f6(int x, S24L s, S36 t) { return (uint32_t)x + CK(s) * 5u + CK(t); } \ + uint32_t P##_f7(S32 s, S32 t) { return CK(s) + 2u * CK(t); } \ + uint32_t P##_f8(S20H s, int x) { return CK(s) + (uint32_t)x; } \ + uint32_t P##_f9(int x, int y, S24L s, int z) { return (uint32_t)(x * 3 + y * 5 + z * 7) + CK(s); } \ + uint32_t P##_f10(int x, S160 s, int y) { return (uint32_t)(x * 11 + y * 13) + CK(s); } \ + uint32_t P##_f11(int x, SA8 s, int y) { return (uint32_t)(x * 17 + y * 19) + CK(s); } \ + uint32_t P##_f12(int x, SP8 s, int y) { return (uint32_t)(x * 23 + y * 29) + CK(s); } \ + S24 P##_r1(int k, S20 s) \ + { \ + S24 r; \ + for (int i = 0; i < 6; i++) \ + r.a[i] = k * i + s.a[i % 5]; \ + return r; \ + } \ + uint32_t P##_v1(int n, ...) \ + { \ + va_list ap; \ + va_start(ap, n); \ + uint32_t h = (uint32_t)n; \ + for (int i = 0; i < n; i++) \ + { \ + S24 s = va_arg(ap, S24); \ + h = h * 33u + CK(s); \ + h ^= (uint32_t)va_arg(ap, int); \ + } \ + va_end(ap); \ + return h; \ + } \ + /* A callee writing its by-value parameter must not touch the caller's. */ \ + uint32_t P##_m1(S32 s) \ + { \ + s.a[0] = 999; \ + s.a[7] ^= 0x55; \ + return CK(s); \ + } + +#define DEFINE_DRIVER(P, O) \ + void run_##P(void) \ + { \ + S17 s17; \ + S20 s20; \ + S24 s24; \ + S24L s24l; \ + S32 s32, t32; \ + S36 s36; \ + S20H s20h; \ + S160 s160; \ + SA8 sa8; \ + SP8 sp8; \ + mix_fill(&s160, sizeof s160, 9); \ + mix_fill(&sa8, sizeof sa8, 10); \ + mix_fill(&sp8, sizeof sp8, 11); \ + mix_fill(&s17, sizeof s17, 1); \ + mix_fill(&s20, sizeof s20, 2); \ + mix_fill(&s24, sizeof s24, 3); \ + mix_fill(&s24l, sizeof s24l, 4); \ + mix_fill(&s32, sizeof s32, 5); \ + mix_fill(&t32, sizeof t32, 6); \ + mix_fill(&s36, sizeof s36, 7); \ + mix_fill(&s20h, sizeof s20h, 8); \ + printf(#P " f: %u %u %u %u %u\n", (unsigned)O##_f1(s17), (unsigned)O##_f2(7, s20), (unsigned)O##_f3(1, 2, 3, s24l), \ + (unsigned)O##_f4(1, 2, 3, 4, s20, 5), (unsigned)O##_f5(s24, s17, 9)); \ + printf(#P " g: %u %u %u %u\n", (unsigned)O##_f6(3, s24l, s36), (unsigned)O##_f7(s32, t32), \ + (unsigned)O##_f8(s20h, 11), (unsigned)O##_f9(4, 5, s24l, 6)); \ + S24 r = O##_r1(5, s20); \ + printf(#P " r1: %u\n", (unsigned)CK(r)); \ + printf(#P " f10: %u\n", (unsigned)O##_f10(3, s160, 4)); \ + printf(#P " a8: %u %u\n", (unsigned)O##_f11(3, sa8, 4), (unsigned)O##_f12(5, sp8, 6)); \ + printf(#P " v1: %u\n", (unsigned)O##_v1(2, s24, 7, s24, 8)); \ + uint32_t before = CK(s32); \ + uint32_t m = O##_m1(s32); \ + printf(#P " m1: %u %d\n", (unsigned)m, CK(s32) == before); \ + } diff --git a/tests/ir_tests/abi_mix_a.c b/tests/ir_tests/abi_mix_a.c new file mode 100644 index 000000000..ee15de5d8 --- /dev/null +++ b/tests/ir_tests/abi_mix_a.c @@ -0,0 +1,5 @@ +#include "abi_mix.h" + +DECLARE_CALLEES(b) +DEFINE_CALLEES(a) +DEFINE_DRIVER(a, b) diff --git a/tests/ir_tests/abi_mix_b.c b/tests/ir_tests/abi_mix_b.c new file mode 100644 index 000000000..aeb6b1624 --- /dev/null +++ b/tests/ir_tests/abi_mix_b.c @@ -0,0 +1,5 @@ +#include "abi_mix.h" + +DECLARE_CALLEES(a) +DEFINE_CALLEES(b) +DEFINE_DRIVER(b, a) diff --git a/tests/ir_tests/abi_mix_main.c b/tests/ir_tests/abi_mix_main.c new file mode 100644 index 000000000..bf19ca0c0 --- /dev/null +++ b/tests/ir_tests/abi_mix_main.c @@ -0,0 +1,10 @@ +/* Driver for abi_mix_a.c / abi_mix_b.c (see abi_mix.h). */ +void run_a(void); +void run_b(void); + +int main(void) +{ + run_a(); + run_b(); + return 0; +} diff --git a/tests/ir_tests/abi_mix_main.expect b/tests/ir_tests/abi_mix_main.expect new file mode 100644 index 000000000..3ad97fd70 --- /dev/null +++ b/tests/ir_tests/abi_mix_main.expect @@ -0,0 +1,14 @@ +a f: 644362173 519956218 639666075 519956096 936906187 +a g: 3269757705 4029519343 3419315040 639666100 +a r1: 3256326057 +a f10: 958096826 +a a8: 3968204900 3267456582 +a v1: 634481709 +a m1: 4027941418 1 +b f: 644362173 519956218 639666075 519956096 936906187 +b g: 3269757705 4029519343 3419315040 639666100 +b r1: 3256326057 +b f10: 958096826 +b a8: 3968204900 3267456582 +b v1: 634481709 +b m1: 4027941418 1 diff --git a/tests/ir_tests/asm/branch_over_pool_window.c b/tests/ir_tests/asm/branch_over_pool_window.c new file mode 100644 index 000000000..f2569144d --- /dev/null +++ b/tests/ir_tests/asm/branch_over_pool_window.c @@ -0,0 +1,188 @@ +/* A forward jump over more code than one literal-pool window (~950 bytes) + * with a few pool entries in between. The old rule kept every such branch + * wide in case a pool dump landed inside the range; this one is short enough + * for a 16-bit B even with every dump the range can take + * (test_branch_over_pool_window_narrows). */ +extern void sink(unsigned); +extern void other(int); +void over_pool_window(int c) +{ + if (c) + { + other(c); + } + else + { + sink(305419896u); + sink(1u); + sink(2u); + sink(3u); + sink(4u); + sink(5u); + sink(6u); + sink(7u); + sink(8u); + sink(9u); + sink(10u); + sink(11u); + sink(12u); + sink(13u); + sink(14u); + sink(15u); + sink(16u); + sink(17u); + sink(18u); + sink(19u); + sink(20u); + sink(21u); + sink(22u); + sink(23u); + sink(24u); + sink(25u); + sink(26u); + sink(27u); + sink(28u); + sink(29u); + sink(30u); + sink(31u); + sink(32u); + sink(33u); + sink(34u); + sink(35u); + sink(36u); + sink(37u); + sink(38u); + sink(39u); + sink(305419936u); + sink(41u); + sink(42u); + sink(43u); + sink(44u); + sink(45u); + sink(46u); + sink(47u); + sink(48u); + sink(49u); + sink(50u); + sink(51u); + sink(52u); + sink(53u); + sink(54u); + sink(55u); + sink(56u); + sink(57u); + sink(58u); + sink(59u); + sink(60u); + sink(61u); + sink(62u); + sink(63u); + sink(64u); + sink(65u); + sink(66u); + sink(67u); + sink(68u); + sink(69u); + sink(70u); + sink(71u); + sink(72u); + sink(73u); + sink(74u); + sink(75u); + sink(76u); + sink(77u); + sink(78u); + sink(79u); + sink(305419976u); + sink(81u); + sink(82u); + sink(83u); + sink(84u); + sink(85u); + sink(86u); + sink(87u); + sink(88u); + sink(89u); + sink(90u); + sink(91u); + sink(92u); + sink(93u); + sink(94u); + sink(95u); + sink(96u); + sink(97u); + sink(98u); + sink(99u); + sink(100u); + sink(101u); + sink(102u); + sink(103u); + sink(104u); + sink(105u); + sink(106u); + sink(107u); + sink(108u); + sink(109u); + sink(110u); + sink(111u); + sink(112u); + sink(113u); + sink(114u); + sink(115u); + sink(116u); + sink(117u); + sink(118u); + sink(119u); + sink(305420016u); + sink(121u); + sink(122u); + sink(123u); + sink(124u); + sink(125u); + sink(126u); + sink(127u); + sink(128u); + sink(129u); + sink(130u); + sink(131u); + sink(132u); + sink(133u); + sink(134u); + sink(135u); + sink(136u); + sink(137u); + sink(138u); + sink(139u); + sink(140u); + sink(141u); + sink(142u); + sink(143u); + sink(144u); + sink(145u); + sink(146u); + sink(147u); + sink(148u); + sink(149u); + sink(150u); + sink(151u); + sink(152u); + sink(153u); + sink(154u); + sink(155u); + sink(156u); + sink(157u); + sink(158u); + sink(159u); + sink(305420056u); + sink(161u); + sink(162u); + sink(163u); + sink(164u); + sink(165u); + sink(166u); + sink(167u); + sink(168u); + sink(169u); + } + other(-1); +} diff --git a/tests/ir_tests/asm/cmp_neg_small.c b/tests/ir_tests/asm/cmp_neg_small.c new file mode 100644 index 000000000..b02650b95 --- /dev/null +++ b/tests/ir_tests/asm/cmp_neg_small.c @@ -0,0 +1,14 @@ +/* CMP Rn, #-k with k = 1..7 is ADDS Rt, Rn, #k into a free low register: + * the same N, Z, C and V in 16 bits (test_cmp_negative_small_is_adds). */ +extern int hit(void); +extern int miss(void); + +int is_none(unsigned x) +{ + return x == 0xFFFFFFFFu ? hit() : miss(); +} + +int below_minus3(int x) +{ + return x < -3 ? hit() : miss(); +} diff --git a/tests/ir_tests/asm/cmp_type_extreme.c b/tests/ir_tests/asm/cmp_type_extreme.c new file mode 100644 index 000000000..f84c29fd0 --- /dev/null +++ b/tests/ir_tests/asm/cmp_type_extreme.c @@ -0,0 +1,12 @@ +/* A compare against an end of its operand's own range is constant + * (test_cmp_type_extreme_folds): VRP folds it written directly and after + * inlining turns a parameter into the extreme. */ +#include + +int u_above_max(uint32_t x) { return x > 0xFFFFFFFFu; } +int s_below_min(int32_t x) { return x < INT32_MIN; } + +static inline int above(uint32_t x, uint32_t m) { return x > m; } +static inline int below(int32_t x, int32_t m) { return x < m; } +extern int use(int); +int inlined(uint32_t a, int32_t b) { return use(above(a, 0xFFFFFFFFu)) + use(below(b, INT32_MIN)); } diff --git a/tests/ir_tests/asm/func_align_literal.c b/tests/ir_tests/asm/func_align_literal.c new file mode 100644 index 000000000..33d62a12a --- /dev/null +++ b/tests/ir_tests/asm/func_align_literal.c @@ -0,0 +1,27 @@ +/* A body that reads its literal pool starts word-aligned, so two identical + * ones compile to the same machine code and fold (test_literal_body_word_aligned). */ +extern void sink(unsigned); + +/* An odd number of halfwords, so what follows would start at 2 mod 4. */ +void pad_before(void) +{ + sink(1); +} + +static __attribute__((noinline)) void lit_a(int x) +{ + sink(0x12345678u + x); + sink(0x9abcdef0u); +} + +static __attribute__((noinline)) void lit_b(int x) +{ + sink(0x12345678u + x); + sink(0x9abcdef0u); +} + +void use_both(int x) +{ + lit_a(x); + lit_b(x + 1); +} diff --git a/tests/ir_tests/asm/has_builtin.c b/tests/ir_tests/asm/has_builtin.c new file mode 100644 index 000000000..30be8c3ec --- /dev/null +++ b/tests/ir_tests/asm/has_builtin.c @@ -0,0 +1,36 @@ +/* __has_builtin answers for what tcc implements, in #if and in running text + * (test_has_builtin): zig.h picks REV16 for its byte swap, and pastes the + * answer into a name to choose between a builtin and a libm import. */ +#define zig_has_builtin(b) __has_builtin(__builtin_##b) +#define CAT2(a, b) a##b +#define CAT(a, b) CAT2(a, b) +#define pick_0 0 +#define pick_1 1 + +unsigned short swap16(unsigned short x) +{ +#if zig_has_builtin(bswap16) + return __builtin_bswap16(x); +#else + return (unsigned short)((x << 8) | (x >> 8)); +#endif +} + +unsigned swap32(unsigned x) +{ +#if zig_has_builtin(bswap32) + return __builtin_bswap32(x); +#else + return (x << 24) | ((x & 0xff00) << 8) | ((x >> 8) & 0xff00) | (x >> 24); +#endif +} + +int answers(void) +{ + /* bswap64 yes, an unknown builtin no, both through a pasted name. */ + return CAT(pick_, zig_has_builtin(bswap64)) * 10 + CAT(pick_, zig_has_builtin(no_such_builtin)); +} + +#if !defined(__has_builtin) +#error "__has_builtin is not defined" +#endif diff --git a/tests/ir_tests/asm/hi_callee_pref.c b/tests/ir_tests/asm/hi_callee_pref.c new file mode 100644 index 000000000..45f1f4399 --- /dev/null +++ b/tests/ir_tests/asm/hi_callee_pref.c @@ -0,0 +1,22 @@ +/* More call-crossing values than r4-r7: the ones only passed from call to + * call take r8-r11, the one compared with constants after every call keeps a + * low register, so its compares are 16-bit (test_hi_callee_pref). */ +extern int step(int); +extern void use(void *, void *, void *, void *, void *); + +int keep_low(void *a, void *b, void *c, void *d, void *e, int n) +{ + int k = step(n); + use(a, b, c, d, e); + if (k == 3) + return 1; + k = step(k); + use(a, b, c, d, e); + if (k == 5) + return 2; + k = step(k); + use(a, b, c, d, e); + if (k == 7) + return 3; + return k == 9; +} diff --git a/tests/ir_tests/asm/it_mov_narrow.c b/tests/ir_tests/asm/it_mov_narrow.c new file mode 100644 index 000000000..c3e750a22 --- /dev/null +++ b/tests/ir_tests/asm/it_mov_narrow.c @@ -0,0 +1,5 @@ +/* A 0/1 or small constant selected inside an IT block is the 16-bit MOV, + * which sets no flags there (test_it_mov_narrow). */ +int sel(int a, int b) { return a == b ? 5 : 9; } +int both(int a, int b) { return a && b; } +int either(int a, int b) { return a || b; } diff --git a/tests/ir_tests/asm/shift64_var.c b/tests/ir_tests/asm/shift64_var.c new file mode 100644 index 000000000..bd3bfb837 --- /dev/null +++ b/tests/ir_tests/asm/shift64_var.c @@ -0,0 +1,6 @@ +/* A 64-bit shift by a variable count: inline at -O2, the __aeabi_* helper + * call at -Os (test_shift64_var_os_calls_helper). */ +unsigned long long shl(unsigned long long x, int n) { return x << n; } +unsigned long long shr(unsigned long long x, int n) { return x >> n; } +long long sar(long long x, int n) { return x >> n; } +unsigned long long shl_const(unsigned long long x) { return x << 5; } diff --git a/tests/ir_tests/asm/size_tier.c b/tests/ir_tests/asm/size_tier.c new file mode 100644 index 000000000..3dde4a9cc --- /dev/null +++ b/tests/ir_tests/asm/size_tier.c @@ -0,0 +1,51 @@ +/* -Os / -Oz: the -O2 pipeline where a choice trades code size for speed + * taken the small way (test_size_tier in test_codegen_asm.py). */ + +int size_macro(void) +{ +#ifdef __OPTIMIZE_SIZE__ + return 1; +#else + return 0; +#endif +} + +/* 160 bytes: larger than the largest copy stub (32 words). */ +struct big +{ + int w[40]; +}; + +void copy_big(struct big *d, const struct big *s) +{ + *d = *s; +} + +/* Loops whose heads -O2 word-aligns with a NOP when they land at 2 mod 4. */ +int sum_pos(const int *p, int n) +{ + int s = 0; + for (int i = 0; i < n; i++) + if (p[i] > 3) + s += p[i]; + else + s -= 1; + return s; +} + +int count_eq(const short *p, int n, int k) +{ + int c = 0; + short x = (short)k; + for (int i = 0; i < n; i++) + c += p[i] == x; + return c; +} + +unsigned mix(const unsigned char *p, int n) +{ + unsigned h = 7; + while (n-- > 0) + h = h * 31 + *p++; + return h; +} diff --git a/tests/ir_tests/asm/volatile_access.c b/tests/ir_tests/asm/volatile_access.c index 0143b0014..1ec93c00f 100644 --- a/tests/ir_tests/asm/volatile_access.c +++ b/tests/ir_tests/asm/volatile_access.c @@ -252,6 +252,16 @@ int v64_1_0(void) { struct VW t = mgw; return t.a; } int v65_2_0(void) { struct VW x = mgw; struct VW y = mgw; return x.a + y.a; } int v66_1_0(void) { struct VW t = mgw; mdead = t.a; return 0; } +/* --- two volatile locals with the same initialiser ---------------------- + * The expression-equality helper and the constant evaluator followed a + * volatile VAR to its definition, so `(a < 0) != (b < 0)` compared two copies + * of the initialiser: cmp_expr_fold folded it to 0 and deleted b's read, and + * frame colouring then gave both objects one slot. */ +int v67_2_2(void) { volatile int a = 5, b = 5; int x = a; return (x < 0) != (b < 0); } +int v68_2_2(void) { volatile long long a = 5, b = 5; long long x = a; return (x < 0) != (b < 0); } +/* a volatile read copied into a local nobody reads is still a read */ +int v69_1_1(void) { volatile int a = 7; int t = a; (void)t; return 0; } + /* --- controls: the same shapes without volatile -------------------------- */ int c01_1_0(int *p) { return *p + *p; } int c02_1_0(int *p) { return p[1] + p[1]; } diff --git a/tests/ir_tests/bug_addr_temp_reuse_before_def.c b/tests/ir_tests/bug_addr_temp_reuse_before_def.c new file mode 100644 index 000000000..5be1e6d6b --- /dev/null +++ b/tests/ir_tests/bug_addr_temp_reuse_before_def.c @@ -0,0 +1,39 @@ +/* A frame address parked in a register must not be reused AHEAD of where it + * was parked. + * + * stackoff_indexed_base_cse rewrites an indexed access's STACKOFF base to a + * TEMP the prologue already loaded with that address, on the grounds that the + * prologue dominates the whole function. It does -- but only from the ASSIGN + * onwards. An access EARLIER in the same straight-line prologue read the temp + * before anything had written it, so at -O2 `arr9[i]` loaded through whatever + * the caller left in the register (with a poisoned one it faulted). + * + * The shape is what the two reads of `m212` build: the second reaches it as + * `*(&m212[a][0] + b)`, which the frontend materialises as an address TEMP, + * and the first is a plain indexed read of the same object earlier in the same + * branch-free run. Reduced from agg_deep fuzz seed 1637. */ +#include +static unsigned csmix(unsigned h, unsigned v) +{ + h ^= v + 0x9e3779b9u + (h << 6) + (h >> 2); + return h * 2654435761u; +} +struct N { unsigned a; unsigned b; }; +struct N2 { struct N n; unsigned t; }; +int main(void) +{ + unsigned cs = 0x12345678u; + char s4 = (char)(863460459u & 0xff); + unsigned u6 = 3478674980u; + unsigned u7 = 2606935024u; + unsigned arr8[8] = { 3266956212u, 2405417304u, 2824046361u, 1788383746u, 3555012821u, 714619954u, 639637460u, 2773089431u }; + unsigned arr9[8] = { 733865748u, 1153198824u, 3056370527u, 560033498u, 1853620413u, 121005395u, 1229005950u, 1711230968u }; + struct N2 n211 = { { 1457867030u, 3834342564u }, 44554686u }; + unsigned m212[4][4] = { { 1994247921u, 1515947663u, 704051171u, 1509225025u }, { 113208333u, 3075160707u, 3725319848u, 316329586u }, { 2695622949u, 1579111367u, 3875827252u, 2455408258u }, { 1504295279u, 697374028u, 3482915202u, 778016309u } }; + unsigned *pa213 = &u6; + unsigned **ppa214 = &pa213; + m212[((unsigned)(2820465294u) & 3u)][((unsigned)(u7) & 3u)] = (unsigned)(((unsigned)(arr9[((unsigned)(u6) & 7u)]) + (unsigned)((-((unsigned)(u7) | 0u))))); + cs = csmix(cs, *(&m212[((unsigned)(2820465294u) & 3u)][0] + ((unsigned)(u7) & 3u))); + cs = csmix(cs, (unsigned)(((unsigned)(((unsigned)((~((unsigned)(((unsigned)(n211.n.b) * (unsigned)(((unsigned)(n211.n.b) ^ cs)))) | 0u))) >> ((unsigned)((((unsigned)(((unsigned)(arr9[((unsigned)(u7) & 7u)]) + (unsigned)((unsigned)(s4)))) & 1u) ? (unsigned)(((unsigned)(u6) % ((unsigned)(arr8[((unsigned)(3839205814u) & 7u)]) | 1u))) : (unsigned)(((unsigned)(m212[((unsigned)(u6) & 3u)][((unsigned)(2229302978u) & 3u)]) & (unsigned)(m212[((unsigned)(u7) & 3u)][((unsigned)(u7) & 3u)]))))) & 31u))) % ((unsigned)(((unsigned)(2546737391u) + (unsigned)(1675086101u))) | 1u)))); + printf("checksum=%08x\n", cs); +} diff --git a/tests/ir_tests/bug_addr_temp_reuse_before_def.expect b/tests/ir_tests/bug_addr_temp_reuse_before_def.expect new file mode 100644 index 000000000..f28224c09 --- /dev/null +++ b/tests/ir_tests/bug_addr_temp_reuse_before_def.expect @@ -0,0 +1 @@ +checksum=64f4107c diff --git a/tests/ir_tests/bug_alive_share_evicted_owner.c b/tests/ir_tests/bug_alive_share_evicted_owner.c new file mode 100644 index 000000000..cc240d46a --- /dev/null +++ b/tests/ir_tests/bug_alive_share_evicted_owner.c @@ -0,0 +1,83 @@ +/* A register alive_share lent out is not free just because its OWNER spilled. + * + * ra:alive_share puts a second live holder on a register the allocator still + * shows as taken by the first -- that is the whole point of it. Eviction then + * picked the owner as its spill victim and handed the register to a new + * interval, but spilling the owner does NOT free the register: the borrower is + * still living there, and its value is destroyed between its definition and + * its use. The expire path states the invariant ("only free a register when + * no surviving interval still holds it"); eviction had no equivalent. + * + * Measured on this program at -O2: alive_share lends R12 to a temp over + * [105,122] while the merge temp owns it across [102,126]; eviction spills the + * owner and gives R12 away over [115,119]. + * + * It takes real register pressure and an if/else merge whose arms both define + * the same value, which is why this is a fuzz reduction (longlong seed 197) + * rather than something written by hand. Same bug: longlong 1449 and + * combo_num 1103. */ +#include +static unsigned csmix(unsigned h, unsigned v) +{ + h ^= v + 0x9e3779b9u + (h << 6) + (h >> 2); + h = (h << 13) | (h >> 19); + return h * 2654435761u; +} +struct S { + unsigned f0; + unsigned f1; + unsigned f2; +}; +int main(void) +{ + unsigned cs = 0x12345678u; + short s1 = (short)(90829728u & 0xffff); + unsigned u2 = 2504597986u; + unsigned u3 = 2081379782u; + unsigned u4 = 2681507826u; + unsigned long long q5 = (((unsigned long long)(u4)) << 32) | (unsigned long long)(u3); + unsigned long long q6 = (((unsigned long long)(u3)) << 32) | (unsigned long long)(u4); + unsigned long long q7 = (((unsigned long long)(u4)) << 32) | (unsigned long long)(u3); + unsigned long long q8 = (((unsigned long long)(u4)) << 32) | (unsigned long long)(u3); + struct S st9 = { 3078485621u, 1041743909u, 1029069764u }; + cs = csmix(cs, (unsigned)(q7) ^ (unsigned)(q7 >> 32)); + cs = csmix(cs, (unsigned)((unsigned)(s1))); + if ((unsigned)(st9.f1) & 1u) { + if ((unsigned)((((unsigned)(u2) & 1u) ? (unsigned)(u3) : (unsigned)(((unsigned)(((unsigned)((((unsigned)(3746058814u) & 1u) ? (unsigned)(3458201546u) : (unsigned)(u3))) ^ (unsigned)(1343907409u))) / ((unsigned)(st9.f1) | 1u))))) & 1u) { + cs = csmix(cs, (unsigned)(((unsigned)((unsigned)(s1)) << ((unsigned)(((unsigned)((unsigned)(s1)) < ((unsigned)(u4) ^ cs))) & 31u)))); + } else { + u2 = (unsigned)(((unsigned)(u2) + (unsigned)(u3))) & 0xffffffffu; + q7 = (((unsigned long long)(unsigned)(((unsigned)((~((unsigned)(u4) | 0u))) | (unsigned)(((unsigned)(u2) % ((unsigned)(u4) | 1u))))))) >> ((unsigned)(((unsigned)((unsigned)(s1)) & (unsigned)(2490539759u))) & 63u); + } + u4 = (unsigned)(((unsigned)(st9.f0) << ((unsigned)(u2) & 31u))) & 0xffffffffu; + if ((unsigned)(((unsigned)(((unsigned)(u2) - (unsigned)(((unsigned)(((unsigned)(1770957371u) & (unsigned)(u2))) - (unsigned)(st9.f1))))) >> ((unsigned)(((unsigned)(((unsigned)(((unsigned)(423991143u) / ((unsigned)(3712597894u) | 1u))) - (unsigned)(((unsigned)(2595007699u) >> ((unsigned)((unsigned)(s1)) & 31u))))) >> ((unsigned)(((unsigned)(((unsigned)(3369889190u) | (unsigned)(u3))) % ((unsigned)(((unsigned)(u4) | (unsigned)(((unsigned)(u4) ^ cs)))) | 1u))) & 31u))) & 31u))) & 1u) { + u2 = (unsigned)((-((unsigned)(((unsigned)(u4) ^ (unsigned)(((unsigned)((-((unsigned)(st9.f1) | 0u))) & (unsigned)(((unsigned)(st9.f2) - (unsigned)(3397749729u))))))) | 0u))) & 0xffffffffu; + cs = csmix(cs, (unsigned)(((unsigned)(((unsigned)((((unsigned)(((unsigned)(st9.f1) | (unsigned)(st9.f0))) & 1u) ? (unsigned)((unsigned)(s1)) : (unsigned)(((unsigned)((unsigned)(s1)) + (unsigned)(((unsigned)((unsigned)(s1)) ^ cs)))))) >> ((unsigned)(((unsigned)(((unsigned)(2073984964u) - (unsigned)(2749828640u))) ^ (unsigned)((-((unsigned)(u3) | 0u))))) & 31u))) == ((unsigned)(u4) ^ cs)))); + } + } else { + cs = csmix(cs, (unsigned)(((unsigned)(u3) % ((unsigned)(((unsigned)(((unsigned)(((unsigned)(2079564487u) + (unsigned)(u2))) % ((unsigned)((((unsigned)(st9.f0) & 1u) ? (unsigned)(u3) : (unsigned)(u4))) | 1u))) >> ((unsigned)((~((unsigned)(u3) | 0u))) & 31u))) | 1u)))); + for (unsigned g11 = 0u; g11 < 5u; g11++) { + unsigned i10 = g11; + } + for (unsigned g13 = 0u; g13 < 5u; g13++) { + unsigned i12 = g13; + cs = csmix(cs, i12); + cs = csmix(cs, (unsigned)(q8) ^ (unsigned)(q8 >> 32)); + cs = csmix(cs, (unsigned)((((unsigned)(3709886646u) & 1u) ? (unsigned)(((unsigned)(((unsigned)(i12) ^ (unsigned)(((unsigned)(i12) ^ cs)))) * (unsigned)((-((unsigned)(((unsigned)((unsigned)(s1)) - (unsigned)(1283018761u))) | 0u))))) : (unsigned)(((unsigned)(u2) < ((unsigned)(((unsigned)(st9.f0) | (unsigned)(((unsigned)(265134004u) & (unsigned)(2867800018u))))) ^ cs)))))); + cs = csmix(cs, ((q8) <= (((q8) ^ (unsigned long long)(cs)))) ? 1u : 0u); + cs = csmix(cs, (unsigned)(u2)); + } + } + cs = csmix(cs, ((q7) == (q5)) ? 1u : 0u); + cs = csmix(cs, u2); + cs = csmix(cs, u3); + cs = csmix(cs, u4); + cs = csmix(cs, (unsigned)(q5) ^ (unsigned)(q5 >> 32)); + cs = csmix(cs, (unsigned)(q6) ^ (unsigned)(q6 >> 32)); + cs = csmix(cs, (unsigned)(q7) ^ (unsigned)(q7 >> 32)); + cs = csmix(cs, (unsigned)(q8) ^ (unsigned)(q8 >> 32)); + cs = csmix(cs, (unsigned)s1); + cs = csmix(cs, st9.f0); + cs = csmix(cs, st9.f1); + printf("checksum=%08x\n", cs); +} diff --git a/tests/ir_tests/bug_alive_share_evicted_owner.expect b/tests/ir_tests/bug_alive_share_evicted_owner.expect new file mode 100644 index 000000000..97c99bf5f --- /dev/null +++ b/tests/ir_tests/bug_alive_share_evicted_owner.expect @@ -0,0 +1 @@ +checksum=fa8a37b7 diff --git a/tests/ir_tests/bug_asm_clobber_live_values.c b/tests/ir_tests/bug_asm_clobber_live_values.c new file mode 100644 index 000000000..d51756357 --- /dev/null +++ b/tests/ir_tests/bug_asm_clobber_live_values.c @@ -0,0 +1,37 @@ +/* Values live across an inline asm statement must survive it. + * + * The register allocator did not know what an asm statement destroys (its + * clobber list, the registers picked for its operands), and asm_gen_code only + * preserves callee-saved r4-r11. A value left in r0-r3/r12 across the asm was + * lost: `clobbered` returned 7 at -O0 and 0 at -O1/-O2. */ +#include + +typedef unsigned int u32; + +/* A value live across the asm while register variables claim r0 and r1. */ +__attribute__((noinline)) static u32 pinned(u32 a, u32 b) +{ + u32 keep = a * 3u + 1u; + register u32 out __asm("r0"); + register u32 in __asm("r1") = b; + __asm volatile("add %[o], %[i], #1" : [o] "=r"(out) : [i] "r"(in)); + return keep + out * 100u; +} + +/* Values live across an asm that clobbers every caller-saved register. */ +__attribute__((noinline)) static u32 clobbered(u32 a, u32 b, u32 c, u32 d) +{ + u32 x = a * 7u, y = b * 11u, z = c * 13u, w = d * 17u; + __asm volatile("movs r0, #0\n movs r1, #0\n movs r2, #0\n movs r3, #0\n mov r12, r0" + : + : + : "r0", "r1", "r2", "r3", "r12", "cc"); + return x + y + z + w; +} + +int main(void) +{ + printf("%u\n", pinned(5, 9)); /* 16 + 10 * 100 */ + printf("%u\n", clobbered(1, 2, 3, 4)); /* 7 + 22 + 39 + 68 */ + return 0; +} diff --git a/tests/ir_tests/bug_asm_clobber_live_values.expect b/tests/ir_tests/bug_asm_clobber_live_values.expect new file mode 100644 index 000000000..0ff8f6fd4 --- /dev/null +++ b/tests/ir_tests/bug_asm_clobber_live_values.expect @@ -0,0 +1,2 @@ +1016 +136 diff --git a/tests/ir_tests/bug_asm_regvar_local.c b/tests/ir_tests/bug_asm_regvar_local.c new file mode 100644 index 000000000..aab031513 --- /dev/null +++ b/tests/ir_tests/bug_asm_regvar_local.c @@ -0,0 +1,56 @@ +/* Local register variables (`register T x __asm("rN")`) feeding inline asm. + * + * Zig's C backend emits its Linux syscall wrappers in exactly this shape. The + * variable used to get no vreg, so its accesses went to a stack slot that was + * never allocated and aliased the previous local: at -O0 `tmp = &t0` + * overwrote t0.array[0], and at -O1 the constraint solver rejected the asm + * with "asm regvar requests register that's taken already". */ +#include + +typedef unsigned int u32; +struct pair +{ + u32 array[2]; +}; + +__attribute__((noinline)) static u32 load_through_pinned(u32 n) +{ + struct pair t0; + t0.array[0] = n; + t0.array[1] = 7; + register u32 ret __asm("r0"); + register struct pair *const tmp __asm("r1") = &t0; + __asm volatile("ldr %[ret], [%[tmp]]" : [ret] "=r"(ret) : [tmp] "r"(tmp) : "memory"); + return ret + t0.array[0] + t0.array[1]; +} + +/* The output and the first input name the same register, as in syscall3. */ +__attribute__((noinline)) static u32 output_shares_input_reg(u32 a1, u32 a2, u32 a3) +{ + register u32 ret __asm("r0"); + register u32 const x1 __asm("r0") = a1; + register u32 const x2 __asm("r1") = a2; + register u32 const x3 __asm("r2") = a3; + __asm volatile("add %[ret], %[a1], %[a2]\n add %[ret], %[ret], %[a3]" + : [ret] "=r"(ret) + : [a1] "r"(x1), [a2] "r"(x2), [a3] "r"(x3)); + return ret; +} + +/* The asm body names r0/r1 itself, as a syscall does: correct only if the + * operands really are in the registers the variables were pinned to. */ +__attribute__((noinline)) static u32 body_names_registers(u32 b) +{ + register u32 in __asm("r1") = b; + register u32 out __asm("r0"); + __asm volatile("adds r0, r1, #1" : "=r"(out) : "r"(in) : "cc"); + return out; +} + +int main(void) +{ + printf("%u\n", load_through_pinned(42)); /* 42 + 42 + 7 */ + printf("%u\n", output_shares_input_reg(1, 20, 300)); /* 321 */ + printf("%u\n", body_names_registers(41)); /* 42 */ + return 0; +} diff --git a/tests/ir_tests/bug_asm_regvar_local.expect b/tests/ir_tests/bug_asm_regvar_local.expect new file mode 100644 index 000000000..96ffa697d --- /dev/null +++ b/tests/ir_tests/bug_asm_regvar_local.expect @@ -0,0 +1,3 @@ +91 +321 +42 diff --git a/tests/ir_tests/bug_block_copy_init_narrow.c b/tests/ir_tests/bug_block_copy_init_narrow.c new file mode 100644 index 000000000..d3ba18bdb --- /dev/null +++ b/tests/ir_tests/bug_block_copy_init_narrow.c @@ -0,0 +1,49 @@ +/* A constant initializer materialized as a copy from .rodata + * (block_copy_init) keeps every store at its own width. + * + * The pass wrote every store below 64 bits as a 32-bit word, so a byte or + * halfword field initialized to a negative constant spilled its sign bits + * into the next fields, which the zero fill had cleared. */ +#include + +struct q +{ + signed char a, b, c, d; + int e; + short f, g; + unsigned char h[3], i; + int pad[20]; +}; + +__attribute__((noinline)) static void show(const struct q *s) +{ + printf("%d %d %d %d %d %d %d %d %d %d %d %d\n", s->a, s->b, s->c, s->d, s->e, s->f, s->g, s->h[0], s->h[1], + s->h[2], s->i, s->pad[19]); +} + +int g = 5; +struct w +{ + int a, b; + float c; + double d; + signed char e; + int pad[10]; +}; + +__attribute__((noinline)) static void show_w(const struct w *s) +{ + printf("%d %d %g %g %d %d\n", s->a, s->b, s->c, s->d, s->e, s->pad[9]); +} + +int main(void) +{ + /* A load from a global ends the constant run; floats keep their bits. */ + struct w u = {g, -1, 1.5f, -2.25, -7, {[9] = 3}}; + show_w(&u); + struct q s = {-1, 0, 0, 0, 5, -2, 0, {0xff, 0, 0}, 0, {[19] = 7}}; + show(&s); + struct q t = {0, -3, 0, -4, 6, 0, -5, {0, 0xfe, 0}, 0x80, {[0] = 9}}; + show(&t); + return 0; +} diff --git a/tests/ir_tests/bug_block_copy_init_narrow.expect b/tests/ir_tests/bug_block_copy_init_narrow.expect new file mode 100644 index 000000000..4e350a837 --- /dev/null +++ b/tests/ir_tests/bug_block_copy_init_narrow.expect @@ -0,0 +1,3 @@ +5 -1 1.5 -2.25 -7 3 +-1 0 0 0 5 -2 0 255 0 0 0 7 +0 -3 0 -4 6 0 -5 0 254 0 128 0 diff --git a/tests/ir_tests/bug_branch_narrow_rehearsal.c b/tests/ir_tests/bug_branch_narrow_rehearsal.c new file mode 100644 index 000000000..6ab3d03b3 --- /dev/null +++ b/tests/ir_tests/bug_branch_narrow_rehearsal.c @@ -0,0 +1,101 @@ +/* Branch narrowing and CBZ fusion trust the rehearsal pass's layout. + * + * The rehearsal is a dry run: it laid out a mid-function return's branch to + * the epilogue as zero bytes (only the real pass emitted it) and an inline asm + * body as zero bytes (dry runs never assemble it). A range holding either was + * longer than modelled, and a fused CBZ, which cannot be widened once + * committed, failed with "CBZ/CBNZ target out of range: offset=128" (a return, + * found compiling the Zig compiler's C) or offset=148 (an asm body). */ +#include + +volatile int g; + +/* A 2-byte body step walks the fused CBZ across its 126-byte limit; one of + * these lands where the return's branch pushed the target to 128. */ +#define ADD1 y += x; +#define ADD2 ADD1 ADD1 +#define ADD4 ADD2 ADD2 +#define ADD8 ADD4 ADD4 +#define ADD16 ADD8 ADD8 +#define ADD32 ADD16 ADD16 +#define EARLY_RETURN(k, body) \ + int adds_##k(int x, int y) \ + { \ + if (x) \ + { \ + body return y; \ + } \ + return g; \ + } + +EARLY_RETURN(40, ADD32 ADD8) +EARLY_RETURN(41, ADD32 ADD8 ADD1) +EARLY_RETURN(42, ADD32 ADD8 ADD2) +EARLY_RETURN(43, ADD32 ADD8 ADD2 ADD1) +EARLY_RETURN(44, ADD32 ADD8 ADD4) +EARLY_RETURN(45, ADD32 ADD8 ADD4 ADD1) +EARLY_RETURN(46, ADD32 ADD8 ADD4 ADD2) +EARLY_RETURN(47, ADD32 ADD8 ADD4 ADD2 ADD1) +EARLY_RETURN(48, ADD32 ADD16) +EARLY_RETURN(49, ADD32 ADD16 ADD1) +EARLY_RETURN(50, ADD32 ADD16 ADD2) +EARLY_RETURN(51, ADD32 ADD16 ADD2 ADD1) +EARLY_RETURN(52, ADD32 ADD16 ADD4) +EARLY_RETURN(53, ADD32 ADD16 ADD4 ADD1) +EARLY_RETURN(54, ADD32 ADD16 ADD4 ADD2) +EARLY_RETURN(55, ADD32 ADD16 ADD4 ADD2 ADD1) +EARLY_RETURN(56, ADD32 ADD16 ADD8) +EARLY_RETURN(57, ADD32 ADD16 ADD8 ADD1) +EARLY_RETURN(58, ADD32 ADD16 ADD8 ADD2) +EARLY_RETURN(59, ADD32 ADD16 ADD8 ADD2 ADD1) +EARLY_RETURN(60, ADD32 ADD16 ADD8 ADD4) +EARLY_RETURN(61, ADD32 ADD16 ADD8 ADD4 ADD1) +EARLY_RETURN(62, ADD32 ADD16 ADD8 ADD4 ADD2) +EARLY_RETURN(63, ADD32 ADD16 ADD8 ADD4 ADD2 ADD1) +EARLY_RETURN(64, ADD32 ADD32) +EARLY_RETURN(65, ADD32 ADD32 ADD1) +EARLY_RETURN(66, ADD32 ADD32 ADD2) +EARLY_RETURN(67, ADD32 ADD32 ADD2 ADD1) +EARLY_RETURN(68, ADD32 ADD32 ADD4) +EARLY_RETURN(69, ADD32 ADD32 ADD4 ADD1) +EARLY_RETURN(70, ADD32 ADD32 ADD4 ADD2) +EARLY_RETURN(71, ADD32 ADD32 ADD4 ADD2 ADD1) +EARLY_RETURN(72, ADD32 ADD32 ADD8) +EARLY_RETURN(73, ADD32 ADD32 ADD8 ADD1) +EARLY_RETURN(74, ADD32 ADD32 ADD8 ADD2) +EARLY_RETURN(75, ADD32 ADD32 ADD8 ADD2 ADD1) +EARLY_RETURN(76, ADD32 ADD32 ADD8 ADD4) +EARLY_RETURN(77, ADD32 ADD32 ADD8 ADD4 ADD1) +EARLY_RETURN(78, ADD32 ADD32 ADD8 ADD4 ADD2) +EARLY_RETURN(79, ADD32 ADD32 ADD8 ADD4 ADD2 ADD1) + +static int (*const early_returns[])(int, int) = { + adds_40, adds_41, adds_42, adds_43, adds_44, adds_45, adds_46, adds_47, + adds_48, adds_49, adds_50, adds_51, adds_52, adds_53, adds_54, adds_55, + adds_56, adds_57, adds_58, adds_59, adds_60, adds_61, adds_62, adds_63, + adds_64, adds_65, adds_66, adds_67, adds_68, adds_69, adds_70, adds_71, + adds_72, adds_73, adds_74, adds_75, adds_76, adds_77, adds_78, adds_79, +}; + +/* 144 bytes of asm body the rehearsal saw as nothing. */ +#define NOP8 "nop\n nop\n nop\n nop\n nop\n nop\n nop\n nop\n" +int over_asm(int x) +{ + if (x) + { + __asm volatile(NOP8 NOP8 NOP8 NOP8 NOP8 NOP8 NOP8 NOP8 NOP8); + g = 1; + } + return g; +} + +int main(void) +{ + int sum = 0; + for (unsigned i = 0; i < sizeof early_returns / sizeof early_returns[0]; i++) + sum += early_returns[i](1, 0) + early_returns[i](0, 0); + printf("%d\n", sum); /* 40 + 41 + ... + 79 */ + printf("%d\n", over_asm(0)); + printf("%d\n", over_asm(1)); + return 0; +} diff --git a/tests/ir_tests/bug_branch_narrow_rehearsal.expect b/tests/ir_tests/bug_branch_narrow_rehearsal.expect new file mode 100644 index 000000000..3ed8bcf68 --- /dev/null +++ b/tests/ir_tests/bug_branch_narrow_rehearsal.expect @@ -0,0 +1,3 @@ +2380 +0 +1 diff --git a/tests/ir_tests/bug_dse_indexed_store_escape.c b/tests/ir_tests/bug_dse_indexed_store_escape.c new file mode 100644 index 000000000..01d7c155e --- /dev/null +++ b/tests/ir_tests/bug_dse_indexed_store_escape.c @@ -0,0 +1,49 @@ +/* Storing an array's address into memory keeps the array's initializer. + * + * dse took `src1 = Addr[array]` with a TEMP dest for "the TEMP is defined as + * that address" -- but in a STORE_INDEXED the dest is the base pointer and in a + * STORE through a pointer the address being written through: the array's + * address was going into memory. When that pointer was only ever written + * through, the address was not marked read, and the stores initializing the + * array were deleted (gcc.c-torture pr47538 once the double guard is gone). */ +#include + +struct S +{ + int a, b, e, f; + int *c; + int d; +}; + +__attribute__((noinline)) static int sum(const struct S *y) +{ + return y->c[0] + y->c[1] * 10 + y->c[3] * 100 + y->d; +} + +__attribute__((noinline)) static int through_field(void) +{ + struct S y; + int c[4] = {1, 2, 3, 4}; + y.a = 10; + y.b = 6; + y.c = c; /* STORE_INDEXED off &y */ + y.d = 3; + return sum(&y); +} + +__attribute__((noinline)) static int through_pointer(void) +{ + struct S y; + struct S *py = &y; + int c[4] = {5, 6, 7, 8}; + py->a = 1; + py->c = c; /* a STORE through a pointer */ + py->d = 9; + return sum(py); +} + +int main(void) +{ + printf("%d %d\n", through_field(), through_pointer()); + return 0; +} diff --git a/tests/ir_tests/bug_dse_indexed_store_escape.expect b/tests/ir_tests/bug_dse_indexed_store_escape.expect new file mode 100644 index 000000000..db21eb6f9 --- /dev/null +++ b/tests/ir_tests/bug_dse_indexed_store_escape.expect @@ -0,0 +1 @@ +424 874 diff --git a/tests/ir_tests/bug_modref_direct_block_copy.c b/tests/ir_tests/bug_modref_direct_block_copy.c new file mode 100644 index 000000000..a02f1e0aa --- /dev/null +++ b/tests/ir_tests/bug_modref_direct_block_copy.c @@ -0,0 +1,37 @@ +/* A direct memmove into a global writes that global. + * + * tcc_ir_call_may_write skipped block-copy helpers without a summary, which + * is right only for one reached through a callee (its effect is in that + * callee's summary). For a direct call it said `g = make()` -- a memmove into + * g -- does not write g, so copy-source load forwarding answered a read of an + * earlier copy of g with g's NEW value (-O1/-O2). */ +#include +#include + +/* Returned in r0, then stored into g with a memmove: 4 single-byte fields + * are more chunks than the inline small-aggregate copy takes. */ +struct quad +{ + uint8_t a, b, c, d; +}; + +struct quad g = {0x34, 2, 3, 4}; + +__attribute__((noinline)) static struct quad make(uint8_t v) +{ + struct quad t = {v, v, v, v}; + return t; +} + +int main(void) +{ + unsigned sum = 0; + + struct quad copy; + copy = g; /* copy the global */ + g = make(9); /* then overwrite it through a memmove */ + sum += copy.a + g.a; + + printf("%u\n", sum); /* 0x34 + 9 = 61 */ + return 0; +} diff --git a/tests/ir_tests/bug_modref_direct_block_copy.expect b/tests/ir_tests/bug_modref_direct_block_copy.expect new file mode 100644 index 000000000..b1e7d265f --- /dev/null +++ b/tests/ir_tests/bug_modref_direct_block_copy.expect @@ -0,0 +1 @@ +61 diff --git a/tests/ir_tests/bug_narrow_field_store_forward.c b/tests/ir_tests/bug_narrow_field_store_forward.c new file mode 100644 index 000000000..4f3b05696 --- /dev/null +++ b/tests/ir_tests/bug_narrow_field_store_forward.c @@ -0,0 +1,60 @@ +/* A narrow struct field's store truncates; forwarding it must too. + * + * sl_forward replaces a load by the value a matching store wrote. For a + * field of a struct the frontend does NOT truncate ahead of that store -- + * `s.a = x` stores x whole and lets the store's width cut it -- so handing the + * register on unmasked passes the bits the store dropped. From -O1 up + * `s.a = x + i; use(s.a)` read x + i instead of (x + i) & 0xff. (A narrow + * LOCAL is safe: the frontend truncates before writing one, which is why this + * only ever showed on fields.) + * + * Both widths, and a second field beside the narrow one so the object is a + * real struct rather than a one-field wrapper. */ +#include + +struct b1 +{ + unsigned char a; + unsigned char b; +}; + +struct b2 +{ + unsigned short a; + unsigned w; +}; + +__attribute__((noinline)) static unsigned byte_field(unsigned x, int n) +{ + struct b1 s; + unsigned r = 0; + s.a = 0; + s.b = 7; + for (int i = 0; i < n; i++) + { + s.a = x + (unsigned)i; /* wraps past 255 */ + r += s.a + s.b; + } + return r; +} + +__attribute__((noinline)) static unsigned half_field(unsigned x, int n) +{ + struct b2 s; + unsigned r = 0; + s.a = x * 3u; + s.w = x; + for (int i = 0; i < n; i++) + { + s.a = (unsigned)i * 70000u; /* wraps past 65535 */ + r += s.a + s.w; + } + return r; +} + +int main(void) +{ + printf("%u %u\n", byte_field(250, 10), byte_field(3, 1)); + printf("%u %u\n", half_field(30000, 5), half_field(1, 2)); + return 0; +} diff --git a/tests/ir_tests/bug_narrow_field_store_forward.expect b/tests/ir_tests/bug_narrow_field_store_forward.expect new file mode 100644 index 000000000..cf9efec4b --- /dev/null +++ b/tests/ir_tests/bug_narrow_field_store_forward.expect @@ -0,0 +1,2 @@ +1591 10 +194640 4466 diff --git a/tests/ir_tests/bug_narrow_local_slot_width.c b/tests/ir_tests/bug_narrow_local_slot_width.c new file mode 100644 index 000000000..55331929a --- /dev/null +++ b/tests/ir_tests/bug_narrow_local_slot_width.c @@ -0,0 +1,85 @@ +/* A byte/halfword local read through its own stack slot keeps its width. + * + * lea_fold turns `T = &local; ... *T` into a direct stack-slot operand, and the + * value paths loaded such a slot with a word load meant for register spill + * slots. Reading a 2-byte struct's u16 field then pulled in the neighbouring + * local's bytes: Zig's C backend returns `!void` as exactly such a struct, and + * `if (t7.error)` took the error branch on a success (-O1/-O2). */ +#include +#include + +struct error_union /* Zig's `!void` */ +{ + uint16_t error; +}; +struct slice +{ + const char *ptr; + unsigned len; +}; +struct flag +{ + uint8_t set; +}; + +static int side; + +__attribute__((noinline)) static struct error_union may_fail(int fail) +{ + struct error_union r = {.error = fail ? 7 : 0}; + return r; +} + +/* The error test must see only the 16-bit field. */ +__attribute__((noinline)) static struct error_union try_step(struct slice const a1, int fail) +{ + struct slice t2; + struct error_union t7; + t2 = a1; + t7 = may_fail(fail); + if (t7.error) + return t7; + side += (int)t2.len; /* the success path must run */ + return (struct error_union){.error = 0}; +} + +/* The same field passed on as an argument must arrive zero-extended. */ +__attribute__((noinline)) static unsigned widen(uint32_t v) +{ + return v; +} +__attribute__((noinline)) static unsigned error_code(struct slice const a1, int fail) +{ + struct slice t2; + struct error_union t7; + t2 = a1; + t7 = may_fail(fail); + return widen(t7.error) + t2.len * 0u; +} + +__attribute__((noinline)) static struct flag get_flag(int v) +{ + struct flag f = {.set = (uint8_t)v}; + return f; +} +__attribute__((noinline)) static int flag_is_set(struct slice const a1, int v) +{ + struct slice t2; + struct flag t5; + t2 = a1; + t5 = get_flag(v); + if (t5.set) + return 1 + (int)(t2.len * 0u); + return 0; +} + +int main(void) +{ + struct slice s = {"abc", 5}; + struct error_union a = try_step(s, 0); + struct error_union b = try_step(s, 1); + printf("%u %u %d\n", a.error, b.error, side); /* 0 7 5 */ + printf("%u %u\n", error_code(s, 0), error_code(s, 1)); /* 0 7 */ + printf("%d %d\n", flag_is_set(s, 0), flag_is_set(s, 1)); /* 0 1 */ + return 0; +} diff --git a/tests/ir_tests/bug_narrow_local_slot_width.expect b/tests/ir_tests/bug_narrow_local_slot_width.expect new file mode 100644 index 000000000..b457f3890 --- /dev/null +++ b/tests/ir_tests/bug_narrow_local_slot_width.expect @@ -0,0 +1,3 @@ +0 7 5 +0 7 +0 1 diff --git a/tests/ir_tests/bug_postinc_call_arg_deref.c b/tests/ir_tests/bug_postinc_call_arg_deref.c new file mode 100644 index 000000000..a179c2d23 --- /dev/null +++ b/tests/ir_tests/bug_postinc_call_arg_deref.c @@ -0,0 +1,47 @@ +/* Regression: ra:load_postinc folded the loop's `++p` into the FIRST `*p` load + * (`ldrb rX,[p],#1`) while a second read of P was still to come. The second + * read was an argument of the outer call: its FUNCPARAMVAL quad sits ABOVE the + * load (params are emitted in source order, the nested call's load after + * them), but the value is read at the outer CALL, below the load. The scan + * between the load and the add only looked at the quads in that window, so it + * never saw the param and the outer call got p+1 -- "YASB" printed as + * "A0S0B0\0 0". FatFs f_setlabel hit it (sdformat: FR_INVALID_NAME). + */ +#include + +__attribute__((noinline)) int ext(int c) { return c == 'S'; } +__attribute__((noinline)) int ext16(int c) { return c == 300; } +__attribute__((noinline)) int take(const char *p, int v) { return *p * 2 + v; } + +static void chars(const char *label) +{ + for (const char *p = label; *p; ++p) + printf("%c%d", *p, ext(*p)); + printf("\n"); +} + +static int ptr_arg(const char *s) +{ + int sum = 0; + for (const char *p = s; *p; ++p) + sum += take(p, ext(*p)); + return sum; +} + +static int shorts(const unsigned short *w) +{ + int sum = 0; + for (const unsigned short *p = w; *p; ++p) + sum = sum * 3 + printf("%d,%d ", *p, ext16(*p)); + printf("\n"); + return sum; +} + +int main(void) +{ + static const unsigned short w[] = {100, 300, 7, 0}; + chars("YASB"); + printf("%d\n", ptr_arg("YASB")); + printf("%d\n", shorts(w)); + return 0; +} diff --git a/tests/ir_tests/bug_postinc_call_arg_deref.expect b/tests/ir_tests/bug_postinc_call_arg_deref.expect new file mode 100644 index 000000000..cb12c8fec --- /dev/null +++ b/tests/ir_tests/bug_postinc_call_arg_deref.expect @@ -0,0 +1,4 @@ +Y0A0S1B0 +607 +100,0 300,1 7,0 +76 diff --git a/tests/ir_tests/bug_sra_narrow_field_undefined.c b/tests/ir_tests/bug_sra_narrow_field_undefined.c new file mode 100644 index 000000000..3f3e4630d --- /dev/null +++ b/tests/ir_tests/bug_sra_narrow_field_undefined.c @@ -0,0 +1,88 @@ +/* A struct field scalar replacement pulls out of the frame has to be DEFINED + * where the struct was written. + * + * SRA gives each field of a promotable frame object its own VAR. SSA only + * counts a full-width slot STORE as a definition of a variable; a narrower one + * updates whatever name is current, in place. A one-byte field written only + * by byte stores therefore got a VAR that was never defined at all, while its + * reads -- which SSA rename turns into whole-register copies -- took the + * undefined entry name. Here `sba25` is built and passed by value, and at -O2 + * the callee read three bytes of whatever was under it; the loop and the + * dirtied stack are what make that show rather than pass by luck. + * + * Reduced from struct_byval fuzz seed 1962. */ +#include +static unsigned csmix(unsigned h, unsigned v) +{ + h ^= v + 0x9e3779b9u + (h << 6) + (h >> 2); + return h * 2654435761u; +} +static unsigned helper1(unsigned pa, unsigned pb) +{ + unsigned lr = pa ^ (pb * 3u); + return (unsigned)(lr) ^ lr; +} +struct SB1 { unsigned char a; }; +struct SB4 { unsigned a; }; +union UB { unsigned w; unsigned char b; }; +static struct SB4 sbh2(struct SB1 p, unsigned x) +{ + struct SB4 r = { (unsigned)(x ^ (p.a * 3u)) & 0xffffffffu }; + r.a = (unsigned)(p.a) & 0xffffffffu; + return r; +} +struct S { + unsigned f0; + unsigned f1; + unsigned f2; +}; +__attribute__((noinline)) static void dirty(int d) +{ + volatile unsigned b[64]; + for (int i = 0; i < 64; i++) + b[i] = 0xDEAD0000u + (unsigned)(d * 64 + i); + if (d) + dirty(d - 1); +} + +__attribute__((noinline)) static unsigned body(void) +{ + unsigned cs = 0x12345678u; + unsigned char s5 = (unsigned char)(1863726238u & 0xff); + unsigned u6 = 844757540u; + unsigned u7 = 2753348224u; + unsigned u8 = 1393046211u; + unsigned u9 = 1308701682u; + unsigned u10 = 2566731158u; + unsigned u11 = 3567405196u; + unsigned arr12[8] = { 1016393539u, 2923799514u, 1914012732u, 4085784861u, 2376755388u, 1239331789u, 1682676424u, 174298200u }; + unsigned arr13[8] = { 3495248964u, 3150037832u, 2562321627u, 3316459955u, 3242667970u, 1555379116u, 3857701763u, 1520688340u }; + struct S st14 = { 3412160013u, 2070586499u, 1763051845u }; + struct S st15 = { 2628473315u, 715880564u, 3847184839u }; + for (unsigned g18 = 0u; g18 < 5u; g18++) { + unsigned i17 = g18; + cs = csmix(cs, i17); + { union UB ub19; ub19.w = (unsigned)(u9); cs = csmix(cs, ub19.w); } + u9 = (unsigned)(u7) & 0xffffffffu; + for (unsigned g21 = 0u; g21 < 8u; g21++) { + unsigned i20 = g21; + cs = csmix(cs, i20); + { union UB ub22; ub22.w = (unsigned)(((unsigned)((unsigned)(s5)) & (unsigned)(((unsigned)(((unsigned)(u8) >> ((unsigned)(589226152u) & 31u))) | (unsigned)(((unsigned)(i17) << ((unsigned)(st15.f1) & 31u))))))); cs = csmix(cs, ub22.w); } + cs = csmix(cs, (unsigned)(((unsigned)(arr13[((unsigned)(u11) & 7u)]) - (unsigned)(((unsigned)(((unsigned)(((unsigned)(i20) - (unsigned)(u10))) ^ (unsigned)(st14.f0))) << ((unsigned)(((unsigned)(((unsigned)(u11) & (unsigned)(4196820644u))) << ((unsigned)((unsigned)(s5)) & 31u))) & 31u)))))); + } + arr13[((unsigned)(u8) & 7u)] = (unsigned)(u7); + } + { struct SB1 sba25 = { (unsigned)(u9) & 0xffu }; + struct SB4 sbt26 = sbh2(sba25, (unsigned)(((unsigned)(((unsigned)((((unsigned)(u11) & 1u) ? (unsigned)(helper1(480138227u, st14.f0)) : (unsigned)(((unsigned)((unsigned)(s5)) / ((unsigned)(u7) | 1u))))) % ((unsigned)((-((unsigned)(u6) | 0u))) | 1u))) % ((unsigned)((~((unsigned)(((unsigned)(((unsigned)(arr12[((unsigned)(4142406418u) & 7u)]) << ((unsigned)(3576979367u) & 31u))) * (unsigned)((unsigned)(s5)))) | 0u))) | 1u)))); + cs = csmix(cs, sbt26.a); + } + cs = csmix(cs, u9); + return cs; +} + +int main(void) +{ + dirty(6); + printf("checksum=%08x\n", body()); + return 0; +} diff --git a/tests/ir_tests/bug_sra_narrow_field_undefined.expect b/tests/ir_tests/bug_sra_narrow_field_undefined.expect new file mode 100644 index 000000000..b4ecfe0e6 --- /dev/null +++ b/tests/ir_tests/bug_sra_narrow_field_undefined.expect @@ -0,0 +1 @@ +checksum=0375ae38 diff --git a/tests/ir_tests/bug_vla_struct_array.c b/tests/ir_tests/bug_vla_struct_array.c new file mode 100644 index 000000000..f5e975524 --- /dev/null +++ b/tests/ir_tests/bug_vla_struct_array.c @@ -0,0 +1,42 @@ +/* An array of structs with a VLA member (GNU C) is itself variable-length. + * + * The array's type was fixed-size, so it got a frame slot sized as if the VLA + * member were empty, and writing its elements ran over the objects next to it + * (gcc.c-torture/execute/align-nest.c overwrote the caller's saved LR once the + * frame layout shifted by a word). */ +#include + +__attribute__((noinline)) static int fill(int n) +{ + volatile int above[4] = {11, 12, 13, 14}; + struct S + { + int i[n]; + int tail; + } s[3]; + volatile int below[4] = {21, 22, 23, 24}; + + for (int k = 0; k < 3; k++) + { + for (int j = 0; j < n; j++) + s[k].i[j] = 100 * k + j; + s[k].tail = -k; + } + int sum = 0; + for (int k = 0; k < 3; k++) + { + for (int j = 0; j < n; j++) + sum += s[k].i[j]; + sum += s[k].tail; + } + for (int j = 0; j < 4; j++) + sum += above[j] * 1000 + below[j] * 100000; + return sum + (int)sizeof s; +} + +int main(void) +{ + printf("%d\n", fill(5)); + printf("%d\n", fill(1)); + return 0; +} diff --git a/tests/ir_tests/bug_vla_struct_array.expect b/tests/ir_tests/bug_vla_struct_array.expect new file mode 100644 index 000000000..8bcffb595 --- /dev/null +++ b/tests/ir_tests/bug_vla_struct_array.expect @@ -0,0 +1,2 @@ +9051599 +9050321 diff --git a/tests/ir_tests/golden/block_copy_init/clear_struct.expected b/tests/ir_tests/golden/block_copy_init/clear_struct.expected index f8df4d0dd..b9240a974 100644 --- a/tests/ir_tests/golden/block_copy_init/clear_struct.expected +++ b/tests/ir_tests/golden/block_copy_init/clear_struct.expected @@ -1,9 +1,4 @@ -0000: StackLoc[-16] <-- #0 [STORE] -0001: StackLoc[-12] <-- #0 [STORE] -0002: StackLoc[-8] <-- #0 [STORE] -0003: T0 <-- P0 [ASSIGN] -0004: T1 <-- Addr[StackLoc[-16]] -0005: PARAM0[call_0] T0 -0006: PARAM1[call_0] T1 -0007: PARAM2[call_0] #12 -0008: CALL GlobalSym(#0) CALL #3 +0000: T0 <-- P0 [ASSIGN] +0001: T0***DEREF*** <-- #0 [STORE] +0002: T0 <-- #0 STORE_INDEXED #4 +0003: T0 <-- #0 STORE_INDEXED #8 diff --git a/tests/ir_tests/golden/branch_cleanup/dead_ternary_diamond.expected b/tests/ir_tests/golden/branch_cleanup/dead_ternary_diamond.expected index f52c5fb5f..32b0175cd 100644 --- a/tests/ir_tests/golden/branch_cleanup/dead_ternary_diamond.expected +++ b/tests/ir_tests/golden/branch_cleanup/dead_ternary_diamond.expected @@ -1,10 +1,11 @@ 0000: CMP P0,P1 -0001: JMP to 5 if ">=S" -0002: PARAM0[call_0] P0 -0003: CALL GlobalSym(#0) --> T0 -0004: JMP to 7 -0005: T1 <-- #0 [ASSIGN] -0006: JMP to 8 -0007: T1 <-- T0 [ASSIGN] -0008: T2 <-- P0 ADD P1 -0009: RETURNVALUE T2 +0001: JMP to 6 if ">=S" +0002: V0 <-- P0 [STORE] +0003: T0 <-- V0 SHL #1 +0004: V1 <-- T0 [STORE] +0005: JMP to 8 +0006: T1 <-- #0 [ASSIGN] +0007: JMP to 9 +0008: T1 <-- V1 [LOAD] +0009: T3 <-- P0 ADD P1 +0010: RETURNVALUE T3 diff --git a/tests/ir_tests/golden/const_cascade/post_forward_arith_chain.expected b/tests/ir_tests/golden/const_cascade/post_forward_arith_chain.expected index e20f78c8f..0ca334990 100644 --- a/tests/ir_tests/golden/const_cascade/post_forward_arith_chain.expected +++ b/tests/ir_tests/golden/const_cascade/post_forward_arith_chain.expected @@ -1,6 +1,6 @@ 0000: T0 <-- P1 AND #3 -0001: Addr[StackLoc[-16]] <-- #1 STORE_INDEXED T0 -0002: StackLoc[-16] <-- #5 [STORE] +0001: Addr[StackLoc[-20]] <-- #1 STORE_INDEXED T0 +0002: StackLoc[-20] <-- #5 [STORE] 0003: V0 <-- #5 [ASSIGN] 0004: V1 <-- #8 [ASSIGN] 0005: V2 <-- #16 [ASSIGN] diff --git a/tests/ir_tests/golden/esp_cleanup/inlined_struct_field_check.expected b/tests/ir_tests/golden/esp_cleanup/inlined_struct_field_check.expected index ff6da481f..b41673829 100644 --- a/tests/ir_tests/golden/esp_cleanup/inlined_struct_field_check.expected +++ b/tests/ir_tests/golden/esp_cleanup/inlined_struct_field_check.expected @@ -1,8 +1,8 @@ -0000: StackLoc[-8] <-- #0 [STORE] -0001: StackLoc[-4] <-- #0 [STORE] -0002: StackLoc[-8] <-- #3 [STORE] -0003: StackLoc[-4] <-- #5 [STORE] -0004: T0 <-- Addr[StackLoc[-8]] +0000: StackLoc[-12] <-- #0 [STORE] +0001: StackLoc[-8] <-- #0 [STORE] +0002: StackLoc[-12] <-- #3 [STORE] +0003: StackLoc[-8] <-- #5 [STORE] +0004: T0 <-- Addr[StackLoc[-12]] 0005: T1 <-- T0 [ASSIGN] 0006: T2 <-- T0 [ASSIGN] 0007: T6 <-- #5 [ASSIGN] diff --git a/tests/ir_tests/golden/kb_cascade/mask_shift_branch_fold.expected b/tests/ir_tests/golden/kb_cascade/mask_shift_branch_fold.expected index 84dd78a48..53f2fb242 100644 --- a/tests/ir_tests/golden/kb_cascade/mask_shift_branch_fold.expected +++ b/tests/ir_tests/golden/kb_cascade/mask_shift_branch_fold.expected @@ -1,6 +1,6 @@ 0000: T0 <-- P1 AND #3 -0001: Addr[StackLoc[-16]] <-- #1 STORE_INDEXED T0 -0002: StackLoc[-16] <-- #5 [STORE] +0001: Addr[StackLoc[-20]] <-- #1 STORE_INDEXED T0 +0002: StackLoc[-20] <-- #5 [STORE] 0003: V0 <-- #5 [ASSIGN] 0004: V1 <-- #0 [ASSIGN] 0005: V2 <-- #0 [ASSIGN] diff --git a/tests/ir_tests/nested_capture_64bit.c b/tests/ir_tests/nested_capture_64bit.c new file mode 100644 index 000000000..b5cc39b99 --- /dev/null +++ b/tests/ir_tests/nested_capture_64bit.c @@ -0,0 +1,63 @@ +/* nested_capture_64bit.c — capturing a 64-bit variable (long long or double) + * from a nested function, as a local and as a parameter. + * + * A captured variable reaches the child as MACH_OP_CHAIN_REL, whose + * needs_deref (from is_lval) means "the value is at [chain+offset]" — the same + * sense as MACH_OP_PARAM_STACK, not "a pointer lives there". Five places in + * the ARM backend carried that rule and every one of them named only + * PARAM_STACK, so a 64-bit capture loaded the value's low word and then + * dereferenced it: `ldr ip,[sl,#-8]; ldrd r0,r1,[ip]`. They now share + * mach_op_64_names_memory(). + */ +#include + +__attribute__((noinline)) static long long ll_loc(int a) +{ + long long q = 42; + __attribute__((noinline)) long long get(void) { return q + 1; } + long long before = get(); + q = 0x100000000LL; + return before + get() + a; +} + +/* Written from both sides, so the frame home is exercised, not a snapshot. */ +__attribute__((noinline)) static long long ll_par(int a, int b, int c, long long q) +{ + __attribute__((noinline)) void bump(long long by) { q += by; } + __attribute__((noinline)) long long get(void) { return q; } + bump(0x100000000LL); + q += 5; + bump(1); + return get() + a + b + c; +} + +/* Soft-float doubles take the FP operand path, which had its own copy. */ +__attribute__((noinline)) static double d_stk(int a, int b, int c, int d, double x) +{ + __attribute__((noinline)) double scale(void) { return x * 2.0; } + double before = scale(); + x = 4.5; + return before + scale() + a + b + c + d; +} + +/* A captured 64-bit value passed as a STACK argument of a further call. */ +__attribute__((noinline)) static long long sink(int a, int b, int c, int d, long long q, long long r) +{ + return q * 10 + r; +} +__attribute__((noinline)) static long long ll_arg(int a, long long q) +{ + __attribute__((noinline)) long long f(void) { return sink(1, 2, 3, 4, q, q + 1); } + long long before = f(); + q = 7; + return before * 100 + f() + a; +} + +int main(void) +{ + printf("ll_loc=%lld\n", ll_loc(1)); + printf("ll_par=%lld\n", ll_par(1, 2, 3, 42LL)); + printf("d_stk=%.2f\n", d_stk(1, 2, 3, 4, 1.5)); + printf("ll_arg=%lld\n", ll_arg(1, 3LL)); + return 0; +} diff --git a/tests/ir_tests/nested_capture_64bit.expect b/tests/ir_tests/nested_capture_64bit.expect new file mode 100644 index 000000000..a8562baeb --- /dev/null +++ b/tests/ir_tests/nested_capture_64bit.expect @@ -0,0 +1,4 @@ +ll_loc=4294967341 +ll_par=4294967350 +d_stk=22.00 +ll_arg=3479 diff --git a/tests/ir_tests/nested_capture_param_byref.c b/tests/ir_tests/nested_capture_param_byref.c new file mode 100644 index 000000000..424f769e6 --- /dev/null +++ b/tests/ir_tests/nested_capture_param_byref.c @@ -0,0 +1,44 @@ +/* nested_capture_param_byref.c — GNU nested functions capture by REFERENCE, and + * that has to hold for parameters too, in both directions. + * + * A captured parameter therefore cannot be snapshotted at the child's + * declaration: it needs a real home in the parent's frame that every parent + * read and write goes through, so a store on either side is seen by the other. + */ +#include + +/* The parent writes a captured stack-passed parameter after declaring the + * child; the child must see the new value. */ +__attribute__((noinline)) static int wr_parent(int a, int b, int c, int d, int e) +{ + __attribute__((noinline)) int get(void) { return e; } + int before = get(); + e = 100; + return before * 1000 + get() + a + b + c + d; +} + +/* The child writes a captured stack-passed parameter; the parent must see it. */ +__attribute__((noinline)) static int wr_child(int a, int b, int c, int d, int e) +{ + __attribute__((noinline)) void bump(int by) { e += by; } + bump(7); + bump(7); + return e + a + b + c + d; +} + +/* The same, for register-passed parameters and a noinline child. */ +__attribute__((noinline)) static int wr_reg(int a, int b) +{ + __attribute__((noinline)) int f(void) { a += b; return a; } + int x = f(); + b = 100; + return x * 1000 + f(); +} + +int main(void) +{ + printf("wr_parent=%d\n", wr_parent(1, 2, 3, 4, 5)); + printf("wr_child=%d\n", wr_child(1, 2, 3, 4, 5)); + printf("wr_reg=%d\n", wr_reg(7, 9)); + return 0; +} diff --git a/tests/ir_tests/nested_capture_param_byref.expect b/tests/ir_tests/nested_capture_param_byref.expect new file mode 100644 index 000000000..10fc4e4cf --- /dev/null +++ b/tests/ir_tests/nested_capture_param_byref.expect @@ -0,0 +1,3 @@ +wr_parent=5110 +wr_child=29 +wr_reg=16116 diff --git a/tests/ir_tests/nested_capture_param_mixed.c b/tests/ir_tests/nested_capture_param_mixed.c new file mode 100644 index 000000000..5721b2c7d --- /dev/null +++ b/tests/ir_tests/nested_capture_param_mixed.c @@ -0,0 +1,30 @@ +/* nested_capture_param_mixed.c — several stack-passed parameters captured at + * once, and a captured parameter the parent reads as well. + * + * The parent's own reads have to go through the frame home too: reading them as + * parameters would add offset_to_args to an offset that is already + * frame-relative, landing in the saved-register area. + */ +#include + +__attribute__((noinline)) static int multi(int a, int b, int c, int d, int e, short f, char g) +{ + __attribute__((noinline)) int sum(void) { return e + f + g; } + return sum() * 10 + a + b + c + d; +} + +__attribute__((noinline)) static int both_read(int a, int b, int c, int d, int e) +{ + __attribute__((noinline)) int h(void) { return e; } + int s = 0; + for (int i = 0; i < 3; i++) + s += e + h(); + return s + a + b + c + d; +} + +int main(void) +{ + printf("multi=%d\n", multi(1, 2, 3, 4, 5, 6, 7)); + printf("both_read=%d\n", both_read(1, 2, 3, 4, 5)); + return 0; +} diff --git a/tests/ir_tests/nested_capture_param_mixed.expect b/tests/ir_tests/nested_capture_param_mixed.expect new file mode 100644 index 000000000..682c62f27 --- /dev/null +++ b/tests/ir_tests/nested_capture_param_mixed.expect @@ -0,0 +1,2 @@ +multi=190 +both_read=40 diff --git a/tests/ir_tests/nested_capture_param_reg_noinline.c b/tests/ir_tests/nested_capture_param_reg_noinline.c new file mode 100644 index 000000000..e23199033 --- /dev/null +++ b/tests/ir_tests/nested_capture_param_reg_noinline.c @@ -0,0 +1,22 @@ +/* nested_capture_param_reg_noinline.c — a noinline nested function capturing + * REGISTER-passed parent parameters. + * + * Nested functions were registered as auto-inline candidates without checking + * __attribute__((noinline)), and gen_function then cleared the captures' + * addrtaken on the strength of that mark — so the parameters never reached a + * frame slot, while the call it assumed away was still emitted. add() read + * both captures from [chain + 0] and returned 10 instead of 26. + */ +#include + +__attribute__((noinline)) static int reg_cap(int a, int b) +{ + __attribute__((noinline)) int add(int x) { return x + a + b; } + return add(10); +} + +int main(void) +{ + printf("reg_cap=%d\n", reg_cap(7, 9)); + return 0; +} diff --git a/tests/ir_tests/nested_capture_param_reg_noinline.expect b/tests/ir_tests/nested_capture_param_reg_noinline.expect new file mode 100644 index 000000000..76ffe4369 --- /dev/null +++ b/tests/ir_tests/nested_capture_param_reg_noinline.expect @@ -0,0 +1 @@ +reg_cap=26 diff --git a/tests/ir_tests/nested_capture_param_stack.c b/tests/ir_tests/nested_capture_param_stack.c new file mode 100644 index 000000000..3e589ed24 --- /dev/null +++ b/tests/ir_tests/nested_capture_param_stack.c @@ -0,0 +1,23 @@ +/* nested_capture_param_stack.c — a nested function capturing a STACK-passed + * parent parameter. + * + * The 5th int argument arrives in the caller's argument area, at an offset + * measured from the parent's offset_to_args — a position the child cannot name, + * because the children are compiled before the parent's prologue fixes it. The + * parameter had no home in the parent's own frame, so the capture resolved to + * chain offset 0 and add() read `d` instead of `e` (24 instead of 25). + * Fixed by giving a captured parameter a frame home the prologue writes into. + */ +#include + +__attribute__((noinline)) static int stk_cap(int a, int b, int c, int d, int e) +{ + int add(int x) { return x + e; } + return add(10) + a + b + c + d; +} + +int main(void) +{ + printf("stk_cap=%d\n", stk_cap(1, 2, 3, 4, 5)); + return 0; +} diff --git a/tests/ir_tests/nested_capture_param_stack.expect b/tests/ir_tests/nested_capture_param_stack.expect new file mode 100644 index 000000000..6a42fafc3 --- /dev/null +++ b/tests/ir_tests/nested_capture_param_stack.expect @@ -0,0 +1 @@ +stk_cap=25 diff --git a/tests/ir_tests/nested_capture_two_level.c b/tests/ir_tests/nested_capture_two_level.c new file mode 100644 index 000000000..8994303e5 --- /dev/null +++ b/tests/ir_tests/nested_capture_two_level.c @@ -0,0 +1,41 @@ +/* nested_capture_two_level.c — a grandchild capturing the grandparent. + * + * inner() reaches `e` by hopping: R10 points at mid's frame top, and mid's + * incoming chain is read from [chain-4]. mid reserved those four bytes + * (gen_function's `loc -= 4`), but the backend's initial min-local-offset scan + * raised `loc` back to 0 because the chain slot is not an IR operand and mid + * has no locals of its own. mid then stored the chain BELOW its own SP, where + * inner's own push landed on it. + */ +#include + +__attribute__((noinline)) static int two_level(int a, int b, int c, int d, int e) +{ + __attribute__((noinline)) int mid(void) + { + __attribute__((noinline)) int inner(void) { return e * 2; } + return inner() + 1; + } + return mid() + a + b + c + d; +} + +/* The same, capturing a grandparent LOCAL, and writing it from the grandchild. */ +__attribute__((noinline)) static int two_level_write(int a) +{ + int e = 5; + __attribute__((noinline)) int mid(void) + { + __attribute__((noinline)) void inner(void) { e += 10; } + inner(); + return e; + } + int m = mid(); + return m * 100 + e + a; +} + +int main(void) +{ + printf("two_level=%d\n", two_level(1, 2, 3, 4, 5)); + printf("two_level_write=%d\n", two_level_write(1)); + return 0; +} diff --git a/tests/ir_tests/nested_capture_two_level.expect b/tests/ir_tests/nested_capture_two_level.expect new file mode 100644 index 000000000..7598eed3b --- /dev/null +++ b/tests/ir_tests/nested_capture_two_level.expect @@ -0,0 +1,2 @@ +two_level=21 +two_level_write=1516 diff --git a/tests/ir_tests/qemu/mps2-an505/Makefile b/tests/ir_tests/qemu/mps2-an505/Makefile index 231c8fd99..dd9b21786 100644 --- a/tests/ir_tests/qemu/mps2-an505/Makefile +++ b/tests/ir_tests/qemu/mps2-an505/Makefile @@ -83,6 +83,8 @@ NEWLIB_LIBC_G = $(NEWLIB_DIR)/libc_g.a NEWLIB_LIBM_G = $(NEWLIB_DIR)/libm_g.a EXTRA_CFLAGS ?= +# Prebuilt objects linked into the test (e.g. gcc-compiled, for ABI interop). +EXTRA_OBJS ?= CFLAGS += -nostdlib -fvisibility=hidden $(GCC_ABI_FLAGS) -ffunction-sections $(EXTRA_CFLAGS) # Two-phase compilation support (reduces memory usage) @@ -166,7 +168,7 @@ $(OUTPUT)/%.o: $(MAKEFILE_DIR)%.S | $(OUTPUT) $(CC_WRAPPER) $(CC) $(CFLAGS) -c $< -o $@ $(TARGET): $(OBJS) $(if $(filter 1,$(USE_NEWLIB_BUILD)),$(NEWLIB_DIR)/libc.a) - $(CC_WRAPPER) $(CC) $(CFLAGS) $(OBJS) $(CRT_LIBS) -o $@ $(LDFLAGS) + $(CC_WRAPPER) $(CC) $(CFLAGS) $(OBJS) $(EXTRA_OBJS) $(CRT_LIBS) -o $@ $(LDFLAGS) # Build newlib from source if not already built (soft ABI by default; other # ABIs normally use the toolchain multilib instead — see USE_NEWLIB_BUILD). diff --git a/tests/ir_tests/qemu/mps2-an505/linker_script.ld b/tests/ir_tests/qemu/mps2-an505/linker_script.ld index 032cfd6fd..e46b49f8e 100644 --- a/tests/ir_tests/qemu/mps2-an505/linker_script.ld +++ b/tests/ir_tests/qemu/mps2-an505/linker_script.ld @@ -7,8 +7,10 @@ MEMORY ENTRY (Reset_Handler) -__stack_size__ = 0x20000; /* 8KB stack */ -__heap_size__ = 0x100000; /* 64KB heap */ +/* 1 MiB: gcc torture tests declare up to 0x80100 bytes (dg-require-stack-size), + * e.g. pr28982b passes a 256 KiB struct by value next to a 256 KiB local. */ +__stack_size__ = 0x100000; /* 1 MiB stack */ +__heap_size__ = 0x100000; /* 1 MiB heap */ SECTIONS diff --git a/tests/ir_tests/qemu_run.py b/tests/ir_tests/qemu_run.py index be46cac0b..61896d453 100644 --- a/tests/ir_tests/qemu_run.py +++ b/tests/ir_tests/qemu_run.py @@ -201,6 +201,9 @@ def expect(self, pattern, timeout: int = 1): self._append_output(chunk) except OSError: pass + # A last line printed without a newline still ends here. + if self._buffer and not self._buffer.endswith("\n"): + self._buffer += "\n" m = regex.search(self._buffer) if m is not None: self.match = m @@ -350,6 +353,7 @@ class CompileConfig: # (TCC_FLOAT_ABI env var, else the Makefile's own default of soft). float_abi: Optional[str] = None fpu: Optional[str] = None + extra_objs: Optional[list] = None # prebuilt objects linked in (e.g. gcc-built, for ABI interop) def __post_init__(self): if self.compiler is None: @@ -406,7 +410,7 @@ def get_test_output_file(test_name, output_dir=None, prefix="", suffix=""): return output_dir / f"{prefix}{Path(primary).stem}{suffix}.elf" -def build_make_command(test_file, machine, compiler, output_dir=None, cflags=None, defines=None, cc_wrapper=None, two_phase=False, output_prefix="", output_suffix="", float_abi=None, fpu=None): +def build_make_command(test_file, machine, compiler, output_dir=None, cflags=None, defines=None, cc_wrapper=None, two_phase=False, output_prefix="", output_suffix="", float_abi=None, fpu=None, extra_objs=None): """Build the make command for compiling a test case.""" make_dir = CURRENT_DIR / 'qemu' / machine test_files = [str(f) for f in _as_file_list(test_file)] @@ -443,6 +447,8 @@ def build_make_command(test_file, machine, compiler, output_dir=None, cflags=Non cmd.append(f"FLOAT_ABI={float_abi}") if fpu: cmd.append(f"FPU={fpu}") + if extra_objs: + cmd.append(f"EXTRA_OBJS={' '.join(str(o) for o in extra_objs)}") return cmd @@ -785,6 +791,7 @@ def compile_testcase(test_file, machine, compiler=None, cflags=None, config=None output_suffix=config.output_suffix, float_abi=config.float_abi or DEFAULT_FLOAT_ABI, fpu=config.fpu or DEFAULT_FPU, + extra_objs=config.extra_objs, ) # Clean if needed diff --git a/tests/ir_tests/sl_forward_load_redefines_var.c b/tests/ir_tests/sl_forward_load_redefines_var.c new file mode 100644 index 000000000..8586311c1 --- /dev/null +++ b/tests/ir_tests/sl_forward_load_redefines_var.c @@ -0,0 +1,46 @@ +/* sl_forward_load_redefines_var.c — store-load forwarding must not forward a + * variable's value across a LOAD that redefines that variable. + * + * `u3` is written both directly and through `**pp` (pp == &p, p == &u3). + * ptr_local_fwd rewrites the pointer accesses into direct `u3` accesses, and + * sl_forward then tracks `u3 <- T` and forwards T into later reads of u3. + * Between them sits `u3 = m[g][2]`, which lowers to `V1 <-- T***DEREF*** + * [LOAD]` — a genuine redefinition of u3. + * + * sl_forward's redefinition scan handles exactly that shape, but the LOAD + * branch reached it only when it had something to forward INTO the load: an + * unresolvable address (here `T25 + 8`, not in the LEA map), a volatile read + * or an address-taken base all did `continue`, skipping the rest of the loop + * body — including the scan. The entry from the earlier store stayed live and + * the second read forwarded a stale value. Those three now `goto + * skip_load_fwd` so the destination's bookkeeping still runs. + * + * Found by tests/fuzz agg_deep seed 3292 (tcc -O2/-Os wrong, -O0/-O1 and both + * gcc levels right), reduced to this. + */ +#include + +static unsigned csmix(unsigned h, unsigned v) +{ + h ^= v + 0x9e3779b9u + (h << 6) + (h >> 2); + return h * 2654435761u; +} + +int main(void) +{ + unsigned cs = 0x12345678u; + unsigned u3 = 140438262u; + unsigned u4 = 2784757854u; + unsigned m28[4][4] = { {1,2,3,4}, {5,6,7,8}, {9,10,11,12}, {13,14,15,16} }; + unsigned *pa29 = &u3; + unsigned **ppa210 = &pa29; + + for (unsigned g = 0u; g < 4u; g++) { + **ppa210 = m28[(u4 & 3u)][(u3 & 3u)]; + cs = csmix(cs, *pa29); + u3 = m28[(g & 3u)][2] & 0xffffffffu; /* redefines u3 via a LOAD */ + cs = csmix(cs, **ppa210); /* must see the redefinition */ + } + printf("checksum=%08x\n", cs); + return 0; +} diff --git a/tests/ir_tests/sl_forward_load_redefines_var.expect b/tests/ir_tests/sl_forward_load_redefines_var.expect new file mode 100644 index 000000000..b58bd1173 --- /dev/null +++ b/tests/ir_tests/sl_forward_load_redefines_var.expect @@ -0,0 +1 @@ +checksum=e8832739 diff --git a/tests/ir_tests/test_alloca_live_size.c b/tests/ir_tests/test_alloca_live_size.c new file mode 100644 index 000000000..c15e068e5 --- /dev/null +++ b/tests/ir_tests/test_alloca_live_size.c @@ -0,0 +1,35 @@ +/* alloca's size can stay live after the allocation: the new stack top was + * computed in the size's own register, so every later use of n read the new + * SP instead -- memset got a stack address as its length and faulted. A VLA + * whose length variable is read afterwards is the same shape. */ +#include +#include + +static volatile int sunk; + +__attribute__((noinline)) static void sink(int x) +{ + sunk += x; +} + +__attribute__((noinline)) static int with_alloca(int n) +{ + char *p = __builtin_alloca(n); + memset(p, 1, n); + sink(p[n - 1]); + return p[0] + n; +} + +__attribute__((noinline)) static int with_vla(int n) +{ + char buf[n]; + memset(buf, 2, n); + sink(buf[n - 1]); + return buf[0] * 100 + n; +} + +int main(void) +{ + printf("%d %d\n", with_alloca(40), with_vla(24)); + return 0; +} diff --git a/tests/ir_tests/test_alloca_live_size.expect b/tests/ir_tests/test_alloca_live_size.expect new file mode 100644 index 000000000..24f8d3369 --- /dev/null +++ b/tests/ir_tests/test_alloca_live_size.expect @@ -0,0 +1 @@ +41 224 diff --git a/tests/ir_tests/test_call_buffer_copy_alias.c b/tests/ir_tests/test_call_buffer_copy_alias.c new file mode 100644 index 000000000..dac5ad04d --- /dev/null +++ b/tests/ir_tests/test_call_buffer_copy_alias.c @@ -0,0 +1,130 @@ +/* The Zig C backend copies a call's struct result into a second local before + * reading it: `t3 = f(); t4 = t3; p = &t4;`. When the call's own buffer is + * written by nothing else and the second local by nothing but the copy, the + * two hold the same bytes from the copy onwards whatever the control flow, so + * the copy goes and the local is the buffer -- anywhere in the function, not + * just the entry block. What must keep its copy: a source read after the + * copy, a destination written by anything else, a copy not next to its call, + * and two locals copied from one buffer. */ +#include +#include + +struct big +{ + uint32_t a[8]; +}; + +__attribute__((noinline)) static struct big make(int x, int y) +{ + struct big b; + for (int i = 0; i < 8; i++) + b.a[i] = (uint32_t)(x * (i + 1) + y); + return b; +} + +__attribute__((noinline)) static void bump(struct big *p, uint32_t k) +{ + p->a[0] += k; +} + +/* the plain shape, outside the entry block */ +__attribute__((noinline)) static uint32_t in_branch(int x, int sel) +{ + struct big t3; + struct big t4; + const struct big *t5; + uint32_t r = 0; + if (sel & 1) + { + r += 7; + t3 = make(x, 1); + t4 = t3; + t5 = (const struct big *)&t4; + r += t5->a[3] + t5->a[7]; + } + else + { + t3 = make(x, 2); + t4 = t3; + r += t4.a[1]; + } + return r; +} + +/* in a loop, with the destination read before the copy runs again */ +__attribute__((noinline)) static uint32_t in_loop(int x, int n) +{ + struct big s; + struct big d; + uint32_t r = 0; + for (int i = 0; i < n; i++) + { + if (i) + r += d.a[2]; /* the previous iteration's value */ + s = make(x + i, 3); + d = s; + r = r * 3 + d.a[0]; + } + return r; +} + +/* the destination is written through a pointer: the writes must stick */ +__attribute__((noinline)) static uint32_t written_through_pointer(int x) +{ + struct big s; + struct big d; + s = make(x, 4); + d = s; + bump(&d, 100); + bump(&d, 1000); + return d.a[0] + d.a[5]; +} + +/* the source is read after the copy: both must keep their own bytes */ +__attribute__((noinline)) static uint32_t source_read_after(int x) +{ + struct big s; + struct big d; + s = make(x, 5); + d = s; + bump(&d, 10000); + return d.a[0] * 3 + s.a[0]; +} + +/* two destinations from one buffer */ +__attribute__((noinline)) static uint32_t two_copies(int x) +{ + struct big s; + struct big d1; + struct big d2; + s = make(x, 6); + d1 = s; + d2 = s; + bump(&d1, 5); + return d1.a[0] * 2 + d2.a[0]; +} + +/* something between the call and the copy */ +__attribute__((noinline)) static uint32_t not_adjacent(int x, uint32_t *out) +{ + struct big s; + struct big d; + s = make(x, 7); + *out = s.a[4]; + d = s; + bump(&d, 3); + return d.a[0] + *out; +} + +int main(void) +{ + unsigned r = 0; + for (int x = 1; x < 6; x++) + r = r * 31 + in_branch(x, x); + printf("%u\n", r); + printf("%u %u\n", in_loop(2, 4), in_loop(3, 1)); + printf("%u %u\n", written_through_pointer(2), source_read_after(3)); + uint32_t out = 0; + printf("%u %u %u\n", two_copies(4), not_adjacent(5, &out), out); + return 0; +} diff --git a/tests/ir_tests/test_call_buffer_copy_alias.expect b/tests/ir_tests/test_call_buffer_copy_alias.expect new file mode 100644 index 000000000..56016f50a --- /dev/null +++ b/tests/ir_tests/test_call_buffer_copy_alias.expect @@ -0,0 +1,4 @@ +19616311 +614 6 +1122 30032 +40 47 32 diff --git a/tests/ir_tests/test_codegen_asm.py b/tests/ir_tests/test_codegen_asm.py index da9a0e425..76149b2f0 100644 --- a/tests/ir_tests/test_codegen_asm.py +++ b/tests/ir_tests/test_codegen_asm.py @@ -212,8 +212,9 @@ def test_struct_packed_9byte_by_value(): load_mnems = {"ldr", "ldr.w", "ldrh", "ldrsh", "ldrsh.w", "ldrb", "ldrsb"} load_count = sum(_count_mnem(consume, m) for m in load_mnems) - # Caller currently copies the by-value struct with __aeabi_memmove. - assert any("__aeabi_memmove" in ops for _, ops in caller), "caller missing __aeabi_memmove copy" + # `struct S s = make();` returns into s's own slot; the store of s onto + # itself used to be an __aeabi_memmove and is now skipped. + assert not any("__aeabi_memmove" in ops for _, ops in caller), "caller copies s onto itself" # Callee loads unaligned packed fields. assert load_count >= 2, f"consume expected at least 2 loads, got {load_count}" # No undefined/breakpoint instructions (i.e. no obviously broken encoding). @@ -368,19 +369,160 @@ def test_size_flags_alias_o2(): `s->optimize = atoi(optarg)` turned both into 0, so -Os meant "no optimization" -- and since -O is a plain option scan, `-O2 ... -Os` (which - is what toybox passes) threw the -O2 away. They alias -O2 until there is - an actual size tier to point them at. + is what toybox passes) threw the -O2 away. They run the -O2 pipeline + (test_size_tier covers where they part from it), and -Oz is -Os. """ o0 = _disassemble(_compile("dead_loop_rotated", extra_cflags=["-O0"])) - o2 = _disassemble(_compile("dead_loop_rotated", extra_cflags=["-O2"])) os_ = _disassemble(_compile("dead_loop_rotated", extra_cflags=["-Os"])) oz = _disassemble(_compile("dead_loop_rotated", extra_cflags=["-Oz"])) + o2_os = _disassemble(_compile("dead_loop_rotated", extra_cflags=["-O2", "-Os"])) - assert os_ == o2, "-Os should select the same tier as -O2" - assert oz == o2, "-Oz should select the same tier as -O2" + assert oz == os_, "-Oz should select the same tier as -Os" + assert o2_os == os_, "a later -Os should win over an earlier -O2" assert os_ != o0, "-Os is still a silent -O0" +def test_size_tier(): + """-Os: where a choice trades code size for speed, the smaller code. + + - no word-alignment NOPs before loop heads and in-loop joins (-O2 pads + them for the M33's fetch; 72 KB of zig.c), + - an aligned block copy longer than the largest copy stub stays a + memmove call instead of an inline LDM/STM loop, + - __OPTIMIZE_SIZE__ is defined, as gcc does. + """ + o2 = _disassemble(_compile("size_tier", extra_cflags=["-O2"])) + os_ = _disassemble(_compile("size_tier", extra_cflags=["-Os"])) + + pads_o2 = sum(_count_mnem_regex(fn, r"^nop") for fn in o2.values()) + pads_os = sum(_count_mnem_regex(fn, r"^nop") for fn in os_.values()) + assert pads_o2 > 0, "the -O2 build should pad some loop head (case no longer exercises alignment)" + assert pads_os == 0, f"-Os padded {pads_os} branch targets" + + assert _count_mnem_regex(o2["copy_big"], r"^ldmia") > 0, "-O2 should copy 160 bytes inline" + assert _count_mnem_regex(os_["copy_big"], r"^bl\b") == 1, f"-Os should call memmove: {os_['copy_big']}" + assert _count_mnem_regex(os_["copy_big"], r"^ldm") == 0, "-Os still copies inline" + + assert os_["size_macro"][0] == ("movs", "r0, #1"), "__OPTIMIZE_SIZE__ not defined at -Os" + assert o2["size_macro"][0] == ("movs", "r0, #0"), "__OPTIMIZE_SIZE__ defined at -O2" + + +def test_branch_over_pool_window_narrows(): + """A forward B over more than a literal-pool window is 16-bit if it fits. + + The then-arm's jump over the 1 KB else-arm used to stay B.W because a pool + dump could land inside the range -- and one does (the arm's second B.W is + that dump's skip branch). A 16-bit B is safe whenever its offset fits with + every dump the range can take; 7,900 branches in zig.c. + """ + fn = _disassemble(_compile("branch_over_pool_window"))["over_pool_window"] + branches = [(m, o) for m, o in fn if re.match(r"^b(\.[nw])?$", m)] + assert branches and branches[0][0] == "b.n", f"jump over the else-arm is wide: {branches[:2]}" + assert any(m == "b.w" for m, _ in branches[1:]), "no pool dump inside the range any more" + + +def test_cmp_negative_small_is_adds(): + """CMP Rn, #-k (k = 1..7, Rn low) is ADDS Rt, Rn, #k into a free low register. + + Rn - (-k) and Rn + k are one sum with one carry and one overflow, so the + flags match, in 16 bits where the compare needs CMP.W (Zig compares + against maxInt(u32), its `none` index, 1,800 times in zig.c). + """ + funcs = _disassemble(_compile("cmp_neg_small")) + for name, k in (("is_none", 1), ("below_minus3", 3)): + fn = funcs[name] + assert _count_mnem_regex(fn, r"^(cmp|cmn)") == 0, f"{name} still compares: {fn}" + assert _count_mnem_regex(fn, rf"^adds r[0-7], r0, #{k}$") == 1, f"{name}: {fn}" + + +def test_literal_body_word_aligned(): + """A body that reads its literal pool starts word-aligned. + + A literal load's offset and the pad before a pool dump depend on the + address mod 4, so the same function at a 2 mod 4 start is different + machine code, and identical code folding compares machine code: lit_b + only folds onto lit_a when both start aligned. + """ + obj = _compile("func_align_literal", extra_cflags=["-fno-function-sections"]) + syms = {} + out = subprocess.run(["arm-none-eabi-nm", str(obj)], stdout=subprocess.PIPE, text=True).stdout + for line in out.splitlines(): + parts = line.split() + if len(parts) == 3: + syms[parts[2]] = int(parts[0], 16) + assert syms["pad_before"] == 0 and syms["lit_a"] % 4 == 0, f"lit_a not word-aligned: {syms}" + assert syms["lit_b"] == syms["lit_a"], f"identical lit_b was not folded onto lit_a: {syms}" + + +def test_hi_callee_pref(): + """More call-crossing values than r4-r7: the compared one keeps a low register. + + Five pointers are only passed from call to call; `k` is compared with a + constant after each. First come first served gave the pointers r4-r7 and + `k` a high register, so every compare was CMP.W; ra:hi_callee_pref gives + values at most one 16-bit-capable op touches r8-r11 first. + """ + fn = _disassemble(_compile("hi_callee_pref"))["keep_low"] + cmps = [(m, o) for m, o in fn if m.startswith("cmp")] + assert len(cmps) == 4, f"expected four compares: {fn}" + assert all(m == "cmp" and re.match(r"r[0-7], #", o) for m, o in cmps), f"a compare is wide: {cmps}" + + +def test_has_builtin(): + """__has_builtin answers for the builtins tcc implements. + + It was `#define __has_builtin(x) 0`, so zig.h built its byte swaps from + shifts and ORs. It must also expand outside #if: zig.h pastes the 0/1 + answer into a name to choose a libm import. + """ + funcs = _disassemble(_compile("has_builtin", extra_cflags=["-nostdinc", f"-I{ROOT / 'include'}"])) + assert _count_mnem_regex(funcs["swap16"], r"^rev16") == 1, f"swap16: {funcs['swap16']}" + assert _count_mnem_regex(funcs["swap32"], r"^rev ") == 1, f"swap32: {funcs['swap32']}" + assert ("movs", "r0, #10") in funcs["answers"], f"answers: {funcs['answers']}" + + +def test_shift64_var_os_calls_helper(): + """-Os: a 64-bit shift by a variable count is the __aeabi_* helper call. + + Inline it is eight instructions at every site (440 of them in zig.c); + -O2 keeps that, -Os takes the call. A constant count stays inline. + """ + o2 = _disassemble(_compile("shift64_var", extra_cflags=["-O2"])) + os_ = _disassemble(_compile("shift64_var", extra_cflags=["-Os"])) + for name, helper in (("shl", "__aeabi_llsl"), ("shr", "__aeabi_llsr"), ("sar", "__aeabi_lasr")): + assert _count_mnem_regex(o2[name], helper) == 0, f"-O2 {name} calls {helper}" + assert _count_mnem_regex(os_[name], helper) == 1, f"-Os {name}: {os_[name]}" + assert _count_mnem_regex(os_["shl_const"], r"__aeabi") == 0, "a constant shift became a call" + + +def test_cmp_type_extreme_folds(): + """A compare against an end of its operand's own range is constant. + + `x > 0xFFFFFFFFu` and `x < INT32_MIN` hold for no x: VRP folds the + CMP whether written directly or made so by inlining -- zig.h's overflow + wrappers check every result against zig_minInt/zig_maxInt of its full + width that way. (A frontend gen_opic fold of the same compares broke + the self-hosted tcc on the device; see the commit.) + """ + funcs = _disassemble(_compile("cmp_type_extreme", + extra_cflags=["-nostdinc", f"-I{ROOT / 'include'}", + f"-I{Path(__file__).parent / 'libc_includes' / 'newlib'}"])) + for name in ("u_above_max", "s_below_min", "inlined"): + assert _count_mnem_regex(funcs[name], r"^cmp") == 0, f"{name} still compares: {funcs[name]}" + + +def test_it_mov_narrow(): + """A small constant moved inside an IT block uses the 16-bit encoding. + + Inside an IT block MOV #imm8 sets no flags, so SELECT arms and the + BOOL_AND/BOOL_OR result need not be the flag-preserving MOV.W. + """ + funcs = _disassemble(_compile("it_mov_narrow")) + for name, fn in funcs.items(): + wide = [(m, o) for m, o in fn if re.match(r"^mov(eq|ne|lt|ge|gt|le|cc|cs|hi|ls)\.w$", m) and re.match(r"r[0-7], #", o)] + assert not wide, f"{name} has a wide conditional move: {fn}" + + def test_switch_const_selector_folds(): """ssa:switch_fold: `switch (7)` picks its arm at compile time. diff --git a/tests/ir_tests/test_complex_param_parts.c b/tests/ir_tests/test_complex_param_parts.c new file mode 100644 index 000000000..7ae069105 --- /dev/null +++ b/tests/ir_tests/test_complex_param_parts.c @@ -0,0 +1,38 @@ +/* Writing one half of a complex parameter: __imag__ x = ... must land in the + * imaginary half whether x arrived wholly on the stack or split across r2-r3 + * and the stack (the IR names both halves through x's vreg, the imaginary one + * at a sub-offset of its slot). */ +#include + +__attribute__((noinline)) static double tw(double v) { return v * 2.0 + 1.0; } + +__attribute__((noinline)) static _Complex double on_stack(int a, int b, int c, int d, _Complex double x) +{ + __imag__ x = tw(__imag__ x) + d; + __real__ x = tw(__real__ x) + a + b + c; + return x; +} + +__attribute__((noinline)) static _Complex double split(_Complex double x) +{ + __imag__ x = tw(__imag__ x); + return x; +} + +__attribute__((noinline)) static _Complex float split_f(int a, int b, int c, _Complex float x) +{ + __imag__ x = (float)tw(__imag__ x) + a; + __real__ x = (float)tw(__real__ x) + b + c; + return x; +} + +int main(void) +{ + _Complex double r = on_stack(1, 2, 3, 4, 5.0 + 7.0i); + printf("%d %d\n", (int)__real__ r, (int)__imag__ r); + r = split(3.0 + 9.0i); + printf("%d %d\n", (int)__real__ r, (int)__imag__ r); + _Complex float f = split_f(1, 2, 3, 6.0f + 8.0fi); + printf("%d %d\n", (int)__real__ f, (int)__imag__ f); + return 0; +} diff --git a/tests/ir_tests/test_complex_param_parts.expect b/tests/ir_tests/test_complex_param_parts.expect new file mode 100644 index 000000000..0602d8e4b --- /dev/null +++ b/tests/ir_tests/test_complex_param_parts.expect @@ -0,0 +1,3 @@ +17 19 +3 19 +18 18 diff --git a/tests/ir_tests/test_cross_jump.c b/tests/ir_tests/test_cross_jump.c new file mode 100644 index 000000000..130673e78 --- /dev/null +++ b/tests/ir_tests/test_cross_jump.c @@ -0,0 +1,116 @@ +/* Identical block tails that jump to the same place share one copy + * (cross_jump). A chain of fallible steps, each failure returning the same + * way; tails with a goto label inside them, whose entry must keep running only + * the part after it; tails that compare and branch; identical continue paths + * in a loop. */ +#include +#include + +struct EU { uint32_t payload; uint16_t error; }; + +__attribute__((noinline)) static struct EU step(uint32_t k, uint32_t fail_at, uint32_t id) +{ + struct EU r = {k * 3 + id, 0}; + if (id == fail_at) + r.error = (uint16_t)(100 + id); + return r; +} + +__attribute__((noinline)) static struct EU chain(uint32_t k, uint32_t fail_at) +{ + struct EU r, t; + t = step(k, fail_at, 1); + if (t.error) { r.payload = 0xaaaaaaaau; r.error = t.error; return r; } + uint32_t a = t.payload; + t = step(a, fail_at, 2); + if (t.error) { r.payload = 0xaaaaaaaau; r.error = t.error; return r; } + uint32_t b = t.payload; + t = step(a + b, fail_at, 3); + if (t.error) { r.payload = 0xaaaaaaaau; r.error = t.error; return r; } + uint32_t c = t.payload; + t = step(c ^ a, fail_at, 4); + if (t.error) { r.payload = 0xaaaaaaaau; r.error = t.error; return r; } + r.payload = t.payload + b + c; + r.error = 0; + return r; +} + +/* The same tail twice, with a label inside the second copy: entering at the + * label must run only what follows it. */ +__attribute__((noinline)) static uint32_t labels(uint32_t k, volatile uint32_t *log) +{ + uint32_t acc = k; + if (k & 1) + { + acc += 11; + log[0] += acc; + log[1] ^= acc; + goto out; + } + if (k & 2) + goto mid; + acc += 11; +mid: + log[0] += acc; + log[1] ^= acc; +out: + return acc * 3 + log[0] + log[1]; +} + +/* Identical tails that compute a comparison before jumping on. */ +__attribute__((noinline)) static uint32_t compares(uint32_t a, uint32_t b, uint32_t c) +{ + uint32_t r; + if (a > b) + { + r = (a < c) + (b < c) * 2; + goto done; + } + if (b > c) + { + r = (a < c) + (b < c) * 2; + goto done; + } + r = 7; +done: + return r * 10 + a; +} + +/* Identical continue paths in a loop. */ +__attribute__((noinline)) static uint32_t loop(uint32_t n) +{ + uint32_t s = 0, t = 1; + for (uint32_t i = 0; i < n; i++) + { + if (i % 3 == 0) + { + s += t; + t = t * 5 + 1; + continue; + } + if (i % 5 == 0) + { + s += t; + t = t * 5 + 1; + continue; + } + s ^= i; + } + return s + t; +} + +int main(void) +{ + for (uint32_t f = 0; f <= 5; f++) + { + struct EU r = chain(7, f); + printf("%u:%u ", r.payload, r.error); + } + printf("\n"); + volatile uint32_t log[2] = {0, 0}; + for (uint32_t k = 0; k < 6; k++) + printf("%u ", labels(k, log)); + printf("\n%u %u %u %u\n", compares(5, 3, 9), compares(1, 4, 2), compares(1, 2, 3), compares(9, 1, 4)); + printf("%u %u\n", loop(20), loop(47)); + return 0; +} diff --git a/tests/ir_tests/test_cross_jump.expect b/tests/ir_tests/test_cross_jump.expect new file mode 100644 index 000000000..355e09a87 --- /dev/null +++ b/tests/ir_tests/test_cross_jump.expect @@ -0,0 +1,4 @@ +1134:0 2863311530:101 2863311530:102 2863311530:103 2863311530:104 1134:0 +55 66 36 92 103 138 +35 11 71 29 +3051747 3169635982 diff --git a/tests/ir_tests/test_drop_unused_statics.py b/tests/ir_tests/test_drop_unused_statics.py new file mode 100644 index 000000000..32a7c5415 --- /dev/null +++ b/tests/ir_tests/test_drop_unused_statics.py @@ -0,0 +1,198 @@ +"""-fdrop-unused-statics: which statics reach the object file. + +At -O1 and above, static functions and initialized static objects are +deferred to the end of the TU, and only those something live refers to are +emitted (prune_unused_statics, frontend/gen/inline/emit.c). Generated C -- +zig's C backend -- defines every helper and constant table it might need; +before this, all of them landed in the object. + +Runtime correctness of the deferred shapes is 475_drop_unused_statics.c; +this checks the symbol table. +""" + +import shutil +import subprocess +from pathlib import Path + +import pytest + +CURRENT_DIR = Path(__file__).parent +TCC = CURRENT_DIR.parent.parent / "armv8m-tcc" +READELF = shutil.which("arm-none-eabi-readelf") + +pytestmark = pytest.mark.skipif( + READELF is None or not TCC.exists(), + reason="needs armv8m-tcc and arm-none-eabi-readelf", +) + +SOURCE = r""" +typedef long long wide; + +/* Unused: every one of these is dropped. */ +__attribute__((noinline)) static int dead_func(int x) { return x + 1; } +static const int dead_table[4] = {1, 2, 3, 4}; +static const char dead_string[] = "never printed"; +static int dead_bss; +static int dead_data = 5; +/* Referenced only from dead code. */ +static int only_from_dead_value = 7; +static int dead_caller(void) { return only_from_dead_value + dead_func(1); } +/* Referenced only from an unused static inline body. */ +static int only_from_inline = 9; +static inline int dead_inline(void) { return only_from_inline; } +/* zig.h's shape: a block-scope prototype, not a call to an undeclared name. */ +static wide dead_libcall_wrapper(wide a, wide b) +{ + extern wide __aeabi_ldivmod_like(wide, wide); + return __aeabi_ldivmod_like(a, b); +} +/* Forward-declared const object, defined later, never used. */ +struct pair { int a, b; }; +static const struct pair dead_fwd_const; +static const struct pair dead_fwd_const = {1, 2}; + +/* Live: reached from the exported function, directly or through data. */ +__attribute__((noinline)) static int live_callee(int x) { return x * 3; } +static int live_via_table(int x) { return x - 1; } +static int (*const live_table[])(int) = {live_via_table}; +static int live_counter = 2; +__attribute__((used)) static int kept_by_used = 3; + +int exported(int x) { return live_callee(x) + live_table[0](x) + live_counter++; } +""" + +DEAD = [ + "dead_func", + "dead_table", + "dead_string", + "dead_bss", + "dead_data", + "only_from_dead_value", + "dead_caller", + "only_from_inline", + "dead_inline", + "dead_libcall_wrapper", + "dead_fwd_const", +] +LIVE = ["exported", "live_callee", "live_via_table", "live_table", "live_counter", "kept_by_used"] + + +def _defined_symbols(tmp_path, *flags): + src = tmp_path / "statics.c" + src.write_text(SOURCE) + obj = tmp_path / "statics.o" + proc = subprocess.run([str(TCC), "-c", *flags, "-o", str(obj), str(src)], capture_output=True, text=True) + assert proc.returncode == 0, proc.stdout + proc.stderr + out = subprocess.run([READELF, "-sW", str(obj)], capture_output=True, text=True, check=True).stdout + names = set() + for line in out.splitlines(): + cols = line.split() + # Num: Value Size Type Bind Vis Ndx Name -- defined = a real section index. + if len(cols) == 8 and cols[6].isdigit(): + names.add(cols[7]) + return names + + +@pytest.mark.parametrize("opt", ["-O1", "-O2"]) +def test_unused_statics_are_dropped(tmp_path, opt): + names = _defined_symbols(tmp_path, opt) + assert not names & set(DEAD), f"emitted though unused: {sorted(names & set(DEAD))}" + assert set(LIVE) <= names, f"live but missing: {sorted(set(LIVE) - names)}" + + +@pytest.mark.parametrize("flags", [("-O0",), ("-O1", "-fno-drop-unused-statics")]) +def test_everything_kept_without_the_option(tmp_path, flags): + names = _defined_symbols(tmp_path, *flags) + # dead_inline is a static inline: never emitted unless referenced. + expected = set(DEAD + LIVE) - {"dead_inline"} + assert expected <= names, f"missing: {sorted(expected - names)}" + + +# A dropped static is still parsed for its diagnostics (check_dropped_statics), +# with nothing it produces reaching the object. + +def _compile(tmp_path, source, *flags): + src = tmp_path / "check.c" + src.write_text(source) + obj = tmp_path / "check.o" + proc = subprocess.run([str(TCC), "-c", "-O1", *flags, "-o", str(obj), str(src)], capture_output=True, text=True) + return proc, obj + + +@pytest.mark.parametrize( + "dead", + [ + "static int dead(int x) { return x + undeclared_name; }", + "static short dead = (int)&main;", + "static void dead(void) { static short s = (int)&main; (void)s; }", + ], +) +def test_error_in_dropped_static_is_reported(tmp_path, dead): + proc, _ = _compile(tmp_path, "int main(void);\n" + dead + "\nint main(void) { return 0; }\n") + assert proc.returncode != 0 and "error" in proc.stderr, proc.stderr + + +def test_warning_in_dropped_function_is_reported(tmp_path): + proc, _ = _compile(tmp_path, "static void dead(void) { int *p = 5; (void)p; }\nint main(void) { return 0; }\n") + assert proc.returncode == 0, proc.stderr + assert "warning" in proc.stderr + + +def test_dropped_statics_leave_nothing_behind(tmp_path): + # Everything a body can put in a section or the symbol table, in dropped + # functions: the object must hold main and nothing else. + source = r""" +extern int only_named_by_dead_code(void); +static const char *tbl[] = {"x", "y"}; +static const char *d1(void) { return "string literal"; } +static void *d2(void) { lab: return &&lab; } +static double d3(double x) { return x * 1.2345; } +static const char *d4(void) { return __func__; } +static int d5(void) { return ((struct { int a; }){7}).a; } +static int d6(void) { static int local_static = 3; return local_static + tbl[0][0]; } +static int d7(void) { char buf[] = "a longer string literal"; return buf[3]; } +static int d8(void) { int big[64] = {1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20}; return big[7]; } +static int (*d9(void))(void) { return only_named_by_dead_code; } +static int d10(int x) { int nested(int y) { return x + y; } int (*p)(int) = nested; return p(1); } +static int d11(int x) { int r; __asm__ volatile("adds %0, %1, #1" : "=r"(r) : "r"(x)); return r; } +int main(void) { return 0; } +""" + proc, obj = _compile(tmp_path, source) + assert proc.returncode == 0, proc.stderr + out = subprocess.run([READELF, "-sW", str(obj)], capture_output=True, text=True, check=True).stdout + names = [cols[7] for cols in (line.split() for line in out.splitlines()) + if len(cols) == 8 and cols[0][:-1].isdigit() and cols[3] not in ("FILE", "SECTION") and cols[7] != "$t"] + assert names == ["main"], names + sizes = subprocess.run(["arm-none-eabi-size", "-A", str(obj)], capture_output=True, text=True, check=True).stdout + for line in sizes.splitlines(): + cols = line.split() + if cols and cols[0] in (".data", ".rodata", ".bss"): + assert cols[1] == "0", line + + +# What the optimizer leaves unreferenced is collected from the object +# (gc_unreferenced_statics): a write-only static, once dead_static_store has +# removed its stores -- including zig's `*(&((T *)&x)->f) = v` field stores +# and struct copies into it -- goes, with the functions only it pointed to. + +def test_write_only_static_and_its_closure_are_collected(tmp_path): + source = r""" +struct big { int v[12]; }; +extern struct big get(void); +static void only_in_initializer(void) {} +struct holder { int a; struct big b; void (*f)(void); }; +static struct holder write_only = {1, {{0}}, only_in_initializer}; +int main(void) +{ + (*(&((struct holder *)&write_only)->a)) = 7; + (*(&((struct holder *)&write_only)->b)) = get(); + return 0; +} +""" + proc, obj = _compile(tmp_path, source, "-O2") + assert proc.returncode == 0, proc.stderr + out = subprocess.run([READELF, "-sW", str(obj)], capture_output=True, text=True, check=True).stdout + defined = [cols[7] for cols in (line.split() for line in out.splitlines()) + if len(cols) == 8 and cols[0][:-1].isdigit() and cols[6].isdigit()] + assert "write_only" not in defined and "only_in_initializer" not in defined, defined + assert "main" in defined diff --git a/tests/ir_tests/test_dse_object_ranges.c b/tests/ir_tests/test_dse_object_ranges.c new file mode 100644 index 000000000..a6273036e --- /dev/null +++ b/tests/ir_tests/test_dse_object_ranges.c @@ -0,0 +1,59 @@ +/* Dead-store elimination bounds a read through an address by the object the + * address points into (dse.c, tcc_ir_frame_object_at). The stores into an + * object read only through a pointer -- passed to a call, indexed at run time, + * reached from an interior pointer -- must stay; the objects around a large + * one whose address is taken must keep theirs too. */ +#include + +struct big +{ + int v[700]; /* past the 2 KiB the byte set can mark */ +}; + +struct pair +{ + int a, b; +}; + +__attribute__((noinline)) static int read_at(const int *p, int i) +{ + return p[i]; +} + +__attribute__((noinline)) static int sum_pair(const struct pair *p) +{ + return p->a * 10 + p->b; +} + +__attribute__((noinline)) static int f(int k) +{ + int before[4]; + struct big b; + struct pair pr; + int after[3]; + + for (int i = 0; i < 4; i++) + before[i] = 100 + i; + for (int i = 0; i < 700; i++) + b.v[i] = i * 3; + pr.a = k; + pr.b = k + 1; + after[0] = 7; + after[1] = 8; + after[2] = 9; + + /* An interior pointer reaching back to the start of its object. */ + const int *mid = &after[1]; + int s = mid[-1] + mid[1]; + + s += read_at(b.v, k * 7); /* the big object, through a call */ + s += read_at(before, k & 3); /* indexed at run time */ + s += sum_pair(&pr); /* a struct through its address */ + return s; +} + +int main(void) +{ + printf("%d %d\n", f(3), f(10)); + return 0; +} diff --git a/tests/ir_tests/test_dse_object_ranges.expect b/tests/ir_tests/test_dse_object_ranges.expect new file mode 100644 index 000000000..c3fea366a --- /dev/null +++ b/tests/ir_tests/test_dse_object_ranges.expect @@ -0,0 +1 @@ +216 439 diff --git a/tests/ir_tests/test_dse_wide.c b/tests/ir_tests/test_dse_wide.c new file mode 100644 index 000000000..9a8ef146d --- /dev/null +++ b/tests/ir_tests/test_dse_wide.c @@ -0,0 +1,44 @@ +/* Dead-store elimination runs on functions with 64-bit values. It used to + * give up on any function with a double or long long operand; the stores into + * objects read through a pointer, a field, or at a partial width must stay, + * and a 64-bit temporary no one reads may go. */ +#include +#include + +struct S +{ + double a, b, *c; + unsigned long long d; +}; + +__attribute__((noinline)) static double sum(const struct S *s) +{ + return s->a + s->b + s->c[0] + s->c[3] + (double)s->d; +} + +__attribute__((noinline)) static long long low_word(const long long *p) +{ + int w; + memcpy(&w, p, sizeof w); /* reads only part of the object */ + return w; +} + +__attribute__((noinline)) static double f(int k) +{ + double c[4] = {1.5, 2.5, 3.5, 4.5}; + struct S s; + long long dead = 12345678901LL * k; /* never read */ + long long half = 0x100000002LL * k; + s.a = k; + s.b = 0.25; + s.c = c; + s.d = 1ULL << 40; + (void)dead; + return sum(&s) + (double)low_word(&half); +} + +int main(void) +{ + printf("%.2f %.2f\n", f(1), f(3)); + return 0; +} diff --git a/tests/ir_tests/test_dse_wide.expect b/tests/ir_tests/test_dse_wide.expect new file mode 100644 index 000000000..197a5a78e --- /dev/null +++ b/tests/ir_tests/test_dse_wide.expect @@ -0,0 +1 @@ +1099511627785.25 1099511627791.25 diff --git a/tests/ir_tests/test_entry_store_var_walk.c b/tests/ir_tests/test_entry_store_var_walk.c new file mode 100644 index 000000000..185708f03 --- /dev/null +++ b/tests/ir_tests/test_entry_store_var_walk.c @@ -0,0 +1,39 @@ +/* A pointer VAR walked over a local array has two definitions -- the start + * and the increment -- and the loads through it must read each element. + * entry_store_prop counted definitions skipping every lvalue destination, + * which also skips a VAR's own (a STACKOFF lvalue), took the walker for a + * single-definition pointer and forwarded the last entry-block store into + * every iteration: 3+4 summed as 4+4. */ +#include + +__attribute__((noinline)) static int walk(void) +{ + int x[2] = {1, 2}; + char gap[64] __attribute__((unused)); + int y[2] = {3, 4}; + const int *e = y + 2; + int s = 0; + for (const int *p = y; p != e; p++) + s += *p; + return s * 100 + x[0] * 10 + x[1]; +} + +__attribute__((noinline)) static int walk_back(void) +{ + int y[2] = {7, 9}; + int s = 0; + const int *p = &y[1]; + const int *const lo = y; + while (p >= lo) + { + s = s * 10 + *p; + p--; + } + return s; +} + +int main(void) +{ + printf("walk=%d back=%d\n", walk(), walk_back()); + return 0; +} diff --git a/tests/ir_tests/test_entry_store_var_walk.expect b/tests/ir_tests/test_entry_store_var_walk.expect new file mode 100644 index 000000000..517dd7e60 --- /dev/null +++ b/tests/ir_tests/test_entry_store_var_walk.expect @@ -0,0 +1 @@ +walk=712 back=97 diff --git a/tests/ir_tests/test_errunion_bool_checks.c b/tests/ir_tests/test_errunion_bool_checks.c new file mode 100644 index 000000000..bb2adee56 --- /dev/null +++ b/tests/ir_tests/test_errunion_bool_checks.c @@ -0,0 +1,135 @@ +/* The Zig C backend's error handling: a call returns a small struct in r0 + * (an error union `{ uint16_t error; }`), the caller copies it and tests the + * field through one `bool` reused by every check. The field is the stored + * register's low bits, zero-extended -- the bits of r0 above it are + * unspecified, and here they are garbage -- and each check branches on its + * own compare. A bool also read outside its test, or reached through a + * pointer, keeps its value. */ +#include +#include +#include + +struct eu16 +{ + uint16_t error; +}; +struct eu8 +{ + uint8_t error; +}; + +union word16 +{ + uint32_t w; + struct eu16 s; +}; +union word8 +{ + uint32_t w; + struct eu8 s; +}; + +static volatile uint32_t junk = 0xABCD0000u; + +/* The returned register holds the struct and garbage above it. */ +__attribute__((noinline)) static struct eu16 fail16(int x) +{ + union word16 u; + u.w = junk | (uint32_t)(x % 3 == 0 ? 0 : 100 + x); + return u.s; +} + +__attribute__((noinline)) static struct eu8 fail8(int x) +{ + union word8 u; + u.w = (junk << 4) | (uint32_t)(x % 4 == 0 ? 0 : 7 + x); + return u.s; +} + +static int seen; +__attribute__((noinline)) static void note(uint16_t e) +{ + seen += e; +} + +__attribute__((noinline)) static uint16_t run(int x) +{ + struct eu16 t0; + struct eu8 t3; + bool t1; + uint16_t t2; + t0 = fail16(x); + t1 = t0.error == UINT16_C(0); + if (t1) + { + goto block_0; + } + t2 = t0.error; + note(t2); + return t2; +block_0:; + t3 = fail8(x); + t1 = t3.error == UINT8_C(0); + if (t1) + { + goto block_1; + } + return (uint16_t)(t3.error + 1000); +block_1:; + t0 = fail16(x + 1); + t1 = t0.error != UINT16_C(0); + if (t1) + { + return (uint16_t)(t0.error + 2000); + } + return 0; +} + +/* The bool is also returned: its checks keep materialising it. */ +__attribute__((noinline)) static int kept(int x) +{ + bool t1; + t1 = fail16(x).error == 0; + if (t1) + x += 10; + t1 = fail16(x + 1).error == 0; + if (t1) + x += 20; + return x * 2 + t1; +} + +__attribute__((noinline)) static void flip(bool *p) +{ + *p = !*p; +} + +/* Reached through a pointer. */ +__attribute__((noinline)) static int through_pointer(int x) +{ + bool t1; + t1 = fail16(x).error == 0; + flip(&t1); + if (t1) + return 1; + t1 = fail8(x).error == 0; + if (t1) + return 2; + return 3; +} + +int main(void) +{ + unsigned sum = 0; + for (int x = 0; x < 24; x++) + sum = sum * 31 + run(x); + printf("%u %d\n", sum, seen); + unsigned k = 0; + for (int x = 0; x < 12; x++) + k = k * 7 + (unsigned)kept(x); + printf("%u\n", k); + unsigned t = 0; + for (int x = 0; x < 12; x++) + t = t * 5 + (unsigned)through_pointer(x); + printf("%u\n", t); + return 0; +} diff --git a/tests/ir_tests/test_errunion_bool_checks.expect b/tests/ir_tests/test_errunion_bool_checks.expect new file mode 100644 index 000000000..eb675bc39 --- /dev/null +++ b/tests/ir_tests/test_errunion_bool_checks.expect @@ -0,0 +1,3 @@ +331362176 1792 +3435758332 +110650831 diff --git a/tests/ir_tests/test_fp_mode_selection.py b/tests/ir_tests/test_fp_mode_selection.py index b0a613028..24b573902 100644 --- a/tests/ir_tests/test_fp_mode_selection.py +++ b/tests/ir_tests/test_fp_mode_selection.py @@ -158,9 +158,11 @@ def test_fp_lib_static_binds_an_archive(tmp_path): # where it put the section, and a writer that moves it moves this too. # Byte offset of YaffHeader.arch_section_offset, counted through the packed -# fields ahead of it. Cross-checked by test_header_layout_is_still_what_this -# _test_assumes below, which reads a value no misaligned read would produce. -ARCH_SECTION_OFFSET_FIELD = 60 +# fields ahead of it: a u32 at 72 since YAFF_VERSION 3 widened the counts +# (a u16 at 60 before; 60 is got_length now). Cross-checked by +# test_header_layout_is_still_what_this_test_assumes below, which reads a +# value no misaligned read would produce. +ARCH_SECTION_OFFSET_FIELD = 72 # YaffArchSection: u16 size, u8 arch, u8 fpu, u8 float_abi, u8 reserved_[3], # u32 required_features. @@ -180,7 +182,7 @@ def test_fp_lib_static_binds_an_archive(tmp_path): def _yaff_arch_section(path): data = path.read_bytes() assert data[:4] == b"YAFF", "not a YAFF module" - offset = struct.unpack_from(" + +struct Q3 { int a[3]; }; +struct Q5 { int a[5]; }; +struct Q7 { int a[7]; }; +struct Q10 { int a[10]; }; + +__attribute__((noinline)) static void touch(void *p, int n) +{ + int *q = p; + for (int i = 0; i < n; i++) + q[i] = q[i] * 3 + i; +} + +#define COPY_TEST(T, N) \ + __attribute__((noinline)) static int copy_##T(int k) \ + { \ + struct T x, y; \ + for (int i = 0; i < N; i++) \ + x.a[i] = k * i + 1; \ + touch(&x, N); \ + int live1 = k * 7, live2 = x.a[0] + 1; \ + y = x; \ + touch(&y, N); \ + int r = live1 + live2; \ + for (int i = 0; i < N; i++) \ + r = r * 5 + y.a[i] - x.a[i]; \ + return r; \ + } +COPY_TEST(Q3, 3) +COPY_TEST(Q5, 5) +COPY_TEST(Q7, 7) +COPY_TEST(Q10, 10) + +__attribute__((noinline)) static int self_assign(int k) +{ + struct Q5 x; + struct Q5 *p = &x; + for (int i = 0; i < 5; i++) + x.a[i] = k + i; + *p = x; + return x.a[0] + x.a[4]; +} + +__attribute__((noinline)) static int to_volatile(int k) +{ + struct Q5 x; + volatile struct Q5 v; + for (int i = 0; i < 5; i++) + x.a[i] = k - i; + touch(&x, 5); + for (int i = 0; i < 5; i++) + v.a[i] = x.a[i]; + return v.a[1] + v.a[3]; +} + +int main(void) +{ + printf("%d %d %d %d\n", copy_Q3(2), copy_Q5(3), copy_Q7(4), copy_Q10(5)); + printf("%d %d\n", self_assign(10), to_volatile(20)); + return 0; +} diff --git a/tests/ir_tests/test_frame_block_copy.expect b/tests/ir_tests/test_frame_block_copy.expect new file mode 100644 index 000000000..589cf7275 --- /dev/null +++ b/tests/ir_tests/test_frame_block_copy.expect @@ -0,0 +1,2 @@ +2541 86885 2748973 415649328 +24 112 diff --git a/tests/ir_tests/test_frame_colour.c b/tests/ir_tests/test_frame_colour.c new file mode 100644 index 000000000..624d3456b --- /dev/null +++ b/tests/ir_tests/test_frame_colour.c @@ -0,0 +1,269 @@ +/* Frame objects with disjoint lifetimes share bytes (frame.c, frame_colour). + * Every shape here must keep the objects that are live at the same time + * apart: values carried around a loop, addresses that escape, one-past-end + * walks, a struct result reached through an array member while another + * struct call runs, backward gotos, switches in loops, setjmp. */ +#include +#include +#include + +struct blk +{ + int v[12]; +}; +struct name +{ + char s[16]; +}; + +static int *kept; + +__attribute__((noinline)) static struct blk mk(int k) +{ + struct blk b; + for (int i = 0; i < 12; i++) + b.v[i] = k * 100 + i; + return b; +} + +__attribute__((noinline)) static struct name nm(const char *s) +{ + struct name n; + memset(&n, 0, sizeof n); + strncpy(n.s, s, sizeof n.s - 1); + return n; +} + +__attribute__((noinline)) static int sum_blk(struct blk b) +{ + int s = 0; + for (int i = 0; i < 12; i++) + s += b.v[i]; + return s; +} + +__attribute__((noinline)) static int sum_range(const int *p, const int *end) +{ + int s = 0; + while (p != end) + s += *p++; + return s; +} + +__attribute__((noinline)) static void keep(int *p) +{ + kept = p; +} + +__attribute__((noinline)) static int two(const char *a, int b) +{ + return (int)strlen(a) * 1000 + b; +} + +/* Sequential struct results: later ones may take earlier ones' bytes. */ +__attribute__((noinline)) static int sequential(int k) +{ + struct blk a = mk(k); + int s = a.v[0] + a.v[11]; + struct blk b = mk(k + 1); + s += sum_blk(b); + struct blk c = mk(k + 2); + s += c.v[5]; + struct blk d = mk(k + 3); + return s + sum_range(d.v, d.v + 12); +} + +/* `acc` is read at the top of each iteration and written at the bottom, so + * it is live around the whole loop; the per-iteration temporaries must not + * land on it. */ +__attribute__((noinline)) static int loop_carried(int n) +{ + struct blk acc = mk(0); + for (int i = 0; i < n; i++) + { + int top = acc.v[i % 12]; + struct blk t = mk(i); + struct blk u = mk(i + 7); + acc.v[i % 12] = top + t.v[3] + u.v[4]; + } + struct blk after = mk(n); + return sum_blk(acc) + after.v[2]; +} + +/* An escaped address keeps its object for the rest of the function. */ +__attribute__((noinline)) static int escaped(int k) +{ + int arr[8]; + for (int i = 0; i < 8; i++) + arr[i] = k + i; + keep(arr); + struct blk x = mk(k); + struct blk y = mk(k + 1); + return kept[3] + kept[7] + x.v[1] + y.v[2]; +} + +/* A struct result's array reached through a pointer-returning call, while + * another struct-returning call runs in the same expression. */ +__attribute__((noinline)) static int temp_through_call(void) +{ + return two(strchr(nm("hello, world").s, 'w'), mk(4).v[3]); +} + +/* A backward goto forms a loop without a loop statement. */ +__attribute__((noinline)) static int goto_loop(int n) +{ + struct blk keepme = mk(9); + int i = 0, s = 0; +again: + { + struct blk t = mk(i); + s += t.v[i % 12] + keepme.v[i % 12]; + } + if (++i < n) + goto again; + struct blk late = mk(n); + return s + late.v[0] + keepme.v[11]; +} + +/* Cases of a switch inside a loop reach back to the loop head. */ +__attribute__((noinline)) static int switch_loop(int n) +{ + struct blk carried = mk(1); + int s = 0; + for (int i = 0; i < n; i++) + { + switch (i % 4) + { + case 0: + { + struct blk a = mk(i); + s += a.v[1]; + break; + } + case 1: + { + struct blk b = mk(i + 1); + carried.v[i % 12] += b.v[2]; + break; + } + case 2: + s += carried.v[(i + 1) % 12]; + break; + default: + { + struct blk c = mk(i + 2); + s -= c.v[3]; + break; + } + } + } + return s + sum_blk(carried); +} + +/* memcpy and strchr keep no pointer, but their results point into their + * first argument: that object lives as long as the result is used. */ +__attribute__((noinline)) static int through_libcalls(int k) +{ + struct blk src = mk(k); + struct blk dst; + int *p = memcpy(&dst, &src, sizeof dst); + struct name n = nm("abc:def"); + char *colon = strchr(n.s, ':'); + struct blk later = mk(k + 5); + struct name other = nm("zzzzzzzzzzzz"); + return p[3] + later.v[1] + (int)(colon - n.s) * 1000 + colon[1] + other.s[0]; +} + +/* Blocks laid out A, C, B where B jumps back to C: x is written in A and read + * in C after y was used in B, so they must not share although C comes before + * B in the source. */ +__attribute__((noinline)) static int layout_order(int k) +{ + struct blk x, y; + int r = 0, s = 0; + if (k) + goto A; + goto end; +A: + x = mk(k); + goto B; +C: + r = x.v[3]; + goto end; +B: + y = mk(k + 1); + s = y.v[4]; + goto C; +end: + return r + s; +} + +/* Objects in different cases of a switch may share; the result must not. */ +__attribute__((noinline)) static int switch_cases(int sel) +{ + struct blk res; + switch (sel) + { + case 0: + { + struct blk a = mk(1); + res = a; + break; + } + case 1: + { + struct blk b = mk(2); + b.v[0] += 5; + res = b; + break; + } + case 2: + { + struct blk c = mk(3); + struct blk d = mk(4); + c.v[0] += d.v[0]; + res = c; + break; + } + default: + res = mk(9); + break; + } + return res.v[0] + res.v[11]; +} + +static jmp_buf env; +__attribute__((noinline)) static void jump(int v) +{ + longjmp(env, v); +} + +/* setjmp returns twice: no object may take another's bytes. */ +__attribute__((noinline)) static int with_setjmp(int k) +{ + struct blk before = mk(k); + volatile int round = 0; + if (setjmp(env) == 0) + { + struct blk first = mk(k + 1); + round = first.v[0]; + jump(1); + } + struct blk second = mk(k + 2); + return before.v[4] + round + second.v[5]; +} + +int main(void) +{ + printf("%d\n", sequential(3)); + printf("%d\n", loop_carried(20)); + printf("%d\n", escaped(5)); + printf("%d\n", temp_through_call()); + printf("%d\n", goto_loop(15)); + printf("%d\n", switch_loop(23)); + printf("%d\n", with_setjmp(2)); + printf("%d\n", through_libcalls(4)); + printf("%d %d\n", layout_order(3), layout_order(0)); + printf("%d %d %d %d\n", switch_cases(0), switch_cases(1), switch_cases(2), switch_cases(7)); + return 0; +} diff --git a/tests/ir_tests/test_frame_colour.expect b/tests/ir_tests/test_frame_colour.expect new file mode 100644 index 000000000..d4c23da85 --- /dev/null +++ b/tests/ir_tests/test_frame_colour.expect @@ -0,0 +1,10 @@ +13248 +54208 +1123 +5403 +26549 +8611 +909 +4526 +707 0 +211 416 1011 1811 diff --git a/tests/ir_tests/test_frame_loop_lifetimes.c b/tests/ir_tests/test_frame_loop_lifetimes.c new file mode 100644 index 000000000..1dfd760f8 --- /dev/null +++ b/tests/ir_tests/test_frame_loop_lifetimes.c @@ -0,0 +1,90 @@ +/* Frame objects inside loops: an object fully overwritten before each use in + * an iteration (struct result, block copy, a store per field) carries nothing + * round the back edge, so objects of different switch cases may share bytes. + * One carried across iterations, read before it is rewritten, or only partly + * rewritten, must not. All locals at the top of the function, as the Zig C + * backend writes them. */ +#include + +struct quad +{ + int v[4]; +}; + +struct pair +{ + int a, b; + short c; +}; + +__attribute__((noinline)) static struct quad make_quad(int x) +{ + struct quad q = {{x, x + 1, x + 2, x + 3}}; + return q; +} + +__attribute__((noinline)) static struct pair make_pair(int x) +{ + struct pair p = {x * 3, x * 5, (short)(x * 7)}; + return p; +} + +__attribute__((noinline)) static int sum_quad(const struct quad *q) +{ + return q->v[0] + q->v[1] + q->v[2] + q->v[3]; +} + +__attribute__((noinline)) static int dispatch(int n) +{ + struct quad carried; + struct quad partial; + struct quad by_call; + struct quad by_fields; + struct quad copy; + struct pair pr; + struct pair pr2; + int total = 0; + + carried = make_quad(100); + partial = make_quad(200); + for (int i = 0; i < n; i++) + { + switch (i % 4) + { + case 0: + by_call = make_quad(i); + total += by_call.v[0] * 3 + by_call.v[3]; + break; + case 1: + by_fields.v[0] = i; + by_fields.v[1] = i * 2; + by_fields.v[2] = i * 3; + by_fields.v[3] = i * 4; + total += sum_quad(&by_fields); + break; + case 2: + pr = make_pair(i); + pr2 = pr; + total += pr2.a - pr2.b + pr2.c; + break; + default: + copy = carried; + total += copy.v[i % 4] - copy.v[0]; + break; + } + /* read before rewrite: the previous iteration's value */ + total += carried.v[i % 4]; + if (i % 5 == 4) + carried = make_quad(i * 10); + /* only partly rewritten: v[1..3] come from before the loop */ + total += partial.v[(i + 1) % 4]; + partial.v[0] = i; + } + return total; +} + +int main(void) +{ + printf("%d %d %d\n", dispatch(1), dispatch(7), dispatch(40)); + return 0; +} diff --git a/tests/ir_tests/test_frame_loop_lifetimes.expect b/tests/ir_tests/test_frame_loop_lifetimes.expect new file mode 100644 index 000000000..ea64dc2c6 --- /dev/null +++ b/tests/ir_tests/test_frame_loop_lifetimes.expect @@ -0,0 +1 @@ +304 1928 17150 diff --git a/tests/ir_tests/test_frame_relayout.c b/tests/ir_tests/test_frame_relayout.c new file mode 100644 index 000000000..3a81adbf1 --- /dev/null +++ b/tests/ir_tests/test_frame_relayout.c @@ -0,0 +1,152 @@ +/* Frame objects the optimizer leaves unreferenced are dropped and the live + * ones packed towards the frame top (frame.c / tcc_ir_frame_relayout). Dead + * aggregates sit between live ones; live objects are reached through escaping + * pointers, one-past-end pointers, struct copies, sret and inline return slots, + * and a frame far past the 16-bit [sp, #imm] reach. Every result is checked. */ +#include +#include + +struct pair +{ + int a, b; +}; +struct big +{ + int v[40]; +}; + +__attribute__((noinline)) static int sum(const int *p, const int *end) +{ + int s = 0; + while (p != end) + s += *p++; + return s; +} + +__attribute__((noinline)) static int sum_n(const int *p, int n) +{ + return sum(p, p + n); +} + +__attribute__((noinline)) static void fill(int *p, int n, int k) +{ + for (int i = 0; i < n; i++) + p[i] = k * i + 1; +} + +__attribute__((noinline)) static struct big make_big(int k) +{ + struct big b; + for (int i = 0; i < 40; i++) + b.v[i] = i ^ k; + return b; +} + +static inline struct pair swap(struct pair p) +{ + struct pair r = {p.b, p.a}; + return r; +} + +/* A never-referenced array between two live ones. */ +__attribute__((noinline)) static int dead_between(int n) +{ + int hi[8]; + char unused[600] __attribute__((unused)); + int lo[8]; + for (int i = 0; i < 8; i++) + { + hi[i] = i + n; + lo[i] = 100 * i; + } + return sum_n(hi, 8) + sum_n(lo, 8); +} + +/* Dead stores only: the zeroed struct is never read. */ +__attribute__((noinline)) static int dead_zeroed(int n) +{ + struct big scratch; + memset(&scratch, 0, sizeof scratch); + int live[4]; + fill(live, 4, n); + return sum_n(live, 4); +} + +/* Struct copies, an sret temporary and an inlined struct return, all moved up + * over a dead odd-sized array. */ +__attribute__((noinline)) static int copies(int k) +{ + char pad[99] __attribute__((unused)); + struct pair p = {k, 2 * k}; + struct pair q = p; + struct pair r = swap(q); + struct big b = make_big(k); + struct big c = b; + return r.a - r.b + c.v[3] + c.v[39] + b.v[0]; +} + +/* Far offsets: a big live array below several dead ones, spills below that. */ +__attribute__((noinline)) static unsigned far_frame(unsigned s) +{ + char dead1[1500] __attribute__((unused)); + int keep[300]; + char dead2[3000] __attribute__((unused)); + int near_[4]; + fill(keep, 300, (int)s); + fill(near_, 4, 3); + unsigned a0 = s + keep[1], a1 = s + keep[299], a2 = s ^ keep[7], a3 = s * 3u, a4 = s + 5, a5 = s + 6; + unsigned a6 = s + 7, a7 = s + 8, a8 = s + 9, a9 = s + 10, a10 = s + 11, a11 = s + 12; + unsigned t = (unsigned)sum_n(keep, 300) + (unsigned)sum_n(near_, 4); + return t + a0 * a11 + a1 * a10 + a2 * a9 + a3 * a8 + a4 * a7 + a5 * a6; +} + +__attribute__((noinline)) static void fill64(long long *p, int n, long long k) +{ + for (int i = 0; i < n; i++) + p[i] = k * i + 1; +} + +/* An 8-byte-aligned object moved over a 13-byte hole stays 8-byte aligned. */ +__attribute__((noinline)) static long long aligned_move(long long k) +{ + char pad[13] __attribute__((unused)); + long long w[3]; + fill64(w, 3, k); + return w[0] + w[1] * w[2] + (long long)((unsigned)(unsigned long)&w[0] & 7u); +} + +/* The by-value parameters' homes are untracked allocations the prologue stores + * to: they stay put while the array below the dead one moves up to them. */ +__attribute__((noinline)) static int by_value(struct pair p, struct big b, int k) +{ + char dead[40] __attribute__((unused)); + int arr[4]; + fill(arr, 4, k); + return p.a * 10 + p.b + sum_n(arr, 4) + b.v[0] + b.v[39]; +} + +/* A walk ends at the lowest array's one-past-end pointer, which lies in the + * dead array above it. */ +__attribute__((noinline)) static int adjacent(void) +{ + int x[2] = {1, 2}; + char gap[64] __attribute__((unused)); + int y[2] = {3, 4}; + const int *e = y + 2; + int s = 0; + for (const int *p = y; p != e; p++) + s += *p; + return s + x[0] + x[1]; +} + +int main(void) +{ + printf("%d\n", dead_between(5)); + printf("%d\n", dead_zeroed(7)); + printf("%d\n", copies(9)); + printf("%u\n", far_frame(11)); + printf("%d\n", adjacent()); + printf("%lld\n", aligned_move(1000000007LL)); + printf("%d\n", by_value((struct pair){3, 4}, make_big(5), 6)); + return 0; +} diff --git a/tests/ir_tests/test_frame_relayout.expect b/tests/ir_tests/test_frame_relayout.expect new file mode 100644 index 000000000..c9561fd62 --- /dev/null +++ b/tests/ir_tests/test_frame_relayout.expect @@ -0,0 +1,7 @@ +2868 +46 +74 +569542 +10 +2000000031000000121 +113 diff --git a/tests/ir_tests/test_icf_folding.c b/tests/ir_tests/test_icf_folding.c new file mode 100644 index 000000000..39f7bf768 --- /dev/null +++ b/tests/ir_tests/test_icf_folding.c @@ -0,0 +1,125 @@ +/* Identical code folding: a static function whose body matches one already + * generated is dropped and its symbol points at the survivor. What must NOT + * fold: bodies differing in a constant, in the function they call, in the + * global they touch, or in a literal-pool value; and any function whose + * address is taken -- by `&`, by naming it, or by passing it on -- since a + * program can compare those. A recursive body calls itself, which is not the + * same as calling another function. */ +#include +#include + +static int g_one = 11; +static int g_two = 22; + +__attribute__((noinline)) static int mul3(const int *p, int n) +{ + int s = 0; + for (int i = 0; i < n; i++) + s += p[i] * 3; + return s; +} + +/* identical to mul3 */ +__attribute__((noinline)) static int mul3_again(const int *p, int n) +{ + int s = 0; + for (int i = 0; i < n; i++) + s += p[i] * 3; + return s; +} + +/* differs by a constant */ +__attribute__((noinline)) static int mul5(const int *p, int n) +{ + int s = 0; + for (int i = 0; i < n; i++) + s += p[i] * 5; + return s; +} + +/* differs by a literal-pool constant */ +__attribute__((noinline)) static int big_a(int x) +{ + return x * 3 + 0x12345678; +} + +__attribute__((noinline)) static int big_b(int x) +{ + return x * 3 + 0x7654321; +} + +/* differ by the global they read */ +__attribute__((noinline)) static int read_one(void) +{ + return g_one * 2; +} + +__attribute__((noinline)) static int read_two(void) +{ + return g_two * 2; +} + +/* differ by the function they call */ +__attribute__((noinline)) static int via_one(const int *p, int n) +{ + return mul3(p, n) + 1; +} + +__attribute__((noinline)) static int via_five(const int *p, int n) +{ + return mul5(p, n) + 1; +} + +/* identical recursion: each calls itself */ +__attribute__((noinline)) static int fact_a(int n) +{ + return n <= 1 ? 1 : n * fact_a(n - 1); +} + +__attribute__((noinline)) static int fact_b(int n) +{ + return n <= 1 ? 1 : n * fact_b(n - 1); +} + +/* identical, but their addresses are compared */ +__attribute__((noinline)) static int key_a(int x) +{ + return x ^ 0x5a; +} + +__attribute__((noinline)) static int key_b(int x) +{ + return x ^ 0x5a; +} + +/* identical, one named (decays to a pointer) without & */ +__attribute__((noinline)) static int named_a(int x) +{ + return x + 77; +} + +__attribute__((noinline)) static int named_b(int x) +{ + return x + 77; +} + +__attribute__((noinline)) static int apply(int (*f)(int), int x) +{ + return f(x); +} + +static int (*volatile ka)(int) = key_a; +static int (*volatile kb)(int) = key_b; + +int main(void) +{ + int v[5] = {1, 2, 3, 4, 5}; + printf("%d %d %d\n", mul3(v, 5), mul3_again(v, 5), mul5(v, 5)); + printf("%d %d\n", big_a(10), big_b(10)); + printf("%d %d\n", read_one(), read_two()); + printf("%d %d\n", via_one(v, 5), via_five(v, 5)); + printf("%d %d\n", fact_a(5), fact_b(6)); + printf("%d %d %d\n", ka(7), kb(7), ka == kb); + printf("%d %d %d\n", apply(named_a, 1), apply(named_b, 2), named_a == named_b); + return 0; +} diff --git a/tests/ir_tests/test_icf_folding.expect b/tests/ir_tests/test_icf_folding.expect new file mode 100644 index 000000000..694b9610e --- /dev/null +++ b/tests/ir_tests/test_icf_folding.expect @@ -0,0 +1,7 @@ +45 45 75 +305419926 124076863 +22 44 +46 76 +120 720 +93 93 0 +78 79 0 diff --git a/tests/ir_tests/test_inline_called_once.c b/tests/ir_tests/test_inline_called_once.c new file mode 100644 index 000000000..171f26d5e --- /dev/null +++ b/tests/ir_tests/test_inline_called_once.c @@ -0,0 +1,130 @@ +/* -finline-functions-called-once (on at -O1): every function body is saved + * until the end of the TU, and a static function with exactly one call site + * and no other reference is expanded at that call. Covers callees defined + * before and after their caller, chains of single calls, struct parameters and + * returns, loops, early returns, callee locals named like caller locals, and + * the functions that stay calls: two call sites, address taken, self + * recursion, and a body that would force a frame pointer on its caller. */ +#include + +struct pair +{ + int a, b; +}; + +struct big +{ + int v[6]; +}; + +static int later_defined(int x); + +static int square_plus(int x) +{ + return x * x + 1; +} + +static int chain_c(int x) +{ + if (x < 0) + return -1; + return x + 3; +} + +static int chain_b(int x) +{ + return chain_c(x * 2) + 1; +} + +static int chain_a(int x) +{ + return chain_b(x + 1) * 10; +} + +static void accumulate(int *acc, const int *vals, int n) +{ + for (int i = 0; i < n; i++) + { + if (vals[i] < 0) + return; + *acc += vals[i]; + } +} + +static struct pair swap_pair(struct pair p) +{ + struct pair r = {p.b, p.a}; + return r; +} + +static int big_sum(struct big b) +{ + int s = 0; + for (int i = 0; i < 6; i++) + s += b.v[i] * (i + 1); + return s; +} + +static int shadow_locals(int index) +{ + int total = 0; + for (int k = 0; k < index; k++) + total += k; + return total; +} + +static int called_twice(int x) +{ + return x - 7; +} + +static int address_taken(int x) +{ + return x ^ 0x55; +} + +static int fact(int n) +{ + return n <= 1 ? 1 : n * fact(n - 1); +} + +__attribute__((noinline)) static int nonnull(void *p) +{ + return p != 0; +} + +static int caller_known(void) +{ + return nonnull(__builtin_return_address(0)); +} + +static int (*volatile fp)(int); + +int main(void) +{ + int acc = 0; + int vals[] = {3, 4, -1, 100}; + struct pair p = {1, 2}; + struct big b = {{1, 2, 3, 4, 5, 6}}; + int total = 0; + + accumulate(&acc, vals, 4); + p = swap_pair(p); + for (int k = 0; k < 3; k++) + total += k; + fp = address_taken; + + printf("%d %d %d %d\n", square_plus(6), chain_a(2), later_defined(5), acc); + printf("%d %d %d %d\n", p.a, p.b, big_sum(b), shadow_locals(5) + total); + printf("%d %d %d %d\n", called_twice(10), called_twice(20), fp(0x0f), fact(5)); + printf("%d\n", caller_known()); + return 0; +} + +static int later_defined(int x) +{ + int r = 0; + while (x-- > 0) + r += 2; + return r; +} diff --git a/tests/ir_tests/test_inline_called_once.expect b/tests/ir_tests/test_inline_called_once.expect new file mode 100644 index 000000000..6b08d759e --- /dev/null +++ b/tests/ir_tests/test_inline_called_once.expect @@ -0,0 +1,4 @@ +37 100 10 7 +2 1 91 13 +3 13 90 120 +1 diff --git a/tests/ir_tests/test_ldm_block_copy.c b/tests/ir_tests/test_ldm_block_copy.c new file mode 100644 index 000000000..35ef38e87 --- /dev/null +++ b/tests/ir_tests/test_ldm_block_copy.c @@ -0,0 +1,163 @@ +/* A small aggregate copied through a pointer comes out of the frontend as + * every word loaded, then every word stored, and the codegen folds the run + * into one LDM and one STM through the registers the loads were given. What + * that must not disturb: a source read again after the copy (the words are not + * dead, so the run is not one), a destination whose address is live past the + * copy (the LDM must not land on it), a copy whose words are not consecutive, + * and a copy of an object onto itself. Offsets on either side, both sides, or + * neither all have to address the same bytes. */ +#include +#include + +struct w3 +{ + uint32_t a[3]; +}; + +struct w4 +{ + uint32_t a[4]; +}; + +struct w6 +{ + uint32_t a[6]; +}; + +struct nested +{ + uint32_t head; + struct w4 mid; + uint32_t tail; +}; + +static uint32_t sum3(const struct w3 *p) { return p->a[0] * 1 + p->a[1] * 3 + p->a[2] * 7; } +static uint32_t sum4(const struct w4 *p) { return p->a[0] * 1 + p->a[1] * 3 + p->a[2] * 7 + p->a[3] * 13; } + +__attribute__((noinline)) static void fill3(struct w3 *p, uint32_t s) +{ + for (int i = 0; i < 3; i++) + p->a[i] = s + (uint32_t)i; +} + +__attribute__((noinline)) static void fill4(struct w4 *p, uint32_t s) +{ + for (int i = 0; i < 4; i++) + p->a[i] = s + (uint32_t)i; +} + +/* pointer -> frame slot, three and four words */ +__attribute__((noinline)) static uint32_t ptr_to_local3(const struct w3 *p) +{ + struct w3 b; + b = *p; + return sum3(&b); +} + +__attribute__((noinline)) static uint32_t ptr_to_local4(const struct w4 *p) +{ + struct w4 b; + b = *p; + return sum4(&b); +} + +/* frame slot -> pointer */ +__attribute__((noinline)) static void local_to_ptr(struct w4 *d, uint32_t s) +{ + struct w4 b; + fill4(&b, s); + *d = b; +} + +/* pointer -> pointer */ +__attribute__((noinline)) static void ptr_to_ptr(struct w4 *d, const struct w4 *s) { *d = *s; } + +/* a displacement on the source, on the destination, and on both */ +__attribute__((noinline)) static void off_src(struct w4 *d, const struct nested *s) { *d = s->mid; } + +__attribute__((noinline)) static void off_dst(struct nested *d, const struct w4 *s) { d->mid = *s; } + +__attribute__((noinline)) static void off_both(struct nested *d, const struct nested *s) { d->mid = s->mid; } + +/* the source is read again afterwards: its words are not dead at their stores */ +__attribute__((noinline)) static uint32_t src_read_after(const struct w4 *p) +{ + struct w4 b; + b = *p; + return sum4(&b) * 3 + p->a[2]; +} + +/* the destination address is live past the copy */ +__attribute__((noinline)) static uint32_t dst_live_after(struct w4 *d, const struct w4 *s) +{ + *d = *s; + return sum4(d) + (d == s ? 1u : 0u); +} + +/* a whole object onto itself */ +__attribute__((noinline)) static uint32_t self_copy(struct w4 *p) +{ + *p = *p; + return sum4(p); +} + +/* more words than one LDM can hold */ +__attribute__((noinline)) static uint32_t six_words(const struct w6 *p) +{ + struct w6 b; + b = *p; + uint32_t r = 0; + for (int i = 0; i < 6; i++) + r = r * 5 + b.a[i]; + return r; +} + +/* the two halves of one object, overlapping in neither direction */ +__attribute__((noinline)) static uint32_t halves(struct w6 *p) +{ + struct w3 lo, hi; + lo = *(const struct w3 *)&p->a[0]; + hi = *(const struct w3 *)&p->a[3]; + return sum3(&lo) * 2 + sum3(&hi); +} + +int main(void) +{ + struct w3 s3; + struct w4 s4, d4; + struct w6 s6; + struct nested n, n2; + + fill3(&s3, 100); + fill4(&s4, 200); + printf("%u %u\n", ptr_to_local3(&s3), ptr_to_local4(&s4)); + + local_to_ptr(&d4, 300); + printf("%u\n", sum4(&d4)); + + ptr_to_ptr(&d4, &s4); + printf("%u\n", sum4(&d4)); + + n.head = 1; + fill4(&n.mid, 400); + n.tail = 2; + off_src(&d4, &n); + printf("%u\n", sum4(&d4)); + + off_dst(&n2, &s4); + printf("%u\n", sum4(&n2.mid)); + + n2.head = 0; + n2.tail = 0; + off_both(&n2, &n); + printf("%u %u %u\n", sum4(&n2.mid), n2.head, n2.tail); + + printf("%u\n", src_read_after(&s4)); + printf("%u\n", dst_live_after(&d4, &s4)); + printf("%u\n", self_copy(&s4)); + + for (int i = 0; i < 6; i++) + s6.a[i] = (uint32_t)(500 + i); + printf("%u %u\n", six_words(&s6), halves(&s6)); + return 0; +} diff --git a/tests/ir_tests/test_ldm_block_copy.expect b/tests/ir_tests/test_ldm_block_copy.expect new file mode 100644 index 000000000..e0d0c7d2c --- /dev/null +++ b/tests/ir_tests/test_ldm_block_copy.expect @@ -0,0 +1,10 @@ +1117 4856 +7256 +4856 +9656 +4856 +9656 0 0 +14770 +4856 +4856 +1953975 16584 diff --git a/tests/ir_tests/test_narrow_call_ext.c b/tests/ir_tests/test_narrow_call_ext.c new file mode 100644 index 000000000..003a7e2d4 --- /dev/null +++ b/tests/ir_tests/test_narrow_call_ext.c @@ -0,0 +1,49 @@ +/* A narrow value that already fits is not extended again (ssa:narrow): a + * byte or halfword load, a mask, a truth value, and the result of a call + * declared to return bool or an unsigned byte or halfword -- the AAPCS has the + * callee extend it. A wider result stored into a narrow local must still be + * truncated, and signed and indirect-call results keep their extension. */ +#include +#include +#include + +__attribute__((noinline)) static uint8_t n8(uint32_t k) { return (uint8_t)(k * 7 + 3); } +__attribute__((noinline)) static uint16_t n16(uint32_t k) { return (uint16_t)(k * 40503u); } +__attribute__((noinline)) static int8_t s8(uint32_t k) { return (int8_t)(k * 37); } +__attribute__((noinline)) static bool odd(uint32_t k) { return k & 1; } +__attribute__((noinline)) static uint32_t wide(uint32_t k) { return k * 0x01010101u + 0x180; } + +static uint8_t (*volatile n8p)(uint32_t) = n8; + +__attribute__((noinline)) static uint32_t calls(uint32_t k) +{ + uint8_t a = n8(k); + uint16_t b = n16(k); + int8_t c = s8(k); + bool d = odd(k); + uint8_t e = (uint8_t)wide(k); /* must truncate */ + uint16_t f = (uint16_t)wide(k + 1); + uint8_t g = n8p(k + 2); /* indirect */ + uint32_t r = 0; + for (uint32_t i = 0; i < 3; i++) + r = r * 31 + (uint32_t)a + b + (uint32_t)(int32_t)c + d + e + f + g + i; + return r; +} + +__attribute__((noinline)) static uint32_t loads(const uint8_t *p, const uint16_t *q, uint32_t m) +{ + uint8_t x = p[m & 3]; + uint16_t y = q[m & 1]; + uint16_t z = (uint16_t)(m & 0x7f); + bool t = m > 100; + return (uint32_t)(uint8_t)x * 3 + (uint16_t)y + (uint16_t)(uint8_t)z + (uint8_t)t; +} + +int main(void) +{ + static const uint8_t p[4] = {0x81, 0xff, 7, 0x80}; + static const uint16_t q[2] = {0xfffe, 0x8001}; + printf("%u %u %u\n", calls(1), calls(200), calls(0xdeadbeef)); + printf("%u %u %u\n", loads(p, q, 0), loads(p, q, 131), loads(p, q, 0xffffffff)); + return 0; +} diff --git a/tests/ir_tests/test_narrow_call_ext.expect b/tests/ir_tests/test_narrow_call_ext.expect new file mode 100644 index 000000000..7c45e881a --- /dev/null +++ b/tests/ir_tests/test_narrow_call_ext.expect @@ -0,0 +1,2 @@ +41310819 91376886 79545291 +65921 33157 33281 diff --git a/tests/ir_tests/test_param_copy_alias.c b/tests/ir_tests/test_param_copy_alias.c new file mode 100644 index 000000000..7a4b9e39f --- /dev/null +++ b/tests/ir_tests/test_param_copy_alias.c @@ -0,0 +1,111 @@ +/* A local filled by a whole copy of a by-value struct parameter becomes the + * parameter's own memory (param_copy_alias). Covers reads and writes through + * the local, a parameter split across r1-r3 and the stack, one wholly on the + * stack, one in registers (copied from its home slot), a large one copied by + * memmove, and the cases that must keep the copy: the parameter still read + * after the local changed, and a copy not in the entry block. */ +#include +#include + +struct M { unsigned a[15]; }; +struct B { unsigned a[40]; }; +struct S { unsigned lo, hi; }; + +__attribute__((noinline)) static unsigned sum_m(struct M m) +{ + unsigned r = 0; + for (int i = 0; i < 15; i++) + r = r * 3 + m.a[i]; + return r; +} + +/* Zig C backend shape: copy, take the address, work through the pointer. */ +__attribute__((noinline)) static unsigned read_through(unsigned k, struct M const a1) +{ + struct M t0; + struct M const *t1; + t0 = a1; + t1 = (struct M const *)&t0; + return t1->a[9] * k + t1->a[10]; +} + +__attribute__((noinline)) static unsigned write_through(int x, struct M a1) +{ + struct M t0; + struct M *t1; + t0 = a1; + t1 = &t0; + t1->a[0] += (unsigned)x; + t1->a[14] = t1->a[13] * 2; + return sum_m(t0); +} + +__attribute__((noinline)) static unsigned on_stack(int a, int b, int c, int d, struct M a1) +{ + struct M t0; + t0 = a1; + t0.a[3] = (unsigned)(a + b + c + d); + return sum_m(t0) ^ t0.a[5]; +} + +__attribute__((noinline)) static unsigned in_regs(struct S a0, unsigned k) +{ + struct S t0; + struct S *t1; + t0 = a0; + t1 = &t0; + t1->lo += k; + return t1->lo * 7 + t1->hi; +} + +__attribute__((noinline)) static unsigned big(struct B a1, unsigned k) +{ + struct B t0; + struct B *t1; + t0 = a1; + t1 = &t0; + for (int i = 0; i < 40; i++) + t1->a[i] += k * (unsigned)i; + unsigned r = 0; + for (int i = 0; i < 40; i++) + r = r * 5 + t1->a[i]; + return r; +} + +/* The parameter is read after the local changed: the copy must stay. */ +__attribute__((noinline)) static unsigned param_after(struct M a1) +{ + struct M t0; + t0 = a1; + t0.a[2] = 1000; + return t0.a[2] + a1.a[2]; +} + +/* The copy is not in the entry block. */ +__attribute__((noinline)) static unsigned late_copy(int flag, struct M a1) +{ + struct M t0; + memset(&t0, 0, sizeof(t0)); + if (flag) + t0 = a1; + return sum_m(t0); +} + +int main(void) +{ + struct M m; + struct B b; + struct S s = {5, 9}; + for (int i = 0; i < 15; i++) + m.a[i] = (unsigned)(i * i + 1); + for (int i = 0; i < 40; i++) + b.a[i] = (unsigned)(i * 3 + 2); + printf("%u\n", read_through(3, m)); + printf("%u %u\n", write_through(4, m), m.a[0]); + printf("%u\n", on_stack(1, 2, 3, 4, m)); + printf("%u %u\n", in_regs(s, 6), s.lo); + printf("%u %u\n", big(b, 3), b.a[39]); + printf("%u\n", param_after(m)); + printf("%u %u\n", late_copy(0, m), late_copy(1, m)); + return 0; +} diff --git a/tests/ir_tests/test_param_copy_alias.expect b/tests/ir_tests/test_param_copy_alias.expect new file mode 100644 index 000000000..ce6ec001a --- /dev/null +++ b/tests/ir_tests/test_param_copy_alias.expect @@ -0,0 +1,7 @@ +347 +33480805 1 +14348776 +86 5 +3057825400 119 +1005 +0 14348786 diff --git a/tests/ir_tests/test_ptr_local_fwd.c b/tests/ir_tests/test_ptr_local_fwd.c new file mode 100644 index 000000000..1109069f8 --- /dev/null +++ b/tests/ir_tests/test_ptr_local_fwd.c @@ -0,0 +1,141 @@ +/* A local pointer whose every definition is `P = &V` for one V holds &V at + * every use, so derefs through it become direct accesses of V + * (ptr_local_fwd). The Zig C backend emits `t0 = call(); t1 = &t0; (*t1)->f` + * and sets t1 again in every block. Pointers that may name two objects, are + * copied from another pointer, or that a nested function redirects must keep + * their deref. */ +#include +#include + +struct S +{ + uint32_t a, b; +}; + +static struct S pool[4] = {{1, 2}, {30, 40}, {500, 600}, {7000, 8000}}; + +__attribute__((noinline)) static struct S *mk(uint32_t k) { return &pool[k & 3]; } +__attribute__((noinline)) static uint32_t use(uint32_t x, uint32_t y) { return x * 3 + y; } +__attribute__((noinline)) static signed char mkc(int k) { return (signed char)(k - 3); } + +/* V set only by a call; P set in the entry block. */ +__attribute__((noinline)) static uint32_t call_only(uint32_t k) +{ + struct S *t0; + struct S *const *t1; + t0 = mk(k); + t1 = (struct S *const *)&t0; + uint32_t t2 = (*t1)->b; + return use(t2, (*t1)->a); +} + +/* The same V re-pointed in every case, as Zig emits it. */ +__attribute__((noinline)) static uint32_t every_block(int k) +{ + struct S *t0; + struct S *const *t1; + uint32_t r; + switch (k) + { + case 0: + t0 = mk(1); + t1 = (struct S *const *)&t0; + r = (*t1)->a; + break; + case 1: + t0 = mk(2); + t1 = (struct S *const *)&t0; + r = (*t1)->b + 1; + break; + default: + t0 = mk((uint32_t)k); + t1 = (struct S *const *)&t0; + t0 = mk(3); /* through t1 this must be seen */ + r = (*t1)->a + (*t1)->b; + break; + } + return r; +} + +/* Stores through P land in V. */ +__attribute__((noinline)) static uint32_t store_through(uint32_t k) +{ + uint32_t v; + uint32_t *p; + v = use(k, 1); + p = &v; + *p += 10; + if (k & 1) + { + p = &v; + *p *= 2; + } + return v + *p; +} + +/* P names V on one path, W on the other: no forwarding. */ +__attribute__((noinline)) static uint32_t two_targets(int k) +{ + uint32_t v, w; + uint32_t *p; + v = use(1, 2); + w = use(3, 4); + if (k) + p = &v; + else + p = &w; + *p += 100; + return v * 1000 + w + *p; +} + +/* P copied from another pointer: no forwarding. */ +__attribute__((noinline)) static uint32_t copied(int k) +{ + uint32_t v, w; + uint32_t *p, *q; + v = use(5, 0); + w = use(6, 0); + q = k ? &w : &v; + p = &v; + *p += 1; + p = q; + *p += 7; + return v * 1000 + w; +} + +/* A nested function redirects the captured P: no forwarding. */ +__attribute__((noinline)) static uint32_t nested(int k) +{ + uint32_t v, w; + uint32_t *p; + void redirect(void) { p = &w; } + v = use(2, 0); + w = use(9, 0); + p = &v; + if (k) + redirect(); + *p += 50; + return v * 1000 + w; +} + +/* A narrow V only set by a call. */ +__attribute__((noinline)) static int narrow(int k) +{ + signed char c; + signed char *p; + c = mkc(k); + p = &c; + return *p * 10 + c; +} + +int main(void) +{ + printf("%u %u\n", call_only(0), call_only(2)); + printf("%u %u %u\n", every_block(0), every_block(1), every_block(2)); + printf("%u %u\n", store_through(2), store_through(3)); + printf("%u %u\n", two_targets(1), two_targets(0)); + printf("%u %u\n", copied(1), copied(0)); + printf("%u %u\n", nested(1), nested(0)); + printf("%d %d\n", narrow(1), narrow(10)); + return 0; +} diff --git a/tests/ir_tests/test_ptr_local_fwd.expect b/tests/ir_tests/test_ptr_local_fwd.expect new file mode 100644 index 000000000..71e4839e3 --- /dev/null +++ b/tests/ir_tests/test_ptr_local_fwd.expect @@ -0,0 +1,7 @@ +7 2300 +30 601 15000 +34 80 +105118 5226 +16025 23018 +6077 56027 +-22 77 diff --git a/tests/ir_tests/test_pure_forward.c b/tests/ir_tests/test_pure_forward.c new file mode 100644 index 000000000..eb51b56e8 --- /dev/null +++ b/tests/ir_tests/test_pure_forward.c @@ -0,0 +1,63 @@ +/* A function that only hands its parameters, untouched, to a callee of the + * same signature becomes a branch to it (pure forward): register, split, + * stack and home-slot struct arguments, 64-bit and double scalars. The + * near-misses must stay ordinary calls: swapped arguments, a modified + * parameter, a narrower parameter type, a struct return. */ +#include + +struct T { int a[11]; }; +struct S { int x, y; }; + +__attribute__((noinline)) static int g1(int k, struct T t) +{ + int r = k; + for (int i = 0; i < 11; i++) + r = r * 3 + t.a[i]; + return r; +} +__attribute__((noinline)) static int f1(int k, struct T t) { return g1(k, t); } + +__attribute__((noinline)) static int g2(struct S s) { return s.x * 100 + s.y; } +__attribute__((noinline)) static int f2(struct S s) { return g2(s); } + +__attribute__((noinline)) static double g3(int a, long long b, double c, int d, int e) +{ + return a + (double)b * 2 + c * 3 + d * 5 + e * 7; +} +__attribute__((noinline)) static double f3(int a, long long b, double c, int d, int e) { return g3(a, b, c, d, e); } + +static int sink; +__attribute__((noinline)) static void g4(int a, int b, int c, int d, struct T t, struct S s) +{ + sink = a + b * 2 + c * 3 + d * 4 + t.a[10] * 5 + s.y * 6; +} +__attribute__((noinline)) static void f4(int a, int b, int c, int d, struct T t, struct S s) { g4(a, b, c, d, t, s); } + +/* Near-misses. */ +__attribute__((noinline)) static int g5(int a, int b) { return a * 10 + b; } +__attribute__((noinline)) static int swapped(int a, int b) { return g5(b, a); } +__attribute__((noinline)) static int modified(int a, int b) +{ + a += 1; + return g5(a, b); +} +__attribute__((noinline)) static int g6(short a) { return a * 2; } +__attribute__((noinline)) static int narrower(int a) { return g6((short)a); } +__attribute__((noinline)) static struct S g7(struct S s) { return (struct S){s.y, s.x}; } +__attribute__((noinline)) static struct S sret(struct S s) { return g7(s); } + +int main(void) +{ + struct T t; + for (int i = 0; i < 11; i++) + t.a[i] = i * i - 5; + struct S s = {7, 9}; + printf("%d %d\n", f1(3, t), f2(s)); + printf("%d\n", (int)f3(1, 1000000000000LL / 1000000, 2.5, 4, 5)); + f4(1, 2, 3, 4, t, s); + printf("%d\n", sink); + printf("%d %d %d\n", swapped(1, 2), modified(1, 2), narrower(70000)); + struct S r = sret(s); + printf("%d %d\n", r.x, r.y); + return 0; +} diff --git a/tests/ir_tests/test_pure_forward.expect b/tests/ir_tests/test_pure_forward.expect new file mode 100644 index 000000000..70d42435b --- /dev/null +++ b/tests/ir_tests/test_pure_forward.expect @@ -0,0 +1,5 @@ +177083 709 +2000063 +559 +21 22 8928 +9 7 diff --git a/tests/ir_tests/test_qemu.py b/tests/ir_tests/test_qemu.py index c730ea04b..9058521b4 100644 --- a/tests/ir_tests/test_qemu.py +++ b/tests/ir_tests/test_qemu.py @@ -14,13 +14,17 @@ def _expect_line(sut, expected_line: str, *, timeout: int = 1, float_tol: float = 1e-5): - """Expect a line from QEMU output. + """Expect a line from QEMU output: a whole output line, not text inside one. + + A substring search let test_frame_relayout's wrong "11" pass: its + expected "10" was found inside the next line, 2000000031000000121. If the expected line ends with a float literal (e.g. "sum=3.500000"), capture the actual float and compare within tolerance. """ if expected_line is None: return + expected_line = expected_line.rstrip() float_matches = list(re.finditer(_FLOAT_RE, expected_line)) if float_matches: @@ -35,7 +39,7 @@ def _expect_line(sut, expected_line: str, *, timeout: int = 1, float_tol: float expected_values.append(float(fm.group(0))) last_end = fm.end() parts.append(re.escape(expected_line[last_end:])) - pattern = "".join(parts) + pattern = _whole_line("".join(parts)) sut.expect(pattern, timeout=timeout) actual_values = [float(sut.match.group(i + 1)) for i in range(len(expected_values))] @@ -46,7 +50,7 @@ def _expect_line(sut, expected_line: str, *, timeout: int = 1, float_tol: float ) return - sut.expect(_escape_regex(expected_line), timeout=timeout) + sut.expect(_whole_line(_escape_regex(expected_line)), timeout=timeout) MACHINE = "mps2-an505" CURRENT_DIR = Path(__file__).parent @@ -163,6 +167,100 @@ def _expect_line(sut, expected_line: str, *, timeout: int = 1, float_tol: float # inline asm operands may reuse their own live registers in IR mode ("bug_inline_asm_reserved_regs.c", 0), + # `register T x __asm("rN")` locals get a vreg (they aliased the previous + # local's stack slot) -- the shape of Zig's C-backend syscall wrappers + ("bug_asm_regvar_local.c", 0), + + # values live across inline asm survive its clobbers and pinned operands + ("bug_asm_clobber_live_values.c", 0), + + # branch narrowing / CBZ fusion: the rehearsal models mid-function returns + # and refuses ranges holding inline asm + ("bug_branch_narrow_rehearsal.c", 0), + + # a byte/halfword local read through its own stack slot (after lea_fold) + # keeps its width -- Zig's 2-byte `!void` error union test + ("bug_narrow_local_slot_width.c", 0), + + # a direct memmove into a global writes it: mod-ref must resolve the + # destination at the call site (copy-source forwarding read the new value) + ("bug_modref_direct_block_copy.c", 0), + + # small aggregates copied as LOAD/STORE chunks instead of __aeabi_memmove + ("test_small_aggregate_copy.c", 0), + + # spill slots reused after their interval expires (incl. eviction victims) + ("test_spill_slot_reuse.c", 0), + + # frame objects left unreferenced are dropped, live ones packed (frame.c) + ("test_frame_relayout.c", 0), + # frame objects with disjoint lifetimes share bytes (frame.c colouring) + ("test_frame_colour.c", 0), + + # an array of structs with a VLA member is itself a VLA + ("bug_vla_struct_array.c", 0), + + # a constant initializer copied from .rodata keeps its narrow fields narrow + ("bug_block_copy_init_narrow.c", 0), + + # an array's address stored into memory keeps the array's initializer + ("bug_dse_indexed_store_escape.c", 0), + + # dead-store elimination bounds reads through an address by its object + ("test_dse_object_ranges.c", 0), + ("test_dse_wide.c", 0), + + # loads/stores through a pointer to a frame object become direct slots + ("test_stack_deref_fold.c", 0), + ("test_ptr_local_fwd.c", 0), + ("test_struct_arg_stack.c", 0), + # struct values more than 32 KiB from the frame base (16-bit operand offset) + ("test_struct_far_frame.c", 0), + # __imag__ x = ... on a complex parameter passed on the stack or split + ("test_complex_param_parts.c", 0), + # a by-value struct parameter passed on by value (memcpy source bias) + ("test_struct_param_pass_on.c", 0), + # a local filled by a whole copy of a by-value struct parameter aliases it + ("test_param_copy_alias.c", 0), + # byte/halfword field stores over a zero-filled local word fold into it + ("test_slot_const_store_fold.c", 0), + # a function only forwarding its parameters becomes a branch to the callee + ("test_pure_forward.c", 0), + # struct stack parts of 3-7 words copied by LDM/STM + ("test_struct_stack_ldm.c", 0), + # static functions with a single call site expand at it (bodies deferred to TU end) + ("test_inline_called_once.c", 0), + # __builtin_return_address(0) off SP, no frame pointer, in every prologue shape + ("test_return_address.c", 0), + # alloca's size stays live after the allocation + ("test_alloca_live_size.c", 0), + # frame objects fully rewritten in each loop iteration share bytes; carried ones do not + ("test_frame_loop_lifetimes.c", 0), + # a pointer VAR walking a local array has two definitions: nothing is forwarded through it + ("test_entry_store_var_walk.c", 0), + # stack-passed parameters read more than once are loaded into registers by the prologue + ("test_stack_params_in_regs.c", 0), + # objects declared first and defined later are laid out once; their bytes are not folded early + ("test_tentative_definitions.c", 0), + # a small struct returned in r0 is read back masked, and one bool per check fuses into the branch + ("test_errunion_bool_checks.c", 0), + # a static function identical to one already generated is dropped for it, unless its address is taken + ("test_icf_folding.c", 0), + # a local copied from a call's own struct buffer is that buffer, anywhere in the function + ("test_call_buffer_copy_alias.c", 0), + ("test_ldm_block_copy.c", 0), + ("bug_addr_temp_reuse_before_def.c", 0), + ("bug_sra_narrow_field_undefined.c", 0), + ("bug_narrow_field_store_forward.c", 0), + ("bug_alive_share_evicted_owner.c", 0), + # word copies between frame slots fused into LDM/STM chunks + ("test_frame_block_copy.c", 0), + ("test_sra.c", 0), + ("test_narrow_call_ext.c", 0), + ("test_cross_jump.c", 0), + ("test_switch_case_merge.c", 0), + ("test_self_store.c", 0), + # mul clobbers base register during struct array indexing (non-power-of-2 element size) ("bug_struct_array_index_mul_clobber.c", 0), @@ -575,6 +673,16 @@ def _expect_line(sut, expected_line: str, *, timeout: int = 1, float_tol: float # loops that should have run (74 of 78 QEMU failures, 2026-08-23). ("471_deref_cse_nop_join.c", 0), + # A 64-bit compare against a constant whose low half is zero compares only + # the high words -- one CMP instead of a CMP and a borrow-folding SBCS. + # Exact for strict order only, so the test carries the refusals (>, <=, + # ==/!=, a non-zero low half, the constant on the left) beside the wins, + # and ties -- equal high words, non-zero low word -- in every seed. + ("525_cmp_hi_only.c", 0), + ("526_bottom_test_ptr_walk.c", 0), + # &&/|| whose last operand is a constant: the chain decides, no stale flags + ("527_landor_const_last_operand.c", 0), + # Compile-time strlen constant folding ("171_strlen_constfold.c", 0), @@ -615,6 +723,14 @@ def _expect_line(sut, expected_line: str, *, timeout: int = 1, float_tol: float ("nested_capture_array.c", 0), ("nested_capture_read.c", 0), ("nested_capture_write.c", 0), + ("nested_capture_param_stack.c", 0), + ("nested_capture_param_reg_noinline.c", 0), + ("nested_capture_param_byref.c", 0), + ("nested_capture_param_mixed.c", 0), + ("nested_capture_64bit.c", 0), + ("nested_capture_two_level.c", 0), + ("sl_forward_load_redefines_var.c", 0), + ("466_alive_share_boundary_coholder.c", 0), ("nested_direct_call_args.c", 0), ("nested_struct_return.c", 0), ("nested_shadowing.c", 0), @@ -1138,6 +1254,204 @@ def _expect_line(sut, expected_line: str, *, timeout: int = 1, float_tol: float # C11 _Pragma operator: pack layout via literal + DO_PRAGMA macro idiom. ("343_pragma_operator.c", 5), + # #pragma pack around bodies saved as tokens (static inline, -O1 deferred + # and called-once bodies): the token after a body's '}' was read with the + # body still capturing, so `f(){} #pragma pack(1) struct S` laid S out + # unpacked; replays now use the pack state at the body's definition. + ("472_pragma_pack_saved_bodies.c", 0), + + # A signed add that absorbed an unsigned (wrapping) inner add kept the + # no-overflow assumption: `(int)(x + 1U) + 1 < (int)x` folded to false. + ("473_reassoc_unsigned_into_signed.c", 0), + + # static inline bodies a call site did not expand were parsed after the + # end-of-TU late_reopt fold and dead-static analysis: a static written only + # there folded to 0, a store read only there was dropped. + ("474_late_reopt_owed_inline_bodies.c", 0), + + # -fdrop-unused-statics: forward-declared const objects, unsized arrays, + # liveness through function-pointer tables, block-scope prototypes, used + # and alias targets -- all deferred to the end of the TU and still defined. + ("475_drop_unused_statics.c", 0), + + # Write-only statics lose their stores and their bytes, but only when no + # path reads them: through a call, a data pointer, a function table, or a + # struct copy. Stores the frontend's addrtaken used to protect. + ("476_write_only_statics.c", 0), + + # Frame copy coalescing: a struct copy whose source dies and whose + # destination is born at the copy shares one slot and the copy goes; a + # source read later or a destination live before must keep theirs. + ("477_frame_copy_coalescing.c", 0), + + # __atomic_load/__atomic_store of 1/2/4 bytes are inline LDR/STR with a + # DMB per the memory order -- no runtime helper (libtcc1 has none). + ("478_inline_atomic_load_store.c", 0), + + # DSE: a copy of a temp holding one of two stack addresses is still an + # address; loads through it keep the stores that filled both objects + # (self-hosted tcc crashed on every 64-bit op in data_processing_mop_impl). + ("479_dse_ambiguous_addr_copy.c", 0), + + # fputs/printf lowering (fwrite, puts) only when the call is a whole + # expression statement: `x = printf(...)` and `int r = fputs(...)` keep + # their value (YasOS libc puts returned 1 for every string). + ("480_fputs_printf_result_used.c", 0), + # An address-taken char/short local passed with no promotion (variadic, + # or to a char/short parameter) is read from its home at its own width, + # not as a word with the slot's stale upper bytes. + ("481_narrow_local_arg_width.c", 0), + # `s = f(s)` with `s` a by-value struct parameter in the argument area: + # the sret pointer is &s, not the struct's first word. + ("482_sret_into_struct_param.c", 0), + # A u8/u16 temporary the allocator spills is stored as a word: its reload + # is a word load, and a byte store left stale bytes for it (pr82524). + ("483_narrow_temp_spill_word.c", 0), + # `f(c ? g() : h())` with a struct result: every word of the selected + # struct reaches f, not just the first (dead_temp_local). + ("484_struct_ternary_arg_copy.c", 0), + # A frame over 32 KiB: colouring keeps an object a struct operand names + # where it is, since the operand's 16-bit offset would wrap below -32768. + ("485_frame_colour_struct_far.c", 0), + # An address-taken local whose address leaves through a call result or a + # store keeps its slot to the function end (Zig autoHash of a u8, -O0). + ("486_addrtaken_escape_slot.c", 0), + # A tail-call-only function with an earlier return still gets its epilogue + # (InternPool.Alignment.max ran into the next function). + ("487_tail_call_early_return.c", 0), + # SCCP does not take `&VAR`'s spill placeholder for a frame slot: a memset + # through &t0 read as a 0 stored to the spilled by-value parameter. + ("488_sccp_var_addr_not_slot.c", 0), + # Post-RA reload elimination pins the dropped load's destination and the + # stored value together; codegen's scratch fixup moved one of them away. + ("489_reload_elim_pins_regs.c", 0), + # dse follows a frame object's address through pointer arithmetic into a + # VAR; the stores filling the object died while it was read through it. + ("490_dse_addr_through_var_add.c", 0), + # A frame word-copy run whose loads share one register is not proof the + # last word dies: it was read again after the LDM/STM block copy. + ("491_block_copy_last_word_live.c", 0), + # A struct-copy round trip whose temporary is also read through its + # address elsewhere is not a private round trip. + ("492_struct_roundtrip_addr_escape.c", 0), + # SCCP does not take a direct VAR store's spill placeholder for a frame + # slot either (CaptureValue.wrap's case 0 payload folded to 0). + ("493_sccp_var_store_not_slot.c", 0), + # dead_addrvar deletes STORE_INDEXEDs through a dead local's LEA along + # with the LEA; left behind they stored through an undefined register. + ("494_dead_addrvar_store_indexed.c", 0), + # A stack-copied struct argument whose base scratch is LR loads its words + # through another register, and not through a live IP. + ("495_stack_struct_arg_base_in_lr.c", 0), + # frame_colour follows a frame address through `T <-- V [LOAD]` (a VAR's + # value), so storing it escapes the object it points into. + ("496_frame_colour_ptr_via_var_load.c", 0), + # The ARMv8-M runtime: arm_mem.S's memcpy/memmove/memset at every + # alignment, tail length and overlap, and the 64-bit division. + ("498_aeabi_mem_div_runtime.c", 0), + # ra:redundant_cmp takes no flags from a compare codegen makes a CBZ (sets + # none) or from a 64-bit compare (lowered per reader: ORRS for != 0). + ("499_redundant_cmp_cbz_i64_flags.c", 0), + # An asm statement clobbering LR (a `bl` in it) makes its function save LR. + ("500_asm_lr_clobber_is_a_call.c", 0), + # Small aggregates copied through a pointer held in a local go inline; + # a struct returned into a local is not copied onto itself. + ("501_struct_copy_through_pointer_local.c", 0), + # ssa:dead_loop's vinfo growth left spare entries claiming instruction 0 as + # their definition; ssa:dce deleted it (a loop counter's initial value). + ("502_dead_loop_vinfo_grow_def_instr.c", 0), + # run_register_coalescing's swap left the per-vreg allocation stale; + # ra:reload_elim deleted a reload into a register the swap had moved. + ("503_coalesce_swap_stale_allocation.c", 0), + # sra: 64-bit fields become VARs, a doubleword also touched a word at a + # time two word VARs (PACK64 to read it whole, split stores); sccp folds a + # PACK64 of constants and ssa:fold cancels pack/split chains. + ("504_sra_wide_doubleword_fields.c", 0), + # 64-bit and narrow-constant slot STOREs are SSA defs; a local only ever + # stored one constant reads as it (inlined `bits`, int64_t parameters). + ("505_const_store_var_forward.c", 0), + # A constant-size __aeabi_memmove4/8 struct copy of up to 128 bytes is + # emitted inline as LDM/STM; a packed-member copy names plain memmove. + ("506_inline_aligned_struct_copy.c", 0), + # UMULL/SMULL fold like MUL: by 0 to 0, both-constant to the product (low + # words, zero-/sign-extended); zig.h's u128 cross terms on a widened u64. + ("507_widening_mul_fold.c", 0), + # UMULL + zero-extended words -> UMAAL before RA (opt/ra/umaal_fusion.c): + # one/two addends, word views of pairs, loads, bignum row, pair pressure. + ("508_umaal_fusion.c", 0), + # A byte/halfword STORE into a register temp is a whole def (MOV, or a + # word store of the extended value when spilled): not live from entry. + ("509_narrow_temp_store_def.c", 0), + # x & M / UBFX x,#0,#w is a copy when x cannot hold a bit outside M: + # unsigned narrow loads, UBFX fields, temp STORE copies; signed keeps it. + ("510_known_zero_extend_fold.c", 0), + # SETIF+TEST_ZERO+JUMPIF fused again after phi resolution: the Zig CBE's + # one reused `bool t` becomes one single-use temp per check under SSA. + ("511_late_setif_branch_fuse.c", 0), + # Pointer-returning helpers inline inside another expansion (Zig CBE + # header()/acquire() under view()); 930725-1's ternary shape, nested. + ("512_nested_inline_ptr_return.c", 0), + ("513_param_across_block_copy.c", 0), + ("514_inline_large_copies.c", 0), + ("515_const_local_table_rodata.c", 0), + ("516_symref_struct_arg_from_table.c", 0), + ("517_sra_pointer_var_roots.c", 0), + ("518_ssa_promote_entry_and_sra_fields.c", 0), + ("519_sra_struct_arg_words.c", 0), + ("520_sra_copy_and_result_buffer.c", 0), + ("521_sra_struct_arg_words_large.c", 0), + ("522_unroll_zig_loop_tables.c", 0), + ("523_zig_accessor_chain_collapse.c", 0), + ("522_sra_narrow_field_read_wider.c", 0), + ("524_sra_far_struct_arg.c", 0), + ("528_sra_partial_fields.c", 0), + ("529_sra_pair_copy.c", 0), + ("530_cbz_over_aligned_branch_target.c", 0), + ("531_dyn_arg_window.c", 0), + ("532_unreachable_branches.c", 0), + ("533_inline_scope_caller_object.c", 0), + ("534_sccp_entry_store_escaped_call.c", 0), + ("535_inline_big_struct_param.c", 0), + ("536_frame_relayout_retry_keeps_copies.c", 0), + ("537_indexed_chain_deref_base.c", 0), + ("538_const_local_table_zero_fill.c", 0), + ("539_cprop_store_copy_inline_helper.c", 0), + ("540_coalesce_high_pressure_latch.c", 0), + ("541_param_home_fwd.c", 0), + ("542_sret_nrvo.c", 0), + ("543_dead_def_cbz_narrow_ext.c", 0), + ("544_frame_word_pair_slot_copy.c", 0), + ("545_cross_jump_alloc.c", 0), + ("546_ssa_symref_store_def.c", 0), + ("548_store_imm_scratch_not_base.c", 0), + # Word-aligned copies of 3..32 words call libtcc1's __tcc_wcopy_N: struct + # assignment, aligned memcpy, by-value stack arguments (tail call too) and + # initialiser images in a leaf; every chain entry, guards around the copy. + ("549_copy_stub_calls.c", 0), + # SRA: a u16 stored into a word field whose other half is struct padding is + # a plain move (Zig error unions), not a read of the old word and a BFI. + ("600_sra_padded_partial_store.c", 0), + # __builtin_{add,sub,mul}_overflow on 8/16/32-bit results: the 32-bit + # add/sub checks and the int-width narrow path, against 64-bit reference. + ("601_overflow_builtins_narrow.c", 0), + # -Os machine outliner: one template instantiated eight times; its windows + # (a jump target at the start, a compare feeding the branch after it, an + # IT block inside) become shared bodies called with BL. + ("602_outline_shared_windows.c", 0), + # awk's record reader: a static helper's out-parameter writes to the + # caller's locals were lost at -O1+, so the whole input was one record + ("603_inline_outparam_record_split.c", 0), + # toysh's expand_arg_nobrace: "if (!ant) ant = &deck;" with the deck + # passed as a stack parameter; "ant != &deck" folded to equal + ("606_param_ne_local_address.c", 0), + # toybox awk's logical NOT, "STKP->num = ! get_set_logical();": a + # comparison still in the flags was converted as its compared operand + ("607_assign_not_of_call_writing_lhs.c", 0), + # toysh's wildcard_matchlen against literal case patterns, and strchr + # finding the NUL terminator: device-only shapes, pinned + ("604_toysh_wildcard_literal.c", 0), + ("605_strchr_nul_terminator.c", 0), + # First-iteration-exit loop elimination (20070824-1.c pointer-chase shape # + runtime control loops); pins behavior across the legacy -> # ssa:first_iter_exit migration. @@ -1449,6 +1763,11 @@ def _expect_line(sut, expected_line: str, *, timeout: int = 1, float_tol: float # ifdef_stack overflow check -> the first `#if` in the predefs reported # "memory full (ifdef)" and the self-hosted compiler couldn't preprocess. ("bug_cmp_ptr_array_alias.c", 0), + # Bug: ra:load_postinc folded `++p` into the first `*p` load while an + # outer call still read *p -- its FUNCPARAMVAL sits above the load, but + # the value is read at the CALL, below it. printf got p+1 (FatFs + # f_setlabel -> FR_INVALID_NAME in sdformat). + ("bug_postinc_call_arg_deref.c", 0), ] @@ -1584,8 +1903,16 @@ def _escape_regex(line): return re.escape(line) +def _whole_line(pattern): + """Anchor a pattern to one whole output line (the stream is consumed up + to each match, so the next line starts right after its newline). Trailing + blanks do not count, as in the upstream tests2 harness: several of those + programs print a space before each newline their .expect files omit.""" + return r"(?m)^" + pattern + r"[ \t]*\r?\n" + + def _run_qemu_test(test_file, expected_exit_code, args=None, defines=None, opt_level="-O0", output_dir=None, timeout=10, - float_abi=None): + float_abi=None, extra_objs=None): expected_lines, expect_exit = load_expect_file(test_file) if expect_exit is not None: expected_exit_code = expect_exit @@ -1593,7 +1920,7 @@ def _run_qemu_test(test_file, expected_exit_code, args=None, defines=None, opt_l if float_abi: opt_suffix += f"_{float_abi}" config = CompileConfig(extra_cflags=opt_level, output_suffix=opt_suffix, output_dir=output_dir, - float_abi=float_abi) + float_abi=float_abi, extra_objs=extra_objs) sut, loglines = run_test(test_file, MACHINE, args, defines=defines, config=config) expected_lines = _strip_compiler_output(expected_lines, loglines) try: @@ -1778,6 +2105,25 @@ def test_libm_float_abi(float_abi, opt_level, tmp_path): float_abi=float_abi) +# AAPCS32 interop with gcc-built code: abi_mix_b.c is compiled by +# arm-none-eabi-gcc, main and abi_mix_a.c by tcc, and each side calls the +# other with structs of 17-36 bytes in registers, split and on the stack +# (see abi_mix.h). Built for the suite's float ABI so the objects link. +@pytest.mark.parametrize("opt_level", OPT_LEVELS, ids=[f"abi_gcc_interop{o}" for o in OPT_LEVELS]) +def test_abi_gcc_interop(opt_level, tmp_path): + import subprocess + from qemu_run import DEFAULT_FLOAT_ABI, DEFAULT_FPU + (tmp_path / "gcc").mkdir() + gcc_obj = tmp_path / "gcc" / "abi_mix_b_gcc.o" # outside the build dir `make clean` removes + fp = [f"-mfloat-abi={DEFAULT_FLOAT_ABI}"] + ([] if DEFAULT_FLOAT_ABI == "soft" else [f"-mfpu={DEFAULT_FPU}"]) + r = subprocess.run(["arm-none-eabi-gcc", "-mcpu=cortex-m33", "-mthumb", *fp, "-O2", "-ffunction-sections", + "-c", str(CURRENT_DIR / "abi_mix_b.c"), "-o", str(gcc_obj)], + capture_output=True, text=True) + assert r.returncode == 0, r.stderr + _run_qemu_test(["abi_mix_main.c", "abi_mix_a.c"], 0, opt_level=opt_level, output_dir=tmp_path / "build", + extra_objs=[gcc_obj]) + + # Nested function xfail tests (not yet implemented) def _generate_nested_xfail_params(): params = [] @@ -1941,6 +2287,45 @@ def test_function_sections_bugs(test_file, expected_exit_code, opt_level, tmp_pa _run_qemu_test(test_file, expected_exit_code, opt_level=cflags, output_dir=tmp_path) +# --------------------------------------------------------------------------- +# Tests requiring every function in one .text (-fno-function-sections) +# --------------------------------------------------------------------------- + +NO_FUNCTION_SECTIONS_TEST_FILES = [ + # Late reopt re-emitting a function takes the statics folded onto it + # (identical code folding) along to its new body. With a section per + # function the erased body leaves its section empty and the new one lands + # where the stale alias still points, so the bug needs one shared .text. + ("497_late_reopt_keeps_icf_alias.c", 0), +] + + +def _generate_no_func_sections_params(): + params = [] + ids = [] + for test_file, expected in NO_FUNCTION_SECTIONS_TEST_FILES: + for opt in OPT_LEVELS: + params.append((test_file, expected, opt)) + ids.append(f"{_test_id(test_file)}{opt}") + return params, ids + + +_NO_FUNC_SECTIONS_PARAMS, _NO_FUNC_SECTIONS_IDS = ( + _generate_no_func_sections_params() if NO_FUNCTION_SECTIONS_TEST_FILES else ([], [])) + + +@pytest.mark.parametrize("test_file,expected_exit_code,opt_level", _NO_FUNC_SECTIONS_PARAMS, + ids=_NO_FUNC_SECTIONS_IDS) +def test_no_function_sections_bugs(test_file, expected_exit_code, opt_level, tmp_path): + """Tests compiled into one .text: the harness Makefile passes + -ffunction-sections, and the -fno- form after it wins.""" + if test_file is None: + pytest.fail("test_file is None") + + cflags = f"{opt_level} -fno-function-sections" + _run_qemu_test(test_file, expected_exit_code, opt_level=cflags, output_dir=tmp_path) + + # --------------------------------------------------------------------------- # Tests requiring -fgnu89-inline # --------------------------------------------------------------------------- @@ -1994,6 +2379,9 @@ def test_gnu89_inline_bugs(test_file, expected_exit_code, opt_level, tmp_path): # set → th_push returns {0,0}. ("bug_struct_mask_copy.c", 0), ("bug_mask_copy_noloop.c", 0), + + # A call to a static function defined further down skips the R9 reload. + ("test_r9_static_forward_call.c", 0), ] diff --git a/tests/ir_tests/test_r9_static_forward_call.c b/tests/ir_tests/test_r9_static_forward_call.c new file mode 100644 index 000000000..fa83d7afc --- /dev/null +++ b/tests/ir_tests/test_r9_static_forward_call.c @@ -0,0 +1,52 @@ +/* Text/data separation: R9 holds the data base. A call to a static function + * -- this module's own even when its definition comes further down -- skips the + * post-call R9 reload. The callees defined later read and write globals, + * recurse, and call each other, so a wrong R9 after any call shows up as a + * wrong global access. */ +#include + +static int counter; +static int table[8] = {3, 1, 4, 1, 5, 9, 2, 6}; +static const char *const names[3] = {"zero", "one", "two"}; + +static int bump(int k); +static int walk(int n); +static int name_len(int i); + +__attribute__((noinline)) static int driver(int k) +{ + int a = bump(k); + int b = walk(7); + int c = name_len(k % 3); + return a * 1000 + b * 10 + c + counter; +} + +__attribute__((noinline)) static int bump(int k) +{ + counter += k; + return table[k & 7] + counter; +} + +__attribute__((noinline)) static int walk(int n) +{ + if (n <= 0) + return table[0]; + return table[n & 7] + walk(n - 1) + bump(1); +} + +__attribute__((noinline)) static int name_len(int i) +{ + int n = 0; + while (names[i][n]) + n++; + return n + bump(0); +} + +int main(void) +{ + printf("%d\n", driver(5)); + int before = counter; + int second = driver(2); + printf("%d %d\n", before, second); + return 0; +} diff --git a/tests/ir_tests/test_r9_static_forward_call.expect b/tests/ir_tests/test_r9_static_forward_call.expect new file mode 100644 index 000000000..26cf23494 --- /dev/null +++ b/tests/ir_tests/test_r9_static_forward_call.expect @@ -0,0 +1,2 @@ +15040 +12 19688 diff --git a/tests/ir_tests/test_return_address.c b/tests/ir_tests/test_return_address.c new file mode 100644 index 000000000..b5e5b1a3b --- /dev/null +++ b/tests/ir_tests/test_return_address.c @@ -0,0 +1,168 @@ +/* __builtin_return_address(0) reads the LR the prologue saved, off SP, without + * a frame pointer. One callee per prologue shape: a leaf (LR saved only for + * the builtin), a non-leaf, stack arguments, a struct straddling r3 and the + * stack (r0-r3 pushed first), variadic (r0-r3 save + frame pointer), alloca + * (frame-record layout), a frame past 4 KB, a read after a call, and a tail + * call (its teardown must pop the LR the builtin made it save). Each + * returns its return address; its caller checks that it lands a little past + * the caller's own PC, read just before the call, and two call sites must + * differ. The PC is read with inline asm, not taken from `&caller`: on YasOS a + * function pointer is the loader's r9-setting thunk, not the code address. */ +#include +#include +#include + +#define NOINLINE __attribute__((noinline)) + +struct five +{ + int v[5]; +}; + +static volatile int sunk; + +NOINLINE static void sink(int x) +{ + sunk += x; +} + +NOINLINE static void *leaf(void) +{ + return __builtin_return_address(0); +} + +NOINLINE static void *nonleaf(int x) +{ + sink(x); + return __builtin_return_address(0); +} + +NOINLINE static void *stack_args(int a, int b, int c, int d, int e, int f) +{ + sink(a + b + c + d + e + f); + return __builtin_return_address(0); +} + +NOINLINE static void *split_struct(int a, int b, int c, struct five s) +{ + sink(a + b + c + s.v[0] + s.v[4]); + return __builtin_return_address(0); +} + +NOINLINE static void *variadic(int n, ...) +{ + va_list ap; + int total = 0; + va_start(ap, n); + while (n-- > 0) + total += va_arg(ap, int); + va_end(ap); + sink(total); + return __builtin_return_address(0); +} + +NOINLINE static void *with_alloca(int n) +{ + char *p = __builtin_alloca(n); + memset(p, 1, n); + sink(p[n - 1]); + return __builtin_return_address(0); +} + +NOINLINE static void *big_frame(int i) +{ + volatile char buf[6000]; + buf[i] = 1; + sink(buf[i]); + return __builtin_return_address(0); +} + +NOINLINE static void *after_call(void) +{ + sink(1); + void *r = __builtin_return_address(0); + sink(2); + return r; +} + +NOINLINE static void *pass(void *p) +{ + return p; +} + +NOINLINE static void *tail_call(void) +{ + return pass(__builtin_return_address(0)); +} + +/* A code address in the caller: the PC, which reads 4 past the mov. */ +#define HERE() ({ void *here_; __asm__ volatile("mov %0, pc" : "=r"(here_)); here_; }) + +/* A return address lands inside its caller, a little past `here`. */ +static int near(void *ra, void *here) +{ + unsigned long d = (unsigned long)ra - (unsigned long)here; + return d > 0 && d < 256; +} + +NOINLINE static int call_leaf(void) +{ + void *here = HERE(); + return near(leaf(), here); +} +NOINLINE static int call_nonleaf(void) +{ + void *here = HERE(); + return near(nonleaf(3), here); +} +NOINLINE static int call_stack_args(void) +{ + void *here = HERE(); + return near(stack_args(1, 2, 3, 4, 5, 6), here); +} +NOINLINE static int call_split_struct(void) +{ + struct five s = {{1, 2, 3, 4, 5}}; + void *here = HERE(); + return near(split_struct(7, 8, 9, s), here); +} +NOINLINE static int call_variadic(void) +{ + void *here = HERE(); + return near(variadic(3, 1, 2, 3), here); +} +NOINLINE static int call_with_alloca(void) +{ + void *here = HERE(); + return near(with_alloca(40), here); +} +NOINLINE static int call_big_frame(void) +{ + void *here = HERE(); + return near(big_frame(5000), here); +} +NOINLINE static int call_after_call(void) +{ + void *here = HERE(); + return near(after_call(), here); +} +NOINLINE static int call_tail_call(void) +{ + void *here = HERE(); + return near(tail_call(), here); +} + +NOINLINE static int two_sites(void) +{ + void *a = leaf(); + void *b = leaf(); + return a != b; +} + +int main(void) +{ + printf("%d %d %d %d\n", call_leaf(), call_nonleaf(), call_stack_args(), call_split_struct()); + printf("%d %d %d %d %d %d\n", call_variadic(), call_with_alloca(), call_big_frame(), call_after_call(), + call_tail_call(), two_sites()); + return 0; +} diff --git a/tests/ir_tests/test_return_address.expect b/tests/ir_tests/test_return_address.expect new file mode 100644 index 000000000..c37ae7e42 --- /dev/null +++ b/tests/ir_tests/test_return_address.expect @@ -0,0 +1,2 @@ +1 1 1 1 +1 1 1 1 1 1 diff --git a/tests/ir_tests/test_self_store.c b/tests/ir_tests/test_self_store.c new file mode 100644 index 000000000..a2f2a7893 --- /dev/null +++ b/tests/ir_tests/test_self_store.c @@ -0,0 +1,134 @@ +/* Copies between objects that frame relayout places on the same bytes: after + * relayout the copy loads and stores the same slot, and the stores (and the + * loads left without a use) are removed. The values must survive: whole-word + * fields, narrow fields, a copy whose source was written on one path only, a + * copy followed by a store through a pointer into the new object, and fields + * read again after the copy. Callees are reached through volatile pointers so + * nothing is known about what they do with the objects. */ +#include + +struct R3 +{ + int *p; + int b; + int c; +}; + +struct N +{ + int *p; + unsigned char f; + short s; +}; + +struct W +{ + int a, b, c, d, e; +}; + +static int sink; + +static struct R3 get3_impl(int *m, int k) +{ + struct R3 r = {m + k, k * 3, k & 1}; + return r; +} +static struct N getn_impl(int *m, int k) +{ + struct N r = {m + k, (unsigned char)(200 + k), (short)(-1000 * k)}; + return r; +} +static struct W getw_impl(int k) +{ + struct W w = {k, k + 1, k + 2, k + 3, k + 4}; + return w; +} +static void use3_impl(struct R3 *r) +{ + sink += r->b + r->c; +} +static void usen_impl(struct N *r) +{ + sink += r->f + r->s; +} +static void usew_impl(struct W *w) +{ + sink += w->a + w->b * 2 + w->c * 3 + w->d * 4 + w->e * 5; +} + +struct R3 (*volatile get3)(int *, int) = get3_impl; +struct N (*volatile getn)(int *, int) = getn_impl; +struct W (*volatile getw_)(int) = getw_impl; +void (*volatile use3)(struct R3 *) = use3_impl; +void (*volatile usen)(struct N *) = usen_impl; +void (*volatile usew)(struct W *) = usew_impl; + +int fields(int *m, int k) +{ + struct R3 t; + struct R3 u; + t = get3(m, k); + u.p = t.p; + u.b = t.b; + u.c = t.c; + use3(&u); + if (u.c) + *u.p = k; + return u.b; +} + +int whole(int *m, int k) +{ + struct R3 t; + struct R3 u; + t = get3(m, k); + u = t; + use3(&u); + return u.b + u.c; +} + +int narrow(int *m, int k) +{ + struct N t; + struct N u; + t = getn(m, k); + u.p = t.p; + u.f = t.f; + u.s = t.s; + usen(&u); + *u.p += u.f; + return u.s; +} + +int wide(int k) +{ + struct W t = getw_(k); + struct W u = t; + usew(&u); + u.c = 100; + usew(&u); + return u.a + u.c + u.e; +} + +int branchy(int k) +{ + struct W t = getw_(k); + if (k & 2) + t.b = 77; + struct W u = t; + usew(&u); + return u.b; +} + +int main(void) +{ + int m[8] = {0}; + printf("fields %d %d\n", fields(m, 2), fields(m, 3)); + printf("whole %d %d\n", whole(m, 4), whole(m, 5)); + printf("narrow %d %d\n", narrow(m, 1), narrow(m, 6)); + printf("wide %d\n", wide(5)); + printf("branchy %d %d\n", branchy(4), branchy(6)); + printf("m %d %d %d %d %d\n", m[1], m[2], m[3], m[5], m[6]); + printf("sink %d\n", sink); + return 0; +} diff --git a/tests/ir_tests/test_self_store.expect b/tests/ir_tests/test_self_store.expect new file mode 100644 index 000000000..826b1a8b1 --- /dev/null +++ b/tests/ir_tests/test_self_store.expect @@ -0,0 +1,7 @@ +fields 6 9 +whole 12 16 +narrow -1000 -6000 +wide 114 +branchy 5 77 +m 201 0 3 0 206 +sink -5670 diff --git a/tests/ir_tests/test_slot_const_store_fold.c b/tests/ir_tests/test_slot_const_store_fold.c new file mode 100644 index 000000000..3d0d65b6c --- /dev/null +++ b/tests/ir_tests/test_slot_const_store_fold.c @@ -0,0 +1,59 @@ +/* Byte and halfword field stores over a zero-filled local word fold into that + * word's store (slot_const_store_fold) -- the compound-literal shape of a + * struct with bool fields passed by value. The cases that must not fold: the + * word read between the stores, a store through a pointer that may reach the + * local, and a volatile local. */ +#include +#include +#include + +struct opt { uint32_t payload; bool is_null; }; +struct O { struct opt w, p; uint8_t fill, align; uint16_t h; }; + +__attribute__((noinline)) static unsigned take(struct O o, int x) +{ + return o.w.payload * 7 + o.w.is_null * 3 + o.p.payload + o.p.is_null * 5 + o.fill * 11 + o.align * 13 + o.h * 17 + + (unsigned)x; +} + +__attribute__((noinline)) static unsigned lit(int x) +{ + return take((struct O){{100, true}, {200, false}, 2, 32, 0x1234}, x); +} + +/* The word is read between the zero fill and the field store. */ +__attribute__((noinline)) static unsigned read_between(int x) +{ + struct O o = {{5, false}, {6, false}, 0, 0, 0}; + unsigned before = *(volatile uint8_t *)&o.fill; + o.fill = (uint8_t)x; + return take(o, 1) + before; +} + +/* A store through a pointer to the same local between the stores. */ +__attribute__((noinline)) static unsigned via_pointer(uint8_t *p, int x) +{ + struct O o = {{1, false}, {2, true}, 0, 0, 0}; + uint8_t *q = p ? p : &o.align; + *q = 9; + o.fill = (uint8_t)x; + return take(o, 2); +} + +__attribute__((noinline)) static unsigned vol(int x) +{ + volatile struct O o = {{3, true}, {4, true}, 0, 0, 0}; + o.fill = 7; + o.h = (uint16_t)x; + return o.fill + o.h + o.w.is_null; +} + +int main(void) +{ + uint8_t sink = 0; + printf("%u %u\n", lit(1), lit(40)); + printf("%u\n", read_between(3)); + printf("%u %u\n", via_pointer(0, 4), via_pointer(&sink, 4)); + printf("%u %u\n", vol(1000), sink); + return 0; +} diff --git a/tests/ir_tests/test_slot_const_store_fold.expect b/tests/ir_tests/test_slot_const_store_fold.expect new file mode 100644 index 000000000..6049849b1 --- /dev/null +++ b/tests/ir_tests/test_slot_const_store_fold.expect @@ -0,0 +1,4 @@ +80562 80601 +75 +177 60 +1008 9 diff --git a/tests/ir_tests/test_small_aggregate_copy.c b/tests/ir_tests/test_small_aggregate_copy.c new file mode 100644 index 000000000..8411d6b9b --- /dev/null +++ b/tests/ir_tests/test_small_aggregate_copy.c @@ -0,0 +1,114 @@ +/* Small aggregate copies inlined as LOAD/STORE chunks instead of + * __aeabi_memmove (vstore.c / small_aggregate_copy_plan). Every shape the + * plan accepts, between frame slots, globals and pointers, including an + * overlapping copy that must keep memmove semantics. */ +#include +#include +#include + +struct eu +{ + uint16_t error; /* Zig's `!void` */ +}; +struct two_bytes +{ + uint8_t a, b; +}; +struct slice +{ + const char *ptr; + uint32_t len; +}; +struct opt_u16 +{ + uint16_t payload; + uint8_t is_null; /* 1 byte of padding follows */ +}; +union word_or_halves +{ + uint32_t w; + uint16_t h[2]; +}; +struct nested +{ + struct two_bytes pair; + uint16_t arr[1]; +}; + +struct eu g_eu = {.error = 0x1234}; +struct slice g_slice = {"hello", 5}; + +__attribute__((noinline)) static struct eu make_eu(int v) +{ + struct eu e = {.error = (uint16_t)v}; + return e; +} +__attribute__((noinline)) static void put_eu(struct eu *out, int v) +{ + struct eu t = make_eu(v); + *out = t; /* frame slot -> pointer */ +} +__attribute__((noinline)) static struct eu get_eu(const struct eu *in) +{ + struct eu t; + t = *in; /* pointer -> frame slot */ + return t; +} +__attribute__((noinline)) static void copy_through(struct opt_u16 *dst, const struct opt_u16 *src) +{ + *dst = *src; /* pointer -> pointer */ +} +__attribute__((noinline)) static unsigned bytes_of(struct two_bytes x) +{ + struct two_bytes y; + y = x; + return y.a * 256u + y.b; +} + +int main(void) +{ + unsigned sum = 0; + + struct eu a, b; + a = make_eu(7); + b = a; /* frame slot -> frame slot */ + sum += b.error; + + struct eu out; + put_eu(&out, 300); + sum += get_eu(&out).error; + + struct eu from_global; + from_global = g_eu; /* global -> frame slot */ + g_eu = make_eu(9); /* frame slot -> global */ + sum += from_global.error + g_eu.error; + + struct slice s; + s = g_slice; + sum += s.len + (unsigned)(s.ptr[1] == 'e'); + + struct opt_u16 o1 = {0xbeef, 1}, o2; + copy_through(&o2, &o1); + sum += o2.payload + o2.is_null; + + union word_or_halves u1, u2; + u1.w = 0x00030004u; + u2 = u1; + sum += u2.h[0] + u2.h[1]; + + struct nested n1 = {{1, 2}, {40}}, n2; + n2 = n1; + sum += n2.pair.a + n2.pair.b + n2.arr[0]; + + sum += bytes_of((struct two_bytes){3, 4}); + + /* Overlapping copy: shift two 2-byte structs down one slot. */ + struct eu arr[3] = {{1}, {2}, {3}}; + struct eu *p = &arr[0], *q = &arr[1]; + *p = *q; + *q = arr[2]; + sum += arr[0].error * 100 + arr[1].error * 10 + arr[2].error; + + printf("%u\n", sum); + return 0; +} diff --git a/tests/ir_tests/test_small_aggregate_copy.expect b/tests/ir_tests/test_small_aggregate_copy.expect new file mode 100644 index 000000000..cc250a060 --- /dev/null +++ b/tests/ir_tests/test_small_aggregate_copy.expect @@ -0,0 +1 @@ +54917 diff --git a/tests/ir_tests/test_spill_slot_reuse.c b/tests/ir_tests/test_spill_slot_reuse.c new file mode 100644 index 000000000..ffdada710 --- /dev/null +++ b/tests/ir_tests/test_spill_slot_reuse.c @@ -0,0 +1,51 @@ +/* Spill slots are reused once their interval has expired (ra_spill_pool_*). + * High register pressure in several phases, so many short-lived values spill + * and hand their slots on; loop-carried values and an eviction victim must + * keep their slot for their whole live range. Every phase is checked. */ +#include + +__attribute__((noinline)) static unsigned mix(unsigned a, unsigned b) { return a * 31u + b; } + +__attribute__((noinline)) static unsigned phases(unsigned s) +{ + unsigned total = 0; + /* Phase 1: 14 simultaneously live values force spills. */ + unsigned a0 = s + 1, a1 = s + 2, a2 = s + 3, a3 = s + 4, a4 = s + 5, a5 = s + 6, a6 = s + 7; + unsigned a7 = s + 8, a8 = s + 9, a9 = s + 10, a10 = s + 11, a11 = s + 12, a12 = s + 13, a13 = s + 14; + total += mix(a0, a13) + mix(a1, a12) + mix(a2, a11) + mix(a3, a10) + mix(a4, a9) + mix(a5, a8) + mix(a6, a7); + /* Phase 2: a new set that may take phase 1's freed slots. */ + unsigned b0 = total ^ 1, b1 = total ^ 2, b2 = total ^ 3, b3 = total ^ 4, b4 = total ^ 5, b5 = total ^ 6; + unsigned b6 = total ^ 7, b7 = total ^ 8, b8 = total ^ 9, b9 = total ^ 10, b10 = total ^ 11, b11 = total ^ 12; + total += mix(b0, b11) + mix(b1, b10) + mix(b2, b9) + mix(b3, b8) + mix(b4, b7) + mix(b5, b6); + /* Phase 3: loop-carried values live across calls while temporaries churn. */ + unsigned c0 = total, c1 = total >> 1, c2 = total >> 2, c3 = total >> 3; + for (unsigned i = 0; i < 10; i++) + { + unsigned t0 = mix(c0, i), t1 = mix(c1, i), t2 = mix(c2, i), t3 = mix(c3, i); + unsigned t4 = t0 ^ t1, t5 = t2 ^ t3, t6 = t4 + t5, t7 = t6 * 3u; + c0 += t7 + t0; + c1 ^= t6 + t1; + c2 += t5 + t2; + c3 ^= t4 + t3; + } + return total + c0 + c1 + c2 + c3; +} + +__attribute__((noinline)) static unsigned long long wide(unsigned long long x) +{ + /* 64-bit values spill as 8-byte slots. */ + unsigned long long w0 = x + 1, w1 = x * 3, w2 = x ^ 0x55, w3 = x << 3, w4 = x >> 2, w5 = x + 77; + unsigned long long r = mix((unsigned)w0, (unsigned)w5) + w1 * w4 + (w2 ^ w3); + unsigned long long v0 = r + 5, v1 = r * 7, v2 = r ^ 0xaa, v3 = r << 1, v4 = r >> 3, v5 = r + 99; + return r + mix((unsigned)v0, (unsigned)v5) + v1 * v4 + (v2 ^ v3); +} + +int main(void) +{ + unsigned acc = 0; + for (unsigned s = 0; s < 4; s++) + acc = acc * 7u + phases(s); + printf("%u\n", acc); + printf("%llu\n", wide(12345)); + return 0; +} diff --git a/tests/ir_tests/test_spill_slot_reuse.expect b/tests/ir_tests/test_spill_slot_reuse.expect new file mode 100644 index 000000000..0a92dc0dd --- /dev/null +++ b/tests/ir_tests/test_spill_slot_reuse.expect @@ -0,0 +1,2 @@ +1715883720 +11530894345689762 diff --git a/tests/ir_tests/test_sra.c b/tests/ir_tests/test_sra.c new file mode 100644 index 000000000..67882c820 --- /dev/null +++ b/tests/ir_tests/test_sra.c @@ -0,0 +1,155 @@ +/* Small struct locals whose fields are only read and written whole live in + * registers (sra). The shapes it meets: one-word wrappers passed and returned + * by value, multi-word values copied between locals and carried round a loop, + * merged from two paths, and narrow fields -- an error union's u16 code, + * bool and u8 tags, a signed short -- which must truncate on store and extend + * on load exactly as memory does. Objects that must stay in memory -- a float + * member passed by value, a volatile struct, one whose address escapes, a + * union read through overlapping or differently signed fields -- keep their + * exact behaviour. */ +#include +#include +#include + +struct Idx { uint32_t v; }; +struct Pair { uint32_t a, b; }; +struct Quad { uint32_t a, b, c, d; }; +struct F1 { float f; }; +struct Tag { uint8_t t; uint8_t pad[3]; }; + +__attribute__((noinline)) static struct Idx next(struct Idx i) { return (struct Idx){i.v * 3 + 1}; } +__attribute__((noinline)) static uint32_t get(struct Idx i) { return i.v; } +__attribute__((noinline)) static uint32_t sum_pair(struct Pair p) { return p.a * 7 + p.b; } +__attribute__((noinline)) static uint32_t sum_quad(struct Quad q) { return q.a + q.b * 2 + q.c * 3 + q.d * 4; } +__attribute__((noinline)) static float half(struct F1 x) { return x.f * 0.5f; } +__attribute__((noinline)) static uint32_t tagv(struct Tag t) { return t.t; } +__attribute__((noinline)) static void poke(uint32_t *p) { *p += 1000; } + +/* One-word values through calls, changed between them. */ +__attribute__((noinline)) static uint32_t chain(uint32_t k) +{ + struct Idx a, b; + a.v = k; + b = next(a); + a = b; + a.v += 2; + b = next(a); + return get(a) * 100 + get(b); +} + +/* A value carried round a loop, merged from two paths each time. */ +__attribute__((noinline)) static uint32_t loop(uint32_t n) +{ + struct Pair p = {1, 2}; + struct Pair t; + for (uint32_t i = 0; i < n; i++) + { + if (i & 1) + { + t.a = p.b; + t.b = p.a + i; + } + else + { + t.a = p.a * 2; + t.b = p.b; + } + p = t; + } + return sum_pair(p); +} + +/* Four words, copied whole and field by field. */ +__attribute__((noinline)) static uint32_t quad(uint32_t k) +{ + struct Quad q = {k, k + 1, k + 2, k + 3}; + struct Quad r = q; + r.c = q.a + q.d; + struct Quad s; + s = r; + s.a ^= 5; + return sum_quad(s) + sum_quad(q); +} + +/* Must stay in memory: a float member passed by value (a VFP register under + * the hard-float ABI), a byte field, a volatile struct, an escaping address. */ +__attribute__((noinline)) static uint32_t stay(uint32_t k) +{ + struct F1 f; + f.f = (float)k; + struct Tag t; + memset(&t, 0, sizeof t); + t.t = (uint8_t)(k + 7); + volatile struct Idx v; + v.v = k; + v.v = v.v + 1; + struct Idx e = {k}; + poke(&e.v); + return (uint32_t)(half(f) * 4.0f) + tagv(t) * 10 + v.v * 100 + e.v; +} + +struct ErrU { uint32_t payload; uint16_t err; }; +struct Tags { uint8_t ok; int8_t delta; int16_t shift; uint16_t code; }; +union Pun { uint32_t w; uint8_t b[4]; int8_t s[4]; }; + +__attribute__((noinline)) static uint32_t fetch(uint32_t k) { return k * 2654435761u; } + +/* An error union built on two paths and checked in a loop. */ +__attribute__((noinline)) static uint32_t err_union(uint32_t n) +{ + struct ErrU e; + uint32_t acc = 0; + for (uint32_t i = 0; i < n; i++) + { + if (fetch(i) & 1) + { + e.payload = 0xaaaaaaaau; + e.err = (uint16_t)(0x10000 + i); /* truncated to i */ + } + else + { + e.payload = fetch(i) >> 8; + e.err = 0; + } + if (e.err != 0) + acc += e.err * 3; + else + acc ^= e.payload; + } + return acc; +} + +/* Narrow fields must truncate on store and extend on load. */ +__attribute__((noinline)) static int32_t tags(int32_t k) +{ + struct Tags t; + t.ok = (uint8_t)(k + 250); /* wraps */ + t.delta = (int8_t)(k * 50); /* sign-extends on read */ + t.shift = (int16_t)(k * 3000); + t.code = (uint16_t)(-k); + int32_t r = t.ok + t.delta + t.shift + t.code; + t.delta = (int8_t)(t.delta - 100); + return r * 7 + t.delta; +} + +/* Read through overlapping or differently signed fields: stays in memory. */ +__attribute__((noinline)) static int32_t pun(uint32_t k) +{ + union Pun p; + p.w = k * 0x01010101u + 0x80; + int32_t r = p.b[0] + p.s[0] + p.b[1]; + p.b[2] = 0xfe; + return r * 1000 + (int32_t)(p.w >> 16) + p.s[2]; +} + +int main(void) +{ + printf("%u %u\n", chain(1), chain(40)); + printf("%u %u %u\n", loop(0), loop(5), loop(12)); + printf("%u %u\n", quad(3), quad(100)); + printf("%u %u\n", stay(2), stay(250)); + printf("%u %u\n", err_union(10), err_union(33)); + printf("%d %d %d\n", tags(1), tags(3), tags(-7)); + printf("%d %d\n", pun(1), pun(200)); + return 0; +} diff --git a/tests/ir_tests/test_sra.expect b/tests/ir_tests/test_sra.expect new file mode 100644 index 000000000..f9d5b4f2c --- /dev/null +++ b/tests/ir_tests/test_sra.expect @@ -0,0 +1,7 @@ +619 12670 +9 49 270 +115 2340 +1396 26860 +244703 2026792 +481802 522810 -145846 +3508 396452 diff --git a/tests/ir_tests/test_stack_deref_fold.c b/tests/ir_tests/test_stack_deref_fold.c new file mode 100644 index 000000000..f309b3aa2 --- /dev/null +++ b/tests/ir_tests/test_stack_deref_fold.c @@ -0,0 +1,110 @@ +/* Loads and stores through a pointer temporary holding a constant frame address + * become direct stack accesses (ssa:stack_deref_fold). The shapes the Zig C + * backend emits: a pointer to each sub-object, set in every arm of a switch; + * narrow fields of either signedness; indexed element accesses; a pointer that + * is also passed on; and one chosen by a merge, which must not be folded. */ +#include +#include + +struct sl +{ + const uint8_t *ptr; + uintptr_t len; +}; +struct ver +{ + uintptr_t major, minor, patch; + struct sl pre, build; +}; +struct range +{ + struct ver min, max; + int8_t tag; + uint16_t flags; +}; + +__attribute__((noinline)) static unsigned keep(const struct range *r) +{ + return (unsigned)(r->min.major * 1000 + r->max.minor * 100 + r->tag + r->flags); +} + +__attribute__((noinline)) static unsigned select_range(int k) +{ + struct range t0; + struct ver *t2; + uintptr_t *t3; + struct sl *t4; + int8_t *tag; + uint16_t *fl; + switch (k) + { + case 1: + t2 = &t0.min; + t3 = &t2->major; + *t3 = 5; + t3 = &t2->minor; + *t3 = 0; + t4 = &t2->pre; + *t4 = (struct sl){NULL, (uintptr_t)0xaaaaaaaau}; + t2 = &t0.max; + t3 = &t2->major; + *t3 = 5; + t3 = &t2->minor; + *t3 = 1; + tag = &t0.tag; + *tag = -3; + fl = &t0.flags; + *fl = 0xfffe; + break; + default: + t2 = &t0.min; + t3 = &t2->major; + *t3 = 27; + t2 = &t0.max; + t3 = &t2->minor; + *t3 = 30; + tag = &t0.tag; + *tag = 7; + fl = &t0.flags; + *fl = 1; + break; + } + /* Read back through fresh pointers, then pass the whole thing on. */ + int8_t *rt = &t0.tag; + uint16_t *rf = &t0.flags; + return keep(&t0) + (unsigned)(*rt * 10) + *rf; +} + +/* Indexed accesses off a pointer into a local array. */ +__attribute__((noinline)) static int indexed(int k) +{ + int16_t a[8]; + int16_t *p = &a[2]; + p[0] = (int16_t)-k; + p[1] = 300; + p[3] = (int16_t)(k * 2); + a[0] = 1; + a[1] = 2; + return a[0] + a[1] + a[2] * 10 + a[3] + a[5] * 100 + p[3]; +} + +/* The pointer comes from a merge: it names one of two objects. */ +__attribute__((noinline)) static int merged(int k) +{ + int x[2] = {1, 2}, y[2] = {30, 40}; + int *p; + if (k) + p = &x[1]; + else + p = &y[0]; + *p += 100; + return x[0] + x[1] + y[0] + y[1] + *p; +} + +int main(void) +{ + printf("%u %u\n", select_range(1), select_range(2)); + printf("%d %d\n", indexed(3), indexed(-4)); + printf("%d %d\n", merged(1), merged(0)); + return 0; +} diff --git a/tests/ir_tests/test_stack_deref_fold.expect b/tests/ir_tests/test_stack_deref_fold.expect new file mode 100644 index 000000000..573e65ff3 --- /dev/null +++ b/tests/ir_tests/test_stack_deref_fold.expect @@ -0,0 +1,3 @@ +136135 30079 +879 -465 +275 303 diff --git a/tests/ir_tests/test_stack_params_in_regs.c b/tests/ir_tests/test_stack_params_in_regs.c new file mode 100644 index 000000000..8ca3a1ea4 --- /dev/null +++ b/tests/ir_tests/test_stack_params_in_regs.c @@ -0,0 +1,120 @@ +/* Stack-passed parameters read more than once stay in the register the + * allocator gave them, loaded by the prologue: 32- and 64-bit ones, near and + * far (a frame larger than the 12-bit immediate), off SP and off the frame + * pointer, rewritten in a loop, live across calls, and in variadic + * functions. A by-value struct's parameter names its memory and stays there, + * whether split across r3 and the stack or wholly on the stack. */ +#include +#include + +__attribute__((noinline)) static int ident(int x) +{ + return x; +} + +__attribute__((noinline)) static int six(int a, int b, int c, int d, int e, int f) +{ + return (a - e) * (b + f) + e * f - (c ^ e) + (d | f); +} + +__attribute__((noinline)) static long long wide(int a, int b, int c, long long x, long long y) +{ + return x * y + (x >> 3) - y * 4 + a + b + c + (x ^ y); +} + +__attribute__((noinline)) static double dbl(int a, int b, int c, int d, double x, double y) +{ + return x * y + x / 4.0 - y + a + b + c + d; +} + +__attribute__((noinline)) static int far_frame(int a, int b, int c, int d, int e, long long w, int f) +{ + volatile char buf[6000]; + for (int i = 0; i < (int)sizeof(buf); i += 97) + buf[i] = (char)((i + e) & 0x3f); + int s = buf[97 * e] + buf[97 * f]; + return s + e * f + (int)(w >> 8) + (int)w + e - f + a + b + c + d; +} + +__attribute__((noinline)) static int rewritten(int a, int b, int c, int d, int e, int n) +{ + for (int i = 0; i < n; i++) + { + e = e * 3 + i; + if (e > 1000) + e %= 997; + } + return e + n + a + b + c + d; +} + +__attribute__((noinline)) static int across_calls(int a, int b, int c, int d, int e, int f) +{ + int s = ident(e) + ident(f); + s += ident(a + e) * f; + return s + e - f + b + c + d; +} + +__attribute__((noinline)) static int with_vla(int a, int b, int c, int d, int e, int f) +{ + int v[e + 1]; + for (int i = 0; i <= e; i++) + v[i] = i * f; + return v[e] + v[e / 2] + e + f + a + b + c + d; +} + +__attribute__((noinline)) static int varsum(int a, int b, int c, int d, int e, int n, ...) +{ + va_list ap; + int s = e * n + e + n; + va_start(ap, n); + for (int i = 0; i < n; i++) + s += va_arg(ap, int) * e; + va_end(ap); + return s + a + b + c + d; +} + +__attribute__((noinline)) static int narrow(int a, int b, int c, int d, signed char e, unsigned short f) +{ + return e * f + e - f + (e < 0) + a + b + c + d; +} + +struct pt +{ + int v[6]; +}; + +__attribute__((noinline)) static int split_struct(int a, int b, int c, struct pt p, int e) +{ + int s = e; + for (int i = 0; i < 6; i++) + s = s * 3 + p.v[i]; + return s + p.v[e % 6] + a + b + c; +} + +/* The Zig C backend's shape: the local copy becomes the parameter's own memory + * (param_copy_alias), read through a pointer and never address-taken. */ +__attribute__((noinline)) static int stack_struct(int a, int b, int c, int d, struct pt p, int e) +{ + struct pt t0; + struct pt const *t1; + t0 = p; + t1 = (struct pt const *)&t0; + return t1->v[1] * e + t1->v[4] - t1->v[0] + a + b + c + d; +} + +int main(void) +{ + struct pt p = {{3, -4, 5, 11, -2, 7}}; + printf("%d\n", six(1, 2, 3, 4, 5, 6)); + printf("%lld\n", wide(1, 2, 3, 123456789012LL, -98765LL)); + printf("%d\n", (int)dbl(1, 2, 3, 4, 2.5, -8.0)); + printf("%d\n", far_frame(1, 2, 3, 4, 5, 0x123456789LL, 7)); + printf("%d\n", rewritten(1, 2, 3, 4, 5, 20)); + printf("%d\n", across_calls(1, 2, 3, 4, 5, 6)); + printf("%d\n", with_vla(1, 2, 3, 4, 9, 3)); + printf("%d\n", varsum(1, 2, 3, 4, 5, 3, 10, 20, 30)); + printf("%d\n", narrow(1, 2, 3, 4, -7, 40000)); + printf("%d\n", split_struct(1, 2, 3, p, 6)); + printf("%d\n", stack_struct(1, 2, 3, 4, p, 6)); + return 0; +} diff --git a/tests/ir_tests/test_stack_params_in_regs.expect b/tests/ir_tests/test_stack_params_in_regs.expect new file mode 100644 index 000000000..bf83e57ad --- /dev/null +++ b/tests/ir_tests/test_stack_params_in_regs.expect @@ -0,0 +1,11 @@ +-2 +-12193317791033185 +-1 +610839921 +225 +55 +61 +333 +-319996 +5023 +-19 diff --git a/tests/ir_tests/test_struct_arg_stack.c b/tests/ir_tests/test_struct_arg_stack.c new file mode 100644 index 000000000..b68c062fc --- /dev/null +++ b/tests/ir_tests/test_struct_arg_stack.c @@ -0,0 +1,76 @@ +/* Structs passed by value from the frame load their words straight off sp/fp + * instead of through a scratch base register. Covers one to five words, a + * struct parameter that arrived on the stack and is passed on, a function with + * a frame pointer (VLA), a struct shorter than a word, and offsets beyond the + * reach of the immediate forms (which must fall back). */ +#include +#include + +struct W1 { uint32_t a; }; +struct W2 { uint32_t a, b; }; +struct W3 { uint32_t a, b, c; }; +struct W4 { uint32_t a, b, c, d; }; +struct W5 { uint32_t a, b, c, d, e; }; +struct B3 { uint8_t x, y, z; }; + +__attribute__((noinline)) static uint32_t s1(uint32_t k, struct W1 w) { return k * 7 + w.a; } +__attribute__((noinline)) static uint32_t s2(struct W2 w) { return w.a * 3 + w.b; } +__attribute__((noinline)) static uint32_t s3(struct W3 w) { return w.a + w.b * 5 + w.c * 11; } +__attribute__((noinline)) static uint32_t s4(struct W4 w) { return w.a + w.b * 2 + w.c * 3 + w.d * 4; } +__attribute__((noinline)) static uint32_t s5(struct W5 w) { return w.a + w.b + w.c + w.d * 9 + w.e * 13; } +__attribute__((noinline)) static uint32_t sb(struct B3 b) { return b.x * 10000u + b.y * 100u + b.z; } +__attribute__((noinline)) static struct W1 mk1(uint32_t k) { return (struct W1){k * 3 + 1}; } + +__attribute__((noinline)) static uint32_t locals(uint32_t k) +{ + struct W1 t0, t1; + t0 = mk1(k); + t1 = t0; + struct W2 w2 = {k, k + 1}; + struct W3 w3 = {k + 2, k + 3, k + 4}; + struct W4 w4 = {k, 2 * k, 3 * k, 4 * k}; + struct W5 w5 = {1, 2, 3, k, k + 5}; + struct B3 b = {(uint8_t)k, (uint8_t)(k + 1), (uint8_t)(k + 2)}; + return s1(k, t1) + s2(w2) + s3(w3) + s4(w4) + s5(w5) + sb(b); +} + +/* q arrives on the stack (r0-r3 are taken) and is passed on. */ +__attribute__((noinline)) static uint32_t pass_on(uint32_t a, uint32_t b, uint32_t c, uint32_t d, struct W3 q, + struct W1 r) +{ + return a + b + c + d + s3(q) + s1(a, r); +} + +/* A VLA forces a frame pointer: fp-relative, possibly negative offsets. */ +__attribute__((noinline)) static uint32_t with_vla(uint32_t n) +{ + volatile uint32_t v[n]; + for (uint32_t i = 0; i < n; i++) + v[i] = i; + struct W1 w1 = {n + v[0]}; + struct W3 w3 = {n, n * 2, n * 3 + v[n - 1]}; + return s1(n, w1) + s3(w3); +} + +/* Structs above big arrays: offsets past 1020 and past 4095. */ +__attribute__((noinline)) static uint32_t far(uint32_t k) +{ + struct W1 w1 = {k}; + struct W2 w2 = {k, k * 2}; + volatile uint8_t pad[1500]; + struct W3 w3 = {k, k + 1, k + 2}; + volatile uint8_t pad2[5000]; + struct W5 w5 = {k, 1, 2, 3, 4}; + pad[0] = (uint8_t)k; + pad2[0] = pad[0]; + return s1(k, w1) + s2(w2) + s3(w3) + s5(w5) + pad2[0]; +} + +int main(void) +{ + printf("%u %u\n", locals(3), locals(1000)); + printf("%u\n", pass_on(1, 2, 3, 4, (struct W3){5, 6, 7}, (struct W1){8})); + printf("%u %u\n", with_vla(1), with_vla(9)); + printf("%u %u\n", far(2), far(77)); + return 0; +} diff --git a/tests/ir_tests/test_struct_arg_stack.expect b/tests/ir_tests/test_struct_arg_stack.expect new file mode 100644 index 000000000..f1ca8aa04 --- /dev/null +++ b/tests/ir_tests/test_struct_arg_stack.expect @@ -0,0 +1,4 @@ +30788 2426668 +137 +52 556 +173 2573 diff --git a/tests/ir_tests/test_struct_far_frame.c b/tests/ir_tests/test_struct_far_frame.c new file mode 100644 index 000000000..f2743f462 --- /dev/null +++ b/tests/ir_tests/test_struct_far_frame.c @@ -0,0 +1,47 @@ +/* Struct values more than 32 KiB from the frame base. A struct-typed stack + * operand keeps its offset in a 16-bit field, so these are named through + * their address instead: a small struct passed in registers, one split across + * r3 and the stack, one passed wholly on the stack, a struct returned into a + * far local, and a by-value parameter that sits behind a big one. */ +#include +#include + +struct S { int a, b, c; }; +struct L { int a[8]; }; +struct Big { char pad[40000]; int tag; }; + +__attribute__((noinline)) static int f(struct S s) { return s.a + s.b * 3 + s.c * 5; } +__attribute__((noinline)) static int g(struct L l) +{ + int r = 0; + for (int i = 0; i < 8; i++) + r += l.a[i] * (i + 1); + return r; +} +__attribute__((noinline)) static int on_stack(int x, int y, int z, int w, struct S s) { return x + y + z + w + f(s); } +__attribute__((noinline)) static struct S mk(int k) { return (struct S){k, k * 2, k * 3}; } + +__attribute__((noinline)) static int far_locals(int k) +{ + volatile char pad[40000]; + struct S s = {k, k + 1, k + 2}; + struct L l; + for (int i = 0; i < 8; i++) + l.a[i] = k + i; + struct S t = mk(k + 10); + pad[0] = (char)k; + pad[39999] = pad[0]; + return f(s) + g(l) + on_stack(1, 2, 3, 4, s) + f(t) + pad[39999]; +} + +__attribute__((noinline)) static int behind_big(struct Big b, struct S s) { return b.tag + f(s) + on_stack(0, 0, 0, b.tag, s); } + +static struct Big big; + +int main(void) +{ + printf("%d %d\n", far_locals(1), far_locals(100)); + big.tag = 7; + printf("%d\n", behind_big(big, (struct S){2, 3, 4})); + return 0; +} diff --git a/tests/ir_tests/test_struct_far_frame.expect b/tests/ir_tests/test_struct_far_frame.expect new file mode 100644 index 000000000..e018aeeda --- /dev/null +++ b/tests/ir_tests/test_struct_far_frame.expect @@ -0,0 +1,2 @@ +501 8124 +76 diff --git a/tests/ir_tests/test_struct_param_pass_on.c b/tests/ir_tests/test_struct_param_pass_on.c new file mode 100644 index 000000000..520129269 --- /dev/null +++ b/tests/ir_tests/test_struct_param_pass_on.c @@ -0,0 +1,47 @@ +/* A by-value struct parameter passed on by value to another call. Large ones + * are copied into the outgoing area by memcpy, around which the caller saves + * r0-r3, IP and LR -- the source address of a parameter must account for that + * push just as a local's does. Covers a struct split across r1-r3 and the + * stack, one wholly on the stack, and small ones in registers. */ +#include + +struct Big { int v[40]; }; +struct Mid { int v[6]; }; + +__attribute__((noinline)) static int sum_big(int k, struct Big b) +{ + int r = k; + for (int i = 0; i < 40; i++) + r += b.v[i] * (i + 1); + return r; +} + +__attribute__((noinline)) static int sum_mid(struct Mid m, int k) +{ + int r = k; + for (int i = 0; i < 6; i++) + r += m.v[i] * (i + 3); + return r; +} + +/* b arrives split across r1-r3 and the stack. */ +__attribute__((noinline)) static int split_on(int k, struct Big b) { return sum_big(k + 1, b) + b.v[39]; } + +/* b arrives wholly on the stack. */ +__attribute__((noinline)) static int stack_on(int a, int c, int d, int e, struct Big b, struct Mid m) +{ + return sum_big(a + c + d + e, b) + sum_mid(m, b.v[0]); +} + +int main(void) +{ + struct Big b; + struct Mid m; + for (int i = 0; i < 40; i++) + b.v[i] = i * 7 - 3; + for (int i = 0; i < 6; i++) + m.v[i] = 100 - i; + printf("%d\n", split_on(5, b)); + printf("%d\n", stack_on(1, 2, 3, 4, b, m)); + return 0; +} diff --git a/tests/ir_tests/test_struct_param_pass_on.expect b/tests/ir_tests/test_struct_param_pass_on.expect new file mode 100644 index 000000000..576cb9173 --- /dev/null +++ b/tests/ir_tests/test_struct_param_pass_on.expect @@ -0,0 +1,2 @@ +147056 +149987 diff --git a/tests/ir_tests/test_struct_stack_ldm.c b/tests/ir_tests/test_struct_stack_ldm.c new file mode 100644 index 000000000..1d240f83b --- /dev/null +++ b/tests/ir_tests/test_struct_stack_ldm.c @@ -0,0 +1,64 @@ +/* A struct argument's stack part of three to seven words is copied by + * LDM/STM when enough registers are free: at the bottom of the outgoing area + * (stmia sp) and above it (a second base register), from locals, with other + * values live across the call setup. */ +#include + +struct W6 { int a[6]; }; +struct W7 { int a[7]; }; +struct W9 { int a[9]; }; +struct W10 { int a[10]; }; + +__attribute__((noinline)) static int h9(int k, struct W9 s) +{ + int r = k; + for (int i = 0; i < 9; i++) + r = r * 5 + s.a[i]; + return r; +} +__attribute__((noinline)) static int h10(struct W10 s, int k) +{ + int r = k; + for (int i = 0; i < 10; i++) + r = r * 7 + s.a[i]; + return r; +} +/* Two struct arguments: the second one's stack part sits above the first's. */ +__attribute__((noinline)) static int h67(int a, int b, int c, int d, struct W6 x, struct W7 y) +{ + int r = a + b * 2 + c * 3 + d * 4; + for (int i = 0; i < 6; i++) + r = r * 3 + x.a[i]; + for (int i = 0; i < 7; i++) + r = r * 11 + y.a[i]; + return r; +} + +__attribute__((noinline)) static int drive(int k) +{ + struct W9 s9; + struct W10 s10; + struct W6 s6; + struct W7 s7; + for (int i = 0; i < 10; i++) + { + if (i < 9) + s9.a[i] = k * i + 1; + s10.a[i] = k - i; + if (i < 6) + s6.a[i] = i * i + k; + if (i < 7) + s7.a[i] = k ^ i; + } + int live1 = k * 3, live2 = k * 5 + 1; + int r = h9(k, s9); + r ^= h10(s10, live1); + r += h67(live1, live2, k, k + 1, s6, s7); + return r + live1 * live2; +} + +int main(void) +{ + printf("%d %d\n", drive(1), drive(1234)); + return 0; +} diff --git a/tests/ir_tests/test_struct_stack_ldm.expect b/tests/ir_tests/test_struct_stack_ldm.expect new file mode 100644 index 000000000..3a430f5e7 --- /dev/null +++ b/tests/ir_tests/test_struct_stack_ldm.expect @@ -0,0 +1 @@ +1776078543 -889699136 diff --git a/tests/ir_tests/test_switch_case_merge.c b/tests/ir_tests/test_switch_case_merge.c new file mode 100644 index 000000000..608c93db7 --- /dev/null +++ b/tests/ir_tests/test_switch_case_merge.c @@ -0,0 +1,221 @@ +/* Switch case bodies that differ only in their own case constant share one + * copy that reads the selector instead (region_merge, case constants): tags + * written back as the value switched on, from 0 and from a non-zero first + * case; and the shapes that must stay apart -- a constant that is not the + * case value, an entry also reached by a goto, a selector changed inside the + * body, a 64-bit selector, negative case values. */ +#include +#include + +struct R { uint8_t tag; uint32_t payload; }; + +__attribute__((noinline)) static uint32_t f(uint32_t k) { return k * 7 + 1; } + +/* Cases from 0: the table indexes with the selector itself. */ +__attribute__((noinline)) static void from0(struct R *r, uint32_t x) +{ + switch (x) + { + case 0: r->payload = f(3); r->tag = 0; break; + case 1: r->payload = f(3); r->tag = 1; break; + case 2: r->payload = f(3); r->tag = 2; break; + case 3: r->payload = f(3); r->tag = 3; break; + case 4: r->payload = f(3); r->tag = 4; break; + case 5: r->payload = f(3); r->tag = 5; break; + default: r->payload = 0; r->tag = 99; break; + } +} + +/* Cases from 3: the table indexes with x - 3. */ +__attribute__((noinline)) static void from3(struct R *r, uint32_t x) +{ + switch (x) + { + case 3: r->payload = f(x); r->tag = 3; break; + case 4: r->payload = f(x); r->tag = 4; break; + case 5: r->payload = f(x); r->tag = 5; break; + case 6: r->payload = f(x); r->tag = 6; break; + case 7: r->payload = f(x); r->tag = 7; break; + case 8: r->payload = f(x); r->tag = 8; break; + default: r->payload = 1; r->tag = 98; break; + } +} + +/* The constant is k + 1, not the case value. */ +__attribute__((noinline)) static void plus1(struct R *r, uint32_t x) +{ + switch (x) + { + case 0: r->payload = f(5); r->tag = 1; break; + case 1: r->payload = f(5); r->tag = 2; break; + case 2: r->payload = f(5); r->tag = 3; break; + case 3: r->payload = f(5); r->tag = 4; break; + case 4: r->payload = f(5); r->tag = 5; break; + default: r->payload = 2; r->tag = 97; break; + } +} + +/* Case 2's body is also reached by a goto, with a selector that is not 2. */ +__attribute__((noinline)) static void jumped(struct R *r, uint32_t x, int skip) +{ + if (skip) + goto two; + switch (x) + { + case 0: r->payload = f(6); r->tag = 0; break; + case 1: r->payload = f(6); r->tag = 1; break; + case 2: + two: + r->payload = f(6); r->tag = 2; break; + case 3: r->payload = f(6); r->tag = 3; break; + case 4: r->payload = f(6); r->tag = 4; break; + default: r->payload = 3; r->tag = 96; break; + } +} + +/* The selector variable is overwritten before the constant is written. */ +__attribute__((noinline)) static void changed(struct R *r, uint32_t x) +{ + switch (x) + { + case 0: x = f(x); r->payload = x; r->tag = 0; break; + case 1: x = f(x); r->payload = x; r->tag = 1; break; + case 2: x = f(x); r->payload = x; r->tag = 2; break; + case 3: x = f(x); r->payload = x; r->tag = 3; break; + case 4: x = f(x); r->payload = x; r->tag = 4; break; + default: r->payload = 4; r->tag = 95; break; + } + r->payload += x; +} + +/* A 64-bit selector. */ +__attribute__((noinline)) static void wide(struct R *r, uint64_t x) +{ + switch (x) + { + case 0: r->payload = f(8); r->tag = 0; break; + case 1: r->payload = f(8); r->tag = 1; break; + case 2: r->payload = f(8); r->tag = 2; break; + case 3: r->payload = f(8); r->tag = 3; break; + case 4: r->payload = f(8); r->tag = 4; break; + default: r->payload = 5; r->tag = 94; break; + } +} + +/* Negative case values through a signed narrow selector. */ +__attribute__((noinline)) static int neg(int8_t x) +{ + volatile int out; + switch (x) + { + case -3: f(1); out = -3; break; + case -2: f(1); out = -2; break; + case -1: f(1); out = -1; break; + case 0: f(1); out = 0; break; + case 1: f(1); out = 1; break; + case 2: f(1); out = 2; break; + default: out = 50; break; + } + return out; +} + +/* Zig's shape: the tag stored into a result union through a pointer, the + * selector a narrow field loaded once. */ +struct U { uint8_t tag; uint32_t a, b; }; +__attribute__((noinline)) static void zigish(struct U *out, const struct U *in) +{ + switch (in->tag) + { + case 0: out->a = f(in->a); out->b = in->b; out->tag = 0; break; + case 1: out->a = f(in->a); out->b = in->b; out->tag = 1; break; + case 2: out->a = f(in->a); out->b = in->b; out->tag = 2; break; + case 3: out->a = f(in->a); out->b = in->b; out->tag = 3; break; + case 4: out->a = f(in->a); out->b = in->b; out->tag = 4; break; + case 5: out->a = f(in->a); out->b = in->b; out->tag = 5; break; + case 6: out->a = 0; out->b = 0; out->tag = 6; break; + default: out->a = 1; out->b = 1; out->tag = 7; break; + } +} + +/* The selector spilled to a frame slot by the calls around it. */ +__attribute__((noinline)) static uint32_t spilled(struct U *out, const struct U *in, uint32_t p, uint32_t q, + uint32_t r, uint32_t s, uint32_t t, uint32_t u, uint32_t v, + uint32_t w) +{ + uint8_t tag = in->tag; + uint32_t x0 = f(p), x1 = f(q), x2 = f(r), x3 = f(s), x4 = f(t), x5 = f(u), x6 = f(v), x7 = f(w); + switch (tag) + { + case 0: out->a = f(in->a); out->b = x0; out->tag = 0; break; + case 1: out->a = f(in->a); out->b = x0; out->tag = 1; break; + case 2: out->a = f(in->a); out->b = x0; out->tag = 2; break; + case 3: out->a = f(in->a); out->b = x0; out->tag = 3; break; + case 4: out->a = f(in->a); out->b = x0; out->tag = 4; break; + default: out->a = 1; out->b = 1; out->tag = 7; break; + } + return x0 + x1 + x2 + x3 + x4 + x5 + x6 + x7 + tag; +} + +int main(void) +{ + struct R r; + uint32_t sum = 0; + for (uint32_t x = 0; x < 8; x++) + { + from0(&r, x); + printf("%u:%u ", r.tag, r.payload); + sum += r.tag; + } + printf("\n"); + for (uint32_t x = 1; x < 11; x++) + { + from3(&r, x); + printf("%u:%u ", r.tag, r.payload); + } + printf("\n"); + for (uint32_t x = 0; x < 6; x++) + { + plus1(&r, x); + printf("%u ", r.tag); + } + printf("\n"); + for (uint32_t x = 0; x < 6; x++) + { + jumped(&r, x, 0); + printf("%u ", r.tag); + jumped(&r, x, 1); + printf("%u ", r.tag); + } + printf("\n"); + for (uint32_t x = 0; x < 6; x++) + { + changed(&r, x); + printf("%u:%u ", r.tag, r.payload); + } + printf("\n"); + for (uint64_t x = 0; x < 6; x++) + { + wide(&r, x + (x == 5 ? 0x100000000ull : 0)); + printf("%u ", r.tag); + } + printf("\n"); + for (int x = -4; x < 4; x++) + printf("%d ", neg((int8_t)x)); + printf("\n"); + for (uint8_t t = 0; t < 9; t++) + { + struct U in = {t, t * 10u, t * 100u}, o; + zigish(&o, &in); + printf("%u:%u:%u ", o.tag, o.a, o.b); + } + printf("\n"); + for (uint8_t t = 0; t < 7; t++) + { + struct U in = {t, t * 10u, 0}, o; + uint32_t v = spilled(&o, &in, t, 1, 2, 3, 4, 5, 6, 7); + printf("%u:%u:%u:%u ", o.tag, o.a, o.b, v); + } + printf("\n"); + printf(sum == 99 + 99 + 15 ? "PASS\n" : "FAIL\n"); + return 0; +} diff --git a/tests/ir_tests/test_switch_case_merge.expect b/tests/ir_tests/test_switch_case_merge.expect new file mode 100644 index 000000000..12ff652eb --- /dev/null +++ b/tests/ir_tests/test_switch_case_merge.expect @@ -0,0 +1,10 @@ +0:22 1:22 2:22 3:22 4:22 5:22 99:0 99:0 +98:1 98:1 3:22 4:29 5:36 6:43 7:50 8:57 98:1 98:1 +1 2 3 4 5 97 +0 2 1 2 2 2 3 2 4 2 96 2 +0:2 1:16 2:30 3:44 4:58 95:9 +0 1 2 3 4 94 +50 -3 -2 -1 0 1 2 50 +0:1:0 1:71:100 2:141:200 3:211:300 4:281:400 5:351:500 6:0:0 7:1:1 7:1:1 +0:1:1:204 1:71:8:212 2:141:15:220 3:211:22:228 4:281:29:236 7:1:1:244 7:1:1:252 +PASS diff --git a/tests/ir_tests/test_tentative_definitions.c b/tests/ir_tests/test_tentative_definitions.c new file mode 100644 index 000000000..4c2e1cd4f --- /dev/null +++ b/tests/ir_tests/test_tentative_definitions.c @@ -0,0 +1,84 @@ +/* File-scope objects declared without an initializer and defined further + * down, the way the Zig C backend declares every static: each is laid out + * once, at the storage its first declaration took (constants) or where its + * definition puts it (the rest). Nothing may fold the bytes of a declaration + * whose definition is still to come, even in a body generated where it stands + * (a variadic one). Tentative definitions never completed are zero-filled + * objects in .bss, static or not, aligned as asked. */ +#include +#include + +struct pair +{ + int a, b; +}; + +struct node +{ + const char *name; + const struct node *next; + int weight; +}; + +static const int tbl[3]; +static const struct pair pr; +static const struct node second; +static const struct node first; +static const int flag_k; +static int counter; +static int counter; +static int base_a; +static int base_b; +static int base_c; +static int never_set; +int global_never_set; +static int aligned_buf[4] __attribute__((aligned(64))); +static int *const counter_ref; + +/* Variadic: generated here, before any definition below. */ +int early_read(int n, ...) +{ + va_list ap; + va_start(ap, n); + int extra = va_arg(ap, int); + va_end(ap); + return tbl[1] * 1000 + pr.b * 100 + flag_k * 10 + n + extra; +} + +/* Stores clustered off one base address, before base_a is defined. */ +void set_bases(int x) +{ + base_a = x + 3; + base_b = x + 1; + base_c = x + 2; +} + +static const int tbl[3] = {1, 2, 3}; +static const struct pair pr = {7, 8}; +static const struct node second = {"second", 0, 20}; +static const struct node first = {"first", &second, 10}; +static const int flag_k = 5; +static int counter = 40; +static int base_a = 99; +static int *const counter_ref = &counter; + +int (*volatile early_p)(int, ...) = early_read; +void (*volatile set_p)(int) = set_bases; + +int main(void) +{ + printf("%d\n", early_p(4, 60)); + set_p(10); + printf("%d %d %d\n", base_a, base_b, base_c); + int w = 0; + for (const struct node *n = &first; n; n = n->next) + { + printf("%s ", n->name); + w += n->weight; + } + printf("%d\n", w); + ++*counter_ref; + printf("%d %d %d\n", counter, never_set, global_never_set); + printf("%d %d\n", (int)((unsigned long)aligned_buf & 63), aligned_buf[3]); + return 0; +} diff --git a/tests/ir_tests/test_tentative_definitions.expect b/tests/ir_tests/test_tentative_definitions.expect new file mode 100644 index 000000000..4ee939905 --- /dev/null +++ b/tests/ir_tests/test_tentative_definitions.expect @@ -0,0 +1,5 @@ +2914 +13 11 12 +first second 30 +41 0 0 +0 0 diff --git a/tests/ir_tests/test_yaff_section_padding.py b/tests/ir_tests/test_yaff_section_padding.py index 040d2b8f8..f5ae0472e 100644 --- a/tests/ir_tests/test_yaff_section_padding.py +++ b/tests/ir_tests/test_yaff_section_padding.py @@ -22,13 +22,14 @@ pytestmark = pytest.mark.skipif(not TCC.exists(), reason="needs armv8m-tcc") -# YaffHeader (source/obj/tccyaff.h) is packed; offsets of the fields read here. +# YaffHeader (source/obj/tccyaff.h) is packed; offsets of the fields read here, +# all u32 since YAFF_VERSION 3 widened the relocation counts and the offsets. CODE_LENGTH = 8 DATA_LENGTH = 16 -GOT_LENGTH = 48 -PLT_LENGTH = 56 -ARCH_SECTION_OFFSET = 60 -TEXT_OFFSET = 70 +GOT_LENGTH = 60 +PLT_LENGTH = 68 +ARCH_SECTION_OFFSET = 72 +TEXT_OFFSET = 92 SOURCES = { # Small .data, far below a page: the case that overflowed. @@ -74,12 +75,12 @@ def test_default_page_layout_links_and_matches_its_header(tmp_path, name): assert data[:4] == b"YAFF" # Rewritten at the very end of tcc_output_yaff; still 0 if the writer # stopped before it. - assert struct.unpack_from("= 12 @@ -674,6 +675,11 @@ def test_yaff_output_structure(tinycc_root): bss_length = unpack_from(" 0 assert data_length > 0 diff --git a/tests/pp/has-attribute.c b/tests/pp/has-attribute.c new file mode 100644 index 000000000..429f6afe7 --- /dev/null +++ b/tests/pp/has-attribute.c @@ -0,0 +1,41 @@ +/* __has_attribute answers what tcc's attribute parser acts on. */ +#if defined(__has_attribute) +defined_yes +#endif +#ifdef __has_attribute +ifdef_yes +#endif +#if __has_attribute(naked) +naked_yes +#endif +#if __has_attribute(noinline) && __has_attribute(__noinline__) +noinline_yes +#endif +#if __has_attribute(aligned) && __has_attribute(section) && __has_attribute(visibility) +aligned_section_visibility_yes +#endif +#if __has_attribute(transparent_union) +transparent_union_yes +#endif +#if __has_attribute(musttail) +musttail_wrong +#else +musttail_no +#endif +#if __has_attribute(stdcall) +stdcall_wrong +#else +stdcall_no +#endif +#define ATTR(x) __has_attribute(x) +#if ATTR(weak) && !ATTR(not_tail_called) +through_macro_yes +#endif +/* the operand is an attribute name and is not expanded even when a macro of + that name exists, as in clang (gcc expands it). Passed as a function-like + macro's argument it is expanded before substitution like any argument -- + which is why headers probe the __noinline__ spelling. */ +#define noinline __attribute__((noinline)) +#if __has_attribute(noinline) && ATTR(__noinline__) +operand_not_expanded_yes +#endif diff --git a/tests/pp/has-attribute.expect b/tests/pp/has-attribute.expect new file mode 100644 index 000000000..1cf09abff --- /dev/null +++ b/tests/pp/has-attribute.expect @@ -0,0 +1,10 @@ +defined_yes +ifdef_yes +naked_yes +noinline_yes +aligned_section_visibility_yes +transparent_union_yes +musttail_no +stdcall_no +through_macro_yes +operand_not_expanded_yes diff --git a/tests/selfhost/test_selfhost_compile.py b/tests/selfhost/test_selfhost_compile.py index fc6309be5..9a2e60afb 100644 --- a/tests/selfhost/test_selfhost_compile.py +++ b/tests/selfhost/test_selfhost_compile.py @@ -29,6 +29,12 @@ "source/obj/tccdbg.c", "source/support/tccdebug.c", "source/obj/tccelf.c", + "source/obj/tccelf_layout.c", + "source/obj/tccelf_ld.c", + "source/obj/tccelf_load.c", + "source/obj/tccelf_output.c", + "source/obj/tccelf_reloc.c", + "source/obj/tccelf_sym.c", "source/ir/tccir_operand.c", "source/obj/tccld.c", "source/machine/tccls.c", @@ -40,9 +46,17 @@ "source/ir/cfg.c", "source/ir/codegen.c", "source/ir/dump.c", + "source/ir/frame.c", + "source/ir/cross_jump.c", + "source/ir/self_store.c", + "source/ir/dead_def.c", "source/ir/machine_op.c", "source/ir/pool.c", "source/ir/regalloc.c", + "source/ir/regalloc_entry.c", + "source/ir/regalloc_phi.c", + "source/ir/regalloc_post.c", + "source/ir/regalloc_scan.c", "source/ir/ssa.c", "source/ir/stack.c", "source/ir/type.c", @@ -70,7 +84,14 @@ # source/backend/arch/arm/thumb/ "source/backend/arch/arm/thumb/arm-thumb-asm.c", "source/backend/arch/arm/thumb/arm-thumb-callsite.c", + "source/backend/arch/arm/thumb/arm-thumb-alu.c", + "source/backend/arch/arm/thumb/arm-thumb-call.c", + "source/backend/arch/arm/thumb/arm-thumb-emit.c", + "source/backend/arch/arm/thumb/arm-thumb-fp.c", + "source/backend/arch/arm/thumb/arm-thumb-frame.c", "source/backend/arch/arm/thumb/arm-thumb-gen.c", + "source/backend/arch/arm/thumb/arm-thumb-mem.c", + "source/backend/arch/arm/thumb/thumb-outline.c", "source/backend/arch/arm/thumb/thop_adr.c", "source/backend/arch/arm/thumb/thop_alu_imm.c", "source/backend/arch/arm/thumb/thop_alu_reg.c", @@ -108,6 +129,7 @@ # source/backend/generators/ "source/backend/generators/function.c", "source/backend/generators/regalloc.c", + "source/backend/generators/icf.c", # source/frontend/gen/builtin/ "source/frontend/gen/builtin/call.c", "source/frontend/gen/builtin/chk.c", @@ -227,6 +249,7 @@ "source/opt/flat/fusion/call_chain_rename.c", "source/opt/flat/fusion/cmp_imm_swap.c", "source/opt/flat/fusion/cmp_narrow_64.c", + "source/opt/flat/fusion/cmp_hi_only.c", "source/opt/flat/fusion/and64_narrow.c", "source/opt/flat/fusion/disp.c", "source/opt/flat/fusion/gens_fusion.c", @@ -271,6 +294,7 @@ "source/opt/flat/memory/bitfield_unit_narrow.c", "source/opt/flat/memory/block_copy_init.c", "source/opt/flat/memory/byte_store_merge.c", + "source/opt/flat/memory/const_local_table.c", "source/opt/flat/memory/copy_source_load_fwd.c", "source/opt/flat/memory/dead_init_via_call.c", "source/opt/flat/memory/dead_lea_store.c", @@ -280,15 +304,17 @@ "source/opt/flat/memory/deref_operand_cse.c", "source/opt/flat/memory/dse.c", "source/opt/flat/memory/entry_store_prop.c", + "source/opt/flat/memory/sret_nrvo.c", "source/opt/flat/memory/global_base_share.c", "source/opt/flat/memory/global_deref_cse.c", "source/opt/flat/memory/invariant_global_load_hoist.c", - "source/opt/flat/memory/invariant_temp_deref_hoist.c", - "source/opt/flat/memory/local_copy_prop.c", "source/opt/flat/memory/mem_init.c", "source/opt/flat/memory/mem_inline.c", "source/opt/flat/memory/memmove_to_indexed_stores.c", "source/opt/flat/memory/ptr_local_fwd.c", + "source/opt/flat/memory/param_copy_alias.c", + "source/opt/flat/memory/slot_const_store_fold.c", + "source/opt/flat/memory/sra.c", "source/opt/flat/memory/rmw_byte_clear.c", "source/opt/flat/memory/sl_forward.c", "source/opt/flat/memory/small_global_memset_to_store.c", @@ -304,26 +330,21 @@ "source/opt/flat/scalar/bitfield.c", "source/opt/flat/scalar/bool.c", "source/opt/flat/scalar/branch.c", - "source/opt/flat/scalar/branch_fold.c", "source/opt/flat/scalar/call_result.c", "source/opt/flat/scalar/cmp_expr_fold.c", "source/opt/flat/scalar/cmp_field_fuse.c", - "source/opt/flat/scalar/complex_const_param_fold.c", "source/opt/flat/scalar/const_aggregate.c", "source/opt/flat/scalar/const_call_fold.c", "source/opt/flat/scalar/const_prop_tmp.c", "source/opt/flat/scalar/const_string_calls.c", "source/opt/flat/scalar/const_var_prop.c", - "source/opt/flat/scalar/float_narrow.c", "source/opt/flat/scalar/global_init_prop.c", "source/opt/flat/scalar/known_bits.c", "source/opt/flat/scalar/narrow_store.c", - "source/opt/flat/scalar/neg_chain_cse.c", "source/opt/flat/scalar/redundant_loop_check.c", "source/opt/flat/scalar/sdiv_pow2.c", "source/opt/flat/scalar/self_arith.c", "source/opt/flat/scalar/self_copy.c", - "source/opt/flat/scalar/single_value_tmp.c", "source/opt/flat/scalar/stack_addr_fold.c", "source/opt/flat/scalar/stack_bool_diamond.c", "source/opt/flat/scalar/switch_collapse.c", @@ -336,6 +357,7 @@ "source/opt/ra/const_branch_fold.c", "source/opt/ra/incomplete_calls.c", "source/opt/ra/phi_const_chain.c", + "source/opt/ra/umaal_fusion.c", # source/opt/ssa/cfg/ "source/opt/ssa/cfg/branch.c", "source/opt/ssa/cfg/switch_fold.c", @@ -370,6 +392,7 @@ "source/opt/ssa/loop/loop_cand.c", "source/opt/ssa/loop/loop_const_sim.c", "source/opt/ssa/loop/loop_rotate.c", + "source/opt/ssa/loop/loop_bottom_test.c", "source/opt/ssa/loop/loop_unroll.c", "source/opt/ssa/loop/ptr_iv_exit_subst.c", # source/opt/ssa/memory/ diff --git a/tests/thumb/armv8m/data_processing_test.py b/tests/thumb/armv8m/data_processing_test.py index cdb8d40b1..16b05f425 100644 --- a/tests/thumb/armv8m/data_processing_test.py +++ b/tests/thumb/armv8m/data_processing_test.py @@ -303,6 +303,9 @@ def test_tst_reg(): def test_udiv(): utils.perform_test_for_file("test_udiv.S") +def test_umaal(): + utils.perform_test_for_file("test_umaal.S") + def test_umlal(): utils.perform_test_for_file("test_umlal.S") diff --git a/tests/thumb/armv8m/test_umaal.S b/tests/thumb/armv8m/test_umaal.S new file mode 100644 index 000000000..aed25aab8 --- /dev/null +++ b/tests/thumb/armv8m/test_umaal.S @@ -0,0 +1,15 @@ +.syntax unified +.thumb + +.global _start +_start: + +.global test_umaal +test_umaal: + umaal r0, r1, r2, r3 + umaal r8, r9, r10, r11 + umaal r3, r12, lr, r4 + + ite eq + umaaleq r0, r1, r2, r3 + umaalne r8, r9, r10, r11 diff --git a/tests/unit/arm/armv8m/Makefile b/tests/unit/arm/armv8m/Makefile index 2c22cef17..9b5eecad0 100644 --- a/tests/unit/arm/armv8m/Makefile +++ b/tests/unit/arm/armv8m/Makefile @@ -87,6 +87,7 @@ UT_MODULE_SRCS := \ $(TOP)/source/opt/ssa/loop/loop_unroll.c \ $(TOP)/source/opt/ssa/loop/iv_strength_reduction.c \ $(TOP)/source/opt/ssa/loop/decrement_to_zero.c \ + $(TOP)/source/opt/ssa/loop/loop_bottom_test.c \ $(TOP)/source/frontend/svalue.c \ $(TOP)/source/backend/arch/arm/arm.c \ $(TOP)/source/backend/arch/arm/thumb/thumb.c \ @@ -133,7 +134,6 @@ UT_MODULE_SRCS := \ $(TOP)/source/opt/util/vreg_query.c \ $(TOP)/source/opt/util/xform.c \ $(TOP)/source/opt/util/ir_insert.c \ - $(TOP)/source/opt/flat/scalar/neg_chain_cse.c \ $(TOP)/source/opt/flat/scalar/known_bits.c \ $(TOP)/source/opt/flat/scalar/const_prop_tmp.c \ $(TOP)/source/opt/flat/scalar/value_tracking.c \ @@ -142,6 +142,7 @@ UT_MODULE_SRCS := \ $(TOP)/source/opt/flat/memory/dse.c \ $(TOP)/source/opt/flat/memory/var_store_elim.c \ $(TOP)/source/opt/flat/memory/entry_store_prop.c \ + $(TOP)/source/opt/flat/memory/sret_nrvo.c \ $(TOP)/source/opt/flat/memory/sl_forward.c \ $(TOP)/source/opt/flat/memory/dead_local_slot.c \ $(TOP)/source/opt/flat/memory/dead_temp_local.c \ @@ -150,13 +151,10 @@ UT_MODULE_SRCS := \ $(TOP)/source/opt/flat/memory/byte_store_merge.c \ $(TOP)/source/opt/flat/memory/mem_inline.c \ $(TOP)/source/opt/ssa/memory/diamond_store_fwd.c \ - $(TOP)/source/opt/flat/scalar/branch_fold.c \ $(TOP)/source/opt/flat/scalar/stack_bool_diamond.c \ $(TOP)/source/opt/flat/scalar/global_init_prop.c \ $(TOP)/source/opt/flat/scalar/symref_const_prop.c \ - $(TOP)/source/opt/flat/scalar/complex_const_param_fold.c \ $(TOP)/source/opt/flat/scalar/stack_addr_fold.c \ - $(TOP)/source/opt/flat/scalar/single_value_tmp.c \ $(TOP)/source/opt/flat/scalar/cmp_field_fuse.c \ $(TOP)/source/opt/analysis/du_chains.c \ $(TOP)/source/opt/flat/scalar/const_string_calls.c \ @@ -211,7 +209,6 @@ UT_MODULE_SRCS := \ $(TOP)/source/opt/flat/loop/reroll.c \ $(TOP)/source/opt/flat/scalar/bitfield.c \ $(TOP)/source/opt/flat/scalar/cmp_expr_fold.c \ - $(TOP)/source/opt/flat/scalar/float_narrow.c \ $(TOP)/source/opt/flat/scalar/const_aggregate.c \ $(TOP)/source/opt/flat/ipa/pure_via_sret.c \ $(TOP)/source/opt/flat/ipa/func_write_summary.c \ @@ -221,17 +218,19 @@ UT_MODULE_SRCS := \ $(TOP)/source/opt/flat/memory/dead_init_via_call.c \ $(TOP)/source/opt/flat/memory/stack_addr_cse.c \ $(TOP)/source/opt/flat/memory/block_copy_init.c \ + $(TOP)/source/opt/flat/memory/const_local_table.c \ $(TOP)/source/opt/flat/memory/small_memset_to_store.c \ $(TOP)/source/opt/flat/memory/small_global_memset_to_store.c \ $(TOP)/source/opt/flat/memory/mem_init.c \ $(TOP)/source/opt/flat/memory/memmove_to_indexed_stores.c \ $(TOP)/source/opt/flat/memory/addrof_var_fwd.c \ $(TOP)/source/opt/flat/memory/ptr_local_fwd.c \ + $(TOP)/source/opt/flat/memory/param_copy_alias.c \ + $(TOP)/source/opt/flat/memory/slot_const_store_fold.c \ + $(TOP)/source/opt/flat/memory/sra.c \ $(TOP)/source/opt/flat/memory/invariant_global_load_hoist.c \ - $(TOP)/source/opt/flat/memory/invariant_temp_deref_hoist.c \ $(TOP)/source/opt/flat/memory/deref_operand_cse.c \ $(TOP)/source/opt/flat/memory/rmw_byte_clear.c \ - $(TOP)/source/opt/flat/memory/local_copy_prop.c \ $(TOP)/source/opt/flat/memory/struct_copy_roundtrip_elim.c \ $(TOP)/source/opt/flat/dce/dead_vla.c \ $(TOP)/source/opt/engine/ctx.c \ @@ -271,9 +270,18 @@ UT_MODULE_SRCS := \ $(TOP)/source/opt/engine/pipeline_table.c \ $(TOP)/source/opt/engine/gen_adapters.c \ $(TOP)/source/ir/regalloc.c \ + $(TOP)/source/ir/regalloc_scan.c \ + $(TOP)/source/ir/regalloc_phi.c \ + $(TOP)/source/ir/regalloc_entry.c \ + $(TOP)/source/ir/regalloc_post.c \ + $(TOP)/source/ir/frame.c \ + $(TOP)/source/ir/cross_jump.c \ + $(TOP)/source/ir/self_store.c \ + $(TOP)/source/ir/dead_def.c \ $(TOP)/source/opt/ra/const_branch_fold.c \ $(TOP)/source/opt/ra/incomplete_calls.c \ $(TOP)/source/opt/ra/phi_const_chain.c \ + $(TOP)/source/opt/ra/umaal_fusion.c \ $(TOP)/source/ir/codegen.c \ $(TOP)/source/ir/machine_op.c \ $(TOP)/source/machine/tccls.c \ @@ -319,9 +327,14 @@ UT_MODULE_OBJS := $(filter-out $(BUILD_DIR)/source/backend/arch/arm/thumb/arm-th # ra_link_stubs.c, tcc_ir_opt_switch_collapse_ex in stubs.c) were deleted. # Trailing entries are cross-build TUs (CORE_FILES / SSA_OPT_SRC / FPU-variant # backend) that no unit suite links today — registered at 0% until one does. +# The Thumb-2 code generator: arm-thumb-gen.c plus the files split out of it. +THUMB_GEN_SRCS := $(TOP)/source/backend/arch/arm/thumb/arm-thumb-gen.c \ + $(foreach p,emit alu mem fp frame call,$(TOP)/source/backend/arch/arm/thumb/arm-thumb-$(p).c) + UT_COVERAGE_ONLY_SRCS := \ - $(TOP)/source/backend/arch/arm/thumb/arm-thumb-gen.c \ + $(THUMB_GEN_SRCS) \ $(TOP)/source/backend/arch/arm/thumb/arm-thumb-callsite.c \ + $(TOP)/source/backend/arch/arm/thumb/thumb-outline.c \ $(TOP)/source/opt/ssa/engine/use_def.c \ $(TOP)/source/opt/ssa/engine/rewrite.c \ $(TOP)/source/opt/ssa/engine/stack_resolve.c \ @@ -447,7 +460,7 @@ UT_LOCAL_SRCS := \ test_arm_aapcs.c \ test_arm_link.c \ test_ld_script.c \ - test_opt_neg_chain.c \ + test_opt_self_store.c \ test_opt_knownbits.c \ test_opt_symaddr_cse.c \ test_opt_cmp_fuse.c \ @@ -487,13 +500,13 @@ UT_LOCAL_SRCS := \ test_opt_promote_extra.c \ test_opt_fusion.c \ test_opt_pipeline_orchestration.c \ - test_opt_float_branch.c \ test_opt_redundant_assign.c \ test_opt_uninit.c \ test_opt_dead_vla.c \ test_opt_xform.c \ test_opt_switch_collapse.c \ test_opt_switch_to_data.c \ + test_opt_region_merge_case.c \ test_metamorphic.c \ test_metamorphic_ssa.c \ test_ssa_opt_arm.c \ @@ -718,8 +731,9 @@ $(COV_STAMP2): # undefined reference here. UT2_MODULE_SRCS := \ $(UT_MODULE_SRCS) \ - $(TOP)/source/backend/arch/arm/thumb/arm-thumb-gen.c \ + $(THUMB_GEN_SRCS) \ $(TOP)/source/backend/arch/arm/thumb/arm-thumb-callsite.c \ + $(TOP)/source/backend/arch/arm/thumb/thumb-outline.c \ $(TOP)/source/backend/arch/fpu/arm/fpv5-sp-d16.c \ $(TOP)/source/backend/arch/fpu/arm/fpv5-d16.c \ $(TOP)/source/backend/arch/fpu/arm/rp2350-dcp.c @@ -1038,8 +1052,12 @@ $(COV_STAMP6): @rm -f $(BUILD_DIR6)/.coverage-* @touch $@ +# The ELF writer/linker: tccelf.c plus the files split out of it. +TCCELF_SRCS := $(TOP)/source/obj/tccelf.c \ + $(foreach p,sym reloc layout output load ld,$(TOP)/source/obj/tccelf_$(p).c) + UT6_MODULE_SRCS := \ - $(TOP)/source/obj/tccelf.c + $(TCCELF_SRCS) UT6_MODULE_OBJS := $(patsubst $(TOP)/%.c,$(BUILD_DIR6)/%.o,$(UT6_MODULE_SRCS)) UT6_LOCAL_SRCS := \ @@ -1271,7 +1289,7 @@ $(COV_STAMP9): UT9_MODULE_SRCS := \ $(TOP)/source/obj/tccyaff.c \ - $(TOP)/source/obj/tccelf.c \ + $(TCCELF_SRCS) \ $(TOP)/source/memory/unique_ptr.c UT9_MODULE_OBJS := $(patsubst $(TOP)/%.c,$(BUILD_DIR9)/%.o,$(UT9_MODULE_SRCS)) diff --git a/tests/unit/arm/armv8m/codegen_mop_stubs.c b/tests/unit/arm/armv8m/codegen_mop_stubs.c index 097686243..198989429 100644 --- a/tests/unit/arm/armv8m/codegen_mop_stubs.c +++ b/tests/unit/arm/armv8m/codegen_mop_stubs.c @@ -301,6 +301,18 @@ void tcc_gen_machine_smull_mop(MachineOperand src1, MachineOperand src2, Machine cgstub_record("smull_mop", (TccIrOp)-1, dest, src1, src2); } +/* 0: the unit build never fuses UMAAL, so its IR expectations stay put. */ +int tcc_machine_has_umaal(void) +{ + return 0; +} + +void tcc_gen_machine_umaal_mop(MachineOperand src1, MachineOperand src2, MachineOperand accum, MachineOperand dest) +{ + (void)accum; + cgstub_record("umaal_mop", (TccIrOp)-1, dest, src1, src2); +} + int tcc_gen_machine_mlal_accum_mop(MachineOperand src1, MachineOperand src2, MachineOperand accum, MachineOperand dest, int is_signed) { @@ -437,6 +449,37 @@ void tcc_gen_machine_spill_block_copy(int32_t src_spill_off, int32_t dst_spill_o cgstub_push("spill_block_copy", (TccIrOp)-1, MACH_OP_NONE, -1, MACH_OP_NONE, -1, MACH_OP_NONE, -1, nwords, 0); } +/* Conservative: every call reloads R9, as before the reload analysis. */ +int tcc_gen_machine_calls_reload_r9(struct TCCIRState *ir) +{ + (void)ir; + return 1; +} + +/* No free-register plan in the stub: callers fall back to their other paths. */ +int tcc_gen_machine_spill_block_copy_free(int32_t src_spill_off, int32_t dst_spill_off, int nwords, uint32_t also_free) +{ + (void)src_spill_off; + (void)dst_spill_off; + (void)nwords; + (void)also_free; + return 0; +} + +/* Likewise: the stub never fuses, so the individual accesses are emitted. */ +int tcc_gen_machine_reg_block_copy(int src_base, int32_t src_off, int dst_base, int32_t dst_off, const int *regs, + int nwords, int end_idx) +{ + (void)end_idx; + (void)src_base; + (void)src_off; + (void)dst_base; + (void)dst_off; + (void)regs; + (void)nwords; + return 0; +} + /* ============================================================================ * Control flow / jumps / switch * ============================================================================ */ @@ -477,6 +520,12 @@ int tcc_gen_machine_pool_entries_total(void) return 0; } +void tcc_gen_machine_align_branch_target(int align) +{ + (void)align; + /* No instruction stream here, so nothing to pad. */ +} + int tcc_gen_machine_cbz_forward_ok(int32_t target_ir, int current_ir_idx) { (void)target_ir; @@ -629,6 +678,44 @@ void tcc_gen_machine_prolog(int leaffunc, uint64_t used_registers, int stack_siz cgstub_push("prolog", (TccIrOp)-1, MACH_OP_NONE, -1, MACH_OP_NONE, -1, MACH_OP_NONE, -1, leaffunc, stack_size); } +/* -Os outliner (thumb-outline.c): never on here. */ +int tcc_gen_machine_prolog_saved_lr(void) +{ + return 0; +} +int tcc_gen_machine_outline_enabled(void) +{ + return 0; +} +void tcc_gen_machine_outline_capture_begin(void) +{ +} +void tcc_gen_machine_outline_capture_end(void) +{ +} +int tcc_gen_machine_outline_analyze(struct TCCIRState *ir, const uint32_t *map, int end) +{ + (void)ir; + (void)map; + (void)end; + return 0; +} +void tcc_gen_machine_outline_op_end(int i) +{ + (void)i; +} +void tcc_gen_machine_outline_op_start(int i) +{ + (void)i; +} +int tcc_gen_machine_outline_suppressing(void) +{ + return 0; +} +void tcc_gen_machine_outline_function_end(void) +{ +} + void tcc_gen_machine_epilog(int leaffunc) { cgstub_push("epilog", (TccIrOp)-1, MACH_OP_NONE, -1, MACH_OP_NONE, -1, MACH_OP_NONE, -1, leaffunc, 0); @@ -772,6 +859,21 @@ void tcc_gen_machine_strldr_cache_set_enabled(int enabled) cgstub_record("strldr_cache_set_enabled", (TccIrOp)-1, CGSTUB_NO_OP, CGSTUB_NO_OP, CGSTUB_NO_OP); } +void tcc_gen_machine_str_elide_set_enabled(int enabled) +{ + (void)enabled; +} + +void tcc_gen_machine_slot_self_copy_set_enabled(int enabled) +{ + (void)enabled; +} + +void tcc_gen_machine_frame_word_pair_set_enabled(int enabled) +{ + (void)enabled; +} + void tcc_gen_machine_imm_cache_reset(void) { cgstub_record("imm_cache_reset", (TccIrOp)-1, CGSTUB_NO_OP, CGSTUB_NO_OP, CGSTUB_NO_OP); @@ -842,6 +944,11 @@ void tcc_gen_machine_builtin_apply_args_mop(MachineOperand dest) cgstub_record("builtin_apply_args_mop", (TccIrOp)-1, dest, CGSTUB_NO_OP, CGSTUB_NO_OP); } +void tcc_gen_machine_return_address_mop(MachineOperand dest) +{ + cgstub_record("return_address_mop", (TccIrOp)-1, dest, CGSTUB_NO_OP, CGSTUB_NO_OP); +} + void tcc_gen_machine_builtin_apply_mop(MachineOperand fn, MachineOperand args, MachineOperand dest) { cgstub_record("builtin_apply_mop", (TccIrOp)-1, dest, fn, args); diff --git a/tests/unit/arm/armv8m/libtcc_api_stubs.c b/tests/unit/arm/armv8m/libtcc_api_stubs.c index 2ac13b379..a8cab81e5 100644 --- a/tests/unit/arm/armv8m/libtcc_api_stubs.c +++ b/tests/unit/arm/armv8m/libtcc_api_stubs.c @@ -68,6 +68,7 @@ int tcc_load_yaff(TCCState *s1, int fd, const char *filename, int level) return -1; } void tcc_yaff_libs_free(TCCState *s1) { (void)s1; } +void tcc_yaff_import_set_free(TCCState *s1) { (void)s1; } int tcc_assemble(TCCState *s1, int do_preprocess) { (void)s1; (void)do_preprocess; return -1; } void ld_script_cleanup(LDScript *ld) { (void)ld; } void arm_deinit(struct TCCState *s) { (void)s; } diff --git a/tests/unit/arm/armv8m/stubs.c b/tests/unit/arm/armv8m/stubs.c index f0c92c2eb..c10df4cff 100644 --- a/tests/unit/arm/armv8m/stubs.c +++ b/tests/unit/arm/armv8m/stubs.c @@ -333,7 +333,7 @@ int put_elf_sym(struct Section *s, addr_t value, unsigned long size, int info, i } /* get_tok_str is declared `const char *get_tok_str(int, CValue*)` in tcc.h and - * used by name-gated optimizer passes (e.g. self_copy_elim, float_narrowing). + * used by name-gated optimizer passes (e.g. self_copy_elim). * The unit-test harness lets individual tests populate a token→name table so * those passes can reach their positive folds. CValue is opaque here (no tcc.h), * hence the void* parameter — the linker resolves by name regardless. */ diff --git a/tests/unit/arm/armv8m/stubs_gen_machine_fallback.c b/tests/unit/arm/armv8m/stubs_gen_machine_fallback.c index 3736f3aef..5faa461ef 100644 --- a/tests/unit/arm/armv8m/stubs_gen_machine_fallback.c +++ b/tests/unit/arm/armv8m/stubs_gen_machine_fallback.c @@ -28,3 +28,9 @@ const char *tcc_get_abi_softcall_name(struct SValue *src1, struct SValue *src2, (void)src1; (void)src2; (void)dest; (void)op; return NULL; } + +/* From arm-thumb-gen.c — DWARF frame base of a frame-pointer function. */ +int tcc_gen_machine_frame_top_offset(void) +{ + return 0; +} diff --git a/tests/unit/arm/armv8m/tcc_state_stub.c b/tests/unit/arm/armv8m/tcc_state_stub.c index dae149fb0..b2a03c49f 100644 --- a/tests/unit/arm/armv8m/tcc_state_stub.c +++ b/tests/unit/arm/armv8m/tcc_state_stub.c @@ -33,6 +33,50 @@ static TCCState ut_tcc_state_storage; TCCState *tcc_state = &ut_tcc_state_storage; +/* Symbols of TUs these binaries do not link, reached from ones they do: + * tccyaff.c's import set reads archives through tccelf.c, sret_nrvo.c + * reads tccgen's current-function type, and the thumb backend names + * helper symbols through strlit_pool.c. Weak, so a binary that links the + * real TU (build_tccelf's tccelf.c) keeps it. */ +__attribute__((weak)) CType func_vt; +__attribute__((weak)) int func_vc; +__attribute__((weak)) ssize_t full_read(int fd, void *buf, size_t count) +{ + (void)fd; (void)buf; (void)count; + return -1; +} +__attribute__((weak)) int tcc_archive_index_load(TCCState *s1, const char *path) +{ + (void)s1; (void)path; + return -1; +} +__attribute__((weak)) int tcc_archive_index_has(TCCState *s1, const char *path, const char *name) +{ + (void)s1; (void)path; (void)name; + return 0; +} +__attribute__((weak)) int tccelf_arm_fp_lib_is_shared(TCCState *s1) +{ + (void)s1; + return 0; +} +__attribute__((weak)) const char *tccelf_get_fp_lib_name(TCCState *s1) +{ + (void)s1; + return NULL; +} +__attribute__((weak)) Sym *external_helper_sym(int v) +{ + (void)v; + return NULL; +} +/* codegen.c's word-aligned body start (cg_word_align_start). */ +__attribute__((weak)) int func_align_pad; +__attribute__((weak)) void gen_fill_nops(int bytes) +{ + (void)bytes; +} + #else /* UT_SSA_OPT_REAL — build_ssaopt (UT11) ===================== */ /* * tcc_state_stub.c - stubs for TCC state initialization diff --git a/tests/unit/arm/armv8m/tccpp_stubs.c b/tests/unit/arm/armv8m/tccpp_stubs.c index 1a49c4a04..183cb63f4 100644 --- a/tests/unit/arm/armv8m/tccpp_stubs.c +++ b/tests/unit/arm/armv8m/tccpp_stubs.c @@ -301,3 +301,11 @@ int normalized_PATHCMP(const char *a, const char *b) { return strcmp(a, b); } + +/* __has_attribute asks the attribute parser (frontend/gen/decl/attribute.c), + which is not linked here; no test depends on its answer. */ +int tcc_attribute_supported(int t) +{ + (void)t; + return 0; +} diff --git a/tests/unit/arm/armv8m/test_arm_aapcs.c b/tests/unit/arm/armv8m/test_arm_aapcs.c index 2b8801038..e3806b391 100644 --- a/tests/unit/arm/armv8m/test_arm_aapcs.c +++ b/tests/unit/arm/armv8m/test_arm_aapcs.c @@ -426,35 +426,37 @@ UT_TEST(test_classify_large_struct_no_arg_flags_stays_by_value) return 0; } -UT_TEST(test_classify_large_struct_with_arg_flags_uses_invisible_ref_in_reg) +/* AAPCS32 passes a composite of any size by value, split between the core + * registers and the stack (C.5); a pointer to a copy is the AArch64 rule. The + * callee (arg_flags allocated) classifies exactly as the caller does. */ +UT_TEST(test_classify_large_struct_with_arg_flags_splits_by_value) { TCCAbiCallLayout layout; layout_init(&layout); - tcc_abi_call_layout_ensure_capacity(&layout, 1); + tcc_abi_call_layout_ensure_capacity(&layout, 2); UT_ASSERT(layout.arg_flags != NULL); TCCAbiArgDesc d = desc_struct(24, 8); TCCAbiArgLoc loc = tcc_abi_classify_argument(&layout, 0, &d); - UT_ASSERT_EQ(loc.kind, TCC_ABI_LOC_REG); + UT_ASSERT_EQ(loc.kind, TCC_ABI_LOC_REG_STACK); UT_ASSERT_EQ(loc.reg_base, 0); - UT_ASSERT_EQ(loc.reg_count, 1); - UT_ASSERT_EQ(loc.size, 4); - UT_ASSERT_EQ(layout.next_reg, 1); - UT_ASSERT_EQ(layout.arg_flags[0] & TCC_ABI_ARG_FLAG_INVISIBLE_REF, - TCC_ABI_ARG_FLAG_INVISIBLE_REF); + UT_ASSERT_EQ(loc.reg_count, 4); + UT_ASSERT_EQ(loc.stack_off, 0); + UT_ASSERT_EQ(loc.stack_size, 8); + UT_ASSERT_EQ(layout.next_reg, 4); + UT_ASSERT_EQ(layout.next_stack_off, 8); + UT_ASSERT_EQ(layout.arg_flags[0], 0) /* passed by value: no lowering flag */; - /* The 8-byte natural alignment must NOT force even-register rounding - * for an invisible reference (it's just a 4-byte pointer) -- confirmed - * by reg_base==0 above (no skip) and by a following scalar32 landing at - * r1, not r2. */ + /* The next argument goes on the stack after the struct's tail. */ TCCAbiArgDesc d32 = desc_scalar32(); TCCAbiArgLoc loc2 = tcc_abi_classify_argument(&layout, 1, &d32); - UT_ASSERT_EQ(loc2.reg_base, 1); + UT_ASSERT_EQ(loc2.kind, TCC_ABI_LOC_STACK); + UT_ASSERT_EQ(loc2.stack_off, 8); return 0; } -UT_TEST(test_classify_large_struct_with_arg_flags_invisible_ref_spills_to_stack) +UT_TEST(test_classify_large_struct_after_four_regs_goes_on_stack_by_value) { TCCAbiCallLayout layout; layout_init(&layout); @@ -469,10 +471,51 @@ UT_TEST(test_classify_large_struct_with_arg_flags_invisible_ref_spills_to_stack) TCCAbiArgLoc loc = tcc_abi_classify_argument(&layout, 4, &dbig); UT_ASSERT_EQ(loc.kind, TCC_ABI_LOC_STACK); UT_ASSERT_EQ(loc.stack_off, 0); - UT_ASSERT_EQ(loc.size, 4); + UT_ASSERT_EQ(loc.size, 32); + UT_ASSERT_EQ(layout.next_stack_off, 32); + UT_ASSERT_EQ(layout.arg_flags[4], 0) /* passed by value: no lowering flag */; + return 0; +} + +UT_TEST(test_classify_8_aligned_struct_after_three_regs_skips_r3) +{ + /* NCRN 3 is odd: an 8-aligned composite starts at an even register, so r3 + * is skipped and, with no pair left, the whole struct goes on the stack. */ + TCCAbiCallLayout layout; + layout_init(&layout); + tcc_abi_call_layout_ensure_capacity(&layout, 5); + TCCAbiArgDesc d32 = desc_scalar32(); + TCCAbiArgDesc d = desc_struct(24, 8); + for (int i = 0; i < 3; i++) + tcc_abi_classify_argument(&layout, i, &d32); + + TCCAbiArgLoc loc = tcc_abi_classify_argument(&layout, 3, &d); + UT_ASSERT_EQ(loc.kind, TCC_ABI_LOC_STACK); + UT_ASSERT_EQ(loc.stack_off, 0); + UT_ASSERT_EQ(layout.next_reg, 4); + UT_ASSERT_EQ(layout.next_stack_off, 24); + + TCCAbiArgLoc loc2 = tcc_abi_classify_argument(&layout, 4, &d32); + UT_ASSERT_EQ(loc2.kind, TCC_ABI_LOC_STACK); + UT_ASSERT_EQ(loc2.stack_off, 24); + return 0; +} + +UT_TEST(test_classify_overaligned_struct_stack_slot_aligned_to_8) +{ + /* C.4 rounds the NSAA up to the composite's alignment, capped at 8. */ + TCCAbiCallLayout layout; + layout_init(&layout); + tcc_abi_call_layout_ensure_capacity(&layout, 6); + TCCAbiArgDesc d32 = desc_scalar32(); + TCCAbiArgDesc d = desc_struct(32, 16); + for (int i = 0; i < 5; i++) + tcc_abi_classify_argument(&layout, i, &d32); UT_ASSERT_EQ(layout.next_stack_off, 4); - UT_ASSERT_EQ(layout.arg_flags[4] & TCC_ABI_ARG_FLAG_INVISIBLE_REF, - TCC_ABI_ARG_FLAG_INVISIBLE_REF); + + TCCAbiArgLoc loc = tcc_abi_classify_argument(&layout, 5, &d); + UT_ASSERT_EQ(loc.kind, TCC_ABI_LOC_STACK); + UT_ASSERT_EQ(loc.stack_off, 8); return 0; } @@ -488,7 +531,7 @@ UT_TEST(test_classify_struct_exactly_16_bytes_not_invisible_ref) TCCAbiArgLoc loc = tcc_abi_classify_argument(&layout, 0, &d); UT_ASSERT_EQ(loc.kind, TCC_ABI_LOC_REG); UT_ASSERT_EQ(loc.reg_count, 4); - UT_ASSERT_EQ(layout.arg_flags[0] & TCC_ABI_ARG_FLAG_INVISIBLE_REF, 0); + UT_ASSERT_EQ(layout.arg_flags[0], 0) /* passed by value: no lowering flag */; return 0; } @@ -649,7 +692,7 @@ UT_TEST(test_ensure_capacity_grow_preserves_existing_data_and_zeros_tail) layout.locs[1].kind = TCC_ABI_LOC_STACK; layout.locs[1].stack_off = 42; - layout.arg_flags[1] = TCC_ABI_ARG_FLAG_INVISIBLE_REF; + layout.arg_flags[1] = 0x5a; layout.args_original[1].size = 99; layout.args_effective[1].size = 77; @@ -659,7 +702,7 @@ UT_TEST(test_ensure_capacity_grow_preserves_existing_data_and_zeros_tail) /* Old data at index 1 survived the realloc. */ UT_ASSERT_EQ(layout.locs[1].kind, TCC_ABI_LOC_STACK); UT_ASSERT_EQ(layout.locs[1].stack_off, 42); - UT_ASSERT_EQ(layout.arg_flags[1], TCC_ABI_ARG_FLAG_INVISIBLE_REF); + UT_ASSERT_EQ(layout.arg_flags[1], 0x5a); UT_ASSERT_EQ(layout.args_original[1].size, 99); UT_ASSERT_EQ(layout.args_effective[1].size, 77); diff --git a/tests/unit/arm/armv8m/test_arm_thumb_asm.c b/tests/unit/arm/armv8m/test_arm_thumb_asm.c index 06b4f37f7..81d0703c3 100644 --- a/tests/unit/arm/armv8m/test_arm_thumb_asm.c +++ b/tests/unit/arm/armv8m/test_arm_thumb_asm.c @@ -248,8 +248,9 @@ static void make_asm_operand(ASMOperand *op, SValue *sv, const char *constraint, sv->r = r_location; } -/* Variant for the specific-register path: VT_LOCAL + a Sym whose r field - * encodes the requested physical register. */ +/* Variant for the specific-register path: a `register T x __asm("rN")` + * operand, which parse_asm_operands records in op->regvar (VT_LOCAL + a Sym + * whose r field names the register is the parse-time shape it comes from). */ static void make_asm_operand_with_local_sym(ASMOperand *op, SValue *sv, Sym *sym, const char *constraint, int forced_reg) { @@ -262,6 +263,7 @@ static void make_asm_operand_with_local_sym(ASMOperand *op, SValue *sv, Sym *sym sv->r = VT_LOCAL; sv->sym = sym; sym->r = (unsigned short)forced_reg; + op->regvar = 1 + forced_reg; } /* Minimal in-memory section setup for g()/gen_le*() tests. */ diff --git a/tests/unit/arm/armv8m/test_gen_dispatch_smoke.c b/tests/unit/arm/armv8m/test_gen_dispatch_smoke.c index 9e2e42b3b..3539764d0 100644 --- a/tests/unit/arm/armv8m/test_gen_dispatch_smoke.c +++ b/tests/unit/arm/armv8m/test_gen_dispatch_smoke.c @@ -688,18 +688,23 @@ UT_TEST(test_cbnz_jump_mop_emits_cbnz) /* ------------------------------------------------------------------ chain helpers */ +extern int allocated_stack_size; + UT_TEST(test_restore_chain_loads_from_chain_slot) { setup_gen(); tcc_state->need_frame_pointer = 1; + allocated_stack_size = 16; tcc_gen_machine_restore_chain(); - /* The static chain register is R10 (a high register), so the LDR from - * [FP, #-4] uses the 32-bit T3 encoding (4 bytes). */ + /* The chain slot is frame offset -4, and the frame pointer is the frame's + * bottom: [FP, #16 - 4]. R10 is a high register, so the LDR uses the + * 32-bit encoding (4 bytes). */ UT_ASSERT_EQ(ind, 4); - UT_ASSERT(bytes_match_opcode(ind, th_ldr_imm(R10, R_FP, 4, 4 /* subtract */, ENFORCE_ENCODING_NONE))); + UT_ASSERT(bytes_match_opcode(ind, th_ldr_imm(R10, R_FP, 12, 6 /* add */, ENFORCE_ENCODING_NONE))); + allocated_stack_size = 0; return 0; } @@ -844,7 +849,9 @@ UT_TEST(test_pending_pool_size_empty_returns_zero) UT_TEST(test_set_chain_emits_mov_r10_fp) { setup_gen(); + allocated_stack_size = 0; + /* An empty frame's top is the frame pointer itself. */ tcc_gen_machine_set_chain(); UT_ASSERT_EQ(ind, 2); UT_ASSERT(bytes_match_opcode(ind, th_mov_reg(R10, R_FP, FLAGS_BEHAVIOUR_NOT_IMPORTANT, THUMB_SHIFT_DEFAULT, @@ -853,6 +860,21 @@ UT_TEST(test_set_chain_emits_mov_r10_fp) return 0; } +UT_TEST(test_set_chain_passes_frame_top) +{ + setup_gen(); + allocated_stack_size = 24; + + /* The chain is the frame's top, FP + frame size: nested functions address + * captured variables at their (negative) frame offsets from it. */ + tcc_gen_machine_set_chain(); + UT_ASSERT_EQ(ind, 4); + UT_ASSERT(bytes_match_opcode(ind, th_add_imm(R10, R_FP, 24, FLAGS_BEHAVIOUR_NOT_IMPORTANT, ENFORCE_ENCODING_NONE))); + + allocated_stack_size = 0; + return 0; +} + /* ------------------------------------------------------------------ store-to-stack helpers */ UT_TEST(test_store_to_stack_fp_small_offset_emits_str) diff --git a/tests/unit/arm/armv8m/test_gen_mem.c b/tests/unit/arm/armv8m/test_gen_mem.c index 4c36ce2c1..04887d464 100644 --- a/tests/unit/arm/armv8m/test_gen_mem.c +++ b/tests/unit/arm/armv8m/test_gen_mem.c @@ -471,3 +471,40 @@ UT_TEST(test_store_mop_spill_halfword_emits_strh) return 0; } + +/* A store is as wide as its destination: a 64-bit integer stored into a word + * is a truncation, and only the low half may go anywhere. The pair path + * wrote the high half as well -- for a plain register destination into its + * absent second register, r1 == -1, which encodes as pc (`mov pc, r7`; the + * tcc-built Zig compiler's InternPool.Key.eql truncating a u64 length). */ +static MachineOperand mop_reg_pair64(int lo, int hi) +{ + MachineOperand m = mop_reg(lo, IROP_BTYPE_INT64); + m.u.reg.r1 = hi; + m.is_64bit = 1; + return m; +} + +UT_TEST(test_store_mop_i64_into_word_reg_moves_low_half_only) +{ + setup_gen(); + + tcc_gen_machine_store_mop(mop_reg(R0, IROP_BTYPE_INT32), mop_reg_pair64(R6, R7), TCCIR_OP_STORE); + + UT_ASSERT(bytes_match_opcode(ind, th_mov_reg(R0, R6, FLAGS_BEHAVIOUR_NOT_IMPORTANT, THUMB_SHIFT_DEFAULT, + ENFORCE_ENCODING_NONE, false))); + + return 0; +} + +UT_TEST(test_store_mop_i64_into_word_slot_stores_one_word) +{ + setup_gen(); + tcc_state->need_frame_pointer = 1; + + tcc_gen_machine_store_mop(mop_spill(-12, IROP_BTYPE_INT32), mop_reg_pair64(R2, R3), TCCIR_OP_STORE); + + UT_ASSERT(bytes_match_opcode(ind, th_str_imm(R2, R_FP, 12, 4 /* subtract */, ENFORCE_ENCODING_NONE))); + + return 0; +} diff --git a/tests/unit/arm/armv8m/test_metamorphic.c b/tests/unit/arm/armv8m/test_metamorphic.c index cab1645c0..ccce4d69f 100644 --- a/tests/unit/arm/armv8m/test_metamorphic.c +++ b/tests/unit/arm/armv8m/test_metamorphic.c @@ -37,13 +37,11 @@ #include /* ── legacy pass entry points (forward-declared; defined in ir/opt_*.c) ── */ -int tcc_ir_opt_neg_chain_cse(TCCIRState *ir); int tcc_ir_opt_known_bits(TCCIRState *ir); int tcc_ir_opt_const_prop_tmp(TCCIRState *ir); int tcc_ir_opt_const_var_prop(TCCIRState *ir); int tcc_ir_opt_add_reassoc(TCCIRState *ir); int tcc_ir_opt_self_arith_fold(TCCIRState *ir); -int tcc_ir_opt_single_value_tmp(TCCIRState *ir); int tcc_ir_opt_cmp_field_fuse(TCCIRState *ir); int tcc_ir_opt_jump_threading(TCCIRState *ir); int tcc_ir_opt_dead_lea_store_elim(TCCIRState *ir); @@ -60,13 +58,11 @@ typedef struct PassEntry /* The passes exercised by the metamorphic loop. All are value/structure * transforms that must preserve straight-line semantics. */ static const PassEntry g_passes[] = { - {"neg_chain_cse", tcc_ir_opt_neg_chain_cse}, {"known_bits", tcc_ir_opt_known_bits}, {"const_prop_tmp", tcc_ir_opt_const_prop_tmp}, {"const_var_prop", tcc_ir_opt_const_var_prop}, {"add_reassoc", tcc_ir_opt_add_reassoc}, {"self_arith_fold", tcc_ir_opt_self_arith_fold}, - {"single_value_tmp", tcc_ir_opt_single_value_tmp}, {"cmp_field_fuse", tcc_ir_opt_cmp_field_fuse}, {"jump_threading", tcc_ir_opt_jump_threading}, {"dead_lea_store_elim", tcc_ir_opt_dead_lea_store_elim}, @@ -720,7 +716,6 @@ UT_TEST(test_shr_sar_known_bits_correct_cases) * ============================================================================ */ -UT_COVERS("neg_chain_cse"); UT_COVERS("known_bits"); UT_COVERS("copy_prop"); UT_COVERS("const_prop"); @@ -728,4 +723,3 @@ UT_COVERS("const_prop_tmp"); UT_COVERS("const_var_prop"); UT_COVERS("add_reassoc"); UT_COVERS("self_arith_fold"); -UT_COVERS("single_value_tmp"); diff --git a/tests/unit/arm/armv8m/test_opt_constfold.c b/tests/unit/arm/armv8m/test_opt_constfold.c index d2c2a53c6..36b298eae 100644 --- a/tests/unit/arm/armv8m/test_opt_constfold.c +++ b/tests/unit/arm/armv8m/test_opt_constfold.c @@ -1,24 +1,17 @@ /* * test_opt_constfold.c - suite for ir/opt_constfold.c * - * Covers two name-gated call-folding passes from ir/opt_constfold.c: + * Covers the name-gated call-folding pass from ir/opt_constfold.c: * * - tcc_ir_opt_self_copy_elim : NOPs a memcpy/memmove (or AAPCS aligned * variant) call whose dst and src arguments are the same pure expression * (a self-copy). FUNCCALLVAL is rewritten to `ASSIGN dst`, FUNCCALLVOID * to NOP, and the param marshalling is NOP'd. - * - tcc_ir_opt_float_narrowing : collects __aeabi_f2d / __aeabi_d2f - * conversion calls and narrows floor()/ceil()/fabs()/... double helpers to - * their float variant when an argument is an f2d result. * * HARNESS NOTES: - * Both passes are name-gated via get_tok_str(callee->v). The unit-test harness - * now provides a settable token->name table (utb_set_tok_str), so self_copy_elim - * can be driven to its positive fold. float_narrowing still cannot complete a - * true positive fold in isolation because change_callee_sym() calls - * external_global_sym(), which is a stubs.c link stub that returns NULL; the - * production pass does not check the return value of change_callee_sym(), so the - * transform is applied partially and is recorded as a suspected bug. + * The pass is name-gated via get_tok_str(callee->v). The unit-test harness + * provides a settable token->name table (utb_set_tok_str), so self_copy_elim + * can be driven to its positive fold. * * These are isolated tests: a hand-built IR sequence is run through the bare * pass entry point and the resulting instructions are inspected directly. @@ -31,7 +24,6 @@ /* Pass entry points (declared in ir/opt.h; forward-declared to avoid pulling in * the optimizer engine headers). */ int tcc_ir_opt_self_copy_elim(TCCIRState *ir); -int tcc_ir_opt_float_narrowing(TCCIRState *ir); int tcc_ir_opt_const_string_calls(TCCIRState *ir); int tcc_ir_opt_const_call_replace(TCCIRState *ir); int tcc_ir_opt_switch_call_replace(TCCIRState *ir); @@ -40,8 +32,7 @@ int tcc_ir_opt_local_addrof_const_fold(TCCIRState *ir); /* Frontend link stubs (sym_push2 / external_global_sym / tok_alloc_const / * global_stack / elfsym) now live in stubs.c so the combined unit-test link - * has a single definition. external_global_sym() returning NULL prevents - * float_narrowing from completing its callee swap in isolation. */ + * has a single definition. */ #define I32 IROP_BTYPE_INT32 #define I64 IROP_BTYPE_INT64 @@ -439,295 +430,6 @@ UT_TEST(test_self_copy_elim_idempotent) return 0; } -/* ---------------------------------------------------- float_narrowing tests */ - -/* GUARD (would-narrow-if-names-matched): a textbook f2d -> floor -> d2f chain - * (the exact Case-1 shape tcc_ir_opt_float_narrowing rewrites). Phase 1 scans - * for __aeabi_f2d / __aeabi_d2f by name; under the "?" stub it finds none, so - * num_f2d == 0 and the pass returns 0 at the early-out, leaving every call and - * param intact. If the f2d/d2f name gate were dropped, the floor call would be - * narrowed and the f2d/d2f calls NOP'd, failing these assertions. - * - * FUNCPARAMVAL Tf(float), param0 (call_id 1) - * Td = FUNCCALLVAL (would be __aeabi_f2d: float->double) - * FUNCPARAMVAL Td(double), param0 (call_id 2) - * Tr = FUNCCALLVAL (would be floor: double->double) - * FUNCPARAMVAL Tr(double), param0 (call_id 3) - * Tf2 = FUNCCALLVAL (would be __aeabi_d2f: double->float) */ -UT_TEST(test_float_narrowing_unmatched_names_no_fold) -{ - TCCIRState *ir = utb_new(); - utb_pools_init(ir); - - static Sym f2d_sym, floor_sym, d2f_sym; - IROperand f2d_callee = utb_callee(ir, &f2d_sym); - IROperand floor_callee = utb_callee(ir, &floor_sym); - IROperand d2f_callee = utb_callee(ir, &d2f_sym); - - /* f2d: float Tf(0) -> double Td(1) */ - utb_emit(ir, TCCIR_OP_FUNCPARAMVAL, UTB_NONE, utb_temp(0, F32), - utb_imm((int32_t)TCCIR_ENCODE_PARAM(1, 0), I32)); - int i_f2d = utb_emit(ir, TCCIR_OP_FUNCCALLVAL, utb_temp(1, F64), f2d_callee, - utb_imm((int32_t)TCCIR_ENCODE_CALL(1, 1), I32)); - /* floor: double Td(1) -> double Tr(2) */ - utb_emit(ir, TCCIR_OP_FUNCPARAMVAL, UTB_NONE, utb_temp(1, F64), - utb_imm((int32_t)TCCIR_ENCODE_PARAM(2, 0), I32)); - int i_floor = utb_emit(ir, TCCIR_OP_FUNCCALLVAL, utb_temp(2, F64), floor_callee, - utb_imm((int32_t)TCCIR_ENCODE_CALL(2, 1), I32)); - /* d2f: double Tr(2) -> float Tf2(3) */ - utb_emit(ir, TCCIR_OP_FUNCPARAMVAL, UTB_NONE, utb_temp(2, F64), - utb_imm((int32_t)TCCIR_ENCODE_PARAM(3, 0), I32)); - int i_d2f = utb_emit(ir, TCCIR_OP_FUNCCALLVAL, utb_temp(3, F32), d2f_callee, - utb_imm((int32_t)TCCIR_ENCODE_CALL(3, 1), I32)); - - int changes = tcc_ir_opt_float_narrowing(ir); - - UT_ASSERT_EQ(changes, 0); - UT_ASSERT_EQ(utb_op(ir, i_f2d), TCCIR_OP_FUNCCALLVAL); - UT_ASSERT_EQ(utb_op(ir, i_floor), TCCIR_OP_FUNCCALLVAL); - UT_ASSERT_EQ(utb_op(ir, i_d2f), TCCIR_OP_FUNCCALLVAL); - - utb_free(ir); - return 0; -} - -/* GUARD: too few instructions. tcc_ir_opt_float_narrowing requires at least 4 - * instructions (n < 4 -> return 0) before doing any scanning. A 2-instruction - * f2d-shaped pair must short-circuit to 0 with the IR untouched. */ -UT_TEST(test_float_narrowing_too_few_instructions_no_fold) -{ - TCCIRState *ir = utb_new(); - utb_pools_init(ir); - - static Sym f2d_sym; - IROperand f2d_callee = utb_callee(ir, &f2d_sym); - - utb_emit(ir, TCCIR_OP_FUNCPARAMVAL, UTB_NONE, utb_temp(0, F32), - utb_imm((int32_t)TCCIR_ENCODE_PARAM(1, 0), I32)); - int i_call = utb_emit(ir, TCCIR_OP_FUNCCALLVAL, utb_temp(1, F64), f2d_callee, - utb_imm((int32_t)TCCIR_ENCODE_CALL(1, 1), I32)); - - int changes = tcc_ir_opt_float_narrowing(ir); - - UT_ASSERT_EQ(changes, 0); - UT_ASSERT_EQ(utb_op(ir, i_call), TCCIR_OP_FUNCCALLVAL); - - utb_free(ir); - return 0; -} - -/* NOTE: a real f2d -> floor -> d2f narrowing positive cannot be tested in this - * isolated harness. Once Phase 2 matches the narrowable middle function it - * calls change_callee_sym(), which calls sym_push2() / external_global_sym(); - * both are stubs in tests/unit/arm/armv8m/stubs.c that return NULL, and - * change_callee_sym() dereferences the NULL sym_push2() result before it can - * report failure. In a real compilation the helper exists and the transform - * completes. This limitation is recorded in the agent conclusion. */ - -/* Negative: f2d and d2f names match but the middle function is not in the - * narrowable table, so Phase 2 never triggers. */ -UT_TEST(test_float_narrowing_non_narrowable_middle_no_fold) -{ - TCCIRState *ir = utb_new(); - utb_pools_init(ir); - - static Sym f2d_sym, middle_sym, d2f_sym; - IROperand f2d_callee = utb_callee_named(ir, &f2d_sym, 20); - IROperand middle_callee = utb_callee_named(ir, &middle_sym, 21); - IROperand d2f_callee = utb_callee_named(ir, &d2f_sym, 22); - - utb_set_tok_str(20, "__aeabi_f2d"); - utb_set_tok_str(21, "some_non_narrowable_func"); - utb_set_tok_str(22, "__aeabi_d2f"); - - /* f2d: T0 -> T1, call_id 1 */ - utb_emit(ir, TCCIR_OP_FUNCPARAMVAL, UTB_NONE, utb_temp(0, F32), - utb_imm((int32_t)TCCIR_ENCODE_PARAM(1, 0), I32)); - int i_f2d = utb_emit(ir, TCCIR_OP_FUNCCALLVAL, utb_temp(1, F64), f2d_callee, - utb_imm((int32_t)TCCIR_ENCODE_CALL(1, 1), I32)); - /* middle: T1 -> T2, call_id 2 */ - utb_emit(ir, TCCIR_OP_FUNCPARAMVAL, UTB_NONE, utb_temp(1, F64), - utb_imm((int32_t)TCCIR_ENCODE_PARAM(2, 0), I32)); - int i_middle = utb_emit(ir, TCCIR_OP_FUNCCALLVAL, utb_temp(2, F64), middle_callee, - utb_imm((int32_t)TCCIR_ENCODE_CALL(2, 1), I32)); - /* d2f: T2 -> T3, call_id 3 */ - utb_emit(ir, TCCIR_OP_FUNCPARAMVAL, UTB_NONE, utb_temp(2, F64), - utb_imm((int32_t)TCCIR_ENCODE_PARAM(3, 0), I32)); - int i_d2f = utb_emit(ir, TCCIR_OP_FUNCCALLVAL, utb_temp(3, F32), d2f_callee, - utb_imm((int32_t)TCCIR_ENCODE_CALL(3, 1), I32)); - - int changes = tcc_ir_opt_float_narrowing(ir); - - UT_ASSERT_EQ(changes, 0); - UT_ASSERT_EQ(utb_op(ir, i_f2d), TCCIR_OP_FUNCCALLVAL); - UT_ASSERT_EQ(utb_op(ir, i_middle), TCCIR_OP_FUNCCALLVAL); - UT_ASSERT_EQ(utb_op(ir, i_d2f), TCCIR_OP_FUNCCALLVAL); - UT_ASSERT_EQ(utb_assert_wellformed(ir, 16), 0); - - utb_set_tok_str(20, NULL); - utb_set_tok_str(21, NULL); - utb_set_tok_str(22, NULL); - utb_free(ir); - return 0; -} - -/* Negative: f2d -> func shape with no trailing d2f. The middle function name - * is not set, so the pass declines even though an f2d call is present. */ -UT_TEST(test_float_narrowing_missing_d2f_no_fold) -{ - TCCIRState *ir = utb_new(); - utb_pools_init(ir); - - static Sym f2d_sym, floor_sym; - IROperand f2d_callee = utb_callee_named(ir, &f2d_sym, 30); - IROperand floor_callee = utb_callee_named(ir, &floor_sym, 31); - - utb_set_tok_str(30, "__aeabi_f2d"); - /* leave floor name as "?" so it does not match the narrowable table */ - - /* f2d: T0 -> T1, call_id 1 */ - utb_emit(ir, TCCIR_OP_FUNCPARAMVAL, UTB_NONE, utb_temp(0, F32), - utb_imm((int32_t)TCCIR_ENCODE_PARAM(1, 0), I32)); - int i_f2d = utb_emit(ir, TCCIR_OP_FUNCCALLVAL, utb_temp(1, F64), f2d_callee, - utb_imm((int32_t)TCCIR_ENCODE_CALL(1, 1), I32)); - /* floor: T1 -> T2, call_id 2 */ - utb_emit(ir, TCCIR_OP_FUNCPARAMVAL, UTB_NONE, utb_temp(1, F64), - utb_imm((int32_t)TCCIR_ENCODE_PARAM(2, 0), I32)); - int i_floor = utb_emit(ir, TCCIR_OP_FUNCCALLVAL, utb_temp(2, F64), floor_callee, - utb_imm((int32_t)TCCIR_ENCODE_CALL(2, 1), I32)); - - int changes = tcc_ir_opt_float_narrowing(ir); - - UT_ASSERT_EQ(changes, 0); - UT_ASSERT_EQ(utb_op(ir, i_f2d), TCCIR_OP_FUNCCALLVAL); - UT_ASSERT_EQ(utb_op(ir, i_floor), TCCIR_OP_FUNCCALLVAL); - UT_ASSERT_EQ(utb_assert_wellformed(ir, 16), 0); - - utb_set_tok_str(30, NULL); - utb_free(ir); - return 0; -} - -/* Negative: the narrowable function's param0 is not the f2d result, so no - * narrowing candidate is found. */ -UT_TEST(test_float_narrowing_f2d_not_consumed_no_fold) -{ - TCCIRState *ir = utb_new(); - utb_pools_init(ir); - - static Sym f2d_sym, floor_sym, d2f_sym; - IROperand f2d_callee = utb_callee_named(ir, &f2d_sym, 40); - IROperand floor_callee = utb_callee_named(ir, &floor_sym, 41); - IROperand d2f_callee = utb_callee_named(ir, &d2f_sym, 42); - - utb_set_tok_str(40, "__aeabi_f2d"); - utb_set_tok_str(41, "floor"); - utb_set_tok_str(42, "__aeabi_d2f"); - - /* f2d: T0 -> T1, call_id 1 (result T1 is unused by floor) */ - utb_emit(ir, TCCIR_OP_FUNCPARAMVAL, UTB_NONE, utb_temp(0, F32), - utb_imm((int32_t)TCCIR_ENCODE_PARAM(1, 0), I32)); - int i_f2d = utb_emit(ir, TCCIR_OP_FUNCCALLVAL, utb_temp(1, F64), f2d_callee, - utb_imm((int32_t)TCCIR_ENCODE_CALL(1, 1), I32)); - /* floor: T4 -> T2, call_id 2 (param0 is T4, not T1) */ - utb_emit(ir, TCCIR_OP_FUNCPARAMVAL, UTB_NONE, utb_temp(4, F64), - utb_imm((int32_t)TCCIR_ENCODE_PARAM(2, 0), I32)); - int i_floor = utb_emit(ir, TCCIR_OP_FUNCCALLVAL, utb_temp(2, F64), floor_callee, - utb_imm((int32_t)TCCIR_ENCODE_CALL(2, 1), I32)); - /* d2f: T2 -> T3, call_id 3 */ - utb_emit(ir, TCCIR_OP_FUNCPARAMVAL, UTB_NONE, utb_temp(2, F64), - utb_imm((int32_t)TCCIR_ENCODE_PARAM(3, 0), I32)); - int i_d2f = utb_emit(ir, TCCIR_OP_FUNCCALLVAL, utb_temp(3, F32), d2f_callee, - utb_imm((int32_t)TCCIR_ENCODE_CALL(3, 1), I32)); - - int changes = tcc_ir_opt_float_narrowing(ir); - - UT_ASSERT_EQ(changes, 0); - UT_ASSERT_EQ(utb_op(ir, i_f2d), TCCIR_OP_FUNCCALLVAL); - UT_ASSERT_EQ(utb_op(ir, i_floor), TCCIR_OP_FUNCCALLVAL); - UT_ASSERT_EQ(utb_op(ir, i_d2f), TCCIR_OP_FUNCCALLVAL); - UT_ASSERT_EQ(utb_assert_wellformed(ir, 16), 0); - - utb_set_tok_str(40, NULL); - utb_set_tok_str(41, NULL); - utb_set_tok_str(42, NULL); - utb_free(ir); - return 0; -} - -/* Negative: no f2d call at all -> num_f2d == 0 and the pass early-outs. */ -UT_TEST(test_float_narrowing_no_f2d_no_fold) -{ - TCCIRState *ir = utb_new(); - utb_pools_init(ir); - - static Sym floor_sym, d2f_sym; - IROperand floor_callee = utb_callee_named(ir, &floor_sym, 50); - IROperand d2f_callee = utb_callee_named(ir, &d2f_sym, 51); - - utb_set_tok_str(50, "floor"); - utb_set_tok_str(51, "__aeabi_d2f"); - - /* floor: T0 -> T1, call_id 1 */ - utb_emit(ir, TCCIR_OP_FUNCPARAMVAL, UTB_NONE, utb_temp(0, F64), - utb_imm((int32_t)TCCIR_ENCODE_PARAM(1, 0), I32)); - int i_floor = utb_emit(ir, TCCIR_OP_FUNCCALLVAL, utb_temp(1, F64), floor_callee, - utb_imm((int32_t)TCCIR_ENCODE_CALL(1, 1), I32)); - /* d2f: T1 -> T2, call_id 2 */ - utb_emit(ir, TCCIR_OP_FUNCPARAMVAL, UTB_NONE, utb_temp(1, F64), - utb_imm((int32_t)TCCIR_ENCODE_PARAM(2, 0), I32)); - int i_d2f = utb_emit(ir, TCCIR_OP_FUNCCALLVAL, utb_temp(2, F32), d2f_callee, - utb_imm((int32_t)TCCIR_ENCODE_CALL(2, 1), I32)); - - int changes = tcc_ir_opt_float_narrowing(ir); - - UT_ASSERT_EQ(changes, 0); - UT_ASSERT_EQ(utb_op(ir, i_floor), TCCIR_OP_FUNCCALLVAL); - UT_ASSERT_EQ(utb_op(ir, i_d2f), TCCIR_OP_FUNCCALLVAL); - UT_ASSERT_EQ(utb_assert_wellformed(ir, 16), 0); - - utb_set_tok_str(50, NULL); - utb_set_tok_str(51, NULL); - utb_free(ir); - return 0; -} - -/* Idempotence: the pass converges on a chain that does not match. */ -UT_TEST(test_float_narrowing_idempotent) -{ - TCCIRState *ir = utb_new(); - utb_pools_init(ir); - - static Sym f2d_sym, floor_sym, d2f_sym; - IROperand f2d_callee = utb_callee(ir, &f2d_sym); - IROperand floor_callee = utb_callee(ir, &floor_sym); - IROperand d2f_callee = utb_callee(ir, &d2f_sym); - - /* f2d: T0 -> T1, call_id 1 */ - utb_emit(ir, TCCIR_OP_FUNCPARAMVAL, UTB_NONE, utb_temp(0, F32), - utb_imm((int32_t)TCCIR_ENCODE_PARAM(1, 0), I32)); - utb_emit(ir, TCCIR_OP_FUNCCALLVAL, utb_temp(1, F64), f2d_callee, - utb_imm((int32_t)TCCIR_ENCODE_CALL(1, 1), I32)); - /* floor: T1 -> T2, call_id 2 */ - utb_emit(ir, TCCIR_OP_FUNCPARAMVAL, UTB_NONE, utb_temp(1, F64), - utb_imm((int32_t)TCCIR_ENCODE_PARAM(2, 0), I32)); - utb_emit(ir, TCCIR_OP_FUNCCALLVAL, utb_temp(2, F64), floor_callee, - utb_imm((int32_t)TCCIR_ENCODE_CALL(2, 1), I32)); - /* d2f: T2 -> T3, call_id 3 */ - utb_emit(ir, TCCIR_OP_FUNCPARAMVAL, UTB_NONE, utb_temp(2, F64), - utb_imm((int32_t)TCCIR_ENCODE_PARAM(3, 0), I32)); - utb_emit(ir, TCCIR_OP_FUNCCALLVAL, utb_temp(3, F32), d2f_callee, - utb_imm((int32_t)TCCIR_ENCODE_CALL(3, 1), I32)); - - int total = utb_run_to_fixpoint(ir, tcc_ir_opt_float_narrowing, 10); - - UT_ASSERT_EQ(total, 0); - UT_ASSERT_EQ(utb_assert_wellformed(ir, 16), 0); - - utb_free(ir); - return 0; -} - /* ================================================================ * const_string_calls / const_call_replace / switch_call_replace / * param_addrof_const_fold / local_addrof_const_fold @@ -2098,7 +1800,6 @@ UT_TEST(test_local_addrof_64bit_store_value_no_fold) } UT_COVERS("self_copy_elim"); -UT_COVERS("float_narrowing"); UT_COVERS("const_string_calls"); UT_COVERS("const_call_replace"); UT_COVERS("switch_call_replace"); diff --git a/tests/unit/arm/armv8m/test_opt_constprop.c b/tests/unit/arm/armv8m/test_opt_constprop.c index 253a71077..d38bc45e0 100644 --- a/tests/unit/arm/armv8m/test_opt_constprop.c +++ b/tests/unit/arm/armv8m/test_opt_constprop.c @@ -52,7 +52,6 @@ int tcc_ir_opt_const_var_prop(TCCIRState *ir); int tcc_ir_opt_const_prop_tmp(TCCIRState *ir); int tcc_ir_opt_global_init_prop(TCCIRState *ir); int tcc_ir_opt_symref_const_prop(TCCIRState *ir); -int tcc_ir_opt_complex_const_param_fold(TCCIRState *ir); int tcc_ir_opt_value_tracking(TCCIRState *ir); int tcc_ir_opt_self_arith_fold(TCCIRState *ir); @@ -368,8 +367,6 @@ UT_TEST(test_constvarprop_stackoff_source_not_propagated) return 0; } - - /* GUARD: a 64-bit immediate assigned into a 32-bit temp must be tracked as the * truncated 32-bit value. This mirrors `(int)(long long)(V2SI){2,2}` after * known_bits folds the 64-bit stack load to `0x0000000200000002`: the following @@ -479,8 +476,6 @@ UT_TEST(test_constproptmp_switch_table_const_index_to_default_jump) return 0; } - - /* ============================================================ global_init_prop * * tcc_ir_opt_global_init_prop reads the initialized value of a const/static @@ -884,148 +879,6 @@ UT_TEST(test_symrefconstprop_idempotent) return 0; } -/* ===================================================== complex_const_param_fold - * - * Folds the {real,imag} pair of a _Complex float local — stored to a stack slot - * as two 4-byte float constants — directly into the FUNCPARAMVAL that passes it - * by value, packing the two component bit patterns into a 64-bit complex-float - * immediate and NOP-ing the two component stores. */ - -/* Build a complex-float lval stack operand at `off` (vreg==-1, F32, is_complex, - * is_lval, not param) — the FUNCPARAM source shape the fold targets. */ -static IROperand utb_cplx_slot(int32_t off) -{ - IROperand op = irop_make_stackoff(0, off, /*is_lval*/ 1, /*is_llocal*/ 0, /*is_param*/ 0, F32); - op.is_complex = 1; - return op; -} - -/* Build a plain float lval stack operand at `off` for a component STORE dest - * (vreg==-1, F32, is_lval, NOT complex, not param). */ -static IROperand utb_f32_slot(int32_t off) -{ - return irop_make_stackoff(0, off, /*is_lval*/ 1, /*is_llocal*/ 0, /*is_param*/ 0, F32); -} - -/* POSITIVE: the canonical 3-op pattern folds. - * STORE slot[-8] <- #real_bits - * STORE slot[-4] <- #imag_bits - * FUNCPARAMVAL slot[-8] (complex lval) - * -> param src1 becomes a packed complex-float i64 immediate; stores NOP'd. */ -UT_TEST(test_cplxparamfold_packs_and_nops_stores) -{ - TCCIRState *ir = utb_new(); - utb_pools_init(ir); - - uint32_t real_bits = 0x3f800000u; /* 1.0f */ - uint32_t imag_bits = 0x40000000u; /* 2.0f */ - - int isr = utb_emit(ir, TCCIR_OP_STORE, utb_f32_slot(-8), utb_imm((int32_t)real_bits, F32), UTB_NONE); - int isi = utb_emit(ir, TCCIR_OP_STORE, utb_f32_slot(-4), utb_imm((int32_t)imag_bits, F32), UTB_NONE); - int ip = utb_emit(ir, TCCIR_OP_FUNCPARAMVAL, UTB_NONE, utb_cplx_slot(-8), - utb_imm((int32_t)TCCIR_ENCODE_PARAM(1, 0), I32)); - - int changes = tcc_ir_opt_complex_const_param_fold(ir); - UT_ASSERT(changes > 0); - - /* Both component stores are dead. */ - UT_ASSERT_EQ(utb_op(ir, isr), TCCIR_OP_NOP); - UT_ASSERT_EQ(utb_op(ir, isi), TCCIR_OP_NOP); - - /* The param source is now a complex-float immediate carrying packed bits: - * real in the low 32 bits, imag in the high 32 bits. */ - IROperand p = utb_src1(ir, ip); - UT_ASSERT_EQ((int)p.is_complex, 1); - UT_ASSERT_EQ((int)p.is_lval, 0); - UT_ASSERT_EQ(irop_is_immediate(p), 1); - uint64_t packed = (uint64_t)real_bits | ((uint64_t)imag_bits << 32); - UT_ASSERT_EQ((long long)irop_get_imm64_ex(ir, p), (long long)(int64_t)packed); - utb_free(ir); - return 0; -} - -/* GUARD (non-complex param): a FUNCPARAMVAL whose source slot is NOT is_complex - * is not a _Complex-by-value pass, so the fold never triggers. */ -UT_TEST(test_cplxparamfold_non_complex_param_no_fold) -{ - TCCIRState *ir = utb_new(); - utb_pools_init(ir); - int isr = utb_emit(ir, TCCIR_OP_STORE, utb_f32_slot(-8), utb_imm(0x3f800000, F32), UTB_NONE); - int isi = utb_emit(ir, TCCIR_OP_STORE, utb_f32_slot(-4), utb_imm(0x40000000, F32), UTB_NONE); - /* Plain (non-complex) float lval source. */ - int ip = utb_emit(ir, TCCIR_OP_FUNCPARAMVAL, UTB_NONE, utb_f32_slot(-8), - utb_imm((int32_t)TCCIR_ENCODE_PARAM(1, 0), I32)); - - int changes = tcc_ir_opt_complex_const_param_fold(ir); - UT_ASSERT_EQ(changes, 0); - UT_ASSERT_EQ(utb_op(ir, isr), TCCIR_OP_STORE); - UT_ASSERT_EQ(utb_op(ir, isi), TCCIR_OP_STORE); - UT_ASSERT_EQ(utb_op(ir, ip), TCCIR_OP_FUNCPARAMVAL); - utb_free(ir); - return 0; -} - -/* GUARD (missing imag store): only the real component is stored, so the 8-byte - * slot is not fully initialized by constants — fold bails (imag_store_idx<0). */ -UT_TEST(test_cplxparamfold_missing_component_no_fold) -{ - TCCIRState *ir = utb_new(); - utb_pools_init(ir); - int isr = utb_emit(ir, TCCIR_OP_STORE, utb_f32_slot(-8), utb_imm(0x3f800000, F32), UTB_NONE); - int ip = utb_emit(ir, TCCIR_OP_FUNCPARAMVAL, UTB_NONE, utb_cplx_slot(-8), - utb_imm((int32_t)TCCIR_ENCODE_PARAM(1, 0), I32)); - - int changes = tcc_ir_opt_complex_const_param_fold(ir); - UT_ASSERT_EQ(changes, 0); - UT_ASSERT_EQ(utb_op(ir, isr), TCCIR_OP_STORE); - UT_ASSERT_EQ(utb_op(ir, ip), TCCIR_OP_FUNCPARAMVAL); - utb_free(ir); - return 0; -} - -/* GUARD (extra read of the slot): a third instruction that also references the - * 8-byte slot (an additional LOAD of slot[-8]) disqualifies the fold — the slot - * is not touched by exactly the three expected ops. */ -UT_TEST(test_cplxparamfold_extra_slot_reference_no_fold) -{ - TCCIRState *ir = utb_new(); - utb_pools_init(ir); - int isr = utb_emit(ir, TCCIR_OP_STORE, utb_f32_slot(-8), utb_imm(0x3f800000, F32), UTB_NONE); - int isi = utb_emit(ir, TCCIR_OP_STORE, utb_f32_slot(-4), utb_imm(0x40000000, F32), UTB_NONE); - /* An extra read of the real slot through src1 of a LOAD. */ - utb_emit(ir, TCCIR_OP_LOAD, utb_temp(0, F32), utb_f32_slot(-8), UTB_NONE); - int ip = utb_emit(ir, TCCIR_OP_FUNCPARAMVAL, UTB_NONE, utb_cplx_slot(-8), - utb_imm((int32_t)TCCIR_ENCODE_PARAM(1, 0), I32)); - - int changes = tcc_ir_opt_complex_const_param_fold(ir); - UT_ASSERT_EQ(changes, 0); - UT_ASSERT_EQ(utb_op(ir, isr), TCCIR_OP_STORE); - UT_ASSERT_EQ(utb_op(ir, isi), TCCIR_OP_STORE); - UT_ASSERT_EQ(utb_op(ir, ip), TCCIR_OP_FUNCPARAMVAL); - utb_free(ir); - return 0; -} - -/* GUARD (non-constant store value): when a component store writes a non-constant - * value (a vreg, not an immediate/float-bits) the slot can't be packed -> bail. */ -UT_TEST(test_cplxparamfold_nonconst_store_no_fold) -{ - TCCIRState *ir = utb_new(); - utb_pools_init(ir); - int isr = utb_emit(ir, TCCIR_OP_STORE, utb_f32_slot(-8), utb_temp(5, F32), UTB_NONE); - int isi = utb_emit(ir, TCCIR_OP_STORE, utb_f32_slot(-4), utb_imm(0x40000000, F32), UTB_NONE); - int ip = utb_emit(ir, TCCIR_OP_FUNCPARAMVAL, UTB_NONE, utb_cplx_slot(-8), - utb_imm((int32_t)TCCIR_ENCODE_PARAM(1, 0), I32)); - - int changes = tcc_ir_opt_complex_const_param_fold(ir); - UT_ASSERT_EQ(changes, 0); - UT_ASSERT_EQ(utb_op(ir, isr), TCCIR_OP_STORE); - UT_ASSERT_EQ(utb_op(ir, isi), TCCIR_OP_STORE); - UT_ASSERT_EQ(utb_op(ir, ip), TCCIR_OP_FUNCPARAMVAL); - utb_free(ir); - return 0; -} - /* ============================================================== value_tracking * * Forward dataflow tracker: propagates compile-time constants through VAR @@ -1748,5 +1601,4 @@ UT_COVERS("const_var_prop"); UT_COVERS("const_prop_tmp"); UT_COVERS("global_init_prop"); UT_COVERS("symref_const_prop"); -UT_COVERS("complex_const_param_fold"); UT_COVERS("value_tracking"); diff --git a/tests/unit/arm/armv8m/test_opt_float_branch.c b/tests/unit/arm/armv8m/test_opt_float_branch.c deleted file mode 100644 index a7904f0a1..000000000 --- a/tests/unit/arm/armv8m/test_opt_float_branch.c +++ /dev/null @@ -1,367 +0,0 @@ -/* - * test_opt_float_branch.c - suite for ir/opt_branch.c float_branch_fold - * - * tcc_ir_opt_float_branch_fold() folds redundant branches after soft-float - * flag-comparison helper calls (__aeabi_cfcmple / __aeabi_cdcmple) and after - * TEST_ZERO patterns. If a first comparison/test proves the result of a - * second identical comparison, the second compare + JUMPIF are either turned - * into an unconditional JUMP or NOP-ed away. - * - * These are isolated tests: a hand-built IR sequence is run through the bare - * pass entry point and the resulting instructions are inspected directly. - */ - -#include "ir_build.h" - -#include "ut.h" - -/* Pass entry point (declared in ir/opt.h; forward-declared to avoid pulling in - * the optimizer engine headers). */ -int tcc_ir_opt_float_branch_fold(TCCIRState *ir); - -#define I32 IROP_BTYPE_INT32 - -/* Bound large enough for encoded vreg values (type<<28 | position). */ -#define UTB_VREG_BOUND 0x30000010 - -/* Condition tokens used in JUMPIF src1 (see evaluate_compare_condition). */ -#define TOK_EQ 0x94 -#define TOK_NE 0x95 - -/* Token used to name the flag-comparison helper via the harness get_tok_str - * table. The pass recognizes __aeabi_cfcmple / __aeabi_cdcmple. */ -#define TOK_FCMP 101 - -/* Distinct call ids so ir_opt_get_call_param_operand does not confuse the - * parameters of the first and second helper calls. */ -#define CALL_ID_1 1 -#define CALL_ID_2 2 - -/* ----------------------------------------------------------------- helpers */ - -/* Read a jump's current target index. */ -static int jump_target(TCCIRState *ir, int i) -{ - return (int)utb_dest(ir, i).u.imm32; -} - -/* Build a SYMREF callee operand whose token is `tok`. Caller must have called - * utb_pools_init(ir) first. */ -static IROperand utb_callee_named(TCCIRState *ir, Sym *sym, int tok) -{ - sym->v = tok; - uint32_t sidx = tcc_ir_pool_add_symref(ir, sym, 0, 0); - return irop_make_symref(0, sidx, 0, 0, 0, I32); -} - -/* Emit a JUMPIF (conditional branch) with condition token `tok`. */ -static int emit_jumpif(TCCIRState *ir, int tgt, int tok) -{ - return utb_emit(ir, TCCIR_OP_JUMPIF, utb_imm(tgt, I32), utb_imm(tok, I32), UTB_NONE); -} - -/* Emit a single-precision flag-comparison helper call: - * __aeabi_cfcmple(a, b) - * The call consumes two FUNCPARAMVAL instructions immediately preceding it. */ -static int emit_fcmp_call(TCCIRState *ir, int call_id, IROperand callee, - IROperand a, IROperand b) -{ - utb_emit(ir, TCCIR_OP_FUNCPARAMVAL, UTB_NONE, a, - utb_imm((int32_t)TCCIR_ENCODE_PARAM(call_id, 0), I32)); - utb_emit(ir, TCCIR_OP_FUNCPARAMVAL, UTB_NONE, b, - utb_imm((int32_t)TCCIR_ENCODE_PARAM(call_id, 1), I32)); - return utb_emit(ir, TCCIR_OP_FUNCCALLVOID, UTB_NONE, callee, - utb_imm((int32_t)TCCIR_ENCODE_CALL(call_id, 2), I32)); -} - -/* ------------------------------------------------------------------ tests */ - -/* POSITIVE: two identical __aeabi_cfcmple helper calls with the same operands. - * First branch is JUMPIF NE; after it falls through the comparison is known to - * be EQ. The second JUMPIF NE can therefore never be taken, so the second - * helper call and its jump are NOP-ed. */ -UT_TEST(test_float_branch_helper_same_ops_ne_then_ne_nop) -{ - TCCIRState *ir = utb_new(); - utb_pools_init(ir); - - static Sym helper; - IROperand callee = utb_callee_named(ir, &helper, TOK_FCMP); - utb_set_tok_str(TOK_FCMP, "__aeabi_cfcmple"); - - IROperand a = utb_temp(0, I32); - IROperand b = utb_temp(1, I32); - - emit_fcmp_call(ir, CALL_ID_1, callee, a, b); - emit_jumpif(ir, 7, TOK_NE); - int icall2 = emit_fcmp_call(ir, CALL_ID_2, callee, a, b); - int ijmp2 = emit_jumpif(ir, 9, TOK_NE); - utb_emit(ir, TCCIR_OP_RETURNVOID, UTB_NONE, UTB_NONE, UTB_NONE); - utb_emit(ir, TCCIR_OP_RETURNVOID, UTB_NONE, UTB_NONE, UTB_NONE); - - int changes = tcc_ir_opt_float_branch_fold(ir); - - UT_ASSERT_EQ(changes, 1); - UT_ASSERT_EQ(utb_op(ir, icall2), TCCIR_OP_NOP); - UT_ASSERT_EQ(utb_op(ir, ijmp2), TCCIR_OP_NOP); - UT_ASSERT_EQ(utb_assert_wellformed(ir, UTB_VREG_BOUND), 0); - - utb_free(ir); - return 0; -} - -/* POSITIVE: same helper/operands, first branch is JUMPIF EQ. After the first - * branch falls through the comparison is known to be NE, so the second JUMPIF - * NE is always true and becomes an unconditional JUMP. */ -UT_TEST(test_float_branch_helper_same_ops_eq_then_ne_jump) -{ - TCCIRState *ir = utb_new(); - utb_pools_init(ir); - - static Sym helper; - IROperand callee = utb_callee_named(ir, &helper, TOK_FCMP); - utb_set_tok_str(TOK_FCMP, "__aeabi_cfcmple"); - - IROperand a = utb_temp(0, I32); - IROperand b = utb_temp(1, I32); - - emit_fcmp_call(ir, CALL_ID_1, callee, a, b); - emit_jumpif(ir, 7, TOK_EQ); - int icall2 = emit_fcmp_call(ir, CALL_ID_2, callee, a, b); - int ijmp2 = emit_jumpif(ir, 9, TOK_NE); - utb_emit(ir, TCCIR_OP_RETURNVOID, UTB_NONE, UTB_NONE, UTB_NONE); - utb_emit(ir, TCCIR_OP_RETURNVOID, UTB_NONE, UTB_NONE, UTB_NONE); - - int changes = tcc_ir_opt_float_branch_fold(ir); - - UT_ASSERT_EQ(changes, 1); - UT_ASSERT_EQ(utb_op(ir, icall2), TCCIR_OP_NOP); - UT_ASSERT_EQ(utb_op(ir, ijmp2), TCCIR_OP_JUMP); - UT_ASSERT_EQ(jump_target(ir, ijmp2), 9); - UT_ASSERT_EQ(utb_assert_wellformed(ir, UTB_VREG_BOUND), 0); - - utb_free(ir); - return 0; -} - -/* GUARD: second helper call uses the same two operands in swapped order, with - * differing jump targets. For a non-strict known fact (here EQ -> known NE) the - * fold pass conservatively declines the swapped case: float comparisons are - * unordered w.r.t. NaN, so cfcmple(a,b) vs cfcmple(b,a) only reason soundly for - * strict orderings (LT/GT/ULT/UGT). No fold occurs. */ -UT_TEST(test_float_branch_helper_swapped_ops_eq_then_eq_no_fold) -{ - TCCIRState *ir = utb_new(); - utb_pools_init(ir); - - static Sym helper; - IROperand callee = utb_callee_named(ir, &helper, TOK_FCMP); - utb_set_tok_str(TOK_FCMP, "__aeabi_cfcmple"); - - IROperand a = utb_temp(0, I32); - IROperand b = utb_temp(1, I32); - - emit_fcmp_call(ir, CALL_ID_1, callee, a, b); - emit_jumpif(ir, 7, TOK_EQ); - int icall2 = emit_fcmp_call(ir, CALL_ID_2, callee, b, a); - int ijmp2 = emit_jumpif(ir, 9, TOK_EQ); - utb_emit(ir, TCCIR_OP_RETURNVOID, UTB_NONE, UTB_NONE, UTB_NONE); - utb_emit(ir, TCCIR_OP_RETURNVOID, UTB_NONE, UTB_NONE, UTB_NONE); - - int changes = tcc_ir_opt_float_branch_fold(ir); - - UT_ASSERT_EQ(changes, 0); - UT_ASSERT_EQ(utb_op(ir, icall2), TCCIR_OP_FUNCCALLVOID); - UT_ASSERT_EQ(utb_op(ir, ijmp2), TCCIR_OP_JUMPIF); - UT_ASSERT_EQ(utb_assert_wellformed(ir, UTB_VREG_BOUND), 0); - - utb_free(ir); - return 0; -} - -/* POSITIVE: TEST_ZERO followed by JUMPIF NE. After the first branch falls - * through the value is known to be zero, so a second TEST_ZERO + JUMPIF NE - * (jump-if-nonzero) can never be taken: both instructions are NOP-ed. */ -UT_TEST(test_float_branch_test_zero_ne_then_ne_nop) -{ - TCCIRState *ir = utb_new(); - - utb_emit(ir, TCCIR_OP_TEST_ZERO, UTB_NONE, utb_temp(0, I32), UTB_NONE); - emit_jumpif(ir, 4, TOK_NE); - int itest2 = utb_emit(ir, TCCIR_OP_TEST_ZERO, UTB_NONE, utb_temp(0, I32), UTB_NONE); - int ijmp2 = emit_jumpif(ir, 5, TOK_NE); - utb_emit(ir, TCCIR_OP_RETURNVOID, UTB_NONE, UTB_NONE, UTB_NONE); - utb_emit(ir, TCCIR_OP_RETURNVOID, UTB_NONE, UTB_NONE, UTB_NONE); - - int changes = tcc_ir_opt_float_branch_fold(ir); - - UT_ASSERT_EQ(changes, 1); - UT_ASSERT_EQ(utb_op(ir, itest2), TCCIR_OP_NOP); - UT_ASSERT_EQ(utb_op(ir, ijmp2), TCCIR_OP_NOP); - UT_ASSERT_EQ(utb_assert_wellformed(ir, UTB_VREG_BOUND), 0); - - utb_free(ir); - return 0; -} - -/* POSITIVE: TEST_ZERO followed by JUMPIF EQ. After the first branch falls - * through the value is known to be non-zero, so a second TEST_ZERO + JUMPIF NE - * (jump-if-nonzero) is always taken and becomes an unconditional JUMP. */ -UT_TEST(test_float_branch_test_zero_eq_then_ne_jump) -{ - TCCIRState *ir = utb_new(); - - utb_emit(ir, TCCIR_OP_TEST_ZERO, UTB_NONE, utb_temp(0, I32), UTB_NONE); - emit_jumpif(ir, 4, TOK_EQ); - int itest2 = utb_emit(ir, TCCIR_OP_TEST_ZERO, UTB_NONE, utb_temp(0, I32), UTB_NONE); - int ijmp2 = emit_jumpif(ir, 5, TOK_NE); - utb_emit(ir, TCCIR_OP_RETURNVOID, UTB_NONE, UTB_NONE, UTB_NONE); - utb_emit(ir, TCCIR_OP_RETURNVOID, UTB_NONE, UTB_NONE, UTB_NONE); - - int changes = tcc_ir_opt_float_branch_fold(ir); - - UT_ASSERT_EQ(changes, 1); - UT_ASSERT_EQ(utb_op(ir, itest2), TCCIR_OP_NOP); - UT_ASSERT_EQ(utb_op(ir, ijmp2), TCCIR_OP_JUMP); - UT_ASSERT_EQ(jump_target(ir, ijmp2), 5); - UT_ASSERT_EQ(utb_assert_wellformed(ir, UTB_VREG_BOUND), 0); - - utb_free(ir); - return 0; -} - -/* GUARD: a non-flag-helper call is ignored entirely. Even though the shape is - * identical, the callee name "foo" is not in the recognized flag-cmp set. */ -UT_TEST(test_float_branch_non_flag_helper_ignored) -{ - TCCIRState *ir = utb_new(); - utb_pools_init(ir); - - static Sym helper; - IROperand callee = utb_callee_named(ir, &helper, TOK_FCMP); - utb_set_tok_str(TOK_FCMP, "foo"); - - IROperand a = utb_temp(0, I32); - IROperand b = utb_temp(1, I32); - - int icall1 = emit_fcmp_call(ir, CALL_ID_1, callee, a, b); - int ijmp1 = emit_jumpif(ir, 7, TOK_EQ); - int icall2 = emit_fcmp_call(ir, CALL_ID_2, callee, a, b); - int ijmp2 = emit_jumpif(ir, 9, TOK_NE); - utb_emit(ir, TCCIR_OP_RETURNVOID, UTB_NONE, UTB_NONE, UTB_NONE); - utb_emit(ir, TCCIR_OP_RETURNVOID, UTB_NONE, UTB_NONE, UTB_NONE); - - int changes = tcc_ir_opt_float_branch_fold(ir); - - UT_ASSERT_EQ(changes, 0); - UT_ASSERT_EQ(utb_op(ir, icall1), TCCIR_OP_FUNCCALLVOID); - UT_ASSERT_EQ(utb_op(ir, ijmp1), TCCIR_OP_JUMPIF); - UT_ASSERT_EQ(utb_op(ir, icall2), TCCIR_OP_FUNCCALLVOID); - UT_ASSERT_EQ(utb_op(ir, ijmp2), TCCIR_OP_JUMPIF); - UT_ASSERT_EQ(utb_assert_wellformed(ir, UTB_VREG_BOUND), 0); - - utb_free(ir); - return 0; -} - -/* GUARD: mismatched helper operands block the fold. The second call compares - * a against c instead of b, so the result is not implied by the first. */ -UT_TEST(test_float_branch_mismatched_operands_no_fold) -{ - TCCIRState *ir = utb_new(); - utb_pools_init(ir); - - static Sym helper; - IROperand callee = utb_callee_named(ir, &helper, TOK_FCMP); - utb_set_tok_str(TOK_FCMP, "__aeabi_cfcmple"); - - IROperand a = utb_temp(0, I32); - IROperand b = utb_temp(1, I32); - IROperand c = utb_temp(2, I32); - - emit_fcmp_call(ir, CALL_ID_1, callee, a, b); - emit_jumpif(ir, 7, TOK_EQ); - int icall2 = emit_fcmp_call(ir, CALL_ID_2, callee, a, c); - int ijmp2 = emit_jumpif(ir, 9, TOK_NE); - utb_emit(ir, TCCIR_OP_RETURNVOID, UTB_NONE, UTB_NONE, UTB_NONE); - utb_emit(ir, TCCIR_OP_RETURNVOID, UTB_NONE, UTB_NONE, UTB_NONE); - - int changes = tcc_ir_opt_float_branch_fold(ir); - - UT_ASSERT_EQ(changes, 0); - UT_ASSERT_EQ(utb_op(ir, icall2), TCCIR_OP_FUNCCALLVOID); - UT_ASSERT_EQ(utb_op(ir, ijmp2), TCCIR_OP_JUMPIF); - UT_ASSERT_EQ(utb_assert_wellformed(ir, UTB_VREG_BOUND), 0); - - utb_free(ir); - return 0; -} - -/* GUARD: a merge point between the first and second helper calls stops the - * scan. The second helper is reachable from the fall-through of the first - * branch and from an unconditional JUMP, so its incoming state is unknown. */ -UT_TEST(test_float_branch_merge_point_blocks_scan) -{ - TCCIRState *ir = utb_new(); - utb_pools_init(ir); - - static Sym helper; - IROperand callee = utb_callee_named(ir, &helper, TOK_FCMP); - utb_set_tok_str(TOK_FCMP, "__aeabi_cfcmple"); - - IROperand a = utb_temp(0, I32); - IROperand b = utb_temp(1, I32); - - emit_fcmp_call(ir, CALL_ID_1, callee, a, b); - emit_jumpif(ir, 8, TOK_EQ); - int icall2 = emit_fcmp_call(ir, CALL_ID_2, callee, a, b); - int ijmp2 = emit_jumpif(ir, 10, TOK_NE); - utb_emit(ir, TCCIR_OP_RETURNVOID, UTB_NONE, UTB_NONE, UTB_NONE); - utb_emit(ir, TCCIR_OP_JUMP, utb_imm(4, I32), UTB_NONE, UTB_NONE); - utb_emit(ir, TCCIR_OP_RETURNVOID, UTB_NONE, UTB_NONE, UTB_NONE); - - int changes = tcc_ir_opt_float_branch_fold(ir); - - UT_ASSERT_EQ(changes, 0); - UT_ASSERT_EQ(utb_op(ir, icall2), TCCIR_OP_FUNCCALLVOID); - UT_ASSERT_EQ(utb_op(ir, ijmp2), TCCIR_OP_JUMPIF); - UT_ASSERT_EQ(utb_assert_wellformed(ir, UTB_VREG_BOUND), 0); - - utb_free(ir); - return 0; -} - -/* GUARD: the instruction immediately following the first helper call must be a - * JUMPIF for the pass to engage. A RETURNVOID there blocks folding. */ -UT_TEST(test_float_branch_non_jumpif_after_helper_blocks_fold) -{ - TCCIRState *ir = utb_new(); - utb_pools_init(ir); - - static Sym helper; - IROperand callee = utb_callee_named(ir, &helper, TOK_FCMP); - utb_set_tok_str(TOK_FCMP, "__aeabi_cfcmple"); - - IROperand a = utb_temp(0, I32); - IROperand b = utb_temp(1, I32); - - int icall1 = emit_fcmp_call(ir, CALL_ID_1, callee, a, b); - utb_emit(ir, TCCIR_OP_RETURNVOID, UTB_NONE, UTB_NONE, UTB_NONE); - int icall2 = emit_fcmp_call(ir, CALL_ID_2, callee, a, b); - int ijmp2 = emit_jumpif(ir, 9, TOK_NE); - utb_emit(ir, TCCIR_OP_RETURNVOID, UTB_NONE, UTB_NONE, UTB_NONE); - utb_emit(ir, TCCIR_OP_RETURNVOID, UTB_NONE, UTB_NONE, UTB_NONE); - - int changes = tcc_ir_opt_float_branch_fold(ir); - - UT_ASSERT_EQ(changes, 0); - UT_ASSERT_EQ(utb_op(ir, icall1), TCCIR_OP_FUNCCALLVOID); - UT_ASSERT_EQ(utb_op(ir, icall2), TCCIR_OP_FUNCCALLVOID); - UT_ASSERT_EQ(utb_op(ir, ijmp2), TCCIR_OP_JUMPIF); - UT_ASSERT_EQ(utb_assert_wellformed(ir, UTB_VREG_BOUND), 0); - - utb_free(ir); - return 0; -} - -UT_COVERS("float_branch"); diff --git a/tests/unit/arm/armv8m/test_opt_loop_utils.c b/tests/unit/arm/armv8m/test_opt_loop_utils.c index ef8c39e88..e2375f9e9 100644 --- a/tests/unit/arm/armv8m/test_opt_loop_utils.c +++ b/tests/unit/arm/armv8m/test_opt_loop_utils.c @@ -484,32 +484,46 @@ UT_TEST(test_unroll_register_body_three_iters) return 0; } -/* A memory-carrying body (STORE [100]=V0) is no longer unrolled: cloning - * stack/aggregate accesses could expose stale initializer values through the - * later forwarding passes (bitfield packed-RMW fuzz class, seed 163176), so - * collect_body_instructions rejects any memory op and the loop is left intact. */ -UT_TEST(test_unroll_store_body_blocks_unroll) +/* STOREs "[100] = #imm" in order; fill vals[]. Returns the count. */ +static int collect_store_imm_values(TCCIRState *ir, int *vals, int max) +{ + int n = 0; + for (int i = 0; i < ir->next_instruction_index && n < max; i++) + { + if (utb_op(ir, i) != TCCIR_OP_STORE) + continue; + IROperand s = utb_src1(ir, i); + if (!irop_is_immediate(s)) + continue; + vals[n++] = (int)irop_get_imm64_ex(ir, s); + } + return n; +} + +/* A body that stores (STORE [100]=V0) unrolls: the copies write memory in + * the loop's order, the IV a constant in each. */ +UT_TEST(test_unroll_store_body_clones_in_order) { TCCIRState *ir = utb_new(); utb_pools_init(ir); emit_unrollable_loop(ir, 0, 3, 1, utb_var(0, I32)); IRLoop L = utb_loop(1, 1, 5, 0); - UT_ASSERT_EQ(try_unroll_loop_ex(ir, &L, NULL, 0), 0); + UT_ASSERT_EQ(try_unroll_loop_ex(ir, &L, NULL, 0), 1); - /* Loop control and the store body are untouched (no clones written). */ - UT_ASSERT_EQ(utb_op(ir, 0), TCCIR_OP_ASSIGN); /* IV init */ - UT_ASSERT_EQ(utb_op(ir, 1), TCCIR_OP_CMP); - UT_ASSERT_EQ(utb_op(ir, 3), TCCIR_OP_STORE); - UT_ASSERT_EQ(utb_op(ir, 5), TCCIR_OP_JUMP); /* back-edge intact */ + int vals[8]; + int n = collect_store_imm_values(ir, vals, 8); + UT_ASSERT_EQ(n, 3); + UT_ASSERT_EQ(vals[0], 0); + UT_ASSERT_EQ(vals[1], 1); + UT_ASSERT_EQ(vals[2], 2); utb_free(ir); return 0; } -UT_TEST(test_unroll_store_body_used_after_blocks_unroll) +UT_TEST(test_unroll_store_body_used_after_sets_final_iv) { - /* Even with a post-loop reader of V0, the memory body blocks the unroll -- - * no clones, and no synthesized final-value ASSIGN V0=#3. */ + /* A post-loop reader of V0 gets the final value ASSIGN V0=#3. */ TCCIRState *ir = utb_new(); utb_pools_init(ir); emit_unrollable_loop(ir, 0, 3, 1, utb_var(0, I32)); @@ -517,10 +531,8 @@ UT_TEST(test_unroll_store_body_used_after_blocks_unroll) utb_emit(ir, TCCIR_OP_ASSIGN, utb_temp(0, I32), utb_var(0, I32), UTB_NONE); /* 7 reads V0 */ IRLoop L = utb_loop(1, 1, 5, 0); - UT_ASSERT_EQ(try_unroll_loop_ex(ir, &L, NULL, 0), 0); + UT_ASSERT_EQ(try_unroll_loop_ex(ir, &L, NULL, 0), 1); - UT_ASSERT_EQ(utb_op(ir, 1), TCCIR_OP_CMP); - UT_ASSERT_EQ(utb_op(ir, 3), TCCIR_OP_STORE); int found_final = 0; for (int i = 0; i < ir->next_instruction_index; i++) { @@ -532,22 +544,48 @@ UT_TEST(test_unroll_store_body_used_after_blocks_unroll) (int)irop_get_imm64_ex(ir, utb_src1(ir, i)) == 3) found_final = 1; } - UT_ASSERT(!found_final); + UT_ASSERT(found_final); utb_free(ir); return 0; } -UT_TEST(test_unroll_store_body_single_iter_blocks_unroll) +UT_TEST(test_unroll_store_body_single_iter) { - /* trip_count = 1 is irrelevant once the body carries memory -- still blocked. */ + /* init=7, limit=8 -> 1 trip: one copy storing #7. */ TCCIRState *ir = utb_new(); utb_pools_init(ir); - emit_unrollable_loop(ir, 7, 8, 1, utb_var(0, I32)); /* init=7, limit=8 -> 1 trip */ + emit_unrollable_loop(ir, 7, 8, 1, utb_var(0, I32)); + IRLoop L = utb_loop(1, 1, 5, 0); + + UT_ASSERT_EQ(try_unroll_loop_ex(ir, &L, NULL, 0), 1); + int vals[4]; + UT_ASSERT_EQ(collect_store_imm_values(ir, vals, 4), 1); + UT_ASSERT_EQ(vals[0], 7); + utb_free(ir); + return 0; +} + +/* A load bound to a VAR by STORE (`V1 <- T0*** [STORE]`, an inlined + * parameter) stays one multi-def TEMP in SSA: sound for one loop body, not + * for N straight-line copies (fuzz 308), so such a body keeps its loop. */ +UT_TEST(test_unroll_var_bound_load_blocks_unroll) +{ + TCCIRState *ir = utb_new(); + utb_pools_init(ir); + utb_emit(ir, TCCIR_OP_ASSIGN, utb_var(0, I32), utb_imm(0, I32), UTB_NONE); /* 0 */ + utb_emit(ir, TCCIR_OP_CMP, UTB_NONE, utb_var(0, I32), utb_imm(3, I32)); /* 1 */ + utb_emit(ir, TCCIR_OP_JUMPIF, utb_imm(6, I32), utb_imm(UT_GE, I32), UTB_NONE); /* 2 exit=6 */ + IROperand deref = utb_temp(0, I32); + deref.is_lval = 1; + utb_emit(ir, TCCIR_OP_STORE, utb_var(1, I32), deref, UTB_NONE); /* 3 V1 <- *T0 */ + utb_emit(ir, TCCIR_OP_ADD, utb_var(0, I32), utb_var(0, I32), utb_imm(1, I32)); /* 4 */ + utb_emit(ir, TCCIR_OP_JUMP, utb_imm(1, I32), UTB_NONE, UTB_NONE); /* 5 */ + utb_emit(ir, TCCIR_OP_RETURNVOID, UTB_NONE, UTB_NONE, UTB_NONE); /* 6 */ IRLoop L = utb_loop(1, 1, 5, 0); UT_ASSERT_EQ(try_unroll_loop_ex(ir, &L, NULL, 0), 0); - UT_ASSERT_EQ(utb_op(ir, 1), TCCIR_OP_CMP); UT_ASSERT_EQ(utb_op(ir, 3), TCCIR_OP_STORE); + UT_ASSERT_EQ(utb_op(ir, 5), TCCIR_OP_JUMP); utb_free(ir); return 0; } @@ -730,6 +768,7 @@ UT_TEST(test_transform_derived_iv_allows_address_slot_deref) div.use_idx = 4; div.shl_idx = 3; div.share_with = -1; + div.off_idx = -1; int out_ptr_vreg = 12345, out_idx_shift = 12345, out_postnop = 12345, out_stride_pos = 12345; int n_before = ir->next_instruction_index; @@ -793,6 +832,7 @@ UT_TEST(test_transform_derived_iv_reduces_register_only_div) div.use_idx = 4; div.shl_idx = 3; div.share_with = -1; + div.off_idx = -1; int out_ptr_vreg = 12345, out_idx_shift = 12345, out_postnop = 12345, out_stride_pos = 12345; @@ -858,6 +898,7 @@ UT_TEST(test_transform_derived_iv_shared_path_refused) div.use_idx = 0; div.shl_idx = -1; div.share_with = -1; + div.off_idx = -1; int ret = transform_derived_iv(ir, &L, &iv, &div, NULL, NULL, NULL, NULL, /*shared_ptr_vreg=*/5); UT_ASSERT_EQ(ret, 0); @@ -1648,6 +1689,7 @@ UT_TEST(test_eliminate_iv_counter_pretest_only_rewrites_to_ptr_cmp) memset(&div, 0, sizeof div); div.stride = 4; div.share_with = -1; + div.off_idx = -1; int ptr_vreg = VR_TEMP(0); int ret = try_eliminate_iv_counter(ir, &L, &iv, &div, ptr_vreg, /*idx_shift=*/0); @@ -1707,6 +1749,7 @@ UT_TEST(test_eliminate_iv_counter_other_use_blocks_elimination) memset(&div, 0, sizeof div); div.stride = 4; div.share_with = -1; + div.off_idx = -1; int n_before = ir->next_instruction_index; UT_ASSERT_EQ(try_eliminate_iv_counter(ir, &L, &iv, &div, VR_TEMP(0), 0), 0); @@ -1742,6 +1785,7 @@ UT_TEST(test_eliminate_iv_counter_no_cmp_found_declines) memset(&div, 0, sizeof div); div.stride = 4; div.share_with = -1; + div.off_idx = -1; UT_ASSERT_EQ(try_eliminate_iv_counter(ir, &L, &iv, &div, VR_TEMP(0), 0), 0); utb_free(ir); diff --git a/tests/unit/arm/armv8m/test_opt_memory_extra.c b/tests/unit/arm/armv8m/test_opt_memory_extra.c index 92588b54c..2cfb2bbb3 100644 --- a/tests/unit/arm/armv8m/test_opt_memory_extra.c +++ b/tests/unit/arm/armv8m/test_opt_memory_extra.c @@ -15,9 +15,7 @@ * * - addrof_var_fwd (tcc_ir_opt_addrof_var_fwd) * - invariant_global_load_hoist (tcc_ir_opt_invariant_global_load_hoist) - * - invariant_temp_deref_hoist (tcc_ir_opt_invariant_temp_deref_hoist) * - rmw_byte_clear (tcc_ir_opt_rmw_byte_clear) - * - local_copy_prop (tcc_ir_opt_local_copy_prop) * - struct_copy_roundtrip_elim (tcc_ir_opt_struct_copy_roundtrip_elim) * - const_memcpy_to_dest (tcc_ir_opt_const_memcpy_to_dest) * @@ -42,9 +40,7 @@ * avoid pulling in the optimizer engine headers). */ int tcc_ir_opt_addrof_var_fwd(TCCIRState *ir); int tcc_ir_opt_invariant_global_load_hoist(TCCIRState *ir); -int tcc_ir_opt_invariant_temp_deref_hoist(TCCIRState *ir); int tcc_ir_opt_rmw_byte_clear(TCCIRState *ir); -int tcc_ir_opt_local_copy_prop(TCCIRState *ir); int tcc_ir_opt_struct_copy_roundtrip_elim(TCCIRState *ir); int tcc_ir_opt_const_memcpy_to_dest(TCCIRState *ir); int tcc_ir_opt_deref_operand_cse(TCCIRState *ir); @@ -93,7 +89,7 @@ static IROperand utb_callee(TCCIRState *ir, Sym *sym, int tok) return irop_make_symref(0, sidx, 0, 0, 0, I32); } -/* tcc_ir_opt_invariant_temp_deref_hoist inserts a new ASSIGN (tcc_ir_pool_add +/* tcc_ir_opt_deref_operand_cse inserts a new ASSIGN (tcc_ir_pool_add * / gsym_cse_insert_before) and allocates a fresh TEMP vreg * (tcc_ir_vreg_alloc_temp). utb_new() leaves iroperand_pool_capacity, * temporary_variables_live_intervals_size and compact_instructions_size at 0; @@ -242,131 +238,6 @@ UT_TEST(test_invariant_global_load_hoist_written_global_kept) return 0; } -/* ================================================================ invariant_temp_deref_hoist */ - -/* POSITIVE: T0 is defined by a LOAD from a stack slot (a "loaded pointer") and - * dereferenced twice with only a pure ALU op between the two derefs and no - * clobber -- a fresh hoisted TEMP is inserted right after T0's def, and BOTH - * deref uses are rewritten to read it instead of re-dereferencing T0. - * (The insertion shifts every instruction from T0's def onward by one slot, - * so this test locates use1/use2 post-pass by their distinguishing dest - * vreg (T1/T2) rather than by a pre-pass instruction index.) */ -UT_TEST(test_invariant_temp_deref_hoist_two_derefs_hoisted) -{ - TCCIRState *ir = utb_hoist_new(3); /* T0..T2 used by hand below */ - - int tdef = utb_emit(ir, TCCIR_OP_LOAD, utb_temp(0, I32), utb_slot_lval(-8, I32), UTB_NONE); - utb_emit(ir, TCCIR_OP_ADD, utb_temp(1, I32), utb_deref_temp(0, I32), utb_imm(1, I32)); - utb_emit(ir, TCCIR_OP_ADD, utb_temp(2, I32), utb_deref_temp(0, I32), utb_imm(2, I32)); - utb_emit(ir, TCCIR_OP_RETURNVALUE, UTB_NONE, utb_temp(2, I32), UTB_NONE); - - int changes = tcc_ir_opt_invariant_temp_deref_hoist(ir); - - UT_ASSERT(changes > 0); - UT_ASSERT_EQ(utb_op(ir, tdef), TCCIR_OP_LOAD); /* the original def is untouched */ - - /* Locate the two ADDs post-pass by their (unaffected) dest vreg. */ - int use1 = -1, use2 = -1; - for (int i = 0; i < ir->next_instruction_index; i++) - { - if (utb_op(ir, i) != TCCIR_OP_ADD) - continue; - int32_t dv = utb_vreg(utb_dest(ir, i)); - if (dv == utb_vreg(utb_temp(1, I32))) - use1 = i; - else if (dv == utb_vreg(utb_temp(2, I32))) - use2 = i; - } - UT_ASSERT(use1 >= 0); - UT_ASSERT(use2 >= 0); - - /* Oracle: neither ADD still dereferences T0 (vreg 0); both read the same - * non-lval hoisted TEMP instead. */ - IROperand s1_use1 = utb_src1(ir, use1); - IROperand s1_use2 = utb_src1(ir, use2); - UT_ASSERT(!s1_use1.is_lval || utb_vreg(s1_use1) != utb_vreg(utb_temp(0, I32))); - UT_ASSERT(!s1_use2.is_lval || utb_vreg(s1_use2) != utb_vreg(utb_temp(0, I32))); - UT_ASSERT_EQ(utb_vreg(s1_use1), utb_vreg(s1_use2)); - - utb_free(ir); - return 0; -} - -UT_TEST(test_invariant_temp_deref_hoist_copy_chain_hoisted) -{ - TCCIRState *ir = utb_hoist_new(5); /* T0..T4 used by hand below */ - - utb_emit(ir, TCCIR_OP_LOAD, utb_temp(0, I32), utb_slot_lval(-8, I32), UTB_NONE); - utb_emit(ir, TCCIR_OP_ASSIGN, utb_temp(1, I32), utb_temp(0, I32), UTB_NONE); - utb_emit(ir, TCCIR_OP_ASSIGN, utb_temp(2, I32), utb_temp(1, I32), UTB_NONE); - utb_emit(ir, TCCIR_OP_ADD, utb_temp(3, I32), utb_deref_temp(1, I32), utb_imm(1, I32)); - utb_emit(ir, TCCIR_OP_ADD, utb_temp(4, I32), utb_deref_temp(2, I32), utb_imm(2, I32)); - utb_emit(ir, TCCIR_OP_RETURNVALUE, UTB_NONE, utb_temp(4, I32), UTB_NONE); - - int changes = tcc_ir_opt_invariant_temp_deref_hoist(ir); - - UT_ASSERT(changes > 0); - - int use1 = -1, use2 = -1; - for (int i = 0; i < ir->next_instruction_index; i++) - { - if (utb_op(ir, i) != TCCIR_OP_ADD) - continue; - int32_t dv = utb_vreg(utb_dest(ir, i)); - if (dv == utb_vreg(utb_temp(3, I32))) - use1 = i; - else if (dv == utb_vreg(utb_temp(4, I32))) - use2 = i; - } - UT_ASSERT(use1 >= 0); - UT_ASSERT(use2 >= 0); - - IROperand s1_use1 = utb_src1(ir, use1); - IROperand s1_use2 = utb_src1(ir, use2); - UT_ASSERT(!s1_use1.is_lval); - UT_ASSERT(!s1_use2.is_lval); - UT_ASSERT_EQ(utb_vreg(s1_use1), utb_vreg(s1_use2)); - UT_ASSERT(utb_vreg(s1_use1) != utb_vreg(utb_temp(1, I32))); - UT_ASSERT(utb_vreg(s1_use2) != utb_vreg(utb_temp(2, I32))); - - utb_free(ir); - return 0; -} - -/* NEGATIVE (guard): a FUNCCALLVOID sits between the two derefs -- the callee - * may write through T0's address, so no hoist may happen across it. */ -UT_TEST(test_invariant_temp_deref_hoist_intervening_call_blocks) -{ - TCCIRState *ir = utb_new(); - utb_pools_init(ir); /* iroperand_pool starts small but tcc_ir_pool_ensure - * grows it via realloc, since capacity is nonzero. */ - ir->temporary_variables_live_intervals_size = 64; - ir->temporary_variables_live_intervals = (IRLiveInterval *)tcc_mallocz(sizeof(IRLiveInterval) * 64); - ir->next_temporary_variable = 3; /* T0..T2 used by hand below */ - ir->compact_instructions_size = UTB_MAX_INSTR; - - static Sym callee_sym; - utb_set_tok_str(TOK_FOO, "foo"); - IROperand callee = utb_callee(ir, &callee_sym, TOK_FOO); - - utb_emit(ir, TCCIR_OP_LOAD, utb_temp(0, I32), utb_slot_lval(-8, I32), UTB_NONE); - int use1 = utb_emit(ir, TCCIR_OP_ADD, utb_temp(1, I32), utb_deref_temp(0, I32), utb_imm(1, I32)); - utb_emit(ir, TCCIR_OP_FUNCCALLVOID, UTB_NONE, callee, utb_imm((int32_t)TCCIR_ENCODE_CALL(1, 0), I32)); - int use2 = utb_emit(ir, TCCIR_OP_ADD, utb_temp(2, I32), utb_deref_temp(0, I32), utb_imm(2, I32)); - utb_emit(ir, TCCIR_OP_RETURNVALUE, UTB_NONE, utb_temp(2, I32), UTB_NONE); - - int changes = tcc_ir_opt_invariant_temp_deref_hoist(ir); - - UT_ASSERT_EQ(changes, 0); - UT_ASSERT_EQ(utb_vreg(utb_src1(ir, use1)), utb_vreg(utb_temp(0, I32))); - UT_ASSERT_EQ(utb_vreg(utb_src1(ir, use2)), utb_vreg(utb_temp(0, I32))); - UT_ASSERT(utb_src1(ir, use1).is_lval); - UT_ASSERT(utb_src1(ir, use2).is_lval); - - utb_free(ir); - return 0; -} - /* ================================================================ deref_operand_cse */ /* Returns the vreg both reads were rewritten onto, or -1 if either still @@ -583,61 +454,6 @@ UT_TEST(test_rmw_byte_clear_multi_use_and_result_kept) return 0; } -/* ================================================================ local_copy_prop */ - -/* POSITIVE: 4 consecutive LOAD(A[k*4])+STORE(B[k*4]) pairs (A at 100, B at - * 200, stride 4) with nothing else touching A -- the pass redirects the - * writes from A to B and NOPs the whole copy chain. */ -UT_TEST(test_local_copy_prop_four_pairs_redirect_writes) -{ - TCCIRState *ir = utb_new(); - - int idx[8]; - int k = 0; - for (int i = 0; i < 4; i++) - { - idx[k++] = utb_emit(ir, TCCIR_OP_LOAD, utb_temp(i, I32), utb_slot_lval(100 + i * 4, I32), UTB_NONE); - idx[k++] = utb_emit(ir, TCCIR_OP_STORE, utb_slot_lval(200 + i * 4, I32), utb_temp(i, I32), UTB_NONE); - } - /* B[0] is read afterwards (the copy's destination is actually used); A is - * never referenced again outside the chain itself. */ - utb_emit(ir, TCCIR_OP_RETURNVALUE, UTB_NONE, utb_slot_lval(200, I32), UTB_NONE); - - int changes = tcc_ir_opt_local_copy_prop(ir); - - UT_ASSERT_EQ(changes, 4); - for (int i = 0; i < 8; i++) - UT_ASSERT_EQ(utb_op(ir, idx[i]), TCCIR_OP_NOP); - - utb_free(ir); - return 0; -} - -/* NEGATIVE (guard): only 3 consecutive pairs (below the count>=4 threshold) - * -- the chain is left completely untouched. */ -UT_TEST(test_local_copy_prop_three_pairs_kept) -{ - TCCIRState *ir = utb_new(); - - int idx[6]; - int k = 0; - for (int i = 0; i < 3; i++) - { - idx[k++] = utb_emit(ir, TCCIR_OP_LOAD, utb_temp(i, I32), utb_slot_lval(100 + i * 4, I32), UTB_NONE); - idx[k++] = utb_emit(ir, TCCIR_OP_STORE, utb_slot_lval(200 + i * 4, I32), utb_temp(i, I32), UTB_NONE); - } - utb_emit(ir, TCCIR_OP_RETURNVALUE, UTB_NONE, utb_imm(0, I32), UTB_NONE); - - int changes = tcc_ir_opt_local_copy_prop(ir); - - UT_ASSERT_EQ(changes, 0); - for (int i = 0; i < 6; i++) - UT_ASSERT_EQ(utb_op(ir, idx[i]), (i % 2 == 0) ? TCCIR_OP_LOAD : TCCIR_OP_STORE); - - utb_free(ir); - return 0; -} - /* ================================================================ struct_copy_roundtrip_elim */ /* POSITIVE: memcpy(&B, &A, N) immediately followed (straight-line, no writes diff --git a/tests/unit/arm/armv8m/test_opt_neg_chain.c b/tests/unit/arm/armv8m/test_opt_neg_chain.c deleted file mode 100644 index 75687fc34..000000000 --- a/tests/unit/arm/armv8m/test_opt_neg_chain.c +++ /dev/null @@ -1,402 +0,0 @@ -/* - * test_opt_neg_chain.c - suite for ir/opt_neg_chain.c (negation-chain CSE) - * - * tcc_ir_opt_neg_chain_cse tracks each TEMP as a canonical (base, sign) pair - * (sign = parity of accumulated negations). When `T_b = #0 SUB T_a` recomputes - * a (base, sign) already produced by an earlier TEMP T_y, the SUB is rewritten - * as `T_b = T_y` (ASSIGN), to be collapsed by a later copy-prop + DCE. - * - * These are isolated tests: a hand-built IR sequence is run through the bare - * pass entry point and the resulting instructions are inspected directly. - */ - -#include "ir_build.h" - -#include "ut.h" - -/* Pass entry point (declared in ir/opt.h; forward-declared to avoid pulling in - * the optimizer engine headers). */ -int tcc_ir_opt_neg_chain_cse(TCCIRState *ir); - -#define I32 IROP_BTYPE_INT32 -#define I64 IROP_BTYPE_INT64 - -/* ------------------------------------------------------------------ tests */ - -/* T0 = 5 (anchor) - * T1 = -T0 (first negation of T0) - * T2 = -T1 == T0 -> folds to T2 = T0 (ASSIGN), because the canonical - * (base=T0, sign=+) was already produced by the anchor. */ -UT_TEST(test_neg_chain_double_negation_folds) -{ - TCCIRState *ir = utb_new(); - - utb_emit(ir, TCCIR_OP_ASSIGN, utb_temp(0, I32), utb_imm(5, I32), UTB_NONE); - utb_emit(ir, TCCIR_OP_SUB, utb_temp(1, I32), utb_imm(0, I32), utb_temp(0, I32)); - int i2 = utb_emit(ir, TCCIR_OP_SUB, utb_temp(2, I32), utb_imm(0, I32), utb_temp(1, I32)); - - int changes = tcc_ir_opt_neg_chain_cse(ir); - - UT_ASSERT_EQ(changes, 1); - UT_ASSERT_EQ(utb_op(ir, i2), TCCIR_OP_ASSIGN); - UT_ASSERT_EQ(utb_vreg(utb_src1(ir, i2)), TCCIR_ENCODE_VREG(TCCIR_VREG_TYPE_TEMP, 0)); - - utb_free(ir); - return 0; -} - -/* A single negation has nothing to CSE against -> no change, SUB preserved. */ -UT_TEST(test_neg_chain_single_negation_no_fold) -{ - TCCIRState *ir = utb_new(); - - utb_emit(ir, TCCIR_OP_ASSIGN, utb_temp(0, I32), utb_imm(5, I32), UTB_NONE); - int i1 = utb_emit(ir, TCCIR_OP_SUB, utb_temp(1, I32), utb_imm(0, I32), utb_temp(0, I32)); - - int changes = tcc_ir_opt_neg_chain_cse(ir); - - UT_ASSERT_EQ(changes, 0); - UT_ASSERT_EQ(utb_op(ir, i1), TCCIR_OP_SUB); - - utb_free(ir); - return 0; -} - -/* `T_b = imm - T_a` with imm != 0 is plain subtraction, not negation -> no fold, - * even when it forms a chain that would otherwise be canonicalizable. */ -UT_TEST(test_neg_chain_nonzero_minuend_no_fold) -{ - TCCIRState *ir = utb_new(); - - utb_emit(ir, TCCIR_OP_ASSIGN, utb_temp(0, I32), utb_imm(5, I32), UTB_NONE); - utb_emit(ir, TCCIR_OP_SUB, utb_temp(1, I32), utb_imm(7, I32), utb_temp(0, I32)); - int i2 = utb_emit(ir, TCCIR_OP_SUB, utb_temp(2, I32), utb_imm(7, I32), utb_temp(1, I32)); - - int changes = tcc_ir_opt_neg_chain_cse(ir); - - UT_ASSERT_EQ(changes, 0); - UT_ASSERT_EQ(utb_op(ir, i2), TCCIR_OP_SUB); - - utb_free(ir); - return 0; -} - -/* Width guard: the canonical match exists, but the final SUB's dest width - * differs from the negated source width, so folding it to an ASSIGN could - * drop/extend bits the SUB wouldn't have -> the pass must NOT fold. */ -UT_TEST(test_neg_chain_width_mismatch_no_fold) -{ - TCCIRState *ir = utb_new(); - - utb_emit(ir, TCCIR_OP_ASSIGN, utb_temp(0, I32), utb_imm(5, I32), UTB_NONE); - utb_emit(ir, TCCIR_OP_SUB, utb_temp(1, I32), utb_imm(0, I32), utb_temp(0, I32)); - /* dest T2 is 64-bit while src2 T1 is 32-bit -> dest_btype != src_btype. */ - int i2 = utb_emit(ir, TCCIR_OP_SUB, utb_temp(2, I64), utb_imm(0, I32), utb_temp(1, I32)); - - int changes = tcc_ir_opt_neg_chain_cse(ir); - - UT_ASSERT_EQ(changes, 0); - UT_ASSERT_EQ(utb_op(ir, i2), TCCIR_OP_SUB); - - utb_free(ir); - return 0; -} - -#define I8 IROP_BTYPE_INT8 -#define I16 IROP_BTYPE_INT16 - -static inline int vreg_temp(int pos) -{ - return TCCIR_ENCODE_VREG(TCCIR_VREG_TYPE_TEMP, pos); -} - -/* ------------------------------------------------------------------ tests */ - -/* INT8 boundary: a narrow negation chain must still CSE correctly. */ -UT_TEST(test_neg_chain_int8_double_negation_folds) -{ - TCCIRState *ir = utb_new(); - - utb_emit(ir, TCCIR_OP_ASSIGN, utb_temp(0, I8), utb_imm(0x80, I8), UTB_NONE); - utb_emit(ir, TCCIR_OP_SUB, utb_temp(1, I8), utb_imm(0, I8), utb_temp(0, I8)); - int i2 = utb_emit(ir, TCCIR_OP_SUB, utb_temp(2, I8), utb_imm(0, I8), utb_temp(1, I8)); - - int changes = tcc_ir_opt_neg_chain_cse(ir); - - UT_ASSERT_EQ(changes, 1); - UT_ASSERT_EQ(utb_op(ir, i2), TCCIR_OP_ASSIGN); - UT_ASSERT_EQ(utb_vreg(utb_src1(ir, i2)), vreg_temp(0)); - UT_ASSERT_EQ(utb_assert_wellformed(ir, 16), 0); - - utb_free(ir); - return 0; -} - -/* INT16 boundary: a 16-bit negation chain must still CSE correctly. */ -UT_TEST(test_neg_chain_int16_double_negation_folds) -{ - TCCIRState *ir = utb_new(); - - utb_emit(ir, TCCIR_OP_ASSIGN, utb_temp(0, I16), utb_imm(0x8000, I16), UTB_NONE); - utb_emit(ir, TCCIR_OP_SUB, utb_temp(1, I16), utb_imm(0, I16), utb_temp(0, I16)); - int i2 = utb_emit(ir, TCCIR_OP_SUB, utb_temp(2, I16), utb_imm(0, I16), utb_temp(1, I16)); - - int changes = tcc_ir_opt_neg_chain_cse(ir); - - UT_ASSERT_EQ(changes, 1); - UT_ASSERT_EQ(utb_op(ir, i2), TCCIR_OP_ASSIGN); - UT_ASSERT_EQ(utb_vreg(utb_src1(ir, i2)), vreg_temp(0)); - UT_ASSERT_EQ(utb_assert_wellformed(ir, 16), 0); - - utb_free(ir); - return 0; -} - -/* INT64 boundary: 64-bit operands are not special-cased away. */ -UT_TEST(test_neg_chain_int64_double_negation_folds) -{ - TCCIRState *ir = utb_new(); - - utb_emit(ir, TCCIR_OP_ASSIGN, utb_temp(0, I64), utb_imm(1, I64), UTB_NONE); - utb_emit(ir, TCCIR_OP_SUB, utb_temp(1, I64), utb_imm(0, I64), utb_temp(0, I64)); - int i2 = utb_emit(ir, TCCIR_OP_SUB, utb_temp(2, I64), utb_imm(0, I64), utb_temp(1, I64)); - - int changes = tcc_ir_opt_neg_chain_cse(ir); - - UT_ASSERT_EQ(changes, 1); - UT_ASSERT_EQ(utb_op(ir, i2), TCCIR_OP_ASSIGN); - UT_ASSERT_EQ(utb_vreg(utb_src1(ir, i2)), vreg_temp(0)); - UT_ASSERT_EQ(utb_assert_wellformed(ir, 16), 0); - - utb_free(ir); - return 0; -} - -/* Chain length 3: T0 -> -T0 -> T0 -> -T0 produces two folds. */ -UT_TEST(test_neg_chain_length_3_folds) -{ - TCCIRState *ir = utb_new(); - - utb_emit(ir, TCCIR_OP_ASSIGN, utb_temp(0, I32), utb_imm(0x80000000, I32), UTB_NONE); - int i1 = utb_emit(ir, TCCIR_OP_SUB, utb_temp(1, I32), utb_imm(0, I32), utb_temp(0, I32)); - int i2 = utb_emit(ir, TCCIR_OP_SUB, utb_temp(2, I32), utb_imm(0, I32), utb_temp(1, I32)); - int i3 = utb_emit(ir, TCCIR_OP_SUB, utb_temp(3, I32), utb_imm(0, I32), utb_temp(2, I32)); - - int changes = tcc_ir_opt_neg_chain_cse(ir); - - UT_ASSERT_EQ(changes, 2); - UT_ASSERT_EQ(utb_op(ir, i1), TCCIR_OP_SUB); - UT_ASSERT_EQ(utb_op(ir, i2), TCCIR_OP_ASSIGN); - UT_ASSERT_EQ(utb_vreg(utb_src1(ir, i2)), vreg_temp(0)); - UT_ASSERT_EQ(utb_op(ir, i3), TCCIR_OP_ASSIGN); - UT_ASSERT_EQ(utb_vreg(utb_src1(ir, i3)), vreg_temp(1)); - UT_ASSERT_EQ(utb_assert_wellformed(ir, 16), 0); - - utb_free(ir); - return 0; -} - -/* Chain length 4/N: alternating negations continue to collapse. */ -UT_TEST(test_neg_chain_length_4_folds) -{ - TCCIRState *ir = utb_new(); - - utb_emit(ir, TCCIR_OP_ASSIGN, utb_temp(0, I32), utb_imm(42, I32), UTB_NONE); - int i1 = utb_emit(ir, TCCIR_OP_SUB, utb_temp(1, I32), utb_imm(0, I32), utb_temp(0, I32)); - int i2 = utb_emit(ir, TCCIR_OP_SUB, utb_temp(2, I32), utb_imm(0, I32), utb_temp(1, I32)); - int i3 = utb_emit(ir, TCCIR_OP_SUB, utb_temp(3, I32), utb_imm(0, I32), utb_temp(2, I32)); - int i4 = utb_emit(ir, TCCIR_OP_SUB, utb_temp(4, I32), utb_imm(0, I32), utb_temp(3, I32)); - - int changes = tcc_ir_opt_neg_chain_cse(ir); - - UT_ASSERT_EQ(changes, 3); - UT_ASSERT_EQ(utb_op(ir, i1), TCCIR_OP_SUB); - UT_ASSERT_EQ(utb_op(ir, i2), TCCIR_OP_ASSIGN); - UT_ASSERT_EQ(utb_vreg(utb_src1(ir, i2)), vreg_temp(0)); - UT_ASSERT_EQ(utb_op(ir, i3), TCCIR_OP_ASSIGN); - UT_ASSERT_EQ(utb_vreg(utb_src1(ir, i3)), vreg_temp(1)); - UT_ASSERT_EQ(utb_op(ir, i4), TCCIR_OP_ASSIGN); - UT_ASSERT_EQ(utb_vreg(utb_src1(ir, i4)), vreg_temp(0)); - UT_ASSERT_EQ(utb_assert_wellformed(ir, 16), 0); - - utb_free(ir); - return 0; -} - -/* ★ Mixed-width link: an INT8 negation of an INT32 TEMP must not fold into the - * 32-bit TEMP. The first link itself cannot fold (widths differ); the second - * link is same-width INT8 and must not be rewritten to an ASSIGN of T0. */ -UT_TEST(test_neg_chain_mixed_width_int8_int32) -{ - TCCIRState *ir = utb_new(); - - utb_emit(ir, TCCIR_OP_ASSIGN, utb_temp(0, I32), utb_imm(5, I32), UTB_NONE); - int i1 = utb_emit(ir, TCCIR_OP_SUB, utb_temp(1, I8), utb_imm(0, I8), utb_temp(0, I32)); - int i2 = utb_emit(ir, TCCIR_OP_SUB, utb_temp(2, I8), utb_imm(0, I8), utb_temp(1, I8)); - - int changes = tcc_ir_opt_neg_chain_cse(ir); - - /* FIXED: a width-changing negation (INT8 = -INT32) is not value-preserving, - * so it must not join the 32-bit base's canonical chain. The second - * same-width INT8 link therefore cannot be folded back to the wide T0; both - * SUBs are preserved and changes == 0. */ - UT_ASSERT_EQ(changes, 0); - UT_ASSERT_EQ(utb_op(ir, i1), TCCIR_OP_SUB); - UT_ASSERT_EQ(utb_op(ir, i2), TCCIR_OP_SUB); - UT_ASSERT_EQ(utb_assert_wellformed(ir, 16), 0); - - utb_free(ir); - return 0; -} - -/* Merge-point reset: a back-edge makes the loop header a merge point, so the - * canonical state accumulated before the loop must be cleared. The post-loop - * negation therefore cannot fold back to the pre-loop anchor. */ -UT_TEST(test_neg_chain_merge_reset_clears_canon) -{ - TCCIRState *ir = utb_new(); - - utb_emit(ir, TCCIR_OP_ASSIGN, utb_temp(0, I32), utb_imm(7, I32), UTB_NONE); /* 0 */ - int i1 = utb_emit(ir, TCCIR_OP_SUB, utb_temp(1, I32), utb_imm(0, I32), utb_temp(0, I32)); /* 1: loop header */ - utb_emit(ir, TCCIR_OP_JUMPIF, utb_imm(1, I32), utb_temp(9, I32), UTB_NONE); /* 2: back-edge */ - int i2 = utb_emit(ir, TCCIR_OP_SUB, utb_temp(2, I32), utb_imm(0, I32), utb_temp(1, I32)); /* 3: after loop */ - - int changes = tcc_ir_opt_neg_chain_cse(ir); - - UT_ASSERT_EQ(changes, 0); - UT_ASSERT_EQ(utb_op(ir, i1), TCCIR_OP_SUB); - UT_ASSERT_EQ(utb_op(ir, i2), TCCIR_OP_SUB); - UT_ASSERT_EQ(utb_assert_wellformed(ir, 16), 0); - - utb_free(ir); - return 0; -} - -/* Reuse after reset: once the canonical tables are cleared at a merge point, a - * new canonical pair can be established and reused later in the same function. */ -UT_TEST(test_neg_chain_reuse_after_reset) -{ - TCCIRState *ir = utb_new(); - - utb_emit(ir, TCCIR_OP_ASSIGN, utb_temp(0, I32), utb_imm(3, I32), UTB_NONE); /* 0 */ - utb_emit(ir, TCCIR_OP_SUB, utb_temp(1, I32), utb_imm(0, I32), utb_temp(0, I32)); /* 1: header */ - utb_emit(ir, TCCIR_OP_JUMPIF, utb_imm(1, I32), utb_temp(9, I32), UTB_NONE); /* 2: back-edge */ - int i2 = utb_emit(ir, TCCIR_OP_SUB, utb_temp(2, I32), utb_imm(0, I32), utb_temp(1, I32)); /* 3 */ - int i3 = utb_emit(ir, TCCIR_OP_SUB, utb_temp(3, I32), utb_imm(0, I32), utb_temp(2, I32)); /* 4 */ - - int changes = tcc_ir_opt_neg_chain_cse(ir); - - UT_ASSERT_EQ(changes, 1); - UT_ASSERT_EQ(utb_op(ir, i2), TCCIR_OP_SUB); - UT_ASSERT_EQ(utb_op(ir, i3), TCCIR_OP_ASSIGN); - UT_ASSERT_EQ(utb_vreg(utb_src1(ir, i3)), vreg_temp(1)); - UT_ASSERT_EQ(utb_assert_wellformed(ir, 16), 0); - - utb_free(ir); - return 0; -} - -/* Non-TEMP source operand: the pass only tracks TEMP negations, so a SUB of a - * VAR must anchor to itself and not fold. */ -UT_TEST(test_neg_chain_var_src_no_fold) -{ - TCCIRState *ir = utb_new(); - - utb_emit(ir, TCCIR_OP_ASSIGN, utb_temp(0, I32), utb_imm(5, I32), UTB_NONE); - int i1 = utb_emit(ir, TCCIR_OP_SUB, utb_temp(1, I32), utb_imm(0, I32), utb_var(0, I32)); - int i2 = utb_emit(ir, TCCIR_OP_SUB, utb_temp(2, I32), utb_imm(0, I32), utb_temp(1, I32)); - - int changes = tcc_ir_opt_neg_chain_cse(ir); - - UT_ASSERT_EQ(changes, 0); - UT_ASSERT_EQ(utb_op(ir, i1), TCCIR_OP_SUB); - UT_ASSERT_EQ(utb_op(ir, i2), TCCIR_OP_SUB); - UT_ASSERT_EQ(utb_assert_wellformed(ir, 16), 0); - - utb_free(ir); - return 0; -} - -/* Non-TEMP destination operand: a VAR destination is ignored entirely, so the - * following TEMP negation has no canonical -T0 to fold against. */ -UT_TEST(test_neg_chain_var_dest_ignored) -{ - TCCIRState *ir = utb_new(); - - utb_emit(ir, TCCIR_OP_ASSIGN, utb_temp(0, I32), utb_imm(5, I32), UTB_NONE); - int i1 = utb_emit(ir, TCCIR_OP_SUB, utb_var(0, I32), utb_imm(0, I32), utb_temp(0, I32)); - int i2 = utb_emit(ir, TCCIR_OP_SUB, utb_temp(1, I32), utb_imm(0, I32), utb_temp(0, I32)); - - int changes = tcc_ir_opt_neg_chain_cse(ir); - - UT_ASSERT_EQ(changes, 0); - UT_ASSERT_EQ(utb_op(ir, i1), TCCIR_OP_SUB); - UT_ASSERT_EQ(utb_op(ir, i2), TCCIR_OP_SUB); - UT_ASSERT_EQ(utb_assert_wellformed(ir, 16), 0); - - utb_free(ir); - return 0; -} - -/* lval on the source operand: a dereferenced value is not a plain negation of - * the underlying TEMP, so the chain must not be canonicalized. */ -UT_TEST(test_neg_chain_lval_src2_no_fold) -{ - TCCIRState *ir = utb_new(); - - utb_emit(ir, TCCIR_OP_ASSIGN, utb_temp(0, I32), utb_imm(5, I32), UTB_NONE); - int i1 = utb_emit(ir, TCCIR_OP_SUB, utb_temp(1, I32), utb_imm(0, I32), utb_lval(utb_temp(0, I32))); - int i2 = utb_emit(ir, TCCIR_OP_SUB, utb_temp(2, I32), utb_imm(0, I32), utb_temp(1, I32)); - - int changes = tcc_ir_opt_neg_chain_cse(ir); - - UT_ASSERT_EQ(changes, 0); - UT_ASSERT_EQ(utb_op(ir, i1), TCCIR_OP_SUB); - UT_ASSERT_EQ(utb_op(ir, i2), TCCIR_OP_SUB); - UT_ASSERT_EQ(utb_assert_wellformed(ir, 16), 0); - - utb_free(ir); - return 0; -} - -/* lval on the destination operand: the pass skips lval destinations, so the - * SUB is not recorded as a negation. */ -UT_TEST(test_neg_chain_lval_dest_ignored) -{ - TCCIRState *ir = utb_new(); - - utb_emit(ir, TCCIR_OP_ASSIGN, utb_temp(0, I32), utb_imm(5, I32), UTB_NONE); - int i1 = utb_emit(ir, TCCIR_OP_SUB, utb_lval(utb_temp(1, I32)), utb_imm(0, I32), utb_temp(0, I32)); - int i2 = utb_emit(ir, TCCIR_OP_SUB, utb_temp(2, I32), utb_imm(0, I32), utb_temp(0, I32)); - - int changes = tcc_ir_opt_neg_chain_cse(ir); - - UT_ASSERT_EQ(changes, 0); - UT_ASSERT_EQ(utb_op(ir, i1), TCCIR_OP_SUB); - UT_ASSERT_EQ(utb_op(ir, i2), TCCIR_OP_SUB); - UT_ASSERT_EQ(utb_assert_wellformed(ir, 16), 0); - - utb_free(ir); - return 0; -} - -/* Idempotence / fixpoint: one application folds all opportunities; a second - * application makes no further changes. */ -UT_TEST(test_neg_chain_idempotent) -{ - TCCIRState *ir = utb_new(); - - utb_emit(ir, TCCIR_OP_ASSIGN, utb_temp(0, I32), utb_imm(5, I32), UTB_NONE); - utb_emit(ir, TCCIR_OP_SUB, utb_temp(1, I32), utb_imm(0, I32), utb_temp(0, I32)); - utb_emit(ir, TCCIR_OP_SUB, utb_temp(2, I32), utb_imm(0, I32), utb_temp(1, I32)); - - int total = utb_run_to_fixpoint(ir, tcc_ir_opt_neg_chain_cse, 10); - UT_ASSERT_EQ(total, 1); - UT_ASSERT_EQ(tcc_ir_opt_neg_chain_cse(ir), 0); - UT_ASSERT_EQ(utb_assert_wellformed(ir, 16), 0); - - utb_free(ir); - return 0; -} - -UT_COVERS("neg_chain_cse"); diff --git a/tests/unit/arm/armv8m/test_opt_region_merge_case.c b/tests/unit/arm/armv8m/test_opt_region_merge_case.c new file mode 100644 index 000000000..56ad38fdb --- /dev/null +++ b/tests/unit/arm/armv8m/test_opt_region_merge_case.c @@ -0,0 +1,205 @@ +/* + * test_opt_region_merge_case.c - suite for ir/cross_jump.c:tcc_ir_region_merge, + * case bodies that differ only in their own case constant. + * + * Post-RA IR built by hand: a SWITCH_TABLE at the end dispatching on T0 + * (allocated to r4, live over the whole function) to three bodies + * `Tk <-- #k [LOAD]; *P0 <-- Tk [STORE]; JUMP end`. The first body is kept + * and reads T0 in place of its constant; the others become jumps to it. + */ + +#include "ir_build.h" + +#include "ut.h" + +int tcc_ir_region_merge(TCCIRState *ir); + +#define I32 IROP_BTYPE_INT32 +#define I64 IROP_BTYPE_INT64 + +enum { NTEMPS = 8 }; + +typedef struct CaseIR +{ + TCCIRState *ir; + TCCIRSwitchTable tab; + int targets[3]; + IRLiveInterval temps[NTEMPS]; + IRLiveInterval params[1]; + int end, sw; +} CaseIR; + +static void set_reg(IRLiveInterval *li, int reg, int start, int end) +{ + li->allocation.r0 = (uint16_t)reg; + li->allocation.r1 = PREG_NONE; + li->allocation.offset = 0; + li->start = (uint32_t)start; + li->end = (uint32_t)end; +} + +typedef struct CaseOpts +{ + int delta; /* constant = case value + delta */ + int goto_mid; /* the default body jumps to the second case body */ + int clobber; /* each body redefines the selector first */ + int sel_btype; /* selector type */ + int body_reg; /* register of the bodies' temps */ +} CaseOpts; + +/* 0: JUMP sw + * 1..: bodies (3 or 4 instructions each) + * default: RETURNVOID or JUMP to body 2 + * sw: SWITCH_TABLE T0, table 0 + * end: RETURNVOID */ +static void build(CaseIR *c, CaseOpts o) +{ + memset(c, 0, sizeof *c); + TCCIRState *ir = c->ir = utb_new(); + ir->iroperand_pool_capacity = UTB_MAX_OPERANDS; + int body_len = o.clobber ? 4 : 3; + c->sw = 1 + 3 * body_len + 1; + c->end = c->sw + 1; + utb_emit(ir, TCCIR_OP_JUMP, utb_imm(c->sw, I32), UTB_NONE, UTB_NONE); + for (int k = 0; k < 3; k++) + { + c->targets[k] = ir->next_instruction_index; + if (o.clobber) + utb_emit(ir, TCCIR_OP_LOAD, utb_temp(0, I32), utb_imm(9, I32), UTB_NONE); + utb_emit(ir, TCCIR_OP_LOAD, utb_temp(1 + k, I32), utb_imm(k + o.delta, I32), UTB_NONE); + utb_emit(ir, TCCIR_OP_STORE, utb_lval(utb_param(0, I32)), utb_temp(1 + k, I32), UTB_NONE); + utb_emit(ir, TCCIR_OP_JUMP, utb_imm(c->end, I32), UTB_NONE, UTB_NONE); + } + if (o.goto_mid) + utb_emit(ir, TCCIR_OP_JUMP, utb_imm(c->targets[1], I32), UTB_NONE, UTB_NONE); + else + utb_emit(ir, TCCIR_OP_RETURNVOID, UTB_NONE, UTB_NONE, UTB_NONE); + int dflt = c->sw - 1; + utb_emit(ir, TCCIR_OP_SWITCH_TABLE, UTB_NONE, utb_temp(0, o.sel_btype), utb_imm(0, I32)); + utb_emit(ir, TCCIR_OP_RETURNVOID, UTB_NONE, UTB_NONE, UTB_NONE); + for (int k = 0; k < 3; k++) + ir->compact_instructions[c->targets[k]].is_jump_target = 1; + ir->compact_instructions[dflt].is_jump_target = 1; + ir->compact_instructions[c->sw].is_jump_target = 1; + ir->compact_instructions[c->end].is_jump_target = 1; + + c->tab.min_val = 0; + c->tab.max_val = 2; + c->tab.default_target = dflt; + c->tab.targets = c->targets; + c->tab.num_entries = 3; + ir->switch_tables = &c->tab; + ir->num_switch_tables = 1; + + set_reg(&c->temps[0], 4, 0, c->sw); + for (int k = 0; k < 3; k++) + set_reg(&c->temps[1 + k], o.body_reg, c->targets[k], c->targets[k] + body_len - 1); + set_reg(&c->params[0], 5, 0, c->sw); + ir->temporary_variables_live_intervals = c->temps; + ir->temporary_variables_live_intervals_size = NTEMPS; + ir->next_temporary_variable = 4; + ir->parameters_live_intervals = c->params; + ir->next_parameter = 1; +} + +static void done(CaseIR *c) +{ + /* The intervals and the table live in CaseIR. */ + c->ir->temporary_variables_live_intervals = NULL; + c->ir->parameters_live_intervals = NULL; + utb_free(c->ir); +} + +static int load_src_is(CaseIR *c, int i, int imm) +{ + IROperand s = utb_src1(c->ir, i); + return s.tag == IROP_TAG_IMM32 && s.u.imm32 == imm; +} + +static CaseOpts plain(void) +{ + CaseOpts o = {0, 0, 0, I32, 0}; + return o; +} + +UT_TEST(test_region_merge_case_merges_bodies) +{ + CaseIR c; + build(&c, plain()); + UT_ASSERT_EQ(tcc_ir_region_merge(c.ir), 2); + int a = c.targets[0]; + UT_ASSERT_EQ(utb_op(c.ir, a), TCCIR_OP_LOAD); + UT_ASSERT_EQ(utb_vreg(utb_src1(c.ir, a)), utb_vreg(utb_temp(0, I32))); + for (int k = 1; k < 3; k++) + { + UT_ASSERT_EQ(utb_op(c.ir, c.targets[k]), TCCIR_OP_JUMP); + UT_ASSERT_EQ(utb_dest(c.ir, c.targets[k]).u.imm32, a); + } + done(&c); + return 0; +} + +UT_TEST(test_region_merge_case_constant_not_case_value) +{ + CaseIR c; + CaseOpts o = plain(); + o.delta = 1; + build(&c, o); + UT_ASSERT_EQ(tcc_ir_region_merge(c.ir), 0); + for (int k = 0; k < 3; k++) + UT_ASSERT(load_src_is(&c, c.targets[k], k + 1)); + done(&c); + return 0; +} + +UT_TEST(test_region_merge_case_entry_also_jumped_to) +{ + CaseIR c; + CaseOpts o = plain(); + o.goto_mid = 1; + build(&c, o); + /* Bodies 0 and 2 merge; body 1 is also reached with another selector. */ + UT_ASSERT_EQ(tcc_ir_region_merge(c.ir), 1); + UT_ASSERT(load_src_is(&c, c.targets[1], 1)); + UT_ASSERT_EQ(utb_op(c.ir, c.targets[2]), TCCIR_OP_JUMP); + UT_ASSERT_EQ(utb_vreg(utb_src1(c.ir, c.targets[0])), utb_vreg(utb_temp(0, I32))); + done(&c); + return 0; +} + +UT_TEST(test_region_merge_case_selector_redefined) +{ + CaseIR c; + CaseOpts o = plain(); + o.clobber = 1; + build(&c, o); + UT_ASSERT_EQ(tcc_ir_region_merge(c.ir), 0); + for (int k = 0; k < 3; k++) + UT_ASSERT(load_src_is(&c, c.targets[k] + 1, k)); + done(&c); + return 0; +} + +UT_TEST(test_region_merge_case_selector_register_shared) +{ + CaseIR c; + CaseOpts o = plain(); + o.body_reg = 4; /* the bodies' temps live in the selector's register */ + build(&c, o); + UT_ASSERT_EQ(tcc_ir_region_merge(c.ir), 0); + done(&c); + return 0; +} + +UT_TEST(test_region_merge_case_wide_selector) +{ + CaseIR c; + CaseOpts o = plain(); + o.sel_btype = I64; + build(&c, o); + UT_ASSERT_EQ(tcc_ir_region_merge(c.ir), 0); + done(&c); + return 0; +} + +UT_COVERS("region_merge"); diff --git a/tests/unit/arm/armv8m/test_opt_self_store.c b/tests/unit/arm/armv8m/test_opt_self_store.c new file mode 100644 index 000000000..852615671 --- /dev/null +++ b/tests/unit/arm/armv8m/test_opt_self_store.c @@ -0,0 +1,152 @@ +/* + * test_opt_self_store.c - suite for ir/self_store.c (self-store removal) + * + * tcc_ir_self_store deletes `StackLoc[X] <-- T [STORE]` when T was loaded from + * StackLoc[X] earlier in the same straight run and nothing between can write + * those bytes; the load goes too when the store was T's only use. The shape + * is what frame relayout leaves of a copy between two objects it placed on + * the same bytes. + */ + +#include "ir_build.h" + +#include "ut.h" + +int tcc_ir_self_store(TCCIRState *ir); + +#define I8 IROP_BTYPE_INT8 +#define I32 IROP_BTYPE_INT32 + +static IROperand slot(int off, int btype) +{ + return utb_stackoff(off, 1, 0, 0, btype); +} + +static TCCIRState *new_ir(int ntemps) +{ + TCCIRState *ir = utb_new(); + ir->next_temporary_variable = ntemps; + return ir; +} + +/* The two-word copy frame relayout leaves: all four instructions go. */ +UT_TEST(test_self_store_two_words_removed) +{ + TCCIRState *ir = new_ir(2); + int l0 = utb_emit(ir, TCCIR_OP_LOAD, utb_temp(0, I32), slot(-16, I32), UTB_NONE); + int l1 = utb_emit(ir, TCCIR_OP_LOAD, utb_temp(1, I32), slot(-12, I32), UTB_NONE); + int s0 = utb_emit(ir, TCCIR_OP_STORE, slot(-16, I32), utb_temp(0, I32), UTB_NONE); + int s1 = utb_emit(ir, TCCIR_OP_STORE, slot(-12, I32), utb_temp(1, I32), UTB_NONE); + + UT_ASSERT_EQ(tcc_ir_self_store(ir), 2); + UT_ASSERT_EQ(utb_op(ir, l0), TCCIR_OP_NOP); + UT_ASSERT_EQ(utb_op(ir, l1), TCCIR_OP_NOP); + UT_ASSERT_EQ(utb_op(ir, s0), TCCIR_OP_NOP); + UT_ASSERT_EQ(utb_op(ir, s1), TCCIR_OP_NOP); + utb_free(ir); + return 0; +} + +/* The loaded value is used again: the store goes, the load stays. */ +UT_TEST(test_self_store_load_kept_for_other_use) +{ + TCCIRState *ir = new_ir(2); + int l0 = utb_emit(ir, TCCIR_OP_LOAD, utb_temp(0, I32), slot(-16, I32), UTB_NONE); + int s0 = utb_emit(ir, TCCIR_OP_STORE, slot(-16, I32), utb_temp(0, I32), UTB_NONE); + int a = utb_emit(ir, TCCIR_OP_ADD, utb_temp(1, I32), utb_temp(0, I32), utb_imm(1, I32)); + + UT_ASSERT_EQ(tcc_ir_self_store(ir), 1); + UT_ASSERT_EQ(utb_op(ir, l0), TCCIR_OP_LOAD); + UT_ASSERT_EQ(utb_op(ir, s0), TCCIR_OP_NOP); + UT_ASSERT_EQ(utb_op(ir, a), TCCIR_OP_ADD); + utb_free(ir); + return 0; +} + +/* A store overlapping the slot between load and store: the bytes changed. */ +UT_TEST(test_self_store_overlapping_store_blocks) +{ + TCCIRState *ir = new_ir(1); + utb_emit(ir, TCCIR_OP_LOAD, utb_temp(0, I32), slot(-16, I32), UTB_NONE); + utb_emit(ir, TCCIR_OP_STORE, slot(-14, I8), utb_imm(5, I8), UTB_NONE); + int s0 = utb_emit(ir, TCCIR_OP_STORE, slot(-16, I32), utb_temp(0, I32), UTB_NONE); + + UT_ASSERT_EQ(tcc_ir_self_store(ir), 0); + UT_ASSERT_EQ(utb_op(ir, s0), TCCIR_OP_STORE); + utb_free(ir); + return 0; +} + +/* A store to disjoint bytes between them does not block. */ +UT_TEST(test_self_store_disjoint_store_allowed) +{ + TCCIRState *ir = new_ir(1); + utb_emit(ir, TCCIR_OP_LOAD, utb_temp(0, I32), slot(-16, I32), UTB_NONE); + int o = utb_emit(ir, TCCIR_OP_STORE, slot(-12, I32), utb_imm(5, I32), UTB_NONE); + int s0 = utb_emit(ir, TCCIR_OP_STORE, slot(-16, I32), utb_temp(0, I32), UTB_NONE); + + UT_ASSERT_EQ(tcc_ir_self_store(ir), 1); + UT_ASSERT_EQ(utb_op(ir, o), TCCIR_OP_STORE); + UT_ASSERT_EQ(utb_op(ir, s0), TCCIR_OP_NOP); + utb_free(ir); + return 0; +} + +/* A call between them may write the slot through an escaped address. */ +UT_TEST(test_self_store_call_blocks) +{ + TCCIRState *ir = new_ir(1); + utb_emit(ir, TCCIR_OP_LOAD, utb_temp(0, I32), slot(-16, I32), UTB_NONE); + utb_emit(ir, TCCIR_OP_FUNCCALLVOID, UTB_NONE, utb_imm(0, I32), utb_imm(0, I32)); + int s0 = utb_emit(ir, TCCIR_OP_STORE, slot(-16, I32), utb_temp(0, I32), UTB_NONE); + + UT_ASSERT_EQ(tcc_ir_self_store(ir), 0); + UT_ASSERT_EQ(utb_op(ir, s0), TCCIR_OP_STORE); + utb_free(ir); + return 0; +} + +/* A store through a pointer between them may alias the slot. */ +UT_TEST(test_self_store_pointer_store_blocks) +{ + TCCIRState *ir = new_ir(2); + utb_emit(ir, TCCIR_OP_LOAD, utb_temp(0, I32), slot(-16, I32), UTB_NONE); + utb_emit(ir, TCCIR_OP_STORE, utb_lval(utb_temp(1, I32)), utb_imm(7, I32), UTB_NONE); + int s0 = utb_emit(ir, TCCIR_OP_STORE, slot(-16, I32), utb_temp(0, I32), UTB_NONE); + + UT_ASSERT_EQ(tcc_ir_self_store(ir), 0); + UT_ASSERT_EQ(utb_op(ir, s0), TCCIR_OP_STORE); + utb_free(ir); + return 0; +} + +/* The store is a jump target: another path may arrive with other bytes. */ +UT_TEST(test_self_store_jump_target_blocks) +{ + TCCIRState *ir = new_ir(1); + utb_emit(ir, TCCIR_OP_LOAD, utb_temp(0, I32), slot(-16, I32), UTB_NONE); + int n = utb_emit(ir, TCCIR_OP_NOP, UTB_NONE, UTB_NONE, UTB_NONE); + int s0 = utb_emit(ir, TCCIR_OP_STORE, slot(-16, I32), utb_temp(0, I32), UTB_NONE); + ir->compact_instructions[n].is_jump_target = 1; + + UT_ASSERT_EQ(tcc_ir_self_store(ir), 0); + UT_ASSERT_EQ(utb_op(ir, s0), TCCIR_OP_STORE); + utb_free(ir); + return 0; +} + +/* Another slot, or a store wider than the load: not a self-copy. */ +UT_TEST(test_self_store_other_slot_or_wider_store_kept) +{ + TCCIRState *ir = new_ir(2); + utb_emit(ir, TCCIR_OP_LOAD, utb_temp(0, I32), slot(-16, I32), UTB_NONE); + int s0 = utb_emit(ir, TCCIR_OP_STORE, slot(-20, I32), utb_temp(0, I32), UTB_NONE); + utb_emit(ir, TCCIR_OP_LOAD, utb_temp(1, I8), slot(-8, I8), UTB_NONE); + int s1 = utb_emit(ir, TCCIR_OP_STORE, slot(-8, I32), utb_temp(1, I32), UTB_NONE); + + UT_ASSERT_EQ(tcc_ir_self_store(ir), 0); + UT_ASSERT_EQ(utb_op(ir, s0), TCCIR_OP_STORE); + UT_ASSERT_EQ(utb_op(ir, s1), TCCIR_OP_STORE); + utb_free(ir); + return 0; +} diff --git a/tests/unit/arm/armv8m/test_opt_utils.c b/tests/unit/arm/armv8m/test_opt_utils.c index ac0fe83d6..ef4f9c47e 100644 --- a/tests/unit/arm/armv8m/test_opt_utils.c +++ b/tests/unit/arm/armv8m/test_opt_utils.c @@ -49,7 +49,7 @@ /* Build a SYMREF callee operand whose token is `tok` (utb_set_tok_str maps * `tok` to a name for get_tok_str()-gated logic). Caller must have called - * utb_pools_init(ir) first. Same pattern as test_opt_float_branch.c / + * utb_pools_init(ir) first. Same pattern as * test_opt_promote_extra.c. */ static IROperand utb_callee_named(TCCIRState *ir, Sym *sym, int tok) { @@ -1679,8 +1679,7 @@ UT_TEST(test_get_constant_string_from_symref_negative_addend_returns_null) * NOTE ON TESTABILITY: both call sym_push2() / external_global_sym(), which * are link stubs in stubs.c that unconditionally return NULL (documented at * their definition and already noted by test_opt_constfold.c's - * float_narrowing / const_string_calls suspected-bug writeups for the SAME - * root cause). change_callee_sym() NULL-checks sym_push2()'s result + * const_string_calls suspected-bug writeups for the SAME root cause). change_callee_sym() NULL-checks sym_push2()'s result * (if (!ftype.ref) return 0;) so its "always returns 0 under this stub" * behavior IS itself a deterministic, sound oracle -- assert it explicitly * here so a change to that guard (e.g. losing the NULL check) is caught. diff --git a/tests/unit/arm/armv8m/test_ssa_opt_dce.c b/tests/unit/arm/armv8m/test_ssa_opt_dce.c index 0f514e5a8..caf6d0caa 100644 --- a/tests/unit/arm/armv8m/test_ssa_opt_dce.c +++ b/tests/unit/arm/armv8m/test_ssa_opt_dce.c @@ -1902,17 +1902,15 @@ UT_TEST(test_dce_var_liveness_struct_and_llong_defs_kept) } /* ======================================================================== - * BUG PIN: store to a dead VAR slot with a volatile lval source + * Store to a dead VAR slot with a volatile lval source * * The fused mem-to-mem copy `STORE lval(V0slot) <- lval(V1)` embodies a * READ of V1. When V1 is volatile, that read is a mandated side effect, so - * the store must survive even though V0 is never read. var_liveness has the - * guard (vl_removable rejects volatile sources); dce_dead_var_stores does - * not, and runs first. This test pins the CURRENT (buggy) outcome; flip the - * assertions to "store kept" once fixed. See docs/bugs/. + * the store must survive even though V0 is never read (dce_instr_reads_volatile + * in every dead-store killer, as vl_removable already did for var_liveness). * ======================================================================== */ -UT_TEST(test_dce_dead_var_store_volatile_lval_src_bug) +UT_TEST(test_dce_dead_var_store_volatile_lval_src) { ssa_ctx c = ssa_ctx_new(/*blocks=*/1, /*temps=*/1); ssa_ctx_init_manual(&c); @@ -1926,10 +1924,8 @@ UT_TEST(test_dce_dead_var_store_volatile_lval_src_bug) tcc_ir_ssa_opt_init(c.ctx, c.ir, c.ssa, c.cfg); int changed = run_dce_with_opt(c.ctx); - /* Buggy behavior: the store (and with it the volatile read of V1) is - * eliminated. Correct behavior: changed == 0, store kept. */ - UT_ASSERT(changed >= 1); - UT_ASSERT_EQ(utb_op(c.ir, store_i), TCCIR_OP_NOP); + UT_ASSERT_EQ(changed, 0); + UT_ASSERT_EQ(utb_op(c.ir, store_i), TCCIR_OP_STORE); ssa_ctx_free(&c); return 0; diff --git a/tests/unit/arm/armv8m/test_ssa_opt_load_cse.c b/tests/unit/arm/armv8m/test_ssa_opt_load_cse.c index ae9c1d26b..3bfc6fd58 100644 --- a/tests/unit/arm/armv8m/test_ssa_opt_load_cse.c +++ b/tests/unit/arm/armv8m/test_ssa_opt_load_cse.c @@ -1691,18 +1691,14 @@ UT_TEST(test_sstore_adjacent_narrow_store_survives) } /* ======================================================================== - * Pinned bugs + * Pointer-deref store sources * ======================================================================== */ -/* Regression lock for docs/bugs.md "load_cse: stack store of a pointer-deref - * source tracks the pointer as the stored value". - * `StackLoc[0] <- *t1` records t1 (the ADDRESS) in the sstore slot because - * the tracker at load_cse.c:1031 lacks the !src.is_lval guard its siblings - * have (load_cse.c:302, :1104, :1181). The later LOAD is then rewritten to - * `ASSIGN t1` — the pointer, not the pointee. Correct behavior: do not track - * (load stays a load). This pins the CURRENT (buggy) result; flip the - * assertions once fixed. */ -UT_TEST(test_stack_store_lval_vreg_src_tracked_bug) +/* `StackLoc[0] <- *t1` stores what t1 points at. The direct stack-slot + * tracker once recorded t1 itself (the ADDRESS) as the slot's value, and the + * later LOAD became `ASSIGN t1` -- the pointer, not the pointee. The load + * must stay a load. */ +UT_TEST(test_stack_store_lval_vreg_src_not_tracked) { setup_tcc_state(); ssa_ctx c = ssa_ctx_new(1, 4); @@ -1718,23 +1714,18 @@ UT_TEST(test_stack_store_lval_vreg_src_tracked_bug) ssa_ctx_build_ssa_plain(&c); ssa_ctx_rebuild(&c); - int changed = ssa_opt_load_cse(c.ctx); - UT_ASSERT(changed >= 1); - UT_ASSERT_EQ(utb_op(c.ir, load_i), TCCIR_OP_ASSIGN); - UT_ASSERT_EQ(utb_vreg(utb_src1(c.ir, load_i)), VR_TMP(1)); + ssa_opt_load_cse(c.ctx); + UT_ASSERT_EQ(utb_op(c.ir, load_i), TCCIR_OP_LOAD); ssa_ctx_free(&c); teardown_tcc_state(); return 0; } -/* Regression lock for the same root cause at the twin call site - * load_cse.c:1076 (TEMP-indir stack store `*T <- val` resolving to - * LEA(StackLoc[N])): an is_lval source is recorded as the stored value. - * `t1 = &StackLoc[0]; *t1 = *t0; t2 = LOAD(StackLoc[0])` currently becomes - * `t2 = t0` (the pointer). Correct behavior: do not track. Pins the CURRENT - * (buggy) result; flip the assertions once fixed. */ -UT_TEST(test_temp_indir_store_lval_src_tracked_bug) +/* The twin site, a TEMP-indir stack store `*T <- val` resolving to + * LEA(StackLoc[N]): `t1 = &StackLoc[0]; *t1 = *t0; t2 = LOAD(StackLoc[0])` + * once became `t2 = t0` (the pointer). The load must stay a load. */ +UT_TEST(test_temp_indir_store_lval_src_not_tracked) { setup_tcc_state(); ssa_ctx c = ssa_ctx_new(1, 4); @@ -1748,10 +1739,8 @@ UT_TEST(test_temp_indir_store_lval_src_tracked_bug) ssa_ctx_build_ssa_plain(&c); ssa_ctx_rebuild(&c); - int changed = ssa_opt_load_cse(c.ctx); - UT_ASSERT(changed >= 1); - UT_ASSERT_EQ(utb_op(c.ir, load_i), TCCIR_OP_ASSIGN); - UT_ASSERT_EQ(utb_vreg(utb_src1(c.ir, load_i)), VR_TMP(0)); + ssa_opt_load_cse(c.ctx); + UT_ASSERT_EQ(utb_op(c.ir, load_i), TCCIR_OP_LOAD); ssa_ctx_free(&c); teardown_tcc_state(); diff --git a/tests/unit/arm/armv8m/test_tccgen.c b/tests/unit/arm/armv8m/test_tccgen.c index b86b9131a..d7644dcee 100644 --- a/tests/unit/arm/armv8m/test_tccgen.c +++ b/tests/unit/arm/armv8m/test_tccgen.c @@ -158,6 +158,27 @@ void tcc_ir_set_original_offset(TCCIRState *ir, int vreg, int offset) (void)offset; } +int tcc_ir_frame_alloc(int loc, int size, int mask) +{ + return (loc - size) & mask; +} + +int tcc_ir_frame_alloc_arg_copy(int loc, int size, int mask) +{ + return (loc - size) & mask; +} + +int tcc_ir_frame_alloc_ret_temp(int loc, int size, int mask) +{ + return (loc - size) & mask; +} + +void tcc_ir_frame_note_sret_call(int call_id, int size) +{ + (void)call_id; + (void)size; +} + void tcc_ir_set_float_type(TCCIRState *ir, int vreg, int is_float, int is_double) { (void)ir; @@ -1335,10 +1356,10 @@ UT_TEST(test_sym_free) reset_symbol_state(); s = sym_push2(&global_stack, SYM_FIRST_ANOM, VT_INT, 0); UT_ASSERT(s != NULL); - s->const_init_data = dummy_data; + sym_facts(s)->const_init_data = dummy_data; sym_free(s); global_stack = NULL; - UT_ASSERT(s->const_init_data == NULL); + UT_ASSERT(s->facts == NULL); UT_ASSERT_EQ((unsigned int)s->v, 0xDEADBEEFU); return 0; } @@ -2401,15 +2422,18 @@ UT_TEST(test_find_sv_const_init) { static unsigned char data[8] = {1, 2, 3, 4, 5, 6, 7, 8}; Sym s; + SymLocalFacts sf; SValue sv; reset_symbol_state(); memset(&s, 0, sizeof(s)); + memset(&sf, 0, sizeof(sf)); s.v = TOK_IDENT + 1; /* a named local */ s.c = 0x20; /* stack offset */ - s.const_init_data = data; - s.const_init_valid = 1; - s.const_init_size = 8; + s.facts = &sf; + sf.const_init_data = data; + sf.const_init_valid = 1; + sf.const_init_size = 8; s.prev = NULL; local_stack = &s; @@ -3533,6 +3557,7 @@ UT_TEST(test_svalue_get_conservative_max_u64) { SValue sv; Sym local; + SymLocalFacts lf = {0}; unsigned long long m; tcc_state = NULL; /* keep the vreg-fact fallback inert */ @@ -3568,8 +3593,9 @@ UT_TEST(test_svalue_get_conservative_max_u64) local.type.t = VT_INT; local.v = TOK_IDENT + 1; local.c = 0x30; - local.objsize_max_valid = 1; - local.objsize_max_value = 500; + local.facts = &lf; + lf.objsize_max_valid = 1; + lf.objsize_max_value = 500; local.prev = NULL; local_stack = &local; @@ -3581,7 +3607,7 @@ UT_TEST(test_svalue_get_conservative_max_u64) UT_ASSERT_EQ((int)m, 500); /* an unbounded local yields nothing */ - local.objsize_max_valid = 0; + lf.objsize_max_valid = 0; UT_ASSERT(!svalue_get_conservative_max_u64(&sv, &m)); local_stack = NULL; @@ -3592,6 +3618,7 @@ UT_TEST(test_svalue_get_conservative_string_bytes_u64) { SValue sv; Sym local; + SymLocalFacts lf = {0}; unsigned long long m; tcc_state = NULL; @@ -3603,8 +3630,9 @@ UT_TEST(test_svalue_get_conservative_string_bytes_u64) local.type.t = VT_INT; local.v = TOK_IDENT + 1; local.c = 0x40; - local.objsize_strlen_valid = 1; - local.objsize_strlen_value = 9; + local.facts = &lf; + lf.objsize_strlen_valid = 1; + lf.objsize_strlen_value = 9; local.prev = NULL; local_stack = &local; @@ -3616,7 +3644,7 @@ UT_TEST(test_svalue_get_conservative_string_bytes_u64) UT_ASSERT_EQ((int)m, 9); /* without a strlen fact there is no bound */ - local.objsize_strlen_valid = 0; + lf.objsize_strlen_valid = 0; UT_ASSERT(!svalue_get_conservative_string_bytes_u64(&sv, &m)); /* a bare (non-local, non-string) constant has no string bound */ diff --git a/tests/unit/arm/armv8m/test_tccls.c b/tests/unit/arm/armv8m/test_tccls.c index aab5bad54..798b825a8 100644 --- a/tests/unit/arm/armv8m/test_tccls.c +++ b/tests/unit/arm/armv8m/test_tccls.c @@ -117,6 +117,38 @@ UT_TEST(test_compact_stack_locations_keeps_negative_spill_base) return 0; } +UT_TEST(test_compact_stack_locations_weighted_puts_hottest_nearest_sp) +{ + LSLiveIntervalState ls; + ls_init(&ls); + + tcc_ls_add_live_interval(&ls, VR_TMP(0), 0, 1, 0, 0, LS_REG_TYPE_INT, 0, -1); + tcc_ls_add_live_interval(&ls, VR_TMP(1), 0, 1, 0, 0, LS_REG_TYPE_INT, 0, -1); + tcc_ls_add_live_interval(&ls, VR_TMP(2), 0, 1, 0, 0, LS_REG_TYPE_INT, 0, -1); + tcc_ls_add_live_interval(&ls, VR_TMP(3), 0, 1, 0, 0, LS_REG_TYPE_INT, 0, -1); + tcc_ls_add_live_interval(&ls, VR_TMP(4), 0, 1, 0, 0, LS_REG_TYPE_LLONG, 0, -1); + + ls.intervals[0].stack_location = 100; + ls.intervals[1].stack_location = 200; + ls.intervals[2].stack_location = 300; + ls.intervals[3].stack_location = 200; /* shares a slot: weights add up */ + ls.intervals[4].stack_location = 400; + const uint32_t weights[] = {1, 10, 5, 3, 20}; + + tcc_ls_compact_stack_locations_weighted(&ls, 0, weights); + + /* Per byte, coldest placed first: slot 100 = 1/4, 300 = 5/4, 400 = 20/8, + * 200 = 13/4. */ + UT_ASSERT_EQ((int)ls.intervals[0].stack_location, -4); + UT_ASSERT_EQ((int)ls.intervals[2].stack_location, -8); + UT_ASSERT_EQ((int)ls.intervals[4].stack_location, -16); + UT_ASSERT_EQ((int)ls.intervals[1].stack_location, -20); + UT_ASSERT_EQ((int)ls.intervals[3].stack_location, -20); + + tcc_ls_deinitialize(&ls); + return 0; +} + /* -------------------------------------------------------------- liveness */ UT_TEST(test_compute_live_regs_counts_integer_intervals_only) diff --git a/tests/unit/arm/armv8m/test_tccyaff.c b/tests/unit/arm/armv8m/test_tccyaff.c index 753b1ac7d..65b541d34 100644 --- a/tests/unit/arm/armv8m/test_tccyaff.c +++ b/tests/unit/arm/armv8m/test_tccyaff.c @@ -247,6 +247,7 @@ static uint8_t *ut_build_yaff(const char *objname, const char *symname, uint32_t { YaffHeader h = {0}; memcpy(h.magic, "YAFF", 4); + h.yaff_version = YAFF_VERSION; h.alignment = 4; h.exported_symbols_amount = 2; /* sentinel at index 0 + one real symbol */ @@ -398,6 +399,32 @@ UT_TEST(test_load_yaff_rejects_bad_magic) return 0; } +/* A library from an older toolchain: the loader would refuse it at run time, + * so the link refuses it now. */ +UT_TEST(test_load_yaff_rejects_other_version) +{ + size_t size; + uint8_t *buf = ut_build_yaff("libold.yaff", "exported_fn", 0x1234, &size); + UT_ASSERT(buf != NULL); + ((YaffHeader *)buf)->yaff_version = YAFF_VERSION - 1; + + char path[] = "/tmp/tccyaff_ut_oldver_XXXXXX"; + int fd = mkstemp(path); + UT_ASSERT(fd >= 0); + UT_ASSERT_EQ(write(fd, buf, size), (ssize_t)size); + lseek(fd, 0, SEEK_SET); + + memset(tcc_state, 0, sizeof(TCCState)); + tcc_state->dynsymtab_section = ut_make_dynsymtab(tcc_state); + UT_ASSERT(tcc_load_yaff(tcc_state, fd, path, 0) != 0); + UT_ASSERT_EQ(tcc_state->nb_yaff_libs, 0); + + close(fd); + unlink(path); + tcc_free(buf); + return 0; +} + UT_TEST(test_load_yaff_and_resolve) { size_t size; @@ -590,6 +617,8 @@ UT_TEST(test_output_yaff_minimal_header) UT_ASSERT_EQ(h.version_minor, 0u); UT_ASSERT_EQ(h.stack_size, 0xFFFFFFFFu); UT_ASSERT_EQ(h.heap_size, 0xFFFFFFFFu); + UT_ASSERT_EQ(h.data_alignment, 8u); /* sections default to 8 */ + UT_ASSERT_EQ(h.data_alignment_offset, 0u); /* Object name follows the header immediately. */ char name[32]; @@ -603,6 +632,36 @@ UT_TEST(test_output_yaff_minimal_header) return 0; } +/* The per-process region starts after the shared .rodata prefix (0x1108) and + * holds a 64-aligned .bss: the loader must start it 8 bytes past a multiple of + * 64, or the .bss object lands where the linker did not put it. */ +UT_TEST(test_output_yaff_data_alignment) +{ + ut_yaff_setup_minimal_output_state(); + tcc_state->share_rodata = 1; + bss_section->sh_addr = 0x1140; + bss_section->sh_size = 16; + bss_section->sh_addralign = 64; + tcc_state->got->sh_addr = 0x1150; + tcc_state->plt->sh_addr = 0x1190; + + char path[] = "/tmp/tccyaff_ut_out_align_XXXXXX"; + FILE *f = ut_yaff_open_temp(path); + + UT_ASSERT_EQ(tcc_output_yaff(tcc_state, f, "align.yaff"), 0); + + YaffHeader h; + ut_yaff_read_header(f, &h); + UT_ASSERT_EQ(h.const_rodata_length, 8u); + UT_ASSERT_EQ(h.data_alignment, 64u); + UT_ASSERT_EQ(h.data_alignment_offset, 8u); + + fclose(f); + unlink(path); + ut_yaff_teardown_output_state(); + return 0; +} + /* Read the architecture section the header points at. Offset 0 lands inside * the header, so it also means "absent"; the callers assert it is not 0. */ static void ut_yaff_read_arch_section(FILE *f, const YaffHeader *h, YaffArchSection *out) From a5cae5834b9b97df0114338ce0929d11fa196ebb Mon Sep 17 00:00:00 2001 From: Mateusz Stadnik Date: Thu, 1 Oct 2026 22:00:39 +0200 Subject: [PATCH 2/4] unity groups by directory, sym facts leaks, barrel-shift join fix - unity: groups follow source directories, with large ones cut by hand in source/unity/groups.txt, instead of packing by line count; predefine headers are generated before gen_unity. - Free Sym facts that outlive their scope or belong to nested-function capture syms (ASan leak in CI building lib/armeabi.c). - barrel_shift: require a single def and no jump target between the shift and its consumer; it runs pre-SSA and miscompiled ctags. Test 608_barrel_shift_join_multidef. Co-Authored-By: Claude Opus 5.5 --- Makefile | 11 +- scripts/gen_unity.py | 149 +++- source/frontend/gen/expr/primary.c | 1 + source/frontend/gen/gen_priv.h | 1 + source/frontend/gen/nested/nested.c | 20 + source/frontend/gen/sym/symtab.c | 30 +- source/include/tcc.h | 1 + source/opt/flat/fusion/barrel_shift.c | 50 +- source/unity/{arm_01.c => arm.c} | 0 source/unity/{arm_02.c => arm_s1.c} | 2 +- source/unity/{thumb_01.c => arm_thumb.c} | 28 +- source/unity/arm_thumb_call.c | 5 + source/unity/{thumb_04.c => arm_thumb_emit.c} | 10 +- source/unity/{thumb_03.c => arm_thumb_gen.c} | 2 +- source/unity/{thumb_02.c => arm_thumb_s1.c} | 56 +- source/unity/frontend_01.c | 11 - source/unity/frontend_05.c | 16 - .../{frontend_02.c => frontend_gen_builtin.c} | 14 +- source/unity/frontend_gen_core.c | 5 + source/unity/frontend_gen_decl.c | 6 + .../{frontend_03.c => frontend_gen_expr.c} | 7 +- .../{frontend_04.c => frontend_gen_inline.c} | 8 - source/unity/frontend_gen_op.c | 15 + source/unity/frontend_gen_stmt.c | 5 + source/unity/frontend_gen_store.c | 6 + source/unity/frontend_gen_sym.c | 4 + .../{frontend_06.c => frontend_gen_type.c} | 4 - source/unity/frontend_gen_value.c | 5 + source/unity/groups.txt | 61 ++ source/unity/{ir_01.c => ir.c} | 40 +- source/unity/ir_02.c | 20 - source/unity/ir_frame.c | 16 + source/unity/{ir_05.c => ir_gen.c} | 24 +- source/unity/{ir_03.c => ir_regalloc.c} | 14 +- source/unity/{ir_04.c => ir_regalloc_ssa.c} | 21 +- source/unity/{machine_01.c => machine_s1.c} | 0 source/unity/{memory_01.c => memory_s1.c} | 0 source/unity/obj_02.c | 11 - source/unity/{obj_01.c => obj_elf_s1.c} | 11 +- source/unity/{obj_03.c => obj_s1.c} | 11 +- source/unity/opt_01.c | 74 -- source/unity/opt_02.c | 56 -- source/unity/opt_04.c | 62 -- source/unity/opt_05.c | 33 - source/unity/opt_07.c | 33 - source/unity/opt_08.c | 49 -- source/unity/opt_09.c | 25 - source/unity/opt_12.c | 44 -- source/unity/opt_14.c | 34 - source/unity/opt_15.c | 55 -- source/unity/opt_analysis.c | 9 + source/unity/opt_engine.c | 28 + source/unity/opt_flat_cfg.c | 18 + source/unity/{opt_11.c => opt_flat_dce.c} | 57 +- source/unity/{opt_10.c => opt_flat_fusion.c} | 66 +- source/unity/opt_flat_ipa.c | 18 + source/unity/{opt_13.c => opt_flat_loop.c} | 56 +- source/unity/opt_flat_loop_iv.c | 19 + source/unity/{opt_16.c => opt_flat_memory.c} | 33 +- source/unity/opt_flat_memory_cse.c | 24 + source/unity/opt_flat_memory_dse.c | 57 ++ source/unity/opt_flat_memory_fwd.c | 32 + source/unity/opt_flat_scalar.c | 29 + .../{opt_06.c => opt_flat_scalar_const.c} | 40 +- source/unity/opt_flat_scalar_known.c | 19 + source/unity/opt_ra.c | 9 + source/unity/opt_ssa_cfg.c | 23 + source/unity/opt_ssa_dce.c | 38 + source/unity/opt_ssa_engine.c | 10 + source/unity/opt_ssa_loop.c | 31 + source/unity/opt_ssa_memory.c | 33 + source/unity/{opt_03.c => opt_ssa_scalar.c} | 27 +- source/unity/opt_ssa_scalar_prop.c | 7 + source/unity/opt_ssa_string.c | 9 + source/unity/opt_util.c | 29 + source/unity/unity.mk | 684 +++++++++--------- .../ir_tests/608_barrel_shift_join_multidef.c | 65 ++ .../608_barrel_shift_join_multidef.expect | 4 + tests/ir_tests/test_qemu.py | 2 + 79 files changed, 1460 insertions(+), 1182 deletions(-) rename source/unity/{arm_01.c => arm.c} (100%) rename source/unity/{arm_02.c => arm_s1.c} (100%) rename source/unity/{thumb_01.c => arm_thumb.c} (79%) create mode 100644 source/unity/arm_thumb_call.c rename source/unity/{thumb_04.c => arm_thumb_emit.c} (56%) rename source/unity/{thumb_03.c => arm_thumb_gen.c} (70%) rename source/unity/{thumb_02.c => arm_thumb_s1.c} (100%) delete mode 100644 source/unity/frontend_01.c delete mode 100644 source/unity/frontend_05.c rename source/unity/{frontend_02.c => frontend_gen_builtin.c} (60%) create mode 100644 source/unity/frontend_gen_core.c create mode 100644 source/unity/frontend_gen_decl.c rename source/unity/{frontend_03.c => frontend_gen_expr.c} (62%) rename source/unity/{frontend_04.c => frontend_gen_inline.c} (56%) create mode 100644 source/unity/frontend_gen_op.c create mode 100644 source/unity/frontend_gen_stmt.c create mode 100644 source/unity/frontend_gen_store.c create mode 100644 source/unity/frontend_gen_sym.c rename source/unity/{frontend_06.c => frontend_gen_type.c} (59%) create mode 100644 source/unity/frontend_gen_value.c create mode 100644 source/unity/groups.txt rename source/unity/{ir_01.c => ir.c} (64%) delete mode 100644 source/unity/ir_02.c create mode 100644 source/unity/ir_frame.c rename source/unity/{ir_05.c => ir_gen.c} (61%) rename source/unity/{ir_03.c => ir_regalloc.c} (54%) rename source/unity/{ir_04.c => ir_regalloc_ssa.c} (93%) rename source/unity/{machine_01.c => machine_s1.c} (100%) rename source/unity/{memory_01.c => memory_s1.c} (100%) delete mode 100644 source/unity/obj_02.c rename source/unity/{obj_01.c => obj_elf_s1.c} (67%) rename source/unity/{obj_03.c => obj_s1.c} (94%) delete mode 100644 source/unity/opt_01.c delete mode 100644 source/unity/opt_02.c delete mode 100644 source/unity/opt_04.c delete mode 100644 source/unity/opt_05.c delete mode 100644 source/unity/opt_07.c delete mode 100644 source/unity/opt_08.c delete mode 100644 source/unity/opt_09.c delete mode 100644 source/unity/opt_12.c delete mode 100644 source/unity/opt_14.c delete mode 100644 source/unity/opt_15.c create mode 100644 source/unity/opt_analysis.c create mode 100644 source/unity/opt_engine.c create mode 100644 source/unity/opt_flat_cfg.c rename source/unity/{opt_11.c => opt_flat_dce.c} (57%) rename source/unity/{opt_10.c => opt_flat_fusion.c} (61%) create mode 100644 source/unity/opt_flat_ipa.c rename source/unity/{opt_13.c => opt_flat_loop.c} (79%) create mode 100644 source/unity/opt_flat_loop_iv.c rename source/unity/{opt_16.c => opt_flat_memory.c} (52%) create mode 100644 source/unity/opt_flat_memory_cse.c create mode 100644 source/unity/opt_flat_memory_dse.c create mode 100644 source/unity/opt_flat_memory_fwd.c create mode 100644 source/unity/opt_flat_scalar.c rename source/unity/{opt_06.c => opt_flat_scalar_const.c} (55%) create mode 100644 source/unity/opt_flat_scalar_known.c create mode 100644 source/unity/opt_ra.c create mode 100644 source/unity/opt_ssa_cfg.c create mode 100644 source/unity/opt_ssa_dce.c create mode 100644 source/unity/opt_ssa_engine.c create mode 100644 source/unity/opt_ssa_loop.c create mode 100644 source/unity/opt_ssa_memory.c rename source/unity/{opt_03.c => opt_ssa_scalar.c} (65%) create mode 100644 source/unity/opt_ssa_scalar_prop.c create mode 100644 source/unity/opt_ssa_string.c create mode 100644 source/unity/opt_util.c create mode 100644 tests/ir_tests/608_barrel_shift_join_multidef.c create mode 100644 tests/ir_tests/608_barrel_shift_join_multidef.expect diff --git a/Makefile b/Makefile index 39df04dc5..293fee3da 100644 --- a/Makefile +++ b/Makefile @@ -372,8 +372,10 @@ TCC_CORE_DEPS = $(wildcard *.h) \ # gcc and tcc. That matters on a Raspberry Pi and far more on the device. # # The groups live in source/unity/ (unity.mk plus one .c per group), generated by -# scripts/gen_unity.py so that no two members of a group can see each other's -# file-local names: see the script for the rules. Sources the manifest does not +# scripts/gen_unity.py from source/unity/groups.txt: a source joins its +# directory's group unless groups.txt names another one, and the script refuses +# a group whose members could see each other's file-local names (see it for the +# rules). Sources the manifest does not # cover -- new files, or files it keeps apart on purpose -- still build as their # own TU, so a stale manifest costs speed, never correctness. # @@ -564,7 +566,10 @@ unity: .PHONY: unity # What scripts/gen_unity.py needs: the compile line and each library's sources. -unity-info: +# It preprocesses every source, so the generated headers they #include must +# exist first -- on a fresh checkout (CI runs it straight after configure) +# nothing has built them yet. +unity-info: $(TCCDEFS_H) tccdefs_table_.h @echo 'CC $(CC) $(DEFINES) $(CFLAGS) $(OPT_PIPELINE_INC) $(BACKEND_GENERATORS_INC)' @$(foreach l,$(UNITY_LIB_NAMES),echo 'LIB $l $(UNITY_SRC_$l)';) .PHONY: unity-info diff --git a/scripts/gen_unity.py b/scripts/gen_unity.py index f699dccab..21afd178e 100755 --- a/scripts/gen_unity.py +++ b/scripts/gen_unity.py @@ -25,8 +25,15 @@ * tcc.h picks TCC_STATE_VAR / TCC_SET_STATE / _tcc_error by whether the includer defined USING_GLOBALS first; a later member cannot change that, so sources only share a group when they agree on it. - * Groups stay within one library and under --max-lines source lines, and a - source bigger than that stands alone. + * Groups stay within one library. + +Which sources share a group is decided by hand, not packed by size: every source +joins the group of its directory (source/opt/flat/cfg/*.c -> opt_flat_cfg) unless +source/unity/groups.txt names it for another group, which is how a directory too +big for one TU is cut into logical pieces. So a file growing or a new file +arriving never moves any other file between groups. A source the macro rules or +the proof below keep apart builds as its own TU; a clash of file-local names +between two members of one group is an error to resolve in groups.txt. Usage (in a configured tree): scripts/gen_unity.py # rewrite source/unity/ @@ -37,6 +44,7 @@ import collections import concurrent.futures import difflib +import fnmatch import os import re import shlex @@ -46,6 +54,7 @@ TOP = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) UNITY_DIR = os.path.join('source', 'unity') +MANIFEST = os.path.join(UNITY_DIR, 'groups.txt') # Macros whose effect the grouping key already captures (see mode_key). tcc.h # consumes USING_GLOBALS and #undefs it, so it is local to the includer. @@ -288,52 +297,114 @@ def analyse(src, flags): ) -def plan(libs, info, max_lines, ungroupable=()): +def read_manifest(): + """[(group, [glob, ...])] from groups.txt, in file order.""" + rules, cur = [], None + with open(os.path.join(TOP, MANIFEST)) as f: + for n, line in enumerate(f, 1): + text = line.split('#', 1)[0].rstrip() + if not text: + continue + if text[0].isspace(): + if cur is None: + sys.exit('gen_unity: %s:%d: continuation without a group' % (MANIFEST, n)) + cur[1].extend(text.split()) + continue + name, sep, pats = text.partition(':') + name = name.strip() + if not sep or not re.fullmatch(r'\w+', name): + sys.exit('gen_unity: %s:%d: expected "group: pattern ..."' % (MANIFEST, n)) + cur = (name, pats.split()) + rules.append(cur) + return rules + + +def group_of(src, rules, used): + rel = os.path.relpath(src, 'source') + for name, pats in rules: + for pat in pats: + if fnmatch.fnmatchcase(rel, pat): + used.add((name, pat)) + return name + d = os.path.dirname(rel) + if d.startswith('backend/arch/'): + d = d[len('backend/arch/'):] + return d.replace('/', '_').replace('-', '_') or 'source' + + +def clashes(i, ch): + """Why source info *i* cannot join the members accumulated in *ch*.""" + local = (i['local_defs'] | i['local_undefs']) - MODE_MACROS + own, hdr = i['names'], i['hdr_names'] + loose = own - hdr + why = [] + if ch['mode'] is not None and i['mode'] != ch['mode']: + why.append('USING_GLOBALS differs') + for what, names in (('both define', own & ch['names']), + ('declared by the other\'s headers', (loose & ch['hdr']) | (ch['loose'] & hdr)), + ('macros the other\'s headers define', (local & ch['hdr_macros']) + | (ch['local'] & i['hdr_macros']))): + if names: + why.append('%s %s' % (what, ' '.join(sorted(names)[:6]))) + return why + + +def plan(libs, info, ungroupable=()): + rules = read_manifest() + used = set() kept_apart = {} - groups = collections.OrderedDict() + errors = [] + found = collections.OrderedDict() # group -> (lib, [members]) for lib, srcs in libs.items(): - chunks = collections.defaultdict(list) # mode -> [chunk] for src in srcs: i = info[src] - # A file that redefines a macro one of its own headers defines stays - # alone; otherwise it only has to avoid the headers of its group. local = (i['local_defs'] | i['local_undefs']) - MODE_MACROS clash = local & i['hdr_macros'] if clash: + # #undef'ing it would take a header's definition from the + # members after it, and the guarded header would not restore it. kept_apart[src] = 'redefines header macros ' + ' '.join(sorted(clash)) continue if src in ungroupable: kept_apart[src] = 'expands differently inside a group' continue - if i['lines'] > max_lines: - kept_apart[src] = '%d lines' % i['lines'] + g = group_of(src, rules, used) + # A file's USING_GLOBALS choice is its own and fixed, so splitting a + # group by it keeps membership stable. (The group checks below + # still compare the mode, in case the analysis ever disagrees.) + # (Whether it does is read from what tcc.h settled on, since a + # header may define USING_GLOBALS for it.) Most sources do. + if 'tcc_state' not in (i['mode'][0] or ''): + g += '_s1' + if g in found and found[g][0] != lib: + errors.append('group %s mixes libraries %s and %s (%s)' % (g, found[g][0], lib, src)) continue - # A file-local name clashes with another member's file-local names, - # and with anything the headers of another member declare -- unless - # this file's own headers declare it too, which it already agrees with. - own, hdr = i['names'], i['hdr_names'] - loose = own - hdr - for ch in chunks[i['mode']]: - if (ch['lines'] + i['lines'] <= max_lines and not (own & ch['names']) - and not (loose & ch['hdr']) and not (ch['loose'] & hdr) - and not (local & ch['hdr_macros']) and not (ch['local'] & i['hdr_macros'])): - break - else: - ch = dict(lines=0, names=set(), hdr=set(), loose=set(), members=[], - local=set(), hdr_macros=set()) - chunks[i['mode']].append(ch) + found.setdefault(g, (lib, []))[1].append(src) + groups = collections.OrderedDict() + for g in sorted(found): + lib, members = found[g] + members.sort() + ch = dict(mode=None, names=set(), hdr=set(), loose=set(), local=set(), hdr_macros=set()) + for src in members: + i = info[src] + why = clashes(i, ch) + if why: + errors.append('group %s: %s clashes with an earlier member: %s' % (g, src, '; '.join(why))) + local = (i['local_defs'] | i['local_undefs']) - MODE_MACROS + ch['mode'] = i['mode'] + ch['names'] |= i['names'] + ch['hdr'] |= i['hdr_names'] + ch['loose'] |= i['names'] - i['hdr_names'] ch['local'] |= local ch['hdr_macros'] |= i['hdr_macros'] - ch['lines'] += i['lines'] - ch['names'] |= own - ch['hdr'] |= hdr - ch['loose'] |= loose - ch['members'].append(src) - # number the groups in source order, so the output is stable - found = [ch for cs in chunks.values() for ch in cs if len(ch['members']) > 1] - found.sort(key=lambda ch: srcs.index(ch['members'][0])) - for k, ch in enumerate(found): - groups['%s_%02d' % (lib, k + 1)] = (lib, ch['members']) + if len(members) > 1: + groups[g] = (lib, members) + for name, pats in rules: + for pat in pats: + if (name, pat) not in used: + errors.append('%s: %s: %s matches no library source' % (MANIFEST, name, pat)) + if errors: + sys.exit('gen_unity: fix source/unity/groups.txt:\n ' + '\n '.join(errors)) return groups, kept_apart @@ -407,11 +478,11 @@ def run(job): return set().union(*ex.map(run, jobs)) if jobs else set() -def render(groups, kept_apart, info, max_lines): +def render(groups, kept_apart, info): files = {} mk = [HEADER.replace('/*', '#').replace(' */', ''), '# Unity-build groups: see the "Unity build" block in the Makefile.\n', - '# Budget %d source lines per group.\n' % max_lines] + '# Membership comes from source/unity/groups.txt.\n'] for src in sorted(kept_apart): mk.append('# own TU: %s (%s)\n' % (src, kept_apart[src])) by_lib = collections.OrderedDict() @@ -431,8 +502,6 @@ def main(): ap = argparse.ArgumentParser(description=__doc__.split('\n\n')[0]) ap.add_argument('--check', action='store_true', help='report whether source/unity/ is up to date; write nothing') - ap.add_argument('--max-lines', type=int, default=6000, - help='source-line budget per group (default %(default)s)') ap.add_argument('--target', default='armv8m', help='CROSS_TARGET to ask make about') ap.add_argument('-v', '--verbose', action='store_true') args = ap.parse_args() @@ -456,18 +525,20 @@ def main(): # and keep out any member whose own code expands differently than alone. ungroupable = set() while True: - groups, kept_apart = plan(libs, info, args.max_lines, ungroupable) + groups, kept_apart = plan(libs, info, ungroupable) bad = verify(groups, info, lib_flags) if not bad: break ungroupable |= bad - files = render(groups, kept_apart, info, args.max_lines) + files = render(groups, kept_apart, info) old = {} udir = os.path.join(TOP, UNITY_DIR) if os.path.isdir(udir): for f in os.listdir(udir): p = os.path.join(UNITY_DIR, f) + if p == MANIFEST: + continue with open(os.path.join(TOP, p)) as fh: old[p] = fh.read() stale = sorted(p for p in set(old) | set(files) if old.get(p) != files.get(p)) diff --git a/source/frontend/gen/expr/primary.c b/source/frontend/gen/expr/primary.c index 90478d81e..ee503b38e 100644 --- a/source/frontend/gen/expr/primary.c +++ b/source/frontend/gen/expr/primary.c @@ -1762,6 +1762,7 @@ __attribute__((noinline)) int unary_primary(void) s->c = nf->captured_offsets[i]; /* Parent's FP offset */ s->vreg = -1; /* No vreg in nested function's IR — pure stack offset via chain */ s->sym_scope = 0; + nested_track_captured_sym(s); goto found_captured_var; } } diff --git a/source/frontend/gen/gen_priv.h b/source/frontend/gen/gen_priv.h index 35312ad8d..8283dc7a3 100644 --- a/source/frontend/gen/gen_priv.h +++ b/source/frontend/gen/gen_priv.h @@ -328,6 +328,7 @@ extern CType func_old_size_t_type; * caller lived in the same TU. */ void vpush_type_size(CType *type, int *a); void compile_nested_functions(Sym *parent_sym); +void nested_track_captured_sym(Sym *s); void pop_local_syms(Sym *b, int keep); /* combine_types() operator classes, shared with op/op.c's gen_op_impl(). */ diff --git a/source/frontend/gen/nested/nested.c b/source/frontend/gen/nested/nested.c index 097e9d9b5..f120e1ad6 100644 --- a/source/frontend/gen/nested/nested.c +++ b/source/frontend/gen/nested/nested.c @@ -180,6 +180,25 @@ typedef struct struct temp_local_variable tmp_vars[MAX_TEMP_LOCAL_VARIABLE_NUMBER]; } ParentSavedState; +/* The stand-in Syms tok_identifier makes for each reference to a captured + * variable sit on no scope stack, so no sym_pop frees the facts a store + * records on them. Chained through prev (unused off a stack) and dropped once + * the body is compiled; the Syms stay, IR may still point at them. */ +static Sym *captured_var_syms; + +void nested_track_captured_sym(Sym *s) +{ + s->prev = captured_var_syms; + captured_var_syms = s; +} + +static void free_captured_sym_facts(void) +{ + for (Sym *s = captured_var_syms; s; s = s->prev) + sym_free_facts(s); + captured_var_syms = NULL; +} + /* Compile all nested functions defined inside a parent function */ void compile_nested_functions(Sym *parent_sym) { @@ -320,6 +339,7 @@ void compile_nested_functions(Sym *parent_sym) } gen_function(nf->sym); + free_captured_sym_facts(); /* Remove parent addr-taken labels from hash table after compilation. */ for (int j = 0; j < nf->nb_addr_labels; j++) diff --git a/source/frontend/gen/sym/symtab.c b/source/frontend/gen/sym/symtab.c index e7d3b3058..dc1c78981 100644 --- a/source/frontend/gen/sym/symtab.c +++ b/source/frontend/gen/sym/symtab.c @@ -69,7 +69,7 @@ SymLocalFacts *sym_facts(Sym *s) return s->facts; } -static void sym_free_facts(Sym *s) +void sym_free_facts(Sym *s) { if (s->facts) { @@ -332,24 +332,20 @@ ST_FUNC void sym_pop(Sym **ptop, Sym *b, int keep) ps = &ts->sym_identifier; *ps = s->prev_tok; } - if (!keep && s->facts && s->facts->const_init_data) - { - tcc_free(s->facts->const_init_data); - s->facts->const_init_data = NULL; - } /* Don't free symbols that have been exported to ELF (sym->c != 0) as they may still be referenced by IR instructions */ - if (!keep && s->c == 0) - { - /* In IR mode the backend may still need Sym pointers (notably for - * VT_SYM address materialization and relocations). Block-scope extern - * declarations create temporary Sym copies that can be referenced by IR - * after the scope ends; freeing them here can lead to missing relocations - * and loads/stores from address 0 at runtime. - */ - if (!(tcc_state->ir && (s->r & VT_SYM))) - sym_free(s); - } + /* In IR mode the backend may still need Sym pointers (notably for + * VT_SYM address materialization and relocations). Block-scope extern + * declarations create temporary Sym copies that can be referenced by IR + * after the scope ends; freeing them here can lead to missing relocations + * and loads/stores from address 0 at runtime. + */ + if (!keep && s->c == 0 && !(tcc_state->ir && (s->r & VT_SYM))) + sym_free(s); + else if (!keep) + /* The Sym outlives its scope (a local's c is its frame offset); its + * facts do not, and nothing frees them later. */ + sym_free_facts(s); s = ss; } if (!keep) diff --git a/source/include/tcc.h b/source/include/tcc.h index 2719f1eb8..2c8f30a34 100644 --- a/source/include/tcc.h +++ b/source/include/tcc.h @@ -708,6 +708,7 @@ typedef struct SymLocalFacts extern const SymLocalFacts sym_no_facts; #define SYM_FACTS(s) ((const SymLocalFacts *)((s)->facts ? (s)->facts : &sym_no_facts)) SymLocalFacts *sym_facts(struct Sym *s); +void sym_free_facts(struct Sym *s); #include "source/ir/machine_op.h" #include "source/ir/tccir.h" diff --git a/source/opt/flat/fusion/barrel_shift.c b/source/opt/flat/fusion/barrel_shift.c index 83f7b97d9..4f0d8379f 100644 --- a/source/opt/flat/fusion/barrel_shift.c +++ b/source/opt/flat/fusion/barrel_shift.c @@ -48,7 +48,49 @@ * This pass used to refuse that case; the refusal cost an instruction every * time and protected nothing (87 such fusions appear across an exhaustive * op x shift-kind x amount matrix, all matching gcc's answers). + * + * Despite running from run_post_ra_optimizations, this sees the flat IR + * BEFORE tcc_ir_ssa_regalloc: a VAR may still have a def per branch, and + * du.def names only the last one. So the shift must be its result's only + * def, and no path may enter between the shift and the consumer: a join there + * reaches the consumer without the shift (ctags' isTagExtraBitMarked computed + * `index` in both arms of an if, and the join read the else arm's value on + * the then path), and a back edge there re-reads a shift source the shift + * read once. */ + +/* Every JUMP/JUMPIF/switch target, plus anything already flagged. */ +static uint8_t *barrel_build_target_map(TCCIRState *ir, int n) +{ + uint8_t *map = tcc_mallocz((size_t)n); + for (int i = 0; i < n; i++) + { + IRQuadCompact *q = &ir->compact_instructions[i]; + if (q->is_jump_target) + map[i] = 1; + if (q->op == TCCIR_OP_JUMP || q->op == TCCIR_OP_JUMPIF) + { + int t = (int)irop_get_imm64_ex(ir, tcc_ir_op_get_dest(ir, q)); + if (t >= 0 && t < n) + map[t] = 1; + } + else if (q->op == TCCIR_OP_SWITCH_TABLE) + { + int table_id = (int)irop_get_imm64_ex(ir, tcc_ir_op_get_src2(ir, q)); + if (table_id >= 0 && table_id < ir->num_switch_tables) + { + TCCIRSwitchTable *table = &ir->switch_tables[table_id]; + for (int j = 0; j < table->num_entries; j++) + if (table->targets[j] >= 0 && table->targets[j] < n) + map[table->targets[j]] = 1; + if (table->default_target >= 0 && table->default_target < n) + map[table->default_target] = 1; + } + } + } + return map; +} + void tcc_ir_barrel_shift_fusion(TCCIRState *ir) { int n = ir->next_instruction_index; @@ -60,6 +102,7 @@ void tcc_ir_barrel_shift_fusion(TCCIRState *ir) IROptDU du; ir_opt_du_build(ir, &du); + uint8_t *is_target = barrel_build_target_map(ir, n); /* Vregs that reach a memory operation's address: an is_lval operand (a real * pointer -- is_local/is_llocal reads a stack variable's value, not an @@ -144,7 +187,7 @@ void tcc_ir_barrel_shift_fusion(TCCIRState *ir) default: continue; } - if (ir_opt_du_uses(&du, vr2) != 1) + if (ir_opt_du_uses(&du, vr2) != 1 || !ir_opt_du_is_single_def(&du, vr2)) continue; IROperand shift_dest = tcc_ir_op_get_dest(ir, sq); @@ -196,11 +239,13 @@ void tcc_ir_barrel_shift_fusion(TCCIRState *ir) if (!irop_has_vreg(other)) continue; - int safe = 1; + int safe = !is_target[i]; for (int j = shift_idx + 1; j < i && safe; j++) { IRQuadCompact *jq = &ir->compact_instructions[j]; TccIrOp bop = jq->op; + if (is_target[j]) + safe = 0; if (bop == TCCIR_OP_JUMP || bop == TCCIR_OP_JUMPIF) safe = 0; if (bop == TCCIR_OP_NOP) @@ -227,6 +272,7 @@ void tcc_ir_barrel_shift_fusion(TCCIRState *ir) } } + tcc_free(is_target); tcc_free(deref_base); tcc_free(du.def); } diff --git a/source/unity/arm_01.c b/source/unity/arm.c similarity index 100% rename from source/unity/arm_01.c rename to source/unity/arm.c diff --git a/source/unity/arm_02.c b/source/unity/arm_s1.c similarity index 100% rename from source/unity/arm_02.c rename to source/unity/arm_s1.c index 2b536b769..8585b1f38 100644 --- a/source/unity/arm_02.c +++ b/source/unity/arm_s1.c @@ -1,4 +1,4 @@ /* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */ +#include "../backend/arch/arm/arm-link.c" #include "../backend/arch/arm/arm_aapcs.c" #include "../backend/arch/arm/arm_regalloc.c" -#include "../backend/arch/arm/arm-link.c" diff --git a/source/unity/thumb_01.c b/source/unity/arm_thumb.c similarity index 79% rename from source/unity/thumb_01.c rename to source/unity/arm_thumb.c index 80400d61e..08baf5e42 100644 --- a/source/unity/thumb_01.c +++ b/source/unity/arm_thumb.c @@ -1,14 +1,4 @@ /* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */ -#include "../backend/arch/arm/thumb/thumb.c" -#undef TH_PACK_CONST_CACHE_SIZE -#undef USING_GLOBALS -#include "../backend/arch/arm/thumb/thop_mem_reg.c" -#undef THOP_MEM_REG_FN -#undef USING_GLOBALS -#undef V_MEM_REG_T1 -#undef V_MEM_REG_T32 -#include "../backend/arch/arm/thumb/thop_mem_imm.c" -#undef USING_GLOBALS #include "../backend/arch/arm/thumb/thop_bitfield.c" #undef USING_GLOBALS #undef V_BFX @@ -16,8 +6,22 @@ #undef V_SSAT_LSL #undef V_USAT_ASR #undef V_USAT_LSL +#include "../backend/arch/arm/thumb/thop_mem_imm.c" +#undef USING_GLOBALS +#include "../backend/arch/arm/thumb/thop_mem_reg.c" +#undef THOP_MEM_REG_FN +#undef USING_GLOBALS +#undef V_MEM_REG_T1 +#undef V_MEM_REG_T32 #include "../backend/arch/arm/thumb/thop_mov.c" #undef USING_GLOBALS -#include "../backend/arch/arm/thumb/arm-thumb-gen.c" -#include "../backend/arch/arm/thumb/arm-thumb-callsite.c" +#include "../backend/arch/arm/thumb/thumb-outline.c" +#undef OUTLINE_MAX_OPS +#undef OUTLINE_MIN_BYTES +#undef R_IS_SP_LR_PC +#undef USING_GLOBALS +#undef WINDOW_EXTEND +#undef WINDOW_SIZE +#include "../backend/arch/arm/thumb/thumb.c" +#undef TH_PACK_CONST_CACHE_SIZE #undef USING_GLOBALS diff --git a/source/unity/arm_thumb_call.c b/source/unity/arm_thumb_call.c new file mode 100644 index 000000000..9c9cf04d0 --- /dev/null +++ b/source/unity/arm_thumb_call.c @@ -0,0 +1,5 @@ +/* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */ +#include "../backend/arch/arm/thumb/arm-thumb-call.c" +#undef COPY_STUB_MAX_WORDS +#include "../backend/arch/arm/thumb/arm-thumb-callsite.c" +#undef USING_GLOBALS diff --git a/source/unity/thumb_04.c b/source/unity/arm_thumb_emit.c similarity index 56% rename from source/unity/thumb_04.c rename to source/unity/arm_thumb_emit.c index 1c48ca4a9..8d0b19a68 100644 --- a/source/unity/thumb_04.c +++ b/source/unity/arm_thumb_emit.c @@ -1,11 +1,5 @@ /* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */ -#include "../backend/arch/arm/thumb/arm-thumb-mem.c" +#include "../backend/arch/arm/thumb/arm-thumb-emit.c" #include "../backend/arch/arm/thumb/arm-thumb-fp.c" #include "../backend/arch/arm/thumb/arm-thumb-frame.c" -#include "../backend/arch/arm/thumb/thumb-outline.c" -#undef OUTLINE_MAX_OPS -#undef OUTLINE_MIN_BYTES -#undef R_IS_SP_LR_PC -#undef USING_GLOBALS -#undef WINDOW_EXTEND -#undef WINDOW_SIZE +#include "../backend/arch/arm/thumb/arm-thumb-mem.c" diff --git a/source/unity/thumb_03.c b/source/unity/arm_thumb_gen.c similarity index 70% rename from source/unity/thumb_03.c rename to source/unity/arm_thumb_gen.c index 1922f8136..074e1a9b8 100644 --- a/source/unity/thumb_03.c +++ b/source/unity/arm_thumb_gen.c @@ -1,3 +1,3 @@ /* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */ -#include "../backend/arch/arm/thumb/arm-thumb-emit.c" #include "../backend/arch/arm/thumb/arm-thumb-alu.c" +#include "../backend/arch/arm/thumb/arm-thumb-gen.c" diff --git a/source/unity/thumb_02.c b/source/unity/arm_thumb_s1.c similarity index 100% rename from source/unity/thumb_02.c rename to source/unity/arm_thumb_s1.c index a2e305d80..06349a024 100644 --- a/source/unity/thumb_02.c +++ b/source/unity/arm_thumb_s1.c @@ -1,4 +1,5 @@ /* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */ +#include "../backend/arch/arm/thumb/thop_adr.c" #include "../backend/arch/arm/thumb/thop_alu_imm.c" #undef THOP_ALU_IMM_FN #undef THOP_ALU_WIDE_FN @@ -13,6 +14,8 @@ #undef V_REG3 #undef V_REGS #undef V_REG_RDN_RM +#include "../backend/arch/arm/thumb/thop_block.c" +#include "../backend/arch/arm/thumb/thop_branch.c" #include "../backend/arch/arm/thumb/thop_cmp.c" #undef THOP_CMP_IMM_FN #undef THOP_CMP_REG_FN @@ -21,6 +24,30 @@ #undef V_CMP_REG #undef V_CMP_REGS #undef V_CMP_REG_T2 +#include "../backend/arch/arm/thumb/thop_coproc.c" +#include "../backend/arch/arm/thumb/thop_dsp.c" +#include "../backend/arch/arm/thumb/thop_extend.c" +#undef THOP_EXTEND_FN +#undef V_EXTEND_T16 +#undef V_EXTEND_T32 +#include "../backend/arch/arm/thumb/thop_ldaex.c" +#include "../backend/arch/arm/thumb/thop_ldr_literal.c" +#include "../backend/arch/arm/thumb/thop_ldrd.c" +#include "../backend/arch/arm/thumb/thop_ldrex.c" +#include "../backend/arch/arm/thumb/thop_mem_exclusive.c" +#undef THOP_EXCLUSIVE_FN +#undef V_EXCLUSIVE +#include "../backend/arch/arm/thumb/thop_mem_unpriv.c" +#undef THOP_MEM_UNPRIV_FN +#undef V_MEM_UNPRIV +#include "../backend/arch/arm/thumb/thop_mrs.c" +#include "../backend/arch/arm/thumb/thop_mul.c" +#include "../backend/arch/arm/thumb/thop_mvn.c" +#include "../backend/arch/arm/thumb/thop_pld.c" +#include "../backend/arch/arm/thumb/thop_rev.c" +#undef V_RBIT_T32 +#undef V_REV_T16 +#undef V_REV_T32 #include "../backend/arch/arm/thumb/thop_shift_imm.c" #undef THOP_SHIFT_IMM_FN #undef V_SHIFT_IMM16 @@ -31,33 +58,6 @@ #undef V_LSL_REG_T1 #undef V_LSR_REG_T1 #undef V_SHIFT_REG32 -#include "../backend/arch/arm/thumb/thop_mem_unpriv.c" -#undef THOP_MEM_UNPRIV_FN -#undef V_MEM_UNPRIV -#include "../backend/arch/arm/thumb/thop_mem_exclusive.c" -#undef THOP_EXCLUSIVE_FN -#undef V_EXCLUSIVE -#include "../backend/arch/arm/thumb/thop_extend.c" -#undef THOP_EXTEND_FN -#undef V_EXTEND_T16 -#undef V_EXTEND_T32 -#include "../backend/arch/arm/thumb/thop_rev.c" -#undef V_RBIT_T32 -#undef V_REV_T16 -#undef V_REV_T32 -#include "../backend/arch/arm/thumb/thop_vfp.c" -#include "../backend/arch/arm/thumb/thop_coproc.c" #include "../backend/arch/arm/thumb/thop_system.c" -#include "../backend/arch/arm/thumb/thop_branch.c" -#include "../backend/arch/arm/thumb/thop_block.c" -#include "../backend/arch/arm/thumb/thop_mul.c" -#include "../backend/arch/arm/thumb/thop_dsp.c" #include "../backend/arch/arm/thumb/thop_tbb.c" -#include "../backend/arch/arm/thumb/thop_ldrd.c" -#include "../backend/arch/arm/thumb/thop_ldrex.c" -#include "../backend/arch/arm/thumb/thop_ldaex.c" -#include "../backend/arch/arm/thumb/thop_mrs.c" -#include "../backend/arch/arm/thumb/thop_pld.c" -#include "../backend/arch/arm/thumb/thop_mvn.c" -#include "../backend/arch/arm/thumb/thop_adr.c" -#include "../backend/arch/arm/thumb/thop_ldr_literal.c" +#include "../backend/arch/arm/thumb/thop_vfp.c" diff --git a/source/unity/frontend_01.c b/source/unity/frontend_01.c deleted file mode 100644 index 59f76fa64..000000000 --- a/source/unity/frontend_01.c +++ /dev/null @@ -1,11 +0,0 @@ -/* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */ -#include "../frontend/tccasm.c" -#undef ASM_MACRO_MAX_ARGS -#undef USING_GLOBALS -#include "../frontend/gen/builtin/call.c" -#undef CONST_LOOP_INLINE_BUDGET -#include "../frontend/gen/builtin/chk.c" -#undef FIND_LOCAL_OBJSIZE -#include "../frontend/gen/builtin/fp2.c" -#include "../frontend/gen/core/predicates.c" -#undef TCC_IS_NATIVE_387 diff --git a/source/unity/frontend_05.c b/source/unity/frontend_05.c deleted file mode 100644 index 67c98165d..000000000 --- a/source/unity/frontend_05.c +++ /dev/null @@ -1,16 +0,0 @@ -/* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */ -#include "../frontend/gen/op/int.c" -#include "../frontend/gen/op/op.c" -#include "../frontend/gen/op/vector.c" -#undef VEC_EMIT_RANGE_MAX -#undef VEC_RECIPE_MAX -#include "../frontend/gen/stmt/block.c" -#include "../frontend/gen/stmt/cleanup.c" -#include "../frontend/gen/stmt/ret.c" -#include "../frontend/gen/stmt/switch.c" -#include "../frontend/gen/store/struct_copy.c" -#include "../frontend/gen/store/vstore.c" -#undef IS_GLOBAL_LVAL -#undef IS_LOCAL_LVAL -#undef IS_REG_DEREF_LVAL -#include "../frontend/gen/sym/attr_merge.c" diff --git a/source/unity/frontend_02.c b/source/unity/frontend_gen_builtin.c similarity index 60% rename from source/unity/frontend_02.c rename to source/unity/frontend_gen_builtin.c index 9f542ec2d..e212d0e4b 100644 --- a/source/unity/frontend_02.c +++ b/source/unity/frontend_gen_builtin.c @@ -1,5 +1,10 @@ /* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */ +#include "../frontend/gen/builtin/call.c" +#undef CONST_LOOP_INLINE_BUDGET +#include "../frontend/gen/builtin/chk.c" +#undef FIND_LOCAL_OBJSIZE #include "../frontend/gen/builtin/fp.c" +#include "../frontend/gen/builtin/fp2.c" #include "../frontend/gen/builtin/misc.c" #undef GCC_TYPE_CLASS_ARRAY #undef GCC_TYPE_CLASS_COMPLEX @@ -14,12 +19,3 @@ #include "../frontend/gen/builtin/overflow.c" #include "../frontend/gen/builtin/simd.c" #include "../frontend/gen/builtin/string.c" -#include "../frontend/gen/core/state.c" -#include "../frontend/gen/core/suppress.c" -#include "../frontend/gen/decl/attribute.c" -#include "../frontend/gen/decl/btype.c" -#include "../frontend/gen/decl/declarator.c" -#include "../frontend/gen/decl/predef_protos.c" -#include "../frontend/gen/expr/atomic.c" -#undef __ATOMIC_SEQ_CST_VALUE -#include "../frontend/gen/value/strlit_pool.c" diff --git a/source/unity/frontend_gen_core.c b/source/unity/frontend_gen_core.c new file mode 100644 index 000000000..c5893e2d5 --- /dev/null +++ b/source/unity/frontend_gen_core.c @@ -0,0 +1,5 @@ +/* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */ +#include "../frontend/gen/core/predicates.c" +#undef TCC_IS_NATIVE_387 +#include "../frontend/gen/core/state.c" +#include "../frontend/gen/core/suppress.c" diff --git a/source/unity/frontend_gen_decl.c b/source/unity/frontend_gen_decl.c new file mode 100644 index 000000000..a2578c6e7 --- /dev/null +++ b/source/unity/frontend_gen_decl.c @@ -0,0 +1,6 @@ +/* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */ +#include "../frontend/gen/decl/attribute.c" +#include "../frontend/gen/decl/btype.c" +#include "../frontend/gen/decl/declarator.c" +#include "../frontend/gen/decl/predef_protos.c" +#include "../frontend/gen/decl/struct.c" diff --git a/source/unity/frontend_03.c b/source/unity/frontend_gen_expr.c similarity index 62% rename from source/unity/frontend_03.c rename to source/unity/frontend_gen_expr.c index a40e72a0a..c69261594 100644 --- a/source/unity/frontend_03.c +++ b/source/unity/frontend_gen_expr.c @@ -1,12 +1,9 @@ /* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */ -#include "../frontend/gen/decl/struct.c" +#include "../frontend/gen/expr/atomic.c" +#undef __ATOMIC_SEQ_CST_VALUE #include "../frontend/gen/expr/cond.c" #include "../frontend/gen/expr/indir.c" #include "../frontend/gen/expr/infix.c" #undef precedence #include "../frontend/gen/expr/primary.c" #include "../frontend/gen/expr/unary.c" -#include "../frontend/gen/init/alloc.c" -#include "../frontend/gen/op/float.c" -#undef gen_negf -#include "../frontend/gen/sym/elfsym.c" diff --git a/source/unity/frontend_04.c b/source/unity/frontend_gen_inline.c similarity index 56% rename from source/unity/frontend_04.c rename to source/unity/frontend_gen_inline.c index 2e3b74159..6943567f6 100644 --- a/source/unity/frontend_04.c +++ b/source/unity/frontend_gen_inline.c @@ -5,11 +5,3 @@ #include "../frontend/gen/inline/emit.c" #undef CALLED_ONCE_LARGE_BODY #undef GC_MARK -#include "../frontend/gen/nested/nested.c" -#include "../frontend/gen/op/complex.c" -#undef PUSH_COMP -#undef PUSH_FCOMP -#undef STORE_COMP -#undef STORE_FCOMP -#include "../frontend/gen/op/fold_math.c" -#undef NUM_FOLDABLE_MATH_FUNCS diff --git a/source/unity/frontend_gen_op.c b/source/unity/frontend_gen_op.c new file mode 100644 index 000000000..e51c9f15c --- /dev/null +++ b/source/unity/frontend_gen_op.c @@ -0,0 +1,15 @@ +/* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */ +#include "../frontend/gen/op/complex.c" +#undef PUSH_COMP +#undef PUSH_FCOMP +#undef STORE_COMP +#undef STORE_FCOMP +#include "../frontend/gen/op/float.c" +#undef gen_negf +#include "../frontend/gen/op/fold_math.c" +#undef NUM_FOLDABLE_MATH_FUNCS +#include "../frontend/gen/op/int.c" +#include "../frontend/gen/op/op.c" +#include "../frontend/gen/op/vector.c" +#undef VEC_EMIT_RANGE_MAX +#undef VEC_RECIPE_MAX diff --git a/source/unity/frontend_gen_stmt.c b/source/unity/frontend_gen_stmt.c new file mode 100644 index 000000000..bbf7bf874 --- /dev/null +++ b/source/unity/frontend_gen_stmt.c @@ -0,0 +1,5 @@ +/* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */ +#include "../frontend/gen/stmt/block.c" +#include "../frontend/gen/stmt/cleanup.c" +#include "../frontend/gen/stmt/ret.c" +#include "../frontend/gen/stmt/switch.c" diff --git a/source/unity/frontend_gen_store.c b/source/unity/frontend_gen_store.c new file mode 100644 index 000000000..a967f4d7e --- /dev/null +++ b/source/unity/frontend_gen_store.c @@ -0,0 +1,6 @@ +/* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */ +#include "../frontend/gen/store/struct_copy.c" +#include "../frontend/gen/store/vstore.c" +#undef IS_GLOBAL_LVAL +#undef IS_LOCAL_LVAL +#undef IS_REG_DEREF_LVAL diff --git a/source/unity/frontend_gen_sym.c b/source/unity/frontend_gen_sym.c new file mode 100644 index 000000000..4c3824220 --- /dev/null +++ b/source/unity/frontend_gen_sym.c @@ -0,0 +1,4 @@ +/* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */ +#include "../frontend/gen/sym/attr_merge.c" +#include "../frontend/gen/sym/elfsym.c" +#include "../frontend/gen/sym/symtab.c" diff --git a/source/unity/frontend_06.c b/source/unity/frontend_gen_type.c similarity index 59% rename from source/unity/frontend_06.c rename to source/unity/frontend_gen_type.c index 936d29269..980a91b63 100644 --- a/source/unity/frontend_06.c +++ b/source/unity/frontend_gen_type.c @@ -1,10 +1,6 @@ /* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */ -#include "../frontend/gen/sym/symtab.c" #include "../frontend/gen/type/assign_check.c" #include "../frontend/gen/type/cast.c" #undef ALLOW_SUBTYPE_ACCESS #undef gen_cvt_itof1 #include "../frontend/gen/type/size.c" -#include "../frontend/gen/value/load.c" -#include "../frontend/gen/value/longlong.c" -#include "../frontend/gen/value/vstack.c" diff --git a/source/unity/frontend_gen_value.c b/source/unity/frontend_gen_value.c new file mode 100644 index 000000000..248aba071 --- /dev/null +++ b/source/unity/frontend_gen_value.c @@ -0,0 +1,5 @@ +/* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */ +#include "../frontend/gen/value/load.c" +#include "../frontend/gen/value/longlong.c" +#include "../frontend/gen/value/strlit_pool.c" +#include "../frontend/gen/value/vstack.c" diff --git a/source/unity/groups.txt b/source/unity/groups.txt new file mode 100644 index 000000000..3dce94b4f --- /dev/null +++ b/source/unity/groups.txt @@ -0,0 +1,61 @@ +# Unity groups that are not simply "one per directory". Hand-written; read by +# scripts/gen_unity.py, which generates the rest of source/unity/ from it. +# +# Every library source joins the group of its directory (source/opt/flat/cfg/*.c +# -> opt_flat_cfg), unless a line below claims it first. Lines are +# group: pattern [pattern ...] +# with shell globs relative to source/; the first line that matches a source +# wins. A directory too big for one TU (the device compiles each group in +# a few MB of heap) is cut here into named pieces; its remaining files keep the +# directory's own group. A source the rules keep apart (see gen_unity.py) +# builds as its own TU wherever it is listed, and members that do not use +# USING_GLOBALS (state through s1->, not tcc_state->) cannot share a TU with +# those that do: they get the group's name plus "_s1". + +# source/ir: the code generator and the register allocator stand on their own. +ir_codegen: ir/codegen.c +ir_regalloc: ir/regalloc.c ir/regalloc_scan.c ir/regalloc_post.c +ir_regalloc_ssa: ir/regalloc_phi.c ir/regalloc_entry.c + +# source/backend/arch/arm/thumb +arm_thumb_gen: backend/arch/arm/thumb/arm-thumb-gen.c backend/arch/arm/thumb/arm-thumb-alu.c +arm_thumb_asm: backend/arch/arm/thumb/arm-thumb-asm.c +arm_thumb_call: backend/arch/arm/thumb/arm-thumb-call*.c +arm_thumb_emit: backend/arch/arm/thumb/arm-thumb-*.c + +# source/obj: the ELF writer/linker apart from debug info, ld scripts and YAFF. +obj_elf: obj/tccelf*.c + +# source/opt/flat/memory +opt_flat_memory_fwd: opt/flat/memory/sl_forward.c opt/flat/memory/copy_source_load_fwd.c + opt/flat/memory/entry_store_prop.c opt/flat/memory/addrof_var_fwd.c + opt/flat/memory/ptr_local_fwd.c opt/flat/memory/param_copy_alias.c +opt_flat_memory_dse: opt/flat/memory/dse.c opt/flat/memory/dead_*.c + opt/flat/memory/var_store_elim.c opt/flat/memory/byte_store_merge.c + opt/flat/memory/rmw_byte_clear.c opt/flat/memory/store_inplace_arith.c + opt/flat/memory/slot_const_store_fold.c opt/flat/memory/bitfield_unit_narrow.c +opt_flat_memory_cse: opt/flat/memory/*_cse.c opt/flat/memory/global_base_share.c + opt/flat/memory/invariant_global_load_hoist.c + +# source/opt/flat/scalar +opt_flat_scalar_const: opt/flat/scalar/const_*.c opt/flat/scalar/addrof_const_fold.c + opt/flat/scalar/cmp_expr_fold.c opt/flat/scalar/symref_const_prop.c + opt/flat/scalar/global_init_prop.c opt/flat/scalar/stack_addr_fold.c + opt/flat/scalar/switch_collapse.c +opt_flat_scalar_known: opt/flat/scalar/known_bits.c opt/flat/scalar/value_tracking.c + opt/flat/scalar/branch.c opt/flat/scalar/redundant_loop_check.c + +# source/opt/flat/loop: induction variables and invariant code motion. +opt_flat_loop_iv: opt/flat/loop/iv_*.c opt/flat/loop/licm.c + opt/flat/loop/decrement_to_zero.c opt/flat/loop/backedge_phi_hoist.c + opt/flat/loop/stack_param_promote.c + +# source/opt/ssa/cfg: its file-local branch_gens is a name the other members' +# headers declare. +opt_ssa_cfg_or_bool: opt/ssa/cfg/or_bool_diamond.c + +# source/ir: frame layout and cross-jumping apart from the IR core. +ir_frame: ir/frame.c ir/cross_jump.c + +# source/opt/ssa/scalar: the propagation engines apart from the folds. +opt_ssa_scalar_prop: opt/ssa/scalar/sccp.c opt/ssa/scalar/cprop.c diff --git a/source/unity/ir_01.c b/source/unity/ir.c similarity index 64% rename from source/unity/ir_01.c rename to source/unity/ir.c index 7b5780ee8..209ba6ee5 100644 --- a/source/unity/ir_01.c +++ b/source/unity/ir.c @@ -1,34 +1,26 @@ /* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */ -#include "../ir/type.c" +#include "../ir/cfg.c" +#undef USING_GLOBALS +#include "../ir/dead_def.c" +#undef DD_MAX_DIST +#include "../ir/dump.c" +#undef SPILL_MARK_BEGIN +#undef SPILL_MARK_END +#undef USING_GLOBALS +#include "../ir/machine_op.c" #undef USING_GLOBALS #include "../ir/pool.c" #undef USING_GLOBALS +#include "../ir/self_store.c" +#undef SC_MAX_DIST +#include "../ir/ssa.c" +#undef DD_VAR +#undef USING_GLOBALS +#include "../ir/type.c" +#undef USING_GLOBALS #include "../ir/vreg.c" #undef IGNORED_VREG_BITS_PER_ENTRY #undef IGNORED_VREG_LOCAL_VAR_BIT #undef IGNORED_VREG_PARAM_BIT #undef IGNORED_VREG_TEMP_BIT #undef USING_GLOBALS -#include "../ir/frame.c" -#undef FRAME_LIVE_MAX_WORDS -#undef FRAME_OBJ_ARG_COPY -#undef FRAME_OBJ_RET_TEMP -#undef TAINT_MULTI -#undef TAINT_NONE -#undef USING_GLOBALS -#include "../ir/cross_jump.c" -#undef CJ_MAX_TAIL -#undef RM_BUCKET_TRIES -#undef RM_MAX_LEN -#undef RM_MAX_PARAMS -#undef RM_MIN_LEN -#undef RM_MULTI -#undef RM_NOCASE -#include "../ir/self_store.c" -#undef SC_MAX_DIST -#include "../ir/dead_def.c" -#undef DD_MAX_DIST -#include "../ir/dump.c" -#undef SPILL_MARK_BEGIN -#undef SPILL_MARK_END -#undef USING_GLOBALS diff --git a/source/unity/ir_02.c b/source/unity/ir_02.c deleted file mode 100644 index d072db47d..000000000 --- a/source/unity/ir_02.c +++ /dev/null @@ -1,20 +0,0 @@ -/* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */ -#include "../ir/codegen.c" -#undef DECODE -#undef DEMOTE_PENDING_SLOT -#undef DEMOTE_REF_NONE -#undef DEMOTE_REF_SLOTS -#undef DYN_ARGS_MAX -#undef MACH_BLOCK_COPY_MAX_WORDS -#undef MOP_CACHE_CHUNK -#undef MOP_CACHE_HDR -#undef SCAN_INTERVALS -#undef SCRATCH_WRAP -#undef USING_GLOBALS -#undef tcc_ir_debug_codegen_generate_entry -#include "../ir/gen/control.c" -#undef USING_GLOBALS -#include "../ir/gen/vla.c" -#undef USING_GLOBALS -#include "../ir/gen/util.c" -#undef USING_GLOBALS diff --git a/source/unity/ir_frame.c b/source/unity/ir_frame.c new file mode 100644 index 000000000..7efc63cba --- /dev/null +++ b/source/unity/ir_frame.c @@ -0,0 +1,16 @@ +/* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */ +#include "../ir/cross_jump.c" +#undef CJ_MAX_TAIL +#undef RM_BUCKET_TRIES +#undef RM_MAX_LEN +#undef RM_MAX_PARAMS +#undef RM_MIN_LEN +#undef RM_MULTI +#undef RM_NOCASE +#include "../ir/frame.c" +#undef FRAME_LIVE_MAX_WORDS +#undef FRAME_OBJ_ARG_COPY +#undef FRAME_OBJ_RET_TEMP +#undef TAINT_MULTI +#undef TAINT_NONE +#undef USING_GLOBALS diff --git a/source/unity/ir_05.c b/source/unity/ir_gen.c similarity index 61% rename from source/unity/ir_05.c rename to source/unity/ir_gen.c index 8aa8376f9..47a465b4c 100644 --- a/source/unity/ir_05.c +++ b/source/unity/ir_gen.c @@ -1,19 +1,31 @@ /* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */ -#include "../ir/gen/put.c" +#include "../ir/gen/arith.c" #undef USING_GLOBALS -#include "../ir/gen/params.c" +#include "../ir/gen/asm.c" #undef USING_GLOBALS -#include "../ir/gen/arith.c" +#include "../ir/gen/config.c" +#undef USING_GLOBALS +#include "../ir/gen/control.c" #undef USING_GLOBALS #include "../ir/gen/float.c" #undef USING_GLOBALS #include "../ir/gen/jump.c" #undef USING_GLOBALS -#include "../ir/gen/asm.c" +#include "../ir/gen/live.c" +#undef USING_GLOBALS +#include "../ir/gen/params.c" +#undef USING_GLOBALS +#include "../ir/gen/put.c" #undef USING_GLOBALS #include "../ir/gen/softfloat.c" #undef USING_GLOBALS -#include "../ir/gen/config.c" +#include "../ir/gen/state.c" +#undef IR_PARAM_INTERVAL_INIT_SIZE +#undef IR_TEMP_INTERVAL_INIT_SIZE +#undef IR_VAR_INTERVAL_INIT_SIZE +#undef QUADRUPLE_INIT_SIZE #undef USING_GLOBALS -#include "../ir/gen/live.c" +#include "../ir/gen/util.c" +#undef USING_GLOBALS +#include "../ir/gen/vla.c" #undef USING_GLOBALS diff --git a/source/unity/ir_03.c b/source/unity/ir_regalloc.c similarity index 54% rename from source/unity/ir_03.c rename to source/unity/ir_regalloc.c index 2b9f2a297..9b5f3006d 100644 --- a/source/unity/ir_03.c +++ b/source/unity/ir_regalloc.c @@ -1,9 +1,4 @@ /* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */ -#include "../ir/cfg.c" -#undef USING_GLOBALS -#include "../ir/ssa.c" -#undef DD_VAR -#undef USING_GLOBALS #include "../ir/regalloc.c" #undef ASSIGN_VREG_IDX #undef BFI_VREG_IDX @@ -13,16 +8,9 @@ #undef RA_EXTEND_BACKEDGE #undef RA_VREG_HAS_ASSIGN_IN_RANGE #undef VREG_IDX +#include "../ir/regalloc_post.c" #include "../ir/regalloc_scan.c" #undef AVIDX #undef AV_FOR_PARAM_USES #undef HINT_IDX #undef RA_EXITPHI_PUSH -#include "../ir/machine_op.c" -#undef USING_GLOBALS -#include "../ir/gen/state.c" -#undef IR_PARAM_INTERVAL_INIT_SIZE -#undef IR_TEMP_INTERVAL_INIT_SIZE -#undef IR_VAR_INTERVAL_INIT_SIZE -#undef QUADRUPLE_INIT_SIZE -#undef USING_GLOBALS diff --git a/source/unity/ir_04.c b/source/unity/ir_regalloc_ssa.c similarity index 93% rename from source/unity/ir_04.c rename to source/unity/ir_regalloc_ssa.c index ae5339965..34644cbfc 100644 --- a/source/unity/ir_04.c +++ b/source/unity/ir_regalloc_ssa.c @@ -1,4 +1,14 @@ /* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */ +#include "../ir/regalloc_entry.c" +#undef REMAP +#undef RUN_SSA +#undef SPLIT_CAND +#undef SPLIT_DEF_BETWEEN +#undef SPLIT_IDX +#undef SPLIT_MEMBER +#undef SPLIT_PUSH +#undef SPLIT_REGION_OK +#undef SPLIT_STRAIGHT #include "../ir/regalloc_phi.c" #undef ADD_CAND #undef DVIDX @@ -10,14 +20,3 @@ #undef RA_DEADCOPY_MARK #undef UF_FIND #undef VIDX -#include "../ir/regalloc_entry.c" -#undef REMAP -#undef RUN_SSA -#undef SPLIT_CAND -#undef SPLIT_DEF_BETWEEN -#undef SPLIT_IDX -#undef SPLIT_MEMBER -#undef SPLIT_PUSH -#undef SPLIT_REGION_OK -#undef SPLIT_STRAIGHT -#include "../ir/regalloc_post.c" diff --git a/source/unity/machine_01.c b/source/unity/machine_s1.c similarity index 100% rename from source/unity/machine_01.c rename to source/unity/machine_s1.c diff --git a/source/unity/memory_01.c b/source/unity/memory_s1.c similarity index 100% rename from source/unity/memory_01.c rename to source/unity/memory_s1.c diff --git a/source/unity/obj_02.c b/source/unity/obj_02.c deleted file mode 100644 index 69507fa11..000000000 --- a/source/unity/obj_02.c +++ /dev/null @@ -1,11 +0,0 @@ -/* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */ -#include "../obj/tccelf_load.c" -#include "../obj/tccelf_ld.c" -#undef DEBUG_VERSION -#undef LD_TOK_EOF -#undef LD_TOK_NAME -#include "../obj/tccld.c" -#undef LDTOK_EOF -#undef LDTOK_NAME -#undef LDTOK_NUM -#undef LDTOK_STRING diff --git a/source/unity/obj_01.c b/source/unity/obj_elf_s1.c similarity index 67% rename from source/unity/obj_01.c rename to source/unity/obj_elf_s1.c index 08f0af7dc..4bb360ebc 100644 --- a/source/unity/obj_01.c +++ b/source/unity/obj_elf_s1.c @@ -1,7 +1,12 @@ /* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */ #include "../obj/tccelf.c" -#include "../obj/tccelf_sym.c" -#undef ELFCLASS_BITS -#include "../obj/tccelf_reloc.c" #include "../obj/tccelf_layout.c" +#include "../obj/tccelf_ld.c" +#undef DEBUG_VERSION +#undef LD_TOK_EOF +#undef LD_TOK_NAME +#include "../obj/tccelf_load.c" #include "../obj/tccelf_output.c" +#include "../obj/tccelf_reloc.c" +#include "../obj/tccelf_sym.c" +#undef ELFCLASS_BITS diff --git a/source/unity/obj_03.c b/source/unity/obj_s1.c similarity index 94% rename from source/unity/obj_03.c rename to source/unity/obj_s1.c index 8688ebf2c..4b738330d 100644 --- a/source/unity/obj_03.c +++ b/source/unity/obj_s1.c @@ -1,7 +1,4 @@ /* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */ -#include "../obj/tccyaff.c" -#undef SHF_DYNSYM -#undef TCC_YAFF_MAX_SYMBOL_ENTRY_SIZE #include "../obj/tccdbg.c" #undef DWARF_ABBREV_ARRAY_TYPE #undef DWARF_ABBREV_BASE_TYPE @@ -61,3 +58,11 @@ #undef new_file #undef section_sym #undef tcov_data +#include "../obj/tccld.c" +#undef LDTOK_EOF +#undef LDTOK_NAME +#undef LDTOK_NUM +#undef LDTOK_STRING +#include "../obj/tccyaff.c" +#undef SHF_DYNSYM +#undef TCC_YAFF_MAX_SYMBOL_ENTRY_SIZE diff --git a/source/unity/opt_01.c b/source/unity/opt_01.c deleted file mode 100644 index 9f8145d92..000000000 --- a/source/unity/opt_01.c +++ /dev/null @@ -1,74 +0,0 @@ -/* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */ -#include "../opt/util/pass_disable.c" -#undef USING_GLOBALS -#include "../opt/util/volatile_access.c" -#undef USING_GLOBALS -#include "../opt/util/const_eval.c" -#undef USING_GLOBALS -#include "../opt/util/const_string_eval.c" -#undef SB_A0 -#undef SB_A1 -#undef USING_GLOBALS -#include "../opt/util/cond_util.c" -#undef USING_GLOBALS -#include "../opt/util/block_scan.c" -#undef USING_GLOBALS -#include "../opt/util/purity.c" -#undef USING_GLOBALS -#include "../opt/util/expr_equal.c" -#undef USING_GLOBALS -#include "../opt/util/call_params.c" -#undef USING_GLOBALS -#include "../opt/util/vreg_query.c" -#undef USING_GLOBALS -#include "../opt/util/vreg_def_use.c" -#undef USING_GLOBALS -#include "../opt/util/xform.c" -#undef USING_GLOBALS -#include "../opt/util/ir_insert.c" -#undef USING_GLOBALS -#include "../opt/analysis/du_chains.c" -#undef USING_GLOBALS -#include "../opt/analysis/alias.c" -#undef USING_GLOBALS -#include "../opt/analysis/memref.c" -#undef USING_GLOBALS -#include "../opt/analysis/mem_ssa.c" -#undef USING_GLOBALS -#include "../opt/engine/ctx.c" -#undef USING_GLOBALS -#include "../opt/engine/run_gens.c" -#undef USING_GLOBALS -#include "../opt/engine/pipeline_run.c" -#undef PIPELINE_MAX_TRACKED -#undef USING_GLOBALS -#include "../opt/engine/pipeline_table.c" -#undef CASCADE_END -#undef COUNTOF -#undef FLAG -#undef PASS -#undef PASS_GATED -#undef USING_GLOBALS -#include "../opt/engine/gen_adapters.c" -#undef USING_GLOBALS -#include "../opt/engine/pass_timing.c" -#undef TCC_PASS_TIMING_MAX -#undef TCC_PT_STACK_MAX -#undef USING_GLOBALS -#include "../opt/engine/fp_cache_shim.c" -#undef USING_GLOBALS -#include "../opt/engine/fp_mat_cache.c" -#undef FP_MAT_CACHE_SIZE -#undef USING_GLOBALS -#include "../opt/engine/pass_registry.c" -#undef USING_GLOBALS -#include "../opt/ra/const_branch_fold.c" -#undef USING_GLOBALS -#include "../opt/ra/incomplete_calls.c" -#undef USING_GLOBALS -#include "../opt/ra/phi_const_chain.c" -#undef USING_GLOBALS -#include "../opt/ssa/string/const_string_fold.c" -#include "../opt/ssa/string/str_strlen.c" -#include "../opt/flat/memory/mem_init.c" -#undef USING_GLOBALS diff --git a/source/unity/opt_02.c b/source/unity/opt_02.c deleted file mode 100644 index 4d626c96a..000000000 --- a/source/unity/opt_02.c +++ /dev/null @@ -1,56 +0,0 @@ -/* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */ -#include "../opt/ra/umaal_fusion.c" -#undef USING_GLOBALS -#include "../opt/ssa/string/str_strcmp.c" -#include "../opt/ssa/string/str_strcpy.c" -#include "../opt/ssa/string/str_strspn.c" -#include "../opt/ssa/string/str_strchr.c" -#include "../opt/ssa/string/str_strstr.c" -#include "../opt/ssa/string/str_memchr.c" -#include "../opt/ssa/memory/global_addr_hoist.c" -#undef GAH_CONST_MIN_MAG -#undef GAH_CONST_TIGHT_BUDGET -#undef GAH_CONST_TIGHT_RELOADS -#undef GAH_LARGE_2ND_RELOADS -#undef GAH_LARGE_FN_INSTRS -#undef GAH_MAX_CALLS_BEFORE -#undef GAH_MAX_HOISTS -#undef GAH_MIN_RELOADS -#undef LAC_MAX_CLASSES -#undef LAC_MAX_FN_INSTRS -#undef LAC_MAX_TEMPS -#undef LAH_CALL_MAX_HOISTS -#undef LAH_CALL_MIN_BUDGET -#undef LAH_MAX_HOISTS -#undef LAH_MAX_LOOPS -#undef LAH_MIN_BUDGET -#include "../opt/ssa/memory/load_cse.c" -#undef GLOAD_MAX -#undef GSTORE_MAX -#undef ILOAD_MAX -#undef SSTORE_MAX -#undef TVSTORE_MAX -#undef USING_GLOBALS -#undef VSLOT_MAX -#undef resolve_lea_stackloc -#include "../opt/ssa/memory/diamond_store_fwd.c" -#undef USING_GLOBALS -#include "../opt/ssa/scalar/bitop_const_fold.c" -#include "../opt/ssa/scalar/strength.c" -#undef USING_GLOBALS -#include "../opt/ssa/scalar/reassoc.c" -#undef USING_GLOBALS -#include "../opt/ssa/scalar/narrow.c" -#undef NARROW_FLOAT_TABLE_COUNT -#undef USING_GLOBALS -#include "../opt/ssa/scalar/var_imm_prop.c" -#undef USING_GLOBALS -#include "../opt/ssa/scalar/bool_norm.c" -#undef K01_MAX_DEPTH -#undef K01_MAX_INSTRS -#undef USING_GLOBALS -#include "../opt/ssa/cfg/switch_fold.c" -#undef USING_GLOBALS -#include "../opt/ssa/dce/unreachable.c" -#undef DCE_MARK -#undef USING_GLOBALS diff --git a/source/unity/opt_04.c b/source/unity/opt_04.c deleted file mode 100644 index 0cbe620e0..000000000 --- a/source/unity/opt_04.c +++ /dev/null @@ -1,62 +0,0 @@ -/* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */ -#include "../opt/ssa/cfg/branch.c" -#undef MARK -#undef SSA_SA_SLOT -#undef USING_GLOBALS -#include "../opt/ssa/cfg/vrp.c" -#undef USING_GLOBALS -#undef VRP_TOK_EQ -#undef VRP_TOK_GE -#undef VRP_TOK_GT -#undef VRP_TOK_LE -#undef VRP_TOK_LT -#undef VRP_TOK_NE -#undef VRP_TOK_UGE -#undef VRP_TOK_UGT -#undef VRP_TOK_ULE -#undef VRP_TOK_ULT -#include "../opt/ssa/dce/stackloc_stores.c" -#undef SL_HASH -#undef SL_HASH_SIZE -#undef SL_SET -#undef SL_TEST -#undef USING_GLOBALS -#include "../opt/ssa/dce/orphan_params.c" -#undef USING_GLOBALS -#include "../opt/ssa/dce/ret_path_frame_store.c" -#undef USING_GLOBALS -#include "../opt/ssa/dce/dead_phi_cycles.c" -#undef BM_SET -#undef BM_TEST -#undef MARK_TEMP_LIVE -#undef USING_GLOBALS -#include "../opt/ssa/dce/dead_overwrite_stores.c" -#undef DOS_PEND_MAX -#undef USING_GLOBALS -#include "../opt/ssa/dce/dead_global_stores.c" -#undef GS_PEND_MAX -#undef USING_GLOBALS -#include "../opt/ssa/dce/var_liveness.c" -#undef USING_GLOBALS -#undef VL_BIT -#undef VL_CLR -#undef VL_SET -#include "../opt/ssa/dce/dce.c" -#undef USING_GLOBALS -#include "../opt/ssa/loop/loop_rotate.c" -#undef SSA_LOOP_ROTATE_MAX_PASSES -#include "../opt/ssa/loop/first_iter_exit.c" -#undef FIE_EXIT_TEST_LOOKAHEAD -#undef LD_MAX_TMPS -#undef LD_MAX_VARS -#undef LD_TOK_EQ -#undef LD_TOK_NE -#undef SSA_FIRST_ITER_EXIT_MAX_PASSES -#include "../opt/ssa/loop/loop_cand.c" -#include "../opt/ssa/loop/loop_const_sim.c" -#undef SSA_LCS_MAX_SPAN -#undef SSA_LOOP_CONST_SIM_MAX_PASSES -#include "../opt/ssa/loop/iv_strength_reduction.c" -#undef SSA_IVSR_MAX_PASSES -#include "../opt/ssa/loop/decrement_to_zero.c" -#undef SSA_DTZ_MAX_PASSES diff --git a/source/unity/opt_05.c b/source/unity/opt_05.c deleted file mode 100644 index c70c2fec7..000000000 --- a/source/unity/opt_05.c +++ /dev/null @@ -1,33 +0,0 @@ -/* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */ -#include "../opt/ssa/loop/ptr_iv_exit_subst.c" -#undef PTRIV_MAX -#undef SSA_PTR_IV_EXIT_SUBST_MAX_PASSES -#include "../opt/ssa/loop/loop_unroll.c" -#undef CZ_MAX_CHAIN -#undef CZ_MAX_SLOTS -#undef SSA_LOOP_UNROLL_MAX_PASSES -#undef SSA_UNROLL_MAX_SPAN -#include "../opt/ssa/loop/loop_bottom_test.c" -#undef SSA_LBT_MAX_REWRITES -#include "../opt/ssa/loop/dead_loop.c" -#undef DL_MAX_ESCAPE_CANDS -#undef USING_GLOBALS -#include "../opt/ssa/scalar/sccp.c" -#undef USING_GLOBALS -#include "../opt/ssa/memory/global_store_dse.c" -#undef USING_GLOBALS -#include "../opt/ssa/memory/ptr_store_dse.c" -#undef USING_GLOBALS -#include "../opt/ssa/cfg/guard_collapse.c" -#undef USING_GLOBALS -#include "../opt/ssa/engine/use_def.c" -#undef USING_GLOBALS -#include "../opt/ssa/engine/rewrite.c" -#undef USING_GLOBALS -#include "../opt/ssa/engine/driver.c" -#undef SSA_RUN -#undef USING_GLOBALS -#include "../opt/flat/scalar/bool.c" -#undef USING_GLOBALS -#include "../opt/flat/scalar/narrow_store.c" -#undef USING_GLOBALS diff --git a/source/unity/opt_07.c b/source/unity/opt_07.c deleted file mode 100644 index becb952e8..000000000 --- a/source/unity/opt_07.c +++ /dev/null @@ -1,33 +0,0 @@ -/* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */ -#include "../opt/flat/scalar/known_bits.c" -#undef KB_MAX_STACK_SLOTS -#undef USING_GLOBALS -#include "../opt/flat/scalar/const_prop_tmp.c" -#undef USING_GLOBALS -#include "../opt/flat/scalar/value_tracking.c" -#undef USING_GLOBALS -#undef VT_BINOP -#undef VT_CLEAR_DEF -#undef VT_HAS_DEF -#undef VT_INVALIDATE -#undef VT_IS_CONST -#undef VT_MAX_ADDRTAKEN -#undef VT_SET_CONST -#undef VT_SET_CONST_DEF -#include "../opt/flat/scalar/const_var_prop.c" -#undef NEG_VREG_MAX -#undef USING_GLOBALS -#include "../opt/flat/scalar/const_aggregate.c" -#undef CAF_ESCAPE -#undef CAF_MAX_SLOTS -#undef CAF_RESOLVE_ADDR_ROOT -#undef CAF_RESOLVE_LVAL -#undef CAF_ROOT_CONFLICT -#undef CAF_ROOT_ESCAPED -#undef CAF_ROOT_NONE -#undef CAF_SLOT_IDX -#undef USING_GLOBALS -#include "../opt/flat/scalar/add_reassoc.c" -#undef USING_GLOBALS -#include "../opt/flat/fusion/pair_reorder.c" -#undef USING_GLOBALS diff --git a/source/unity/opt_08.c b/source/unity/opt_08.c deleted file mode 100644 index 2e9cf595f..000000000 --- a/source/unity/opt_08.c +++ /dev/null @@ -1,49 +0,0 @@ -/* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */ -#include "../opt/flat/scalar/cmp_field_fuse.c" -#undef LOG_CMPFUSE -#undef USING_GLOBALS -#include "../opt/flat/scalar/cmp_expr_fold.c" -#undef USING_GLOBALS -#include "../opt/flat/scalar/bitfield.c" -#undef LOG_BITFIELD -#undef USING_GLOBALS -#include "../opt/flat/memory/symaddr_cse.c" -#undef SYMADDR_CSE_MAX_HOISTS -#undef USING_GLOBALS -#include "../opt/flat/memory/dse.c" -#undef DSE_CASCADE_PURE_CALL_PARAMS -#undef DSE_DEC_SOURCES -#undef DSE_DEC_TEMP -#undef DSE_ENSURE_CAP -#undef DSE_INC_USE -#undef DSE_IS_DEAD_ELIGIBLE -#undef DSE_IS_PURE_CALL -#undef DSE_SOURCE_IS_PURE -#undef GET_ORIGIN -#undef MARK_ORIGIN_READ -#undef MARK_STACKLOC_OP -#undef MARK_VAR_USED -#undef STACKLOC_HASH -#undef STACKLOC_HASH_SIZE -#undef STACKLOC_SET -#undef STACKLOC_TEST -#undef USING_GLOBALS -#include "../opt/flat/memory/var_store_elim.c" -#undef USING_GLOBALS -#include "../opt/flat/memory/entry_store_prop.c" -#undef ESTORE_COMPACT -#undef LEA_SET -#undef MAX_ADDRTAKEN_BASES -#undef MAX_BC_RANGES -#undef MAX_ENTRY_STORES -#undef PHF_MAX -#undef USING_GLOBALS -#include "../opt/flat/memory/sret_nrvo.c" -#undef NRVO_FAIL -#undef NRVO_IDX -#undef NRVO_MAX_OBJ -#undef NRVO_MAX_REFS -#undef NRVO_OWNED -#undef USING_GLOBALS -#include "../opt/flat/memory/dead_temp_local.c" -#undef USING_GLOBALS diff --git a/source/unity/opt_09.c b/source/unity/opt_09.c deleted file mode 100644 index c6d366b04..000000000 --- a/source/unity/opt_09.c +++ /dev/null @@ -1,25 +0,0 @@ -/* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */ -#include "../opt/flat/memory/sl_forward.c" -#undef CHECK_ADDR_ALIAS -#undef CHECK_WIDTH_OVERLAP -#undef GSLFWD_MAX_ENTRIES -#undef SL_FWD_MAX_DEAD_STORES -#undef USING_GLOBALS -#include "../opt/flat/memory/copy_source_load_fwd.c" -#undef CSF_ALL64 -#undef CSF_DBG -#undef CSF_MAX_BLOCK_COPY -#undef CSF_MAX_FIELDS -#undef USING_GLOBALS -#include "../opt/flat/memory/dead_local_slot.c" -#undef DLS_LIVE_ADD -#undef DLS_NONTAME_RANGE_OVERLAPS -#undef TAME_FIND -#undef TAME_FIND_OR_ADD -#undef USING_GLOBALS -#undef VR_FLAT -#undef VR_SLOT_AMBIG -#include "../opt/flat/memory/dead_static_store.c" -#undef DSS_VREG_MAP_MAX -#undef DSS_VREG_MAP_OVERFLOW -#undef USING_GLOBALS diff --git a/source/unity/opt_12.c b/source/unity/opt_12.c deleted file mode 100644 index 73aede6d9..000000000 --- a/source/unity/opt_12.c +++ /dev/null @@ -1,44 +0,0 @@ -/* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */ -#include "../opt/flat/dce/ub_only_body_elide.c" -#undef SRC_IS_UNINIT_READ -#undef TEMP_IS_TAINTED -#undef UB_ELIDE_MAX_STACK_OFFS -#undef UB_ELIDE_MAX_TEMPS -#undef UB_ELIDE_MAX_VAR_POS -#undef USING_GLOBALS -#include "../opt/flat/dce/const_return_uninit_elide.c" -#undef CRUE_MAX_STORE_OFFSETS -#undef CRUE_MAX_VAR_POS -#undef USING_GLOBALS -#include "../opt/flat/dce/null_store_dom_return.c" -#undef NSDR_MAX_TEMP -#undef NSDR_MAX_VAR -#undef USING_GLOBALS -#undef VREG_IS_KNOWN_ZERO -#include "../opt/flat/dce/dce.c" -#undef MARK_REACHABLE -#undef USING_GLOBALS -#include "../opt/flat/dce/compact_nops.c" -#undef USING_GLOBALS -#include "../opt/flat/dce/orphan_cmp_elim.c" -#undef USING_GLOBALS -#include "../opt/flat/dce/redundant_cmp.c" -#undef USING_GLOBALS -#include "../opt/flat/dce/zero_vla_elim.c" -#undef SLOT_USED_BY -#undef USING_GLOBALS -#include "../opt/flat/loop/loop_ir_mutate.c" -#undef USING_GLOBALS -#include "../opt/flat/loop/iv_analysis.c" -#undef IV_AFFINE_MAX -#undef USING_GLOBALS -#include "../opt/flat/loop/iv_strength_reduction.c" -#undef APPLY_SHIFT -#undef SR_TAINT_MAX -#undef USING_GLOBALS -#include "../opt/flat/loop/loop_exit_analysis.c" -#undef USING_GLOBALS -#include "../opt/flat/loop/loop_eliminate.c" -#undef USING_GLOBALS -#include "../opt/flat/loop/decrement_to_zero.c" -#undef USING_GLOBALS diff --git a/source/unity/opt_14.c b/source/unity/opt_14.c deleted file mode 100644 index 7112b6fc2..000000000 --- a/source/unity/opt_14.c +++ /dev/null @@ -1,34 +0,0 @@ -/* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */ -#include "../opt/flat/dce/dead_vla.c" -#undef CLASSIFY -#undef USING_GLOBALS -#include "../opt/flat/cfg/jumpif_invert.c" -#undef USING_GLOBALS -#include "../opt/flat/scalar/var_to_tmp.c" -#undef USING_GLOBALS -#include "../opt/flat/loop/backedge_phi_hoist.c" -#undef USING_GLOBALS -#include "../opt/flat/cfg/abort_tail_merge.c" -#undef USING_GLOBALS -#include "../opt/flat/loop/loop_detect.c" -#undef BUDGET_MAX_VREGS -#undef WINDOW_SIZE -#include "../opt/flat/loop/func_purity.c" -#undef NUM_PURE_FUNCS -#include "../opt/flat/loop/licm.c" -#undef GLH_MAX_PER_LOOP -#undef GLH_MAX_WRITTEN -#undef LICM_MAX_ADDR_TEMPS -#undef LICM_MAX_MEM_PARAMS -#undef MAX_HOISTABLE_CALLS -#include "../opt/flat/loop/stack_param_promote.c" -#include "../opt/flat/memory/invariant_global_load_hoist.c" -#undef IGLH_MAX_TRACKED -#undef IGLH_MAX_WRITTEN -#undef USING_GLOBALS -#include "../opt/flat/memory/global_deref_cse.c" -#undef GDCSE_MAX_ENTRIES -#undef GDCSE_MAX_USES -#undef USING_GLOBALS -#include "../opt/flat/ipa/tu_noreturn.c" -#undef USING_GLOBALS diff --git a/source/unity/opt_15.c b/source/unity/opt_15.c deleted file mode 100644 index b090ce3ee..000000000 --- a/source/unity/opt_15.c +++ /dev/null @@ -1,55 +0,0 @@ -/* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */ -#include "../opt/flat/memory/deref_operand_cse.c" -#undef DOCSE_MAX_ENTRIES -#undef DOCSE_MAX_USES -#undef USING_GLOBALS -#include "../opt/flat/memory/bitfield_unit_narrow.c" -#undef BFUN_MAX_EXTRACTS -#undef USING_GLOBALS -#include "../opt/flat/memory/rmw_byte_clear.c" -#undef USING_GLOBALS -#include "../opt/flat/memory/struct_copy_roundtrip_elim.c" -#undef MGLF_MAX -#undef USING_GLOBALS -#include "../opt/flat/fusion/barrel_shift.c" -#undef USING_GLOBALS -#include "../opt/flat/fusion/shift_pair_ubfx.c" -#undef USING_GLOBALS -#include "../opt/flat/fusion/shift64_dead_half.c" -#undef USING_GLOBALS -#include "../opt/flat/fusion/zero_half64.c" -#undef USING_GLOBALS -#undef ZH64_D_HI -#undef ZH64_D_LO -#undef ZH64_S1_HI -#undef ZH64_S1_LO -#undef ZH64_S2_HI -#undef ZH64_S2_LO -#undef ZK_HI -#undef ZK_LO -#include "../opt/flat/fusion/shift64_extract_ubfx.c" -#undef USING_GLOBALS -#include "../opt/flat/ipa/pure_via_sret.c" -#undef SRET_MARK -#undef SRET_TEST -#undef SRET_VR_TO_BIT -#undef USING_GLOBALS -#include "../opt/flat/ipa/func_write_summary.c" -#undef USING_GLOBALS -#undef VR_BIT -#include "../opt/flat/ipa/tu_func_summary.c" -#undef TU_MODREF_MAX_VISIT -#undef TU_VREG_MAP_MAX -#undef TU_VREG_MAP_OVERFLOW -#undef USING_GLOBALS -#include "../opt/flat/ipa/tu_dead_statics.c" -#undef USING_GLOBALS -#include "../opt/flat/memory/dead_init_via_call.c" -#undef USING_GLOBALS -#include "../opt/flat/memory/stack_addr_cse.c" -#undef STACK_CSE_MAX_ENTRIES -#undef USING_GLOBALS -#include "../opt/flat/memory/block_copy_init.c" -#undef USING_GLOBALS -#include "../opt/flat/memory/small_memset_to_store.c" -#undef USING_GLOBALS diff --git a/source/unity/opt_analysis.c b/source/unity/opt_analysis.c new file mode 100644 index 000000000..621ca75b0 --- /dev/null +++ b/source/unity/opt_analysis.c @@ -0,0 +1,9 @@ +/* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */ +#include "../opt/analysis/alias.c" +#undef USING_GLOBALS +#include "../opt/analysis/du_chains.c" +#undef USING_GLOBALS +#include "../opt/analysis/mem_ssa.c" +#undef USING_GLOBALS +#include "../opt/analysis/memref.c" +#undef USING_GLOBALS diff --git a/source/unity/opt_engine.c b/source/unity/opt_engine.c new file mode 100644 index 000000000..54231e025 --- /dev/null +++ b/source/unity/opt_engine.c @@ -0,0 +1,28 @@ +/* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */ +#include "../opt/engine/ctx.c" +#undef USING_GLOBALS +#include "../opt/engine/fp_cache_shim.c" +#undef USING_GLOBALS +#include "../opt/engine/fp_mat_cache.c" +#undef FP_MAT_CACHE_SIZE +#undef USING_GLOBALS +#include "../opt/engine/gen_adapters.c" +#undef USING_GLOBALS +#include "../opt/engine/pass_registry.c" +#undef USING_GLOBALS +#include "../opt/engine/pass_timing.c" +#undef TCC_PASS_TIMING_MAX +#undef TCC_PT_STACK_MAX +#undef USING_GLOBALS +#include "../opt/engine/pipeline_run.c" +#undef PIPELINE_MAX_TRACKED +#undef USING_GLOBALS +#include "../opt/engine/pipeline_table.c" +#undef CASCADE_END +#undef COUNTOF +#undef FLAG +#undef PASS +#undef PASS_GATED +#undef USING_GLOBALS +#include "../opt/engine/run_gens.c" +#undef USING_GLOBALS diff --git a/source/unity/opt_flat_cfg.c b/source/unity/opt_flat_cfg.c new file mode 100644 index 000000000..a14b391c9 --- /dev/null +++ b/source/unity/opt_flat_cfg.c @@ -0,0 +1,18 @@ +/* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */ +#include "../opt/flat/cfg/abort_tail_merge.c" +#undef USING_GLOBALS +#include "../opt/flat/cfg/bool_diamond_branch.c" +#undef USING_GLOBALS +#include "../opt/flat/cfg/if_convert.c" +#undef JT_HAS_OTHER +#undef JT_NOP_JUMP +#undef USING_GLOBALS +#include "../opt/flat/cfg/jump_thread.c" +#undef USING_GLOBALS +#include "../opt/flat/cfg/jumpif_invert.c" +#undef USING_GLOBALS +#include "../opt/flat/cfg/returnvalue_merge.c" +#undef USING_GLOBALS +#include "../opt/flat/cfg/switch_to_data.c" +#undef SWITCHDATA_MAX_BODY_OPS +#undef USING_GLOBALS diff --git a/source/unity/opt_11.c b/source/unity/opt_flat_dce.c similarity index 57% rename from source/unity/opt_11.c rename to source/unity/opt_flat_dce.c index 107b59471..06cb617fa 100644 --- a/source/unity/opt_11.c +++ b/source/unity/opt_flat_dce.c @@ -1,22 +1,4 @@ /* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */ -#include "../opt/flat/fusion/and64_narrow.c" -#undef USING_GLOBALS -#include "../opt/flat/fusion/shl32_or_chain.c" -#undef USING_GLOBALS -#include "../opt/flat/scalar/sdiv_pow2.c" -#undef USING_GLOBALS -#include "../opt/flat/memory/dead_lea_store.c" -#undef ADD_READ -#undef RESOLVE_LVAL_SLOT -#undef USING_GLOBALS -#include "../opt/flat/memory/store_inplace_arith.c" -#undef USING_GLOBALS -#include "../opt/flat/scalar/redundant_loop_check.c" -#undef USING_GLOBALS -#include "../opt/flat/cfg/if_convert.c" -#undef JT_HAS_OTHER -#undef JT_NOP_JUMP -#undef USING_GLOBALS #include "../opt/flat/dce/body_essential.c" #undef GETBIT #undef IS_LOCAL_PTR_OP @@ -28,7 +10,21 @@ #undef PUSH #undef SETBIT #undef USING_GLOBALS -#include "../opt/flat/dce/noreturn_collapse.c" +#include "../opt/flat/dce/compact_nops.c" +#undef USING_GLOBALS +#include "../opt/flat/dce/const_return_uninit_elide.c" +#undef CRUE_MAX_STORE_OFFSETS +#undef CRUE_MAX_VAR_POS +#undef USING_GLOBALS +#include "../opt/flat/dce/dce.c" +#undef MARK_REACHABLE +#undef USING_GLOBALS +#include "../opt/flat/dce/dead_addrvar_elim.c" +#undef USING_GLOBALS +#include "../opt/flat/dce/dead_trailing_addrvar_store.c" +#undef USING_GLOBALS +#include "../opt/flat/dce/dead_vla.c" +#undef CLASSIFY #undef USING_GLOBALS #include "../opt/flat/dce/infinite_loop_simplify.c" #undef MAX_HOIST @@ -37,9 +33,25 @@ #undef USING_GLOBALS #include "../opt/flat/dce/noreturn_call_epilogue.c" #undef USING_GLOBALS -#include "../opt/flat/dce/dead_addrvar_elim.c" +#include "../opt/flat/dce/noreturn_collapse.c" #undef USING_GLOBALS -#include "../opt/flat/dce/dead_trailing_addrvar_store.c" +#include "../opt/flat/dce/null_store_dom_return.c" +#undef NSDR_MAX_TEMP +#undef NSDR_MAX_VAR +#undef USING_GLOBALS +#undef VREG_IS_KNOWN_ZERO +#include "../opt/flat/dce/orphan_cmp_elim.c" +#undef USING_GLOBALS +#include "../opt/flat/dce/redundant_cmp.c" +#undef USING_GLOBALS +#include "../opt/flat/dce/trap_only_body_suppress.c" +#undef USING_GLOBALS +#include "../opt/flat/dce/ub_only_body_elide.c" +#undef SRC_IS_UNINIT_READ +#undef TEMP_IS_TAINTED +#undef UB_ELIDE_MAX_STACK_OFFS +#undef UB_ELIDE_MAX_TEMPS +#undef UB_ELIDE_MAX_VAR_POS #undef USING_GLOBALS #include "../opt/flat/dce/uninit_ub.c" #undef UDR_MAX_VAR_POS @@ -47,5 +59,6 @@ #undef UDR_RR_GET #undef UNINIT_MAX_VAR_POS #undef USING_GLOBALS -#include "../opt/flat/cfg/returnvalue_merge.c" +#include "../opt/flat/dce/zero_vla_elim.c" +#undef SLOT_USED_BY #undef USING_GLOBALS diff --git a/source/unity/opt_10.c b/source/unity/opt_flat_fusion.c similarity index 61% rename from source/unity/opt_10.c rename to source/unity/opt_flat_fusion.c index a51d7eec2..a73b8ca9b 100644 --- a/source/unity/opt_10.c +++ b/source/unity/opt_flat_fusion.c @@ -1,49 +1,57 @@ /* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */ -#include "../opt/flat/memory/global_base_share.c" -#undef GBS_DELTA_MAX -#undef GBS_DELTA_MIN -#undef GBS_MAX_CLUSTER -#undef USING_GLOBALS -#include "../opt/flat/memory/byte_store_merge.c" -#undef CMD_MAX_BYTES -#undef CMD_OVL -#undef RSE_DEF_MULTI -#undef RSE_MAX_COPY_SRC -#undef USING_GLOBALS -#include "../opt/flat/memory/mem_inline.c" -#undef MI_MAX_PIECES -#undef MI_MEMCPY_MAX -#undef MI_MEMCPY_PIECES -#undef MI_MEMSET_MAX -#undef MI_MEMSET_PIECES +#include "../opt/flat/fusion/add_deref_fold.c" #undef USING_GLOBALS -#include "../opt/flat/fusion/disp.c" +#include "../opt/flat/fusion/and64_narrow.c" #undef USING_GLOBALS -#include "../opt/flat/fusion/gens_fusion.c" +#include "../opt/flat/fusion/assign_fuse.c" #undef USING_GLOBALS -#include "../opt/flat/fusion/add_deref_fold.c" +#include "../opt/flat/fusion/barrel_shift.c" #undef USING_GLOBALS #include "../opt/flat/fusion/call_chain_rename.c" #undef USING_GLOBALS -#include "../opt/flat/fusion/stackoff_addr_cse.c" -#undef SAC_MAX_OFFSETS -#undef SIB_MAX_OFFSETS +#include "../opt/flat/fusion/cmp_hi_only.c" +#undef USING_GLOBALS +#undef ZH64_CMP_HI +#include "../opt/flat/fusion/cmp_imm_swap.c" +#undef USING_GLOBALS +#include "../opt/flat/fusion/cmp_narrow_64.c" +#undef USING_GLOBALS +#include "../opt/flat/fusion/disp.c" +#undef USING_GLOBALS +#include "../opt/flat/fusion/gens_fusion.c" +#undef USING_GLOBALS +#include "../opt/flat/fusion/indexed_chain.c" #undef USING_GLOBALS #include "../opt/flat/fusion/lea_fold.c" #undef USING_GLOBALS #include "../opt/flat/fusion/lea_rmw_fold.c" #undef LEA_RMW_MAX_SITES #undef USING_GLOBALS -#include "../opt/flat/fusion/assign_fuse.c" -#undef USING_GLOBALS #include "../opt/flat/fusion/pack64.c" #undef USING_GLOBALS #include "../opt/flat/fusion/pack64_tautology.c" #undef USING_GLOBALS -#include "../opt/flat/fusion/cmp_narrow_64.c" +#include "../opt/flat/fusion/pair_reorder.c" #undef USING_GLOBALS -#include "../opt/flat/fusion/cmp_hi_only.c" +#include "../opt/flat/fusion/shift64_dead_half.c" #undef USING_GLOBALS -#undef ZH64_CMP_HI -#include "../opt/flat/fusion/cmp_imm_swap.c" +#include "../opt/flat/fusion/shift64_extract_ubfx.c" +#undef USING_GLOBALS +#include "../opt/flat/fusion/shift_pair_ubfx.c" +#undef USING_GLOBALS +#include "../opt/flat/fusion/shl32_or_chain.c" +#undef USING_GLOBALS +#include "../opt/flat/fusion/stackoff_addr_cse.c" +#undef SAC_MAX_OFFSETS +#undef SIB_MAX_OFFSETS +#undef USING_GLOBALS +#include "../opt/flat/fusion/zero_half64.c" #undef USING_GLOBALS +#undef ZH64_D_HI +#undef ZH64_D_LO +#undef ZH64_S1_HI +#undef ZH64_S1_LO +#undef ZH64_S2_HI +#undef ZH64_S2_LO +#undef ZK_HI +#undef ZK_LO diff --git a/source/unity/opt_flat_ipa.c b/source/unity/opt_flat_ipa.c new file mode 100644 index 000000000..5375c628f --- /dev/null +++ b/source/unity/opt_flat_ipa.c @@ -0,0 +1,18 @@ +/* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */ +#include "../opt/flat/ipa/func_write_summary.c" +#undef USING_GLOBALS +#undef VR_BIT +#include "../opt/flat/ipa/pure_via_sret.c" +#undef SRET_MARK +#undef SRET_TEST +#undef SRET_VR_TO_BIT +#undef USING_GLOBALS +#include "../opt/flat/ipa/tu_dead_statics.c" +#undef USING_GLOBALS +#include "../opt/flat/ipa/tu_func_summary.c" +#undef TU_MODREF_MAX_VISIT +#undef TU_VREG_MAP_MAX +#undef TU_VREG_MAP_OVERFLOW +#undef USING_GLOBALS +#include "../opt/flat/ipa/tu_noreturn.c" +#undef USING_GLOBALS diff --git a/source/unity/opt_13.c b/source/unity/opt_flat_loop.c similarity index 79% rename from source/unity/opt_13.c rename to source/unity/opt_flat_loop.c index 2fc874273..f69b69d3a 100644 --- a/source/unity/opt_13.c +++ b/source/unity/opt_flat_loop.c @@ -1,24 +1,4 @@ /* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */ -#include "../opt/flat/loop/loop_unroll.c" -#undef UNROLL_MAX_FOLD -#undef UNROLL_MAX_RENAME -#undef USING_GLOBALS -#include "../opt/flat/loop/loop_rotate.c" -#undef ROT_LVAL_IS_INDIRECT -#undef ROT_NOTE_DEF -#undef ROT_NOTE_READ -#undef ROT_VAR_DIRECT -#undef USING_GLOBALS -#undef _ROFF -#include "../opt/flat/loop/loop_relayout.c" -#undef RJ -#undef USING_GLOBALS -#include "../opt/flat/loop/seq_guard_elim.c" -#undef SEQ_MAX_INSTRS -#undef SEQ_MAX_TRACK -#undef SEQ_ROTATED -#undef SEQ_TOP_TESTED -#undef USING_GLOBALS #include "../opt/flat/loop/const_sim.c" #undef LCS_MAX_CALLS #undef LCS_MAX_ITER_STEPS @@ -31,6 +11,31 @@ #undef LCS_MAX_VAR_POS #undef LCS_RECORD_EXIT #undef USING_GLOBALS +#include "../opt/flat/loop/func_purity.c" +#undef NUM_PURE_FUNCS +#include "../opt/flat/loop/loop_detect.c" +#undef BUDGET_MAX_VREGS +#undef WINDOW_SIZE +#include "../opt/flat/loop/loop_eliminate.c" +#undef USING_GLOBALS +#include "../opt/flat/loop/loop_exit_analysis.c" +#undef USING_GLOBALS +#include "../opt/flat/loop/loop_ir_mutate.c" +#undef USING_GLOBALS +#include "../opt/flat/loop/loop_relayout.c" +#undef RJ +#undef USING_GLOBALS +#include "../opt/flat/loop/loop_rotate.c" +#undef ROT_LVAL_IS_INDIRECT +#undef ROT_NOTE_DEF +#undef ROT_NOTE_READ +#undef ROT_VAR_DIRECT +#undef USING_GLOBALS +#undef _ROFF +#include "../opt/flat/loop/loop_unroll.c" +#undef UNROLL_MAX_FOLD +#undef UNROLL_MAX_RENAME +#undef USING_GLOBALS #include "../opt/flat/loop/reroll.c" #undef LOG_REROLL #undef REROLL_MAX_PERIOD @@ -38,10 +43,9 @@ #undef REROLL_MIN_REPEATS #undef RS_HASH #undef USING_GLOBALS -#include "../opt/flat/cfg/switch_to_data.c" -#undef SWITCHDATA_MAX_BODY_OPS -#undef USING_GLOBALS -#include "../opt/flat/cfg/jump_thread.c" -#undef USING_GLOBALS -#include "../opt/flat/cfg/bool_diamond_branch.c" +#include "../opt/flat/loop/seq_guard_elim.c" +#undef SEQ_MAX_INSTRS +#undef SEQ_MAX_TRACK +#undef SEQ_ROTATED +#undef SEQ_TOP_TESTED #undef USING_GLOBALS diff --git a/source/unity/opt_flat_loop_iv.c b/source/unity/opt_flat_loop_iv.c new file mode 100644 index 000000000..77039cbc1 --- /dev/null +++ b/source/unity/opt_flat_loop_iv.c @@ -0,0 +1,19 @@ +/* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */ +#include "../opt/flat/loop/backedge_phi_hoist.c" +#undef USING_GLOBALS +#include "../opt/flat/loop/decrement_to_zero.c" +#undef USING_GLOBALS +#include "../opt/flat/loop/iv_analysis.c" +#undef IV_AFFINE_MAX +#undef USING_GLOBALS +#include "../opt/flat/loop/iv_strength_reduction.c" +#undef APPLY_SHIFT +#undef SR_TAINT_MAX +#undef USING_GLOBALS +#include "../opt/flat/loop/licm.c" +#undef GLH_MAX_PER_LOOP +#undef GLH_MAX_WRITTEN +#undef LICM_MAX_ADDR_TEMPS +#undef LICM_MAX_MEM_PARAMS +#undef MAX_HOISTABLE_CALLS +#include "../opt/flat/loop/stack_param_promote.c" diff --git a/source/unity/opt_16.c b/source/unity/opt_flat_memory.c similarity index 52% rename from source/unity/opt_16.c rename to source/unity/opt_flat_memory.c index 6bd97b35e..4432e3bbd 100644 --- a/source/unity/opt_16.c +++ b/source/unity/opt_flat_memory.c @@ -1,23 +1,24 @@ /* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */ +#include "../opt/flat/memory/block_copy_init.c" +#undef USING_GLOBALS #include "../opt/flat/memory/const_local_table.c" #undef CLT_MAX_OBJ #undef CLT_MAX_SIZE #undef USING_GLOBALS -#include "../opt/flat/memory/small_global_memset_to_store.c" +#include "../opt/flat/memory/mem_init.c" #undef USING_GLOBALS -#include "../opt/flat/memory/addrof_var_fwd.c" -#undef ADDROF_VAR_MAX_ALIASES -#undef ADDROF_VAR_MAX_REWRITES +#include "../opt/flat/memory/mem_inline.c" +#undef MI_MAX_PIECES +#undef MI_MEMCPY_MAX +#undef MI_MEMCPY_PIECES +#undef MI_MEMSET_MAX +#undef MI_MEMSET_PIECES #undef USING_GLOBALS -#include "../opt/flat/memory/ptr_local_fwd.c" -#undef PLF_CONFLICT -#undef PLF_UNKNOWN +#include "../opt/flat/memory/memmove_to_indexed_stores.c" #undef USING_GLOBALS -#include "../opt/flat/memory/param_copy_alias.c" -#undef PCA_MAX_WORDS +#include "../opt/flat/memory/small_global_memset_to_store.c" #undef USING_GLOBALS -#include "../opt/flat/memory/slot_const_store_fold.c" -#undef SCSF_MAX +#include "../opt/flat/memory/small_memset_to_store.c" #undef USING_GLOBALS #include "../opt/flat/memory/sra.c" #undef SRA_DF_MAX @@ -27,5 +28,13 @@ #undef SRA_NO_ADDR #undef SRA_UNKNOWN #undef SRA_VMIX -#include "../opt/flat/memory/memmove_to_indexed_stores.c" +#include "../opt/flat/memory/sret_nrvo.c" +#undef NRVO_FAIL +#undef NRVO_IDX +#undef NRVO_MAX_OBJ +#undef NRVO_MAX_REFS +#undef NRVO_OWNED +#undef USING_GLOBALS +#include "../opt/flat/memory/struct_copy_roundtrip_elim.c" +#undef MGLF_MAX #undef USING_GLOBALS diff --git a/source/unity/opt_flat_memory_cse.c b/source/unity/opt_flat_memory_cse.c new file mode 100644 index 000000000..e6014aee4 --- /dev/null +++ b/source/unity/opt_flat_memory_cse.c @@ -0,0 +1,24 @@ +/* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */ +#include "../opt/flat/memory/deref_operand_cse.c" +#undef DOCSE_MAX_ENTRIES +#undef DOCSE_MAX_USES +#undef USING_GLOBALS +#include "../opt/flat/memory/global_base_share.c" +#undef GBS_DELTA_MAX +#undef GBS_DELTA_MIN +#undef GBS_MAX_CLUSTER +#undef USING_GLOBALS +#include "../opt/flat/memory/global_deref_cse.c" +#undef GDCSE_MAX_ENTRIES +#undef GDCSE_MAX_USES +#undef USING_GLOBALS +#include "../opt/flat/memory/invariant_global_load_hoist.c" +#undef IGLH_MAX_TRACKED +#undef IGLH_MAX_WRITTEN +#undef USING_GLOBALS +#include "../opt/flat/memory/stack_addr_cse.c" +#undef STACK_CSE_MAX_ENTRIES +#undef USING_GLOBALS +#include "../opt/flat/memory/symaddr_cse.c" +#undef SYMADDR_CSE_MAX_HOISTS +#undef USING_GLOBALS diff --git a/source/unity/opt_flat_memory_dse.c b/source/unity/opt_flat_memory_dse.c new file mode 100644 index 000000000..73c419b7b --- /dev/null +++ b/source/unity/opt_flat_memory_dse.c @@ -0,0 +1,57 @@ +/* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */ +#include "../opt/flat/memory/bitfield_unit_narrow.c" +#undef BFUN_MAX_EXTRACTS +#undef USING_GLOBALS +#include "../opt/flat/memory/byte_store_merge.c" +#undef CMD_MAX_BYTES +#undef CMD_OVL +#undef RSE_DEF_MULTI +#undef RSE_MAX_COPY_SRC +#undef USING_GLOBALS +#include "../opt/flat/memory/dead_init_via_call.c" +#undef USING_GLOBALS +#include "../opt/flat/memory/dead_lea_store.c" +#undef ADD_READ +#undef RESOLVE_LVAL_SLOT +#undef USING_GLOBALS +#include "../opt/flat/memory/dead_local_slot.c" +#undef DLS_LIVE_ADD +#undef DLS_NONTAME_RANGE_OVERLAPS +#undef TAME_FIND +#undef TAME_FIND_OR_ADD +#undef USING_GLOBALS +#undef VR_FLAT +#undef VR_SLOT_AMBIG +#include "../opt/flat/memory/dead_static_store.c" +#undef DSS_VREG_MAP_MAX +#undef DSS_VREG_MAP_OVERFLOW +#undef USING_GLOBALS +#include "../opt/flat/memory/dead_temp_local.c" +#undef USING_GLOBALS +#include "../opt/flat/memory/dse.c" +#undef DSE_CASCADE_PURE_CALL_PARAMS +#undef DSE_DEC_SOURCES +#undef DSE_DEC_TEMP +#undef DSE_ENSURE_CAP +#undef DSE_INC_USE +#undef DSE_IS_DEAD_ELIGIBLE +#undef DSE_IS_PURE_CALL +#undef DSE_SOURCE_IS_PURE +#undef GET_ORIGIN +#undef MARK_ORIGIN_READ +#undef MARK_STACKLOC_OP +#undef MARK_VAR_USED +#undef STACKLOC_HASH +#undef STACKLOC_HASH_SIZE +#undef STACKLOC_SET +#undef STACKLOC_TEST +#undef USING_GLOBALS +#include "../opt/flat/memory/rmw_byte_clear.c" +#undef USING_GLOBALS +#include "../opt/flat/memory/slot_const_store_fold.c" +#undef SCSF_MAX +#undef USING_GLOBALS +#include "../opt/flat/memory/store_inplace_arith.c" +#undef USING_GLOBALS +#include "../opt/flat/memory/var_store_elim.c" +#undef USING_GLOBALS diff --git a/source/unity/opt_flat_memory_fwd.c b/source/unity/opt_flat_memory_fwd.c new file mode 100644 index 000000000..e0bd4bf4a --- /dev/null +++ b/source/unity/opt_flat_memory_fwd.c @@ -0,0 +1,32 @@ +/* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */ +#include "../opt/flat/memory/addrof_var_fwd.c" +#undef ADDROF_VAR_MAX_ALIASES +#undef ADDROF_VAR_MAX_REWRITES +#undef USING_GLOBALS +#include "../opt/flat/memory/copy_source_load_fwd.c" +#undef CSF_ALL64 +#undef CSF_DBG +#undef CSF_MAX_BLOCK_COPY +#undef CSF_MAX_FIELDS +#undef USING_GLOBALS +#include "../opt/flat/memory/entry_store_prop.c" +#undef ESTORE_COMPACT +#undef LEA_SET +#undef MAX_ADDRTAKEN_BASES +#undef MAX_BC_RANGES +#undef MAX_ENTRY_STORES +#undef PHF_MAX +#undef USING_GLOBALS +#include "../opt/flat/memory/param_copy_alias.c" +#undef PCA_MAX_WORDS +#undef USING_GLOBALS +#include "../opt/flat/memory/ptr_local_fwd.c" +#undef PLF_CONFLICT +#undef PLF_UNKNOWN +#undef USING_GLOBALS +#include "../opt/flat/memory/sl_forward.c" +#undef CHECK_ADDR_ALIAS +#undef CHECK_WIDTH_OVERLAP +#undef GSLFWD_MAX_ENTRIES +#undef SL_FWD_MAX_DEAD_STORES +#undef USING_GLOBALS diff --git a/source/unity/opt_flat_scalar.c b/source/unity/opt_flat_scalar.c new file mode 100644 index 000000000..babab9817 --- /dev/null +++ b/source/unity/opt_flat_scalar.c @@ -0,0 +1,29 @@ +/* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */ +#include "../opt/flat/scalar/add_reassoc.c" +#undef USING_GLOBALS +#include "../opt/flat/scalar/bitfield.c" +#undef LOG_BITFIELD +#undef USING_GLOBALS +#include "../opt/flat/scalar/bool.c" +#undef USING_GLOBALS +#include "../opt/flat/scalar/call_result.c" +#undef USING_GLOBALS +#include "../opt/flat/scalar/cmp_field_fuse.c" +#undef LOG_CMPFUSE +#undef USING_GLOBALS +#include "../opt/flat/scalar/inline_param_copy.c" +#undef USING_GLOBALS +#include "../opt/flat/scalar/narrow_store.c" +#undef USING_GLOBALS +#include "../opt/flat/scalar/sdiv_pow2.c" +#undef USING_GLOBALS +#include "../opt/flat/scalar/self_arith.c" +#undef USING_GLOBALS +#include "../opt/flat/scalar/self_copy.c" +#undef USING_GLOBALS +#include "../opt/flat/scalar/stack_bool_diamond.c" +#undef USING_GLOBALS +#include "../opt/flat/scalar/var_tmp_fwd.c" +#undef USING_GLOBALS +#include "../opt/flat/scalar/var_to_tmp.c" +#undef USING_GLOBALS diff --git a/source/unity/opt_06.c b/source/unity/opt_flat_scalar_const.c similarity index 55% rename from source/unity/opt_06.c rename to source/unity/opt_flat_scalar_const.c index a77b022af..18010d895 100644 --- a/source/unity/opt_06.c +++ b/source/unity/opt_flat_scalar_const.c @@ -1,39 +1,35 @@ /* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */ -#include "../opt/ssa/engine/stack_resolve.c" -#undef USING_GLOBALS -#include "../opt/flat/scalar/branch.c" -#undef REMAT_MAX_USES +#include "../opt/flat/scalar/addrof_const_fold.c" #undef USING_GLOBALS -#include "../opt/flat/scalar/stack_bool_diamond.c" +#include "../opt/flat/scalar/cmp_expr_fold.c" #undef USING_GLOBALS -#include "../opt/flat/scalar/const_string_calls.c" +#include "../opt/flat/scalar/const_aggregate.c" +#undef CAF_ESCAPE +#undef CAF_MAX_SLOTS +#undef CAF_RESOLVE_ADDR_ROOT +#undef CAF_RESOLVE_LVAL +#undef CAF_ROOT_CONFLICT +#undef CAF_ROOT_ESCAPED +#undef CAF_ROOT_NONE +#undef CAF_SLOT_IDX #undef USING_GLOBALS #include "../opt/flat/scalar/const_call_fold.c" #undef SWITCH_FUNC_MAX_OPS #undef SWITCH_FUNC_SIM_MAX_REPLAY #undef SWITCH_FUNC_SIM_MAX_VREGS #undef USING_GLOBALS -#include "../opt/flat/scalar/addrof_const_fold.c" -#undef USING_GLOBALS -#include "../opt/flat/scalar/global_init_prop.c" -#undef USING_GLOBALS -#include "../opt/flat/scalar/symref_const_prop.c" -#undef USING_GLOBALS -#include "../opt/flat/scalar/stack_addr_fold.c" -#undef USING_GLOBALS -#include "../opt/flat/scalar/call_result.c" +#include "../opt/flat/scalar/const_prop_tmp.c" #undef USING_GLOBALS -#include "../opt/flat/scalar/self_arith.c" +#include "../opt/flat/scalar/const_string_calls.c" #undef USING_GLOBALS -#include "../opt/flat/scalar/self_copy.c" +#include "../opt/flat/scalar/const_var_prop.c" +#undef NEG_VREG_MAX #undef USING_GLOBALS -#include "../opt/flat/scalar/var_tmp_fwd.c" +#include "../opt/flat/scalar/global_init_prop.c" #undef USING_GLOBALS -#include "../opt/flat/scalar/inline_param_copy.c" +#include "../opt/flat/scalar/stack_addr_fold.c" #undef USING_GLOBALS #include "../opt/flat/scalar/switch_collapse.c" #undef USING_GLOBALS -#include "../opt/flat/fusion/indexed_chain.c" -#undef USING_GLOBALS -#include "../opt/flat/dce/trap_only_body_suppress.c" +#include "../opt/flat/scalar/symref_const_prop.c" #undef USING_GLOBALS diff --git a/source/unity/opt_flat_scalar_known.c b/source/unity/opt_flat_scalar_known.c new file mode 100644 index 000000000..1ede91f26 --- /dev/null +++ b/source/unity/opt_flat_scalar_known.c @@ -0,0 +1,19 @@ +/* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */ +#include "../opt/flat/scalar/branch.c" +#undef REMAT_MAX_USES +#undef USING_GLOBALS +#include "../opt/flat/scalar/known_bits.c" +#undef KB_MAX_STACK_SLOTS +#undef USING_GLOBALS +#include "../opt/flat/scalar/redundant_loop_check.c" +#undef USING_GLOBALS +#include "../opt/flat/scalar/value_tracking.c" +#undef USING_GLOBALS +#undef VT_BINOP +#undef VT_CLEAR_DEF +#undef VT_HAS_DEF +#undef VT_INVALIDATE +#undef VT_IS_CONST +#undef VT_MAX_ADDRTAKEN +#undef VT_SET_CONST +#undef VT_SET_CONST_DEF diff --git a/source/unity/opt_ra.c b/source/unity/opt_ra.c new file mode 100644 index 000000000..5d7ddbab0 --- /dev/null +++ b/source/unity/opt_ra.c @@ -0,0 +1,9 @@ +/* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */ +#include "../opt/ra/const_branch_fold.c" +#undef USING_GLOBALS +#include "../opt/ra/incomplete_calls.c" +#undef USING_GLOBALS +#include "../opt/ra/phi_const_chain.c" +#undef USING_GLOBALS +#include "../opt/ra/umaal_fusion.c" +#undef USING_GLOBALS diff --git a/source/unity/opt_ssa_cfg.c b/source/unity/opt_ssa_cfg.c new file mode 100644 index 000000000..f9d5a96ed --- /dev/null +++ b/source/unity/opt_ssa_cfg.c @@ -0,0 +1,23 @@ +/* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */ +#include "../opt/ssa/cfg/branch.c" +#undef MARK +#undef SSA_SA_SLOT +#undef USING_GLOBALS +#include "../opt/ssa/cfg/guard_collapse.c" +#undef USING_GLOBALS +#include "../opt/ssa/cfg/phi.c" +#undef USING_GLOBALS +#include "../opt/ssa/cfg/switch_fold.c" +#undef USING_GLOBALS +#include "../opt/ssa/cfg/vrp.c" +#undef USING_GLOBALS +#undef VRP_TOK_EQ +#undef VRP_TOK_GE +#undef VRP_TOK_GT +#undef VRP_TOK_LE +#undef VRP_TOK_LT +#undef VRP_TOK_NE +#undef VRP_TOK_UGE +#undef VRP_TOK_UGT +#undef VRP_TOK_ULE +#undef VRP_TOK_ULT diff --git a/source/unity/opt_ssa_dce.c b/source/unity/opt_ssa_dce.c new file mode 100644 index 000000000..f4b3299c6 --- /dev/null +++ b/source/unity/opt_ssa_dce.c @@ -0,0 +1,38 @@ +/* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */ +#include "../opt/ssa/dce/dce.c" +#undef USING_GLOBALS +#include "../opt/ssa/dce/dce_common.c" +#undef USING_GLOBALS +#include "../opt/ssa/dce/dead_global_stores.c" +#undef GS_PEND_MAX +#undef USING_GLOBALS +#include "../opt/ssa/dce/dead_overwrite_stores.c" +#undef DOS_PEND_MAX +#undef USING_GLOBALS +#include "../opt/ssa/dce/dead_phi_cycles.c" +#undef BM_SET +#undef BM_TEST +#undef MARK_TEMP_LIVE +#undef USING_GLOBALS +#include "../opt/ssa/dce/dead_var_stores.c" +#undef USING_GLOBALS +#include "../opt/ssa/dce/orphan_params.c" +#undef USING_GLOBALS +#include "../opt/ssa/dce/ret_path_frame_store.c" +#undef USING_GLOBALS +#include "../opt/ssa/dce/stackloc_stores.c" +#undef SL_HASH +#undef SL_HASH_SIZE +#undef SL_SET +#undef SL_TEST +#undef USING_GLOBALS +#include "../opt/ssa/dce/temp_worklist.c" +#undef USING_GLOBALS +#include "../opt/ssa/dce/unreachable.c" +#undef DCE_MARK +#undef USING_GLOBALS +#include "../opt/ssa/dce/var_liveness.c" +#undef USING_GLOBALS +#undef VL_BIT +#undef VL_CLR +#undef VL_SET diff --git a/source/unity/opt_ssa_engine.c b/source/unity/opt_ssa_engine.c new file mode 100644 index 000000000..55f9d705c --- /dev/null +++ b/source/unity/opt_ssa_engine.c @@ -0,0 +1,10 @@ +/* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */ +#include "../opt/ssa/engine/driver.c" +#undef SSA_RUN +#undef USING_GLOBALS +#include "../opt/ssa/engine/rewrite.c" +#undef USING_GLOBALS +#include "../opt/ssa/engine/stack_resolve.c" +#undef USING_GLOBALS +#include "../opt/ssa/engine/use_def.c" +#undef USING_GLOBALS diff --git a/source/unity/opt_ssa_loop.c b/source/unity/opt_ssa_loop.c new file mode 100644 index 000000000..35cb742e9 --- /dev/null +++ b/source/unity/opt_ssa_loop.c @@ -0,0 +1,31 @@ +/* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */ +#include "../opt/ssa/loop/dead_loop.c" +#undef DL_MAX_ESCAPE_CANDS +#undef USING_GLOBALS +#include "../opt/ssa/loop/decrement_to_zero.c" +#undef SSA_DTZ_MAX_PASSES +#include "../opt/ssa/loop/first_iter_exit.c" +#undef FIE_EXIT_TEST_LOOKAHEAD +#undef LD_MAX_TMPS +#undef LD_MAX_VARS +#undef LD_TOK_EQ +#undef LD_TOK_NE +#undef SSA_FIRST_ITER_EXIT_MAX_PASSES +#include "../opt/ssa/loop/iv_strength_reduction.c" +#undef SSA_IVSR_MAX_PASSES +#include "../opt/ssa/loop/loop_bottom_test.c" +#undef SSA_LBT_MAX_REWRITES +#include "../opt/ssa/loop/loop_cand.c" +#include "../opt/ssa/loop/loop_const_sim.c" +#undef SSA_LCS_MAX_SPAN +#undef SSA_LOOP_CONST_SIM_MAX_PASSES +#include "../opt/ssa/loop/loop_rotate.c" +#undef SSA_LOOP_ROTATE_MAX_PASSES +#include "../opt/ssa/loop/loop_unroll.c" +#undef CZ_MAX_CHAIN +#undef CZ_MAX_SLOTS +#undef SSA_LOOP_UNROLL_MAX_PASSES +#undef SSA_UNROLL_MAX_SPAN +#include "../opt/ssa/loop/ptr_iv_exit_subst.c" +#undef PTRIV_MAX +#undef SSA_PTR_IV_EXIT_SUBST_MAX_PASSES diff --git a/source/unity/opt_ssa_memory.c b/source/unity/opt_ssa_memory.c new file mode 100644 index 000000000..76a465ec5 --- /dev/null +++ b/source/unity/opt_ssa_memory.c @@ -0,0 +1,33 @@ +/* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */ +#include "../opt/ssa/memory/diamond_store_fwd.c" +#undef USING_GLOBALS +#include "../opt/ssa/memory/global_addr_hoist.c" +#undef GAH_CONST_MIN_MAG +#undef GAH_CONST_TIGHT_BUDGET +#undef GAH_CONST_TIGHT_RELOADS +#undef GAH_LARGE_2ND_RELOADS +#undef GAH_LARGE_FN_INSTRS +#undef GAH_MAX_CALLS_BEFORE +#undef GAH_MAX_HOISTS +#undef GAH_MIN_RELOADS +#undef LAC_MAX_CLASSES +#undef LAC_MAX_FN_INSTRS +#undef LAC_MAX_TEMPS +#undef LAH_CALL_MAX_HOISTS +#undef LAH_CALL_MIN_BUDGET +#undef LAH_MAX_HOISTS +#undef LAH_MAX_LOOPS +#undef LAH_MIN_BUDGET +#include "../opt/ssa/memory/global_store_dse.c" +#undef USING_GLOBALS +#include "../opt/ssa/memory/load_cse.c" +#undef GLOAD_MAX +#undef GSTORE_MAX +#undef ILOAD_MAX +#undef SSTORE_MAX +#undef TVSTORE_MAX +#undef USING_GLOBALS +#undef VSLOT_MAX +#undef resolve_lea_stackloc +#include "../opt/ssa/memory/ptr_store_dse.c" +#undef USING_GLOBALS diff --git a/source/unity/opt_03.c b/source/unity/opt_ssa_scalar.c similarity index 65% rename from source/unity/opt_03.c rename to source/unity/opt_ssa_scalar.c index 465e2dc22..3ab5a11cf 100644 --- a/source/unity/opt_03.c +++ b/source/unity/opt_ssa_scalar.c @@ -1,4 +1,11 @@ /* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */ +#include "../opt/ssa/scalar/bitop_const_fold.c" +#include "../opt/ssa/scalar/bool_norm.c" +#undef K01_MAX_DEPTH +#undef K01_MAX_INSTRS +#undef USING_GLOBALS +#include "../opt/ssa/scalar/cmp_offset_fold.c" +#undef USING_GLOBALS #include "../opt/ssa/scalar/fold.c" #undef USING_GLOBALS #include "../opt/ssa/scalar/gvn.c" @@ -8,24 +15,16 @@ #undef GVN_PURE_CALL_MAX_ARGS #undef GVN_SRC_OK #undef USING_GLOBALS -#include "../opt/ssa/scalar/cprop.c" +#include "../opt/ssa/scalar/narrow.c" +#undef NARROW_FLOAT_TABLE_COUNT +#undef USING_GLOBALS +#include "../opt/ssa/scalar/reassoc.c" #undef USING_GLOBALS -#undef VARF_BIT -#undef VARF_SET #include "../opt/ssa/scalar/setif_or_taut.c" #undef USING_GLOBALS -#include "../opt/ssa/scalar/cmp_offset_fold.c" +#include "../opt/ssa/scalar/strength.c" #undef USING_GLOBALS #include "../opt/ssa/scalar/tmp_block_const.c" #undef USING_GLOBALS -#include "../opt/ssa/cfg/phi.c" -#undef USING_GLOBALS -#include "../opt/ssa/cfg/or_bool_diamond.c" -#undef ORBD_REFS_SLOT -#undef USING_GLOBALS -#include "../opt/ssa/dce/temp_worklist.c" -#undef USING_GLOBALS -#include "../opt/ssa/dce/dead_var_stores.c" -#undef USING_GLOBALS -#include "../opt/ssa/dce/dce_common.c" +#include "../opt/ssa/scalar/var_imm_prop.c" #undef USING_GLOBALS diff --git a/source/unity/opt_ssa_scalar_prop.c b/source/unity/opt_ssa_scalar_prop.c new file mode 100644 index 000000000..d77aab2ec --- /dev/null +++ b/source/unity/opt_ssa_scalar_prop.c @@ -0,0 +1,7 @@ +/* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */ +#include "../opt/ssa/scalar/cprop.c" +#undef USING_GLOBALS +#undef VARF_BIT +#undef VARF_SET +#include "../opt/ssa/scalar/sccp.c" +#undef USING_GLOBALS diff --git a/source/unity/opt_ssa_string.c b/source/unity/opt_ssa_string.c new file mode 100644 index 000000000..6c4e5d87e --- /dev/null +++ b/source/unity/opt_ssa_string.c @@ -0,0 +1,9 @@ +/* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */ +#include "../opt/ssa/string/const_string_fold.c" +#include "../opt/ssa/string/str_memchr.c" +#include "../opt/ssa/string/str_strchr.c" +#include "../opt/ssa/string/str_strcmp.c" +#include "../opt/ssa/string/str_strcpy.c" +#include "../opt/ssa/string/str_strlen.c" +#include "../opt/ssa/string/str_strspn.c" +#include "../opt/ssa/string/str_strstr.c" diff --git a/source/unity/opt_util.c b/source/unity/opt_util.c new file mode 100644 index 000000000..970b07bb3 --- /dev/null +++ b/source/unity/opt_util.c @@ -0,0 +1,29 @@ +/* Generated by scripts/gen_unity.py -- do not edit; rerun it instead. */ +#include "../opt/util/block_scan.c" +#undef USING_GLOBALS +#include "../opt/util/call_params.c" +#undef USING_GLOBALS +#include "../opt/util/cond_util.c" +#undef USING_GLOBALS +#include "../opt/util/const_eval.c" +#undef USING_GLOBALS +#include "../opt/util/const_string_eval.c" +#undef SB_A0 +#undef SB_A1 +#undef USING_GLOBALS +#include "../opt/util/expr_equal.c" +#undef USING_GLOBALS +#include "../opt/util/ir_insert.c" +#undef USING_GLOBALS +#include "../opt/util/pass_disable.c" +#undef USING_GLOBALS +#include "../opt/util/purity.c" +#undef USING_GLOBALS +#include "../opt/util/volatile_access.c" +#undef USING_GLOBALS +#include "../opt/util/vreg_def_use.c" +#undef USING_GLOBALS +#include "../opt/util/vreg_query.c" +#undef USING_GLOBALS +#include "../opt/util/xform.c" +#undef USING_GLOBALS diff --git a/source/unity/unity.mk b/source/unity/unity.mk index 47b4a07c2..530bab464 100644 --- a/source/unity/unity.mk +++ b/source/unity/unity.mk @@ -1,6 +1,6 @@ # Generated by scripts/gen_unity.py -- do not edit; rerun it instead. # Unity-build groups: see the "Unity build" block in the Makefile. -# Budget 6000 source lines per group. +# Membership comes from source/unity/groups.txt. # own TU: source/backend/generators/regalloc.c (expands differently inside a group) # own TU: source/driver/libtcc.c (redefines header macros WARN_ON _tcc_error free gnu_ext realloc tcc_malloc tcc_mallocz tcc_realloc) # own TU: source/frontend/gen/decl/decl.c (redefines header macros gjmp gjmp_addr) @@ -11,412 +11,422 @@ # own TU: source/ir/tccir_operand.c (redefines header macros INT32_MAX INT32_MIN UINT32_MAX) # own TU: source/opt/ssa/cfg/cmp_eq.c (redefines header macros TOK_EQ TOK_NE) -UNITY_GROUPS_frontend = frontend_01 frontend_02 frontend_03 frontend_04 frontend_05 frontend_06 -UNITY_MEMBERS_frontend_01 = \ - source/frontend/tccasm.c \ +UNITY_GROUPS_arm = arm arm_s1 +UNITY_MEMBERS_arm = \ + source/backend/arch/arm/arm.c \ + source/backend/arch/arm/ssa_opt_arm.c +UNITY_MEMBERS_arm_s1 = \ + source/backend/arch/arm/arm-link.c \ + source/backend/arch/arm/arm_aapcs.c \ + source/backend/arch/arm/arm_regalloc.c + +UNITY_GROUPS_thumb = arm_thumb arm_thumb_call arm_thumb_emit arm_thumb_gen arm_thumb_s1 +UNITY_MEMBERS_arm_thumb = \ + source/backend/arch/arm/thumb/thop_bitfield.c \ + source/backend/arch/arm/thumb/thop_mem_imm.c \ + source/backend/arch/arm/thumb/thop_mem_reg.c \ + source/backend/arch/arm/thumb/thop_mov.c \ + source/backend/arch/arm/thumb/thumb-outline.c \ + source/backend/arch/arm/thumb/thumb.c +UNITY_MEMBERS_arm_thumb_call = \ + source/backend/arch/arm/thumb/arm-thumb-call.c \ + source/backend/arch/arm/thumb/arm-thumb-callsite.c +UNITY_MEMBERS_arm_thumb_emit = \ + source/backend/arch/arm/thumb/arm-thumb-emit.c \ + source/backend/arch/arm/thumb/arm-thumb-fp.c \ + source/backend/arch/arm/thumb/arm-thumb-frame.c \ + source/backend/arch/arm/thumb/arm-thumb-mem.c +UNITY_MEMBERS_arm_thumb_gen = \ + source/backend/arch/arm/thumb/arm-thumb-alu.c \ + source/backend/arch/arm/thumb/arm-thumb-gen.c +UNITY_MEMBERS_arm_thumb_s1 = \ + source/backend/arch/arm/thumb/thop_adr.c \ + source/backend/arch/arm/thumb/thop_alu_imm.c \ + source/backend/arch/arm/thumb/thop_alu_reg.c \ + source/backend/arch/arm/thumb/thop_block.c \ + source/backend/arch/arm/thumb/thop_branch.c \ + source/backend/arch/arm/thumb/thop_cmp.c \ + source/backend/arch/arm/thumb/thop_coproc.c \ + source/backend/arch/arm/thumb/thop_dsp.c \ + source/backend/arch/arm/thumb/thop_extend.c \ + source/backend/arch/arm/thumb/thop_ldaex.c \ + source/backend/arch/arm/thumb/thop_ldr_literal.c \ + source/backend/arch/arm/thumb/thop_ldrd.c \ + source/backend/arch/arm/thumb/thop_ldrex.c \ + source/backend/arch/arm/thumb/thop_mem_exclusive.c \ + source/backend/arch/arm/thumb/thop_mem_unpriv.c \ + source/backend/arch/arm/thumb/thop_mrs.c \ + source/backend/arch/arm/thumb/thop_mul.c \ + source/backend/arch/arm/thumb/thop_mvn.c \ + source/backend/arch/arm/thumb/thop_pld.c \ + source/backend/arch/arm/thumb/thop_rev.c \ + source/backend/arch/arm/thumb/thop_shift_imm.c \ + source/backend/arch/arm/thumb/thop_shift_reg.c \ + source/backend/arch/arm/thumb/thop_system.c \ + source/backend/arch/arm/thumb/thop_tbb.c \ + source/backend/arch/arm/thumb/thop_vfp.c + +UNITY_GROUPS_frontend = frontend_gen_builtin frontend_gen_core frontend_gen_decl frontend_gen_expr frontend_gen_inline frontend_gen_op frontend_gen_stmt frontend_gen_store frontend_gen_sym frontend_gen_type frontend_gen_value +UNITY_MEMBERS_frontend_gen_builtin = \ source/frontend/gen/builtin/call.c \ source/frontend/gen/builtin/chk.c \ - source/frontend/gen/builtin/fp2.c \ - source/frontend/gen/core/predicates.c -UNITY_MEMBERS_frontend_02 = \ source/frontend/gen/builtin/fp.c \ + source/frontend/gen/builtin/fp2.c \ source/frontend/gen/builtin/misc.c \ source/frontend/gen/builtin/overflow.c \ source/frontend/gen/builtin/simd.c \ - source/frontend/gen/builtin/string.c \ + source/frontend/gen/builtin/string.c +UNITY_MEMBERS_frontend_gen_core = \ + source/frontend/gen/core/predicates.c \ source/frontend/gen/core/state.c \ - source/frontend/gen/core/suppress.c \ + source/frontend/gen/core/suppress.c +UNITY_MEMBERS_frontend_gen_decl = \ source/frontend/gen/decl/attribute.c \ source/frontend/gen/decl/btype.c \ source/frontend/gen/decl/declarator.c \ source/frontend/gen/decl/predef_protos.c \ + source/frontend/gen/decl/struct.c +UNITY_MEMBERS_frontend_gen_expr = \ source/frontend/gen/expr/atomic.c \ - source/frontend/gen/value/strlit_pool.c -UNITY_MEMBERS_frontend_03 = \ - source/frontend/gen/decl/struct.c \ source/frontend/gen/expr/cond.c \ source/frontend/gen/expr/indir.c \ source/frontend/gen/expr/infix.c \ source/frontend/gen/expr/primary.c \ - source/frontend/gen/expr/unary.c \ - source/frontend/gen/init/alloc.c \ - source/frontend/gen/op/float.c \ - source/frontend/gen/sym/elfsym.c -UNITY_MEMBERS_frontend_04 = \ + source/frontend/gen/expr/unary.c +UNITY_MEMBERS_frontend_gen_inline = \ source/frontend/gen/inline/analysis.c \ source/frontend/gen/inline/const_eval.c \ - source/frontend/gen/inline/emit.c \ - source/frontend/gen/nested/nested.c \ + source/frontend/gen/inline/emit.c +UNITY_MEMBERS_frontend_gen_op = \ source/frontend/gen/op/complex.c \ - source/frontend/gen/op/fold_math.c -UNITY_MEMBERS_frontend_05 = \ + source/frontend/gen/op/float.c \ + source/frontend/gen/op/fold_math.c \ source/frontend/gen/op/int.c \ source/frontend/gen/op/op.c \ - source/frontend/gen/op/vector.c \ + source/frontend/gen/op/vector.c +UNITY_MEMBERS_frontend_gen_stmt = \ source/frontend/gen/stmt/block.c \ source/frontend/gen/stmt/cleanup.c \ source/frontend/gen/stmt/ret.c \ - source/frontend/gen/stmt/switch.c \ + source/frontend/gen/stmt/switch.c +UNITY_MEMBERS_frontend_gen_store = \ source/frontend/gen/store/struct_copy.c \ - source/frontend/gen/store/vstore.c \ - source/frontend/gen/sym/attr_merge.c -UNITY_MEMBERS_frontend_06 = \ - source/frontend/gen/sym/symtab.c \ + source/frontend/gen/store/vstore.c +UNITY_MEMBERS_frontend_gen_sym = \ + source/frontend/gen/sym/attr_merge.c \ + source/frontend/gen/sym/elfsym.c \ + source/frontend/gen/sym/symtab.c +UNITY_MEMBERS_frontend_gen_type = \ source/frontend/gen/type/assign_check.c \ source/frontend/gen/type/cast.c \ - source/frontend/gen/type/size.c \ + source/frontend/gen/type/size.c +UNITY_MEMBERS_frontend_gen_value = \ source/frontend/gen/value/load.c \ source/frontend/gen/value/longlong.c \ + source/frontend/gen/value/strlit_pool.c \ source/frontend/gen/value/vstack.c -UNITY_GROUPS_ir = ir_01 ir_02 ir_03 ir_04 ir_05 -UNITY_MEMBERS_ir_01 = \ - source/ir/type.c \ +UNITY_GROUPS_ir = ir ir_frame ir_gen ir_regalloc ir_regalloc_ssa +UNITY_MEMBERS_ir = \ + source/ir/cfg.c \ + source/ir/dead_def.c \ + source/ir/dump.c \ + source/ir/machine_op.c \ source/ir/pool.c \ - source/ir/vreg.c \ - source/ir/frame.c \ - source/ir/cross_jump.c \ source/ir/self_store.c \ - source/ir/dead_def.c \ - source/ir/dump.c -UNITY_MEMBERS_ir_02 = \ - source/ir/codegen.c \ - source/ir/gen/control.c \ - source/ir/gen/vla.c \ - source/ir/gen/util.c -UNITY_MEMBERS_ir_03 = \ - source/ir/cfg.c \ source/ir/ssa.c \ - source/ir/regalloc.c \ - source/ir/regalloc_scan.c \ - source/ir/machine_op.c \ - source/ir/gen/state.c -UNITY_MEMBERS_ir_04 = \ - source/ir/regalloc_phi.c \ - source/ir/regalloc_entry.c \ - source/ir/regalloc_post.c -UNITY_MEMBERS_ir_05 = \ - source/ir/gen/put.c \ - source/ir/gen/params.c \ + source/ir/type.c \ + source/ir/vreg.c +UNITY_MEMBERS_ir_frame = \ + source/ir/cross_jump.c \ + source/ir/frame.c +UNITY_MEMBERS_ir_gen = \ source/ir/gen/arith.c \ + source/ir/gen/asm.c \ + source/ir/gen/config.c \ + source/ir/gen/control.c \ source/ir/gen/float.c \ source/ir/gen/jump.c \ - source/ir/gen/asm.c \ + source/ir/gen/live.c \ + source/ir/gen/params.c \ + source/ir/gen/put.c \ source/ir/gen/softfloat.c \ - source/ir/gen/config.c \ - source/ir/gen/live.c + source/ir/gen/state.c \ + source/ir/gen/util.c \ + source/ir/gen/vla.c +UNITY_MEMBERS_ir_regalloc = \ + source/ir/regalloc.c \ + source/ir/regalloc_post.c \ + source/ir/regalloc_scan.c +UNITY_MEMBERS_ir_regalloc_ssa = \ + source/ir/regalloc_entry.c \ + source/ir/regalloc_phi.c -UNITY_GROUPS_machine = machine_01 -UNITY_MEMBERS_machine_01 = \ +UNITY_GROUPS_machine = machine_s1 +UNITY_MEMBERS_machine_s1 = \ source/machine/tccls.c \ source/machine/tccmachine.c -UNITY_GROUPS_memory = memory_01 -UNITY_MEMBERS_memory_01 = \ +UNITY_GROUPS_memory = memory_s1 +UNITY_MEMBERS_memory_s1 = \ source/memory/unique_ptr.c \ source/memory/vector.c -UNITY_GROUPS_obj = obj_01 obj_02 obj_03 -UNITY_MEMBERS_obj_01 = \ +UNITY_GROUPS_obj = obj_elf_s1 obj_s1 +UNITY_MEMBERS_obj_elf_s1 = \ source/obj/tccelf.c \ - source/obj/tccelf_sym.c \ - source/obj/tccelf_reloc.c \ source/obj/tccelf_layout.c \ - source/obj/tccelf_output.c -UNITY_MEMBERS_obj_02 = \ - source/obj/tccelf_load.c \ source/obj/tccelf_ld.c \ - source/obj/tccld.c -UNITY_MEMBERS_obj_03 = \ - source/obj/tccyaff.c \ - source/obj/tccdbg.c + source/obj/tccelf_load.c \ + source/obj/tccelf_output.c \ + source/obj/tccelf_reloc.c \ + source/obj/tccelf_sym.c +UNITY_MEMBERS_obj_s1 = \ + source/obj/tccdbg.c \ + source/obj/tccld.c \ + source/obj/tccyaff.c -UNITY_GROUPS_opt = opt_01 opt_02 opt_03 opt_04 opt_05 opt_06 opt_07 opt_08 opt_09 opt_10 opt_11 opt_12 opt_13 opt_14 opt_15 opt_16 -UNITY_MEMBERS_opt_01 = \ - source/opt/util/pass_disable.c \ - source/opt/util/volatile_access.c \ - source/opt/util/const_eval.c \ - source/opt/util/const_string_eval.c \ - source/opt/util/cond_util.c \ - source/opt/util/block_scan.c \ - source/opt/util/purity.c \ - source/opt/util/expr_equal.c \ - source/opt/util/call_params.c \ - source/opt/util/vreg_query.c \ - source/opt/util/vreg_def_use.c \ - source/opt/util/xform.c \ - source/opt/util/ir_insert.c \ - source/opt/analysis/du_chains.c \ +UNITY_GROUPS_opt = opt_analysis opt_engine opt_flat_cfg opt_flat_dce opt_flat_fusion opt_flat_ipa opt_flat_loop opt_flat_loop_iv opt_flat_memory opt_flat_memory_cse opt_flat_memory_dse opt_flat_memory_fwd opt_flat_scalar opt_flat_scalar_const opt_flat_scalar_known opt_ra opt_ssa_cfg opt_ssa_dce opt_ssa_engine opt_ssa_loop opt_ssa_memory opt_ssa_scalar opt_ssa_scalar_prop opt_ssa_string opt_util +UNITY_MEMBERS_opt_analysis = \ source/opt/analysis/alias.c \ - source/opt/analysis/memref.c \ + source/opt/analysis/du_chains.c \ source/opt/analysis/mem_ssa.c \ + source/opt/analysis/memref.c +UNITY_MEMBERS_opt_engine = \ source/opt/engine/ctx.c \ - source/opt/engine/run_gens.c \ - source/opt/engine/pipeline_run.c \ - source/opt/engine/pipeline_table.c \ - source/opt/engine/gen_adapters.c \ - source/opt/engine/pass_timing.c \ source/opt/engine/fp_cache_shim.c \ source/opt/engine/fp_mat_cache.c \ + source/opt/engine/gen_adapters.c \ source/opt/engine/pass_registry.c \ - source/opt/ra/const_branch_fold.c \ - source/opt/ra/incomplete_calls.c \ - source/opt/ra/phi_const_chain.c \ - source/opt/ssa/string/const_string_fold.c \ - source/opt/ssa/string/str_strlen.c \ - source/opt/flat/memory/mem_init.c -UNITY_MEMBERS_opt_02 = \ - source/opt/ra/umaal_fusion.c \ - source/opt/ssa/string/str_strcmp.c \ - source/opt/ssa/string/str_strcpy.c \ - source/opt/ssa/string/str_strspn.c \ - source/opt/ssa/string/str_strchr.c \ - source/opt/ssa/string/str_strstr.c \ - source/opt/ssa/string/str_memchr.c \ - source/opt/ssa/memory/global_addr_hoist.c \ - source/opt/ssa/memory/load_cse.c \ - source/opt/ssa/memory/diamond_store_fwd.c \ - source/opt/ssa/scalar/bitop_const_fold.c \ - source/opt/ssa/scalar/strength.c \ - source/opt/ssa/scalar/reassoc.c \ - source/opt/ssa/scalar/narrow.c \ - source/opt/ssa/scalar/var_imm_prop.c \ - source/opt/ssa/scalar/bool_norm.c \ - source/opt/ssa/cfg/switch_fold.c \ - source/opt/ssa/dce/unreachable.c -UNITY_MEMBERS_opt_03 = \ - source/opt/ssa/scalar/fold.c \ - source/opt/ssa/scalar/gvn.c \ - source/opt/ssa/scalar/cprop.c \ - source/opt/ssa/scalar/setif_or_taut.c \ - source/opt/ssa/scalar/cmp_offset_fold.c \ - source/opt/ssa/scalar/tmp_block_const.c \ - source/opt/ssa/cfg/phi.c \ - source/opt/ssa/cfg/or_bool_diamond.c \ - source/opt/ssa/dce/temp_worklist.c \ - source/opt/ssa/dce/dead_var_stores.c \ - source/opt/ssa/dce/dce_common.c -UNITY_MEMBERS_opt_04 = \ - source/opt/ssa/cfg/branch.c \ - source/opt/ssa/cfg/vrp.c \ - source/opt/ssa/dce/stackloc_stores.c \ - source/opt/ssa/dce/orphan_params.c \ - source/opt/ssa/dce/ret_path_frame_store.c \ - source/opt/ssa/dce/dead_phi_cycles.c \ - source/opt/ssa/dce/dead_overwrite_stores.c \ - source/opt/ssa/dce/dead_global_stores.c \ - source/opt/ssa/dce/var_liveness.c \ - source/opt/ssa/dce/dce.c \ - source/opt/ssa/loop/loop_rotate.c \ - source/opt/ssa/loop/first_iter_exit.c \ - source/opt/ssa/loop/loop_cand.c \ - source/opt/ssa/loop/loop_const_sim.c \ - source/opt/ssa/loop/iv_strength_reduction.c \ - source/opt/ssa/loop/decrement_to_zero.c -UNITY_MEMBERS_opt_05 = \ - source/opt/ssa/loop/ptr_iv_exit_subst.c \ - source/opt/ssa/loop/loop_unroll.c \ - source/opt/ssa/loop/loop_bottom_test.c \ - source/opt/ssa/loop/dead_loop.c \ - source/opt/ssa/scalar/sccp.c \ - source/opt/ssa/memory/global_store_dse.c \ - source/opt/ssa/memory/ptr_store_dse.c \ - source/opt/ssa/cfg/guard_collapse.c \ - source/opt/ssa/engine/use_def.c \ - source/opt/ssa/engine/rewrite.c \ - source/opt/ssa/engine/driver.c \ - source/opt/flat/scalar/bool.c \ - source/opt/flat/scalar/narrow_store.c -UNITY_MEMBERS_opt_06 = \ - source/opt/ssa/engine/stack_resolve.c \ - source/opt/flat/scalar/branch.c \ - source/opt/flat/scalar/stack_bool_diamond.c \ - source/opt/flat/scalar/const_string_calls.c \ - source/opt/flat/scalar/const_call_fold.c \ - source/opt/flat/scalar/addrof_const_fold.c \ - source/opt/flat/scalar/global_init_prop.c \ - source/opt/flat/scalar/symref_const_prop.c \ - source/opt/flat/scalar/stack_addr_fold.c \ - source/opt/flat/scalar/call_result.c \ - source/opt/flat/scalar/self_arith.c \ - source/opt/flat/scalar/self_copy.c \ - source/opt/flat/scalar/var_tmp_fwd.c \ - source/opt/flat/scalar/inline_param_copy.c \ - source/opt/flat/scalar/switch_collapse.c \ - source/opt/flat/fusion/indexed_chain.c \ - source/opt/flat/dce/trap_only_body_suppress.c -UNITY_MEMBERS_opt_07 = \ - source/opt/flat/scalar/known_bits.c \ - source/opt/flat/scalar/const_prop_tmp.c \ - source/opt/flat/scalar/value_tracking.c \ - source/opt/flat/scalar/const_var_prop.c \ - source/opt/flat/scalar/const_aggregate.c \ - source/opt/flat/scalar/add_reassoc.c \ - source/opt/flat/fusion/pair_reorder.c -UNITY_MEMBERS_opt_08 = \ - source/opt/flat/scalar/cmp_field_fuse.c \ - source/opt/flat/scalar/cmp_expr_fold.c \ - source/opt/flat/scalar/bitfield.c \ - source/opt/flat/memory/symaddr_cse.c \ - source/opt/flat/memory/dse.c \ - source/opt/flat/memory/var_store_elim.c \ - source/opt/flat/memory/entry_store_prop.c \ - source/opt/flat/memory/sret_nrvo.c \ - source/opt/flat/memory/dead_temp_local.c -UNITY_MEMBERS_opt_09 = \ - source/opt/flat/memory/sl_forward.c \ - source/opt/flat/memory/copy_source_load_fwd.c \ - source/opt/flat/memory/dead_local_slot.c \ - source/opt/flat/memory/dead_static_store.c -UNITY_MEMBERS_opt_10 = \ - source/opt/flat/memory/global_base_share.c \ - source/opt/flat/memory/byte_store_merge.c \ - source/opt/flat/memory/mem_inline.c \ - source/opt/flat/fusion/disp.c \ - source/opt/flat/fusion/gens_fusion.c \ - source/opt/flat/fusion/add_deref_fold.c \ - source/opt/flat/fusion/call_chain_rename.c \ - source/opt/flat/fusion/stackoff_addr_cse.c \ - source/opt/flat/fusion/lea_fold.c \ - source/opt/flat/fusion/lea_rmw_fold.c \ - source/opt/flat/fusion/assign_fuse.c \ - source/opt/flat/fusion/pack64.c \ - source/opt/flat/fusion/pack64_tautology.c \ - source/opt/flat/fusion/cmp_narrow_64.c \ - source/opt/flat/fusion/cmp_hi_only.c \ - source/opt/flat/fusion/cmp_imm_swap.c -UNITY_MEMBERS_opt_11 = \ - source/opt/flat/fusion/and64_narrow.c \ - source/opt/flat/fusion/shl32_or_chain.c \ - source/opt/flat/scalar/sdiv_pow2.c \ - source/opt/flat/memory/dead_lea_store.c \ - source/opt/flat/memory/store_inplace_arith.c \ - source/opt/flat/scalar/redundant_loop_check.c \ + source/opt/engine/pass_timing.c \ + source/opt/engine/pipeline_run.c \ + source/opt/engine/pipeline_table.c \ + source/opt/engine/run_gens.c +UNITY_MEMBERS_opt_flat_cfg = \ + source/opt/flat/cfg/abort_tail_merge.c \ + source/opt/flat/cfg/bool_diamond_branch.c \ source/opt/flat/cfg/if_convert.c \ + source/opt/flat/cfg/jump_thread.c \ + source/opt/flat/cfg/jumpif_invert.c \ + source/opt/flat/cfg/returnvalue_merge.c \ + source/opt/flat/cfg/switch_to_data.c +UNITY_MEMBERS_opt_flat_dce = \ source/opt/flat/dce/body_essential.c \ - source/opt/flat/dce/noreturn_collapse.c \ + source/opt/flat/dce/compact_nops.c \ + source/opt/flat/dce/const_return_uninit_elide.c \ + source/opt/flat/dce/dce.c \ + source/opt/flat/dce/dead_addrvar_elim.c \ + source/opt/flat/dce/dead_trailing_addrvar_store.c \ + source/opt/flat/dce/dead_vla.c \ source/opt/flat/dce/infinite_loop_simplify.c \ source/opt/flat/dce/infinite_self_recursion.c \ source/opt/flat/dce/noreturn_call_epilogue.c \ - source/opt/flat/dce/dead_addrvar_elim.c \ - source/opt/flat/dce/dead_trailing_addrvar_store.c \ - source/opt/flat/dce/uninit_ub.c \ - source/opt/flat/cfg/returnvalue_merge.c -UNITY_MEMBERS_opt_12 = \ - source/opt/flat/dce/ub_only_body_elide.c \ - source/opt/flat/dce/const_return_uninit_elide.c \ + source/opt/flat/dce/noreturn_collapse.c \ source/opt/flat/dce/null_store_dom_return.c \ - source/opt/flat/dce/dce.c \ - source/opt/flat/dce/compact_nops.c \ source/opt/flat/dce/orphan_cmp_elim.c \ source/opt/flat/dce/redundant_cmp.c \ - source/opt/flat/dce/zero_vla_elim.c \ - source/opt/flat/loop/loop_ir_mutate.c \ - source/opt/flat/loop/iv_analysis.c \ - source/opt/flat/loop/iv_strength_reduction.c \ - source/opt/flat/loop/loop_exit_analysis.c \ - source/opt/flat/loop/loop_eliminate.c \ - source/opt/flat/loop/decrement_to_zero.c -UNITY_MEMBERS_opt_13 = \ - source/opt/flat/loop/loop_unroll.c \ - source/opt/flat/loop/loop_rotate.c \ - source/opt/flat/loop/loop_relayout.c \ - source/opt/flat/loop/seq_guard_elim.c \ - source/opt/flat/loop/const_sim.c \ - source/opt/flat/loop/reroll.c \ - source/opt/flat/cfg/switch_to_data.c \ - source/opt/flat/cfg/jump_thread.c \ - source/opt/flat/cfg/bool_diamond_branch.c -UNITY_MEMBERS_opt_14 = \ - source/opt/flat/dce/dead_vla.c \ - source/opt/flat/cfg/jumpif_invert.c \ - source/opt/flat/scalar/var_to_tmp.c \ - source/opt/flat/loop/backedge_phi_hoist.c \ - source/opt/flat/cfg/abort_tail_merge.c \ - source/opt/flat/loop/loop_detect.c \ - source/opt/flat/loop/func_purity.c \ - source/opt/flat/loop/licm.c \ - source/opt/flat/loop/stack_param_promote.c \ - source/opt/flat/memory/invariant_global_load_hoist.c \ - source/opt/flat/memory/global_deref_cse.c \ - source/opt/flat/ipa/tu_noreturn.c -UNITY_MEMBERS_opt_15 = \ - source/opt/flat/memory/deref_operand_cse.c \ - source/opt/flat/memory/bitfield_unit_narrow.c \ - source/opt/flat/memory/rmw_byte_clear.c \ - source/opt/flat/memory/struct_copy_roundtrip_elim.c \ + source/opt/flat/dce/trap_only_body_suppress.c \ + source/opt/flat/dce/ub_only_body_elide.c \ + source/opt/flat/dce/uninit_ub.c \ + source/opt/flat/dce/zero_vla_elim.c +UNITY_MEMBERS_opt_flat_fusion = \ + source/opt/flat/fusion/add_deref_fold.c \ + source/opt/flat/fusion/and64_narrow.c \ + source/opt/flat/fusion/assign_fuse.c \ source/opt/flat/fusion/barrel_shift.c \ - source/opt/flat/fusion/shift_pair_ubfx.c \ + source/opt/flat/fusion/call_chain_rename.c \ + source/opt/flat/fusion/cmp_hi_only.c \ + source/opt/flat/fusion/cmp_imm_swap.c \ + source/opt/flat/fusion/cmp_narrow_64.c \ + source/opt/flat/fusion/disp.c \ + source/opt/flat/fusion/gens_fusion.c \ + source/opt/flat/fusion/indexed_chain.c \ + source/opt/flat/fusion/lea_fold.c \ + source/opt/flat/fusion/lea_rmw_fold.c \ + source/opt/flat/fusion/pack64.c \ + source/opt/flat/fusion/pack64_tautology.c \ + source/opt/flat/fusion/pair_reorder.c \ source/opt/flat/fusion/shift64_dead_half.c \ - source/opt/flat/fusion/zero_half64.c \ source/opt/flat/fusion/shift64_extract_ubfx.c \ - source/opt/flat/ipa/pure_via_sret.c \ + source/opt/flat/fusion/shift_pair_ubfx.c \ + source/opt/flat/fusion/shl32_or_chain.c \ + source/opt/flat/fusion/stackoff_addr_cse.c \ + source/opt/flat/fusion/zero_half64.c +UNITY_MEMBERS_opt_flat_ipa = \ source/opt/flat/ipa/func_write_summary.c \ - source/opt/flat/ipa/tu_func_summary.c \ + source/opt/flat/ipa/pure_via_sret.c \ source/opt/flat/ipa/tu_dead_statics.c \ - source/opt/flat/memory/dead_init_via_call.c \ - source/opt/flat/memory/stack_addr_cse.c \ + source/opt/flat/ipa/tu_func_summary.c \ + source/opt/flat/ipa/tu_noreturn.c +UNITY_MEMBERS_opt_flat_loop = \ + source/opt/flat/loop/const_sim.c \ + source/opt/flat/loop/func_purity.c \ + source/opt/flat/loop/loop_detect.c \ + source/opt/flat/loop/loop_eliminate.c \ + source/opt/flat/loop/loop_exit_analysis.c \ + source/opt/flat/loop/loop_ir_mutate.c \ + source/opt/flat/loop/loop_relayout.c \ + source/opt/flat/loop/loop_rotate.c \ + source/opt/flat/loop/loop_unroll.c \ + source/opt/flat/loop/reroll.c \ + source/opt/flat/loop/seq_guard_elim.c +UNITY_MEMBERS_opt_flat_loop_iv = \ + source/opt/flat/loop/backedge_phi_hoist.c \ + source/opt/flat/loop/decrement_to_zero.c \ + source/opt/flat/loop/iv_analysis.c \ + source/opt/flat/loop/iv_strength_reduction.c \ + source/opt/flat/loop/licm.c \ + source/opt/flat/loop/stack_param_promote.c +UNITY_MEMBERS_opt_flat_memory = \ source/opt/flat/memory/block_copy_init.c \ - source/opt/flat/memory/small_memset_to_store.c -UNITY_MEMBERS_opt_16 = \ source/opt/flat/memory/const_local_table.c \ + source/opt/flat/memory/mem_init.c \ + source/opt/flat/memory/mem_inline.c \ + source/opt/flat/memory/memmove_to_indexed_stores.c \ source/opt/flat/memory/small_global_memset_to_store.c \ + source/opt/flat/memory/small_memset_to_store.c \ + source/opt/flat/memory/sra.c \ + source/opt/flat/memory/sret_nrvo.c \ + source/opt/flat/memory/struct_copy_roundtrip_elim.c +UNITY_MEMBERS_opt_flat_memory_cse = \ + source/opt/flat/memory/deref_operand_cse.c \ + source/opt/flat/memory/global_base_share.c \ + source/opt/flat/memory/global_deref_cse.c \ + source/opt/flat/memory/invariant_global_load_hoist.c \ + source/opt/flat/memory/stack_addr_cse.c \ + source/opt/flat/memory/symaddr_cse.c +UNITY_MEMBERS_opt_flat_memory_dse = \ + source/opt/flat/memory/bitfield_unit_narrow.c \ + source/opt/flat/memory/byte_store_merge.c \ + source/opt/flat/memory/dead_init_via_call.c \ + source/opt/flat/memory/dead_lea_store.c \ + source/opt/flat/memory/dead_local_slot.c \ + source/opt/flat/memory/dead_static_store.c \ + source/opt/flat/memory/dead_temp_local.c \ + source/opt/flat/memory/dse.c \ + source/opt/flat/memory/rmw_byte_clear.c \ + source/opt/flat/memory/slot_const_store_fold.c \ + source/opt/flat/memory/store_inplace_arith.c \ + source/opt/flat/memory/var_store_elim.c +UNITY_MEMBERS_opt_flat_memory_fwd = \ source/opt/flat/memory/addrof_var_fwd.c \ - source/opt/flat/memory/ptr_local_fwd.c \ + source/opt/flat/memory/copy_source_load_fwd.c \ + source/opt/flat/memory/entry_store_prop.c \ source/opt/flat/memory/param_copy_alias.c \ - source/opt/flat/memory/slot_const_store_fold.c \ - source/opt/flat/memory/sra.c \ - source/opt/flat/memory/memmove_to_indexed_stores.c - -UNITY_GROUPS_arm = arm_01 arm_02 -UNITY_MEMBERS_arm_01 = \ - source/backend/arch/arm/arm.c \ - source/backend/arch/arm/ssa_opt_arm.c -UNITY_MEMBERS_arm_02 = \ - source/backend/arch/arm/arm_aapcs.c \ - source/backend/arch/arm/arm_regalloc.c \ - source/backend/arch/arm/arm-link.c - -UNITY_GROUPS_thumb = thumb_01 thumb_02 thumb_03 thumb_04 -UNITY_MEMBERS_thumb_01 = \ - source/backend/arch/arm/thumb/thumb.c \ - source/backend/arch/arm/thumb/thop_mem_reg.c \ - source/backend/arch/arm/thumb/thop_mem_imm.c \ - source/backend/arch/arm/thumb/thop_bitfield.c \ - source/backend/arch/arm/thumb/thop_mov.c \ - source/backend/arch/arm/thumb/arm-thumb-gen.c \ - source/backend/arch/arm/thumb/arm-thumb-callsite.c -UNITY_MEMBERS_thumb_02 = \ - source/backend/arch/arm/thumb/thop_alu_imm.c \ - source/backend/arch/arm/thumb/thop_alu_reg.c \ - source/backend/arch/arm/thumb/thop_cmp.c \ - source/backend/arch/arm/thumb/thop_shift_imm.c \ - source/backend/arch/arm/thumb/thop_shift_reg.c \ - source/backend/arch/arm/thumb/thop_mem_unpriv.c \ - source/backend/arch/arm/thumb/thop_mem_exclusive.c \ - source/backend/arch/arm/thumb/thop_extend.c \ - source/backend/arch/arm/thumb/thop_rev.c \ - source/backend/arch/arm/thumb/thop_vfp.c \ - source/backend/arch/arm/thumb/thop_coproc.c \ - source/backend/arch/arm/thumb/thop_system.c \ - source/backend/arch/arm/thumb/thop_branch.c \ - source/backend/arch/arm/thumb/thop_block.c \ - source/backend/arch/arm/thumb/thop_mul.c \ - source/backend/arch/arm/thumb/thop_dsp.c \ - source/backend/arch/arm/thumb/thop_tbb.c \ - source/backend/arch/arm/thumb/thop_ldrd.c \ - source/backend/arch/arm/thumb/thop_ldrex.c \ - source/backend/arch/arm/thumb/thop_ldaex.c \ - source/backend/arch/arm/thumb/thop_mrs.c \ - source/backend/arch/arm/thumb/thop_pld.c \ - source/backend/arch/arm/thumb/thop_mvn.c \ - source/backend/arch/arm/thumb/thop_adr.c \ - source/backend/arch/arm/thumb/thop_ldr_literal.c -UNITY_MEMBERS_thumb_03 = \ - source/backend/arch/arm/thumb/arm-thumb-emit.c \ - source/backend/arch/arm/thumb/arm-thumb-alu.c -UNITY_MEMBERS_thumb_04 = \ - source/backend/arch/arm/thumb/arm-thumb-mem.c \ - source/backend/arch/arm/thumb/arm-thumb-fp.c \ - source/backend/arch/arm/thumb/arm-thumb-frame.c \ - source/backend/arch/arm/thumb/thumb-outline.c + source/opt/flat/memory/ptr_local_fwd.c \ + source/opt/flat/memory/sl_forward.c +UNITY_MEMBERS_opt_flat_scalar = \ + source/opt/flat/scalar/add_reassoc.c \ + source/opt/flat/scalar/bitfield.c \ + source/opt/flat/scalar/bool.c \ + source/opt/flat/scalar/call_result.c \ + source/opt/flat/scalar/cmp_field_fuse.c \ + source/opt/flat/scalar/inline_param_copy.c \ + source/opt/flat/scalar/narrow_store.c \ + source/opt/flat/scalar/sdiv_pow2.c \ + source/opt/flat/scalar/self_arith.c \ + source/opt/flat/scalar/self_copy.c \ + source/opt/flat/scalar/stack_bool_diamond.c \ + source/opt/flat/scalar/var_tmp_fwd.c \ + source/opt/flat/scalar/var_to_tmp.c +UNITY_MEMBERS_opt_flat_scalar_const = \ + source/opt/flat/scalar/addrof_const_fold.c \ + source/opt/flat/scalar/cmp_expr_fold.c \ + source/opt/flat/scalar/const_aggregate.c \ + source/opt/flat/scalar/const_call_fold.c \ + source/opt/flat/scalar/const_prop_tmp.c \ + source/opt/flat/scalar/const_string_calls.c \ + source/opt/flat/scalar/const_var_prop.c \ + source/opt/flat/scalar/global_init_prop.c \ + source/opt/flat/scalar/stack_addr_fold.c \ + source/opt/flat/scalar/switch_collapse.c \ + source/opt/flat/scalar/symref_const_prop.c +UNITY_MEMBERS_opt_flat_scalar_known = \ + source/opt/flat/scalar/branch.c \ + source/opt/flat/scalar/known_bits.c \ + source/opt/flat/scalar/redundant_loop_check.c \ + source/opt/flat/scalar/value_tracking.c +UNITY_MEMBERS_opt_ra = \ + source/opt/ra/const_branch_fold.c \ + source/opt/ra/incomplete_calls.c \ + source/opt/ra/phi_const_chain.c \ + source/opt/ra/umaal_fusion.c +UNITY_MEMBERS_opt_ssa_cfg = \ + source/opt/ssa/cfg/branch.c \ + source/opt/ssa/cfg/guard_collapse.c \ + source/opt/ssa/cfg/phi.c \ + source/opt/ssa/cfg/switch_fold.c \ + source/opt/ssa/cfg/vrp.c +UNITY_MEMBERS_opt_ssa_dce = \ + source/opt/ssa/dce/dce.c \ + source/opt/ssa/dce/dce_common.c \ + source/opt/ssa/dce/dead_global_stores.c \ + source/opt/ssa/dce/dead_overwrite_stores.c \ + source/opt/ssa/dce/dead_phi_cycles.c \ + source/opt/ssa/dce/dead_var_stores.c \ + source/opt/ssa/dce/orphan_params.c \ + source/opt/ssa/dce/ret_path_frame_store.c \ + source/opt/ssa/dce/stackloc_stores.c \ + source/opt/ssa/dce/temp_worklist.c \ + source/opt/ssa/dce/unreachable.c \ + source/opt/ssa/dce/var_liveness.c +UNITY_MEMBERS_opt_ssa_engine = \ + source/opt/ssa/engine/driver.c \ + source/opt/ssa/engine/rewrite.c \ + source/opt/ssa/engine/stack_resolve.c \ + source/opt/ssa/engine/use_def.c +UNITY_MEMBERS_opt_ssa_loop = \ + source/opt/ssa/loop/dead_loop.c \ + source/opt/ssa/loop/decrement_to_zero.c \ + source/opt/ssa/loop/first_iter_exit.c \ + source/opt/ssa/loop/iv_strength_reduction.c \ + source/opt/ssa/loop/loop_bottom_test.c \ + source/opt/ssa/loop/loop_cand.c \ + source/opt/ssa/loop/loop_const_sim.c \ + source/opt/ssa/loop/loop_rotate.c \ + source/opt/ssa/loop/loop_unroll.c \ + source/opt/ssa/loop/ptr_iv_exit_subst.c +UNITY_MEMBERS_opt_ssa_memory = \ + source/opt/ssa/memory/diamond_store_fwd.c \ + source/opt/ssa/memory/global_addr_hoist.c \ + source/opt/ssa/memory/global_store_dse.c \ + source/opt/ssa/memory/load_cse.c \ + source/opt/ssa/memory/ptr_store_dse.c +UNITY_MEMBERS_opt_ssa_scalar = \ + source/opt/ssa/scalar/bitop_const_fold.c \ + source/opt/ssa/scalar/bool_norm.c \ + source/opt/ssa/scalar/cmp_offset_fold.c \ + source/opt/ssa/scalar/fold.c \ + source/opt/ssa/scalar/gvn.c \ + source/opt/ssa/scalar/narrow.c \ + source/opt/ssa/scalar/reassoc.c \ + source/opt/ssa/scalar/setif_or_taut.c \ + source/opt/ssa/scalar/strength.c \ + source/opt/ssa/scalar/tmp_block_const.c \ + source/opt/ssa/scalar/var_imm_prop.c +UNITY_MEMBERS_opt_ssa_scalar_prop = \ + source/opt/ssa/scalar/cprop.c \ + source/opt/ssa/scalar/sccp.c +UNITY_MEMBERS_opt_ssa_string = \ + source/opt/ssa/string/const_string_fold.c \ + source/opt/ssa/string/str_memchr.c \ + source/opt/ssa/string/str_strchr.c \ + source/opt/ssa/string/str_strcmp.c \ + source/opt/ssa/string/str_strcpy.c \ + source/opt/ssa/string/str_strlen.c \ + source/opt/ssa/string/str_strspn.c \ + source/opt/ssa/string/str_strstr.c +UNITY_MEMBERS_opt_util = \ + source/opt/util/block_scan.c \ + source/opt/util/call_params.c \ + source/opt/util/cond_util.c \ + source/opt/util/const_eval.c \ + source/opt/util/const_string_eval.c \ + source/opt/util/expr_equal.c \ + source/opt/util/ir_insert.c \ + source/opt/util/pass_disable.c \ + source/opt/util/purity.c \ + source/opt/util/volatile_access.c \ + source/opt/util/vreg_def_use.c \ + source/opt/util/vreg_query.c \ + source/opt/util/xform.c diff --git a/tests/ir_tests/608_barrel_shift_join_multidef.c b/tests/ir_tests/608_barrel_shift_join_multidef.c new file mode 100644 index 000000000..3c062c189 --- /dev/null +++ b/tests/ir_tests/608_barrel_shift_join_multidef.c @@ -0,0 +1,65 @@ +// ctags' isTagExtraBitMarked: `index` is a signed divide by 8 in both arms of +// an if, and the join adds it to a pointer. The barrel-shift fusion sees the +// flat IR before SSA, where `index` has one def per arm; it folded the else +// arm's `asr #3` into the join's ADD, so the then arm reached it with the +// shift's source register never written and ctags faulted on the device. +#include + +typedef struct +{ + int pad[9]; + unsigned char extra[2]; + unsigned char *dyn; +} entry; + +__attribute__((noinline)) static int marked(const entry *e, int bit) +{ + unsigned index, offset; + const unsigned char *slot; + + if (bit < 9) + { + index = bit / 8; + offset = bit % 8; + slot = e->extra; + } + else if (!e->dyn) + return 0; + else + { + index = (bit - 9) / 8; + offset = (bit - 9) % 8; + slot = e->dyn; + } + return !!(slot[index] & (1 << offset)); +} + +/* The same join through a loop: the shift source changes on the back edge. */ +__attribute__((noinline)) static int sum_shifted(const int *v, int n) +{ + int s = 0, x = v[0]; + int t = x >> 3; + for (int i = 0; i < n; i++) + { + s += t; + x = v[i + 1]; + t = x >> 3; + } + return s; +} + +int main(void) +{ + static unsigned char dyn[4] = {0x00, 0x24, 0x00, 0x80}; + entry e = {{0}, {0x05, 0x01}, dyn}; + unsigned bits = 0; + for (int b = 0; b < 9 + 32; b++) + bits = bits * 3 + marked(&e, b); + printf("bits=%u\n", bits); + printf("then0=%d then8=%d else18=%d\n", marked(&e, 0), marked(&e, 8), marked(&e, 9 + 13)); + e.dyn = 0; + printf("nodyn=%d\n", marked(&e, 12)); + int v[5] = {80, 160, -24, 7, 0}; + printf("sum=%d\n", sum_shifted(v, 4)); + return 0; +} diff --git a/tests/ir_tests/608_barrel_shift_join_multidef.expect b/tests/ir_tests/608_barrel_shift_join_multidef.expect new file mode 100644 index 000000000..0935ee7ed --- /dev/null +++ b/tests/ir_tests/608_barrel_shift_join_multidef.expect @@ -0,0 +1,4 @@ +bits=2045904888 +then0=1 then8=1 else18=1 +nodyn=0 +sum=27 diff --git a/tests/ir_tests/test_qemu.py b/tests/ir_tests/test_qemu.py index 9058521b4..a29cc8ed5 100644 --- a/tests/ir_tests/test_qemu.py +++ b/tests/ir_tests/test_qemu.py @@ -1447,6 +1447,8 @@ def _expect_line(sut, expected_line: str, *, timeout: int = 1, float_tol: float # toybox awk's logical NOT, "STKP->num = ! get_set_logical();": a # comparison still in the flags was converted as its compared operand ("607_assign_not_of_call_writing_lhs.c", 0), + # barrel-shift fusion across a join: a VAR with a def per arm + ("608_barrel_shift_join_multidef.c", 0), # toysh's wildcard_matchlen against literal case patterns, and strchr # finding the NUL terminator: device-only shapes, pinned ("604_toysh_wildcard_literal.c", 0), From 6c494c69294ace0480fff7fca0a9405f549e7be7 Mon Sep 17 00:00:00 2001 From: Mateusz Stadnik Date: Thu, 1 Oct 2026 22:24:57 +0200 Subject: [PATCH 3/4] golden: keep esp_cleanup's case a stack object const_local_table now promotes the constant-initialised struct to a read-only table, the reads fold before entry_store and esp_cleanup never ran. Re-storing one field keeps it on the stack. Co-Authored-By: Claude Opus 5.5 --- .../esp_cleanup/inlined_struct_field_check.c | 8 +++++++- .../inlined_struct_field_check.expected | 19 ++++++++++--------- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/tests/ir_tests/golden/esp_cleanup/inlined_struct_field_check.c b/tests/ir_tests/golden/esp_cleanup/inlined_struct_field_check.c index be7296357..0cde5fbac 100644 --- a/tests/ir_tests/golden/esp_cleanup/inlined_struct_field_check.c +++ b/tests/ir_tests/golden/esp_cleanup/inlined_struct_field_check.c @@ -15,7 +15,12 @@ * a foldable consumer -- the original `return x->i == 0 && x->j == 5;` -- the * earlier propagation group collapses the whole diamond to `return 1` and * entry_store arrives with nothing left to forward, so the pass under test - * never executes. An opaque sink keeps the struct reads alive that far. */ + * never executes. An opaque sink keeps the struct reads alive that far. + * + * The re-store of `a.i` keeps `a` a stack object. A local initialised with + * constants and never written again is promoted to a read-only table + * (const_local_table), after which the reads fold to #8 before entry_store + * runs -- and the pass under test never executes either. */ struct A { int i; int j; }; extern int sink(int); @@ -26,5 +31,6 @@ static int add_fields(struct A *x) { int main(void) { struct A a = { .i = 3, .j = 5 }; + a.i = 3; return sink(add_fields(&a)); } diff --git a/tests/ir_tests/golden/esp_cleanup/inlined_struct_field_check.expected b/tests/ir_tests/golden/esp_cleanup/inlined_struct_field_check.expected index b41673829..58a8ef49e 100644 --- a/tests/ir_tests/golden/esp_cleanup/inlined_struct_field_check.expected +++ b/tests/ir_tests/golden/esp_cleanup/inlined_struct_field_check.expected @@ -2,12 +2,13 @@ 0001: StackLoc[-8] <-- #0 [STORE] 0002: StackLoc[-12] <-- #3 [STORE] 0003: StackLoc[-8] <-- #5 [STORE] -0004: T0 <-- Addr[StackLoc[-12]] -0005: T1 <-- T0 [ASSIGN] -0006: T2 <-- T0 [ASSIGN] -0007: T6 <-- #5 [ASSIGN] -0008: T4 <-- #3 ADD #5 -0009: JMP to 10 -0010: PARAM0[call_0] T4 -0011: CALL GlobalSym(#0) --> T5 -0012: RETURNVALUE T5 +0004: StackLoc[-12] <-- #3 [STORE] +0005: T1 <-- Addr[StackLoc[-12]] +0006: T2 <-- T1 [ASSIGN] +0007: T3 <-- T1 [ASSIGN] +0008: T7 <-- #5 [ASSIGN] +0009: T5 <-- #3 ADD #5 +0010: JMP to 11 +0011: PARAM0[call_0] T5 +0012: CALL GlobalSym(#0) --> T6 +0013: RETURNVALUE T6 From d1ca9e227bc5932b1db8a7d9cab4e2cd17f934a8 Mon Sep 17 00:00:00 2001 From: Mateusz Stadnik Date: Thu, 1 Oct 2026 22:44:06 +0200 Subject: [PATCH 4/4] gcctestsuite: apply the ASan timeout multiplier to discovered cases Every discovery site passed timeout=30, overriding GCCTestCase's 30 * ASAN_TIMEOUT_MULTIPLIER default, so ASan CI got 30 s and limits-caselabels (21-30 s on the runner) timed out. Co-Authored-By: Claude Opus 5.5 --- tests/gcctestsuite/conftest.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/gcctestsuite/conftest.py b/tests/gcctestsuite/conftest.py index 636fcdf0c..29f186f5a 100644 --- a/tests/gcctestsuite/conftest.py +++ b/tests/gcctestsuite/conftest.py @@ -444,7 +444,7 @@ def discover_gcc_compile_tests() -> List[GCCTestCase]: tests.append(GCCTestCase( source=c_file, category="gcc_compile", - timeout=30, + timeout=30 * ASAN_TIMEOUT_MULTIPLIER, dg_options=parse_dg_options(c_file), expected_compile_failure=bool(dg_errors), expected_error_patterns=dg_errors, @@ -474,7 +474,7 @@ def discover_gcc_execute_tests() -> List[GCCTestCase]: tests.append(GCCTestCase( source=c_file, category="gcc_execute", - timeout=30, + timeout=30 * ASAN_TIMEOUT_MULTIPLIER, dg_options=parse_dg_options(c_file) )) @@ -485,7 +485,7 @@ def discover_gcc_execute_tests() -> List[GCCTestCase]: tests.append(GCCTestCase( source=c_file, category="gcc_execute", - timeout=30, + timeout=30 * ASAN_TIMEOUT_MULTIPLIER, dg_options=parse_dg_options(c_file) )) @@ -516,7 +516,7 @@ def discover_gcc_execute_tests() -> List[GCCTestCase]: tests.append(GCCTestCase( source=c_file, category="gcc_execute", - timeout=30, + timeout=30 * ASAN_TIMEOUT_MULTIPLIER, dg_options=parse_dg_options(c_file), extra_sources=extra ))