summaryrefslogtreecommitdiff
path: root/rjit_c.c
blob: 9e442f5f6fa421b4fd2e9a4a23c5695220621bad (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
/**********************************************************************

  rjit_c.c - C helpers for RJIT

  Copyright (C) 2017 Takashi Kokubun <k0kubun@ruby-lang.org>.

**********************************************************************/

#include "rjit.h" // defines USE_RJIT

#if USE_RJIT

#include "rjit_c.h"
#include "include/ruby/assert.h"
#include "include/ruby/debug.h"
#include "internal.h"
#include "internal/compile.h"
#include "internal/fixnum.h"
#include "internal/hash.h"
#include "internal/sanitizers.h"
#include "internal/gc.h"
#include "yjit.h"
#include "vm_insnhelper.h"
#include "probes.h"
#include "probes_helper.h"

#include "insns.inc"
#include "insns_info.inc"

// For mmapp(), sysconf()
#ifndef _WIN32
#include <unistd.h>
#include <sys/mman.h>
#endif

#include <errno.h>

#if defined(MAP_FIXED_NOREPLACE) && defined(_SC_PAGESIZE)
// Align the current write position to a multiple of bytes
static uint8_t *
align_ptr(uint8_t *ptr, uint32_t multiple)
{
    // Compute the pointer modulo the given alignment boundary
    uint32_t rem = ((uint32_t)(uintptr_t)ptr) % multiple;

    // If the pointer is already aligned, stop
    if (rem == 0)
        return ptr;

    // Pad the pointer by the necessary amount to align it
    uint32_t pad = multiple - rem;

    return ptr + pad;
}
#endif

// Address space reservation. Memory pages are mapped on an as needed basis.
// See the Rust mm module for details.
static uint8_t *
rjit_reserve_addr_space(uint32_t mem_size)
{
#ifndef _WIN32
    uint8_t *mem_block;

    // On Linux
    #if defined(MAP_FIXED_NOREPLACE) && defined(_SC_PAGESIZE)
        uint32_t const page_size = (uint32_t)sysconf(_SC_PAGESIZE);
        uint8_t *const cfunc_sample_addr = (void *)&rjit_reserve_addr_space;
        uint8_t *const probe_region_end = cfunc_sample_addr + INT32_MAX;
        // Align the requested address to page size
        uint8_t *req_addr = align_ptr(cfunc_sample_addr, page_size);

        // Probe for addresses close to this function using MAP_FIXED_NOREPLACE
        // to improve odds of being in range for 32-bit relative call instructions.
        do {
            mem_block = mmap(
                req_addr,
                mem_size,
                PROT_NONE,
                MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE,
                -1,
                0
            );

            // If we succeeded, stop
            if (mem_block != MAP_FAILED) {
                break;
            }

            // +4MB
            req_addr += 4 * 1024 * 1024;
        } while (req_addr < probe_region_end);

    // On MacOS and other platforms
    #else
        // Try to map a chunk of memory as executable
        mem_block = mmap(
            (void *)rjit_reserve_addr_space,
            mem_size,
            PROT_NONE,
            MAP_PRIVATE | MAP_ANONYMOUS,
            -1,
            0
        );
    #endif

    // Fallback
    if (mem_block == MAP_FAILED) {
        // Try again without the address hint (e.g., valgrind)
        mem_block = mmap(
            NULL,
            mem_size,
            PROT_NONE,
            MAP_PRIVATE | MAP_ANONYMOUS,
            -1,
            0
        );
    }

    // Check that the memory mapping was successful
    if (mem_block == MAP_FAILED) {
        perror("ruby: yjit: mmap:");
        if(errno == ENOMEM) {
            // No crash report if it's only insufficient memory
            exit(EXIT_FAILURE);
        }
        rb_bug("mmap failed");
    }

    return mem_block;
#else
    // Windows not supported for now
    return NULL;
#endif
}

static VALUE
mprotect_write(rb_execution_context_t *ec, VALUE self, VALUE rb_mem_block, VALUE rb_mem_size)
{
    void *mem_block = (void *)NUM2SIZET(rb_mem_block);
    uint32_t mem_size = NUM2UINT(rb_mem_size);
    return RBOOL(mprotect(mem_block, mem_size, PROT_READ | PROT_WRITE) == 0);
}

static VALUE
mprotect_exec(rb_execution_context_t *ec, VALUE self, VALUE rb_mem_block, VALUE rb_mem_size)
{
    void *mem_block = (void *)NUM2SIZET(rb_mem_block);
    uint32_t mem_size = NUM2UINT(rb_mem_size);
    if (mem_size == 0) return Qfalse; // Some platforms return an error for mem_size 0.

    if (mprotect(mem_block, mem_size, PROT_READ | PROT_EXEC)) {
        rb_bug("Couldn't make JIT page (%p, %lu bytes) executable, errno: %s\n",
            mem_block, (unsigned long)mem_size, strerror(errno));
    }
    return Qtrue;
}

static VALUE
rjit_optimized_call(VALUE *recv, rb_execution_context_t *ec, int argc, VALUE *argv, int kw_splat, VALUE block_handler)
{
    rb_proc_t *proc;
    GetProcPtr(recv, proc);
    return rb_vm_invoke_proc(ec, proc, argc, argv, kw_splat, block_handler);
}

static VALUE
rjit_str_neq_internal(VALUE str1, VALUE str2)
{
    return rb_str_eql_internal(str1, str2) == Qtrue ? Qfalse : Qtrue;
}

static VALUE
rjit_str_simple_append(VALUE str1, VALUE str2)
{
    return rb_str_cat(str1, RSTRING_PTR(str2), RSTRING_LEN(str2));
}

static VALUE
rjit_rb_ary_subseq_length(VALUE ary, long beg)
{
    long len = RARRAY_LEN(ary);
    return rb_ary_subseq(ary, beg, len);
}

// The code we generate in gen_send_cfunc() doesn't fire the c_return TracePoint event
// like the interpreter. When tracing for c_return is enabled, we patch the code after
// the C method return to call into this to fire the event.
static void
rjit_full_cfunc_return(rb_execution_context_t *ec, VALUE return_value)
{
    rb_control_frame_t *cfp = ec->cfp;
    RUBY_ASSERT_ALWAYS(cfp == GET_EC()->cfp);
    const rb_callable_method_entry_t *me = rb_vm_frame_method_entry(cfp);

    RUBY_ASSERT_ALWAYS(RUBYVM_CFUNC_FRAME_P(cfp));
    RUBY_ASSERT_ALWAYS(me->def->type == VM_METHOD_TYPE_CFUNC);

    // CHECK_CFP_CONSISTENCY("full_cfunc_return"); TODO revive this

    // Pop the C func's frame and fire the c_return TracePoint event
    // Note that this is the same order as vm_call_cfunc_with_frame().
    rb_vm_pop_frame(ec);
    EXEC_EVENT_HOOK(ec, RUBY_EVENT_C_RETURN, cfp->self, me->def->original_id, me->called_id, me->owner, return_value);
    // Note, this deviates from the interpreter in that users need to enable
    // a c_return TracePoint for this DTrace hook to work. A reasonable change
    // since the Ruby return event works this way as well.
    RUBY_DTRACE_CMETHOD_RETURN_HOOK(ec, me->owner, me->def->original_id);

    // Push return value into the caller's stack. We know that it's a frame that
    // uses cfp->sp because we are patching a call done with gen_send_cfunc().
    ec->cfp->sp[0] = return_value;
    ec->cfp->sp++;
}

static rb_proc_t *
rjit_get_proc_ptr(VALUE procv)
{
    rb_proc_t *proc;
    GetProcPtr(procv, proc);
    return proc;
}

// Use the same buffer size as Stackprof.
#define BUFF_LEN 2048

extern VALUE rb_rjit_raw_samples;
extern VALUE rb_rjit_line_samples;

static void
rjit_record_exit_stack(const VALUE *exit_pc)
{
    // Let Primitive.rjit_stop_stats stop this
    if (!rb_rjit_call_p) return;

    // Get the opcode from the encoded insn handler at this PC
    int insn = rb_vm_insn_addr2opcode((void *)*exit_pc);

    // Create 2 array buffers to be used to collect frames and lines.
    VALUE frames_buffer[BUFF_LEN] = { 0 };
    int lines_buffer[BUFF_LEN] = { 0 };

    // Records call frame and line information for each method entry into two
    // temporary buffers. Returns the number of times we added to the buffer (ie
    // the length of the stack).
    //
    // Call frame info is stored in the frames_buffer, line number information
    // in the lines_buffer. The first argument is the start point and the second
    // argument is the buffer limit, set at 2048.
    int stack_length = rb_profile_frames(0, BUFF_LEN, frames_buffer, lines_buffer);
    int samples_length = stack_length + 3; // 3: length, insn, count

    // If yjit_raw_samples is less than or equal to the current length of the samples
    // we might have seen this stack trace previously.
    int prev_stack_len_index = (int)RARRAY_LEN(rb_rjit_raw_samples) - samples_length;
    VALUE prev_stack_len_obj;
    if (RARRAY_LEN(rb_rjit_raw_samples) >= samples_length && FIXNUM_P(prev_stack_len_obj = RARRAY_AREF(rb_rjit_raw_samples, prev_stack_len_index))) {
        int prev_stack_len = NUM2INT(prev_stack_len_obj);
        int idx = stack_length - 1;
        int prev_frame_idx = 0;
        bool seen_already = true;

        // If the previous stack length and current stack length are equal,
        // loop and compare the current frame to the previous frame. If they are
        // not equal, set seen_already to false and break out of the loop.
        if (prev_stack_len == stack_length) {
            while (idx >= 0) {
                VALUE current_frame = frames_buffer[idx];
                VALUE prev_frame = RARRAY_AREF(rb_rjit_raw_samples, prev_stack_len_index + prev_frame_idx + 1);

                // If the current frame and previous frame are not equal, set
                // seen_already to false and break out of the loop.
                if (current_frame != prev_frame) {
                    seen_already = false;
                    break;
                }

                idx--;
                prev_frame_idx++;
            }

            // If we know we've seen this stack before, increment the counter by 1.
            if (seen_already) {
                int prev_idx = (int)RARRAY_LEN(rb_rjit_raw_samples) - 1;
                int prev_count = NUM2INT(RARRAY_AREF(rb_rjit_raw_samples, prev_idx));
                int new_count = prev_count + 1;

                rb_ary_store(rb_rjit_raw_samples, prev_idx, INT2NUM(new_count));
                rb_ary_store(rb_rjit_line_samples, prev_idx, INT2NUM(new_count));
                return;
            }
        }
    }

    rb_ary_push(rb_rjit_raw_samples, INT2NUM(stack_length));
    rb_ary_push(rb_rjit_line_samples, INT2NUM(stack_length));

    int idx = stack_length - 1;

    while (idx >= 0) {
        VALUE frame = frames_buffer[idx];
        int line = lines_buffer[idx];

        rb_ary_push(rb_rjit_raw_samples, frame);
        rb_ary_push(rb_rjit_line_samples, INT2NUM(line));

        idx--;
    }

    // Push the insn value into the yjit_raw_samples Vec.
    rb_ary_push(rb_rjit_raw_samples, INT2NUM(insn));

    // Push the current line onto the yjit_line_samples Vec. This
    // points to the line in insns.def.
    int line = (int)RARRAY_LEN(rb_rjit_line_samples) - 1;
    rb_ary_push(rb_rjit_line_samples, INT2NUM(line));

    // Push number of times seen onto the stack, which is 1
    // because it's the first time we've seen it.
    rb_ary_push(rb_rjit_raw_samples, INT2NUM(1));
    rb_ary_push(rb_rjit_line_samples, INT2NUM(1));
}

// For a given raw_sample (frame), set the hash with the caller's
// name, file, and line number. Return the  hash with collected frame_info.
static void
rjit_add_frame(VALUE hash, VALUE frame)
{
    VALUE frame_id = SIZET2NUM(frame);

    if (RTEST(rb_hash_aref(hash, frame_id))) {
        return;
    }
    else {
        VALUE frame_info = rb_hash_new();
        // Full label for the frame
        VALUE name = rb_profile_frame_full_label(frame);
        // Absolute path of the frame from rb_iseq_realpath
        VALUE file = rb_profile_frame_absolute_path(frame);
        // Line number of the frame
        VALUE line = rb_profile_frame_first_lineno(frame);

        // If absolute path isn't available use the rb_iseq_path
        if (NIL_P(file)) {
            file = rb_profile_frame_path(frame);
        }

        rb_hash_aset(frame_info, ID2SYM(rb_intern("name")), name);
        rb_hash_aset(frame_info, ID2SYM(rb_intern("file")), file);
        rb_hash_aset(frame_info, ID2SYM(rb_intern("samples")), INT2NUM(0));
        rb_hash_aset(frame_info, ID2SYM(rb_intern("total_samples")), INT2NUM(0));
        rb_hash_aset(frame_info, ID2SYM(rb_intern("edges")), rb_hash_new());
        rb_hash_aset(frame_info, ID2SYM(rb_intern("lines")), rb_hash_new());

        if (line != INT2FIX(0)) {
            rb_hash_aset(frame_info, ID2SYM(rb_intern("line")), line);
        }

       rb_hash_aset(hash, frame_id, frame_info);
    }
}

static VALUE
rjit_exit_traces(void)
{
    int samples_len = (int)RARRAY_LEN(rb_rjit_raw_samples);
    RUBY_ASSERT(samples_len == RARRAY_LEN(rb_rjit_line_samples));

    VALUE result = rb_hash_new();
    VALUE raw_samples = rb_ary_new_capa(samples_len);
    VALUE line_samples = rb_ary_new_capa(samples_len);
    VALUE frames = rb_hash_new();
    int idx = 0;

    // While the index is less than samples_len, parse yjit_raw_samples and
    // yjit_line_samples, then add casted values to raw_samples and line_samples array.
    while (idx < samples_len) {
        int num = NUM2INT(RARRAY_AREF(rb_rjit_raw_samples, idx));
        int line_num = NUM2INT(RARRAY_AREF(rb_rjit_line_samples, idx));
        idx++;

        rb_ary_push(raw_samples, SIZET2NUM(num));
        rb_ary_push(line_samples, INT2NUM(line_num));

        // Loop through the length of samples_len and add data to the
        // frames hash. Also push the current value onto the raw_samples
        // and line_samples array respectively.
        for (int o = 0; o < num; o++) {
            rjit_add_frame(frames, RARRAY_AREF(rb_rjit_raw_samples, idx));
            rb_ary_push(raw_samples, SIZET2NUM(RARRAY_AREF(rb_rjit_raw_samples, idx)));
            rb_ary_push(line_samples, RARRAY_AREF(rb_rjit_line_samples, idx));
            idx++;
        }

        // insn BIN and lineno
        rb_ary_push(raw_samples, RARRAY_AREF(rb_rjit_raw_samples, idx));
        rb_ary_push(line_samples, RARRAY_AREF(rb_rjit_line_samples, idx));
        idx++;

        // Number of times seen
        rb_ary_push(raw_samples, RARRAY_AREF(rb_rjit_raw_samples, idx));
        rb_ary_push(line_samples, RARRAY_AREF(rb_rjit_line_samples, idx));
        idx++;
    }

    // Set add the raw_samples, line_samples, and frames to the results
    // hash.
    rb_hash_aset(result, ID2SYM(rb_intern("raw")), raw_samples);
    rb_hash_aset(result, ID2SYM(rb_intern("lines")), line_samples);
    rb_hash_aset(result, ID2SYM(rb_intern("frames")), frames);

    return result;
}

// An offsetof implementation that works for unnamed struct and union.
// Multiplying 8 for compatibility with libclang's offsetof.
#define OFFSETOF(ptr, member) RB_SIZE2NUM(((char *)&ptr.member - (char*)&ptr) * 8)

#define SIZEOF(type) RB_SIZE2NUM(sizeof(type))
#define SIGNED_TYPE_P(type) RBOOL((type)(-1) < (type)(1))

// Insn side exit counters
static size_t rjit_insn_exits[VM_INSTRUCTION_SIZE] = { 0 };

// macOS: brew install capstone
// Ubuntu/Debian: apt-get install libcapstone-dev
// Fedora: dnf -y install capstone-devel
#ifdef HAVE_LIBCAPSTONE
#include <capstone/capstone.h>
#endif

// Return an array of [address, mnemonic, op_str]
static VALUE
dump_disasm(rb_execution_context_t *ec, VALUE self, VALUE from, VALUE to, VALUE test)
{
    VALUE result = rb_ary_new();
#ifdef HAVE_LIBCAPSTONE
    // Prepare for calling cs_disasm
    static csh handle;
    if (cs_open(CS_ARCH_X86, CS_MODE_64, &handle) != CS_ERR_OK) {
        rb_raise(rb_eRuntimeError, "failed to make Capstone handle");
    }
    size_t from_addr = NUM2SIZET(from);
    size_t to_addr = NUM2SIZET(to);

    // Call cs_disasm and convert results to a Ruby array
    cs_insn *insns;
    size_t base_addr = RTEST(test) ? 0 : from_addr; // On tests, start from 0 for output stability.
    size_t count = cs_disasm(handle, (const uint8_t *)from_addr, to_addr - from_addr, base_addr, 0, &insns);
    for (size_t i = 0; i < count; i++) {
        VALUE vals = rb_ary_new_from_args(3, LONG2NUM(insns[i].address), rb_str_new2(insns[i].mnemonic), rb_str_new2(insns[i].op_str));
        rb_ary_push(result, vals);
    }

    // Free memory used by capstone
    cs_free(insns, count);
    cs_close(&handle);
#endif
    return result;
}

// Same as `RubyVM::RJIT.enabled?`, but this is used before it's defined.
static VALUE
rjit_enabled_p(rb_execution_context_t *ec, VALUE self)
{
    return RBOOL(rb_rjit_enabled);
}

static int
for_each_iseq_i(void *vstart, void *vend, size_t stride, void *data)
{
    VALUE block = (VALUE)data;
    VALUE v = (VALUE)vstart;
    for (; v != (VALUE)vend; v += stride) {
        void *ptr = asan_poisoned_object_p(v);
        asan_unpoison_object(v, false);

        if (rb_obj_is_iseq(v)) {
            extern VALUE rb_rjit_iseq_new(rb_iseq_t *iseq);
            rb_iseq_t *iseq = (rb_iseq_t *)v;
            rb_funcall(block, rb_intern("call"), 1, rb_rjit_iseq_new(iseq));
        }

        asan_poison_object_if(ptr, v);
    }
    return 0;
}

static VALUE
rjit_for_each_iseq(rb_execution_context_t *ec, VALUE self, VALUE block)
{
    rb_objspace_each_objects(for_each_iseq_i, (void *)block);
    return Qnil;
}

// bindgen funcs
extern ID rb_get_symbol_id(VALUE name);
extern VALUE rb_fix_aref(VALUE fix, VALUE idx);
extern VALUE rb_str_getbyte(VALUE str, VALUE index);
extern VALUE rb_vm_concat_array(VALUE ary1, VALUE ary2st);
extern VALUE rb_vm_get_ev_const(rb_execution_context_t *ec, VALUE orig_klass, ID id, VALUE allow_nil);
extern VALUE rb_vm_getclassvariable(const rb_iseq_t *iseq, const rb_control_frame_t *cfp, ID id, ICVARC ic);
extern VALUE rb_vm_opt_newarray_min(rb_execution_context_t *ec, rb_num_t num, const VALUE *ptr);
extern VALUE rb_vm_splat_array(VALUE flag, VALUE array);
extern bool rb_simple_iseq_p(const rb_iseq_t *iseq);
extern bool rb_vm_defined(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, rb_num_t op_type, VALUE obj, VALUE v);
extern bool rb_vm_ic_hit_p(IC ic, const VALUE *reg_ep);
extern rb_event_flag_t rb_rjit_global_events;
extern void rb_vm_setinstancevariable(const rb_iseq_t *iseq, VALUE obj, ID id, VALUE val, IVC ic);
extern VALUE rb_vm_throw(const rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, rb_num_t throw_state, VALUE throwobj);
extern VALUE rb_reg_new_ary(VALUE ary, int opt);
extern void rb_vm_setclassvariable(const rb_iseq_t *iseq, const rb_control_frame_t *cfp, ID id, VALUE val, ICVARC ic);
extern VALUE rb_str_bytesize(VALUE str);
extern const rb_callable_method_entry_t *rb_callable_method_entry_or_negative(VALUE klass, ID mid);
extern VALUE rb_vm_yield_with_cfunc(rb_execution_context_t *ec, const struct rb_captured_block *captured, int argc, const VALUE *argv);
extern VALUE rb_vm_set_ivar_id(VALUE obj, ID id, VALUE val);
extern VALUE rb_ary_unshift_m(int argc, VALUE *argv, VALUE ary);

#include "rjit_c.rbinc"

#endif // USE_RJIT