cpus.c 44.9 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25
/*
 * QEMU System Emulator
 *
 * Copyright (c) 2003-2008 Fabrice Bellard
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE.
 */

/* Needed early for CONFIG_BSD etc. */
Peter Maydell's avatar
Peter Maydell committed
26
#include "qemu/osdep.h"
27 28
#include "qemu-common.h"
#include "cpu.h"
29
#include "monitor/monitor.h"
Wenchao Xia's avatar
Wenchao Xia committed
30
#include "qapi/qmp/qerror.h"
31
#include "qemu/error-report.h"
32
#include "sysemu/sysemu.h"
33
#include "sysemu/block-backend.h"
34
#include "exec/gdbstub.h"
35 36
#include "sysemu/dma.h"
#include "sysemu/kvm.h"
37
#include "qmp-commands.h"
38
#include "exec/exec-all.h"
39

40
#include "qemu/thread.h"
41 42
#include "sysemu/cpus.h"
#include "sysemu/qtest.h"
43 44
#include "qemu/main-loop.h"
#include "qemu/bitmap.h"
45
#include "qemu/seqlock.h"
Wenchao Xia's avatar
Wenchao Xia committed
46
#include "qapi-event.h"
47
#include "hw/nmi.h"
48
#include "sysemu/replay.h"
49 50

#ifndef _WIN32
51
#include "qemu/compatfd.h"
52
#endif
53

54 55 56 57
#ifdef CONFIG_LINUX

#include <sys/prctl.h>

58 59 60 61
#ifndef PR_MCE_KILL
#define PR_MCE_KILL 33
#endif

62 63 64 65 66 67 68 69 70 71
#ifndef PR_MCE_KILL_SET
#define PR_MCE_KILL_SET 1
#endif

#ifndef PR_MCE_KILL_EARLY
#define PR_MCE_KILL_EARLY 1
#endif

#endif /* CONFIG_LINUX */

72
static CPUState *next_cpu;
73 74
int64_t max_delay;
int64_t max_advance;
75

76 77 78 79 80 81 82 83
/* vcpu throttling controls */
static QEMUTimer *throttle_timer;
static unsigned int throttle_percentage;

#define CPU_THROTTLE_PCT_MIN 1
#define CPU_THROTTLE_PCT_MAX 99
#define CPU_THROTTLE_TIMESLICE_NS 10000000

84 85 86 87 88
bool cpu_is_stopped(CPUState *cpu)
{
    return cpu->stopped || !runstate_is_running();
}

89
static bool cpu_thread_is_idle(CPUState *cpu)
90
{
91
    if (cpu->stop || cpu->queued_work_first) {
92 93
        return false;
    }
94
    if (cpu_is_stopped(cpu)) {
95 96
        return true;
    }
97
    if (!cpu->halted || cpu_has_work(cpu) ||
98
        kvm_halt_in_kernel()) {
99 100 101 102 103 104 105
        return false;
    }
    return true;
}

static bool all_cpu_threads_idle(void)
{
106
    CPUState *cpu;
107

108
    CPU_FOREACH(cpu) {
109
        if (!cpu_thread_is_idle(cpu)) {
110 111 112 113 114 115
            return false;
        }
    }
    return true;
}

116 117 118
/***********************************************************/
/* guest cycle counter */

119 120
/* Protected by TimersState seqlock */

121
static bool icount_sleep = true;
122
static int64_t vm_clock_warp_start = -1;
123 124 125 126
/* Conversion factor from emulated instructions to virtual clock ticks.  */
static int icount_time_shift;
/* Arbitrarily pick 1MIPS as the minimum allowable speed.  */
#define MAX_ICOUNT_SHIFT 10
127

128 129 130 131 132
static QEMUTimer *icount_rt_timer;
static QEMUTimer *icount_vm_timer;
static QEMUTimer *icount_warp_timer;

typedef struct TimersState {
133
    /* Protected by BQL.  */
134 135
    int64_t cpu_ticks_prev;
    int64_t cpu_ticks_offset;
136 137 138 139 140

    /* cpu_clock_offset can be read out of BQL, so protect it with
     * this lock.
     */
    QemuSeqLock vm_clock_seqlock;
141 142 143
    int64_t cpu_clock_offset;
    int32_t cpu_ticks_enabled;
    int64_t dummy;
144 145 146 147 148

    /* Compensate for varying guest execution speed.  */
    int64_t qemu_icount_bias;
    /* Only written by TCG thread */
    int64_t qemu_icount;
149 150
} TimersState;

151
static TimersState timers_state;
152

153
int64_t cpu_get_icount_raw(void)
154 155
{
    int64_t icount;
156
    CPUState *cpu = current_cpu;
157

158
    icount = timers_state.qemu_icount;
159
    if (cpu) {
160
        if (!cpu->can_do_io) {
161 162
            fprintf(stderr, "Bad icount read\n");
            exit(1);
163
        }
164
        icount -= (cpu->icount_decr.u16.low + cpu->icount_extra);
165
    }
166 167 168 169 170 171 172
    return icount;
}

/* Return the virtual CPU time, based on the instruction counter.  */
static int64_t cpu_get_icount_locked(void)
{
    int64_t icount = cpu_get_icount_raw();
173
    return timers_state.qemu_icount_bias + cpu_icount_to_ns(icount);
174 175
}

176 177 178 179 180 181 182 183 184 185 186 187 188
int64_t cpu_get_icount(void)
{
    int64_t icount;
    unsigned start;

    do {
        start = seqlock_read_begin(&timers_state.vm_clock_seqlock);
        icount = cpu_get_icount_locked();
    } while (seqlock_read_retry(&timers_state.vm_clock_seqlock, start));

    return icount;
}

189 190 191 192 193
int64_t cpu_icount_to_ns(int64_t icount)
{
    return icount << icount_time_shift;
}

194
/* return the host CPU cycle counter and handle stop/restart */
195
/* Caller must hold the BQL */
196 197
int64_t cpu_get_ticks(void)
{
198 199
    int64_t ticks;

200 201 202
    if (use_icount) {
        return cpu_get_icount();
    }
203 204 205

    ticks = timers_state.cpu_ticks_offset;
    if (timers_state.cpu_ticks_enabled) {
206
        ticks += cpu_get_host_ticks();
207 208 209 210 211 212 213
    }

    if (timers_state.cpu_ticks_prev > ticks) {
        /* Note: non increasing ticks may happen if the host uses
           software suspend */
        timers_state.cpu_ticks_offset += timers_state.cpu_ticks_prev - ticks;
        ticks = timers_state.cpu_ticks_prev;
214
    }
215 216 217

    timers_state.cpu_ticks_prev = ticks;
    return ticks;
218 219
}

220
static int64_t cpu_get_clock_locked(void)
221
{
222
    int64_t ticks;
223

224 225 226
    ticks = timers_state.cpu_clock_offset;
    if (timers_state.cpu_ticks_enabled) {
        ticks += get_clock();
227
    }
228

229
    return ticks;
230 231 232 233 234 235 236 237 238 239 240 241 242 243
}

/* return the host CPU monotonic timer and handle stop/restart */
int64_t cpu_get_clock(void)
{
    int64_t ti;
    unsigned start;

    do {
        start = seqlock_read_begin(&timers_state.vm_clock_seqlock);
        ti = cpu_get_clock_locked();
    } while (seqlock_read_retry(&timers_state.vm_clock_seqlock, start));

    return ti;
244 245
}

246 247 248
/* enable cpu_get_ticks()
 * Caller must hold BQL which server as mutex for vm_clock_seqlock.
 */
249 250
void cpu_enable_ticks(void)
{
251
    /* Here, the really thing protected by seqlock is cpu_clock_offset. */
252
    seqlock_write_begin(&timers_state.vm_clock_seqlock);
253
    if (!timers_state.cpu_ticks_enabled) {
254
        timers_state.cpu_ticks_offset -= cpu_get_host_ticks();
255 256 257
        timers_state.cpu_clock_offset -= get_clock();
        timers_state.cpu_ticks_enabled = 1;
    }
258
    seqlock_write_end(&timers_state.vm_clock_seqlock);
259 260 261
}

/* disable cpu_get_ticks() : the clock is stopped. You must not call
262 263 264
 * cpu_get_ticks() after that.
 * Caller must hold BQL which server as mutex for vm_clock_seqlock.
 */
265 266
void cpu_disable_ticks(void)
{
267
    /* Here, the really thing protected by seqlock is cpu_clock_offset. */
268
    seqlock_write_begin(&timers_state.vm_clock_seqlock);
269
    if (timers_state.cpu_ticks_enabled) {
270
        timers_state.cpu_ticks_offset += cpu_get_host_ticks();
271
        timers_state.cpu_clock_offset = cpu_get_clock_locked();
272 273
        timers_state.cpu_ticks_enabled = 0;
    }
274
    seqlock_write_end(&timers_state.vm_clock_seqlock);
275 276 277 278 279 280
}

/* Correlation between real and virtual time is always going to be
   fairly approximate, so ignore small variation.
   When the guest is idle real and virtual time will be aligned in
   the IO wait loop.  */
281
#define ICOUNT_WOBBLE (NANOSECONDS_PER_SECOND / 10)
282 283 284 285 286 287

static void icount_adjust(void)
{
    int64_t cur_time;
    int64_t cur_icount;
    int64_t delta;
288 289

    /* Protected by TimersState mutex.  */
290
    static int64_t last_delta;
291

292 293 294 295
    /* If the VM is not running, then do nothing.  */
    if (!runstate_is_running()) {
        return;
    }
296

297
    seqlock_write_begin(&timers_state.vm_clock_seqlock);
298 299
    cur_time = cpu_get_clock_locked();
    cur_icount = cpu_get_icount_locked();
300

301 302 303 304 305 306 307 308 309 310 311 312 313 314 315
    delta = cur_icount - cur_time;
    /* FIXME: This is a very crude algorithm, somewhat prone to oscillation.  */
    if (delta > 0
        && last_delta + ICOUNT_WOBBLE < delta * 2
        && icount_time_shift > 0) {
        /* The guest is getting too far ahead.  Slow time down.  */
        icount_time_shift--;
    }
    if (delta < 0
        && last_delta - ICOUNT_WOBBLE > delta * 2
        && icount_time_shift < MAX_ICOUNT_SHIFT) {
        /* The guest is getting too far behind.  Speed time up.  */
        icount_time_shift++;
    }
    last_delta = delta;
316 317
    timers_state.qemu_icount_bias = cur_icount
                              - (timers_state.qemu_icount << icount_time_shift);
318
    seqlock_write_end(&timers_state.vm_clock_seqlock);
319 320 321 322
}

static void icount_adjust_rt(void *opaque)
{
323
    timer_mod(icount_rt_timer,
324
              qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL_RT) + 1000);
325 326 327 328 329
    icount_adjust();
}

static void icount_adjust_vm(void *opaque)
{
330 331
    timer_mod(icount_vm_timer,
                   qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) +
332
                   NANOSECONDS_PER_SECOND / 10);
333 334 335 336 337 338 339 340
    icount_adjust();
}

static int64_t qemu_icount_round(int64_t count)
{
    return (count + (1 << icount_time_shift) - 1) >> icount_time_shift;
}

341
static void icount_warp_rt(void)
342
{
343 344 345
    unsigned seq;
    int64_t warp_start;

346 347 348
    /* The icount_warp_timer is rescheduled soon after vm_clock_warp_start
     * changes from -1 to another value, so the race here is okay.
     */
349 350 351 352 353 354
    do {
        seq = seqlock_read_begin(&timers_state.vm_clock_seqlock);
        warp_start = vm_clock_warp_start;
    } while (seqlock_read_retry(&timers_state.vm_clock_seqlock, seq));

    if (warp_start == -1) {
355 356 357
        return;
    }

358
    seqlock_write_begin(&timers_state.vm_clock_seqlock);
359
    if (runstate_is_running()) {
360 361
        int64_t clock = REPLAY_CLOCK(REPLAY_CLOCK_VIRTUAL_RT,
                                     cpu_get_clock_locked());
362 363 364 365
        int64_t warp_delta;

        warp_delta = clock - vm_clock_warp_start;
        if (use_icount == 2) {
366
            /*
367
             * In adaptive mode, do not let QEMU_CLOCK_VIRTUAL run too
368 369
             * far ahead of real time.
             */
370
            int64_t cur_icount = cpu_get_icount_locked();
371
            int64_t delta = clock - cur_icount;
372
            warp_delta = MIN(warp_delta, delta);
373
        }
374
        timers_state.qemu_icount_bias += warp_delta;
375 376
    }
    vm_clock_warp_start = -1;
377
    seqlock_write_end(&timers_state.vm_clock_seqlock);
378 379 380 381

    if (qemu_clock_expired(QEMU_CLOCK_VIRTUAL)) {
        qemu_clock_notify(QEMU_CLOCK_VIRTUAL);
    }
382 383
}

384
static void icount_timer_cb(void *opaque)
385
{
386 387 388 389
    /* No need for a checkpoint because the timer already synchronizes
     * with CHECKPOINT_CLOCK_VIRTUAL_RT.
     */
    icount_warp_rt();
390 391
}

392 393
void qtest_clock_warp(int64_t dest)
{
394
    int64_t clock = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
395
    AioContext *aio_context;
396
    assert(qtest_enabled());
397
    aio_context = qemu_get_aio_context();
398
    while (clock < dest) {
399
        int64_t deadline = qemu_clock_deadline_ns_all(QEMU_CLOCK_VIRTUAL);
400
        int64_t warp = qemu_soonest_timeout(dest - clock, deadline);
401

402
        seqlock_write_begin(&timers_state.vm_clock_seqlock);
403
        timers_state.qemu_icount_bias += warp;
404
        seqlock_write_end(&timers_state.vm_clock_seqlock);
405

406
        qemu_clock_run_timers(QEMU_CLOCK_VIRTUAL);
407
        timerlist_run_timers(aio_context->tlg.tl[QEMU_CLOCK_VIRTUAL]);
408
        clock = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
409
    }
410
    qemu_clock_notify(QEMU_CLOCK_VIRTUAL);
411 412
}

413
void qemu_start_warp_timer(void)
414
{
415
    int64_t clock;
416 417
    int64_t deadline;

418
    if (!use_icount) {
419 420 421
        return;
    }

Pavel Dovgalyuk's avatar
Pavel Dovgalyuk committed
422 423 424 425 426 427 428 429
    /* Nothing to do if the VM is stopped: QEMU_CLOCK_VIRTUAL timers
     * do not fire, so computing the deadline does not make sense.
     */
    if (!runstate_is_running()) {
        return;
    }

    /* warp clock deterministically in record/replay mode */
430
    if (!replay_checkpoint(CHECKPOINT_CLOCK_WARP_START)) {
Pavel Dovgalyuk's avatar
Pavel Dovgalyuk committed
431 432 433
        return;
    }

434
    if (!all_cpu_threads_idle()) {
435 436 437
        return;
    }

438 439
    if (qtest_enabled()) {
        /* When testing, qtest commands advance icount.  */
440
        return;
441 442
    }

443
    /* We want to use the earliest deadline from ALL vm_clocks */
444
    clock = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL_RT);
445
    deadline = qemu_clock_deadline_ns_all(QEMU_CLOCK_VIRTUAL);
446
    if (deadline < 0) {
447 448 449 450 451
        static bool notified;
        if (!icount_sleep && !notified) {
            error_report("WARNING: icount sleep disabled and no active timers");
            notified = true;
        }
452
        return;
453 454
    }

455 456
    if (deadline > 0) {
        /*
457
         * Ensure QEMU_CLOCK_VIRTUAL proceeds even when the virtual CPU goes to
458 459 460
         * sleep.  Otherwise, the CPU might be waiting for a future timer
         * interrupt to wake it up, but the interrupt never comes because
         * the vCPU isn't running any insns and thus doesn't advance the
461
         * QEMU_CLOCK_VIRTUAL.
462
         */
463 464 465 466 467 468 469 470
        if (!icount_sleep) {
            /*
             * We never let VCPUs sleep in no sleep icount mode.
             * If there is a pending QEMU_CLOCK_VIRTUAL timer we just advance
             * to the next QEMU_CLOCK_VIRTUAL event and notify it.
             * It is useful when we want a deterministic execution time,
             * isolated from host latencies.
             */
471
            seqlock_write_begin(&timers_state.vm_clock_seqlock);
472
            timers_state.qemu_icount_bias += deadline;
473
            seqlock_write_end(&timers_state.vm_clock_seqlock);
474 475 476 477 478 479 480 481 482 483
            qemu_clock_notify(QEMU_CLOCK_VIRTUAL);
        } else {
            /*
             * We do stop VCPUs and only advance QEMU_CLOCK_VIRTUAL after some
             * "real" time, (related to the time left until the next event) has
             * passed. The QEMU_CLOCK_VIRTUAL_RT clock will do this.
             * This avoids that the warps are visible externally; for example,
             * you will not be sending network packets continuously instead of
             * every 100ms.
             */
484
            seqlock_write_begin(&timers_state.vm_clock_seqlock);
485 486 487
            if (vm_clock_warp_start == -1 || vm_clock_warp_start > clock) {
                vm_clock_warp_start = clock;
            }
488
            seqlock_write_end(&timers_state.vm_clock_seqlock);
489
            timer_mod_anticipate(icount_warp_timer, clock + deadline);
490
        }
491
    } else if (deadline == 0) {
492
        qemu_clock_notify(QEMU_CLOCK_VIRTUAL);
493 494 495
    }
}

496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517
static void qemu_account_warp_timer(void)
{
    if (!use_icount || !icount_sleep) {
        return;
    }

    /* Nothing to do if the VM is stopped: QEMU_CLOCK_VIRTUAL timers
     * do not fire, so computing the deadline does not make sense.
     */
    if (!runstate_is_running()) {
        return;
    }

    /* warp clock deterministically in record/replay mode */
    if (!replay_checkpoint(CHECKPOINT_CLOCK_WARP_ACCOUNT)) {
        return;
    }

    timer_del(icount_warp_timer);
    icount_warp_rt();
}

518 519 520 521 522 523 524 525 526 527 528 529
static bool icount_state_needed(void *opaque)
{
    return use_icount;
}

/*
 * This is a subsection for icount migration.
 */
static const VMStateDescription icount_vmstate_timers = {
    .name = "timer/icount",
    .version_id = 1,
    .minimum_version_id = 1,
530
    .needed = icount_state_needed,
531 532 533 534 535 536 537
    .fields = (VMStateField[]) {
        VMSTATE_INT64(qemu_icount_bias, TimersState),
        VMSTATE_INT64(qemu_icount, TimersState),
        VMSTATE_END_OF_LIST()
    }
};

538 539 540 541
static const VMStateDescription vmstate_timers = {
    .name = "timer",
    .version_id = 2,
    .minimum_version_id = 1,
542
    .fields = (VMStateField[]) {
543 544 545 546
        VMSTATE_INT64(cpu_ticks_offset, TimersState),
        VMSTATE_INT64(dummy, TimersState),
        VMSTATE_INT64_V(cpu_clock_offset, TimersState, 2),
        VMSTATE_END_OF_LIST()
547
    },
548 549 550
    .subsections = (const VMStateDescription*[]) {
        &icount_vmstate_timers,
        NULL
551 552 553
    }
};

554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621
static void cpu_throttle_thread(void *opaque)
{
    CPUState *cpu = opaque;
    double pct;
    double throttle_ratio;
    long sleeptime_ns;

    if (!cpu_throttle_get_percentage()) {
        return;
    }

    pct = (double)cpu_throttle_get_percentage()/100;
    throttle_ratio = pct / (1 - pct);
    sleeptime_ns = (long)(throttle_ratio * CPU_THROTTLE_TIMESLICE_NS);

    qemu_mutex_unlock_iothread();
    atomic_set(&cpu->throttle_thread_scheduled, 0);
    g_usleep(sleeptime_ns / 1000); /* Convert ns to us for usleep call */
    qemu_mutex_lock_iothread();
}

static void cpu_throttle_timer_tick(void *opaque)
{
    CPUState *cpu;
    double pct;

    /* Stop the timer if needed */
    if (!cpu_throttle_get_percentage()) {
        return;
    }
    CPU_FOREACH(cpu) {
        if (!atomic_xchg(&cpu->throttle_thread_scheduled, 1)) {
            async_run_on_cpu(cpu, cpu_throttle_thread, cpu);
        }
    }

    pct = (double)cpu_throttle_get_percentage()/100;
    timer_mod(throttle_timer, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL_RT) +
                                   CPU_THROTTLE_TIMESLICE_NS / (1-pct));
}

void cpu_throttle_set(int new_throttle_pct)
{
    /* Ensure throttle percentage is within valid range */
    new_throttle_pct = MIN(new_throttle_pct, CPU_THROTTLE_PCT_MAX);
    new_throttle_pct = MAX(new_throttle_pct, CPU_THROTTLE_PCT_MIN);

    atomic_set(&throttle_percentage, new_throttle_pct);

    timer_mod(throttle_timer, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL_RT) +
                                       CPU_THROTTLE_TIMESLICE_NS);
}

void cpu_throttle_stop(void)
{
    atomic_set(&throttle_percentage, 0);
}

bool cpu_throttle_active(void)
{
    return (cpu_throttle_get_percentage() != 0);
}

int cpu_throttle_get_percentage(void)
{
    return atomic_read(&throttle_percentage);
}

622 623
void cpu_ticks_init(void)
{
624
    seqlock_init(&timers_state.vm_clock_seqlock);
625
    vmstate_register(NULL, 0, &vmstate_timers, &timers_state);
626 627
    throttle_timer = timer_new_ns(QEMU_CLOCK_VIRTUAL_RT,
                                           cpu_throttle_timer_tick, NULL);
628 629
}

630
void configure_icount(QemuOpts *opts, Error **errp)
631
{
632
    const char *option;
633
    char *rem_str = NULL;
634 635

    option = qemu_opt_get(opts, "shift");
636
    if (!option) {
637 638 639
        if (qemu_opt_get(opts, "align") != NULL) {
            error_setg(errp, "Please specify shift option when using align");
        }
640 641
        return;
    }
642 643

    icount_sleep = qemu_opt_get_bool(opts, "sleep", true);
644 645
    if (icount_sleep) {
        icount_warp_timer = timer_new_ns(QEMU_CLOCK_VIRTUAL_RT,
646
                                         icount_timer_cb, NULL);
647
    }
648

649
    icount_align_option = qemu_opt_get_bool(opts, "align", false);
650 651

    if (icount_align_option && !icount_sleep) {
652
        error_setg(errp, "align=on and sleep=off are incompatible");
653
    }
654
    if (strcmp(option, "auto") != 0) {
655 656 657 658 659
        errno = 0;
        icount_time_shift = strtol(option, &rem_str, 0);
        if (errno != 0 || *rem_str != '\0' || !strlen(option)) {
            error_setg(errp, "icount: Invalid shift value");
        }
660 661
        use_icount = 1;
        return;
662 663
    } else if (icount_align_option) {
        error_setg(errp, "shift=auto and align=on are incompatible");
664
    } else if (!icount_sleep) {
665
        error_setg(errp, "shift=auto and sleep=off are incompatible");
666 667 668 669 670 671 672 673 674 675 676 677 678
    }

    use_icount = 2;

    /* 125MIPS seems a reasonable initial guess at the guest speed.
       It will be corrected fairly quickly anyway.  */
    icount_time_shift = 3;

    /* Have both realtime and virtual time triggers for speed adjustment.
       The realtime trigger catches emulated time passing too slowly,
       the virtual time trigger catches emulated time passing too fast.
       Realtime triggers occur even when idle, so use them less frequently
       than VM triggers.  */
679 680
    icount_rt_timer = timer_new_ms(QEMU_CLOCK_VIRTUAL_RT,
                                   icount_adjust_rt, NULL);
681
    timer_mod(icount_rt_timer,
682
                   qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL_RT) + 1000);
683 684 685 686
    icount_vm_timer = timer_new_ns(QEMU_CLOCK_VIRTUAL,
                                        icount_adjust_vm, NULL);
    timer_mod(icount_vm_timer,
                   qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) +
687
                   NANOSECONDS_PER_SECOND / 10);
688 689
}

690 691 692 693
/***********************************************************/
void hw_error(const char *fmt, ...)
{
    va_list ap;
694
    CPUState *cpu;
695 696 697 698 699

    va_start(ap, fmt);
    fprintf(stderr, "qemu: hardware error: ");
    vfprintf(stderr, fmt, ap);
    fprintf(stderr, "\n");
700
    CPU_FOREACH(cpu) {
701
        fprintf(stderr, "CPU #%d:\n", cpu->cpu_index);
702
        cpu_dump_state(cpu, stderr, fprintf, CPU_DUMP_FPU);
703 704 705 706 707 708 709
    }
    va_end(ap);
    abort();
}

void cpu_synchronize_all_states(void)
{
710
    CPUState *cpu;
711

712
    CPU_FOREACH(cpu) {
713
        cpu_synchronize_state(cpu);
714 715 716 717 718
    }
}

void cpu_synchronize_all_post_reset(void)
{
719
    CPUState *cpu;
720

721
    CPU_FOREACH(cpu) {
722
        cpu_synchronize_post_reset(cpu);
723 724 725 726 727
    }
}

void cpu_synchronize_all_post_init(void)
{
728
    CPUState *cpu;
729

730
    CPU_FOREACH(cpu) {
731
        cpu_synchronize_post_init(cpu);
732 733 734
    }
}

735
static int do_vm_stop(RunState state)
736
{
737 738
    int ret = 0;

739
    if (runstate_is_running()) {
740 741
        cpu_disable_ticks();
        pause_all_vcpus();
742
        runstate_set(state);
743
        vm_state_notify(0, state);
Wenchao Xia's avatar
Wenchao Xia committed
744
        qapi_event_send_stop(&error_abort);
745
    }
746

747
    bdrv_drain_all();
748
    ret = blk_flush_all();
749

750
    return ret;
751 752
}

753
static bool cpu_can_run(CPUState *cpu)
754
{
755
    if (cpu->stop) {
756
        return false;
757
    }
758
    if (cpu_is_stopped(cpu)) {
759
        return false;
760
    }
761
    return true;
762 763
}

764
static void cpu_handle_guest_debug(CPUState *cpu)
765
{
766
    gdb_set_stop_cpu(cpu);
767
    qemu_system_debug_request();
768
    cpu->stopped = true;
769 770
}

771 772 773 774 775 776 777 778 779 780 781 782
#ifdef CONFIG_LINUX
static void sigbus_reraise(void)
{
    sigset_t set;
    struct sigaction action;

    memset(&action, 0, sizeof(action));
    action.sa_handler = SIG_DFL;
    if (!sigaction(SIGBUS, &action, NULL)) {
        raise(SIGBUS);
        sigemptyset(&set);
        sigaddset(&set, SIGBUS);
783
        pthread_sigmask(SIG_UNBLOCK, &set, NULL);
784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809
    }
    perror("Failed to re-raise SIGBUS!\n");
    abort();
}

static void sigbus_handler(int n, struct qemu_signalfd_siginfo *siginfo,
                           void *ctx)
{
    if (kvm_on_sigbus(siginfo->ssi_code,
                      (void *)(intptr_t)siginfo->ssi_addr)) {
        sigbus_reraise();
    }
}

static void qemu_init_sigbus(void)
{
    struct sigaction action;

    memset(&action, 0, sizeof(action));
    action.sa_flags = SA_SIGINFO;
    action.sa_sigaction = (void (*)(int, siginfo_t*, void*))sigbus_handler;
    sigaction(SIGBUS, &action, NULL);

    prctl(PR_MCE_KILL, PR_MCE_KILL_SET, PR_MCE_KILL_EARLY, 0, 0);
}

810
static void qemu_kvm_eat_signals(CPUState *cpu)
811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830
{
    struct timespec ts = { 0, 0 };
    siginfo_t siginfo;
    sigset_t waitset;
    sigset_t chkset;
    int r;

    sigemptyset(&waitset);
    sigaddset(&waitset, SIG_IPI);
    sigaddset(&waitset, SIGBUS);

    do {
        r = sigtimedwait(&waitset, &siginfo, &ts);
        if (r == -1 && !(errno == EAGAIN || errno == EINTR)) {
            perror("sigtimedwait");
            exit(1);
        }

        switch (r) {
        case SIGBUS:
831
            if (kvm_on_sigbus_vcpu(cpu, siginfo.si_code, siginfo.si_addr)) {
832 833 834 835 836 837 838 839 840 841 842 843 844 845 846
                sigbus_reraise();
            }
            break;
        default:
            break;
        }

        r = sigpending(&chkset);
        if (r == -1) {
            perror("sigpending");
            exit(1);
        }
    } while (sigismember(&chkset, SIG_IPI) || sigismember(&chkset, SIGBUS));
}

847 848 849 850 851
#else /* !CONFIG_LINUX */

static void qemu_init_sigbus(void)
{
}
852

853
static void qemu_kvm_eat_signals(CPUState *cpu)
854 855
{
}
856 857
#endif /* !CONFIG_LINUX */

858
#ifndef _WIN32
859 860 861 862
static void dummy_signal(int sig)
{
}

863
static void qemu_kvm_init_cpu_signals(CPUState *cpu)
864 865 866 867 868 869 870 871 872 873 874 875
{
    int r;
    sigset_t set;
    struct sigaction sigact;

    memset(&sigact, 0, sizeof(sigact));
    sigact.sa_handler = dummy_signal;
    sigaction(SIG_IPI, &sigact, NULL);

    pthread_sigmask(SIG_BLOCK, NULL, &set);
    sigdelset(&set, SIG_IPI);
    sigdelset(&set, SIGBUS);
876
    r = kvm_set_signal_mask(cpu, &set);
877 878 879 880 881 882
    if (r) {
        fprintf(stderr, "kvm_set_signal_mask: %s\n", strerror(-r));
        exit(1);
    }
}

883
#else /* _WIN32 */
884
static void qemu_kvm_init_cpu_signals(CPUState *cpu)
885
{
886 887 888
    abort();
}
#endif /* _WIN32 */
889

890
static QemuMutex qemu_global_mutex;
891
static QemuCond qemu_io_proceeded_cond;
892
static unsigned iothread_requesting_mutex;
893 894 895 896 897 898 899

static QemuThread io_thread;

/* cpu creation */
static QemuCond qemu_cpu_cond;
/* system init */
static QemuCond qemu_pause_cond;
900
static QemuCond qemu_work_cond;
901

902
void qemu_init_cpu_loop(void)
903
{
904
    qemu_init_sigbus();
905 906 907
    qemu_cond_init(&qemu_cpu_cond);
    qemu_cond_init(&qemu_pause_cond);
    qemu_cond_init(&qemu_work_cond);
908
    qemu_cond_init(&qemu_io_proceeded_cond);
909 910
    qemu_mutex_init(&qemu_global_mutex);

911
    qemu_thread_get_self(&io_thread);
912 913
}

914
void run_on_cpu(CPUState *cpu, void (*func)(void *data), void *data)
915 916 917
{
    struct qemu_work_item wi;

918
    if (qemu_cpu_is_self(cpu)) {
919 920 921 922 923 924
        func(data);
        return;
    }

    wi.func = func;
    wi.data = data;
925
    wi.free = false;
926 927

    qemu_mutex_lock(&cpu->work_mutex);
928 929
    if (cpu->queued_work_first == NULL) {
        cpu->queued_work_first = &wi;
930
    } else {
931
        cpu->queued_work_last->next = &wi;
932
    }
933
    cpu->queued_work_last = &wi;
934 935
    wi.next = NULL;
    wi.done = false;
936
    qemu_mutex_unlock(&cpu->work_mutex);
937

938
    qemu_cpu_kick(cpu);
939
    while (!atomic_mb_read(&wi.done)) {
940
        CPUState *self_cpu = current_cpu;
941 942

        qemu_cond_wait(&qemu_work_cond, &qemu_global_mutex);
943
        current_cpu = self_cpu;
944 945 946
    }
}

947 948 949 950 951 952 953 954 955 956 957 958 959
void async_run_on_cpu(CPUState *cpu, void (*func)(void *data), void *data)
{
    struct qemu_work_item *wi;

    if (qemu_cpu_is_self(cpu)) {
        func(data);
        return;
    }

    wi = g_malloc0(sizeof(struct qemu_work_item));
    wi->func = func;
    wi->data = data;
    wi->free = true;
960 961

    qemu_mutex_lock(&cpu->work_mutex);
962 963 964 965 966 967 968 969
    if (cpu->queued_work_first == NULL) {
        cpu->queued_work_first = wi;
    } else {
        cpu->queued_work_last->next = wi;
    }
    cpu->queued_work_last = wi;
    wi->next = NULL;
    wi->done = false;
970
    qemu_mutex_unlock(&cpu->work_mutex);
971 972 973 974

    qemu_cpu_kick(cpu);
}

Gu Zheng's avatar
Gu Zheng committed
975 976 977 978 979 980 981 982 983 984 985 986
static void qemu_kvm_destroy_vcpu(CPUState *cpu)
{
    if (kvm_destroy_vcpu(cpu) < 0) {
        error_report("kvm_destroy_vcpu failed");
        exit(EXIT_FAILURE);
    }
}

static void qemu_tcg_destroy_vcpu(CPUState *cpu)
{
}

987
static void flush_queued_work(CPUState *cpu)
988 989 990
{
    struct qemu_work_item *wi;

991
    if (cpu->queued_work_first == NULL) {
992
        return;
993
    }
994

995 996 997
    qemu_mutex_lock(&cpu->work_mutex);
    while (cpu->queued_work_first != NULL) {
        wi = cpu->queued_work_first;
998
        cpu->queued_work_first = wi->next;
999 1000 1001 1002
        if (!cpu->queued_work_first) {
            cpu->queued_work_last = NULL;
        }
        qemu_mutex_unlock(&cpu->work_mutex);
1003
        wi->func(wi->data);
1004
        qemu_mutex_lock(&cpu->work_mutex);
1005 1006
        if (wi->free) {
            g_free(wi);
1007 1008
        } else {
            atomic_mb_set(&wi->done, true);
1009
        }
1010
    }
1011
    qemu_mutex_unlock(&cpu->work_mutex);
1012 1013 1014
    qemu_cond_broadcast(&qemu_work_cond);
}

1015
static void qemu_wait_io_event_common(CPUState *cpu)
1016
{
1017 1018
    if (cpu->stop) {
        cpu->stop = false;
1019
        cpu->stopped = true;
1020
        qemu_cond_broadcast(&qemu_pause_cond);
1021
    }
1022
    flush_queued_work(cpu);
1023
    cpu->thread_kicked = false;
1024 1025
}

1026
static void qemu_tcg_wait_io_event(CPUState *cpu)
1027
{
1028
    while (all_cpu_threads_idle()) {
1029
        qemu_cond_wait(cpu->halt_cond, &qemu_global_mutex);
1030
    }
1031

1032 1033 1034
    while (iothread_requesting_mutex) {
        qemu_cond_wait(&qemu_io_proceeded_cond, &qemu_global_mutex);
    }
1035

1036
    CPU_FOREACH(cpu) {
1037
        qemu_wait_io_event_common(cpu);
1038
    }
1039 1040
}

1041
static void qemu_kvm_wait_io_event(CPUState *cpu)
1042
{
1043
    while (cpu_thread_is_idle(cpu)) {
1044
        qemu_cond_wait(cpu->halt_cond, &qemu_global_mutex);
1045
    }
1046

1047
    qemu_kvm_eat_signals(cpu);
1048
    qemu_wait_io_event_common(cpu);
1049 1050
}

1051
static void *qemu_kvm_cpu_thread_fn(void *arg)
1052
{
1053
    CPUState *cpu = arg;
1054
    int r;
1055

1056 1057
    rcu_register_thread();

1058
    qemu_mutex_lock_iothread();
1059
    qemu_thread_get_self(cpu->thread);
1060
    cpu->thread_id = qemu_get_thread_id();
1061
    cpu->can_do_io = 1;
1062
    current_cpu = cpu;
1063

1064
    r = kvm_init_vcpu(cpu);
1065 1066 1067 1068
    if (r < 0) {
        fprintf(stderr, "kvm_init_vcpu failed: %s\n", strerror(-r));
        exit(1);
    }
1069

1070
    qemu_kvm_init_cpu_signals(cpu);
1071 1072

    /* signal CPU creation */
1073
    cpu->created = true;
1074 1075
    qemu_cond_signal(&qemu_cpu_cond);

Gu Zheng's avatar
Gu Zheng committed
1076
    do {
1077
        if (cpu_can_run(cpu)) {
1078
            r = kvm_cpu_exec(cpu);
1079
            if (r == EXCP_DEBUG) {
1080
                cpu_handle_guest_debug(cpu);
1081
            }
1082
        }
1083
        qemu_kvm_wait_io_event(cpu);
Gu Zheng's avatar
Gu Zheng committed
1084
    } while (!cpu->unplug || cpu_can_run(cpu));
1085

Gu Zheng's avatar
Gu Zheng committed
1086
    qemu_kvm_destroy_vcpu(cpu);
1087 1088
    cpu->created = false;
    qemu_cond_signal(&qemu_cpu_cond);
Gu Zheng's avatar
Gu Zheng committed
1089
    qemu_mutex_unlock_iothread();
1090 1091 1092
    return NULL;
}

1093 1094 1095 1096 1097 1098
static void *qemu_dummy_cpu_thread_fn(void *arg)
{
#ifdef _WIN32
    fprintf(stderr, "qtest is not supported under Windows\n");
    exit(1);
#else
1099
    CPUState *cpu = arg;
1100 1101 1102
    sigset_t waitset;
    int r;

1103 1104
    rcu_register_thread();

1105
    qemu_mutex_lock_iothread();
1106
    qemu_thread_get_self(cpu->thread);
1107
    cpu->thread_id = qemu_get_thread_id();
1108
    cpu->can_do_io = 1;
1109 1110 1111 1112 1113

    sigemptyset(&waitset);
    sigaddset(&waitset, SIG_IPI);

    /* signal CPU creation */
1114
    cpu->created = true;
1115 1116
    qemu_cond_signal(&qemu_cpu_cond);

1117
    current_cpu = cpu;
1118
    while (1) {
1119
        current_cpu = NULL;
1120 1121 1122 1123 1124 1125 1126 1127 1128 1129
        qemu_mutex_unlock_iothread();
        do {
            int sig;
            r = sigwait(&waitset, &sig);
        } while (r == -1 && (errno == EAGAIN || errno == EINTR));
        if (r == -1) {
            perror("sigwait");
            exit(1);
        }
        qemu_mutex_lock_iothread();
1130
        current_cpu = cpu;
1131
        qemu_wait_io_event_common(cpu);
1132 1133 1134 1135 1136 1137
    }

    return NULL;
#endif
}

1138 1139
static void tcg_exec_all(void);

1140
static void *qemu_tcg_cpu_thread_fn(void *arg)
1141
{
1142
    CPUState *cpu = arg;
Gu Zheng's avatar
Gu Zheng committed
1143
    CPUState *remove_cpu = NULL;
1144

1145 1146
    rcu_register_thread();

1147
    qemu_mutex_lock_iothread();
1148
    qemu_thread_get_self(cpu->thread);
1149

1150 1151 1152
    CPU_FOREACH(cpu) {
        cpu->thread_id = qemu_get_thread_id();
        cpu->created = true;
1153
        cpu->can_do_io = 1;
1154
    }
1155 1156
    qemu_cond_signal(&qemu_cpu_cond);

1157
    /* wait for initial kick-off after machine start */
1158
    while (first_cpu->stopped) {
1159
        qemu_cond_wait(first_cpu->halt_cond, &qemu_global_mutex);
1160 1161

        /* process any pending work */
1162
        CPU_FOREACH(cpu) {
1163
            qemu_wait_io_event_common(cpu);
1164
        }
1165
    }
1166

1167
    /* process any pending work */
1168
    atomic_mb_set(&exit_request, 1);
1169

1170
    while (1) {
1171
        tcg_exec_all();
1172 1173

        if (use_icount) {
1174
            int64_t deadline = qemu_clock_deadline_ns_all(QEMU_CLOCK_VIRTUAL);
1175 1176

            if (deadline == 0) {
1177
                qemu_clock_notify(QEMU_CLOCK_VIRTUAL);
1178
            }
1179
        }
1180
        qemu_tcg_wait_io_event(QTAILQ_FIRST(&cpus));
Gu Zheng's avatar
Gu Zheng committed
1181 1182 1183 1184 1185 1186 1187 1188
        CPU_FOREACH(cpu) {
            if (cpu->unplug && !cpu_can_run(cpu)) {
                remove_cpu = cpu;
                break;
            }
        }
        if (remove_cpu) {
            qemu_tcg_destroy_vcpu(remove_cpu);
1189 1190
            cpu->created = false;
            qemu_cond_signal(&qemu_cpu_cond);
Gu Zheng's avatar
Gu Zheng committed
1191 1192
            remove_cpu = NULL;
        }
1193 1194 1195 1196 1197
    }

    return NULL;
}

1198
static void qemu_cpu_kick_thread(CPUState *cpu)
Paolo Bonzini's avatar
Paolo Bonzini committed
1199 1200 1201 1202
{
#ifndef _WIN32
    int err;

1203 1204
    if (cpu->thread_kicked) {
        return;
1205
    }
1206
    cpu->thread_kicked = true;
1207
    err = pthread_kill(cpu->thread->thread, SIG_IPI);
Paolo Bonzini's avatar
Paolo Bonzini committed
1208 1209 1210 1211 1212
    if (err) {
        fprintf(stderr, "qemu:%s: %s", __func__, strerror(err));
        exit(1);
    }
#else /* _WIN32 */
1213 1214 1215
    abort();
#endif
}
1216

1217 1218 1219 1220 1221 1222 1223 1224 1225 1226
static void qemu_cpu_kick_no_halt(void)
{
    CPUState *cpu;
    /* Ensure whatever caused the exit has reached the CPU threads before
     * writing exit_request.
     */
    atomic_mb_set(&exit_request, 1);
    cpu = atomic_mb_read(&tcg_current_cpu);
    if (cpu) {
        cpu_exit(cpu);
Paolo Bonzini's avatar
Paolo Bonzini committed
1227 1228 1229
    }
}

1230
void qemu_cpu_kick(CPUState *cpu)
1231
{
1232
    qemu_cond_broadcast(cpu->halt_cond);
1233 1234 1235 1236 1237
    if (tcg_enabled()) {
        qemu_cpu_kick_no_halt();
    } else {
        qemu_cpu_kick_thread(cpu);
    }
1238 1239
}

1240
void qemu_cpu_kick_self(void)
1241
{
1242
    assert(current_cpu);
1243
    qemu_cpu_kick_thread(current_cpu);
1244 1245
}

1246
bool qemu_cpu_is_self(CPUState *cpu)
1247
{
1248
    return qemu_thread_is_self(cpu->thread);
1249 1250
}

1251
bool qemu_in_vcpu_thread(void)
1252
{
1253
    return current_cpu && qemu_cpu_is_self(current_cpu);
1254 1255
}

1256 1257 1258 1259 1260 1261 1262
static __thread bool iothread_locked = false;

bool qemu_mutex_iothread_locked(void)
{
    return iothread_locked;
}

1263 1264
void qemu_mutex_lock_iothread(void)
{
1265
    atomic_inc(&iothread_requesting_mutex);
1266