2 * handling kvm guest interrupts
4 * Copyright IBM Corp. 2008
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License (version 2 only)
8 * as published by the Free Software Foundation.
10 * Author(s): Carsten Otte <cotte@de.ibm.com>
13 #include <linux/interrupt.h>
14 #include <linux/kvm_host.h>
15 #include <linux/hrtimer.h>
16 #include <linux/signal.h>
17 #include <linux/slab.h>
18 #include <asm/asm-offsets.h>
19 #include <asm/uaccess.h>
22 #include "trace-s390.h"
24 #define IOINT_SCHID_MASK 0x0000ffff
25 #define IOINT_SSID_MASK 0x00030000
26 #define IOINT_CSSID_MASK 0x03fc0000
27 #define IOINT_AI_MASK 0x04000000
29 static int is_ioint(u64 type)
31 return ((type & 0xfffe0000u) != 0xfffe0000u);
34 int psw_extint_disabled(struct kvm_vcpu *vcpu)
36 return !(vcpu->arch.sie_block->gpsw.mask & PSW_MASK_EXT);
39 static int psw_ioint_disabled(struct kvm_vcpu *vcpu)
41 return !(vcpu->arch.sie_block->gpsw.mask & PSW_MASK_IO);
44 static int psw_mchk_disabled(struct kvm_vcpu *vcpu)
46 return !(vcpu->arch.sie_block->gpsw.mask & PSW_MASK_MCHECK);
49 static int psw_interrupts_disabled(struct kvm_vcpu *vcpu)
51 if ((vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PER) ||
52 (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_IO) ||
53 (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_EXT))
58 static u64 int_word_to_isc_bits(u32 int_word)
60 u8 isc = (int_word & 0x38000000) >> 27;
62 return (0x80 >> isc) << 24;
65 static int __interrupt_is_deliverable(struct kvm_vcpu *vcpu,
66 struct kvm_s390_interrupt_info *inti)
69 case KVM_S390_INT_EXTERNAL_CALL:
70 if (psw_extint_disabled(vcpu))
72 if (vcpu->arch.sie_block->gcr[0] & 0x2000ul)
74 case KVM_S390_INT_EMERGENCY:
75 if (psw_extint_disabled(vcpu))
77 if (vcpu->arch.sie_block->gcr[0] & 0x4000ul)
80 case KVM_S390_INT_SERVICE:
81 case KVM_S390_INT_PFAULT_INIT:
82 case KVM_S390_INT_PFAULT_DONE:
83 case KVM_S390_INT_VIRTIO:
84 if (psw_extint_disabled(vcpu))
86 if (vcpu->arch.sie_block->gcr[0] & 0x200ul)
89 case KVM_S390_PROGRAM_INT:
90 case KVM_S390_SIGP_STOP:
91 case KVM_S390_SIGP_SET_PREFIX:
92 case KVM_S390_RESTART:
95 if (psw_mchk_disabled(vcpu))
97 if (vcpu->arch.sie_block->gcr[14] & inti->mchk.cr14)
100 case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
101 if (psw_ioint_disabled(vcpu))
103 if (vcpu->arch.sie_block->gcr[6] &
104 int_word_to_isc_bits(inti->io.io_int_word))
108 printk(KERN_WARNING "illegal interrupt type %llx\n",
115 static void __set_cpu_idle(struct kvm_vcpu *vcpu)
117 BUG_ON(vcpu->vcpu_id > KVM_MAX_VCPUS - 1);
118 atomic_set_mask(CPUSTAT_WAIT, &vcpu->arch.sie_block->cpuflags);
119 set_bit(vcpu->vcpu_id, vcpu->arch.local_int.float_int->idle_mask);
122 static void __unset_cpu_idle(struct kvm_vcpu *vcpu)
124 BUG_ON(vcpu->vcpu_id > KVM_MAX_VCPUS - 1);
125 atomic_clear_mask(CPUSTAT_WAIT, &vcpu->arch.sie_block->cpuflags);
126 clear_bit(vcpu->vcpu_id, vcpu->arch.local_int.float_int->idle_mask);
129 static void __reset_intercept_indicators(struct kvm_vcpu *vcpu)
131 atomic_clear_mask(CPUSTAT_ECALL_PEND |
132 CPUSTAT_IO_INT | CPUSTAT_EXT_INT | CPUSTAT_STOP_INT,
133 &vcpu->arch.sie_block->cpuflags);
134 vcpu->arch.sie_block->lctl = 0x0000;
135 vcpu->arch.sie_block->ictl &= ~ICTL_LPSW;
138 static void __set_cpuflag(struct kvm_vcpu *vcpu, u32 flag)
140 atomic_set_mask(flag, &vcpu->arch.sie_block->cpuflags);
143 static void __set_intercept_indicator(struct kvm_vcpu *vcpu,
144 struct kvm_s390_interrupt_info *inti)
146 switch (inti->type) {
147 case KVM_S390_INT_EXTERNAL_CALL:
148 case KVM_S390_INT_EMERGENCY:
149 case KVM_S390_INT_SERVICE:
150 case KVM_S390_INT_PFAULT_INIT:
151 case KVM_S390_INT_PFAULT_DONE:
152 case KVM_S390_INT_VIRTIO:
153 if (psw_extint_disabled(vcpu))
154 __set_cpuflag(vcpu, CPUSTAT_EXT_INT);
156 vcpu->arch.sie_block->lctl |= LCTL_CR0;
158 case KVM_S390_SIGP_STOP:
159 __set_cpuflag(vcpu, CPUSTAT_STOP_INT);
162 if (psw_mchk_disabled(vcpu))
163 vcpu->arch.sie_block->ictl |= ICTL_LPSW;
165 vcpu->arch.sie_block->lctl |= LCTL_CR14;
167 case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
168 if (psw_ioint_disabled(vcpu))
169 __set_cpuflag(vcpu, CPUSTAT_IO_INT);
171 vcpu->arch.sie_block->lctl |= LCTL_CR6;
178 static void __do_deliver_interrupt(struct kvm_vcpu *vcpu,
179 struct kvm_s390_interrupt_info *inti)
181 const unsigned short table[] = { 2, 4, 4, 6 };
184 switch (inti->type) {
185 case KVM_S390_INT_EMERGENCY:
186 VCPU_EVENT(vcpu, 4, "%s", "interrupt: sigp emerg");
187 vcpu->stat.deliver_emergency_signal++;
188 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, inti->type,
189 inti->emerg.code, 0);
190 rc = put_guest(vcpu, 0x1201, (u16 __user *)__LC_EXT_INT_CODE);
191 rc |= put_guest(vcpu, inti->emerg.code,
192 (u16 __user *)__LC_EXT_CPU_ADDR);
193 rc |= copy_to_guest(vcpu, __LC_EXT_OLD_PSW,
194 &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
195 rc |= copy_from_guest(vcpu, &vcpu->arch.sie_block->gpsw,
196 __LC_EXT_NEW_PSW, sizeof(psw_t));
198 case KVM_S390_INT_EXTERNAL_CALL:
199 VCPU_EVENT(vcpu, 4, "%s", "interrupt: sigp ext call");
200 vcpu->stat.deliver_external_call++;
201 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, inti->type,
202 inti->extcall.code, 0);
203 rc = put_guest(vcpu, 0x1202, (u16 __user *)__LC_EXT_INT_CODE);
204 rc |= put_guest(vcpu, inti->extcall.code,
205 (u16 __user *)__LC_EXT_CPU_ADDR);
206 rc |= copy_to_guest(vcpu, __LC_EXT_OLD_PSW,
207 &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
208 rc |= copy_from_guest(vcpu, &vcpu->arch.sie_block->gpsw,
209 __LC_EXT_NEW_PSW, sizeof(psw_t));
211 case KVM_S390_INT_SERVICE:
212 VCPU_EVENT(vcpu, 4, "interrupt: sclp parm:%x",
213 inti->ext.ext_params);
214 vcpu->stat.deliver_service_signal++;
215 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, inti->type,
216 inti->ext.ext_params, 0);
217 rc = put_guest(vcpu, 0x2401, (u16 __user *)__LC_EXT_INT_CODE);
218 rc |= copy_to_guest(vcpu, __LC_EXT_OLD_PSW,
219 &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
220 rc |= copy_from_guest(vcpu, &vcpu->arch.sie_block->gpsw,
221 __LC_EXT_NEW_PSW, sizeof(psw_t));
222 rc |= put_guest(vcpu, inti->ext.ext_params,
223 (u32 __user *)__LC_EXT_PARAMS);
225 case KVM_S390_INT_PFAULT_INIT:
226 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, inti->type, 0,
227 inti->ext.ext_params2);
228 rc = put_guest(vcpu, 0x2603, (u16 __user *) __LC_EXT_INT_CODE);
229 rc |= put_guest(vcpu, 0x0600, (u16 __user *) __LC_EXT_CPU_ADDR);
230 rc |= copy_to_guest(vcpu, __LC_EXT_OLD_PSW,
231 &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
232 rc |= copy_from_guest(vcpu, &vcpu->arch.sie_block->gpsw,
233 __LC_EXT_NEW_PSW, sizeof(psw_t));
234 rc |= put_guest(vcpu, inti->ext.ext_params2,
235 (u64 __user *) __LC_EXT_PARAMS2);
237 case KVM_S390_INT_PFAULT_DONE:
238 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, inti->type, 0,
239 inti->ext.ext_params2);
240 rc = put_guest(vcpu, 0x2603, (u16 __user *) __LC_EXT_INT_CODE);
241 rc |= put_guest(vcpu, 0x0680, (u16 __user *) __LC_EXT_CPU_ADDR);
242 rc |= copy_to_guest(vcpu, __LC_EXT_OLD_PSW,
243 &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
244 rc |= copy_from_guest(vcpu, &vcpu->arch.sie_block->gpsw,
245 __LC_EXT_NEW_PSW, sizeof(psw_t));
246 rc |= put_guest(vcpu, inti->ext.ext_params2,
247 (u64 __user *) __LC_EXT_PARAMS2);
249 case KVM_S390_INT_VIRTIO:
250 VCPU_EVENT(vcpu, 4, "interrupt: virtio parm:%x,parm64:%llx",
251 inti->ext.ext_params, inti->ext.ext_params2);
252 vcpu->stat.deliver_virtio_interrupt++;
253 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, inti->type,
254 inti->ext.ext_params,
255 inti->ext.ext_params2);
256 rc = put_guest(vcpu, 0x2603, (u16 __user *)__LC_EXT_INT_CODE);
257 rc |= put_guest(vcpu, 0x0d00, (u16 __user *)__LC_EXT_CPU_ADDR);
258 rc |= copy_to_guest(vcpu, __LC_EXT_OLD_PSW,
259 &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
260 rc |= copy_from_guest(vcpu, &vcpu->arch.sie_block->gpsw,
261 __LC_EXT_NEW_PSW, sizeof(psw_t));
262 rc |= put_guest(vcpu, inti->ext.ext_params,
263 (u32 __user *)__LC_EXT_PARAMS);
264 rc |= put_guest(vcpu, inti->ext.ext_params2,
265 (u64 __user *)__LC_EXT_PARAMS2);
267 case KVM_S390_SIGP_STOP:
268 VCPU_EVENT(vcpu, 4, "%s", "interrupt: cpu stop");
269 vcpu->stat.deliver_stop_signal++;
270 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, inti->type,
272 __set_intercept_indicator(vcpu, inti);
275 case KVM_S390_SIGP_SET_PREFIX:
276 VCPU_EVENT(vcpu, 4, "interrupt: set prefix to %x",
277 inti->prefix.address);
278 vcpu->stat.deliver_prefix_signal++;
279 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, inti->type,
280 inti->prefix.address, 0);
281 kvm_s390_set_prefix(vcpu, inti->prefix.address);
284 case KVM_S390_RESTART:
285 VCPU_EVENT(vcpu, 4, "%s", "interrupt: cpu restart");
286 vcpu->stat.deliver_restart_signal++;
287 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, inti->type,
289 rc = copy_to_guest(vcpu,
290 offsetof(struct _lowcore, restart_old_psw),
291 &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
292 rc |= copy_from_guest(vcpu, &vcpu->arch.sie_block->gpsw,
293 offsetof(struct _lowcore, restart_psw),
295 atomic_clear_mask(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
297 case KVM_S390_PROGRAM_INT:
298 VCPU_EVENT(vcpu, 4, "interrupt: pgm check code:%x, ilc:%x",
300 table[vcpu->arch.sie_block->ipa >> 14]);
301 vcpu->stat.deliver_program_int++;
302 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, inti->type,
304 rc = put_guest(vcpu, inti->pgm.code, (u16 __user *)__LC_PGM_INT_CODE);
305 rc |= put_guest(vcpu, table[vcpu->arch.sie_block->ipa >> 14],
306 (u16 __user *)__LC_PGM_ILC);
307 rc |= copy_to_guest(vcpu, __LC_PGM_OLD_PSW,
308 &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
309 rc |= copy_from_guest(vcpu, &vcpu->arch.sie_block->gpsw,
310 __LC_PGM_NEW_PSW, sizeof(psw_t));
314 VCPU_EVENT(vcpu, 4, "interrupt: machine check mcic=%llx",
316 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, inti->type,
319 rc = kvm_s390_vcpu_store_status(vcpu,
320 KVM_S390_STORE_STATUS_PREFIXED);
321 rc |= put_guest(vcpu, inti->mchk.mcic, (u64 __user *) __LC_MCCK_CODE);
322 rc |= copy_to_guest(vcpu, __LC_MCK_OLD_PSW,
323 &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
324 rc |= copy_from_guest(vcpu, &vcpu->arch.sie_block->gpsw,
325 __LC_MCK_NEW_PSW, sizeof(psw_t));
328 case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
330 __u32 param0 = ((__u32)inti->io.subchannel_id << 16) |
331 inti->io.subchannel_nr;
332 __u64 param1 = ((__u64)inti->io.io_int_parm << 32) |
333 inti->io.io_int_word;
334 VCPU_EVENT(vcpu, 4, "interrupt: I/O %llx", inti->type);
335 vcpu->stat.deliver_io_int++;
336 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, inti->type,
338 rc = put_guest(vcpu, inti->io.subchannel_id,
339 (u16 __user *) __LC_SUBCHANNEL_ID);
340 rc |= put_guest(vcpu, inti->io.subchannel_nr,
341 (u16 __user *) __LC_SUBCHANNEL_NR);
342 rc |= put_guest(vcpu, inti->io.io_int_parm,
343 (u32 __user *) __LC_IO_INT_PARM);
344 rc |= put_guest(vcpu, inti->io.io_int_word,
345 (u32 __user *) __LC_IO_INT_WORD);
346 rc |= copy_to_guest(vcpu, __LC_IO_OLD_PSW,
347 &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
348 rc |= copy_from_guest(vcpu, &vcpu->arch.sie_block->gpsw,
349 __LC_IO_NEW_PSW, sizeof(psw_t));
356 printk("kvm: The guest lowcore is not mapped during interrupt "
357 "delivery, killing userspace\n");
362 static int __try_deliver_ckc_interrupt(struct kvm_vcpu *vcpu)
366 if (psw_extint_disabled(vcpu))
368 if (!(vcpu->arch.sie_block->gcr[0] & 0x800ul))
370 rc = put_guest(vcpu, 0x1004, (u16 __user *)__LC_EXT_INT_CODE);
371 rc |= copy_to_guest(vcpu, __LC_EXT_OLD_PSW,
372 &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
373 rc |= copy_from_guest(vcpu, &vcpu->arch.sie_block->gpsw,
374 __LC_EXT_NEW_PSW, sizeof(psw_t));
376 printk("kvm: The guest lowcore is not mapped during interrupt "
377 "delivery, killing userspace\n");
383 int kvm_cpu_has_interrupt(struct kvm_vcpu *vcpu)
385 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
386 struct kvm_s390_float_interrupt *fi = vcpu->arch.local_int.float_int;
387 struct kvm_s390_interrupt_info *inti;
390 if (atomic_read(&li->active)) {
391 spin_lock_bh(&li->lock);
392 list_for_each_entry(inti, &li->list, list)
393 if (__interrupt_is_deliverable(vcpu, inti)) {
397 spin_unlock_bh(&li->lock);
400 if ((!rc) && atomic_read(&fi->active)) {
401 spin_lock(&fi->lock);
402 list_for_each_entry(inti, &fi->list, list)
403 if (__interrupt_is_deliverable(vcpu, inti)) {
407 spin_unlock(&fi->lock);
410 if ((!rc) && (vcpu->arch.sie_block->ckc <
411 get_tod_clock_fast() + vcpu->arch.sie_block->epoch)) {
412 if ((!psw_extint_disabled(vcpu)) &&
413 (vcpu->arch.sie_block->gcr[0] & 0x800ul))
420 int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu)
425 int kvm_s390_handle_wait(struct kvm_vcpu *vcpu)
428 DECLARE_WAITQUEUE(wait, current);
430 vcpu->stat.exit_wait_state++;
431 if (kvm_cpu_has_interrupt(vcpu))
434 __set_cpu_idle(vcpu);
435 spin_lock_bh(&vcpu->arch.local_int.lock);
436 vcpu->arch.local_int.timer_due = 0;
437 spin_unlock_bh(&vcpu->arch.local_int.lock);
439 if (psw_interrupts_disabled(vcpu)) {
440 VCPU_EVENT(vcpu, 3, "%s", "disabled wait");
441 __unset_cpu_idle(vcpu);
442 return -EOPNOTSUPP; /* disabled wait */
445 if (psw_extint_disabled(vcpu) ||
446 (!(vcpu->arch.sie_block->gcr[0] & 0x800ul))) {
447 VCPU_EVENT(vcpu, 3, "%s", "enabled wait w/o timer");
451 now = get_tod_clock_fast() + vcpu->arch.sie_block->epoch;
452 if (vcpu->arch.sie_block->ckc < now) {
453 __unset_cpu_idle(vcpu);
457 sltime = tod_to_ns(vcpu->arch.sie_block->ckc - now);
459 hrtimer_start(&vcpu->arch.ckc_timer, ktime_set (0, sltime) , HRTIMER_MODE_REL);
460 VCPU_EVENT(vcpu, 5, "enabled wait via clock comparator: %llx ns", sltime);
462 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
463 spin_lock(&vcpu->arch.local_int.float_int->lock);
464 spin_lock_bh(&vcpu->arch.local_int.lock);
465 add_wait_queue(&vcpu->wq, &wait);
466 while (list_empty(&vcpu->arch.local_int.list) &&
467 list_empty(&vcpu->arch.local_int.float_int->list) &&
468 (!vcpu->arch.local_int.timer_due) &&
469 !signal_pending(current)) {
470 set_current_state(TASK_INTERRUPTIBLE);
471 spin_unlock_bh(&vcpu->arch.local_int.lock);
472 spin_unlock(&vcpu->arch.local_int.float_int->lock);
474 spin_lock(&vcpu->arch.local_int.float_int->lock);
475 spin_lock_bh(&vcpu->arch.local_int.lock);
477 __unset_cpu_idle(vcpu);
478 __set_current_state(TASK_RUNNING);
479 remove_wait_queue(&vcpu->wq, &wait);
480 spin_unlock_bh(&vcpu->arch.local_int.lock);
481 spin_unlock(&vcpu->arch.local_int.float_int->lock);
482 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
484 hrtimer_try_to_cancel(&vcpu->arch.ckc_timer);
488 void kvm_s390_tasklet(unsigned long parm)
490 struct kvm_vcpu *vcpu = (struct kvm_vcpu *) parm;
492 spin_lock(&vcpu->arch.local_int.lock);
493 vcpu->arch.local_int.timer_due = 1;
494 if (waitqueue_active(&vcpu->wq))
495 wake_up_interruptible(&vcpu->wq);
496 spin_unlock(&vcpu->arch.local_int.lock);
500 * low level hrtimer wake routine. Because this runs in hardirq context
501 * we schedule a tasklet to do the real work.
503 enum hrtimer_restart kvm_s390_idle_wakeup(struct hrtimer *timer)
505 struct kvm_vcpu *vcpu;
507 vcpu = container_of(timer, struct kvm_vcpu, arch.ckc_timer);
508 tasklet_schedule(&vcpu->arch.tasklet);
510 return HRTIMER_NORESTART;
513 void kvm_s390_deliver_pending_interrupts(struct kvm_vcpu *vcpu)
515 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
516 struct kvm_s390_float_interrupt *fi = vcpu->arch.local_int.float_int;
517 struct kvm_s390_interrupt_info *n, *inti = NULL;
520 __reset_intercept_indicators(vcpu);
521 if (atomic_read(&li->active)) {
524 spin_lock_bh(&li->lock);
525 list_for_each_entry_safe(inti, n, &li->list, list) {
526 if (__interrupt_is_deliverable(vcpu, inti)) {
527 list_del(&inti->list);
531 __set_intercept_indicator(vcpu, inti);
533 if (list_empty(&li->list))
534 atomic_set(&li->active, 0);
535 spin_unlock_bh(&li->lock);
537 __do_deliver_interrupt(vcpu, inti);
543 if ((vcpu->arch.sie_block->ckc <
544 get_tod_clock_fast() + vcpu->arch.sie_block->epoch))
545 __try_deliver_ckc_interrupt(vcpu);
547 if (atomic_read(&fi->active)) {
550 spin_lock(&fi->lock);
551 list_for_each_entry_safe(inti, n, &fi->list, list) {
552 if (__interrupt_is_deliverable(vcpu, inti)) {
553 list_del(&inti->list);
558 __set_intercept_indicator(vcpu, inti);
560 if (list_empty(&fi->list))
561 atomic_set(&fi->active, 0);
562 spin_unlock(&fi->lock);
564 __do_deliver_interrupt(vcpu, inti);
571 void kvm_s390_deliver_pending_machine_checks(struct kvm_vcpu *vcpu)
573 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
574 struct kvm_s390_float_interrupt *fi = vcpu->arch.local_int.float_int;
575 struct kvm_s390_interrupt_info *n, *inti = NULL;
578 __reset_intercept_indicators(vcpu);
579 if (atomic_read(&li->active)) {
582 spin_lock_bh(&li->lock);
583 list_for_each_entry_safe(inti, n, &li->list, list) {
584 if ((inti->type == KVM_S390_MCHK) &&
585 __interrupt_is_deliverable(vcpu, inti)) {
586 list_del(&inti->list);
590 __set_intercept_indicator(vcpu, inti);
592 if (list_empty(&li->list))
593 atomic_set(&li->active, 0);
594 spin_unlock_bh(&li->lock);
596 __do_deliver_interrupt(vcpu, inti);
602 if (atomic_read(&fi->active)) {
605 spin_lock(&fi->lock);
606 list_for_each_entry_safe(inti, n, &fi->list, list) {
607 if ((inti->type == KVM_S390_MCHK) &&
608 __interrupt_is_deliverable(vcpu, inti)) {
609 list_del(&inti->list);
614 __set_intercept_indicator(vcpu, inti);
616 if (list_empty(&fi->list))
617 atomic_set(&fi->active, 0);
618 spin_unlock(&fi->lock);
620 __do_deliver_interrupt(vcpu, inti);
627 int kvm_s390_inject_program_int(struct kvm_vcpu *vcpu, u16 code)
629 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
630 struct kvm_s390_interrupt_info *inti;
632 inti = kzalloc(sizeof(*inti), GFP_KERNEL);
636 inti->type = KVM_S390_PROGRAM_INT;
637 inti->pgm.code = code;
639 VCPU_EVENT(vcpu, 3, "inject: program check %d (from kernel)", code);
640 trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, inti->type, code, 0, 1);
641 spin_lock_bh(&li->lock);
642 list_add(&inti->list, &li->list);
643 atomic_set(&li->active, 1);
644 BUG_ON(waitqueue_active(li->wq));
645 spin_unlock_bh(&li->lock);
649 struct kvm_s390_interrupt_info *kvm_s390_get_io_int(struct kvm *kvm,
652 struct kvm_s390_float_interrupt *fi;
653 struct kvm_s390_interrupt_info *inti, *iter;
655 if ((!schid && !cr6) || (schid && cr6))
657 mutex_lock(&kvm->lock);
658 fi = &kvm->arch.float_int;
659 spin_lock(&fi->lock);
661 list_for_each_entry(iter, &fi->list, list) {
662 if (!is_ioint(iter->type))
665 ((cr6 & int_word_to_isc_bits(iter->io.io_int_word)) == 0))
668 if (((schid & 0x00000000ffff0000) >> 16) !=
669 iter->io.subchannel_id)
671 if ((schid & 0x000000000000ffff) !=
672 iter->io.subchannel_nr)
679 list_del_init(&inti->list);
682 if (list_empty(&fi->list))
683 atomic_set(&fi->active, 0);
684 spin_unlock(&fi->lock);
685 mutex_unlock(&kvm->lock);
689 static int __inject_vm(struct kvm *kvm, struct kvm_s390_interrupt_info *inti)
691 struct kvm_s390_local_interrupt *li;
692 struct kvm_s390_float_interrupt *fi;
693 struct kvm_s390_interrupt_info *iter;
697 mutex_lock(&kvm->lock);
698 fi = &kvm->arch.float_int;
699 spin_lock(&fi->lock);
700 if (fi->irq_count >= KVM_S390_MAX_FLOAT_IRQS) {
705 if (!is_ioint(inti->type)) {
706 list_add_tail(&inti->list, &fi->list);
708 u64 isc_bits = int_word_to_isc_bits(inti->io.io_int_word);
710 /* Keep I/O interrupts sorted in isc order. */
711 list_for_each_entry(iter, &fi->list, list) {
712 if (!is_ioint(iter->type))
714 if (int_word_to_isc_bits(iter->io.io_int_word)
719 list_add_tail(&inti->list, &iter->list);
721 atomic_set(&fi->active, 1);
722 sigcpu = find_first_bit(fi->idle_mask, KVM_MAX_VCPUS);
723 if (sigcpu == KVM_MAX_VCPUS) {
725 sigcpu = fi->next_rr_cpu++;
726 if (sigcpu == KVM_MAX_VCPUS)
727 sigcpu = fi->next_rr_cpu = 0;
728 } while (fi->local_int[sigcpu] == NULL);
730 li = fi->local_int[sigcpu];
731 spin_lock_bh(&li->lock);
732 atomic_set_mask(CPUSTAT_EXT_INT, li->cpuflags);
733 if (waitqueue_active(li->wq))
734 wake_up_interruptible(li->wq);
735 spin_unlock_bh(&li->lock);
737 spin_unlock(&fi->lock);
738 mutex_unlock(&kvm->lock);
742 int kvm_s390_inject_vm(struct kvm *kvm,
743 struct kvm_s390_interrupt *s390int)
745 struct kvm_s390_interrupt_info *inti;
747 inti = kzalloc(sizeof(*inti), GFP_KERNEL);
751 inti->type = s390int->type;
752 switch (inti->type) {
753 case KVM_S390_INT_VIRTIO:
754 VM_EVENT(kvm, 5, "inject: virtio parm:%x,parm64:%llx",
755 s390int->parm, s390int->parm64);
756 inti->ext.ext_params = s390int->parm;
757 inti->ext.ext_params2 = s390int->parm64;
759 case KVM_S390_INT_SERVICE:
760 VM_EVENT(kvm, 5, "inject: sclp parm:%x", s390int->parm);
761 inti->ext.ext_params = s390int->parm;
763 case KVM_S390_INT_PFAULT_DONE:
764 inti->type = s390int->type;
765 inti->ext.ext_params2 = s390int->parm64;
768 VM_EVENT(kvm, 5, "inject: machine check parm64:%llx",
770 inti->mchk.cr14 = s390int->parm; /* upper bits are not used */
771 inti->mchk.mcic = s390int->parm64;
773 case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
774 if (inti->type & IOINT_AI_MASK)
775 VM_EVENT(kvm, 5, "%s", "inject: I/O (AI)");
777 VM_EVENT(kvm, 5, "inject: I/O css %x ss %x schid %04x",
778 s390int->type & IOINT_CSSID_MASK,
779 s390int->type & IOINT_SSID_MASK,
780 s390int->type & IOINT_SCHID_MASK);
781 inti->io.subchannel_id = s390int->parm >> 16;
782 inti->io.subchannel_nr = s390int->parm & 0x0000ffffu;
783 inti->io.io_int_parm = s390int->parm64 >> 32;
784 inti->io.io_int_word = s390int->parm64 & 0x00000000ffffffffull;
790 trace_kvm_s390_inject_vm(s390int->type, s390int->parm, s390int->parm64,
793 return __inject_vm(kvm, inti);
796 int kvm_s390_inject_vcpu(struct kvm_vcpu *vcpu,
797 struct kvm_s390_interrupt *s390int)
799 struct kvm_s390_local_interrupt *li;
800 struct kvm_s390_interrupt_info *inti;
802 inti = kzalloc(sizeof(*inti), GFP_KERNEL);
806 switch (s390int->type) {
807 case KVM_S390_PROGRAM_INT:
808 if (s390int->parm & 0xffff0000) {
812 inti->type = s390int->type;
813 inti->pgm.code = s390int->parm;
814 VCPU_EVENT(vcpu, 3, "inject: program check %d (from user)",
817 case KVM_S390_SIGP_SET_PREFIX:
818 inti->prefix.address = s390int->parm;
819 inti->type = s390int->type;
820 VCPU_EVENT(vcpu, 3, "inject: set prefix to %x (from user)",
823 case KVM_S390_SIGP_STOP:
824 case KVM_S390_RESTART:
825 VCPU_EVENT(vcpu, 3, "inject: type %x", s390int->type);
826 inti->type = s390int->type;
828 case KVM_S390_INT_EXTERNAL_CALL:
829 if (s390int->parm & 0xffff0000) {
833 VCPU_EVENT(vcpu, 3, "inject: external call source-cpu:%u",
835 inti->type = s390int->type;
836 inti->extcall.code = s390int->parm;
838 case KVM_S390_INT_EMERGENCY:
839 if (s390int->parm & 0xffff0000) {
843 VCPU_EVENT(vcpu, 3, "inject: emergency %u\n", s390int->parm);
844 inti->type = s390int->type;
845 inti->emerg.code = s390int->parm;
848 VCPU_EVENT(vcpu, 5, "inject: machine check parm64:%llx",
850 inti->type = s390int->type;
851 inti->mchk.mcic = s390int->parm64;
853 case KVM_S390_INT_PFAULT_INIT:
854 inti->type = s390int->type;
855 inti->ext.ext_params2 = s390int->parm64;
857 case KVM_S390_INT_VIRTIO:
858 case KVM_S390_INT_SERVICE:
859 case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
864 trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, s390int->type, s390int->parm,
867 mutex_lock(&vcpu->kvm->lock);
868 li = &vcpu->arch.local_int;
869 spin_lock_bh(&li->lock);
870 if (inti->type == KVM_S390_PROGRAM_INT)
871 list_add(&inti->list, &li->list);
873 list_add_tail(&inti->list, &li->list);
874 atomic_set(&li->active, 1);
875 if (inti->type == KVM_S390_SIGP_STOP)
876 li->action_bits |= ACTION_STOP_ON_STOP;
877 atomic_set_mask(CPUSTAT_EXT_INT, li->cpuflags);
878 if (waitqueue_active(&vcpu->wq))
879 wake_up_interruptible(&vcpu->wq);
880 spin_unlock_bh(&li->lock);
881 mutex_unlock(&vcpu->kvm->lock);
885 static void clear_floating_interrupts(struct kvm *kvm)
887 struct kvm_s390_float_interrupt *fi;
888 struct kvm_s390_interrupt_info *n, *inti = NULL;
890 mutex_lock(&kvm->lock);
891 fi = &kvm->arch.float_int;
892 spin_lock(&fi->lock);
893 list_for_each_entry_safe(inti, n, &fi->list, list) {
894 list_del(&inti->list);
898 atomic_set(&fi->active, 0);
899 spin_unlock(&fi->lock);
900 mutex_unlock(&kvm->lock);
903 static inline int copy_irq_to_user(struct kvm_s390_interrupt_info *inti,
906 struct kvm_s390_irq __user *uptr = (struct kvm_s390_irq __user *) addr;
907 struct kvm_s390_irq irq = {0};
909 irq.type = inti->type;
910 switch (inti->type) {
911 case KVM_S390_INT_PFAULT_INIT:
912 case KVM_S390_INT_PFAULT_DONE:
913 case KVM_S390_INT_VIRTIO:
914 case KVM_S390_INT_SERVICE:
915 irq.u.ext = inti->ext;
917 case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
921 irq.u.mchk = inti->mchk;
927 if (copy_to_user(uptr, &irq, sizeof(irq)))
933 static int get_all_floating_irqs(struct kvm *kvm, __u8 *buf, __u64 len)
935 struct kvm_s390_interrupt_info *inti;
936 struct kvm_s390_float_interrupt *fi;
940 mutex_lock(&kvm->lock);
941 fi = &kvm->arch.float_int;
942 spin_lock(&fi->lock);
944 list_for_each_entry(inti, &fi->list, list) {
945 if (len < sizeof(struct kvm_s390_irq)) {
946 /* signal userspace to try again */
950 ret = copy_irq_to_user(inti, buf);
953 buf += sizeof(struct kvm_s390_irq);
954 len -= sizeof(struct kvm_s390_irq);
958 spin_unlock(&fi->lock);
959 mutex_unlock(&kvm->lock);
961 return ret < 0 ? ret : n;
964 static int flic_get_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
968 switch (attr->group) {
969 case KVM_DEV_FLIC_GET_ALL_IRQS:
970 r = get_all_floating_irqs(dev->kvm, (u8 *) attr->addr,
980 static inline int copy_irq_from_user(struct kvm_s390_interrupt_info *inti,
983 struct kvm_s390_irq __user *uptr = (struct kvm_s390_irq __user *) addr;
988 if (get_user(inti->type, (u64 __user *)addr))
991 switch (inti->type) {
992 case KVM_S390_INT_PFAULT_INIT:
993 case KVM_S390_INT_PFAULT_DONE:
994 case KVM_S390_INT_VIRTIO:
995 case KVM_S390_INT_SERVICE:
996 target = (void *) &inti->ext;
997 source = &uptr->u.ext;
998 size = sizeof(inti->ext);
1000 case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
1001 target = (void *) &inti->io;
1002 source = &uptr->u.io;
1003 size = sizeof(inti->io);
1006 target = (void *) &inti->mchk;
1007 source = &uptr->u.mchk;
1008 size = sizeof(inti->mchk);
1014 if (copy_from_user(target, source, size))
1020 static int enqueue_floating_irq(struct kvm_device *dev,
1021 struct kvm_device_attr *attr)
1023 struct kvm_s390_interrupt_info *inti = NULL;
1025 int len = attr->attr;
1027 if (len % sizeof(struct kvm_s390_irq) != 0)
1029 else if (len > KVM_S390_FLIC_MAX_BUFFER)
1032 while (len >= sizeof(struct kvm_s390_irq)) {
1033 inti = kzalloc(sizeof(*inti), GFP_KERNEL);
1037 r = copy_irq_from_user(inti, attr->addr);
1042 r = __inject_vm(dev->kvm, inti);
1047 len -= sizeof(struct kvm_s390_irq);
1048 attr->addr += sizeof(struct kvm_s390_irq);
1054 static int flic_set_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
1058 struct kvm_vcpu *vcpu;
1060 switch (attr->group) {
1061 case KVM_DEV_FLIC_ENQUEUE:
1062 r = enqueue_floating_irq(dev, attr);
1064 case KVM_DEV_FLIC_CLEAR_IRQS:
1066 clear_floating_interrupts(dev->kvm);
1068 case KVM_DEV_FLIC_APF_ENABLE:
1069 dev->kvm->arch.gmap->pfault_enabled = 1;
1071 case KVM_DEV_FLIC_APF_DISABLE_WAIT:
1072 dev->kvm->arch.gmap->pfault_enabled = 0;
1074 * Make sure no async faults are in transition when
1075 * clearing the queues. So we don't need to worry
1076 * about late coming workers.
1078 synchronize_srcu(&dev->kvm->srcu);
1079 kvm_for_each_vcpu(i, vcpu, dev->kvm)
1080 kvm_clear_async_pf_completion_queue(vcpu);
1089 static int flic_create(struct kvm_device *dev, u32 type)
1093 if (dev->kvm->arch.flic)
1095 dev->kvm->arch.flic = dev;
1099 static void flic_destroy(struct kvm_device *dev)
1101 dev->kvm->arch.flic = NULL;
1105 /* s390 floating irq controller (flic) */
1106 struct kvm_device_ops kvm_flic_ops = {
1108 .get_attr = flic_get_attr,
1109 .set_attr = flic_set_attr,
1110 .create = flic_create,
1111 .destroy = flic_destroy,