arch/s390/kvm/kvm-s390.c

   1 /*
   2  * hosting zSeries kernel virtual machines
   3  *
   4  * Copyright IBM Corp. 2008, 2009
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License (version 2 only)
   8  * as published by the Free Software Foundation.
   9  *
  10  *    Author(s): Carsten Otte <cotte@de.ibm.com>
  11  *               Christian Borntraeger <borntraeger@de.ibm.com>
  12  *               Heiko Carstens <heiko.carstens@de.ibm.com>
  13  *               Christian Ehrhardt <ehrhardt@de.ibm.com>
  14  *               Jason J. Herne <jjherne@us.ibm.com>
  15  */
  16
  17 #include <linux/compiler.h>
  18 #include <linux/err.h>
  19 #include <linux/fs.h>
  20 #include <linux/hrtimer.h>
  21 #include <linux/init.h>
  22 #include <linux/kvm.h>
  23 #include <linux/kvm_host.h>
  24 #include <linux/module.h>
  25 #include <linux/slab.h>
  26 #include <linux/timer.h>
  27 #include <asm/asm-offsets.h>
  28 #include <asm/lowcore.h>
  29 #include <asm/pgtable.h>
  30 #include <asm/nmi.h>
  31 #include <asm/switch_to.h>
  32 #include <asm/facility.h>
  33 #include <asm/sclp.h>
  34 #include "kvm-s390.h"
  35 #include "gaccess.h"
  36
  37 #define CREATE_TRACE_POINTS
  38 #include "trace.h"
  39 #include "trace-s390.h"
  40
  41 #define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
  42
  43 struct kvm_stats_debugfs_item debugfs_entries[] = {
  44         { "userspace_handled", VCPU_STAT(exit_userspace) },
  45         { "exit_null", VCPU_STAT(exit_null) },
  46         { "exit_validity", VCPU_STAT(exit_validity) },
  47         { "exit_stop_request", VCPU_STAT(exit_stop_request) },
  48         { "exit_external_request", VCPU_STAT(exit_external_request) },
  49         { "exit_external_interrupt", VCPU_STAT(exit_external_interrupt) },
  50         { "exit_instruction", VCPU_STAT(exit_instruction) },
  51         { "exit_program_interruption", VCPU_STAT(exit_program_interruption) },
  52         { "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program) },
  53         { "instruction_lctlg", VCPU_STAT(instruction_lctlg) },
  54         { "instruction_lctl", VCPU_STAT(instruction_lctl) },
  55         { "instruction_stctl", VCPU_STAT(instruction_stctl) },
  56         { "instruction_stctg", VCPU_STAT(instruction_stctg) },
  57         { "deliver_emergency_signal", VCPU_STAT(deliver_emergency_signal) },
  58         { "deliver_external_call", VCPU_STAT(deliver_external_call) },
  59         { "deliver_service_signal", VCPU_STAT(deliver_service_signal) },
  60         { "deliver_virtio_interrupt", VCPU_STAT(deliver_virtio_interrupt) },
  61         { "deliver_stop_signal", VCPU_STAT(deliver_stop_signal) },
  62         { "deliver_prefix_signal", VCPU_STAT(deliver_prefix_signal) },
  63         { "deliver_restart_signal", VCPU_STAT(deliver_restart_signal) },
  64         { "deliver_program_interruption", VCPU_STAT(deliver_program_int) },
  65         { "exit_wait_state", VCPU_STAT(exit_wait_state) },
  66         { "instruction_pfmf", VCPU_STAT(instruction_pfmf) },
  67         { "instruction_stidp", VCPU_STAT(instruction_stidp) },
  68         { "instruction_spx", VCPU_STAT(instruction_spx) },
  69         { "instruction_stpx", VCPU_STAT(instruction_stpx) },
  70         { "instruction_stap", VCPU_STAT(instruction_stap) },
  71         { "instruction_storage_key", VCPU_STAT(instruction_storage_key) },
  72         { "instruction_ipte_interlock", VCPU_STAT(instruction_ipte_interlock) },
  73         { "instruction_stsch", VCPU_STAT(instruction_stsch) },
  74         { "instruction_chsc", VCPU_STAT(instruction_chsc) },
  75         { "instruction_essa", VCPU_STAT(instruction_essa) },
  76         { "instruction_stsi", VCPU_STAT(instruction_stsi) },
  77         { "instruction_stfl", VCPU_STAT(instruction_stfl) },
  78         { "instruction_tprot", VCPU_STAT(instruction_tprot) },
  79         { "instruction_sigp_sense", VCPU_STAT(instruction_sigp_sense) },
  80         { "instruction_sigp_sense_running", VCPU_STAT(instruction_sigp_sense_running) },
  81         { "instruction_sigp_external_call", VCPU_STAT(instruction_sigp_external_call) },
  82         { "instruction_sigp_emergency", VCPU_STAT(instruction_sigp_emergency) },
  83         { "instruction_sigp_stop", VCPU_STAT(instruction_sigp_stop) },
  84         { "instruction_sigp_set_arch", VCPU_STAT(instruction_sigp_arch) },
  85         { "instruction_sigp_set_prefix", VCPU_STAT(instruction_sigp_prefix) },
  86         { "instruction_sigp_restart", VCPU_STAT(instruction_sigp_restart) },
  87         { "diagnose_10", VCPU_STAT(diagnose_10) },
  88         { "diagnose_44", VCPU_STAT(diagnose_44) },
  89         { "diagnose_9c", VCPU_STAT(diagnose_9c) },
  90         { NULL }
  91 };
  92
  93 unsigned long *vfacilities;
  94 static struct gmap_notifier gmap_notifier;
  95
  96 /* test availability of vfacility */
  97 int test_vfacility(unsigned long nr)
  98 {
  99         return __test_facility(nr, (void *) vfacilities);
 100 }
 101
 102 /* Section: not file related */
 103 int kvm_arch_hardware_enable(void *garbage)
 104 {
 105         /* every s390 is virtualization enabled ;-) */
 106         return 0;
 107 }
 108
 109 void kvm_arch_hardware_disable(void *garbage)
 110 {
 111 }
 112
 113 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address);
 114
 115 int kvm_arch_hardware_setup(void)
 116 {
 117         gmap_notifier.notifier_call = kvm_gmap_notifier;
 118         gmap_register_ipte_notifier(&gmap_notifier);
 119         return 0;
 120 }
 121
 122 void kvm_arch_hardware_unsetup(void)
 123 {
 124         gmap_unregister_ipte_notifier(&gmap_notifier);
 125 }
 126
 127 void kvm_arch_check_processor_compat(void *rtn)
 128 {
 129 }
 130
 131 int kvm_arch_init(void *opaque)
 132 {
 133         return 0;
 134 }
 135
 136 void kvm_arch_exit(void)
 137 {
 138 }
 139
 140 /* Section: device related */
 141 long kvm_arch_dev_ioctl(struct file *filp,
 142                         unsigned int ioctl, unsigned long arg)
 143 {
 144         if (ioctl == KVM_S390_ENABLE_SIE)
 145                 return s390_enable_sie();
 146         return -EINVAL;
 147 }
 148
 149 int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
 150 {
 151         int r;
 152
 153         switch (ext) {
 154         case KVM_CAP_S390_PSW:
 155         case KVM_CAP_S390_GMAP:
 156         case KVM_CAP_SYNC_MMU:
 157 #ifdef CONFIG_KVM_S390_UCONTROL
 158         case KVM_CAP_S390_UCONTROL:
 159 #endif
 160         case KVM_CAP_ASYNC_PF:
 161         case KVM_CAP_SYNC_REGS:
 162         case KVM_CAP_ONE_REG:
 163         case KVM_CAP_ENABLE_CAP:
 164         case KVM_CAP_S390_CSS_SUPPORT:
 165         case KVM_CAP_IRQFD:
 166         case KVM_CAP_IOEVENTFD:
 167         case KVM_CAP_DEVICE_CTRL:
 168         case KVM_CAP_ENABLE_CAP_VM:
 169         case KVM_CAP_S390_IRQCHIP:
 170         case KVM_CAP_VM_ATTRIBUTES:
 171         case KVM_CAP_MP_STATE:
 172                 r = 1;
 173                 break;
 174         case KVM_CAP_NR_VCPUS:
 175         case KVM_CAP_MAX_VCPUS:
 176                 r = KVM_MAX_VCPUS;
 177                 break;
 178         case KVM_CAP_NR_MEMSLOTS:
 179                 r = KVM_USER_MEM_SLOTS;
 180                 break;
 181         case KVM_CAP_S390_COW:
 182                 r = MACHINE_HAS_ESOP;
 183                 break;
 184         default:
 185                 r = 0;
 186         }
 187         return r;
 188 }
 189
 190 static void kvm_s390_sync_dirty_log(struct kvm *kvm,
 191                                         struct kvm_memory_slot *memslot)
 192 {
 193         gfn_t cur_gfn, last_gfn;
 194         unsigned long address;
 195         struct gmap *gmap = kvm->arch.gmap;
 196
 197         down_read(&gmap->mm->mmap_sem);
 198         /* Loop over all guest pages */
 199         last_gfn = memslot->base_gfn + memslot->npages;
 200         for (cur_gfn = memslot->base_gfn; cur_gfn <= last_gfn; cur_gfn++) {
 201                 address = gfn_to_hva_memslot(memslot, cur_gfn);
 202
 203                 if (gmap_test_and_clear_dirty(address, gmap))
 204                         mark_page_dirty(kvm, cur_gfn);
 205         }
 206         up_read(&gmap->mm->mmap_sem);
 207 }
 208
 209 /* Section: vm related */
 210 /*
 211  * Get (and clear) the dirty memory log for a memory slot.
 212  */
 213 int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
 214                                struct kvm_dirty_log *log)
 215 {
 216         int r;
 217         unsigned long n;
 218         struct kvm_memory_slot *memslot;
 219         int is_dirty = 0;
 220
 221         mutex_lock(&kvm->slots_lock);
 222
 223         r = -EINVAL;
 224         if (log->slot >= KVM_USER_MEM_SLOTS)
 225                 goto out;
 226
 227         memslot = id_to_memslot(kvm->memslots, log->slot);
 228         r = -ENOENT;
 229         if (!memslot->dirty_bitmap)
 230                 goto out;
 231
 232         kvm_s390_sync_dirty_log(kvm, memslot);
 233         r = kvm_get_dirty_log(kvm, log, &is_dirty);
 234         if (r)
 235                 goto out;
 236
 237         /* Clear the dirty log */
 238         if (is_dirty) {
 239                 n = kvm_dirty_bitmap_bytes(memslot);
 240                 memset(memslot->dirty_bitmap, 0, n);
 241         }
 242         r = 0;
 243 out:
 244         mutex_unlock(&kvm->slots_lock);
 245         return r;
 246 }
 247
 248 static int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap)
 249 {
 250         int r;
 251
 252         if (cap->flags)
 253                 return -EINVAL;
 254
 255         switch (cap->cap) {
 256         case KVM_CAP_S390_IRQCHIP:
 257                 kvm->arch.use_irqchip = 1;
 258                 r = 0;
 259                 break;
 260         default:
 261                 r = -EINVAL;
 262                 break;
 263         }
 264         return r;
 265 }
 266
 267 static int kvm_s390_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
 268 {
 269         int ret;
 270         unsigned int idx;
 271         switch (attr->attr) {
 272         case KVM_S390_VM_MEM_ENABLE_CMMA:
 273                 ret = -EBUSY;
 274                 mutex_lock(&kvm->lock);
 275                 if (atomic_read(&kvm->online_vcpus) == 0) {
 276                         kvm->arch.use_cmma = 1;
 277                         ret = 0;
 278                 }
 279                 mutex_unlock(&kvm->lock);
 280                 break;
 281         case KVM_S390_VM_MEM_CLR_CMMA:
 282                 mutex_lock(&kvm->lock);
 283                 idx = srcu_read_lock(&kvm->srcu);
 284                 page_table_reset_pgste(kvm->arch.gmap->mm, 0, TASK_SIZE, false);
 285                 srcu_read_unlock(&kvm->srcu, idx);
 286                 mutex_unlock(&kvm->lock);
 287                 ret = 0;
 288                 break;
 289         default:
 290                 ret = -ENXIO;
 291                 break;
 292         }
 293         return ret;
 294 }
 295
 296 static int kvm_s390_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr)
 297 {
 298         int ret;
 299
 300         switch (attr->group) {
 301         case KVM_S390_VM_MEM_CTRL:
 302                 ret = kvm_s390_mem_control(kvm, attr);
 303                 break;
 304         default:
 305                 ret = -ENXIO;
 306                 break;
 307         }
 308
 309         return ret;
 310 }
 311
 312 static int kvm_s390_vm_get_attr(struct kvm *kvm, struct kvm_device_attr *attr)
 313 {
 314         return -ENXIO;
 315 }
 316
 317 static int kvm_s390_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr)
 318 {
 319         int ret;
 320
 321         switch (attr->group) {
 322         case KVM_S390_VM_MEM_CTRL:
 323                 switch (attr->attr) {
 324                 case KVM_S390_VM_MEM_ENABLE_CMMA:
 325                 case KVM_S390_VM_MEM_CLR_CMMA:
 326                         ret = 0;
 327                         break;
 328                 default:
 329                         ret = -ENXIO;
 330                         break;
 331                 }
 332                 break;
 333         default:
 334                 ret = -ENXIO;
 335                 break;
 336         }
 337
 338         return ret;
 339 }
 340
 341 long kvm_arch_vm_ioctl(struct file *filp,
 342                        unsigned int ioctl, unsigned long arg)
 343 {
 344         struct kvm *kvm = filp->private_data;
 345         void __user *argp = (void __user *)arg;
 346         struct kvm_device_attr attr;
 347         int r;
 348
 349         switch (ioctl) {
 350         case KVM_S390_INTERRUPT: {
 351                 struct kvm_s390_interrupt s390int;
 352
 353                 r = -EFAULT;
 354                 if (copy_from_user(&s390int, argp, sizeof(s390int)))
 355                         break;
 356                 r = kvm_s390_inject_vm(kvm, &s390int);
 357                 break;
 358         }
 359         case KVM_ENABLE_CAP: {
 360                 struct kvm_enable_cap cap;
 361                 r = -EFAULT;
 362                 if (copy_from_user(&cap, argp, sizeof(cap)))
 363                         break;
 364                 r = kvm_vm_ioctl_enable_cap(kvm, &cap);
 365                 break;
 366         }
 367         case KVM_CREATE_IRQCHIP: {
 368                 struct kvm_irq_routing_entry routing;
 369
 370                 r = -EINVAL;
 371                 if (kvm->arch.use_irqchip) {
 372                         /* Set up dummy routing. */
 373                         memset(&routing, 0, sizeof(routing));
 374                         kvm_set_irq_routing(kvm, &routing, 0, 0);
 375                         r = 0;
 376                 }
 377                 break;
 378         }
 379         case KVM_SET_DEVICE_ATTR: {
 380                 r = -EFAULT;
 381                 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
 382                         break;
 383                 r = kvm_s390_vm_set_attr(kvm, &attr);
 384                 break;
 385         }
 386         case KVM_GET_DEVICE_ATTR: {
 387                 r = -EFAULT;
 388                 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
 389                         break;
 390                 r = kvm_s390_vm_get_attr(kvm, &attr);
 391                 break;
 392         }
 393         case KVM_HAS_DEVICE_ATTR: {
 394                 r = -EFAULT;
 395                 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
 396                         break;
 397                 r = kvm_s390_vm_has_attr(kvm, &attr);
 398                 break;
 399         }
 400         default:
 401                 r = -ENOTTY;
 402         }
 403
 404         return r;
 405 }
 406
 407 int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
 408 {
 409         int rc;
 410         char debug_name[16];
 411         static unsigned long sca_offset;
 412
 413         rc = -EINVAL;
 414 #ifdef CONFIG_KVM_S390_UCONTROL
 415         if (type & ~KVM_VM_S390_UCONTROL)
 416                 goto out_err;
 417         if ((type & KVM_VM_S390_UCONTROL) && (!capable(CAP_SYS_ADMIN)))
 418                 goto out_err;
 419 #else
 420         if (type)
 421                 goto out_err;
 422 #endif
 423
 424         rc = s390_enable_sie();
 425         if (rc)
 426                 goto out_err;
 427
 428         rc = -ENOMEM;
 429
 430         kvm->arch.sca = (struct sca_block *) get_zeroed_page(GFP_KERNEL);
 431         if (!kvm->arch.sca)
 432                 goto out_err;
 433         spin_lock(&kvm_lock);
 434         sca_offset = (sca_offset + 16) & 0x7f0;
 435         kvm->arch.sca = (struct sca_block *) ((char *) kvm->arch.sca + sca_offset);
 436         spin_unlock(&kvm_lock);
 437
 438         sprintf(debug_name, "kvm-%u", current->pid);
 439
 440         kvm->arch.dbf = debug_register(debug_name, 8, 2, 8 * sizeof(long));
 441         if (!kvm->arch.dbf)
 442                 goto out_nodbf;
 443
 444         spin_lock_init(&kvm->arch.float_int.lock);
 445         INIT_LIST_HEAD(&kvm->arch.float_int.list);
 446         init_waitqueue_head(&kvm->arch.ipte_wq);
 447
 448         debug_register_view(kvm->arch.dbf, &debug_sprintf_view);
 449         VM_EVENT(kvm, 3, "%s", "vm created");
 450
 451         if (type & KVM_VM_S390_UCONTROL) {
 452                 kvm->arch.gmap = NULL;
 453         } else {
 454                 kvm->arch.gmap = gmap_alloc(current->mm);
 455                 if (!kvm->arch.gmap)
 456                         goto out_nogmap;
 457                 kvm->arch.gmap->private = kvm;
 458                 kvm->arch.gmap->pfault_enabled = 0;
 459         }
 460
 461         kvm->arch.css_support = 0;
 462         kvm->arch.use_irqchip = 0;
 463
 464         spin_lock_init(&kvm->arch.start_stop_lock);
 465
 466         return 0;
 467 out_nogmap:
 468         debug_unregister(kvm->arch.dbf);
 469 out_nodbf:
 470         free_page((unsigned long)(kvm->arch.sca));
 471 out_err:
 472         return rc;
 473 }
 474
 475 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
 476 {
 477         VCPU_EVENT(vcpu, 3, "%s", "free cpu");
 478         trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id);
 479         kvm_s390_clear_local_irqs(vcpu);
 480         kvm_clear_async_pf_completion_queue(vcpu);
 481         if (!kvm_is_ucontrol(vcpu->kvm)) {
 482                 clear_bit(63 - vcpu->vcpu_id,
 483                           (unsigned long *) &vcpu->kvm->arch.sca->mcn);
 484                 if (vcpu->kvm->arch.sca->cpu[vcpu->vcpu_id].sda ==
 485                     (__u64) vcpu->arch.sie_block)
 486                         vcpu->kvm->arch.sca->cpu[vcpu->vcpu_id].sda = 0;
 487         }
 488         smp_mb();
 489
 490         if (kvm_is_ucontrol(vcpu->kvm))
 491                 gmap_free(vcpu->arch.gmap);
 492
 493         if (kvm_s390_cmma_enabled(vcpu->kvm))
 494                 kvm_s390_vcpu_unsetup_cmma(vcpu);
 495         free_page((unsigned long)(vcpu->arch.sie_block));
 496
 497         kvm_vcpu_uninit(vcpu);
 498         kmem_cache_free(kvm_vcpu_cache, vcpu);
 499 }
 500
 501 static void kvm_free_vcpus(struct kvm *kvm)
 502 {
 503         unsigned int i;
 504         struct kvm_vcpu *vcpu;
 505
 506         kvm_for_each_vcpu(i, vcpu, kvm)
 507                 kvm_arch_vcpu_destroy(vcpu);
 508
 509         mutex_lock(&kvm->lock);
 510         for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
 511                 kvm->vcpus[i] = NULL;
 512
 513         atomic_set(&kvm->online_vcpus, 0);
 514         mutex_unlock(&kvm->lock);
 515 }
 516
 517 void kvm_arch_sync_events(struct kvm *kvm)
 518 {
 519 }
 520
 521 void kvm_arch_destroy_vm(struct kvm *kvm)
 522 {
 523         kvm_free_vcpus(kvm);
 524         free_page((unsigned long)(kvm->arch.sca));
 525         debug_unregister(kvm->arch.dbf);
 526         if (!kvm_is_ucontrol(kvm))
 527                 gmap_free(kvm->arch.gmap);
 528         kvm_s390_destroy_adapters(kvm);
 529         kvm_s390_clear_float_irqs(kvm);
 530 }
 531
 532 /* Section: vcpu related */
 533 int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
 534 {
 535         vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
 536         kvm_clear_async_pf_completion_queue(vcpu);
 537         if (kvm_is_ucontrol(vcpu->kvm)) {
 538                 vcpu->arch.gmap = gmap_alloc(current->mm);
 539                 if (!vcpu->arch.gmap)
 540                         return -ENOMEM;
 541                 vcpu->arch.gmap->private = vcpu->kvm;
 542                 return 0;
 543         }
 544
 545         vcpu->arch.gmap = vcpu->kvm->arch.gmap;
 546         vcpu->run->kvm_valid_regs = KVM_SYNC_PREFIX |
 547                                     KVM_SYNC_GPRS |
 548                                     KVM_SYNC_ACRS |
 549                                     KVM_SYNC_CRS;
 550         return 0;
 551 }
 552
 553 void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu)
 554 {
 555         /* Nothing todo */
 556 }
 557
 558 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
 559 {
 560         save_fp_ctl(&vcpu->arch.host_fpregs.fpc);
 561         save_fp_regs(vcpu->arch.host_fpregs.fprs);
 562         save_access_regs(vcpu->arch.host_acrs);
 563         restore_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
 564         restore_fp_regs(vcpu->arch.guest_fpregs.fprs);
 565         restore_access_regs(vcpu->run->s.regs.acrs);
 566         gmap_enable(vcpu->arch.gmap);
 567         atomic_set_mask(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
 568 }
 569
 570 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
 571 {
 572         atomic_clear_mask(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
 573         gmap_disable(vcpu->arch.gmap);
 574         save_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
 575         save_fp_regs(vcpu->arch.guest_fpregs.fprs);
 576         save_access_regs(vcpu->run->s.regs.acrs);
 577         restore_fp_ctl(&vcpu->arch.host_fpregs.fpc);
 578         restore_fp_regs(vcpu->arch.host_fpregs.fprs);
 579         restore_access_regs(vcpu->arch.host_acrs);
 580 }
 581
 582 static void kvm_s390_vcpu_initial_reset(struct kvm_vcpu *vcpu)
 583 {
 584         /* this equals initial cpu reset in pop, but we don't switch to ESA */
 585         vcpu->arch.sie_block->gpsw.mask = 0UL;
 586         vcpu->arch.sie_block->gpsw.addr = 0UL;
 587         kvm_s390_set_prefix(vcpu, 0);
 588         vcpu->arch.sie_block->cputm     = 0UL;
 589         vcpu->arch.sie_block->ckc       = 0UL;
 590         vcpu->arch.sie_block->todpr     = 0;
 591         memset(vcpu->arch.sie_block->gcr, 0, 16 * sizeof(__u64));
 592         vcpu->arch.sie_block->gcr[0]  = 0xE0UL;
 593         vcpu->arch.sie_block->gcr[14] = 0xC2000000UL;
 594         vcpu->arch.guest_fpregs.fpc = 0;
 595         asm volatile("lfpc %0" : : "Q" (vcpu->arch.guest_fpregs.fpc));
 596         vcpu->arch.sie_block->gbea = 1;
 597         vcpu->arch.sie_block->pp = 0;
 598         vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
 599         kvm_clear_async_pf_completion_queue(vcpu);
 600         if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm))
 601                 kvm_s390_vcpu_stop(vcpu);
 602         kvm_s390_clear_local_irqs(vcpu);
 603 }
 604
 605 int kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
 606 {
 607         return 0;
 608 }
 609
 610 void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu)
 611 {
 612         free_page(vcpu->arch.sie_block->cbrlo);
 613         vcpu->arch.sie_block->cbrlo = 0;
 614 }
 615
 616 int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu *vcpu)
 617 {
 618         vcpu->arch.sie_block->cbrlo = get_zeroed_page(GFP_KERNEL);
 619         if (!vcpu->arch.sie_block->cbrlo)
 620                 return -ENOMEM;
 621
 622         vcpu->arch.sie_block->ecb2 |= 0x80;
 623         vcpu->arch.sie_block->ecb2 &= ~0x08;
 624         return 0;
 625 }
 626
 627 int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
 628 {
 629         int rc = 0;
 630
 631         atomic_set(&vcpu->arch.sie_block->cpuflags, CPUSTAT_ZARCH |
 632                                                     CPUSTAT_SM |
 633                                                     CPUSTAT_STOPPED |
 634                                                     CPUSTAT_GED);
 635         vcpu->arch.sie_block->ecb   = 6;
 636         if (test_vfacility(50) && test_vfacility(73))
 637                 vcpu->arch.sie_block->ecb |= 0x10;
 638
 639         vcpu->arch.sie_block->ecb2  = 8;
 640         vcpu->arch.sie_block->eca   = 0xD1002000U;
 641         if (sclp_has_siif())
 642                 vcpu->arch.sie_block->eca |= 1;
 643         vcpu->arch.sie_block->fac   = (int) (long) vfacilities;
 644         vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE |
 645                                       ICTL_TPROT;
 646
 647         if (kvm_s390_cmma_enabled(vcpu->kvm)) {
 648                 rc = kvm_s390_vcpu_setup_cmma(vcpu);
 649                 if (rc)
 650                         return rc;
 651         }
 652         hrtimer_init(&vcpu->arch.ckc_timer, CLOCK_REALTIME, HRTIMER_MODE_ABS);
 653         vcpu->arch.ckc_timer.function = kvm_s390_idle_wakeup;
 654         get_cpu_id(&vcpu->arch.cpu_id);
 655         vcpu->arch.cpu_id.version = 0xff;
 656         return rc;
 657 }
 658
 659 struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
 660                                       unsigned int id)
 661 {
 662         struct kvm_vcpu *vcpu;
 663         struct sie_page *sie_page;
 664         int rc = -EINVAL;
 665
 666         if (id >= KVM_MAX_VCPUS)
 667                 goto out;
 668
 669         rc = -ENOMEM;
 670
 671         vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
 672         if (!vcpu)
 673                 goto out;
 674
 675         sie_page = (struct sie_page *) get_zeroed_page(GFP_KERNEL);
 676         if (!sie_page)
 677                 goto out_free_cpu;
 678
 679         vcpu->arch.sie_block = &sie_page->sie_block;
 680         vcpu->arch.sie_block->itdba = (unsigned long) &sie_page->itdb;
 681
 682         vcpu->arch.sie_block->icpua = id;
 683         if (!kvm_is_ucontrol(kvm)) {
 684                 if (!kvm->arch.sca) {
 685                         WARN_ON_ONCE(1);
 686                         goto out_free_cpu;
 687                 }
 688                 if (!kvm->arch.sca->cpu[id].sda)
 689                         kvm->arch.sca->cpu[id].sda =
 690                                 (__u64) vcpu->arch.sie_block;
 691                 vcpu->arch.sie_block->scaoh =
 692                         (__u32)(((__u64)kvm->arch.sca) >> 32);
 693                 vcpu->arch.sie_block->scaol = (__u32)(__u64)kvm->arch.sca;
 694                 set_bit(63 - id, (unsigned long *) &kvm->arch.sca->mcn);
 695         }
 696
 697         spin_lock_init(&vcpu->arch.local_int.lock);
 698         INIT_LIST_HEAD(&vcpu->arch.local_int.list);
 699         vcpu->arch.local_int.float_int = &kvm->arch.float_int;
 700         vcpu->arch.local_int.wq = &vcpu->wq;
 701         vcpu->arch.local_int.cpuflags = &vcpu->arch.sie_block->cpuflags;
 702
 703         rc = kvm_vcpu_init(vcpu, kvm, id);
 704         if (rc)
 705                 goto out_free_sie_block;
 706         VM_EVENT(kvm, 3, "create cpu %d at %p, sie block at %p", id, vcpu,
 707                  vcpu->arch.sie_block);
 708         trace_kvm_s390_create_vcpu(id, vcpu, vcpu->arch.sie_block);
 709
 710         return vcpu;
 711 out_free_sie_block:
 712         free_page((unsigned long)(vcpu->arch.sie_block));
 713 out_free_cpu:
 714         kmem_cache_free(kvm_vcpu_cache, vcpu);
 715 out:
 716         return ERR_PTR(rc);
 717 }
 718
 719 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
 720 {
 721         return kvm_cpu_has_interrupt(vcpu);
 722 }
 723
 724 void s390_vcpu_block(struct kvm_vcpu *vcpu)
 725 {
 726         atomic_set_mask(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
 727 }
 728
 729 void s390_vcpu_unblock(struct kvm_vcpu *vcpu)
 730 {
 731         atomic_clear_mask(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
 732 }
 733
 734 /*
 735  * Kick a guest cpu out of SIE and wait until SIE is not running.
 736  * If the CPU is not running (e.g. waiting as idle) the function will
 737  * return immediately. */
 738 void exit_sie(struct kvm_vcpu *vcpu)
 739 {
 740         atomic_set_mask(CPUSTAT_STOP_INT, &vcpu->arch.sie_block->cpuflags);
 741         while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE)
 742                 cpu_relax();
 743 }
 744
 745 /* Kick a guest cpu out of SIE and prevent SIE-reentry */
 746 void exit_sie_sync(struct kvm_vcpu *vcpu)
 747 {
 748         s390_vcpu_block(vcpu);
 749         exit_sie(vcpu);
 750 }
 751
 752 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address)
 753 {
 754         int i;
 755         struct kvm *kvm = gmap->private;
 756         struct kvm_vcpu *vcpu;
 757
 758         kvm_for_each_vcpu(i, vcpu, kvm) {
 759                 /* match against both prefix pages */
 760                 if (kvm_s390_get_prefix(vcpu) == (address & ~0x1000UL)) {
 761                         VCPU_EVENT(vcpu, 2, "gmap notifier for %lx", address);
 762                         kvm_make_request(KVM_REQ_MMU_RELOAD, vcpu);
 763                         exit_sie_sync(vcpu);
 764                 }
 765         }
 766 }
 767
 768 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
 769 {
 770         /* kvm common code refers to this, but never calls it */
 771         BUG();
 772         return 0;
 773 }
 774
 775 static int kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu,
 776                                            struct kvm_one_reg *reg)
 777 {
 778         int r = -EINVAL;
 779
 780         switch (reg->id) {
 781         case KVM_REG_S390_TODPR:
 782                 r = put_user(vcpu->arch.sie_block->todpr,
 783                              (u32 __user *)reg->addr);
 784                 break;
 785         case KVM_REG_S390_EPOCHDIFF:
 786                 r = put_user(vcpu->arch.sie_block->epoch,
 787                              (u64 __user *)reg->addr);
 788                 break;
 789         case KVM_REG_S390_CPU_TIMER:
 790                 r = put_user(vcpu->arch.sie_block->cputm,
 791                              (u64 __user *)reg->addr);
 792                 break;
 793         case KVM_REG_S390_CLOCK_COMP:
 794                 r = put_user(vcpu->arch.sie_block->ckc,
 795                              (u64 __user *)reg->addr);
 796                 break;
 797         case KVM_REG_S390_PFTOKEN:
 798                 r = put_user(vcpu->arch.pfault_token,
 799                              (u64 __user *)reg->addr);
 800                 break;
 801         case KVM_REG_S390_PFCOMPARE:
 802                 r = put_user(vcpu->arch.pfault_compare,
 803                              (u64 __user *)reg->addr);
 804                 break;
 805         case KVM_REG_S390_PFSELECT:
 806                 r = put_user(vcpu->arch.pfault_select,
 807                              (u64 __user *)reg->addr);
 808                 break;
 809         case KVM_REG_S390_PP:
 810                 r = put_user(vcpu->arch.sie_block->pp,
 811                              (u64 __user *)reg->addr);
 812                 break;
 813         case KVM_REG_S390_GBEA:
 814                 r = put_user(vcpu->arch.sie_block->gbea,
 815                              (u64 __user *)reg->addr);
 816                 break;
 817         default:
 818                 break;
 819         }
 820
 821         return r;
 822 }
 823
 824 static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu,
 825                                            struct kvm_one_reg *reg)
 826 {
 827         int r = -EINVAL;
 828
 829         switch (reg->id) {
 830         case KVM_REG_S390_TODPR:
 831                 r = get_user(vcpu->arch.sie_block->todpr,
 832                              (u32 __user *)reg->addr);
 833                 break;
 834         case KVM_REG_S390_EPOCHDIFF:
 835                 r = get_user(vcpu->arch.sie_block->epoch,
 836                              (u64 __user *)reg->addr);
 837                 break;
 838         case KVM_REG_S390_CPU_TIMER:
 839                 r = get_user(vcpu->arch.sie_block->cputm,
 840                              (u64 __user *)reg->addr);
 841                 break;
 842         case KVM_REG_S390_CLOCK_COMP:
 843                 r = get_user(vcpu->arch.sie_block->ckc,
 844                              (u64 __user *)reg->addr);
 845                 break;
 846         case KVM_REG_S390_PFTOKEN:
 847                 r = get_user(vcpu->arch.pfault_token,
 848                              (u64 __user *)reg->addr);
 849                 break;
 850         case KVM_REG_S390_PFCOMPARE:
 851                 r = get_user(vcpu->arch.pfault_compare,
 852                              (u64 __user *)reg->addr);
 853                 break;
 854         case KVM_REG_S390_PFSELECT:
 855                 r = get_user(vcpu->arch.pfault_select,
 856                              (u64 __user *)reg->addr);
 857                 break;
 858         case KVM_REG_S390_PP:
 859                 r = get_user(vcpu->arch.sie_block->pp,
 860                              (u64 __user *)reg->addr);
 861                 break;
 862         case KVM_REG_S390_GBEA:
 863                 r = get_user(vcpu->arch.sie_block->gbea,
 864                              (u64 __user *)reg->addr);
 865                 break;
 866         default:
 867                 break;
 868         }
 869
 870         return r;
 871 }
 872
 873 static int kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu)
 874 {
 875         kvm_s390_vcpu_initial_reset(vcpu);
 876         return 0;
 877 }
 878
 879 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
 880 {
 881         memcpy(&vcpu->run->s.regs.gprs, &regs->gprs, sizeof(regs->gprs));
 882         return 0;
 883 }
 884
 885 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
 886 {
 887         memcpy(&regs->gprs, &vcpu->run->s.regs.gprs, sizeof(regs->gprs));
 888         return 0;
 889 }
 890
 891 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
 892                                   struct kvm_sregs *sregs)
 893 {
 894         memcpy(&vcpu->run->s.regs.acrs, &sregs->acrs, sizeof(sregs->acrs));
 895         memcpy(&vcpu->arch.sie_block->gcr, &sregs->crs, sizeof(sregs->crs));
 896         restore_access_regs(vcpu->run->s.regs.acrs);
 897         return 0;
 898 }
 899
 900 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
 901                                   struct kvm_sregs *sregs)
 902 {
 903         memcpy(&sregs->acrs, &vcpu->run->s.regs.acrs, sizeof(sregs->acrs));
 904         memcpy(&sregs->crs, &vcpu->arch.sie_block->gcr, sizeof(sregs->crs));
 905         return 0;
 906 }
 907
 908 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
 909 {
 910         if (test_fp_ctl(fpu->fpc))
 911                 return -EINVAL;
 912         memcpy(&vcpu->arch.guest_fpregs.fprs, &fpu->fprs, sizeof(fpu->fprs));
 913         vcpu->arch.guest_fpregs.fpc = fpu->fpc;
 914         restore_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
 915         restore_fp_regs(vcpu->arch.guest_fpregs.fprs);
 916         return 0;
 917 }
 918
 919 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
 920 {
 921         memcpy(&fpu->fprs, &vcpu->arch.guest_fpregs.fprs, sizeof(fpu->fprs));
 922         fpu->fpc = vcpu->arch.guest_fpregs.fpc;
 923         return 0;
 924 }
 925
 926 static int kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu *vcpu, psw_t psw)
 927 {
 928         int rc = 0;
 929
 930         if (!is_vcpu_stopped(vcpu))
 931                 rc = -EBUSY;
 932         else {
 933                 vcpu->run->psw_mask = psw.mask;
 934                 vcpu->run->psw_addr = psw.addr;
 935         }
 936         return rc;
 937 }
 938
 939 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
 940                                   struct kvm_translation *tr)
 941 {
 942         return -EINVAL; /* not implemented yet */
 943 }
 944
 945 #define VALID_GUESTDBG_FLAGS (KVM_GUESTDBG_SINGLESTEP | \
 946                               KVM_GUESTDBG_USE_HW_BP | \
 947                               KVM_GUESTDBG_ENABLE)
 948
 949 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
 950                                         struct kvm_guest_debug *dbg)
 951 {
 952         int rc = 0;
 953
 954         vcpu->guest_debug = 0;
 955         kvm_s390_clear_bp_data(vcpu);
 956
 957         if (dbg->control & ~VALID_GUESTDBG_FLAGS)
 958                 return -EINVAL;
 959
 960         if (dbg->control & KVM_GUESTDBG_ENABLE) {
 961                 vcpu->guest_debug = dbg->control;
 962                 /* enforce guest PER */
 963                 atomic_set_mask(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
 964
 965                 if (dbg->control & KVM_GUESTDBG_USE_HW_BP)
 966                         rc = kvm_s390_import_bp_data(vcpu, dbg);
 967         } else {
 968                 atomic_clear_mask(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
 969                 vcpu->arch.guestdbg.last_bp = 0;
 970         }
 971
 972         if (rc) {
 973                 vcpu->guest_debug = 0;
 974                 kvm_s390_clear_bp_data(vcpu);
 975                 atomic_clear_mask(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
 976         }
 977
 978         return rc;
 979 }
 980
 981 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
 982                                     struct kvm_mp_state *mp_state)
 983 {
 984         /* CHECK_STOP and LOAD are not supported yet */
 985         return is_vcpu_stopped(vcpu) ? KVM_MP_STATE_STOPPED :
 986                                        KVM_MP_STATE_OPERATING;
 987 }
 988
 989 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
 990                                     struct kvm_mp_state *mp_state)
 991 {
 992         int rc = 0;
 993
 994         /* user space knows about this interface - let it control the state */
 995         vcpu->kvm->arch.user_cpu_state_ctrl = 1;
 996
 997         switch (mp_state->mp_state) {
 998         case KVM_MP_STATE_STOPPED:
 999                 kvm_s390_vcpu_stop(vcpu);
1000                 break;
1001         case KVM_MP_STATE_OPERATING:
1002                 kvm_s390_vcpu_start(vcpu);
1003                 break;
1004         case KVM_MP_STATE_LOAD:
1005         case KVM_MP_STATE_CHECK_STOP:
1006                 /* fall through - CHECK_STOP and LOAD are not supported yet */
1007         default:
1008                 rc = -ENXIO;
1009         }
1010
1011         return rc;
1012 }
1013
1014 bool kvm_s390_cmma_enabled(struct kvm *kvm)
1015 {
1016         if (!MACHINE_IS_LPAR)
1017                 return false;
1018         /* only enable for z10 and later */
1019         if (!MACHINE_HAS_EDAT1)
1020                 return false;
1021         if (!kvm->arch.use_cmma)
1022                 return false;
1023         return true;
1024 }
1025
1026 static bool ibs_enabled(struct kvm_vcpu *vcpu)
1027 {
1028         return atomic_read(&vcpu->arch.sie_block->cpuflags) & CPUSTAT_IBS;
1029 }
1030
1031 static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu)
1032 {
1033 retry:
1034         s390_vcpu_unblock(vcpu);
1035         /*
1036          * We use MMU_RELOAD just to re-arm the ipte notifier for the
1037          * guest prefix page. gmap_ipte_notify will wait on the ptl lock.
1038          * This ensures that the ipte instruction for this request has
1039          * already finished. We might race against a second unmapper that
1040          * wants to set the blocking bit. Lets just retry the request loop.
1041          */
1042         if (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu)) {
1043                 int rc;
1044                 rc = gmap_ipte_notify(vcpu->arch.gmap,
1045                                       kvm_s390_get_prefix(vcpu),
1046                                       PAGE_SIZE * 2);
1047                 if (rc)
1048                         return rc;
1049                 goto retry;
1050         }
1051
1052         if (kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu)) {
1053                 if (!ibs_enabled(vcpu)) {
1054                         trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 1);
1055                         atomic_set_mask(CPUSTAT_IBS,
1056                                         &vcpu->arch.sie_block->cpuflags);
1057                 }
1058                 goto retry;
1059         }
1060
1061         if (kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu)) {
1062                 if (ibs_enabled(vcpu)) {
1063                         trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 0);
1064                         atomic_clear_mask(CPUSTAT_IBS,
1065                                           &vcpu->arch.sie_block->cpuflags);
1066                 }
1067                 goto retry;
1068         }
1069
1070         /* nothing to do, just clear the request */
1071         clear_bit(KVM_REQ_UNHALT, &vcpu->requests);
1072
1073         return 0;
1074 }
1075
1076 /**
1077  * kvm_arch_fault_in_page - fault-in guest page if necessary
1078  * @vcpu: The corresponding virtual cpu
1079  * @gpa: Guest physical address
1080  * @writable: Whether the page should be writable or not
1081  *
1082  * Make sure that a guest page has been faulted-in on the host.
1083  *
1084  * Return: Zero on success, negative error code otherwise.
1085  */
1086 long kvm_arch_fault_in_page(struct kvm_vcpu *vcpu, gpa_t gpa, int writable)
1087 {
1088         struct mm_struct *mm = current->mm;
1089         hva_t hva;
1090         long rc;
1091
1092         hva = gmap_fault(gpa, vcpu->arch.gmap);
1093         if (IS_ERR_VALUE(hva))
1094                 return (long)hva;
1095         down_read(&mm->mmap_sem);
1096         rc = get_user_pages(current, mm, hva, 1, writable, 0, NULL, NULL);
1097         up_read(&mm->mmap_sem);
1098
1099         return rc < 0 ? rc : 0;
1100 }
1101
1102 static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token,
1103                                       unsigned long token)
1104 {
1105         struct kvm_s390_interrupt inti;
1106         inti.parm64 = token;
1107
1108         if (start_token) {
1109                 inti.type = KVM_S390_INT_PFAULT_INIT;
1110                 WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu, &inti));
1111         } else {
1112                 inti.type = KVM_S390_INT_PFAULT_DONE;
1113                 WARN_ON_ONCE(kvm_s390_inject_vm(vcpu->kvm, &inti));
1114         }
1115 }
1116
1117 void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
1118                                      struct kvm_async_pf *work)
1119 {
1120         trace_kvm_s390_pfault_init(vcpu, work->arch.pfault_token);
1121         __kvm_inject_pfault_token(vcpu, true, work->arch.pfault_token);
1122 }
1123
1124 void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
1125                                  struct kvm_async_pf *work)
1126 {
1127         trace_kvm_s390_pfault_done(vcpu, work->arch.pfault_token);
1128         __kvm_inject_pfault_token(vcpu, false, work->arch.pfault_token);
1129 }
1130
1131 void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu,
1132                                struct kvm_async_pf *work)
1133 {
1134         /* s390 will always inject the page directly */
1135 }
1136
1137 bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu)
1138 {
1139         /*
1140          * s390 will always inject the page directly,
1141          * but we still want check_async_completion to cleanup
1142          */
1143         return true;
1144 }
1145
1146 static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu)
1147 {
1148         hva_t hva;
1149         struct kvm_arch_async_pf arch;
1150         int rc;
1151
1152         if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
1153                 return 0;
1154         if ((vcpu->arch.sie_block->gpsw.mask & vcpu->arch.pfault_select) !=
1155             vcpu->arch.pfault_compare)
1156                 return 0;
1157         if (psw_extint_disabled(vcpu))
1158                 return 0;
1159         if (kvm_cpu_has_interrupt(vcpu))
1160                 return 0;
1161         if (!(vcpu->arch.sie_block->gcr[0] & 0x200ul))
1162                 return 0;
1163         if (!vcpu->arch.gmap->pfault_enabled)
1164                 return 0;
1165
1166         hva = gfn_to_hva(vcpu->kvm, gpa_to_gfn(current->thread.gmap_addr));
1167         hva += current->thread.gmap_addr & ~PAGE_MASK;
1168         if (read_guest_real(vcpu, vcpu->arch.pfault_token, &arch.pfault_token, 8))
1169                 return 0;
1170
1171         rc = kvm_setup_async_pf(vcpu, current->thread.gmap_addr, hva, &arch);
1172         return rc;
1173 }
1174
1175 static int vcpu_pre_run(struct kvm_vcpu *vcpu)
1176 {
1177         int rc, cpuflags;
1178
1179         /*
1180          * On s390 notifications for arriving pages will be delivered directly
1181          * to the guest but the house keeping for completed pfaults is
1182          * handled outside the worker.
1183          */
1184         kvm_check_async_pf_completion(vcpu);
1185
1186         memcpy(&vcpu->arch.sie_block->gg14, &vcpu->run->s.regs.gprs[14], 16);
1187
1188         if (need_resched())
1189                 schedule();
1190
1191         if (test_cpu_flag(CIF_MCCK_PENDING))
1192                 s390_handle_mcck();
1193
1194         if (!kvm_is_ucontrol(vcpu->kvm))
1195                 kvm_s390_deliver_pending_interrupts(vcpu);
1196
1197         rc = kvm_s390_handle_requests(vcpu);
1198         if (rc)
1199                 return rc;
1200
1201         if (guestdbg_enabled(vcpu)) {
1202                 kvm_s390_backup_guest_per_regs(vcpu);
1203                 kvm_s390_patch_guest_per_regs(vcpu);
1204         }
1205
1206         vcpu->arch.sie_block->icptcode = 0;
1207         cpuflags = atomic_read(&vcpu->arch.sie_block->cpuflags);
1208         VCPU_EVENT(vcpu, 6, "entering sie flags %x", cpuflags);
1209         trace_kvm_s390_sie_enter(vcpu, cpuflags);
1210
1211         return 0;
1212 }
1213
1214 static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason)
1215 {
1216         int rc = -1;
1217
1218         VCPU_EVENT(vcpu, 6, "exit sie icptcode %d",
1219                    vcpu->arch.sie_block->icptcode);
1220         trace_kvm_s390_sie_exit(vcpu, vcpu->arch.sie_block->icptcode);
1221
1222         if (guestdbg_enabled(vcpu))
1223                 kvm_s390_restore_guest_per_regs(vcpu);
1224
1225         if (exit_reason >= 0) {
1226                 rc = 0;
1227         } else if (kvm_is_ucontrol(vcpu->kvm)) {
1228                 vcpu->run->exit_reason = KVM_EXIT_S390_UCONTROL;
1229                 vcpu->run->s390_ucontrol.trans_exc_code =
1230                                                 current->thread.gmap_addr;
1231                 vcpu->run->s390_ucontrol.pgm_code = 0x10;
1232                 rc = -EREMOTE;
1233
1234         } else if (current->thread.gmap_pfault) {
1235                 trace_kvm_s390_major_guest_pfault(vcpu);
1236                 current->thread.gmap_pfault = 0;
1237                 if (kvm_arch_setup_async_pf(vcpu)) {
1238                         rc = 0;
1239                 } else {
1240                         gpa_t gpa = current->thread.gmap_addr;
1241                         rc = kvm_arch_fault_in_page(vcpu, gpa, 1);
1242                 }
1243         }
1244
1245         if (rc == -1) {
1246                 VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction");
1247                 trace_kvm_s390_sie_fault(vcpu);
1248                 rc = kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);
1249         }
1250
1251         memcpy(&vcpu->run->s.regs.gprs[14], &vcpu->arch.sie_block->gg14, 16);
1252
1253         if (rc == 0) {
1254                 if (kvm_is_ucontrol(vcpu->kvm))
1255                         /* Don't exit for host interrupts. */
1256                         rc = vcpu->arch.sie_block->icptcode ? -EOPNOTSUPP : 0;
1257                 else
1258                         rc = kvm_handle_sie_intercept(vcpu);
1259         }
1260
1261         return rc;
1262 }
1263
1264 static int __vcpu_run(struct kvm_vcpu *vcpu)
1265 {
1266         int rc, exit_reason;
1267
1268         /*
1269          * We try to hold kvm->srcu during most of vcpu_run (except when run-
1270          * ning the guest), so that memslots (and other stuff) are protected
1271          */
1272         vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
1273
1274         do {
1275                 rc = vcpu_pre_run(vcpu);
1276                 if (rc)
1277                         break;
1278
1279                 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
1280                 /*
1281                  * As PF_VCPU will be used in fault handler, between
1282                  * guest_enter and guest_exit should be no uaccess.
1283                  */
1284                 preempt_disable();
1285                 kvm_guest_enter();
1286                 preempt_enable();
1287                 exit_reason = sie64a(vcpu->arch.sie_block,
1288                                      vcpu->run->s.regs.gprs);
1289                 kvm_guest_exit();
1290                 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
1291
1292                 rc = vcpu_post_run(vcpu, exit_reason);
1293         } while (!signal_pending(current) && !guestdbg_exit_pending(vcpu) && !rc);
1294
1295         srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
1296         return rc;
1297 }
1298
1299 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
1300 {
1301         int rc;
1302         sigset_t sigsaved;
1303
1304         if (guestdbg_exit_pending(vcpu)) {
1305                 kvm_s390_prepare_debug_exit(vcpu);
1306                 return 0;
1307         }
1308
1309         if (vcpu->sigset_active)
1310                 sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);
1311
1312         if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm)) {
1313                 kvm_s390_vcpu_start(vcpu);
1314         } else if (is_vcpu_stopped(vcpu)) {
1315                 pr_err_ratelimited("kvm-s390: can't run stopped vcpu %d\n",
1316                                    vcpu->vcpu_id);
1317                 return -EINVAL;
1318         }
1319
1320         switch (kvm_run->exit_reason) {
1321         case KVM_EXIT_S390_SIEIC:
1322         case KVM_EXIT_UNKNOWN:
1323         case KVM_EXIT_INTR:
1324         case KVM_EXIT_S390_RESET:
1325         case KVM_EXIT_S390_UCONTROL:
1326         case KVM_EXIT_S390_TSCH:
1327         case KVM_EXIT_DEBUG:
1328                 break;
1329         default:
1330                 BUG();
1331         }
1332
1333         vcpu->arch.sie_block->gpsw.mask = kvm_run->psw_mask;
1334         vcpu->arch.sie_block->gpsw.addr = kvm_run->psw_addr;
1335         if (kvm_run->kvm_dirty_regs & KVM_SYNC_PREFIX) {
1336                 kvm_run->kvm_dirty_regs &= ~KVM_SYNC_PREFIX;
1337                 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix);
1338         }
1339         if (kvm_run->kvm_dirty_regs & KVM_SYNC_CRS) {
1340                 kvm_run->kvm_dirty_regs &= ~KVM_SYNC_CRS;
1341                 memcpy(&vcpu->arch.sie_block->gcr, &kvm_run->s.regs.crs, 128);
1342                 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix);
1343         }
1344
1345         might_fault();
1346         rc = __vcpu_run(vcpu);
1347
1348         if (signal_pending(current) && !rc) {
1349                 kvm_run->exit_reason = KVM_EXIT_INTR;
1350                 rc = -EINTR;
1351         }
1352
1353         if (guestdbg_exit_pending(vcpu) && !rc)  {
1354                 kvm_s390_prepare_debug_exit(vcpu);
1355                 rc = 0;
1356         }
1357
1358         if (rc == -EOPNOTSUPP) {
1359                 /* intercept cannot be handled in-kernel, prepare kvm-run */
1360                 kvm_run->exit_reason         = KVM_EXIT_S390_SIEIC;
1361                 kvm_run->s390_sieic.icptcode = vcpu->arch.sie_block->icptcode;
1362                 kvm_run->s390_sieic.ipa      = vcpu->arch.sie_block->ipa;
1363                 kvm_run->s390_sieic.ipb      = vcpu->arch.sie_block->ipb;
1364                 rc = 0;
1365         }
1366
1367         if (rc == -EREMOTE) {
1368                 /* intercept was handled, but userspace support is needed
1369                  * kvm_run has been prepared by the handler */
1370                 rc = 0;
1371         }
1372
1373         kvm_run->psw_mask     = vcpu->arch.sie_block->gpsw.mask;
1374         kvm_run->psw_addr     = vcpu->arch.sie_block->gpsw.addr;
1375         kvm_run->s.regs.prefix = kvm_s390_get_prefix(vcpu);
1376         memcpy(&kvm_run->s.regs.crs, &vcpu->arch.sie_block->gcr, 128);
1377
1378         if (vcpu->sigset_active)
1379                 sigprocmask(SIG_SETMASK, &sigsaved, NULL);
1380
1381         vcpu->stat.exit_userspace++;
1382         return rc;
1383 }
1384
1385 /*
1386  * store status at address
1387  * we use have two special cases:
1388  * KVM_S390_STORE_STATUS_NOADDR: -> 0x1200 on 64 bit
1389  * KVM_S390_STORE_STATUS_PREFIXED: -> prefix
1390  */
1391 int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long gpa)
1392 {
1393         unsigned char archmode = 1;
1394         unsigned int px;
1395         u64 clkcomp;
1396         int rc;
1397
1398         if (gpa == KVM_S390_STORE_STATUS_NOADDR) {
1399                 if (write_guest_abs(vcpu, 163, &archmode, 1))
1400                         return -EFAULT;
1401                 gpa = SAVE_AREA_BASE;
1402         } else if (gpa == KVM_S390_STORE_STATUS_PREFIXED) {
1403                 if (write_guest_real(vcpu, 163, &archmode, 1))
1404                         return -EFAULT;
1405                 gpa = kvm_s390_real_to_abs(vcpu, SAVE_AREA_BASE);
1406         }
1407         rc = write_guest_abs(vcpu, gpa + offsetof(struct save_area, fp_regs),
1408                              vcpu->arch.guest_fpregs.fprs, 128);
1409         rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, gp_regs),
1410                               vcpu->run->s.regs.gprs, 128);
1411         rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, psw),
1412                               &vcpu->arch.sie_block->gpsw, 16);
1413         px = kvm_s390_get_prefix(vcpu);
1414         rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, pref_reg),
1415                               &px, 4);
1416         rc |= write_guest_abs(vcpu,
1417                               gpa + offsetof(struct save_area, fp_ctrl_reg),
1418                               &vcpu->arch.guest_fpregs.fpc, 4);
1419         rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, tod_reg),
1420                               &vcpu->arch.sie_block->todpr, 4);
1421         rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, timer),
1422                               &vcpu->arch.sie_block->cputm, 8);
1423         clkcomp = vcpu->arch.sie_block->ckc >> 8;
1424         rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, clk_cmp),
1425                               &clkcomp, 8);
1426         rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, acc_regs),
1427                               &vcpu->run->s.regs.acrs, 64);
1428         rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, ctrl_regs),
1429                               &vcpu->arch.sie_block->gcr, 128);
1430         return rc ? -EFAULT : 0;
1431 }
1432
1433 int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr)
1434 {
1435         /*
1436          * The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy
1437          * copying in vcpu load/put. Lets update our copies before we save
1438          * it into the save area
1439          */
1440         save_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
1441         save_fp_regs(vcpu->arch.guest_fpregs.fprs);
1442         save_access_regs(vcpu->run->s.regs.acrs);
1443
1444         return kvm_s390_store_status_unloaded(vcpu, addr);
1445 }
1446
1447 static void __disable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
1448 {
1449         kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu);
1450         kvm_make_request(KVM_REQ_DISABLE_IBS, vcpu);
1451         exit_sie_sync(vcpu);
1452 }
1453
1454 static void __disable_ibs_on_all_vcpus(struct kvm *kvm)
1455 {
1456         unsigned int i;
1457         struct kvm_vcpu *vcpu;
1458
1459         kvm_for_each_vcpu(i, vcpu, kvm) {
1460                 __disable_ibs_on_vcpu(vcpu);
1461         }
1462 }
1463
1464 static void __enable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
1465 {
1466         kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu);
1467         kvm_make_request(KVM_REQ_ENABLE_IBS, vcpu);
1468         exit_sie_sync(vcpu);
1469 }
1470
1471 void kvm_s390_vcpu_start(struct kvm_vcpu *vcpu)
1472 {
1473         int i, online_vcpus, started_vcpus = 0;
1474
1475         if (!is_vcpu_stopped(vcpu))
1476                 return;
1477
1478         trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 1);
1479         /* Only one cpu at a time may enter/leave the STOPPED state. */
1480         spin_lock(&vcpu->kvm->arch.start_stop_lock);
1481         online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
1482
1483         for (i = 0; i < online_vcpus; i++) {
1484                 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i]))
1485                         started_vcpus++;
1486         }
1487
1488         if (started_vcpus == 0) {
1489                 /* we're the only active VCPU -> speed it up */
1490                 __enable_ibs_on_vcpu(vcpu);
1491         } else if (started_vcpus == 1) {
1492                 /*
1493                  * As we are starting a second VCPU, we have to disable
1494                  * the IBS facility on all VCPUs to remove potentially
1495                  * oustanding ENABLE requests.
1496                  */
1497                 __disable_ibs_on_all_vcpus(vcpu->kvm);
1498         }
1499
1500         atomic_clear_mask(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
1501         /*
1502          * Another VCPU might have used IBS while we were offline.
1503          * Let's play safe and flush the VCPU at startup.
1504          */
1505         vcpu->arch.sie_block->ihcpu  = 0xffff;
1506         spin_unlock(&vcpu->kvm->arch.start_stop_lock);
1507         return;
1508 }
1509
1510 void kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu)
1511 {
1512         int i, online_vcpus, started_vcpus = 0;
1513         struct kvm_vcpu *started_vcpu = NULL;
1514
1515         if (is_vcpu_stopped(vcpu))
1516                 return;
1517
1518         trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 0);
1519         /* Only one cpu at a time may enter/leave the STOPPED state. */
1520         spin_lock(&vcpu->kvm->arch.start_stop_lock);
1521         online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
1522
1523         /* Need to lock access to action_bits to avoid a SIGP race condition */
1524         spin_lock(&vcpu->arch.local_int.lock);
1525         atomic_set_mask(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
1526
1527         /* SIGP STOP and SIGP STOP AND STORE STATUS has been fully processed */
1528         vcpu->arch.local_int.action_bits &=
1529                                  ~(ACTION_STOP_ON_STOP | ACTION_STORE_ON_STOP);
1530         spin_unlock(&vcpu->arch.local_int.lock);
1531
1532         __disable_ibs_on_vcpu(vcpu);
1533
1534         for (i = 0; i < online_vcpus; i++) {
1535                 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i])) {
1536                         started_vcpus++;
1537                         started_vcpu = vcpu->kvm->vcpus[i];
1538                 }
1539         }
1540
1541         if (started_vcpus == 1) {
1542                 /*
1543                  * As we only have one VCPU left, we want to enable the
1544                  * IBS facility for that VCPU to speed it up.
1545                  */
1546                 __enable_ibs_on_vcpu(started_vcpu);
1547         }
1548
1549         spin_unlock(&vcpu->kvm->arch.start_stop_lock);
1550         return;
1551 }
1552
1553 static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
1554                                      struct kvm_enable_cap *cap)
1555 {
1556         int r;
1557
1558         if (cap->flags)
1559                 return -EINVAL;
1560
1561         switch (cap->cap) {
1562         case KVM_CAP_S390_CSS_SUPPORT:
1563                 if (!vcpu->kvm->arch.css_support) {
1564                         vcpu->kvm->arch.css_support = 1;
1565                         trace_kvm_s390_enable_css(vcpu->kvm);
1566                 }
1567                 r = 0;
1568                 break;
1569         default:
1570                 r = -EINVAL;
1571                 break;
1572         }
1573         return r;
1574 }
1575
1576 long kvm_arch_vcpu_ioctl(struct file *filp,
1577                          unsigned int ioctl, unsigned long arg)
1578 {
1579         struct kvm_vcpu *vcpu = filp->private_data;
1580         void __user *argp = (void __user *)arg;
1581         int idx;
1582         long r;
1583
1584         switch (ioctl) {
1585         case KVM_S390_INTERRUPT: {
1586                 struct kvm_s390_interrupt s390int;
1587
1588                 r = -EFAULT;
1589                 if (copy_from_user(&s390int, argp, sizeof(s390int)))
1590                         break;
1591                 r = kvm_s390_inject_vcpu(vcpu, &s390int);
1592                 break;
1593         }
1594         case KVM_S390_STORE_STATUS:
1595                 idx = srcu_read_lock(&vcpu->kvm->srcu);
1596                 r = kvm_s390_vcpu_store_status(vcpu, arg);
1597                 srcu_read_unlock(&vcpu->kvm->srcu, idx);
1598                 break;
1599         case KVM_S390_SET_INITIAL_PSW: {
1600                 psw_t psw;
1601
1602                 r = -EFAULT;
1603                 if (copy_from_user(&psw, argp, sizeof(psw)))
1604                         break;
1605                 r = kvm_arch_vcpu_ioctl_set_initial_psw(vcpu, psw);
1606                 break;
1607         }
1608         case KVM_S390_INITIAL_RESET:
1609                 r = kvm_arch_vcpu_ioctl_initial_reset(vcpu);
1610                 break;
1611         case KVM_SET_ONE_REG:
1612         case KVM_GET_ONE_REG: {
1613                 struct kvm_one_reg reg;
1614                 r = -EFAULT;
1615                 if (copy_from_user(&reg, argp, sizeof(reg)))
1616                         break;
1617                 if (ioctl == KVM_SET_ONE_REG)
1618                         r = kvm_arch_vcpu_ioctl_set_one_reg(vcpu, &reg);
1619                 else
1620                         r = kvm_arch_vcpu_ioctl_get_one_reg(vcpu, &reg);
1621                 break;
1622         }
1623 #ifdef CONFIG_KVM_S390_UCONTROL
1624         case KVM_S390_UCAS_MAP: {
1625                 struct kvm_s390_ucas_mapping ucasmap;
1626
1627                 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
1628                         r = -EFAULT;
1629                         break;
1630                 }
1631
1632                 if (!kvm_is_ucontrol(vcpu->kvm)) {
1633                         r = -EINVAL;
1634                         break;
1635                 }
1636
1637                 r = gmap_map_segment(vcpu->arch.gmap, ucasmap.user_addr,
1638                                      ucasmap.vcpu_addr, ucasmap.length);
1639                 break;
1640         }
1641         case KVM_S390_UCAS_UNMAP: {
1642                 struct kvm_s390_ucas_mapping ucasmap;
1643
1644                 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
1645                         r = -EFAULT;
1646                         break;
1647                 }
1648
1649                 if (!kvm_is_ucontrol(vcpu->kvm)) {
1650                         r = -EINVAL;
1651                         break;
1652                 }
1653
1654                 r = gmap_unmap_segment(vcpu->arch.gmap, ucasmap.vcpu_addr,
1655                         ucasmap.length);
1656                 break;
1657         }
1658 #endif
1659         case KVM_S390_VCPU_FAULT: {
1660                 r = gmap_fault(arg, vcpu->arch.gmap);
1661                 if (!IS_ERR_VALUE(r))
1662                         r = 0;
1663                 break;
1664         }
1665         case KVM_ENABLE_CAP:
1666         {
1667                 struct kvm_enable_cap cap;
1668                 r = -EFAULT;
1669                 if (copy_from_user(&cap, argp, sizeof(cap)))
1670                         break;
1671                 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
1672                 break;
1673         }
1674         default:
1675                 r = -ENOTTY;
1676         }
1677         return r;
1678 }
1679
1680 int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
1681 {
1682 #ifdef CONFIG_KVM_S390_UCONTROL
1683         if ((vmf->pgoff == KVM_S390_SIE_PAGE_OFFSET)
1684                  && (kvm_is_ucontrol(vcpu->kvm))) {
1685                 vmf->page = virt_to_page(vcpu->arch.sie_block);
1686                 get_page(vmf->page);
1687                 return 0;
1688         }
1689 #endif
1690         return VM_FAULT_SIGBUS;
1691 }
1692
1693 void kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *free,
1694                            struct kvm_memory_slot *dont)
1695 {
1696 }
1697
1698 int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot,
1699                             unsigned long npages)
1700 {
1701         return 0;
1702 }
1703
1704 void kvm_arch_memslots_updated(struct kvm *kvm)
1705 {
1706 }
1707
1708 /* Section: memory related */
1709 int kvm_arch_prepare_memory_region(struct kvm *kvm,
1710                                    struct kvm_memory_slot *memslot,
1711                                    struct kvm_userspace_memory_region *mem,
1712                                    enum kvm_mr_change change)
1713 {
1714         /* A few sanity checks. We can have memory slots which have to be
1715            located/ended at a segment boundary (1MB). The memory in userland is
1716            ok to be fragmented into various different vmas. It is okay to mmap()
1717            and munmap() stuff in this slot after doing this call at any time */
1718
1719         if (mem->userspace_addr & 0xffffful)
1720                 return -EINVAL;
1721
1722         if (mem->memory_size & 0xffffful)
1723                 return -EINVAL;
1724
1725         return 0;
1726 }
1727
1728 void kvm_arch_commit_memory_region(struct kvm *kvm,
1729                                 struct kvm_userspace_memory_region *mem,
1730                                 const struct kvm_memory_slot *old,
1731                                 enum kvm_mr_change change)
1732 {
1733         int rc;
1734
1735         /* If the basics of the memslot do not change, we do not want
1736          * to update the gmap. Every update causes several unnecessary
1737          * segment translation exceptions. This is usually handled just
1738          * fine by the normal fault handler + gmap, but it will also
1739          * cause faults on the prefix page of running guest CPUs.
1740          */
1741         if (old->userspace_addr == mem->userspace_addr &&
1742             old->base_gfn * PAGE_SIZE == mem->guest_phys_addr &&
1743             old->npages * PAGE_SIZE == mem->memory_size)
1744                 return;
1745
1746         rc = gmap_map_segment(kvm->arch.gmap, mem->userspace_addr,
1747                 mem->guest_phys_addr, mem->memory_size);
1748         if (rc)
1749                 printk(KERN_WARNING "kvm-s390: failed to commit memory region\n");
1750         return;
1751 }
1752
1753 void kvm_arch_flush_shadow_all(struct kvm *kvm)
1754 {
1755 }
1756
1757 void kvm_arch_flush_shadow_memslot(struct kvm *kvm,
1758                                    struct kvm_memory_slot *slot)
1759 {
1760 }
1761
1762 static int __init kvm_s390_init(void)
1763 {
1764         int ret;
1765         ret = kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
1766         if (ret)
1767                 return ret;
1768
1769         /*
1770          * guests can ask for up to 255+1 double words, we need a full page
1771          * to hold the maximum amount of facilities. On the other hand, we
1772          * only set facilities that are known to work in KVM.
1773          */
1774         vfacilities = (unsigned long *) get_zeroed_page(GFP_KERNEL|GFP_DMA);
1775         if (!vfacilities) {
1776                 kvm_exit();
1777                 return -ENOMEM;
1778         }
1779         memcpy(vfacilities, S390_lowcore.stfle_fac_list, 16);
1780         vfacilities[0] &= 0xff82fff3f4fc2000UL;
1781         vfacilities[1] &= 0x005c000000000000UL;
1782         return 0;
1783 }
1784
1785 static void __exit kvm_s390_exit(void)
1786 {
1787         free_page((unsigned long) vfacilities);
1788         kvm_exit();
1789 }
1790
1791 module_init(kvm_s390_init);
1792 module_exit(kvm_s390_exit);
1793
1794 /*
1795  * Enable autoloading of the kvm module.
1796  * Note that we add the module alias here instead of virt/kvm/kvm_main.c
1797  * since x86 takes a different approach.
1798  */
1799 #include <linux/miscdevice.h>
1800 MODULE_ALIAS_MISCDEV(KVM_MINOR);
1801 MODULE_ALIAS("devname:kvm");