// SPDX-License-Identifier: GPL-2.0 /* * Copyright (c) 2017-2019, IBM Corporation. */ #define pr_fmt(fmt) "xive-kvm: " fmt #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "book3s_xive.h" static u8 xive_vm_esb_load(struct xive_irq_data *xd, u32 offset) { u64 val; /* * The KVM XIVE native device does not use the XIVE_ESB_SET_PQ_10 * load operation, so there is no need to enforce load-after-store * ordering. */ val = in_be64(xd->eoi_mmio + offset); return (u8)val; } static void kvmppc_xive_native_cleanup_queue(struct kvm_vcpu *vcpu, int prio) { struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu; struct xive_q *q = &xc->queues[prio]; xive_native_disable_queue(xc->vp_id, q, prio); if (q->qpage) { put_page(virt_to_page(q->qpage)); q->qpage = NULL; } } static int kvmppc_xive_native_configure_queue(u32 vp_id, struct xive_q *q, u8 prio, __be32 *qpage, u32 order, bool can_escalate) { int rc; __be32 *qpage_prev = q->qpage; rc = xive_native_configure_queue(vp_id, q, prio, qpage, order, can_escalate); if (rc) return rc; if (qpage_prev) put_page(virt_to_page(qpage_prev)); return rc; } void kvmppc_xive_native_cleanup_vcpu(struct kvm_vcpu *vcpu) { struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu; int i; if (!kvmppc_xive_enabled(vcpu)) return; if (!xc) return; pr_devel("native_cleanup_vcpu(cpu=%d)\n", xc->server_num); /* Ensure no interrupt is still routed to that VP */ xc->valid = false; kvmppc_xive_disable_vcpu_interrupts(vcpu); /* Free escalations */ for (i = 0; i < KVMPPC_XIVE_Q_COUNT; i++) { /* Free the escalation irq */ if (xc->esc_virq[i]) { if (kvmppc_xive_has_single_escalation(xc->xive)) xive_cleanup_single_escalation(vcpu, xc, xc->esc_virq[i]); free_irq(xc->esc_virq[i], vcpu); irq_dispose_mapping(xc->esc_virq[i]); kfree(xc->esc_virq_names[i]); xc->esc_virq[i] = 0; } } /* Disable the VP */ xive_native_disable_vp(xc->vp_id); /* Clear the cam word so guest entry won't try to push context */ vcpu->arch.xive_cam_word = 0; /* Free the queues */ for (i = 0; i < KVMPPC_XIVE_Q_COUNT; i++) { kvmppc_xive_native_cleanup_queue(vcpu, i); } /* Free the VP */ kfree(xc); /* Cleanup the vcpu */ vcpu->arch.irq_type = KVMPPC_IRQ_DEFAULT; vcpu->arch.xive_vcpu = NULL; } int kvmppc_xive_native_connect_vcpu(struct kvm_device *dev, struct kvm_vcpu *vcpu, u32 server_num) { struct kvmppc_xive *xive = dev->private; struct kvmppc_xive_vcpu *xc = NULL; int rc; u32 vp_id; pr_devel("native_connect_vcpu(server=%d)\n", server_num); if (dev->ops != &kvm_xive_native_ops) { pr_devel("Wrong ops !\n"); return -EPERM; } if (xive->kvm != vcpu->kvm) return -EPERM; if (vcpu->arch.irq_type != KVMPPC_IRQ_DEFAULT) return -EBUSY; mutex_lock(&xive->lock); rc = kvmppc_xive_compute_vp_id(xive, server_num, &vp_id); if (rc) goto bail; xc = kzalloc(sizeof(*xc), GFP_KERNEL); if (!xc) { rc = -ENOMEM; goto bail; } vcpu->arch.xive_vcpu = xc; xc->xive = xive; xc->vcpu = vcpu; xc->server_num = server_num; xc->vp_id = vp_id; xc->valid = true; vcpu->arch.irq_type = KVMPPC_IRQ_XIVE; rc = xive_native_get_vp_info(xc->vp_id, &xc->vp_cam, &xc->vp_chip_id); if (rc) { pr_err("Failed to get VP info from OPAL: %d\n", rc); goto bail; } if (!kvmppc_xive_check_save_restore(vcpu)) { pr_err("inconsistent save-restore setup for VCPU %d\n", server_num); rc = -EIO; goto bail; } /* * Enable the VP first as the single escalation mode will * affect escalation interrupts numbering */ rc = xive_native_enable_vp(xc->vp_id, kvmppc_xive_has_single_escalation(xive)); if (rc) { pr_err("Failed to enable VP in OPAL: %d\n", rc); goto bail; } /* Configure VCPU fields for use by assembly push/pull */ vcpu->arch.xive_saved_state.w01 = cpu_to_be64(0xff000000); vcpu->arch.xive_cam_word = cpu_to_be32(xc->vp_cam | TM_QW1W2_VO); /* TODO: reset all queues to a clean state ? */ bail: mutex_unlock(&xive->lock); if (rc) kvmppc_xive_native_cleanup_vcpu(vcpu); return rc; } /* * Device passthrough support */ static int kvmppc_xive_native_reset_mapped(struct kvm *kvm, unsigned long irq) { struct kvmppc_xive *xive = kvm->arch.xive; pgoff_t esb_pgoff = KVM_XIVE_ESB_PAGE_OFFSET + irq * 2; if (irq >= KVMPPC_XIVE_NR_IRQS) return -EINVAL; /* * Clear the ESB pages of the IRQ number being mapped (or * unmapped) into the guest and let the the VM fault handler * repopulate with the appropriate ESB pages (device or IC) */ pr_debug("clearing esb pages for girq 0x%lx\n", irq); mutex_lock(&xive->mapping_lock); if (xive->mapping) unmap_mapping_range(xive->mapping, esb_pgoff << PAGE_SHIFT, 2ull << PAGE_SHIFT, 1); mutex_unlock(&xive->mapping_lock); return 0; } static struct kvmppc_xive_ops kvmppc_xive_native_ops = { .reset_mapped = kvmppc_xive_native_reset_mapped, }; static vm_fault_t xive_native_esb_fault(struct vm_fault *vmf) { struct vm_area_struct *vma = vmf->vma; struct kvm_device *dev = vma->vm_file->private_data; struct kvmppc_xive *xive = dev->private; struct kvmppc_xive_src_block *sb; struct kvmppc_xive_irq_state *state; struct xive_irq_data *xd; u32 hw_num; u16 src; u64 page; unsigned long irq; u64 page_offset; /* * Linux/KVM uses a two pages ESB setting, one for trigger and * one for EOI */ page_offset = vmf->pgoff - vma->vm_pgoff; irq = page_offset / 2; sb = kvmppc_xive_find_source(xive, irq, &src); if (!sb) { pr_devel("%s: source %lx not found !\n", __func__, irq); return VM_FAULT_SIGBUS; } state = &sb->irq_state[src]; /* Some sanity checking */ if (!state->valid) { pr_devel("%s: source %lx invalid !\n", __func__, irq); return VM_FAULT_SIGBUS; } kvmppc_xive_select_irq(state, &hw_num, &xd); arch_spin_lock(&sb->lock); /* * first/even page is for trigger * second/odd page is for EOI and management. */ page = page_offset % 2 ? xd->eoi_page : xd->trig_page; arch_spin_unlock(&sb->lock); if (WARN_ON(!page)) { pr_err("%s: accessing invalid ESB page for source %lx !\n", __func__, irq); return VM_FAULT_SIGBUS; } vmf_insert_pfn(vma, vmf->address, page >> PAGE_SHIFT); return VM_FAULT_NOPAGE; } static const struct vm_operations_struct xive_native_esb_vmops = { .fault = xive_native_esb_fault, }; static vm_fault_t xive_native_tima_fault(struct vm_fault *vmf) { struct vm_area_struct *vma = vmf->vma; switch (vmf->pgoff - vma->vm_pgoff) { case 0: /* HW - forbid access */ case 1: /* HV - forbid access */ return VM_FAULT_SIGBUS; case 2: /* OS */ vmf_insert_pfn(vma, vmf->address, xive_tima_os >> PAGE_SHIFT); return VM_FAULT_NOPAGE; case 3: /* USER - TODO */ default: return VM_FAULT_SIGBUS; } } static const struct vm_operations_struct xive_native_tima_vmops = { .fault = xive_native_tima_fault, }; static int kvmppc_xive_native_mmap(struct kvm_device *dev, struct vm_area_struct *vma) { struct kvmppc_xive *xive = dev->private; /* We only allow mappings at fixed offset for now */ if (vma->vm_pgoff == KVM_XIVE_TIMA_PAGE_OFFSET) { if (vma_pages(vma) > 4) return -EINVAL; vma->vm_ops = &xive_native_tima_vmops; } else if (vma->vm_pgoff == KVM_XIVE_ESB_PAGE_OFFSET) { if (vma_pages(vma) > KVMPPC_XIVE_NR_IRQS * 2) return -EINVAL; vma->vm_ops = &xive_native_esb_vmops; } else { return -EINVAL; } vma->vm_flags |= VM_IO | VM_PFNMAP; vma->vm_page_prot = pgprot_noncached_wc(vma->vm_page_prot); /* * Grab the KVM device file address_space to be able to clear * the ESB pages mapping when a device is passed-through into * the guest. */ xive->mapping = vma->vm_file->f_mapping; return 0; } static int kvmppc_xive_native_set_source(struct kvmppc_xive *xive, long irq, u64 addr) { struct kvmppc_xive_src_block *sb; struct kvmppc_xive_irq_state *state; u64 __user *ubufp = (u64 __user *) addr; u64 val; u16 idx; int rc; pr_devel("%s irq=0x%lx\n", __func__, irq); if (irq < KVMPPC_XIVE_FIRST_IRQ || irq >= KVMPPC_XIVE_NR_IRQS) return -E2BIG; sb = kvmppc_xive_find_source(xive, irq, &idx); if (!sb) { pr_debug("No source, creating source block...\n"); sb = kvmppc_xive_create_src_block(xive, irq); if (!sb) { pr_err("Failed to create block...\n"); return -ENOMEM; } } state = &sb->irq_state[idx]; if (get_user(val, ubufp)) { pr_err("fault getting user info !\n"); return -EFAULT; } arch_spin_lock(&sb->lock); /* * If the source doesn't already have an IPI, allocate * one and get the corresponding data */ if (!state->ipi_number) { state->ipi_number = xive_native_alloc_irq(); if (state->ipi_number == 0) { pr_err("Failed to allocate IRQ !\n"); rc = -ENXIO; goto unlock; } xive_native_populate_irq_data(state->ipi_number, &state->ipi_data); pr_debug("%s allocated hw_irq=0x%x for irq=0x%lx\n", __func__, state->ipi_number, irq); } /* Restore LSI state */ if (val & KVM_XIVE_LEVEL_SENSITIVE) { state->lsi = true; if (val & KVM_XIVE_LEVEL_ASSERTED) state->asserted = true; pr_devel(" LSI ! Asserted=%d\n", state->asserted); } /* Mask IRQ to start with */ state->act_server = 0; state->act_priority = MASKED; xive_vm_esb_load(&state->ipi_data, XIVE_ESB_SET_PQ_01); xive_native_configure_irq(state->ipi_number, 0, MASKED, 0); /* Increment the number of valid sources and mark this one valid */ if (!state->valid) xive->src_count++; state->valid = true; rc = 0; unlock: arch_spin_unlock(&sb->lock); return rc; } static int kvmppc_xive_native_update_source_config(struct kvmppc_xive *xive, struct kvmppc_xive_src_block *sb, struct kvmppc_xive_irq_state *state, u32 server, u8 priority, bool masked, u32 eisn) { struct kvm *kvm = xive->kvm; u32 hw_num; int rc = 0; arch_spin_lock(&sb->lock); if (state->act_server == server && state->act_priority == priority && state->eisn == eisn) goto unlock; pr_devel("new_act_prio=%d new_act_server=%d mask=%d act_server=%d act_prio=%d\n", priority, server, masked, state->act_server, state->act_priority); kvmppc_xive_select_irq(state, &hw_num, NULL); if (priority != MASKED && !masked) { rc = kvmppc_xive_select_target(kvm, &server, priority); if (rc) goto unlock; state->act_priority = priority; state->act_server = server; state->eisn = eisn; rc = xive_native_configure_irq(hw_num, kvmppc_xive_vp(xive, server), priority, eisn); } else { state->act_priority = MASKED; state->act_server = 0; state->eisn = 0; rc = xive_native_configure_irq(hw_num, 0, MASKED, 0); } unlock: arch_spin_unlock(&sb->lock); return rc; } static int kvmppc_xive_native_set_source_config(struct kvmppc_xive *xive, long irq, u64 addr) { struct kvmppc_xive_src_block *sb; struct kvmppc_xive_irq_state *state; u64 __user *ubufp = (u64 __user *) addr; u16 src; u64 kvm_cfg; u32 server; u8 priority; bool masked; u32 eisn; sb = kvmppc_xive_find_source(xive, irq, &src); if (!sb) return -ENOENT; state = &sb->irq_state[src]; if (!state->valid) return -EINVAL; if (get_user(kvm_cfg, ubufp)) return -EFAULT; pr_devel("%s irq=0x%lx cfg=%016llx\n", __func__, irq, kvm_cfg); priority = (kvm_cfg & KVM_XIVE_SOURCE_PRIORITY_MASK) >> KVM_XIVE_SOURCE_PRIORITY_SHIFT; server = (kvm_cfg & KVM_XIVE_SOURCE_SERVER_MASK) >> KVM_XIVE_SOURCE_SERVER_SHIFT; masked = (kvm_cfg & KVM_XIVE_SOURCE_MASKED_MASK) >> KVM_XIVE_SOURCE_MASKED_SHIFT; eisn = (kvm_cfg & KVM_XIVE_SOURCE_EISN_MASK) >> KVM_XIVE_SOURCE_EISN_SHIFT; if (priority != xive_prio_from_guest(priority)) { pr_err("invalid priority for queue %d for VCPU %d\n", priority, server); return -EINVAL; } return kvmppc_xive_native_update_source_config(xive, sb, state, server, priority, masked, eisn); } static int kvmppc_xive_native_sync_source(struct kvmppc_xive *xive, long irq, u64 addr) { struct kvmppc_xive_src_block *sb; struct kvmppc_xive_irq_state *state; struct xive_irq_data *xd; u32 hw_num; u16 src; int rc = 0; pr_devel("%s irq=0x%lx", __func__, irq); sb = kvmppc_xive_find_source(xive, irq, &src); if (!sb) return -ENOENT; state = &sb->irq_state[src]; rc = -EINVAL; arch_spin_lock(&sb->lock); if (state->valid) { kvmppc_xive_select_irq(state, &hw_num, &xd); xive_native_sync_source(hw_num); rc = 0; } arch_spin_unlock(&sb->lock); return rc; } static int xive_native_validate_queue_size(u32 qshift) { /* * We only support 64K pages for the moment. This is also * advertised in the DT property "ibm,xive-eq-sizes" */ switch (qshift) { case 0: /* EQ reset */ case 16: return 0; case 12: case 21: case 24: default: return -EINVAL; } } static int kvmppc_xive_native_set_queue_config(struct kvmppc_xive *xive, long eq_idx, u64 addr) { struct kvm *kvm = xive->kvm; struct kvm_vcpu *vcpu; struct kvmppc_xive_vcpu *xc; void __user *ubufp = (void __user *) addr; u32 server; u8 priority; struct kvm_ppc_xive_eq kvm_eq; int rc; __be32 *qaddr = 0; struct page *page; struct xive_q *q; gfn_t gfn; unsigned long page_size; int srcu_idx; /* * Demangle priority/server tuple from the EQ identifier */ priority = (eq_idx & KVM_XIVE_EQ_PRIORITY_MASK) >> KVM_XIVE_EQ_PRIORITY_SHIFT; server = (eq_idx & KVM_XIVE_EQ_SERVER_MASK) >> KVM_XIVE_EQ_SERVER_SHIFT; if (copy_from_user(&kvm_eq, ubufp, sizeof(kvm_eq))) return -EFAULT; vcpu = kvmppc_xive_find_server(kvm, server); if (!vcpu) { pr_err("Can't find server %d\n", server); return -ENOENT; } xc = vcpu->arch.xive_vcpu; if (priority != xive_prio_from_guest(priority)) { pr_err("Trying to restore invalid queue %d for VCPU %d\n", priority, server); return -EINVAL; } q = &xc->queues[priority]; pr_devel("%s VCPU %d priority %d fl:%x shift:%d addr:%llx g:%d idx:%d\n", __func__, server, priority, kvm_eq.flags, kvm_eq.qshift, kvm_eq.qaddr, kvm_eq.qtoggle, kvm_eq.qindex); /* reset queue and disable queueing */ if (!kvm_eq.qshift) { q->guest_qaddr = 0; q->guest_qshift = 0; rc = kvmppc_xive_native_configure_queue(xc->vp_id, q, priority, NULL, 0, true); if (rc) { pr_err("Failed to reset queue %d for VCPU %d: %d\n", priority, xc->server_num, rc); return rc; } return 0; } /* * sPAPR specifies a "Unconditional Notify (n) flag" for the * H_INT_SET_QUEUE_CONFIG hcall which forces notification * without using the coalescing mechanisms provided by the * XIVE END ESBs. This is required on KVM as notification * using the END ESBs is not supported. */ if (kvm_eq.flags != KVM_XIVE_EQ_ALWAYS_NOTIFY) { pr_err("invalid flags %d\n", kvm_eq.flags); return -EINVAL; } rc = xive_native_validate_queue_size(kvm_eq.qshift); if (rc) { pr_err("invalid queue size %d\n", kvm_eq.qshift); return rc; } if (kvm_eq.qaddr & ((1ull << kvm_eq.qshift) - 1)) { pr_err("queue page is not aligned %llx/%llx\n", kvm_eq.qaddr, 1ull << kvm_eq.qshift); return -EINVAL; } srcu_idx = srcu_read_lock(&kvm->srcu); gfn = gpa_to_gfn(kvm_eq.qaddr); page_size = kvm_host_page_size(vcpu, gfn); if (1ull << kvm_eq.qshift > page_size) { srcu_read_unlock(&kvm->srcu, srcu_idx); pr_warn("Incompatible host page size %lx!\n", page_size); return -EINVAL; } page = gfn_to_page(kvm, gfn); if (is_error_page(page)) { srcu_read_unlock(&kvm->srcu, srcu_idx); pr_err("Couldn't get queue page %llx!\n", kvm_eq.qaddr); return -EINVAL; } qaddr = page_to_virt(page) + (kvm_eq.qaddr & ~PAGE_MASK); srcu_read_unlock(&kvm->srcu, srcu_idx); /* * Backup the queue page guest address to the mark EQ page * dirty for migration. */ q->guest_qaddr = kvm_eq.qaddr; q->guest_qshift = kvm_eq.qshift; /* * Unconditional Notification is forced by default at the * OPAL level because the use of END ESBs is not supported by * Linux. */ rc = kvmppc_xive_native_configure_queue(xc->vp_id, q, priority, (__be32 *) qaddr, kvm_eq.qshift, true); if (rc) { pr_err("Failed to configure queue %d for VCPU %d: %d\n", priority, xc->server_num, rc); put_page(page); return rc; } /* * Only restore the queue state when needed. When doing the * H_INT_SET_SOURCE_CONFIG hcall, it should not. */ if (kvm_eq.qtoggle != 1 || kvm_eq.qindex != 0) { rc = xive_native_set_queue_state(xc->vp_id, priority, kvm_eq.qtoggle, kvm_eq.qindex); if (rc) goto error; } rc = kvmppc_xive_attach_escalation(vcpu, priority, kvmppc_xive_has_single_escalation(xive)); error: if (rc) kvmppc_xive_native_cleanup_queue(vcpu, priority); return rc; } static int kvmppc_xive_native_get_queue_config(struct kvmppc_xive *xive, long eq_idx, u64 addr) { struct kvm *kvm = xive->kvm; struct kvm_vcpu *vcpu; struct kvmppc_xive_vcpu *xc; struct xive_q *q; void __user *ubufp = (u64 __user *) addr; u32 server; u8 priority; struct kvm_ppc_xive_eq kvm_eq; u64 qaddr; u64 qshift; u64 qeoi_page; u32 escalate_irq; u64 qflags; int rc; /* * Demangle priority/server tuple from the EQ identifier */ priority = (eq_idx & KVM_XIVE_EQ_PRIORITY_MASK) >> KVM_XIVE_EQ_PRIORITY_SHIFT; server = (eq_idx & KVM_XIVE_EQ_SERVER_MASK) >> KVM_XIVE_EQ_SERVER_SHIFT; vcpu = kvmppc_xive_find_server(kvm, server); if (!vcpu) { pr_err("Can't find server %d\n", server); return -ENOENT; } xc = vcpu->arch.xive_vcpu; if (priority != xive_prio_from_guest(priority)) { pr_err("invalid priority for queue %d for VCPU %d\n", priority, server); return -EINVAL; } q = &xc->queues[priority]; memset(&kvm_eq, 0, sizeof(kvm_eq)); if (!q->qpage) return 0; rc = xive_native_get_queue_info(xc->vp_id, priority, &qaddr, &qshift, &qeoi_page, &escalate_irq, &qflags); if (rc) return rc; kvm_eq.flags = 0; if (qflags & OPAL_XIVE_EQ_ALWAYS_NOTIFY) kvm_eq.flags |= KVM_XIVE_EQ_ALWAYS_NOTIFY; kvm_eq.qshift = q->guest_qshift; kvm_eq.qaddr = q->guest_qaddr; rc = xive_native_get_queue_state(xc->vp_id, priority, &kvm_eq.qtoggle, &kvm_eq.qindex); if (rc) return rc; pr_devel("%s VCPU %d priority %d fl:%x shift:%d addr:%llx g:%d idx:%d\n", __func__, server, priority, kvm_eq.flags, kvm_eq.qshift, kvm_eq.qaddr, kvm_eq.qtoggle, kvm_eq.qindex); if (copy_to_user(ubufp, &kvm_eq, sizeof(kvm_eq))) return -EFAULT; return 0; } static void kvmppc_xive_reset_sources(struct kvmppc_xive_src_block *sb) { int i; for (i = 0; i < KVMPPC_XICS_IRQ_PER_ICS; i++) { struct kvmppc_xive_irq_state *state = &sb->irq_state[i]; if (!state->valid) continue; if (state->act_priority == MASKED) continue; state->eisn = 0; state->act_server = 0; state->act_priority = MASKED; xive_vm_esb_load(&state->ipi_data, XIVE_ESB_SET_PQ_01); xive_native_configure_irq(state->ipi_number, 0, MASKED, 0); if (state->pt_number) { xive_vm_esb_load(state->pt_data, XIVE_ESB_SET_PQ_01); xive_native_configure_irq(state->pt_number, 0, MASKED, 0); } } } static int kvmppc_xive_reset(struct kvmppc_xive *xive) { struct kvm *kvm = xive->kvm; struct kvm_vcpu *vcpu; unsigned long i; pr_devel("%s\n", __func__); mutex_lock(&xive->lock); kvm_for_each_vcpu(i, vcpu, kvm) { struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu; unsigned int prio; if (!xc) continue; kvmppc_xive_disable_vcpu_interrupts(vcpu); for (prio = 0; prio < KVMPPC_XIVE_Q_COUNT; prio++) { /* Single escalation, no queue 7 */ if (prio == 7 && kvmppc_xive_has_single_escalation(xive)) break; if (xc->esc_virq[prio]) { free_irq(xc->esc_virq[prio], vcpu); irq_dispose_mapping(xc->esc_virq[prio]); kfree(xc->esc_virq_names[prio]); xc->esc_virq[prio] = 0; } kvmppc_xive_native_cleanup_queue(vcpu, prio); } } for (i = 0; i <= xive->max_sbid; i++) { struct kvmppc_xive_src_block *sb = xive->src_blocks[i]; if (sb) { arch_spin_lock(&sb->lock); kvmppc_xive_reset_sources(sb); arch_spin_unlock(&sb->lock); } } mutex_unlock(&xive->lock); return 0; } static void kvmppc_xive_native_sync_sources(struct kvmppc_xive_src_block *sb) { int j; for (j = 0; j < KVMPPC_XICS_IRQ_PER_ICS; j++) { struct kvmppc_xive_irq_state *state = &sb->irq_state[j]; struct xive_irq_data *xd; u32 hw_num; if (!state->valid) continue; /* * The struct kvmppc_xive_irq_state reflects the state * of the EAS configuration and not the state of the * source. The source is masked setting the PQ bits to * '-Q', which is what is being done before calling * the KVM_DEV_XIVE_EQ_SYNC control. * * If a source EAS is configured, OPAL syncs the XIVE * IC of the source and the XIVE IC of the previous * target if any. * * So it should be fine ignoring MASKED sources as * they have been synced already. */ if (state->act_priority == MASKED) continue; kvmppc_xive_select_irq(state, &hw_num, &xd); xive_native_sync_source(hw_num); xive_native_sync_queue(hw_num); } } static int kvmppc_xive_native_vcpu_eq_sync(struct kvm_vcpu *vcpu) { struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu; unsigned int prio; int srcu_idx; if (!xc) return -ENOENT; for (prio = 0; prio < KVMPPC_XIVE_Q_COUNT; prio++) { struct xive_q *q = &xc->queues[prio]; if (!q->qpage) continue; /* Mark EQ page dirty for migration */ srcu_idx = srcu_read_lock(&vcpu->kvm->srcu); mark_page_dirty(vcpu->kvm, gpa_to_gfn(q->guest_qaddr)); srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx); } return 0; } static int kvmppc_xive_native_eq_sync(struct kvmppc_xive *xive) { struct kvm *kvm = xive->kvm; struct kvm_vcpu *vcpu; unsigned long i; pr_devel("%s\n", __func__); mutex_lock(&xive->lock); for (i = 0; i <= xive->max_sbid; i++) { struct kvmppc_xive_src_block *sb = xive->src_blocks[i]; if (sb) { arch_spin_lock(&sb->lock); kvmppc_xive_native_sync_sources(sb); arch_spin_unlock(&sb->lock); } } kvm_for_each_vcpu(i, vcpu, kvm) { kvmppc_xive_native_vcpu_eq_sync(vcpu); } mutex_unlock(&xive->lock); return 0; } static int kvmppc_xive_native_set_attr(struct kvm_device *dev, struct kvm_device_attr *attr) { struct kvmppc_xive *xive = dev->private; switch (attr->group) { case KVM_DEV_XIVE_GRP_CTRL: switch (attr->attr) { case KVM_DEV_XIVE_RESET: return kvmppc_xive_reset(xive); case KVM_DEV_XIVE_EQ_SYNC: return kvmppc_xive_native_eq_sync(xive); case KVM_DEV_XIVE_NR_SERVERS: return kvmppc_xive_set_nr_servers(xive, attr->addr); } break; case KVM_DEV_XIVE_GRP_SOURCE: return kvmppc_xive_native_set_source(xive, attr->attr, attr->addr); case KVM_DEV_XIVE_GRP_SOURCE_CONFIG: return kvmppc_xive_native_set_source_config(xive, attr->attr, attr->addr); case KVM_DEV_XIVE_GRP_EQ_CONFIG: return kvmppc_xive_native_set_queue_config(xive, attr->attr, attr->addr); case KVM_DEV_XIVE_GRP_SOURCE_SYNC: return kvmppc_xive_native_sync_source(xive, attr->attr, attr->addr); } return -ENXIO; } static int kvmppc_xive_native_get_attr(struct kvm_device *dev, struct kvm_device_attr *attr) { struct kvmppc_xive *xive = dev->private; switch (attr->group) { case KVM_DEV_XIVE_GRP_EQ_CONFIG: return kvmppc_xive_native_get_queue_config(xive, attr->attr, attr->addr); } return -ENXIO; } static int kvmppc_xive_native_has_attr(struct kvm_device *dev, struct kvm_device_attr *attr) { switch (attr->group) { case KVM_DEV_XIVE_GRP_CTRL: switch (attr->attr) { case KVM_DEV_XIVE_RESET: case KVM_DEV_XIVE_EQ_SYNC: case KVM_DEV_XIVE_NR_SERVERS: return 0; } break; case KVM_DEV_XIVE_GRP_SOURCE: case KVM_DEV_XIVE_GRP_SOURCE_CONFIG: case KVM_DEV_XIVE_GRP_SOURCE_SYNC: if (attr->attr >= KVMPPC_XIVE_FIRST_IRQ && attr->attr < KVMPPC_XIVE_NR_IRQS) return 0; break; case KVM_DEV_XIVE_GRP_EQ_CONFIG: return 0; } return -ENXIO; } /* * Called when device fd is closed. kvm->lock is held. */ static void kvmppc_xive_native_release(struct kvm_device *dev) { struct kvmppc_xive *xive = dev->private; struct kvm *kvm = xive->kvm; struct kvm_vcpu *vcpu; unsigned long i; pr_devel("Releasing xive native device\n"); /* * Clear the KVM device file address_space which is used to * unmap the ESB pages when a device is passed-through. */ mutex_lock(&xive->mapping_lock); xive->mapping = NULL; mutex_unlock(&xive->mapping_lock); /* * Since this is the device release function, we know that * userspace does not have any open fd or mmap referring to * the device. Therefore there can not be any of the * device attribute set/get, mmap, or page fault functions * being executed concurrently, and similarly, the * connect_vcpu and set/clr_mapped functions also cannot * be being executed. */ debugfs_remove(xive->dentry); /* * We should clean up the vCPU interrupt presenters first. */ kvm_for_each_vcpu(i, vcpu, kvm) { /* * Take vcpu->mutex to ensure that no one_reg get/set ioctl * (i.e. kvmppc_xive_native_[gs]et_vp) can be being done. * Holding the vcpu->mutex also means that the vcpu cannot * be executing the KVM_RUN ioctl, and therefore it cannot * be executing the XIVE push or pull code or accessing * the XIVE MMIO regions. */ mutex_lock(&vcpu->mutex); kvmppc_xive_native_cleanup_vcpu(vcpu); mutex_unlock(&vcpu->mutex); } /* * Now that we have cleared vcpu->arch.xive_vcpu, vcpu->arch.irq_type * and vcpu->arch.xive_esc_[vr]addr on each vcpu, we are safe * against xive code getting called during vcpu execution or * set/get one_reg operations. */ kvm->arch.xive = NULL; for (i = 0; i <= xive->max_sbid; i++) { if (xive->src_blocks[i]) kvmppc_xive_free_sources(xive->src_blocks[i]); kfree(xive->src_blocks[i]); xive->src_blocks[i] = NULL; } if (xive->vp_base != XIVE_INVALID_VP) xive_native_free_vp_block(xive->vp_base); /* * A reference of the kvmppc_xive pointer is now kept under * the xive_devices struct of the machine for reuse. It is * freed when the VM is destroyed for now until we fix all the * execution paths. */ kfree(dev); } /* * Create a XIVE device. kvm->lock is held. */ static int kvmppc_xive_native_create(struct kvm_device *dev, u32 type) { struct kvmppc_xive *xive; struct kvm *kvm = dev->kvm; pr_devel("Creating xive native device\n"); if (kvm->arch.xive) return -EEXIST; xive = kvmppc_xive_get_device(kvm, type); if (!xive) return -ENOMEM; dev->private = xive; xive->dev = dev; xive->kvm = kvm; mutex_init(&xive->mapping_lock); mutex_init(&xive->lock); /* VP allocation is delayed to the first call to connect_vcpu */ xive->vp_base = XIVE_INVALID_VP; /* KVM_MAX_VCPUS limits the number of VMs to roughly 64 per sockets * on a POWER9 system. */ xive->nr_servers = KVM_MAX_VCPUS; if (xive_native_has_single_escalation()) xive->flags |= KVMPPC_XIVE_FLAG_SINGLE_ESCALATION; if (xive_native_has_save_restore()) xive->flags |= KVMPPC_XIVE_FLAG_SAVE_RESTORE; xive->ops = &kvmppc_xive_native_ops; kvm->arch.xive = xive; return 0; } /* * Interrupt Pending Buffer (IPB) offset */ #define TM_IPB_SHIFT 40 #define TM_IPB_MASK (((u64) 0xFF) << TM_IPB_SHIFT) int kvmppc_xive_native_get_vp(struct kvm_vcpu *vcpu, union kvmppc_one_reg *val) { struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu; u64 opal_state; int rc; if (!kvmppc_xive_enabled(vcpu)) return -EPERM; if (!xc) return -ENOENT; /* Thread context registers. We only care about IPB and CPPR */ val->xive_timaval[0] = vcpu->arch.xive_saved_state.w01; /* Get the VP state from OPAL */ rc = xive_native_get_vp_state(xc->vp_id, &opal_state); if (rc) return rc; /* * Capture the backup of IPB register in the NVT structure and * merge it in our KVM VP state. */ val->xive_timaval[0] |= cpu_to_be64(opal_state & TM_IPB_MASK); pr_devel("%s NSR=%02x CPPR=%02x IBP=%02x PIPR=%02x w01=%016llx w2=%08x opal=%016llx\n", __func__, vcpu->arch.xive_saved_state.nsr, vcpu->arch.xive_saved_state.cppr, vcpu->arch.xive_saved_state.ipb, vcpu->arch.xive_saved_state.pipr, vcpu->arch.xive_saved_state.w01, (u32) vcpu->arch.xive_cam_word, opal_state); return 0; } int kvmppc_xive_native_set_vp(struct kvm_vcpu *vcpu, union kvmppc_one_reg *val) { struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu; struct kvmppc_xive *xive = vcpu->kvm->arch.xive; pr_devel("%s w01=%016llx vp=%016llx\n", __func__, val->xive_timaval[0], val->xive_timaval[1]); if (!kvmppc_xive_enabled(vcpu)) return -EPERM; if (!xc || !xive) return -ENOENT; /* We can't update the state of a "pushed" VCPU */ if (WARN_ON(vcpu->arch.xive_pushed)) return -EBUSY; /* * Restore the thread context registers. IPB and CPPR should * be the only ones that matter. */ vcpu->arch.xive_saved_state.w01 = val->xive_timaval[0]; /* * There is no need to restore the XIVE internal state (IPB * stored in the NVT) as the IPB register was merged in KVM VP * state when captured. */ return 0; } bool kvmppc_xive_native_supported(void) { return xive_native_has_queue_state_support(); } static int xive_native_debug_show(struct seq_file *m, void *private) { struct kvmppc_xive *xive = m->private; struct kvm *kvm = xive->kvm; struct kvm_vcpu *vcpu; unsigned long i; if (!kvm) return 0; seq_puts(m, "=========\nVCPU state\n=========\n"); kvm_for_each_vcpu(i, vcpu, kvm) { struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu; if (!xc) continue; seq_printf(m, "VCPU %d: VP=%#x/%02x\n" " NSR=%02x CPPR=%02x IBP=%02x PIPR=%02x w01=%016llx w2=%08x\n", xc->server_num, xc->vp_id, xc->vp_chip_id, vcpu->arch.xive_saved_state.nsr, vcpu->arch.xive_saved_state.cppr, vcpu->arch.xive_saved_state.ipb, vcpu->arch.xive_saved_state.pipr, be64_to_cpu(vcpu->arch.xive_saved_state.w01), be32_to_cpu(vcpu->arch.xive_cam_word)); kvmppc_xive_debug_show_queues(m, vcpu); } seq_puts(m, "=========\nSources\n=========\n"); for (i = 0; i <= xive->max_sbid; i++) { struct kvmppc_xive_src_block *sb = xive->src_blocks[i]; if (sb) { arch_spin_lock(&sb->lock); kvmppc_xive_debug_show_sources(m, sb); arch_spin_unlock(&sb->lock); } } return 0; } DEFINE_SHOW_ATTRIBUTE(xive_native_debug); static void xive_native_debugfs_init(struct kvmppc_xive *xive) { xive->dentry = debugfs_create_file("xive", 0444, xive->kvm->debugfs_dentry, xive, &xive_native_debug_fops); pr_debug("%s: created\n", __func__); } static void kvmppc_xive_native_init(struct kvm_device *dev) { struct kvmppc_xive *xive = (struct kvmppc_xive *)dev->private; /* Register some debug interfaces */ xive_native_debugfs_init(xive); } struct kvm_device_ops kvm_xive_native_ops = { .name = "kvm-xive-native", .create = kvmppc_xive_native_create, .init = kvmppc_xive_native_init, .release = kvmppc_xive_native_release, .set_attr = kvmppc_xive_native_set_attr, .get_attr = kvmppc_xive_native_get_attr, .has_attr = kvmppc_xive_native_has_attr, .mmap = kvmppc_xive_native_mmap, };