On Fri, 2008-10-31 at 14:44 -0500, Hollis Blanchard wrote: > > * Yu's idea about TLB handling was great, and I saw about a 20% > performance improvement on a couple small workloads on 440. Very > happy about that one. :) This patch, which I'll call the "direct TLB" patch, improved performance for me, but I didn't test it hard enough. Christian saw a 20% *regression* with it, because although it does slightly reduce exit handling time across the board, it also dramatically increases the number of TLB misses. I don't think he's posted his data here yet though. I guess this is because we don't make any effort to reload the TLB when re-entering the guest, so it's far more likely to fault. However, I was actually just thinking that we may be able to take a hybrid approach: keep the shadow TLB, but only save/restore it on vcpu_get() and vcpu_put(). That way, we wouldn't pay the per-exit cost, but we'd also have a more populated TLB present when we enter the guest. I may try that as soon as I get out of virtio page size hell. Anyways, the patch was posted at http://penguinppc.org/~hollisb/kvm/patches/kvmppc_no_shadow_tlb.diff, but I'll repost it here in case anybody has any thoughts. (Because it can degrade performance, I did not include it in my recent push to Avi.) ----- kvm: ppc: insert shadow mappings directly into the MMU The overhead of checking for TLB updates after every exit handler outweighs any benefits. Instead of keeping a shadow TLB data structure (which wouldn't work well for larger TLBs anyways), insert shadow mappings directly into the hardware TLB. (This means that host mappings may overwrite shadow mappings, which on a subsequent fault will cause the shadow mapping to be reinserted elsewhere (and a corresponding increase to the guest page's refcount). However, we won't lose the page pointer, so we still properly free it.) The net is about a 20% performance improvement (!). Thanks to Liu Yu <yu.liu@xxxxxxxxxxxxx> for the idea. Signed-off-by: Hollis Blanchard <hollisb@xxxxxxxxxx> diff --git a/arch/powerpc/include/asm/kvm_44x.h b/arch/powerpc/include/asm/kvm_44x.h --- a/arch/powerpc/include/asm/kvm_44x.h +++ b/arch/powerpc/include/asm/kvm_44x.h @@ -22,19 +22,25 @@ #include <linux/kvm_host.h> -/* XXX Can't include mmu-44x.h because it redefines struct mm_context. */ #define PPC44x_TLB_SIZE 64 + +/* If the guest is expecting it, this can be as large as we like; we'd just + * need to find some way of advertising it. */ +#define KVM44x_GUEST_TLB_SIZE 64 + +struct kvmppc_44x_shadow_ref { + struct page *page; + u16 gtlb_index; + u8 writeable; + u8 tid; +}; struct kvmppc_vcpu_44x { /* Unmodified copy of the guest's TLB. */ - struct kvmppc_44x_tlbe guest_tlb[PPC44x_TLB_SIZE]; - /* TLB that's actually used when the guest is running. */ - struct kvmppc_44x_tlbe shadow_tlb[PPC44x_TLB_SIZE]; - /* Pages which are referenced in the shadow TLB. */ - struct page *shadow_pages[PPC44x_TLB_SIZE]; + struct kvmppc_44x_tlbe guest_tlb[KVM44x_GUEST_TLB_SIZE]; - /* Track which TLB entries we've modified in the current exit. */ - u8 shadow_tlb_mod[PPC44x_TLB_SIZE]; + /* Shadow pages referenced in the real TLB. */ + struct kvmppc_44x_shadow_ref tlb_shadow[PPC44x_TLB_SIZE]; struct kvm_vcpu vcpu; }; diff --git a/arch/powerpc/include/asm/kvm_ppc.h b/arch/powerpc/include/asm/kvm_ppc.h --- a/arch/powerpc/include/asm/kvm_ppc.h +++ b/arch/powerpc/include/asm/kvm_ppc.h @@ -52,8 +52,8 @@ extern int kvmppc_emulate_mmio(struct kv extern int kvmppc_emulate_mmio(struct kvm_run *run, struct kvm_vcpu *vcpu); extern void kvmppc_emulate_dec(struct kvm_vcpu *vcpu); -extern void kvmppc_mmu_map(struct kvm_vcpu *vcpu, u64 gvaddr, gfn_t gfn, - u64 asid, u32 flags); +extern void kvmppc_mmu_map(struct kvm_vcpu *vcpu, unsigned int gtlb_idx, + gva_t gvaddr, gfn_t gfn, u64 asid, u32 flags); extern void kvmppc_mmu_priv_switch(struct kvm_vcpu *vcpu, int usermode); extern void kvmppc_mmu_switch_pid(struct kvm_vcpu *vcpu, u32 pid); diff --git a/arch/powerpc/kernel/asm-offsets.c b/arch/powerpc/kernel/asm-offsets.c --- a/arch/powerpc/kernel/asm-offsets.c +++ b/arch/powerpc/kernel/asm-offsets.c @@ -357,12 +357,6 @@ int main(void) #ifdef CONFIG_KVM DEFINE(TLBE_BYTES, sizeof(struct kvmppc_44x_tlbe)); - DEFINE(VCPU_TO_44X, offsetof(struct kvmppc_vcpu_44x, vcpu)); - DEFINE(VCPU44x_SHADOW_TLB, - offsetof(struct kvmppc_vcpu_44x, shadow_tlb)); - DEFINE(VCPU44x_SHADOW_MOD, - offsetof(struct kvmppc_vcpu_44x, shadow_tlb_mod)); - DEFINE(VCPU_HOST_STACK, offsetof(struct kvm_vcpu, arch.host_stack)); DEFINE(VCPU_HOST_PID, offsetof(struct kvm_vcpu, arch.host_pid)); DEFINE(VCPU_GPRS, offsetof(struct kvm_vcpu, arch.gpr)); diff --git a/arch/powerpc/kvm/44x.c b/arch/powerpc/kvm/44x.c --- a/arch/powerpc/kvm/44x.c +++ b/arch/powerpc/kvm/44x.c @@ -96,21 +96,14 @@ void kvmppc_core_load_guest_debugstate(s void kvmppc_core_vcpu_load(struct kvm_vcpu *vcpu, int cpu) { - int i; - - /* Mark every guest entry in the shadow TLB entry modified, so that they - * will all be reloaded on the next vcpu run (instead of being - * demand-faulted). */ - for (i = 0; i <= tlb_44x_hwater; i++) - kvmppc_tlbe_set_modified(vcpu, i); } void kvmppc_core_vcpu_put(struct kvm_vcpu *vcpu) { - /* Don't leave guest TLB entries resident when being de-scheduled. */ - /* XXX It would be nice to differentiate between heavyweight exit and - * sched_out here, since we could avoid the TLB flush for heavyweight - * exits. */ + /* XXX Since every guest uses TS=1 TID=0/1 mappings, we can't leave any TLB + * entries around when we're descheduled, so we must completely flush the + * TLB of all guest mappings. On the other hand, if there is only one + * guest, this flush is completely unnecessary. */ _tlbia(); } @@ -130,6 +123,7 @@ int kvmppc_core_vcpu_setup(struct kvm_vc { struct kvmppc_vcpu_44x *vcpu_44x = to_44x(vcpu); struct kvmppc_44x_tlbe *tlbe = &vcpu_44x->guest_tlb[0]; + int i; tlbe->tid = 0; tlbe->word0 = PPC44x_TLB_16M | PPC44x_TLB_VALID; @@ -147,6 +141,9 @@ int kvmppc_core_vcpu_setup(struct kvm_vc * real timebase frequency. Accordingly, it must see the state of * CCR1[TCS]. */ vcpu->arch.ccr1 = mfspr(SPRN_CCR1); + + for (i = 0; i < ARRAY_SIZE(vcpu_44x->tlb_shadow); i++) + vcpu_44x->tlb_shadow[i].gtlb_index = -1; return 0; } diff --git a/arch/powerpc/kvm/44x_tlb.c b/arch/powerpc/kvm/44x_tlb.c --- a/arch/powerpc/kvm/44x_tlb.c +++ b/arch/powerpc/kvm/44x_tlb.c @@ -22,7 +22,9 @@ #include <linux/kvm.h> #include <linux/kvm_host.h> #include <linux/highmem.h> + #include <asm/mmu-44x.h> +#include <asm/tlbflush.h> #include <asm/kvm_ppc.h> #include <asm/kvm_44x.h> @@ -30,8 +32,6 @@ #define PPC44x_TLB_USER_PERM_MASK (PPC44x_TLB_UX|PPC44x_TLB_UR|PPC44x_TLB_UW) #define PPC44x_TLB_SUPER_PERM_MASK (PPC44x_TLB_SX|PPC44x_TLB_SR|PPC44x_TLB_SW) - -static unsigned int kvmppc_tlb_44x_pos; #ifdef DEBUG void kvmppc_dump_tlbs(struct kvm_vcpu *vcpu) @@ -43,23 +43,51 @@ void kvmppc_dump_tlbs(struct kvm_vcpu *v printk("| %2s | %3s | %8s | %8s | %8s |\n", "nr", "tid", "word0", "word1", "word2"); - for (i = 0; i < PPC44x_TLB_SIZE; i++) { + for (i = 0; i < ARRAY_SIZE(vcpu_44x->guest_tlb); i++) { tlbe = &vcpu_44x->guest_tlb[i]; if (tlbe->word0 & PPC44x_TLB_VALID) printk(" G%2d | %02X | %08X | %08X | %08X |\n", i, tlbe->tid, tlbe->word0, tlbe->word1, tlbe->word2); } - - for (i = 0; i < PPC44x_TLB_SIZE; i++) { - tlbe = &vcpu_44x->shadow_tlb[i]; - if (tlbe->word0 & PPC44x_TLB_VALID) - printk(" S%2d | %02X | %08X | %08X | %08X |\n", - i, tlbe->tid, tlbe->word0, tlbe->word1, - tlbe->word2); - } } #endif + +static inline void kvmppc_44x_tlbie(unsigned int index) +{ + /* 0 <= index < 64, so the V bit is clear and we can use the index as + * word0. */ + asm volatile( + "tlbwe %[index], %[index], 0\n" + : + : [index] "r"(index) + ); +} + +static inline void kvmppc_44x_tlbwe(unsigned int index, + u8 tid, + u32 word0, + u32 word1, + u32 word2) +{ + unsigned long tmp; + + asm volatile( + "mfspr %[tmp], %[sprn_mmucr]\n" + "rlwimi %[tmp], %[pid], 0, 0xff\n" + "mtspr %[sprn_mmucr], %[tmp]\n" + "tlbwe %[word0], %[index], 0\n" + "tlbwe %[word1], %[index], 1\n" + "tlbwe %[word2], %[index], 2\n" + : [tmp] "=&r"(tmp) + : [word0] "r"(word0), + [word1] "r"(word1), + [word2] "r"(word2), + [pid] "r"(tid), + [index] "r"(index), + [sprn_mmucr] "i"(SPRN_MMUCR) + ); +} static u32 kvmppc_44x_tlb_shadow_attrib(u32 attrib, int usermode) { @@ -87,7 +115,7 @@ int kvmppc_44x_tlb_index(struct kvm_vcpu int i; /* XXX Replace loop with fancy data structures. */ - for (i = 0; i < PPC44x_TLB_SIZE; i++) { + for (i = 0; i < ARRAY_SIZE(vcpu_44x->guest_tlb); i++) { struct kvmppc_44x_tlbe *tlbe = &vcpu_44x->guest_tlb[i]; unsigned int tid; @@ -113,76 +141,69 @@ int kvmppc_44x_tlb_index(struct kvm_vcpu return -1; } -struct kvmppc_44x_tlbe *kvmppc_44x_itlb_search(struct kvm_vcpu *vcpu, - gva_t eaddr) +int kvmppc_44x_itlb_index(struct kvm_vcpu *vcpu, gva_t eaddr) { - struct kvmppc_vcpu_44x *vcpu_44x = to_44x(vcpu); unsigned int as = !!(vcpu->arch.msr & MSR_IS); - unsigned int index; - index = kvmppc_44x_tlb_index(vcpu, eaddr, vcpu->arch.pid, as); - if (index == -1) - return NULL; - return &vcpu_44x->guest_tlb[index]; + return kvmppc_44x_tlb_index(vcpu, eaddr, vcpu->arch.pid, as); } -struct kvmppc_44x_tlbe *kvmppc_44x_dtlb_search(struct kvm_vcpu *vcpu, - gva_t eaddr) +int kvmppc_44x_dtlb_index(struct kvm_vcpu *vcpu, gva_t eaddr) { - struct kvmppc_vcpu_44x *vcpu_44x = to_44x(vcpu); unsigned int as = !!(vcpu->arch.msr & MSR_DS); - unsigned int index; - index = kvmppc_44x_tlb_index(vcpu, eaddr, vcpu->arch.pid, as); - if (index == -1) - return NULL; - return &vcpu_44x->guest_tlb[index]; + return kvmppc_44x_tlb_index(vcpu, eaddr, vcpu->arch.pid, as); } -static int kvmppc_44x_tlbe_is_writable(struct kvmppc_44x_tlbe *tlbe) +static void kvmppc_44x_shadow_release(struct kvmppc_vcpu_44x *vcpu_44x, + unsigned int stlb_index) { - return tlbe->word2 & (PPC44x_TLB_SW|PPC44x_TLB_UW); -} + struct kvmppc_44x_shadow_ref *ref = &vcpu_44x->tlb_shadow[stlb_index]; -static void kvmppc_44x_shadow_release(struct kvm_vcpu *vcpu, - unsigned int index) -{ - struct kvmppc_vcpu_44x *vcpu_44x = to_44x(vcpu); - struct kvmppc_44x_tlbe *stlbe = &vcpu_44x->shadow_tlb[index]; - struct page *page = vcpu_44x->shadow_pages[index]; + if (!ref->page) + return; - if (get_tlb_v(stlbe)) { - if (kvmppc_44x_tlbe_is_writable(stlbe)) - kvm_release_page_dirty(page); - else - kvm_release_page_clean(page); - } -} + /* Discard from the TLB. */ + /* Note: we could actually invalidate a host mapping, if the host overwrote + * this TLB entry since we inserted a guest mapping. */ + kvmppc_44x_tlbie(stlb_index); -void kvmppc_tlbe_set_modified(struct kvm_vcpu *vcpu, unsigned int i) -{ - struct kvmppc_vcpu_44x *vcpu_44x = to_44x(vcpu); + /* Now release the page. */ + if (ref->writeable) + kvm_release_page_dirty(ref->page); + else + kvm_release_page_clean(ref->page); - vcpu_44x->shadow_tlb_mod[i] = 1; + ref->page = NULL; + + /* XXX set tlb_44x_index to stlb_index? */ + + KVMTRACE_1D(STLB_INVAL, &vcpu_44x->vcpu, stlb_index, handler); } /* Caller must ensure that the specified guest TLB entry is safe to insert into * the shadow TLB. */ -void kvmppc_mmu_map(struct kvm_vcpu *vcpu, u64 gvaddr, gfn_t gfn, u64 asid, - u32 flags) +void kvmppc_mmu_map(struct kvm_vcpu *vcpu, unsigned int gtlb_index, + gva_t gvaddr, gfn_t gfn, u64 asid, u32 flags) { struct kvmppc_vcpu_44x *vcpu_44x = to_44x(vcpu); + struct kvmppc_44x_shadow_ref *ref; struct page *new_page; - struct kvmppc_44x_tlbe *stlbe; hpa_t hpaddr; unsigned int victim; + u32 word0; + u32 word1; + u32 word2; + u8 tid; - /* Future optimization: don't overwrite the TLB entry containing the - * current PC (or stack?). */ - victim = kvmppc_tlb_44x_pos++; - if (kvmppc_tlb_44x_pos > tlb_44x_hwater) - kvmppc_tlb_44x_pos = 0; - stlbe = &vcpu_44x->shadow_tlb[victim]; + /* Select TLB entry to clobber. Indirectly guard against races with the TLB + * miss handler by disabling interrupts. */ + local_irq_disable(); + victim = ++tlb_44x_index; + if (victim > tlb_44x_hwater) + victim = 0; + tlb_44x_index = victim; + local_irq_enable(); /* Get reference to new page. */ new_page = gfn_to_page(vcpu->kvm, gfn); @@ -191,12 +212,9 @@ void kvmppc_mmu_map(struct kvm_vcpu *vcp kvm_release_page_clean(new_page); return; } - hpaddr = page_to_phys(new_page); - /* Drop reference to old page. */ - kvmppc_44x_shadow_release(vcpu, victim); - - vcpu_44x->shadow_pages[victim] = new_page; + /* Invalidate any previous shadow mappings. */ + kvmppc_44x_shadow_release(vcpu_44x, victim); /* XXX Make sure (va, size) doesn't overlap any other * entries. 440x6 user manual says the result would be @@ -204,78 +222,60 @@ void kvmppc_mmu_map(struct kvm_vcpu *vcp /* XXX what about AS? */ - stlbe->tid = !(asid & 0xff); - /* Force TS=1 for all guest mappings. */ /* For now we hardcode 4KB mappings, but it will be important to * use host large pages in the future. */ - stlbe->word0 = (gvaddr & PAGE_MASK) | PPC44x_TLB_VALID | PPC44x_TLB_TS - | PPC44x_TLB_4K; - stlbe->word1 = (hpaddr & 0xfffffc00) | ((hpaddr >> 32) & 0xf); - stlbe->word2 = kvmppc_44x_tlb_shadow_attrib(flags, - vcpu->arch.msr & MSR_PR); - kvmppc_tlbe_set_modified(vcpu, victim); + word0 = (gvaddr & PAGE_MASK) | PPC44x_TLB_VALID | PPC44x_TLB_TS + | PPC44x_TLB_4K; + hpaddr = page_to_phys(new_page); + word1 = (hpaddr & 0xfffffc00) | ((hpaddr >> 32) & 0xf); + word2 = kvmppc_44x_tlb_shadow_attrib(flags, vcpu->arch.msr & MSR_PR); + tid = !(asid & 0xff); - KVMTRACE_5D(STLB_WRITE, vcpu, victim, - stlbe->tid, stlbe->word0, stlbe->word1, stlbe->word2, - handler); + /* Keep track of the reference so we can properly release it later. */ + ref = &vcpu_44x->tlb_shadow[victim]; + ref->page = new_page; + ref->gtlb_index = gtlb_index; + ref->writeable = !!(word2 & PPC44x_TLB_UW); + ref->tid = tid; + + /* Insert shadow mapping into hardware TLB. */ + kvmppc_44x_tlbwe(victim, tid, word0, word1, word2); + KVMTRACE_5D(STLB_WRITE, vcpu, victim, tid, word0, word1, word2, handler); } -static void kvmppc_mmu_invalidate(struct kvm_vcpu *vcpu, gva_t eaddr, - gva_t eend, u32 asid) +/* For a particular guest TLB entry, invalidate the corresponding host TLB + * mappings and release the host pages. */ +static void kvmppc_44x_invalidate(struct kvm_vcpu *vcpu, + unsigned int gtlb_index) { - struct kvmppc_vcpu_44x *vcpu_44x = to_44x(vcpu); - unsigned int pid = !(asid & 0xff); - int i; + struct kvmppc_vcpu_44x *vcpu_44x = to_44x(vcpu); + int i; - /* XXX Replace loop with fancy data structures. */ - for (i = 0; i <= tlb_44x_hwater; i++) { - struct kvmppc_44x_tlbe *stlbe = &vcpu_44x->shadow_tlb[i]; - unsigned int tid; - - if (!get_tlb_v(stlbe)) - continue; - - if (eend < get_tlb_eaddr(stlbe)) - continue; - - if (eaddr > get_tlb_end(stlbe)) - continue; - - tid = get_tlb_tid(stlbe); - if (tid && (tid != pid)) - continue; - - kvmppc_44x_shadow_release(vcpu, i); - stlbe->word0 = 0; - kvmppc_tlbe_set_modified(vcpu, i); - KVMTRACE_5D(STLB_INVAL, vcpu, i, - stlbe->tid, stlbe->word0, stlbe->word1, - stlbe->word2, handler); - } + for (i = 0; i < ARRAY_SIZE(vcpu_44x->tlb_shadow); i++) { + struct kvmppc_44x_shadow_ref *ref = &vcpu_44x->tlb_shadow[i]; + if (ref->gtlb_index == gtlb_index) + kvmppc_44x_shadow_release(vcpu_44x, i); + } } -/* Invalidate all mappings on the privilege switch after PID has been changed. - * The guest always runs with PID=1, so we must clear the entire TLB when - * switching address spaces. */ +/* Invalidate user mappings on the privilege switch after PID has been changed. + * Guest userspace runs with TID=0 mappings and PID=0, to make sure it can't + * access guest kernel mappings (TID=1). When we switch to a new guest PID, + * which will also use host PID=0, we must discard the old guest userspace + * mappings. */ void kvmppc_mmu_priv_switch(struct kvm_vcpu *vcpu, int usermode) { - struct kvmppc_vcpu_44x *vcpu_44x = to_44x(vcpu); - int i; + if (vcpu->arch.swap_pid) { + struct kvmppc_vcpu_44x *vcpu_44x = to_44x(vcpu); + int i; - if (vcpu->arch.swap_pid) { - /* XXX Replace loop with fancy data structures. */ - for (i = 0; i <= tlb_44x_hwater; i++) { - struct kvmppc_44x_tlbe *stlbe = &vcpu_44x->shadow_tlb[i]; + for (i = 0; i < ARRAY_SIZE(vcpu_44x->tlb_shadow); i++) { + struct kvmppc_44x_shadow_ref *ref = &vcpu_44x->tlb_shadow[i]; + if (ref->tid == 0) + kvmppc_44x_shadow_release(vcpu_44x, i); + } - /* Future optimization: clear only userspace mappings. */ - kvmppc_44x_shadow_release(vcpu, i); - stlbe->word0 = 0; - kvmppc_tlbe_set_modified(vcpu, i); - KVMTRACE_5D(STLB_INVAL, vcpu, i, - stlbe->tid, stlbe->word0, stlbe->word1, - stlbe->word2, handler); - } vcpu->arch.swap_pid = 0; } @@ -306,28 +306,21 @@ int kvmppc_44x_emul_tlbwe(struct kvm_vcp int kvmppc_44x_emul_tlbwe(struct kvm_vcpu *vcpu, u8 ra, u8 rs, u8 ws) { struct kvmppc_vcpu_44x *vcpu_44x = to_44x(vcpu); - u64 eaddr; - u64 raddr; - u64 asid; - u32 flags; struct kvmppc_44x_tlbe *tlbe; - unsigned int index; + unsigned int gtlb_index; - index = vcpu->arch.gpr[ra]; - if (index > PPC44x_TLB_SIZE) { - printk("%s: index %d\n", __func__, index); + gtlb_index = vcpu->arch.gpr[ra]; + if (gtlb_index >= ARRAY_SIZE(vcpu_44x->guest_tlb)) { + printk("%s: index %d\n", __func__, gtlb_index); kvmppc_dump_vcpu(vcpu); return EMULATE_FAIL; } - tlbe = &vcpu_44x->guest_tlb[index]; + tlbe = &vcpu_44x->guest_tlb[gtlb_index]; - /* Invalidate shadow mappings for the about-to-be-clobbered TLBE. */ - if (tlbe->word0 & PPC44x_TLB_VALID) { - eaddr = get_tlb_eaddr(tlbe); - asid = (tlbe->word0 & PPC44x_TLB_TS) | tlbe->tid; - kvmppc_mmu_invalidate(vcpu, eaddr, get_tlb_end(tlbe), asid); - } + /* Invalidate shadow mappings for the about-to-be-clobbered TLB entry. */ + if (tlbe->word0 & PPC44x_TLB_VALID) + kvmppc_44x_invalidate(vcpu, gtlb_index); switch (ws) { case PPC44x_TLB_PAGEID: @@ -348,18 +341,18 @@ int kvmppc_44x_emul_tlbwe(struct kvm_vcp } if (tlbe_is_host_safe(vcpu, tlbe)) { - eaddr = get_tlb_eaddr(tlbe); - raddr = get_tlb_raddr(tlbe); - asid = (tlbe->word0 & PPC44x_TLB_TS) | tlbe->tid; - flags = tlbe->word2 & 0xffff; + gva_t eaddr = get_tlb_eaddr(tlbe); + gfn_t gfn = get_tlb_raddr(tlbe) >> PAGE_SHIFT; + u64 asid = (tlbe->word0 & PPC44x_TLB_TS) | tlbe->tid; + u32 flags = tlbe->word2 & 0xffff; /* Create a 4KB mapping on the host. If the guest wanted a * large page, only the first 4KB is mapped here and the rest * are mapped on the fly. */ - kvmppc_mmu_map(vcpu, eaddr, raddr >> PAGE_SHIFT, asid, flags); + kvmppc_mmu_map(vcpu, gtlb_index, eaddr, gfn, asid, flags); } - KVMTRACE_5D(GTLB_WRITE, vcpu, index, + KVMTRACE_5D(GTLB_WRITE, vcpu, gtlb_index, tlbe->tid, tlbe->word0, tlbe->word1, tlbe->word2, handler); @@ -369,7 +362,7 @@ int kvmppc_44x_emul_tlbsx(struct kvm_vcp int kvmppc_44x_emul_tlbsx(struct kvm_vcpu *vcpu, u8 rt, u8 ra, u8 rb, u8 rc) { u32 ea; - int index; + int gtlb_index; unsigned int as = get_mmucr_sts(vcpu); unsigned int pid = get_mmucr_stid(vcpu); @@ -377,14 +370,14 @@ int kvmppc_44x_emul_tlbsx(struct kvm_vcp if (ra) ea += vcpu->arch.gpr[ra]; - index = kvmppc_44x_tlb_index(vcpu, ea, pid, as); + gtlb_index = kvmppc_44x_tlb_index(vcpu, ea, pid, as); if (rc) { - if (index < 0) + if (gtlb_index < 0) vcpu->arch.cr &= ~0x20000000; else vcpu->arch.cr |= 0x20000000; } - vcpu->arch.gpr[rt] = index; + vcpu->arch.gpr[rt] = gtlb_index; return EMULATE_DONE; } diff --git a/arch/powerpc/kvm/44x_tlb.h b/arch/powerpc/kvm/44x_tlb.h --- a/arch/powerpc/kvm/44x_tlb.h +++ b/arch/powerpc/kvm/44x_tlb.h @@ -25,11 +25,8 @@ extern int kvmppc_44x_tlb_index(struct kvm_vcpu *vcpu, gva_t eaddr, unsigned int pid, unsigned int as); -extern struct kvmppc_44x_tlbe *kvmppc_44x_dtlb_search(struct kvm_vcpu *vcpu, - gva_t eaddr); -extern struct kvmppc_44x_tlbe *kvmppc_44x_itlb_search(struct kvm_vcpu *vcpu, - gva_t eaddr); -extern void kvmppc_tlbe_set_modified(struct kvm_vcpu *vcpu, unsigned int i); +extern int kvmppc_44x_dtlb_index(struct kvm_vcpu *vcpu, gva_t eaddr); +extern int kvmppc_44x_itlb_index(struct kvm_vcpu *vcpu, gva_t eaddr); extern int kvmppc_44x_emul_tlbsx(struct kvm_vcpu *vcpu, u8 rt, u8 ra, u8 rb, u8 rc); diff --git a/arch/powerpc/kvm/booke.c b/arch/powerpc/kvm/booke.c --- a/arch/powerpc/kvm/booke.c +++ b/arch/powerpc/kvm/booke.c @@ -24,10 +24,12 @@ #include <linux/module.h> #include <linux/vmalloc.h> #include <linux/fs.h> + #include <asm/cputable.h> #include <asm/uaccess.h> +#include <asm/cacheflush.h> #include <asm/kvm_ppc.h> -#include <asm/cacheflush.h> +#include <asm/kvm_44x.h> #include "booke.h" #include "44x_tlb.h" @@ -206,11 +208,6 @@ int kvmppc_handle_exit(struct kvm_run *r /* Since we switched IVPR back to the host's value, the host * handled this interrupt the moment we enabled interrupts. * Now we just offer it a chance to reschedule the guest. */ - - /* XXX At this point the TLB still holds our shadow TLB, so if - * we do reschedule the host will fault over it. Perhaps we - * should politely restore the host's entries to minimize - * misses before ceding control. */ vcpu->stat.dec_exits++; if (need_resched()) cond_resched(); @@ -281,14 +278,17 @@ int kvmppc_handle_exit(struct kvm_run *r r = RESUME_GUEST; break; + /* XXX move to a 440-specific file. */ case BOOKE_INTERRUPT_DTLB_MISS: { + struct kvmppc_vcpu_44x *vcpu_44x = to_44x(vcpu); struct kvmppc_44x_tlbe *gtlbe; unsigned long eaddr = vcpu->arch.fault_dear; + int gtlb_index; gfn_t gfn; /* Check the guest TLB. */ - gtlbe = kvmppc_44x_dtlb_search(vcpu, eaddr); - if (!gtlbe) { + gtlb_index = kvmppc_44x_dtlb_index(vcpu, eaddr); + if (gtlb_index < 0) { /* The guest didn't have a mapping for it. */ kvmppc_booke_queue_irqprio(vcpu, BOOKE_IRQPRIO_DTLB_MISS); vcpu->arch.dear = vcpu->arch.fault_dear; @@ -298,6 +298,7 @@ int kvmppc_handle_exit(struct kvm_run *r break; } + gtlbe = &vcpu_44x->guest_tlb[gtlb_index]; vcpu->arch.paddr_accessed = tlb_xlate(gtlbe, eaddr); gfn = vcpu->arch.paddr_accessed >> PAGE_SHIFT; @@ -308,7 +309,7 @@ int kvmppc_handle_exit(struct kvm_run *r * b) the guest used a large mapping which we're faking * Either way, we need to satisfy the fault without * invoking the guest. */ - kvmppc_mmu_map(vcpu, eaddr, gfn, gtlbe->tid, + kvmppc_mmu_map(vcpu, gtlb_index, eaddr, gfn, gtlbe->tid, gtlbe->word2); vcpu->stat.dtlb_virt_miss_exits++; r = RESUME_GUEST; @@ -322,16 +323,19 @@ int kvmppc_handle_exit(struct kvm_run *r break; } + /* XXX move to a 440-specific file. */ case BOOKE_INTERRUPT_ITLB_MISS: { + struct kvmppc_vcpu_44x *vcpu_44x = to_44x(vcpu); struct kvmppc_44x_tlbe *gtlbe; unsigned long eaddr = vcpu->arch.pc; gfn_t gfn; + int gtlb_index; r = RESUME_GUEST; /* Check the guest TLB. */ - gtlbe = kvmppc_44x_itlb_search(vcpu, eaddr); - if (!gtlbe) { + gtlb_index = kvmppc_44x_itlb_index(vcpu, eaddr); + if (gtlb_index < 0) { /* The guest didn't have a mapping for it. */ kvmppc_booke_queue_irqprio(vcpu, BOOKE_IRQPRIO_ITLB_MISS); vcpu->stat.itlb_real_miss_exits++; @@ -340,6 +344,7 @@ int kvmppc_handle_exit(struct kvm_run *r vcpu->stat.itlb_virt_miss_exits++; + gtlbe = &vcpu_44x->guest_tlb[gtlb_index]; gfn = tlb_xlate(gtlbe, eaddr) >> PAGE_SHIFT; if (kvm_is_visible_gfn(vcpu->kvm, gfn)) { @@ -349,7 +354,7 @@ int kvmppc_handle_exit(struct kvm_run *r * b) the guest used a large mapping which we're faking * Either way, we need to satisfy the fault without * invoking the guest. */ - kvmppc_mmu_map(vcpu, eaddr, gfn, gtlbe->tid, + kvmppc_mmu_map(vcpu, gtlb_index, eaddr, gfn, gtlbe->tid, gtlbe->word2); } else { /* Guest mapped and leaped at non-RAM! */ diff --git a/arch/powerpc/kvm/booke_interrupts.S b/arch/powerpc/kvm/booke_interrupts.S --- a/arch/powerpc/kvm/booke_interrupts.S +++ b/arch/powerpc/kvm/booke_interrupts.S @@ -335,54 +335,6 @@ lightweight_exit: lwz r3, VCPU_SHADOW_PID(r4) mtspr SPRN_PID, r3 - /* Prevent all asynchronous TLB updates. */ - mfmsr r5 - lis r6, (MSR_EE|MSR_CE|MSR_ME|MSR_DE)@h - ori r6, r6, (MSR_EE|MSR_CE|MSR_ME|MSR_DE)@l - andc r6, r5, r6 - mtmsr r6 - - /* Load the guest mappings, leaving the host's "pinned" kernel mappings - * in place. */ - mfspr r10, SPRN_MMUCR /* Save host MMUCR. */ - li r5, PPC44x_TLB_SIZE - lis r5, tlb_44x_hwater@ha - lwz r5, tlb_44x_hwater@l(r5) - mtctr r5 - addi r9, r4, -VCPU_TO_44X + VCPU44x_SHADOW_TLB - addi r5, r4, -VCPU_TO_44X + VCPU44x_SHADOW_MOD - li r3, 0 -1: - lbzx r7, r3, r5 - cmpwi r7, 0 - beq 3f - - /* Load guest entry. */ - mulli r11, r3, TLBE_BYTES - add r11, r11, r9 - lwz r7, 0(r11) - mtspr SPRN_MMUCR, r7 - lwz r7, 4(r11) - tlbwe r7, r3, PPC44x_TLB_PAGEID - lwz r7, 8(r11) - tlbwe r7, r3, PPC44x_TLB_XLAT - lwz r7, 12(r11) - tlbwe r7, r3, PPC44x_TLB_ATTRIB -3: - addi r3, r3, 1 /* Increment index. */ - bdnz 1b - - mtspr SPRN_MMUCR, r10 /* Restore host MMUCR. */ - - /* Clear bitmap of modified TLB entries */ - li r5, PPC44x_TLB_SIZE>>2 - mtctr r5 - addi r5, r4, -VCPU_TO_44X + VCPU44x_SHADOW_MOD - 4 - li r6, 0 -1: - stwu r6, 4(r5) - bdnz 1b - iccci 0, 0 /* XXX hack */ /* Load some guest volatiles. */ -- Hollis Blanchard IBM Linux Technology Center -- To unsubscribe from this list: send the line "unsubscribe kvm-ppc" in the body of a message to majordomo@xxxxxxxxxxxxxxx More majordomo info at http://vger.kernel.org/majordomo-info.html
