Author: avg Date: Sat Mar 25 18:45:09 2017 New Revision: 315959 URL: https://svnweb.freebsd.org/changeset/base/315959
Log: specific end of interrupt implementation for AMD Local APIC The change is more intrusive than I would like because the feature requires that a vector number is written to a special register. Thus, now the vector number has to be provided to lapic_eoi(). It was readily available in the IO-APIC and MSI cases, but the IPI handlers required more work. Also, we now store the VMM IPI number in a global variable, so that it is available to the justreturn handler for the same reason. Reviewed by: kib MFC after: 6 weeks Differential Revision: https://reviews.freebsd.org/D9880 Modified: head/sys/amd64/amd64/apic_vector.S head/sys/amd64/amd64/genassym.c head/sys/amd64/vmm/amd/svm.c head/sys/amd64/vmm/vmm.c head/sys/amd64/vmm/vmm_host.h head/sys/i386/i386/apic_vector.s head/sys/i386/i386/genassym.c head/sys/x86/include/apicvar.h head/sys/x86/x86/io_apic.c head/sys/x86/x86/local_apic.c head/sys/x86/x86/msi.c head/sys/x86/xen/xen_apic.c Modified: head/sys/amd64/amd64/apic_vector.S ============================================================================== --- head/sys/amd64/amd64/apic_vector.S Sat Mar 25 15:57:47 2017 (r315958) +++ head/sys/amd64/amd64/apic_vector.S Sat Mar 25 18:45:09 2017 (r315959) @@ -50,22 +50,6 @@ #define LK #endif - .text - SUPERALIGN_TEXT - /* End Of Interrupt to APIC */ -as_lapic_eoi: - cmpl $0,x2apic_mode - jne 1f - movq lapic_map,%rax - movl $0,LA_EOI(%rax) - ret -1: - movl $MSR_APIC_EOI,%ecx - xorl %eax,%eax - xorl %edx,%edx - wrmsr - ret - /* * I/O Interrupt Entry Point. Rather than having one entry point for * each interrupt source, we use one entry point for each 32-bit word @@ -182,7 +166,7 @@ IDTVEC(xen_intr_upcall) SUPERALIGN_TEXT invltlb_ret: - call as_lapic_eoi + call native_lapic_eoi POP_FRAME jmp doreti_iret @@ -191,18 +175,21 @@ IDTVEC(invltlb) PUSH_FRAME call invltlb_handler + movl $IPI_INVLTLB, %edi jmp invltlb_ret IDTVEC(invltlb_pcid) PUSH_FRAME call invltlb_pcid_handler + movl $IPI_INVLTLB, %edi jmp invltlb_ret IDTVEC(invltlb_invpcid) PUSH_FRAME call invltlb_invpcid_handler + movl $IPI_INVLTLB, %edi jmp invltlb_ret /* @@ -215,6 +202,7 @@ IDTVEC(invlpg) PUSH_FRAME call invlpg_handler + movl $IPI_INVLPG, %edi jmp invltlb_ret /* @@ -226,6 +214,7 @@ IDTVEC(invlrng) PUSH_FRAME call invlrng_handler + movl $IPI_INVLRNG, %edi jmp invltlb_ret /* @@ -237,6 +226,7 @@ IDTVEC(invlcache) PUSH_FRAME call invlcache_handler + movl $IPI_INVLCACHE, %edi jmp invltlb_ret /* @@ -247,7 +237,8 @@ IDTVEC(invlcache) IDTVEC(ipi_intr_bitmap_handler) PUSH_FRAME - call as_lapic_eoi + movl $IPI_BITMAP_VECTOR, %edi + call native_lapic_eoi FAKE_MCOUNT(TF_RIP(%rsp)) @@ -263,7 +254,8 @@ IDTVEC(ipi_intr_bitmap_handler) IDTVEC(cpustop) PUSH_FRAME - call as_lapic_eoi + movl $IPI_STOP, %edi + call native_lapic_eoi call cpustop_handler jmp doreti @@ -277,7 +269,8 @@ IDTVEC(cpususpend) PUSH_FRAME call cpususpend_handler - call as_lapic_eoi + movl $IPI_SUSPEND, %edi + call native_lapic_eoi jmp doreti /* @@ -295,37 +288,22 @@ IDTVEC(rendezvous) incq (%rax) #endif call smp_rendezvous_action - call as_lapic_eoi + movl $IPI_RENDEZVOUS, %edi + call native_lapic_eoi jmp doreti /* * IPI handler whose purpose is to interrupt the CPU with minimum overhead. * This is used by bhyve to force a host cpu executing in guest context to * trap into the hypervisor. - * - * This handler is different from other IPI handlers in the following aspects: - * - * 1. It doesn't push a trapframe on the stack. - * - * This implies that a DDB backtrace involving 'justreturn' will skip the - * function that was interrupted by this handler. - * - * 2. It doesn't 'swapgs' when userspace is interrupted. - * - * The 'justreturn' handler does not access any pcpu data so it is not an - * issue. Moreover the 'justreturn' handler can only be interrupted by an NMI - * whose handler already doesn't trust GS.base when kernel code is interrupted. */ .text SUPERALIGN_TEXT IDTVEC(justreturn) - pushq %rax - pushq %rcx - pushq %rdx - call as_lapic_eoi - popq %rdx - popq %rcx - popq %rax + PUSH_FRAME + movl vmm_ipinum, %edi + call native_lapic_eoi + POP_FRAME jmp doreti_iret #endif /* SMP */ Modified: head/sys/amd64/amd64/genassym.c ============================================================================== --- head/sys/amd64/amd64/genassym.c Sat Mar 25 15:57:47 2017 (r315958) +++ head/sys/amd64/amd64/genassym.c Sat Mar 25 18:45:09 2017 (r315959) @@ -44,6 +44,7 @@ __FBSDID("$FreeBSD$"); #include <sys/assym.h> #include <sys/bio.h> #include <sys/buf.h> +#include <sys/bus.h> #include <sys/proc.h> #ifdef HWPMC_HOOKS #include <sys/pmckern.h> @@ -61,7 +62,9 @@ __FBSDID("$FreeBSD$"); #include <vm/pmap.h> #include <vm/vm_map.h> #include <sys/proc.h> +#include <machine/intr_machdep.h> #include <x86/apicreg.h> +#include <x86/apicvar.h> #include <machine/cpu.h> #include <machine/pcb.h> #include <machine/sigframe.h> @@ -215,10 +218,19 @@ ASSYM(PC_LDT, offsetof(struct pcpu, pc_l ASSYM(PC_COMMONTSSP, offsetof(struct pcpu, pc_commontssp)); ASSYM(PC_TSS, offsetof(struct pcpu, pc_tss)); ASSYM(PC_PM_SAVE_CNT, offsetof(struct pcpu, pc_pm_save_cnt)); - + ASSYM(LA_EOI, LAPIC_EOI * LAPIC_MEM_MUL); ASSYM(LA_ISR, LAPIC_ISR0 * LAPIC_MEM_MUL); +ASSYM(IPI_INVLTLB, IPI_INVLTLB); +ASSYM(IPI_INVLPG, IPI_INVLPG); +ASSYM(IPI_INVLRNG, IPI_INVLRNG); +ASSYM(IPI_INVLCACHE, IPI_INVLCACHE); +ASSYM(IPI_BITMAP_VECTOR, IPI_BITMAP_VECTOR); +ASSYM(IPI_STOP, IPI_STOP); +ASSYM(IPI_SUSPEND, IPI_SUSPEND); +ASSYM(IPI_RENDEZVOUS, IPI_RENDEZVOUS); + ASSYM(KCSEL, GSEL(GCODE_SEL, SEL_KPL)); ASSYM(KDSEL, GSEL(GDATA_SEL, SEL_KPL)); ASSYM(KUCSEL, GSEL(GUCODE_SEL, SEL_UPL)); Modified: head/sys/amd64/vmm/amd/svm.c ============================================================================== --- head/sys/amd64/vmm/amd/svm.c Sat Mar 25 15:57:47 2017 (r315958) +++ head/sys/amd64/vmm/amd/svm.c Sat Mar 25 18:45:09 2017 (r315959) @@ -48,6 +48,7 @@ __FBSDID("$FreeBSD$"); #include <machine/vmm_dev.h> #include <machine/vmm_instruction_emul.h> +#include "vmm_host.h" #include "vmm_lapic.h" #include "vmm_stat.h" #include "vmm_ktr.h" @@ -1620,7 +1621,7 @@ svm_inj_interrupts(struct svm_softc *sc, * Although not explicitly specified in APMv2 the * relative priorities were verified empirically. */ - ipi_cpu(curcpu, IPI_AST); /* XXX vmm_ipinum? */ + ipi_cpu(curcpu, vmm_ipinum); } else { vm_nmi_clear(sc->vm, vcpu); Modified: head/sys/amd64/vmm/vmm.c ============================================================================== --- head/sys/amd64/vmm/vmm.c Sat Mar 25 15:57:47 2017 (r315958) +++ head/sys/amd64/vmm/vmm.c Sat Mar 25 18:45:09 2017 (r315959) @@ -215,7 +215,6 @@ SYSCTL_INT(_hw_vmm, OID_AUTO, halt_detec &halt_detection_enabled, 0, "Halt VM if all vcpus execute HLT with interrupts disabled"); -static int vmm_ipinum; SYSCTL_INT(_hw_vmm, OID_AUTO, ipinum, CTLFLAG_RD, &vmm_ipinum, 0, "IPI vector used for vcpu notifications"); Modified: head/sys/amd64/vmm/vmm_host.h ============================================================================== --- head/sys/amd64/vmm/vmm_host.h Sat Mar 25 15:57:47 2017 (r315958) +++ head/sys/amd64/vmm/vmm_host.h Sat Mar 25 18:45:09 2017 (r315959) @@ -33,6 +33,8 @@ #error "no user-serviceable parts inside" #endif +extern int vmm_ipinum; + struct xsave_limits { int xsave_enabled; uint64_t xcr0_allowed; Modified: head/sys/i386/i386/apic_vector.s ============================================================================== --- head/sys/i386/i386/apic_vector.s Sat Mar 25 15:57:47 2017 (r315958) +++ head/sys/i386/i386/apic_vector.s Sat Mar 25 18:45:09 2017 (r315959) @@ -44,22 +44,6 @@ #include "assym.s" - .text - SUPERALIGN_TEXT - /* End Of Interrupt to APIC */ -as_lapic_eoi: - cmpl $0,x2apic_mode - jne 1f - movl lapic_map,%eax - movl $0,LA_EOI(%eax) - ret -1: - movl $MSR_APIC_EOI,%ecx - xorl %eax,%eax - xorl %edx,%edx - wrmsr - ret - /* * I/O Interrupt Entry Point. Rather than having one entry point for * each interrupt source, we use one entry point for each 32-bit word @@ -188,7 +172,8 @@ IDTVEC(xen_intr_upcall) .text SUPERALIGN_TEXT invltlb_ret: - call as_lapic_eoi + call native_lapic_eoi + add $4, %esp POP_FRAME iret @@ -200,6 +185,7 @@ IDTVEC(invltlb) call invltlb_handler + pushl $IPI_INVLTLB jmp invltlb_ret /* @@ -214,6 +200,7 @@ IDTVEC(invlpg) call invlpg_handler + pushl $IPI_INVLPG jmp invltlb_ret /* @@ -228,6 +215,7 @@ IDTVEC(invlrng) call invlrng_handler + pushl $IPI_INVLRNG jmp invltlb_ret /* @@ -242,6 +230,7 @@ IDTVEC(invlcache) call invlcache_handler + pushl $IPI_INVLCACHE jmp invltlb_ret /* @@ -254,8 +243,10 @@ IDTVEC(ipi_intr_bitmap_handler) SET_KERNEL_SREGS cld - call as_lapic_eoi - + pushl $IPI_BITMAP_VECTOR + call native_lapic_eoi + add $4, %esp + FAKE_MCOUNT(TF_EIP(%esp)) call ipi_bitmap_handler @@ -272,7 +263,9 @@ IDTVEC(cpustop) SET_KERNEL_SREGS cld - call as_lapic_eoi + pushl $IPI_STOP + call native_lapic_eoi + add $4, %esp call cpustop_handler POP_FRAME @@ -288,7 +281,9 @@ IDTVEC(cpususpend) SET_KERNEL_SREGS cld - call as_lapic_eoi + pushl $IPI_SUSPEND + call native_lapic_eoi + add $4, %esp call cpususpend_handler POP_FRAME @@ -313,7 +308,9 @@ IDTVEC(rendezvous) #endif call smp_rendezvous_action - call as_lapic_eoi + pushl $IPI_RENDEZVOUS + call native_lapic_eoi + add $4, %esp POP_FRAME iret Modified: head/sys/i386/i386/genassym.c ============================================================================== --- head/sys/i386/i386/genassym.c Sat Mar 25 15:57:47 2017 (r315958) +++ head/sys/i386/i386/genassym.c Sat Mar 25 18:45:09 2017 (r315959) @@ -70,7 +70,10 @@ __FBSDID("$FreeBSD$"); #include <nfsclient/nfs.h> #include <nfs/nfsdiskless.h> #ifdef DEV_APIC +#include <sys/bus.h> +#include <machine/intr_machdep.h> #include <x86/apicreg.h> +#include <x86/apicvar.h> #endif #include <machine/cpu.h> #include <machine/pcb.h> @@ -219,6 +222,15 @@ ASSYM(PC_PRIVATE_TSS, offsetof(struct pc #ifdef DEV_APIC ASSYM(LA_EOI, LAPIC_EOI * LAPIC_MEM_MUL); ASSYM(LA_ISR, LAPIC_ISR0 * LAPIC_MEM_MUL); + +ASSYM(IPI_INVLTLB, IPI_INVLTLB); +ASSYM(IPI_INVLPG, IPI_INVLPG); +ASSYM(IPI_INVLRNG, IPI_INVLRNG); +ASSYM(IPI_INVLCACHE, IPI_INVLCACHE); +ASSYM(IPI_BITMAP_VECTOR, IPI_BITMAP_VECTOR); +ASSYM(IPI_STOP, IPI_STOP); +ASSYM(IPI_SUSPEND, IPI_SUSPEND); +ASSYM(IPI_RENDEZVOUS, IPI_RENDEZVOUS); #endif ASSYM(KCSEL, GSEL(GCODE_SEL, SEL_KPL)); Modified: head/sys/x86/include/apicvar.h ============================================================================== --- head/sys/x86/include/apicvar.h Sat Mar 25 15:57:47 2017 (r315958) +++ head/sys/x86/include/apicvar.h Sat Mar 25 18:45:09 2017 (r315959) @@ -210,7 +210,7 @@ struct apic_ops { void (*setup)(int); void (*dump)(const char *); void (*disable)(void); - void (*eoi)(void); + void (*eoi)(u_int vector); int (*id)(void); int (*intr_pending)(u_int); void (*set_logical_id)(u_int, u_int, u_int); @@ -301,10 +301,10 @@ lapic_disable(void) } static inline void -lapic_eoi(void) +lapic_eoi(u_int vector) { - apic_ops.eoi(); + apic_ops.eoi(vector); } static inline int @@ -469,6 +469,7 @@ lapic_set_lvt_triggermode(u_int apic_id, return (apic_ops.set_lvt_triggermode(apic_id, lvt, trigger)); } +void native_lapic_eoi(u_int vector); void lapic_handle_cmc(void); void lapic_handle_error(void); void lapic_handle_intr(int vector, struct trapframe *frame); Modified: head/sys/x86/x86/io_apic.c ============================================================================== --- head/sys/x86/x86/io_apic.c Sat Mar 25 15:57:47 2017 (r315958) +++ head/sys/x86/x86/io_apic.c Sat Mar 25 18:45:09 2017 (r315959) @@ -151,10 +151,10 @@ _ioapic_eoi_source(struct intsrc *isrc, volatile uint32_t *apic_eoi; uint32_t low1; - lapic_eoi(); + src = (struct ioapic_intsrc *)isrc; + lapic_eoi(src->io_vector); if (!lapic_eoi_suppression) return; - src = (struct ioapic_intsrc *)isrc; if (src->io_edgetrigger) return; io = (struct ioapic *)isrc->is_pic; Modified: head/sys/x86/x86/local_apic.c ============================================================================== --- head/sys/x86/x86/local_apic.c Sat Mar 25 15:57:47 2017 (r315958) +++ head/sys/x86/x86/local_apic.c Sat Mar 25 18:45:09 2017 (r315959) @@ -83,6 +83,9 @@ __FBSDID("$FreeBSD$"); #define GSEL_APIC GSEL(GCODE_SEL, SEL_KPL) #endif +#define INTEL_SEOI 1 +#define AMD_SEOI 2 + /* Sanity checks on IDT vectors. */ CTASSERT(APIC_IO_INTS + APIC_NUM_IOINTS == APIC_TIMER_INT); CTASSERT(APIC_TIMER_INT < APIC_LOCAL_INTS); @@ -184,6 +187,10 @@ static struct eventtimer lapic_et; #ifdef SMP static uint64_t lapic_ipi_wait_mult; #endif +#ifdef __amd64__ +/* IPI vector used for VMM VCPU notifications. */ +int vmm_ipinum; +#endif SYSCTL_NODE(_hw, OID_AUTO, apic, CTLFLAG_RD, 0, "APIC options"); SYSCTL_INT(_hw_apic, OID_AUTO, x2apic_mode, CTLFLAG_RD, &x2apic_mode, 0, ""); @@ -312,7 +319,6 @@ static void native_lapic_xapic_mode(void static void native_lapic_setup(int boot); static void native_lapic_dump(const char *str); static void native_lapic_disable(void); -static void native_lapic_eoi(void); static int native_lapic_id(void); static int native_lapic_intr_pending(u_int vector); static u_int native_apic_cpuid(u_int apic_id); @@ -447,6 +453,31 @@ elvt_mode(struct lapic *la, u_int idx, u return (lvt_mode_impl(la, elvt, idx, value)); } +static inline uint32_t +amd_read_ext_features(void) +{ + uint32_t version; + + if (cpu_vendor_id != CPU_VENDOR_AMD) + return (0); + version = lapic_read32(LAPIC_VERSION); + if ((version & APIC_VER_AMD_EXT_SPACE) != 0) + return (lapic_read32(LAPIC_EXT_FEATURES)); + return (0); +} + +static inline uint32_t +amd_read_elvt_count(void) +{ + uint32_t extf; + uint32_t count; + + extf = amd_read_ext_features(); + count = (extf & APIC_EXTF_ELVT_MASK) >> APIC_EXTF_ELVT_SHIFT; + count = min(count, APIC_ELVT_MAX + 1); + return (count); +} + /* * Map the local APIC and setup necessary interrupt vectors. */ @@ -456,9 +487,9 @@ native_lapic_init(vm_paddr_t addr) #ifdef SMP uint64_t r, r1, r2, rx; #endif - uint32_t ver; + uint32_t extf, ver; u_int regs[4]; - int i, arat; + int i, arat, seoi_enable; /* * Enable x2APIC mode if possible. Map the local APIC @@ -546,16 +577,27 @@ native_lapic_init(vm_paddr_t addr) */ ver = lapic_read32(LAPIC_VERSION); if ((ver & APIC_VER_EOI_SUPPRESSION) != 0) { - lapic_eoi_suppression = 1; + lapic_eoi_suppression = INTEL_SEOI; + } else { + extf = amd_read_ext_features(); + if ((extf & APIC_EXTF_SEIO_CAP) != 0) + lapic_eoi_suppression = AMD_SEOI; + } + if (lapic_eoi_suppression != 0) { + seoi_enable = 1; if (vm_guest == VM_GUEST_KVM) { if (bootverbose) printf( "KVM -- disabling lapic eoi suppression\n"); - lapic_eoi_suppression = 0; + seoi_enable = 0; } TUNABLE_INT_FETCH("hw.lapic_eoi_suppression", - &lapic_eoi_suppression); + &seoi_enable); + if (seoi_enable == 0) + lapic_eoi_suppression = 0; } + if (lapic_eoi_suppression != 0) + printf("LAPIC specific EOI enabled\n"); #ifdef SMP #define LOOPS 100000 @@ -642,32 +684,6 @@ native_lapic_create(u_int apic_id, int b #endif } -static inline uint32_t -amd_read_ext_features(void) -{ - uint32_t version; - - if (cpu_vendor_id != CPU_VENDOR_AMD) - return (0); - version = lapic_read32(LAPIC_VERSION); - if ((version & APIC_VER_AMD_EXT_SPACE) != 0) - return (lapic_read32(LAPIC_EXT_FEATURES)); - else - return (0); -} - -static inline uint32_t -amd_read_elvt_count(void) -{ - uint32_t extf; - uint32_t count; - - extf = amd_read_ext_features(); - count = (extf & APIC_EXTF_ELVT_MASK) >> APIC_EXTF_ELVT_SHIFT; - count = min(count, APIC_ELVT_MAX + 1); - return (count); -} - /* * Dump contents of local APIC registers */ @@ -702,9 +718,11 @@ native_lapic_dump(const char* str) extf = amd_read_ext_features(); if (extf != 0) { printf(" AMD ext features: 0x%08x\n", extf); + extf = lapic_read32(LAPIC_EXT_CTRL); + printf(" AMD ext control: 0x%08x\n", extf); elvt_count = amd_read_elvt_count(); for (i = 0; i < elvt_count; i++) - printf(" AMD elvt%d: 0x%08x\n", i, + printf(" AMD elvt%d: 0x%08x\n", i, lapic_read32(LAPIC_EXT_LVT0 + i)); } } @@ -1022,9 +1040,15 @@ lapic_enable(void) value = lapic_read32(LAPIC_SVR); value &= ~(APIC_SVR_VECTOR | APIC_SVR_FOCUS); value |= APIC_SVR_FEN | APIC_SVR_SWEN | APIC_SPURIOUS_INT; - if (lapic_eoi_suppression) + if (lapic_eoi_suppression == INTEL_SEOI) value |= APIC_SVR_EOI_SUPPRESSION; lapic_write32(LAPIC_SVR, value); + + if (lapic_eoi_suppression == AMD_SEOI) { + value = lapic_read32(LAPIC_EXT_CTRL); + value |= APIC_EXTF_SEIO_CAP; + lapic_write32(LAPIC_EXT_CTRL, value); + } } /* Reset the local APIC on the BSP during resume. */ @@ -1227,11 +1251,14 @@ lapic_set_tpr(u_int vector) #endif } -static void -native_lapic_eoi(void) +void +native_lapic_eoi(u_int vector) { - lapic_write32_nofence(LAPIC_EOI, 0); + if (lapic_eoi_suppression == AMD_SEOI) + lapic_write32(LAPIC_EXT_SEOI, vector); + else + lapic_write32_nofence(LAPIC_EOI, 0); } void @@ -1252,7 +1279,7 @@ lapic_handle_timer(struct trapframe *fra struct thread *td; /* Send EOI first thing. */ - lapic_eoi(); + lapic_eoi(APIC_TIMER_INT); #if defined(SMP) && !defined(SCHED_ULE) /* @@ -1373,7 +1400,7 @@ void lapic_handle_cmc(void) { - lapic_eoi(); + lapic_eoi(APIC_CMC_INT); cmc_intr(); } @@ -1447,7 +1474,7 @@ lapic_handle_error(void) esr = lapic_read32(LAPIC_ESR); printf("CPU%d: local APIC error 0x%x\n", PCPU_GET(cpuid), esr); - lapic_eoi(); + lapic_eoi(APIC_ERROR_INT); } static u_int Modified: head/sys/x86/x86/msi.c ============================================================================== --- head/sys/x86/x86/msi.c Sat Mar 25 15:57:47 2017 (r315958) +++ head/sys/x86/x86/msi.c Sat Mar 25 18:45:09 2017 (r315959) @@ -177,16 +177,18 @@ msi_enable_source(struct intsrc *isrc) static void msi_disable_source(struct intsrc *isrc, int eoi) { + struct msi_intsrc *msi = (struct msi_intsrc *)isrc; if (eoi == PIC_EOI) - lapic_eoi(); + lapic_eoi(msi->msi_vector); } static void msi_eoi_source(struct intsrc *isrc) { + struct msi_intsrc *msi = (struct msi_intsrc *)isrc; - lapic_eoi(); + lapic_eoi(msi->msi_vector); } static void Modified: head/sys/x86/xen/xen_apic.c ============================================================================== --- head/sys/x86/xen/xen_apic.c Sat Mar 25 15:57:47 2017 (r315958) +++ head/sys/x86/xen/xen_apic.c Sat Mar 25 18:45:09 2017 (r315959) @@ -147,7 +147,7 @@ xen_pv_lapic_is_x2apic(void) } static void -xen_pv_lapic_eoi(void) +xen_pv_lapic_eoi(u_int vector) { XEN_APIC_UNSUPPORTED; _______________________________________________ svn-src-all@freebsd.org mailing list https://lists.freebsd.org/mailman/listinfo/svn-src-all To unsubscribe, send any mail to "svn-src-all-unsubscr...@freebsd.org"
