Author: jeff Date: Fri Jan 12 23:34:16 2018 New Revision: 327901 URL: https://svnweb.freebsd.org/changeset/base/327901
Log: Add support for NUMA domains to bus dma tags. This causes all memory allocated with a tag to come from the specified domain if it meets the other constraints provided by the tag. Automatically create a tag at the root of each bus specifying the domain local to that bus if available. Reviewed by: jhb, kib Tested by: pho Sponsored by: Netflix, Dell/EMC Isilon Differential Revision: https://reviews.freebsd.org/D13545 Modified: head/sys/arm/arm/busdma_machdep-v4.c head/sys/arm/arm/busdma_machdep-v6.c head/sys/arm64/arm64/busdma_machdep.c head/sys/dev/acpica/acpi_pcib_acpi.c head/sys/mips/mips/busdma_machdep.c head/sys/powerpc/powerpc/busdma_machdep.c head/sys/sys/bus_dma.h head/sys/vm/vm_phys.c head/sys/vm/vm_phys.h head/sys/x86/include/busdma_impl.h head/sys/x86/iommu/busdma_dmar.c head/sys/x86/x86/busdma_bounce.c head/sys/x86/x86/busdma_machdep.c Modified: head/sys/arm/arm/busdma_machdep-v4.c ============================================================================== --- head/sys/arm/arm/busdma_machdep-v4.c Fri Jan 12 23:25:05 2018 (r327900) +++ head/sys/arm/arm/busdma_machdep-v4.c Fri Jan 12 23:34:16 2018 (r327901) @@ -500,6 +500,13 @@ bus_dma_tag_create(bus_dma_tag_t parent, bus_size_t al } int +bus_dma_tag_set_domain(bus_dma_tag_t dmat, int domain) +{ + + return (0); +} + +int bus_dma_tag_destroy(bus_dma_tag_t dmat) { bus_dma_tag_t dmat_copy; Modified: head/sys/arm/arm/busdma_machdep-v6.c ============================================================================== --- head/sys/arm/arm/busdma_machdep-v6.c Fri Jan 12 23:25:05 2018 (r327900) +++ head/sys/arm/arm/busdma_machdep-v6.c Fri Jan 12 23:34:16 2018 (r327901) @@ -563,6 +563,13 @@ bus_dma_tag_create(bus_dma_tag_t parent, bus_size_t al } int +bus_dma_tag_set_domain(bus_dma_tag_t dmat, int domain) +{ + + return (0); +} + +int bus_dma_tag_destroy(bus_dma_tag_t dmat) { bus_dma_tag_t dmat_copy; Modified: head/sys/arm64/arm64/busdma_machdep.c ============================================================================== --- head/sys/arm64/arm64/busdma_machdep.c Fri Jan 12 23:25:05 2018 (r327900) +++ head/sys/arm64/arm64/busdma_machdep.c Fri Jan 12 23:34:16 2018 (r327901) @@ -223,3 +223,9 @@ bus_dma_tag_destroy(bus_dma_tag_t dmat) return (tc->impl->tag_destroy(dmat)); } +int +bus_dma_tag_set_domain(bus_dma_tag_t dmat, int domain) +{ + + return (0); +} Modified: head/sys/dev/acpica/acpi_pcib_acpi.c ============================================================================== --- head/sys/dev/acpica/acpi_pcib_acpi.c Fri Jan 12 23:25:05 2018 (r327900) +++ head/sys/dev/acpica/acpi_pcib_acpi.c Fri Jan 12 23:34:16 2018 (r327901) @@ -60,6 +60,7 @@ ACPI_MODULE_NAME("PCI_ACPI") struct acpi_hpcib_softc { device_t ap_dev; ACPI_HANDLE ap_handle; + bus_dma_tag_t ap_dma_tag; int ap_flags; uint32_t ap_osc_ctl; @@ -108,6 +109,7 @@ static int acpi_pcib_acpi_release_resource(device_t d #endif static int acpi_pcib_request_feature(device_t pcib, device_t dev, enum pci_feature feature); +static bus_dma_tag_t acpi_pcib_get_dma_tag(device_t bus, device_t child); static device_method_t acpi_pcib_acpi_methods[] = { /* Device interface */ @@ -136,6 +138,7 @@ static device_method_t acpi_pcib_acpi_methods[] = { DEVMETHOD(bus_setup_intr, bus_generic_setup_intr), DEVMETHOD(bus_teardown_intr, bus_generic_teardown_intr), DEVMETHOD(bus_get_cpus, acpi_pcib_get_cpus), + DEVMETHOD(bus_get_dma_tag, acpi_pcib_get_dma_tag), /* pcib interface */ DEVMETHOD(pcib_maxslots, pcib_maxslots), @@ -366,6 +369,7 @@ acpi_pcib_acpi_attach(device_t dev) rman_res_t start; int rid; #endif + int error, domain; uint8_t busno; ACPI_FUNCTION_TRACE((char *)(uintptr_t)__func__); @@ -537,15 +541,33 @@ acpi_pcib_acpi_attach(device_t dev) acpi_pcib_fetch_prt(dev, &sc->ap_prt); + error = bus_dma_tag_create(bus_get_dma_tag(dev), 1, + 0, BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, + NULL, NULL, BUS_SPACE_MAXSIZE, BUS_SPACE_UNRESTRICTED, + BUS_SPACE_MAXSIZE, 0, NULL, NULL, &sc->ap_dma_tag); + if (error != 0) + goto errout; + error = bus_get_domain(dev, &domain); + if (error == 0) + error = bus_dma_tag_set_domain(sc->ap_dma_tag, domain); + /* Don't fail to attach if the domain can't be queried or set. */ + error = 0; + bus_generic_probe(dev); if (device_add_child(dev, "pci", -1) == NULL) { - device_printf(device_get_parent(dev), "couldn't attach pci bus\n"); -#if defined(NEW_PCIB) && defined(PCI_RES_BUS) - pcib_host_res_free(dev, &sc->ap_host_res); -#endif - return (ENXIO); + bus_dma_tag_destroy(sc->ap_dma_tag); + sc->ap_dma_tag = NULL; + error = ENXIO; + goto errout; } return (bus_generic_attach(dev)); + +errout: + device_printf(device_get_parent(dev), "couldn't attach pci bus\n"); +#if defined(NEW_PCIB) && defined(PCI_RES_BUS) + pcib_host_res_free(dev, &sc->ap_host_res); +#endif + return (error); } /* @@ -752,4 +774,14 @@ acpi_pcib_request_feature(device_t pcib, device_t dev, } return (acpi_pcib_osc(sc, osc_ctl)); +} + +static bus_dma_tag_t +acpi_pcib_get_dma_tag(device_t bus, device_t child) +{ + struct acpi_hpcib_softc *sc; + + sc = device_get_softc(bus); + + return (sc->ap_dma_tag); } Modified: head/sys/mips/mips/busdma_machdep.c ============================================================================== --- head/sys/mips/mips/busdma_machdep.c Fri Jan 12 23:25:05 2018 (r327900) +++ head/sys/mips/mips/busdma_machdep.c Fri Jan 12 23:34:16 2018 (r327901) @@ -481,6 +481,13 @@ bus_dma_tag_create(bus_dma_tag_t parent, bus_size_t al } int +bus_dma_tag_set_domain(bus_dma_tag_t dmat, int domain) +{ + + return (0); +} + +int bus_dma_tag_destroy(bus_dma_tag_t dmat) { #ifdef KTR Modified: head/sys/powerpc/powerpc/busdma_machdep.c ============================================================================== --- head/sys/powerpc/powerpc/busdma_machdep.c Fri Jan 12 23:25:05 2018 (r327900) +++ head/sys/powerpc/powerpc/busdma_machdep.c Fri Jan 12 23:34:16 2018 (r327901) @@ -341,6 +341,13 @@ bus_dma_tag_create(bus_dma_tag_t parent, bus_size_t al } int +bus_dma_tag_set_domain(bus_dma_tag_t dmat, int domain) +{ + + return (0); +} + +int bus_dma_tag_destroy(bus_dma_tag_t dmat) { bus_dma_tag_t dmat_copy; Modified: head/sys/sys/bus_dma.h ============================================================================== --- head/sys/sys/bus_dma.h Fri Jan 12 23:25:05 2018 (r327900) +++ head/sys/sys/bus_dma.h Fri Jan 12 23:34:16 2018 (r327901) @@ -176,6 +176,14 @@ int bus_dma_tag_create(bus_dma_tag_t parent, bus_size_ bus_size_t maxsegsz, int flags, bus_dma_lock_t *lockfunc, void *lockfuncarg, bus_dma_tag_t *dmat); +/* + * Set the memory domain to be used for allocations. + * + * Automatic for PCI devices. Must be set prior to creating maps or + * allocating memory. + */ +int bus_dma_tag_set_domain(bus_dma_tag_t dmat, int domain); + int bus_dma_tag_destroy(bus_dma_tag_t dmat); /* Modified: head/sys/vm/vm_phys.c ============================================================================== --- head/sys/vm/vm_phys.c Fri Jan 12 23:25:05 2018 (r327900) +++ head/sys/vm/vm_phys.c Fri Jan 12 23:34:16 2018 (r327901) @@ -198,20 +198,32 @@ vm_phys_fictitious_cmp(struct vm_phys_fictitious_seg * (uintmax_t)p1->end, (uintmax_t)p2->start, (uintmax_t)p2->end); } -boolean_t -vm_phys_domain_intersects(long mask, vm_paddr_t low, vm_paddr_t high) +int +vm_phys_domain_match(int prefer, vm_paddr_t low, vm_paddr_t high) { - struct vm_phys_seg *s; - int idx; +#ifdef VM_NUMA_ALLOC + domainset_t mask; + int i; - while ((idx = ffsl(mask)) != 0) { - idx--; /* ffsl counts from 1 */ - mask &= ~(1UL << idx); - s = &vm_phys_segs[idx]; - if (low < s->end && high > s->start) - return (TRUE); - } - return (FALSE); + if (vm_ndomains == 1 || mem_affinity == NULL) + return (0); + + DOMAINSET_ZERO(&mask); + /* + * Check for any memory that overlaps low, high. + */ + for (i = 0; mem_affinity[i].end != 0; i++) + if (mem_affinity[i].start <= high && + mem_affinity[i].end >= low) + DOMAINSET_SET(mem_affinity[i].domain, &mask); + if (prefer != -1 && DOMAINSET_ISSET(prefer, &mask)) + return (prefer); + if (DOMAINSET_EMPTY(&mask)) + panic("vm_phys_domain_match: Impossible constraint"); + return (DOMAINSET_FFS(&mask) - 1); +#else + return (0); +#endif } /* Modified: head/sys/vm/vm_phys.h ============================================================================== --- head/sys/vm/vm_phys.h Fri Jan 12 23:25:05 2018 (r327900) +++ head/sys/vm/vm_phys.h Fri Jan 12 23:34:16 2018 (r327901) @@ -77,7 +77,7 @@ vm_page_t vm_phys_alloc_contig(int domain, u_long npag vm_page_t vm_phys_alloc_freelist_pages(int domain, int freelist, int pool, int order); vm_page_t vm_phys_alloc_pages(int domain, int pool, int order); -boolean_t vm_phys_domain_intersects(long mask, vm_paddr_t low, vm_paddr_t high); +int vm_phys_domain_match(int prefer, vm_paddr_t low, vm_paddr_t high); int vm_phys_fictitious_reg_range(vm_paddr_t start, vm_paddr_t end, vm_memattr_t memattr); void vm_phys_fictitious_unreg_range(vm_paddr_t start, vm_paddr_t end); Modified: head/sys/x86/include/busdma_impl.h ============================================================================== --- head/sys/x86/include/busdma_impl.h Fri Jan 12 23:25:05 2018 (r327900) +++ head/sys/x86/include/busdma_impl.h Fri Jan 12 23:34:16 2018 (r327901) @@ -50,6 +50,7 @@ struct bus_dma_tag_common { bus_dma_lock_t *lockfunc; void *lockfuncarg; int ref_count; + int domain; }; struct bus_dma_impl { @@ -60,6 +61,7 @@ struct bus_dma_impl { bus_size_t maxsegsz, int flags, bus_dma_lock_t *lockfunc, void *lockfuncarg, bus_dma_tag_t *dmat); int (*tag_destroy)(bus_dma_tag_t dmat); + int (*tag_set_domain)(bus_dma_tag_t); int (*map_create)(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp); int (*map_destroy)(bus_dma_tag_t dmat, bus_dmamap_t map); int (*mem_alloc)(bus_dma_tag_t dmat, void** vaddr, int flags, Modified: head/sys/x86/iommu/busdma_dmar.c ============================================================================== --- head/sys/x86/iommu/busdma_dmar.c Fri Jan 12 23:25:05 2018 (r327900) +++ head/sys/x86/iommu/busdma_dmar.c Fri Jan 12 23:34:16 2018 (r327901) @@ -326,6 +326,13 @@ out: } static int +dmar_bus_dma_tag_set_domain(bus_dma_tag_t dmat) +{ + + return (0); +} + +static int dmar_bus_dma_tag_destroy(bus_dma_tag_t dmat1) { struct bus_dma_tag_dmar *dmat, *dmat_copy, *parent; @@ -345,7 +352,7 @@ dmar_bus_dma_tag_destroy(bus_dma_tag_t dmat1) 1) { if (dmat == &dmat->ctx->ctx_tag) dmar_free_ctx(dmat->ctx); - free(dmat->segments, M_DMAR_DMAMAP); + free_domain(dmat->segments, M_DMAR_DMAMAP); free(dmat, M_DEVBUF); dmat = parent; } else @@ -366,16 +373,18 @@ dmar_bus_dmamap_create(bus_dma_tag_t dmat, int flags, WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, "%s", __func__); tag = (struct bus_dma_tag_dmar *)dmat; - map = malloc(sizeof(*map), M_DMAR_DMAMAP, M_NOWAIT | M_ZERO); + map = malloc_domain(sizeof(*map), M_DMAR_DMAMAP, + tag->common.domain, M_NOWAIT | M_ZERO); if (map == NULL) { *mapp = NULL; return (ENOMEM); } if (tag->segments == NULL) { - tag->segments = malloc(sizeof(bus_dma_segment_t) * - tag->common.nsegments, M_DMAR_DMAMAP, M_NOWAIT); + tag->segments = malloc_domain(sizeof(bus_dma_segment_t) * + tag->common.nsegments, M_DMAR_DMAMAP, + tag->common.domain, M_NOWAIT); if (tag->segments == NULL) { - free(map, M_DMAR_DMAMAP); + free_domain(map, M_DMAR_DMAMAP); *mapp = NULL; return (ENOMEM); } @@ -407,7 +416,7 @@ dmar_bus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap return (EBUSY); } DMAR_DOMAIN_UNLOCK(domain); - free(map, M_DMAR_DMAMAP); + free_domain(map, M_DMAR_DMAMAP); } tag->map_count--; return (0); @@ -438,10 +447,11 @@ dmar_bus_dmamem_alloc(bus_dma_tag_t dmat, void** vaddr if (tag->common.maxsize < PAGE_SIZE && tag->common.alignment <= tag->common.maxsize && attr == VM_MEMATTR_DEFAULT) { - *vaddr = malloc(tag->common.maxsize, M_DEVBUF, mflags); + *vaddr = malloc_domain(tag->common.maxsize, M_DEVBUF, + tag->common.domain, mflags); map->flags |= BUS_DMAMAP_DMAR_MALLOC; } else { - *vaddr = (void *)kmem_alloc_attr(kernel_arena, + *vaddr = (void *)kmem_alloc_attr_domain(tag->common.domain, tag->common.maxsize, mflags, 0ul, BUS_SPACE_MAXADDR, attr); map->flags |= BUS_DMAMAP_DMAR_KMEM_ALLOC; @@ -464,7 +474,7 @@ dmar_bus_dmamem_free(bus_dma_tag_t dmat, void *vaddr, map = (struct bus_dmamap_dmar *)map1; if ((map->flags & BUS_DMAMAP_DMAR_MALLOC) != 0) { - free(vaddr, M_DEVBUF); + free_domain(vaddr, M_DEVBUF); map->flags &= ~BUS_DMAMAP_DMAR_MALLOC; } else { KASSERT((map->flags & BUS_DMAMAP_DMAR_KMEM_ALLOC) != 0, @@ -832,6 +842,7 @@ dmar_bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t struct bus_dma_impl bus_dma_dmar_impl = { .tag_create = dmar_bus_dma_tag_create, .tag_destroy = dmar_bus_dma_tag_destroy, + .tag_set_domain = dmar_bus_dma_tag_set_domain, .map_create = dmar_bus_dmamap_create, .map_destroy = dmar_bus_dmamap_destroy, .mem_alloc = dmar_bus_dmamem_alloc, @@ -842,7 +853,7 @@ struct bus_dma_impl bus_dma_dmar_impl = { .map_waitok = dmar_bus_dmamap_waitok, .map_complete = dmar_bus_dmamap_complete, .map_unload = dmar_bus_dmamap_unload, - .map_sync = dmar_bus_dmamap_sync + .map_sync = dmar_bus_dmamap_sync, }; static void Modified: head/sys/x86/x86/busdma_bounce.c ============================================================================== --- head/sys/x86/x86/busdma_bounce.c Fri Jan 12 23:25:05 2018 (r327900) +++ head/sys/x86/x86/busdma_bounce.c Fri Jan 12 23:34:16 2018 (r327901) @@ -99,6 +99,7 @@ struct bounce_zone { int total_bounced; int total_deferred; int map_count; + int domain; bus_size_t alignment; bus_addr_t lowaddr; char zoneid[8]; @@ -150,6 +151,32 @@ static void _bus_dmamap_count_phys(bus_dma_tag_t dmat, static int _bus_dmamap_reserve_pages(bus_dma_tag_t dmat, bus_dmamap_t map, int flags); +static int +bounce_bus_dma_zone_setup(bus_dma_tag_t dmat) +{ + struct bounce_zone *bz; + int error; + + /* Must bounce */ + if ((error = alloc_bounce_zone(dmat)) != 0) + return (error); + bz = dmat->bounce_zone; + + if (ptoa(bz->total_bpages) < dmat->common.maxsize) { + int pages; + + pages = atop(dmat->common.maxsize) - bz->total_bpages; + + /* Add pages to our bounce pool */ + if (alloc_bounce_pages(dmat, pages) < pages) + return (ENOMEM); + } + /* Performed initial allocation */ + dmat->bounce_flags |= BUS_DMA_MIN_ALLOC_COMP; + + return (0); +} + /* * Allocate a device specific dma_tag. */ @@ -184,28 +211,9 @@ bounce_bus_dma_tag_create(bus_dma_tag_t parent, bus_si newtag->bounce_flags |= BUS_DMA_COULD_BOUNCE; if (((newtag->bounce_flags & BUS_DMA_COULD_BOUNCE) != 0) && - (flags & BUS_DMA_ALLOCNOW) != 0) { - struct bounce_zone *bz; - - /* Must bounce */ - if ((error = alloc_bounce_zone(newtag)) != 0) { - free(newtag, M_DEVBUF); - return (error); - } - bz = newtag->bounce_zone; - - if (ptoa(bz->total_bpages) < maxsize) { - int pages; - - pages = atop(maxsize) - bz->total_bpages; - - /* Add pages to our bounce pool */ - if (alloc_bounce_pages(newtag, pages) < pages) - error = ENOMEM; - } - /* Performed initial allocation */ - newtag->bounce_flags |= BUS_DMA_MIN_ALLOC_COMP; - } else + (flags & BUS_DMA_ALLOCNOW) != 0) + error = bounce_bus_dma_zone_setup(newtag); + else error = 0; if (error != 0) @@ -218,7 +226,24 @@ bounce_bus_dma_tag_create(bus_dma_tag_t parent, bus_si return (error); } +/* + * Update the domain for the tag. We may need to reallocate the zone and + * bounce pages. + */ static int +bounce_bus_dma_tag_set_domain(bus_dma_tag_t dmat) +{ + + KASSERT(dmat->map_count == 0, + ("bounce_bus_dma_tag_set_domain: Domain set after use.\n")); + if ((dmat->bounce_flags & BUS_DMA_COULD_BOUNCE) == 0 || + dmat->bounce_zone == NULL) + return (0); + dmat->bounce_flags &= ~BUS_DMA_MIN_ALLOC_COMP; + return (bounce_bus_dma_zone_setup(dmat)); +} + +static int bounce_bus_dma_tag_destroy(bus_dma_tag_t dmat) { bus_dma_tag_t dmat_copy, parent; @@ -237,7 +262,7 @@ bounce_bus_dma_tag_destroy(bus_dma_tag_t dmat) atomic_subtract_int(&dmat->common.ref_count, 1); if (dmat->common.ref_count == 0) { if (dmat->segments != NULL) - free(dmat->segments, M_DEVBUF); + free_domain(dmat->segments, M_DEVBUF); free(dmat, M_DEVBUF); /* * Last reference count, so @@ -269,9 +294,9 @@ bounce_bus_dmamap_create(bus_dma_tag_t dmat, int flags error = 0; if (dmat->segments == NULL) { - dmat->segments = (bus_dma_segment_t *)malloc( + dmat->segments = (bus_dma_segment_t *)malloc_domain( sizeof(bus_dma_segment_t) * dmat->common.nsegments, - M_DEVBUF, M_NOWAIT); + M_DEVBUF, dmat->common.domain, M_NOWAIT); if (dmat->segments == NULL) { CTR3(KTR_BUSDMA, "%s: tag %p error %d", __func__, dmat, ENOMEM); @@ -292,8 +317,8 @@ bounce_bus_dmamap_create(bus_dma_tag_t dmat, int flags } bz = dmat->bounce_zone; - *mapp = (bus_dmamap_t)malloc(sizeof(**mapp), M_DEVBUF, - M_NOWAIT | M_ZERO); + *mapp = (bus_dmamap_t)malloc_domain(sizeof(**mapp), M_DEVBUF, + dmat->common.domain, M_NOWAIT | M_ZERO); if (*mapp == NULL) { CTR3(KTR_BUSDMA, "%s: tag %p error %d", __func__, dmat, ENOMEM); @@ -355,7 +380,7 @@ bounce_bus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmam } if (dmat->bounce_zone) dmat->bounce_zone->map_count--; - free(map, M_DEVBUF); + free_domain(map, M_DEVBUF); } dmat->map_count--; CTR2(KTR_BUSDMA, "%s: tag %p error 0", __func__, dmat); @@ -386,9 +411,9 @@ bounce_bus_dmamem_alloc(bus_dma_tag_t dmat, void** vad *mapp = NULL; if (dmat->segments == NULL) { - dmat->segments = (bus_dma_segment_t *)malloc( + dmat->segments = (bus_dma_segment_t *)malloc_domain( sizeof(bus_dma_segment_t) * dmat->common.nsegments, - M_DEVBUF, mflags); + M_DEVBUF, dmat->common.domain, mflags); if (dmat->segments == NULL) { CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", __func__, dmat, dmat->common.flags, ENOMEM); @@ -427,18 +452,19 @@ bounce_bus_dmamem_alloc(bus_dma_tag_t dmat, void** vad (dmat->common.alignment <= dmat->common.maxsize) && dmat->common.lowaddr >= ptoa((vm_paddr_t)Maxmem) && attr == VM_MEMATTR_DEFAULT) { - *vaddr = malloc(dmat->common.maxsize, M_DEVBUF, mflags); + *vaddr = malloc_domain(dmat->common.maxsize, M_DEVBUF, + dmat->common.domain, mflags); } else if (dmat->common.nsegments >= howmany(dmat->common.maxsize, MIN(dmat->common.maxsegsz, PAGE_SIZE)) && dmat->common.alignment <= PAGE_SIZE && (dmat->common.boundary % PAGE_SIZE) == 0) { /* Page-based multi-segment allocations allowed */ - *vaddr = (void *)kmem_alloc_attr(kernel_arena, + *vaddr = (void *)kmem_alloc_attr_domain(dmat->common.domain, dmat->common.maxsize, mflags, 0ul, dmat->common.lowaddr, attr); dmat->bounce_flags |= BUS_DMA_KMEM_ALLOC; } else { - *vaddr = (void *)kmem_alloc_contig(kernel_arena, + *vaddr = (void *)kmem_alloc_contig_domain(dmat->common.domain, dmat->common.maxsize, mflags, 0ul, dmat->common.lowaddr, dmat->common.alignment != 0 ? dmat->common.alignment : 1ul, dmat->common.boundary, attr); @@ -471,7 +497,7 @@ bounce_bus_dmamem_free(bus_dma_tag_t dmat, void *vaddr if (map != NULL) panic("bus_dmamem_free: Invalid map freed\n"); if ((dmat->bounce_flags & BUS_DMA_KMEM_ALLOC) == 0) - free(vaddr, M_DEVBUF); + free_domain(vaddr, M_DEVBUF); else kmem_free(kernel_arena, (vm_offset_t)vaddr, dmat->common.maxsize); @@ -1041,7 +1067,8 @@ alloc_bounce_zone(bus_dma_tag_t dmat) /* Check to see if we already have a suitable zone */ STAILQ_FOREACH(bz, &bounce_zone_list, links) { if ((dmat->common.alignment <= bz->alignment) && - (dmat->common.lowaddr >= bz->lowaddr)) { + (dmat->common.lowaddr >= bz->lowaddr) && + (dmat->common.domain == bz->domain)) { dmat->bounce_zone = bz; return (0); } @@ -1058,6 +1085,7 @@ alloc_bounce_zone(bus_dma_tag_t dmat) bz->lowaddr = dmat->common.lowaddr; bz->alignment = MAX(dmat->common.alignment, PAGE_SIZE); bz->map_count = 0; + bz->domain = dmat->common.domain; snprintf(bz->zoneid, 8, "zone%d", busdma_zonecount); busdma_zonecount++; snprintf(bz->lowaddrid, 18, "%#jx", (uintmax_t)bz->lowaddr); @@ -1103,6 +1131,10 @@ alloc_bounce_zone(bus_dma_tag_t dmat) SYSCTL_ADD_UAUTO(busdma_sysctl_tree(bz), SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, "alignment", CTLFLAG_RD, &bz->alignment, ""); + SYSCTL_ADD_INT(busdma_sysctl_tree(bz), + SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, + "domain", CTLFLAG_RD, &bz->domain, 0, + "memory domain"); return (0); } @@ -1118,18 +1150,16 @@ alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages) while (numpages > 0) { struct bounce_page *bpage; - bpage = (struct bounce_page *)malloc(sizeof(*bpage), M_DEVBUF, - M_NOWAIT | M_ZERO); + bpage = (struct bounce_page *)malloc_domain(sizeof(*bpage), + M_DEVBUF, dmat->common.domain, M_NOWAIT | M_ZERO); if (bpage == NULL) break; - bpage->vaddr = (vm_offset_t)contigmalloc(PAGE_SIZE, M_DEVBUF, - M_NOWAIT, 0ul, - bz->lowaddr, - PAGE_SIZE, - 0); + bpage->vaddr = (vm_offset_t)contigmalloc_domain(PAGE_SIZE, + M_DEVBUF, dmat->common.domain, M_NOWAIT, 0ul, + bz->lowaddr, PAGE_SIZE, 0); if (bpage->vaddr == 0) { - free(bpage, M_DEVBUF); + free_domain(bpage, M_DEVBUF); break; } bpage->busaddr = pmap_kextract(bpage->vaddr); @@ -1271,6 +1301,7 @@ busdma_swi(void) struct bus_dma_impl bus_dma_bounce_impl = { .tag_create = bounce_bus_dma_tag_create, .tag_destroy = bounce_bus_dma_tag_destroy, + .tag_set_domain = bounce_bus_dma_tag_set_domain, .map_create = bounce_bus_dmamap_create, .map_destroy = bounce_bus_dmamap_destroy, .mem_alloc = bounce_bus_dmamem_alloc, @@ -1281,5 +1312,5 @@ struct bus_dma_impl bus_dma_bounce_impl = { .map_waitok = bounce_bus_dmamap_waitok, .map_complete = bounce_bus_dmamap_complete, .map_unload = bounce_bus_dmamap_unload, - .map_sync = bounce_bus_dmamap_sync + .map_sync = bounce_bus_dmamap_sync, }; Modified: head/sys/x86/x86/busdma_machdep.c ============================================================================== --- head/sys/x86/x86/busdma_machdep.c Fri Jan 12 23:25:05 2018 (r327900) +++ head/sys/x86/x86/busdma_machdep.c Fri Jan 12 23:34:16 2018 (r327901) @@ -43,8 +43,12 @@ __FBSDID("$FreeBSD$"); #include <sys/memdesc.h> #include <sys/mutex.h> #include <sys/uio.h> +#include <sys/vmmeter.h> #include <vm/vm.h> #include <vm/vm_extern.h> +#include <vm/vm_param.h> +#include <vm/vm_page.h> +#include <vm/vm_phys.h> #include <vm/pmap.h> #include <machine/bus.h> #include <x86/include/busdma_impl.h> @@ -180,10 +184,27 @@ common_bus_dma_tag_create(struct bus_dma_tag_common *p common->filterarg = parent->filterarg; common->parent = parent->parent; } + common->domain = parent->domain; atomic_add_int(&parent->ref_count, 1); } + common->domain = vm_phys_domain_match(common->domain, 0ul, + common->lowaddr); *dmat = common; return (0); +} + +int +bus_dma_tag_set_domain(bus_dma_tag_t dmat, int domain) +{ + struct bus_dma_tag_common *tc; + + tc = (struct bus_dma_tag_common *)dmat; + domain = vm_phys_domain_match(domain, 0ul, tc->lowaddr); + /* Only call the callback if it changes. */ + if (domain == tc->domain) + return (0); + tc->domain = domain; + return (tc->impl->tag_set_domain(dmat)); } /* _______________________________________________ svn-src-head@freebsd.org mailing list https://lists.freebsd.org/mailman/listinfo/svn-src-head To unsubscribe, send any mail to "svn-src-head-unsubscr...@freebsd.org"
