From b1d2dc009dece4cd7e629419b52266ba51960a6b Mon Sep 17 00:00:00 2001 From: Nicolin Chen Date: Thu, 23 May 2019 21:06:32 -0700 Subject: dma-contiguous: add dma_{alloc,free}_contiguous() helpers Both dma_alloc_from_contiguous() and dma_release_from_contiguous() are very simply implemented, but requiring callers to pass certain parameters like count and align, and taking a boolean parameter to check __GFP_NOWARN in the allocation flags. So every function call duplicates similar work: unsigned long order = get_order(size); size_t count = size >> PAGE_SHIFT; page = dma_alloc_from_contiguous(dev, count, order, gfp & __GFP_NOWARN); [...] dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT); Additionally, as CMA can be used only in the context which permits sleeping, most of callers do a gfpflags_allow_blocking() check and a corresponding fallback allocation of normal pages upon any false result: if (gfpflags_allow_blocking(flag)) page = dma_alloc_from_contiguous(); if (!page) page = alloc_pages(); [...] if (!dma_release_from_contiguous(dev, page, count)) __free_pages(page, get_order(size)); So this patch simplifies those function calls by abstracting these operations into the two new functions: dma_{alloc,free}_contiguous. As some callers of dma_{alloc,release}_from_contiguous() might be complicated, this patch just implements these two new functions to kernel/dma/direct.c only as an initial step. Suggested-by: Christoph Hellwig Signed-off-by: Nicolin Chen Tested-by: dann frazier Signed-off-by: Christoph Hellwig --- include/linux/dma-contiguous.h | 13 +++++++++++++ 1 file changed, 13 insertions(+) (limited to 'include') diff --git a/include/linux/dma-contiguous.h b/include/linux/dma-contiguous.h index 6665fa03c0d1..428f3b7b1c42 100644 --- a/include/linux/dma-contiguous.h +++ b/include/linux/dma-contiguous.h @@ -111,6 +111,8 @@ struct page *dma_alloc_from_contiguous(struct device *dev, size_t count, unsigned int order, bool no_warn); bool dma_release_from_contiguous(struct device *dev, struct page *pages, int count); +struct page *dma_alloc_contiguous(struct device *dev, size_t size, gfp_t gfp); +void dma_free_contiguous(struct device *dev, struct page *page, size_t size); #else @@ -153,6 +155,17 @@ bool dma_release_from_contiguous(struct device *dev, struct page *pages, return false; } +static inline struct page *dma_alloc_contiguous(struct device *dev, size_t size, + gfp_t gfp) +{ + return NULL; +} + +static inline void dma_free_contiguous(struct device *dev, struct page *page, + size_t size) +{ +} + #endif #endif -- cgit v1.2.3 From dd3dcede9fa0a0b661ac1f24843f4a1b1317fdb6 Mon Sep 17 00:00:00 2001 From: Nicolin Chen Date: Wed, 29 May 2019 17:54:25 -0700 Subject: dma-contiguous: fix !CONFIG_DMA_CMA version of dma_{alloc, free}_contiguous() Commit fdaeec198ada ("dma-contiguous: add dma_{alloc,free}_contiguous() helpers") adds a pair of new helper functions so as to abstract code in the dma-direct (and other places in the future), however it breaks QEMU boot feature using x86_64 defconfig. That's because x86_64 defconfig has CONFIG_DMA_CMA=n so those two newly introduced helper functions are empty in their !CONFIG_DMA_CMA version, while previously the platform independent dma-direct code had fallback alloc_pages_node() and __free_pages(). So this patch fixes it by adding alloc_pages_node() and __free_pages() in the !CONFIG_DMA_CMA version of the two helper functions. Tested with below QEMU command: qemu-system-x86_64 -m 512m \ -drive file=images/x86_64/rootfs.ext4,format=raw,if=ide \ -append 'console=ttyS0 root=/dev/sda' -nographic \ -kernel arch/x86_64/boot/bzImage with the rootfs from the below link: https://github.com/ClangBuiltLinux/continuous-integration/raw/master/images/x86_64/rootfs.ext4 Fixes: fdaeec198ada ("dma-contiguous: add dma_{alloc,free}_contiguous() helpers") Reported-by: Nathan Chancellor Signed-off-by: Nicolin Chen Signed-off-by: Christoph Hellwig --- include/linux/dma-contiguous.h | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'include') diff --git a/include/linux/dma-contiguous.h b/include/linux/dma-contiguous.h index 428f3b7b1c42..c05d4e661489 100644 --- a/include/linux/dma-contiguous.h +++ b/include/linux/dma-contiguous.h @@ -50,6 +50,7 @@ #ifdef __KERNEL__ #include +#include struct cma; struct page; @@ -155,15 +156,20 @@ bool dma_release_from_contiguous(struct device *dev, struct page *pages, return false; } +/* Use fallback alloc() and free() when CONFIG_DMA_CMA=n */ static inline struct page *dma_alloc_contiguous(struct device *dev, size_t size, gfp_t gfp) { - return NULL; + int node = dev ? dev_to_node(dev) : NUMA_NO_NODE; + size_t align = get_order(PAGE_ALIGN(size)); + + return alloc_pages_node(node, gfp, align); } static inline void dma_free_contiguous(struct device *dev, struct page *page, size_t size) { + __free_pages(page, get_order(size)); } #endif -- cgit v1.2.3 From da83a722959a82733c3ca60030cc364ca2318c5a Mon Sep 17 00:00:00 2001 From: Fredrik Noring Date: Wed, 29 May 2019 13:28:39 +0300 Subject: lib/genalloc: add gen_pool_dma_zalloc() for zeroed DMA allocations gen_pool_dma_zalloc() is a zeroed memory variant of gen_pool_dma_alloc(). Also document the return values of both, and indicate NULL as a "%NULL" constant. Signed-off-by: Fredrik Noring Reviewed-by: Greg Kroah-Hartman Signed-off-by: Christoph Hellwig --- include/linux/genalloc.h | 1 + lib/genalloc.c | 29 ++++++++++++++++++++++++++++- 2 files changed, 29 insertions(+), 1 deletion(-) (limited to 'include') diff --git a/include/linux/genalloc.h b/include/linux/genalloc.h index dd0a452373e7..6c62eeca754f 100644 --- a/include/linux/genalloc.h +++ b/include/linux/genalloc.h @@ -121,6 +121,7 @@ extern unsigned long gen_pool_alloc_algo(struct gen_pool *, size_t, genpool_algo_t algo, void *data); extern void *gen_pool_dma_alloc(struct gen_pool *pool, size_t size, dma_addr_t *dma); +void *gen_pool_dma_zalloc(struct gen_pool *pool, size_t size, dma_addr_t *dma); extern void gen_pool_free(struct gen_pool *, unsigned long, size_t); extern void gen_pool_for_each_chunk(struct gen_pool *, void (*)(struct gen_pool *, struct gen_pool_chunk *, void *), void *); diff --git a/lib/genalloc.c b/lib/genalloc.c index 7e85d1e37a6e..5db43476a19b 100644 --- a/lib/genalloc.c +++ b/lib/genalloc.c @@ -337,12 +337,14 @@ EXPORT_SYMBOL(gen_pool_alloc_algo); * gen_pool_dma_alloc - allocate special memory from the pool for DMA usage * @pool: pool to allocate from * @size: number of bytes to allocate from the pool - * @dma: dma-view physical address return value. Use NULL if unneeded. + * @dma: dma-view physical address return value. Use %NULL if unneeded. * * Allocate the requested number of bytes from the specified pool. * Uses the pool allocation function (with first-fit algorithm by default). * Can not be used in NMI handler on architectures without * NMI-safe cmpxchg implementation. + * + * Return: virtual address of the allocated memory, or %NULL on failure */ void *gen_pool_dma_alloc(struct gen_pool *pool, size_t size, dma_addr_t *dma) { @@ -362,6 +364,31 @@ void *gen_pool_dma_alloc(struct gen_pool *pool, size_t size, dma_addr_t *dma) } EXPORT_SYMBOL(gen_pool_dma_alloc); +/** + * gen_pool_dma_zalloc - allocate special zeroed memory from the pool for + * DMA usage + * @pool: pool to allocate from + * @size: number of bytes to allocate from the pool + * @dma: dma-view physical address return value. Use %NULL if unneeded. + * + * Allocate the requested number of zeroed bytes from the specified pool. + * Uses the pool allocation function (with first-fit algorithm by default). + * Can not be used in NMI handler on architectures without + * NMI-safe cmpxchg implementation. + * + * Return: virtual address of the allocated zeroed memory, or %NULL on failure + */ +void *gen_pool_dma_zalloc(struct gen_pool *pool, size_t size, dma_addr_t *dma) +{ + void *vaddr = gen_pool_dma_alloc(pool, size, dma); + + if (vaddr) + memset(vaddr, 0, size); + + return vaddr; +} +EXPORT_SYMBOL(gen_pool_dma_zalloc); + /** * gen_pool_free - free allocated special memory back to the pool * @pool: pool to free to -- cgit v1.2.3 From b0310c2f09bbe8aebefb97ed67949a3a7092aca6 Mon Sep 17 00:00:00 2001 From: Laurentiu Tudor Date: Wed, 29 May 2019 13:28:40 +0300 Subject: USB: use genalloc for USB HCs with local memory For HCs that have local memory, replace the current DMA API usage with a genalloc generic allocator to manage the mappings for these devices. To help users, introduce a new HCD API, usb_hcd_setup_local_mem() that will setup up the genalloc backing up the device local memory. It will be used in subsequent patches. This is in preparation for dropping the existing "coherent" dma mem declaration APIs. The current implementation was relying on a short circuit in the DMA API that in the end, was acting as an allocator for these type of devices. Signed-off-by: Laurentiu Tudor Tested-by: Fredrik Noring Reviewed-by: Greg Kroah-Hartman Signed-off-by: Christoph Hellwig --- drivers/usb/Kconfig | 1 + drivers/usb/core/buffer.c | 9 +++++++++ drivers/usb/core/hcd.c | 36 ++++++++++++++++++++++++++++++++++++ drivers/usb/host/ohci-hcd.c | 23 ++++++++++++++++++----- drivers/usb/host/ohci-mem.c | 35 +++++++++++++++++++++++++++++++---- drivers/usb/host/ohci.h | 2 ++ include/linux/usb/hcd.h | 5 +++++ 7 files changed, 102 insertions(+), 9 deletions(-) (limited to 'include') diff --git a/drivers/usb/Kconfig b/drivers/usb/Kconfig index e4b27413f528..389c57d8eba7 100644 --- a/drivers/usb/Kconfig +++ b/drivers/usb/Kconfig @@ -45,6 +45,7 @@ config USB_ARCH_HAS_HCD config USB tristate "Support for Host-side USB" depends on USB_ARCH_HAS_HCD + select GENERIC_ALLOCATOR select USB_COMMON select NLS # for UTF-8 strings ---help--- diff --git a/drivers/usb/core/buffer.c b/drivers/usb/core/buffer.c index f641342cdec0..d2064ad7ad14 100644 --- a/drivers/usb/core/buffer.c +++ b/drivers/usb/core/buffer.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include @@ -124,6 +125,9 @@ void *hcd_buffer_alloc( if (size == 0) return NULL; + if (hcd->localmem_pool) + return gen_pool_dma_alloc(hcd->localmem_pool, size, dma); + /* some USB hosts just use PIO */ if (!IS_ENABLED(CONFIG_HAS_DMA) || (!is_device_dma_capable(bus->sysdev) && @@ -152,6 +156,11 @@ void hcd_buffer_free( if (!addr) return; + if (hcd->localmem_pool) { + gen_pool_free(hcd->localmem_pool, (unsigned long)addr, size); + return; + } + if (!IS_ENABLED(CONFIG_HAS_DMA) || (!is_device_dma_capable(bus->sysdev) && !(hcd->driver->flags & HCD_LOCAL_MEM))) { diff --git a/drivers/usb/core/hcd.c b/drivers/usb/core/hcd.c index 94d22551fc1b..29b96e5e8621 100644 --- a/drivers/usb/core/hcd.c +++ b/drivers/usb/core/hcd.c @@ -29,6 +29,8 @@ #include #include #include +#include +#include #include #include @@ -3039,6 +3041,40 @@ usb_hcd_platform_shutdown(struct platform_device *dev) } EXPORT_SYMBOL_GPL(usb_hcd_platform_shutdown); +int usb_hcd_setup_local_mem(struct usb_hcd *hcd, phys_addr_t phys_addr, + dma_addr_t dma, size_t size) +{ + int err; + void *local_mem; + + hcd->localmem_pool = devm_gen_pool_create(hcd->self.sysdev, PAGE_SHIFT, + dev_to_node(hcd->self.sysdev), + dev_name(hcd->self.sysdev)); + if (IS_ERR(hcd->localmem_pool)) + return PTR_ERR(hcd->localmem_pool); + + local_mem = devm_memremap(hcd->self.sysdev, phys_addr, + size, MEMREMAP_WC); + if (!local_mem) + return -ENOMEM; + + /* + * Here we pass a dma_addr_t but the arg type is a phys_addr_t. + * It's not backed by system memory and thus there's no kernel mapping + * for it. + */ + err = gen_pool_add_virt(hcd->localmem_pool, (unsigned long)local_mem, + dma, size, dev_to_node(hcd->self.sysdev)); + if (err < 0) { + dev_err(hcd->self.sysdev, "gen_pool_add_virt failed with %d\n", + err); + return err; + } + + return 0; +} +EXPORT_SYMBOL_GPL(usb_hcd_setup_local_mem); + /*-------------------------------------------------------------------------*/ #if IS_ENABLED(CONFIG_USB_MON) diff --git a/drivers/usb/host/ohci-hcd.c b/drivers/usb/host/ohci-hcd.c index 210181fd98d2..b200b19b44fa 100644 --- a/drivers/usb/host/ohci-hcd.c +++ b/drivers/usb/host/ohci-hcd.c @@ -40,6 +40,7 @@ #include #include #include +#include #include #include @@ -505,8 +506,15 @@ static int ohci_init (struct ohci_hcd *ohci) timer_setup(&ohci->io_watchdog, io_watchdog_func, 0); ohci->prev_frame_no = IO_WATCHDOG_OFF; - ohci->hcca = dma_alloc_coherent (hcd->self.controller, - sizeof(*ohci->hcca), &ohci->hcca_dma, GFP_KERNEL); + if (hcd->localmem_pool) + ohci->hcca = gen_pool_dma_alloc(hcd->localmem_pool, + sizeof(*ohci->hcca), + &ohci->hcca_dma); + else + ohci->hcca = dma_alloc_coherent(hcd->self.controller, + sizeof(*ohci->hcca), + &ohci->hcca_dma, + GFP_KERNEL); if (!ohci->hcca) return -ENOMEM; @@ -990,9 +998,14 @@ static void ohci_stop (struct usb_hcd *hcd) remove_debug_files (ohci); ohci_mem_cleanup (ohci); if (ohci->hcca) { - dma_free_coherent (hcd->self.controller, - sizeof *ohci->hcca, - ohci->hcca, ohci->hcca_dma); + if (hcd->localmem_pool) + gen_pool_free(hcd->localmem_pool, + (unsigned long)ohci->hcca, + sizeof(*ohci->hcca)); + else + dma_free_coherent(hcd->self.controller, + sizeof(*ohci->hcca), + ohci->hcca, ohci->hcca_dma); ohci->hcca = NULL; ohci->hcca_dma = 0; } diff --git a/drivers/usb/host/ohci-mem.c b/drivers/usb/host/ohci-mem.c index 3965ac0341eb..4afe27cc7e46 100644 --- a/drivers/usb/host/ohci-mem.c +++ b/drivers/usb/host/ohci-mem.c @@ -36,6 +36,13 @@ static void ohci_hcd_init (struct ohci_hcd *ohci) static int ohci_mem_init (struct ohci_hcd *ohci) { + /* + * HCs with local memory allocate from localmem_pool so there's + * no need to create the below dma pools. + */ + if (ohci_to_hcd(ohci)->localmem_pool) + return 0; + ohci->td_cache = dma_pool_create ("ohci_td", ohci_to_hcd(ohci)->self.controller, sizeof (struct td), @@ -84,8 +91,12 @@ td_alloc (struct ohci_hcd *hc, gfp_t mem_flags) { dma_addr_t dma; struct td *td; + struct usb_hcd *hcd = ohci_to_hcd(hc); - td = dma_pool_zalloc (hc->td_cache, mem_flags, &dma); + if (hcd->localmem_pool) + td = gen_pool_dma_zalloc(hcd->localmem_pool, sizeof(*td), &dma); + else + td = dma_pool_zalloc(hc->td_cache, mem_flags, &dma); if (td) { /* in case hc fetches it, make it look dead */ td->hwNextTD = cpu_to_hc32 (hc, dma); @@ -99,6 +110,7 @@ static void td_free (struct ohci_hcd *hc, struct td *td) { struct td **prev = &hc->td_hash [TD_HASH_FUNC (td->td_dma)]; + struct usb_hcd *hcd = ohci_to_hcd(hc); while (*prev && *prev != td) prev = &(*prev)->td_hash; @@ -106,7 +118,12 @@ td_free (struct ohci_hcd *hc, struct td *td) *prev = td->td_hash; else if ((td->hwINFO & cpu_to_hc32(hc, TD_DONE)) != 0) ohci_dbg (hc, "no hash for td %p\n", td); - dma_pool_free (hc->td_cache, td, td->td_dma); + + if (hcd->localmem_pool) + gen_pool_free(hcd->localmem_pool, (unsigned long)td, + sizeof(*td)); + else + dma_pool_free(hc->td_cache, td, td->td_dma); } /*-------------------------------------------------------------------------*/ @@ -117,8 +134,12 @@ ed_alloc (struct ohci_hcd *hc, gfp_t mem_flags) { dma_addr_t dma; struct ed *ed; + struct usb_hcd *hcd = ohci_to_hcd(hc); - ed = dma_pool_zalloc (hc->ed_cache, mem_flags, &dma); + if (hcd->localmem_pool) + ed = gen_pool_dma_zalloc(hcd->localmem_pool, sizeof(*ed), &dma); + else + ed = dma_pool_zalloc(hc->ed_cache, mem_flags, &dma); if (ed) { INIT_LIST_HEAD (&ed->td_list); ed->dma = dma; @@ -129,6 +150,12 @@ ed_alloc (struct ohci_hcd *hc, gfp_t mem_flags) static void ed_free (struct ohci_hcd *hc, struct ed *ed) { - dma_pool_free (hc->ed_cache, ed, ed->dma); + struct usb_hcd *hcd = ohci_to_hcd(hc); + + if (hcd->localmem_pool) + gen_pool_free(hcd->localmem_pool, (unsigned long)ed, + sizeof(*ed)); + else + dma_pool_free(hc->ed_cache, ed, ed->dma); } diff --git a/drivers/usb/host/ohci.h b/drivers/usb/host/ohci.h index ef4813bfc5bf..b015b00774b2 100644 --- a/drivers/usb/host/ohci.h +++ b/drivers/usb/host/ohci.h @@ -385,6 +385,8 @@ struct ohci_hcd { /* * memory management for queue data structures + * + * @td_cache and @ed_cache are %NULL if &usb_hcd.localmem_pool is used. */ struct dma_pool *td_cache; struct dma_pool *ed_cache; diff --git a/include/linux/usb/hcd.h b/include/linux/usb/hcd.h index bb57b5af4700..127560a4bfa0 100644 --- a/include/linux/usb/hcd.h +++ b/include/linux/usb/hcd.h @@ -216,6 +216,9 @@ struct usb_hcd { #define HC_IS_RUNNING(state) ((state) & __ACTIVE) #define HC_IS_SUSPENDED(state) ((state) & __SUSPEND) + /* memory pool for HCs having local memory, or %NULL */ + struct gen_pool *localmem_pool; + /* more shared queuing code would be good; it should support * smarter scheduling, handle transaction translators, etc; * input size of periodic table to an interrupt scheduler. @@ -461,6 +464,8 @@ extern int usb_add_hcd(struct usb_hcd *hcd, unsigned int irqnum, unsigned long irqflags); extern void usb_remove_hcd(struct usb_hcd *hcd); extern int usb_hcd_find_raw_port_number(struct usb_hcd *hcd, int port1); +int usb_hcd_setup_local_mem(struct usb_hcd *hcd, phys_addr_t phys_addr, + dma_addr_t dma, size_t size); struct platform_device; extern void usb_hcd_platform_shutdown(struct platform_device *dev); -- cgit v1.2.3 From 2d7a3dc3e24f43504b1f25eae8195e600f4cce8b Mon Sep 17 00:00:00 2001 From: Laurentiu Tudor Date: Wed, 29 May 2019 13:28:43 +0300 Subject: USB: drop HCD_LOCAL_MEM flag With the addition of the local memory allocator, the HCD_LOCAL_MEM flag can be dropped and the checks against it replaced with a check for the localmem_pool ptr being initialized. Signed-off-by: Laurentiu Tudor Tested-by: Fredrik Noring Reviewed-by: Greg Kroah-Hartman Signed-off-by: Christoph Hellwig --- drivers/usb/core/buffer.c | 8 +++----- drivers/usb/core/hcd.c | 15 ++++++--------- drivers/usb/host/ehci-hcd.c | 2 +- drivers/usb/host/fotg210-hcd.c | 2 +- drivers/usb/host/ohci-hcd.c | 2 +- drivers/usb/host/ohci-sm501.c | 5 +++-- drivers/usb/host/ohci-tmio.c | 2 +- drivers/usb/host/uhci-hcd.c | 2 +- include/linux/usb/hcd.h | 1 - 9 files changed, 17 insertions(+), 22 deletions(-) (limited to 'include') diff --git a/drivers/usb/core/buffer.c b/drivers/usb/core/buffer.c index d2064ad7ad14..1359b78a624e 100644 --- a/drivers/usb/core/buffer.c +++ b/drivers/usb/core/buffer.c @@ -68,7 +68,7 @@ int hcd_buffer_create(struct usb_hcd *hcd) if (!IS_ENABLED(CONFIG_HAS_DMA) || (!is_device_dma_capable(hcd->self.sysdev) && - !(hcd->driver->flags & HCD_LOCAL_MEM))) + !hcd->localmem_pool)) return 0; for (i = 0; i < HCD_BUFFER_POOLS; i++) { @@ -130,8 +130,7 @@ void *hcd_buffer_alloc( /* some USB hosts just use PIO */ if (!IS_ENABLED(CONFIG_HAS_DMA) || - (!is_device_dma_capable(bus->sysdev) && - !(hcd->driver->flags & HCD_LOCAL_MEM))) { + !is_device_dma_capable(bus->sysdev)) { *dma = ~(dma_addr_t) 0; return kmalloc(size, mem_flags); } @@ -162,8 +161,7 @@ void hcd_buffer_free( } if (!IS_ENABLED(CONFIG_HAS_DMA) || - (!is_device_dma_capable(bus->sysdev) && - !(hcd->driver->flags & HCD_LOCAL_MEM))) { + !is_device_dma_capable(bus->sysdev)) { kfree(addr); return; } diff --git a/drivers/usb/core/hcd.c b/drivers/usb/core/hcd.c index 29b96e5e8621..fe631d18c1ed 100644 --- a/drivers/usb/core/hcd.c +++ b/drivers/usb/core/hcd.c @@ -1347,14 +1347,14 @@ EXPORT_SYMBOL_GPL(usb_hcd_unlink_urb_from_ep); * using regular system memory - like pci devices doing bus mastering. * * To support host controllers with limited dma capabilities we provide dma - * bounce buffers. This feature can be enabled using the HCD_LOCAL_MEM flag. + * bounce buffers. This feature can be enabled by initializing + * hcd->localmem_pool using usb_hcd_setup_local_mem(). * For this to work properly the host controller code must first use the * function dma_declare_coherent_memory() to point out which memory area * that should be used for dma allocations. * - * The HCD_LOCAL_MEM flag then tells the usb code to allocate all data for - * dma using dma_alloc_coherent() which in turn allocates from the memory - * area pointed out with dma_declare_coherent_memory(). + * The initialized hcd->localmem_pool then tells the usb code to allocate all + * data for dma using the genalloc API. * * So, to summarize... * @@ -1364,9 +1364,6 @@ EXPORT_SYMBOL_GPL(usb_hcd_unlink_urb_from_ep); * (a) "normal" kernel memory is no good, and * (b) there's not enough to share * - * - The only *portable* hook for such stuff in the - * DMA framework is dma_declare_coherent_memory() - * * - So we use that, even though the primary requirement * is that the memory be "local" (hence addressable * by that device), not "coherent". @@ -1533,7 +1530,7 @@ int usb_hcd_map_urb_for_dma(struct usb_hcd *hcd, struct urb *urb, urb->setup_dma)) return -EAGAIN; urb->transfer_flags |= URB_SETUP_MAP_SINGLE; - } else if (hcd->driver->flags & HCD_LOCAL_MEM) { + } else if (hcd->localmem_pool) { ret = hcd_alloc_coherent( urb->dev->bus, mem_flags, &urb->setup_dma, @@ -1603,7 +1600,7 @@ int usb_hcd_map_urb_for_dma(struct usb_hcd *hcd, struct urb *urb, else urb->transfer_flags |= URB_DMA_MAP_SINGLE; } - } else if (hcd->driver->flags & HCD_LOCAL_MEM) { + } else if (hcd->localmem_pool) { ret = hcd_alloc_coherent( urb->dev->bus, mem_flags, &urb->transfer_dma, diff --git a/drivers/usb/host/ehci-hcd.c b/drivers/usb/host/ehci-hcd.c index cdafa97f632d..9da7e22848c9 100644 --- a/drivers/usb/host/ehci-hcd.c +++ b/drivers/usb/host/ehci-hcd.c @@ -559,7 +559,7 @@ static int ehci_init(struct usb_hcd *hcd) ehci->command = temp; /* Accept arbitrarily long scatter-gather lists */ - if (!(hcd->driver->flags & HCD_LOCAL_MEM)) + if (!hcd->localmem_pool) hcd->self.sg_tablesize = ~0; /* Prepare for unlinking active QHs */ diff --git a/drivers/usb/host/fotg210-hcd.c b/drivers/usb/host/fotg210-hcd.c index 0da68df259c8..5d74ff61fa4c 100644 --- a/drivers/usb/host/fotg210-hcd.c +++ b/drivers/usb/host/fotg210-hcd.c @@ -4995,7 +4995,7 @@ static int hcd_fotg210_init(struct usb_hcd *hcd) fotg210->command = temp; /* Accept arbitrarily long scatter-gather lists */ - if (!(hcd->driver->flags & HCD_LOCAL_MEM)) + if (!hcd->localmem_pool) hcd->self.sg_tablesize = ~0; return 0; } diff --git a/drivers/usb/host/ohci-hcd.c b/drivers/usb/host/ohci-hcd.c index b200b19b44fa..5801858d867e 100644 --- a/drivers/usb/host/ohci-hcd.c +++ b/drivers/usb/host/ohci-hcd.c @@ -448,7 +448,7 @@ static int ohci_init (struct ohci_hcd *ohci) struct usb_hcd *hcd = ohci_to_hcd(ohci); /* Accept arbitrarily long scatter-gather lists */ - if (!(hcd->driver->flags & HCD_LOCAL_MEM)) + if (!hcd->localmem_pool) hcd->self.sg_tablesize = ~0; if (distrust_firmware) diff --git a/drivers/usb/host/ohci-sm501.c b/drivers/usb/host/ohci-sm501.c index b710e100aec9..c158cda9e4b9 100644 --- a/drivers/usb/host/ohci-sm501.c +++ b/drivers/usb/host/ohci-sm501.c @@ -49,7 +49,7 @@ static const struct hc_driver ohci_sm501_hc_driver = { * generic hardware linkage */ .irq = ohci_irq, - .flags = HCD_USB11 | HCD_MEMORY | HCD_LOCAL_MEM, + .flags = HCD_USB11 | HCD_MEMORY, /* * basic lifecycle operations @@ -153,7 +153,8 @@ static int ohci_hcd_sm501_drv_probe(struct platform_device *pdev) * fine. This is however not always the case - buffers may be allocated * using kmalloc() - so the usb core needs to be told that it must copy * data into our local memory if the buffers happen to be placed in - * regular memory. The HCD_LOCAL_MEM flag does just that. + * regular memory. A non-null hcd->localmem_pool initialized by the + * the call to usb_hcd_setup_local_mem() below does just that. */ if (usb_hcd_setup_local_mem(hcd, mem->start, diff --git a/drivers/usb/host/ohci-tmio.c b/drivers/usb/host/ohci-tmio.c index 3b84ce0c3f29..d5a293a707b6 100644 --- a/drivers/usb/host/ohci-tmio.c +++ b/drivers/usb/host/ohci-tmio.c @@ -153,7 +153,7 @@ static const struct hc_driver ohci_tmio_hc_driver = { /* generic hardware linkage */ .irq = ohci_irq, - .flags = HCD_USB11 | HCD_MEMORY | HCD_LOCAL_MEM, + .flags = HCD_USB11 | HCD_MEMORY, /* basic lifecycle operations */ .start = ohci_tmio_start, diff --git a/drivers/usb/host/uhci-hcd.c b/drivers/usb/host/uhci-hcd.c index 98deb5f64268..03bc59755123 100644 --- a/drivers/usb/host/uhci-hcd.c +++ b/drivers/usb/host/uhci-hcd.c @@ -581,7 +581,7 @@ static int uhci_start(struct usb_hcd *hcd) hcd->uses_new_polling = 1; /* Accept arbitrarily long scatter-gather lists */ - if (!(hcd->driver->flags & HCD_LOCAL_MEM)) + if (!hcd->localmem_pool) hcd->self.sg_tablesize = ~0; spin_lock_init(&uhci->lock); diff --git a/include/linux/usb/hcd.h b/include/linux/usb/hcd.h index 127560a4bfa0..bab27ccc8ff5 100644 --- a/include/linux/usb/hcd.h +++ b/include/linux/usb/hcd.h @@ -256,7 +256,6 @@ struct hc_driver { int flags; #define HCD_MEMORY 0x0001 /* HC regs use memory (else I/O) */ -#define HCD_LOCAL_MEM 0x0002 /* HC needs local memory */ #define HCD_SHARED 0x0004 /* Two (or more) usb_hcds share HW */ #define HCD_USB11 0x0010 /* USB 1.1 */ #define HCD_USB2 0x0020 /* USB 2.0 */ -- cgit v1.2.3 From c30700db9eaabb35e0b123301df35a6846e6b6b4 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 3 Jun 2019 08:43:51 +0200 Subject: dma-direct: provide generic support for uncached kernel segments A few architectures support uncached kernel segments. In that case we get an uncached mapping for a given physica address by using an offset in the uncached segement. Implement support for this scheme in the generic dma-direct code instead of duplicating it in arch hooks. Signed-off-by: Christoph Hellwig --- arch/Kconfig | 8 ++++++++ include/linux/dma-noncoherent.h | 3 +++ kernel/dma/direct.c | 17 +++++++++++++++-- 3 files changed, 26 insertions(+), 2 deletions(-) (limited to 'include') diff --git a/arch/Kconfig b/arch/Kconfig index c47b328eada0..e8d19c3cb91f 100644 --- a/arch/Kconfig +++ b/arch/Kconfig @@ -260,6 +260,14 @@ config ARCH_HAS_SET_MEMORY config ARCH_HAS_SET_DIRECT_MAP bool +# +# Select if arch has an uncached kernel segment and provides the +# uncached_kernel_address / cached_kernel_address symbols to use it +# +config ARCH_HAS_UNCACHED_SEGMENT + select ARCH_HAS_DMA_PREP_COHERENT + bool + # Select if arch init_task must go in the __init_task_data section config ARCH_TASK_STRUCT_ON_STACK bool diff --git a/include/linux/dma-noncoherent.h b/include/linux/dma-noncoherent.h index 9741767e400f..7e0126a04e02 100644 --- a/include/linux/dma-noncoherent.h +++ b/include/linux/dma-noncoherent.h @@ -80,4 +80,7 @@ static inline void arch_dma_prep_coherent(struct page *page, size_t size) } #endif /* CONFIG_ARCH_HAS_DMA_PREP_COHERENT */ +void *uncached_kernel_address(void *addr); +void *cached_kernel_address(void *addr); + #endif /* _LINUX_DMA_NONCOHERENT_H */ diff --git a/kernel/dma/direct.c b/kernel/dma/direct.c index 0816c1e8b05a..b67f0aa08aa3 100644 --- a/kernel/dma/direct.c +++ b/kernel/dma/direct.c @@ -158,6 +158,13 @@ void *dma_direct_alloc_pages(struct device *dev, size_t size, *dma_handle = phys_to_dma(dev, page_to_phys(page)); } memset(ret, 0, size); + + if (IS_ENABLED(CONFIG_ARCH_HAS_UNCACHED_SEGMENT) && + !dev_is_dma_coherent(dev) && !(attrs & DMA_ATTR_NON_CONSISTENT)) { + arch_dma_prep_coherent(page, size); + ret = uncached_kernel_address(ret); + } + return ret; } @@ -173,13 +180,18 @@ void dma_direct_free_pages(struct device *dev, size_t size, void *cpu_addr, if (force_dma_unencrypted()) set_memory_encrypted((unsigned long)cpu_addr, 1 << page_order); + + if (IS_ENABLED(CONFIG_ARCH_HAS_UNCACHED_SEGMENT) && + !dev_is_dma_coherent(dev) && !(attrs & DMA_ATTR_NON_CONSISTENT)) + cpu_addr = cached_kernel_address(cpu_addr); __dma_direct_free_pages(dev, size, virt_to_page(cpu_addr)); } void *dma_direct_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs) { - if (!dev_is_dma_coherent(dev)) + if (!IS_ENABLED(CONFIG_ARCH_HAS_UNCACHED_SEGMENT) && + !dev_is_dma_coherent(dev)) return arch_dma_alloc(dev, size, dma_handle, gfp, attrs); return dma_direct_alloc_pages(dev, size, dma_handle, gfp, attrs); } @@ -187,7 +199,8 @@ void *dma_direct_alloc(struct device *dev, size_t size, void dma_direct_free(struct device *dev, size_t size, void *cpu_addr, dma_addr_t dma_addr, unsigned long attrs) { - if (!dev_is_dma_coherent(dev)) + if (!IS_ENABLED(CONFIG_ARCH_HAS_UNCACHED_SEGMENT) && + !dev_is_dma_coherent(dev)) arch_dma_free(dev, size, cpu_addr, dma_addr, attrs); else dma_direct_free_pages(dev, size, cpu_addr, dma_addr, attrs); -- cgit v1.2.3 From 4b85faed211ccfbcc7f3adf1cd62f0b00d1a172b Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Fri, 14 Jun 2019 16:06:10 +0200 Subject: dma-mapping: add a dma_alloc_need_uncached helper Check if we need to allocate uncached memory for a device given the allocation flags. Switch over the uncached segment check to this helper to deal with architectures that do not support the dma_cache_sync operation and thus should not returned cacheable memory for DMA_ATTR_NON_CONSISTENT allocations. Signed-off-by: Christoph Hellwig --- include/linux/dma-noncoherent.h | 14 ++++++++++++++ kernel/dma/direct.c | 4 ++-- 2 files changed, 16 insertions(+), 2 deletions(-) (limited to 'include') diff --git a/include/linux/dma-noncoherent.h b/include/linux/dma-noncoherent.h index 7e0126a04e02..732919ac5c11 100644 --- a/include/linux/dma-noncoherent.h +++ b/include/linux/dma-noncoherent.h @@ -20,6 +20,20 @@ static inline bool dev_is_dma_coherent(struct device *dev) } #endif /* CONFIG_ARCH_HAS_DMA_COHERENCE_H */ +/* + * Check if an allocation needs to be marked uncached to be coherent. + */ +static inline bool dma_alloc_need_uncached(struct device *dev, + unsigned long attrs) +{ + if (dev_is_dma_coherent(dev)) + return false; + if (IS_ENABLED(CONFIG_DMA_NONCOHERENT_CACHE_SYNC) && + (attrs & DMA_ATTR_NON_CONSISTENT)) + return false; + return true; +} + void *arch_dma_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs); void arch_dma_free(struct device *dev, size_t size, void *cpu_addr, diff --git a/kernel/dma/direct.c b/kernel/dma/direct.c index b67f0aa08aa3..c2893713bf80 100644 --- a/kernel/dma/direct.c +++ b/kernel/dma/direct.c @@ -160,7 +160,7 @@ void *dma_direct_alloc_pages(struct device *dev, size_t size, memset(ret, 0, size); if (IS_ENABLED(CONFIG_ARCH_HAS_UNCACHED_SEGMENT) && - !dev_is_dma_coherent(dev) && !(attrs & DMA_ATTR_NON_CONSISTENT)) { + dma_alloc_need_uncached(dev, attrs)) { arch_dma_prep_coherent(page, size); ret = uncached_kernel_address(ret); } @@ -182,7 +182,7 @@ void dma_direct_free_pages(struct device *dev, size_t size, void *cpu_addr, set_memory_encrypted((unsigned long)cpu_addr, 1 << page_order); if (IS_ENABLED(CONFIG_ARCH_HAS_UNCACHED_SEGMENT) && - !dev_is_dma_coherent(dev) && !(attrs & DMA_ATTR_NON_CONSISTENT)) + dma_alloc_need_uncached(dev, attrs)) cpu_addr = cached_kernel_address(cpu_addr); __dma_direct_free_pages(dev, size, virt_to_page(cpu_addr)); } -- cgit v1.2.3 From d98849aff87911013aadb730138ab728b52fc547 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Fri, 14 Jun 2019 16:17:27 +0200 Subject: dma-direct: handle DMA_ATTR_NO_KERNEL_MAPPING in common code DMA_ATTR_NO_KERNEL_MAPPING is generally implemented by allocating normal cacheable pages or CMA memory, and then returning the page pointer as the opaque handle. Lift that code from the xtensa and generic dma remapping implementations into the generic dma-direct code so that we don't even call arch_dma_alloc for these allocations. Signed-off-by: Christoph Hellwig --- arch/xtensa/kernel/pci-dma.c | 8 +------- include/linux/dma-noncoherent.h | 2 ++ kernel/dma/direct.c | 14 ++++++++++++++ kernel/dma/remap.c | 13 ++----------- 4 files changed, 19 insertions(+), 18 deletions(-) (limited to 'include') diff --git a/arch/xtensa/kernel/pci-dma.c b/arch/xtensa/kernel/pci-dma.c index 9171bff76fc4..206771277dff 100644 --- a/arch/xtensa/kernel/pci-dma.c +++ b/arch/xtensa/kernel/pci-dma.c @@ -167,10 +167,6 @@ void *arch_dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, *handle = phys_to_dma(dev, page_to_phys(page)); - if (attrs & DMA_ATTR_NO_KERNEL_MAPPING) { - return page; - } - #ifdef CONFIG_MMU if (PageHighMem(page)) { void *p; @@ -196,9 +192,7 @@ void arch_dma_free(struct device *dev, size_t size, void *vaddr, unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT; struct page *page; - if (attrs & DMA_ATTR_NO_KERNEL_MAPPING) { - page = vaddr; - } else if (platform_vaddr_uncached(vaddr)) { + if (platform_vaddr_uncached(vaddr)) { page = virt_to_page(platform_vaddr_to_cached(vaddr)); } else { #ifdef CONFIG_MMU diff --git a/include/linux/dma-noncoherent.h b/include/linux/dma-noncoherent.h index 732919ac5c11..53ee36ecdf37 100644 --- a/include/linux/dma-noncoherent.h +++ b/include/linux/dma-noncoherent.h @@ -28,6 +28,8 @@ static inline bool dma_alloc_need_uncached(struct device *dev, { if (dev_is_dma_coherent(dev)) return false; + if (attrs & DMA_ATTR_NO_KERNEL_MAPPING) + return false; if (IS_ENABLED(CONFIG_DMA_NONCOHERENT_CACHE_SYNC) && (attrs & DMA_ATTR_NON_CONSISTENT)) return false; diff --git a/kernel/dma/direct.c b/kernel/dma/direct.c index fc354f4f490b..b90e1aede743 100644 --- a/kernel/dma/direct.c +++ b/kernel/dma/direct.c @@ -138,6 +138,14 @@ void *dma_direct_alloc_pages(struct device *dev, size_t size, if (!page) return NULL; + if (attrs & DMA_ATTR_NO_KERNEL_MAPPING) { + /* remove any dirty cache lines on the kernel alias */ + if (!PageHighMem(page)) + arch_dma_prep_coherent(page, size); + /* return the page pointer as the opaque cookie */ + return page; + } + if (PageHighMem(page)) { /* * Depending on the cma= arguments and per-arch setup @@ -178,6 +186,12 @@ void dma_direct_free_pages(struct device *dev, size_t size, void *cpu_addr, { unsigned int page_order = get_order(size); + if (attrs & DMA_ATTR_NO_KERNEL_MAPPING) { + /* cpu_addr is a struct page cookie, not a kernel address */ + __dma_direct_free_pages(dev, size, cpu_addr); + return; + } + if (force_dma_unencrypted()) set_memory_encrypted((unsigned long)cpu_addr, 1 << page_order); diff --git a/kernel/dma/remap.c b/kernel/dma/remap.c index 0207e3764d52..a594aec07882 100644 --- a/kernel/dma/remap.c +++ b/kernel/dma/remap.c @@ -202,8 +202,7 @@ void *arch_dma_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle, size = PAGE_ALIGN(size); - if (!gfpflags_allow_blocking(flags) && - !(attrs & DMA_ATTR_NO_KERNEL_MAPPING)) { + if (!gfpflags_allow_blocking(flags)) { ret = dma_alloc_from_pool(size, &page, flags); if (!ret) return NULL; @@ -217,11 +216,6 @@ void *arch_dma_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle, /* remove any dirty cache lines on the kernel alias */ arch_dma_prep_coherent(page, size); - if (attrs & DMA_ATTR_NO_KERNEL_MAPPING) { - ret = page; /* opaque cookie */ - goto done; - } - /* create a coherent mapping */ ret = dma_common_contiguous_remap(page, size, VM_USERMAP, arch_dma_mmap_pgprot(dev, PAGE_KERNEL, attrs), @@ -240,10 +234,7 @@ done: void arch_dma_free(struct device *dev, size_t size, void *vaddr, dma_addr_t dma_handle, unsigned long attrs) { - if (attrs & DMA_ATTR_NO_KERNEL_MAPPING) { - /* vaddr is a struct page cookie, not a kernel address */ - __dma_direct_free_pages(dev, size, vaddr); - } else if (!dma_free_from_pool(vaddr, PAGE_ALIGN(size))) { + if (!dma_free_from_pool(vaddr, PAGE_ALIGN(size))) { phys_addr_t phys = dma_to_phys(dev, dma_handle); struct page *page = pfn_to_page(__phys_to_pfn(phys)); -- cgit v1.2.3 From cf394fc5f7155c24efb584979e81427575ab3539 Mon Sep 17 00:00:00 2001 From: Fredrik Noring Date: Tue, 25 Jun 2019 17:05:58 +0200 Subject: lib/genalloc.c: Add algorithm, align and zeroed family of DMA allocators Provide the algorithm option to DMA allocators as well, along with convenience variants for zeroed and aligned memory. The following four functions are added: - gen_pool_dma_alloc_algo() - gen_pool_dma_alloc_align() - gen_pool_dma_zalloc_algo() - gen_pool_dma_zalloc_align() Signed-off-by: Fredrik Noring Tested-by: Guenter Roeck Signed-off-by: Christoph Hellwig --- include/linux/genalloc.h | 10 ++++- lib/genalloc.c | 100 +++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 105 insertions(+), 5 deletions(-) (limited to 'include') diff --git a/include/linux/genalloc.h b/include/linux/genalloc.h index 6c62eeca754f..ed641337df87 100644 --- a/include/linux/genalloc.h +++ b/include/linux/genalloc.h @@ -121,7 +121,15 @@ extern unsigned long gen_pool_alloc_algo(struct gen_pool *, size_t, genpool_algo_t algo, void *data); extern void *gen_pool_dma_alloc(struct gen_pool *pool, size_t size, dma_addr_t *dma); -void *gen_pool_dma_zalloc(struct gen_pool *pool, size_t size, dma_addr_t *dma); +extern void *gen_pool_dma_alloc_algo(struct gen_pool *pool, size_t size, + dma_addr_t *dma, genpool_algo_t algo, void *data); +extern void *gen_pool_dma_alloc_align(struct gen_pool *pool, size_t size, + dma_addr_t *dma, int align); +extern void *gen_pool_dma_zalloc(struct gen_pool *pool, size_t size, dma_addr_t *dma); +extern void *gen_pool_dma_zalloc_algo(struct gen_pool *pool, size_t size, + dma_addr_t *dma, genpool_algo_t algo, void *data); +extern void *gen_pool_dma_zalloc_align(struct gen_pool *pool, size_t size, + dma_addr_t *dma, int align); extern void gen_pool_free(struct gen_pool *, unsigned long, size_t); extern void gen_pool_for_each_chunk(struct gen_pool *, void (*)(struct gen_pool *, struct gen_pool_chunk *, void *), void *); diff --git a/lib/genalloc.c b/lib/genalloc.c index 5db43476a19b..512623fbac51 100644 --- a/lib/genalloc.c +++ b/lib/genalloc.c @@ -347,13 +347,35 @@ EXPORT_SYMBOL(gen_pool_alloc_algo); * Return: virtual address of the allocated memory, or %NULL on failure */ void *gen_pool_dma_alloc(struct gen_pool *pool, size_t size, dma_addr_t *dma) +{ + return gen_pool_dma_alloc_algo(pool, size, dma, pool->algo, pool->data); +} +EXPORT_SYMBOL(gen_pool_dma_alloc); + +/** + * gen_pool_dma_alloc_algo - allocate special memory from the pool for DMA + * usage with the given pool algorithm + * @pool: pool to allocate from + * @size: number of bytes to allocate from the pool + * @dma: DMA-view physical address return value. Use %NULL if unneeded. + * @algo: algorithm passed from caller + * @data: data passed to algorithm + * + * Allocate the requested number of bytes from the specified pool. Uses the + * given pool allocation function. Can not be used in NMI handler on + * architectures without NMI-safe cmpxchg implementation. + * + * Return: virtual address of the allocated memory, or %NULL on failure + */ +void *gen_pool_dma_alloc_algo(struct gen_pool *pool, size_t size, + dma_addr_t *dma, genpool_algo_t algo, void *data) { unsigned long vaddr; if (!pool) return NULL; - vaddr = gen_pool_alloc(pool, size); + vaddr = gen_pool_alloc_algo(pool, size, algo, data); if (!vaddr) return NULL; @@ -362,7 +384,31 @@ void *gen_pool_dma_alloc(struct gen_pool *pool, size_t size, dma_addr_t *dma) return (void *)vaddr; } -EXPORT_SYMBOL(gen_pool_dma_alloc); +EXPORT_SYMBOL(gen_pool_dma_alloc_algo); + +/** + * gen_pool_dma_alloc_align - allocate special memory from the pool for DMA + * usage with the given alignment + * @pool: pool to allocate from + * @size: number of bytes to allocate from the pool + * @dma: DMA-view physical address return value. Use %NULL if unneeded. + * @align: alignment in bytes for starting address + * + * Allocate the requested number bytes from the specified pool, with the given + * alignment restriction. Can not be used in NMI handler on architectures + * without NMI-safe cmpxchg implementation. + * + * Return: virtual address of the allocated memory, or %NULL on failure + */ +void *gen_pool_dma_alloc_align(struct gen_pool *pool, size_t size, + dma_addr_t *dma, int align) +{ + struct genpool_data_align data = { .align = align }; + + return gen_pool_dma_alloc_algo(pool, size, dma, + gen_pool_first_fit_align, &data); +} +EXPORT_SYMBOL(gen_pool_dma_alloc_align); /** * gen_pool_dma_zalloc - allocate special zeroed memory from the pool for @@ -380,14 +426,60 @@ EXPORT_SYMBOL(gen_pool_dma_alloc); */ void *gen_pool_dma_zalloc(struct gen_pool *pool, size_t size, dma_addr_t *dma) { - void *vaddr = gen_pool_dma_alloc(pool, size, dma); + return gen_pool_dma_zalloc_algo(pool, size, dma, pool->algo, pool->data); +} +EXPORT_SYMBOL(gen_pool_dma_zalloc); + +/** + * gen_pool_dma_zalloc_algo - allocate special zeroed memory from the pool for + * DMA usage with the given pool algorithm + * @pool: pool to allocate from + * @size: number of bytes to allocate from the pool + * @dma: DMA-view physical address return value. Use %NULL if unneeded. + * @algo: algorithm passed from caller + * @data: data passed to algorithm + * + * Allocate the requested number of zeroed bytes from the specified pool. Uses + * the given pool allocation function. Can not be used in NMI handler on + * architectures without NMI-safe cmpxchg implementation. + * + * Return: virtual address of the allocated zeroed memory, or %NULL on failure + */ +void *gen_pool_dma_zalloc_algo(struct gen_pool *pool, size_t size, + dma_addr_t *dma, genpool_algo_t algo, void *data) +{ + void *vaddr = gen_pool_dma_alloc_algo(pool, size, dma, algo, data); if (vaddr) memset(vaddr, 0, size); return vaddr; } -EXPORT_SYMBOL(gen_pool_dma_zalloc); +EXPORT_SYMBOL(gen_pool_dma_zalloc_algo); + +/** + * gen_pool_dma_zalloc_align - allocate special zeroed memory from the pool for + * DMA usage with the given alignment + * @pool: pool to allocate from + * @size: number of bytes to allocate from the pool + * @dma: DMA-view physical address return value. Use %NULL if unneeded. + * @align: alignment in bytes for starting address + * + * Allocate the requested number of zeroed bytes from the specified pool, + * with the given alignment restriction. Can not be used in NMI handler on + * architectures without NMI-safe cmpxchg implementation. + * + * Return: virtual address of the allocated zeroed memory, or %NULL on failure + */ +void *gen_pool_dma_zalloc_align(struct gen_pool *pool, size_t size, + dma_addr_t *dma, int align) +{ + struct genpool_data_align data = { .align = align }; + + return gen_pool_dma_zalloc_algo(pool, size, dma, + gen_pool_first_fit_align, &data); +} +EXPORT_SYMBOL(gen_pool_dma_zalloc_align); /** * gen_pool_free - free allocated special memory back to the pool -- cgit v1.2.3 From 15ffe5e1acf5fe1512e98b20702e46ce9f25e2f7 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 8 Jul 2019 12:55:27 -0700 Subject: dma-mapping: mark dma_alloc_need_uncached as __always_inline Without the __always_inline at least i386 configs that have CONFIG_OPTIMIZE_INLINING set seem fail to inline dma_alloc_need_uncached, leading to a linker error because of undefined symbols. Reported-by: Randy Dunlap Signed-off-by: Christoph Hellwig Acked-by: Randy Dunlap # build-tested --- include/linux/dma-noncoherent.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include') diff --git a/include/linux/dma-noncoherent.h b/include/linux/dma-noncoherent.h index 53ee36ecdf37..3813211a9aad 100644 --- a/include/linux/dma-noncoherent.h +++ b/include/linux/dma-noncoherent.h @@ -23,7 +23,7 @@ static inline bool dev_is_dma_coherent(struct device *dev) /* * Check if an allocation needs to be marked uncached to be coherent. */ -static inline bool dma_alloc_need_uncached(struct device *dev, +static __always_inline bool dma_alloc_need_uncached(struct device *dev, unsigned long attrs) { if (dev_is_dma_coherent(dev)) -- cgit v1.2.3