diff options
Diffstat (limited to 'drivers/gpu/drm/ttm')
-rw-r--r-- | drivers/gpu/drm/ttm/ttm_agp_backend.c | 9 | ||||
-rw-r--r-- | drivers/gpu/drm/ttm/ttm_bo.c | 140 | ||||
-rw-r--r-- | drivers/gpu/drm/ttm/ttm_bo_manager.c | 2 | ||||
-rw-r--r-- | drivers/gpu/drm/ttm/ttm_bo_util.c | 110 | ||||
-rw-r--r-- | drivers/gpu/drm/ttm/ttm_bo_vm.c | 5 | ||||
-rw-r--r-- | drivers/gpu/drm/ttm/ttm_tt.c | 12 |
6 files changed, 120 insertions, 158 deletions
diff --git a/drivers/gpu/drm/ttm/ttm_agp_backend.c b/drivers/gpu/drm/ttm/ttm_agp_backend.c index 6050dc846894..38f1351140e2 100644 --- a/drivers/gpu/drm/ttm/ttm_agp_backend.c +++ b/drivers/gpu/drm/ttm/ttm_agp_backend.c @@ -82,17 +82,18 @@ static int ttm_agp_bind(struct ttm_tt *ttm, struct ttm_mem_reg *bo_mem) return ret; } -static int ttm_agp_unbind(struct ttm_tt *ttm) +static void ttm_agp_unbind(struct ttm_tt *ttm) { struct ttm_agp_backend *agp_be = container_of(ttm, struct ttm_agp_backend, ttm); if (agp_be->mem) { - if (agp_be->mem->is_bound) - return agp_unbind_memory(agp_be->mem); + if (agp_be->mem->is_bound) { + agp_unbind_memory(agp_be->mem); + return; + } agp_free_memory(agp_be->mem); agp_be->mem = NULL; } - return 0; } static void ttm_agp_destroy(struct ttm_tt *ttm) diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c index 0f20e14a4cfd..f297fd5e02d4 100644 --- a/drivers/gpu/drm/ttm/ttm_bo.c +++ b/drivers/gpu/drm/ttm/ttm_bo.c @@ -85,7 +85,6 @@ static void ttm_mem_type_debug(struct ttm_bo_device *bdev, struct drm_printer *p drm_printf(p, " has_type: %d\n", man->has_type); drm_printf(p, " use_type: %d\n", man->use_type); drm_printf(p, " flags: 0x%08X\n", man->flags); - drm_printf(p, " gpu_offset: 0x%08llX\n", man->gpu_offset); drm_printf(p, " size: %llu\n", man->size); drm_printf(p, " available_caching: 0x%08X\n", man->available_caching); drm_printf(p, " default_caching: 0x%08X\n", man->default_caching); @@ -273,32 +272,26 @@ static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo, struct ttm_operation_ctx *ctx) { struct ttm_bo_device *bdev = bo->bdev; - bool old_is_pci = ttm_mem_reg_is_pci(bdev, &bo->mem); - bool new_is_pci = ttm_mem_reg_is_pci(bdev, mem); struct ttm_mem_type_manager *old_man = &bdev->man[bo->mem.mem_type]; struct ttm_mem_type_manager *new_man = &bdev->man[mem->mem_type]; - int ret = 0; + int ret; - if (old_is_pci || new_is_pci || - ((mem->placement & bo->mem.placement & TTM_PL_MASK_CACHING) == 0)) { - ret = ttm_mem_io_lock(old_man, true); - if (unlikely(ret != 0)) - goto out_err; - ttm_bo_unmap_virtual_locked(bo); - ttm_mem_io_unlock(old_man); - } + ret = ttm_mem_io_lock(old_man, true); + if (unlikely(ret != 0)) + goto out_err; + ttm_bo_unmap_virtual_locked(bo); + ttm_mem_io_unlock(old_man); /* * Create and bind a ttm if required. */ if (!(new_man->flags & TTM_MEMTYPE_FLAG_FIXED)) { - if (bo->ttm == NULL) { - bool zero = !(old_man->flags & TTM_MEMTYPE_FLAG_FIXED); - ret = ttm_tt_create(bo, zero); - if (ret) - goto out_err; - } + bool zero = !(old_man->flags & TTM_MEMTYPE_FLAG_FIXED); + + ret = ttm_tt_create(bo, zero); + if (ret) + goto out_err; ret = ttm_tt_set_placement_caching(bo->ttm, mem->placement); if (ret) @@ -314,7 +307,6 @@ static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo, if (bdev->driver->move_notify) bdev->driver->move_notify(bo, evict, mem); bo->mem = *mem; - mem->mm_node = NULL; goto moved; } } @@ -343,12 +335,6 @@ static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo, moved: bo->evicted = false; - if (bo->mem.mm_node) - bo->offset = (bo->mem.start << PAGE_SHIFT) + - bdev->man[bo->mem.mem_type].gpu_offset; - else - bo->offset = 0; - ctx->bytes_moved += bo->num_pages << PAGE_SHIFT; return 0; @@ -624,7 +610,6 @@ static void ttm_bo_release(struct kref *kref) ttm_bo_cleanup_memtype_use(bo); dma_resv_unlock(bo->base.resv); - BUG_ON(bo->mem.mm_node != NULL); atomic_dec(&ttm_bo_glob.bo_count); dma_fence_put(bo->moving); if (!ttm_bo_uses_embedded_gem_object(bo)) @@ -667,13 +652,8 @@ static int ttm_bo_evict(struct ttm_buffer_object *bo, placement.num_busy_placement = 0; bdev->driver->evict_flags(bo, &placement); - if (!placement.num_placement && !placement.num_busy_placement) { - ret = ttm_bo_pipeline_gutting(bo); - if (ret) - return ret; - - return ttm_tt_create(bo, false); - } + if (!placement.num_placement && !placement.num_busy_placement) + return ttm_bo_pipeline_gutting(bo); evict_mem = bo->mem; evict_mem.mm_node = NULL; @@ -856,12 +836,29 @@ static int ttm_mem_evict_first(struct ttm_bo_device *bdev, return ret; } +static int ttm_bo_mem_get(struct ttm_buffer_object *bo, + const struct ttm_place *place, + struct ttm_mem_reg *mem) +{ + struct ttm_mem_type_manager *man = &bo->bdev->man[mem->mem_type]; + + mem->mm_node = NULL; + if (!man->func || !man->func->get_node) + return 0; + + return man->func->get_node(man, bo, place, mem); +} + void ttm_bo_mem_put(struct ttm_buffer_object *bo, struct ttm_mem_reg *mem) { struct ttm_mem_type_manager *man = &bo->bdev->man[mem->mem_type]; - if (mem->mm_node) - (*man->func->put_node)(man, mem); + if (!man->func || !man->func->put_node) + return; + + man->func->put_node(man, mem); + mem->mm_node = NULL; + mem->mem_type = TTM_PL_SYSTEM; } EXPORT_SYMBOL(ttm_bo_mem_put); @@ -917,11 +914,11 @@ static int ttm_bo_mem_force_space(struct ttm_buffer_object *bo, ticket = dma_resv_locking_ctx(bo->base.resv); do { - ret = (*man->func->get_node)(man, bo, place, mem); - if (unlikely(ret != 0)) - return ret; - if (mem->mm_node) + ret = ttm_bo_mem_get(bo, place, mem); + if (likely(!ret)) break; + if (unlikely(ret != -ENOSPC)) + return ret; ret = ttm_mem_evict_first(bdev, mem->mem_type, place, ctx, ticket); if (unlikely(ret != 0)) @@ -1047,7 +1044,6 @@ int ttm_bo_mem_space(struct ttm_buffer_object *bo, if (unlikely(ret)) return ret; - mem->mm_node = NULL; for (i = 0; i < placement->num_placement; ++i) { const struct ttm_place *place = &placement->placement[i]; struct ttm_mem_type_manager *man; @@ -1059,21 +1055,16 @@ int ttm_bo_mem_space(struct ttm_buffer_object *bo, goto error; type_found = true; - mem->mm_node = NULL; - if (mem->mem_type == TTM_PL_SYSTEM) - return 0; - - man = &bdev->man[mem->mem_type]; - ret = (*man->func->get_node)(man, bo, place, mem); + ret = ttm_bo_mem_get(bo, place, mem); + if (ret == -ENOSPC) + continue; if (unlikely(ret)) goto error; - if (!mem->mm_node) - continue; - + man = &bdev->man[mem->mem_type]; ret = ttm_bo_add_move_fence(bo, man, mem, ctx->no_wait_gpu); if (unlikely(ret)) { - (*man->func->put_node)(man, mem); + ttm_bo_mem_put(bo, mem); if (ret == -EBUSY) continue; @@ -1092,12 +1083,8 @@ int ttm_bo_mem_space(struct ttm_buffer_object *bo, goto error; type_found = true; - mem->mm_node = NULL; - if (mem->mem_type == TTM_PL_SYSTEM) - return 0; - ret = ttm_bo_mem_force_space(bo, place, mem, ctx); - if (ret == 0 && mem->mm_node) + if (likely(!ret)) return 0; if (ret && ret != -EBUSY) @@ -1135,6 +1122,8 @@ static int ttm_bo_move_buffer(struct ttm_buffer_object *bo, mem.page_alignment = bo->mem.page_alignment; mem.bus.io_reserved_vm = false; mem.bus.io_reserved_count = 0; + mem.mm_node = NULL; + /* * Determine where to move the buffer. */ @@ -1143,7 +1132,7 @@ static int ttm_bo_move_buffer(struct ttm_buffer_object *bo, goto out_unlock; ret = ttm_bo_handle_move_mem(bo, &mem, false, ctx); out_unlock: - if (ret && mem.mm_node) + if (ret) ttm_bo_mem_put(bo, &mem); return ret; } @@ -1158,7 +1147,7 @@ static bool ttm_bo_places_compat(const struct ttm_place *places, for (i = 0; i < num_placement; i++) { const struct ttm_place *heap = &places[i]; - if (mem->mm_node && (mem->start < heap->fpfn || + if ((mem->start < heap->fpfn || (heap->lpfn != 0 && (mem->start + mem->num_pages) > heap->lpfn))) continue; @@ -1203,13 +1192,8 @@ int ttm_bo_validate(struct ttm_buffer_object *bo, /* * Remove the backing store if no placement is given. */ - if (!placement->num_placement && !placement->num_busy_placement) { - ret = ttm_bo_pipeline_gutting(bo); - if (ret) - return ret; - - return ttm_tt_create(bo, false); - } + if (!placement->num_placement && !placement->num_busy_placement) + return ttm_bo_pipeline_gutting(bo); /* * Check whether we need to move buffer. @@ -1226,14 +1210,6 @@ int ttm_bo_validate(struct ttm_buffer_object *bo, ttm_flag_masked(&bo->mem.placement, new_flags, ~TTM_PL_MASK_MEMTYPE); } - /* - * We might need to add a TTM. - */ - if (bo->mem.mem_type == TTM_PL_SYSTEM && bo->ttm == NULL) { - ret = ttm_tt_create(bo, true); - if (ret) - return ret; - } return 0; } EXPORT_SYMBOL(ttm_bo_validate); @@ -1542,7 +1518,6 @@ int ttm_bo_init_mm(struct ttm_bo_device *bdev, unsigned type, BUG_ON(type >= TTM_NUM_MEM_TYPES); man = &bdev->man[type]; BUG_ON(man->has_type); - man->io_reserve_fastpath = true; man->use_io_reserve_lru = false; mutex_init(&man->io_reserve_mutex); spin_lock_init(&man->move_lock); @@ -1720,23 +1695,6 @@ EXPORT_SYMBOL(ttm_bo_device_init); * buffer object vm functions. */ -bool ttm_mem_reg_is_pci(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem) -{ - struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type]; - - if (!(man->flags & TTM_MEMTYPE_FLAG_FIXED)) { - if (mem->mem_type == TTM_PL_SYSTEM) - return false; - - if (man->flags & TTM_MEMTYPE_FLAG_CMA) - return false; - - if (mem->placement & TTM_PL_FLAG_CACHED) - return false; - } - return true; -} - void ttm_bo_unmap_virtual_locked(struct ttm_buffer_object *bo) { struct ttm_bo_device *bdev = bo->bdev; @@ -1880,7 +1838,7 @@ out: } EXPORT_SYMBOL(ttm_bo_swapout); -void ttm_bo_swapout_all(struct ttm_bo_device *bdev) +void ttm_bo_swapout_all(void) { struct ttm_operation_ctx ctx = { .interruptible = false, diff --git a/drivers/gpu/drm/ttm/ttm_bo_manager.c b/drivers/gpu/drm/ttm/ttm_bo_manager.c index 18d3debcc949..facd3049c3aa 100644 --- a/drivers/gpu/drm/ttm/ttm_bo_manager.c +++ b/drivers/gpu/drm/ttm/ttm_bo_manager.c @@ -86,7 +86,7 @@ static int ttm_bo_man_get_node(struct ttm_mem_type_manager *man, mem->start = node->start; } - return 0; + return ret; } static void ttm_bo_man_put_node(struct ttm_mem_type_manager *man, diff --git a/drivers/gpu/drm/ttm/ttm_bo_util.c b/drivers/gpu/drm/ttm/ttm_bo_util.c index f09b096ba4fd..7fb3e0bcbab4 100644 --- a/drivers/gpu/drm/ttm/ttm_bo_util.c +++ b/drivers/gpu/drm/ttm/ttm_bo_util.c @@ -93,7 +93,7 @@ EXPORT_SYMBOL(ttm_bo_move_ttm); int ttm_mem_io_lock(struct ttm_mem_type_manager *man, bool interruptible) { - if (likely(man->io_reserve_fastpath)) + if (likely(!man->use_io_reserve_lru)) return 0; if (interruptible) @@ -105,7 +105,7 @@ int ttm_mem_io_lock(struct ttm_mem_type_manager *man, bool interruptible) void ttm_mem_io_unlock(struct ttm_mem_type_manager *man) { - if (likely(man->io_reserve_fastpath)) + if (likely(!man->use_io_reserve_lru)) return; mutex_unlock(&man->io_reserve_mutex); @@ -115,39 +115,35 @@ static int ttm_mem_io_evict(struct ttm_mem_type_manager *man) { struct ttm_buffer_object *bo; - if (!man->use_io_reserve_lru || list_empty(&man->io_reserve_lru)) - return -EAGAIN; + bo = list_first_entry_or_null(&man->io_reserve_lru, + struct ttm_buffer_object, + io_reserve_lru); + if (!bo) + return -ENOSPC; - bo = list_first_entry(&man->io_reserve_lru, - struct ttm_buffer_object, - io_reserve_lru); list_del_init(&bo->io_reserve_lru); ttm_bo_unmap_virtual_locked(bo); - return 0; } - int ttm_mem_io_reserve(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem) { struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type]; - int ret = 0; + int ret; + + if (mem->bus.io_reserved_count++) + return 0; if (!bdev->driver->io_mem_reserve) return 0; - if (likely(man->io_reserve_fastpath)) - return bdev->driver->io_mem_reserve(bdev, mem); - if (bdev->driver->io_mem_reserve && - mem->bus.io_reserved_count++ == 0) { retry: - ret = bdev->driver->io_mem_reserve(bdev, mem); - if (ret == -EAGAIN) { - ret = ttm_mem_io_evict(man); - if (ret == 0) - goto retry; - } + ret = bdev->driver->io_mem_reserve(bdev, mem); + if (ret == -ENOSPC) { + ret = ttm_mem_io_evict(man); + if (ret == 0) + goto retry; } return ret; } @@ -155,35 +151,31 @@ retry: void ttm_mem_io_free(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem) { - struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type]; - - if (likely(man->io_reserve_fastpath)) + if (--mem->bus.io_reserved_count) return; - if (bdev->driver->io_mem_reserve && - --mem->bus.io_reserved_count == 0 && - bdev->driver->io_mem_free) - bdev->driver->io_mem_free(bdev, mem); + if (!bdev->driver->io_mem_free) + return; + bdev->driver->io_mem_free(bdev, mem); } int ttm_mem_io_reserve_vm(struct ttm_buffer_object *bo) { + struct ttm_mem_type_manager *man = &bo->bdev->man[bo->mem.mem_type]; struct ttm_mem_reg *mem = &bo->mem; int ret; - if (!mem->bus.io_reserved_vm) { - struct ttm_mem_type_manager *man = - &bo->bdev->man[mem->mem_type]; + if (mem->bus.io_reserved_vm) + return 0; - ret = ttm_mem_io_reserve(bo->bdev, mem); - if (unlikely(ret != 0)) - return ret; - mem->bus.io_reserved_vm = true; - if (man->use_io_reserve_lru) - list_add_tail(&bo->io_reserve_lru, - &man->io_reserve_lru); - } + ret = ttm_mem_io_reserve(bo->bdev, mem); + if (unlikely(ret != 0)) + return ret; + mem->bus.io_reserved_vm = true; + if (man->use_io_reserve_lru) + list_add_tail(&bo->io_reserve_lru, + &man->io_reserve_lru); return 0; } @@ -191,15 +183,17 @@ void ttm_mem_io_free_vm(struct ttm_buffer_object *bo) { struct ttm_mem_reg *mem = &bo->mem; - if (mem->bus.io_reserved_vm) { - mem->bus.io_reserved_vm = false; - list_del_init(&bo->io_reserve_lru); - ttm_mem_io_free(bo->bdev, mem); - } + if (!mem->bus.io_reserved_vm) + return; + + mem->bus.io_reserved_vm = false; + list_del_init(&bo->io_reserve_lru); + ttm_mem_io_free(bo->bdev, mem); } -static int ttm_mem_reg_ioremap(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem, - void **virtual) +static int ttm_mem_reg_ioremap(struct ttm_bo_device *bdev, + struct ttm_mem_reg *mem, + void **virtual) { struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type]; int ret; @@ -216,9 +210,11 @@ static int ttm_mem_reg_ioremap(struct ttm_bo_device *bdev, struct ttm_mem_reg *m addr = mem->bus.addr; } else { if (mem->placement & TTM_PL_FLAG_WC) - addr = ioremap_wc(mem->bus.base + mem->bus.offset, mem->bus.size); + addr = ioremap_wc(mem->bus.base + mem->bus.offset, + mem->bus.size); else - addr = ioremap(mem->bus.base + mem->bus.offset, mem->bus.size); + addr = ioremap(mem->bus.base + mem->bus.offset, + mem->bus.size); if (!addr) { (void) ttm_mem_io_lock(man, false); ttm_mem_io_free(bdev, mem); @@ -230,8 +226,9 @@ static int ttm_mem_reg_ioremap(struct ttm_bo_device *bdev, struct ttm_mem_reg *m return 0; } -static void ttm_mem_reg_iounmap(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem, - void *virtual) +static void ttm_mem_reg_iounmap(struct ttm_bo_device *bdev, + struct ttm_mem_reg *mem, + void *virtual) { struct ttm_mem_type_manager *man; @@ -513,11 +510,13 @@ static int ttm_bo_ioremap(struct ttm_buffer_object *bo, } else { map->bo_kmap_type = ttm_bo_map_iomap; if (mem->placement & TTM_PL_FLAG_WC) - map->virtual = ioremap_wc(bo->mem.bus.base + bo->mem.bus.offset + offset, + map->virtual = ioremap_wc(bo->mem.bus.base + + bo->mem.bus.offset + offset, size); else - map->virtual = ioremap(bo->mem.bus.base + bo->mem.bus.offset + offset, - size); + map->virtual = ioremap(bo->mem.bus.base + + bo->mem.bus.offset + offset, + size); } return (!map->virtual) ? -ENOMEM : 0; } @@ -532,12 +531,15 @@ static int ttm_bo_kmap_ttm(struct ttm_buffer_object *bo, .interruptible = false, .no_wait_gpu = false }; - struct ttm_tt *ttm = bo->ttm; + struct ttm_tt *ttm; pgprot_t prot; int ret; - BUG_ON(!ttm); + ret = ttm_tt_create(bo, true); + if (ret) + return ret; + ttm = bo->ttm; ret = ttm_tt_populate(ttm, &ctx); if (ret) return ret; diff --git a/drivers/gpu/drm/ttm/ttm_bo_vm.c b/drivers/gpu/drm/ttm/ttm_bo_vm.c index fa03fab02076..d7a6537dd6ee 100644 --- a/drivers/gpu/drm/ttm/ttm_bo_vm.c +++ b/drivers/gpu/drm/ttm/ttm_bo_vm.c @@ -351,6 +351,11 @@ vm_fault_t ttm_bo_vm_fault_reserved(struct vm_fault *vmf, }; + if (ttm_tt_create(bo, true)) { + ret = VM_FAULT_OOM; + goto out_io_unlock; + } + ttm = bo->ttm; if (ttm_tt_populate(bo->ttm, &ctx)) { ret = VM_FAULT_OOM; diff --git a/drivers/gpu/drm/ttm/ttm_tt.c b/drivers/gpu/drm/ttm/ttm_tt.c index 2ec448e1d663..9d1c7177384c 100644 --- a/drivers/gpu/drm/ttm/ttm_tt.c +++ b/drivers/gpu/drm/ttm/ttm_tt.c @@ -50,6 +50,9 @@ int ttm_tt_create(struct ttm_buffer_object *bo, bool zero_alloc) dma_resv_assert_held(bo->base.resv); + if (bo->ttm) + return 0; + if (bdev->need_dma32) page_flags |= TTM_PAGE_FLAG_DMA32; @@ -67,7 +70,6 @@ int ttm_tt_create(struct ttm_buffer_object *bo, bool zero_alloc) page_flags |= TTM_PAGE_FLAG_SG; break; default: - bo->ttm = NULL; pr_err("Illegal buffer object type\n"); return -EINVAL; } @@ -242,7 +244,6 @@ int ttm_tt_init(struct ttm_tt *ttm, struct ttm_buffer_object *bo, ttm_tt_init_fields(ttm, bo, page_flags); if (ttm_tt_alloc_page_directory(ttm)) { - ttm_tt_destroy(ttm); pr_err("Failed allocating page table\n"); return -ENOMEM; } @@ -266,7 +267,6 @@ int ttm_dma_tt_init(struct ttm_dma_tt *ttm_dma, struct ttm_buffer_object *bo, INIT_LIST_HEAD(&ttm_dma->pages_list); if (ttm_dma_tt_alloc_page_directory(ttm_dma)) { - ttm_tt_destroy(ttm); pr_err("Failed allocating page table\n"); return -ENOMEM; } @@ -288,7 +288,6 @@ int ttm_sg_tt_init(struct ttm_dma_tt *ttm_dma, struct ttm_buffer_object *bo, else ret = ttm_dma_tt_alloc_page_directory(ttm_dma); if (ret) { - ttm_tt_destroy(ttm); pr_err("Failed allocating page table\n"); return -ENOMEM; } @@ -311,11 +310,8 @@ EXPORT_SYMBOL(ttm_dma_tt_fini); void ttm_tt_unbind(struct ttm_tt *ttm) { - int ret; - if (ttm->state == tt_bound) { - ret = ttm->func->unbind(ttm); - BUG_ON(ret); + ttm->func->unbind(ttm); ttm->state = tt_unbound; } } |