From e6aabee05f41c9d18e0b92194819edd84f352ac9 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Wed, 27 May 2015 09:26:09 +0200 Subject: iommu/amd: Handle integer overflow in dma_ops_area_alloc Handle this case to make sure boundary_size does not become 0 and trigger a BUG_ON later. Signed-off-by: Joerg Roedel --- drivers/iommu/amd_iommu.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'drivers/iommu/amd_iommu.c') diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index aa710b095f1a..4dfadcfed34a 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -1699,14 +1699,16 @@ static unsigned long dma_ops_area_alloc(struct device *dev, unsigned long next_bit = dom->next_address % APERTURE_RANGE_SIZE; int max_index = dom->aperture_size >> APERTURE_RANGE_SHIFT; int i = start >> APERTURE_RANGE_SHIFT; - unsigned long boundary_size; + unsigned long boundary_size, mask; unsigned long address = -1; unsigned long limit; next_bit >>= PAGE_SHIFT; - boundary_size = ALIGN(dma_get_seg_boundary(dev) + 1, - PAGE_SIZE) >> PAGE_SHIFT; + mask = dma_get_seg_boundary(dev); + + boundary_size = mask + 1 ? ALIGN(mask + 1, PAGE_SIZE) >> PAGE_SHIFT : + 1UL << (BITS_PER_LONG - PAGE_SHIFT); for (;i < max_index; ++i) { unsigned long offset = dom->aperture[i]->offset >> PAGE_SHIFT; -- cgit v1.2.3 From 35cf248f8860b8d2956516d6cdf7e57ff4ed4cbb Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 28 May 2015 18:41:37 +0200 Subject: iommu/amd: Implement dm_region call-backs Add the get_dm_regions and put_dm_regions callbacks to the iommu_ops of the AMD IOMMU driver. Signed-off-by: Joerg Roedel --- drivers/iommu/amd_iommu.c | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) (limited to 'drivers/iommu/amd_iommu.c') diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index e43d48956dea..22d38e3cdc7a 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -3412,6 +3412,47 @@ static bool amd_iommu_capable(enum iommu_cap cap) return false; } +static void amd_iommu_get_dm_regions(struct device *dev, + struct list_head *head) +{ + struct unity_map_entry *entry; + u16 devid; + + devid = get_device_id(dev); + + list_for_each_entry(entry, &amd_iommu_unity_map, list) { + struct iommu_dm_region *region; + + if (devid < entry->devid_start || devid > entry->devid_end) + continue; + + region = kzalloc(sizeof(*region), GFP_KERNEL); + if (!region) { + pr_err("Out of memory allocating dm-regions for %s\n", + dev_name(dev)); + return; + } + + region->start = entry->address_start; + region->length = entry->address_end - entry->address_start; + if (entry->prot & IOMMU_PROT_IR) + region->prot |= IOMMU_READ; + if (entry->prot & IOMMU_PROT_IW) + region->prot |= IOMMU_WRITE; + + list_add_tail(®ion->list, head); + } +} + +static void amd_iommu_put_dm_regions(struct device *dev, + struct list_head *head) +{ + struct iommu_dm_region *entry, *next; + + list_for_each_entry_safe(entry, next, head, list) + kfree(entry); +} + static const struct iommu_ops amd_iommu_ops = { .capable = amd_iommu_capable, .domain_alloc = amd_iommu_domain_alloc, @@ -3422,6 +3463,8 @@ static const struct iommu_ops amd_iommu_ops = { .unmap = amd_iommu_unmap, .map_sg = default_iommu_map_sg, .iova_to_phys = amd_iommu_iova_to_phys, + .get_dm_regions = amd_iommu_get_dm_regions, + .put_dm_regions = amd_iommu_put_dm_regions, .pgsize_bitmap = AMD_IOMMU_PGSIZES, }; -- cgit v1.2.3 From 063071dff53858027e95d3cfcedb1780952302ad Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 28 May 2015 18:41:38 +0200 Subject: iommu/amd: Use default domain if available for DMA-API Signed-off-by: Joerg Roedel --- drivers/iommu/amd_iommu.c | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'drivers/iommu/amd_iommu.c') diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index 22d38e3cdc7a..ae7d636d3099 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -2495,12 +2495,19 @@ void amd_iommu_init_notifier(void) static struct protection_domain *get_domain(struct device *dev) { struct protection_domain *domain; + struct iommu_domain *io_domain; struct dma_ops_domain *dma_dom; u16 devid = get_device_id(dev); if (!check_device(dev)) return ERR_PTR(-EINVAL); + io_domain = iommu_get_domain_for_dev(dev); + if (io_domain) { + domain = to_pdomain(io_domain); + return domain; + } + domain = domain_for_device(dev); if (domain != NULL && !dma_ops_domain(domain)) return ERR_PTR(-EBUSY); -- cgit v1.2.3 From aafd8ba0ca74894b9397e412bbd7f8ea2662ead8 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 28 May 2015 18:41:39 +0200 Subject: iommu/amd: Implement add_device and remove_device Implement these two iommu-ops call-backs to make use of the initialization and notifier features of the iommu core. Signed-off-by: Joerg Roedel --- drivers/iommu/amd_iommu.c | 210 +++++++++++------------------------------ drivers/iommu/amd_iommu_init.c | 31 ++---- 2 files changed, 63 insertions(+), 178 deletions(-) (limited to 'drivers/iommu/amd_iommu.c') diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index ae7d636d3099..0f8776940bf5 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -119,7 +119,7 @@ struct iommu_cmd { struct kmem_cache *amd_iommu_irq_cache; static void update_domain(struct protection_domain *domain); -static int __init alloc_passthrough_domain(void); +static int alloc_passthrough_domain(void); /**************************************************************************** * @@ -434,64 +434,15 @@ static void iommu_uninit_device(struct device *dev) /* Unlink from alias, it may change if another device is re-plugged */ dev_data->alias_data = NULL; + /* Remove dma-ops */ + dev->archdata.dma_ops = NULL; + /* * We keep dev_data around for unplugged devices and reuse it when the * device is re-plugged - not doing so would introduce a ton of races. */ } -void __init amd_iommu_uninit_devices(void) -{ - struct iommu_dev_data *dev_data, *n; - struct pci_dev *pdev = NULL; - - for_each_pci_dev(pdev) { - - if (!check_device(&pdev->dev)) - continue; - - iommu_uninit_device(&pdev->dev); - } - - /* Free all of our dev_data structures */ - list_for_each_entry_safe(dev_data, n, &dev_data_list, dev_data_list) - free_dev_data(dev_data); -} - -int __init amd_iommu_init_devices(void) -{ - struct pci_dev *pdev = NULL; - int ret = 0; - - for_each_pci_dev(pdev) { - - if (!check_device(&pdev->dev)) - continue; - - ret = iommu_init_device(&pdev->dev); - if (ret == -ENOTSUPP) - iommu_ignore_device(&pdev->dev); - else if (ret) - goto out_free; - } - - /* - * Initialize IOMMU groups only after iommu_init_device() has - * had a chance to populate any IVRS defined aliases. - */ - for_each_pci_dev(pdev) { - if (check_device(&pdev->dev)) - init_iommu_group(&pdev->dev); - } - - return 0; - -out_free: - - amd_iommu_uninit_devices(); - - return ret; -} #ifdef CONFIG_AMD_IOMMU_STATS /* @@ -2402,81 +2353,79 @@ static struct protection_domain *domain_for_device(struct device *dev) return dom; } -static int device_change_notifier(struct notifier_block *nb, - unsigned long action, void *data) +static int amd_iommu_add_device(struct device *dev) { struct dma_ops_domain *dma_domain; struct protection_domain *domain; struct iommu_dev_data *dev_data; - struct device *dev = data; struct amd_iommu *iommu; unsigned long flags; u16 devid; + int ret; - if (!check_device(dev)) + if (!check_device(dev) || get_dev_data(dev)) return 0; - devid = get_device_id(dev); - iommu = amd_iommu_rlookup_table[devid]; - dev_data = get_dev_data(dev); - - switch (action) { - case BUS_NOTIFY_ADD_DEVICE: - - iommu_init_device(dev); - init_iommu_group(dev); + devid = get_device_id(dev); + iommu = amd_iommu_rlookup_table[devid]; - /* - * dev_data is still NULL and - * got initialized in iommu_init_device - */ - dev_data = get_dev_data(dev); + ret = iommu_init_device(dev); + if (ret == -ENOTSUPP) { + iommu_ignore_device(dev); + goto out; + } + init_iommu_group(dev); - if (iommu_pass_through || dev_data->iommu_v2) { - dev_data->passthrough = true; - attach_device(dev, pt_domain); - break; - } + dev_data = get_dev_data(dev); - domain = domain_for_device(dev); + if (iommu_pass_through || dev_data->iommu_v2) { + /* Make sure passthrough domain is allocated */ + alloc_passthrough_domain(); + dev_data->passthrough = true; + attach_device(dev, pt_domain); + goto out; + } - /* allocate a protection domain if a device is added */ - dma_domain = find_protection_domain(devid); - if (!dma_domain) { - dma_domain = dma_ops_domain_alloc(); - if (!dma_domain) - goto out; - dma_domain->target_dev = devid; + domain = domain_for_device(dev); - spin_lock_irqsave(&iommu_pd_list_lock, flags); - list_add_tail(&dma_domain->list, &iommu_pd_list); - spin_unlock_irqrestore(&iommu_pd_list_lock, flags); - } + /* allocate a protection domain if a device is added */ + dma_domain = find_protection_domain(devid); + if (!dma_domain) { + dma_domain = dma_ops_domain_alloc(); + if (!dma_domain) + goto out; + dma_domain->target_dev = devid; - dev->archdata.dma_ops = &amd_iommu_dma_ops; + init_unity_mappings_for_device(dma_domain, devid); - break; - case BUS_NOTIFY_REMOVED_DEVICE: + spin_lock_irqsave(&iommu_pd_list_lock, flags); + list_add_tail(&dma_domain->list, &iommu_pd_list); + spin_unlock_irqrestore(&iommu_pd_list_lock, flags); + } - iommu_uninit_device(dev); + attach_device(dev, &dma_domain->domain); - default: - goto out; - } + dev->archdata.dma_ops = &amd_iommu_dma_ops; +out: iommu_completion_wait(iommu); -out: return 0; } -static struct notifier_block device_nb = { - .notifier_call = device_change_notifier, -}; - -void amd_iommu_init_notifier(void) +static void amd_iommu_remove_device(struct device *dev) { - bus_register_notifier(&pci_bus_type, &device_nb); + struct amd_iommu *iommu; + u16 devid; + + if (!check_device(dev)) + return; + + devid = get_device_id(dev); + iommu = amd_iommu_rlookup_table[devid]; + + iommu_uninit_device(dev); + iommu_completion_wait(iommu); } /***************************************************************************** @@ -3018,54 +2967,6 @@ static int amd_iommu_dma_supported(struct device *dev, u64 mask) return check_device(dev); } -/* - * The function for pre-allocating protection domains. - * - * If the driver core informs the DMA layer if a driver grabs a device - * we don't need to preallocate the protection domains anymore. - * For now we have to. - */ -static void __init prealloc_protection_domains(void) -{ - struct iommu_dev_data *dev_data; - struct dma_ops_domain *dma_dom; - struct pci_dev *dev = NULL; - u16 devid; - - for_each_pci_dev(dev) { - - /* Do we handle this device? */ - if (!check_device(&dev->dev)) - continue; - - dev_data = get_dev_data(&dev->dev); - if (!amd_iommu_force_isolation && dev_data->iommu_v2) { - /* Make sure passthrough domain is allocated */ - alloc_passthrough_domain(); - dev_data->passthrough = true; - attach_device(&dev->dev, pt_domain); - pr_info("AMD-Vi: Using passthrough domain for device %s\n", - dev_name(&dev->dev)); - } - - /* Is there already any domain for it? */ - if (domain_for_device(&dev->dev)) - continue; - - devid = get_device_id(&dev->dev); - - dma_dom = dma_ops_domain_alloc(); - if (!dma_dom) - continue; - init_unity_mappings_for_device(dma_dom, devid); - dma_dom->target_dev = devid; - - attach_device(&dev->dev, &dma_dom->domain); - - list_add_tail(&dma_dom->list, &iommu_pd_list); - } -} - static struct dma_map_ops amd_iommu_dma_ops = { .alloc = alloc_coherent, .free = free_coherent, @@ -3131,11 +3032,6 @@ int __init amd_iommu_init_dma_ops(void) goto free_domains; } - /* - * Pre-allocate the protection domains for each device. - */ - prealloc_protection_domains(); - iommu_detected = 1; swiotlb = 0; @@ -3228,7 +3124,7 @@ out_err: return NULL; } -static int __init alloc_passthrough_domain(void) +static int alloc_passthrough_domain(void) { if (pt_domain != NULL) return 0; @@ -3470,6 +3366,8 @@ static const struct iommu_ops amd_iommu_ops = { .unmap = amd_iommu_unmap, .map_sg = default_iommu_map_sg, .iova_to_phys = amd_iommu_iova_to_phys, + .add_device = amd_iommu_add_device, + .remove_device = amd_iommu_remove_device, .get_dm_regions = amd_iommu_get_dm_regions, .put_dm_regions = amd_iommu_put_dm_regions, .pgsize_bitmap = AMD_IOMMU_PGSIZES, diff --git a/drivers/iommu/amd_iommu_init.c b/drivers/iommu/amd_iommu_init.c index 450ef5001a65..e4a6e405e35d 100644 --- a/drivers/iommu/amd_iommu_init.c +++ b/drivers/iommu/amd_iommu_init.c @@ -226,6 +226,7 @@ static enum iommu_init_state init_state = IOMMU_START_STATE; static int amd_iommu_enable_interrupts(void); static int __init iommu_go_to_state(enum iommu_init_state state); +static void init_device_table_dma(void); static inline void update_last_devid(u16 devid) { @@ -1385,7 +1386,12 @@ static int __init amd_iommu_init_pci(void) break; } - ret = amd_iommu_init_devices(); + init_device_table_dma(); + + for_each_iommu(iommu) + iommu_flush_all_caches(iommu); + + amd_iommu_init_api(); print_iommu_info(); @@ -1825,8 +1831,6 @@ static bool __init check_ioapic_information(void) static void __init free_dma_resources(void) { - amd_iommu_uninit_devices(); - free_pages((unsigned long)amd_iommu_pd_alloc_bitmap, get_order(MAX_DOMAIN_ID/8)); @@ -2019,27 +2023,10 @@ static bool detect_ivrs(void) static int amd_iommu_init_dma(void) { - struct amd_iommu *iommu; - int ret; - if (iommu_pass_through) - ret = amd_iommu_init_passthrough(); + return amd_iommu_init_passthrough(); else - ret = amd_iommu_init_dma_ops(); - - if (ret) - return ret; - - init_device_table_dma(); - - for_each_iommu(iommu) - iommu_flush_all_caches(iommu); - - amd_iommu_init_api(); - - amd_iommu_init_notifier(); - - return 0; + return amd_iommu_init_dma_ops(); } /**************************************************************************** -- cgit v1.2.3 From 0bb6e243d7fbb39fced5bd4a4c83eb49c6e820ce Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 28 May 2015 18:41:40 +0200 Subject: iommu/amd: Support IOMMU_DOMAIN_DMA type allocation This enables allocation of DMA-API default domains from the IOMMU core and switches allocation of domain dma-api domain to the IOMMU core too. Signed-off-by: Joerg Roedel --- drivers/iommu/amd_iommu.c | 311 ++++++++++------------------------------ drivers/iommu/amd_iommu_types.h | 3 - 2 files changed, 73 insertions(+), 241 deletions(-) (limited to 'drivers/iommu/amd_iommu.c') diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index 0f8776940bf5..27300aece203 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -64,10 +64,6 @@ static DEFINE_RWLOCK(amd_iommu_devtable_lock); -/* A list of preallocated protection domains */ -static LIST_HEAD(iommu_pd_list); -static DEFINE_SPINLOCK(iommu_pd_list_lock); - /* List of all available dev_data structures */ static LIST_HEAD(dev_data_list); static DEFINE_SPINLOCK(dev_data_list_lock); @@ -234,31 +230,38 @@ static bool pdev_pri_erratum(struct pci_dev *pdev, u32 erratum) } /* - * In this function the list of preallocated protection domains is traversed to - * find the domain for a specific device + * This function actually applies the mapping to the page table of the + * dma_ops domain. */ -static struct dma_ops_domain *find_protection_domain(u16 devid) +static void alloc_unity_mapping(struct dma_ops_domain *dma_dom, + struct unity_map_entry *e) { - struct dma_ops_domain *entry, *ret = NULL; - unsigned long flags; - u16 alias = amd_iommu_alias_table[devid]; - - if (list_empty(&iommu_pd_list)) - return NULL; - - spin_lock_irqsave(&iommu_pd_list_lock, flags); + u64 addr; - list_for_each_entry(entry, &iommu_pd_list, list) { - if (entry->target_dev == devid || - entry->target_dev == alias) { - ret = entry; - break; - } + for (addr = e->address_start; addr < e->address_end; + addr += PAGE_SIZE) { + if (addr < dma_dom->aperture_size) + __set_bit(addr >> PAGE_SHIFT, + dma_dom->aperture[0]->bitmap); } +} + +/* + * Inits the unity mappings required for a specific device + */ +static void init_unity_mappings_for_device(struct device *dev, + struct dma_ops_domain *dma_dom) +{ + struct unity_map_entry *e; + u16 devid; - spin_unlock_irqrestore(&iommu_pd_list_lock, flags); + devid = get_device_id(dev); - return ret; + list_for_each_entry(e, &amd_iommu_unity_map, list) { + if (!(devid >= e->devid_start && devid <= e->devid_end)) + continue; + alloc_unity_mapping(dma_dom, e); + } } /* @@ -290,11 +293,23 @@ static bool check_device(struct device *dev) static void init_iommu_group(struct device *dev) { + struct dma_ops_domain *dma_domain; + struct iommu_domain *domain; struct iommu_group *group; group = iommu_group_get_for_dev(dev); - if (!IS_ERR(group)) - iommu_group_put(group); + if (IS_ERR(group)) + return; + + domain = iommu_group_default_domain(group); + if (!domain) + goto out; + + dma_domain = to_pdomain(domain)->priv; + + init_unity_mappings_for_device(dev, dma_domain); +out: + iommu_group_put(group); } static int __last_alias(struct pci_dev *pdev, u16 alias, void *data) @@ -1414,94 +1429,6 @@ static unsigned long iommu_unmap_page(struct protection_domain *dom, return unmapped; } -/* - * This function checks if a specific unity mapping entry is needed for - * this specific IOMMU. - */ -static int iommu_for_unity_map(struct amd_iommu *iommu, - struct unity_map_entry *entry) -{ - u16 bdf, i; - - for (i = entry->devid_start; i <= entry->devid_end; ++i) { - bdf = amd_iommu_alias_table[i]; - if (amd_iommu_rlookup_table[bdf] == iommu) - return 1; - } - - return 0; -} - -/* - * This function actually applies the mapping to the page table of the - * dma_ops domain. - */ -static int dma_ops_unity_map(struct dma_ops_domain *dma_dom, - struct unity_map_entry *e) -{ - u64 addr; - int ret; - - for (addr = e->address_start; addr < e->address_end; - addr += PAGE_SIZE) { - ret = iommu_map_page(&dma_dom->domain, addr, addr, e->prot, - PAGE_SIZE); - if (ret) - return ret; - /* - * if unity mapping is in aperture range mark the page - * as allocated in the aperture - */ - if (addr < dma_dom->aperture_size) - __set_bit(addr >> PAGE_SHIFT, - dma_dom->aperture[0]->bitmap); - } - - return 0; -} - -/* - * Init the unity mappings for a specific IOMMU in the system - * - * Basically iterates over all unity mapping entries and applies them to - * the default domain DMA of that IOMMU if necessary. - */ -static int iommu_init_unity_mappings(struct amd_iommu *iommu) -{ - struct unity_map_entry *entry; - int ret; - - list_for_each_entry(entry, &amd_iommu_unity_map, list) { - if (!iommu_for_unity_map(iommu, entry)) - continue; - ret = dma_ops_unity_map(iommu->default_dom, entry); - if (ret) - return ret; - } - - return 0; -} - -/* - * Inits the unity mappings required for a specific device - */ -static int init_unity_mappings_for_device(struct dma_ops_domain *dma_dom, - u16 devid) -{ - struct unity_map_entry *e; - int ret; - - list_for_each_entry(e, &amd_iommu_unity_map, list) { - if (!(devid >= e->devid_start && devid <= e->devid_end)) - continue; - ret = dma_ops_unity_map(dma_dom, e); - if (ret) - return ret; - } - - return 0; -} - /**************************************************************************** * * The next functions belong to the address allocator for the dma_ops @@ -2324,42 +2251,9 @@ static void detach_device(struct device *dev) dev_data->ats.enabled = false; } -/* - * Find out the protection domain structure for a given PCI device. This - * will give us the pointer to the page table root for example. - */ -static struct protection_domain *domain_for_device(struct device *dev) -{ - struct iommu_dev_data *dev_data; - struct protection_domain *dom = NULL; - unsigned long flags; - - dev_data = get_dev_data(dev); - - if (dev_data->domain) - return dev_data->domain; - - if (dev_data->alias_data != NULL) { - struct iommu_dev_data *alias_data = dev_data->alias_data; - - read_lock_irqsave(&amd_iommu_devtable_lock, flags); - if (alias_data->domain != NULL) { - __attach_device(dev_data, alias_data->domain); - dom = alias_data->domain; - } - read_unlock_irqrestore(&amd_iommu_devtable_lock, flags); - } - - return dom; -} - static int amd_iommu_add_device(struct device *dev) { - struct dma_ops_domain *dma_domain; - struct protection_domain *domain; - struct iommu_dev_data *dev_data; struct amd_iommu *iommu; - unsigned long flags; u16 devid; int ret; @@ -2376,35 +2270,6 @@ static int amd_iommu_add_device(struct device *dev) } init_iommu_group(dev); - dev_data = get_dev_data(dev); - - if (iommu_pass_through || dev_data->iommu_v2) { - /* Make sure passthrough domain is allocated */ - alloc_passthrough_domain(); - dev_data->passthrough = true; - attach_device(dev, pt_domain); - goto out; - } - - domain = domain_for_device(dev); - - /* allocate a protection domain if a device is added */ - dma_domain = find_protection_domain(devid); - if (!dma_domain) { - dma_domain = dma_ops_domain_alloc(); - if (!dma_domain) - goto out; - dma_domain->target_dev = devid; - - init_unity_mappings_for_device(dma_domain, devid); - - spin_lock_irqsave(&iommu_pd_list_lock, flags); - list_add_tail(&dma_domain->list, &iommu_pd_list); - spin_unlock_irqrestore(&iommu_pd_list_lock, flags); - } - - attach_device(dev, &dma_domain->domain); - dev->archdata.dma_ops = &amd_iommu_dma_ops; out: @@ -2445,34 +2310,19 @@ static struct protection_domain *get_domain(struct device *dev) { struct protection_domain *domain; struct iommu_domain *io_domain; - struct dma_ops_domain *dma_dom; - u16 devid = get_device_id(dev); if (!check_device(dev)) return ERR_PTR(-EINVAL); io_domain = iommu_get_domain_for_dev(dev); - if (io_domain) { - domain = to_pdomain(io_domain); - return domain; - } + if (!io_domain) + return NULL; - domain = domain_for_device(dev); - if (domain != NULL && !dma_ops_domain(domain)) + domain = to_pdomain(io_domain); + if (!dma_ops_domain(domain)) return ERR_PTR(-EBUSY); - if (domain != NULL) - return domain; - - /* Device not bound yet - bind it */ - dma_dom = find_protection_domain(devid); - if (!dma_dom) - dma_dom = amd_iommu_rlookup_table[devid]->default_dom; - attach_device(dev, &dma_dom->domain); - DUMP_printk("Using protection domain %d for device %s\n", - dma_dom->domain.id, dev_name(dev)); - - return &dma_dom->domain; + return domain; } static void update_device_table(struct protection_domain *domain) @@ -3014,23 +2864,7 @@ void __init amd_iommu_init_api(void) int __init amd_iommu_init_dma_ops(void) { - struct amd_iommu *iommu; - int ret, unhandled; - - /* - * first allocate a default protection domain for every IOMMU we - * found in the system. Devices not assigned to any other - * protection domain will be assigned to the default one. - */ - for_each_iommu(iommu) { - iommu->default_dom = dma_ops_domain_alloc(); - if (iommu->default_dom == NULL) - return -ENOMEM; - iommu->default_dom->domain.flags |= PD_DEFAULT_MASK; - ret = iommu_init_unity_mappings(iommu); - if (ret) - goto free_domains; - } + int unhandled; iommu_detected = 1; swiotlb = 0; @@ -3050,14 +2884,6 @@ int __init amd_iommu_init_dma_ops(void) pr_info("AMD-Vi: Lazy IO/TLB flushing enabled\n"); return 0; - -free_domains: - - for_each_iommu(iommu) { - dma_ops_domain_free(iommu->default_dom); - } - - return ret; } /***************************************************************************** @@ -3142,30 +2968,39 @@ static int alloc_passthrough_domain(void) static struct iommu_domain *amd_iommu_domain_alloc(unsigned type) { struct protection_domain *pdomain; + struct dma_ops_domain *dma_domain; - /* We only support unmanaged domains for now */ - if (type != IOMMU_DOMAIN_UNMANAGED) - return NULL; + switch (type) { + case IOMMU_DOMAIN_UNMANAGED: + pdomain = protection_domain_alloc(); + if (!pdomain) + return NULL; - pdomain = protection_domain_alloc(); - if (!pdomain) - goto out_free; + pdomain->mode = PAGE_MODE_3_LEVEL; + pdomain->pt_root = (void *)get_zeroed_page(GFP_KERNEL); + if (!pdomain->pt_root) { + protection_domain_free(pdomain); + return NULL; + } - pdomain->mode = PAGE_MODE_3_LEVEL; - pdomain->pt_root = (void *)get_zeroed_page(GFP_KERNEL); - if (!pdomain->pt_root) - goto out_free; + pdomain->domain.geometry.aperture_start = 0; + pdomain->domain.geometry.aperture_end = ~0ULL; + pdomain->domain.geometry.force_aperture = true; - pdomain->domain.geometry.aperture_start = 0; - pdomain->domain.geometry.aperture_end = ~0ULL; - pdomain->domain.geometry.force_aperture = true; + break; + case IOMMU_DOMAIN_DMA: + dma_domain = dma_ops_domain_alloc(); + if (!dma_domain) { + pr_err("AMD-Vi: Failed to allocate\n"); + return NULL; + } + pdomain = &dma_domain->domain; + break; + default: + return NULL; + } return &pdomain->domain; - -out_free: - protection_domain_free(pdomain); - - return NULL; } static void amd_iommu_domain_free(struct iommu_domain *dom) diff --git a/drivers/iommu/amd_iommu_types.h b/drivers/iommu/amd_iommu_types.h index 05030e523771..fe796cfc62ca 100644 --- a/drivers/iommu/amd_iommu_types.h +++ b/drivers/iommu/amd_iommu_types.h @@ -552,9 +552,6 @@ struct amd_iommu { /* if one, we need to send a completion wait command */ bool need_sync; - /* default dma_ops domain for that IOMMU */ - struct dma_ops_domain *default_dom; - /* IOMMU sysfs device */ struct device *iommu_dev; -- cgit v1.2.3 From 07f643a35d6b50f0f091444f07db1353188e787e Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 28 May 2015 18:41:41 +0200 Subject: iommu/amd: Support IOMMU_DOMAIN_IDENTITY type allocation Add support to allocate direct mapped domains through the IOMMU-API. Signed-off-by: Joerg Roedel --- drivers/iommu/amd_iommu.c | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'drivers/iommu/amd_iommu.c') diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index 27300aece203..188b81d56ac4 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -2996,6 +2996,13 @@ static struct iommu_domain *amd_iommu_domain_alloc(unsigned type) } pdomain = &dma_domain->domain; break; + case IOMMU_DOMAIN_IDENTITY: + pdomain = protection_domain_alloc(); + if (!pdomain) + return NULL; + + pdomain->mode = PAGE_MODE_NONE; + break; default: return NULL; } -- cgit v1.2.3 From 07ee86948c9111d49583be462500042fedfecb4a Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 28 May 2015 18:41:42 +0200 Subject: iommu/amd: Put IOMMUv2 devices in a direct mapped domain A device that might be used for HSA needs to be in a direct mapped domain so that all DMA-API mappings stay alive when the IOMMUv2 stack is used. Signed-off-by: Joerg Roedel --- drivers/iommu/amd_iommu.c | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) (limited to 'drivers/iommu/amd_iommu.c') diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index 188b81d56ac4..623be17e28d7 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -2253,6 +2253,8 @@ static void detach_device(struct device *dev) static int amd_iommu_add_device(struct device *dev) { + struct iommu_dev_data *dev_data; + struct iommu_domain *domain; struct amd_iommu *iommu; u16 devid; int ret; @@ -2270,7 +2272,18 @@ static int amd_iommu_add_device(struct device *dev) } init_iommu_group(dev); - dev->archdata.dma_ops = &amd_iommu_dma_ops; + dev_data = get_dev_data(dev); + if (dev_data && dev_data->iommu_v2) + iommu_request_dm_for_dev(dev); + + /* Domains are initialized for this device - have a look what we ended up with */ + domain = iommu_get_domain_for_dev(dev); + if (domain->type == IOMMU_DOMAIN_IDENTITY) { + dev_data->passthrough = true; + dev->archdata.dma_ops = &nommu_dma_ops; + } else { + dev->archdata.dma_ops = &amd_iommu_dma_ops; + } out: iommu_completion_wait(iommu); -- cgit v1.2.3 From 343e9cac9c4a2715e8fa6b89b04e31e9fcbae5e3 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 28 May 2015 18:41:43 +0200 Subject: iommu/amd: Get rid of device_dma_ops_init() With device intialization done in the add_device call-back now there is no reason for this function anymore. Signed-off-by: Joerg Roedel --- drivers/iommu/amd_iommu.c | 40 +--------------------------------------- 1 file changed, 1 insertion(+), 39 deletions(-) (limited to 'drivers/iommu/amd_iommu.c') diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index 623be17e28d7..fadda2c0cd18 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -2268,6 +2268,7 @@ static int amd_iommu_add_device(struct device *dev) ret = iommu_init_device(dev); if (ret == -ENOTSUPP) { iommu_ignore_device(dev); + dev->archdata.dma_ops = &nommu_dma_ops; goto out; } init_iommu_group(dev); @@ -2840,36 +2841,6 @@ static struct dma_map_ops amd_iommu_dma_ops = { .dma_supported = amd_iommu_dma_supported, }; -static unsigned device_dma_ops_init(void) -{ - struct iommu_dev_data *dev_data; - struct pci_dev *pdev = NULL; - unsigned unhandled = 0; - - for_each_pci_dev(pdev) { - if (!check_device(&pdev->dev)) { - - iommu_ignore_device(&pdev->dev); - - unhandled += 1; - continue; - } - - dev_data = get_dev_data(&pdev->dev); - - if (!dev_data->passthrough) - pdev->dev.archdata.dma_ops = &amd_iommu_dma_ops; - else - pdev->dev.archdata.dma_ops = &nommu_dma_ops; - } - - return unhandled; -} - -/* - * The function which clues the AMD IOMMU driver into dma_ops. - */ - void __init amd_iommu_init_api(void) { bus_set_iommu(&pci_bus_type, &amd_iommu_ops); @@ -2877,18 +2848,9 @@ void __init amd_iommu_init_api(void) int __init amd_iommu_init_dma_ops(void) { - int unhandled; - iommu_detected = 1; swiotlb = 0; - /* Make the driver finally visible to the drivers */ - unhandled = device_dma_ops_init(); - if (unhandled && max_pfn > MAX_DMA32_PFN) { - /* There are unhandled devices - initialize swiotlb for them */ - swiotlb = 1; - } - amd_iommu_stats_init(); if (amd_iommu_unmap_flush) -- cgit v1.2.3 From 2870b0a4911038fd6aed9093cda2dbe80fd0ee2e Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 28 May 2015 18:41:44 +0200 Subject: iommu/amd: Remove unused fields from struct dma_ops_domain The list_head and target_dev members are not used anymore. Remove them. Signed-off-by: Joerg Roedel --- drivers/iommu/amd_iommu.c | 1 - drivers/iommu/amd_iommu_types.h | 8 -------- 2 files changed, 9 deletions(-) (limited to 'drivers/iommu/amd_iommu.c') diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index fadda2c0cd18..96390b9b7842 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -1886,7 +1886,6 @@ static struct dma_ops_domain *dma_ops_domain_alloc(void) goto free_dma_dom; dma_dom->need_flush = false; - dma_dom->target_dev = 0xffff; add_domain_to_list(&dma_dom->domain); diff --git a/drivers/iommu/amd_iommu_types.h b/drivers/iommu/amd_iommu_types.h index fe796cfc62ca..bb56560ac6ca 100644 --- a/drivers/iommu/amd_iommu_types.h +++ b/drivers/iommu/amd_iommu_types.h @@ -446,8 +446,6 @@ struct aperture_range { * Data container for a dma_ops specific protection domain */ struct dma_ops_domain { - struct list_head list; - /* generic protection domain information */ struct protection_domain domain; @@ -462,12 +460,6 @@ struct dma_ops_domain { /* This will be set to true when TLB needs to be flushed */ bool need_flush; - - /* - * if this is a preallocated domain, keep the device for which it was - * preallocated in this variable - */ - u16 target_dev; }; /* -- cgit v1.2.3 From 3a18404cd952ae529651f72a13e5d6ffee824c2e Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 28 May 2015 18:41:45 +0200 Subject: iommu/amd: Propagate errors from amd_iommu_init_api This function can fail. Propagate any errors back to the initialization state machine. Signed-off-by: Joerg Roedel --- drivers/iommu/amd_iommu.c | 4 ++-- drivers/iommu/amd_iommu_init.c | 5 +++-- drivers/iommu/amd_iommu_proto.h | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) (limited to 'drivers/iommu/amd_iommu.c') diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index 96390b9b7842..8bc6f40edf5e 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -2840,9 +2840,9 @@ static struct dma_map_ops amd_iommu_dma_ops = { .dma_supported = amd_iommu_dma_supported, }; -void __init amd_iommu_init_api(void) +int __init amd_iommu_init_api(void) { - bus_set_iommu(&pci_bus_type, &amd_iommu_ops); + return bus_set_iommu(&pci_bus_type, &amd_iommu_ops); } int __init amd_iommu_init_dma_ops(void) diff --git a/drivers/iommu/amd_iommu_init.c b/drivers/iommu/amd_iommu_init.c index e4a6e405e35d..dbac49cea7a1 100644 --- a/drivers/iommu/amd_iommu_init.c +++ b/drivers/iommu/amd_iommu_init.c @@ -1391,9 +1391,10 @@ static int __init amd_iommu_init_pci(void) for_each_iommu(iommu) iommu_flush_all_caches(iommu); - amd_iommu_init_api(); + ret = amd_iommu_init_api(); - print_iommu_info(); + if (!ret) + print_iommu_info(); return ret; } diff --git a/drivers/iommu/amd_iommu_proto.h b/drivers/iommu/amd_iommu_proto.h index 72b0fd455e24..9ed1c4330551 100644 --- a/drivers/iommu/amd_iommu_proto.h +++ b/drivers/iommu/amd_iommu_proto.h @@ -30,7 +30,7 @@ extern void amd_iommu_reset_cmd_buffer(struct amd_iommu *iommu); extern int amd_iommu_init_devices(void); extern void amd_iommu_uninit_devices(void); extern void amd_iommu_init_notifier(void); -extern void amd_iommu_init_api(void); +extern int amd_iommu_init_api(void); /* Needed for interrupt remapping */ extern int amd_iommu_prepare(void); -- cgit v1.2.3 From 4d58b8a6de6b8c3611f10124f83e90e5a2406437 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 11 Jun 2015 09:21:39 +0200 Subject: iommu/amd: Handle errors returned from iommu_init_device Without this patch only -ENOTSUPP is handled, but there are other possible errors. Handle them too. Reported-by: Dan Carpenter Signed-off-by: Joerg Roedel --- drivers/iommu/amd_iommu.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) (limited to 'drivers/iommu/amd_iommu.c') diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index 8bc6f40edf5e..e5b77d398e9b 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -2265,7 +2265,11 @@ static int amd_iommu_add_device(struct device *dev) iommu = amd_iommu_rlookup_table[devid]; ret = iommu_init_device(dev); - if (ret == -ENOTSUPP) { + if (ret) { + if (ret != -ENOTSUPP) + pr_err("Failed to initialize device %s - trying to proceed anyway\n", + dev_name(dev)); + iommu_ignore_device(dev); dev->archdata.dma_ops = &nommu_dma_ops; goto out; @@ -2273,7 +2277,10 @@ static int amd_iommu_add_device(struct device *dev) init_iommu_group(dev); dev_data = get_dev_data(dev); - if (dev_data && dev_data->iommu_v2) + + BUG_ON(!dev_data); + + if (dev_data->iommu_v2) iommu_request_dm_for_dev(dev); /* Domains are initialized for this device - have a look what we ended up with */ -- cgit v1.2.3 From 0b3fff54bc01e8e6064d222a33e6fa7adabd94cd Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 18 Jun 2015 10:48:34 +0200 Subject: iommu/amd: Handle large pages correctly in free_pagetable Make sure that we are skipping over large PTEs while walking the page-table tree. Cc: stable@kernel.org Fixes: 5c34c403b723 ("iommu/amd: Fix memory leak in free_pagetable") Signed-off-by: Joerg Roedel --- drivers/iommu/amd_iommu.c | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'drivers/iommu/amd_iommu.c') diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index 4dfadcfed34a..31e90c472c76 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -1866,9 +1866,15 @@ static void free_pt_##LVL (unsigned long __pt) \ pt = (u64 *)__pt; \ \ for (i = 0; i < 512; ++i) { \ + /* PTE present? */ \ if (!IOMMU_PTE_PRESENT(pt[i])) \ continue; \ \ + /* Large PTE? */ \ + if (PM_PTE_LEVEL(pt[i]) == 0 || \ + PM_PTE_LEVEL(pt[i]) == 7) \ + continue; \ + \ p = (unsigned long)IOMMU_PTE_PAGE(pt[i]); \ FN(p); \ } \ -- cgit v1.2.3