diff --git a/drivers/dma/tegra210-adma.c b/drivers/dma/tegra210-adma.c index a838ad440ec4..66906a91aa62 100644 --- a/drivers/dma/tegra210-adma.c +++ b/drivers/dma/tegra210-adma.c @@ -415,17 +415,17 @@ static irqreturn_t tegra_adma_isr(int irq, void *dev_id) struct tegra_adma_chan *tdc = dev_id; unsigned long status; - spin_lock(&tdc->vc.lock); + dma_vchan_lock(&tdc->vc); status = tegra_adma_irq_clear(tdc); if (status == 0 || !tdc->desc) { - spin_unlock(&tdc->vc.lock); + dma_vchan_unlock(&tdc->vc); return IRQ_NONE; } vchan_cyclic_callback(&tdc->desc->vd); - spin_unlock(&tdc->vc.lock); + dma_vchan_unlock(&tdc->vc); return IRQ_HANDLED; } @@ -435,14 +435,14 @@ static void tegra_adma_issue_pending(struct dma_chan *dc) struct tegra_adma_chan *tdc = to_tegra_adma_chan(dc); unsigned long flags; - spin_lock_irqsave(&tdc->vc.lock, flags); + dma_vchan_lock_irqsave(&tdc->vc, flags); if (vchan_issue_pending(&tdc->vc)) { if (!tdc->desc) tegra_adma_start(tdc); } - spin_unlock_irqrestore(&tdc->vc.lock, flags); + dma_vchan_unlock_irqrestore(&tdc->vc, flags); } static bool tegra_adma_is_paused(struct tegra_adma_chan *tdc) @@ -496,14 +496,14 @@ static int tegra_adma_terminate_all(struct dma_chan *dc) unsigned long flags; LIST_HEAD(head); - spin_lock_irqsave(&tdc->vc.lock, flags); + dma_vchan_lock_irqsave(&tdc->vc, flags); if (tdc->desc) tegra_adma_stop(tdc); tegra_adma_request_free(tdc); vchan_get_all_descriptors(&tdc->vc, &head); - spin_unlock_irqrestore(&tdc->vc.lock, flags); + dma_vchan_unlock_irqrestore(&tdc->vc, flags); vchan_dma_desc_free_list(&tdc->vc, &head); return 0; @@ -524,7 +524,7 @@ static enum dma_status tegra_adma_tx_status(struct dma_chan *dc, if (ret == DMA_COMPLETE || !txstate) return ret; - spin_lock_irqsave(&tdc->vc.lock, flags); + dma_vchan_lock_irqsave(&tdc->vc, flags); vd = vchan_find_desc(&tdc->vc, cookie); if (vd) { @@ -536,7 +536,7 @@ static enum dma_status tegra_adma_tx_status(struct dma_chan *dc, residual = 0; } - spin_unlock_irqrestore(&tdc->vc.lock, flags); + dma_vchan_unlock_irqrestore(&tdc->vc, flags); dma_set_residue(txstate, residual); @@ -936,6 +936,11 @@ static int tegra_adma_probe(struct platform_device *pdev) goto irq_dispose; } + if (tdma->is_virtualized) + tdc->vc.rt_spinlock_fix_enabled = DMA_RT_SPINLOCK_FIX_ENABLED; + else + tdc->vc.rt_spinlock_fix_enabled = 0; + vchan_init(&tdc->vc, &tdma->dma_dev); tdc->vc.desc_free = tegra_adma_desc_free; tdc->tdma = tdma; diff --git a/drivers/dma/virt-dma.c b/drivers/dma/virt-dma.c index a6f4265be0c9..5436729f7934 100644 --- a/drivers/dma/virt-dma.c +++ b/drivers/dma/virt-dma.c @@ -23,11 +23,11 @@ dma_cookie_t vchan_tx_submit(struct dma_async_tx_descriptor *tx) unsigned long flags; dma_cookie_t cookie; - spin_lock_irqsave(&vc->lock, flags); + dma_vchan_lock_irqsave(vc, flags); cookie = dma_cookie_assign(tx); list_move_tail(&vd->node, &vc->desc_submitted); - spin_unlock_irqrestore(&vc->lock, flags); + dma_vchan_unlock_irqrestore(vc, flags); dev_dbg(vc->chan.device->dev, "vchan %p: txd %p[%x]: submitted\n", vc, vd, cookie); @@ -52,9 +52,9 @@ int vchan_tx_desc_free(struct dma_async_tx_descriptor *tx) struct virt_dma_desc *vd = to_virt_desc(tx); unsigned long flags; - spin_lock_irqsave(&vc->lock, flags); + dma_vchan_lock_irqsave(vc, flags); list_del(&vd->node); - spin_unlock_irqrestore(&vc->lock, flags); + dma_vchan_unlock_irqrestore(vc, flags); dev_dbg(vc->chan.device->dev, "vchan %p: txd %p[%x]: freeing\n", vc, vd, vd->tx.cookie); @@ -87,7 +87,7 @@ static void vchan_complete(struct tasklet_struct *t) struct dmaengine_desc_callback cb; LIST_HEAD(head); - spin_lock_irq(&vc->lock); + dma_vchan_lock_irq(vc); list_splice_tail_init(&vc->desc_completed, &head); vd = vc->cyclic; if (vd) { @@ -96,7 +96,7 @@ static void vchan_complete(struct tasklet_struct *t) } else { memset(&cb, 0, sizeof(cb)); } - spin_unlock_irq(&vc->lock); + dma_vchan_unlock_irq(vc); dmaengine_desc_callback_invoke(&cb, &vd->tx_result); @@ -124,7 +124,7 @@ void vchan_init(struct virt_dma_chan *vc, struct dma_device *dmadev) { dma_cookie_init(&vc->chan); - spin_lock_init(&vc->lock); + dma_vchan_lock_init(vc); INIT_LIST_HEAD(&vc->desc_allocated); INIT_LIST_HEAD(&vc->desc_submitted); INIT_LIST_HEAD(&vc->desc_issued); diff --git a/drivers/dma/virt-dma.h b/drivers/dma/virt-dma.h index e9f5250fbe4d..b19a0f9cee92 100644 --- a/drivers/dma/virt-dma.h +++ b/drivers/dma/virt-dma.h @@ -34,13 +34,75 @@ struct virt_dma_chan { struct list_head desc_terminated; struct virt_dma_desc *cyclic; + + /* To-do: Upstream review for this WAR */ + raw_spinlock_t rawlock; + uint32_t rt_spinlock_fix_enabled; }; +#define DMA_RT_SPINLOCK_FIX_ENABLED 0xAF5FA2B1 + static inline struct virt_dma_chan *to_virt_chan(struct dma_chan *chan) { return container_of(chan, struct virt_dma_chan, chan); } +#define dma_vchan_lock_init(vc) \ +do { \ + if (DMA_RT_SPINLOCK_FIX_ENABLED == (vc)->rt_spinlock_fix_enabled) \ + raw_spin_lock_init(&(vc)->rawlock); \ + else \ + spin_lock_init(&(vc)->lock); \ +} while (0) + +#define dma_vchan_lock(vc) \ +do { \ + if (DMA_RT_SPINLOCK_FIX_ENABLED == (vc)->rt_spinlock_fix_enabled) \ + raw_spin_lock(&(vc)->rawlock); \ + else \ + spin_lock(&(vc)->lock); \ +} while (0) + +#define dma_vchan_unlock(vc) \ +do { \ + if (DMA_RT_SPINLOCK_FIX_ENABLED == (vc)->rt_spinlock_fix_enabled) \ + raw_spin_unlock(&(vc)->rawlock); \ + else \ + spin_unlock(&(vc)->lock); \ +} while (0) + +#define dma_vchan_lock_irq(vc) \ +do { \ + if (DMA_RT_SPINLOCK_FIX_ENABLED == (vc)->rt_spinlock_fix_enabled) \ + raw_spin_lock_irq(&(vc)->rawlock); \ + else \ + spin_lock_irq(&(vc)->lock); \ +} while (0) + +#define dma_vchan_unlock_irq(vc) \ +do { \ + if (DMA_RT_SPINLOCK_FIX_ENABLED == (vc)->rt_spinlock_fix_enabled) \ + raw_spin_unlock_irq(&(vc)->rawlock); \ + else \ + spin_unlock_irq(&(vc)->lock); \ +} while (0) + +#define dma_vchan_lock_irqsave(vc, flags) \ +do { \ + if (DMA_RT_SPINLOCK_FIX_ENABLED == (vc)->rt_spinlock_fix_enabled) \ + raw_spin_lock_irqsave(&(vc)->rawlock, flags); \ + else \ + spin_lock_irqsave(&(vc)->lock, flags); \ +} while (0) + +#define dma_vchan_unlock_irqrestore(vc, flags) \ +do { \ + if (DMA_RT_SPINLOCK_FIX_ENABLED == (vc)->rt_spinlock_fix_enabled) \ + raw_spin_unlock_irqrestore(&(vc)->rawlock, flags); \ + else \ + spin_unlock_irqrestore(&(vc)->lock, flags); \ +} while (0) + void vchan_dma_desc_free_list(struct virt_dma_chan *vc, struct list_head *head); void vchan_init(struct virt_dma_chan *vc, struct dma_device *dmadev); struct virt_dma_desc *vchan_find_desc(struct virt_dma_chan *, dma_cookie_t); @@ -66,9 +128,9 @@ static inline struct dma_async_tx_descriptor *vchan_tx_prep(struct virt_dma_chan vd->tx_result.result = DMA_TRANS_NOERROR; vd->tx_result.residue = 0; - spin_lock_irqsave(&vc->lock, flags); + dma_vchan_lock_irqsave(vc, flags); list_add_tail(&vd->node, &vc->desc_allocated); - spin_unlock_irqrestore(&vc->lock, flags); + dma_vchan_unlock_irqrestore(vc, flags); return &vd->tx; } @@ -116,9 +178,9 @@ static inline void vchan_vdesc_fini(struct virt_dma_desc *vd) if (dmaengine_desc_test_reuse(&vd->tx)) { unsigned long flags; - spin_lock_irqsave(&vc->lock, flags); + dma_vchan_lock_irqsave(vc, flags); list_add(&vd->node, &vc->desc_allocated); - spin_unlock_irqrestore(&vc->lock, flags); + dma_vchan_unlock_irqrestore(vc, flags); } else { vc->desc_free(vd); } @@ -190,11 +252,11 @@ static inline void vchan_free_chan_resources(struct virt_dma_chan *vc) unsigned long flags; LIST_HEAD(head); - spin_lock_irqsave(&vc->lock, flags); + dma_vchan_lock_irqsave(vc, flags); vchan_get_all_descriptors(vc, &head); list_for_each_entry(vd, &head, node) dmaengine_desc_clear_reuse(&vd->tx); - spin_unlock_irqrestore(&vc->lock, flags); + dma_vchan_unlock_irqrestore(vc, flags); vchan_dma_desc_free_list(vc, &head); } @@ -215,11 +277,11 @@ static inline void vchan_synchronize(struct virt_dma_chan *vc) tasklet_kill(&vc->task); - spin_lock_irqsave(&vc->lock, flags); + dma_vchan_lock_irqsave(vc, flags); list_splice_tail_init(&vc->desc_terminated, &head); - spin_unlock_irqrestore(&vc->lock, flags); + dma_vchan_unlock_irqrestore(vc, flags); vchan_dma_desc_free_list(vc, &head); }