Merge tag 'qcom-soc-for-3.16-2' of git://git.kernel.org/pub/scm/linux/kernel/git...

[pandora-kernel.git] / drivers / infiniband / hw / cxgb4 / t4.h

diff --git a/drivers/infiniband/hw/cxgb4/t4.h b/drivers/infiniband/hw/cxgb4/t4.h
index eeca8b1..2178f31 100644 (file)
--- a/drivers/infiniband/hw/cxgb4/t4.h
+++ b/drivers/infiniband/hw/cxgb4/t4.h
@@ -84,7 +84,14 @@ struct t4_status_page {
                        sizeof(struct fw_ri_isgl)) / sizeof(struct fw_ri_sge))
 #define T4_MAX_FR_IMMD ((T4_SQ_NUM_BYTES - sizeof(struct fw_ri_fr_nsmr_wr) - \
                        sizeof(struct fw_ri_immd)) & ~31UL)
-#define T4_MAX_FR_DEPTH (1024 / sizeof(u64))
+#define T4_MAX_FR_IMMD_DEPTH (T4_MAX_FR_IMMD / sizeof(u64))
+#define T4_MAX_FR_DSGL 1024
+#define T4_MAX_FR_DSGL_DEPTH (T4_MAX_FR_DSGL / sizeof(u64))
+
+static inline int t4_max_fr_depth(int use_dsgl)
+{
+       return use_dsgl ? T4_MAX_FR_DSGL_DEPTH : T4_MAX_FR_IMMD_DEPTH;
+}
 
 #define T4_RQ_NUM_SLOTS 2
 #define T4_RQ_NUM_BYTES (T4_EQ_ENTRY_SIZE * T4_RQ_NUM_SLOTS)
@@ -292,7 +299,7 @@ struct t4_sq {
        unsigned long phys_addr;
        struct t4_swsqe *sw_sq;
        struct t4_swsqe *oldest_read;
-       u64 udb;
+       u64 __iomem *udb;
        size_t memsize;
        u32 qid;
        u16 in_use;
@@ -314,7 +321,7 @@ struct t4_rq {
        dma_addr_t dma_addr;
        DEFINE_DMA_UNMAP_ADDR(mapping);
        struct t4_swrqe *sw_rq;
-       u64 udb;
+       u64 __iomem *udb;
        size_t memsize;
        u32 qid;
        u32 msn;
@@ -435,15 +442,67 @@ static inline u16 t4_sq_wq_size(struct t4_wq *wq)
                return wq->sq.size * T4_SQ_NUM_SLOTS;
 }
 
-static inline void t4_ring_sq_db(struct t4_wq *wq, u16 inc)
+/* This function copies 64 byte coalesced work request to memory
+ * mapped BAR2 space. For coalesced WRs, the SGE fetches data
+ * from the FIFO instead of from Host.
+ */
+static inline void pio_copy(u64 __iomem *dst, u64 *src)
+{
+       int count = 8;
+
+       while (count) {
+               writeq(*src, dst);
+               src++;
+               dst++;
+               count--;
+       }
+}
+
+static inline void t4_ring_sq_db(struct t4_wq *wq, u16 inc, u8 t5,
+                                union t4_wr *wqe)
 {
+
+       /* Flush host queue memory writes. */
        wmb();
+       if (t5) {
+               if (inc == 1 && wqe) {
+                       PDBG("%s: WC wq->sq.pidx = %d\n",
+                            __func__, wq->sq.pidx);
+                       pio_copy(wq->sq.udb + 7, (void *)wqe);
+               } else {
+                       PDBG("%s: DB wq->sq.pidx = %d\n",
+                            __func__, wq->sq.pidx);
+                       writel(PIDX_T5(inc), wq->sq.udb);
+               }
+
+               /* Flush user doorbell area writes. */
+               wmb();
+               return;
+       }
        writel(QID(wq->sq.qid) | PIDX(inc), wq->db);
 }
 
-static inline void t4_ring_rq_db(struct t4_wq *wq, u16 inc)
+static inline void t4_ring_rq_db(struct t4_wq *wq, u16 inc, u8 t5,
+                                union t4_recv_wr *wqe)
 {
+
+       /* Flush host queue memory writes. */
        wmb();
+       if (t5) {
+               if (inc == 1 && wqe) {
+                       PDBG("%s: WC wq->rq.pidx = %d\n",
+                            __func__, wq->rq.pidx);
+                       pio_copy(wq->rq.udb + 7, (void *)wqe);
+               } else {
+                       PDBG("%s: DB wq->rq.pidx = %d\n",
+                            __func__, wq->rq.pidx);
+                       writel(PIDX_T5(inc), wq->rq.udb);
+               }
+
+               /* Flush user doorbell area writes. */
+               wmb();
+               return;
+       }
        writel(QID(wq->rq.qid) | PIDX(inc), wq->db);
 }
 
@@ -568,6 +627,9 @@ static inline int t4_next_hw_cqe(struct t4_cq *cq, struct t4_cqe **cqe)
                printk(KERN_ERR MOD "cq overflow cqid %u\n", cq->cqid);
                BUG_ON(1);
        } else if (t4_valid_cqe(cq, &cq->queue[cq->cidx])) {
+
+               /* Ensure CQE is flushed to memory */
+               rmb();
                *cqe = &cq->queue[cq->cidx];
                ret = 0;
        } else