drivers/hv/ring_buffer.c

   1 /*
   2  *
   3  * Copyright (c) 2009, Microsoft Corporation.
   4  *
   5  * This program is free software; you can redistribute it and/or modify it
   6  * under the terms and conditions of the GNU General Public License,
   7  * version 2, as published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  12  * more details.
  13  *
  14  * You should have received a copy of the GNU General Public License along with
  15  * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
  16  * Place - Suite 330, Boston, MA 02111-1307 USA.
  17  *
  18  * Authors:
  19  *   Haiyang Zhang <haiyangz@microsoft.com>
  20  *   Hank Janssen  <hjanssen@microsoft.com>
  21  *   K. Y. Srinivasan <kys@microsoft.com>
  22  *
  23  */
  24 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  25
  26 #include <linux/kernel.h>
  27 #include <linux/mm.h>
  28 #include <linux/hyperv.h>
  29
  30 #include "hyperv_vmbus.h"
  31
  32
  33 /* #defines */
  34
  35
  36 /* Amount of space to write to */
  37 #define BYTES_AVAIL_TO_WRITE(r, w, z) \
  38         ((w) >= (r)) ? ((z) - ((w) - (r))) : ((r) - (w))
  39
  40
  41 /*
  42  *
  43  * hv_get_ringbuffer_availbytes()
  44  *
  45  * Get number of bytes available to read and to write to
  46  * for the specified ring buffer
  47  */
  48 static inline void
  49 hv_get_ringbuffer_availbytes(struct hv_ring_buffer_info *rbi,
  50                           u32 *read, u32 *write)
  51 {
  52         u32 read_loc, write_loc;
  53
  54         smp_read_barrier_depends();
  55
  56         /* Capture the read/write indices before they changed */
  57         read_loc = rbi->ring_buffer->read_index;
  58         write_loc = rbi->ring_buffer->write_index;
  59
  60         *write = BYTES_AVAIL_TO_WRITE(read_loc, write_loc, rbi->ring_datasize);
  61         *read = rbi->ring_datasize - *write;
  62 }
  63
  64 /*
  65  * hv_get_next_write_location()
  66  *
  67  * Get the next write location for the specified ring buffer
  68  *
  69  */
  70 static inline u32
  71 hv_get_next_write_location(struct hv_ring_buffer_info *ring_info)
  72 {
  73         u32 next = ring_info->ring_buffer->write_index;
  74
  75         return next;
  76 }
  77
  78 /*
  79  * hv_set_next_write_location()
  80  *
  81  * Set the next write location for the specified ring buffer
  82  *
  83  */
  84 static inline void
  85 hv_set_next_write_location(struct hv_ring_buffer_info *ring_info,
  86                      u32 next_write_location)
  87 {
  88         ring_info->ring_buffer->write_index = next_write_location;
  89 }
  90
  91 /*
  92  * hv_get_next_read_location()
  93  *
  94  * Get the next read location for the specified ring buffer
  95  */
  96 static inline u32
  97 hv_get_next_read_location(struct hv_ring_buffer_info *ring_info)
  98 {
  99         u32 next = ring_info->ring_buffer->read_index;
 100
 101         return next;
 102 }
 103
 104 /*
 105  * hv_get_next_readlocation_withoffset()
 106  *
 107  * Get the next read location + offset for the specified ring buffer.
 108  * This allows the caller to skip
 109  */
 110 static inline u32
 111 hv_get_next_readlocation_withoffset(struct hv_ring_buffer_info *ring_info,
 112                                  u32 offset)
 113 {
 114         u32 next = ring_info->ring_buffer->read_index;
 115
 116         next += offset;
 117         next %= ring_info->ring_datasize;
 118
 119         return next;
 120 }
 121
 122 /*
 123  *
 124  * hv_set_next_read_location()
 125  *
 126  * Set the next read location for the specified ring buffer
 127  *
 128  */
 129 static inline void
 130 hv_set_next_read_location(struct hv_ring_buffer_info *ring_info,
 131                     u32 next_read_location)
 132 {
 133         ring_info->ring_buffer->read_index = next_read_location;
 134 }
 135
 136
 137 /*
 138  *
 139  * hv_get_ring_buffer()
 140  *
 141  * Get the start of the ring buffer
 142  */
 143 static inline void *
 144 hv_get_ring_buffer(struct hv_ring_buffer_info *ring_info)
 145 {
 146         return (void *)ring_info->ring_buffer->buffer;
 147 }
 148
 149
 150 /*
 151  *
 152  * hv_get_ring_buffersize()
 153  *
 154  * Get the size of the ring buffer
 155  */
 156 static inline u32
 157 hv_get_ring_buffersize(struct hv_ring_buffer_info *ring_info)
 158 {
 159         return ring_info->ring_datasize;
 160 }
 161
 162 /*
 163  *
 164  * hv_get_ring_bufferindices()
 165  *
 166  * Get the read and write indices as u64 of the specified ring buffer
 167  *
 168  */
 169 static inline u64
 170 hv_get_ring_bufferindices(struct hv_ring_buffer_info *ring_info)
 171 {
 172         return (u64)ring_info->ring_buffer->write_index << 32;
 173 }
 174
 175
 176 /*
 177  *
 178  * hv_dump_ring_info()
 179  *
 180  * Dump out to console the ring buffer info
 181  *
 182  */
 183 void hv_dump_ring_info(struct hv_ring_buffer_info *ring_info, char *prefix)
 184 {
 185         u32 bytes_avail_towrite;
 186         u32 bytes_avail_toread;
 187
 188         hv_get_ringbuffer_availbytes(ring_info,
 189         &bytes_avail_toread,
 190         &bytes_avail_towrite);
 191
 192         DPRINT(VMBUS,
 193                 DEBUG_RING_LVL,
 194                 "%s <<ringinfo %p buffer %p avail write %u "
 195                 "avail read %u read idx %u write idx %u>>",
 196                 prefix,
 197                 ring_info,
 198                 ring_info->ring_buffer->buffer,
 199                 bytes_avail_towrite,
 200                 bytes_avail_toread,
 201                 ring_info->ring_buffer->read_index,
 202                 ring_info->ring_buffer->write_index);
 203 }
 204
 205
 206 /*
 207  *
 208  * hv_copyfrom_ringbuffer()
 209  *
 210  * Helper routine to copy to source from ring buffer.
 211  * Assume there is enough room. Handles wrap-around in src case only!!
 212  *
 213  */
 214 static u32 hv_copyfrom_ringbuffer(
 215         struct hv_ring_buffer_info      *ring_info,
 216         void                            *dest,
 217         u32                             destlen,
 218         u32                             start_read_offset)
 219 {
 220         void *ring_buffer = hv_get_ring_buffer(ring_info);
 221         u32 ring_buffer_size = hv_get_ring_buffersize(ring_info);
 222
 223         u32 frag_len;
 224
 225         /* wrap-around detected at the src */
 226         if (destlen > ring_buffer_size - start_read_offset) {
 227                 frag_len = ring_buffer_size - start_read_offset;
 228
 229                 memcpy(dest, ring_buffer + start_read_offset, frag_len);
 230                 memcpy(dest + frag_len, ring_buffer, destlen - frag_len);
 231         } else
 232
 233                 memcpy(dest, ring_buffer + start_read_offset, destlen);
 234
 235
 236         start_read_offset += destlen;
 237         start_read_offset %= ring_buffer_size;
 238
 239         return start_read_offset;
 240 }
 241
 242
 243 /*
 244  *
 245  * hv_copyto_ringbuffer()
 246  *
 247  * Helper routine to copy from source to ring buffer.
 248  * Assume there is enough room. Handles wrap-around in dest case only!!
 249  *
 250  */
 251 static u32 hv_copyto_ringbuffer(
 252         struct hv_ring_buffer_info      *ring_info,
 253         u32                             start_write_offset,
 254         void                            *src,
 255         u32                             srclen)
 256 {
 257         void *ring_buffer = hv_get_ring_buffer(ring_info);
 258         u32 ring_buffer_size = hv_get_ring_buffersize(ring_info);
 259         u32 frag_len;
 260
 261         /* wrap-around detected! */
 262         if (srclen > ring_buffer_size - start_write_offset) {
 263                 frag_len = ring_buffer_size - start_write_offset;
 264                 memcpy(ring_buffer + start_write_offset, src, frag_len);
 265                 memcpy(ring_buffer, src + frag_len, srclen - frag_len);
 266         } else
 267                 memcpy(ring_buffer + start_write_offset, src, srclen);
 268
 269         start_write_offset += srclen;
 270         start_write_offset %= ring_buffer_size;
 271
 272         return start_write_offset;
 273 }
 274
 275 /*
 276  *
 277  * hv_ringbuffer_get_debuginfo()
 278  *
 279  * Get various debug metrics for the specified ring buffer
 280  *
 281  */
 282 void hv_ringbuffer_get_debuginfo(struct hv_ring_buffer_info *ring_info,
 283                             struct hv_ring_buffer_debug_info *debug_info)
 284 {
 285         u32 bytes_avail_towrite;
 286         u32 bytes_avail_toread;
 287
 288         if (ring_info->ring_buffer) {
 289                 hv_get_ringbuffer_availbytes(ring_info,
 290                                         &bytes_avail_toread,
 291                                         &bytes_avail_towrite);
 292
 293                 debug_info->bytes_avail_toread = bytes_avail_toread;
 294                 debug_info->bytes_avail_towrite = bytes_avail_towrite;
 295                 debug_info->current_read_index =
 296                         ring_info->ring_buffer->read_index;
 297                 debug_info->current_write_index =
 298                         ring_info->ring_buffer->write_index;
 299                 debug_info->current_interrupt_mask =
 300                         ring_info->ring_buffer->interrupt_mask;
 301         }
 302 }
 303
 304
 305 /*
 306  *
 307  * hv_get_ringbuffer_interrupt_mask()
 308  *
 309  * Get the interrupt mask for the specified ring buffer
 310  *
 311  */
 312 u32 hv_get_ringbuffer_interrupt_mask(struct hv_ring_buffer_info *rbi)
 313 {
 314         return rbi->ring_buffer->interrupt_mask;
 315 }
 316
 317 /*
 318  *
 319  * hv_ringbuffer_init()
 320  *
 321  *Initialize the ring buffer
 322  *
 323  */
 324 int hv_ringbuffer_init(struct hv_ring_buffer_info *ring_info,
 325                    void *buffer, u32 buflen)
 326 {
 327         if (sizeof(struct hv_ring_buffer) != PAGE_SIZE)
 328                 return -EINVAL;
 329
 330         memset(ring_info, 0, sizeof(struct hv_ring_buffer_info));
 331
 332         ring_info->ring_buffer = (struct hv_ring_buffer *)buffer;
 333         ring_info->ring_buffer->read_index =
 334                 ring_info->ring_buffer->write_index = 0;
 335
 336         ring_info->ring_size = buflen;
 337         ring_info->ring_datasize = buflen - sizeof(struct hv_ring_buffer);
 338
 339         spin_lock_init(&ring_info->ring_lock);
 340
 341         return 0;
 342 }
 343
 344 /*
 345  *
 346  * hv_ringbuffer_cleanup()
 347  *
 348  * Cleanup the ring buffer
 349  *
 350  */
 351 void hv_ringbuffer_cleanup(struct hv_ring_buffer_info *ring_info)
 352 {
 353 }
 354
 355 /*
 356  *
 357  * hv_ringbuffer_write()
 358  *
 359  * Write to the ring buffer
 360  *
 361  */
 362 int hv_ringbuffer_write(struct hv_ring_buffer_info *outring_info,
 363                     struct scatterlist *sglist, u32 sgcount)
 364 {
 365         int i = 0;
 366         u32 bytes_avail_towrite;
 367         u32 bytes_avail_toread;
 368         u32 totalbytes_towrite = 0;
 369
 370         struct scatterlist *sg;
 371         u32 next_write_location;
 372         u64 prev_indices = 0;
 373         unsigned long flags;
 374
 375         for_each_sg(sglist, sg, sgcount, i)
 376         {
 377                 totalbytes_towrite += sg->length;
 378         }
 379
 380         totalbytes_towrite += sizeof(u64);
 381
 382         spin_lock_irqsave(&outring_info->ring_lock, flags);
 383
 384         hv_get_ringbuffer_availbytes(outring_info,
 385                                 &bytes_avail_toread,
 386                                 &bytes_avail_towrite);
 387
 388
 389         /* If there is only room for the packet, assume it is full. */
 390         /* Otherwise, the next time around, we think the ring buffer */
 391         /* is empty since the read index == write index */
 392         if (bytes_avail_towrite <= totalbytes_towrite) {
 393                 spin_unlock_irqrestore(&outring_info->ring_lock, flags);
 394                 return -EAGAIN;
 395         }
 396
 397         /* Write to the ring buffer */
 398         next_write_location = hv_get_next_write_location(outring_info);
 399
 400         for_each_sg(sglist, sg, sgcount, i)
 401         {
 402                 next_write_location = hv_copyto_ringbuffer(outring_info,
 403                                                      next_write_location,
 404                                                      sg_virt(sg),
 405                                                      sg->length);
 406         }
 407
 408         /* Set previous packet start */
 409         prev_indices = hv_get_ring_bufferindices(outring_info);
 410
 411         next_write_location = hv_copyto_ringbuffer(outring_info,
 412                                              next_write_location,
 413                                              &prev_indices,
 414                                              sizeof(u64));
 415
 416         /* Make sure we flush all writes before updating the writeIndex */
 417         smp_wmb();
 418
 419         /* Now, update the write location */
 420         hv_set_next_write_location(outring_info, next_write_location);
 421
 422
 423         spin_unlock_irqrestore(&outring_info->ring_lock, flags);
 424         return 0;
 425 }
 426
 427
 428 /*
 429  *
 430  * hv_ringbuffer_peek()
 431  *
 432  * Read without advancing the read index
 433  *
 434  */
 435 int hv_ringbuffer_peek(struct hv_ring_buffer_info *Inring_info,
 436                    void *Buffer, u32 buflen)
 437 {
 438         u32 bytes_avail_towrite;
 439         u32 bytes_avail_toread;
 440         u32 next_read_location = 0;
 441         unsigned long flags;
 442
 443         spin_lock_irqsave(&Inring_info->ring_lock, flags);
 444
 445         hv_get_ringbuffer_availbytes(Inring_info,
 446                                 &bytes_avail_toread,
 447                                 &bytes_avail_towrite);
 448
 449         /* Make sure there is something to read */
 450         if (bytes_avail_toread < buflen) {
 451
 452                 spin_unlock_irqrestore(&Inring_info->ring_lock, flags);
 453
 454                 return -EAGAIN;
 455         }
 456
 457         /* Convert to byte offset */
 458         next_read_location = hv_get_next_read_location(Inring_info);
 459
 460         next_read_location = hv_copyfrom_ringbuffer(Inring_info,
 461                                                 Buffer,
 462                                                 buflen,
 463                                                 next_read_location);
 464
 465         spin_unlock_irqrestore(&Inring_info->ring_lock, flags);
 466
 467         return 0;
 468 }
 469
 470
 471 /*
 472  *
 473  * hv_ringbuffer_read()
 474  *
 475  * Read and advance the read index
 476  *
 477  */
 478 int hv_ringbuffer_read(struct hv_ring_buffer_info *inring_info, void *buffer,
 479                    u32 buflen, u32 offset)
 480 {
 481         u32 bytes_avail_towrite;
 482         u32 bytes_avail_toread;
 483         u32 next_read_location = 0;
 484         u64 prev_indices = 0;
 485         unsigned long flags;
 486
 487         if (buflen <= 0)
 488                 return -EINVAL;
 489
 490         spin_lock_irqsave(&inring_info->ring_lock, flags);
 491
 492         hv_get_ringbuffer_availbytes(inring_info,
 493                                 &bytes_avail_toread,
 494                                 &bytes_avail_towrite);
 495
 496         /* Make sure there is something to read */
 497         if (bytes_avail_toread < buflen) {
 498                 spin_unlock_irqrestore(&inring_info->ring_lock, flags);
 499
 500                 return -EAGAIN;
 501         }
 502
 503         next_read_location =
 504                 hv_get_next_readlocation_withoffset(inring_info, offset);
 505
 506         next_read_location = hv_copyfrom_ringbuffer(inring_info,
 507                                                 buffer,
 508                                                 buflen,
 509                                                 next_read_location);
 510
 511         next_read_location = hv_copyfrom_ringbuffer(inring_info,
 512                                                 &prev_indices,
 513                                                 sizeof(u64),
 514                                                 next_read_location);
 515
 516         /* Make sure all reads are done before we update the read index since */
 517         /* the writer may start writing to the read area once the read index */
 518         /*is updated */
 519         smp_mb();
 520
 521         /* Update the read index */
 522         hv_set_next_read_location(inring_info, next_read_location);
 523
 524         spin_unlock_irqrestore(&inring_info->ring_lock, flags);
 525
 526         return 0;
 527 }