drivers/infiniband/hw/ehca/ehca_mrmw.c

   1 /*
   2  *  IBM eServer eHCA Infiniband device driver for Linux on POWER
   3  *
   4  *  MR/MW functions
   5  *
   6  *  Authors: Dietmar Decker <ddecker@de.ibm.com>
   7  *           Christoph Raisch <raisch@de.ibm.com>
   8  *
   9  *  Copyright (c) 2005 IBM Corporation
  10  *
  11  *  All rights reserved.
  12  *
  13  *  This source code is distributed under a dual license of GPL v2.0 and OpenIB
  14  *  BSD.
  15  *
  16  * OpenIB BSD License
  17  *
  18  * Redistribution and use in source and binary forms, with or without
  19  * modification, are permitted provided that the following conditions are met:
  20  *
  21  * Redistributions of source code must retain the above copyright notice, this
  22  * list of conditions and the following disclaimer.
  23  *
  24  * Redistributions in binary form must reproduce the above copyright notice,
  25  * this list of conditions and the following disclaimer in the documentation
  26  * and/or other materials
  27  * provided with the distribution.
  28  *
  29  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  30  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  31  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  32  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  33  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  34  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  35  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
  36  * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
  37  * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  38  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  39  * POSSIBILITY OF SUCH DAMAGE.
  40  */
  41
  42 #include <asm/current.h>
  43
  44 #include "ehca_iverbs.h"
  45 #include "ehca_mrmw.h"
  46 #include "hcp_if.h"
  47 #include "hipz_hw.h"
  48
  49 static struct kmem_cache *mr_cache;
  50 static struct kmem_cache *mw_cache;
  51
  52 static struct ehca_mr *ehca_mr_new(void)
  53 {
  54         struct ehca_mr *me;
  55
  56         me = kmem_cache_alloc(mr_cache, GFP_KERNEL);
  57         if (me) {
  58                 memset(me, 0, sizeof(struct ehca_mr));
  59                 spin_lock_init(&me->mrlock);
  60         } else
  61                 ehca_gen_err("alloc failed");
  62
  63         return me;
  64 }
  65
  66 static void ehca_mr_delete(struct ehca_mr *me)
  67 {
  68         kmem_cache_free(mr_cache, me);
  69 }
  70
  71 static struct ehca_mw *ehca_mw_new(void)
  72 {
  73         struct ehca_mw *me;
  74
  75         me = kmem_cache_alloc(mw_cache, GFP_KERNEL);
  76         if (me) {
  77                 memset(me, 0, sizeof(struct ehca_mw));
  78                 spin_lock_init(&me->mwlock);
  79         } else
  80                 ehca_gen_err("alloc failed");
  81
  82         return me;
  83 }
  84
  85 static void ehca_mw_delete(struct ehca_mw *me)
  86 {
  87         kmem_cache_free(mw_cache, me);
  88 }
  89
  90 /*----------------------------------------------------------------------*/
  91
  92 struct ib_mr *ehca_get_dma_mr(struct ib_pd *pd, int mr_access_flags)
  93 {
  94         struct ib_mr *ib_mr;
  95         int ret;
  96         struct ehca_mr *e_maxmr;
  97         struct ehca_pd *e_pd = container_of(pd, struct ehca_pd, ib_pd);
  98         struct ehca_shca *shca =
  99                 container_of(pd->device, struct ehca_shca, ib_device);
 100
 101         if (shca->maxmr) {
 102                 e_maxmr = ehca_mr_new();
 103                 if (!e_maxmr) {
 104                         ehca_err(&shca->ib_device, "out of memory");
 105                         ib_mr = ERR_PTR(-ENOMEM);
 106                         goto get_dma_mr_exit0;
 107                 }
 108
 109                 ret = ehca_reg_maxmr(shca, e_maxmr, (u64*)KERNELBASE,
 110                                      mr_access_flags, e_pd,
 111                                      &e_maxmr->ib.ib_mr.lkey,
 112                                      &e_maxmr->ib.ib_mr.rkey);
 113                 if (ret) {
 114                         ib_mr = ERR_PTR(ret);
 115                         goto get_dma_mr_exit0;
 116                 }
 117                 ib_mr = &e_maxmr->ib.ib_mr;
 118         } else {
 119                 ehca_err(&shca->ib_device, "no internal max-MR exist!");
 120                 ib_mr = ERR_PTR(-EINVAL);
 121                 goto get_dma_mr_exit0;
 122         }
 123
 124 get_dma_mr_exit0:
 125         if (IS_ERR(ib_mr))
 126                 ehca_err(&shca->ib_device, "rc=%lx pd=%p mr_access_flags=%x ",
 127                          PTR_ERR(ib_mr), pd, mr_access_flags);
 128         return ib_mr;
 129 } /* end ehca_get_dma_mr() */
 130
 131 /*----------------------------------------------------------------------*/
 132
 133 struct ib_mr *ehca_reg_phys_mr(struct ib_pd *pd,
 134                                struct ib_phys_buf *phys_buf_array,
 135                                int num_phys_buf,
 136                                int mr_access_flags,
 137                                u64 *iova_start)
 138 {
 139         struct ib_mr *ib_mr;
 140         int ret;
 141         struct ehca_mr *e_mr;
 142         struct ehca_shca *shca =
 143                 container_of(pd->device, struct ehca_shca, ib_device);
 144         struct ehca_pd *e_pd = container_of(pd, struct ehca_pd, ib_pd);
 145
 146         u64 size;
 147         struct ehca_mr_pginfo pginfo={0,0,0,0,0,0,0,NULL,0,NULL,NULL,0,NULL,0};
 148         u32 num_pages_mr;
 149         u32 num_pages_4k; /* 4k portion "pages" */
 150
 151         if ((num_phys_buf <= 0) || !phys_buf_array) {
 152                 ehca_err(pd->device, "bad input values: num_phys_buf=%x "
 153                          "phys_buf_array=%p", num_phys_buf, phys_buf_array);
 154                 ib_mr = ERR_PTR(-EINVAL);
 155                 goto reg_phys_mr_exit0;
 156         }
 157         if (((mr_access_flags & IB_ACCESS_REMOTE_WRITE) &&
 158              !(mr_access_flags & IB_ACCESS_LOCAL_WRITE)) ||
 159             ((mr_access_flags & IB_ACCESS_REMOTE_ATOMIC) &&
 160              !(mr_access_flags & IB_ACCESS_LOCAL_WRITE))) {
 161                 /*
 162                  * Remote Write Access requires Local Write Access
 163                  * Remote Atomic Access requires Local Write Access
 164                  */
 165                 ehca_err(pd->device, "bad input values: mr_access_flags=%x",
 166                          mr_access_flags);
 167                 ib_mr = ERR_PTR(-EINVAL);
 168                 goto reg_phys_mr_exit0;
 169         }
 170
 171         /* check physical buffer list and calculate size */
 172         ret = ehca_mr_chk_buf_and_calc_size(phys_buf_array, num_phys_buf,
 173                                             iova_start, &size);
 174         if (ret) {
 175                 ib_mr = ERR_PTR(ret);
 176                 goto reg_phys_mr_exit0;
 177         }
 178         if ((size == 0) ||
 179             (((u64)iova_start + size) < (u64)iova_start)) {
 180                 ehca_err(pd->device, "bad input values: size=%lx iova_start=%p",
 181                          size, iova_start);
 182                 ib_mr = ERR_PTR(-EINVAL);
 183                 goto reg_phys_mr_exit0;
 184         }
 185
 186         e_mr = ehca_mr_new();
 187         if (!e_mr) {
 188                 ehca_err(pd->device, "out of memory");
 189                 ib_mr = ERR_PTR(-ENOMEM);
 190                 goto reg_phys_mr_exit0;
 191         }
 192
 193         /* determine number of MR pages */
 194         num_pages_mr = ((((u64)iova_start % PAGE_SIZE) + size +
 195                          PAGE_SIZE - 1) / PAGE_SIZE);
 196         num_pages_4k = ((((u64)iova_start % EHCA_PAGESIZE) + size +
 197                          EHCA_PAGESIZE - 1) / EHCA_PAGESIZE);
 198
 199         /* register MR on HCA */
 200         if (ehca_mr_is_maxmr(size, iova_start)) {
 201                 e_mr->flags |= EHCA_MR_FLAG_MAXMR;
 202                 ret = ehca_reg_maxmr(shca, e_mr, iova_start, mr_access_flags,
 203                                      e_pd, &e_mr->ib.ib_mr.lkey,
 204                                      &e_mr->ib.ib_mr.rkey);
 205                 if (ret) {
 206                         ib_mr = ERR_PTR(ret);
 207                         goto reg_phys_mr_exit1;
 208                 }
 209         } else {
 210                 pginfo.type           = EHCA_MR_PGI_PHYS;
 211                 pginfo.num_pages      = num_pages_mr;
 212                 pginfo.num_4k         = num_pages_4k;
 213                 pginfo.num_phys_buf   = num_phys_buf;
 214                 pginfo.phys_buf_array = phys_buf_array;
 215                 pginfo.next_4k        = (((u64)iova_start & ~PAGE_MASK) /
 216                                          EHCA_PAGESIZE);
 217
 218                 ret = ehca_reg_mr(shca, e_mr, iova_start, size, mr_access_flags,
 219                                   e_pd, &pginfo, &e_mr->ib.ib_mr.lkey,
 220                                   &e_mr->ib.ib_mr.rkey);
 221                 if (ret) {
 222                         ib_mr = ERR_PTR(ret);
 223                         goto reg_phys_mr_exit1;
 224                 }
 225         }
 226
 227         /* successful registration of all pages */
 228         return &e_mr->ib.ib_mr;
 229
 230 reg_phys_mr_exit1:
 231         ehca_mr_delete(e_mr);
 232 reg_phys_mr_exit0:
 233         if (IS_ERR(ib_mr))
 234                 ehca_err(pd->device, "rc=%lx pd=%p phys_buf_array=%p "
 235                          "num_phys_buf=%x mr_access_flags=%x iova_start=%p",
 236                          PTR_ERR(ib_mr), pd, phys_buf_array,
 237                          num_phys_buf, mr_access_flags, iova_start);
 238         return ib_mr;
 239 } /* end ehca_reg_phys_mr() */
 240
 241 /*----------------------------------------------------------------------*/
 242
 243 struct ib_mr *ehca_reg_user_mr(struct ib_pd *pd,
 244                                struct ib_umem *region,
 245                                int mr_access_flags,
 246                                struct ib_udata *udata)
 247 {
 248         struct ib_mr *ib_mr;
 249         struct ehca_mr *e_mr;
 250         struct ehca_shca *shca =
 251                 container_of(pd->device, struct ehca_shca, ib_device);
 252         struct ehca_pd *e_pd = container_of(pd, struct ehca_pd, ib_pd);
 253         struct ehca_mr_pginfo pginfo={0,0,0,0,0,0,0,NULL,0,NULL,NULL,0,NULL,0};
 254         int ret;
 255         u32 num_pages_mr;
 256         u32 num_pages_4k; /* 4k portion "pages" */
 257
 258         if (!pd) {
 259                 ehca_gen_err("bad pd=%p", pd);
 260                 return ERR_PTR(-EFAULT);
 261         }
 262         if (!region) {
 263                 ehca_err(pd->device, "bad input values: region=%p", region);
 264                 ib_mr = ERR_PTR(-EINVAL);
 265                 goto reg_user_mr_exit0;
 266         }
 267         if (((mr_access_flags & IB_ACCESS_REMOTE_WRITE) &&
 268              !(mr_access_flags & IB_ACCESS_LOCAL_WRITE)) ||
 269             ((mr_access_flags & IB_ACCESS_REMOTE_ATOMIC) &&
 270              !(mr_access_flags & IB_ACCESS_LOCAL_WRITE))) {
 271                 /*
 272                  * Remote Write Access requires Local Write Access
 273                  * Remote Atomic Access requires Local Write Access
 274                  */
 275                 ehca_err(pd->device, "bad input values: mr_access_flags=%x",
 276                          mr_access_flags);
 277                 ib_mr = ERR_PTR(-EINVAL);
 278                 goto reg_user_mr_exit0;
 279         }
 280         if (region->page_size != PAGE_SIZE) {
 281                 ehca_err(pd->device, "page size not supported, "
 282                          "region->page_size=%x", region->page_size);
 283                 ib_mr = ERR_PTR(-EINVAL);
 284                 goto reg_user_mr_exit0;
 285         }
 286
 287         if ((region->length == 0) ||
 288             ((region->virt_base + region->length) < region->virt_base)) {
 289                 ehca_err(pd->device, "bad input values: length=%lx "
 290                          "virt_base=%lx", region->length, region->virt_base);
 291                 ib_mr = ERR_PTR(-EINVAL);
 292                 goto reg_user_mr_exit0;
 293         }
 294
 295         e_mr = ehca_mr_new();
 296         if (!e_mr) {
 297                 ehca_err(pd->device, "out of memory");
 298                 ib_mr = ERR_PTR(-ENOMEM);
 299                 goto reg_user_mr_exit0;
 300         }
 301
 302         /* determine number of MR pages */
 303         num_pages_mr = (((region->virt_base % PAGE_SIZE) + region->length +
 304                          PAGE_SIZE - 1) / PAGE_SIZE);
 305         num_pages_4k = (((region->virt_base % EHCA_PAGESIZE) + region->length +
 306                          EHCA_PAGESIZE - 1) / EHCA_PAGESIZE);
 307
 308         /* register MR on HCA */
 309         pginfo.type       = EHCA_MR_PGI_USER;
 310         pginfo.num_pages  = num_pages_mr;
 311         pginfo.num_4k     = num_pages_4k;
 312         pginfo.region     = region;
 313         pginfo.next_4k    = region->offset / EHCA_PAGESIZE;
 314         pginfo.next_chunk = list_prepare_entry(pginfo.next_chunk,
 315                                                (&region->chunk_list),
 316                                                list);
 317
 318         ret = ehca_reg_mr(shca, e_mr, (u64*)region->virt_base,
 319                           region->length, mr_access_flags, e_pd, &pginfo,
 320                           &e_mr->ib.ib_mr.lkey, &e_mr->ib.ib_mr.rkey);
 321         if (ret) {
 322                 ib_mr = ERR_PTR(ret);
 323                 goto reg_user_mr_exit1;
 324         }
 325
 326         /* successful registration of all pages */
 327         return &e_mr->ib.ib_mr;
 328
 329 reg_user_mr_exit1:
 330         ehca_mr_delete(e_mr);
 331 reg_user_mr_exit0:
 332         if (IS_ERR(ib_mr))
 333                 ehca_err(pd->device, "rc=%lx pd=%p region=%p mr_access_flags=%x"
 334                          " udata=%p",
 335                          PTR_ERR(ib_mr), pd, region, mr_access_flags, udata);
 336         return ib_mr;
 337 } /* end ehca_reg_user_mr() */
 338
 339 /*----------------------------------------------------------------------*/
 340
 341 int ehca_rereg_phys_mr(struct ib_mr *mr,
 342                        int mr_rereg_mask,
 343                        struct ib_pd *pd,
 344                        struct ib_phys_buf *phys_buf_array,
 345                        int num_phys_buf,
 346                        int mr_access_flags,
 347                        u64 *iova_start)
 348 {
 349         int ret;
 350
 351         struct ehca_shca *shca =
 352                 container_of(mr->device, struct ehca_shca, ib_device);
 353         struct ehca_mr *e_mr = container_of(mr, struct ehca_mr, ib.ib_mr);
 354         struct ehca_pd *my_pd = container_of(mr->pd, struct ehca_pd, ib_pd);
 355         u64 new_size;
 356         u64 *new_start;
 357         u32 new_acl;
 358         struct ehca_pd *new_pd;
 359         u32 tmp_lkey, tmp_rkey;
 360         unsigned long sl_flags;
 361         u32 num_pages_mr = 0;
 362         u32 num_pages_4k = 0; /* 4k portion "pages" */
 363         struct ehca_mr_pginfo pginfo={0,0,0,0,0,0,0,NULL,0,NULL,NULL,0,NULL,0};
 364         u32 cur_pid = current->tgid;
 365
 366         if (my_pd->ib_pd.uobject && my_pd->ib_pd.uobject->context &&
 367             (my_pd->ownpid != cur_pid)) {
 368                 ehca_err(mr->device, "Invalid caller pid=%x ownpid=%x",
 369                          cur_pid, my_pd->ownpid);
 370                 ret = -EINVAL;
 371                 goto rereg_phys_mr_exit0;
 372         }
 373
 374         if (!(mr_rereg_mask & IB_MR_REREG_TRANS)) {
 375                 /* TODO not supported, because PHYP rereg hCall needs pages */
 376                 ehca_err(mr->device, "rereg without IB_MR_REREG_TRANS not "
 377                          "supported yet, mr_rereg_mask=%x", mr_rereg_mask);
 378                 ret = -EINVAL;
 379                 goto rereg_phys_mr_exit0;
 380         }
 381
 382         if (mr_rereg_mask & IB_MR_REREG_PD) {
 383                 if (!pd) {
 384                         ehca_err(mr->device, "rereg with bad pd, pd=%p "
 385                                  "mr_rereg_mask=%x", pd, mr_rereg_mask);
 386                         ret = -EINVAL;
 387                         goto rereg_phys_mr_exit0;
 388                 }
 389         }
 390
 391         if ((mr_rereg_mask &
 392              ~(IB_MR_REREG_TRANS | IB_MR_REREG_PD | IB_MR_REREG_ACCESS)) ||
 393             (mr_rereg_mask == 0)) {
 394                 ret = -EINVAL;
 395                 goto rereg_phys_mr_exit0;
 396         }
 397
 398         /* check other parameters */
 399         if (e_mr == shca->maxmr) {
 400                 /* should be impossible, however reject to be sure */
 401                 ehca_err(mr->device, "rereg internal max-MR impossible, mr=%p "
 402                          "shca->maxmr=%p mr->lkey=%x",
 403                          mr, shca->maxmr, mr->lkey);
 404                 ret = -EINVAL;
 405                 goto rereg_phys_mr_exit0;
 406         }
 407         if (mr_rereg_mask & IB_MR_REREG_TRANS) { /* transl., i.e. addr/size */
 408                 if (e_mr->flags & EHCA_MR_FLAG_FMR) {
 409                         ehca_err(mr->device, "not supported for FMR, mr=%p "
 410                                  "flags=%x", mr, e_mr->flags);
 411                         ret = -EINVAL;
 412                         goto rereg_phys_mr_exit0;
 413                 }
 414                 if (!phys_buf_array || num_phys_buf <= 0) {
 415                         ehca_err(mr->device, "bad input values: mr_rereg_mask=%x"
 416                                  " phys_buf_array=%p num_phys_buf=%x",
 417                                  mr_rereg_mask, phys_buf_array, num_phys_buf);
 418                         ret = -EINVAL;
 419                         goto rereg_phys_mr_exit0;
 420                 }
 421         }
 422         if ((mr_rereg_mask & IB_MR_REREG_ACCESS) &&     /* change ACL */
 423             (((mr_access_flags & IB_ACCESS_REMOTE_WRITE) &&
 424               !(mr_access_flags & IB_ACCESS_LOCAL_WRITE)) ||
 425              ((mr_access_flags & IB_ACCESS_REMOTE_ATOMIC) &&
 426               !(mr_access_flags & IB_ACCESS_LOCAL_WRITE)))) {
 427                 /*
 428                  * Remote Write Access requires Local Write Access
 429                  * Remote Atomic Access requires Local Write Access
 430                  */
 431                 ehca_err(mr->device, "bad input values: mr_rereg_mask=%x "
 432                          "mr_access_flags=%x", mr_rereg_mask, mr_access_flags);
 433                 ret = -EINVAL;
 434                 goto rereg_phys_mr_exit0;
 435         }
 436
 437         /* set requested values dependent on rereg request */
 438         spin_lock_irqsave(&e_mr->mrlock, sl_flags);
 439         new_start = e_mr->start;  /* new == old address */
 440         new_size  = e_mr->size;   /* new == old length */
 441         new_acl   = e_mr->acl;    /* new == old access control */
 442         new_pd    = container_of(mr->pd,struct ehca_pd,ib_pd); /*new == old PD*/
 443
 444         if (mr_rereg_mask & IB_MR_REREG_TRANS) {
 445                 new_start = iova_start; /* change address */
 446                 /* check physical buffer list and calculate size */
 447                 ret = ehca_mr_chk_buf_and_calc_size(phys_buf_array,
 448                                                     num_phys_buf, iova_start,
 449                                                     &new_size);
 450                 if (ret)
 451                         goto rereg_phys_mr_exit1;
 452                 if ((new_size == 0) ||
 453                     (((u64)iova_start + new_size) < (u64)iova_start)) {
 454                         ehca_err(mr->device, "bad input values: new_size=%lx "
 455                                  "iova_start=%p", new_size, iova_start);
 456                         ret = -EINVAL;
 457                         goto rereg_phys_mr_exit1;
 458                 }
 459                 num_pages_mr = ((((u64)new_start % PAGE_SIZE) + new_size +
 460                                  PAGE_SIZE - 1) / PAGE_SIZE);
 461                 num_pages_4k = ((((u64)new_start % EHCA_PAGESIZE) + new_size +
 462                                  EHCA_PAGESIZE - 1) / EHCA_PAGESIZE);
 463                 pginfo.type           = EHCA_MR_PGI_PHYS;
 464                 pginfo.num_pages      = num_pages_mr;
 465                 pginfo.num_4k         = num_pages_4k;
 466                 pginfo.num_phys_buf   = num_phys_buf;
 467                 pginfo.phys_buf_array = phys_buf_array;
 468                 pginfo.next_4k        = (((u64)iova_start & ~PAGE_MASK) /
 469                                          EHCA_PAGESIZE);
 470         }
 471         if (mr_rereg_mask & IB_MR_REREG_ACCESS)
 472                 new_acl = mr_access_flags;
 473         if (mr_rereg_mask & IB_MR_REREG_PD)
 474                 new_pd = container_of(pd, struct ehca_pd, ib_pd);
 475
 476         ret = ehca_rereg_mr(shca, e_mr, new_start, new_size, new_acl,
 477                             new_pd, &pginfo, &tmp_lkey, &tmp_rkey);
 478         if (ret)
 479                 goto rereg_phys_mr_exit1;
 480
 481         /* successful reregistration */
 482         if (mr_rereg_mask & IB_MR_REREG_PD)
 483                 mr->pd = pd;
 484         mr->lkey = tmp_lkey;
 485         mr->rkey = tmp_rkey;
 486
 487 rereg_phys_mr_exit1:
 488         spin_unlock_irqrestore(&e_mr->mrlock, sl_flags);
 489 rereg_phys_mr_exit0:
 490         if (ret)
 491                 ehca_err(mr->device, "ret=%x mr=%p mr_rereg_mask=%x pd=%p "
 492                          "phys_buf_array=%p num_phys_buf=%x mr_access_flags=%x "
 493                          "iova_start=%p",
 494                          ret, mr, mr_rereg_mask, pd, phys_buf_array,
 495                          num_phys_buf, mr_access_flags, iova_start);
 496         return ret;
 497 } /* end ehca_rereg_phys_mr() */
 498
 499 /*----------------------------------------------------------------------*/
 500
 501 int ehca_query_mr(struct ib_mr *mr, struct ib_mr_attr *mr_attr)
 502 {
 503         int ret = 0;
 504         u64 h_ret;
 505         struct ehca_shca *shca =
 506                 container_of(mr->device, struct ehca_shca, ib_device);
 507         struct ehca_mr *e_mr = container_of(mr, struct ehca_mr, ib.ib_mr);
 508         struct ehca_pd *my_pd = container_of(mr->pd, struct ehca_pd, ib_pd);
 509         u32 cur_pid = current->tgid;
 510         unsigned long sl_flags;
 511         struct ehca_mr_hipzout_parms hipzout = {{0},0,0,0,0,0};
 512
 513         if (my_pd->ib_pd.uobject && my_pd->ib_pd.uobject->context &&
 514             (my_pd->ownpid != cur_pid)) {
 515                 ehca_err(mr->device, "Invalid caller pid=%x ownpid=%x",
 516                          cur_pid, my_pd->ownpid);
 517                 ret = -EINVAL;
 518                 goto query_mr_exit0;
 519         }
 520
 521         if ((e_mr->flags & EHCA_MR_FLAG_FMR)) {
 522                 ehca_err(mr->device, "not supported for FMR, mr=%p e_mr=%p "
 523                          "e_mr->flags=%x", mr, e_mr, e_mr->flags);
 524                 ret = -EINVAL;
 525                 goto query_mr_exit0;
 526         }
 527
 528         memset(mr_attr, 0, sizeof(struct ib_mr_attr));
 529         spin_lock_irqsave(&e_mr->mrlock, sl_flags);
 530
 531         h_ret = hipz_h_query_mr(shca->ipz_hca_handle, e_mr, &hipzout);
 532         if (h_ret != H_SUCCESS) {
 533                 ehca_err(mr->device, "hipz_mr_query failed, h_ret=%lx mr=%p "
 534                          "hca_hndl=%lx mr_hndl=%lx lkey=%x",
 535                          h_ret, mr, shca->ipz_hca_handle.handle,
 536                          e_mr->ipz_mr_handle.handle, mr->lkey);
 537                 ret = ehca_mrmw_map_hrc_query_mr(h_ret);
 538                 goto query_mr_exit1;
 539         }
 540         mr_attr->pd               = mr->pd;
 541         mr_attr->device_virt_addr = hipzout.vaddr;
 542         mr_attr->size             = hipzout.len;
 543         mr_attr->lkey             = hipzout.lkey;
 544         mr_attr->rkey             = hipzout.rkey;
 545         ehca_mrmw_reverse_map_acl(&hipzout.acl, &mr_attr->mr_access_flags);
 546
 547 query_mr_exit1:
 548         spin_unlock_irqrestore(&e_mr->mrlock, sl_flags);
 549 query_mr_exit0:
 550         if (ret)
 551                 ehca_err(mr->device, "ret=%x mr=%p mr_attr=%p",
 552                          ret, mr, mr_attr);
 553         return ret;
 554 } /* end ehca_query_mr() */
 555
 556 /*----------------------------------------------------------------------*/
 557
 558 int ehca_dereg_mr(struct ib_mr *mr)
 559 {
 560         int ret = 0;
 561         u64 h_ret;
 562         struct ehca_shca *shca =
 563                 container_of(mr->device, struct ehca_shca, ib_device);
 564         struct ehca_mr *e_mr = container_of(mr, struct ehca_mr, ib.ib_mr);
 565         struct ehca_pd *my_pd = container_of(mr->pd, struct ehca_pd, ib_pd);
 566         u32 cur_pid = current->tgid;
 567
 568         if (my_pd->ib_pd.uobject && my_pd->ib_pd.uobject->context &&
 569             (my_pd->ownpid != cur_pid)) {
 570                 ehca_err(mr->device, "Invalid caller pid=%x ownpid=%x",
 571                          cur_pid, my_pd->ownpid);
 572                 ret = -EINVAL;
 573                 goto dereg_mr_exit0;
 574         }
 575
 576         if ((e_mr->flags & EHCA_MR_FLAG_FMR)) {
 577                 ehca_err(mr->device, "not supported for FMR, mr=%p e_mr=%p "
 578                          "e_mr->flags=%x", mr, e_mr, e_mr->flags);
 579                 ret = -EINVAL;
 580                 goto dereg_mr_exit0;
 581         } else if (e_mr == shca->maxmr) {
 582                 /* should be impossible, however reject to be sure */
 583                 ehca_err(mr->device, "dereg internal max-MR impossible, mr=%p "
 584                          "shca->maxmr=%p mr->lkey=%x",
 585                          mr, shca->maxmr, mr->lkey);
 586                 ret = -EINVAL;
 587                 goto dereg_mr_exit0;
 588         }
 589
 590         /* TODO: BUSY: MR still has bound window(s) */
 591         h_ret = hipz_h_free_resource_mr(shca->ipz_hca_handle, e_mr);
 592         if (h_ret != H_SUCCESS) {
 593                 ehca_err(mr->device, "hipz_free_mr failed, h_ret=%lx shca=%p "
 594                          "e_mr=%p hca_hndl=%lx mr_hndl=%lx mr->lkey=%x",
 595                          h_ret, shca, e_mr, shca->ipz_hca_handle.handle,
 596                          e_mr->ipz_mr_handle.handle, mr->lkey);
 597                 ret = ehca_mrmw_map_hrc_free_mr(h_ret);
 598                 goto dereg_mr_exit0;
 599         }
 600
 601         /* successful deregistration */
 602         ehca_mr_delete(e_mr);
 603
 604 dereg_mr_exit0:
 605         if (ret)
 606                 ehca_err(mr->device, "ret=%x mr=%p", ret, mr);
 607         return ret;
 608 } /* end ehca_dereg_mr() */
 609
 610 /*----------------------------------------------------------------------*/
 611
 612 struct ib_mw *ehca_alloc_mw(struct ib_pd *pd)
 613 {
 614         struct ib_mw *ib_mw;
 615         u64 h_ret;
 616         struct ehca_mw *e_mw;
 617         struct ehca_pd *e_pd = container_of(pd, struct ehca_pd, ib_pd);
 618         struct ehca_shca *shca =
 619                 container_of(pd->device, struct ehca_shca, ib_device);
 620         struct ehca_mw_hipzout_parms hipzout = {{0},0};
 621
 622         e_mw = ehca_mw_new();
 623         if (!e_mw) {
 624                 ib_mw = ERR_PTR(-ENOMEM);
 625                 goto alloc_mw_exit0;
 626         }
 627
 628         h_ret = hipz_h_alloc_resource_mw(shca->ipz_hca_handle, e_mw,
 629                                          e_pd->fw_pd, &hipzout);
 630         if (h_ret != H_SUCCESS) {
 631                 ehca_err(pd->device, "hipz_mw_allocate failed, h_ret=%lx "
 632                          "shca=%p hca_hndl=%lx mw=%p",
 633                          h_ret, shca, shca->ipz_hca_handle.handle, e_mw);
 634                 ib_mw = ERR_PTR(ehca_mrmw_map_hrc_alloc(h_ret));
 635                 goto alloc_mw_exit1;
 636         }
 637         /* successful MW allocation */
 638         e_mw->ipz_mw_handle = hipzout.handle;
 639         e_mw->ib_mw.rkey    = hipzout.rkey;
 640         return &e_mw->ib_mw;
 641
 642 alloc_mw_exit1:
 643         ehca_mw_delete(e_mw);
 644 alloc_mw_exit0:
 645         if (IS_ERR(ib_mw))
 646                 ehca_err(pd->device, "rc=%lx pd=%p", PTR_ERR(ib_mw), pd);
 647         return ib_mw;
 648 } /* end ehca_alloc_mw() */
 649
 650 /*----------------------------------------------------------------------*/
 651
 652 int ehca_bind_mw(struct ib_qp *qp,
 653                  struct ib_mw *mw,
 654                  struct ib_mw_bind *mw_bind)
 655 {
 656         /* TODO: not supported up to now */
 657         ehca_gen_err("bind MW currently not supported by HCAD");
 658
 659         return -EPERM;
 660 } /* end ehca_bind_mw() */
 661
 662 /*----------------------------------------------------------------------*/
 663
 664 int ehca_dealloc_mw(struct ib_mw *mw)
 665 {
 666         u64 h_ret;
 667         struct ehca_shca *shca =
 668                 container_of(mw->device, struct ehca_shca, ib_device);
 669         struct ehca_mw *e_mw = container_of(mw, struct ehca_mw, ib_mw);
 670
 671         h_ret = hipz_h_free_resource_mw(shca->ipz_hca_handle, e_mw);
 672         if (h_ret != H_SUCCESS) {
 673                 ehca_err(mw->device, "hipz_free_mw failed, h_ret=%lx shca=%p "
 674                          "mw=%p rkey=%x hca_hndl=%lx mw_hndl=%lx",
 675                          h_ret, shca, mw, mw->rkey, shca->ipz_hca_handle.handle,
 676                          e_mw->ipz_mw_handle.handle);
 677                 return ehca_mrmw_map_hrc_free_mw(h_ret);
 678         }
 679         /* successful deallocation */
 680         ehca_mw_delete(e_mw);
 681         return 0;
 682 } /* end ehca_dealloc_mw() */
 683
 684 /*----------------------------------------------------------------------*/
 685
 686 struct ib_fmr *ehca_alloc_fmr(struct ib_pd *pd,
 687                               int mr_access_flags,
 688                               struct ib_fmr_attr *fmr_attr)
 689 {
 690         struct ib_fmr *ib_fmr;
 691         struct ehca_shca *shca =
 692                 container_of(pd->device, struct ehca_shca, ib_device);
 693         struct ehca_pd *e_pd = container_of(pd, struct ehca_pd, ib_pd);
 694         struct ehca_mr *e_fmr;
 695         int ret;
 696         u32 tmp_lkey, tmp_rkey;
 697         struct ehca_mr_pginfo pginfo={0,0,0,0,0,0,0,NULL,0,NULL,NULL,0,NULL,0};
 698
 699         /* check other parameters */
 700         if (((mr_access_flags & IB_ACCESS_REMOTE_WRITE) &&
 701              !(mr_access_flags & IB_ACCESS_LOCAL_WRITE)) ||
 702             ((mr_access_flags & IB_ACCESS_REMOTE_ATOMIC) &&
 703              !(mr_access_flags & IB_ACCESS_LOCAL_WRITE))) {
 704                 /*
 705                  * Remote Write Access requires Local Write Access
 706                  * Remote Atomic Access requires Local Write Access
 707                  */
 708                 ehca_err(pd->device, "bad input values: mr_access_flags=%x",
 709                          mr_access_flags);
 710                 ib_fmr = ERR_PTR(-EINVAL);
 711                 goto alloc_fmr_exit0;
 712         }
 713         if (mr_access_flags & IB_ACCESS_MW_BIND) {
 714                 ehca_err(pd->device, "bad input values: mr_access_flags=%x",
 715                          mr_access_flags);
 716                 ib_fmr = ERR_PTR(-EINVAL);
 717                 goto alloc_fmr_exit0;
 718         }
 719         if ((fmr_attr->max_pages == 0) || (fmr_attr->max_maps == 0)) {
 720                 ehca_err(pd->device, "bad input values: fmr_attr->max_pages=%x "
 721                          "fmr_attr->max_maps=%x fmr_attr->page_shift=%x",
 722                          fmr_attr->max_pages, fmr_attr->max_maps,
 723                          fmr_attr->page_shift);
 724                 ib_fmr = ERR_PTR(-EINVAL);
 725                 goto alloc_fmr_exit0;
 726         }
 727         if (((1 << fmr_attr->page_shift) != EHCA_PAGESIZE) &&
 728             ((1 << fmr_attr->page_shift) != PAGE_SIZE)) {
 729                 ehca_err(pd->device, "unsupported fmr_attr->page_shift=%x",
 730                          fmr_attr->page_shift);
 731                 ib_fmr = ERR_PTR(-EINVAL);
 732                 goto alloc_fmr_exit0;
 733         }
 734
 735         e_fmr = ehca_mr_new();
 736         if (!e_fmr) {
 737                 ib_fmr = ERR_PTR(-ENOMEM);
 738                 goto alloc_fmr_exit0;
 739         }
 740         e_fmr->flags |= EHCA_MR_FLAG_FMR;
 741
 742         /* register MR on HCA */
 743         ret = ehca_reg_mr(shca, e_fmr, NULL,
 744                           fmr_attr->max_pages * (1 << fmr_attr->page_shift),
 745                           mr_access_flags, e_pd, &pginfo,
 746                           &tmp_lkey, &tmp_rkey);
 747         if (ret) {
 748                 ib_fmr = ERR_PTR(ret);
 749                 goto alloc_fmr_exit1;
 750         }
 751
 752         /* successful */
 753         e_fmr->fmr_page_size = 1 << fmr_attr->page_shift;
 754         e_fmr->fmr_max_pages = fmr_attr->max_pages;
 755         e_fmr->fmr_max_maps = fmr_attr->max_maps;
 756         e_fmr->fmr_map_cnt = 0;
 757         return &e_fmr->ib.ib_fmr;
 758
 759 alloc_fmr_exit1:
 760         ehca_mr_delete(e_fmr);
 761 alloc_fmr_exit0:
 762         if (IS_ERR(ib_fmr))
 763                 ehca_err(pd->device, "rc=%lx pd=%p mr_access_flags=%x "
 764                          "fmr_attr=%p", PTR_ERR(ib_fmr), pd,
 765                          mr_access_flags, fmr_attr);
 766         return ib_fmr;
 767 } /* end ehca_alloc_fmr() */
 768
 769 /*----------------------------------------------------------------------*/
 770
 771 int ehca_map_phys_fmr(struct ib_fmr *fmr,
 772                       u64 *page_list,
 773                       int list_len,
 774                       u64 iova)
 775 {
 776         int ret;
 777         struct ehca_shca *shca =
 778                 container_of(fmr->device, struct ehca_shca, ib_device);
 779         struct ehca_mr *e_fmr = container_of(fmr, struct ehca_mr, ib.ib_fmr);
 780         struct ehca_pd *e_pd = container_of(fmr->pd, struct ehca_pd, ib_pd);
 781         struct ehca_mr_pginfo pginfo={0,0,0,0,0,0,0,NULL,0,NULL,NULL,0,NULL,0};
 782         u32 tmp_lkey, tmp_rkey;
 783
 784         if (!(e_fmr->flags & EHCA_MR_FLAG_FMR)) {
 785                 ehca_err(fmr->device, "not a FMR, e_fmr=%p e_fmr->flags=%x",
 786                          e_fmr, e_fmr->flags);
 787                 ret = -EINVAL;
 788                 goto map_phys_fmr_exit0;
 789         }
 790         ret = ehca_fmr_check_page_list(e_fmr, page_list, list_len);
 791         if (ret)
 792                 goto map_phys_fmr_exit0;
 793         if (iova % e_fmr->fmr_page_size) {
 794                 /* only whole-numbered pages */
 795                 ehca_err(fmr->device, "bad iova, iova=%lx fmr_page_size=%x",
 796                          iova, e_fmr->fmr_page_size);
 797                 ret = -EINVAL;
 798                 goto map_phys_fmr_exit0;
 799         }
 800         if (e_fmr->fmr_map_cnt >= e_fmr->fmr_max_maps) {
 801                 /* HCAD does not limit the maps, however trace this anyway */
 802                 ehca_info(fmr->device, "map limit exceeded, fmr=%p "
 803                           "e_fmr->fmr_map_cnt=%x e_fmr->fmr_max_maps=%x",
 804                           fmr, e_fmr->fmr_map_cnt, e_fmr->fmr_max_maps);
 805         }
 806
 807         pginfo.type      = EHCA_MR_PGI_FMR;
 808         pginfo.num_pages = list_len;
 809         pginfo.num_4k    = list_len * (e_fmr->fmr_page_size / EHCA_PAGESIZE);
 810         pginfo.page_list = page_list;
 811         pginfo.next_4k   = ((iova & (e_fmr->fmr_page_size-1)) /
 812                             EHCA_PAGESIZE);
 813
 814         ret = ehca_rereg_mr(shca, e_fmr, (u64*)iova,
 815                             list_len * e_fmr->fmr_page_size,
 816                             e_fmr->acl, e_pd, &pginfo, &tmp_lkey, &tmp_rkey);
 817         if (ret)
 818                 goto map_phys_fmr_exit0;
 819
 820         /* successful reregistration */
 821         e_fmr->fmr_map_cnt++;
 822         e_fmr->ib.ib_fmr.lkey = tmp_lkey;
 823         e_fmr->ib.ib_fmr.rkey = tmp_rkey;
 824         return 0;
 825
 826 map_phys_fmr_exit0:
 827         if (ret)
 828                 ehca_err(fmr->device, "ret=%x fmr=%p page_list=%p list_len=%x "
 829                          "iova=%lx",
 830                          ret, fmr, page_list, list_len, iova);
 831         return ret;
 832 } /* end ehca_map_phys_fmr() */
 833
 834 /*----------------------------------------------------------------------*/
 835
 836 int ehca_unmap_fmr(struct list_head *fmr_list)
 837 {
 838         int ret = 0;
 839         struct ib_fmr *ib_fmr;
 840         struct ehca_shca *shca = NULL;
 841         struct ehca_shca *prev_shca;
 842         struct ehca_mr *e_fmr;
 843         u32 num_fmr = 0;
 844         u32 unmap_fmr_cnt = 0;
 845
 846         /* check all FMR belong to same SHCA, and check internal flag */
 847         list_for_each_entry(ib_fmr, fmr_list, list) {
 848                 prev_shca = shca;
 849                 if (!ib_fmr) {
 850                         ehca_gen_err("bad fmr=%p in list", ib_fmr);
 851                         ret = -EINVAL;
 852                         goto unmap_fmr_exit0;
 853                 }
 854                 shca = container_of(ib_fmr->device, struct ehca_shca,
 855                                     ib_device);
 856                 e_fmr = container_of(ib_fmr, struct ehca_mr, ib.ib_fmr);
 857                 if ((shca != prev_shca) && prev_shca) {
 858                         ehca_err(&shca->ib_device, "SHCA mismatch, shca=%p "
 859                                  "prev_shca=%p e_fmr=%p",
 860                                  shca, prev_shca, e_fmr);
 861                         ret = -EINVAL;
 862                         goto unmap_fmr_exit0;
 863                 }
 864                 if (!(e_fmr->flags & EHCA_MR_FLAG_FMR)) {
 865                         ehca_err(&shca->ib_device, "not a FMR, e_fmr=%p "
 866                                  "e_fmr->flags=%x", e_fmr, e_fmr->flags);
 867                         ret = -EINVAL;
 868                         goto unmap_fmr_exit0;
 869                 }
 870                 num_fmr++;
 871         }
 872
 873         /* loop over all FMRs to unmap */
 874         list_for_each_entry(ib_fmr, fmr_list, list) {
 875                 unmap_fmr_cnt++;
 876                 e_fmr = container_of(ib_fmr, struct ehca_mr, ib.ib_fmr);
 877                 shca = container_of(ib_fmr->device, struct ehca_shca,
 878                                     ib_device);
 879                 ret = ehca_unmap_one_fmr(shca, e_fmr);
 880                 if (ret) {
 881                         /* unmap failed, stop unmapping of rest of FMRs */
 882                         ehca_err(&shca->ib_device, "unmap of one FMR failed, "
 883                                  "stop rest, e_fmr=%p num_fmr=%x "
 884                                  "unmap_fmr_cnt=%x lkey=%x", e_fmr, num_fmr,
 885                                  unmap_fmr_cnt, e_fmr->ib.ib_fmr.lkey);
 886                         goto unmap_fmr_exit0;
 887                 }
 888         }
 889
 890 unmap_fmr_exit0:
 891         if (ret)
 892                 ehca_gen_err("ret=%x fmr_list=%p num_fmr=%x unmap_fmr_cnt=%x",
 893                              ret, fmr_list, num_fmr, unmap_fmr_cnt);
 894         return ret;
 895 } /* end ehca_unmap_fmr() */
 896
 897 /*----------------------------------------------------------------------*/
 898
 899 int ehca_dealloc_fmr(struct ib_fmr *fmr)
 900 {
 901         int ret;
 902         u64 h_ret;
 903         struct ehca_shca *shca =
 904                 container_of(fmr->device, struct ehca_shca, ib_device);
 905         struct ehca_mr *e_fmr = container_of(fmr, struct ehca_mr, ib.ib_fmr);
 906
 907         if (!(e_fmr->flags & EHCA_MR_FLAG_FMR)) {
 908                 ehca_err(fmr->device, "not a FMR, e_fmr=%p e_fmr->flags=%x",
 909                          e_fmr, e_fmr->flags);
 910                 ret = -EINVAL;
 911                 goto free_fmr_exit0;
 912         }
 913
 914         h_ret = hipz_h_free_resource_mr(shca->ipz_hca_handle, e_fmr);
 915         if (h_ret != H_SUCCESS) {
 916                 ehca_err(fmr->device, "hipz_free_mr failed, h_ret=%lx e_fmr=%p "
 917                          "hca_hndl=%lx fmr_hndl=%lx fmr->lkey=%x",
 918                          h_ret, e_fmr, shca->ipz_hca_handle.handle,
 919                          e_fmr->ipz_mr_handle.handle, fmr->lkey);
 920                 ret = ehca_mrmw_map_hrc_free_mr(h_ret);
 921                 goto free_fmr_exit0;
 922         }
 923         /* successful deregistration */
 924         ehca_mr_delete(e_fmr);
 925         return 0;
 926
 927 free_fmr_exit0:
 928         if (ret)
 929                 ehca_err(&shca->ib_device, "ret=%x fmr=%p", ret, fmr);
 930         return ret;
 931 } /* end ehca_dealloc_fmr() */
 932
 933 /*----------------------------------------------------------------------*/
 934
 935 int ehca_reg_mr(struct ehca_shca *shca,
 936                 struct ehca_mr *e_mr,
 937                 u64 *iova_start,
 938                 u64 size,
 939                 int acl,
 940                 struct ehca_pd *e_pd,
 941                 struct ehca_mr_pginfo *pginfo,
 942                 u32 *lkey, /*OUT*/
 943                 u32 *rkey) /*OUT*/
 944 {
 945         int ret;
 946         u64 h_ret;
 947         u32 hipz_acl;
 948         struct ehca_mr_hipzout_parms hipzout = {{0},0,0,0,0,0};
 949
 950         ehca_mrmw_map_acl(acl, &hipz_acl);
 951         ehca_mrmw_set_pgsize_hipz_acl(&hipz_acl);
 952         if (ehca_use_hp_mr == 1)
 953                 hipz_acl |= 0x00000001;
 954
 955         h_ret = hipz_h_alloc_resource_mr(shca->ipz_hca_handle, e_mr,
 956                                          (u64)iova_start, size, hipz_acl,
 957                                          e_pd->fw_pd, &hipzout);
 958         if (h_ret != H_SUCCESS) {
 959                 ehca_err(&shca->ib_device, "hipz_alloc_mr failed, h_ret=%lx "
 960                          "hca_hndl=%lx", h_ret, shca->ipz_hca_handle.handle);
 961                 ret = ehca_mrmw_map_hrc_alloc(h_ret);
 962                 goto ehca_reg_mr_exit0;
 963         }
 964
 965         e_mr->ipz_mr_handle = hipzout.handle;
 966
 967         ret = ehca_reg_mr_rpages(shca, e_mr, pginfo);
 968         if (ret)
 969                 goto ehca_reg_mr_exit1;
 970
 971         /* successful registration */
 972         e_mr->num_pages = pginfo->num_pages;
 973         e_mr->num_4k    = pginfo->num_4k;
 974         e_mr->start     = iova_start;
 975         e_mr->size      = size;
 976         e_mr->acl       = acl;
 977         *lkey = hipzout.lkey;
 978         *rkey = hipzout.rkey;
 979         return 0;
 980
 981 ehca_reg_mr_exit1:
 982         h_ret = hipz_h_free_resource_mr(shca->ipz_hca_handle, e_mr);
 983         if (h_ret != H_SUCCESS) {
 984                 ehca_err(&shca->ib_device, "h_ret=%lx shca=%p e_mr=%p "
 985                          "iova_start=%p size=%lx acl=%x e_pd=%p lkey=%x "
 986                          "pginfo=%p num_pages=%lx num_4k=%lx ret=%x",
 987                          h_ret, shca, e_mr, iova_start, size, acl, e_pd,
 988                          hipzout.lkey, pginfo, pginfo->num_pages,
 989                          pginfo->num_4k, ret);
 990                 ehca_err(&shca->ib_device, "internal error in ehca_reg_mr, "
 991                          "not recoverable");
 992         }
 993 ehca_reg_mr_exit0:
 994         if (ret)
 995                 ehca_err(&shca->ib_device, "ret=%x shca=%p e_mr=%p "
 996                          "iova_start=%p size=%lx acl=%x e_pd=%p pginfo=%p "
 997                          "num_pages=%lx num_4k=%lx",
 998                          ret, shca, e_mr, iova_start, size, acl, e_pd, pginfo,
 999                          pginfo->num_pages, pginfo->num_4k);
1000         return ret;
1001 } /* end ehca_reg_mr() */
1002
1003 /*----------------------------------------------------------------------*/
1004
1005 int ehca_reg_mr_rpages(struct ehca_shca *shca,
1006                        struct ehca_mr *e_mr,
1007                        struct ehca_mr_pginfo *pginfo)
1008 {
1009         int ret = 0;
1010         u64 h_ret;
1011         u32 rnum;
1012         u64 rpage;
1013         u32 i;
1014         u64 *kpage;
1015
1016         kpage = ehca_alloc_fw_ctrlblock();
1017         if (!kpage) {
1018                 ehca_err(&shca->ib_device, "kpage alloc failed");
1019                 ret = -ENOMEM;
1020                 goto ehca_reg_mr_rpages_exit0;
1021         }
1022
1023         /* max 512 pages per shot */
1024         for (i = 0; i < ((pginfo->num_4k + 512 - 1) / 512); i++) {
1025
1026                 if (i == ((pginfo->num_4k + 512 - 1) / 512) - 1) {
1027                         rnum = pginfo->num_4k % 512; /* last shot */
1028                         if (rnum == 0)
1029                                 rnum = 512;      /* last shot is full */
1030                 } else
1031                         rnum = 512;
1032
1033                 if (rnum > 1) {
1034                         ret = ehca_set_pagebuf(e_mr, pginfo, rnum, kpage);
1035                         if (ret) {
1036                                 ehca_err(&shca->ib_device, "ehca_set_pagebuf "
1037                                          "bad rc, ret=%x rnum=%x kpage=%p",
1038                                          ret, rnum, kpage);
1039                                 ret = -EFAULT;
1040                                 goto ehca_reg_mr_rpages_exit1;
1041                         }
1042                         rpage = virt_to_abs(kpage);
1043                         if (!rpage) {
1044                                 ehca_err(&shca->ib_device, "kpage=%p i=%x",
1045                                          kpage, i);
1046                                 ret = -EFAULT;
1047                                 goto ehca_reg_mr_rpages_exit1;
1048                         }
1049                 } else {  /* rnum==1 */
1050                         ret = ehca_set_pagebuf_1(e_mr, pginfo, &rpage);
1051                         if (ret) {
1052                                 ehca_err(&shca->ib_device, "ehca_set_pagebuf_1 "
1053                                          "bad rc, ret=%x i=%x", ret, i);
1054                                 ret = -EFAULT;
1055                                 goto ehca_reg_mr_rpages_exit1;
1056                         }
1057                 }
1058
1059                 h_ret = hipz_h_register_rpage_mr(shca->ipz_hca_handle, e_mr,
1060                                                  0, /* pagesize 4k */
1061                                                  0, rpage, rnum);
1062
1063                 if (i == ((pginfo->num_4k + 512 - 1) / 512) - 1) {
1064                         /*
1065                          * check for 'registration complete'==H_SUCCESS
1066                          * and for 'page registered'==H_PAGE_REGISTERED
1067                          */
1068                         if (h_ret != H_SUCCESS) {
1069                                 ehca_err(&shca->ib_device, "last "
1070                                          "hipz_reg_rpage_mr failed, h_ret=%lx "
1071                                          "e_mr=%p i=%x hca_hndl=%lx mr_hndl=%lx"
1072                                          " lkey=%x", h_ret, e_mr, i,
1073                                          shca->ipz_hca_handle.handle,
1074                                          e_mr->ipz_mr_handle.handle,
1075                                          e_mr->ib.ib_mr.lkey);
1076                                 ret = ehca_mrmw_map_hrc_rrpg_last(h_ret);
1077                                 break;
1078                         } else
1079                                 ret = 0;
1080                 } else if (h_ret != H_PAGE_REGISTERED) {
1081                         ehca_err(&shca->ib_device, "hipz_reg_rpage_mr failed, "
1082                                  "h_ret=%lx e_mr=%p i=%x lkey=%x hca_hndl=%lx "
1083                                  "mr_hndl=%lx", h_ret, e_mr, i,
1084                                  e_mr->ib.ib_mr.lkey,
1085                                  shca->ipz_hca_handle.handle,
1086                                  e_mr->ipz_mr_handle.handle);
1087                         ret = ehca_mrmw_map_hrc_rrpg_notlast(h_ret);
1088                         break;
1089                 } else
1090                         ret = 0;
1091         } /* end for(i) */
1092
1093
1094 ehca_reg_mr_rpages_exit1:
1095         ehca_free_fw_ctrlblock(kpage);
1096 ehca_reg_mr_rpages_exit0:
1097         if (ret)
1098                 ehca_err(&shca->ib_device, "ret=%x shca=%p e_mr=%p pginfo=%p "
1099                          "num_pages=%lx num_4k=%lx", ret, shca, e_mr, pginfo,
1100                          pginfo->num_pages, pginfo->num_4k);
1101         return ret;
1102 } /* end ehca_reg_mr_rpages() */
1103
1104 /*----------------------------------------------------------------------*/
1105
1106 inline int ehca_rereg_mr_rereg1(struct ehca_shca *shca,
1107                                 struct ehca_mr *e_mr,
1108                                 u64 *iova_start,
1109                                 u64 size,
1110                                 u32 acl,
1111                                 struct ehca_pd *e_pd,
1112                                 struct ehca_mr_pginfo *pginfo,
1113                                 u32 *lkey, /*OUT*/
1114                                 u32 *rkey) /*OUT*/
1115 {
1116         int ret;
1117         u64 h_ret;
1118         u32 hipz_acl;
1119         u64 *kpage;
1120         u64 rpage;
1121         struct ehca_mr_pginfo pginfo_save;
1122         struct ehca_mr_hipzout_parms hipzout = {{0},0,0,0,0,0};
1123
1124         ehca_mrmw_map_acl(acl, &hipz_acl);
1125         ehca_mrmw_set_pgsize_hipz_acl(&hipz_acl);
1126
1127         kpage = ehca_alloc_fw_ctrlblock();
1128         if (!kpage) {
1129                 ehca_err(&shca->ib_device, "kpage alloc failed");
1130                 ret = -ENOMEM;
1131                 goto ehca_rereg_mr_rereg1_exit0;
1132         }
1133
1134         pginfo_save = *pginfo;
1135         ret = ehca_set_pagebuf(e_mr, pginfo, pginfo->num_4k, kpage);
1136         if (ret) {
1137                 ehca_err(&shca->ib_device, "set pagebuf failed, e_mr=%p "
1138                          "pginfo=%p type=%x num_pages=%lx num_4k=%lx kpage=%p",
1139                          e_mr, pginfo, pginfo->type, pginfo->num_pages,
1140                          pginfo->num_4k,kpage);
1141                 goto ehca_rereg_mr_rereg1_exit1;
1142         }
1143         rpage = virt_to_abs(kpage);
1144         if (!rpage) {
1145                 ehca_err(&shca->ib_device, "kpage=%p", kpage);
1146                 ret = -EFAULT;
1147                 goto ehca_rereg_mr_rereg1_exit1;
1148         }
1149         h_ret = hipz_h_reregister_pmr(shca->ipz_hca_handle, e_mr,
1150                                       (u64)iova_start, size, hipz_acl,
1151                                       e_pd->fw_pd, rpage, &hipzout);
1152         if (h_ret != H_SUCCESS) {
1153                 /*
1154                  * reregistration unsuccessful, try it again with the 3 hCalls,
1155                  * e.g. this is required in case H_MR_CONDITION
1156                  * (MW bound or MR is shared)
1157                  */
1158                 ehca_warn(&shca->ib_device, "hipz_h_reregister_pmr failed "
1159                           "(Rereg1), h_ret=%lx e_mr=%p", h_ret, e_mr);
1160                 *pginfo = pginfo_save;
1161                 ret = -EAGAIN;
1162         } else if ((u64*)hipzout.vaddr != iova_start) {
1163                 ehca_err(&shca->ib_device, "PHYP changed iova_start in "
1164                          "rereg_pmr, iova_start=%p iova_start_out=%lx e_mr=%p "
1165                          "mr_handle=%lx lkey=%x lkey_out=%x", iova_start,
1166                          hipzout.vaddr, e_mr, e_mr->ipz_mr_handle.handle,
1167                          e_mr->ib.ib_mr.lkey, hipzout.lkey);
1168                 ret = -EFAULT;
1169         } else {
1170                 /*
1171                  * successful reregistration
1172                  * note: start and start_out are identical for eServer HCAs
1173                  */
1174                 e_mr->num_pages = pginfo->num_pages;
1175                 e_mr->num_4k    = pginfo->num_4k;
1176                 e_mr->start     = iova_start;
1177                 e_mr->size      = size;
1178                 e_mr->acl       = acl;
1179                 *lkey = hipzout.lkey;
1180                 *rkey = hipzout.rkey;
1181         }
1182
1183 ehca_rereg_mr_rereg1_exit1:
1184         ehca_free_fw_ctrlblock(kpage);
1185 ehca_rereg_mr_rereg1_exit0:
1186         if ( ret && (ret != -EAGAIN) )
1187                 ehca_err(&shca->ib_device, "ret=%x lkey=%x rkey=%x "
1188                          "pginfo=%p num_pages=%lx num_4k=%lx",
1189                          ret, *lkey, *rkey, pginfo, pginfo->num_pages,
1190                          pginfo->num_4k);
1191         return ret;
1192 } /* end ehca_rereg_mr_rereg1() */
1193
1194 /*----------------------------------------------------------------------*/
1195
1196 int ehca_rereg_mr(struct ehca_shca *shca,
1197                   struct ehca_mr *e_mr,
1198                   u64 *iova_start,
1199                   u64 size,
1200                   int acl,
1201                   struct ehca_pd *e_pd,
1202                   struct ehca_mr_pginfo *pginfo,
1203                   u32 *lkey,
1204                   u32 *rkey)
1205 {
1206         int ret = 0;
1207         u64 h_ret;
1208         int rereg_1_hcall = 1; /* 1: use hipz_h_reregister_pmr directly */
1209         int rereg_3_hcall = 0; /* 1: use 3 hipz calls for reregistration */
1210
1211         /* first determine reregistration hCall(s) */
1212         if ((pginfo->num_4k > 512) || (e_mr->num_4k > 512) ||
1213             (pginfo->num_4k > e_mr->num_4k)) {
1214                 ehca_dbg(&shca->ib_device, "Rereg3 case, pginfo->num_4k=%lx "
1215                          "e_mr->num_4k=%x", pginfo->num_4k, e_mr->num_4k);
1216                 rereg_1_hcall = 0;
1217                 rereg_3_hcall = 1;
1218         }
1219
1220         if (e_mr->flags & EHCA_MR_FLAG_MAXMR) { /* check for max-MR */
1221                 rereg_1_hcall = 0;
1222                 rereg_3_hcall = 1;
1223                 e_mr->flags &= ~EHCA_MR_FLAG_MAXMR;
1224                 ehca_err(&shca->ib_device, "Rereg MR for max-MR! e_mr=%p",
1225                          e_mr);
1226         }
1227
1228         if (rereg_1_hcall) {
1229                 ret = ehca_rereg_mr_rereg1(shca, e_mr, iova_start, size,
1230                                            acl, e_pd, pginfo, lkey, rkey);
1231                 if (ret) {
1232                         if (ret == -EAGAIN)
1233                                 rereg_3_hcall = 1;
1234                         else
1235                                 goto ehca_rereg_mr_exit0;
1236                 }
1237         }
1238
1239         if (rereg_3_hcall) {
1240                 struct ehca_mr save_mr;
1241
1242                 /* first deregister old MR */
1243                 h_ret = hipz_h_free_resource_mr(shca->ipz_hca_handle, e_mr);
1244                 if (h_ret != H_SUCCESS) {
1245                         ehca_err(&shca->ib_device, "hipz_free_mr failed, "
1246                                  "h_ret=%lx e_mr=%p hca_hndl=%lx mr_hndl=%lx "
1247                                  "mr->lkey=%x",
1248                                  h_ret, e_mr, shca->ipz_hca_handle.handle,
1249                                  e_mr->ipz_mr_handle.handle,
1250                                  e_mr->ib.ib_mr.lkey);
1251                         ret = ehca_mrmw_map_hrc_free_mr(h_ret);
1252                         goto ehca_rereg_mr_exit0;
1253                 }
1254                 /* clean ehca_mr_t, without changing struct ib_mr and lock */
1255                 save_mr = *e_mr;
1256                 ehca_mr_deletenew(e_mr);
1257
1258                 /* set some MR values */
1259                 e_mr->flags = save_mr.flags;
1260                 e_mr->fmr_page_size = save_mr.fmr_page_size;
1261                 e_mr->fmr_max_pages = save_mr.fmr_max_pages;
1262                 e_mr->fmr_max_maps = save_mr.fmr_max_maps;
1263                 e_mr->fmr_map_cnt = save_mr.fmr_map_cnt;
1264
1265                 ret = ehca_reg_mr(shca, e_mr, iova_start, size, acl,
1266                                       e_pd, pginfo, lkey, rkey);
1267                 if (ret) {
1268                         u32 offset = (u64)(&e_mr->flags) - (u64)e_mr;
1269                         memcpy(&e_mr->flags, &(save_mr.flags),
1270                                sizeof(struct ehca_mr) - offset);
1271                         goto ehca_rereg_mr_exit0;
1272                 }
1273         }
1274
1275 ehca_rereg_mr_exit0:
1276         if (ret)
1277                 ehca_err(&shca->ib_device, "ret=%x shca=%p e_mr=%p "
1278                          "iova_start=%p size=%lx acl=%x e_pd=%p pginfo=%p "
1279                          "num_pages=%lx lkey=%x rkey=%x rereg_1_hcall=%x "
1280                          "rereg_3_hcall=%x", ret, shca, e_mr, iova_start, size,
1281                          acl, e_pd, pginfo, pginfo->num_pages, *lkey, *rkey,
1282                          rereg_1_hcall, rereg_3_hcall);
1283         return ret;
1284 } /* end ehca_rereg_mr() */
1285
1286 /*----------------------------------------------------------------------*/
1287
1288 int ehca_unmap_one_fmr(struct ehca_shca *shca,
1289                        struct ehca_mr *e_fmr)
1290 {
1291         int ret = 0;
1292         u64 h_ret;
1293         int rereg_1_hcall = 1; /* 1: use hipz_mr_reregister directly */
1294         int rereg_3_hcall = 0; /* 1: use 3 hipz calls for unmapping */
1295         struct ehca_pd *e_pd =
1296                 container_of(e_fmr->ib.ib_fmr.pd, struct ehca_pd, ib_pd);
1297         struct ehca_mr save_fmr;
1298         u32 tmp_lkey, tmp_rkey;
1299         struct ehca_mr_pginfo pginfo={0,0,0,0,0,0,0,NULL,0,NULL,NULL,0,NULL,0};
1300         struct ehca_mr_hipzout_parms hipzout = {{0},0,0,0,0,0};
1301
1302         /* first check if reregistration hCall can be used for unmap */
1303         if (e_fmr->fmr_max_pages > 512) {
1304                 rereg_1_hcall = 0;
1305                 rereg_3_hcall = 1;
1306         }
1307
1308         if (rereg_1_hcall) {
1309                 /*
1310                  * note: after using rereg hcall with len=0,
1311                  * rereg hcall must be used again for registering pages
1312                  */
1313                 h_ret = hipz_h_reregister_pmr(shca->ipz_hca_handle, e_fmr, 0,
1314                                               0, 0, e_pd->fw_pd, 0, &hipzout);
1315                 if (h_ret != H_SUCCESS) {
1316                         /*
1317                          * should not happen, because length checked above,
1318                          * FMRs are not shared and no MW bound to FMRs
1319                          */
1320                         ehca_err(&shca->ib_device, "hipz_reregister_pmr failed "
1321                                  "(Rereg1), h_ret=%lx e_fmr=%p hca_hndl=%lx "
1322                                  "mr_hndl=%lx lkey=%x lkey_out=%x",
1323                                  h_ret, e_fmr, shca->ipz_hca_handle.handle,
1324                                  e_fmr->ipz_mr_handle.handle,
1325                                  e_fmr->ib.ib_fmr.lkey, hipzout.lkey);
1326                         rereg_3_hcall = 1;
1327                 } else {
1328                         /* successful reregistration */
1329                         e_fmr->start = NULL;
1330                         e_fmr->size = 0;
1331                         tmp_lkey = hipzout.lkey;
1332                         tmp_rkey = hipzout.rkey;
1333                 }
1334         }
1335
1336         if (rereg_3_hcall) {
1337                 struct ehca_mr save_mr;
1338
1339                 /* first free old FMR */
1340                 h_ret = hipz_h_free_resource_mr(shca->ipz_hca_handle, e_fmr);
1341                 if (h_ret != H_SUCCESS) {
1342                         ehca_err(&shca->ib_device, "hipz_free_mr failed, "
1343                                  "h_ret=%lx e_fmr=%p hca_hndl=%lx mr_hndl=%lx "
1344                                  "lkey=%x",
1345                                  h_ret, e_fmr, shca->ipz_hca_handle.handle,
1346                                  e_fmr->ipz_mr_handle.handle,
1347                                  e_fmr->ib.ib_fmr.lkey);
1348                         ret = ehca_mrmw_map_hrc_free_mr(h_ret);
1349                         goto ehca_unmap_one_fmr_exit0;
1350                 }
1351                 /* clean ehca_mr_t, without changing lock */
1352                 save_fmr = *e_fmr;
1353                 ehca_mr_deletenew(e_fmr);
1354
1355                 /* set some MR values */
1356                 e_fmr->flags = save_fmr.flags;
1357                 e_fmr->fmr_page_size = save_fmr.fmr_page_size;
1358                 e_fmr->fmr_max_pages = save_fmr.fmr_max_pages;
1359                 e_fmr->fmr_max_maps = save_fmr.fmr_max_maps;
1360                 e_fmr->fmr_map_cnt = save_fmr.fmr_map_cnt;
1361                 e_fmr->acl = save_fmr.acl;
1362
1363                 pginfo.type      = EHCA_MR_PGI_FMR;
1364                 pginfo.num_pages = 0;
1365                 pginfo.num_4k    = 0;
1366                 ret = ehca_reg_mr(shca, e_fmr, NULL,
1367                                   (e_fmr->fmr_max_pages * e_fmr->fmr_page_size),
1368                                   e_fmr->acl, e_pd, &pginfo, &tmp_lkey,
1369                                   &tmp_rkey);
1370                 if (ret) {
1371                         u32 offset = (u64)(&e_fmr->flags) - (u64)e_fmr;
1372                         memcpy(&e_fmr->flags, &(save_mr.flags),
1373                                sizeof(struct ehca_mr) - offset);
1374                         goto ehca_unmap_one_fmr_exit0;
1375                 }
1376         }
1377
1378 ehca_unmap_one_fmr_exit0:
1379         if (ret)
1380                 ehca_err(&shca->ib_device, "ret=%x tmp_lkey=%x tmp_rkey=%x "
1381                          "fmr_max_pages=%x rereg_1_hcall=%x rereg_3_hcall=%x",
1382                          ret, tmp_lkey, tmp_rkey, e_fmr->fmr_max_pages,
1383                          rereg_1_hcall, rereg_3_hcall);
1384         return ret;
1385 } /* end ehca_unmap_one_fmr() */
1386
1387 /*----------------------------------------------------------------------*/
1388
1389 int ehca_reg_smr(struct ehca_shca *shca,
1390                  struct ehca_mr *e_origmr,
1391                  struct ehca_mr *e_newmr,
1392                  u64 *iova_start,
1393                  int acl,
1394                  struct ehca_pd *e_pd,
1395                  u32 *lkey, /*OUT*/
1396                  u32 *rkey) /*OUT*/
1397 {
1398         int ret = 0;
1399         u64 h_ret;
1400         u32 hipz_acl;
1401         struct ehca_mr_hipzout_parms hipzout = {{0},0,0,0,0,0};
1402
1403         ehca_mrmw_map_acl(acl, &hipz_acl);
1404         ehca_mrmw_set_pgsize_hipz_acl(&hipz_acl);
1405
1406         h_ret = hipz_h_register_smr(shca->ipz_hca_handle, e_newmr, e_origmr,
1407                                     (u64)iova_start, hipz_acl, e_pd->fw_pd,
1408                                     &hipzout);
1409         if (h_ret != H_SUCCESS) {
1410                 ehca_err(&shca->ib_device, "hipz_reg_smr failed, h_ret=%lx "
1411                          "shca=%p e_origmr=%p e_newmr=%p iova_start=%p acl=%x "
1412                          "e_pd=%p hca_hndl=%lx mr_hndl=%lx lkey=%x",
1413                          h_ret, shca, e_origmr, e_newmr, iova_start, acl, e_pd,
1414                          shca->ipz_hca_handle.handle,
1415                          e_origmr->ipz_mr_handle.handle,
1416                          e_origmr->ib.ib_mr.lkey);
1417                 ret = ehca_mrmw_map_hrc_reg_smr(h_ret);
1418                 goto ehca_reg_smr_exit0;
1419         }
1420         /* successful registration */
1421         e_newmr->num_pages     = e_origmr->num_pages;
1422         e_newmr->num_4k        = e_origmr->num_4k;
1423         e_newmr->start         = iova_start;
1424         e_newmr->size          = e_origmr->size;
1425         e_newmr->acl           = acl;
1426         e_newmr->ipz_mr_handle = hipzout.handle;
1427         *lkey = hipzout.lkey;
1428         *rkey = hipzout.rkey;
1429         return 0;
1430
1431 ehca_reg_smr_exit0:
1432         if (ret)
1433                 ehca_err(&shca->ib_device, "ret=%x shca=%p e_origmr=%p "
1434                          "e_newmr=%p iova_start=%p acl=%x e_pd=%p",
1435                          ret, shca, e_origmr, e_newmr, iova_start, acl, e_pd);
1436         return ret;
1437 } /* end ehca_reg_smr() */
1438
1439 /*----------------------------------------------------------------------*/
1440
1441 /* register internal max-MR to internal SHCA */
1442 int ehca_reg_internal_maxmr(
1443         struct ehca_shca *shca,
1444         struct ehca_pd *e_pd,
1445         struct ehca_mr **e_maxmr)  /*OUT*/
1446 {
1447         int ret;
1448         struct ehca_mr *e_mr;
1449         u64 *iova_start;
1450         u64 size_maxmr;
1451         struct ehca_mr_pginfo pginfo={0,0,0,0,0,0,0,NULL,0,NULL,NULL,0,NULL,0};
1452         struct ib_phys_buf ib_pbuf;
1453         u32 num_pages_mr;
1454         u32 num_pages_4k; /* 4k portion "pages" */
1455
1456         e_mr = ehca_mr_new();
1457         if (!e_mr) {
1458                 ehca_err(&shca->ib_device, "out of memory");
1459                 ret = -ENOMEM;
1460                 goto ehca_reg_internal_maxmr_exit0;
1461         }
1462         e_mr->flags |= EHCA_MR_FLAG_MAXMR;
1463
1464         /* register internal max-MR on HCA */
1465         size_maxmr = (u64)high_memory - PAGE_OFFSET;
1466         iova_start = (u64*)KERNELBASE;
1467         ib_pbuf.addr = 0;
1468         ib_pbuf.size = size_maxmr;
1469         num_pages_mr = ((((u64)iova_start % PAGE_SIZE) + size_maxmr +
1470                          PAGE_SIZE - 1) / PAGE_SIZE);
1471         num_pages_4k = ((((u64)iova_start % EHCA_PAGESIZE) + size_maxmr +
1472                          EHCA_PAGESIZE - 1) / EHCA_PAGESIZE);
1473
1474         pginfo.type           = EHCA_MR_PGI_PHYS;
1475         pginfo.num_pages      = num_pages_mr;
1476         pginfo.num_4k         = num_pages_4k;
1477         pginfo.num_phys_buf   = 1;
1478         pginfo.phys_buf_array = &ib_pbuf;
1479
1480         ret = ehca_reg_mr(shca, e_mr, iova_start, size_maxmr, 0, e_pd,
1481                           &pginfo, &e_mr->ib.ib_mr.lkey,
1482                           &e_mr->ib.ib_mr.rkey);
1483         if (ret) {
1484                 ehca_err(&shca->ib_device, "reg of internal max MR failed, "
1485                          "e_mr=%p iova_start=%p size_maxmr=%lx num_pages_mr=%x "
1486                          "num_pages_4k=%x", e_mr, iova_start, size_maxmr,
1487                          num_pages_mr, num_pages_4k);
1488                 goto ehca_reg_internal_maxmr_exit1;
1489         }
1490
1491         /* successful registration of all pages */
1492         e_mr->ib.ib_mr.device = e_pd->ib_pd.device;
1493         e_mr->ib.ib_mr.pd = &e_pd->ib_pd;
1494         e_mr->ib.ib_mr.uobject = NULL;
1495         atomic_inc(&(e_pd->ib_pd.usecnt));
1496         atomic_set(&(e_mr->ib.ib_mr.usecnt), 0);
1497         *e_maxmr = e_mr;
1498         return 0;
1499
1500 ehca_reg_internal_maxmr_exit1:
1501         ehca_mr_delete(e_mr);
1502 ehca_reg_internal_maxmr_exit0:
1503         if (ret)
1504                 ehca_err(&shca->ib_device, "ret=%x shca=%p e_pd=%p e_maxmr=%p",
1505                          ret, shca, e_pd, e_maxmr);
1506         return ret;
1507 } /* end ehca_reg_internal_maxmr() */
1508
1509 /*----------------------------------------------------------------------*/
1510
1511 int ehca_reg_maxmr(struct ehca_shca *shca,
1512                    struct ehca_mr *e_newmr,
1513                    u64 *iova_start,
1514                    int acl,
1515                    struct ehca_pd *e_pd,
1516                    u32 *lkey,
1517                    u32 *rkey)
1518 {
1519         u64 h_ret;
1520         struct ehca_mr *e_origmr = shca->maxmr;
1521         u32 hipz_acl;
1522         struct ehca_mr_hipzout_parms hipzout = {{0},0,0,0,0,0};
1523
1524         ehca_mrmw_map_acl(acl, &hipz_acl);
1525         ehca_mrmw_set_pgsize_hipz_acl(&hipz_acl);
1526
1527         h_ret = hipz_h_register_smr(shca->ipz_hca_handle, e_newmr, e_origmr,
1528                                     (u64)iova_start, hipz_acl, e_pd->fw_pd,
1529                                     &hipzout);
1530         if (h_ret != H_SUCCESS) {
1531                 ehca_err(&shca->ib_device, "hipz_reg_smr failed, h_ret=%lx "
1532                          "e_origmr=%p hca_hndl=%lx mr_hndl=%lx lkey=%x",
1533                          h_ret, e_origmr, shca->ipz_hca_handle.handle,
1534                          e_origmr->ipz_mr_handle.handle,
1535                          e_origmr->ib.ib_mr.lkey);
1536                 return ehca_mrmw_map_hrc_reg_smr(h_ret);
1537         }
1538         /* successful registration */
1539         e_newmr->num_pages     = e_origmr->num_pages;
1540         e_newmr->num_4k        = e_origmr->num_4k;
1541         e_newmr->start         = iova_start;
1542         e_newmr->size          = e_origmr->size;
1543         e_newmr->acl           = acl;
1544         e_newmr->ipz_mr_handle = hipzout.handle;
1545         *lkey = hipzout.lkey;
1546         *rkey = hipzout.rkey;
1547         return 0;
1548 } /* end ehca_reg_maxmr() */
1549
1550 /*----------------------------------------------------------------------*/
1551
1552 int ehca_dereg_internal_maxmr(struct ehca_shca *shca)
1553 {
1554         int ret;
1555         struct ehca_mr *e_maxmr;
1556         struct ib_pd *ib_pd;
1557
1558         if (!shca->maxmr) {
1559                 ehca_err(&shca->ib_device, "bad call, shca=%p", shca);
1560                 ret = -EINVAL;
1561                 goto ehca_dereg_internal_maxmr_exit0;
1562         }
1563
1564         e_maxmr = shca->maxmr;
1565         ib_pd = e_maxmr->ib.ib_mr.pd;
1566         shca->maxmr = NULL; /* remove internal max-MR indication from SHCA */
1567
1568         ret = ehca_dereg_mr(&e_maxmr->ib.ib_mr);
1569         if (ret) {
1570                 ehca_err(&shca->ib_device, "dereg internal max-MR failed, "
1571                          "ret=%x e_maxmr=%p shca=%p lkey=%x",
1572                          ret, e_maxmr, shca, e_maxmr->ib.ib_mr.lkey);
1573                 shca->maxmr = e_maxmr;
1574                 goto ehca_dereg_internal_maxmr_exit0;
1575         }
1576
1577         atomic_dec(&ib_pd->usecnt);
1578
1579 ehca_dereg_internal_maxmr_exit0:
1580         if (ret)
1581                 ehca_err(&shca->ib_device, "ret=%x shca=%p shca->maxmr=%p",
1582                          ret, shca, shca->maxmr);
1583         return ret;
1584 } /* end ehca_dereg_internal_maxmr() */
1585
1586 /*----------------------------------------------------------------------*/
1587
1588 /*
1589  * check physical buffer array of MR verbs for validness and
1590  * calculates MR size
1591  */
1592 int ehca_mr_chk_buf_and_calc_size(struct ib_phys_buf *phys_buf_array,
1593                                   int num_phys_buf,
1594                                   u64 *iova_start,
1595                                   u64 *size)
1596 {
1597         struct ib_phys_buf *pbuf = phys_buf_array;
1598         u64 size_count = 0;
1599         u32 i;
1600
1601         if (num_phys_buf == 0) {
1602                 ehca_gen_err("bad phys buf array len, num_phys_buf=0");
1603                 return -EINVAL;
1604         }
1605         /* check first buffer */
1606         if (((u64)iova_start & ~PAGE_MASK) != (pbuf->addr & ~PAGE_MASK)) {
1607                 ehca_gen_err("iova_start/addr mismatch, iova_start=%p "
1608                              "pbuf->addr=%lx pbuf->size=%lx",
1609                              iova_start, pbuf->addr, pbuf->size);
1610                 return -EINVAL;
1611         }
1612         if (((pbuf->addr + pbuf->size) % PAGE_SIZE) &&
1613             (num_phys_buf > 1)) {
1614                 ehca_gen_err("addr/size mismatch in 1st buf, pbuf->addr=%lx "
1615                              "pbuf->size=%lx", pbuf->addr, pbuf->size);
1616                 return -EINVAL;
1617         }
1618
1619         for (i = 0; i < num_phys_buf; i++) {
1620                 if ((i > 0) && (pbuf->addr % PAGE_SIZE)) {
1621                         ehca_gen_err("bad address, i=%x pbuf->addr=%lx "
1622                                      "pbuf->size=%lx",
1623                                      i, pbuf->addr, pbuf->size);
1624                         return -EINVAL;
1625                 }
1626                 if (((i > 0) && /* not 1st */
1627                      (i < (num_phys_buf - 1)) &&        /* not last */
1628                      (pbuf->size % PAGE_SIZE)) || (pbuf->size == 0)) {
1629                         ehca_gen_err("bad size, i=%x pbuf->size=%lx",
1630                                      i, pbuf->size);
1631                         return -EINVAL;
1632                 }
1633                 size_count += pbuf->size;
1634                 pbuf++;
1635         }
1636
1637         *size = size_count;
1638         return 0;
1639 } /* end ehca_mr_chk_buf_and_calc_size() */
1640
1641 /*----------------------------------------------------------------------*/
1642
1643 /* check page list of map FMR verb for validness */
1644 int ehca_fmr_check_page_list(struct ehca_mr *e_fmr,
1645                              u64 *page_list,
1646                              int list_len)
1647 {
1648         u32 i;
1649         u64 *page;
1650
1651         if ((list_len == 0) || (list_len > e_fmr->fmr_max_pages)) {
1652                 ehca_gen_err("bad list_len, list_len=%x "
1653                              "e_fmr->fmr_max_pages=%x fmr=%p",
1654                              list_len, e_fmr->fmr_max_pages, e_fmr);
1655                 return -EINVAL;
1656         }
1657
1658         /* each page must be aligned */
1659         page = page_list;
1660         for (i = 0; i < list_len; i++) {
1661                 if (*page % e_fmr->fmr_page_size) {
1662                         ehca_gen_err("bad page, i=%x *page=%lx page=%p fmr=%p "
1663                                      "fmr_page_size=%x", i, *page, page, e_fmr,
1664                                      e_fmr->fmr_page_size);
1665                         return -EINVAL;
1666                 }
1667                 page++;
1668         }
1669
1670         return 0;
1671 } /* end ehca_fmr_check_page_list() */
1672
1673 /*----------------------------------------------------------------------*/
1674
1675 /* setup page buffer from page info */
1676 int ehca_set_pagebuf(struct ehca_mr *e_mr,
1677                      struct ehca_mr_pginfo *pginfo,
1678                      u32 number,
1679                      u64 *kpage)
1680 {
1681         int ret = 0;
1682         struct ib_umem_chunk *prev_chunk;
1683         struct ib_umem_chunk *chunk;
1684         struct ib_phys_buf *pbuf;
1685         u64 *fmrlist;
1686         u64 num4k, pgaddr, offs4k;
1687         u32 i = 0;
1688         u32 j = 0;
1689
1690         if (pginfo->type == EHCA_MR_PGI_PHYS) {
1691                 /* loop over desired phys_buf_array entries */
1692                 while (i < number) {
1693                         pbuf   = pginfo->phys_buf_array + pginfo->next_buf;
1694                         num4k  = ((pbuf->addr % EHCA_PAGESIZE) + pbuf->size +
1695                                   EHCA_PAGESIZE - 1) / EHCA_PAGESIZE;
1696                         offs4k = (pbuf->addr & ~PAGE_MASK) / EHCA_PAGESIZE;
1697                         while (pginfo->next_4k < offs4k + num4k) {
1698                                 /* sanity check */
1699                                 if ((pginfo->page_cnt >= pginfo->num_pages) ||
1700                                     (pginfo->page_4k_cnt >= pginfo->num_4k)) {
1701                                         ehca_gen_err("page_cnt >= num_pages, "
1702                                                      "page_cnt=%lx "
1703                                                      "num_pages=%lx "
1704                                                      "page_4k_cnt=%lx "
1705                                                      "num_4k=%lx i=%x",
1706                                                      pginfo->page_cnt,
1707                                                      pginfo->num_pages,
1708                                                      pginfo->page_4k_cnt,
1709                                                      pginfo->num_4k, i);
1710                                         ret = -EFAULT;
1711                                         goto ehca_set_pagebuf_exit0;
1712                                 }
1713                                 *kpage = phys_to_abs(
1714                                         (pbuf->addr & EHCA_PAGEMASK)
1715                                         + (pginfo->next_4k * EHCA_PAGESIZE));
1716                                 if ( !(*kpage) && pbuf->addr ) {
1717                                         ehca_gen_err("pbuf->addr=%lx "
1718                                                      "pbuf->size=%lx "
1719                                                      "next_4k=%lx", pbuf->addr,
1720                                                      pbuf->size,
1721                                                      pginfo->next_4k);
1722                                         ret = -EFAULT;
1723                                         goto ehca_set_pagebuf_exit0;
1724                                 }
1725                                 (pginfo->page_4k_cnt)++;
1726                                 (pginfo->next_4k)++;
1727                                 if (pginfo->next_4k %
1728                                     (PAGE_SIZE / EHCA_PAGESIZE) == 0)
1729                                         (pginfo->page_cnt)++;
1730                                 kpage++;
1731                                 i++;
1732                                 if (i >= number) break;
1733                         }
1734                         if (pginfo->next_4k >= offs4k + num4k) {
1735                                 (pginfo->next_buf)++;
1736                                 pginfo->next_4k = 0;
1737                         }
1738                 }
1739         } else if (pginfo->type == EHCA_MR_PGI_USER) {
1740                 /* loop over desired chunk entries */
1741                 chunk      = pginfo->next_chunk;
1742                 prev_chunk = pginfo->next_chunk;
1743                 list_for_each_entry_continue(chunk,
1744                                              (&(pginfo->region->chunk_list)),
1745                                              list) {
1746                         for (i = pginfo->next_nmap; i < chunk->nmap; ) {
1747                                 pgaddr = ( page_to_pfn(chunk->page_list[i].page)
1748                                            << PAGE_SHIFT );
1749                                 *kpage = phys_to_abs(pgaddr +
1750                                                      (pginfo->next_4k *
1751                                                       EHCA_PAGESIZE));
1752                                 if ( !(*kpage) ) {
1753                                         ehca_gen_err("pgaddr=%lx "
1754                                                      "chunk->page_list[i]=%lx "
1755                                                      "i=%x next_4k=%lx mr=%p",
1756                                                      pgaddr,
1757                                                      (u64)sg_dma_address(
1758                                                              &chunk->
1759                                                              page_list[i]),
1760                                                      i, pginfo->next_4k, e_mr);
1761                                         ret = -EFAULT;
1762                                         goto ehca_set_pagebuf_exit0;
1763                                 }
1764                                 (pginfo->page_4k_cnt)++;
1765                                 (pginfo->next_4k)++;
1766                                 kpage++;
1767                                 if (pginfo->next_4k %
1768                                     (PAGE_SIZE / EHCA_PAGESIZE) == 0) {
1769                                         (pginfo->page_cnt)++;
1770                                         (pginfo->next_nmap)++;
1771                                         pginfo->next_4k = 0;
1772                                         i++;
1773                                 }
1774                                 j++;
1775                                 if (j >= number) break;
1776                         }
1777                         if ((pginfo->next_nmap >= chunk->nmap) &&
1778                             (j >= number)) {
1779                                 pginfo->next_nmap = 0;
1780                                 prev_chunk = chunk;
1781                                 break;
1782                         } else if (pginfo->next_nmap >= chunk->nmap) {
1783                                 pginfo->next_nmap = 0;
1784                                 prev_chunk = chunk;
1785                         } else if (j >= number)
1786                                 break;
1787                         else
1788                                 prev_chunk = chunk;
1789                 }
1790                 pginfo->next_chunk =
1791                         list_prepare_entry(prev_chunk,
1792                                            (&(pginfo->region->chunk_list)),
1793                                            list);
1794         } else if (pginfo->type == EHCA_MR_PGI_FMR) {
1795                 /* loop over desired page_list entries */
1796                 fmrlist = pginfo->page_list + pginfo->next_listelem;
1797                 for (i = 0; i < number; i++) {
1798                         *kpage = phys_to_abs((*fmrlist & EHCA_PAGEMASK) +
1799                                              pginfo->next_4k * EHCA_PAGESIZE);
1800                         if ( !(*kpage) ) {
1801                                 ehca_gen_err("*fmrlist=%lx fmrlist=%p "
1802                                              "next_listelem=%lx next_4k=%lx",
1803                                              *fmrlist, fmrlist,
1804                                              pginfo->next_listelem,
1805                                              pginfo->next_4k);
1806                                 ret = -EFAULT;
1807                                 goto ehca_set_pagebuf_exit0;
1808                         }
1809                         (pginfo->page_4k_cnt)++;
1810                         (pginfo->next_4k)++;
1811                         kpage++;
1812                         if (pginfo->next_4k %
1813                             (e_mr->fmr_page_size / EHCA_PAGESIZE) == 0) {
1814                                 (pginfo->page_cnt)++;
1815                                 (pginfo->next_listelem)++;
1816                                 fmrlist++;
1817                                 pginfo->next_4k = 0;
1818                         }
1819                 }
1820         } else {
1821                 ehca_gen_err("bad pginfo->type=%x", pginfo->type);
1822                 ret = -EFAULT;
1823                 goto ehca_set_pagebuf_exit0;
1824         }
1825
1826 ehca_set_pagebuf_exit0:
1827         if (ret)
1828                 ehca_gen_err("ret=%x e_mr=%p pginfo=%p type=%x num_pages=%lx "
1829                              "num_4k=%lx next_buf=%lx next_4k=%lx number=%x "
1830                              "kpage=%p page_cnt=%lx page_4k_cnt=%lx i=%x "
1831                              "next_listelem=%lx region=%p next_chunk=%p "
1832                              "next_nmap=%lx", ret, e_mr, pginfo, pginfo->type,
1833                              pginfo->num_pages, pginfo->num_4k,
1834                              pginfo->next_buf, pginfo->next_4k, number, kpage,
1835                              pginfo->page_cnt, pginfo->page_4k_cnt, i,
1836                              pginfo->next_listelem, pginfo->region,
1837                              pginfo->next_chunk, pginfo->next_nmap);
1838         return ret;
1839 } /* end ehca_set_pagebuf() */
1840
1841 /*----------------------------------------------------------------------*/
1842
1843 /* setup 1 page from page info page buffer */
1844 int ehca_set_pagebuf_1(struct ehca_mr *e_mr,
1845                        struct ehca_mr_pginfo *pginfo,
1846                        u64 *rpage)
1847 {
1848         int ret = 0;
1849         struct ib_phys_buf *tmp_pbuf;
1850         u64 *fmrlist;
1851         struct ib_umem_chunk *chunk;
1852         struct ib_umem_chunk *prev_chunk;
1853         u64 pgaddr, num4k, offs4k;
1854
1855         if (pginfo->type == EHCA_MR_PGI_PHYS) {
1856                 /* sanity check */
1857                 if ((pginfo->page_cnt >= pginfo->num_pages) ||
1858                     (pginfo->page_4k_cnt >= pginfo->num_4k)) {
1859                         ehca_gen_err("page_cnt >= num_pages, page_cnt=%lx "
1860                                      "num_pages=%lx page_4k_cnt=%lx num_4k=%lx",
1861                                      pginfo->page_cnt, pginfo->num_pages,
1862                                      pginfo->page_4k_cnt, pginfo->num_4k);
1863                         ret = -EFAULT;
1864                         goto ehca_set_pagebuf_1_exit0;
1865                 }
1866                 tmp_pbuf = pginfo->phys_buf_array + pginfo->next_buf;
1867                 num4k  = ((tmp_pbuf->addr % EHCA_PAGESIZE) + tmp_pbuf->size +
1868                           EHCA_PAGESIZE - 1) / EHCA_PAGESIZE;
1869                 offs4k = (tmp_pbuf->addr & ~PAGE_MASK) / EHCA_PAGESIZE;
1870                 *rpage = phys_to_abs((tmp_pbuf->addr & EHCA_PAGEMASK) +
1871                                      (pginfo->next_4k * EHCA_PAGESIZE));
1872                 if ( !(*rpage) && tmp_pbuf->addr ) {
1873                         ehca_gen_err("tmp_pbuf->addr=%lx"
1874                                      " tmp_pbuf->size=%lx next_4k=%lx",
1875                                      tmp_pbuf->addr, tmp_pbuf->size,
1876                                      pginfo->next_4k);
1877                         ret = -EFAULT;
1878                         goto ehca_set_pagebuf_1_exit0;
1879                 }
1880                 (pginfo->page_4k_cnt)++;
1881                 (pginfo->next_4k)++;
1882                 if (pginfo->next_4k % (PAGE_SIZE / EHCA_PAGESIZE) == 0)
1883                         (pginfo->page_cnt)++;
1884                 if (pginfo->next_4k >= offs4k + num4k) {
1885                         (pginfo->next_buf)++;
1886                         pginfo->next_4k = 0;
1887                 }
1888         } else if (pginfo->type == EHCA_MR_PGI_USER) {
1889                 chunk      = pginfo->next_chunk;
1890                 prev_chunk = pginfo->next_chunk;
1891                 list_for_each_entry_continue(chunk,
1892                                              (&(pginfo->region->chunk_list)),
1893                                              list) {
1894                         pgaddr = ( page_to_pfn(chunk->page_list[
1895                                                        pginfo->next_nmap].page)
1896                                    << PAGE_SHIFT);
1897                         *rpage = phys_to_abs(pgaddr +
1898                                              (pginfo->next_4k * EHCA_PAGESIZE));
1899                         if ( !(*rpage) ) {
1900                                 ehca_gen_err("pgaddr=%lx chunk->page_list[]=%lx"
1901                                              " next_nmap=%lx next_4k=%lx mr=%p",
1902                                              pgaddr, (u64)sg_dma_address(
1903                                                      &chunk->page_list[
1904                                                              pginfo->
1905                                                              next_nmap]),
1906                                              pginfo->next_nmap, pginfo->next_4k,
1907                                              e_mr);
1908                                 ret = -EFAULT;
1909                                 goto ehca_set_pagebuf_1_exit0;
1910                         }
1911                         (pginfo->page_4k_cnt)++;
1912                         (pginfo->next_4k)++;
1913                         if (pginfo->next_4k %
1914                             (PAGE_SIZE / EHCA_PAGESIZE) == 0) {
1915                                 (pginfo->page_cnt)++;
1916                                 (pginfo->next_nmap)++;
1917                                 pginfo->next_4k = 0;
1918                         }
1919                         if (pginfo->next_nmap >= chunk->nmap) {
1920                                 pginfo->next_nmap = 0;
1921                                 prev_chunk = chunk;
1922                         }
1923                         break;
1924                 }
1925                 pginfo->next_chunk =
1926                         list_prepare_entry(prev_chunk,
1927                                            (&(pginfo->region->chunk_list)),
1928                                            list);
1929         } else if (pginfo->type == EHCA_MR_PGI_FMR) {
1930                 fmrlist = pginfo->page_list + pginfo->next_listelem;
1931                 *rpage = phys_to_abs((*fmrlist & EHCA_PAGEMASK) +
1932                                      pginfo->next_4k * EHCA_PAGESIZE);
1933                 if ( !(*rpage) ) {
1934                         ehca_gen_err("*fmrlist=%lx fmrlist=%p "
1935                                      "next_listelem=%lx next_4k=%lx",
1936                                      *fmrlist, fmrlist, pginfo->next_listelem,
1937                                      pginfo->next_4k);
1938                         ret = -EFAULT;
1939                         goto ehca_set_pagebuf_1_exit0;
1940                 }
1941                 (pginfo->page_4k_cnt)++;
1942                 (pginfo->next_4k)++;
1943                 if (pginfo->next_4k %
1944                     (e_mr->fmr_page_size / EHCA_PAGESIZE) == 0) {
1945                         (pginfo->page_cnt)++;
1946                         (pginfo->next_listelem)++;
1947                         pginfo->next_4k = 0;
1948                 }
1949         } else {
1950                 ehca_gen_err("bad pginfo->type=%x", pginfo->type);
1951                 ret = -EFAULT;
1952                 goto ehca_set_pagebuf_1_exit0;
1953         }
1954
1955 ehca_set_pagebuf_1_exit0:
1956         if (ret)
1957                 ehca_gen_err("ret=%x e_mr=%p pginfo=%p type=%x num_pages=%lx "
1958                              "num_4k=%lx next_buf=%lx next_4k=%lx rpage=%p "
1959                              "page_cnt=%lx page_4k_cnt=%lx next_listelem=%lx "
1960                              "region=%p next_chunk=%p next_nmap=%lx", ret, e_mr,
1961                              pginfo, pginfo->type, pginfo->num_pages,
1962                              pginfo->num_4k, pginfo->next_buf, pginfo->next_4k,
1963                              rpage, pginfo->page_cnt, pginfo->page_4k_cnt,
1964                              pginfo->next_listelem, pginfo->region,
1965                              pginfo->next_chunk, pginfo->next_nmap);
1966         return ret;
1967 } /* end ehca_set_pagebuf_1() */
1968
1969 /*----------------------------------------------------------------------*/
1970
1971 /*
1972  * check MR if it is a max-MR, i.e. uses whole memory
1973  * in case it's a max-MR 1 is returned, else 0
1974  */
1975 int ehca_mr_is_maxmr(u64 size,
1976                      u64 *iova_start)
1977 {
1978         /* a MR is treated as max-MR only if it fits following: */
1979         if ((size == ((u64)high_memory - PAGE_OFFSET)) &&
1980             (iova_start == (void*)KERNELBASE)) {
1981                 ehca_gen_dbg("this is a max-MR");
1982                 return 1;
1983         } else
1984                 return 0;
1985 } /* end ehca_mr_is_maxmr() */
1986
1987 /*----------------------------------------------------------------------*/
1988
1989 /* map access control for MR/MW. This routine is used for MR and MW. */
1990 void ehca_mrmw_map_acl(int ib_acl,
1991                        u32 *hipz_acl)
1992 {
1993         *hipz_acl = 0;
1994         if (ib_acl & IB_ACCESS_REMOTE_READ)
1995                 *hipz_acl |= HIPZ_ACCESSCTRL_R_READ;
1996         if (ib_acl & IB_ACCESS_REMOTE_WRITE)
1997                 *hipz_acl |= HIPZ_ACCESSCTRL_R_WRITE;
1998         if (ib_acl & IB_ACCESS_REMOTE_ATOMIC)
1999                 *hipz_acl |= HIPZ_ACCESSCTRL_R_ATOMIC;
2000         if (ib_acl & IB_ACCESS_LOCAL_WRITE)
2001                 *hipz_acl |= HIPZ_ACCESSCTRL_L_WRITE;
2002         if (ib_acl & IB_ACCESS_MW_BIND)
2003                 *hipz_acl |= HIPZ_ACCESSCTRL_MW_BIND;
2004 } /* end ehca_mrmw_map_acl() */
2005
2006 /*----------------------------------------------------------------------*/
2007
2008 /* sets page size in hipz access control for MR/MW. */
2009 void ehca_mrmw_set_pgsize_hipz_acl(u32 *hipz_acl) /*INOUT*/
2010 {
2011         return; /* HCA supports only 4k */
2012 } /* end ehca_mrmw_set_pgsize_hipz_acl() */
2013
2014 /*----------------------------------------------------------------------*/
2015
2016 /*
2017  * reverse map access control for MR/MW.
2018  * This routine is used for MR and MW.
2019  */
2020 void ehca_mrmw_reverse_map_acl(const u32 *hipz_acl,
2021                                int *ib_acl) /*OUT*/
2022 {
2023         *ib_acl = 0;
2024         if (*hipz_acl & HIPZ_ACCESSCTRL_R_READ)
2025                 *ib_acl |= IB_ACCESS_REMOTE_READ;
2026         if (*hipz_acl & HIPZ_ACCESSCTRL_R_WRITE)
2027                 *ib_acl |= IB_ACCESS_REMOTE_WRITE;
2028         if (*hipz_acl & HIPZ_ACCESSCTRL_R_ATOMIC)
2029                 *ib_acl |= IB_ACCESS_REMOTE_ATOMIC;
2030         if (*hipz_acl & HIPZ_ACCESSCTRL_L_WRITE)
2031                 *ib_acl |= IB_ACCESS_LOCAL_WRITE;
2032         if (*hipz_acl & HIPZ_ACCESSCTRL_MW_BIND)
2033                 *ib_acl |= IB_ACCESS_MW_BIND;
2034 } /* end ehca_mrmw_reverse_map_acl() */
2035
2036
2037 /*----------------------------------------------------------------------*/
2038
2039 /*
2040  * map HIPZ rc to IB retcodes for MR/MW allocations
2041  * Used for hipz_mr_reg_alloc and hipz_mw_alloc.
2042  */
2043 int ehca_mrmw_map_hrc_alloc(const u64 hipz_rc)
2044 {
2045         switch (hipz_rc) {
2046         case H_SUCCESS:              /* successful completion */
2047                 return 0;
2048         case H_ADAPTER_PARM:         /* invalid adapter handle */
2049         case H_RT_PARM:              /* invalid resource type */
2050         case H_NOT_ENOUGH_RESOURCES: /* insufficient resources */
2051         case H_MLENGTH_PARM:         /* invalid memory length */
2052         case H_MEM_ACCESS_PARM:      /* invalid access controls */
2053         case H_CONSTRAINED:          /* resource constraint */
2054                 return -EINVAL;
2055         case H_BUSY:                 /* long busy */
2056                 return -EBUSY;
2057         default:
2058                 return -EINVAL;
2059         }
2060 } /* end ehca_mrmw_map_hrc_alloc() */
2061
2062 /*----------------------------------------------------------------------*/
2063
2064 /*
2065  * map HIPZ rc to IB retcodes for MR register rpage
2066  * Used for hipz_h_register_rpage_mr at registering last page
2067  */
2068 int ehca_mrmw_map_hrc_rrpg_last(const u64 hipz_rc)
2069 {
2070         switch (hipz_rc) {
2071         case H_SUCCESS:         /* registration complete */
2072                 return 0;
2073         case H_PAGE_REGISTERED: /* page registered */
2074         case H_ADAPTER_PARM:    /* invalid adapter handle */
2075         case H_RH_PARM:         /* invalid resource handle */
2076 /*      case H_QT_PARM:            invalid queue type */
2077         case H_PARAMETER:       /*
2078                                  * invalid logical address,
2079                                  * or count zero or greater 512
2080                                  */
2081         case H_TABLE_FULL:      /* page table full */
2082         case H_HARDWARE:        /* HCA not operational */
2083                 return -EINVAL;
2084         case H_BUSY:            /* long busy */
2085                 return -EBUSY;
2086         default:
2087                 return -EINVAL;
2088         }
2089 } /* end ehca_mrmw_map_hrc_rrpg_last() */
2090
2091 /*----------------------------------------------------------------------*/
2092
2093 /*
2094  * map HIPZ rc to IB retcodes for MR register rpage
2095  * Used for hipz_h_register_rpage_mr at registering one page, but not last page
2096  */
2097 int ehca_mrmw_map_hrc_rrpg_notlast(const u64 hipz_rc)
2098 {
2099         switch (hipz_rc) {
2100         case H_PAGE_REGISTERED: /* page registered */
2101                 return 0;
2102         case H_SUCCESS:         /* registration complete */
2103         case H_ADAPTER_PARM:    /* invalid adapter handle */
2104         case H_RH_PARM:         /* invalid resource handle */
2105 /*      case H_QT_PARM:            invalid queue type */
2106         case H_PARAMETER:       /*
2107                                  * invalid logical address,
2108                                  * or count zero or greater 512
2109                                  */
2110         case H_TABLE_FULL:      /* page table full */
2111         case H_HARDWARE:        /* HCA not operational */
2112                 return -EINVAL;
2113         case H_BUSY:            /* long busy */
2114                 return -EBUSY;
2115         default:
2116                 return -EINVAL;
2117         }
2118 } /* end ehca_mrmw_map_hrc_rrpg_notlast() */
2119
2120 /*----------------------------------------------------------------------*/
2121
2122 /* map HIPZ rc to IB retcodes for MR query. Used for hipz_mr_query. */
2123 int ehca_mrmw_map_hrc_query_mr(const u64 hipz_rc)
2124 {
2125         switch (hipz_rc) {
2126         case H_SUCCESS:              /* successful completion */
2127                 return 0;
2128         case H_ADAPTER_PARM:         /* invalid adapter handle */
2129         case H_RH_PARM:              /* invalid resource handle */
2130                 return -EINVAL;
2131         case H_BUSY:                 /* long busy */
2132                 return -EBUSY;
2133         default:
2134                 return -EINVAL;
2135         }
2136 } /* end ehca_mrmw_map_hrc_query_mr() */
2137
2138 /*----------------------------------------------------------------------*/
2139 /*----------------------------------------------------------------------*/
2140
2141 /*
2142  * map HIPZ rc to IB retcodes for freeing MR resource
2143  * Used for hipz_h_free_resource_mr
2144  */
2145 int ehca_mrmw_map_hrc_free_mr(const u64 hipz_rc)
2146 {
2147         switch (hipz_rc) {
2148         case H_SUCCESS:      /* resource freed */
2149                 return 0;
2150         case H_ADAPTER_PARM: /* invalid adapter handle */
2151         case H_RH_PARM:      /* invalid resource handle */
2152         case H_R_STATE:      /* invalid resource state */
2153         case H_HARDWARE:     /* HCA not operational */
2154                 return -EINVAL;
2155         case H_RESOURCE:     /* Resource in use */
2156         case H_BUSY:         /* long busy */
2157                 return -EBUSY;
2158         default:
2159                 return -EINVAL;
2160         }
2161 } /* end ehca_mrmw_map_hrc_free_mr() */
2162
2163 /*----------------------------------------------------------------------*/
2164
2165 /*
2166  * map HIPZ rc to IB retcodes for freeing MW resource
2167  * Used for hipz_h_free_resource_mw
2168  */
2169 int ehca_mrmw_map_hrc_free_mw(const u64 hipz_rc)
2170 {
2171         switch (hipz_rc) {
2172         case H_SUCCESS:      /* resource freed */
2173                 return 0;
2174         case H_ADAPTER_PARM: /* invalid adapter handle */
2175         case H_RH_PARM:      /* invalid resource handle */
2176         case H_R_STATE:      /* invalid resource state */
2177         case H_HARDWARE:     /* HCA not operational */
2178                 return -EINVAL;
2179         case H_RESOURCE:     /* Resource in use */
2180         case H_BUSY:         /* long busy */
2181                 return -EBUSY;
2182         default:
2183                 return -EINVAL;
2184         }
2185 } /* end ehca_mrmw_map_hrc_free_mw() */
2186
2187 /*----------------------------------------------------------------------*/
2188
2189 /*
2190  * map HIPZ rc to IB retcodes for SMR registrations
2191  * Used for hipz_h_register_smr.
2192  */
2193 int ehca_mrmw_map_hrc_reg_smr(const u64 hipz_rc)
2194 {
2195         switch (hipz_rc) {
2196         case H_SUCCESS:              /* successful completion */
2197                 return 0;
2198         case H_ADAPTER_PARM:         /* invalid adapter handle */
2199         case H_RH_PARM:              /* invalid resource handle */
2200         case H_MEM_PARM:             /* invalid MR virtual address */
2201         case H_MEM_ACCESS_PARM:      /* invalid access controls */
2202         case H_NOT_ENOUGH_RESOURCES: /* insufficient resources */
2203                 return -EINVAL;
2204         case H_BUSY:                 /* long busy */
2205                 return -EBUSY;
2206         default:
2207                 return -EINVAL;
2208         }
2209 } /* end ehca_mrmw_map_hrc_reg_smr() */
2210
2211 /*----------------------------------------------------------------------*/
2212
2213 /*
2214  * MR destructor and constructor
2215  * used in Reregister MR verb, sets all fields in ehca_mr_t to 0,
2216  * except struct ib_mr and spinlock
2217  */
2218 void ehca_mr_deletenew(struct ehca_mr *mr)
2219 {
2220         mr->flags         = 0;
2221         mr->num_pages     = 0;
2222         mr->num_4k        = 0;
2223         mr->acl           = 0;
2224         mr->start         = NULL;
2225         mr->fmr_page_size = 0;
2226         mr->fmr_max_pages = 0;
2227         mr->fmr_max_maps  = 0;
2228         mr->fmr_map_cnt   = 0;
2229         memset(&mr->ipz_mr_handle, 0, sizeof(mr->ipz_mr_handle));
2230         memset(&mr->galpas, 0, sizeof(mr->galpas));
2231         mr->nr_of_pages   = 0;
2232         mr->pagearray     = NULL;
2233 } /* end ehca_mr_deletenew() */
2234
2235 int ehca_init_mrmw_cache(void)
2236 {
2237         mr_cache = kmem_cache_create("ehca_cache_mr",
2238                                      sizeof(struct ehca_mr), 0,
2239                                      SLAB_HWCACHE_ALIGN,
2240                                      NULL, NULL);
2241         if (!mr_cache)
2242                 return -ENOMEM;
2243         mw_cache = kmem_cache_create("ehca_cache_mw",
2244                                      sizeof(struct ehca_mw), 0,
2245                                      SLAB_HWCACHE_ALIGN,
2246                                      NULL, NULL);
2247         if (!mw_cache) {
2248                 kmem_cache_destroy(mr_cache);
2249                 mr_cache = NULL;
2250                 return -ENOMEM;
2251         }
2252         return 0;
2253 }
2254
2255 void ehca_cleanup_mrmw_cache(void)
2256 {
2257         if (mr_cache)
2258                 kmem_cache_destroy(mr_cache);
2259         if (mw_cache)
2260                 kmem_cache_destroy(mw_cache);
2261 }