drivers/scsi/ufs/ufshcd.c

   1 /*
   2  * Universal Flash Storage Host controller driver
   3  *
   4  * This code is based on drivers/scsi/ufs/ufshcd.c
   5  * Copyright (C) 2011-2012 Samsung India Software Operations
   6  *
   7  * Santosh Yaraganavi <santosh.sy@samsung.com>
   8  * Vinayak Holikatti <h.vinayak@samsung.com>
   9  *
  10  * This program is free software; you can redistribute it and/or
  11  * modify it under the terms of the GNU General Public License
  12  * as published by the Free Software Foundation; either version 2
  13  * of the License, or (at your option) any later version.
  14  *
  15  * This program is distributed in the hope that it will be useful,
  16  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  17  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  18  * GNU General Public License for more details.
  19  *
  20  * NO WARRANTY
  21  * THE PROGRAM IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OR
  22  * CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED INCLUDING, WITHOUT
  23  * LIMITATION, ANY WARRANTIES OR CONDITIONS OF TITLE, NON-INFRINGEMENT,
  24  * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Each Recipient is
  25  * solely responsible for determining the appropriateness of using and
  26  * distributing the Program and assumes all risks associated with its
  27  * exercise of rights under this Agreement, including but not limited to
  28  * the risks and costs of program errors, damage to or loss of data,
  29  * programs or equipment, and unavailability or interruption of operations.
  30
  31  * DISCLAIMER OF LIABILITY
  32  * NEITHER RECIPIENT NOR ANY CONTRIBUTORS SHALL HAVE ANY LIABILITY FOR ANY
  33  * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  34  * DAMAGES (INCLUDING WITHOUT LIMITATION LOST PROFITS), HOWEVER CAUSED AND
  35  * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
  36  * TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
  37  * USE OR DISTRIBUTION OF THE PROGRAM OR THE EXERCISE OF ANY RIGHTS GRANTED
  38  * HEREUNDER, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGES
  39
  40  * You should have received a copy of the GNU General Public License
  41  * along with this program; if not, write to the Free Software
  42  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301,
  43  * USA.
  44  */
  45
  46 #include <linux/module.h>
  47 #include <linux/kernel.h>
  48 #include <linux/init.h>
  49 #include <linux/pci.h>
  50 #include <linux/interrupt.h>
  51 #include <linux/io.h>
  52 #include <linux/delay.h>
  53 #include <linux/slab.h>
  54 #include <linux/spinlock.h>
  55 #include <linux/workqueue.h>
  56 #include <linux/errno.h>
  57 #include <linux/types.h>
  58 #include <linux/wait.h>
  59 #include <linux/bitops.h>
  60
  61 #include <asm/irq.h>
  62 #include <asm/byteorder.h>
  63 #include <scsi/scsi.h>
  64 #include <scsi/scsi_cmnd.h>
  65 #include <scsi/scsi_host.h>
  66 #include <scsi/scsi_tcq.h>
  67 #include <scsi/scsi_dbg.h>
  68 #include <scsi/scsi_eh.h>
  69
  70 #include "ufs.h"
  71 #include "ufshci.h"
  72
  73 #define UFSHCD "ufshcd"
  74 #define UFSHCD_DRIVER_VERSION "0.1"
  75
  76 enum {
  77         UFSHCD_MAX_CHANNEL      = 0,
  78         UFSHCD_MAX_ID           = 1,
  79         UFSHCD_MAX_LUNS         = 8,
  80         UFSHCD_CMD_PER_LUN      = 32,
  81         UFSHCD_CAN_QUEUE        = 32,
  82 };
  83
  84 /* UFSHCD states */
  85 enum {
  86         UFSHCD_STATE_OPERATIONAL,
  87         UFSHCD_STATE_RESET,
  88         UFSHCD_STATE_ERROR,
  89 };
  90
  91 /* Interrupt configuration options */
  92 enum {
  93         UFSHCD_INT_DISABLE,
  94         UFSHCD_INT_ENABLE,
  95         UFSHCD_INT_CLEAR,
  96 };
  97
  98 /* Interrupt aggregation options */
  99 enum {
 100         INT_AGGR_RESET,
 101         INT_AGGR_CONFIG,
 102 };
 103
 104 /**
 105  * struct uic_command - UIC command structure
 106  * @command: UIC command
 107  * @argument1: UIC command argument 1
 108  * @argument2: UIC command argument 2
 109  * @argument3: UIC command argument 3
 110  * @cmd_active: Indicate if UIC command is outstanding
 111  * @result: UIC command result
 112  */
 113 struct uic_command {
 114         u32 command;
 115         u32 argument1;
 116         u32 argument2;
 117         u32 argument3;
 118         int cmd_active;
 119         int result;
 120 };
 121
 122 /**
 123  * struct ufs_hba - per adapter private structure
 124  * @mmio_base: UFSHCI base register address
 125  * @ucdl_base_addr: UFS Command Descriptor base address
 126  * @utrdl_base_addr: UTP Transfer Request Descriptor base address
 127  * @utmrdl_base_addr: UTP Task Management Descriptor base address
 128  * @ucdl_dma_addr: UFS Command Descriptor DMA address
 129  * @utrdl_dma_addr: UTRDL DMA address
 130  * @utmrdl_dma_addr: UTMRDL DMA address
 131  * @host: Scsi_Host instance of the driver
 132  * @pdev: PCI device handle
 133  * @lrb: local reference block
 134  * @outstanding_tasks: Bits representing outstanding task requests
 135  * @outstanding_reqs: Bits representing outstanding transfer requests
 136  * @capabilities: UFS Controller Capabilities
 137  * @nutrs: Transfer Request Queue depth supported by controller
 138  * @nutmrs: Task Management Queue depth supported by controller
 139  * @active_uic_cmd: handle of active UIC command
 140  * @ufshcd_tm_wait_queue: wait queue for task management
 141  * @tm_condition: condition variable for task management
 142  * @ufshcd_state: UFSHCD states
 143  * @int_enable_mask: Interrupt Mask Bits
 144  * @uic_workq: Work queue for UIC completion handling
 145  * @feh_workq: Work queue for fatal controller error handling
 146  * @errors: HBA errors
 147  */
 148 struct ufs_hba {
 149         void __iomem *mmio_base;
 150
 151         /* Virtual memory reference */
 152         struct utp_transfer_cmd_desc *ucdl_base_addr;
 153         struct utp_transfer_req_desc *utrdl_base_addr;
 154         struct utp_task_req_desc *utmrdl_base_addr;
 155
 156         /* DMA memory reference */
 157         dma_addr_t ucdl_dma_addr;
 158         dma_addr_t utrdl_dma_addr;
 159         dma_addr_t utmrdl_dma_addr;
 160
 161         struct Scsi_Host *host;
 162         struct pci_dev *pdev;
 163
 164         struct ufshcd_lrb *lrb;
 165
 166         unsigned long outstanding_tasks;
 167         unsigned long outstanding_reqs;
 168
 169         u32 capabilities;
 170         int nutrs;
 171         int nutmrs;
 172         u32 ufs_version;
 173
 174         struct uic_command active_uic_cmd;
 175         wait_queue_head_t ufshcd_tm_wait_queue;
 176         unsigned long tm_condition;
 177
 178         u32 ufshcd_state;
 179         u32 int_enable_mask;
 180
 181         /* Work Queues */
 182         struct work_struct uic_workq;
 183         struct work_struct feh_workq;
 184
 185         /* HBA Errors */
 186         u32 errors;
 187 };
 188
 189 /**
 190  * struct ufshcd_lrb - local reference block
 191  * @utr_descriptor_ptr: UTRD address of the command
 192  * @ucd_cmd_ptr: UCD address of the command
 193  * @ucd_rsp_ptr: Response UPIU address for this command
 194  * @ucd_prdt_ptr: PRDT address of the command
 195  * @cmd: pointer to SCSI command
 196  * @sense_buffer: pointer to sense buffer address of the SCSI command
 197  * @sense_bufflen: Length of the sense buffer
 198  * @scsi_status: SCSI status of the command
 199  * @command_type: SCSI, UFS, Query.
 200  * @task_tag: Task tag of the command
 201  * @lun: LUN of the command
 202  */
 203 struct ufshcd_lrb {
 204         struct utp_transfer_req_desc *utr_descriptor_ptr;
 205         struct utp_upiu_cmd *ucd_cmd_ptr;
 206         struct utp_upiu_rsp *ucd_rsp_ptr;
 207         struct ufshcd_sg_entry *ucd_prdt_ptr;
 208
 209         struct scsi_cmnd *cmd;
 210         u8 *sense_buffer;
 211         unsigned int sense_bufflen;
 212         int scsi_status;
 213
 214         int command_type;
 215         int task_tag;
 216         unsigned int lun;
 217 };
 218
 219 /**
 220  * ufshcd_get_ufs_version - Get the UFS version supported by the HBA
 221  * @hba - Pointer to adapter instance
 222  *
 223  * Returns UFSHCI version supported by the controller
 224  */
 225 static inline u32 ufshcd_get_ufs_version(struct ufs_hba *hba)
 226 {
 227         return readl(hba->mmio_base + REG_UFS_VERSION);
 228 }
 229
 230 /**
 231  * ufshcd_is_device_present - Check if any device connected to
 232  *                            the host controller
 233  * @reg_hcs - host controller status register value
 234  *
 235  * Returns 1 if device present, 0 if no device detected
 236  */
 237 static inline int ufshcd_is_device_present(u32 reg_hcs)
 238 {
 239         return (DEVICE_PRESENT & reg_hcs) ? 1 : 0;
 240 }
 241
 242 /**
 243  * ufshcd_get_tr_ocs - Get the UTRD Overall Command Status
 244  * @lrb: pointer to local command reference block
 245  *
 246  * This function is used to get the OCS field from UTRD
 247  * Returns the OCS field in the UTRD
 248  */
 249 static inline int ufshcd_get_tr_ocs(struct ufshcd_lrb *lrbp)
 250 {
 251         return lrbp->utr_descriptor_ptr->header.dword_2 & MASK_OCS;
 252 }
 253
 254 /**
 255  * ufshcd_get_tmr_ocs - Get the UTMRD Overall Command Status
 256  * @task_req_descp: pointer to utp_task_req_desc structure
 257  *
 258  * This function is used to get the OCS field from UTMRD
 259  * Returns the OCS field in the UTMRD
 260  */
 261 static inline int
 262 ufshcd_get_tmr_ocs(struct utp_task_req_desc *task_req_descp)
 263 {
 264         return task_req_descp->header.dword_2 & MASK_OCS;
 265 }
 266
 267 /**
 268  * ufshcd_get_tm_free_slot - get a free slot for task management request
 269  * @hba: per adapter instance
 270  *
 271  * Returns maximum number of task management request slots in case of
 272  * task management queue full or returns the free slot number
 273  */
 274 static inline int ufshcd_get_tm_free_slot(struct ufs_hba *hba)
 275 {
 276         return find_first_zero_bit(&hba->outstanding_tasks, hba->nutmrs);
 277 }
 278
 279 /**
 280  * ufshcd_utrl_clear - Clear a bit in UTRLCLR register
 281  * @hba: per adapter instance
 282  * @pos: position of the bit to be cleared
 283  */
 284 static inline void ufshcd_utrl_clear(struct ufs_hba *hba, u32 pos)
 285 {
 286         writel(~(1 << pos),
 287                 (hba->mmio_base + REG_UTP_TRANSFER_REQ_LIST_CLEAR));
 288 }
 289
 290 /**
 291  * ufshcd_get_lists_status - Check UCRDY, UTRLRDY and UTMRLRDY
 292  * @reg: Register value of host controller status
 293  *
 294  * Returns integer, 0 on Success and positive value if failed
 295  */
 296 static inline int ufshcd_get_lists_status(u32 reg)
 297 {
 298         /*
 299          * The mask 0xFF is for the following HCS register bits
 300          * Bit          Description
 301          *  0           Device Present
 302          *  1           UTRLRDY
 303          *  2           UTMRLRDY
 304          *  3           UCRDY
 305          *  4           HEI
 306          *  5           DEI
 307          * 6-7          reserved
 308          */
 309         return (((reg) & (0xFF)) >> 1) ^ (0x07);
 310 }
 311
 312 /**
 313  * ufshcd_get_uic_cmd_result - Get the UIC command result
 314  * @hba: Pointer to adapter instance
 315  *
 316  * This function gets the result of UIC command completion
 317  * Returns 0 on success, non zero value on error
 318  */
 319 static inline int ufshcd_get_uic_cmd_result(struct ufs_hba *hba)
 320 {
 321         return readl(hba->mmio_base + REG_UIC_COMMAND_ARG_2) &
 322                MASK_UIC_COMMAND_RESULT;
 323 }
 324
 325 /**
 326  * ufshcd_free_hba_memory - Free allocated memory for LRB, request
 327  *                          and task lists
 328  * @hba: Pointer to adapter instance
 329  */
 330 static inline void ufshcd_free_hba_memory(struct ufs_hba *hba)
 331 {
 332         size_t utmrdl_size, utrdl_size, ucdl_size;
 333
 334         kfree(hba->lrb);
 335
 336         if (hba->utmrdl_base_addr) {
 337                 utmrdl_size = sizeof(struct utp_task_req_desc) * hba->nutmrs;
 338                 dma_free_coherent(&hba->pdev->dev, utmrdl_size,
 339                                   hba->utmrdl_base_addr, hba->utmrdl_dma_addr);
 340         }
 341
 342         if (hba->utrdl_base_addr) {
 343                 utrdl_size =
 344                 (sizeof(struct utp_transfer_req_desc) * hba->nutrs);
 345                 dma_free_coherent(&hba->pdev->dev, utrdl_size,
 346                                   hba->utrdl_base_addr, hba->utrdl_dma_addr);
 347         }
 348
 349         if (hba->ucdl_base_addr) {
 350                 ucdl_size =
 351                 (sizeof(struct utp_transfer_cmd_desc) * hba->nutrs);
 352                 dma_free_coherent(&hba->pdev->dev, ucdl_size,
 353                                   hba->ucdl_base_addr, hba->ucdl_dma_addr);
 354         }
 355 }
 356
 357 /**
 358  * ufshcd_is_valid_req_rsp - checks if controller TR response is valid
 359  * @ucd_rsp_ptr: pointer to response UPIU
 360  *
 361  * This function checks the response UPIU for valid transaction type in
 362  * response field
 363  * Returns 0 on success, non-zero on failure
 364  */
 365 static inline int
 366 ufshcd_is_valid_req_rsp(struct utp_upiu_rsp *ucd_rsp_ptr)
 367 {
 368         return ((be32_to_cpu(ucd_rsp_ptr->header.dword_0) >> 24) ==
 369                  UPIU_TRANSACTION_RESPONSE) ? 0 : DID_ERROR << 16;
 370 }
 371
 372 /**
 373  * ufshcd_get_rsp_upiu_result - Get the result from response UPIU
 374  * @ucd_rsp_ptr: pointer to response UPIU
 375  *
 376  * This function gets the response status and scsi_status from response UPIU
 377  * Returns the response result code.
 378  */
 379 static inline int
 380 ufshcd_get_rsp_upiu_result(struct utp_upiu_rsp *ucd_rsp_ptr)
 381 {
 382         return be32_to_cpu(ucd_rsp_ptr->header.dword_1) & MASK_RSP_UPIU_RESULT;
 383 }
 384
 385 /**
 386  * ufshcd_config_int_aggr - Configure interrupt aggregation values.
 387  *              Currently there is no use case where we want to configure
 388  *              interrupt aggregation dynamically. So to configure interrupt
 389  *              aggregation, #define INT_AGGR_COUNTER_THRESHOLD_VALUE and
 390  *              INT_AGGR_TIMEOUT_VALUE are used.
 391  * @hba: per adapter instance
 392  * @option: Interrupt aggregation option
 393  */
 394 static inline void
 395 ufshcd_config_int_aggr(struct ufs_hba *hba, int option)
 396 {
 397         switch (option) {
 398         case INT_AGGR_RESET:
 399                 writel((INT_AGGR_ENABLE |
 400                         INT_AGGR_COUNTER_AND_TIMER_RESET),
 401                         (hba->mmio_base +
 402                          REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL));
 403                 break;
 404         case INT_AGGR_CONFIG:
 405                 writel((INT_AGGR_ENABLE |
 406                         INT_AGGR_PARAM_WRITE |
 407                         INT_AGGR_COUNTER_THRESHOLD_VALUE |
 408                         INT_AGGR_TIMEOUT_VALUE),
 409                         (hba->mmio_base +
 410                          REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL));
 411                 break;
 412         }
 413 }
 414
 415 /**
 416  * ufshcd_enable_run_stop_reg - Enable run-stop registers,
 417  *                      When run-stop registers are set to 1, it indicates the
 418  *                      host controller that it can process the requests
 419  * @hba: per adapter instance
 420  */
 421 static void ufshcd_enable_run_stop_reg(struct ufs_hba *hba)
 422 {
 423         writel(UTP_TASK_REQ_LIST_RUN_STOP_BIT,
 424                (hba->mmio_base +
 425                 REG_UTP_TASK_REQ_LIST_RUN_STOP));
 426         writel(UTP_TRANSFER_REQ_LIST_RUN_STOP_BIT,
 427                (hba->mmio_base +
 428                 REG_UTP_TRANSFER_REQ_LIST_RUN_STOP));
 429 }
 430
 431 /**
 432  * ufshcd_hba_stop - Send controller to reset state
 433  * @hba: per adapter instance
 434  */
 435 static inline void ufshcd_hba_stop(struct ufs_hba *hba)
 436 {
 437         writel(CONTROLLER_DISABLE, (hba->mmio_base + REG_CONTROLLER_ENABLE));
 438 }
 439
 440 /**
 441  * ufshcd_hba_start - Start controller initialization sequence
 442  * @hba: per adapter instance
 443  */
 444 static inline void ufshcd_hba_start(struct ufs_hba *hba)
 445 {
 446         writel(CONTROLLER_ENABLE , (hba->mmio_base + REG_CONTROLLER_ENABLE));
 447 }
 448
 449 /**
 450  * ufshcd_is_hba_active - Get controller state
 451  * @hba: per adapter instance
 452  *
 453  * Returns zero if controller is active, 1 otherwise
 454  */
 455 static inline int ufshcd_is_hba_active(struct ufs_hba *hba)
 456 {
 457         return (readl(hba->mmio_base + REG_CONTROLLER_ENABLE) & 0x1) ? 0 : 1;
 458 }
 459
 460 /**
 461  * ufshcd_send_command - Send SCSI or device management commands
 462  * @hba: per adapter instance
 463  * @task_tag: Task tag of the command
 464  */
 465 static inline
 466 void ufshcd_send_command(struct ufs_hba *hba, unsigned int task_tag)
 467 {
 468         __set_bit(task_tag, &hba->outstanding_reqs);
 469         writel((1 << task_tag),
 470                (hba->mmio_base + REG_UTP_TRANSFER_REQ_DOOR_BELL));
 471 }
 472
 473 /**
 474  * ufshcd_copy_sense_data - Copy sense data in case of check condition
 475  * @lrb - pointer to local reference block
 476  */
 477 static inline void ufshcd_copy_sense_data(struct ufshcd_lrb *lrbp)
 478 {
 479         int len;
 480         if (lrbp->sense_buffer) {
 481                 len = be16_to_cpu(lrbp->ucd_rsp_ptr->sense_data_len);
 482                 memcpy(lrbp->sense_buffer,
 483                         lrbp->ucd_rsp_ptr->sense_data,
 484                         min_t(int, len, SCSI_SENSE_BUFFERSIZE));
 485         }
 486 }
 487
 488 /**
 489  * ufshcd_hba_capabilities - Read controller capabilities
 490  * @hba: per adapter instance
 491  */
 492 static inline void ufshcd_hba_capabilities(struct ufs_hba *hba)
 493 {
 494         hba->capabilities =
 495                 readl(hba->mmio_base + REG_CONTROLLER_CAPABILITIES);
 496
 497         /* nutrs and nutmrs are 0 based values */
 498         hba->nutrs = (hba->capabilities & MASK_TRANSFER_REQUESTS_SLOTS) + 1;
 499         hba->nutmrs =
 500         ((hba->capabilities & MASK_TASK_MANAGEMENT_REQUEST_SLOTS) >> 16) + 1;
 501 }
 502
 503 /**
 504  * ufshcd_send_uic_command - Send UIC commands to unipro layers
 505  * @hba: per adapter instance
 506  * @uic_command: UIC command
 507  */
 508 static inline void
 509 ufshcd_send_uic_command(struct ufs_hba *hba, struct uic_command *uic_cmnd)
 510 {
 511         /* Write Args */
 512         writel(uic_cmnd->argument1,
 513               (hba->mmio_base + REG_UIC_COMMAND_ARG_1));
 514         writel(uic_cmnd->argument2,
 515               (hba->mmio_base + REG_UIC_COMMAND_ARG_2));
 516         writel(uic_cmnd->argument3,
 517               (hba->mmio_base + REG_UIC_COMMAND_ARG_3));
 518
 519         /* Write UIC Cmd */
 520         writel((uic_cmnd->command & COMMAND_OPCODE_MASK),
 521                (hba->mmio_base + REG_UIC_COMMAND));
 522 }
 523
 524 /**
 525  * ufshcd_map_sg - Map scatter-gather list to prdt
 526  * @lrbp - pointer to local reference block
 527  *
 528  * Returns 0 in case of success, non-zero value in case of failure
 529  */
 530 static int ufshcd_map_sg(struct ufshcd_lrb *lrbp)
 531 {
 532         struct ufshcd_sg_entry *prd_table;
 533         struct scatterlist *sg;
 534         struct scsi_cmnd *cmd;
 535         int sg_segments;
 536         int i;
 537
 538         cmd = lrbp->cmd;
 539         sg_segments = scsi_dma_map(cmd);
 540         if (sg_segments < 0)
 541                 return sg_segments;
 542
 543         if (sg_segments) {
 544                 lrbp->utr_descriptor_ptr->prd_table_length =
 545                                         cpu_to_le16((u16) (sg_segments));
 546
 547                 prd_table = (struct ufshcd_sg_entry *)lrbp->ucd_prdt_ptr;
 548
 549                 scsi_for_each_sg(cmd, sg, sg_segments, i) {
 550                         prd_table[i].size  =
 551                                 cpu_to_le32(((u32) sg_dma_len(sg))-1);
 552                         prd_table[i].base_addr =
 553                                 cpu_to_le32(lower_32_bits(sg->dma_address));
 554                         prd_table[i].upper_addr =
 555                                 cpu_to_le32(upper_32_bits(sg->dma_address));
 556                 }
 557         } else {
 558                 lrbp->utr_descriptor_ptr->prd_table_length = 0;
 559         }
 560
 561         return 0;
 562 }
 563
 564 /**
 565  * ufshcd_int_config - enable/disable interrupts
 566  * @hba: per adapter instance
 567  * @option: interrupt option
 568  */
 569 static void ufshcd_int_config(struct ufs_hba *hba, u32 option)
 570 {
 571         switch (option) {
 572         case UFSHCD_INT_ENABLE:
 573                 writel(hba->int_enable_mask,
 574                       (hba->mmio_base + REG_INTERRUPT_ENABLE));
 575                 break;
 576         case UFSHCD_INT_DISABLE:
 577                 if (hba->ufs_version == UFSHCI_VERSION_10)
 578                         writel(INTERRUPT_DISABLE_MASK_10,
 579                               (hba->mmio_base + REG_INTERRUPT_ENABLE));
 580                 else
 581                         writel(INTERRUPT_DISABLE_MASK_11,
 582                                (hba->mmio_base + REG_INTERRUPT_ENABLE));
 583                 break;
 584         }
 585 }
 586
 587 /**
 588  * ufshcd_compose_upiu - form UFS Protocol Information Unit(UPIU)
 589  * @lrb - pointer to local reference block
 590  */
 591 static void ufshcd_compose_upiu(struct ufshcd_lrb *lrbp)
 592 {
 593         struct utp_transfer_req_desc *req_desc;
 594         struct utp_upiu_cmd *ucd_cmd_ptr;
 595         u32 data_direction;
 596         u32 upiu_flags;
 597
 598         ucd_cmd_ptr = lrbp->ucd_cmd_ptr;
 599         req_desc = lrbp->utr_descriptor_ptr;
 600
 601         switch (lrbp->command_type) {
 602         case UTP_CMD_TYPE_SCSI:
 603                 if (lrbp->cmd->sc_data_direction == DMA_FROM_DEVICE) {
 604                         data_direction = UTP_DEVICE_TO_HOST;
 605                         upiu_flags = UPIU_CMD_FLAGS_READ;
 606                 } else if (lrbp->cmd->sc_data_direction == DMA_TO_DEVICE) {
 607                         data_direction = UTP_HOST_TO_DEVICE;
 608                         upiu_flags = UPIU_CMD_FLAGS_WRITE;
 609                 } else {
 610                         data_direction = UTP_NO_DATA_TRANSFER;
 611                         upiu_flags = UPIU_CMD_FLAGS_NONE;
 612                 }
 613
 614                 /* Transfer request descriptor header fields */
 615                 req_desc->header.dword_0 =
 616                         cpu_to_le32(data_direction | UTP_SCSI_COMMAND);
 617
 618                 /*
 619                  * assigning invalid value for command status. Controller
 620                  * updates OCS on command completion, with the command
 621                  * status
 622                  */
 623                 req_desc->header.dword_2 =
 624                         cpu_to_le32(OCS_INVALID_COMMAND_STATUS);
 625
 626                 /* command descriptor fields */
 627                 ucd_cmd_ptr->header.dword_0 =
 628                         cpu_to_be32(UPIU_HEADER_DWORD(UPIU_TRANSACTION_COMMAND,
 629                                                       upiu_flags,
 630                                                       lrbp->lun,
 631                                                       lrbp->task_tag));
 632                 ucd_cmd_ptr->header.dword_1 =
 633                         cpu_to_be32(
 634                                 UPIU_HEADER_DWORD(UPIU_COMMAND_SET_TYPE_SCSI,
 635                                                   0,
 636                                                   0,
 637                                                   0));
 638
 639                 /* Total EHS length and Data segment length will be zero */
 640                 ucd_cmd_ptr->header.dword_2 = 0;
 641
 642                 ucd_cmd_ptr->exp_data_transfer_len =
 643                         cpu_to_be32(lrbp->cmd->transfersize);
 644
 645                 memcpy(ucd_cmd_ptr->cdb,
 646                        lrbp->cmd->cmnd,
 647                        (min_t(unsigned short,
 648                               lrbp->cmd->cmd_len,
 649                               MAX_CDB_SIZE)));
 650                 break;
 651         case UTP_CMD_TYPE_DEV_MANAGE:
 652                 /* For query function implementation */
 653                 break;
 654         case UTP_CMD_TYPE_UFS:
 655                 /* For UFS native command implementation */
 656                 break;
 657         } /* end of switch */
 658 }
 659
 660 /**
 661  * ufshcd_queuecommand - main entry point for SCSI requests
 662  * @cmd: command from SCSI Midlayer
 663  * @done: call back function
 664  *
 665  * Returns 0 for success, non-zero in case of failure
 666  */
 667 static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd)
 668 {
 669         struct ufshcd_lrb *lrbp;
 670         struct ufs_hba *hba;
 671         unsigned long flags;
 672         int tag;
 673         int err = 0;
 674
 675         hba = shost_priv(host);
 676
 677         tag = cmd->request->tag;
 678
 679         if (hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL) {
 680                 err = SCSI_MLQUEUE_HOST_BUSY;
 681                 goto out;
 682         }
 683
 684         lrbp = &hba->lrb[tag];
 685
 686         lrbp->cmd = cmd;
 687         lrbp->sense_bufflen = SCSI_SENSE_BUFFERSIZE;
 688         lrbp->sense_buffer = cmd->sense_buffer;
 689         lrbp->task_tag = tag;
 690         lrbp->lun = cmd->device->lun;
 691
 692         lrbp->command_type = UTP_CMD_TYPE_SCSI;
 693
 694         /* form UPIU before issuing the command */
 695         ufshcd_compose_upiu(lrbp);
 696         err = ufshcd_map_sg(lrbp);
 697         if (err)
 698                 goto out;
 699
 700         /* issue command to the controller */
 701         spin_lock_irqsave(hba->host->host_lock, flags);
 702         ufshcd_send_command(hba, tag);
 703         spin_unlock_irqrestore(hba->host->host_lock, flags);
 704 out:
 705         return err;
 706 }
 707
 708 /**
 709  * ufshcd_memory_alloc - allocate memory for host memory space data structures
 710  * @hba: per adapter instance
 711  *
 712  * 1. Allocate DMA memory for Command Descriptor array
 713  *      Each command descriptor consist of Command UPIU, Response UPIU and PRDT
 714  * 2. Allocate DMA memory for UTP Transfer Request Descriptor List (UTRDL).
 715  * 3. Allocate DMA memory for UTP Task Management Request Descriptor List
 716  *      (UTMRDL)
 717  * 4. Allocate memory for local reference block(lrb).
 718  *
 719  * Returns 0 for success, non-zero in case of failure
 720  */
 721 static int ufshcd_memory_alloc(struct ufs_hba *hba)
 722 {
 723         size_t utmrdl_size, utrdl_size, ucdl_size;
 724
 725         /* Allocate memory for UTP command descriptors */
 726         ucdl_size = (sizeof(struct utp_transfer_cmd_desc) * hba->nutrs);
 727         hba->ucdl_base_addr = dma_alloc_coherent(&hba->pdev->dev,
 728                                                  ucdl_size,
 729                                                  &hba->ucdl_dma_addr,
 730                                                  GFP_KERNEL);
 731
 732         /*
 733          * UFSHCI requires UTP command descriptor to be 128 byte aligned.
 734          * make sure hba->ucdl_dma_addr is aligned to PAGE_SIZE
 735          * if hba->ucdl_dma_addr is aligned to PAGE_SIZE, then it will
 736          * be aligned to 128 bytes as well
 737          */
 738         if (!hba->ucdl_base_addr ||
 739             WARN_ON(hba->ucdl_dma_addr & (PAGE_SIZE - 1))) {
 740                 dev_err(&hba->pdev->dev,
 741                         "Command Descriptor Memory allocation failed\n");
 742                 goto out;
 743         }
 744
 745         /*
 746          * Allocate memory for UTP Transfer descriptors
 747          * UFSHCI requires 1024 byte alignment of UTRD
 748          */
 749         utrdl_size = (sizeof(struct utp_transfer_req_desc) * hba->nutrs);
 750         hba->utrdl_base_addr = dma_alloc_coherent(&hba->pdev->dev,
 751                                                   utrdl_size,
 752                                                   &hba->utrdl_dma_addr,
 753                                                   GFP_KERNEL);
 754         if (!hba->utrdl_base_addr ||
 755             WARN_ON(hba->utrdl_dma_addr & (PAGE_SIZE - 1))) {
 756                 dev_err(&hba->pdev->dev,
 757                         "Transfer Descriptor Memory allocation failed\n");
 758                 goto out;
 759         }
 760
 761         /*
 762          * Allocate memory for UTP Task Management descriptors
 763          * UFSHCI requires 1024 byte alignment of UTMRD
 764          */
 765         utmrdl_size = sizeof(struct utp_task_req_desc) * hba->nutmrs;
 766         hba->utmrdl_base_addr = dma_alloc_coherent(&hba->pdev->dev,
 767                                                    utmrdl_size,
 768                                                    &hba->utmrdl_dma_addr,
 769                                                    GFP_KERNEL);
 770         if (!hba->utmrdl_base_addr ||
 771             WARN_ON(hba->utmrdl_dma_addr & (PAGE_SIZE - 1))) {
 772                 dev_err(&hba->pdev->dev,
 773                 "Task Management Descriptor Memory allocation failed\n");
 774                 goto out;
 775         }
 776
 777         /* Allocate memory for local reference block */
 778         hba->lrb = kcalloc(hba->nutrs, sizeof(struct ufshcd_lrb), GFP_KERNEL);
 779         if (!hba->lrb) {
 780                 dev_err(&hba->pdev->dev, "LRB Memory allocation failed\n");
 781                 goto out;
 782         }
 783         return 0;
 784 out:
 785         ufshcd_free_hba_memory(hba);
 786         return -ENOMEM;
 787 }
 788
 789 /**
 790  * ufshcd_host_memory_configure - configure local reference block with
 791  *                              memory offsets
 792  * @hba: per adapter instance
 793  *
 794  * Configure Host memory space
 795  * 1. Update Corresponding UTRD.UCDBA and UTRD.UCDBAU with UCD DMA
 796  * address.
 797  * 2. Update each UTRD with Response UPIU offset, Response UPIU length
 798  * and PRDT offset.
 799  * 3. Save the corresponding addresses of UTRD, UCD.CMD, UCD.RSP and UCD.PRDT
 800  * into local reference block.
 801  */
 802 static void ufshcd_host_memory_configure(struct ufs_hba *hba)
 803 {
 804         struct utp_transfer_cmd_desc *cmd_descp;
 805         struct utp_transfer_req_desc *utrdlp;
 806         dma_addr_t cmd_desc_dma_addr;
 807         dma_addr_t cmd_desc_element_addr;
 808         u16 response_offset;
 809         u16 prdt_offset;
 810         int cmd_desc_size;
 811         int i;
 812
 813         utrdlp = hba->utrdl_base_addr;
 814         cmd_descp = hba->ucdl_base_addr;
 815
 816         response_offset =
 817                 offsetof(struct utp_transfer_cmd_desc, response_upiu);
 818         prdt_offset =
 819                 offsetof(struct utp_transfer_cmd_desc, prd_table);
 820
 821         cmd_desc_size = sizeof(struct utp_transfer_cmd_desc);
 822         cmd_desc_dma_addr = hba->ucdl_dma_addr;
 823
 824         for (i = 0; i < hba->nutrs; i++) {
 825                 /* Configure UTRD with command descriptor base address */
 826                 cmd_desc_element_addr =
 827                                 (cmd_desc_dma_addr + (cmd_desc_size * i));
 828                 utrdlp[i].command_desc_base_addr_lo =
 829                                 cpu_to_le32(lower_32_bits(cmd_desc_element_addr));
 830                 utrdlp[i].command_desc_base_addr_hi =
 831                                 cpu_to_le32(upper_32_bits(cmd_desc_element_addr));
 832
 833                 /* Response upiu and prdt offset should be in double words */
 834                 utrdlp[i].response_upiu_offset =
 835                                 cpu_to_le16((response_offset >> 2));
 836                 utrdlp[i].prd_table_offset =
 837                                 cpu_to_le16((prdt_offset >> 2));
 838                 utrdlp[i].response_upiu_length =
 839                                 cpu_to_le16(ALIGNED_UPIU_SIZE);
 840
 841                 hba->lrb[i].utr_descriptor_ptr = (utrdlp + i);
 842                 hba->lrb[i].ucd_cmd_ptr =
 843                         (struct utp_upiu_cmd *)(cmd_descp + i);
 844                 hba->lrb[i].ucd_rsp_ptr =
 845                         (struct utp_upiu_rsp *)cmd_descp[i].response_upiu;
 846                 hba->lrb[i].ucd_prdt_ptr =
 847                         (struct ufshcd_sg_entry *)cmd_descp[i].prd_table;
 848         }
 849 }
 850
 851 /**
 852  * ufshcd_dme_link_startup - Notify Unipro to perform link startup
 853  * @hba: per adapter instance
 854  *
 855  * UIC_CMD_DME_LINK_STARTUP command must be issued to Unipro layer,
 856  * in order to initialize the Unipro link startup procedure.
 857  * Once the Unipro links are up, the device connected to the controller
 858  * is detected.
 859  *
 860  * Returns 0 on success, non-zero value on failure
 861  */
 862 static int ufshcd_dme_link_startup(struct ufs_hba *hba)
 863 {
 864         struct uic_command *uic_cmd;
 865         unsigned long flags;
 866
 867         /* check if controller is ready to accept UIC commands */
 868         if (((readl(hba->mmio_base + REG_CONTROLLER_STATUS)) &
 869             UIC_COMMAND_READY) == 0x0) {
 870                 dev_err(&hba->pdev->dev,
 871                         "Controller not ready"
 872                         " to accept UIC commands\n");
 873                 return -EIO;
 874         }
 875
 876         spin_lock_irqsave(hba->host->host_lock, flags);
 877
 878         /* form UIC command */
 879         uic_cmd = &hba->active_uic_cmd;
 880         uic_cmd->command = UIC_CMD_DME_LINK_STARTUP;
 881         uic_cmd->argument1 = 0;
 882         uic_cmd->argument2 = 0;
 883         uic_cmd->argument3 = 0;
 884
 885         /* enable UIC related interrupts */
 886         hba->int_enable_mask |= UIC_COMMAND_COMPL;
 887         ufshcd_int_config(hba, UFSHCD_INT_ENABLE);
 888
 889         /* sending UIC commands to controller */
 890         ufshcd_send_uic_command(hba, uic_cmd);
 891         spin_unlock_irqrestore(hba->host->host_lock, flags);
 892         return 0;
 893 }
 894
 895 /**
 896  * ufshcd_make_hba_operational - Make UFS controller operational
 897  * @hba: per adapter instance
 898  *
 899  * To bring UFS host controller to operational state,
 900  * 1. Check if device is present
 901  * 2. Configure run-stop-registers
 902  * 3. Enable required interrupts
 903  * 4. Configure interrupt aggregation
 904  *
 905  * Returns 0 on success, non-zero value on failure
 906  */
 907 static int ufshcd_make_hba_operational(struct ufs_hba *hba)
 908 {
 909         int err = 0;
 910         u32 reg;
 911
 912         /* check if device present */
 913         reg = readl((hba->mmio_base + REG_CONTROLLER_STATUS));
 914         if (!ufshcd_is_device_present(reg)) {
 915                 dev_err(&hba->pdev->dev, "cc: Device not present\n");
 916                 err = -ENXIO;
 917                 goto out;
 918         }
 919
 920         /*
 921          * UCRDY, UTMRLDY and UTRLRDY bits must be 1
 922          * DEI, HEI bits must be 0
 923          */
 924         if (!(ufshcd_get_lists_status(reg))) {
 925                 ufshcd_enable_run_stop_reg(hba);
 926         } else {
 927                 dev_err(&hba->pdev->dev,
 928                         "Host controller not ready to process requests");
 929                 err = -EIO;
 930                 goto out;
 931         }
 932
 933         /* Enable required interrupts */
 934         hba->int_enable_mask |= (UTP_TRANSFER_REQ_COMPL |
 935                                  UIC_ERROR |
 936                                  UTP_TASK_REQ_COMPL |
 937                                  DEVICE_FATAL_ERROR |
 938                                  CONTROLLER_FATAL_ERROR |
 939                                  SYSTEM_BUS_FATAL_ERROR);
 940         ufshcd_int_config(hba, UFSHCD_INT_ENABLE);
 941
 942         /* Configure interrupt aggregation */
 943         ufshcd_config_int_aggr(hba, INT_AGGR_CONFIG);
 944
 945         if (hba->ufshcd_state == UFSHCD_STATE_RESET)
 946                 scsi_unblock_requests(hba->host);
 947
 948         hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL;
 949         scsi_scan_host(hba->host);
 950 out:
 951         return err;
 952 }
 953
 954 /**
 955  * ufshcd_hba_enable - initialize the controller
 956  * @hba: per adapter instance
 957  *
 958  * The controller resets itself and controller firmware initialization
 959  * sequence kicks off. When controller is ready it will set
 960  * the Host Controller Enable bit to 1.
 961  *
 962  * Returns 0 on success, non-zero value on failure
 963  */
 964 static int ufshcd_hba_enable(struct ufs_hba *hba)
 965 {
 966         int retry;
 967
 968         /*
 969          * msleep of 1 and 5 used in this function might result in msleep(20),
 970          * but it was necessary to send the UFS FPGA to reset mode during
 971          * development and testing of this driver. msleep can be changed to
 972          * mdelay and retry count can be reduced based on the controller.
 973          */
 974         if (!ufshcd_is_hba_active(hba)) {
 975
 976                 /* change controller state to "reset state" */
 977                 ufshcd_hba_stop(hba);
 978
 979                 /*
 980                  * This delay is based on the testing done with UFS host
 981                  * controller FPGA. The delay can be changed based on the
 982                  * host controller used.
 983                  */
 984                 msleep(5);
 985         }
 986
 987         /* start controller initialization sequence */
 988         ufshcd_hba_start(hba);
 989
 990         /*
 991          * To initialize a UFS host controller HCE bit must be set to 1.
 992          * During initialization the HCE bit value changes from 1->0->1.
 993          * When the host controller completes initialization sequence
 994          * it sets the value of HCE bit to 1. The same HCE bit is read back
 995          * to check if the controller has completed initialization sequence.
 996          * So without this delay the value HCE = 1, set in the previous
 997          * instruction might be read back.
 998          * This delay can be changed based on the controller.
 999          */
1000         msleep(1);
1001
1002         /* wait for the host controller to complete initialization */
1003         retry = 10;
1004         while (ufshcd_is_hba_active(hba)) {
1005                 if (retry) {
1006                         retry--;
1007                 } else {
1008                         dev_err(&hba->pdev->dev,
1009                                 "Controller enable failed\n");
1010                         return -EIO;
1011                 }
1012                 msleep(5);
1013         }
1014         return 0;
1015 }
1016
1017 /**
1018  * ufshcd_initialize_hba - start the initialization process
1019  * @hba: per adapter instance
1020  *
1021  * 1. Enable the controller via ufshcd_hba_enable.
1022  * 2. Program the Transfer Request List Address with the starting address of
1023  * UTRDL.
1024  * 3. Program the Task Management Request List Address with starting address
1025  * of UTMRDL.
1026  *
1027  * Returns 0 on success, non-zero value on failure.
1028  */
1029 static int ufshcd_initialize_hba(struct ufs_hba *hba)
1030 {
1031         if (ufshcd_hba_enable(hba))
1032                 return -EIO;
1033
1034         /* Configure UTRL and UTMRL base address registers */
1035         writel(lower_32_bits(hba->utrdl_dma_addr),
1036                (hba->mmio_base + REG_UTP_TRANSFER_REQ_LIST_BASE_L));
1037         writel(upper_32_bits(hba->utrdl_dma_addr),
1038                (hba->mmio_base + REG_UTP_TRANSFER_REQ_LIST_BASE_H));
1039         writel(lower_32_bits(hba->utmrdl_dma_addr),
1040                (hba->mmio_base + REG_UTP_TASK_REQ_LIST_BASE_L));
1041         writel(upper_32_bits(hba->utmrdl_dma_addr),
1042                (hba->mmio_base + REG_UTP_TASK_REQ_LIST_BASE_H));
1043
1044         /* Initialize unipro link startup procedure */
1045         return ufshcd_dme_link_startup(hba);
1046 }
1047
1048 /**
1049  * ufshcd_do_reset - reset the host controller
1050  * @hba: per adapter instance
1051  *
1052  * Returns SUCCESS/FAILED
1053  */
1054 static int ufshcd_do_reset(struct ufs_hba *hba)
1055 {
1056         struct ufshcd_lrb *lrbp;
1057         unsigned long flags;
1058         int tag;
1059
1060         /* block commands from midlayer */
1061         scsi_block_requests(hba->host);
1062
1063         spin_lock_irqsave(hba->host->host_lock, flags);
1064         hba->ufshcd_state = UFSHCD_STATE_RESET;
1065
1066         /* send controller to reset state */
1067         ufshcd_hba_stop(hba);
1068         spin_unlock_irqrestore(hba->host->host_lock, flags);
1069
1070         /* abort outstanding commands */
1071         for (tag = 0; tag < hba->nutrs; tag++) {
1072                 if (test_bit(tag, &hba->outstanding_reqs)) {
1073                         lrbp = &hba->lrb[tag];
1074                         scsi_dma_unmap(lrbp->cmd);
1075                         lrbp->cmd->result = DID_RESET << 16;
1076                         lrbp->cmd->scsi_done(lrbp->cmd);
1077                         lrbp->cmd = NULL;
1078                 }
1079         }
1080
1081         /* clear outstanding request/task bit maps */
1082         hba->outstanding_reqs = 0;
1083         hba->outstanding_tasks = 0;
1084
1085         /* start the initialization process */
1086         if (ufshcd_initialize_hba(hba)) {
1087                 dev_err(&hba->pdev->dev,
1088                         "Reset: Controller initialization failed\n");
1089                 return FAILED;
1090         }
1091         return SUCCESS;
1092 }
1093
1094 /**
1095  * ufshcd_slave_alloc - handle initial SCSI device configurations
1096  * @sdev: pointer to SCSI device
1097  *
1098  * Returns success
1099  */
1100 static int ufshcd_slave_alloc(struct scsi_device *sdev)
1101 {
1102         struct ufs_hba *hba;
1103
1104         hba = shost_priv(sdev->host);
1105         sdev->tagged_supported = 1;
1106
1107         /* Mode sense(6) is not supported by UFS, so use Mode sense(10) */
1108         sdev->use_10_for_ms = 1;
1109         scsi_set_tag_type(sdev, MSG_SIMPLE_TAG);
1110
1111         /*
1112          * Inform SCSI Midlayer that the LUN queue depth is same as the
1113          * controller queue depth. If a LUN queue depth is less than the
1114          * controller queue depth and if the LUN reports
1115          * SAM_STAT_TASK_SET_FULL, the LUN queue depth will be adjusted
1116          * with scsi_adjust_queue_depth.
1117          */
1118         scsi_activate_tcq(sdev, hba->nutrs);
1119         return 0;
1120 }
1121
1122 /**
1123  * ufshcd_slave_destroy - remove SCSI device configurations
1124  * @sdev: pointer to SCSI device
1125  */
1126 static void ufshcd_slave_destroy(struct scsi_device *sdev)
1127 {
1128         struct ufs_hba *hba;
1129
1130         hba = shost_priv(sdev->host);
1131         scsi_deactivate_tcq(sdev, hba->nutrs);
1132 }
1133
1134 /**
1135  * ufshcd_task_req_compl - handle task management request completion
1136  * @hba: per adapter instance
1137  * @index: index of the completed request
1138  *
1139  * Returns SUCCESS/FAILED
1140  */
1141 static int ufshcd_task_req_compl(struct ufs_hba *hba, u32 index)
1142 {
1143         struct utp_task_req_desc *task_req_descp;
1144         struct utp_upiu_task_rsp *task_rsp_upiup;
1145         unsigned long flags;
1146         int ocs_value;
1147         int task_result;
1148
1149         spin_lock_irqsave(hba->host->host_lock, flags);
1150
1151         /* Clear completed tasks from outstanding_tasks */
1152         __clear_bit(index, &hba->outstanding_tasks);
1153
1154         task_req_descp = hba->utmrdl_base_addr;
1155         ocs_value = ufshcd_get_tmr_ocs(&task_req_descp[index]);
1156
1157         if (ocs_value == OCS_SUCCESS) {
1158                 task_rsp_upiup = (struct utp_upiu_task_rsp *)
1159                                 task_req_descp[index].task_rsp_upiu;
1160                 task_result = be32_to_cpu(task_rsp_upiup->header.dword_1);
1161                 task_result = ((task_result & MASK_TASK_RESPONSE) >> 8);
1162
1163                 if (task_result != UPIU_TASK_MANAGEMENT_FUNC_COMPL &&
1164                     task_result != UPIU_TASK_MANAGEMENT_FUNC_SUCCEEDED)
1165                         task_result = FAILED;
1166                 else
1167                         task_result = SUCCESS;
1168         } else {
1169                 task_result = FAILED;
1170                 dev_err(&hba->pdev->dev,
1171                         "trc: Invalid ocs = %x\n", ocs_value);
1172         }
1173         spin_unlock_irqrestore(hba->host->host_lock, flags);
1174         return task_result;
1175 }
1176
1177 /**
1178  * ufshcd_adjust_lun_qdepth - Update LUN queue depth if device responds with
1179  *                            SAM_STAT_TASK_SET_FULL SCSI command status.
1180  * @cmd: pointer to SCSI command
1181  */
1182 static void ufshcd_adjust_lun_qdepth(struct scsi_cmnd *cmd)
1183 {
1184         struct ufs_hba *hba;
1185         int i;
1186         int lun_qdepth = 0;
1187
1188         hba = shost_priv(cmd->device->host);
1189
1190         /*
1191          * LUN queue depth can be obtained by counting outstanding commands
1192          * on the LUN.
1193          */
1194         for (i = 0; i < hba->nutrs; i++) {
1195                 if (test_bit(i, &hba->outstanding_reqs)) {
1196
1197                         /*
1198                          * Check if the outstanding command belongs
1199                          * to the LUN which reported SAM_STAT_TASK_SET_FULL.
1200                          */
1201                         if (cmd->device->lun == hba->lrb[i].lun)
1202                                 lun_qdepth++;
1203                 }
1204         }
1205
1206         /*
1207          * LUN queue depth will be total outstanding commands, except the
1208          * command for which the LUN reported SAM_STAT_TASK_SET_FULL.
1209          */
1210         scsi_adjust_queue_depth(cmd->device, MSG_SIMPLE_TAG, lun_qdepth - 1);
1211 }
1212
1213 /**
1214  * ufshcd_scsi_cmd_status - Update SCSI command result based on SCSI status
1215  * @lrb: pointer to local reference block of completed command
1216  * @scsi_status: SCSI command status
1217  *
1218  * Returns value base on SCSI command status
1219  */
1220 static inline int
1221 ufshcd_scsi_cmd_status(struct ufshcd_lrb *lrbp, int scsi_status)
1222 {
1223         int result = 0;
1224
1225         switch (scsi_status) {
1226         case SAM_STAT_GOOD:
1227                 result |= DID_OK << 16 |
1228                           COMMAND_COMPLETE << 8 |
1229                           SAM_STAT_GOOD;
1230                 break;
1231         case SAM_STAT_CHECK_CONDITION:
1232                 result |= DID_OK << 16 |
1233                           COMMAND_COMPLETE << 8 |
1234                           SAM_STAT_CHECK_CONDITION;
1235                 ufshcd_copy_sense_data(lrbp);
1236                 break;
1237         case SAM_STAT_BUSY:
1238                 result |= SAM_STAT_BUSY;
1239                 break;
1240         case SAM_STAT_TASK_SET_FULL:
1241
1242                 /*
1243                  * If a LUN reports SAM_STAT_TASK_SET_FULL, then the LUN queue
1244                  * depth needs to be adjusted to the exact number of
1245                  * outstanding commands the LUN can handle at any given time.
1246                  */
1247                 ufshcd_adjust_lun_qdepth(lrbp->cmd);
1248                 result |= SAM_STAT_TASK_SET_FULL;
1249                 break;
1250         case SAM_STAT_TASK_ABORTED:
1251                 result |= SAM_STAT_TASK_ABORTED;
1252                 break;
1253         default:
1254                 result |= DID_ERROR << 16;
1255                 break;
1256         } /* end of switch */
1257
1258         return result;
1259 }
1260
1261 /**
1262  * ufshcd_transfer_rsp_status - Get overall status of the response
1263  * @hba: per adapter instance
1264  * @lrb: pointer to local reference block of completed command
1265  *
1266  * Returns result of the command to notify SCSI midlayer
1267  */
1268 static inline int
1269 ufshcd_transfer_rsp_status(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
1270 {
1271         int result = 0;
1272         int scsi_status;
1273         int ocs;
1274
1275         /* overall command status of utrd */
1276         ocs = ufshcd_get_tr_ocs(lrbp);
1277
1278         switch (ocs) {
1279         case OCS_SUCCESS:
1280
1281                 /* check if the returned transfer response is valid */
1282                 result = ufshcd_is_valid_req_rsp(lrbp->ucd_rsp_ptr);
1283                 if (result) {
1284                         dev_err(&hba->pdev->dev,
1285                                 "Invalid response = %x\n", result);
1286                         break;
1287                 }
1288
1289                 /*
1290                  * get the response UPIU result to extract
1291                  * the SCSI command status
1292                  */
1293                 result = ufshcd_get_rsp_upiu_result(lrbp->ucd_rsp_ptr);
1294
1295                 /*
1296                  * get the result based on SCSI status response
1297                  * to notify the SCSI midlayer of the command status
1298                  */
1299                 scsi_status = result & MASK_SCSI_STATUS;
1300                 result = ufshcd_scsi_cmd_status(lrbp, scsi_status);
1301                 break;
1302         case OCS_ABORTED:
1303                 result |= DID_ABORT << 16;
1304                 break;
1305         case OCS_INVALID_CMD_TABLE_ATTR:
1306         case OCS_INVALID_PRDT_ATTR:
1307         case OCS_MISMATCH_DATA_BUF_SIZE:
1308         case OCS_MISMATCH_RESP_UPIU_SIZE:
1309         case OCS_PEER_COMM_FAILURE:
1310         case OCS_FATAL_ERROR:
1311         default:
1312                 result |= DID_ERROR << 16;
1313                 dev_err(&hba->pdev->dev,
1314                 "OCS error from controller = %x\n", ocs);
1315                 break;
1316         } /* end of switch */
1317
1318         return result;
1319 }
1320
1321 /**
1322  * ufshcd_transfer_req_compl - handle SCSI and query command completion
1323  * @hba: per adapter instance
1324  */
1325 static void ufshcd_transfer_req_compl(struct ufs_hba *hba)
1326 {
1327         struct ufshcd_lrb *lrb;
1328         unsigned long completed_reqs;
1329         u32 tr_doorbell;
1330         int result;
1331         int index;
1332
1333         lrb = hba->lrb;
1334         tr_doorbell =
1335                 readl(hba->mmio_base + REG_UTP_TRANSFER_REQ_DOOR_BELL);
1336         completed_reqs = tr_doorbell ^ hba->outstanding_reqs;
1337
1338         for (index = 0; index < hba->nutrs; index++) {
1339                 if (test_bit(index, &completed_reqs)) {
1340
1341                         result = ufshcd_transfer_rsp_status(hba, &lrb[index]);
1342
1343                         if (lrb[index].cmd) {
1344                                 scsi_dma_unmap(lrb[index].cmd);
1345                                 lrb[index].cmd->result = result;
1346                                 lrb[index].cmd->scsi_done(lrb[index].cmd);
1347
1348                                 /* Mark completed command as NULL in LRB */
1349                                 lrb[index].cmd = NULL;
1350                         }
1351                 } /* end of if */
1352         } /* end of for */
1353
1354         /* clear corresponding bits of completed commands */
1355         hba->outstanding_reqs ^= completed_reqs;
1356
1357         /* Reset interrupt aggregation counters */
1358         ufshcd_config_int_aggr(hba, INT_AGGR_RESET);
1359 }
1360
1361 /**
1362  * ufshcd_uic_cc_handler - handle UIC command completion
1363  * @work: pointer to a work queue structure
1364  *
1365  * Returns 0 on success, non-zero value on failure
1366  */
1367 static void ufshcd_uic_cc_handler (struct work_struct *work)
1368 {
1369         struct ufs_hba *hba;
1370
1371         hba = container_of(work, struct ufs_hba, uic_workq);
1372
1373         if ((hba->active_uic_cmd.command == UIC_CMD_DME_LINK_STARTUP) &&
1374             !(ufshcd_get_uic_cmd_result(hba))) {
1375
1376                 if (ufshcd_make_hba_operational(hba))
1377                         dev_err(&hba->pdev->dev,
1378                                 "cc: hba not operational state\n");
1379                 return;
1380         }
1381 }
1382
1383 /**
1384  * ufshcd_fatal_err_handler - handle fatal errors
1385  * @hba: per adapter instance
1386  */
1387 static void ufshcd_fatal_err_handler(struct work_struct *work)
1388 {
1389         struct ufs_hba *hba;
1390         hba = container_of(work, struct ufs_hba, feh_workq);
1391
1392         /* check if reset is already in progress */
1393         if (hba->ufshcd_state != UFSHCD_STATE_RESET)
1394                 ufshcd_do_reset(hba);
1395 }
1396
1397 /**
1398  * ufshcd_err_handler - Check for fatal errors
1399  * @work: pointer to a work queue structure
1400  */
1401 static void ufshcd_err_handler(struct ufs_hba *hba)
1402 {
1403         u32 reg;
1404
1405         if (hba->errors & INT_FATAL_ERRORS)
1406                 goto fatal_eh;
1407
1408         if (hba->errors & UIC_ERROR) {
1409
1410                 reg = readl(hba->mmio_base +
1411                             REG_UIC_ERROR_CODE_PHY_ADAPTER_LAYER);
1412                 if (reg & UIC_DATA_LINK_LAYER_ERROR_PA_INIT)
1413                         goto fatal_eh;
1414         }
1415         return;
1416 fatal_eh:
1417         hba->ufshcd_state = UFSHCD_STATE_ERROR;
1418         schedule_work(&hba->feh_workq);
1419 }
1420
1421 /**
1422  * ufshcd_tmc_handler - handle task management function completion
1423  * @hba: per adapter instance
1424  */
1425 static void ufshcd_tmc_handler(struct ufs_hba *hba)
1426 {
1427         u32 tm_doorbell;
1428
1429         tm_doorbell = readl(hba->mmio_base + REG_UTP_TASK_REQ_DOOR_BELL);
1430         hba->tm_condition = tm_doorbell ^ hba->outstanding_tasks;
1431         wake_up_interruptible(&hba->ufshcd_tm_wait_queue);
1432 }
1433
1434 /**
1435  * ufshcd_sl_intr - Interrupt service routine
1436  * @hba: per adapter instance
1437  * @intr_status: contains interrupts generated by the controller
1438  */
1439 static void ufshcd_sl_intr(struct ufs_hba *hba, u32 intr_status)
1440 {
1441         hba->errors = UFSHCD_ERROR_MASK & intr_status;
1442         if (hba->errors)
1443                 ufshcd_err_handler(hba);
1444
1445         if (intr_status & UIC_COMMAND_COMPL)
1446                 schedule_work(&hba->uic_workq);
1447
1448         if (intr_status & UTP_TASK_REQ_COMPL)
1449                 ufshcd_tmc_handler(hba);
1450
1451         if (intr_status & UTP_TRANSFER_REQ_COMPL)
1452                 ufshcd_transfer_req_compl(hba);
1453 }
1454
1455 /**
1456  * ufshcd_intr - Main interrupt service routine
1457  * @irq: irq number
1458  * @__hba: pointer to adapter instance
1459  *
1460  * Returns IRQ_HANDLED - If interrupt is valid
1461  *              IRQ_NONE - If invalid interrupt
1462  */
1463 static irqreturn_t ufshcd_intr(int irq, void *__hba)
1464 {
1465         u32 intr_status;
1466         irqreturn_t retval = IRQ_NONE;
1467         struct ufs_hba *hba = __hba;
1468
1469         spin_lock(hba->host->host_lock);
1470         intr_status = readl(hba->mmio_base + REG_INTERRUPT_STATUS);
1471
1472         if (intr_status) {
1473                 ufshcd_sl_intr(hba, intr_status);
1474
1475                 /* If UFSHCI 1.0 then clear interrupt status register */
1476                 if (hba->ufs_version == UFSHCI_VERSION_10)
1477                         writel(intr_status,
1478                                (hba->mmio_base + REG_INTERRUPT_STATUS));
1479                 retval = IRQ_HANDLED;
1480         }
1481         spin_unlock(hba->host->host_lock);
1482         return retval;
1483 }
1484
1485 /**
1486  * ufshcd_issue_tm_cmd - issues task management commands to controller
1487  * @hba: per adapter instance
1488  * @lrbp: pointer to local reference block
1489  *
1490  * Returns SUCCESS/FAILED
1491  */
1492 static int
1493 ufshcd_issue_tm_cmd(struct ufs_hba *hba,
1494                     struct ufshcd_lrb *lrbp,
1495                     u8 tm_function)
1496 {
1497         struct utp_task_req_desc *task_req_descp;
1498         struct utp_upiu_task_req *task_req_upiup;
1499         struct Scsi_Host *host;
1500         unsigned long flags;
1501         int free_slot = 0;
1502         int err;
1503
1504         host = hba->host;
1505
1506         spin_lock_irqsave(host->host_lock, flags);
1507
1508         /* If task management queue is full */
1509         free_slot = ufshcd_get_tm_free_slot(hba);
1510         if (free_slot >= hba->nutmrs) {
1511                 spin_unlock_irqrestore(host->host_lock, flags);
1512                 dev_err(&hba->pdev->dev, "Task management queue full\n");
1513                 err = FAILED;
1514                 goto out;
1515         }
1516
1517         task_req_descp = hba->utmrdl_base_addr;
1518         task_req_descp += free_slot;
1519
1520         /* Configure task request descriptor */
1521         task_req_descp->header.dword_0 = cpu_to_le32(UTP_REQ_DESC_INT_CMD);
1522         task_req_descp->header.dword_2 =
1523                         cpu_to_le32(OCS_INVALID_COMMAND_STATUS);
1524
1525         /* Configure task request UPIU */
1526         task_req_upiup =
1527                 (struct utp_upiu_task_req *) task_req_descp->task_req_upiu;
1528         task_req_upiup->header.dword_0 =
1529                 cpu_to_be32(UPIU_HEADER_DWORD(UPIU_TRANSACTION_TASK_REQ, 0,
1530                                               lrbp->lun, lrbp->task_tag));
1531         task_req_upiup->header.dword_1 =
1532         cpu_to_be32(UPIU_HEADER_DWORD(0, tm_function, 0, 0));
1533
1534         task_req_upiup->input_param1 = lrbp->lun;
1535         task_req_upiup->input_param1 =
1536                 cpu_to_be32(task_req_upiup->input_param1);
1537         task_req_upiup->input_param2 = lrbp->task_tag;
1538         task_req_upiup->input_param2 =
1539                 cpu_to_be32(task_req_upiup->input_param2);
1540
1541         /* send command to the controller */
1542         __set_bit(free_slot, &hba->outstanding_tasks);
1543         writel((1 << free_slot),
1544                (hba->mmio_base + REG_UTP_TASK_REQ_DOOR_BELL));
1545
1546         spin_unlock_irqrestore(host->host_lock, flags);
1547
1548         /* wait until the task management command is completed */
1549         err =
1550         wait_event_interruptible_timeout(hba->ufshcd_tm_wait_queue,
1551                                          (test_bit(free_slot,
1552                                          &hba->tm_condition) != 0),
1553                                          60 * HZ);
1554         if (!err) {
1555                 dev_err(&hba->pdev->dev,
1556                         "Task management command timed-out\n");
1557                 err = FAILED;
1558                 goto out;
1559         }
1560         clear_bit(free_slot, &hba->tm_condition);
1561         err = ufshcd_task_req_compl(hba, free_slot);
1562 out:
1563         return err;
1564 }
1565
1566 /**
1567  * ufshcd_device_reset - reset device and abort all the pending commands
1568  * @cmd: SCSI command pointer
1569  *
1570  * Returns SUCCESS/FAILED
1571  */
1572 static int ufshcd_device_reset(struct scsi_cmnd *cmd)
1573 {
1574         struct Scsi_Host *host;
1575         struct ufs_hba *hba;
1576         unsigned int tag;
1577         u32 pos;
1578         int err;
1579
1580         host = cmd->device->host;
1581         hba = shost_priv(host);
1582         tag = cmd->request->tag;
1583
1584         err = ufshcd_issue_tm_cmd(hba, &hba->lrb[tag], UFS_LOGICAL_RESET);
1585         if (err == FAILED)
1586                 goto out;
1587
1588         for (pos = 0; pos < hba->nutrs; pos++) {
1589                 if (test_bit(pos, &hba->outstanding_reqs) &&
1590                     (hba->lrb[tag].lun == hba->lrb[pos].lun)) {
1591
1592                         /* clear the respective UTRLCLR register bit */
1593                         ufshcd_utrl_clear(hba, pos);
1594
1595                         clear_bit(pos, &hba->outstanding_reqs);
1596
1597                         if (hba->lrb[pos].cmd) {
1598                                 scsi_dma_unmap(hba->lrb[pos].cmd);
1599                                 hba->lrb[pos].cmd->result =
1600                                                 DID_ABORT << 16;
1601                                 hba->lrb[pos].cmd->scsi_done(cmd);
1602                                 hba->lrb[pos].cmd = NULL;
1603                         }
1604                 }
1605         } /* end of for */
1606 out:
1607         return err;
1608 }
1609
1610 /**
1611  * ufshcd_host_reset - Main reset function registered with scsi layer
1612  * @cmd: SCSI command pointer
1613  *
1614  * Returns SUCCESS/FAILED
1615  */
1616 static int ufshcd_host_reset(struct scsi_cmnd *cmd)
1617 {
1618         struct ufs_hba *hba;
1619
1620         hba = shost_priv(cmd->device->host);
1621
1622         if (hba->ufshcd_state == UFSHCD_STATE_RESET)
1623                 return SUCCESS;
1624
1625         return ufshcd_do_reset(hba);
1626 }
1627
1628 /**
1629  * ufshcd_abort - abort a specific command
1630  * @cmd: SCSI command pointer
1631  *
1632  * Returns SUCCESS/FAILED
1633  */
1634 static int ufshcd_abort(struct scsi_cmnd *cmd)
1635 {
1636         struct Scsi_Host *host;
1637         struct ufs_hba *hba;
1638         unsigned long flags;
1639         unsigned int tag;
1640         int err;
1641
1642         host = cmd->device->host;
1643         hba = shost_priv(host);
1644         tag = cmd->request->tag;
1645
1646         spin_lock_irqsave(host->host_lock, flags);
1647
1648         /* check if command is still pending */
1649         if (!(test_bit(tag, &hba->outstanding_reqs))) {
1650                 err = FAILED;
1651                 spin_unlock_irqrestore(host->host_lock, flags);
1652                 goto out;
1653         }
1654         spin_unlock_irqrestore(host->host_lock, flags);
1655
1656         err = ufshcd_issue_tm_cmd(hba, &hba->lrb[tag], UFS_ABORT_TASK);
1657         if (err == FAILED)
1658                 goto out;
1659
1660         scsi_dma_unmap(cmd);
1661
1662         spin_lock_irqsave(host->host_lock, flags);
1663
1664         /* clear the respective UTRLCLR register bit */
1665         ufshcd_utrl_clear(hba, tag);
1666
1667         __clear_bit(tag, &hba->outstanding_reqs);
1668         hba->lrb[tag].cmd = NULL;
1669         spin_unlock_irqrestore(host->host_lock, flags);
1670 out:
1671         return err;
1672 }
1673
1674 static struct scsi_host_template ufshcd_driver_template = {
1675         .module                 = THIS_MODULE,
1676         .name                   = UFSHCD,
1677         .proc_name              = UFSHCD,
1678         .queuecommand           = ufshcd_queuecommand,
1679         .slave_alloc            = ufshcd_slave_alloc,
1680         .slave_destroy          = ufshcd_slave_destroy,
1681         .eh_abort_handler       = ufshcd_abort,
1682         .eh_device_reset_handler = ufshcd_device_reset,
1683         .eh_host_reset_handler  = ufshcd_host_reset,
1684         .this_id                = -1,
1685         .sg_tablesize           = SG_ALL,
1686         .cmd_per_lun            = UFSHCD_CMD_PER_LUN,
1687         .can_queue              = UFSHCD_CAN_QUEUE,
1688 };
1689
1690 /**
1691  * ufshcd_shutdown - main function to put the controller in reset state
1692  * @pdev: pointer to PCI device handle
1693  */
1694 static void ufshcd_shutdown(struct pci_dev *pdev)
1695 {
1696         ufshcd_hba_stop((struct ufs_hba *)pci_get_drvdata(pdev));
1697 }
1698
1699 #ifdef CONFIG_PM
1700 /**
1701  * ufshcd_suspend - suspend power management function
1702  * @pdev: pointer to PCI device handle
1703  * @state: power state
1704  *
1705  * Returns -ENOSYS
1706  */
1707 static int ufshcd_suspend(struct pci_dev *pdev, pm_message_t state)
1708 {
1709         /*
1710          * TODO:
1711          * 1. Block SCSI requests from SCSI midlayer
1712          * 2. Change the internal driver state to non operational
1713          * 3. Set UTRLRSR and UTMRLRSR bits to zero
1714          * 4. Wait until outstanding commands are completed
1715          * 5. Set HCE to zero to send the UFS host controller to reset state
1716          */
1717
1718         return -ENOSYS;
1719 }
1720
1721 /**
1722  * ufshcd_resume - resume power management function
1723  * @pdev: pointer to PCI device handle
1724  *
1725  * Returns -ENOSYS
1726  */
1727 static int ufshcd_resume(struct pci_dev *pdev)
1728 {
1729         /*
1730          * TODO:
1731          * 1. Set HCE to 1, to start the UFS host controller
1732          * initialization process
1733          * 2. Set UTRLRSR and UTMRLRSR bits to 1
1734          * 3. Change the internal driver state to operational
1735          * 4. Unblock SCSI requests from SCSI midlayer
1736          */
1737
1738         return -ENOSYS;
1739 }
1740 #endif /* CONFIG_PM */
1741
1742 /**
1743  * ufshcd_hba_free - free allocated memory for
1744  *                      host memory space data structures
1745  * @hba: per adapter instance
1746  */
1747 static void ufshcd_hba_free(struct ufs_hba *hba)
1748 {
1749         iounmap(hba->mmio_base);
1750         ufshcd_free_hba_memory(hba);
1751         pci_release_regions(hba->pdev);
1752 }
1753
1754 /**
1755  * ufshcd_remove - de-allocate PCI/SCSI host and host memory space
1756  *              data structure memory
1757  * @pdev - pointer to PCI handle
1758  */
1759 static void ufshcd_remove(struct pci_dev *pdev)
1760 {
1761         struct ufs_hba *hba = pci_get_drvdata(pdev);
1762
1763         /* disable interrupts */
1764         ufshcd_int_config(hba, UFSHCD_INT_DISABLE);
1765         free_irq(pdev->irq, hba);
1766
1767         ufshcd_hba_stop(hba);
1768         ufshcd_hba_free(hba);
1769
1770         scsi_remove_host(hba->host);
1771         scsi_host_put(hba->host);
1772         pci_set_drvdata(pdev, NULL);
1773         pci_clear_master(pdev);
1774         pci_disable_device(pdev);
1775 }
1776
1777 /**
1778  * ufshcd_set_dma_mask - Set dma mask based on the controller
1779  *                       addressing capability
1780  * @pdev: PCI device structure
1781  *
1782  * Returns 0 for success, non-zero for failure
1783  */
1784 static int ufshcd_set_dma_mask(struct ufs_hba *hba)
1785 {
1786         int err;
1787         u64 dma_mask;
1788
1789         /*
1790          * If controller supports 64 bit addressing mode, then set the DMA
1791          * mask to 64-bit, else set the DMA mask to 32-bit
1792          */
1793         if (hba->capabilities & MASK_64_ADDRESSING_SUPPORT)
1794                 dma_mask = DMA_BIT_MASK(64);
1795         else
1796                 dma_mask = DMA_BIT_MASK(32);
1797
1798         err = pci_set_dma_mask(hba->pdev, dma_mask);
1799         if (err)
1800                 return err;
1801
1802         err = pci_set_consistent_dma_mask(hba->pdev, dma_mask);
1803
1804         return err;
1805 }
1806
1807 /**
1808  * ufshcd_probe - probe routine of the driver
1809  * @pdev: pointer to PCI device handle
1810  * @id: PCI device id
1811  *
1812  * Returns 0 on success, non-zero value on failure
1813  */
1814 static int ufshcd_probe(struct pci_dev *pdev, const struct pci_device_id *id)
1815 {
1816         struct Scsi_Host *host;
1817         struct ufs_hba *hba;
1818         int err;
1819
1820         err = pci_enable_device(pdev);
1821         if (err) {
1822                 dev_err(&pdev->dev, "pci_enable_device failed\n");
1823                 goto out_error;
1824         }
1825
1826         pci_set_master(pdev);
1827
1828         host = scsi_host_alloc(&ufshcd_driver_template,
1829                                 sizeof(struct ufs_hba));
1830         if (!host) {
1831                 dev_err(&pdev->dev, "scsi_host_alloc failed\n");
1832                 err = -ENOMEM;
1833                 goto out_disable;
1834         }
1835         hba = shost_priv(host);
1836
1837         err = pci_request_regions(pdev, UFSHCD);
1838         if (err < 0) {
1839                 dev_err(&pdev->dev, "request regions failed\n");
1840                 goto out_host_put;
1841         }
1842
1843         hba->mmio_base = pci_ioremap_bar(pdev, 0);
1844         if (!hba->mmio_base) {
1845                 dev_err(&pdev->dev, "memory map failed\n");
1846                 err = -ENOMEM;
1847                 goto out_release_regions;
1848         }
1849
1850         hba->host = host;
1851         hba->pdev = pdev;
1852
1853         /* Read capabilities registers */
1854         ufshcd_hba_capabilities(hba);
1855
1856         /* Get UFS version supported by the controller */
1857         hba->ufs_version = ufshcd_get_ufs_version(hba);
1858
1859         err = ufshcd_set_dma_mask(hba);
1860         if (err) {
1861                 dev_err(&pdev->dev, "set dma mask failed\n");
1862                 goto out_iounmap;
1863         }
1864
1865         /* Allocate memory for host memory space */
1866         err = ufshcd_memory_alloc(hba);
1867         if (err) {
1868                 dev_err(&pdev->dev, "Memory allocation failed\n");
1869                 goto out_iounmap;
1870         }
1871
1872         /* Configure LRB */
1873         ufshcd_host_memory_configure(hba);
1874
1875         host->can_queue = hba->nutrs;
1876         host->cmd_per_lun = hba->nutrs;
1877         host->max_id = UFSHCD_MAX_ID;
1878         host->max_lun = UFSHCD_MAX_LUNS;
1879         host->max_channel = UFSHCD_MAX_CHANNEL;
1880         host->unique_id = host->host_no;
1881         host->max_cmd_len = MAX_CDB_SIZE;
1882
1883         /* Initailize wait queue for task management */
1884         init_waitqueue_head(&hba->ufshcd_tm_wait_queue);
1885
1886         /* Initialize work queues */
1887         INIT_WORK(&hba->uic_workq, ufshcd_uic_cc_handler);
1888         INIT_WORK(&hba->feh_workq, ufshcd_fatal_err_handler);
1889
1890         /* IRQ registration */
1891         err = request_irq(pdev->irq, ufshcd_intr, IRQF_SHARED, UFSHCD, hba);
1892         if (err) {
1893                 dev_err(&pdev->dev, "request irq failed\n");
1894                 goto out_lrb_free;
1895         }
1896
1897         /* Enable SCSI tag mapping */
1898         err = scsi_init_shared_tag_map(host, host->can_queue);
1899         if (err) {
1900                 dev_err(&pdev->dev, "init shared queue failed\n");
1901                 goto out_free_irq;
1902         }
1903
1904         pci_set_drvdata(pdev, hba);
1905
1906         err = scsi_add_host(host, &pdev->dev);
1907         if (err) {
1908                 dev_err(&pdev->dev, "scsi_add_host failed\n");
1909                 goto out_free_irq;
1910         }
1911
1912         /* Initialization routine */
1913         err = ufshcd_initialize_hba(hba);
1914         if (err) {
1915                 dev_err(&pdev->dev, "Initialization failed\n");
1916                 goto out_free_irq;
1917         }
1918
1919         return 0;
1920
1921 out_free_irq:
1922         free_irq(pdev->irq, hba);
1923 out_lrb_free:
1924         ufshcd_free_hba_memory(hba);
1925 out_iounmap:
1926         iounmap(hba->mmio_base);
1927 out_release_regions:
1928         pci_release_regions(pdev);
1929 out_host_put:
1930         scsi_host_put(host);
1931 out_disable:
1932         pci_clear_master(pdev);
1933         pci_disable_device(pdev);
1934 out_error:
1935         return err;
1936 }
1937
1938 static DEFINE_PCI_DEVICE_TABLE(ufshcd_pci_tbl) = {
1939         { PCI_VENDOR_ID_SAMSUNG, 0xC00C, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 },
1940         { }     /* terminate list */
1941 };
1942
1943 MODULE_DEVICE_TABLE(pci, ufshcd_pci_tbl);
1944
1945 static struct pci_driver ufshcd_pci_driver = {
1946         .name = UFSHCD,
1947         .id_table = ufshcd_pci_tbl,
1948         .probe = ufshcd_probe,
1949         .remove = ufshcd_remove,
1950         .shutdown = ufshcd_shutdown,
1951 #ifdef CONFIG_PM
1952         .suspend = ufshcd_suspend,
1953         .resume = ufshcd_resume,
1954 #endif
1955 };
1956
1957 module_pci_driver(ufshcd_pci_driver);
1958
1959 MODULE_AUTHOR("Santosh Yaragnavi <santosh.sy@samsung.com>, "
1960               "Vinayak Holikatti <h.vinayak@samsung.com>");
1961 MODULE_DESCRIPTION("Generic UFS host controller driver");
1962 MODULE_LICENSE("GPL");
1963 MODULE_VERSION(UFSHCD_DRIVER_VERSION);