arch/mips/cavium-octeon/octeon-irq.c

   1 /*
   2  * This file is subject to the terms and conditions of the GNU General Public
   3  * License.  See the file "COPYING" in the main directory of this archive
   4  * for more details.
   5  *
   6  * Copyright (C) 2004-2008, 2009, 2010, 2011 Cavium Networks
   7  */
   8
   9 #include <linux/interrupt.h>
  10 #include <linux/bitops.h>
  11 #include <linux/percpu.h>
  12 #include <linux/irq.h>
  13 #include <linux/smp.h>
  14
  15 #include <asm/octeon/octeon.h>
  16
  17 static DEFINE_RAW_SPINLOCK(octeon_irq_ciu0_lock);
  18 static DEFINE_RAW_SPINLOCK(octeon_irq_ciu1_lock);
  19
  20 static DEFINE_PER_CPU(unsigned long, octeon_irq_ciu0_en_mirror);
  21 static DEFINE_PER_CPU(unsigned long, octeon_irq_ciu1_en_mirror);
  22
  23 static __read_mostly u8 octeon_irq_ciu_to_irq[8][64];
  24
  25 union octeon_ciu_chip_data {
  26         void *p;
  27         unsigned long l;
  28         struct {
  29                 unsigned int line:6;
  30                 unsigned int bit:6;
  31         } s;
  32 };
  33
  34 struct octeon_core_chip_data {
  35         struct mutex core_irq_mutex;
  36         bool current_en;
  37         bool desired_en;
  38         u8 bit;
  39 };
  40
  41 #define MIPS_CORE_IRQ_LINES 8
  42
  43 static struct octeon_core_chip_data octeon_irq_core_chip_data[MIPS_CORE_IRQ_LINES];
  44
  45 static void __init octeon_irq_set_ciu_mapping(int irq, int line, int bit,
  46                                               struct irq_chip *chip,
  47                                               irq_flow_handler_t handler)
  48 {
  49         union octeon_ciu_chip_data cd;
  50
  51         irq_set_chip_and_handler(irq, chip, handler);
  52
  53         cd.l = 0;
  54         cd.s.line = line;
  55         cd.s.bit = bit;
  56
  57         irq_set_chip_data(irq, cd.p);
  58         octeon_irq_ciu_to_irq[line][bit] = irq;
  59 }
  60
  61 static int octeon_coreid_for_cpu(int cpu)
  62 {
  63 #ifdef CONFIG_SMP
  64         return cpu_logical_map(cpu);
  65 #else
  66         return cvmx_get_core_num();
  67 #endif
  68 }
  69
  70 static int octeon_cpu_for_coreid(int coreid)
  71 {
  72 #ifdef CONFIG_SMP
  73         return cpu_number_map(coreid);
  74 #else
  75         return smp_processor_id();
  76 #endif
  77 }
  78
  79 static void octeon_irq_core_ack(struct irq_data *data)
  80 {
  81         struct octeon_core_chip_data *cd = irq_data_get_irq_chip_data(data);
  82         unsigned int bit = cd->bit;
  83
  84         /*
  85          * We don't need to disable IRQs to make these atomic since
  86          * they are already disabled earlier in the low level
  87          * interrupt code.
  88          */
  89         clear_c0_status(0x100 << bit);
  90         /* The two user interrupts must be cleared manually. */
  91         if (bit < 2)
  92                 clear_c0_cause(0x100 << bit);
  93 }
  94
  95 static void octeon_irq_core_eoi(struct irq_data *data)
  96 {
  97         struct octeon_core_chip_data *cd = irq_data_get_irq_chip_data(data);
  98
  99         /*
 100          * We don't need to disable IRQs to make these atomic since
 101          * they are already disabled earlier in the low level
 102          * interrupt code.
 103          */
 104         set_c0_status(0x100 << cd->bit);
 105 }
 106
 107 static void octeon_irq_core_set_enable_local(void *arg)
 108 {
 109         struct irq_data *data = arg;
 110         struct octeon_core_chip_data *cd = irq_data_get_irq_chip_data(data);
 111         unsigned int mask = 0x100 << cd->bit;
 112
 113         /*
 114          * Interrupts are already disabled, so these are atomic.
 115          */
 116         if (cd->desired_en)
 117                 set_c0_status(mask);
 118         else
 119                 clear_c0_status(mask);
 120
 121 }
 122
 123 static void octeon_irq_core_disable(struct irq_data *data)
 124 {
 125         struct octeon_core_chip_data *cd = irq_data_get_irq_chip_data(data);
 126         cd->desired_en = false;
 127 }
 128
 129 static void octeon_irq_core_enable(struct irq_data *data)
 130 {
 131         struct octeon_core_chip_data *cd = irq_data_get_irq_chip_data(data);
 132         cd->desired_en = true;
 133 }
 134
 135 static void octeon_irq_core_bus_lock(struct irq_data *data)
 136 {
 137         struct octeon_core_chip_data *cd = irq_data_get_irq_chip_data(data);
 138
 139         mutex_lock(&cd->core_irq_mutex);
 140 }
 141
 142 static void octeon_irq_core_bus_sync_unlock(struct irq_data *data)
 143 {
 144         struct octeon_core_chip_data *cd = irq_data_get_irq_chip_data(data);
 145
 146         if (cd->desired_en != cd->current_en) {
 147                 on_each_cpu(octeon_irq_core_set_enable_local, data, 1);
 148
 149                 cd->current_en = cd->desired_en;
 150         }
 151
 152         mutex_unlock(&cd->core_irq_mutex);
 153 }
 154
 155 static struct irq_chip octeon_irq_chip_core = {
 156         .name = "Core",
 157         .irq_enable = octeon_irq_core_enable,
 158         .irq_disable = octeon_irq_core_disable,
 159         .irq_ack = octeon_irq_core_ack,
 160         .irq_eoi = octeon_irq_core_eoi,
 161         .irq_bus_lock = octeon_irq_core_bus_lock,
 162         .irq_bus_sync_unlock = octeon_irq_core_bus_sync_unlock,
 163
 164         .irq_cpu_online = octeon_irq_core_eoi,
 165         .irq_cpu_offline = octeon_irq_core_ack,
 166         .flags = IRQCHIP_ONOFFLINE_ENABLED,
 167 };
 168
 169 static void __init octeon_irq_init_core(void)
 170 {
 171         int i;
 172         int irq;
 173         struct octeon_core_chip_data *cd;
 174
 175         for (i = 0; i < MIPS_CORE_IRQ_LINES; i++) {
 176                 cd = &octeon_irq_core_chip_data[i];
 177                 cd->current_en = false;
 178                 cd->desired_en = false;
 179                 cd->bit = i;
 180                 mutex_init(&cd->core_irq_mutex);
 181
 182                 irq = OCTEON_IRQ_SW0 + i;
 183                 switch (irq) {
 184                 case OCTEON_IRQ_TIMER:
 185                 case OCTEON_IRQ_SW0:
 186                 case OCTEON_IRQ_SW1:
 187                 case OCTEON_IRQ_5:
 188                 case OCTEON_IRQ_PERF:
 189                         irq_set_chip_data(irq, cd);
 190                         irq_set_chip_and_handler(irq, &octeon_irq_chip_core,
 191                                                  handle_percpu_irq);
 192                         break;
 193                 default:
 194                         break;
 195                 }
 196         }
 197 }
 198
 199 static int next_cpu_for_irq(struct irq_data *data)
 200 {
 201
 202 #ifdef CONFIG_SMP
 203         int cpu;
 204         int weight = cpumask_weight(data->affinity);
 205
 206         if (weight > 1) {
 207                 cpu = smp_processor_id();
 208                 for (;;) {
 209                         cpu = cpumask_next(cpu, data->affinity);
 210                         if (cpu >= nr_cpu_ids) {
 211                                 cpu = -1;
 212                                 continue;
 213                         } else if (cpumask_test_cpu(cpu, cpu_online_mask)) {
 214                                 break;
 215                         }
 216                 }
 217         } else if (weight == 1) {
 218                 cpu = cpumask_first(data->affinity);
 219         } else {
 220                 cpu = smp_processor_id();
 221         }
 222         return cpu;
 223 #else
 224         return smp_processor_id();
 225 #endif
 226 }
 227
 228 static void octeon_irq_ciu_enable(struct irq_data *data)
 229 {
 230         int cpu = next_cpu_for_irq(data);
 231         int coreid = octeon_coreid_for_cpu(cpu);
 232         unsigned long *pen;
 233         unsigned long flags;
 234         union octeon_ciu_chip_data cd;
 235
 236         cd.p = irq_data_get_irq_chip_data(data);
 237
 238         if (cd.s.line == 0) {
 239                 raw_spin_lock_irqsave(&octeon_irq_ciu0_lock, flags);
 240                 pen = &per_cpu(octeon_irq_ciu0_en_mirror, cpu);
 241                 set_bit(cd.s.bit, pen);
 242                 cvmx_write_csr(CVMX_CIU_INTX_EN0(coreid * 2), *pen);
 243                 raw_spin_unlock_irqrestore(&octeon_irq_ciu0_lock, flags);
 244         } else {
 245                 raw_spin_lock_irqsave(&octeon_irq_ciu1_lock, flags);
 246                 pen = &per_cpu(octeon_irq_ciu1_en_mirror, cpu);
 247                 set_bit(cd.s.bit, pen);
 248                 cvmx_write_csr(CVMX_CIU_INTX_EN1(coreid * 2 + 1), *pen);
 249                 raw_spin_unlock_irqrestore(&octeon_irq_ciu1_lock, flags);
 250         }
 251 }
 252
 253 static void octeon_irq_ciu_enable_local(struct irq_data *data)
 254 {
 255         unsigned long *pen;
 256         unsigned long flags;
 257         union octeon_ciu_chip_data cd;
 258
 259         cd.p = irq_data_get_irq_chip_data(data);
 260
 261         if (cd.s.line == 0) {
 262                 raw_spin_lock_irqsave(&octeon_irq_ciu0_lock, flags);
 263                 pen = &__get_cpu_var(octeon_irq_ciu0_en_mirror);
 264                 set_bit(cd.s.bit, pen);
 265                 cvmx_write_csr(CVMX_CIU_INTX_EN0(cvmx_get_core_num() * 2), *pen);
 266                 raw_spin_unlock_irqrestore(&octeon_irq_ciu0_lock, flags);
 267         } else {
 268                 raw_spin_lock_irqsave(&octeon_irq_ciu1_lock, flags);
 269                 pen = &__get_cpu_var(octeon_irq_ciu1_en_mirror);
 270                 set_bit(cd.s.bit, pen);
 271                 cvmx_write_csr(CVMX_CIU_INTX_EN1(cvmx_get_core_num() * 2 + 1), *pen);
 272                 raw_spin_unlock_irqrestore(&octeon_irq_ciu1_lock, flags);
 273         }
 274 }
 275
 276 static void octeon_irq_ciu_disable_local(struct irq_data *data)
 277 {
 278         unsigned long *pen;
 279         unsigned long flags;
 280         union octeon_ciu_chip_data cd;
 281
 282         cd.p = irq_data_get_irq_chip_data(data);
 283
 284         if (cd.s.line == 0) {
 285                 raw_spin_lock_irqsave(&octeon_irq_ciu0_lock, flags);
 286                 pen = &__get_cpu_var(octeon_irq_ciu0_en_mirror);
 287                 clear_bit(cd.s.bit, pen);
 288                 cvmx_write_csr(CVMX_CIU_INTX_EN0(cvmx_get_core_num() * 2), *pen);
 289                 raw_spin_unlock_irqrestore(&octeon_irq_ciu0_lock, flags);
 290         } else {
 291                 raw_spin_lock_irqsave(&octeon_irq_ciu1_lock, flags);
 292                 pen = &__get_cpu_var(octeon_irq_ciu1_en_mirror);
 293                 clear_bit(cd.s.bit, pen);
 294                 cvmx_write_csr(CVMX_CIU_INTX_EN1(cvmx_get_core_num() * 2 + 1), *pen);
 295                 raw_spin_unlock_irqrestore(&octeon_irq_ciu1_lock, flags);
 296         }
 297 }
 298
 299 static void octeon_irq_ciu_disable_all(struct irq_data *data)
 300 {
 301         unsigned long flags;
 302         unsigned long *pen;
 303         int cpu;
 304         union octeon_ciu_chip_data cd;
 305
 306         wmb(); /* Make sure flag changes arrive before register updates. */
 307
 308         cd.p = irq_data_get_irq_chip_data(data);
 309
 310         if (cd.s.line == 0) {
 311                 raw_spin_lock_irqsave(&octeon_irq_ciu0_lock, flags);
 312                 for_each_online_cpu(cpu) {
 313                         int coreid = octeon_coreid_for_cpu(cpu);
 314                         pen = &per_cpu(octeon_irq_ciu0_en_mirror, cpu);
 315                         clear_bit(cd.s.bit, pen);
 316                         cvmx_write_csr(CVMX_CIU_INTX_EN0(coreid * 2), *pen);
 317                 }
 318                 raw_spin_unlock_irqrestore(&octeon_irq_ciu0_lock, flags);
 319         } else {
 320                 raw_spin_lock_irqsave(&octeon_irq_ciu1_lock, flags);
 321                 for_each_online_cpu(cpu) {
 322                         int coreid = octeon_coreid_for_cpu(cpu);
 323                         pen = &per_cpu(octeon_irq_ciu1_en_mirror, cpu);
 324                         clear_bit(cd.s.bit, pen);
 325                         cvmx_write_csr(CVMX_CIU_INTX_EN1(coreid * 2 + 1), *pen);
 326                 }
 327                 raw_spin_unlock_irqrestore(&octeon_irq_ciu1_lock, flags);
 328         }
 329 }
 330
 331 static void octeon_irq_ciu_enable_all(struct irq_data *data)
 332 {
 333         unsigned long flags;
 334         unsigned long *pen;
 335         int cpu;
 336         union octeon_ciu_chip_data cd;
 337
 338         cd.p = irq_data_get_irq_chip_data(data);
 339
 340         if (cd.s.line == 0) {
 341                 raw_spin_lock_irqsave(&octeon_irq_ciu0_lock, flags);
 342                 for_each_online_cpu(cpu) {
 343                         int coreid = octeon_coreid_for_cpu(cpu);
 344                         pen = &per_cpu(octeon_irq_ciu0_en_mirror, cpu);
 345                         set_bit(cd.s.bit, pen);
 346                         cvmx_write_csr(CVMX_CIU_INTX_EN0(coreid * 2), *pen);
 347                 }
 348                 raw_spin_unlock_irqrestore(&octeon_irq_ciu0_lock, flags);
 349         } else {
 350                 raw_spin_lock_irqsave(&octeon_irq_ciu1_lock, flags);
 351                 for_each_online_cpu(cpu) {
 352                         int coreid = octeon_coreid_for_cpu(cpu);
 353                         pen = &per_cpu(octeon_irq_ciu1_en_mirror, cpu);
 354                         set_bit(cd.s.bit, pen);
 355                         cvmx_write_csr(CVMX_CIU_INTX_EN1(coreid * 2 + 1), *pen);
 356                 }
 357                 raw_spin_unlock_irqrestore(&octeon_irq_ciu1_lock, flags);
 358         }
 359 }
 360
 361 /*
 362  * Enable the irq on the next core in the affinity set for chips that
 363  * have the EN*_W1{S,C} registers.
 364  */
 365 static void octeon_irq_ciu_enable_v2(struct irq_data *data)
 366 {
 367         u64 mask;
 368         int cpu = next_cpu_for_irq(data);
 369         union octeon_ciu_chip_data cd;
 370
 371         cd.p = irq_data_get_irq_chip_data(data);
 372         mask = 1ull << (cd.s.bit);
 373
 374         /*
 375          * Called under the desc lock, so these should never get out
 376          * of sync.
 377          */
 378         if (cd.s.line == 0) {
 379                 int index = octeon_coreid_for_cpu(cpu) * 2;
 380                 set_bit(cd.s.bit, &per_cpu(octeon_irq_ciu0_en_mirror, cpu));
 381                 cvmx_write_csr(CVMX_CIU_INTX_EN0_W1S(index), mask);
 382         } else {
 383                 int index = octeon_coreid_for_cpu(cpu) * 2 + 1;
 384                 set_bit(cd.s.bit, &per_cpu(octeon_irq_ciu1_en_mirror, cpu));
 385                 cvmx_write_csr(CVMX_CIU_INTX_EN1_W1S(index), mask);
 386         }
 387 }
 388
 389 /*
 390  * Enable the irq on the current CPU for chips that
 391  * have the EN*_W1{S,C} registers.
 392  */
 393 static void octeon_irq_ciu_enable_local_v2(struct irq_data *data)
 394 {
 395         u64 mask;
 396         union octeon_ciu_chip_data cd;
 397
 398         cd.p = irq_data_get_irq_chip_data(data);
 399         mask = 1ull << (cd.s.bit);
 400
 401         if (cd.s.line == 0) {
 402                 int index = cvmx_get_core_num() * 2;
 403                 set_bit(cd.s.bit, &__get_cpu_var(octeon_irq_ciu0_en_mirror));
 404                 cvmx_write_csr(CVMX_CIU_INTX_EN0_W1S(index), mask);
 405         } else {
 406                 int index = cvmx_get_core_num() * 2 + 1;
 407                 set_bit(cd.s.bit, &__get_cpu_var(octeon_irq_ciu1_en_mirror));
 408                 cvmx_write_csr(CVMX_CIU_INTX_EN1_W1S(index), mask);
 409         }
 410 }
 411
 412 static void octeon_irq_ciu_disable_local_v2(struct irq_data *data)
 413 {
 414         u64 mask;
 415         union octeon_ciu_chip_data cd;
 416
 417         cd.p = irq_data_get_irq_chip_data(data);
 418         mask = 1ull << (cd.s.bit);
 419
 420         if (cd.s.line == 0) {
 421                 int index = cvmx_get_core_num() * 2;
 422                 clear_bit(cd.s.bit, &__get_cpu_var(octeon_irq_ciu0_en_mirror));
 423                 cvmx_write_csr(CVMX_CIU_INTX_EN0_W1C(index), mask);
 424         } else {
 425                 int index = cvmx_get_core_num() * 2 + 1;
 426                 clear_bit(cd.s.bit, &__get_cpu_var(octeon_irq_ciu1_en_mirror));
 427                 cvmx_write_csr(CVMX_CIU_INTX_EN1_W1C(index), mask);
 428         }
 429 }
 430
 431 /*
 432  * Write to the W1C bit in CVMX_CIU_INTX_SUM0 to clear the irq.
 433  */
 434 static void octeon_irq_ciu_ack(struct irq_data *data)
 435 {
 436         u64 mask;
 437         union octeon_ciu_chip_data cd;
 438
 439         cd.p = data->chip_data;
 440         mask = 1ull << (cd.s.bit);
 441
 442         if (cd.s.line == 0) {
 443                 int index = cvmx_get_core_num() * 2;
 444                 cvmx_write_csr(CVMX_CIU_INTX_SUM0(index), mask);
 445         } else {
 446                 cvmx_write_csr(CVMX_CIU_INT_SUM1, mask);
 447         }
 448 }
 449
 450 /*
 451  * Disable the irq on the all cores for chips that have the EN*_W1{S,C}
 452  * registers.
 453  */
 454 static void octeon_irq_ciu_disable_all_v2(struct irq_data *data)
 455 {
 456         int cpu;
 457         u64 mask;
 458         union octeon_ciu_chip_data cd;
 459
 460         wmb(); /* Make sure flag changes arrive before register updates. */
 461
 462         cd.p = data->chip_data;
 463         mask = 1ull << (cd.s.bit);
 464
 465         if (cd.s.line == 0) {
 466                 for_each_online_cpu(cpu) {
 467                         int index = octeon_coreid_for_cpu(cpu) * 2;
 468                         clear_bit(cd.s.bit, &per_cpu(octeon_irq_ciu0_en_mirror, cpu));
 469                         cvmx_write_csr(CVMX_CIU_INTX_EN0_W1C(index), mask);
 470                 }
 471         } else {
 472                 for_each_online_cpu(cpu) {
 473                         int index = octeon_coreid_for_cpu(cpu) * 2 + 1;
 474                         clear_bit(cd.s.bit, &per_cpu(octeon_irq_ciu1_en_mirror, cpu));
 475                         cvmx_write_csr(CVMX_CIU_INTX_EN1_W1C(index), mask);
 476                 }
 477         }
 478 }
 479
 480 /*
 481  * Enable the irq on the all cores for chips that have the EN*_W1{S,C}
 482  * registers.
 483  */
 484 static void octeon_irq_ciu_enable_all_v2(struct irq_data *data)
 485 {
 486         int cpu;
 487         u64 mask;
 488         union octeon_ciu_chip_data cd;
 489
 490         cd.p = data->chip_data;
 491         mask = 1ull << (cd.s.bit);
 492
 493         if (cd.s.line == 0) {
 494                 for_each_online_cpu(cpu) {
 495                         int index = octeon_coreid_for_cpu(cpu) * 2;
 496                         set_bit(cd.s.bit, &per_cpu(octeon_irq_ciu0_en_mirror, cpu));
 497                         cvmx_write_csr(CVMX_CIU_INTX_EN0_W1S(index), mask);
 498                 }
 499         } else {
 500                 for_each_online_cpu(cpu) {
 501                         int index = octeon_coreid_for_cpu(cpu) * 2 + 1;
 502                         set_bit(cd.s.bit, &per_cpu(octeon_irq_ciu1_en_mirror, cpu));
 503                         cvmx_write_csr(CVMX_CIU_INTX_EN1_W1S(index), mask);
 504                 }
 505         }
 506 }
 507
 508 #ifdef CONFIG_SMP
 509
 510 static void octeon_irq_cpu_offline_ciu(struct irq_data *data)
 511 {
 512         int cpu = smp_processor_id();
 513         cpumask_t new_affinity;
 514
 515         if (!cpumask_test_cpu(cpu, data->affinity))
 516                 return;
 517
 518         if (cpumask_weight(data->affinity) > 1) {
 519                 /*
 520                  * It has multi CPU affinity, just remove this CPU
 521                  * from the affinity set.
 522                  */
 523                 cpumask_copy(&new_affinity, data->affinity);
 524                 cpumask_clear_cpu(cpu, &new_affinity);
 525         } else {
 526                 /* Otherwise, put it on lowest numbered online CPU. */
 527                 cpumask_clear(&new_affinity);
 528                 cpumask_set_cpu(cpumask_first(cpu_online_mask), &new_affinity);
 529         }
 530         __irq_set_affinity_locked(data, &new_affinity);
 531 }
 532
 533 static int octeon_irq_ciu_set_affinity(struct irq_data *data,
 534                                        const struct cpumask *dest, bool force)
 535 {
 536         int cpu;
 537         bool enable_one = !irqd_irq_disabled(data) && !irqd_irq_masked(data);
 538         unsigned long flags;
 539         union octeon_ciu_chip_data cd;
 540
 541         cd.p = data->chip_data;
 542
 543         /*
 544          * For non-v2 CIU, we will allow only single CPU affinity.
 545          * This removes the need to do locking in the .ack/.eoi
 546          * functions.
 547          */
 548         if (cpumask_weight(dest) != 1)
 549                 return -EINVAL;
 550
 551         if (!enable_one)
 552                 return 0;
 553
 554         if (cd.s.line == 0) {
 555                 raw_spin_lock_irqsave(&octeon_irq_ciu0_lock, flags);
 556                 for_each_online_cpu(cpu) {
 557                         int coreid = octeon_coreid_for_cpu(cpu);
 558                         unsigned long *pen = &per_cpu(octeon_irq_ciu0_en_mirror, cpu);
 559
 560                         if (cpumask_test_cpu(cpu, dest) && enable_one) {
 561                                 enable_one = false;
 562                                 set_bit(cd.s.bit, pen);
 563                         } else {
 564                                 clear_bit(cd.s.bit, pen);
 565                         }
 566                         cvmx_write_csr(CVMX_CIU_INTX_EN0(coreid * 2), *pen);
 567                 }
 568                 raw_spin_unlock_irqrestore(&octeon_irq_ciu0_lock, flags);
 569         } else {
 570                 raw_spin_lock_irqsave(&octeon_irq_ciu1_lock, flags);
 571                 for_each_online_cpu(cpu) {
 572                         int coreid = octeon_coreid_for_cpu(cpu);
 573                         unsigned long *pen = &per_cpu(octeon_irq_ciu1_en_mirror, cpu);
 574
 575                         if (cpumask_test_cpu(cpu, dest) && enable_one) {
 576                                 enable_one = false;
 577                                 set_bit(cd.s.bit, pen);
 578                         } else {
 579                                 clear_bit(cd.s.bit, pen);
 580                         }
 581                         cvmx_write_csr(CVMX_CIU_INTX_EN1(coreid * 2 + 1), *pen);
 582                 }
 583                 raw_spin_unlock_irqrestore(&octeon_irq_ciu1_lock, flags);
 584         }
 585         return 0;
 586 }
 587
 588 /*
 589  * Set affinity for the irq for chips that have the EN*_W1{S,C}
 590  * registers.
 591  */
 592 static int octeon_irq_ciu_set_affinity_v2(struct irq_data *data,
 593                                           const struct cpumask *dest,
 594                                           bool force)
 595 {
 596         int cpu;
 597         bool enable_one = !irqd_irq_disabled(data) && !irqd_irq_masked(data);
 598         u64 mask;
 599         union octeon_ciu_chip_data cd;
 600
 601         if (!enable_one)
 602                 return 0;
 603
 604         cd.p = data->chip_data;
 605         mask = 1ull << cd.s.bit;
 606
 607         if (cd.s.line == 0) {
 608                 for_each_online_cpu(cpu) {
 609                         unsigned long *pen = &per_cpu(octeon_irq_ciu0_en_mirror, cpu);
 610                         int index = octeon_coreid_for_cpu(cpu) * 2;
 611                         if (cpumask_test_cpu(cpu, dest) && enable_one) {
 612                                 enable_one = false;
 613                                 set_bit(cd.s.bit, pen);
 614                                 cvmx_write_csr(CVMX_CIU_INTX_EN0_W1S(index), mask);
 615                         } else {
 616                                 clear_bit(cd.s.bit, pen);
 617                                 cvmx_write_csr(CVMX_CIU_INTX_EN0_W1C(index), mask);
 618                         }
 619                 }
 620         } else {
 621                 for_each_online_cpu(cpu) {
 622                         unsigned long *pen = &per_cpu(octeon_irq_ciu1_en_mirror, cpu);
 623                         int index = octeon_coreid_for_cpu(cpu) * 2 + 1;
 624                         if (cpumask_test_cpu(cpu, dest) && enable_one) {
 625                                 enable_one = false;
 626                                 set_bit(cd.s.bit, pen);
 627                                 cvmx_write_csr(CVMX_CIU_INTX_EN1_W1S(index), mask);
 628                         } else {
 629                                 clear_bit(cd.s.bit, pen);
 630                                 cvmx_write_csr(CVMX_CIU_INTX_EN1_W1C(index), mask);
 631                         }
 632                 }
 633         }
 634         return 0;
 635 }
 636 #endif
 637
 638 /*
 639  * The v1 CIU code already masks things, so supply a dummy version to
 640  * the core chip code.
 641  */
 642 static void octeon_irq_dummy_mask(struct irq_data *data)
 643 {
 644 }
 645
 646 /*
 647  * Newer octeon chips have support for lockless CIU operation.
 648  */
 649 static struct irq_chip octeon_irq_chip_ciu_v2 = {
 650         .name = "CIU",
 651         .irq_enable = octeon_irq_ciu_enable_v2,
 652         .irq_disable = octeon_irq_ciu_disable_all_v2,
 653         .irq_mask = octeon_irq_ciu_disable_local_v2,
 654         .irq_unmask = octeon_irq_ciu_enable_v2,
 655 #ifdef CONFIG_SMP
 656         .irq_set_affinity = octeon_irq_ciu_set_affinity_v2,
 657         .irq_cpu_offline = octeon_irq_cpu_offline_ciu,
 658 #endif
 659 };
 660
 661 static struct irq_chip octeon_irq_chip_ciu_edge_v2 = {
 662         .name = "CIU-E",
 663         .irq_enable = octeon_irq_ciu_enable_v2,
 664         .irq_disable = octeon_irq_ciu_disable_all_v2,
 665         .irq_ack = octeon_irq_ciu_ack,
 666         .irq_mask = octeon_irq_ciu_disable_local_v2,
 667         .irq_unmask = octeon_irq_ciu_enable_v2,
 668 #ifdef CONFIG_SMP
 669         .irq_set_affinity = octeon_irq_ciu_set_affinity_v2,
 670         .irq_cpu_offline = octeon_irq_cpu_offline_ciu,
 671 #endif
 672 };
 673
 674 static struct irq_chip octeon_irq_chip_ciu = {
 675         .name = "CIU",
 676         .irq_enable = octeon_irq_ciu_enable,
 677         .irq_disable = octeon_irq_ciu_disable_all,
 678         .irq_mask = octeon_irq_dummy_mask,
 679 #ifdef CONFIG_SMP
 680         .irq_set_affinity = octeon_irq_ciu_set_affinity,
 681         .irq_cpu_offline = octeon_irq_cpu_offline_ciu,
 682 #endif
 683 };
 684
 685 static struct irq_chip octeon_irq_chip_ciu_edge = {
 686         .name = "CIU-E",
 687         .irq_enable = octeon_irq_ciu_enable,
 688         .irq_disable = octeon_irq_ciu_disable_all,
 689         .irq_mask = octeon_irq_dummy_mask,
 690         .irq_ack = octeon_irq_ciu_ack,
 691 #ifdef CONFIG_SMP
 692         .irq_set_affinity = octeon_irq_ciu_set_affinity,
 693         .irq_cpu_offline = octeon_irq_cpu_offline_ciu,
 694 #endif
 695 };
 696
 697 /* The mbox versions don't do any affinity or round-robin. */
 698 static struct irq_chip octeon_irq_chip_ciu_mbox_v2 = {
 699         .name = "CIU-M",
 700         .irq_enable = octeon_irq_ciu_enable_all_v2,
 701         .irq_disable = octeon_irq_ciu_disable_all_v2,
 702         .irq_ack = octeon_irq_ciu_disable_local_v2,
 703         .irq_eoi = octeon_irq_ciu_enable_local_v2,
 704
 705         .irq_cpu_online = octeon_irq_ciu_enable_local_v2,
 706         .irq_cpu_offline = octeon_irq_ciu_disable_local_v2,
 707         .flags = IRQCHIP_ONOFFLINE_ENABLED,
 708 };
 709
 710 static struct irq_chip octeon_irq_chip_ciu_mbox = {
 711         .name = "CIU-M",
 712         .irq_enable = octeon_irq_ciu_enable_all,
 713         .irq_disable = octeon_irq_ciu_disable_all,
 714
 715         .irq_cpu_online = octeon_irq_ciu_enable_local,
 716         .irq_cpu_offline = octeon_irq_ciu_disable_local,
 717         .flags = IRQCHIP_ONOFFLINE_ENABLED,
 718 };
 719
 720 /*
 721  * Watchdog interrupts are special.  They are associated with a single
 722  * core, so we hardwire the affinity to that core.
 723  */
 724 static void octeon_irq_ciu_wd_enable(struct irq_data *data)
 725 {
 726         unsigned long flags;
 727         unsigned long *pen;
 728         int coreid = data->irq - OCTEON_IRQ_WDOG0;      /* Bit 0-63 of EN1 */
 729         int cpu = octeon_cpu_for_coreid(coreid);
 730
 731         raw_spin_lock_irqsave(&octeon_irq_ciu1_lock, flags);
 732         pen = &per_cpu(octeon_irq_ciu1_en_mirror, cpu);
 733         set_bit(coreid, pen);
 734         cvmx_write_csr(CVMX_CIU_INTX_EN1(coreid * 2 + 1), *pen);
 735         raw_spin_unlock_irqrestore(&octeon_irq_ciu1_lock, flags);
 736 }
 737
 738 /*
 739  * Watchdog interrupts are special.  They are associated with a single
 740  * core, so we hardwire the affinity to that core.
 741  */
 742 static void octeon_irq_ciu1_wd_enable_v2(struct irq_data *data)
 743 {
 744         int coreid = data->irq - OCTEON_IRQ_WDOG0;
 745         int cpu = octeon_cpu_for_coreid(coreid);
 746
 747         set_bit(coreid, &per_cpu(octeon_irq_ciu1_en_mirror, cpu));
 748         cvmx_write_csr(CVMX_CIU_INTX_EN1_W1S(coreid * 2 + 1), 1ull << coreid);
 749 }
 750
 751
 752 static struct irq_chip octeon_irq_chip_ciu_wd_v2 = {
 753         .name = "CIU-W",
 754         .irq_enable = octeon_irq_ciu1_wd_enable_v2,
 755         .irq_disable = octeon_irq_ciu_disable_all_v2,
 756         .irq_mask = octeon_irq_ciu_disable_local_v2,
 757         .irq_unmask = octeon_irq_ciu_enable_local_v2,
 758 };
 759
 760 static struct irq_chip octeon_irq_chip_ciu_wd = {
 761         .name = "CIU-W",
 762         .irq_enable = octeon_irq_ciu_wd_enable,
 763         .irq_disable = octeon_irq_ciu_disable_all,
 764         .irq_mask = octeon_irq_dummy_mask,
 765 };
 766
 767 static void octeon_irq_ip2_v1(void)
 768 {
 769         const unsigned long core_id = cvmx_get_core_num();
 770         u64 ciu_sum = cvmx_read_csr(CVMX_CIU_INTX_SUM0(core_id * 2));
 771
 772         ciu_sum &= __get_cpu_var(octeon_irq_ciu0_en_mirror);
 773         clear_c0_status(STATUSF_IP2);
 774         if (likely(ciu_sum)) {
 775                 int bit = fls64(ciu_sum) - 1;
 776                 int irq = octeon_irq_ciu_to_irq[0][bit];
 777                 if (likely(irq))
 778                         do_IRQ(irq);
 779                 else
 780                         spurious_interrupt();
 781         } else {
 782                 spurious_interrupt();
 783         }
 784         set_c0_status(STATUSF_IP2);
 785 }
 786
 787 static void octeon_irq_ip2_v2(void)
 788 {
 789         const unsigned long core_id = cvmx_get_core_num();
 790         u64 ciu_sum = cvmx_read_csr(CVMX_CIU_INTX_SUM0(core_id * 2));
 791
 792         ciu_sum &= __get_cpu_var(octeon_irq_ciu0_en_mirror);
 793         if (likely(ciu_sum)) {
 794                 int bit = fls64(ciu_sum) - 1;
 795                 int irq = octeon_irq_ciu_to_irq[0][bit];
 796                 if (likely(irq))
 797                         do_IRQ(irq);
 798                 else
 799                         spurious_interrupt();
 800         } else {
 801                 spurious_interrupt();
 802         }
 803 }
 804 static void octeon_irq_ip3_v1(void)
 805 {
 806         u64 ciu_sum = cvmx_read_csr(CVMX_CIU_INT_SUM1);
 807
 808         ciu_sum &= __get_cpu_var(octeon_irq_ciu1_en_mirror);
 809         clear_c0_status(STATUSF_IP3);
 810         if (likely(ciu_sum)) {
 811                 int bit = fls64(ciu_sum) - 1;
 812                 int irq = octeon_irq_ciu_to_irq[1][bit];
 813                 if (likely(irq))
 814                         do_IRQ(irq);
 815                 else
 816                         spurious_interrupt();
 817         } else {
 818                 spurious_interrupt();
 819         }
 820         set_c0_status(STATUSF_IP3);
 821 }
 822
 823 static void octeon_irq_ip3_v2(void)
 824 {
 825         u64 ciu_sum = cvmx_read_csr(CVMX_CIU_INT_SUM1);
 826
 827         ciu_sum &= __get_cpu_var(octeon_irq_ciu1_en_mirror);
 828         if (likely(ciu_sum)) {
 829                 int bit = fls64(ciu_sum) - 1;
 830                 int irq = octeon_irq_ciu_to_irq[1][bit];
 831                 if (likely(irq))
 832                         do_IRQ(irq);
 833                 else
 834                         spurious_interrupt();
 835         } else {
 836                 spurious_interrupt();
 837         }
 838 }
 839
 840 static void octeon_irq_ip4_mask(void)
 841 {
 842         clear_c0_status(STATUSF_IP4);
 843         spurious_interrupt();
 844 }
 845
 846 static void (*octeon_irq_ip2)(void);
 847 static void (*octeon_irq_ip3)(void);
 848 static void (*octeon_irq_ip4)(void);
 849
 850 void __cpuinitdata (*octeon_irq_setup_secondary)(void);
 851
 852 static void __cpuinit octeon_irq_percpu_enable(void)
 853 {
 854         irq_cpu_online();
 855 }
 856
 857 static void __cpuinit octeon_irq_init_ciu_percpu(void)
 858 {
 859         int coreid = cvmx_get_core_num();
 860         /*
 861          * Disable All CIU Interrupts. The ones we need will be
 862          * enabled later.  Read the SUM register so we know the write
 863          * completed.
 864          */
 865         cvmx_write_csr(CVMX_CIU_INTX_EN0((coreid * 2)), 0);
 866         cvmx_write_csr(CVMX_CIU_INTX_EN0((coreid * 2 + 1)), 0);
 867         cvmx_write_csr(CVMX_CIU_INTX_EN1((coreid * 2)), 0);
 868         cvmx_write_csr(CVMX_CIU_INTX_EN1((coreid * 2 + 1)), 0);
 869         cvmx_read_csr(CVMX_CIU_INTX_SUM0((coreid * 2)));
 870 }
 871
 872 static void __cpuinit octeon_irq_setup_secondary_ciu(void)
 873 {
 874
 875         __get_cpu_var(octeon_irq_ciu0_en_mirror) = 0;
 876         __get_cpu_var(octeon_irq_ciu1_en_mirror) = 0;
 877
 878         octeon_irq_init_ciu_percpu();
 879         octeon_irq_percpu_enable();
 880
 881         /* Enable the CIU lines */
 882         set_c0_status(STATUSF_IP3 | STATUSF_IP2);
 883         clear_c0_status(STATUSF_IP4);
 884 }
 885
 886 static void __init octeon_irq_init_ciu(void)
 887 {
 888         unsigned int i;
 889         struct irq_chip *chip;
 890         struct irq_chip *chip_edge;
 891         struct irq_chip *chip_mbox;
 892         struct irq_chip *chip_wd;
 893
 894         octeon_irq_init_ciu_percpu();
 895         octeon_irq_setup_secondary = octeon_irq_setup_secondary_ciu;
 896
 897         if (OCTEON_IS_MODEL(OCTEON_CN58XX_PASS2_X) ||
 898             OCTEON_IS_MODEL(OCTEON_CN56XX_PASS2_X) ||
 899             OCTEON_IS_MODEL(OCTEON_CN52XX_PASS2_X) ||
 900             OCTEON_IS_MODEL(OCTEON_CN6XXX)) {
 901                 octeon_irq_ip2 = octeon_irq_ip2_v2;
 902                 octeon_irq_ip3 = octeon_irq_ip3_v2;
 903                 chip = &octeon_irq_chip_ciu_v2;
 904                 chip_edge = &octeon_irq_chip_ciu_edge_v2;
 905                 chip_mbox = &octeon_irq_chip_ciu_mbox_v2;
 906                 chip_wd = &octeon_irq_chip_ciu_wd_v2;
 907         } else {
 908                 octeon_irq_ip2 = octeon_irq_ip2_v1;
 909                 octeon_irq_ip3 = octeon_irq_ip3_v1;
 910                 chip = &octeon_irq_chip_ciu;
 911                 chip_edge = &octeon_irq_chip_ciu_edge;
 912                 chip_mbox = &octeon_irq_chip_ciu_mbox;
 913                 chip_wd = &octeon_irq_chip_ciu_wd;
 914         }
 915         octeon_irq_ip4 = octeon_irq_ip4_mask;
 916
 917         /* Mips internal */
 918         octeon_irq_init_core();
 919
 920         /* CIU_0 */
 921         for (i = 0; i < 16; i++)
 922                 octeon_irq_set_ciu_mapping(i + OCTEON_IRQ_WORKQ0, 0, i + 0, chip, handle_level_irq);
 923         for (i = 0; i < 16; i++)
 924                 octeon_irq_set_ciu_mapping(i + OCTEON_IRQ_GPIO0, 0, i + 16, chip, handle_level_irq);
 925
 926         octeon_irq_set_ciu_mapping(OCTEON_IRQ_MBOX0, 0, 32, chip_mbox, handle_percpu_irq);
 927         octeon_irq_set_ciu_mapping(OCTEON_IRQ_MBOX1, 0, 33, chip_mbox, handle_percpu_irq);
 928
 929         octeon_irq_set_ciu_mapping(OCTEON_IRQ_UART0, 0, 34, chip, handle_level_irq);
 930         octeon_irq_set_ciu_mapping(OCTEON_IRQ_UART1, 0, 35, chip, handle_level_irq);
 931
 932         for (i = 0; i < 4; i++)
 933                 octeon_irq_set_ciu_mapping(i + OCTEON_IRQ_PCI_INT0, 0, i + 36, chip, handle_level_irq);
 934         for (i = 0; i < 4; i++)
 935                 octeon_irq_set_ciu_mapping(i + OCTEON_IRQ_PCI_MSI0, 0, i + 40, chip, handle_level_irq);
 936
 937         octeon_irq_set_ciu_mapping(OCTEON_IRQ_TWSI, 0, 45, chip, handle_level_irq);
 938         octeon_irq_set_ciu_mapping(OCTEON_IRQ_RML, 0, 46, chip, handle_level_irq);
 939         octeon_irq_set_ciu_mapping(OCTEON_IRQ_TRACE0, 0, 47, chip, handle_level_irq);
 940
 941         for (i = 0; i < 2; i++)
 942                 octeon_irq_set_ciu_mapping(i + OCTEON_IRQ_GMX_DRP0, 0, i + 48, chip_edge, handle_edge_irq);
 943
 944         octeon_irq_set_ciu_mapping(OCTEON_IRQ_IPD_DRP, 0, 50, chip_edge, handle_edge_irq);
 945         octeon_irq_set_ciu_mapping(OCTEON_IRQ_KEY_ZERO, 0, 51, chip_edge, handle_edge_irq);
 946
 947         for (i = 0; i < 4; i++)
 948                 octeon_irq_set_ciu_mapping(i + OCTEON_IRQ_TIMER0, 0, i + 52, chip_edge, handle_edge_irq);
 949
 950         octeon_irq_set_ciu_mapping(OCTEON_IRQ_USB0, 0, 56, chip, handle_level_irq);
 951         octeon_irq_set_ciu_mapping(OCTEON_IRQ_PCM, 0, 57, chip, handle_level_irq);
 952         octeon_irq_set_ciu_mapping(OCTEON_IRQ_MPI, 0, 58, chip, handle_level_irq);
 953         octeon_irq_set_ciu_mapping(OCTEON_IRQ_TWSI2, 0, 59, chip, handle_level_irq);
 954         octeon_irq_set_ciu_mapping(OCTEON_IRQ_POWIQ, 0, 60, chip, handle_level_irq);
 955         octeon_irq_set_ciu_mapping(OCTEON_IRQ_IPDPPTHR, 0, 61, chip, handle_level_irq);
 956         octeon_irq_set_ciu_mapping(OCTEON_IRQ_MII0, 0, 62, chip, handle_level_irq);
 957         octeon_irq_set_ciu_mapping(OCTEON_IRQ_BOOTDMA, 0, 63, chip, handle_level_irq);
 958
 959         /* CIU_1 */
 960         for (i = 0; i < 16; i++)
 961                 octeon_irq_set_ciu_mapping(i + OCTEON_IRQ_WDOG0, 1, i + 0, chip_wd, handle_level_irq);
 962
 963         octeon_irq_set_ciu_mapping(OCTEON_IRQ_UART2, 1, 16, chip, handle_level_irq);
 964         octeon_irq_set_ciu_mapping(OCTEON_IRQ_USB1, 1, 17, chip, handle_level_irq);
 965         octeon_irq_set_ciu_mapping(OCTEON_IRQ_MII1, 1, 18, chip, handle_level_irq);
 966         octeon_irq_set_ciu_mapping(OCTEON_IRQ_NAND, 1, 19, chip, handle_level_irq);
 967         octeon_irq_set_ciu_mapping(OCTEON_IRQ_MIO, 1, 20, chip, handle_level_irq);
 968         octeon_irq_set_ciu_mapping(OCTEON_IRQ_IOB, 1, 21, chip, handle_level_irq);
 969         octeon_irq_set_ciu_mapping(OCTEON_IRQ_FPA, 1, 22, chip, handle_level_irq);
 970         octeon_irq_set_ciu_mapping(OCTEON_IRQ_POW, 1, 23, chip, handle_level_irq);
 971         octeon_irq_set_ciu_mapping(OCTEON_IRQ_L2C, 1, 24, chip, handle_level_irq);
 972         octeon_irq_set_ciu_mapping(OCTEON_IRQ_IPD, 1, 25, chip, handle_level_irq);
 973         octeon_irq_set_ciu_mapping(OCTEON_IRQ_PIP, 1, 26, chip, handle_level_irq);
 974         octeon_irq_set_ciu_mapping(OCTEON_IRQ_PKO, 1, 27, chip, handle_level_irq);
 975         octeon_irq_set_ciu_mapping(OCTEON_IRQ_ZIP, 1, 28, chip, handle_level_irq);
 976         octeon_irq_set_ciu_mapping(OCTEON_IRQ_TIM, 1, 29, chip, handle_level_irq);
 977         octeon_irq_set_ciu_mapping(OCTEON_IRQ_RAD, 1, 30, chip, handle_level_irq);
 978         octeon_irq_set_ciu_mapping(OCTEON_IRQ_KEY, 1, 31, chip, handle_level_irq);
 979         octeon_irq_set_ciu_mapping(OCTEON_IRQ_DFA, 1, 32, chip, handle_level_irq);
 980         octeon_irq_set_ciu_mapping(OCTEON_IRQ_USBCTL, 1, 33, chip, handle_level_irq);
 981         octeon_irq_set_ciu_mapping(OCTEON_IRQ_SLI, 1, 34, chip, handle_level_irq);
 982         octeon_irq_set_ciu_mapping(OCTEON_IRQ_DPI, 1, 35, chip, handle_level_irq);
 983
 984         octeon_irq_set_ciu_mapping(OCTEON_IRQ_AGX0, 1, 36, chip, handle_level_irq);
 985
 986         octeon_irq_set_ciu_mapping(OCTEON_IRQ_AGL, 1, 46, chip, handle_level_irq);
 987
 988         octeon_irq_set_ciu_mapping(OCTEON_IRQ_PTP, 1, 47, chip_edge, handle_edge_irq);
 989
 990         octeon_irq_set_ciu_mapping(OCTEON_IRQ_PEM0, 1, 48, chip, handle_level_irq);
 991         octeon_irq_set_ciu_mapping(OCTEON_IRQ_PEM1, 1, 49, chip, handle_level_irq);
 992         octeon_irq_set_ciu_mapping(OCTEON_IRQ_SRIO0, 1, 50, chip, handle_level_irq);
 993         octeon_irq_set_ciu_mapping(OCTEON_IRQ_SRIO1, 1, 51, chip, handle_level_irq);
 994         octeon_irq_set_ciu_mapping(OCTEON_IRQ_LMC0, 1, 52, chip, handle_level_irq);
 995         octeon_irq_set_ciu_mapping(OCTEON_IRQ_DFM, 1, 56, chip, handle_level_irq);
 996         octeon_irq_set_ciu_mapping(OCTEON_IRQ_RST, 1, 63, chip, handle_level_irq);
 997
 998         /* Enable the CIU lines */
 999         set_c0_status(STATUSF_IP3 | STATUSF_IP2);
1000         clear_c0_status(STATUSF_IP4);
1001 }
1002
1003 void __init arch_init_irq(void)
1004 {
1005 #ifdef CONFIG_SMP
1006         /* Set the default affinity to the boot cpu. */
1007         cpumask_clear(irq_default_affinity);
1008         cpumask_set_cpu(smp_processor_id(), irq_default_affinity);
1009 #endif
1010         octeon_irq_init_ciu();
1011 }
1012
1013 asmlinkage void plat_irq_dispatch(void)
1014 {
1015         unsigned long cop0_cause;
1016         unsigned long cop0_status;
1017
1018         while (1) {
1019                 cop0_cause = read_c0_cause();
1020                 cop0_status = read_c0_status();
1021                 cop0_cause &= cop0_status;
1022                 cop0_cause &= ST0_IM;
1023
1024                 if (unlikely(cop0_cause & STATUSF_IP2))
1025                         octeon_irq_ip2();
1026                 else if (unlikely(cop0_cause & STATUSF_IP3))
1027                         octeon_irq_ip3();
1028                 else if (unlikely(cop0_cause & STATUSF_IP4))
1029                         octeon_irq_ip4();
1030                 else if (likely(cop0_cause))
1031                         do_IRQ(fls(cop0_cause) - 9 + MIPS_CPU_IRQ_BASE);
1032                 else
1033                         break;
1034         }
1035 }
1036
1037 #ifdef CONFIG_HOTPLUG_CPU
1038
1039 void fixup_irqs(void)
1040 {
1041         irq_cpu_offline();
1042 }
1043
1044 #endif /* CONFIG_HOTPLUG_CPU */