drivers/net/sfc/filter.c

   1 /****************************************************************************
   2  * Driver for Solarflare Solarstorm network controllers and boards
   3  * Copyright 2005-2010 Solarflare Communications Inc.
   4  *
   5  * This program is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 as published
   7  * by the Free Software Foundation, incorporated herein by reference.
   8  */
   9
  10 #include <linux/in.h>
  11 #include <net/ip.h>
  12 #include "efx.h"
  13 #include "filter.h"
  14 #include "io.h"
  15 #include "nic.h"
  16 #include "regs.h"
  17
  18 /* "Fudge factors" - difference between programmed value and actual depth.
  19  * Due to pipelined implementation we need to program H/W with a value that
  20  * is larger than the hop limit we want.
  21  */
  22 #define FILTER_CTL_SRCH_FUDGE_WILD 3
  23 #define FILTER_CTL_SRCH_FUDGE_FULL 1
  24
  25 /* Hard maximum hop limit.  Hardware will time-out beyond 200-something.
  26  * We also need to avoid infinite loops in efx_filter_search() when the
  27  * table is full.
  28  */
  29 #define FILTER_CTL_SRCH_MAX 200
  30
  31 /* Don't try very hard to find space for performance hints, as this is
  32  * counter-productive. */
  33 #define FILTER_CTL_SRCH_HINT_MAX 5
  34
  35 enum efx_filter_table_id {
  36         EFX_FILTER_TABLE_RX_IP = 0,
  37         EFX_FILTER_TABLE_RX_MAC,
  38         EFX_FILTER_TABLE_COUNT,
  39 };
  40
  41 struct efx_filter_table {
  42         enum efx_filter_table_id id;
  43         u32             offset;         /* address of table relative to BAR */
  44         unsigned        size;           /* number of entries */
  45         unsigned        step;           /* step between entries */
  46         unsigned        used;           /* number currently used */
  47         unsigned long   *used_bitmap;
  48         struct efx_filter_spec *spec;
  49         unsigned        search_depth[EFX_FILTER_TYPE_COUNT];
  50 };
  51
  52 struct efx_filter_state {
  53         spinlock_t      lock;
  54         struct efx_filter_table table[EFX_FILTER_TABLE_COUNT];
  55 #ifdef CONFIG_RFS_ACCEL
  56         u32             *rps_flow_id;
  57         unsigned        rps_expire_index;
  58 #endif
  59 };
  60
  61 /* The filter hash function is LFSR polynomial x^16 + x^3 + 1 of a 32-bit
  62  * key derived from the n-tuple.  The initial LFSR state is 0xffff. */
  63 static u16 efx_filter_hash(u32 key)
  64 {
  65         u16 tmp;
  66
  67         /* First 16 rounds */
  68         tmp = 0x1fff ^ key >> 16;
  69         tmp = tmp ^ tmp >> 3 ^ tmp >> 6;
  70         tmp = tmp ^ tmp >> 9;
  71         /* Last 16 rounds */
  72         tmp = tmp ^ tmp << 13 ^ key;
  73         tmp = tmp ^ tmp >> 3 ^ tmp >> 6;
  74         return tmp ^ tmp >> 9;
  75 }
  76
  77 /* To allow for hash collisions, filter search continues at these
  78  * increments from the first possible entry selected by the hash. */
  79 static u16 efx_filter_increment(u32 key)
  80 {
  81         return key * 2 - 1;
  82 }
  83
  84 static enum efx_filter_table_id
  85 efx_filter_spec_table_id(const struct efx_filter_spec *spec)
  86 {
  87         BUILD_BUG_ON(EFX_FILTER_TABLE_RX_IP != (EFX_FILTER_TCP_FULL >> 2));
  88         BUILD_BUG_ON(EFX_FILTER_TABLE_RX_IP != (EFX_FILTER_TCP_WILD >> 2));
  89         BUILD_BUG_ON(EFX_FILTER_TABLE_RX_IP != (EFX_FILTER_UDP_FULL >> 2));
  90         BUILD_BUG_ON(EFX_FILTER_TABLE_RX_IP != (EFX_FILTER_UDP_WILD >> 2));
  91         BUILD_BUG_ON(EFX_FILTER_TABLE_RX_MAC != (EFX_FILTER_MAC_FULL >> 2));
  92         BUILD_BUG_ON(EFX_FILTER_TABLE_RX_MAC != (EFX_FILTER_MAC_WILD >> 2));
  93         EFX_BUG_ON_PARANOID(spec->type == EFX_FILTER_UNSPEC);
  94         return spec->type >> 2;
  95 }
  96
  97 static struct efx_filter_table *
  98 efx_filter_spec_table(struct efx_filter_state *state,
  99                       const struct efx_filter_spec *spec)
 100 {
 101         if (spec->type == EFX_FILTER_UNSPEC)
 102                 return NULL;
 103         else
 104                 return &state->table[efx_filter_spec_table_id(spec)];
 105 }
 106
 107 static void efx_filter_table_reset_search_depth(struct efx_filter_table *table)
 108 {
 109         memset(table->search_depth, 0, sizeof(table->search_depth));
 110 }
 111
 112 static void efx_filter_push_rx_limits(struct efx_nic *efx)
 113 {
 114         struct efx_filter_state *state = efx->filter_state;
 115         struct efx_filter_table *table;
 116         efx_oword_t filter_ctl;
 117
 118         efx_reado(efx, &filter_ctl, FR_BZ_RX_FILTER_CTL);
 119
 120         table = &state->table[EFX_FILTER_TABLE_RX_IP];
 121         EFX_SET_OWORD_FIELD(filter_ctl, FRF_BZ_TCP_FULL_SRCH_LIMIT,
 122                             table->search_depth[EFX_FILTER_TCP_FULL] +
 123                             FILTER_CTL_SRCH_FUDGE_FULL);
 124         EFX_SET_OWORD_FIELD(filter_ctl, FRF_BZ_TCP_WILD_SRCH_LIMIT,
 125                             table->search_depth[EFX_FILTER_TCP_WILD] +
 126                             FILTER_CTL_SRCH_FUDGE_WILD);
 127         EFX_SET_OWORD_FIELD(filter_ctl, FRF_BZ_UDP_FULL_SRCH_LIMIT,
 128                             table->search_depth[EFX_FILTER_UDP_FULL] +
 129                             FILTER_CTL_SRCH_FUDGE_FULL);
 130         EFX_SET_OWORD_FIELD(filter_ctl, FRF_BZ_UDP_WILD_SRCH_LIMIT,
 131                             table->search_depth[EFX_FILTER_UDP_WILD] +
 132                             FILTER_CTL_SRCH_FUDGE_WILD);
 133
 134         table = &state->table[EFX_FILTER_TABLE_RX_MAC];
 135         if (table->size) {
 136                 EFX_SET_OWORD_FIELD(
 137                         filter_ctl, FRF_CZ_ETHERNET_FULL_SEARCH_LIMIT,
 138                         table->search_depth[EFX_FILTER_MAC_FULL] +
 139                         FILTER_CTL_SRCH_FUDGE_FULL);
 140                 EFX_SET_OWORD_FIELD(
 141                         filter_ctl, FRF_CZ_ETHERNET_WILDCARD_SEARCH_LIMIT,
 142                         table->search_depth[EFX_FILTER_MAC_WILD] +
 143                         FILTER_CTL_SRCH_FUDGE_WILD);
 144         }
 145
 146         efx_writeo(efx, &filter_ctl, FR_BZ_RX_FILTER_CTL);
 147 }
 148
 149 static inline void __efx_filter_set_ipv4(struct efx_filter_spec *spec,
 150                                          __be32 host1, __be16 port1,
 151                                          __be32 host2, __be16 port2)
 152 {
 153         spec->data[0] = ntohl(host1) << 16 | ntohs(port1);
 154         spec->data[1] = ntohs(port2) << 16 | ntohl(host1) >> 16;
 155         spec->data[2] = ntohl(host2);
 156 }
 157
 158 /**
 159  * efx_filter_set_ipv4_local - specify IPv4 host, transport protocol and port
 160  * @spec: Specification to initialise
 161  * @proto: Transport layer protocol number
 162  * @host: Local host address (network byte order)
 163  * @port: Local port (network byte order)
 164  */
 165 int efx_filter_set_ipv4_local(struct efx_filter_spec *spec, u8 proto,
 166                               __be32 host, __be16 port)
 167 {
 168         __be32 host1;
 169         __be16 port1;
 170
 171         EFX_BUG_ON_PARANOID(!(spec->flags & EFX_FILTER_FLAG_RX));
 172
 173         /* This cannot currently be combined with other filtering */
 174         if (spec->type != EFX_FILTER_UNSPEC)
 175                 return -EPROTONOSUPPORT;
 176
 177         if (port == 0)
 178                 return -EINVAL;
 179
 180         switch (proto) {
 181         case IPPROTO_TCP:
 182                 spec->type = EFX_FILTER_TCP_WILD;
 183                 break;
 184         case IPPROTO_UDP:
 185                 spec->type = EFX_FILTER_UDP_WILD;
 186                 break;
 187         default:
 188                 return -EPROTONOSUPPORT;
 189         }
 190
 191         /* Filter is constructed in terms of source and destination,
 192          * with the odd wrinkle that the ports are swapped in a UDP
 193          * wildcard filter.  We need to convert from local and remote
 194          * (= zero for wildcard) addresses.
 195          */
 196         host1 = 0;
 197         if (proto != IPPROTO_UDP) {
 198                 port1 = 0;
 199         } else {
 200                 port1 = port;
 201                 port = 0;
 202         }
 203
 204         __efx_filter_set_ipv4(spec, host1, port1, host, port);
 205         return 0;
 206 }
 207
 208 /**
 209  * efx_filter_set_ipv4_full - specify IPv4 hosts, transport protocol and ports
 210  * @spec: Specification to initialise
 211  * @proto: Transport layer protocol number
 212  * @host: Local host address (network byte order)
 213  * @port: Local port (network byte order)
 214  * @rhost: Remote host address (network byte order)
 215  * @rport: Remote port (network byte order)
 216  */
 217 int efx_filter_set_ipv4_full(struct efx_filter_spec *spec, u8 proto,
 218                              __be32 host, __be16 port,
 219                              __be32 rhost, __be16 rport)
 220 {
 221         EFX_BUG_ON_PARANOID(!(spec->flags & EFX_FILTER_FLAG_RX));
 222
 223         /* This cannot currently be combined with other filtering */
 224         if (spec->type != EFX_FILTER_UNSPEC)
 225                 return -EPROTONOSUPPORT;
 226
 227         if (port == 0 || rport == 0)
 228                 return -EINVAL;
 229
 230         switch (proto) {
 231         case IPPROTO_TCP:
 232                 spec->type = EFX_FILTER_TCP_FULL;
 233                 break;
 234         case IPPROTO_UDP:
 235                 spec->type = EFX_FILTER_UDP_FULL;
 236                 break;
 237         default:
 238                 return -EPROTONOSUPPORT;
 239         }
 240
 241         __efx_filter_set_ipv4(spec, rhost, rport, host, port);
 242         return 0;
 243 }
 244
 245 /**
 246  * efx_filter_set_eth_local - specify local Ethernet address and optional VID
 247  * @spec: Specification to initialise
 248  * @vid: VLAN ID to match, or %EFX_FILTER_VID_UNSPEC
 249  * @addr: Local Ethernet MAC address
 250  */
 251 int efx_filter_set_eth_local(struct efx_filter_spec *spec,
 252                              u16 vid, const u8 *addr)
 253 {
 254         EFX_BUG_ON_PARANOID(!(spec->flags & EFX_FILTER_FLAG_RX));
 255
 256         /* This cannot currently be combined with other filtering */
 257         if (spec->type != EFX_FILTER_UNSPEC)
 258                 return -EPROTONOSUPPORT;
 259
 260         if (vid == EFX_FILTER_VID_UNSPEC) {
 261                 spec->type = EFX_FILTER_MAC_WILD;
 262                 spec->data[0] = 0;
 263         } else {
 264                 spec->type = EFX_FILTER_MAC_FULL;
 265                 spec->data[0] = vid;
 266         }
 267
 268         spec->data[1] = addr[2] << 24 | addr[3] << 16 | addr[4] << 8 | addr[5];
 269         spec->data[2] = addr[0] << 8 | addr[1];
 270         return 0;
 271 }
 272
 273 /* Build a filter entry and return its n-tuple key. */
 274 static u32 efx_filter_build(efx_oword_t *filter, struct efx_filter_spec *spec)
 275 {
 276         u32 data3;
 277
 278         switch (efx_filter_spec_table_id(spec)) {
 279         case EFX_FILTER_TABLE_RX_IP: {
 280                 bool is_udp = (spec->type == EFX_FILTER_UDP_FULL ||
 281                                spec->type == EFX_FILTER_UDP_WILD);
 282                 EFX_POPULATE_OWORD_7(
 283                         *filter,
 284                         FRF_BZ_RSS_EN,
 285                         !!(spec->flags & EFX_FILTER_FLAG_RX_RSS),
 286                         FRF_BZ_SCATTER_EN,
 287                         !!(spec->flags & EFX_FILTER_FLAG_RX_SCATTER),
 288                         FRF_BZ_TCP_UDP, is_udp,
 289                         FRF_BZ_RXQ_ID, spec->dmaq_id,
 290                         EFX_DWORD_2, spec->data[2],
 291                         EFX_DWORD_1, spec->data[1],
 292                         EFX_DWORD_0, spec->data[0]);
 293                 data3 = is_udp;
 294                 break;
 295         }
 296
 297         case EFX_FILTER_TABLE_RX_MAC: {
 298                 bool is_wild = spec->type == EFX_FILTER_MAC_WILD;
 299                 EFX_POPULATE_OWORD_8(
 300                         *filter,
 301                         FRF_CZ_RMFT_RSS_EN,
 302                         !!(spec->flags & EFX_FILTER_FLAG_RX_RSS),
 303                         FRF_CZ_RMFT_SCATTER_EN,
 304                         !!(spec->flags & EFX_FILTER_FLAG_RX_SCATTER),
 305                         FRF_CZ_RMFT_IP_OVERRIDE,
 306                         !!(spec->flags & EFX_FILTER_FLAG_RX_OVERRIDE_IP),
 307                         FRF_CZ_RMFT_RXQ_ID, spec->dmaq_id,
 308                         FRF_CZ_RMFT_WILDCARD_MATCH, is_wild,
 309                         FRF_CZ_RMFT_DEST_MAC_HI, spec->data[2],
 310                         FRF_CZ_RMFT_DEST_MAC_LO, spec->data[1],
 311                         FRF_CZ_RMFT_VLAN_ID, spec->data[0]);
 312                 data3 = is_wild;
 313                 break;
 314         }
 315
 316         default:
 317                 BUG();
 318         }
 319
 320         return spec->data[0] ^ spec->data[1] ^ spec->data[2] ^ data3;
 321 }
 322
 323 static bool efx_filter_equal(const struct efx_filter_spec *left,
 324                              const struct efx_filter_spec *right)
 325 {
 326         if (left->type != right->type ||
 327             memcmp(left->data, right->data, sizeof(left->data)))
 328                 return false;
 329
 330         return true;
 331 }
 332
 333 static int efx_filter_search(struct efx_filter_table *table,
 334                              struct efx_filter_spec *spec, u32 key,
 335                              bool for_insert, int *depth_required)
 336 {
 337         unsigned hash, incr, filter_idx, depth, depth_max;
 338         struct efx_filter_spec *cmp;
 339
 340         hash = efx_filter_hash(key);
 341         incr = efx_filter_increment(key);
 342         depth_max = (spec->priority <= EFX_FILTER_PRI_HINT ?
 343                      FILTER_CTL_SRCH_HINT_MAX : FILTER_CTL_SRCH_MAX);
 344
 345         for (depth = 1, filter_idx = hash & (table->size - 1);
 346              depth <= depth_max && test_bit(filter_idx, table->used_bitmap);
 347              ++depth) {
 348                 cmp = &table->spec[filter_idx];
 349                 if (efx_filter_equal(spec, cmp))
 350                         goto found;
 351                 filter_idx = (filter_idx + incr) & (table->size - 1);
 352         }
 353         if (!for_insert)
 354                 return -ENOENT;
 355         if (depth > depth_max)
 356                 return -EBUSY;
 357 found:
 358         *depth_required = depth;
 359         return filter_idx;
 360 }
 361
 362 /* Construct/deconstruct external filter IDs */
 363
 364 static inline int
 365 efx_filter_make_id(enum efx_filter_table_id table_id, unsigned index)
 366 {
 367         return table_id << 16 | index;
 368 }
 369
 370 /**
 371  * efx_filter_insert_filter - add or replace a filter
 372  * @efx: NIC in which to insert the filter
 373  * @spec: Specification for the filter
 374  * @replace: Flag for whether the specified filter may replace a filter
 375  *      with an identical match expression and equal or lower priority
 376  *
 377  * On success, return the filter ID.
 378  * On failure, return a negative error code.
 379  */
 380 int efx_filter_insert_filter(struct efx_nic *efx, struct efx_filter_spec *spec,
 381                              bool replace)
 382 {
 383         struct efx_filter_state *state = efx->filter_state;
 384         struct efx_filter_table *table = efx_filter_spec_table(state, spec);
 385         struct efx_filter_spec *saved_spec;
 386         efx_oword_t filter;
 387         int filter_idx, depth;
 388         u32 key;
 389         int rc;
 390
 391         if (!table || table->size == 0)
 392                 return -EINVAL;
 393
 394         key = efx_filter_build(&filter, spec);
 395
 396         netif_vdbg(efx, hw, efx->net_dev,
 397                    "%s: type %d search_depth=%d", __func__, spec->type,
 398                    table->search_depth[spec->type]);
 399
 400         spin_lock_bh(&state->lock);
 401
 402         rc = efx_filter_search(table, spec, key, true, &depth);
 403         if (rc < 0)
 404                 goto out;
 405         filter_idx = rc;
 406         BUG_ON(filter_idx >= table->size);
 407         saved_spec = &table->spec[filter_idx];
 408
 409         if (test_bit(filter_idx, table->used_bitmap)) {
 410                 /* Should we replace the existing filter? */
 411                 if (!replace) {
 412                         rc = -EEXIST;
 413                         goto out;
 414                 }
 415                 if (spec->priority < saved_spec->priority) {
 416                         rc = -EPERM;
 417                         goto out;
 418                 }
 419         } else {
 420                 __set_bit(filter_idx, table->used_bitmap);
 421                 ++table->used;
 422         }
 423         *saved_spec = *spec;
 424
 425         if (table->search_depth[spec->type] < depth) {
 426                 table->search_depth[spec->type] = depth;
 427                 efx_filter_push_rx_limits(efx);
 428         }
 429
 430         efx_writeo(efx, &filter, table->offset + table->step * filter_idx);
 431
 432         netif_vdbg(efx, hw, efx->net_dev,
 433                    "%s: filter type %d index %d rxq %u set",
 434                    __func__, spec->type, filter_idx, spec->dmaq_id);
 435         rc = efx_filter_make_id(table->id, filter_idx);
 436
 437 out:
 438         spin_unlock_bh(&state->lock);
 439         return rc;
 440 }
 441
 442 static void efx_filter_table_clear_entry(struct efx_nic *efx,
 443                                          struct efx_filter_table *table,
 444                                          int filter_idx)
 445 {
 446         static efx_oword_t filter;
 447
 448         if (test_bit(filter_idx, table->used_bitmap)) {
 449                 __clear_bit(filter_idx, table->used_bitmap);
 450                 --table->used;
 451                 memset(&table->spec[filter_idx], 0, sizeof(table->spec[0]));
 452
 453                 efx_writeo(efx, &filter,
 454                            table->offset + table->step * filter_idx);
 455         }
 456 }
 457
 458 /**
 459  * efx_filter_remove_filter - remove a filter by specification
 460  * @efx: NIC from which to remove the filter
 461  * @spec: Specification for the filter
 462  *
 463  * On success, return zero.
 464  * On failure, return a negative error code.
 465  */
 466 int efx_filter_remove_filter(struct efx_nic *efx, struct efx_filter_spec *spec)
 467 {
 468         struct efx_filter_state *state = efx->filter_state;
 469         struct efx_filter_table *table = efx_filter_spec_table(state, spec);
 470         struct efx_filter_spec *saved_spec;
 471         efx_oword_t filter;
 472         int filter_idx, depth;
 473         u32 key;
 474         int rc;
 475
 476         if (!table)
 477                 return -EINVAL;
 478
 479         key = efx_filter_build(&filter, spec);
 480
 481         spin_lock_bh(&state->lock);
 482
 483         rc = efx_filter_search(table, spec, key, false, &depth);
 484         if (rc < 0)
 485                 goto out;
 486         filter_idx = rc;
 487         saved_spec = &table->spec[filter_idx];
 488
 489         if (spec->priority < saved_spec->priority) {
 490                 rc = -EPERM;
 491                 goto out;
 492         }
 493
 494         efx_filter_table_clear_entry(efx, table, filter_idx);
 495         if (table->used == 0)
 496                 efx_filter_table_reset_search_depth(table);
 497         rc = 0;
 498
 499 out:
 500         spin_unlock_bh(&state->lock);
 501         return rc;
 502 }
 503
 504 static void efx_filter_table_clear(struct efx_nic *efx,
 505                                    enum efx_filter_table_id table_id,
 506                                    enum efx_filter_priority priority)
 507 {
 508         struct efx_filter_state *state = efx->filter_state;
 509         struct efx_filter_table *table = &state->table[table_id];
 510         int filter_idx;
 511
 512         spin_lock_bh(&state->lock);
 513
 514         for (filter_idx = 0; filter_idx < table->size; ++filter_idx)
 515                 if (table->spec[filter_idx].priority <= priority)
 516                         efx_filter_table_clear_entry(efx, table, filter_idx);
 517         if (table->used == 0)
 518                 efx_filter_table_reset_search_depth(table);
 519
 520         spin_unlock_bh(&state->lock);
 521 }
 522
 523 /**
 524  * efx_filter_clear_rx - remove RX filters by priority
 525  * @efx: NIC from which to remove the filters
 526  * @priority: Maximum priority to remove
 527  */
 528 void efx_filter_clear_rx(struct efx_nic *efx, enum efx_filter_priority priority)
 529 {
 530         efx_filter_table_clear(efx, EFX_FILTER_TABLE_RX_IP, priority);
 531         efx_filter_table_clear(efx, EFX_FILTER_TABLE_RX_MAC, priority);
 532 }
 533
 534 /* Restore filter stater after reset */
 535 void efx_restore_filters(struct efx_nic *efx)
 536 {
 537         struct efx_filter_state *state = efx->filter_state;
 538         enum efx_filter_table_id table_id;
 539         struct efx_filter_table *table;
 540         efx_oword_t filter;
 541         int filter_idx;
 542
 543         spin_lock_bh(&state->lock);
 544
 545         for (table_id = 0; table_id < EFX_FILTER_TABLE_COUNT; table_id++) {
 546                 table = &state->table[table_id];
 547                 for (filter_idx = 0; filter_idx < table->size; filter_idx++) {
 548                         if (!test_bit(filter_idx, table->used_bitmap))
 549                                 continue;
 550                         efx_filter_build(&filter, &table->spec[filter_idx]);
 551                         efx_writeo(efx, &filter,
 552                                    table->offset + table->step * filter_idx);
 553                 }
 554         }
 555
 556         efx_filter_push_rx_limits(efx);
 557
 558         spin_unlock_bh(&state->lock);
 559 }
 560
 561 int efx_probe_filters(struct efx_nic *efx)
 562 {
 563         struct efx_filter_state *state;
 564         struct efx_filter_table *table;
 565         unsigned table_id;
 566
 567         state = kzalloc(sizeof(*efx->filter_state), GFP_KERNEL);
 568         if (!state)
 569                 return -ENOMEM;
 570         efx->filter_state = state;
 571
 572         spin_lock_init(&state->lock);
 573
 574         if (efx_nic_rev(efx) >= EFX_REV_FALCON_B0) {
 575 #ifdef CONFIG_RFS_ACCEL
 576                 state->rps_flow_id = kcalloc(FR_BZ_RX_FILTER_TBL0_ROWS,
 577                                              sizeof(*state->rps_flow_id),
 578                                              GFP_KERNEL);
 579                 if (!state->rps_flow_id)
 580                         goto fail;
 581 #endif
 582                 table = &state->table[EFX_FILTER_TABLE_RX_IP];
 583                 table->id = EFX_FILTER_TABLE_RX_IP;
 584                 table->offset = FR_BZ_RX_FILTER_TBL0;
 585                 table->size = FR_BZ_RX_FILTER_TBL0_ROWS;
 586                 table->step = FR_BZ_RX_FILTER_TBL0_STEP;
 587         }
 588
 589         if (efx_nic_rev(efx) >= EFX_REV_SIENA_A0) {
 590                 table = &state->table[EFX_FILTER_TABLE_RX_MAC];
 591                 table->id = EFX_FILTER_TABLE_RX_MAC;
 592                 table->offset = FR_CZ_RX_MAC_FILTER_TBL0;
 593                 table->size = FR_CZ_RX_MAC_FILTER_TBL0_ROWS;
 594                 table->step = FR_CZ_RX_MAC_FILTER_TBL0_STEP;
 595         }
 596
 597         for (table_id = 0; table_id < EFX_FILTER_TABLE_COUNT; table_id++) {
 598                 table = &state->table[table_id];
 599                 if (table->size == 0)
 600                         continue;
 601                 table->used_bitmap = kcalloc(BITS_TO_LONGS(table->size),
 602                                              sizeof(unsigned long),
 603                                              GFP_KERNEL);
 604                 if (!table->used_bitmap)
 605                         goto fail;
 606                 table->spec = vzalloc(table->size * sizeof(*table->spec));
 607                 if (!table->spec)
 608                         goto fail;
 609         }
 610
 611         return 0;
 612
 613 fail:
 614         efx_remove_filters(efx);
 615         return -ENOMEM;
 616 }
 617
 618 void efx_remove_filters(struct efx_nic *efx)
 619 {
 620         struct efx_filter_state *state = efx->filter_state;
 621         enum efx_filter_table_id table_id;
 622
 623         for (table_id = 0; table_id < EFX_FILTER_TABLE_COUNT; table_id++) {
 624                 kfree(state->table[table_id].used_bitmap);
 625                 vfree(state->table[table_id].spec);
 626         }
 627 #ifdef CONFIG_RFS_ACCEL
 628         kfree(state->rps_flow_id);
 629 #endif
 630         kfree(state);
 631 }
 632
 633 #ifdef CONFIG_RFS_ACCEL
 634
 635 int efx_filter_rfs(struct net_device *net_dev, const struct sk_buff *skb,
 636                    u16 rxq_index, u32 flow_id)
 637 {
 638         struct efx_nic *efx = netdev_priv(net_dev);
 639         struct efx_channel *channel;
 640         struct efx_filter_state *state = efx->filter_state;
 641         struct efx_filter_spec spec;
 642         const struct iphdr *ip;
 643         const __be16 *ports;
 644         int nhoff;
 645         int rc;
 646
 647         nhoff = skb_network_offset(skb);
 648
 649         if (skb->protocol != htons(ETH_P_IP))
 650                 return -EPROTONOSUPPORT;
 651
 652         /* RFS must validate the IP header length before calling us */
 653         EFX_BUG_ON_PARANOID(!pskb_may_pull(skb, nhoff + sizeof(*ip)));
 654         ip = (const struct iphdr *)(skb->data + nhoff);
 655         if (ip->frag_off & htons(IP_MF | IP_OFFSET))
 656                 return -EPROTONOSUPPORT;
 657         EFX_BUG_ON_PARANOID(!pskb_may_pull(skb, nhoff + 4 * ip->ihl + 4));
 658         ports = (const __be16 *)(skb->data + nhoff + 4 * ip->ihl);
 659
 660         efx_filter_init_rx(&spec, EFX_FILTER_PRI_HINT, 0, rxq_index);
 661         rc = efx_filter_set_ipv4_full(&spec, ip->protocol,
 662                                       ip->daddr, ports[1], ip->saddr, ports[0]);
 663         if (rc)
 664                 return rc;
 665
 666         rc = efx_filter_insert_filter(efx, &spec, true);
 667         if (rc < 0)
 668                 return rc;
 669
 670         /* Remember this so we can check whether to expire the filter later */
 671         state->rps_flow_id[rc] = flow_id;
 672         channel = efx_get_channel(efx, skb_get_rx_queue(skb));
 673         ++channel->rfs_filters_added;
 674
 675         netif_info(efx, rx_status, efx->net_dev,
 676                    "steering %s %pI4:%u:%pI4:%u to queue %u [flow %u filter %d]\n",
 677                    (ip->protocol == IPPROTO_TCP) ? "TCP" : "UDP",
 678                    &ip->saddr, ntohs(ports[0]), &ip->daddr, ntohs(ports[1]),
 679                    rxq_index, flow_id, rc);
 680
 681         return rc;
 682 }
 683
 684 bool __efx_filter_rfs_expire(struct efx_nic *efx, unsigned quota)
 685 {
 686         struct efx_filter_state *state = efx->filter_state;
 687         struct efx_filter_table *table = &state->table[EFX_FILTER_TABLE_RX_IP];
 688         unsigned mask = table->size - 1;
 689         unsigned index;
 690         unsigned stop;
 691
 692         if (!spin_trylock_bh(&state->lock))
 693                 return false;
 694
 695         index = state->rps_expire_index;
 696         stop = (index + quota) & mask;
 697
 698         while (index != stop) {
 699                 if (test_bit(index, table->used_bitmap) &&
 700                     table->spec[index].priority == EFX_FILTER_PRI_HINT &&
 701                     rps_may_expire_flow(efx->net_dev,
 702                                         table->spec[index].dmaq_id,
 703                                         state->rps_flow_id[index], index)) {
 704                         netif_info(efx, rx_status, efx->net_dev,
 705                                    "expiring filter %d [flow %u]\n",
 706                                    index, state->rps_flow_id[index]);
 707                         efx_filter_table_clear_entry(efx, table, index);
 708                 }
 709                 index = (index + 1) & mask;
 710         }
 711
 712         state->rps_expire_index = stop;
 713         if (table->used == 0)
 714                 efx_filter_table_reset_search_depth(table);
 715
 716         spin_unlock_bh(&state->lock);
 717         return true;
 718 }
 719
 720 #endif /* CONFIG_RFS_ACCEL */