drivers/net/sfc/filter.c

   1 /****************************************************************************
   2  * Driver for Solarflare Solarstorm network controllers and boards
   3  * Copyright 2005-2010 Solarflare Communications Inc.
   4  *
   5  * This program is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 as published
   7  * by the Free Software Foundation, incorporated herein by reference.
   8  */
   9
  10 #include <linux/in.h>
  11 #include <net/ip.h>
  12 #include "efx.h"
  13 #include "filter.h"
  14 #include "io.h"
  15 #include "nic.h"
  16 #include "regs.h"
  17
  18 /* "Fudge factors" - difference between programmed value and actual depth.
  19  * Due to pipelined implementation we need to program H/W with a value that
  20  * is larger than the hop limit we want.
  21  */
  22 #define FILTER_CTL_SRCH_FUDGE_WILD 3
  23 #define FILTER_CTL_SRCH_FUDGE_FULL 1
  24
  25 /* Hard maximum hop limit.  Hardware will time-out beyond 200-something.
  26  * We also need to avoid infinite loops in efx_filter_search() when the
  27  * table is full.
  28  */
  29 #define FILTER_CTL_SRCH_MAX 200
  30
  31 /* Don't try very hard to find space for performance hints, as this is
  32  * counter-productive. */
  33 #define FILTER_CTL_SRCH_HINT_MAX 5
  34
  35 enum efx_filter_table_id {
  36         EFX_FILTER_TABLE_RX_IP = 0,
  37         EFX_FILTER_TABLE_RX_MAC,
  38         EFX_FILTER_TABLE_COUNT,
  39 };
  40
  41 struct efx_filter_table {
  42         enum efx_filter_table_id id;
  43         u32             offset;         /* address of table relative to BAR */
  44         unsigned        size;           /* number of entries */
  45         unsigned        step;           /* step between entries */
  46         unsigned        used;           /* number currently used */
  47         unsigned long   *used_bitmap;
  48         struct efx_filter_spec *spec;
  49         unsigned        search_depth[EFX_FILTER_TYPE_COUNT];
  50 };
  51
  52 struct efx_filter_state {
  53         spinlock_t      lock;
  54         struct efx_filter_table table[EFX_FILTER_TABLE_COUNT];
  55 #ifdef CONFIG_RFS_ACCEL
  56         u32             *rps_flow_id;
  57         unsigned        rps_expire_index;
  58 #endif
  59 };
  60
  61 /* The filter hash function is LFSR polynomial x^16 + x^3 + 1 of a 32-bit
  62  * key derived from the n-tuple.  The initial LFSR state is 0xffff. */
  63 static u16 efx_filter_hash(u32 key)
  64 {
  65         u16 tmp;
  66
  67         /* First 16 rounds */
  68         tmp = 0x1fff ^ key >> 16;
  69         tmp = tmp ^ tmp >> 3 ^ tmp >> 6;
  70         tmp = tmp ^ tmp >> 9;
  71         /* Last 16 rounds */
  72         tmp = tmp ^ tmp << 13 ^ key;
  73         tmp = tmp ^ tmp >> 3 ^ tmp >> 6;
  74         return tmp ^ tmp >> 9;
  75 }
  76
  77 /* To allow for hash collisions, filter search continues at these
  78  * increments from the first possible entry selected by the hash. */
  79 static u16 efx_filter_increment(u32 key)
  80 {
  81         return key * 2 - 1;
  82 }
  83
  84 static enum efx_filter_table_id
  85 efx_filter_spec_table_id(const struct efx_filter_spec *spec)
  86 {
  87         BUILD_BUG_ON(EFX_FILTER_TABLE_RX_IP != (EFX_FILTER_TCP_FULL >> 2));
  88         BUILD_BUG_ON(EFX_FILTER_TABLE_RX_IP != (EFX_FILTER_TCP_WILD >> 2));
  89         BUILD_BUG_ON(EFX_FILTER_TABLE_RX_IP != (EFX_FILTER_UDP_FULL >> 2));
  90         BUILD_BUG_ON(EFX_FILTER_TABLE_RX_IP != (EFX_FILTER_UDP_WILD >> 2));
  91         BUILD_BUG_ON(EFX_FILTER_TABLE_RX_MAC != (EFX_FILTER_MAC_FULL >> 2));
  92         BUILD_BUG_ON(EFX_FILTER_TABLE_RX_MAC != (EFX_FILTER_MAC_WILD >> 2));
  93         EFX_BUG_ON_PARANOID(spec->type == EFX_FILTER_UNSPEC);
  94         return spec->type >> 2;
  95 }
  96
  97 static struct efx_filter_table *
  98 efx_filter_spec_table(struct efx_filter_state *state,
  99                       const struct efx_filter_spec *spec)
 100 {
 101         if (spec->type == EFX_FILTER_UNSPEC)
 102                 return NULL;
 103         else
 104                 return &state->table[efx_filter_spec_table_id(spec)];
 105 }
 106
 107 static void efx_filter_table_reset_search_depth(struct efx_filter_table *table)
 108 {
 109         memset(table->search_depth, 0, sizeof(table->search_depth));
 110 }
 111
 112 static void efx_filter_push_rx_limits(struct efx_nic *efx)
 113 {
 114         struct efx_filter_state *state = efx->filter_state;
 115         struct efx_filter_table *table;
 116         efx_oword_t filter_ctl;
 117
 118         efx_reado(efx, &filter_ctl, FR_BZ_RX_FILTER_CTL);
 119
 120         table = &state->table[EFX_FILTER_TABLE_RX_IP];
 121         EFX_SET_OWORD_FIELD(filter_ctl, FRF_BZ_TCP_FULL_SRCH_LIMIT,
 122                             table->search_depth[EFX_FILTER_TCP_FULL] +
 123                             FILTER_CTL_SRCH_FUDGE_FULL);
 124         EFX_SET_OWORD_FIELD(filter_ctl, FRF_BZ_TCP_WILD_SRCH_LIMIT,
 125                             table->search_depth[EFX_FILTER_TCP_WILD] +
 126                             FILTER_CTL_SRCH_FUDGE_WILD);
 127         EFX_SET_OWORD_FIELD(filter_ctl, FRF_BZ_UDP_FULL_SRCH_LIMIT,
 128                             table->search_depth[EFX_FILTER_UDP_FULL] +
 129                             FILTER_CTL_SRCH_FUDGE_FULL);
 130         EFX_SET_OWORD_FIELD(filter_ctl, FRF_BZ_UDP_WILD_SRCH_LIMIT,
 131                             table->search_depth[EFX_FILTER_UDP_WILD] +
 132                             FILTER_CTL_SRCH_FUDGE_WILD);
 133
 134         table = &state->table[EFX_FILTER_TABLE_RX_MAC];
 135         if (table->size) {
 136                 EFX_SET_OWORD_FIELD(
 137                         filter_ctl, FRF_CZ_ETHERNET_FULL_SEARCH_LIMIT,
 138                         table->search_depth[EFX_FILTER_MAC_FULL] +
 139                         FILTER_CTL_SRCH_FUDGE_FULL);
 140                 EFX_SET_OWORD_FIELD(
 141                         filter_ctl, FRF_CZ_ETHERNET_WILDCARD_SEARCH_LIMIT,
 142                         table->search_depth[EFX_FILTER_MAC_WILD] +
 143                         FILTER_CTL_SRCH_FUDGE_WILD);
 144         }
 145
 146         efx_writeo(efx, &filter_ctl, FR_BZ_RX_FILTER_CTL);
 147 }
 148
 149 static inline void __efx_filter_set_ipv4(struct efx_filter_spec *spec,
 150                                          __be32 host1, __be16 port1,
 151                                          __be32 host2, __be16 port2)
 152 {
 153         spec->data[0] = ntohl(host1) << 16 | ntohs(port1);
 154         spec->data[1] = ntohs(port2) << 16 | ntohl(host1) >> 16;
 155         spec->data[2] = ntohl(host2);
 156 }
 157
 158 /**
 159  * efx_filter_set_ipv4_local - specify IPv4 host, transport protocol and port
 160  * @spec: Specification to initialise
 161  * @proto: Transport layer protocol number
 162  * @host: Local host address (network byte order)
 163  * @port: Local port (network byte order)
 164  */
 165 int efx_filter_set_ipv4_local(struct efx_filter_spec *spec, u8 proto,
 166                               __be32 host, __be16 port)
 167 {
 168         __be32 host1;
 169         __be16 port1;
 170
 171         EFX_BUG_ON_PARANOID(!(spec->flags & EFX_FILTER_FLAG_RX));
 172
 173         /* This cannot currently be combined with other filtering */
 174         if (spec->type != EFX_FILTER_UNSPEC)
 175                 return -EPROTONOSUPPORT;
 176
 177         if (port == 0)
 178                 return -EINVAL;
 179
 180         switch (proto) {
 181         case IPPROTO_TCP:
 182                 spec->type = EFX_FILTER_TCP_WILD;
 183                 break;
 184         case IPPROTO_UDP:
 185                 spec->type = EFX_FILTER_UDP_WILD;
 186                 break;
 187         default:
 188                 return -EPROTONOSUPPORT;
 189         }
 190
 191         /* Filter is constructed in terms of source and destination,
 192          * with the odd wrinkle that the ports are swapped in a UDP
 193          * wildcard filter.  We need to convert from local and remote
 194          * (= zero for wildcard) addresses.
 195          */
 196         host1 = 0;
 197         if (proto != IPPROTO_UDP) {
 198                 port1 = 0;
 199         } else {
 200                 port1 = port;
 201                 port = 0;
 202         }
 203
 204         __efx_filter_set_ipv4(spec, host1, port1, host, port);
 205         return 0;
 206 }
 207
 208 /**
 209  * efx_filter_set_ipv4_full - specify IPv4 hosts, transport protocol and ports
 210  * @spec: Specification to initialise
 211  * @proto: Transport layer protocol number
 212  * @host: Local host address (network byte order)
 213  * @port: Local port (network byte order)
 214  * @rhost: Remote host address (network byte order)
 215  * @rport: Remote port (network byte order)
 216  */
 217 int efx_filter_set_ipv4_full(struct efx_filter_spec *spec, u8 proto,
 218                              __be32 host, __be16 port,
 219                              __be32 rhost, __be16 rport)
 220 {
 221         EFX_BUG_ON_PARANOID(!(spec->flags & EFX_FILTER_FLAG_RX));
 222
 223         /* This cannot currently be combined with other filtering */
 224         if (spec->type != EFX_FILTER_UNSPEC)
 225                 return -EPROTONOSUPPORT;
 226
 227         if (port == 0 || rport == 0)
 228                 return -EINVAL;
 229
 230         switch (proto) {
 231         case IPPROTO_TCP:
 232                 spec->type = EFX_FILTER_TCP_FULL;
 233                 break;
 234         case IPPROTO_UDP:
 235                 spec->type = EFX_FILTER_UDP_FULL;
 236                 break;
 237         default:
 238                 return -EPROTONOSUPPORT;
 239         }
 240
 241         __efx_filter_set_ipv4(spec, rhost, rport, host, port);
 242         return 0;
 243 }
 244
 245 /**
 246  * efx_filter_set_eth_local - specify local Ethernet address and optional VID
 247  * @spec: Specification to initialise
 248  * @vid: VLAN ID to match, or %EFX_FILTER_VID_UNSPEC
 249  * @addr: Local Ethernet MAC address
 250  */
 251 int efx_filter_set_eth_local(struct efx_filter_spec *spec,
 252                              u16 vid, const u8 *addr)
 253 {
 254         EFX_BUG_ON_PARANOID(!(spec->flags & EFX_FILTER_FLAG_RX));
 255
 256         /* This cannot currently be combined with other filtering */
 257         if (spec->type != EFX_FILTER_UNSPEC)
 258                 return -EPROTONOSUPPORT;
 259
 260         if (vid == EFX_FILTER_VID_UNSPEC) {
 261                 spec->type = EFX_FILTER_MAC_WILD;
 262                 spec->data[0] = 0;
 263         } else {
 264                 spec->type = EFX_FILTER_MAC_FULL;
 265                 spec->data[0] = vid;
 266         }
 267
 268         spec->data[1] = addr[2] << 24 | addr[3] << 16 | addr[4] << 8 | addr[5];
 269         spec->data[2] = addr[0] << 8 | addr[1];
 270         return 0;
 271 }
 272
 273 /* Build a filter entry and return its n-tuple key. */
 274 static u32 efx_filter_build(efx_oword_t *filter, struct efx_filter_spec *spec)
 275 {
 276         u32 data3;
 277
 278         switch (efx_filter_spec_table_id(spec)) {
 279         case EFX_FILTER_TABLE_RX_IP: {
 280                 bool is_udp = (spec->type == EFX_FILTER_UDP_FULL ||
 281                                spec->type == EFX_FILTER_UDP_WILD);
 282                 EFX_POPULATE_OWORD_7(
 283                         *filter,
 284                         FRF_BZ_RSS_EN,
 285                         !!(spec->flags & EFX_FILTER_FLAG_RX_RSS),
 286                         FRF_BZ_SCATTER_EN,
 287                         !!(spec->flags & EFX_FILTER_FLAG_RX_SCATTER),
 288                         FRF_BZ_TCP_UDP, is_udp,
 289                         FRF_BZ_RXQ_ID, spec->dmaq_id,
 290                         EFX_DWORD_2, spec->data[2],
 291                         EFX_DWORD_1, spec->data[1],
 292                         EFX_DWORD_0, spec->data[0]);
 293                 data3 = is_udp;
 294                 break;
 295         }
 296
 297         case EFX_FILTER_TABLE_RX_MAC: {
 298                 bool is_wild = spec->type == EFX_FILTER_MAC_WILD;
 299                 EFX_POPULATE_OWORD_8(
 300                         *filter,
 301                         FRF_CZ_RMFT_RSS_EN,
 302                         !!(spec->flags & EFX_FILTER_FLAG_RX_RSS),
 303                         FRF_CZ_RMFT_SCATTER_EN,
 304                         !!(spec->flags & EFX_FILTER_FLAG_RX_SCATTER),
 305                         FRF_CZ_RMFT_IP_OVERRIDE,
 306                         !!(spec->flags & EFX_FILTER_FLAG_RX_OVERRIDE_IP),
 307                         FRF_CZ_RMFT_RXQ_ID, spec->dmaq_id,
 308                         FRF_CZ_RMFT_WILDCARD_MATCH, is_wild,
 309                         FRF_CZ_RMFT_DEST_MAC_HI, spec->data[2],
 310                         FRF_CZ_RMFT_DEST_MAC_LO, spec->data[1],
 311                         FRF_CZ_RMFT_VLAN_ID, spec->data[0]);
 312                 data3 = is_wild;
 313                 break;
 314         }
 315
 316         default:
 317                 BUG();
 318         }
 319
 320         return spec->data[0] ^ spec->data[1] ^ spec->data[2] ^ data3;
 321 }
 322
 323 static bool efx_filter_equal(const struct efx_filter_spec *left,
 324                              const struct efx_filter_spec *right)
 325 {
 326         if (left->type != right->type ||
 327             memcmp(left->data, right->data, sizeof(left->data)))
 328                 return false;
 329
 330         return true;
 331 }
 332
 333 static int efx_filter_search(struct efx_filter_table *table,
 334                              struct efx_filter_spec *spec, u32 key,
 335                              bool for_insert, int *depth_required)
 336 {
 337         unsigned hash, incr, filter_idx, depth, depth_max;
 338
 339         hash = efx_filter_hash(key);
 340         incr = efx_filter_increment(key);
 341
 342         filter_idx = hash & (table->size - 1);
 343         depth = 1;
 344         depth_max = (for_insert ?
 345                      (spec->priority <= EFX_FILTER_PRI_HINT ?
 346                       FILTER_CTL_SRCH_HINT_MAX : FILTER_CTL_SRCH_MAX) :
 347                      table->search_depth[spec->type]);
 348
 349         for (;;) {
 350                 /* Return success if entry is used and matches this spec
 351                  * or entry is unused and we are trying to insert.
 352                  */
 353                 if (test_bit(filter_idx, table->used_bitmap) ?
 354                     efx_filter_equal(spec, &table->spec[filter_idx]) :
 355                     for_insert) {
 356                         *depth_required = depth;
 357                         return filter_idx;
 358                 }
 359
 360                 /* Return failure if we reached the maximum search depth */
 361                 if (depth == depth_max)
 362                         return for_insert ? -EBUSY : -ENOENT;
 363
 364                 filter_idx = (filter_idx + incr) & (table->size - 1);
 365                 ++depth;
 366         }
 367 }
 368
 369 /* Construct/deconstruct external filter IDs */
 370
 371 static inline int
 372 efx_filter_make_id(enum efx_filter_table_id table_id, unsigned index)
 373 {
 374         return table_id << 16 | index;
 375 }
 376
 377 /**
 378  * efx_filter_insert_filter - add or replace a filter
 379  * @efx: NIC in which to insert the filter
 380  * @spec: Specification for the filter
 381  * @replace: Flag for whether the specified filter may replace a filter
 382  *      with an identical match expression and equal or lower priority
 383  *
 384  * On success, return the filter ID.
 385  * On failure, return a negative error code.
 386  */
 387 int efx_filter_insert_filter(struct efx_nic *efx, struct efx_filter_spec *spec,
 388                              bool replace)
 389 {
 390         struct efx_filter_state *state = efx->filter_state;
 391         struct efx_filter_table *table = efx_filter_spec_table(state, spec);
 392         struct efx_filter_spec *saved_spec;
 393         efx_oword_t filter;
 394         int filter_idx, depth;
 395         u32 key;
 396         int rc;
 397
 398         if (!table || table->size == 0)
 399                 return -EINVAL;
 400
 401         key = efx_filter_build(&filter, spec);
 402
 403         netif_vdbg(efx, hw, efx->net_dev,
 404                    "%s: type %d search_depth=%d", __func__, spec->type,
 405                    table->search_depth[spec->type]);
 406
 407         spin_lock_bh(&state->lock);
 408
 409         rc = efx_filter_search(table, spec, key, true, &depth);
 410         if (rc < 0)
 411                 goto out;
 412         filter_idx = rc;
 413         BUG_ON(filter_idx >= table->size);
 414         saved_spec = &table->spec[filter_idx];
 415
 416         if (test_bit(filter_idx, table->used_bitmap)) {
 417                 /* Should we replace the existing filter? */
 418                 if (!replace) {
 419                         rc = -EEXIST;
 420                         goto out;
 421                 }
 422                 if (spec->priority < saved_spec->priority) {
 423                         rc = -EPERM;
 424                         goto out;
 425                 }
 426         } else {
 427                 __set_bit(filter_idx, table->used_bitmap);
 428                 ++table->used;
 429         }
 430         *saved_spec = *spec;
 431
 432         if (table->search_depth[spec->type] < depth) {
 433                 table->search_depth[spec->type] = depth;
 434                 efx_filter_push_rx_limits(efx);
 435         }
 436
 437         efx_writeo(efx, &filter, table->offset + table->step * filter_idx);
 438
 439         netif_vdbg(efx, hw, efx->net_dev,
 440                    "%s: filter type %d index %d rxq %u set",
 441                    __func__, spec->type, filter_idx, spec->dmaq_id);
 442         rc = efx_filter_make_id(table->id, filter_idx);
 443
 444 out:
 445         spin_unlock_bh(&state->lock);
 446         return rc;
 447 }
 448
 449 static void efx_filter_table_clear_entry(struct efx_nic *efx,
 450                                          struct efx_filter_table *table,
 451                                          int filter_idx)
 452 {
 453         static efx_oword_t filter;
 454
 455         if (test_bit(filter_idx, table->used_bitmap)) {
 456                 __clear_bit(filter_idx, table->used_bitmap);
 457                 --table->used;
 458                 memset(&table->spec[filter_idx], 0, sizeof(table->spec[0]));
 459
 460                 efx_writeo(efx, &filter,
 461                            table->offset + table->step * filter_idx);
 462         }
 463 }
 464
 465 /**
 466  * efx_filter_remove_filter - remove a filter by specification
 467  * @efx: NIC from which to remove the filter
 468  * @spec: Specification for the filter
 469  *
 470  * On success, return zero.
 471  * On failure, return a negative error code.
 472  */
 473 int efx_filter_remove_filter(struct efx_nic *efx, struct efx_filter_spec *spec)
 474 {
 475         struct efx_filter_state *state = efx->filter_state;
 476         struct efx_filter_table *table = efx_filter_spec_table(state, spec);
 477         struct efx_filter_spec *saved_spec;
 478         efx_oword_t filter;
 479         int filter_idx, depth;
 480         u32 key;
 481         int rc;
 482
 483         if (!table)
 484                 return -EINVAL;
 485
 486         key = efx_filter_build(&filter, spec);
 487
 488         spin_lock_bh(&state->lock);
 489
 490         rc = efx_filter_search(table, spec, key, false, &depth);
 491         if (rc < 0)
 492                 goto out;
 493         filter_idx = rc;
 494         saved_spec = &table->spec[filter_idx];
 495
 496         if (spec->priority < saved_spec->priority) {
 497                 rc = -EPERM;
 498                 goto out;
 499         }
 500
 501         efx_filter_table_clear_entry(efx, table, filter_idx);
 502         if (table->used == 0)
 503                 efx_filter_table_reset_search_depth(table);
 504         rc = 0;
 505
 506 out:
 507         spin_unlock_bh(&state->lock);
 508         return rc;
 509 }
 510
 511 static void efx_filter_table_clear(struct efx_nic *efx,
 512                                    enum efx_filter_table_id table_id,
 513                                    enum efx_filter_priority priority)
 514 {
 515         struct efx_filter_state *state = efx->filter_state;
 516         struct efx_filter_table *table = &state->table[table_id];
 517         int filter_idx;
 518
 519         spin_lock_bh(&state->lock);
 520
 521         for (filter_idx = 0; filter_idx < table->size; ++filter_idx)
 522                 if (table->spec[filter_idx].priority <= priority)
 523                         efx_filter_table_clear_entry(efx, table, filter_idx);
 524         if (table->used == 0)
 525                 efx_filter_table_reset_search_depth(table);
 526
 527         spin_unlock_bh(&state->lock);
 528 }
 529
 530 /**
 531  * efx_filter_clear_rx - remove RX filters by priority
 532  * @efx: NIC from which to remove the filters
 533  * @priority: Maximum priority to remove
 534  */
 535 void efx_filter_clear_rx(struct efx_nic *efx, enum efx_filter_priority priority)
 536 {
 537         efx_filter_table_clear(efx, EFX_FILTER_TABLE_RX_IP, priority);
 538         efx_filter_table_clear(efx, EFX_FILTER_TABLE_RX_MAC, priority);
 539 }
 540
 541 /* Restore filter stater after reset */
 542 void efx_restore_filters(struct efx_nic *efx)
 543 {
 544         struct efx_filter_state *state = efx->filter_state;
 545         enum efx_filter_table_id table_id;
 546         struct efx_filter_table *table;
 547         efx_oword_t filter;
 548         int filter_idx;
 549
 550         spin_lock_bh(&state->lock);
 551
 552         for (table_id = 0; table_id < EFX_FILTER_TABLE_COUNT; table_id++) {
 553                 table = &state->table[table_id];
 554                 for (filter_idx = 0; filter_idx < table->size; filter_idx++) {
 555                         if (!test_bit(filter_idx, table->used_bitmap))
 556                                 continue;
 557                         efx_filter_build(&filter, &table->spec[filter_idx]);
 558                         efx_writeo(efx, &filter,
 559                                    table->offset + table->step * filter_idx);
 560                 }
 561         }
 562
 563         efx_filter_push_rx_limits(efx);
 564
 565         spin_unlock_bh(&state->lock);
 566 }
 567
 568 int efx_probe_filters(struct efx_nic *efx)
 569 {
 570         struct efx_filter_state *state;
 571         struct efx_filter_table *table;
 572         unsigned table_id;
 573
 574         state = kzalloc(sizeof(*efx->filter_state), GFP_KERNEL);
 575         if (!state)
 576                 return -ENOMEM;
 577         efx->filter_state = state;
 578
 579         spin_lock_init(&state->lock);
 580
 581         if (efx_nic_rev(efx) >= EFX_REV_FALCON_B0) {
 582 #ifdef CONFIG_RFS_ACCEL
 583                 state->rps_flow_id = kcalloc(FR_BZ_RX_FILTER_TBL0_ROWS,
 584                                              sizeof(*state->rps_flow_id),
 585                                              GFP_KERNEL);
 586                 if (!state->rps_flow_id)
 587                         goto fail;
 588 #endif
 589                 table = &state->table[EFX_FILTER_TABLE_RX_IP];
 590                 table->id = EFX_FILTER_TABLE_RX_IP;
 591                 table->offset = FR_BZ_RX_FILTER_TBL0;
 592                 table->size = FR_BZ_RX_FILTER_TBL0_ROWS;
 593                 table->step = FR_BZ_RX_FILTER_TBL0_STEP;
 594         }
 595
 596         if (efx_nic_rev(efx) >= EFX_REV_SIENA_A0) {
 597                 table = &state->table[EFX_FILTER_TABLE_RX_MAC];
 598                 table->id = EFX_FILTER_TABLE_RX_MAC;
 599                 table->offset = FR_CZ_RX_MAC_FILTER_TBL0;
 600                 table->size = FR_CZ_RX_MAC_FILTER_TBL0_ROWS;
 601                 table->step = FR_CZ_RX_MAC_FILTER_TBL0_STEP;
 602         }
 603
 604         for (table_id = 0; table_id < EFX_FILTER_TABLE_COUNT; table_id++) {
 605                 table = &state->table[table_id];
 606                 if (table->size == 0)
 607                         continue;
 608                 table->used_bitmap = kcalloc(BITS_TO_LONGS(table->size),
 609                                              sizeof(unsigned long),
 610                                              GFP_KERNEL);
 611                 if (!table->used_bitmap)
 612                         goto fail;
 613                 table->spec = vzalloc(table->size * sizeof(*table->spec));
 614                 if (!table->spec)
 615                         goto fail;
 616         }
 617
 618         return 0;
 619
 620 fail:
 621         efx_remove_filters(efx);
 622         return -ENOMEM;
 623 }
 624
 625 void efx_remove_filters(struct efx_nic *efx)
 626 {
 627         struct efx_filter_state *state = efx->filter_state;
 628         enum efx_filter_table_id table_id;
 629
 630         for (table_id = 0; table_id < EFX_FILTER_TABLE_COUNT; table_id++) {
 631                 kfree(state->table[table_id].used_bitmap);
 632                 vfree(state->table[table_id].spec);
 633         }
 634 #ifdef CONFIG_RFS_ACCEL
 635         kfree(state->rps_flow_id);
 636 #endif
 637         kfree(state);
 638 }
 639
 640 #ifdef CONFIG_RFS_ACCEL
 641
 642 int efx_filter_rfs(struct net_device *net_dev, const struct sk_buff *skb,
 643                    u16 rxq_index, u32 flow_id)
 644 {
 645         struct efx_nic *efx = netdev_priv(net_dev);
 646         struct efx_channel *channel;
 647         struct efx_filter_state *state = efx->filter_state;
 648         struct efx_filter_spec spec;
 649         const struct iphdr *ip;
 650         const __be16 *ports;
 651         int nhoff;
 652         int rc;
 653
 654         nhoff = skb_network_offset(skb);
 655
 656         if (skb->protocol != htons(ETH_P_IP))
 657                 return -EPROTONOSUPPORT;
 658
 659         /* RFS must validate the IP header length before calling us */
 660         EFX_BUG_ON_PARANOID(skb_headlen(skb) < nhoff + sizeof(*ip));
 661         ip = (const struct iphdr *)(skb->data + nhoff);
 662         if (ip_is_fragment(ip))
 663                 return -EPROTONOSUPPORT;
 664         EFX_BUG_ON_PARANOID(skb_headlen(skb) < nhoff + 4 * ip->ihl + 4);
 665         ports = (const __be16 *)(skb->data + nhoff + 4 * ip->ihl);
 666
 667         efx_filter_init_rx(&spec, EFX_FILTER_PRI_HINT, 0, rxq_index);
 668         rc = efx_filter_set_ipv4_full(&spec, ip->protocol,
 669                                       ip->daddr, ports[1], ip->saddr, ports[0]);
 670         if (rc)
 671                 return rc;
 672
 673         rc = efx_filter_insert_filter(efx, &spec, true);
 674         if (rc < 0)
 675                 return rc;
 676
 677         /* Remember this so we can check whether to expire the filter later */
 678         state->rps_flow_id[rc] = flow_id;
 679         channel = efx_get_channel(efx, skb_get_rx_queue(skb));
 680         ++channel->rfs_filters_added;
 681
 682         netif_info(efx, rx_status, efx->net_dev,
 683                    "steering %s %pI4:%u:%pI4:%u to queue %u [flow %u filter %d]\n",
 684                    (ip->protocol == IPPROTO_TCP) ? "TCP" : "UDP",
 685                    &ip->saddr, ntohs(ports[0]), &ip->daddr, ntohs(ports[1]),
 686                    rxq_index, flow_id, rc);
 687
 688         return rc;
 689 }
 690
 691 bool __efx_filter_rfs_expire(struct efx_nic *efx, unsigned quota)
 692 {
 693         struct efx_filter_state *state = efx->filter_state;
 694         struct efx_filter_table *table = &state->table[EFX_FILTER_TABLE_RX_IP];
 695         unsigned mask = table->size - 1;
 696         unsigned index;
 697         unsigned stop;
 698
 699         if (!spin_trylock_bh(&state->lock))
 700                 return false;
 701
 702         index = state->rps_expire_index;
 703         stop = (index + quota) & mask;
 704
 705         while (index != stop) {
 706                 if (test_bit(index, table->used_bitmap) &&
 707                     table->spec[index].priority == EFX_FILTER_PRI_HINT &&
 708                     rps_may_expire_flow(efx->net_dev,
 709                                         table->spec[index].dmaq_id,
 710                                         state->rps_flow_id[index], index)) {
 711                         netif_info(efx, rx_status, efx->net_dev,
 712                                    "expiring filter %d [flow %u]\n",
 713                                    index, state->rps_flow_id[index]);
 714                         efx_filter_table_clear_entry(efx, table, index);
 715                 }
 716                 index = (index + 1) & mask;
 717         }
 718
 719         state->rps_expire_index = stop;
 720         if (table->used == 0)
 721                 efx_filter_table_reset_search_depth(table);
 722
 723         spin_unlock_bh(&state->lock);
 724         return true;
 725 }
 726
 727 #endif /* CONFIG_RFS_ACCEL */