1 /****************************************************************************
2 * Driver for Solarflare Solarstorm network controllers and boards
3 * Copyright 2005-2010 Solarflare Communications Inc.
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 as published
7 * by the Free Software Foundation, incorporated herein by reference.
18 /* "Fudge factors" - difference between programmed value and actual depth.
19 * Due to pipelined implementation we need to program H/W with a value that
20 * is larger than the hop limit we want.
22 #define FILTER_CTL_SRCH_FUDGE_WILD 3
23 #define FILTER_CTL_SRCH_FUDGE_FULL 1
25 /* Hard maximum hop limit. Hardware will time-out beyond 200-something.
26 * We also need to avoid infinite loops in efx_filter_search() when the
29 #define FILTER_CTL_SRCH_MAX 200
31 /* Don't try very hard to find space for performance hints, as this is
32 * counter-productive. */
33 #define FILTER_CTL_SRCH_HINT_MAX 5
35 enum efx_filter_table_id {
36 EFX_FILTER_TABLE_RX_IP = 0,
37 EFX_FILTER_TABLE_RX_MAC,
38 EFX_FILTER_TABLE_RX_DEF,
39 EFX_FILTER_TABLE_TX_MAC,
40 EFX_FILTER_TABLE_COUNT,
43 enum efx_filter_index {
44 EFX_FILTER_INDEX_UC_DEF,
45 EFX_FILTER_INDEX_MC_DEF,
46 EFX_FILTER_SIZE_RX_DEF,
49 struct efx_filter_table {
50 enum efx_filter_table_id id;
51 u32 offset; /* address of table relative to BAR */
52 unsigned size; /* number of entries */
53 unsigned step; /* step between entries */
54 unsigned used; /* number currently used */
55 unsigned long *used_bitmap;
56 struct efx_filter_spec *spec;
57 unsigned search_depth[EFX_FILTER_TYPE_COUNT];
60 struct efx_filter_state {
62 struct efx_filter_table table[EFX_FILTER_TABLE_COUNT];
63 #ifdef CONFIG_RFS_ACCEL
65 unsigned rps_expire_index;
69 static void efx_filter_table_clear_entry(struct efx_nic *efx,
70 struct efx_filter_table *table,
71 unsigned int filter_idx);
73 /* The filter hash function is LFSR polynomial x^16 + x^3 + 1 of a 32-bit
74 * key derived from the n-tuple. The initial LFSR state is 0xffff. */
75 static u16 efx_filter_hash(u32 key)
80 tmp = 0x1fff ^ key >> 16;
81 tmp = tmp ^ tmp >> 3 ^ tmp >> 6;
84 tmp = tmp ^ tmp << 13 ^ key;
85 tmp = tmp ^ tmp >> 3 ^ tmp >> 6;
86 return tmp ^ tmp >> 9;
89 /* To allow for hash collisions, filter search continues at these
90 * increments from the first possible entry selected by the hash. */
91 static u16 efx_filter_increment(u32 key)
96 static enum efx_filter_table_id
97 efx_filter_spec_table_id(const struct efx_filter_spec *spec)
99 BUILD_BUG_ON(EFX_FILTER_TABLE_RX_IP != (EFX_FILTER_TCP_FULL >> 2));
100 BUILD_BUG_ON(EFX_FILTER_TABLE_RX_IP != (EFX_FILTER_TCP_WILD >> 2));
101 BUILD_BUG_ON(EFX_FILTER_TABLE_RX_IP != (EFX_FILTER_UDP_FULL >> 2));
102 BUILD_BUG_ON(EFX_FILTER_TABLE_RX_IP != (EFX_FILTER_UDP_WILD >> 2));
103 BUILD_BUG_ON(EFX_FILTER_TABLE_RX_MAC != (EFX_FILTER_MAC_FULL >> 2));
104 BUILD_BUG_ON(EFX_FILTER_TABLE_RX_MAC != (EFX_FILTER_MAC_WILD >> 2));
105 BUILD_BUG_ON(EFX_FILTER_TABLE_TX_MAC != EFX_FILTER_TABLE_RX_MAC + 2);
106 EFX_BUG_ON_PARANOID(spec->type == EFX_FILTER_UNSPEC);
107 return (spec->type >> 2) + ((spec->flags & EFX_FILTER_FLAG_TX) ? 2 : 0);
110 static struct efx_filter_table *
111 efx_filter_spec_table(struct efx_filter_state *state,
112 const struct efx_filter_spec *spec)
114 if (spec->type == EFX_FILTER_UNSPEC)
117 return &state->table[efx_filter_spec_table_id(spec)];
120 static void efx_filter_table_reset_search_depth(struct efx_filter_table *table)
122 memset(table->search_depth, 0, sizeof(table->search_depth));
125 static void efx_filter_push_rx_config(struct efx_nic *efx)
127 struct efx_filter_state *state = efx->filter_state;
128 struct efx_filter_table *table;
129 efx_oword_t filter_ctl;
131 efx_reado(efx, &filter_ctl, FR_BZ_RX_FILTER_CTL);
133 table = &state->table[EFX_FILTER_TABLE_RX_IP];
134 EFX_SET_OWORD_FIELD(filter_ctl, FRF_BZ_TCP_FULL_SRCH_LIMIT,
135 table->search_depth[EFX_FILTER_TCP_FULL] +
136 FILTER_CTL_SRCH_FUDGE_FULL);
137 EFX_SET_OWORD_FIELD(filter_ctl, FRF_BZ_TCP_WILD_SRCH_LIMIT,
138 table->search_depth[EFX_FILTER_TCP_WILD] +
139 FILTER_CTL_SRCH_FUDGE_WILD);
140 EFX_SET_OWORD_FIELD(filter_ctl, FRF_BZ_UDP_FULL_SRCH_LIMIT,
141 table->search_depth[EFX_FILTER_UDP_FULL] +
142 FILTER_CTL_SRCH_FUDGE_FULL);
143 EFX_SET_OWORD_FIELD(filter_ctl, FRF_BZ_UDP_WILD_SRCH_LIMIT,
144 table->search_depth[EFX_FILTER_UDP_WILD] +
145 FILTER_CTL_SRCH_FUDGE_WILD);
147 table = &state->table[EFX_FILTER_TABLE_RX_MAC];
150 filter_ctl, FRF_CZ_ETHERNET_FULL_SEARCH_LIMIT,
151 table->search_depth[EFX_FILTER_MAC_FULL] +
152 FILTER_CTL_SRCH_FUDGE_FULL);
154 filter_ctl, FRF_CZ_ETHERNET_WILDCARD_SEARCH_LIMIT,
155 table->search_depth[EFX_FILTER_MAC_WILD] +
156 FILTER_CTL_SRCH_FUDGE_WILD);
159 table = &state->table[EFX_FILTER_TABLE_RX_DEF];
162 filter_ctl, FRF_CZ_UNICAST_NOMATCH_Q_ID,
163 table->spec[EFX_FILTER_INDEX_UC_DEF].dmaq_id);
165 filter_ctl, FRF_CZ_UNICAST_NOMATCH_RSS_ENABLED,
166 !!(table->spec[EFX_FILTER_INDEX_UC_DEF].flags &
167 EFX_FILTER_FLAG_RX_RSS));
169 filter_ctl, FRF_CZ_MULTICAST_NOMATCH_Q_ID,
170 table->spec[EFX_FILTER_INDEX_MC_DEF].dmaq_id);
172 filter_ctl, FRF_CZ_MULTICAST_NOMATCH_RSS_ENABLED,
173 !!(table->spec[EFX_FILTER_INDEX_MC_DEF].flags &
174 EFX_FILTER_FLAG_RX_RSS));
177 efx_writeo(efx, &filter_ctl, FR_BZ_RX_FILTER_CTL);
180 static void efx_filter_push_tx_limits(struct efx_nic *efx)
182 struct efx_filter_state *state = efx->filter_state;
183 struct efx_filter_table *table;
186 efx_reado(efx, &tx_cfg, FR_AZ_TX_CFG);
188 table = &state->table[EFX_FILTER_TABLE_TX_MAC];
191 tx_cfg, FRF_CZ_TX_ETH_FILTER_FULL_SEARCH_RANGE,
192 table->search_depth[EFX_FILTER_MAC_FULL] +
193 FILTER_CTL_SRCH_FUDGE_FULL);
195 tx_cfg, FRF_CZ_TX_ETH_FILTER_WILD_SEARCH_RANGE,
196 table->search_depth[EFX_FILTER_MAC_WILD] +
197 FILTER_CTL_SRCH_FUDGE_WILD);
200 efx_writeo(efx, &tx_cfg, FR_AZ_TX_CFG);
203 static inline void __efx_filter_set_ipv4(struct efx_filter_spec *spec,
204 __be32 host1, __be16 port1,
205 __be32 host2, __be16 port2)
207 spec->data[0] = ntohl(host1) << 16 | ntohs(port1);
208 spec->data[1] = ntohs(port2) << 16 | ntohl(host1) >> 16;
209 spec->data[2] = ntohl(host2);
212 static inline void __efx_filter_get_ipv4(const struct efx_filter_spec *spec,
213 __be32 *host1, __be16 *port1,
214 __be32 *host2, __be16 *port2)
216 *host1 = htonl(spec->data[0] >> 16 | spec->data[1] << 16);
217 *port1 = htons(spec->data[0]);
218 *host2 = htonl(spec->data[2]);
219 *port2 = htons(spec->data[1] >> 16);
223 * efx_filter_set_ipv4_local - specify IPv4 host, transport protocol and port
224 * @spec: Specification to initialise
225 * @proto: Transport layer protocol number
226 * @host: Local host address (network byte order)
227 * @port: Local port (network byte order)
229 int efx_filter_set_ipv4_local(struct efx_filter_spec *spec, u8 proto,
230 __be32 host, __be16 port)
235 EFX_BUG_ON_PARANOID(!(spec->flags & EFX_FILTER_FLAG_RX));
237 /* This cannot currently be combined with other filtering */
238 if (spec->type != EFX_FILTER_UNSPEC)
239 return -EPROTONOSUPPORT;
246 spec->type = EFX_FILTER_TCP_WILD;
249 spec->type = EFX_FILTER_UDP_WILD;
252 return -EPROTONOSUPPORT;
255 /* Filter is constructed in terms of source and destination,
256 * with the odd wrinkle that the ports are swapped in a UDP
257 * wildcard filter. We need to convert from local and remote
258 * (= zero for wildcard) addresses.
261 if (proto != IPPROTO_UDP) {
268 __efx_filter_set_ipv4(spec, host1, port1, host, port);
272 int efx_filter_get_ipv4_local(const struct efx_filter_spec *spec,
273 u8 *proto, __be32 *host, __be16 *port)
278 switch (spec->type) {
279 case EFX_FILTER_TCP_WILD:
280 *proto = IPPROTO_TCP;
281 __efx_filter_get_ipv4(spec, &host1, &port1, host, port);
283 case EFX_FILTER_UDP_WILD:
284 *proto = IPPROTO_UDP;
285 __efx_filter_get_ipv4(spec, &host1, port, host, &port1);
293 * efx_filter_set_ipv4_full - specify IPv4 hosts, transport protocol and ports
294 * @spec: Specification to initialise
295 * @proto: Transport layer protocol number
296 * @host: Local host address (network byte order)
297 * @port: Local port (network byte order)
298 * @rhost: Remote host address (network byte order)
299 * @rport: Remote port (network byte order)
301 int efx_filter_set_ipv4_full(struct efx_filter_spec *spec, u8 proto,
302 __be32 host, __be16 port,
303 __be32 rhost, __be16 rport)
305 EFX_BUG_ON_PARANOID(!(spec->flags & EFX_FILTER_FLAG_RX));
307 /* This cannot currently be combined with other filtering */
308 if (spec->type != EFX_FILTER_UNSPEC)
309 return -EPROTONOSUPPORT;
311 if (port == 0 || rport == 0)
316 spec->type = EFX_FILTER_TCP_FULL;
319 spec->type = EFX_FILTER_UDP_FULL;
322 return -EPROTONOSUPPORT;
325 __efx_filter_set_ipv4(spec, rhost, rport, host, port);
329 int efx_filter_get_ipv4_full(const struct efx_filter_spec *spec,
330 u8 *proto, __be32 *host, __be16 *port,
331 __be32 *rhost, __be16 *rport)
333 switch (spec->type) {
334 case EFX_FILTER_TCP_FULL:
335 *proto = IPPROTO_TCP;
337 case EFX_FILTER_UDP_FULL:
338 *proto = IPPROTO_UDP;
344 __efx_filter_get_ipv4(spec, rhost, rport, host, port);
349 * efx_filter_set_eth_local - specify local Ethernet address and optional VID
350 * @spec: Specification to initialise
351 * @vid: VLAN ID to match, or %EFX_FILTER_VID_UNSPEC
352 * @addr: Local Ethernet MAC address
354 int efx_filter_set_eth_local(struct efx_filter_spec *spec,
355 u16 vid, const u8 *addr)
357 EFX_BUG_ON_PARANOID(!(spec->flags &
358 (EFX_FILTER_FLAG_RX | EFX_FILTER_FLAG_TX)));
360 /* This cannot currently be combined with other filtering */
361 if (spec->type != EFX_FILTER_UNSPEC)
362 return -EPROTONOSUPPORT;
364 if (vid == EFX_FILTER_VID_UNSPEC) {
365 spec->type = EFX_FILTER_MAC_WILD;
368 spec->type = EFX_FILTER_MAC_FULL;
372 spec->data[1] = addr[2] << 24 | addr[3] << 16 | addr[4] << 8 | addr[5];
373 spec->data[2] = addr[0] << 8 | addr[1];
378 * efx_filter_set_uc_def - specify matching otherwise-unmatched unicast
379 * @spec: Specification to initialise
381 int efx_filter_set_uc_def(struct efx_filter_spec *spec)
383 EFX_BUG_ON_PARANOID(!(spec->flags &
384 (EFX_FILTER_FLAG_RX | EFX_FILTER_FLAG_TX)));
386 if (spec->type != EFX_FILTER_UNSPEC)
389 spec->type = EFX_FILTER_UC_DEF;
390 memset(spec->data, 0, sizeof(spec->data)); /* ensure equality */
395 * efx_filter_set_mc_def - specify matching otherwise-unmatched multicast
396 * @spec: Specification to initialise
398 int efx_filter_set_mc_def(struct efx_filter_spec *spec)
400 EFX_BUG_ON_PARANOID(!(spec->flags &
401 (EFX_FILTER_FLAG_RX | EFX_FILTER_FLAG_TX)));
403 if (spec->type != EFX_FILTER_UNSPEC)
406 spec->type = EFX_FILTER_MC_DEF;
407 memset(spec->data, 0, sizeof(spec->data)); /* ensure equality */
411 static void efx_filter_reset_rx_def(struct efx_nic *efx, unsigned filter_idx)
413 struct efx_filter_state *state = efx->filter_state;
414 struct efx_filter_table *table = &state->table[EFX_FILTER_TABLE_RX_DEF];
415 struct efx_filter_spec *spec = &table->spec[filter_idx];
417 efx_filter_init_rx(spec, EFX_FILTER_PRI_MANUAL,
418 EFX_FILTER_FLAG_RX_RSS, 0);
419 spec->type = EFX_FILTER_UC_DEF + filter_idx;
420 table->used_bitmap[0] |= 1 << filter_idx;
423 int efx_filter_get_eth_local(const struct efx_filter_spec *spec,
426 switch (spec->type) {
427 case EFX_FILTER_MAC_WILD:
428 *vid = EFX_FILTER_VID_UNSPEC;
430 case EFX_FILTER_MAC_FULL:
431 *vid = spec->data[0];
437 addr[0] = spec->data[2] >> 8;
438 addr[1] = spec->data[2];
439 addr[2] = spec->data[1] >> 24;
440 addr[3] = spec->data[1] >> 16;
441 addr[4] = spec->data[1] >> 8;
442 addr[5] = spec->data[1];
446 /* Build a filter entry and return its n-tuple key. */
447 static u32 efx_filter_build(efx_oword_t *filter, struct efx_filter_spec *spec)
451 switch (efx_filter_spec_table_id(spec)) {
452 case EFX_FILTER_TABLE_RX_IP: {
453 bool is_udp = (spec->type == EFX_FILTER_UDP_FULL ||
454 spec->type == EFX_FILTER_UDP_WILD);
455 EFX_POPULATE_OWORD_7(
458 !!(spec->flags & EFX_FILTER_FLAG_RX_RSS),
460 !!(spec->flags & EFX_FILTER_FLAG_RX_SCATTER),
461 FRF_BZ_TCP_UDP, is_udp,
462 FRF_BZ_RXQ_ID, spec->dmaq_id,
463 EFX_DWORD_2, spec->data[2],
464 EFX_DWORD_1, spec->data[1],
465 EFX_DWORD_0, spec->data[0]);
470 case EFX_FILTER_TABLE_RX_MAC: {
471 bool is_wild = spec->type == EFX_FILTER_MAC_WILD;
472 EFX_POPULATE_OWORD_7(
475 !!(spec->flags & EFX_FILTER_FLAG_RX_RSS),
476 FRF_CZ_RMFT_SCATTER_EN,
477 !!(spec->flags & EFX_FILTER_FLAG_RX_SCATTER),
478 FRF_CZ_RMFT_RXQ_ID, spec->dmaq_id,
479 FRF_CZ_RMFT_WILDCARD_MATCH, is_wild,
480 FRF_CZ_RMFT_DEST_MAC_HI, spec->data[2],
481 FRF_CZ_RMFT_DEST_MAC_LO, spec->data[1],
482 FRF_CZ_RMFT_VLAN_ID, spec->data[0]);
487 case EFX_FILTER_TABLE_TX_MAC: {
488 bool is_wild = spec->type == EFX_FILTER_MAC_WILD;
489 EFX_POPULATE_OWORD_5(*filter,
490 FRF_CZ_TMFT_TXQ_ID, spec->dmaq_id,
491 FRF_CZ_TMFT_WILDCARD_MATCH, is_wild,
492 FRF_CZ_TMFT_SRC_MAC_HI, spec->data[2],
493 FRF_CZ_TMFT_SRC_MAC_LO, spec->data[1],
494 FRF_CZ_TMFT_VLAN_ID, spec->data[0]);
495 data3 = is_wild | spec->dmaq_id << 1;
503 return spec->data[0] ^ spec->data[1] ^ spec->data[2] ^ data3;
506 static bool efx_filter_equal(const struct efx_filter_spec *left,
507 const struct efx_filter_spec *right)
509 if (left->type != right->type ||
510 memcmp(left->data, right->data, sizeof(left->data)))
513 if (left->flags & EFX_FILTER_FLAG_TX &&
514 left->dmaq_id != right->dmaq_id)
521 * Construct/deconstruct external filter IDs. At least the RX filter
522 * IDs must be ordered by matching priority, for RX NFC semantics.
524 * Deconstruction needs to be robust against invalid IDs so that
525 * efx_filter_remove_id_safe() and efx_filter_get_filter_safe() can
526 * accept user-provided IDs.
529 #define EFX_FILTER_MATCH_PRI_COUNT 5
531 static const u8 efx_filter_type_match_pri[EFX_FILTER_TYPE_COUNT] = {
532 [EFX_FILTER_TCP_FULL] = 0,
533 [EFX_FILTER_UDP_FULL] = 0,
534 [EFX_FILTER_TCP_WILD] = 1,
535 [EFX_FILTER_UDP_WILD] = 1,
536 [EFX_FILTER_MAC_FULL] = 2,
537 [EFX_FILTER_MAC_WILD] = 3,
538 [EFX_FILTER_UC_DEF] = 4,
539 [EFX_FILTER_MC_DEF] = 4,
542 static const enum efx_filter_table_id efx_filter_range_table[] = {
543 EFX_FILTER_TABLE_RX_IP, /* RX match pri 0 */
544 EFX_FILTER_TABLE_RX_IP,
545 EFX_FILTER_TABLE_RX_MAC,
546 EFX_FILTER_TABLE_RX_MAC,
547 EFX_FILTER_TABLE_RX_DEF, /* RX match pri 4 */
548 EFX_FILTER_TABLE_COUNT, /* TX match pri 0; invalid */
549 EFX_FILTER_TABLE_COUNT, /* invalid */
550 EFX_FILTER_TABLE_TX_MAC,
551 EFX_FILTER_TABLE_TX_MAC, /* TX match pri 3 */
554 #define EFX_FILTER_INDEX_WIDTH 13
555 #define EFX_FILTER_INDEX_MASK ((1 << EFX_FILTER_INDEX_WIDTH) - 1)
558 efx_filter_make_id(const struct efx_filter_spec *spec, unsigned int index)
562 range = efx_filter_type_match_pri[spec->type];
563 if (!(spec->flags & EFX_FILTER_FLAG_RX))
564 range += EFX_FILTER_MATCH_PRI_COUNT;
566 return range << EFX_FILTER_INDEX_WIDTH | index;
569 static inline enum efx_filter_table_id efx_filter_id_table_id(u32 id)
571 unsigned int range = id >> EFX_FILTER_INDEX_WIDTH;
573 if (range < ARRAY_SIZE(efx_filter_range_table))
574 return efx_filter_range_table[range];
576 return EFX_FILTER_TABLE_COUNT; /* invalid */
579 static inline unsigned int efx_filter_id_index(u32 id)
581 return id & EFX_FILTER_INDEX_MASK;
584 static inline u8 efx_filter_id_flags(u32 id)
586 unsigned int range = id >> EFX_FILTER_INDEX_WIDTH;
588 if (range < EFX_FILTER_MATCH_PRI_COUNT)
589 return EFX_FILTER_FLAG_RX;
591 return EFX_FILTER_FLAG_TX;
594 u32 efx_filter_get_rx_id_limit(struct efx_nic *efx)
596 struct efx_filter_state *state = efx->filter_state;
597 unsigned int range = EFX_FILTER_MATCH_PRI_COUNT - 1;
598 enum efx_filter_table_id table_id;
601 table_id = efx_filter_range_table[range];
602 if (state->table[table_id].size != 0)
603 return range << EFX_FILTER_INDEX_WIDTH |
604 state->table[table_id].size;
611 * efx_filter_insert_filter - add or replace a filter
612 * @efx: NIC in which to insert the filter
613 * @spec: Specification for the filter
614 * @replace_equal: Flag for whether the specified filter may replace an
615 * existing filter with equal priority
617 * On success, return the filter ID.
618 * On failure, return a negative error code.
620 * If an existing filter has equal match values to the new filter
621 * spec, then the new filter might replace it, depending on the
622 * relative priorities. If the existing filter has lower priority, or
623 * if @replace_equal is set and it has equal priority, then it is
624 * replaced. Otherwise the function fails, returning -%EPERM if
625 * the existing filter has higher priority or -%EEXIST if it has
628 s32 efx_filter_insert_filter(struct efx_nic *efx, struct efx_filter_spec *spec,
631 struct efx_filter_state *state = efx->filter_state;
632 struct efx_filter_table *table = efx_filter_spec_table(state, spec);
634 int rep_index, ins_index;
635 unsigned int depth = 0;
638 if (!table || table->size == 0)
641 netif_vdbg(efx, hw, efx->net_dev,
642 "%s: type %d search_depth=%d", __func__, spec->type,
643 table->search_depth[spec->type]);
645 if (table->id == EFX_FILTER_TABLE_RX_DEF) {
646 /* One filter spec per type */
647 BUILD_BUG_ON(EFX_FILTER_INDEX_UC_DEF != 0);
648 BUILD_BUG_ON(EFX_FILTER_INDEX_MC_DEF !=
649 EFX_FILTER_MC_DEF - EFX_FILTER_UC_DEF);
650 rep_index = spec->type - EFX_FILTER_INDEX_UC_DEF;
651 ins_index = rep_index;
653 spin_lock_bh(&state->lock);
655 /* Search concurrently for
656 * (1) a filter to be replaced (rep_index): any filter
657 * with the same match values, up to the current
658 * search depth for this type, and
659 * (2) the insertion point (ins_index): (1) or any
660 * free slot before it or up to the maximum search
661 * depth for this priority
662 * We fail if we cannot find (2).
664 * We can stop once either
665 * (a) we find (1), in which case we have definitely
666 * found (2) as well; or
667 * (b) we have searched exhaustively for (1), and have
668 * either found (2) or searched exhaustively for it
670 u32 key = efx_filter_build(&filter, spec);
671 unsigned int hash = efx_filter_hash(key);
672 unsigned int incr = efx_filter_increment(key);
673 unsigned int max_rep_depth = table->search_depth[spec->type];
674 unsigned int max_ins_depth =
675 spec->priority <= EFX_FILTER_PRI_HINT ?
676 FILTER_CTL_SRCH_HINT_MAX : FILTER_CTL_SRCH_MAX;
677 unsigned int i = hash & (table->size - 1);
682 spin_lock_bh(&state->lock);
685 if (!test_bit(i, table->used_bitmap)) {
688 } else if (efx_filter_equal(spec, &table->spec[i])) {
696 if (depth >= max_rep_depth &&
697 (ins_index >= 0 || depth >= max_ins_depth)) {
707 i = (i + incr) & (table->size - 1);
712 /* If we found a filter to be replaced, check whether we
715 if (rep_index >= 0) {
716 struct efx_filter_spec *saved_spec = &table->spec[rep_index];
718 if (spec->priority == saved_spec->priority && !replace_equal) {
722 if (spec->priority < saved_spec->priority) {
728 /* Insert the filter */
729 if (ins_index != rep_index) {
730 __set_bit(ins_index, table->used_bitmap);
733 table->spec[ins_index] = *spec;
735 if (table->id == EFX_FILTER_TABLE_RX_DEF) {
736 efx_filter_push_rx_config(efx);
738 if (table->search_depth[spec->type] < depth) {
739 table->search_depth[spec->type] = depth;
740 if (spec->flags & EFX_FILTER_FLAG_TX)
741 efx_filter_push_tx_limits(efx);
743 efx_filter_push_rx_config(efx);
746 efx_writeo(efx, &filter,
747 table->offset + table->step * ins_index);
749 /* If we were able to replace a filter by inserting
750 * at a lower depth, clear the replaced filter
752 if (ins_index != rep_index && rep_index >= 0)
753 efx_filter_table_clear_entry(efx, table, rep_index);
756 netif_vdbg(efx, hw, efx->net_dev,
757 "%s: filter type %d index %d rxq %u set",
758 __func__, spec->type, ins_index, spec->dmaq_id);
759 rc = efx_filter_make_id(spec, ins_index);
762 spin_unlock_bh(&state->lock);
766 static void efx_filter_table_clear_entry(struct efx_nic *efx,
767 struct efx_filter_table *table,
768 unsigned int filter_idx)
770 static efx_oword_t filter;
772 if (table->id == EFX_FILTER_TABLE_RX_DEF) {
773 /* RX default filters must always exist */
774 efx_filter_reset_rx_def(efx, filter_idx);
775 efx_filter_push_rx_config(efx);
776 } else if (test_bit(filter_idx, table->used_bitmap)) {
777 __clear_bit(filter_idx, table->used_bitmap);
779 memset(&table->spec[filter_idx], 0, sizeof(table->spec[0]));
781 efx_writeo(efx, &filter,
782 table->offset + table->step * filter_idx);
787 * efx_filter_remove_id_safe - remove a filter by ID, carefully
788 * @efx: NIC from which to remove the filter
789 * @priority: Priority of filter, as passed to @efx_filter_insert_filter
790 * @filter_id: ID of filter, as returned by @efx_filter_insert_filter
792 * This function will range-check @filter_id, so it is safe to call
793 * with a value passed from userland.
795 int efx_filter_remove_id_safe(struct efx_nic *efx,
796 enum efx_filter_priority priority,
799 struct efx_filter_state *state = efx->filter_state;
800 enum efx_filter_table_id table_id;
801 struct efx_filter_table *table;
802 unsigned int filter_idx;
803 struct efx_filter_spec *spec;
807 table_id = efx_filter_id_table_id(filter_id);
808 if ((unsigned int)table_id >= EFX_FILTER_TABLE_COUNT)
810 table = &state->table[table_id];
812 filter_idx = efx_filter_id_index(filter_id);
813 if (filter_idx >= table->size)
815 spec = &table->spec[filter_idx];
817 filter_flags = efx_filter_id_flags(filter_id);
819 spin_lock_bh(&state->lock);
821 if (test_bit(filter_idx, table->used_bitmap) &&
822 spec->priority == priority) {
823 efx_filter_table_clear_entry(efx, table, filter_idx);
824 if (table->used == 0)
825 efx_filter_table_reset_search_depth(table);
831 spin_unlock_bh(&state->lock);
837 * efx_filter_get_filter_safe - retrieve a filter by ID, carefully
838 * @efx: NIC from which to remove the filter
839 * @priority: Priority of filter, as passed to @efx_filter_insert_filter
840 * @filter_id: ID of filter, as returned by @efx_filter_insert_filter
841 * @spec: Buffer in which to store filter specification
843 * This function will range-check @filter_id, so it is safe to call
844 * with a value passed from userland.
846 int efx_filter_get_filter_safe(struct efx_nic *efx,
847 enum efx_filter_priority priority,
848 u32 filter_id, struct efx_filter_spec *spec_buf)
850 struct efx_filter_state *state = efx->filter_state;
851 enum efx_filter_table_id table_id;
852 struct efx_filter_table *table;
853 struct efx_filter_spec *spec;
854 unsigned int filter_idx;
858 table_id = efx_filter_id_table_id(filter_id);
859 if ((unsigned int)table_id >= EFX_FILTER_TABLE_COUNT)
861 table = &state->table[table_id];
863 filter_idx = efx_filter_id_index(filter_id);
864 if (filter_idx >= table->size)
866 spec = &table->spec[filter_idx];
868 filter_flags = efx_filter_id_flags(filter_id);
870 spin_lock_bh(&state->lock);
872 if (test_bit(filter_idx, table->used_bitmap) &&
873 spec->priority == priority) {
880 spin_unlock_bh(&state->lock);
885 static void efx_filter_table_clear(struct efx_nic *efx,
886 enum efx_filter_table_id table_id,
887 enum efx_filter_priority priority)
889 struct efx_filter_state *state = efx->filter_state;
890 struct efx_filter_table *table = &state->table[table_id];
891 unsigned int filter_idx;
893 spin_lock_bh(&state->lock);
895 for (filter_idx = 0; filter_idx < table->size; ++filter_idx)
896 if (table->spec[filter_idx].priority <= priority)
897 efx_filter_table_clear_entry(efx, table, filter_idx);
898 if (table->used == 0)
899 efx_filter_table_reset_search_depth(table);
901 spin_unlock_bh(&state->lock);
905 * efx_filter_clear_rx - remove RX filters by priority
906 * @efx: NIC from which to remove the filters
907 * @priority: Maximum priority to remove
909 void efx_filter_clear_rx(struct efx_nic *efx, enum efx_filter_priority priority)
911 efx_filter_table_clear(efx, EFX_FILTER_TABLE_RX_IP, priority);
912 efx_filter_table_clear(efx, EFX_FILTER_TABLE_RX_MAC, priority);
915 u32 efx_filter_count_rx_used(struct efx_nic *efx,
916 enum efx_filter_priority priority)
918 struct efx_filter_state *state = efx->filter_state;
919 enum efx_filter_table_id table_id;
920 struct efx_filter_table *table;
921 unsigned int filter_idx;
924 spin_lock_bh(&state->lock);
926 for (table_id = EFX_FILTER_TABLE_RX_IP;
927 table_id <= EFX_FILTER_TABLE_RX_DEF;
929 table = &state->table[table_id];
930 for (filter_idx = 0; filter_idx < table->size; filter_idx++) {
931 if (test_bit(filter_idx, table->used_bitmap) &&
932 table->spec[filter_idx].priority == priority)
937 spin_unlock_bh(&state->lock);
942 s32 efx_filter_get_rx_ids(struct efx_nic *efx,
943 enum efx_filter_priority priority,
946 struct efx_filter_state *state = efx->filter_state;
947 enum efx_filter_table_id table_id;
948 struct efx_filter_table *table;
949 unsigned int filter_idx;
952 spin_lock_bh(&state->lock);
954 for (table_id = EFX_FILTER_TABLE_RX_IP;
955 table_id <= EFX_FILTER_TABLE_RX_DEF;
957 table = &state->table[table_id];
958 for (filter_idx = 0; filter_idx < table->size; filter_idx++) {
959 if (test_bit(filter_idx, table->used_bitmap) &&
960 table->spec[filter_idx].priority == priority) {
965 buf[count++] = efx_filter_make_id(
966 &table->spec[filter_idx], filter_idx);
971 spin_unlock_bh(&state->lock);
976 /* Restore filter stater after reset */
977 void efx_restore_filters(struct efx_nic *efx)
979 struct efx_filter_state *state = efx->filter_state;
980 enum efx_filter_table_id table_id;
981 struct efx_filter_table *table;
983 unsigned int filter_idx;
985 spin_lock_bh(&state->lock);
987 for (table_id = 0; table_id < EFX_FILTER_TABLE_COUNT; table_id++) {
988 table = &state->table[table_id];
990 /* Check whether this is a regular register table */
991 if (table->step == 0)
994 for (filter_idx = 0; filter_idx < table->size; filter_idx++) {
995 if (!test_bit(filter_idx, table->used_bitmap))
997 efx_filter_build(&filter, &table->spec[filter_idx]);
998 efx_writeo(efx, &filter,
999 table->offset + table->step * filter_idx);
1003 efx_filter_push_rx_config(efx);
1004 efx_filter_push_tx_limits(efx);
1006 spin_unlock_bh(&state->lock);
1009 int efx_probe_filters(struct efx_nic *efx)
1011 struct efx_filter_state *state;
1012 struct efx_filter_table *table;
1015 state = kzalloc(sizeof(*efx->filter_state), GFP_KERNEL);
1018 efx->filter_state = state;
1020 spin_lock_init(&state->lock);
1022 if (efx_nic_rev(efx) >= EFX_REV_FALCON_B0) {
1023 #ifdef CONFIG_RFS_ACCEL
1024 state->rps_flow_id = kcalloc(FR_BZ_RX_FILTER_TBL0_ROWS,
1025 sizeof(*state->rps_flow_id),
1027 if (!state->rps_flow_id)
1030 table = &state->table[EFX_FILTER_TABLE_RX_IP];
1031 table->id = EFX_FILTER_TABLE_RX_IP;
1032 table->offset = FR_BZ_RX_FILTER_TBL0;
1033 table->size = FR_BZ_RX_FILTER_TBL0_ROWS;
1034 table->step = FR_BZ_RX_FILTER_TBL0_STEP;
1037 if (efx_nic_rev(efx) >= EFX_REV_SIENA_A0) {
1038 table = &state->table[EFX_FILTER_TABLE_RX_MAC];
1039 table->id = EFX_FILTER_TABLE_RX_MAC;
1040 table->offset = FR_CZ_RX_MAC_FILTER_TBL0;
1041 table->size = FR_CZ_RX_MAC_FILTER_TBL0_ROWS;
1042 table->step = FR_CZ_RX_MAC_FILTER_TBL0_STEP;
1044 table = &state->table[EFX_FILTER_TABLE_RX_DEF];
1045 table->id = EFX_FILTER_TABLE_RX_DEF;
1046 table->size = EFX_FILTER_SIZE_RX_DEF;
1048 table = &state->table[EFX_FILTER_TABLE_TX_MAC];
1049 table->id = EFX_FILTER_TABLE_TX_MAC;
1050 table->offset = FR_CZ_TX_MAC_FILTER_TBL0;
1051 table->size = FR_CZ_TX_MAC_FILTER_TBL0_ROWS;
1052 table->step = FR_CZ_TX_MAC_FILTER_TBL0_STEP;
1055 for (table_id = 0; table_id < EFX_FILTER_TABLE_COUNT; table_id++) {
1056 table = &state->table[table_id];
1057 if (table->size == 0)
1059 table->used_bitmap = kcalloc(BITS_TO_LONGS(table->size),
1060 sizeof(unsigned long),
1062 if (!table->used_bitmap)
1064 table->spec = vzalloc(table->size * sizeof(*table->spec));
1069 if (state->table[EFX_FILTER_TABLE_RX_DEF].size) {
1070 /* RX default filters must always exist */
1072 for (i = 0; i < EFX_FILTER_SIZE_RX_DEF; i++)
1073 efx_filter_reset_rx_def(efx, i);
1076 efx_filter_push_rx_config(efx);
1081 efx_remove_filters(efx);
1085 void efx_remove_filters(struct efx_nic *efx)
1087 struct efx_filter_state *state = efx->filter_state;
1088 enum efx_filter_table_id table_id;
1090 for (table_id = 0; table_id < EFX_FILTER_TABLE_COUNT; table_id++) {
1091 kfree(state->table[table_id].used_bitmap);
1092 vfree(state->table[table_id].spec);
1094 #ifdef CONFIG_RFS_ACCEL
1095 kfree(state->rps_flow_id);
1100 #ifdef CONFIG_RFS_ACCEL
1102 int efx_filter_rfs(struct net_device *net_dev, const struct sk_buff *skb,
1103 u16 rxq_index, u32 flow_id)
1105 struct efx_nic *efx = netdev_priv(net_dev);
1106 struct efx_channel *channel;
1107 struct efx_filter_state *state = efx->filter_state;
1108 struct efx_filter_spec spec;
1109 const struct iphdr *ip;
1110 const __be16 *ports;
1114 nhoff = skb_network_offset(skb);
1116 if (skb->protocol != htons(ETH_P_IP))
1117 return -EPROTONOSUPPORT;
1119 /* RFS must validate the IP header length before calling us */
1120 EFX_BUG_ON_PARANOID(skb_headlen(skb) < nhoff + sizeof(*ip));
1121 ip = (const struct iphdr *)(skb->data + nhoff);
1122 if (ip_is_fragment(ip))
1123 return -EPROTONOSUPPORT;
1124 EFX_BUG_ON_PARANOID(skb_headlen(skb) < nhoff + 4 * ip->ihl + 4);
1125 ports = (const __be16 *)(skb->data + nhoff + 4 * ip->ihl);
1127 efx_filter_init_rx(&spec, EFX_FILTER_PRI_HINT, 0, rxq_index);
1128 rc = efx_filter_set_ipv4_full(&spec, ip->protocol,
1129 ip->daddr, ports[1], ip->saddr, ports[0]);
1133 rc = efx_filter_insert_filter(efx, &spec, true);
1137 /* Remember this so we can check whether to expire the filter later */
1138 state->rps_flow_id[rc] = flow_id;
1139 channel = efx_get_channel(efx, skb_get_rx_queue(skb));
1140 ++channel->rfs_filters_added;
1142 netif_info(efx, rx_status, efx->net_dev,
1143 "steering %s %pI4:%u:%pI4:%u to queue %u [flow %u filter %d]\n",
1144 (ip->protocol == IPPROTO_TCP) ? "TCP" : "UDP",
1145 &ip->saddr, ntohs(ports[0]), &ip->daddr, ntohs(ports[1]),
1146 rxq_index, flow_id, rc);
1151 bool __efx_filter_rfs_expire(struct efx_nic *efx, unsigned quota)
1153 struct efx_filter_state *state = efx->filter_state;
1154 struct efx_filter_table *table = &state->table[EFX_FILTER_TABLE_RX_IP];
1155 unsigned mask = table->size - 1;
1159 if (!spin_trylock_bh(&state->lock))
1162 index = state->rps_expire_index;
1163 stop = (index + quota) & mask;
1165 while (index != stop) {
1166 if (test_bit(index, table->used_bitmap) &&
1167 table->spec[index].priority == EFX_FILTER_PRI_HINT &&
1168 rps_may_expire_flow(efx->net_dev,
1169 table->spec[index].dmaq_id,
1170 state->rps_flow_id[index], index)) {
1171 netif_info(efx, rx_status, efx->net_dev,
1172 "expiring filter %d [flow %u]\n",
1173 index, state->rps_flow_id[index]);
1174 efx_filter_table_clear_entry(efx, table, index);
1176 index = (index + 1) & mask;
1179 state->rps_expire_index = stop;
1180 if (table->used == 0)
1181 efx_filter_table_reset_search_depth(table);
1183 spin_unlock_bh(&state->lock);
1187 #endif /* CONFIG_RFS_ACCEL */