net/core/filter.c

   1 /*
   2  * Linux Socket Filter - Kernel level socket filtering
   3  *
   4  * Based on the design of the Berkeley Packet Filter. The new
   5  * internal format has been designed by PLUMgrid:
   6  *
   7  *      Copyright (c) 2011 - 2014 PLUMgrid, http://plumgrid.com
   8  *
   9  * Authors:
  10  *
  11  *      Jay Schulist <jschlst@samba.org>
  12  *      Alexei Starovoitov <ast@plumgrid.com>
  13  *      Daniel Borkmann <dborkman@redhat.com>
  14  *
  15  * This program is free software; you can redistribute it and/or
  16  * modify it under the terms of the GNU General Public License
  17  * as published by the Free Software Foundation; either version
  18  * 2 of the License, or (at your option) any later version.
  19  *
  20  * Andi Kleen - Fix a few bad bugs and races.
  21  * Kris Katterjohn - Added many additional checks in sk_chk_filter()
  22  */
  23
  24 #include <linux/module.h>
  25 #include <linux/types.h>
  26 #include <linux/mm.h>
  27 #include <linux/fcntl.h>
  28 #include <linux/socket.h>
  29 #include <linux/in.h>
  30 #include <linux/inet.h>
  31 #include <linux/netdevice.h>
  32 #include <linux/if_packet.h>
  33 #include <linux/gfp.h>
  34 #include <net/ip.h>
  35 #include <net/protocol.h>
  36 #include <net/netlink.h>
  37 #include <linux/skbuff.h>
  38 #include <net/sock.h>
  39 #include <linux/errno.h>
  40 #include <linux/timer.h>
  41 #include <asm/uaccess.h>
  42 #include <asm/unaligned.h>
  43 #include <linux/filter.h>
  44 #include <linux/ratelimit.h>
  45 #include <linux/seccomp.h>
  46 #include <linux/if_vlan.h>
  47
  48 /* No hurry in this branch
  49  *
  50  * Exported for the bpf jit load helper.
  51  */
  52 void *bpf_internal_load_pointer_neg_helper(const struct sk_buff *skb, int k, unsigned int size)
  53 {
  54         u8 *ptr = NULL;
  55
  56         if (k >= SKF_NET_OFF)
  57                 ptr = skb_network_header(skb) + k - SKF_NET_OFF;
  58         else if (k >= SKF_LL_OFF)
  59                 ptr = skb_mac_header(skb) + k - SKF_LL_OFF;
  60
  61         if (ptr >= skb->head && ptr + size <= skb_tail_pointer(skb))
  62                 return ptr;
  63         return NULL;
  64 }
  65
  66 static inline void *load_pointer(const struct sk_buff *skb, int k,
  67                                  unsigned int size, void *buffer)
  68 {
  69         if (k >= 0)
  70                 return skb_header_pointer(skb, k, size, buffer);
  71         return bpf_internal_load_pointer_neg_helper(skb, k, size);
  72 }
  73
  74 /**
  75  *      sk_filter - run a packet through a socket filter
  76  *      @sk: sock associated with &sk_buff
  77  *      @skb: buffer to filter
  78  *
  79  * Run the filter code and then cut skb->data to correct size returned by
  80  * sk_run_filter. If pkt_len is 0 we toss packet. If skb->len is smaller
  81  * than pkt_len we keep whole skb->data. This is the socket level
  82  * wrapper to sk_run_filter. It returns 0 if the packet should
  83  * be accepted or -EPERM if the packet should be tossed.
  84  *
  85  */
  86 int sk_filter(struct sock *sk, struct sk_buff *skb)
  87 {
  88         int err;
  89         struct sk_filter *filter;
  90
  91         /*
  92          * If the skb was allocated from pfmemalloc reserves, only
  93          * allow SOCK_MEMALLOC sockets to use it as this socket is
  94          * helping free memory
  95          */
  96         if (skb_pfmemalloc(skb) && !sock_flag(sk, SOCK_MEMALLOC))
  97                 return -ENOMEM;
  98
  99         err = security_sock_rcv_skb(sk, skb);
 100         if (err)
 101                 return err;
 102
 103         rcu_read_lock();
 104         filter = rcu_dereference(sk->sk_filter);
 105         if (filter) {
 106                 unsigned int pkt_len = SK_RUN_FILTER(filter, skb);
 107
 108                 err = pkt_len ? pskb_trim(skb, pkt_len) : -EPERM;
 109         }
 110         rcu_read_unlock();
 111
 112         return err;
 113 }
 114 EXPORT_SYMBOL(sk_filter);
 115
 116 /* Base function for offset calculation. Needs to go into .text section,
 117  * therefore keeping it non-static as well; will also be used by JITs
 118  * anyway later on, so do not let the compiler omit it.
 119  */
 120 noinline u64 __bpf_call_base(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5)
 121 {
 122         return 0;
 123 }
 124
 125 /* Register mappings for user programs. */
 126 #define A_REG           0
 127 #define X_REG           7
 128 #define TMP_REG         8
 129 #define ARG2_REG        2
 130 #define ARG3_REG        3
 131
 132 /**
 133  *      __sk_run_filter - run a filter on a given context
 134  *      @ctx: buffer to run the filter on
 135  *      @insn: filter to apply
 136  *
 137  * Decode and apply filter instructions to the skb->data. Return length to
 138  * keep, 0 for none. @ctx is the data we are operating on, @insn is the
 139  * array of filter instructions.
 140  */
 141 unsigned int __sk_run_filter(void *ctx, const struct sock_filter_int *insn)
 142 {
 143         u64 stack[MAX_BPF_STACK / sizeof(u64)];
 144         u64 regs[MAX_BPF_REG], tmp;
 145         void *ptr;
 146         int off;
 147
 148 #define K  insn->imm
 149 #define A  regs[insn->a_reg]
 150 #define X  regs[insn->x_reg]
 151 #define R0 regs[0]
 152
 153 #define CONT     ({insn++; goto select_insn; })
 154 #define CONT_JMP ({insn++; goto select_insn; })
 155
 156         static const void *jumptable[256] = {
 157                 [0 ... 255] = &&default_label,
 158                 /* Now overwrite non-defaults ... */
 159 #define DL(A, B, C)     [A|B|C] = &&A##_##B##_##C
 160                 DL(BPF_ALU, BPF_ADD, BPF_X),
 161                 DL(BPF_ALU, BPF_ADD, BPF_K),
 162                 DL(BPF_ALU, BPF_SUB, BPF_X),
 163                 DL(BPF_ALU, BPF_SUB, BPF_K),
 164                 DL(BPF_ALU, BPF_AND, BPF_X),
 165                 DL(BPF_ALU, BPF_AND, BPF_K),
 166                 DL(BPF_ALU, BPF_OR, BPF_X),
 167                 DL(BPF_ALU, BPF_OR, BPF_K),
 168                 DL(BPF_ALU, BPF_LSH, BPF_X),
 169                 DL(BPF_ALU, BPF_LSH, BPF_K),
 170                 DL(BPF_ALU, BPF_RSH, BPF_X),
 171                 DL(BPF_ALU, BPF_RSH, BPF_K),
 172                 DL(BPF_ALU, BPF_XOR, BPF_X),
 173                 DL(BPF_ALU, BPF_XOR, BPF_K),
 174                 DL(BPF_ALU, BPF_MUL, BPF_X),
 175                 DL(BPF_ALU, BPF_MUL, BPF_K),
 176                 DL(BPF_ALU, BPF_MOV, BPF_X),
 177                 DL(BPF_ALU, BPF_MOV, BPF_K),
 178                 DL(BPF_ALU, BPF_DIV, BPF_X),
 179                 DL(BPF_ALU, BPF_DIV, BPF_K),
 180                 DL(BPF_ALU, BPF_MOD, BPF_X),
 181                 DL(BPF_ALU, BPF_MOD, BPF_K),
 182                 DL(BPF_ALU, BPF_NEG, 0),
 183                 DL(BPF_ALU, BPF_END, BPF_TO_BE),
 184                 DL(BPF_ALU, BPF_END, BPF_TO_LE),
 185                 DL(BPF_ALU64, BPF_ADD, BPF_X),
 186                 DL(BPF_ALU64, BPF_ADD, BPF_K),
 187                 DL(BPF_ALU64, BPF_SUB, BPF_X),
 188                 DL(BPF_ALU64, BPF_SUB, BPF_K),
 189                 DL(BPF_ALU64, BPF_AND, BPF_X),
 190                 DL(BPF_ALU64, BPF_AND, BPF_K),
 191                 DL(BPF_ALU64, BPF_OR, BPF_X),
 192                 DL(BPF_ALU64, BPF_OR, BPF_K),
 193                 DL(BPF_ALU64, BPF_LSH, BPF_X),
 194                 DL(BPF_ALU64, BPF_LSH, BPF_K),
 195                 DL(BPF_ALU64, BPF_RSH, BPF_X),
 196                 DL(BPF_ALU64, BPF_RSH, BPF_K),
 197                 DL(BPF_ALU64, BPF_XOR, BPF_X),
 198                 DL(BPF_ALU64, BPF_XOR, BPF_K),
 199                 DL(BPF_ALU64, BPF_MUL, BPF_X),
 200                 DL(BPF_ALU64, BPF_MUL, BPF_K),
 201                 DL(BPF_ALU64, BPF_MOV, BPF_X),
 202                 DL(BPF_ALU64, BPF_MOV, BPF_K),
 203                 DL(BPF_ALU64, BPF_ARSH, BPF_X),
 204                 DL(BPF_ALU64, BPF_ARSH, BPF_K),
 205                 DL(BPF_ALU64, BPF_DIV, BPF_X),
 206                 DL(BPF_ALU64, BPF_DIV, BPF_K),
 207                 DL(BPF_ALU64, BPF_MOD, BPF_X),
 208                 DL(BPF_ALU64, BPF_MOD, BPF_K),
 209                 DL(BPF_ALU64, BPF_NEG, 0),
 210                 DL(BPF_JMP, BPF_CALL, 0),
 211                 DL(BPF_JMP, BPF_JA, 0),
 212                 DL(BPF_JMP, BPF_JEQ, BPF_X),
 213                 DL(BPF_JMP, BPF_JEQ, BPF_K),
 214                 DL(BPF_JMP, BPF_JNE, BPF_X),
 215                 DL(BPF_JMP, BPF_JNE, BPF_K),
 216                 DL(BPF_JMP, BPF_JGT, BPF_X),
 217                 DL(BPF_JMP, BPF_JGT, BPF_K),
 218                 DL(BPF_JMP, BPF_JGE, BPF_X),
 219                 DL(BPF_JMP, BPF_JGE, BPF_K),
 220                 DL(BPF_JMP, BPF_JSGT, BPF_X),
 221                 DL(BPF_JMP, BPF_JSGT, BPF_K),
 222                 DL(BPF_JMP, BPF_JSGE, BPF_X),
 223                 DL(BPF_JMP, BPF_JSGE, BPF_K),
 224                 DL(BPF_JMP, BPF_JSET, BPF_X),
 225                 DL(BPF_JMP, BPF_JSET, BPF_K),
 226                 DL(BPF_JMP, BPF_EXIT, 0),
 227                 DL(BPF_STX, BPF_MEM, BPF_B),
 228                 DL(BPF_STX, BPF_MEM, BPF_H),
 229                 DL(BPF_STX, BPF_MEM, BPF_W),
 230                 DL(BPF_STX, BPF_MEM, BPF_DW),
 231                 DL(BPF_STX, BPF_XADD, BPF_W),
 232                 DL(BPF_STX, BPF_XADD, BPF_DW),
 233                 DL(BPF_ST, BPF_MEM, BPF_B),
 234                 DL(BPF_ST, BPF_MEM, BPF_H),
 235                 DL(BPF_ST, BPF_MEM, BPF_W),
 236                 DL(BPF_ST, BPF_MEM, BPF_DW),
 237                 DL(BPF_LDX, BPF_MEM, BPF_B),
 238                 DL(BPF_LDX, BPF_MEM, BPF_H),
 239                 DL(BPF_LDX, BPF_MEM, BPF_W),
 240                 DL(BPF_LDX, BPF_MEM, BPF_DW),
 241                 DL(BPF_LD, BPF_ABS, BPF_W),
 242                 DL(BPF_LD, BPF_ABS, BPF_H),
 243                 DL(BPF_LD, BPF_ABS, BPF_B),
 244                 DL(BPF_LD, BPF_IND, BPF_W),
 245                 DL(BPF_LD, BPF_IND, BPF_H),
 246                 DL(BPF_LD, BPF_IND, BPF_B),
 247 #undef DL
 248         };
 249
 250         regs[FP_REG]  = (u64) (unsigned long) &stack[ARRAY_SIZE(stack)];
 251         regs[ARG1_REG] = (u64) (unsigned long) ctx;
 252         regs[A_REG] = 0;
 253         regs[X_REG] = 0;
 254
 255 select_insn:
 256         goto *jumptable[insn->code];
 257
 258         /* ALU */
 259 #define ALU(OPCODE, OP)                 \
 260         BPF_ALU64_##OPCODE##_BPF_X:     \
 261                 A = A OP X;             \
 262                 CONT;                   \
 263         BPF_ALU_##OPCODE##_BPF_X:       \
 264                 A = (u32) A OP (u32) X; \
 265                 CONT;                   \
 266         BPF_ALU64_##OPCODE##_BPF_K:     \
 267                 A = A OP K;             \
 268                 CONT;                   \
 269         BPF_ALU_##OPCODE##_BPF_K:       \
 270                 A = (u32) A OP (u32) K; \
 271                 CONT;
 272
 273         ALU(BPF_ADD,  +)
 274         ALU(BPF_SUB,  -)
 275         ALU(BPF_AND,  &)
 276         ALU(BPF_OR,   |)
 277         ALU(BPF_LSH, <<)
 278         ALU(BPF_RSH, >>)
 279         ALU(BPF_XOR,  ^)
 280         ALU(BPF_MUL,  *)
 281 #undef ALU
 282         BPF_ALU_BPF_NEG_0:
 283                 A = (u32) -A;
 284                 CONT;
 285         BPF_ALU64_BPF_NEG_0:
 286                 A = -A;
 287                 CONT;
 288         BPF_ALU_BPF_MOV_BPF_X:
 289                 A = (u32) X;
 290                 CONT;
 291         BPF_ALU_BPF_MOV_BPF_K:
 292                 A = (u32) K;
 293                 CONT;
 294         BPF_ALU64_BPF_MOV_BPF_X:
 295                 A = X;
 296                 CONT;
 297         BPF_ALU64_BPF_MOV_BPF_K:
 298                 A = K;
 299                 CONT;
 300         BPF_ALU64_BPF_ARSH_BPF_X:
 301                 (*(s64 *) &A) >>= X;
 302                 CONT;
 303         BPF_ALU64_BPF_ARSH_BPF_K:
 304                 (*(s64 *) &A) >>= K;
 305                 CONT;
 306         BPF_ALU64_BPF_MOD_BPF_X:
 307                 if (unlikely(X == 0))
 308                         return 0;
 309                 tmp = A;
 310                 A = do_div(tmp, X);
 311                 CONT;
 312         BPF_ALU_BPF_MOD_BPF_X:
 313                 if (unlikely(X == 0))
 314                         return 0;
 315                 tmp = (u32) A;
 316                 A = do_div(tmp, (u32) X);
 317                 CONT;
 318         BPF_ALU64_BPF_MOD_BPF_K:
 319                 tmp = A;
 320                 A = do_div(tmp, K);
 321                 CONT;
 322         BPF_ALU_BPF_MOD_BPF_K:
 323                 tmp = (u32) A;
 324                 A = do_div(tmp, (u32) K);
 325                 CONT;
 326         BPF_ALU64_BPF_DIV_BPF_X:
 327                 if (unlikely(X == 0))
 328                         return 0;
 329                 do_div(A, X);
 330                 CONT;
 331         BPF_ALU_BPF_DIV_BPF_X:
 332                 if (unlikely(X == 0))
 333                         return 0;
 334                 tmp = (u32) A;
 335                 do_div(tmp, (u32) X);
 336                 A = (u32) tmp;
 337                 CONT;
 338         BPF_ALU64_BPF_DIV_BPF_K:
 339                 do_div(A, K);
 340                 CONT;
 341         BPF_ALU_BPF_DIV_BPF_K:
 342                 tmp = (u32) A;
 343                 do_div(tmp, (u32) K);
 344                 A = (u32) tmp;
 345                 CONT;
 346         BPF_ALU_BPF_END_BPF_TO_BE:
 347                 switch (K) {
 348                 case 16:
 349                         A = (__force u16) cpu_to_be16(A);
 350                         break;
 351                 case 32:
 352                         A = (__force u32) cpu_to_be32(A);
 353                         break;
 354                 case 64:
 355                         A = (__force u64) cpu_to_be64(A);
 356                         break;
 357                 }
 358                 CONT;
 359         BPF_ALU_BPF_END_BPF_TO_LE:
 360                 switch (K) {
 361                 case 16:
 362                         A = (__force u16) cpu_to_le16(A);
 363                         break;
 364                 case 32:
 365                         A = (__force u32) cpu_to_le32(A);
 366                         break;
 367                 case 64:
 368                         A = (__force u64) cpu_to_le64(A);
 369                         break;
 370                 }
 371                 CONT;
 372
 373         /* CALL */
 374         BPF_JMP_BPF_CALL_0:
 375                 /* Function call scratches R1-R5 registers, preserves R6-R9,
 376                  * and stores return value into R0.
 377                  */
 378                 R0 = (__bpf_call_base + insn->imm)(regs[1], regs[2], regs[3],
 379                                                    regs[4], regs[5]);
 380                 CONT;
 381
 382         /* JMP */
 383         BPF_JMP_BPF_JA_0:
 384                 insn += insn->off;
 385                 CONT;
 386         BPF_JMP_BPF_JEQ_BPF_X:
 387                 if (A == X) {
 388                         insn += insn->off;
 389                         CONT_JMP;
 390                 }
 391                 CONT;
 392         BPF_JMP_BPF_JEQ_BPF_K:
 393                 if (A == K) {
 394                         insn += insn->off;
 395                         CONT_JMP;
 396                 }
 397                 CONT;
 398         BPF_JMP_BPF_JNE_BPF_X:
 399                 if (A != X) {
 400                         insn += insn->off;
 401                         CONT_JMP;
 402                 }
 403                 CONT;
 404         BPF_JMP_BPF_JNE_BPF_K:
 405                 if (A != K) {
 406                         insn += insn->off;
 407                         CONT_JMP;
 408                 }
 409                 CONT;
 410         BPF_JMP_BPF_JGT_BPF_X:
 411                 if (A > X) {
 412                         insn += insn->off;
 413                         CONT_JMP;
 414                 }
 415                 CONT;
 416         BPF_JMP_BPF_JGT_BPF_K:
 417                 if (A > K) {
 418                         insn += insn->off;
 419                         CONT_JMP;
 420                 }
 421                 CONT;
 422         BPF_JMP_BPF_JGE_BPF_X:
 423                 if (A >= X) {
 424                         insn += insn->off;
 425                         CONT_JMP;
 426                 }
 427                 CONT;
 428         BPF_JMP_BPF_JGE_BPF_K:
 429                 if (A >= K) {
 430                         insn += insn->off;
 431                         CONT_JMP;
 432                 }
 433                 CONT;
 434         BPF_JMP_BPF_JSGT_BPF_X:
 435                 if (((s64)A) > ((s64)X)) {
 436                         insn += insn->off;
 437                         CONT_JMP;
 438                 }
 439                 CONT;
 440         BPF_JMP_BPF_JSGT_BPF_K:
 441                 if (((s64)A) > ((s64)K)) {
 442                         insn += insn->off;
 443                         CONT_JMP;
 444                 }
 445                 CONT;
 446         BPF_JMP_BPF_JSGE_BPF_X:
 447                 if (((s64)A) >= ((s64)X)) {
 448                         insn += insn->off;
 449                         CONT_JMP;
 450                 }
 451                 CONT;
 452         BPF_JMP_BPF_JSGE_BPF_K:
 453                 if (((s64)A) >= ((s64)K)) {
 454                         insn += insn->off;
 455                         CONT_JMP;
 456                 }
 457                 CONT;
 458         BPF_JMP_BPF_JSET_BPF_X:
 459                 if (A & X) {
 460                         insn += insn->off;
 461                         CONT_JMP;
 462                 }
 463                 CONT;
 464         BPF_JMP_BPF_JSET_BPF_K:
 465                 if (A & K) {
 466                         insn += insn->off;
 467                         CONT_JMP;
 468                 }
 469                 CONT;
 470         BPF_JMP_BPF_EXIT_0:
 471                 return R0;
 472
 473         /* STX and ST and LDX*/
 474 #define LDST(SIZEOP, SIZE)                                      \
 475         BPF_STX_BPF_MEM_##SIZEOP:                               \
 476                 *(SIZE *)(unsigned long) (A + insn->off) = X;   \
 477                 CONT;                                           \
 478         BPF_ST_BPF_MEM_##SIZEOP:                                \
 479                 *(SIZE *)(unsigned long) (A + insn->off) = K;   \
 480                 CONT;                                           \
 481         BPF_LDX_BPF_MEM_##SIZEOP:                               \
 482                 A = *(SIZE *)(unsigned long) (X + insn->off);   \
 483                 CONT;
 484
 485         LDST(BPF_B,   u8)
 486         LDST(BPF_H,  u16)
 487         LDST(BPF_W,  u32)
 488         LDST(BPF_DW, u64)
 489 #undef LDST
 490         BPF_STX_BPF_XADD_BPF_W: /* lock xadd *(u32 *)(A + insn->off) += X */
 491                 atomic_add((u32) X, (atomic_t *)(unsigned long)
 492                            (A + insn->off));
 493                 CONT;
 494         BPF_STX_BPF_XADD_BPF_DW: /* lock xadd *(u64 *)(A + insn->off) += X */
 495                 atomic64_add((u64) X, (atomic64_t *)(unsigned long)
 496                              (A + insn->off));
 497                 CONT;
 498         BPF_LD_BPF_ABS_BPF_W: /* R0 = ntohl(*(u32 *) (skb->data + K)) */
 499                 off = K;
 500 load_word:
 501                 /* BPF_LD + BPD_ABS and BPF_LD + BPF_IND insns are only
 502                  * appearing in the programs where ctx == skb. All programs
 503                  * keep 'ctx' in regs[CTX_REG] == R6, sk_convert_filter()
 504                  * saves it in R6, internal BPF verifier will check that
 505                  * R6 == ctx.
 506                  *
 507                  * BPF_ABS and BPF_IND are wrappers of function calls, so
 508                  * they scratch R1-R5 registers, preserve R6-R9, and store
 509                  * return value into R0.
 510                  *
 511                  * Implicit input:
 512                  *   ctx
 513                  *
 514                  * Explicit input:
 515                  *   X == any register
 516                  *   K == 32-bit immediate
 517                  *
 518                  * Output:
 519                  *   R0 - 8/16/32-bit skb data converted to cpu endianness
 520                  */
 521                 ptr = load_pointer((struct sk_buff *) ctx, off, 4, &tmp);
 522                 if (likely(ptr != NULL)) {
 523                         R0 = get_unaligned_be32(ptr);
 524                         CONT;
 525                 }
 526                 return 0;
 527         BPF_LD_BPF_ABS_BPF_H: /* R0 = ntohs(*(u16 *) (skb->data + K)) */
 528                 off = K;
 529 load_half:
 530                 ptr = load_pointer((struct sk_buff *) ctx, off, 2, &tmp);
 531                 if (likely(ptr != NULL)) {
 532                         R0 = get_unaligned_be16(ptr);
 533                         CONT;
 534                 }
 535                 return 0;
 536         BPF_LD_BPF_ABS_BPF_B: /* R0 = *(u8 *) (ctx + K) */
 537                 off = K;
 538 load_byte:
 539                 ptr = load_pointer((struct sk_buff *) ctx, off, 1, &tmp);
 540                 if (likely(ptr != NULL)) {
 541                         R0 = *(u8 *)ptr;
 542                         CONT;
 543                 }
 544                 return 0;
 545         BPF_LD_BPF_IND_BPF_W: /* R0 = ntohl(*(u32 *) (skb->data + X + K)) */
 546                 off = K + X;
 547                 goto load_word;
 548         BPF_LD_BPF_IND_BPF_H: /* R0 = ntohs(*(u16 *) (skb->data + X + K)) */
 549                 off = K + X;
 550                 goto load_half;
 551         BPF_LD_BPF_IND_BPF_B: /* R0 = *(u8 *) (skb->data + X + K) */
 552                 off = K + X;
 553                 goto load_byte;
 554
 555         default_label:
 556                 /* If we ever reach this, we have a bug somewhere. */
 557                 WARN_RATELIMIT(1, "unknown opcode %02x\n", insn->code);
 558                 return 0;
 559 #undef CONT_JMP
 560 #undef CONT
 561
 562 #undef R0
 563 #undef X
 564 #undef A
 565 #undef K
 566 }
 567
 568 u32 sk_run_filter_int_seccomp(const struct seccomp_data *ctx,
 569                               const struct sock_filter_int *insni)
 570     __attribute__ ((alias ("__sk_run_filter")));
 571
 572 u32 sk_run_filter_int_skb(const struct sk_buff *ctx,
 573                           const struct sock_filter_int *insni)
 574     __attribute__ ((alias ("__sk_run_filter")));
 575 EXPORT_SYMBOL_GPL(sk_run_filter_int_skb);
 576
 577 /* Helper to find the offset of pkt_type in sk_buff structure. We want
 578  * to make sure its still a 3bit field starting at a byte boundary;
 579  * taken from arch/x86/net/bpf_jit_comp.c.
 580  */
 581 #define PKT_TYPE_MAX    7
 582 static unsigned int pkt_type_offset(void)
 583 {
 584         struct sk_buff skb_probe = { .pkt_type = ~0, };
 585         u8 *ct = (u8 *) &skb_probe;
 586         unsigned int off;
 587
 588         for (off = 0; off < sizeof(struct sk_buff); off++) {
 589                 if (ct[off] == PKT_TYPE_MAX)
 590                         return off;
 591         }
 592
 593         pr_err_once("Please fix %s, as pkt_type couldn't be found!\n", __func__);
 594         return -1;
 595 }
 596
 597 static u64 __skb_get_pay_offset(u64 ctx, u64 A, u64 X, u64 r4, u64 r5)
 598 {
 599         struct sk_buff *skb = (struct sk_buff *)(long) ctx;
 600
 601         return __skb_get_poff(skb);
 602 }
 603
 604 static u64 __skb_get_nlattr(u64 ctx, u64 A, u64 X, u64 r4, u64 r5)
 605 {
 606         struct sk_buff *skb = (struct sk_buff *)(long) ctx;
 607         struct nlattr *nla;
 608
 609         if (skb_is_nonlinear(skb))
 610                 return 0;
 611
 612         if (skb->len < sizeof(struct nlattr))
 613                 return 0;
 614
 615         if (A > skb->len - sizeof(struct nlattr))
 616                 return 0;
 617
 618         nla = nla_find((struct nlattr *) &skb->data[A], skb->len - A, X);
 619         if (nla)
 620                 return (void *) nla - (void *) skb->data;
 621
 622         return 0;
 623 }
 624
 625 static u64 __skb_get_nlattr_nest(u64 ctx, u64 A, u64 X, u64 r4, u64 r5)
 626 {
 627         struct sk_buff *skb = (struct sk_buff *)(long) ctx;
 628         struct nlattr *nla;
 629
 630         if (skb_is_nonlinear(skb))
 631                 return 0;
 632
 633         if (skb->len < sizeof(struct nlattr))
 634                 return 0;
 635
 636         if (A > skb->len - sizeof(struct nlattr))
 637                 return 0;
 638
 639         nla = (struct nlattr *) &skb->data[A];
 640         if (nla->nla_len > skb->len - A)
 641                 return 0;
 642
 643         nla = nla_find_nested(nla, X);
 644         if (nla)
 645                 return (void *) nla - (void *) skb->data;
 646
 647         return 0;
 648 }
 649
 650 static u64 __get_raw_cpu_id(u64 ctx, u64 A, u64 X, u64 r4, u64 r5)
 651 {
 652         return raw_smp_processor_id();
 653 }
 654
 655 static bool convert_bpf_extensions(struct sock_filter *fp,
 656                                    struct sock_filter_int **insnp)
 657 {
 658         struct sock_filter_int *insn = *insnp;
 659
 660         switch (fp->k) {
 661         case SKF_AD_OFF + SKF_AD_PROTOCOL:
 662                 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, protocol) != 2);
 663
 664                 insn->code = BPF_LDX | BPF_MEM | BPF_H;
 665                 insn->a_reg = A_REG;
 666                 insn->x_reg = CTX_REG;
 667                 insn->off = offsetof(struct sk_buff, protocol);
 668                 insn++;
 669
 670                 /* A = ntohs(A) [emitting a nop or swap16] */
 671                 insn->code = BPF_ALU | BPF_END | BPF_FROM_BE;
 672                 insn->a_reg = A_REG;
 673                 insn->imm = 16;
 674                 break;
 675
 676         case SKF_AD_OFF + SKF_AD_PKTTYPE:
 677                 insn->code = BPF_LDX | BPF_MEM | BPF_B;
 678                 insn->a_reg = A_REG;
 679                 insn->x_reg = CTX_REG;
 680                 insn->off = pkt_type_offset();
 681                 if (insn->off < 0)
 682                         return false;
 683                 insn++;
 684
 685                 insn->code = BPF_ALU | BPF_AND | BPF_K;
 686                 insn->a_reg = A_REG;
 687                 insn->imm = PKT_TYPE_MAX;
 688                 break;
 689
 690         case SKF_AD_OFF + SKF_AD_IFINDEX:
 691         case SKF_AD_OFF + SKF_AD_HATYPE:
 692                 if (FIELD_SIZEOF(struct sk_buff, dev) == 8)
 693                         insn->code = BPF_LDX | BPF_MEM | BPF_DW;
 694                 else
 695                         insn->code = BPF_LDX | BPF_MEM | BPF_W;
 696                 insn->a_reg = TMP_REG;
 697                 insn->x_reg = CTX_REG;
 698                 insn->off = offsetof(struct sk_buff, dev);
 699                 insn++;
 700
 701                 insn->code = BPF_JMP | BPF_JNE | BPF_K;
 702                 insn->a_reg = TMP_REG;
 703                 insn->imm = 0;
 704                 insn->off = 1;
 705                 insn++;
 706
 707                 insn->code = BPF_JMP | BPF_EXIT;
 708                 insn++;
 709
 710                 BUILD_BUG_ON(FIELD_SIZEOF(struct net_device, ifindex) != 4);
 711                 BUILD_BUG_ON(FIELD_SIZEOF(struct net_device, type) != 2);
 712
 713                 insn->a_reg = A_REG;
 714                 insn->x_reg = TMP_REG;
 715
 716                 if (fp->k == SKF_AD_OFF + SKF_AD_IFINDEX) {
 717                         insn->code = BPF_LDX | BPF_MEM | BPF_W;
 718                         insn->off = offsetof(struct net_device, ifindex);
 719                 } else {
 720                         insn->code = BPF_LDX | BPF_MEM | BPF_H;
 721                         insn->off = offsetof(struct net_device, type);
 722                 }
 723                 break;
 724
 725         case SKF_AD_OFF + SKF_AD_MARK:
 726                 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, mark) != 4);
 727
 728                 insn->code = BPF_LDX | BPF_MEM | BPF_W;
 729                 insn->a_reg = A_REG;
 730                 insn->x_reg = CTX_REG;
 731                 insn->off = offsetof(struct sk_buff, mark);
 732                 break;
 733
 734         case SKF_AD_OFF + SKF_AD_RXHASH:
 735                 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, hash) != 4);
 736
 737                 insn->code = BPF_LDX | BPF_MEM | BPF_W;
 738                 insn->a_reg = A_REG;
 739                 insn->x_reg = CTX_REG;
 740                 insn->off = offsetof(struct sk_buff, hash);
 741                 break;
 742
 743         case SKF_AD_OFF + SKF_AD_QUEUE:
 744                 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, queue_mapping) != 2);
 745
 746                 insn->code = BPF_LDX | BPF_MEM | BPF_H;
 747                 insn->a_reg = A_REG;
 748                 insn->x_reg = CTX_REG;
 749                 insn->off = offsetof(struct sk_buff, queue_mapping);
 750                 break;
 751
 752         case SKF_AD_OFF + SKF_AD_VLAN_TAG:
 753         case SKF_AD_OFF + SKF_AD_VLAN_TAG_PRESENT:
 754                 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, vlan_tci) != 2);
 755
 756                 insn->code = BPF_LDX | BPF_MEM | BPF_H;
 757                 insn->a_reg = A_REG;
 758                 insn->x_reg = CTX_REG;
 759                 insn->off = offsetof(struct sk_buff, vlan_tci);
 760                 insn++;
 761
 762                 BUILD_BUG_ON(VLAN_TAG_PRESENT != 0x1000);
 763
 764                 if (fp->k == SKF_AD_OFF + SKF_AD_VLAN_TAG) {
 765                         insn->code = BPF_ALU | BPF_AND | BPF_K;
 766                         insn->a_reg = A_REG;
 767                         insn->imm = ~VLAN_TAG_PRESENT;
 768                 } else {
 769                         insn->code = BPF_ALU | BPF_RSH | BPF_K;
 770                         insn->a_reg = A_REG;
 771                         insn->imm = 12;
 772                         insn++;
 773
 774                         insn->code = BPF_ALU | BPF_AND | BPF_K;
 775                         insn->a_reg = A_REG;
 776                         insn->imm = 1;
 777                 }
 778                 break;
 779
 780         case SKF_AD_OFF + SKF_AD_PAY_OFFSET:
 781         case SKF_AD_OFF + SKF_AD_NLATTR:
 782         case SKF_AD_OFF + SKF_AD_NLATTR_NEST:
 783         case SKF_AD_OFF + SKF_AD_CPU:
 784                 /* arg1 = ctx */
 785                 insn->code = BPF_ALU64 | BPF_MOV | BPF_X;
 786                 insn->a_reg = ARG1_REG;
 787                 insn->x_reg = CTX_REG;
 788                 insn++;
 789
 790                 /* arg2 = A */
 791                 insn->code = BPF_ALU64 | BPF_MOV | BPF_X;
 792                 insn->a_reg = ARG2_REG;
 793                 insn->x_reg = A_REG;
 794                 insn++;
 795
 796                 /* arg3 = X */
 797                 insn->code = BPF_ALU64 | BPF_MOV | BPF_X;
 798                 insn->a_reg = ARG3_REG;
 799                 insn->x_reg = X_REG;
 800                 insn++;
 801
 802                 /* Emit call(ctx, arg2=A, arg3=X) */
 803                 insn->code = BPF_JMP | BPF_CALL;
 804                 switch (fp->k) {
 805                 case SKF_AD_OFF + SKF_AD_PAY_OFFSET:
 806                         insn->imm = __skb_get_pay_offset - __bpf_call_base;
 807                         break;
 808                 case SKF_AD_OFF + SKF_AD_NLATTR:
 809                         insn->imm = __skb_get_nlattr - __bpf_call_base;
 810                         break;
 811                 case SKF_AD_OFF + SKF_AD_NLATTR_NEST:
 812                         insn->imm = __skb_get_nlattr_nest - __bpf_call_base;
 813                         break;
 814                 case SKF_AD_OFF + SKF_AD_CPU:
 815                         insn->imm = __get_raw_cpu_id - __bpf_call_base;
 816                         break;
 817                 }
 818                 break;
 819
 820         case SKF_AD_OFF + SKF_AD_ALU_XOR_X:
 821                 insn->code = BPF_ALU | BPF_XOR | BPF_X;
 822                 insn->a_reg = A_REG;
 823                 insn->x_reg = X_REG;
 824                 break;
 825
 826         default:
 827                 /* This is just a dummy call to avoid letting the compiler
 828                  * evict __bpf_call_base() as an optimization. Placed here
 829                  * where no-one bothers.
 830                  */
 831                 BUG_ON(__bpf_call_base(0, 0, 0, 0, 0) != 0);
 832                 return false;
 833         }
 834
 835         *insnp = insn;
 836         return true;
 837 }
 838
 839 /**
 840  *      sk_convert_filter - convert filter program
 841  *      @prog: the user passed filter program
 842  *      @len: the length of the user passed filter program
 843  *      @new_prog: buffer where converted program will be stored
 844  *      @new_len: pointer to store length of converted program
 845  *
 846  * Remap 'sock_filter' style BPF instruction set to 'sock_filter_ext' style.
 847  * Conversion workflow:
 848  *
 849  * 1) First pass for calculating the new program length:
 850  *   sk_convert_filter(old_prog, old_len, NULL, &new_len)
 851  *
 852  * 2) 2nd pass to remap in two passes: 1st pass finds new
 853  *    jump offsets, 2nd pass remapping:
 854  *   new_prog = kmalloc(sizeof(struct sock_filter_int) * new_len);
 855  *   sk_convert_filter(old_prog, old_len, new_prog, &new_len);
 856  *
 857  * User BPF's register A is mapped to our BPF register 6, user BPF
 858  * register X is mapped to BPF register 7; frame pointer is always
 859  * register 10; Context 'void *ctx' is stored in register 1, that is,
 860  * for socket filters: ctx == 'struct sk_buff *', for seccomp:
 861  * ctx == 'struct seccomp_data *'.
 862  */
 863 int sk_convert_filter(struct sock_filter *prog, int len,
 864                       struct sock_filter_int *new_prog, int *new_len)
 865 {
 866         int new_flen = 0, pass = 0, target, i;
 867         struct sock_filter_int *new_insn;
 868         struct sock_filter *fp;
 869         int *addrs = NULL;
 870         u8 bpf_src;
 871
 872         BUILD_BUG_ON(BPF_MEMWORDS * sizeof(u32) > MAX_BPF_STACK);
 873         BUILD_BUG_ON(FP_REG + 1 != MAX_BPF_REG);
 874
 875         if (len <= 0 || len >= BPF_MAXINSNS)
 876                 return -EINVAL;
 877
 878         if (new_prog) {
 879                 addrs = kzalloc(len * sizeof(*addrs), GFP_KERNEL);
 880                 if (!addrs)
 881                         return -ENOMEM;
 882         }
 883
 884 do_pass:
 885         new_insn = new_prog;
 886         fp = prog;
 887
 888         if (new_insn) {
 889                 new_insn->code = BPF_ALU64 | BPF_MOV | BPF_X;
 890                 new_insn->a_reg = CTX_REG;
 891                 new_insn->x_reg = ARG1_REG;
 892         }
 893         new_insn++;
 894
 895         for (i = 0; i < len; fp++, i++) {
 896                 struct sock_filter_int tmp_insns[6] = { };
 897                 struct sock_filter_int *insn = tmp_insns;
 898
 899                 if (addrs)
 900                         addrs[i] = new_insn - new_prog;
 901
 902                 switch (fp->code) {
 903                 /* All arithmetic insns and skb loads map as-is. */
 904                 case BPF_ALU | BPF_ADD | BPF_X:
 905                 case BPF_ALU | BPF_ADD | BPF_K:
 906                 case BPF_ALU | BPF_SUB | BPF_X:
 907                 case BPF_ALU | BPF_SUB | BPF_K:
 908                 case BPF_ALU | BPF_AND | BPF_X:
 909                 case BPF_ALU | BPF_AND | BPF_K:
 910                 case BPF_ALU | BPF_OR | BPF_X:
 911                 case BPF_ALU | BPF_OR | BPF_K:
 912                 case BPF_ALU | BPF_LSH | BPF_X:
 913                 case BPF_ALU | BPF_LSH | BPF_K:
 914                 case BPF_ALU | BPF_RSH | BPF_X:
 915                 case BPF_ALU | BPF_RSH | BPF_K:
 916                 case BPF_ALU | BPF_XOR | BPF_X:
 917                 case BPF_ALU | BPF_XOR | BPF_K:
 918                 case BPF_ALU | BPF_MUL | BPF_X:
 919                 case BPF_ALU | BPF_MUL | BPF_K:
 920                 case BPF_ALU | BPF_DIV | BPF_X:
 921                 case BPF_ALU | BPF_DIV | BPF_K:
 922                 case BPF_ALU | BPF_MOD | BPF_X:
 923                 case BPF_ALU | BPF_MOD | BPF_K:
 924                 case BPF_ALU | BPF_NEG:
 925                 case BPF_LD | BPF_ABS | BPF_W:
 926                 case BPF_LD | BPF_ABS | BPF_H:
 927                 case BPF_LD | BPF_ABS | BPF_B:
 928                 case BPF_LD | BPF_IND | BPF_W:
 929                 case BPF_LD | BPF_IND | BPF_H:
 930                 case BPF_LD | BPF_IND | BPF_B:
 931                         /* Check for overloaded BPF extension and
 932                          * directly convert it if found, otherwise
 933                          * just move on with mapping.
 934                          */
 935                         if (BPF_CLASS(fp->code) == BPF_LD &&
 936                             BPF_MODE(fp->code) == BPF_ABS &&
 937                             convert_bpf_extensions(fp, &insn))
 938                                 break;
 939
 940                         insn->code = fp->code;
 941                         insn->a_reg = A_REG;
 942                         insn->x_reg = X_REG;
 943                         insn->imm = fp->k;
 944                         break;
 945
 946                 /* Jump opcodes map as-is, but offsets need adjustment. */
 947                 case BPF_JMP | BPF_JA:
 948                         target = i + fp->k + 1;
 949                         insn->code = fp->code;
 950 #define EMIT_JMP                                                        \
 951         do {                                                            \
 952                 if (target >= len || target < 0)                        \
 953                         goto err;                                       \
 954                 insn->off = addrs ? addrs[target] - addrs[i] - 1 : 0;   \
 955                 /* Adjust pc relative offset for 2nd or 3rd insn. */    \
 956                 insn->off -= insn - tmp_insns;                          \
 957         } while (0)
 958
 959                         EMIT_JMP;
 960                         break;
 961
 962                 case BPF_JMP | BPF_JEQ | BPF_K:
 963                 case BPF_JMP | BPF_JEQ | BPF_X:
 964                 case BPF_JMP | BPF_JSET | BPF_K:
 965                 case BPF_JMP | BPF_JSET | BPF_X:
 966                 case BPF_JMP | BPF_JGT | BPF_K:
 967                 case BPF_JMP | BPF_JGT | BPF_X:
 968                 case BPF_JMP | BPF_JGE | BPF_K:
 969                 case BPF_JMP | BPF_JGE | BPF_X:
 970                         if (BPF_SRC(fp->code) == BPF_K && (int) fp->k < 0) {
 971                                 /* BPF immediates are signed, zero extend
 972                                  * immediate into tmp register and use it
 973                                  * in compare insn.
 974                                  */
 975                                 insn->code = BPF_ALU | BPF_MOV | BPF_K;
 976                                 insn->a_reg = TMP_REG;
 977                                 insn->imm = fp->k;
 978                                 insn++;
 979
 980                                 insn->a_reg = A_REG;
 981                                 insn->x_reg = TMP_REG;
 982                                 bpf_src = BPF_X;
 983                         } else {
 984                                 insn->a_reg = A_REG;
 985                                 insn->x_reg = X_REG;
 986                                 insn->imm = fp->k;
 987                                 bpf_src = BPF_SRC(fp->code);
 988                         }
 989
 990                         /* Common case where 'jump_false' is next insn. */
 991                         if (fp->jf == 0) {
 992                                 insn->code = BPF_JMP | BPF_OP(fp->code) | bpf_src;
 993                                 target = i + fp->jt + 1;
 994                                 EMIT_JMP;
 995                                 break;
 996                         }
 997
 998                         /* Convert JEQ into JNE when 'jump_true' is next insn. */
 999                         if (fp->jt == 0 && BPF_OP(fp->code) == BPF_JEQ) {
1000                                 insn->code = BPF_JMP | BPF_JNE | bpf_src;
1001                                 target = i + fp->jf + 1;
1002                                 EMIT_JMP;
1003                                 break;
1004                         }
1005
1006                         /* Other jumps are mapped into two insns: Jxx and JA. */
1007                         target = i + fp->jt + 1;
1008                         insn->code = BPF_JMP | BPF_OP(fp->code) | bpf_src;
1009                         EMIT_JMP;
1010                         insn++;
1011
1012                         insn->code = BPF_JMP | BPF_JA;
1013                         target = i + fp->jf + 1;
1014                         EMIT_JMP;
1015                         break;
1016
1017                 /* ldxb 4 * ([14] & 0xf) is remaped into 6 insns. */
1018                 case BPF_LDX | BPF_MSH | BPF_B:
1019                         insn->code = BPF_ALU64 | BPF_MOV | BPF_X;
1020                         insn->a_reg = TMP_REG;
1021                         insn->x_reg = A_REG;
1022                         insn++;
1023
1024                         insn->code = BPF_LD | BPF_ABS | BPF_B;
1025                         insn->a_reg = A_REG;
1026                         insn->imm = fp->k;
1027                         insn++;
1028
1029                         insn->code = BPF_ALU | BPF_AND | BPF_K;
1030                         insn->a_reg = A_REG;
1031                         insn->imm = 0xf;
1032                         insn++;
1033
1034                         insn->code = BPF_ALU | BPF_LSH | BPF_K;
1035                         insn->a_reg = A_REG;
1036                         insn->imm = 2;
1037                         insn++;
1038
1039                         insn->code = BPF_ALU64 | BPF_MOV | BPF_X;
1040                         insn->a_reg = X_REG;
1041                         insn->x_reg = A_REG;
1042                         insn++;
1043
1044                         insn->code = BPF_ALU64 | BPF_MOV | BPF_X;
1045                         insn->a_reg = A_REG;
1046                         insn->x_reg = TMP_REG;
1047                         break;
1048
1049                 /* RET_K, RET_A are remaped into 2 insns. */
1050                 case BPF_RET | BPF_A:
1051                 case BPF_RET | BPF_K:
1052                         insn->code = BPF_ALU | BPF_MOV |
1053                                      (BPF_RVAL(fp->code) == BPF_K ?
1054                                       BPF_K : BPF_X);
1055                         insn->a_reg = 0;
1056                         insn->x_reg = A_REG;
1057                         insn->imm = fp->k;
1058                         insn++;
1059
1060                         insn->code = BPF_JMP | BPF_EXIT;
1061                         break;
1062
1063                 /* Store to stack. */
1064                 case BPF_ST:
1065                 case BPF_STX:
1066                         insn->code = BPF_STX | BPF_MEM | BPF_W;
1067                         insn->a_reg = FP_REG;
1068                         insn->x_reg = fp->code == BPF_ST ? A_REG : X_REG;
1069                         insn->off = -(BPF_MEMWORDS - fp->k) * 4;
1070                         break;
1071
1072                 /* Load from stack. */
1073                 case BPF_LD | BPF_MEM:
1074                 case BPF_LDX | BPF_MEM:
1075                         insn->code = BPF_LDX | BPF_MEM | BPF_W;
1076                         insn->a_reg = BPF_CLASS(fp->code) == BPF_LD ?
1077                                       A_REG : X_REG;
1078                         insn->x_reg = FP_REG;
1079                         insn->off = -(BPF_MEMWORDS - fp->k) * 4;
1080                         break;
1081
1082                 /* A = K or X = K */
1083                 case BPF_LD | BPF_IMM:
1084                 case BPF_LDX | BPF_IMM:
1085                         insn->code = BPF_ALU | BPF_MOV | BPF_K;
1086                         insn->a_reg = BPF_CLASS(fp->code) == BPF_LD ?
1087                                       A_REG : X_REG;
1088                         insn->imm = fp->k;
1089                         break;
1090
1091                 /* X = A */
1092                 case BPF_MISC | BPF_TAX:
1093                         insn->code = BPF_ALU64 | BPF_MOV | BPF_X;
1094                         insn->a_reg = X_REG;
1095                         insn->x_reg = A_REG;
1096                         break;
1097
1098                 /* A = X */
1099                 case BPF_MISC | BPF_TXA:
1100                         insn->code = BPF_ALU64 | BPF_MOV | BPF_X;
1101                         insn->a_reg = A_REG;
1102                         insn->x_reg = X_REG;
1103                         break;
1104
1105                 /* A = skb->len or X = skb->len */
1106                 case BPF_LD | BPF_W | BPF_LEN:
1107                 case BPF_LDX | BPF_W | BPF_LEN:
1108                         insn->code = BPF_LDX | BPF_MEM | BPF_W;
1109                         insn->a_reg = BPF_CLASS(fp->code) == BPF_LD ?
1110                                       A_REG : X_REG;
1111                         insn->x_reg = CTX_REG;
1112                         insn->off = offsetof(struct sk_buff, len);
1113                         break;
1114
1115                 /* access seccomp_data fields */
1116                 case BPF_LDX | BPF_ABS | BPF_W:
1117                         insn->code = BPF_LDX | BPF_MEM | BPF_W;
1118                         insn->a_reg = A_REG;
1119                         insn->x_reg = CTX_REG;
1120                         insn->off = fp->k;
1121                         break;
1122
1123                 default:
1124                         goto err;
1125                 }
1126
1127                 insn++;
1128                 if (new_prog)
1129                         memcpy(new_insn, tmp_insns,
1130                                sizeof(*insn) * (insn - tmp_insns));
1131
1132                 new_insn += insn - tmp_insns;
1133         }
1134
1135         if (!new_prog) {
1136                 /* Only calculating new length. */
1137                 *new_len = new_insn - new_prog;
1138                 return 0;
1139         }
1140
1141         pass++;
1142         if (new_flen != new_insn - new_prog) {
1143                 new_flen = new_insn - new_prog;
1144                 if (pass > 2)
1145                         goto err;
1146
1147                 goto do_pass;
1148         }
1149
1150         kfree(addrs);
1151         BUG_ON(*new_len != new_flen);
1152         return 0;
1153 err:
1154         kfree(addrs);
1155         return -EINVAL;
1156 }
1157
1158 /* Security:
1159  *
1160  * A BPF program is able to use 16 cells of memory to store intermediate
1161  * values (check u32 mem[BPF_MEMWORDS] in sk_run_filter()).
1162  *
1163  * As we dont want to clear mem[] array for each packet going through
1164  * sk_run_filter(), we check that filter loaded by user never try to read
1165  * a cell if not previously written, and we check all branches to be sure
1166  * a malicious user doesn't try to abuse us.
1167  */
1168 static int check_load_and_stores(struct sock_filter *filter, int flen)
1169 {
1170         u16 *masks, memvalid = 0; /* one bit per cell, 16 cells */
1171         int pc, ret = 0;
1172
1173         BUILD_BUG_ON(BPF_MEMWORDS > 16);
1174         masks = kmalloc(flen * sizeof(*masks), GFP_KERNEL);
1175         if (!masks)
1176                 return -ENOMEM;
1177         memset(masks, 0xff, flen * sizeof(*masks));
1178
1179         for (pc = 0; pc < flen; pc++) {
1180                 memvalid &= masks[pc];
1181
1182                 switch (filter[pc].code) {
1183                 case BPF_S_ST:
1184                 case BPF_S_STX:
1185                         memvalid |= (1 << filter[pc].k);
1186                         break;
1187                 case BPF_S_LD_MEM:
1188                 case BPF_S_LDX_MEM:
1189                         if (!(memvalid & (1 << filter[pc].k))) {
1190                                 ret = -EINVAL;
1191                                 goto error;
1192                         }
1193                         break;
1194                 case BPF_S_JMP_JA:
1195                         /* a jump must set masks on target */
1196                         masks[pc + 1 + filter[pc].k] &= memvalid;
1197                         memvalid = ~0;
1198                         break;
1199                 case BPF_S_JMP_JEQ_K:
1200                 case BPF_S_JMP_JEQ_X:
1201                 case BPF_S_JMP_JGE_K:
1202                 case BPF_S_JMP_JGE_X:
1203                 case BPF_S_JMP_JGT_K:
1204                 case BPF_S_JMP_JGT_X:
1205                 case BPF_S_JMP_JSET_X:
1206                 case BPF_S_JMP_JSET_K:
1207                         /* a jump must set masks on targets */
1208                         masks[pc + 1 + filter[pc].jt] &= memvalid;
1209                         masks[pc + 1 + filter[pc].jf] &= memvalid;
1210                         memvalid = ~0;
1211                         break;
1212                 }
1213         }
1214 error:
1215         kfree(masks);
1216         return ret;
1217 }
1218
1219 /**
1220  *      sk_chk_filter - verify socket filter code
1221  *      @filter: filter to verify
1222  *      @flen: length of filter
1223  *
1224  * Check the user's filter code. If we let some ugly
1225  * filter code slip through kaboom! The filter must contain
1226  * no references or jumps that are out of range, no illegal
1227  * instructions, and must end with a RET instruction.
1228  *
1229  * All jumps are forward as they are not signed.
1230  *
1231  * Returns 0 if the rule set is legal or -EINVAL if not.
1232  */
1233 int sk_chk_filter(struct sock_filter *filter, unsigned int flen)
1234 {
1235         /*
1236          * Valid instructions are initialized to non-0.
1237          * Invalid instructions are initialized to 0.
1238          */
1239         static const u8 codes[] = {
1240                 [BPF_ALU|BPF_ADD|BPF_K]  = BPF_S_ALU_ADD_K,
1241                 [BPF_ALU|BPF_ADD|BPF_X]  = BPF_S_ALU_ADD_X,
1242                 [BPF_ALU|BPF_SUB|BPF_K]  = BPF_S_ALU_SUB_K,
1243                 [BPF_ALU|BPF_SUB|BPF_X]  = BPF_S_ALU_SUB_X,
1244                 [BPF_ALU|BPF_MUL|BPF_K]  = BPF_S_ALU_MUL_K,
1245                 [BPF_ALU|BPF_MUL|BPF_X]  = BPF_S_ALU_MUL_X,
1246                 [BPF_ALU|BPF_DIV|BPF_X]  = BPF_S_ALU_DIV_X,
1247                 [BPF_ALU|BPF_MOD|BPF_K]  = BPF_S_ALU_MOD_K,
1248                 [BPF_ALU|BPF_MOD|BPF_X]  = BPF_S_ALU_MOD_X,
1249                 [BPF_ALU|BPF_AND|BPF_K]  = BPF_S_ALU_AND_K,
1250                 [BPF_ALU|BPF_AND|BPF_X]  = BPF_S_ALU_AND_X,
1251                 [BPF_ALU|BPF_OR|BPF_K]   = BPF_S_ALU_OR_K,
1252                 [BPF_ALU|BPF_OR|BPF_X]   = BPF_S_ALU_OR_X,
1253                 [BPF_ALU|BPF_XOR|BPF_K]  = BPF_S_ALU_XOR_K,
1254                 [BPF_ALU|BPF_XOR|BPF_X]  = BPF_S_ALU_XOR_X,
1255                 [BPF_ALU|BPF_LSH|BPF_K]  = BPF_S_ALU_LSH_K,
1256                 [BPF_ALU|BPF_LSH|BPF_X]  = BPF_S_ALU_LSH_X,
1257                 [BPF_ALU|BPF_RSH|BPF_K]  = BPF_S_ALU_RSH_K,
1258                 [BPF_ALU|BPF_RSH|BPF_X]  = BPF_S_ALU_RSH_X,
1259                 [BPF_ALU|BPF_NEG]        = BPF_S_ALU_NEG,
1260                 [BPF_LD|BPF_W|BPF_ABS]   = BPF_S_LD_W_ABS,
1261                 [BPF_LD|BPF_H|BPF_ABS]   = BPF_S_LD_H_ABS,
1262                 [BPF_LD|BPF_B|BPF_ABS]   = BPF_S_LD_B_ABS,
1263                 [BPF_LD|BPF_W|BPF_LEN]   = BPF_S_LD_W_LEN,
1264                 [BPF_LD|BPF_W|BPF_IND]   = BPF_S_LD_W_IND,
1265                 [BPF_LD|BPF_H|BPF_IND]   = BPF_S_LD_H_IND,
1266                 [BPF_LD|BPF_B|BPF_IND]   = BPF_S_LD_B_IND,
1267                 [BPF_LD|BPF_IMM]         = BPF_S_LD_IMM,
1268                 [BPF_LDX|BPF_W|BPF_LEN]  = BPF_S_LDX_W_LEN,
1269                 [BPF_LDX|BPF_B|BPF_MSH]  = BPF_S_LDX_B_MSH,
1270                 [BPF_LDX|BPF_IMM]        = BPF_S_LDX_IMM,
1271                 [BPF_MISC|BPF_TAX]       = BPF_S_MISC_TAX,
1272                 [BPF_MISC|BPF_TXA]       = BPF_S_MISC_TXA,
1273                 [BPF_RET|BPF_K]          = BPF_S_RET_K,
1274                 [BPF_RET|BPF_A]          = BPF_S_RET_A,
1275                 [BPF_ALU|BPF_DIV|BPF_K]  = BPF_S_ALU_DIV_K,
1276                 [BPF_LD|BPF_MEM]         = BPF_S_LD_MEM,
1277                 [BPF_LDX|BPF_MEM]        = BPF_S_LDX_MEM,
1278                 [BPF_ST]                 = BPF_S_ST,
1279                 [BPF_STX]                = BPF_S_STX,
1280                 [BPF_JMP|BPF_JA]         = BPF_S_JMP_JA,
1281                 [BPF_JMP|BPF_JEQ|BPF_K]  = BPF_S_JMP_JEQ_K,
1282                 [BPF_JMP|BPF_JEQ|BPF_X]  = BPF_S_JMP_JEQ_X,
1283                 [BPF_JMP|BPF_JGE|BPF_K]  = BPF_S_JMP_JGE_K,
1284                 [BPF_JMP|BPF_JGE|BPF_X]  = BPF_S_JMP_JGE_X,
1285                 [BPF_JMP|BPF_JGT|BPF_K]  = BPF_S_JMP_JGT_K,
1286                 [BPF_JMP|BPF_JGT|BPF_X]  = BPF_S_JMP_JGT_X,
1287                 [BPF_JMP|BPF_JSET|BPF_K] = BPF_S_JMP_JSET_K,
1288                 [BPF_JMP|BPF_JSET|BPF_X] = BPF_S_JMP_JSET_X,
1289         };
1290         int pc;
1291         bool anc_found;
1292
1293         if (flen == 0 || flen > BPF_MAXINSNS)
1294                 return -EINVAL;
1295
1296         /* check the filter code now */
1297         for (pc = 0; pc < flen; pc++) {
1298                 struct sock_filter *ftest = &filter[pc];
1299                 u16 code = ftest->code;
1300
1301                 if (code >= ARRAY_SIZE(codes))
1302                         return -EINVAL;
1303                 code = codes[code];
1304                 if (!code)
1305                         return -EINVAL;
1306                 /* Some instructions need special checks */
1307                 switch (code) {
1308                 case BPF_S_ALU_DIV_K:
1309                 case BPF_S_ALU_MOD_K:
1310                         /* check for division by zero */
1311                         if (ftest->k == 0)
1312                                 return -EINVAL;
1313                         break;
1314                 case BPF_S_LD_MEM:
1315                 case BPF_S_LDX_MEM:
1316                 case BPF_S_ST:
1317                 case BPF_S_STX:
1318                         /* check for invalid memory addresses */
1319                         if (ftest->k >= BPF_MEMWORDS)
1320                                 return -EINVAL;
1321                         break;
1322                 case BPF_S_JMP_JA:
1323                         /*
1324                          * Note, the large ftest->k might cause loops.
1325                          * Compare this with conditional jumps below,
1326                          * where offsets are limited. --ANK (981016)
1327                          */
1328                         if (ftest->k >= (unsigned int)(flen-pc-1))
1329                                 return -EINVAL;
1330                         break;
1331                 case BPF_S_JMP_JEQ_K:
1332                 case BPF_S_JMP_JEQ_X:
1333                 case BPF_S_JMP_JGE_K:
1334                 case BPF_S_JMP_JGE_X:
1335                 case BPF_S_JMP_JGT_K:
1336                 case BPF_S_JMP_JGT_X:
1337                 case BPF_S_JMP_JSET_X:
1338                 case BPF_S_JMP_JSET_K:
1339                         /* for conditionals both must be safe */
1340                         if (pc + ftest->jt + 1 >= flen ||
1341                             pc + ftest->jf + 1 >= flen)
1342                                 return -EINVAL;
1343                         break;
1344                 case BPF_S_LD_W_ABS:
1345                 case BPF_S_LD_H_ABS:
1346                 case BPF_S_LD_B_ABS:
1347                         anc_found = false;
1348 #define ANCILLARY(CODE) case SKF_AD_OFF + SKF_AD_##CODE:        \
1349                                 code = BPF_S_ANC_##CODE;        \
1350                                 anc_found = true;               \
1351                                 break
1352                         switch (ftest->k) {
1353                         ANCILLARY(PROTOCOL);
1354                         ANCILLARY(PKTTYPE);
1355                         ANCILLARY(IFINDEX);
1356                         ANCILLARY(NLATTR);
1357                         ANCILLARY(NLATTR_NEST);
1358                         ANCILLARY(MARK);
1359                         ANCILLARY(QUEUE);
1360                         ANCILLARY(HATYPE);
1361                         ANCILLARY(RXHASH);
1362                         ANCILLARY(CPU);
1363                         ANCILLARY(ALU_XOR_X);
1364                         ANCILLARY(VLAN_TAG);
1365                         ANCILLARY(VLAN_TAG_PRESENT);
1366                         ANCILLARY(PAY_OFFSET);
1367                         }
1368
1369                         /* ancillary operation unknown or unsupported */
1370                         if (anc_found == false && ftest->k >= SKF_AD_OFF)
1371                                 return -EINVAL;
1372                 }
1373                 ftest->code = code;
1374         }
1375
1376         /* last instruction must be a RET code */
1377         switch (filter[flen - 1].code) {
1378         case BPF_S_RET_K:
1379         case BPF_S_RET_A:
1380                 return check_load_and_stores(filter, flen);
1381         }
1382         return -EINVAL;
1383 }
1384 EXPORT_SYMBOL(sk_chk_filter);
1385
1386 static int sk_store_orig_filter(struct sk_filter *fp,
1387                                 const struct sock_fprog *fprog)
1388 {
1389         unsigned int fsize = sk_filter_proglen(fprog);
1390         struct sock_fprog_kern *fkprog;
1391
1392         fp->orig_prog = kmalloc(sizeof(*fkprog), GFP_KERNEL);
1393         if (!fp->orig_prog)
1394                 return -ENOMEM;
1395
1396         fkprog = fp->orig_prog;
1397         fkprog->len = fprog->len;
1398         fkprog->filter = kmemdup(fp->insns, fsize, GFP_KERNEL);
1399         if (!fkprog->filter) {
1400                 kfree(fp->orig_prog);
1401                 return -ENOMEM;
1402         }
1403
1404         return 0;
1405 }
1406
1407 static void sk_release_orig_filter(struct sk_filter *fp)
1408 {
1409         struct sock_fprog_kern *fprog = fp->orig_prog;
1410
1411         if (fprog) {
1412                 kfree(fprog->filter);
1413                 kfree(fprog);
1414         }
1415 }
1416
1417 /**
1418  *      sk_filter_release_rcu - Release a socket filter by rcu_head
1419  *      @rcu: rcu_head that contains the sk_filter to free
1420  */
1421 static void sk_filter_release_rcu(struct rcu_head *rcu)
1422 {
1423         struct sk_filter *fp = container_of(rcu, struct sk_filter, rcu);
1424
1425         sk_release_orig_filter(fp);
1426         bpf_jit_free(fp);
1427 }
1428
1429 /**
1430  *      sk_filter_release - release a socket filter
1431  *      @fp: filter to remove
1432  *
1433  *      Remove a filter from a socket and release its resources.
1434  */
1435 static void sk_filter_release(struct sk_filter *fp)
1436 {
1437         if (atomic_dec_and_test(&fp->refcnt))
1438                 call_rcu(&fp->rcu, sk_filter_release_rcu);
1439 }
1440
1441 void sk_filter_uncharge(struct sock *sk, struct sk_filter *fp)
1442 {
1443         atomic_sub(sk_filter_size(fp->len), &sk->sk_omem_alloc);
1444         sk_filter_release(fp);
1445 }
1446
1447 void sk_filter_charge(struct sock *sk, struct sk_filter *fp)
1448 {
1449         atomic_inc(&fp->refcnt);
1450         atomic_add(sk_filter_size(fp->len), &sk->sk_omem_alloc);
1451 }
1452
1453 static struct sk_filter *__sk_migrate_realloc(struct sk_filter *fp,
1454                                               struct sock *sk,
1455                                               unsigned int len)
1456 {
1457         struct sk_filter *fp_new;
1458
1459         if (sk == NULL)
1460                 return krealloc(fp, len, GFP_KERNEL);
1461
1462         fp_new = sock_kmalloc(sk, len, GFP_KERNEL);
1463         if (fp_new) {
1464                 memcpy(fp_new, fp, sizeof(struct sk_filter));
1465                 /* As we're kepping orig_prog in fp_new along,
1466                  * we need to make sure we're not evicting it
1467                  * from the old fp.
1468                  */
1469                 fp->orig_prog = NULL;
1470                 sk_filter_uncharge(sk, fp);
1471         }
1472
1473         return fp_new;
1474 }
1475
1476 static struct sk_filter *__sk_migrate_filter(struct sk_filter *fp,
1477                                              struct sock *sk)
1478 {
1479         struct sock_filter *old_prog;
1480         struct sk_filter *old_fp;
1481         int i, err, new_len, old_len = fp->len;
1482
1483         /* We are free to overwrite insns et al right here as it
1484          * won't be used at this point in time anymore internally
1485          * after the migration to the internal BPF instruction
1486          * representation.
1487          */
1488         BUILD_BUG_ON(sizeof(struct sock_filter) !=
1489                      sizeof(struct sock_filter_int));
1490
1491         /* For now, we need to unfiddle BPF_S_* identifiers in place.
1492          * This can sooner or later on be subject to removal, e.g. when
1493          * JITs have been converted.
1494          */
1495         for (i = 0; i < fp->len; i++)
1496                 sk_decode_filter(&fp->insns[i], &fp->insns[i]);
1497
1498         /* Conversion cannot happen on overlapping memory areas,
1499          * so we need to keep the user BPF around until the 2nd
1500          * pass. At this time, the user BPF is stored in fp->insns.
1501          */
1502         old_prog = kmemdup(fp->insns, old_len * sizeof(struct sock_filter),
1503                            GFP_KERNEL);
1504         if (!old_prog) {
1505                 err = -ENOMEM;
1506                 goto out_err;
1507         }
1508
1509         /* 1st pass: calculate the new program length. */
1510         err = sk_convert_filter(old_prog, old_len, NULL, &new_len);
1511         if (err)
1512                 goto out_err_free;
1513
1514         /* Expand fp for appending the new filter representation. */
1515         old_fp = fp;
1516         fp = __sk_migrate_realloc(old_fp, sk, sk_filter_size(new_len));
1517         if (!fp) {
1518                 /* The old_fp is still around in case we couldn't
1519                  * allocate new memory, so uncharge on that one.
1520                  */
1521                 fp = old_fp;
1522                 err = -ENOMEM;
1523                 goto out_err_free;
1524         }
1525
1526         fp->bpf_func = sk_run_filter_int_skb;
1527         fp->len = new_len;
1528
1529         /* 2nd pass: remap sock_filter insns into sock_filter_int insns. */
1530         err = sk_convert_filter(old_prog, old_len, fp->insnsi, &new_len);
1531         if (err)
1532                 /* 2nd sk_convert_filter() can fail only if it fails
1533                  * to allocate memory, remapping must succeed. Note,
1534                  * that at this time old_fp has already been released
1535                  * by __sk_migrate_realloc().
1536                  */
1537                 goto out_err_free;
1538
1539         kfree(old_prog);
1540         return fp;
1541
1542 out_err_free:
1543         kfree(old_prog);
1544 out_err:
1545         /* Rollback filter setup. */
1546         if (sk != NULL)
1547                 sk_filter_uncharge(sk, fp);
1548         else
1549                 kfree(fp);
1550         return ERR_PTR(err);
1551 }
1552
1553 static struct sk_filter *__sk_prepare_filter(struct sk_filter *fp,
1554                                              struct sock *sk)
1555 {
1556         int err;
1557
1558         fp->bpf_func = NULL;
1559         fp->jited = 0;
1560
1561         err = sk_chk_filter(fp->insns, fp->len);
1562         if (err)
1563                 return ERR_PTR(err);
1564
1565         /* Probe if we can JIT compile the filter and if so, do
1566          * the compilation of the filter.
1567          */
1568         bpf_jit_compile(fp);
1569
1570         /* JIT compiler couldn't process this filter, so do the
1571          * internal BPF translation for the optimized interpreter.
1572          */
1573         if (!fp->jited)
1574                 fp = __sk_migrate_filter(fp, sk);
1575
1576         return fp;
1577 }
1578
1579 /**
1580  *      sk_unattached_filter_create - create an unattached filter
1581  *      @fprog: the filter program
1582  *      @pfp: the unattached filter that is created
1583  *
1584  * Create a filter independent of any socket. We first run some
1585  * sanity checks on it to make sure it does not explode on us later.
1586  * If an error occurs or there is insufficient memory for the filter
1587  * a negative errno code is returned. On success the return is zero.
1588  */
1589 int sk_unattached_filter_create(struct sk_filter **pfp,
1590                                 struct sock_fprog *fprog)
1591 {
1592         unsigned int fsize = sk_filter_proglen(fprog);
1593         struct sk_filter *fp;
1594
1595         /* Make sure new filter is there and in the right amounts. */
1596         if (fprog->filter == NULL)
1597                 return -EINVAL;
1598
1599         fp = kmalloc(sk_filter_size(fprog->len), GFP_KERNEL);
1600         if (!fp)
1601                 return -ENOMEM;
1602
1603         memcpy(fp->insns, fprog->filter, fsize);
1604
1605         atomic_set(&fp->refcnt, 1);
1606         fp->len = fprog->len;
1607         /* Since unattached filters are not copied back to user
1608          * space through sk_get_filter(), we do not need to hold
1609          * a copy here, and can spare us the work.
1610          */
1611         fp->orig_prog = NULL;
1612
1613         /* __sk_prepare_filter() already takes care of uncharging
1614          * memory in case something goes wrong.
1615          */
1616         fp = __sk_prepare_filter(fp, NULL);
1617         if (IS_ERR(fp))
1618                 return PTR_ERR(fp);
1619
1620         *pfp = fp;
1621         return 0;
1622 }
1623 EXPORT_SYMBOL_GPL(sk_unattached_filter_create);
1624
1625 void sk_unattached_filter_destroy(struct sk_filter *fp)
1626 {
1627         sk_filter_release(fp);
1628 }
1629 EXPORT_SYMBOL_GPL(sk_unattached_filter_destroy);
1630
1631 /**
1632  *      sk_attach_filter - attach a socket filter
1633  *      @fprog: the filter program
1634  *      @sk: the socket to use
1635  *
1636  * Attach the user's filter code. We first run some sanity checks on
1637  * it to make sure it does not explode on us later. If an error
1638  * occurs or there is insufficient memory for the filter a negative
1639  * errno code is returned. On success the return is zero.
1640  */
1641 int sk_attach_filter(struct sock_fprog *fprog, struct sock *sk)
1642 {
1643         struct sk_filter *fp, *old_fp;
1644         unsigned int fsize = sk_filter_proglen(fprog);
1645         unsigned int sk_fsize = sk_filter_size(fprog->len);
1646         int err;
1647
1648         if (sock_flag(sk, SOCK_FILTER_LOCKED))
1649                 return -EPERM;
1650
1651         /* Make sure new filter is there and in the right amounts. */
1652         if (fprog->filter == NULL)
1653                 return -EINVAL;
1654
1655         fp = sock_kmalloc(sk, sk_fsize, GFP_KERNEL);
1656         if (!fp)
1657                 return -ENOMEM;
1658
1659         if (copy_from_user(fp->insns, fprog->filter, fsize)) {
1660                 sock_kfree_s(sk, fp, sk_fsize);
1661                 return -EFAULT;
1662         }
1663
1664         atomic_set(&fp->refcnt, 1);
1665         fp->len = fprog->len;
1666
1667         err = sk_store_orig_filter(fp, fprog);
1668         if (err) {
1669                 sk_filter_uncharge(sk, fp);
1670                 return -ENOMEM;
1671         }
1672
1673         /* __sk_prepare_filter() already takes care of uncharging
1674          * memory in case something goes wrong.
1675          */
1676         fp = __sk_prepare_filter(fp, sk);
1677         if (IS_ERR(fp))
1678                 return PTR_ERR(fp);
1679
1680         old_fp = rcu_dereference_protected(sk->sk_filter,
1681                                            sock_owned_by_user(sk));
1682         rcu_assign_pointer(sk->sk_filter, fp);
1683
1684         if (old_fp)
1685                 sk_filter_uncharge(sk, old_fp);
1686
1687         return 0;
1688 }
1689 EXPORT_SYMBOL_GPL(sk_attach_filter);
1690
1691 int sk_detach_filter(struct sock *sk)
1692 {
1693         int ret = -ENOENT;
1694         struct sk_filter *filter;
1695
1696         if (sock_flag(sk, SOCK_FILTER_LOCKED))
1697                 return -EPERM;
1698
1699         filter = rcu_dereference_protected(sk->sk_filter,
1700                                            sock_owned_by_user(sk));
1701         if (filter) {
1702                 RCU_INIT_POINTER(sk->sk_filter, NULL);
1703                 sk_filter_uncharge(sk, filter);
1704                 ret = 0;
1705         }
1706
1707         return ret;
1708 }
1709 EXPORT_SYMBOL_GPL(sk_detach_filter);
1710
1711 void sk_decode_filter(struct sock_filter *filt, struct sock_filter *to)
1712 {
1713         static const u16 decodes[] = {
1714                 [BPF_S_ALU_ADD_K]       = BPF_ALU|BPF_ADD|BPF_K,
1715                 [BPF_S_ALU_ADD_X]       = BPF_ALU|BPF_ADD|BPF_X,
1716                 [BPF_S_ALU_SUB_K]       = BPF_ALU|BPF_SUB|BPF_K,
1717                 [BPF_S_ALU_SUB_X]       = BPF_ALU|BPF_SUB|BPF_X,
1718                 [BPF_S_ALU_MUL_K]       = BPF_ALU|BPF_MUL|BPF_K,
1719                 [BPF_S_ALU_MUL_X]       = BPF_ALU|BPF_MUL|BPF_X,
1720                 [BPF_S_ALU_DIV_X]       = BPF_ALU|BPF_DIV|BPF_X,
1721                 [BPF_S_ALU_MOD_K]       = BPF_ALU|BPF_MOD|BPF_K,
1722                 [BPF_S_ALU_MOD_X]       = BPF_ALU|BPF_MOD|BPF_X,
1723                 [BPF_S_ALU_AND_K]       = BPF_ALU|BPF_AND|BPF_K,
1724                 [BPF_S_ALU_AND_X]       = BPF_ALU|BPF_AND|BPF_X,
1725                 [BPF_S_ALU_OR_K]        = BPF_ALU|BPF_OR|BPF_K,
1726                 [BPF_S_ALU_OR_X]        = BPF_ALU|BPF_OR|BPF_X,
1727                 [BPF_S_ALU_XOR_K]       = BPF_ALU|BPF_XOR|BPF_K,
1728                 [BPF_S_ALU_XOR_X]       = BPF_ALU|BPF_XOR|BPF_X,
1729                 [BPF_S_ALU_LSH_K]       = BPF_ALU|BPF_LSH|BPF_K,
1730                 [BPF_S_ALU_LSH_X]       = BPF_ALU|BPF_LSH|BPF_X,
1731                 [BPF_S_ALU_RSH_K]       = BPF_ALU|BPF_RSH|BPF_K,
1732                 [BPF_S_ALU_RSH_X]       = BPF_ALU|BPF_RSH|BPF_X,
1733                 [BPF_S_ALU_NEG]         = BPF_ALU|BPF_NEG,
1734                 [BPF_S_LD_W_ABS]        = BPF_LD|BPF_W|BPF_ABS,
1735                 [BPF_S_LD_H_ABS]        = BPF_LD|BPF_H|BPF_ABS,
1736                 [BPF_S_LD_B_ABS]        = BPF_LD|BPF_B|BPF_ABS,
1737                 [BPF_S_ANC_PROTOCOL]    = BPF_LD|BPF_B|BPF_ABS,
1738                 [BPF_S_ANC_PKTTYPE]     = BPF_LD|BPF_B|BPF_ABS,
1739                 [BPF_S_ANC_IFINDEX]     = BPF_LD|BPF_B|BPF_ABS,
1740                 [BPF_S_ANC_NLATTR]      = BPF_LD|BPF_B|BPF_ABS,
1741                 [BPF_S_ANC_NLATTR_NEST] = BPF_LD|BPF_B|BPF_ABS,
1742                 [BPF_S_ANC_MARK]        = BPF_LD|BPF_B|BPF_ABS,
1743                 [BPF_S_ANC_QUEUE]       = BPF_LD|BPF_B|BPF_ABS,
1744                 [BPF_S_ANC_HATYPE]      = BPF_LD|BPF_B|BPF_ABS,
1745                 [BPF_S_ANC_RXHASH]      = BPF_LD|BPF_B|BPF_ABS,
1746                 [BPF_S_ANC_CPU]         = BPF_LD|BPF_B|BPF_ABS,
1747                 [BPF_S_ANC_ALU_XOR_X]   = BPF_LD|BPF_B|BPF_ABS,
1748                 [BPF_S_ANC_VLAN_TAG]    = BPF_LD|BPF_B|BPF_ABS,
1749                 [BPF_S_ANC_VLAN_TAG_PRESENT] = BPF_LD|BPF_B|BPF_ABS,
1750                 [BPF_S_ANC_PAY_OFFSET]  = BPF_LD|BPF_B|BPF_ABS,
1751                 [BPF_S_LD_W_LEN]        = BPF_LD|BPF_W|BPF_LEN,
1752                 [BPF_S_LD_W_IND]        = BPF_LD|BPF_W|BPF_IND,
1753                 [BPF_S_LD_H_IND]        = BPF_LD|BPF_H|BPF_IND,
1754                 [BPF_S_LD_B_IND]        = BPF_LD|BPF_B|BPF_IND,
1755                 [BPF_S_LD_IMM]          = BPF_LD|BPF_IMM,
1756                 [BPF_S_LDX_W_LEN]       = BPF_LDX|BPF_W|BPF_LEN,
1757                 [BPF_S_LDX_B_MSH]       = BPF_LDX|BPF_B|BPF_MSH,
1758                 [BPF_S_LDX_IMM]         = BPF_LDX|BPF_IMM,
1759                 [BPF_S_MISC_TAX]        = BPF_MISC|BPF_TAX,
1760                 [BPF_S_MISC_TXA]        = BPF_MISC|BPF_TXA,
1761                 [BPF_S_RET_K]           = BPF_RET|BPF_K,
1762                 [BPF_S_RET_A]           = BPF_RET|BPF_A,
1763                 [BPF_S_ALU_DIV_K]       = BPF_ALU|BPF_DIV|BPF_K,
1764                 [BPF_S_LD_MEM]          = BPF_LD|BPF_MEM,
1765                 [BPF_S_LDX_MEM]         = BPF_LDX|BPF_MEM,
1766                 [BPF_S_ST]              = BPF_ST,
1767                 [BPF_S_STX]             = BPF_STX,
1768                 [BPF_S_JMP_JA]          = BPF_JMP|BPF_JA,
1769                 [BPF_S_JMP_JEQ_K]       = BPF_JMP|BPF_JEQ|BPF_K,
1770                 [BPF_S_JMP_JEQ_X]       = BPF_JMP|BPF_JEQ|BPF_X,
1771                 [BPF_S_JMP_JGE_K]       = BPF_JMP|BPF_JGE|BPF_K,
1772                 [BPF_S_JMP_JGE_X]       = BPF_JMP|BPF_JGE|BPF_X,
1773                 [BPF_S_JMP_JGT_K]       = BPF_JMP|BPF_JGT|BPF_K,
1774                 [BPF_S_JMP_JGT_X]       = BPF_JMP|BPF_JGT|BPF_X,
1775                 [BPF_S_JMP_JSET_K]      = BPF_JMP|BPF_JSET|BPF_K,
1776                 [BPF_S_JMP_JSET_X]      = BPF_JMP|BPF_JSET|BPF_X,
1777         };
1778         u16 code;
1779
1780         code = filt->code;
1781
1782         to->code = decodes[code];
1783         to->jt = filt->jt;
1784         to->jf = filt->jf;
1785         to->k = filt->k;
1786 }
1787
1788 int sk_get_filter(struct sock *sk, struct sock_filter __user *ubuf,
1789                   unsigned int len)
1790 {
1791         struct sock_fprog_kern *fprog;
1792         struct sk_filter *filter;
1793         int ret = 0;
1794
1795         lock_sock(sk);
1796         filter = rcu_dereference_protected(sk->sk_filter,
1797                                            sock_owned_by_user(sk));
1798         if (!filter)
1799                 goto out;
1800
1801         /* We're copying the filter that has been originally attached,
1802          * so no conversion/decode needed anymore.
1803          */
1804         fprog = filter->orig_prog;
1805
1806         ret = fprog->len;
1807         if (!len)
1808                 /* User space only enquires number of filter blocks. */
1809                 goto out;
1810
1811         ret = -EINVAL;
1812         if (len < fprog->len)
1813                 goto out;
1814
1815         ret = -EFAULT;
1816         if (copy_to_user(ubuf, fprog->filter, sk_filter_proglen(fprog)))
1817                 goto out;
1818
1819         /* Instead of bytes, the API requests to return the number
1820          * of filter blocks.
1821          */
1822         ret = fprog->len;
1823 out:
1824         release_sock(sk);
1825         return ret;
1826 }