1// SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0 2/* Copyright (c) 2016-2018 Mellanox Technologies. All rights reserved */ 3 4#include <linux/kernel.h> 5#include <linux/types.h> 6#include <linux/rhashtable.h> 7#include <linux/bitops.h> 8#include <linux/in6.h> 9#include <linux/notifier.h> 10#include <linux/inetdevice.h> 11#include <linux/netdevice.h> 12#include <linux/if_bridge.h> 13#include <linux/socket.h> 14#include <linux/route.h> 15#include <linux/gcd.h> 16#include <linux/if_macvlan.h> 17#include <linux/refcount.h> 18#include <linux/jhash.h> 19#include <linux/net_namespace.h> 20#include <linux/mutex.h> 21#include <linux/genalloc.h> 22#include <net/netevent.h> 23#include <net/neighbour.h> 24#include <net/arp.h> 25#include <net/inet_dscp.h> 26#include <net/ip_fib.h> 27#include <net/ip6_fib.h> 28#include <net/nexthop.h> 29#include <net/fib_rules.h> 30#include <net/ip_tunnels.h> 31#include <net/l3mdev.h> 32#include <net/addrconf.h> 33#include <net/ndisc.h> 34#include <net/ipv6.h> 35#include <net/fib_notifier.h> 36#include <net/switchdev.h> 37 38#include "spectrum.h" 39#include "core.h" 40#include "reg.h" 41#include "spectrum_cnt.h" 42#include "spectrum_dpipe.h" 43#include "spectrum_ipip.h" 44#include "spectrum_mr.h" 45#include "spectrum_mr_tcam.h" 46#include "spectrum_router.h" 47#include "spectrum_span.h" 48 49struct mlxsw_sp_fib; 50struct mlxsw_sp_vr; 51struct mlxsw_sp_lpm_tree; 52struct mlxsw_sp_rif_ops; 53 54struct mlxsw_sp_crif_key { 55 struct net_device *dev; 56}; 57 58struct mlxsw_sp_crif { 59 struct mlxsw_sp_crif_key key; 60 struct rhash_head ht_node; 61 bool can_destroy; 62 struct list_head nexthop_list; 63 struct mlxsw_sp_rif *rif; 64}; 65 66static const struct rhashtable_params mlxsw_sp_crif_ht_params = { 67 .key_offset = offsetof(struct mlxsw_sp_crif, key), 68 .key_len = sizeof_field(struct mlxsw_sp_crif, key), 69 .head_offset = offsetof(struct mlxsw_sp_crif, ht_node), 70}; 71 72struct mlxsw_sp_rif { 73 struct mlxsw_sp_crif *crif; /* NULL for underlay RIF */ 74 netdevice_tracker dev_tracker; 75 struct list_head neigh_list; 76 struct mlxsw_sp_fid *fid; 77 unsigned char addr[ETH_ALEN]; 78 int mtu; 79 u16 rif_index; 80 u8 mac_profile_id; 81 u8 rif_entries; 82 u16 vr_id; 83 const struct mlxsw_sp_rif_ops *ops; 84 struct mlxsw_sp *mlxsw_sp; 85 86 unsigned int counter_ingress; 87 bool counter_ingress_valid; 88 unsigned int counter_egress; 89 bool counter_egress_valid; 90}; 91 92static struct net_device *mlxsw_sp_rif_dev(const struct mlxsw_sp_rif *rif) 93{ 94 if (!rif->crif) 95 return NULL; 96 return rif->crif->key.dev; 97} 98 99struct mlxsw_sp_rif_params { 100 struct net_device *dev; 101 union { 102 u16 system_port; 103 u16 lag_id; 104 }; 105 u16 vid; 106 bool lag; 107 bool double_entry; 108}; 109 110struct mlxsw_sp_rif_subport { 111 struct mlxsw_sp_rif common; 112 refcount_t ref_count; 113 union { 114 u16 system_port; 115 u16 lag_id; 116 }; 117 u16 vid; 118 bool lag; 119}; 120 121struct mlxsw_sp_rif_ipip_lb { 122 struct mlxsw_sp_rif common; 123 struct mlxsw_sp_rif_ipip_lb_config lb_config; 124 u16 ul_vr_id; /* Spectrum-1. */ 125 u16 ul_rif_id; /* Spectrum-2+. */ 126}; 127 128struct mlxsw_sp_rif_params_ipip_lb { 129 struct mlxsw_sp_rif_params common; 130 struct mlxsw_sp_rif_ipip_lb_config lb_config; 131}; 132 133struct mlxsw_sp_rif_ops { 134 enum mlxsw_sp_rif_type type; 135 size_t rif_size; 136 137 void (*setup)(struct mlxsw_sp_rif *rif, 138 const struct mlxsw_sp_rif_params *params); 139 int (*configure)(struct mlxsw_sp_rif *rif, 140 struct netlink_ext_ack *extack); 141 void (*deconfigure)(struct mlxsw_sp_rif *rif); 142 struct mlxsw_sp_fid * (*fid_get)(struct mlxsw_sp_rif *rif, 143 const struct mlxsw_sp_rif_params *params, 144 struct netlink_ext_ack *extack); 145 void (*fdb_del)(struct mlxsw_sp_rif *rif, const char *mac); 146}; 147 148struct mlxsw_sp_rif_mac_profile { 149 unsigned char mac_prefix[ETH_ALEN]; 150 refcount_t ref_count; 151 u8 id; 152}; 153 154struct mlxsw_sp_router_ops { 155 int (*init)(struct mlxsw_sp *mlxsw_sp); 156 int (*ipips_init)(struct mlxsw_sp *mlxsw_sp); 157}; 158 159static struct mlxsw_sp_rif * 160mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp, 161 const struct net_device *dev); 162static void mlxsw_sp_rif_destroy(struct mlxsw_sp_rif *rif); 163static void mlxsw_sp_lpm_tree_hold(struct mlxsw_sp_lpm_tree *lpm_tree); 164static void mlxsw_sp_lpm_tree_put(struct mlxsw_sp *mlxsw_sp, 165 struct mlxsw_sp_lpm_tree *lpm_tree); 166static int mlxsw_sp_vr_lpm_tree_bind(struct mlxsw_sp *mlxsw_sp, 167 const struct mlxsw_sp_fib *fib, 168 u8 tree_id); 169static int mlxsw_sp_vr_lpm_tree_unbind(struct mlxsw_sp *mlxsw_sp, 170 const struct mlxsw_sp_fib *fib); 171 172static unsigned int * 173mlxsw_sp_rif_p_counter_get(struct mlxsw_sp_rif *rif, 174 enum mlxsw_sp_rif_counter_dir dir) 175{ 176 switch (dir) { 177 case MLXSW_SP_RIF_COUNTER_EGRESS: 178 return &rif->counter_egress; 179 case MLXSW_SP_RIF_COUNTER_INGRESS: 180 return &rif->counter_ingress; 181 } 182 return NULL; 183} 184 185static bool 186mlxsw_sp_rif_counter_valid_get(struct mlxsw_sp_rif *rif, 187 enum mlxsw_sp_rif_counter_dir dir) 188{ 189 switch (dir) { 190 case MLXSW_SP_RIF_COUNTER_EGRESS: 191 return rif->counter_egress_valid; 192 case MLXSW_SP_RIF_COUNTER_INGRESS: 193 return rif->counter_ingress_valid; 194 } 195 return false; 196} 197 198static void 199mlxsw_sp_rif_counter_valid_set(struct mlxsw_sp_rif *rif, 200 enum mlxsw_sp_rif_counter_dir dir, 201 bool valid) 202{ 203 switch (dir) { 204 case MLXSW_SP_RIF_COUNTER_EGRESS: 205 rif->counter_egress_valid = valid; 206 break; 207 case MLXSW_SP_RIF_COUNTER_INGRESS: 208 rif->counter_ingress_valid = valid; 209 break; 210 } 211} 212 213static int mlxsw_sp_rif_counter_edit(struct mlxsw_sp *mlxsw_sp, u16 rif_index, 214 unsigned int counter_index, bool enable, 215 enum mlxsw_sp_rif_counter_dir dir) 216{ 217 char ritr_pl[MLXSW_REG_RITR_LEN]; 218 bool is_egress = false; 219 int err; 220 221 if (dir == MLXSW_SP_RIF_COUNTER_EGRESS) 222 is_egress = true; 223 mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index); 224 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 225 if (err) 226 return err; 227 228 mlxsw_reg_ritr_counter_pack(ritr_pl, counter_index, enable, 229 is_egress); 230 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 231} 232 233int mlxsw_sp_rif_counter_value_get(struct mlxsw_sp *mlxsw_sp, 234 struct mlxsw_sp_rif *rif, 235 enum mlxsw_sp_rif_counter_dir dir, u64 *cnt) 236{ 237 char ricnt_pl[MLXSW_REG_RICNT_LEN]; 238 unsigned int *p_counter_index; 239 bool valid; 240 int err; 241 242 valid = mlxsw_sp_rif_counter_valid_get(rif, dir); 243 if (!valid) 244 return -EINVAL; 245 246 p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir); 247 if (!p_counter_index) 248 return -EINVAL; 249 mlxsw_reg_ricnt_pack(ricnt_pl, *p_counter_index, 250 MLXSW_REG_RICNT_OPCODE_NOP); 251 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl); 252 if (err) 253 return err; 254 *cnt = mlxsw_reg_ricnt_good_unicast_packets_get(ricnt_pl); 255 return 0; 256} 257 258struct mlxsw_sp_rif_counter_set_basic { 259 u64 good_unicast_packets; 260 u64 good_multicast_packets; 261 u64 good_broadcast_packets; 262 u64 good_unicast_bytes; 263 u64 good_multicast_bytes; 264 u64 good_broadcast_bytes; 265 u64 error_packets; 266 u64 discard_packets; 267 u64 error_bytes; 268 u64 discard_bytes; 269}; 270 271static int 272mlxsw_sp_rif_counter_fetch_clear(struct mlxsw_sp_rif *rif, 273 enum mlxsw_sp_rif_counter_dir dir, 274 struct mlxsw_sp_rif_counter_set_basic *set) 275{ 276 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 277 char ricnt_pl[MLXSW_REG_RICNT_LEN]; 278 unsigned int *p_counter_index; 279 int err; 280 281 if (!mlxsw_sp_rif_counter_valid_get(rif, dir)) 282 return -EINVAL; 283 284 p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir); 285 if (!p_counter_index) 286 return -EINVAL; 287 288 mlxsw_reg_ricnt_pack(ricnt_pl, *p_counter_index, 289 MLXSW_REG_RICNT_OPCODE_CLEAR); 290 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl); 291 if (err) 292 return err; 293 294 if (!set) 295 return 0; 296 297#define MLXSW_SP_RIF_COUNTER_EXTRACT(NAME) \ 298 (set->NAME = mlxsw_reg_ricnt_ ## NAME ## _get(ricnt_pl)) 299 300 MLXSW_SP_RIF_COUNTER_EXTRACT(good_unicast_packets); 301 MLXSW_SP_RIF_COUNTER_EXTRACT(good_multicast_packets); 302 MLXSW_SP_RIF_COUNTER_EXTRACT(good_broadcast_packets); 303 MLXSW_SP_RIF_COUNTER_EXTRACT(good_unicast_bytes); 304 MLXSW_SP_RIF_COUNTER_EXTRACT(good_multicast_bytes); 305 MLXSW_SP_RIF_COUNTER_EXTRACT(good_broadcast_bytes); 306 MLXSW_SP_RIF_COUNTER_EXTRACT(error_packets); 307 MLXSW_SP_RIF_COUNTER_EXTRACT(discard_packets); 308 MLXSW_SP_RIF_COUNTER_EXTRACT(error_bytes); 309 MLXSW_SP_RIF_COUNTER_EXTRACT(discard_bytes); 310 311#undef MLXSW_SP_RIF_COUNTER_EXTRACT 312 313 return 0; 314} 315 316static int mlxsw_sp_rif_counter_clear(struct mlxsw_sp *mlxsw_sp, 317 unsigned int counter_index) 318{ 319 char ricnt_pl[MLXSW_REG_RICNT_LEN]; 320 321 mlxsw_reg_ricnt_pack(ricnt_pl, counter_index, 322 MLXSW_REG_RICNT_OPCODE_CLEAR); 323 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl); 324} 325 326int mlxsw_sp_rif_counter_alloc(struct mlxsw_sp_rif *rif, 327 enum mlxsw_sp_rif_counter_dir dir) 328{ 329 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 330 unsigned int *p_counter_index; 331 int err; 332 333 if (mlxsw_sp_rif_counter_valid_get(rif, dir)) 334 return 0; 335 336 p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir); 337 if (!p_counter_index) 338 return -EINVAL; 339 340 err = mlxsw_sp_counter_alloc(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF, 341 p_counter_index); 342 if (err) 343 return err; 344 345 err = mlxsw_sp_rif_counter_clear(mlxsw_sp, *p_counter_index); 346 if (err) 347 goto err_counter_clear; 348 349 err = mlxsw_sp_rif_counter_edit(mlxsw_sp, rif->rif_index, 350 *p_counter_index, true, dir); 351 if (err) 352 goto err_counter_edit; 353 mlxsw_sp_rif_counter_valid_set(rif, dir, true); 354 return 0; 355 356err_counter_edit: 357err_counter_clear: 358 mlxsw_sp_counter_free(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF, 359 *p_counter_index); 360 return err; 361} 362 363void mlxsw_sp_rif_counter_free(struct mlxsw_sp_rif *rif, 364 enum mlxsw_sp_rif_counter_dir dir) 365{ 366 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 367 unsigned int *p_counter_index; 368 369 if (!mlxsw_sp_rif_counter_valid_get(rif, dir)) 370 return; 371 372 p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir); 373 if (WARN_ON(!p_counter_index)) 374 return; 375 mlxsw_sp_rif_counter_edit(mlxsw_sp, rif->rif_index, 376 *p_counter_index, false, dir); 377 mlxsw_sp_counter_free(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF, 378 *p_counter_index); 379 mlxsw_sp_rif_counter_valid_set(rif, dir, false); 380} 381 382static void mlxsw_sp_rif_counters_alloc(struct mlxsw_sp_rif *rif) 383{ 384 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 385 struct devlink *devlink; 386 387 devlink = priv_to_devlink(mlxsw_sp->core); 388 if (!devlink_dpipe_table_counter_enabled(devlink, 389 MLXSW_SP_DPIPE_TABLE_NAME_ERIF)) 390 return; 391 mlxsw_sp_rif_counter_alloc(rif, MLXSW_SP_RIF_COUNTER_EGRESS); 392} 393 394static void mlxsw_sp_rif_counters_free(struct mlxsw_sp_rif *rif) 395{ 396 mlxsw_sp_rif_counter_free(rif, MLXSW_SP_RIF_COUNTER_EGRESS); 397} 398 399#define MLXSW_SP_PREFIX_COUNT (sizeof(struct in6_addr) * BITS_PER_BYTE + 1) 400 401struct mlxsw_sp_prefix_usage { 402 DECLARE_BITMAP(b, MLXSW_SP_PREFIX_COUNT); 403}; 404 405#define mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) \ 406 for_each_set_bit(prefix, (prefix_usage)->b, MLXSW_SP_PREFIX_COUNT) 407 408static bool 409mlxsw_sp_prefix_usage_eq(struct mlxsw_sp_prefix_usage *prefix_usage1, 410 struct mlxsw_sp_prefix_usage *prefix_usage2) 411{ 412 return !memcmp(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1)); 413} 414 415static void 416mlxsw_sp_prefix_usage_cpy(struct mlxsw_sp_prefix_usage *prefix_usage1, 417 struct mlxsw_sp_prefix_usage *prefix_usage2) 418{ 419 memcpy(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1)); 420} 421 422static void 423mlxsw_sp_prefix_usage_set(struct mlxsw_sp_prefix_usage *prefix_usage, 424 unsigned char prefix_len) 425{ 426 set_bit(prefix_len, prefix_usage->b); 427} 428 429static void 430mlxsw_sp_prefix_usage_clear(struct mlxsw_sp_prefix_usage *prefix_usage, 431 unsigned char prefix_len) 432{ 433 clear_bit(prefix_len, prefix_usage->b); 434} 435 436struct mlxsw_sp_fib_key { 437 unsigned char addr[sizeof(struct in6_addr)]; 438 unsigned char prefix_len; 439}; 440 441enum mlxsw_sp_fib_entry_type { 442 MLXSW_SP_FIB_ENTRY_TYPE_REMOTE, 443 MLXSW_SP_FIB_ENTRY_TYPE_LOCAL, 444 MLXSW_SP_FIB_ENTRY_TYPE_TRAP, 445 MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE, 446 MLXSW_SP_FIB_ENTRY_TYPE_UNREACHABLE, 447 448 /* This is a special case of local delivery, where a packet should be 449 * decapsulated on reception. Note that there is no corresponding ENCAP, 450 * because that's a type of next hop, not of FIB entry. (There can be 451 * several next hops in a REMOTE entry, and some of them may be 452 * encapsulating entries.) 453 */ 454 MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP, 455 MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP, 456}; 457 458struct mlxsw_sp_nexthop_group_info; 459struct mlxsw_sp_nexthop_group; 460struct mlxsw_sp_fib_entry; 461 462struct mlxsw_sp_fib_node { 463 struct mlxsw_sp_fib_entry *fib_entry; 464 struct list_head list; 465 struct rhash_head ht_node; 466 struct mlxsw_sp_fib *fib; 467 struct mlxsw_sp_fib_key key; 468}; 469 470struct mlxsw_sp_fib_entry_decap { 471 struct mlxsw_sp_ipip_entry *ipip_entry; 472 u32 tunnel_index; 473}; 474 475struct mlxsw_sp_fib_entry { 476 struct mlxsw_sp_fib_node *fib_node; 477 enum mlxsw_sp_fib_entry_type type; 478 struct list_head nexthop_group_node; 479 struct mlxsw_sp_nexthop_group *nh_group; 480 struct mlxsw_sp_fib_entry_decap decap; /* Valid for decap entries. */ 481}; 482 483struct mlxsw_sp_fib4_entry { 484 struct mlxsw_sp_fib_entry common; 485 struct fib_info *fi; 486 u32 tb_id; 487 dscp_t dscp; 488 u8 type; 489}; 490 491struct mlxsw_sp_fib6_entry { 492 struct mlxsw_sp_fib_entry common; 493 struct list_head rt6_list; 494 unsigned int nrt6; 495}; 496 497struct mlxsw_sp_rt6 { 498 struct list_head list; 499 struct fib6_info *rt; 500}; 501 502struct mlxsw_sp_lpm_tree { 503 u8 id; /* tree ID */ 504 unsigned int ref_count; 505 enum mlxsw_sp_l3proto proto; 506 unsigned long prefix_ref_count[MLXSW_SP_PREFIX_COUNT]; 507 struct mlxsw_sp_prefix_usage prefix_usage; 508}; 509 510struct mlxsw_sp_fib { 511 struct rhashtable ht; 512 struct list_head node_list; 513 struct mlxsw_sp_vr *vr; 514 struct mlxsw_sp_lpm_tree *lpm_tree; 515 enum mlxsw_sp_l3proto proto; 516}; 517 518struct mlxsw_sp_vr { 519 u16 id; /* virtual router ID */ 520 u32 tb_id; /* kernel fib table id */ 521 unsigned int rif_count; 522 struct mlxsw_sp_fib *fib4; 523 struct mlxsw_sp_fib *fib6; 524 struct mlxsw_sp_mr_table *mr_table[MLXSW_SP_L3_PROTO_MAX]; 525 struct mlxsw_sp_rif *ul_rif; 526 refcount_t ul_rif_refcnt; 527}; 528 529static const struct rhashtable_params mlxsw_sp_fib_ht_params; 530 531static struct mlxsw_sp_fib *mlxsw_sp_fib_create(struct mlxsw_sp *mlxsw_sp, 532 struct mlxsw_sp_vr *vr, 533 enum mlxsw_sp_l3proto proto) 534{ 535 struct mlxsw_sp_lpm_tree *lpm_tree; 536 struct mlxsw_sp_fib *fib; 537 int err; 538 539 lpm_tree = mlxsw_sp->router->lpm.proto_trees[proto]; 540 fib = kzalloc(sizeof(*fib), GFP_KERNEL); 541 if (!fib) 542 return ERR_PTR(-ENOMEM); 543 err = rhashtable_init(&fib->ht, &mlxsw_sp_fib_ht_params); 544 if (err) 545 goto err_rhashtable_init; 546 INIT_LIST_HEAD(&fib->node_list); 547 fib->proto = proto; 548 fib->vr = vr; 549 fib->lpm_tree = lpm_tree; 550 mlxsw_sp_lpm_tree_hold(lpm_tree); 551 err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib, lpm_tree->id); 552 if (err) 553 goto err_lpm_tree_bind; 554 return fib; 555 556err_lpm_tree_bind: 557 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree); 558err_rhashtable_init: 559 kfree(fib); 560 return ERR_PTR(err); 561} 562 563static void mlxsw_sp_fib_destroy(struct mlxsw_sp *mlxsw_sp, 564 struct mlxsw_sp_fib *fib) 565{ 566 mlxsw_sp_vr_lpm_tree_unbind(mlxsw_sp, fib); 567 mlxsw_sp_lpm_tree_put(mlxsw_sp, fib->lpm_tree); 568 WARN_ON(!list_empty(&fib->node_list)); 569 rhashtable_destroy(&fib->ht); 570 kfree(fib); 571} 572 573static struct mlxsw_sp_lpm_tree * 574mlxsw_sp_lpm_tree_find_unused(struct mlxsw_sp *mlxsw_sp) 575{ 576 static struct mlxsw_sp_lpm_tree *lpm_tree; 577 int i; 578 579 for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) { 580 lpm_tree = &mlxsw_sp->router->lpm.trees[i]; 581 if (lpm_tree->ref_count == 0) 582 return lpm_tree; 583 } 584 return NULL; 585} 586 587static int mlxsw_sp_lpm_tree_alloc(struct mlxsw_sp *mlxsw_sp, 588 struct mlxsw_sp_lpm_tree *lpm_tree) 589{ 590 char ralta_pl[MLXSW_REG_RALTA_LEN]; 591 592 mlxsw_reg_ralta_pack(ralta_pl, true, 593 (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto, 594 lpm_tree->id); 595 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl); 596} 597 598static void mlxsw_sp_lpm_tree_free(struct mlxsw_sp *mlxsw_sp, 599 struct mlxsw_sp_lpm_tree *lpm_tree) 600{ 601 char ralta_pl[MLXSW_REG_RALTA_LEN]; 602 603 mlxsw_reg_ralta_pack(ralta_pl, false, 604 (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto, 605 lpm_tree->id); 606 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl); 607} 608 609static int 610mlxsw_sp_lpm_tree_left_struct_set(struct mlxsw_sp *mlxsw_sp, 611 struct mlxsw_sp_prefix_usage *prefix_usage, 612 struct mlxsw_sp_lpm_tree *lpm_tree) 613{ 614 char ralst_pl[MLXSW_REG_RALST_LEN]; 615 u8 root_bin = 0; 616 u8 prefix; 617 u8 last_prefix = MLXSW_REG_RALST_BIN_NO_CHILD; 618 619 mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) 620 root_bin = prefix; 621 622 mlxsw_reg_ralst_pack(ralst_pl, root_bin, lpm_tree->id); 623 mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) { 624 if (prefix == 0) 625 continue; 626 mlxsw_reg_ralst_bin_pack(ralst_pl, prefix, last_prefix, 627 MLXSW_REG_RALST_BIN_NO_CHILD); 628 last_prefix = prefix; 629 } 630 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralst), ralst_pl); 631} 632 633static struct mlxsw_sp_lpm_tree * 634mlxsw_sp_lpm_tree_create(struct mlxsw_sp *mlxsw_sp, 635 struct mlxsw_sp_prefix_usage *prefix_usage, 636 enum mlxsw_sp_l3proto proto) 637{ 638 struct mlxsw_sp_lpm_tree *lpm_tree; 639 int err; 640 641 lpm_tree = mlxsw_sp_lpm_tree_find_unused(mlxsw_sp); 642 if (!lpm_tree) 643 return ERR_PTR(-EBUSY); 644 lpm_tree->proto = proto; 645 err = mlxsw_sp_lpm_tree_alloc(mlxsw_sp, lpm_tree); 646 if (err) 647 return ERR_PTR(err); 648 649 err = mlxsw_sp_lpm_tree_left_struct_set(mlxsw_sp, prefix_usage, 650 lpm_tree); 651 if (err) 652 goto err_left_struct_set; 653 memcpy(&lpm_tree->prefix_usage, prefix_usage, 654 sizeof(lpm_tree->prefix_usage)); 655 memset(&lpm_tree->prefix_ref_count, 0, 656 sizeof(lpm_tree->prefix_ref_count)); 657 lpm_tree->ref_count = 1; 658 return lpm_tree; 659 660err_left_struct_set: 661 mlxsw_sp_lpm_tree_free(mlxsw_sp, lpm_tree); 662 return ERR_PTR(err); 663} 664 665static void mlxsw_sp_lpm_tree_destroy(struct mlxsw_sp *mlxsw_sp, 666 struct mlxsw_sp_lpm_tree *lpm_tree) 667{ 668 mlxsw_sp_lpm_tree_free(mlxsw_sp, lpm_tree); 669} 670 671static struct mlxsw_sp_lpm_tree * 672mlxsw_sp_lpm_tree_get(struct mlxsw_sp *mlxsw_sp, 673 struct mlxsw_sp_prefix_usage *prefix_usage, 674 enum mlxsw_sp_l3proto proto) 675{ 676 struct mlxsw_sp_lpm_tree *lpm_tree; 677 int i; 678 679 for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) { 680 lpm_tree = &mlxsw_sp->router->lpm.trees[i]; 681 if (lpm_tree->ref_count != 0 && 682 lpm_tree->proto == proto && 683 mlxsw_sp_prefix_usage_eq(&lpm_tree->prefix_usage, 684 prefix_usage)) { 685 mlxsw_sp_lpm_tree_hold(lpm_tree); 686 return lpm_tree; 687 } 688 } 689 return mlxsw_sp_lpm_tree_create(mlxsw_sp, prefix_usage, proto); 690} 691 692static void mlxsw_sp_lpm_tree_hold(struct mlxsw_sp_lpm_tree *lpm_tree) 693{ 694 lpm_tree->ref_count++; 695} 696 697static void mlxsw_sp_lpm_tree_put(struct mlxsw_sp *mlxsw_sp, 698 struct mlxsw_sp_lpm_tree *lpm_tree) 699{ 700 if (--lpm_tree->ref_count == 0) 701 mlxsw_sp_lpm_tree_destroy(mlxsw_sp, lpm_tree); 702} 703 704#define MLXSW_SP_LPM_TREE_MIN 1 /* tree 0 is reserved */ 705 706static int mlxsw_sp_lpm_init(struct mlxsw_sp *mlxsw_sp) 707{ 708 struct mlxsw_sp_prefix_usage req_prefix_usage = {{ 0 } }; 709 struct mlxsw_sp_lpm_tree *lpm_tree; 710 u64 max_trees; 711 int err, i; 712 713 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_LPM_TREES)) 714 return -EIO; 715 716 max_trees = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_LPM_TREES); 717 mlxsw_sp->router->lpm.tree_count = max_trees - MLXSW_SP_LPM_TREE_MIN; 718 mlxsw_sp->router->lpm.trees = kcalloc(mlxsw_sp->router->lpm.tree_count, 719 sizeof(struct mlxsw_sp_lpm_tree), 720 GFP_KERNEL); 721 if (!mlxsw_sp->router->lpm.trees) 722 return -ENOMEM; 723 724 for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) { 725 lpm_tree = &mlxsw_sp->router->lpm.trees[i]; 726 lpm_tree->id = i + MLXSW_SP_LPM_TREE_MIN; 727 } 728 729 lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage, 730 MLXSW_SP_L3_PROTO_IPV4); 731 if (IS_ERR(lpm_tree)) { 732 err = PTR_ERR(lpm_tree); 733 goto err_ipv4_tree_get; 734 } 735 mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV4] = lpm_tree; 736 737 lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage, 738 MLXSW_SP_L3_PROTO_IPV6); 739 if (IS_ERR(lpm_tree)) { 740 err = PTR_ERR(lpm_tree); 741 goto err_ipv6_tree_get; 742 } 743 mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV6] = lpm_tree; 744 745 return 0; 746 747err_ipv6_tree_get: 748 lpm_tree = mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV4]; 749 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree); 750err_ipv4_tree_get: 751 kfree(mlxsw_sp->router->lpm.trees); 752 return err; 753} 754 755static void mlxsw_sp_lpm_fini(struct mlxsw_sp *mlxsw_sp) 756{ 757 struct mlxsw_sp_lpm_tree *lpm_tree; 758 759 lpm_tree = mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV6]; 760 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree); 761 762 lpm_tree = mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV4]; 763 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree); 764 765 kfree(mlxsw_sp->router->lpm.trees); 766} 767 768static bool mlxsw_sp_vr_is_used(const struct mlxsw_sp_vr *vr) 769{ 770 return !!vr->fib4 || !!vr->fib6 || 771 !!vr->mr_table[MLXSW_SP_L3_PROTO_IPV4] || 772 !!vr->mr_table[MLXSW_SP_L3_PROTO_IPV6]; 773} 774 775static struct mlxsw_sp_vr *mlxsw_sp_vr_find_unused(struct mlxsw_sp *mlxsw_sp) 776{ 777 int max_vrs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); 778 struct mlxsw_sp_vr *vr; 779 int i; 780 781 for (i = 0; i < max_vrs; i++) { 782 vr = &mlxsw_sp->router->vrs[i]; 783 if (!mlxsw_sp_vr_is_used(vr)) 784 return vr; 785 } 786 return NULL; 787} 788 789static int mlxsw_sp_vr_lpm_tree_bind(struct mlxsw_sp *mlxsw_sp, 790 const struct mlxsw_sp_fib *fib, u8 tree_id) 791{ 792 char raltb_pl[MLXSW_REG_RALTB_LEN]; 793 794 mlxsw_reg_raltb_pack(raltb_pl, fib->vr->id, 795 (enum mlxsw_reg_ralxx_protocol) fib->proto, 796 tree_id); 797 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), raltb_pl); 798} 799 800static int mlxsw_sp_vr_lpm_tree_unbind(struct mlxsw_sp *mlxsw_sp, 801 const struct mlxsw_sp_fib *fib) 802{ 803 char raltb_pl[MLXSW_REG_RALTB_LEN]; 804 805 /* Bind to tree 0 which is default */ 806 mlxsw_reg_raltb_pack(raltb_pl, fib->vr->id, 807 (enum mlxsw_reg_ralxx_protocol) fib->proto, 0); 808 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), raltb_pl); 809} 810 811static u32 mlxsw_sp_fix_tb_id(u32 tb_id) 812{ 813 /* For our purpose, squash main, default and local tables into one */ 814 if (tb_id == RT_TABLE_LOCAL || tb_id == RT_TABLE_DEFAULT) 815 tb_id = RT_TABLE_MAIN; 816 return tb_id; 817} 818 819static struct mlxsw_sp_vr *mlxsw_sp_vr_find(struct mlxsw_sp *mlxsw_sp, 820 u32 tb_id) 821{ 822 int max_vrs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); 823 struct mlxsw_sp_vr *vr; 824 int i; 825 826 tb_id = mlxsw_sp_fix_tb_id(tb_id); 827 828 for (i = 0; i < max_vrs; i++) { 829 vr = &mlxsw_sp->router->vrs[i]; 830 if (mlxsw_sp_vr_is_used(vr) && vr->tb_id == tb_id) 831 return vr; 832 } 833 return NULL; 834} 835 836int mlxsw_sp_router_tb_id_vr_id(struct mlxsw_sp *mlxsw_sp, u32 tb_id, 837 u16 *vr_id) 838{ 839 struct mlxsw_sp_vr *vr; 840 int err = 0; 841 842 mutex_lock(&mlxsw_sp->router->lock); 843 vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id); 844 if (!vr) { 845 err = -ESRCH; 846 goto out; 847 } 848 *vr_id = vr->id; 849out: 850 mutex_unlock(&mlxsw_sp->router->lock); 851 return err; 852} 853 854static struct mlxsw_sp_fib *mlxsw_sp_vr_fib(const struct mlxsw_sp_vr *vr, 855 enum mlxsw_sp_l3proto proto) 856{ 857 switch (proto) { 858 case MLXSW_SP_L3_PROTO_IPV4: 859 return vr->fib4; 860 case MLXSW_SP_L3_PROTO_IPV6: 861 return vr->fib6; 862 } 863 return NULL; 864} 865 866static struct mlxsw_sp_vr *mlxsw_sp_vr_create(struct mlxsw_sp *mlxsw_sp, 867 u32 tb_id, 868 struct netlink_ext_ack *extack) 869{ 870 struct mlxsw_sp_mr_table *mr4_table, *mr6_table; 871 struct mlxsw_sp_fib *fib4; 872 struct mlxsw_sp_fib *fib6; 873 struct mlxsw_sp_vr *vr; 874 int err; 875 876 vr = mlxsw_sp_vr_find_unused(mlxsw_sp); 877 if (!vr) { 878 NL_SET_ERR_MSG_MOD(extack, "Exceeded number of supported virtual routers"); 879 return ERR_PTR(-EBUSY); 880 } 881 fib4 = mlxsw_sp_fib_create(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV4); 882 if (IS_ERR(fib4)) 883 return ERR_CAST(fib4); 884 fib6 = mlxsw_sp_fib_create(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV6); 885 if (IS_ERR(fib6)) { 886 err = PTR_ERR(fib6); 887 goto err_fib6_create; 888 } 889 mr4_table = mlxsw_sp_mr_table_create(mlxsw_sp, vr->id, 890 MLXSW_SP_L3_PROTO_IPV4); 891 if (IS_ERR(mr4_table)) { 892 err = PTR_ERR(mr4_table); 893 goto err_mr4_table_create; 894 } 895 mr6_table = mlxsw_sp_mr_table_create(mlxsw_sp, vr->id, 896 MLXSW_SP_L3_PROTO_IPV6); 897 if (IS_ERR(mr6_table)) { 898 err = PTR_ERR(mr6_table); 899 goto err_mr6_table_create; 900 } 901 902 vr->fib4 = fib4; 903 vr->fib6 = fib6; 904 vr->mr_table[MLXSW_SP_L3_PROTO_IPV4] = mr4_table; 905 vr->mr_table[MLXSW_SP_L3_PROTO_IPV6] = mr6_table; 906 vr->tb_id = tb_id; 907 return vr; 908 909err_mr6_table_create: 910 mlxsw_sp_mr_table_destroy(mr4_table); 911err_mr4_table_create: 912 mlxsw_sp_fib_destroy(mlxsw_sp, fib6); 913err_fib6_create: 914 mlxsw_sp_fib_destroy(mlxsw_sp, fib4); 915 return ERR_PTR(err); 916} 917 918static void mlxsw_sp_vr_destroy(struct mlxsw_sp *mlxsw_sp, 919 struct mlxsw_sp_vr *vr) 920{ 921 mlxsw_sp_mr_table_destroy(vr->mr_table[MLXSW_SP_L3_PROTO_IPV6]); 922 vr->mr_table[MLXSW_SP_L3_PROTO_IPV6] = NULL; 923 mlxsw_sp_mr_table_destroy(vr->mr_table[MLXSW_SP_L3_PROTO_IPV4]); 924 vr->mr_table[MLXSW_SP_L3_PROTO_IPV4] = NULL; 925 mlxsw_sp_fib_destroy(mlxsw_sp, vr->fib6); 926 vr->fib6 = NULL; 927 mlxsw_sp_fib_destroy(mlxsw_sp, vr->fib4); 928 vr->fib4 = NULL; 929} 930 931static struct mlxsw_sp_vr *mlxsw_sp_vr_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id, 932 struct netlink_ext_ack *extack) 933{ 934 struct mlxsw_sp_vr *vr; 935 936 tb_id = mlxsw_sp_fix_tb_id(tb_id); 937 vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id); 938 if (!vr) 939 vr = mlxsw_sp_vr_create(mlxsw_sp, tb_id, extack); 940 return vr; 941} 942 943static void mlxsw_sp_vr_put(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_vr *vr) 944{ 945 if (!vr->rif_count && list_empty(&vr->fib4->node_list) && 946 list_empty(&vr->fib6->node_list) && 947 mlxsw_sp_mr_table_empty(vr->mr_table[MLXSW_SP_L3_PROTO_IPV4]) && 948 mlxsw_sp_mr_table_empty(vr->mr_table[MLXSW_SP_L3_PROTO_IPV6])) 949 mlxsw_sp_vr_destroy(mlxsw_sp, vr); 950} 951 952static bool 953mlxsw_sp_vr_lpm_tree_should_replace(struct mlxsw_sp_vr *vr, 954 enum mlxsw_sp_l3proto proto, u8 tree_id) 955{ 956 struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto); 957 958 if (!mlxsw_sp_vr_is_used(vr)) 959 return false; 960 if (fib->lpm_tree->id == tree_id) 961 return true; 962 return false; 963} 964 965static int mlxsw_sp_vr_lpm_tree_replace(struct mlxsw_sp *mlxsw_sp, 966 struct mlxsw_sp_fib *fib, 967 struct mlxsw_sp_lpm_tree *new_tree) 968{ 969 struct mlxsw_sp_lpm_tree *old_tree = fib->lpm_tree; 970 int err; 971 972 fib->lpm_tree = new_tree; 973 mlxsw_sp_lpm_tree_hold(new_tree); 974 err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib, new_tree->id); 975 if (err) 976 goto err_tree_bind; 977 mlxsw_sp_lpm_tree_put(mlxsw_sp, old_tree); 978 return 0; 979 980err_tree_bind: 981 mlxsw_sp_lpm_tree_put(mlxsw_sp, new_tree); 982 fib->lpm_tree = old_tree; 983 return err; 984} 985 986static int mlxsw_sp_vrs_lpm_tree_replace(struct mlxsw_sp *mlxsw_sp, 987 struct mlxsw_sp_fib *fib, 988 struct mlxsw_sp_lpm_tree *new_tree) 989{ 990 int max_vrs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); 991 enum mlxsw_sp_l3proto proto = fib->proto; 992 struct mlxsw_sp_lpm_tree *old_tree; 993 u8 old_id, new_id = new_tree->id; 994 struct mlxsw_sp_vr *vr; 995 int i, err; 996 997 old_tree = mlxsw_sp->router->lpm.proto_trees[proto]; 998 old_id = old_tree->id; 999 1000 for (i = 0; i < max_vrs; i++) { 1001 vr = &mlxsw_sp->router->vrs[i]; 1002 if (!mlxsw_sp_vr_lpm_tree_should_replace(vr, proto, old_id)) 1003 continue; 1004 err = mlxsw_sp_vr_lpm_tree_replace(mlxsw_sp, 1005 mlxsw_sp_vr_fib(vr, proto), 1006 new_tree); 1007 if (err) 1008 goto err_tree_replace; 1009 } 1010 1011 memcpy(new_tree->prefix_ref_count, old_tree->prefix_ref_count, 1012 sizeof(new_tree->prefix_ref_count)); 1013 mlxsw_sp->router->lpm.proto_trees[proto] = new_tree; 1014 mlxsw_sp_lpm_tree_put(mlxsw_sp, old_tree); 1015 1016 return 0; 1017 1018err_tree_replace: 1019 for (i--; i >= 0; i--) { 1020 if (!mlxsw_sp_vr_lpm_tree_should_replace(vr, proto, new_id)) 1021 continue; 1022 mlxsw_sp_vr_lpm_tree_replace(mlxsw_sp, 1023 mlxsw_sp_vr_fib(vr, proto), 1024 old_tree); 1025 } 1026 return err; 1027} 1028 1029static int mlxsw_sp_vrs_init(struct mlxsw_sp *mlxsw_sp) 1030{ 1031 struct mlxsw_sp_vr *vr; 1032 u64 max_vrs; 1033 int i; 1034 1035 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_VRS)) 1036 return -EIO; 1037 1038 max_vrs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); 1039 mlxsw_sp->router->vrs = kcalloc(max_vrs, sizeof(struct mlxsw_sp_vr), 1040 GFP_KERNEL); 1041 if (!mlxsw_sp->router->vrs) 1042 return -ENOMEM; 1043 1044 for (i = 0; i < max_vrs; i++) { 1045 vr = &mlxsw_sp->router->vrs[i]; 1046 vr->id = i; 1047 } 1048 1049 return 0; 1050} 1051 1052static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp); 1053 1054static void mlxsw_sp_vrs_fini(struct mlxsw_sp *mlxsw_sp) 1055{ 1056 /* At this stage we're guaranteed not to have new incoming 1057 * FIB notifications and the work queue is free from FIBs 1058 * sitting on top of mlxsw netdevs. However, we can still 1059 * have other FIBs queued. Flush the queue before flushing 1060 * the device's tables. No need for locks, as we're the only 1061 * writer. 1062 */ 1063 mlxsw_core_flush_owq(); 1064 mlxsw_sp_router_fib_flush(mlxsw_sp); 1065 kfree(mlxsw_sp->router->vrs); 1066} 1067 1068u32 mlxsw_sp_ipip_dev_ul_tb_id(const struct net_device *ol_dev) 1069{ 1070 struct net_device *d; 1071 u32 tb_id; 1072 1073 rcu_read_lock(); 1074 d = mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev); 1075 if (d) 1076 tb_id = l3mdev_fib_table(d) ? : RT_TABLE_MAIN; 1077 else 1078 tb_id = RT_TABLE_MAIN; 1079 rcu_read_unlock(); 1080 1081 return tb_id; 1082} 1083 1084static void 1085mlxsw_sp_crif_init(struct mlxsw_sp_crif *crif, struct net_device *dev) 1086{ 1087 crif->key.dev = dev; 1088 INIT_LIST_HEAD(&crif->nexthop_list); 1089} 1090 1091static struct mlxsw_sp_crif * 1092mlxsw_sp_crif_alloc(struct net_device *dev) 1093{ 1094 struct mlxsw_sp_crif *crif; 1095 1096 crif = kzalloc(sizeof(*crif), GFP_KERNEL); 1097 if (!crif) 1098 return NULL; 1099 1100 mlxsw_sp_crif_init(crif, dev); 1101 return crif; 1102} 1103 1104static void mlxsw_sp_crif_free(struct mlxsw_sp_crif *crif) 1105{ 1106 if (WARN_ON(crif->rif)) 1107 return; 1108 1109 WARN_ON(!list_empty(&crif->nexthop_list)); 1110 kfree(crif); 1111} 1112 1113static int mlxsw_sp_crif_insert(struct mlxsw_sp_router *router, 1114 struct mlxsw_sp_crif *crif) 1115{ 1116 return rhashtable_insert_fast(&router->crif_ht, &crif->ht_node, 1117 mlxsw_sp_crif_ht_params); 1118} 1119 1120static void mlxsw_sp_crif_remove(struct mlxsw_sp_router *router, 1121 struct mlxsw_sp_crif *crif) 1122{ 1123 rhashtable_remove_fast(&router->crif_ht, &crif->ht_node, 1124 mlxsw_sp_crif_ht_params); 1125} 1126 1127static struct mlxsw_sp_crif * 1128mlxsw_sp_crif_lookup(struct mlxsw_sp_router *router, 1129 const struct net_device *dev) 1130{ 1131 struct mlxsw_sp_crif_key key = { 1132 .dev = (struct net_device *)dev, 1133 }; 1134 1135 return rhashtable_lookup_fast(&router->crif_ht, &key, 1136 mlxsw_sp_crif_ht_params); 1137} 1138 1139static struct mlxsw_sp_rif * 1140mlxsw_sp_rif_create(struct mlxsw_sp *mlxsw_sp, 1141 const struct mlxsw_sp_rif_params *params, 1142 struct netlink_ext_ack *extack); 1143 1144static struct mlxsw_sp_rif_ipip_lb * 1145mlxsw_sp_ipip_ol_ipip_lb_create(struct mlxsw_sp *mlxsw_sp, 1146 enum mlxsw_sp_ipip_type ipipt, 1147 struct net_device *ol_dev, 1148 struct netlink_ext_ack *extack) 1149{ 1150 struct mlxsw_sp_rif_params_ipip_lb lb_params; 1151 const struct mlxsw_sp_ipip_ops *ipip_ops; 1152 struct mlxsw_sp_rif *rif; 1153 1154 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipipt]; 1155 lb_params = (struct mlxsw_sp_rif_params_ipip_lb) { 1156 .common.dev = ol_dev, 1157 .common.lag = false, 1158 .common.double_entry = ipip_ops->double_rif_entry, 1159 .lb_config = ipip_ops->ol_loopback_config(mlxsw_sp, ol_dev), 1160 }; 1161 1162 rif = mlxsw_sp_rif_create(mlxsw_sp, &lb_params.common, extack); 1163 if (IS_ERR(rif)) 1164 return ERR_CAST(rif); 1165 return container_of(rif, struct mlxsw_sp_rif_ipip_lb, common); 1166} 1167 1168static struct mlxsw_sp_ipip_entry * 1169mlxsw_sp_ipip_entry_alloc(struct mlxsw_sp *mlxsw_sp, 1170 enum mlxsw_sp_ipip_type ipipt, 1171 struct net_device *ol_dev) 1172{ 1173 const struct mlxsw_sp_ipip_ops *ipip_ops; 1174 struct mlxsw_sp_ipip_entry *ipip_entry; 1175 struct mlxsw_sp_ipip_entry *ret = NULL; 1176 int err; 1177 1178 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipipt]; 1179 ipip_entry = kzalloc(sizeof(*ipip_entry), GFP_KERNEL); 1180 if (!ipip_entry) 1181 return ERR_PTR(-ENOMEM); 1182 1183 ipip_entry->ol_lb = mlxsw_sp_ipip_ol_ipip_lb_create(mlxsw_sp, ipipt, 1184 ol_dev, NULL); 1185 if (IS_ERR(ipip_entry->ol_lb)) { 1186 ret = ERR_CAST(ipip_entry->ol_lb); 1187 goto err_ol_ipip_lb_create; 1188 } 1189 1190 ipip_entry->ipipt = ipipt; 1191 ipip_entry->ol_dev = ol_dev; 1192 ipip_entry->parms = ipip_ops->parms_init(ol_dev); 1193 1194 err = ipip_ops->rem_ip_addr_set(mlxsw_sp, ipip_entry); 1195 if (err) { 1196 ret = ERR_PTR(err); 1197 goto err_rem_ip_addr_set; 1198 } 1199 1200 return ipip_entry; 1201 1202err_rem_ip_addr_set: 1203 mlxsw_sp_rif_destroy(&ipip_entry->ol_lb->common); 1204err_ol_ipip_lb_create: 1205 kfree(ipip_entry); 1206 return ret; 1207} 1208 1209static void mlxsw_sp_ipip_entry_dealloc(struct mlxsw_sp *mlxsw_sp, 1210 struct mlxsw_sp_ipip_entry *ipip_entry) 1211{ 1212 const struct mlxsw_sp_ipip_ops *ipip_ops = 1213 mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt]; 1214 1215 ipip_ops->rem_ip_addr_unset(mlxsw_sp, ipip_entry); 1216 mlxsw_sp_rif_destroy(&ipip_entry->ol_lb->common); 1217 kfree(ipip_entry); 1218} 1219 1220static bool 1221mlxsw_sp_ipip_entry_saddr_matches(struct mlxsw_sp *mlxsw_sp, 1222 const enum mlxsw_sp_l3proto ul_proto, 1223 union mlxsw_sp_l3addr saddr, 1224 u32 ul_tb_id, 1225 struct mlxsw_sp_ipip_entry *ipip_entry) 1226{ 1227 u32 tun_ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ipip_entry->ol_dev); 1228 enum mlxsw_sp_ipip_type ipipt = ipip_entry->ipipt; 1229 union mlxsw_sp_l3addr tun_saddr; 1230 1231 if (mlxsw_sp->router->ipip_ops_arr[ipipt]->ul_proto != ul_proto) 1232 return false; 1233 1234 tun_saddr = mlxsw_sp_ipip_netdev_saddr(ul_proto, ipip_entry->ol_dev); 1235 return tun_ul_tb_id == ul_tb_id && 1236 mlxsw_sp_l3addr_eq(&tun_saddr, &saddr); 1237} 1238 1239static int mlxsw_sp_ipip_decap_parsing_depth_inc(struct mlxsw_sp *mlxsw_sp, 1240 enum mlxsw_sp_ipip_type ipipt) 1241{ 1242 const struct mlxsw_sp_ipip_ops *ipip_ops; 1243 1244 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipipt]; 1245 1246 /* Not all tunnels require to increase the default pasing depth 1247 * (96 bytes). 1248 */ 1249 if (ipip_ops->inc_parsing_depth) 1250 return mlxsw_sp_parsing_depth_inc(mlxsw_sp); 1251 1252 return 0; 1253} 1254 1255static void mlxsw_sp_ipip_decap_parsing_depth_dec(struct mlxsw_sp *mlxsw_sp, 1256 enum mlxsw_sp_ipip_type ipipt) 1257{ 1258 const struct mlxsw_sp_ipip_ops *ipip_ops = 1259 mlxsw_sp->router->ipip_ops_arr[ipipt]; 1260 1261 if (ipip_ops->inc_parsing_depth) 1262 mlxsw_sp_parsing_depth_dec(mlxsw_sp); 1263} 1264 1265static int 1266mlxsw_sp_fib_entry_decap_init(struct mlxsw_sp *mlxsw_sp, 1267 struct mlxsw_sp_fib_entry *fib_entry, 1268 struct mlxsw_sp_ipip_entry *ipip_entry) 1269{ 1270 u32 tunnel_index; 1271 int err; 1272 1273 err = mlxsw_sp_kvdl_alloc(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1274 1, &tunnel_index); 1275 if (err) 1276 return err; 1277 1278 err = mlxsw_sp_ipip_decap_parsing_depth_inc(mlxsw_sp, 1279 ipip_entry->ipipt); 1280 if (err) 1281 goto err_parsing_depth_inc; 1282 1283 ipip_entry->decap_fib_entry = fib_entry; 1284 fib_entry->decap.ipip_entry = ipip_entry; 1285 fib_entry->decap.tunnel_index = tunnel_index; 1286 1287 return 0; 1288 1289err_parsing_depth_inc: 1290 mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1, 1291 fib_entry->decap.tunnel_index); 1292 return err; 1293} 1294 1295static void mlxsw_sp_fib_entry_decap_fini(struct mlxsw_sp *mlxsw_sp, 1296 struct mlxsw_sp_fib_entry *fib_entry) 1297{ 1298 enum mlxsw_sp_ipip_type ipipt = fib_entry->decap.ipip_entry->ipipt; 1299 1300 /* Unlink this node from the IPIP entry that it's the decap entry of. */ 1301 fib_entry->decap.ipip_entry->decap_fib_entry = NULL; 1302 fib_entry->decap.ipip_entry = NULL; 1303 mlxsw_sp_ipip_decap_parsing_depth_dec(mlxsw_sp, ipipt); 1304 mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1305 1, fib_entry->decap.tunnel_index); 1306} 1307 1308static struct mlxsw_sp_fib_node * 1309mlxsw_sp_fib_node_lookup(struct mlxsw_sp_fib *fib, const void *addr, 1310 size_t addr_len, unsigned char prefix_len); 1311static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp, 1312 struct mlxsw_sp_fib_entry *fib_entry); 1313 1314static void 1315mlxsw_sp_ipip_entry_demote_decap(struct mlxsw_sp *mlxsw_sp, 1316 struct mlxsw_sp_ipip_entry *ipip_entry) 1317{ 1318 struct mlxsw_sp_fib_entry *fib_entry = ipip_entry->decap_fib_entry; 1319 1320 mlxsw_sp_fib_entry_decap_fini(mlxsw_sp, fib_entry); 1321 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP; 1322 1323 mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry); 1324} 1325 1326static void 1327mlxsw_sp_ipip_entry_promote_decap(struct mlxsw_sp *mlxsw_sp, 1328 struct mlxsw_sp_ipip_entry *ipip_entry, 1329 struct mlxsw_sp_fib_entry *decap_fib_entry) 1330{ 1331 if (mlxsw_sp_fib_entry_decap_init(mlxsw_sp, decap_fib_entry, 1332 ipip_entry)) 1333 return; 1334 decap_fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP; 1335 1336 if (mlxsw_sp_fib_entry_update(mlxsw_sp, decap_fib_entry)) 1337 mlxsw_sp_ipip_entry_demote_decap(mlxsw_sp, ipip_entry); 1338} 1339 1340static struct mlxsw_sp_fib_entry * 1341mlxsw_sp_router_ip2me_fib_entry_find(struct mlxsw_sp *mlxsw_sp, u32 tb_id, 1342 enum mlxsw_sp_l3proto proto, 1343 const union mlxsw_sp_l3addr *addr, 1344 enum mlxsw_sp_fib_entry_type type) 1345{ 1346 struct mlxsw_sp_fib_node *fib_node; 1347 unsigned char addr_prefix_len; 1348 struct mlxsw_sp_fib *fib; 1349 struct mlxsw_sp_vr *vr; 1350 const void *addrp; 1351 size_t addr_len; 1352 u32 addr4; 1353 1354 vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id); 1355 if (!vr) 1356 return NULL; 1357 fib = mlxsw_sp_vr_fib(vr, proto); 1358 1359 switch (proto) { 1360 case MLXSW_SP_L3_PROTO_IPV4: 1361 addr4 = be32_to_cpu(addr->addr4); 1362 addrp = &addr4; 1363 addr_len = 4; 1364 addr_prefix_len = 32; 1365 break; 1366 case MLXSW_SP_L3_PROTO_IPV6: 1367 addrp = &addr->addr6; 1368 addr_len = 16; 1369 addr_prefix_len = 128; 1370 break; 1371 default: 1372 WARN_ON(1); 1373 return NULL; 1374 } 1375 1376 fib_node = mlxsw_sp_fib_node_lookup(fib, addrp, addr_len, 1377 addr_prefix_len); 1378 if (!fib_node || fib_node->fib_entry->type != type) 1379 return NULL; 1380 1381 return fib_node->fib_entry; 1382} 1383 1384/* Given an IPIP entry, find the corresponding decap route. */ 1385static struct mlxsw_sp_fib_entry * 1386mlxsw_sp_ipip_entry_find_decap(struct mlxsw_sp *mlxsw_sp, 1387 struct mlxsw_sp_ipip_entry *ipip_entry) 1388{ 1389 static struct mlxsw_sp_fib_node *fib_node; 1390 const struct mlxsw_sp_ipip_ops *ipip_ops; 1391 unsigned char saddr_prefix_len; 1392 union mlxsw_sp_l3addr saddr; 1393 struct mlxsw_sp_fib *ul_fib; 1394 struct mlxsw_sp_vr *ul_vr; 1395 const void *saddrp; 1396 size_t saddr_len; 1397 u32 ul_tb_id; 1398 u32 saddr4; 1399 1400 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt]; 1401 1402 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ipip_entry->ol_dev); 1403 ul_vr = mlxsw_sp_vr_find(mlxsw_sp, ul_tb_id); 1404 if (!ul_vr) 1405 return NULL; 1406 1407 ul_fib = mlxsw_sp_vr_fib(ul_vr, ipip_ops->ul_proto); 1408 saddr = mlxsw_sp_ipip_netdev_saddr(ipip_ops->ul_proto, 1409 ipip_entry->ol_dev); 1410 1411 switch (ipip_ops->ul_proto) { 1412 case MLXSW_SP_L3_PROTO_IPV4: 1413 saddr4 = be32_to_cpu(saddr.addr4); 1414 saddrp = &saddr4; 1415 saddr_len = 4; 1416 saddr_prefix_len = 32; 1417 break; 1418 case MLXSW_SP_L3_PROTO_IPV6: 1419 saddrp = &saddr.addr6; 1420 saddr_len = 16; 1421 saddr_prefix_len = 128; 1422 break; 1423 default: 1424 WARN_ON(1); 1425 return NULL; 1426 } 1427 1428 fib_node = mlxsw_sp_fib_node_lookup(ul_fib, saddrp, saddr_len, 1429 saddr_prefix_len); 1430 if (!fib_node || 1431 fib_node->fib_entry->type != MLXSW_SP_FIB_ENTRY_TYPE_TRAP) 1432 return NULL; 1433 1434 return fib_node->fib_entry; 1435} 1436 1437static struct mlxsw_sp_ipip_entry * 1438mlxsw_sp_ipip_entry_create(struct mlxsw_sp *mlxsw_sp, 1439 enum mlxsw_sp_ipip_type ipipt, 1440 struct net_device *ol_dev) 1441{ 1442 struct mlxsw_sp_ipip_entry *ipip_entry; 1443 1444 ipip_entry = mlxsw_sp_ipip_entry_alloc(mlxsw_sp, ipipt, ol_dev); 1445 if (IS_ERR(ipip_entry)) 1446 return ipip_entry; 1447 1448 list_add_tail(&ipip_entry->ipip_list_node, 1449 &mlxsw_sp->router->ipip_list); 1450 1451 return ipip_entry; 1452} 1453 1454static void 1455mlxsw_sp_ipip_entry_destroy(struct mlxsw_sp *mlxsw_sp, 1456 struct mlxsw_sp_ipip_entry *ipip_entry) 1457{ 1458 list_del(&ipip_entry->ipip_list_node); 1459 mlxsw_sp_ipip_entry_dealloc(mlxsw_sp, ipip_entry); 1460} 1461 1462static bool 1463mlxsw_sp_ipip_entry_matches_decap(struct mlxsw_sp *mlxsw_sp, 1464 const struct net_device *ul_dev, 1465 enum mlxsw_sp_l3proto ul_proto, 1466 union mlxsw_sp_l3addr ul_dip, 1467 struct mlxsw_sp_ipip_entry *ipip_entry) 1468{ 1469 u32 ul_tb_id = l3mdev_fib_table(ul_dev) ? : RT_TABLE_MAIN; 1470 enum mlxsw_sp_ipip_type ipipt = ipip_entry->ipipt; 1471 1472 if (mlxsw_sp->router->ipip_ops_arr[ipipt]->ul_proto != ul_proto) 1473 return false; 1474 1475 return mlxsw_sp_ipip_entry_saddr_matches(mlxsw_sp, ul_proto, ul_dip, 1476 ul_tb_id, ipip_entry); 1477} 1478 1479/* Given decap parameters, find the corresponding IPIP entry. */ 1480static struct mlxsw_sp_ipip_entry * 1481mlxsw_sp_ipip_entry_find_by_decap(struct mlxsw_sp *mlxsw_sp, int ul_dev_ifindex, 1482 enum mlxsw_sp_l3proto ul_proto, 1483 union mlxsw_sp_l3addr ul_dip) 1484{ 1485 struct mlxsw_sp_ipip_entry *ipip_entry = NULL; 1486 struct net_device *ul_dev; 1487 1488 rcu_read_lock(); 1489 1490 ul_dev = dev_get_by_index_rcu(mlxsw_sp_net(mlxsw_sp), ul_dev_ifindex); 1491 if (!ul_dev) 1492 goto out_unlock; 1493 1494 list_for_each_entry(ipip_entry, &mlxsw_sp->router->ipip_list, 1495 ipip_list_node) 1496 if (mlxsw_sp_ipip_entry_matches_decap(mlxsw_sp, ul_dev, 1497 ul_proto, ul_dip, 1498 ipip_entry)) 1499 goto out_unlock; 1500 1501 rcu_read_unlock(); 1502 1503 return NULL; 1504 1505out_unlock: 1506 rcu_read_unlock(); 1507 return ipip_entry; 1508} 1509 1510static bool mlxsw_sp_netdev_ipip_type(const struct mlxsw_sp *mlxsw_sp, 1511 const struct net_device *dev, 1512 enum mlxsw_sp_ipip_type *p_type) 1513{ 1514 struct mlxsw_sp_router *router = mlxsw_sp->router; 1515 const struct mlxsw_sp_ipip_ops *ipip_ops; 1516 enum mlxsw_sp_ipip_type ipipt; 1517 1518 for (ipipt = 0; ipipt < MLXSW_SP_IPIP_TYPE_MAX; ++ipipt) { 1519 ipip_ops = router->ipip_ops_arr[ipipt]; 1520 if (dev->type == ipip_ops->dev_type) { 1521 if (p_type) 1522 *p_type = ipipt; 1523 return true; 1524 } 1525 } 1526 return false; 1527} 1528 1529static bool mlxsw_sp_netdev_is_ipip_ol(const struct mlxsw_sp *mlxsw_sp, 1530 const struct net_device *dev) 1531{ 1532 return mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, NULL); 1533} 1534 1535static struct mlxsw_sp_ipip_entry * 1536mlxsw_sp_ipip_entry_find_by_ol_dev(struct mlxsw_sp *mlxsw_sp, 1537 const struct net_device *ol_dev) 1538{ 1539 struct mlxsw_sp_ipip_entry *ipip_entry; 1540 1541 list_for_each_entry(ipip_entry, &mlxsw_sp->router->ipip_list, 1542 ipip_list_node) 1543 if (ipip_entry->ol_dev == ol_dev) 1544 return ipip_entry; 1545 1546 return NULL; 1547} 1548 1549static struct mlxsw_sp_ipip_entry * 1550mlxsw_sp_ipip_entry_find_by_ul_dev(const struct mlxsw_sp *mlxsw_sp, 1551 const struct net_device *ul_dev, 1552 struct mlxsw_sp_ipip_entry *start) 1553{ 1554 struct mlxsw_sp_ipip_entry *ipip_entry; 1555 1556 ipip_entry = list_prepare_entry(start, &mlxsw_sp->router->ipip_list, 1557 ipip_list_node); 1558 list_for_each_entry_continue(ipip_entry, &mlxsw_sp->router->ipip_list, 1559 ipip_list_node) { 1560 struct net_device *ol_dev = ipip_entry->ol_dev; 1561 struct net_device *ipip_ul_dev; 1562 1563 rcu_read_lock(); 1564 ipip_ul_dev = mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev); 1565 rcu_read_unlock(); 1566 1567 if (ipip_ul_dev == ul_dev) 1568 return ipip_entry; 1569 } 1570 1571 return NULL; 1572} 1573 1574static bool mlxsw_sp_netdev_is_ipip_ul(struct mlxsw_sp *mlxsw_sp, 1575 const struct net_device *dev) 1576{ 1577 return mlxsw_sp_ipip_entry_find_by_ul_dev(mlxsw_sp, dev, NULL); 1578} 1579 1580static bool mlxsw_sp_netdevice_ipip_can_offload(struct mlxsw_sp *mlxsw_sp, 1581 const struct net_device *ol_dev, 1582 enum mlxsw_sp_ipip_type ipipt) 1583{ 1584 const struct mlxsw_sp_ipip_ops *ops 1585 = mlxsw_sp->router->ipip_ops_arr[ipipt]; 1586 1587 return ops->can_offload(mlxsw_sp, ol_dev); 1588} 1589 1590static int mlxsw_sp_netdevice_ipip_ol_reg_event(struct mlxsw_sp *mlxsw_sp, 1591 struct net_device *ol_dev) 1592{ 1593 enum mlxsw_sp_ipip_type ipipt = MLXSW_SP_IPIP_TYPE_MAX; 1594 struct mlxsw_sp_ipip_entry *ipip_entry; 1595 enum mlxsw_sp_l3proto ul_proto; 1596 union mlxsw_sp_l3addr saddr; 1597 u32 ul_tb_id; 1598 1599 mlxsw_sp_netdev_ipip_type(mlxsw_sp, ol_dev, &ipipt); 1600 if (mlxsw_sp_netdevice_ipip_can_offload(mlxsw_sp, ol_dev, ipipt)) { 1601 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ol_dev); 1602 ul_proto = mlxsw_sp->router->ipip_ops_arr[ipipt]->ul_proto; 1603 saddr = mlxsw_sp_ipip_netdev_saddr(ul_proto, ol_dev); 1604 if (!mlxsw_sp_ipip_demote_tunnel_by_saddr(mlxsw_sp, ul_proto, 1605 saddr, ul_tb_id, 1606 NULL)) { 1607 ipip_entry = mlxsw_sp_ipip_entry_create(mlxsw_sp, ipipt, 1608 ol_dev); 1609 if (IS_ERR(ipip_entry)) 1610 return PTR_ERR(ipip_entry); 1611 } 1612 } 1613 1614 return 0; 1615} 1616 1617static void mlxsw_sp_netdevice_ipip_ol_unreg_event(struct mlxsw_sp *mlxsw_sp, 1618 struct net_device *ol_dev) 1619{ 1620 struct mlxsw_sp_ipip_entry *ipip_entry; 1621 1622 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev); 1623 if (ipip_entry) 1624 mlxsw_sp_ipip_entry_destroy(mlxsw_sp, ipip_entry); 1625} 1626 1627static void 1628mlxsw_sp_ipip_entry_ol_up_event(struct mlxsw_sp *mlxsw_sp, 1629 struct mlxsw_sp_ipip_entry *ipip_entry) 1630{ 1631 struct mlxsw_sp_fib_entry *decap_fib_entry; 1632 1633 decap_fib_entry = mlxsw_sp_ipip_entry_find_decap(mlxsw_sp, ipip_entry); 1634 if (decap_fib_entry) 1635 mlxsw_sp_ipip_entry_promote_decap(mlxsw_sp, ipip_entry, 1636 decap_fib_entry); 1637} 1638 1639static int 1640mlxsw_sp_rif_ipip_lb_op(struct mlxsw_sp_rif_ipip_lb *lb_rif, u16 ul_vr_id, 1641 u16 ul_rif_id, bool enable) 1642{ 1643 struct mlxsw_sp_rif_ipip_lb_config lb_cf = lb_rif->lb_config; 1644 struct net_device *dev = mlxsw_sp_rif_dev(&lb_rif->common); 1645 enum mlxsw_reg_ritr_loopback_ipip_options ipip_options; 1646 struct mlxsw_sp_rif *rif = &lb_rif->common; 1647 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 1648 char ritr_pl[MLXSW_REG_RITR_LEN]; 1649 struct in6_addr *saddr6; 1650 u32 saddr4; 1651 1652 ipip_options = MLXSW_REG_RITR_LOOPBACK_IPIP_OPTIONS_GRE_KEY_PRESET; 1653 switch (lb_cf.ul_protocol) { 1654 case MLXSW_SP_L3_PROTO_IPV4: 1655 saddr4 = be32_to_cpu(lb_cf.saddr.addr4); 1656 mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_LOOPBACK_IF, 1657 rif->rif_index, rif->vr_id, dev->mtu); 1658 mlxsw_reg_ritr_loopback_ipip4_pack(ritr_pl, lb_cf.lb_ipipt, 1659 ipip_options, ul_vr_id, 1660 ul_rif_id, saddr4, 1661 lb_cf.okey); 1662 break; 1663 1664 case MLXSW_SP_L3_PROTO_IPV6: 1665 saddr6 = &lb_cf.saddr.addr6; 1666 mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_LOOPBACK_IF, 1667 rif->rif_index, rif->vr_id, dev->mtu); 1668 mlxsw_reg_ritr_loopback_ipip6_pack(ritr_pl, lb_cf.lb_ipipt, 1669 ipip_options, ul_vr_id, 1670 ul_rif_id, saddr6, 1671 lb_cf.okey); 1672 break; 1673 } 1674 1675 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 1676} 1677 1678static int mlxsw_sp_netdevice_ipip_ol_update_mtu(struct mlxsw_sp *mlxsw_sp, 1679 struct net_device *ol_dev) 1680{ 1681 struct mlxsw_sp_ipip_entry *ipip_entry; 1682 struct mlxsw_sp_rif_ipip_lb *lb_rif; 1683 int err = 0; 1684 1685 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev); 1686 if (ipip_entry) { 1687 lb_rif = ipip_entry->ol_lb; 1688 err = mlxsw_sp_rif_ipip_lb_op(lb_rif, lb_rif->ul_vr_id, 1689 lb_rif->ul_rif_id, true); 1690 if (err) 1691 goto out; 1692 lb_rif->common.mtu = ol_dev->mtu; 1693 } 1694 1695out: 1696 return err; 1697} 1698 1699static void mlxsw_sp_netdevice_ipip_ol_up_event(struct mlxsw_sp *mlxsw_sp, 1700 struct net_device *ol_dev) 1701{ 1702 struct mlxsw_sp_ipip_entry *ipip_entry; 1703 1704 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev); 1705 if (ipip_entry) 1706 mlxsw_sp_ipip_entry_ol_up_event(mlxsw_sp, ipip_entry); 1707} 1708 1709static void 1710mlxsw_sp_ipip_entry_ol_down_event(struct mlxsw_sp *mlxsw_sp, 1711 struct mlxsw_sp_ipip_entry *ipip_entry) 1712{ 1713 if (ipip_entry->decap_fib_entry) 1714 mlxsw_sp_ipip_entry_demote_decap(mlxsw_sp, ipip_entry); 1715} 1716 1717static void mlxsw_sp_netdevice_ipip_ol_down_event(struct mlxsw_sp *mlxsw_sp, 1718 struct net_device *ol_dev) 1719{ 1720 struct mlxsw_sp_ipip_entry *ipip_entry; 1721 1722 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev); 1723 if (ipip_entry) 1724 mlxsw_sp_ipip_entry_ol_down_event(mlxsw_sp, ipip_entry); 1725} 1726 1727static void mlxsw_sp_nexthop_rif_update(struct mlxsw_sp *mlxsw_sp, 1728 struct mlxsw_sp_rif *rif); 1729 1730static void mlxsw_sp_rif_migrate_destroy(struct mlxsw_sp *mlxsw_sp, 1731 struct mlxsw_sp_rif *old_rif, 1732 struct mlxsw_sp_rif *new_rif, 1733 bool migrate_nhs) 1734{ 1735 struct mlxsw_sp_crif *crif = old_rif->crif; 1736 struct mlxsw_sp_crif mock_crif = {}; 1737 1738 if (migrate_nhs) 1739 mlxsw_sp_nexthop_rif_update(mlxsw_sp, new_rif); 1740 1741 /* Plant a mock CRIF so that destroying the old RIF doesn't unoffload 1742 * our nexthops and IPIP tunnels, and doesn't sever the crif->rif link. 1743 */ 1744 mlxsw_sp_crif_init(&mock_crif, crif->key.dev); 1745 old_rif->crif = &mock_crif; 1746 mock_crif.rif = old_rif; 1747 mlxsw_sp_rif_destroy(old_rif); 1748} 1749 1750static int 1751mlxsw_sp_ipip_entry_ol_lb_update(struct mlxsw_sp *mlxsw_sp, 1752 struct mlxsw_sp_ipip_entry *ipip_entry, 1753 bool keep_encap, 1754 struct netlink_ext_ack *extack) 1755{ 1756 struct mlxsw_sp_rif_ipip_lb *old_lb_rif = ipip_entry->ol_lb; 1757 struct mlxsw_sp_rif_ipip_lb *new_lb_rif; 1758 1759 new_lb_rif = mlxsw_sp_ipip_ol_ipip_lb_create(mlxsw_sp, 1760 ipip_entry->ipipt, 1761 ipip_entry->ol_dev, 1762 extack); 1763 if (IS_ERR(new_lb_rif)) 1764 return PTR_ERR(new_lb_rif); 1765 ipip_entry->ol_lb = new_lb_rif; 1766 1767 mlxsw_sp_rif_migrate_destroy(mlxsw_sp, &old_lb_rif->common, 1768 &new_lb_rif->common, keep_encap); 1769 return 0; 1770} 1771 1772/** 1773 * __mlxsw_sp_ipip_entry_update_tunnel - Update offload related to IPIP entry. 1774 * @mlxsw_sp: mlxsw_sp. 1775 * @ipip_entry: IPIP entry. 1776 * @recreate_loopback: Recreates the associated loopback RIF. 1777 * @keep_encap: Updates next hops that use the tunnel netdevice. This is only 1778 * relevant when recreate_loopback is true. 1779 * @update_nexthops: Updates next hops, keeping the current loopback RIF. This 1780 * is only relevant when recreate_loopback is false. 1781 * @extack: extack. 1782 * 1783 * Return: Non-zero value on failure. 1784 */ 1785int __mlxsw_sp_ipip_entry_update_tunnel(struct mlxsw_sp *mlxsw_sp, 1786 struct mlxsw_sp_ipip_entry *ipip_entry, 1787 bool recreate_loopback, 1788 bool keep_encap, 1789 bool update_nexthops, 1790 struct netlink_ext_ack *extack) 1791{ 1792 int err; 1793 1794 /* RIFs can't be edited, so to update loopback, we need to destroy and 1795 * recreate it. That creates a window of opportunity where RALUE and 1796 * RATR registers end up referencing a RIF that's already gone. RATRs 1797 * are handled in mlxsw_sp_ipip_entry_ol_lb_update(), and to take care 1798 * of RALUE, demote the decap route back. 1799 */ 1800 if (ipip_entry->decap_fib_entry) 1801 mlxsw_sp_ipip_entry_demote_decap(mlxsw_sp, ipip_entry); 1802 1803 if (recreate_loopback) { 1804 err = mlxsw_sp_ipip_entry_ol_lb_update(mlxsw_sp, ipip_entry, 1805 keep_encap, extack); 1806 if (err) 1807 return err; 1808 } else if (update_nexthops) { 1809 mlxsw_sp_nexthop_rif_update(mlxsw_sp, 1810 &ipip_entry->ol_lb->common); 1811 } 1812 1813 if (ipip_entry->ol_dev->flags & IFF_UP) 1814 mlxsw_sp_ipip_entry_ol_up_event(mlxsw_sp, ipip_entry); 1815 1816 return 0; 1817} 1818 1819static int mlxsw_sp_netdevice_ipip_ol_vrf_event(struct mlxsw_sp *mlxsw_sp, 1820 struct net_device *ol_dev, 1821 struct netlink_ext_ack *extack) 1822{ 1823 struct mlxsw_sp_ipip_entry *ipip_entry = 1824 mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev); 1825 1826 if (!ipip_entry) 1827 return 0; 1828 1829 return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry, 1830 true, false, false, extack); 1831} 1832 1833static int 1834mlxsw_sp_netdevice_ipip_ul_vrf_event(struct mlxsw_sp *mlxsw_sp, 1835 struct mlxsw_sp_ipip_entry *ipip_entry, 1836 struct net_device *ul_dev, 1837 bool *demote_this, 1838 struct netlink_ext_ack *extack) 1839{ 1840 u32 ul_tb_id = l3mdev_fib_table(ul_dev) ? : RT_TABLE_MAIN; 1841 enum mlxsw_sp_l3proto ul_proto; 1842 union mlxsw_sp_l3addr saddr; 1843 1844 /* Moving underlay to a different VRF might cause local address 1845 * conflict, and the conflicting tunnels need to be demoted. 1846 */ 1847 ul_proto = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt]->ul_proto; 1848 saddr = mlxsw_sp_ipip_netdev_saddr(ul_proto, ipip_entry->ol_dev); 1849 if (mlxsw_sp_ipip_demote_tunnel_by_saddr(mlxsw_sp, ul_proto, 1850 saddr, ul_tb_id, 1851 ipip_entry)) { 1852 *demote_this = true; 1853 return 0; 1854 } 1855 1856 return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry, 1857 true, true, false, extack); 1858} 1859 1860static int 1861mlxsw_sp_netdevice_ipip_ul_up_event(struct mlxsw_sp *mlxsw_sp, 1862 struct mlxsw_sp_ipip_entry *ipip_entry, 1863 struct net_device *ul_dev) 1864{ 1865 return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry, 1866 false, false, true, NULL); 1867} 1868 1869static int 1870mlxsw_sp_netdevice_ipip_ul_down_event(struct mlxsw_sp *mlxsw_sp, 1871 struct mlxsw_sp_ipip_entry *ipip_entry, 1872 struct net_device *ul_dev) 1873{ 1874 /* A down underlay device causes encapsulated packets to not be 1875 * forwarded, but decap still works. So refresh next hops without 1876 * touching anything else. 1877 */ 1878 return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry, 1879 false, false, true, NULL); 1880} 1881 1882static int 1883mlxsw_sp_netdevice_ipip_ol_change_event(struct mlxsw_sp *mlxsw_sp, 1884 struct net_device *ol_dev, 1885 struct netlink_ext_ack *extack) 1886{ 1887 const struct mlxsw_sp_ipip_ops *ipip_ops; 1888 struct mlxsw_sp_ipip_entry *ipip_entry; 1889 int err; 1890 1891 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev); 1892 if (!ipip_entry) 1893 /* A change might make a tunnel eligible for offloading, but 1894 * that is currently not implemented. What falls to slow path 1895 * stays there. 1896 */ 1897 return 0; 1898 1899 /* A change might make a tunnel not eligible for offloading. */ 1900 if (!mlxsw_sp_netdevice_ipip_can_offload(mlxsw_sp, ol_dev, 1901 ipip_entry->ipipt)) { 1902 mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry); 1903 return 0; 1904 } 1905 1906 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt]; 1907 err = ipip_ops->ol_netdev_change(mlxsw_sp, ipip_entry, extack); 1908 return err; 1909} 1910 1911void mlxsw_sp_ipip_entry_demote_tunnel(struct mlxsw_sp *mlxsw_sp, 1912 struct mlxsw_sp_ipip_entry *ipip_entry) 1913{ 1914 struct net_device *ol_dev = ipip_entry->ol_dev; 1915 1916 if (ol_dev->flags & IFF_UP) 1917 mlxsw_sp_ipip_entry_ol_down_event(mlxsw_sp, ipip_entry); 1918 mlxsw_sp_ipip_entry_destroy(mlxsw_sp, ipip_entry); 1919} 1920 1921/* The configuration where several tunnels have the same local address in the 1922 * same underlay table needs special treatment in the HW. That is currently not 1923 * implemented in the driver. This function finds and demotes the first tunnel 1924 * with a given source address, except the one passed in the argument 1925 * `except'. 1926 */ 1927bool 1928mlxsw_sp_ipip_demote_tunnel_by_saddr(struct mlxsw_sp *mlxsw_sp, 1929 enum mlxsw_sp_l3proto ul_proto, 1930 union mlxsw_sp_l3addr saddr, 1931 u32 ul_tb_id, 1932 const struct mlxsw_sp_ipip_entry *except) 1933{ 1934 struct mlxsw_sp_ipip_entry *ipip_entry, *tmp; 1935 1936 list_for_each_entry_safe(ipip_entry, tmp, &mlxsw_sp->router->ipip_list, 1937 ipip_list_node) { 1938 if (ipip_entry != except && 1939 mlxsw_sp_ipip_entry_saddr_matches(mlxsw_sp, ul_proto, saddr, 1940 ul_tb_id, ipip_entry)) { 1941 mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry); 1942 return true; 1943 } 1944 } 1945 1946 return false; 1947} 1948 1949static void mlxsw_sp_ipip_demote_tunnel_by_ul_netdev(struct mlxsw_sp *mlxsw_sp, 1950 struct net_device *ul_dev) 1951{ 1952 struct mlxsw_sp_ipip_entry *ipip_entry, *tmp; 1953 1954 list_for_each_entry_safe(ipip_entry, tmp, &mlxsw_sp->router->ipip_list, 1955 ipip_list_node) { 1956 struct net_device *ol_dev = ipip_entry->ol_dev; 1957 struct net_device *ipip_ul_dev; 1958 1959 rcu_read_lock(); 1960 ipip_ul_dev = mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev); 1961 rcu_read_unlock(); 1962 if (ipip_ul_dev == ul_dev) 1963 mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry); 1964 } 1965} 1966 1967static int mlxsw_sp_netdevice_ipip_ol_event(struct mlxsw_sp *mlxsw_sp, 1968 struct net_device *ol_dev, 1969 unsigned long event, 1970 struct netdev_notifier_info *info) 1971{ 1972 struct netdev_notifier_changeupper_info *chup; 1973 struct netlink_ext_ack *extack; 1974 int err = 0; 1975 1976 switch (event) { 1977 case NETDEV_REGISTER: 1978 err = mlxsw_sp_netdevice_ipip_ol_reg_event(mlxsw_sp, ol_dev); 1979 break; 1980 case NETDEV_UNREGISTER: 1981 mlxsw_sp_netdevice_ipip_ol_unreg_event(mlxsw_sp, ol_dev); 1982 break; 1983 case NETDEV_UP: 1984 mlxsw_sp_netdevice_ipip_ol_up_event(mlxsw_sp, ol_dev); 1985 break; 1986 case NETDEV_DOWN: 1987 mlxsw_sp_netdevice_ipip_ol_down_event(mlxsw_sp, ol_dev); 1988 break; 1989 case NETDEV_CHANGEUPPER: 1990 chup = container_of(info, typeof(*chup), info); 1991 extack = info->extack; 1992 if (netif_is_l3_master(chup->upper_dev)) 1993 err = mlxsw_sp_netdevice_ipip_ol_vrf_event(mlxsw_sp, 1994 ol_dev, 1995 extack); 1996 break; 1997 case NETDEV_CHANGE: 1998 extack = info->extack; 1999 err = mlxsw_sp_netdevice_ipip_ol_change_event(mlxsw_sp, 2000 ol_dev, extack); 2001 break; 2002 case NETDEV_CHANGEMTU: 2003 err = mlxsw_sp_netdevice_ipip_ol_update_mtu(mlxsw_sp, ol_dev); 2004 break; 2005 } 2006 return err; 2007} 2008 2009static int 2010__mlxsw_sp_netdevice_ipip_ul_event(struct mlxsw_sp *mlxsw_sp, 2011 struct mlxsw_sp_ipip_entry *ipip_entry, 2012 struct net_device *ul_dev, 2013 bool *demote_this, 2014 unsigned long event, 2015 struct netdev_notifier_info *info) 2016{ 2017 struct netdev_notifier_changeupper_info *chup; 2018 struct netlink_ext_ack *extack; 2019 2020 switch (event) { 2021 case NETDEV_CHANGEUPPER: 2022 chup = container_of(info, typeof(*chup), info); 2023 extack = info->extack; 2024 if (netif_is_l3_master(chup->upper_dev)) 2025 return mlxsw_sp_netdevice_ipip_ul_vrf_event(mlxsw_sp, 2026 ipip_entry, 2027 ul_dev, 2028 demote_this, 2029 extack); 2030 break; 2031 2032 case NETDEV_UP: 2033 return mlxsw_sp_netdevice_ipip_ul_up_event(mlxsw_sp, ipip_entry, 2034 ul_dev); 2035 case NETDEV_DOWN: 2036 return mlxsw_sp_netdevice_ipip_ul_down_event(mlxsw_sp, 2037 ipip_entry, 2038 ul_dev); 2039 } 2040 return 0; 2041} 2042 2043static int 2044mlxsw_sp_netdevice_ipip_ul_event(struct mlxsw_sp *mlxsw_sp, 2045 struct net_device *ul_dev, 2046 unsigned long event, 2047 struct netdev_notifier_info *info) 2048{ 2049 struct mlxsw_sp_ipip_entry *ipip_entry = NULL; 2050 int err; 2051 2052 while ((ipip_entry = mlxsw_sp_ipip_entry_find_by_ul_dev(mlxsw_sp, 2053 ul_dev, 2054 ipip_entry))) { 2055 struct mlxsw_sp_ipip_entry *prev; 2056 bool demote_this = false; 2057 2058 err = __mlxsw_sp_netdevice_ipip_ul_event(mlxsw_sp, ipip_entry, 2059 ul_dev, &demote_this, 2060 event, info); 2061 if (err) { 2062 mlxsw_sp_ipip_demote_tunnel_by_ul_netdev(mlxsw_sp, 2063 ul_dev); 2064 return err; 2065 } 2066 2067 if (demote_this) { 2068 if (list_is_first(&ipip_entry->ipip_list_node, 2069 &mlxsw_sp->router->ipip_list)) 2070 prev = NULL; 2071 else 2072 /* This can't be cached from previous iteration, 2073 * because that entry could be gone now. 2074 */ 2075 prev = list_prev_entry(ipip_entry, 2076 ipip_list_node); 2077 mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry); 2078 ipip_entry = prev; 2079 } 2080 } 2081 2082 return 0; 2083} 2084 2085int mlxsw_sp_router_nve_promote_decap(struct mlxsw_sp *mlxsw_sp, u32 ul_tb_id, 2086 enum mlxsw_sp_l3proto ul_proto, 2087 const union mlxsw_sp_l3addr *ul_sip, 2088 u32 tunnel_index) 2089{ 2090 enum mlxsw_sp_fib_entry_type type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP; 2091 struct mlxsw_sp_router *router = mlxsw_sp->router; 2092 struct mlxsw_sp_fib_entry *fib_entry; 2093 int err = 0; 2094 2095 mutex_lock(&mlxsw_sp->router->lock); 2096 2097 if (WARN_ON_ONCE(router->nve_decap_config.valid)) { 2098 err = -EINVAL; 2099 goto out; 2100 } 2101 2102 router->nve_decap_config.ul_tb_id = ul_tb_id; 2103 router->nve_decap_config.tunnel_index = tunnel_index; 2104 router->nve_decap_config.ul_proto = ul_proto; 2105 router->nve_decap_config.ul_sip = *ul_sip; 2106 router->nve_decap_config.valid = true; 2107 2108 /* It is valid to create a tunnel with a local IP and only later 2109 * assign this IP address to a local interface 2110 */ 2111 fib_entry = mlxsw_sp_router_ip2me_fib_entry_find(mlxsw_sp, ul_tb_id, 2112 ul_proto, ul_sip, 2113 type); 2114 if (!fib_entry) 2115 goto out; 2116 2117 fib_entry->decap.tunnel_index = tunnel_index; 2118 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP; 2119 2120 err = mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry); 2121 if (err) 2122 goto err_fib_entry_update; 2123 2124 goto out; 2125 2126err_fib_entry_update: 2127 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP; 2128 mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry); 2129out: 2130 mutex_unlock(&mlxsw_sp->router->lock); 2131 return err; 2132} 2133 2134void mlxsw_sp_router_nve_demote_decap(struct mlxsw_sp *mlxsw_sp, u32 ul_tb_id, 2135 enum mlxsw_sp_l3proto ul_proto, 2136 const union mlxsw_sp_l3addr *ul_sip) 2137{ 2138 enum mlxsw_sp_fib_entry_type type = MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP; 2139 struct mlxsw_sp_router *router = mlxsw_sp->router; 2140 struct mlxsw_sp_fib_entry *fib_entry; 2141 2142 mutex_lock(&mlxsw_sp->router->lock); 2143 2144 if (WARN_ON_ONCE(!router->nve_decap_config.valid)) 2145 goto out; 2146 2147 router->nve_decap_config.valid = false; 2148 2149 fib_entry = mlxsw_sp_router_ip2me_fib_entry_find(mlxsw_sp, ul_tb_id, 2150 ul_proto, ul_sip, 2151 type); 2152 if (!fib_entry) 2153 goto out; 2154 2155 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP; 2156 mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry); 2157out: 2158 mutex_unlock(&mlxsw_sp->router->lock); 2159} 2160 2161static bool mlxsw_sp_router_nve_is_decap(struct mlxsw_sp *mlxsw_sp, 2162 u32 ul_tb_id, 2163 enum mlxsw_sp_l3proto ul_proto, 2164 const union mlxsw_sp_l3addr *ul_sip) 2165{ 2166 struct mlxsw_sp_router *router = mlxsw_sp->router; 2167 2168 return router->nve_decap_config.valid && 2169 router->nve_decap_config.ul_tb_id == ul_tb_id && 2170 router->nve_decap_config.ul_proto == ul_proto && 2171 !memcmp(&router->nve_decap_config.ul_sip, ul_sip, 2172 sizeof(*ul_sip)); 2173} 2174 2175struct mlxsw_sp_neigh_key { 2176 struct neighbour *n; 2177}; 2178 2179struct mlxsw_sp_neigh_entry { 2180 struct list_head rif_list_node; 2181 struct rhash_head ht_node; 2182 struct mlxsw_sp_neigh_key key; 2183 u16 rif; 2184 bool connected; 2185 unsigned char ha[ETH_ALEN]; 2186 struct list_head nexthop_list; /* list of nexthops using 2187 * this neigh entry 2188 */ 2189 struct list_head nexthop_neighs_list_node; 2190 unsigned int counter_index; 2191 bool counter_valid; 2192}; 2193 2194static const struct rhashtable_params mlxsw_sp_neigh_ht_params = { 2195 .key_offset = offsetof(struct mlxsw_sp_neigh_entry, key), 2196 .head_offset = offsetof(struct mlxsw_sp_neigh_entry, ht_node), 2197 .key_len = sizeof(struct mlxsw_sp_neigh_key), 2198}; 2199 2200struct mlxsw_sp_neigh_entry * 2201mlxsw_sp_rif_neigh_next(struct mlxsw_sp_rif *rif, 2202 struct mlxsw_sp_neigh_entry *neigh_entry) 2203{ 2204 if (!neigh_entry) { 2205 if (list_empty(&rif->neigh_list)) 2206 return NULL; 2207 else 2208 return list_first_entry(&rif->neigh_list, 2209 typeof(*neigh_entry), 2210 rif_list_node); 2211 } 2212 if (list_is_last(&neigh_entry->rif_list_node, &rif->neigh_list)) 2213 return NULL; 2214 return list_next_entry(neigh_entry, rif_list_node); 2215} 2216 2217int mlxsw_sp_neigh_entry_type(struct mlxsw_sp_neigh_entry *neigh_entry) 2218{ 2219 return neigh_entry->key.n->tbl->family; 2220} 2221 2222unsigned char * 2223mlxsw_sp_neigh_entry_ha(struct mlxsw_sp_neigh_entry *neigh_entry) 2224{ 2225 return neigh_entry->ha; 2226} 2227 2228u32 mlxsw_sp_neigh4_entry_dip(struct mlxsw_sp_neigh_entry *neigh_entry) 2229{ 2230 struct neighbour *n; 2231 2232 n = neigh_entry->key.n; 2233 return ntohl(*((__be32 *) n->primary_key)); 2234} 2235 2236struct in6_addr * 2237mlxsw_sp_neigh6_entry_dip(struct mlxsw_sp_neigh_entry *neigh_entry) 2238{ 2239 struct neighbour *n; 2240 2241 n = neigh_entry->key.n; 2242 return (struct in6_addr *) &n->primary_key; 2243} 2244 2245int mlxsw_sp_neigh_counter_get(struct mlxsw_sp *mlxsw_sp, 2246 struct mlxsw_sp_neigh_entry *neigh_entry, 2247 u64 *p_counter) 2248{ 2249 if (!neigh_entry->counter_valid) 2250 return -EINVAL; 2251 2252 return mlxsw_sp_flow_counter_get(mlxsw_sp, neigh_entry->counter_index, 2253 p_counter, NULL); 2254} 2255 2256static struct mlxsw_sp_neigh_entry * 2257mlxsw_sp_neigh_entry_alloc(struct mlxsw_sp *mlxsw_sp, struct neighbour *n, 2258 u16 rif) 2259{ 2260 struct mlxsw_sp_neigh_entry *neigh_entry; 2261 2262 neigh_entry = kzalloc(sizeof(*neigh_entry), GFP_KERNEL); 2263 if (!neigh_entry) 2264 return NULL; 2265 2266 neigh_entry->key.n = n; 2267 neigh_entry->rif = rif; 2268 INIT_LIST_HEAD(&neigh_entry->nexthop_list); 2269 2270 return neigh_entry; 2271} 2272 2273static void mlxsw_sp_neigh_entry_free(struct mlxsw_sp_neigh_entry *neigh_entry) 2274{ 2275 kfree(neigh_entry); 2276} 2277 2278static int 2279mlxsw_sp_neigh_entry_insert(struct mlxsw_sp *mlxsw_sp, 2280 struct mlxsw_sp_neigh_entry *neigh_entry) 2281{ 2282 return rhashtable_insert_fast(&mlxsw_sp->router->neigh_ht, 2283 &neigh_entry->ht_node, 2284 mlxsw_sp_neigh_ht_params); 2285} 2286 2287static void 2288mlxsw_sp_neigh_entry_remove(struct mlxsw_sp *mlxsw_sp, 2289 struct mlxsw_sp_neigh_entry *neigh_entry) 2290{ 2291 rhashtable_remove_fast(&mlxsw_sp->router->neigh_ht, 2292 &neigh_entry->ht_node, 2293 mlxsw_sp_neigh_ht_params); 2294} 2295 2296static bool 2297mlxsw_sp_neigh_counter_should_alloc(struct mlxsw_sp *mlxsw_sp, 2298 struct mlxsw_sp_neigh_entry *neigh_entry) 2299{ 2300 struct devlink *devlink; 2301 const char *table_name; 2302 2303 switch (mlxsw_sp_neigh_entry_type(neigh_entry)) { 2304 case AF_INET: 2305 table_name = MLXSW_SP_DPIPE_TABLE_NAME_HOST4; 2306 break; 2307 case AF_INET6: 2308 table_name = MLXSW_SP_DPIPE_TABLE_NAME_HOST6; 2309 break; 2310 default: 2311 WARN_ON(1); 2312 return false; 2313 } 2314 2315 devlink = priv_to_devlink(mlxsw_sp->core); 2316 return devlink_dpipe_table_counter_enabled(devlink, table_name); 2317} 2318 2319static void 2320mlxsw_sp_neigh_counter_alloc(struct mlxsw_sp *mlxsw_sp, 2321 struct mlxsw_sp_neigh_entry *neigh_entry) 2322{ 2323 if (!mlxsw_sp_neigh_counter_should_alloc(mlxsw_sp, neigh_entry)) 2324 return; 2325 2326 if (mlxsw_sp_flow_counter_alloc(mlxsw_sp, &neigh_entry->counter_index)) 2327 return; 2328 2329 neigh_entry->counter_valid = true; 2330} 2331 2332static void 2333mlxsw_sp_neigh_counter_free(struct mlxsw_sp *mlxsw_sp, 2334 struct mlxsw_sp_neigh_entry *neigh_entry) 2335{ 2336 if (!neigh_entry->counter_valid) 2337 return; 2338 mlxsw_sp_flow_counter_free(mlxsw_sp, 2339 neigh_entry->counter_index); 2340 neigh_entry->counter_valid = false; 2341} 2342 2343static struct mlxsw_sp_neigh_entry * 2344mlxsw_sp_neigh_entry_create(struct mlxsw_sp *mlxsw_sp, struct neighbour *n) 2345{ 2346 struct mlxsw_sp_neigh_entry *neigh_entry; 2347 struct mlxsw_sp_rif *rif; 2348 int err; 2349 2350 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, n->dev); 2351 if (!rif) 2352 return ERR_PTR(-EINVAL); 2353 2354 neigh_entry = mlxsw_sp_neigh_entry_alloc(mlxsw_sp, n, rif->rif_index); 2355 if (!neigh_entry) 2356 return ERR_PTR(-ENOMEM); 2357 2358 err = mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry); 2359 if (err) 2360 goto err_neigh_entry_insert; 2361 2362 mlxsw_sp_neigh_counter_alloc(mlxsw_sp, neigh_entry); 2363 atomic_inc(&mlxsw_sp->router->neighs_update.neigh_count); 2364 list_add(&neigh_entry->rif_list_node, &rif->neigh_list); 2365 2366 return neigh_entry; 2367 2368err_neigh_entry_insert: 2369 mlxsw_sp_neigh_entry_free(neigh_entry); 2370 return ERR_PTR(err); 2371} 2372 2373static void 2374mlxsw_sp_neigh_entry_destroy(struct mlxsw_sp *mlxsw_sp, 2375 struct mlxsw_sp_neigh_entry *neigh_entry) 2376{ 2377 list_del(&neigh_entry->rif_list_node); 2378 atomic_dec(&mlxsw_sp->router->neighs_update.neigh_count); 2379 mlxsw_sp_neigh_counter_free(mlxsw_sp, neigh_entry); 2380 mlxsw_sp_neigh_entry_remove(mlxsw_sp, neigh_entry); 2381 mlxsw_sp_neigh_entry_free(neigh_entry); 2382} 2383 2384static struct mlxsw_sp_neigh_entry * 2385mlxsw_sp_neigh_entry_lookup(struct mlxsw_sp *mlxsw_sp, struct neighbour *n) 2386{ 2387 struct mlxsw_sp_neigh_key key; 2388 2389 key.n = n; 2390 return rhashtable_lookup_fast(&mlxsw_sp->router->neigh_ht, 2391 &key, mlxsw_sp_neigh_ht_params); 2392} 2393 2394static void 2395mlxsw_sp_router_neighs_update_interval_init(struct mlxsw_sp *mlxsw_sp) 2396{ 2397 unsigned long interval; 2398 2399#if IS_ENABLED(CONFIG_IPV6) 2400 interval = min_t(unsigned long, 2401 NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME), 2402 NEIGH_VAR(&nd_tbl.parms, DELAY_PROBE_TIME)); 2403#else 2404 interval = NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME); 2405#endif 2406 mlxsw_sp->router->neighs_update.interval = jiffies_to_msecs(interval); 2407} 2408 2409static void mlxsw_sp_router_neigh_ent_ipv4_process(struct mlxsw_sp *mlxsw_sp, 2410 char *rauhtd_pl, 2411 int ent_index) 2412{ 2413 u64 max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); 2414 struct net_device *dev; 2415 struct neighbour *n; 2416 __be32 dipn; 2417 u32 dip; 2418 u16 rif; 2419 2420 mlxsw_reg_rauhtd_ent_ipv4_unpack(rauhtd_pl, ent_index, &rif, &dip); 2421 2422 if (WARN_ON_ONCE(rif >= max_rifs)) 2423 return; 2424 if (!mlxsw_sp->router->rifs[rif]) { 2425 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Incorrect RIF in neighbour entry\n"); 2426 return; 2427 } 2428 2429 dipn = htonl(dip); 2430 dev = mlxsw_sp_rif_dev(mlxsw_sp->router->rifs[rif]); 2431 n = neigh_lookup(&arp_tbl, &dipn, dev); 2432 if (!n) 2433 return; 2434 2435 netdev_dbg(dev, "Updating neighbour with IP=%pI4h\n", &dip); 2436 neigh_event_send(n, NULL); 2437 neigh_release(n); 2438} 2439 2440#if IS_ENABLED(CONFIG_IPV6) 2441static void mlxsw_sp_router_neigh_ent_ipv6_process(struct mlxsw_sp *mlxsw_sp, 2442 char *rauhtd_pl, 2443 int rec_index) 2444{ 2445 struct net_device *dev; 2446 struct neighbour *n; 2447 struct in6_addr dip; 2448 u16 rif; 2449 2450 mlxsw_reg_rauhtd_ent_ipv6_unpack(rauhtd_pl, rec_index, &rif, 2451 (char *) &dip); 2452 2453 if (!mlxsw_sp->router->rifs[rif]) { 2454 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Incorrect RIF in neighbour entry\n"); 2455 return; 2456 } 2457 2458 dev = mlxsw_sp_rif_dev(mlxsw_sp->router->rifs[rif]); 2459 n = neigh_lookup(&nd_tbl, &dip, dev); 2460 if (!n) 2461 return; 2462 2463 netdev_dbg(dev, "Updating neighbour with IP=%pI6c\n", &dip); 2464 neigh_event_send(n, NULL); 2465 neigh_release(n); 2466} 2467#else 2468static void mlxsw_sp_router_neigh_ent_ipv6_process(struct mlxsw_sp *mlxsw_sp, 2469 char *rauhtd_pl, 2470 int rec_index) 2471{ 2472} 2473#endif 2474 2475static void mlxsw_sp_router_neigh_rec_ipv4_process(struct mlxsw_sp *mlxsw_sp, 2476 char *rauhtd_pl, 2477 int rec_index) 2478{ 2479 u8 num_entries; 2480 int i; 2481 2482 num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl, 2483 rec_index); 2484 /* Hardware starts counting at 0, so add 1. */ 2485 num_entries++; 2486 2487 /* Each record consists of several neighbour entries. */ 2488 for (i = 0; i < num_entries; i++) { 2489 int ent_index; 2490 2491 ent_index = rec_index * MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC + i; 2492 mlxsw_sp_router_neigh_ent_ipv4_process(mlxsw_sp, rauhtd_pl, 2493 ent_index); 2494 } 2495 2496} 2497 2498static void mlxsw_sp_router_neigh_rec_ipv6_process(struct mlxsw_sp *mlxsw_sp, 2499 char *rauhtd_pl, 2500 int rec_index) 2501{ 2502 /* One record contains one entry. */ 2503 mlxsw_sp_router_neigh_ent_ipv6_process(mlxsw_sp, rauhtd_pl, 2504 rec_index); 2505} 2506 2507static void mlxsw_sp_router_neigh_rec_process(struct mlxsw_sp *mlxsw_sp, 2508 char *rauhtd_pl, int rec_index) 2509{ 2510 switch (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, rec_index)) { 2511 case MLXSW_REG_RAUHTD_TYPE_IPV4: 2512 mlxsw_sp_router_neigh_rec_ipv4_process(mlxsw_sp, rauhtd_pl, 2513 rec_index); 2514 break; 2515 case MLXSW_REG_RAUHTD_TYPE_IPV6: 2516 mlxsw_sp_router_neigh_rec_ipv6_process(mlxsw_sp, rauhtd_pl, 2517 rec_index); 2518 break; 2519 } 2520} 2521 2522static bool mlxsw_sp_router_rauhtd_is_full(char *rauhtd_pl) 2523{ 2524 u8 num_rec, last_rec_index, num_entries; 2525 2526 num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl); 2527 last_rec_index = num_rec - 1; 2528 2529 if (num_rec < MLXSW_REG_RAUHTD_REC_MAX_NUM) 2530 return false; 2531 if (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, last_rec_index) == 2532 MLXSW_REG_RAUHTD_TYPE_IPV6) 2533 return true; 2534 2535 num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl, 2536 last_rec_index); 2537 if (++num_entries == MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC) 2538 return true; 2539 return false; 2540} 2541 2542static int 2543__mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp, 2544 char *rauhtd_pl, 2545 enum mlxsw_reg_rauhtd_type type) 2546{ 2547 int i, num_rec; 2548 int err; 2549 2550 /* Ensure the RIF we read from the device does not change mid-dump. */ 2551 mutex_lock(&mlxsw_sp->router->lock); 2552 do { 2553 mlxsw_reg_rauhtd_pack(rauhtd_pl, type); 2554 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(rauhtd), 2555 rauhtd_pl); 2556 if (err) { 2557 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Failed to dump neighbour table\n"); 2558 break; 2559 } 2560 num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl); 2561 for (i = 0; i < num_rec; i++) 2562 mlxsw_sp_router_neigh_rec_process(mlxsw_sp, rauhtd_pl, 2563 i); 2564 } while (mlxsw_sp_router_rauhtd_is_full(rauhtd_pl)); 2565 mutex_unlock(&mlxsw_sp->router->lock); 2566 2567 return err; 2568} 2569 2570static int mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp) 2571{ 2572 enum mlxsw_reg_rauhtd_type type; 2573 char *rauhtd_pl; 2574 int err; 2575 2576 if (!atomic_read(&mlxsw_sp->router->neighs_update.neigh_count)) 2577 return 0; 2578 2579 rauhtd_pl = kmalloc(MLXSW_REG_RAUHTD_LEN, GFP_KERNEL); 2580 if (!rauhtd_pl) 2581 return -ENOMEM; 2582 2583 type = MLXSW_REG_RAUHTD_TYPE_IPV4; 2584 err = __mlxsw_sp_router_neighs_update_rauhtd(mlxsw_sp, rauhtd_pl, type); 2585 if (err) 2586 goto out; 2587 2588 type = MLXSW_REG_RAUHTD_TYPE_IPV6; 2589 err = __mlxsw_sp_router_neighs_update_rauhtd(mlxsw_sp, rauhtd_pl, type); 2590out: 2591 kfree(rauhtd_pl); 2592 return err; 2593} 2594 2595static void mlxsw_sp_router_neighs_update_nh(struct mlxsw_sp *mlxsw_sp) 2596{ 2597 struct mlxsw_sp_neigh_entry *neigh_entry; 2598 2599 mutex_lock(&mlxsw_sp->router->lock); 2600 list_for_each_entry(neigh_entry, &mlxsw_sp->router->nexthop_neighs_list, 2601 nexthop_neighs_list_node) 2602 /* If this neigh have nexthops, make the kernel think this neigh 2603 * is active regardless of the traffic. 2604 */ 2605 neigh_event_send(neigh_entry->key.n, NULL); 2606 mutex_unlock(&mlxsw_sp->router->lock); 2607} 2608 2609static void 2610mlxsw_sp_router_neighs_update_work_schedule(struct mlxsw_sp *mlxsw_sp) 2611{ 2612 unsigned long interval = mlxsw_sp->router->neighs_update.interval; 2613 2614 mlxsw_core_schedule_dw(&mlxsw_sp->router->neighs_update.dw, 2615 msecs_to_jiffies(interval)); 2616} 2617 2618static void mlxsw_sp_router_neighs_update_work(struct work_struct *work) 2619{ 2620 struct mlxsw_sp_router *router; 2621 int err; 2622 2623 router = container_of(work, struct mlxsw_sp_router, 2624 neighs_update.dw.work); 2625 err = mlxsw_sp_router_neighs_update_rauhtd(router->mlxsw_sp); 2626 if (err) 2627 dev_err(router->mlxsw_sp->bus_info->dev, "Could not update kernel for neigh activity"); 2628 2629 mlxsw_sp_router_neighs_update_nh(router->mlxsw_sp); 2630 2631 mlxsw_sp_router_neighs_update_work_schedule(router->mlxsw_sp); 2632} 2633 2634static void mlxsw_sp_router_probe_unresolved_nexthops(struct work_struct *work) 2635{ 2636 struct mlxsw_sp_neigh_entry *neigh_entry; 2637 struct mlxsw_sp_router *router; 2638 2639 router = container_of(work, struct mlxsw_sp_router, 2640 nexthop_probe_dw.work); 2641 /* Iterate over nexthop neighbours, find those who are unresolved and 2642 * send arp on them. This solves the chicken-egg problem when 2643 * the nexthop wouldn't get offloaded until the neighbor is resolved 2644 * but it wouldn't get resolved ever in case traffic is flowing in HW 2645 * using different nexthop. 2646 */ 2647 mutex_lock(&router->lock); 2648 list_for_each_entry(neigh_entry, &router->nexthop_neighs_list, 2649 nexthop_neighs_list_node) 2650 if (!neigh_entry->connected) 2651 neigh_event_send(neigh_entry->key.n, NULL); 2652 mutex_unlock(&router->lock); 2653 2654 mlxsw_core_schedule_dw(&router->nexthop_probe_dw, 2655 MLXSW_SP_UNRESOLVED_NH_PROBE_INTERVAL); 2656} 2657 2658static void 2659mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp, 2660 struct mlxsw_sp_neigh_entry *neigh_entry, 2661 bool removing, bool dead); 2662 2663static enum mlxsw_reg_rauht_op mlxsw_sp_rauht_op(bool adding) 2664{ 2665 return adding ? MLXSW_REG_RAUHT_OP_WRITE_ADD : 2666 MLXSW_REG_RAUHT_OP_WRITE_DELETE; 2667} 2668 2669static int 2670mlxsw_sp_router_neigh_entry_op4(struct mlxsw_sp *mlxsw_sp, 2671 struct mlxsw_sp_neigh_entry *neigh_entry, 2672 enum mlxsw_reg_rauht_op op) 2673{ 2674 struct neighbour *n = neigh_entry->key.n; 2675 u32 dip = ntohl(*((__be32 *) n->primary_key)); 2676 char rauht_pl[MLXSW_REG_RAUHT_LEN]; 2677 2678 mlxsw_reg_rauht_pack4(rauht_pl, op, neigh_entry->rif, neigh_entry->ha, 2679 dip); 2680 if (neigh_entry->counter_valid) 2681 mlxsw_reg_rauht_pack_counter(rauht_pl, 2682 neigh_entry->counter_index); 2683 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl); 2684} 2685 2686static int 2687mlxsw_sp_router_neigh_entry_op6(struct mlxsw_sp *mlxsw_sp, 2688 struct mlxsw_sp_neigh_entry *neigh_entry, 2689 enum mlxsw_reg_rauht_op op) 2690{ 2691 struct neighbour *n = neigh_entry->key.n; 2692 char rauht_pl[MLXSW_REG_RAUHT_LEN]; 2693 const char *dip = n->primary_key; 2694 2695 mlxsw_reg_rauht_pack6(rauht_pl, op, neigh_entry->rif, neigh_entry->ha, 2696 dip); 2697 if (neigh_entry->counter_valid) 2698 mlxsw_reg_rauht_pack_counter(rauht_pl, 2699 neigh_entry->counter_index); 2700 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl); 2701} 2702 2703bool mlxsw_sp_neigh_ipv6_ignore(struct mlxsw_sp_neigh_entry *neigh_entry) 2704{ 2705 struct neighbour *n = neigh_entry->key.n; 2706 2707 /* Packets with a link-local destination address are trapped 2708 * after LPM lookup and never reach the neighbour table, so 2709 * there is no need to program such neighbours to the device. 2710 */ 2711 if (ipv6_addr_type((struct in6_addr *) &n->primary_key) & 2712 IPV6_ADDR_LINKLOCAL) 2713 return true; 2714 return false; 2715} 2716 2717static void 2718mlxsw_sp_neigh_entry_update(struct mlxsw_sp *mlxsw_sp, 2719 struct mlxsw_sp_neigh_entry *neigh_entry, 2720 bool adding) 2721{ 2722 enum mlxsw_reg_rauht_op op = mlxsw_sp_rauht_op(adding); 2723 int err; 2724 2725 if (!adding && !neigh_entry->connected) 2726 return; 2727 neigh_entry->connected = adding; 2728 if (neigh_entry->key.n->tbl->family == AF_INET) { 2729 err = mlxsw_sp_router_neigh_entry_op4(mlxsw_sp, neigh_entry, 2730 op); 2731 if (err) 2732 return; 2733 } else if (neigh_entry->key.n->tbl->family == AF_INET6) { 2734 if (mlxsw_sp_neigh_ipv6_ignore(neigh_entry)) 2735 return; 2736 err = mlxsw_sp_router_neigh_entry_op6(mlxsw_sp, neigh_entry, 2737 op); 2738 if (err) 2739 return; 2740 } else { 2741 WARN_ON_ONCE(1); 2742 return; 2743 } 2744 2745 if (adding) 2746 neigh_entry->key.n->flags |= NTF_OFFLOADED; 2747 else 2748 neigh_entry->key.n->flags &= ~NTF_OFFLOADED; 2749} 2750 2751void 2752mlxsw_sp_neigh_entry_counter_update(struct mlxsw_sp *mlxsw_sp, 2753 struct mlxsw_sp_neigh_entry *neigh_entry, 2754 bool adding) 2755{ 2756 if (adding) 2757 mlxsw_sp_neigh_counter_alloc(mlxsw_sp, neigh_entry); 2758 else 2759 mlxsw_sp_neigh_counter_free(mlxsw_sp, neigh_entry); 2760 mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, true); 2761} 2762 2763struct mlxsw_sp_netevent_work { 2764 struct work_struct work; 2765 struct mlxsw_sp *mlxsw_sp; 2766 struct neighbour *n; 2767}; 2768 2769static void mlxsw_sp_router_neigh_event_work(struct work_struct *work) 2770{ 2771 struct mlxsw_sp_netevent_work *net_work = 2772 container_of(work, struct mlxsw_sp_netevent_work, work); 2773 struct mlxsw_sp *mlxsw_sp = net_work->mlxsw_sp; 2774 struct mlxsw_sp_neigh_entry *neigh_entry; 2775 struct neighbour *n = net_work->n; 2776 unsigned char ha[ETH_ALEN]; 2777 bool entry_connected; 2778 u8 nud_state, dead; 2779 2780 /* If these parameters are changed after we release the lock, 2781 * then we are guaranteed to receive another event letting us 2782 * know about it. 2783 */ 2784 read_lock_bh(&n->lock); 2785 memcpy(ha, n->ha, ETH_ALEN); 2786 nud_state = n->nud_state; 2787 dead = n->dead; 2788 read_unlock_bh(&n->lock); 2789 2790 mutex_lock(&mlxsw_sp->router->lock); 2791 mlxsw_sp_span_respin(mlxsw_sp); 2792 2793 entry_connected = nud_state & NUD_VALID && !dead; 2794 neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n); 2795 if (!entry_connected && !neigh_entry) 2796 goto out; 2797 if (!neigh_entry) { 2798 neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n); 2799 if (IS_ERR(neigh_entry)) 2800 goto out; 2801 } 2802 2803 if (neigh_entry->connected && entry_connected && 2804 !memcmp(neigh_entry->ha, ha, ETH_ALEN)) 2805 goto out; 2806 2807 memcpy(neigh_entry->ha, ha, ETH_ALEN); 2808 mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, entry_connected); 2809 mlxsw_sp_nexthop_neigh_update(mlxsw_sp, neigh_entry, !entry_connected, 2810 dead); 2811 2812 if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list)) 2813 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry); 2814 2815out: 2816 mutex_unlock(&mlxsw_sp->router->lock); 2817 neigh_release(n); 2818 kfree(net_work); 2819} 2820 2821static int mlxsw_sp_mp_hash_init(struct mlxsw_sp *mlxsw_sp); 2822 2823static void mlxsw_sp_router_mp_hash_event_work(struct work_struct *work) 2824{ 2825 struct mlxsw_sp_netevent_work *net_work = 2826 container_of(work, struct mlxsw_sp_netevent_work, work); 2827 struct mlxsw_sp *mlxsw_sp = net_work->mlxsw_sp; 2828 2829 mlxsw_sp_mp_hash_init(mlxsw_sp); 2830 kfree(net_work); 2831} 2832 2833static int __mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp); 2834 2835static void mlxsw_sp_router_update_priority_work(struct work_struct *work) 2836{ 2837 struct mlxsw_sp_netevent_work *net_work = 2838 container_of(work, struct mlxsw_sp_netevent_work, work); 2839 struct mlxsw_sp *mlxsw_sp = net_work->mlxsw_sp; 2840 2841 __mlxsw_sp_router_init(mlxsw_sp); 2842 kfree(net_work); 2843} 2844 2845static int mlxsw_sp_router_schedule_work(struct net *net, 2846 struct mlxsw_sp_router *router, 2847 struct neighbour *n, 2848 void (*cb)(struct work_struct *)) 2849{ 2850 struct mlxsw_sp_netevent_work *net_work; 2851 2852 if (!net_eq(net, mlxsw_sp_net(router->mlxsw_sp))) 2853 return NOTIFY_DONE; 2854 2855 net_work = kzalloc(sizeof(*net_work), GFP_ATOMIC); 2856 if (!net_work) 2857 return NOTIFY_BAD; 2858 2859 INIT_WORK(&net_work->work, cb); 2860 net_work->mlxsw_sp = router->mlxsw_sp; 2861 net_work->n = n; 2862 mlxsw_core_schedule_work(&net_work->work); 2863 return NOTIFY_DONE; 2864} 2865 2866static bool mlxsw_sp_dev_lower_is_port(struct net_device *dev) 2867{ 2868 struct mlxsw_sp_port *mlxsw_sp_port; 2869 2870 rcu_read_lock(); 2871 mlxsw_sp_port = mlxsw_sp_port_dev_lower_find_rcu(dev); 2872 rcu_read_unlock(); 2873 return !!mlxsw_sp_port; 2874} 2875 2876static int mlxsw_sp_router_schedule_neigh_work(struct mlxsw_sp_router *router, 2877 struct neighbour *n) 2878{ 2879 struct net *net; 2880 2881 net = neigh_parms_net(n->parms); 2882 2883 /* Take a reference to ensure the neighbour won't be destructed until we 2884 * drop the reference in delayed work. 2885 */ 2886 neigh_clone(n); 2887 return mlxsw_sp_router_schedule_work(net, router, n, 2888 mlxsw_sp_router_neigh_event_work); 2889} 2890 2891static int mlxsw_sp_router_netevent_event(struct notifier_block *nb, 2892 unsigned long event, void *ptr) 2893{ 2894 struct mlxsw_sp_router *router; 2895 unsigned long interval; 2896 struct neigh_parms *p; 2897 struct neighbour *n; 2898 2899 router = container_of(nb, struct mlxsw_sp_router, netevent_nb); 2900 2901 switch (event) { 2902 case NETEVENT_DELAY_PROBE_TIME_UPDATE: 2903 p = ptr; 2904 2905 /* We don't care about changes in the default table. */ 2906 if (!p->dev || (p->tbl->family != AF_INET && 2907 p->tbl->family != AF_INET6)) 2908 return NOTIFY_DONE; 2909 2910 /* We are in atomic context and can't take RTNL mutex, 2911 * so use RCU variant to walk the device chain. 2912 */ 2913 if (!mlxsw_sp_dev_lower_is_port(p->dev)) 2914 return NOTIFY_DONE; 2915 2916 interval = jiffies_to_msecs(NEIGH_VAR(p, DELAY_PROBE_TIME)); 2917 router->neighs_update.interval = interval; 2918 break; 2919 case NETEVENT_NEIGH_UPDATE: 2920 n = ptr; 2921 2922 if (n->tbl->family != AF_INET && n->tbl->family != AF_INET6) 2923 return NOTIFY_DONE; 2924 2925 if (!mlxsw_sp_dev_lower_is_port(n->dev)) 2926 return NOTIFY_DONE; 2927 2928 return mlxsw_sp_router_schedule_neigh_work(router, n); 2929 2930 case NETEVENT_IPV4_MPATH_HASH_UPDATE: 2931 case NETEVENT_IPV6_MPATH_HASH_UPDATE: 2932 return mlxsw_sp_router_schedule_work(ptr, router, NULL, 2933 mlxsw_sp_router_mp_hash_event_work); 2934 2935 case NETEVENT_IPV4_FWD_UPDATE_PRIORITY_UPDATE: 2936 return mlxsw_sp_router_schedule_work(ptr, router, NULL, 2937 mlxsw_sp_router_update_priority_work); 2938 } 2939 2940 return NOTIFY_DONE; 2941} 2942 2943static int mlxsw_sp_neigh_init(struct mlxsw_sp *mlxsw_sp) 2944{ 2945 int err; 2946 2947 err = rhashtable_init(&mlxsw_sp->router->neigh_ht, 2948 &mlxsw_sp_neigh_ht_params); 2949 if (err) 2950 return err; 2951 2952 /* Initialize the polling interval according to the default 2953 * table. 2954 */ 2955 mlxsw_sp_router_neighs_update_interval_init(mlxsw_sp); 2956 2957 /* Create the delayed works for the activity_update */ 2958 INIT_DELAYED_WORK(&mlxsw_sp->router->neighs_update.dw, 2959 mlxsw_sp_router_neighs_update_work); 2960 INIT_DELAYED_WORK(&mlxsw_sp->router->nexthop_probe_dw, 2961 mlxsw_sp_router_probe_unresolved_nexthops); 2962 atomic_set(&mlxsw_sp->router->neighs_update.neigh_count, 0); 2963 mlxsw_core_schedule_dw(&mlxsw_sp->router->neighs_update.dw, 0); 2964 mlxsw_core_schedule_dw(&mlxsw_sp->router->nexthop_probe_dw, 0); 2965 return 0; 2966} 2967 2968static void mlxsw_sp_neigh_fini(struct mlxsw_sp *mlxsw_sp) 2969{ 2970 cancel_delayed_work_sync(&mlxsw_sp->router->neighs_update.dw); 2971 cancel_delayed_work_sync(&mlxsw_sp->router->nexthop_probe_dw); 2972 rhashtable_destroy(&mlxsw_sp->router->neigh_ht); 2973} 2974 2975static void mlxsw_sp_neigh_rif_gone_sync(struct mlxsw_sp *mlxsw_sp, 2976 struct mlxsw_sp_rif *rif) 2977{ 2978 struct mlxsw_sp_neigh_entry *neigh_entry, *tmp; 2979 2980 list_for_each_entry_safe(neigh_entry, tmp, &rif->neigh_list, 2981 rif_list_node) { 2982 mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, false); 2983 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry); 2984 } 2985} 2986 2987struct mlxsw_sp_neigh_rif_made_sync { 2988 struct mlxsw_sp *mlxsw_sp; 2989 struct mlxsw_sp_rif *rif; 2990 int err; 2991}; 2992 2993static void mlxsw_sp_neigh_rif_made_sync_each(struct neighbour *n, void *data) 2994{ 2995 struct mlxsw_sp_neigh_rif_made_sync *rms = data; 2996 int rc; 2997 2998 if (rms->err) 2999 return; 3000 if (n->dev != mlxsw_sp_rif_dev(rms->rif)) 3001 return; 3002 rc = mlxsw_sp_router_schedule_neigh_work(rms->mlxsw_sp->router, n); 3003 if (rc != NOTIFY_DONE) 3004 rms->err = -ENOMEM; 3005} 3006 3007static int mlxsw_sp_neigh_rif_made_sync(struct mlxsw_sp *mlxsw_sp, 3008 struct mlxsw_sp_rif *rif) 3009{ 3010 struct mlxsw_sp_neigh_rif_made_sync rms = { 3011 .mlxsw_sp = mlxsw_sp, 3012 .rif = rif, 3013 }; 3014 3015 neigh_for_each(&arp_tbl, mlxsw_sp_neigh_rif_made_sync_each, &rms); 3016 if (rms.err) 3017 goto err_arp; 3018 3019#if IS_ENABLED(CONFIG_IPV6) 3020 neigh_for_each(&nd_tbl, mlxsw_sp_neigh_rif_made_sync_each, &rms); 3021#endif 3022 if (rms.err) 3023 goto err_nd; 3024 3025 return 0; 3026 3027err_nd: 3028err_arp: 3029 mlxsw_sp_neigh_rif_gone_sync(mlxsw_sp, rif); 3030 return rms.err; 3031} 3032 3033enum mlxsw_sp_nexthop_type { 3034 MLXSW_SP_NEXTHOP_TYPE_ETH, 3035 MLXSW_SP_NEXTHOP_TYPE_IPIP, 3036}; 3037 3038enum mlxsw_sp_nexthop_action { 3039 /* Nexthop forwards packets to an egress RIF */ 3040 MLXSW_SP_NEXTHOP_ACTION_FORWARD, 3041 /* Nexthop discards packets */ 3042 MLXSW_SP_NEXTHOP_ACTION_DISCARD, 3043 /* Nexthop traps packets */ 3044 MLXSW_SP_NEXTHOP_ACTION_TRAP, 3045}; 3046 3047struct mlxsw_sp_nexthop_key { 3048 struct fib_nh *fib_nh; 3049}; 3050 3051struct mlxsw_sp_nexthop { 3052 struct list_head neigh_list_node; /* member of neigh entry list */ 3053 struct list_head crif_list_node; 3054 struct list_head router_list_node; 3055 struct mlxsw_sp_nexthop_group_info *nhgi; /* pointer back to the group 3056 * this nexthop belongs to 3057 */ 3058 struct rhash_head ht_node; 3059 struct neigh_table *neigh_tbl; 3060 struct mlxsw_sp_nexthop_key key; 3061 unsigned char gw_addr[sizeof(struct in6_addr)]; 3062 int ifindex; 3063 int nh_weight; 3064 int norm_nh_weight; 3065 int num_adj_entries; 3066 struct mlxsw_sp_crif *crif; 3067 u8 should_offload:1, /* set indicates this nexthop should be written 3068 * to the adjacency table. 3069 */ 3070 offloaded:1, /* set indicates this nexthop was written to the 3071 * adjacency table. 3072 */ 3073 update:1; /* set indicates this nexthop should be updated in the 3074 * adjacency table (f.e., its MAC changed). 3075 */ 3076 enum mlxsw_sp_nexthop_action action; 3077 enum mlxsw_sp_nexthop_type type; 3078 union { 3079 struct mlxsw_sp_neigh_entry *neigh_entry; 3080 struct mlxsw_sp_ipip_entry *ipip_entry; 3081 }; 3082 unsigned int counter_index; 3083 bool counter_valid; 3084}; 3085 3086static struct net_device * 3087mlxsw_sp_nexthop_dev(const struct mlxsw_sp_nexthop *nh) 3088{ 3089 if (!nh->crif) 3090 return NULL; 3091 return nh->crif->key.dev; 3092} 3093 3094enum mlxsw_sp_nexthop_group_type { 3095 MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4, 3096 MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6, 3097 MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ, 3098}; 3099 3100struct mlxsw_sp_nexthop_group_info { 3101 struct mlxsw_sp_nexthop_group *nh_grp; 3102 u32 adj_index; 3103 u16 ecmp_size; 3104 u16 count; 3105 int sum_norm_weight; 3106 u8 adj_index_valid:1, 3107 gateway:1, /* routes using the group use a gateway */ 3108 is_resilient:1; 3109 struct list_head list; /* member in nh_res_grp_list */ 3110 struct mlxsw_sp_nexthop nexthops[]; 3111}; 3112 3113static struct mlxsw_sp_rif * 3114mlxsw_sp_nhgi_rif(const struct mlxsw_sp_nexthop_group_info *nhgi) 3115{ 3116 struct mlxsw_sp_crif *crif = nhgi->nexthops[0].crif; 3117 3118 if (!crif) 3119 return NULL; 3120 return crif->rif; 3121} 3122 3123struct mlxsw_sp_nexthop_group_vr_key { 3124 u16 vr_id; 3125 enum mlxsw_sp_l3proto proto; 3126}; 3127 3128struct mlxsw_sp_nexthop_group_vr_entry { 3129 struct list_head list; /* member in vr_list */ 3130 struct rhash_head ht_node; /* member in vr_ht */ 3131 refcount_t ref_count; 3132 struct mlxsw_sp_nexthop_group_vr_key key; 3133}; 3134 3135struct mlxsw_sp_nexthop_group { 3136 struct rhash_head ht_node; 3137 struct list_head fib_list; /* list of fib entries that use this group */ 3138 union { 3139 struct { 3140 struct fib_info *fi; 3141 } ipv4; 3142 struct { 3143 u32 id; 3144 } obj; 3145 }; 3146 struct mlxsw_sp_nexthop_group_info *nhgi; 3147 struct list_head vr_list; 3148 struct rhashtable vr_ht; 3149 enum mlxsw_sp_nexthop_group_type type; 3150 bool can_destroy; 3151}; 3152 3153void mlxsw_sp_nexthop_counter_alloc(struct mlxsw_sp *mlxsw_sp, 3154 struct mlxsw_sp_nexthop *nh) 3155{ 3156 struct devlink *devlink; 3157 3158 devlink = priv_to_devlink(mlxsw_sp->core); 3159 if (!devlink_dpipe_table_counter_enabled(devlink, 3160 MLXSW_SP_DPIPE_TABLE_NAME_ADJ)) 3161 return; 3162 3163 if (mlxsw_sp_flow_counter_alloc(mlxsw_sp, &nh->counter_index)) 3164 return; 3165 3166 nh->counter_valid = true; 3167} 3168 3169void mlxsw_sp_nexthop_counter_free(struct mlxsw_sp *mlxsw_sp, 3170 struct mlxsw_sp_nexthop *nh) 3171{ 3172 if (!nh->counter_valid) 3173 return; 3174 mlxsw_sp_flow_counter_free(mlxsw_sp, nh->counter_index); 3175 nh->counter_valid = false; 3176} 3177 3178int mlxsw_sp_nexthop_counter_get(struct mlxsw_sp *mlxsw_sp, 3179 struct mlxsw_sp_nexthop *nh, u64 *p_counter) 3180{ 3181 if (!nh->counter_valid) 3182 return -EINVAL; 3183 3184 return mlxsw_sp_flow_counter_get(mlxsw_sp, nh->counter_index, 3185 p_counter, NULL); 3186} 3187 3188struct mlxsw_sp_nexthop *mlxsw_sp_nexthop_next(struct mlxsw_sp_router *router, 3189 struct mlxsw_sp_nexthop *nh) 3190{ 3191 if (!nh) { 3192 if (list_empty(&router->nexthop_list)) 3193 return NULL; 3194 else 3195 return list_first_entry(&router->nexthop_list, 3196 typeof(*nh), router_list_node); 3197 } 3198 if (list_is_last(&nh->router_list_node, &router->nexthop_list)) 3199 return NULL; 3200 return list_next_entry(nh, router_list_node); 3201} 3202 3203bool mlxsw_sp_nexthop_is_forward(const struct mlxsw_sp_nexthop *nh) 3204{ 3205 return nh->offloaded && nh->action == MLXSW_SP_NEXTHOP_ACTION_FORWARD; 3206} 3207 3208unsigned char *mlxsw_sp_nexthop_ha(struct mlxsw_sp_nexthop *nh) 3209{ 3210 if (nh->type != MLXSW_SP_NEXTHOP_TYPE_ETH || 3211 !mlxsw_sp_nexthop_is_forward(nh)) 3212 return NULL; 3213 return nh->neigh_entry->ha; 3214} 3215 3216int mlxsw_sp_nexthop_indexes(struct mlxsw_sp_nexthop *nh, u32 *p_adj_index, 3217 u32 *p_adj_size, u32 *p_adj_hash_index) 3218{ 3219 struct mlxsw_sp_nexthop_group_info *nhgi = nh->nhgi; 3220 u32 adj_hash_index = 0; 3221 int i; 3222 3223 if (!nh->offloaded || !nhgi->adj_index_valid) 3224 return -EINVAL; 3225 3226 *p_adj_index = nhgi->adj_index; 3227 *p_adj_size = nhgi->ecmp_size; 3228 3229 for (i = 0; i < nhgi->count; i++) { 3230 struct mlxsw_sp_nexthop *nh_iter = &nhgi->nexthops[i]; 3231 3232 if (nh_iter == nh) 3233 break; 3234 if (nh_iter->offloaded) 3235 adj_hash_index += nh_iter->num_adj_entries; 3236 } 3237 3238 *p_adj_hash_index = adj_hash_index; 3239 return 0; 3240} 3241 3242struct mlxsw_sp_rif *mlxsw_sp_nexthop_rif(struct mlxsw_sp_nexthop *nh) 3243{ 3244 if (WARN_ON(!nh->crif)) 3245 return NULL; 3246 return nh->crif->rif; 3247} 3248 3249bool mlxsw_sp_nexthop_group_has_ipip(struct mlxsw_sp_nexthop *nh) 3250{ 3251 struct mlxsw_sp_nexthop_group_info *nhgi = nh->nhgi; 3252 int i; 3253 3254 for (i = 0; i < nhgi->count; i++) { 3255 struct mlxsw_sp_nexthop *nh_iter = &nhgi->nexthops[i]; 3256 3257 if (nh_iter->type == MLXSW_SP_NEXTHOP_TYPE_IPIP) 3258 return true; 3259 } 3260 return false; 3261} 3262 3263static const struct rhashtable_params mlxsw_sp_nexthop_group_vr_ht_params = { 3264 .key_offset = offsetof(struct mlxsw_sp_nexthop_group_vr_entry, key), 3265 .head_offset = offsetof(struct mlxsw_sp_nexthop_group_vr_entry, ht_node), 3266 .key_len = sizeof(struct mlxsw_sp_nexthop_group_vr_key), 3267 .automatic_shrinking = true, 3268}; 3269 3270static struct mlxsw_sp_nexthop_group_vr_entry * 3271mlxsw_sp_nexthop_group_vr_entry_lookup(struct mlxsw_sp_nexthop_group *nh_grp, 3272 const struct mlxsw_sp_fib *fib) 3273{ 3274 struct mlxsw_sp_nexthop_group_vr_key key; 3275 3276 memset(&key, 0, sizeof(key)); 3277 key.vr_id = fib->vr->id; 3278 key.proto = fib->proto; 3279 return rhashtable_lookup_fast(&nh_grp->vr_ht, &key, 3280 mlxsw_sp_nexthop_group_vr_ht_params); 3281} 3282 3283static int 3284mlxsw_sp_nexthop_group_vr_entry_create(struct mlxsw_sp_nexthop_group *nh_grp, 3285 const struct mlxsw_sp_fib *fib) 3286{ 3287 struct mlxsw_sp_nexthop_group_vr_entry *vr_entry; 3288 int err; 3289 3290 vr_entry = kzalloc(sizeof(*vr_entry), GFP_KERNEL); 3291 if (!vr_entry) 3292 return -ENOMEM; 3293 3294 vr_entry->key.vr_id = fib->vr->id; 3295 vr_entry->key.proto = fib->proto; 3296 refcount_set(&vr_entry->ref_count, 1); 3297 3298 err = rhashtable_insert_fast(&nh_grp->vr_ht, &vr_entry->ht_node, 3299 mlxsw_sp_nexthop_group_vr_ht_params); 3300 if (err) 3301 goto err_hashtable_insert; 3302 3303 list_add(&vr_entry->list, &nh_grp->vr_list); 3304 3305 return 0; 3306 3307err_hashtable_insert: 3308 kfree(vr_entry); 3309 return err; 3310} 3311 3312static void 3313mlxsw_sp_nexthop_group_vr_entry_destroy(struct mlxsw_sp_nexthop_group *nh_grp, 3314 struct mlxsw_sp_nexthop_group_vr_entry *vr_entry) 3315{ 3316 list_del(&vr_entry->list); 3317 rhashtable_remove_fast(&nh_grp->vr_ht, &vr_entry->ht_node, 3318 mlxsw_sp_nexthop_group_vr_ht_params); 3319 kfree(vr_entry); 3320} 3321 3322static int 3323mlxsw_sp_nexthop_group_vr_link(struct mlxsw_sp_nexthop_group *nh_grp, 3324 const struct mlxsw_sp_fib *fib) 3325{ 3326 struct mlxsw_sp_nexthop_group_vr_entry *vr_entry; 3327 3328 vr_entry = mlxsw_sp_nexthop_group_vr_entry_lookup(nh_grp, fib); 3329 if (vr_entry) { 3330 refcount_inc(&vr_entry->ref_count); 3331 return 0; 3332 } 3333 3334 return mlxsw_sp_nexthop_group_vr_entry_create(nh_grp, fib); 3335} 3336 3337static void 3338mlxsw_sp_nexthop_group_vr_unlink(struct mlxsw_sp_nexthop_group *nh_grp, 3339 const struct mlxsw_sp_fib *fib) 3340{ 3341 struct mlxsw_sp_nexthop_group_vr_entry *vr_entry; 3342 3343 vr_entry = mlxsw_sp_nexthop_group_vr_entry_lookup(nh_grp, fib); 3344 if (WARN_ON_ONCE(!vr_entry)) 3345 return; 3346 3347 if (!refcount_dec_and_test(&vr_entry->ref_count)) 3348 return; 3349 3350 mlxsw_sp_nexthop_group_vr_entry_destroy(nh_grp, vr_entry); 3351} 3352 3353struct mlxsw_sp_nexthop_group_cmp_arg { 3354 enum mlxsw_sp_nexthop_group_type type; 3355 union { 3356 struct fib_info *fi; 3357 struct mlxsw_sp_fib6_entry *fib6_entry; 3358 u32 id; 3359 }; 3360}; 3361 3362static bool 3363mlxsw_sp_nexthop6_group_has_nexthop(const struct mlxsw_sp_nexthop_group *nh_grp, 3364 const struct in6_addr *gw, int ifindex, 3365 int weight) 3366{ 3367 int i; 3368 3369 for (i = 0; i < nh_grp->nhgi->count; i++) { 3370 const struct mlxsw_sp_nexthop *nh; 3371 3372 nh = &nh_grp->nhgi->nexthops[i]; 3373 if (nh->ifindex == ifindex && nh->nh_weight == weight && 3374 ipv6_addr_equal(gw, (struct in6_addr *) nh->gw_addr)) 3375 return true; 3376 } 3377 3378 return false; 3379} 3380 3381static bool 3382mlxsw_sp_nexthop6_group_cmp(const struct mlxsw_sp_nexthop_group *nh_grp, 3383 const struct mlxsw_sp_fib6_entry *fib6_entry) 3384{ 3385 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 3386 3387 if (nh_grp->nhgi->count != fib6_entry->nrt6) 3388 return false; 3389 3390 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) { 3391 struct fib6_nh *fib6_nh = mlxsw_sp_rt6->rt->fib6_nh; 3392 struct in6_addr *gw; 3393 int ifindex, weight; 3394 3395 ifindex = fib6_nh->fib_nh_dev->ifindex; 3396 weight = fib6_nh->fib_nh_weight; 3397 gw = &fib6_nh->fib_nh_gw6; 3398 if (!mlxsw_sp_nexthop6_group_has_nexthop(nh_grp, gw, ifindex, 3399 weight)) 3400 return false; 3401 } 3402 3403 return true; 3404} 3405 3406static int 3407mlxsw_sp_nexthop_group_cmp(struct rhashtable_compare_arg *arg, const void *ptr) 3408{ 3409 const struct mlxsw_sp_nexthop_group_cmp_arg *cmp_arg = arg->key; 3410 const struct mlxsw_sp_nexthop_group *nh_grp = ptr; 3411 3412 if (nh_grp->type != cmp_arg->type) 3413 return 1; 3414 3415 switch (cmp_arg->type) { 3416 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4: 3417 return cmp_arg->fi != nh_grp->ipv4.fi; 3418 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6: 3419 return !mlxsw_sp_nexthop6_group_cmp(nh_grp, 3420 cmp_arg->fib6_entry); 3421 case MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ: 3422 return cmp_arg->id != nh_grp->obj.id; 3423 default: 3424 WARN_ON(1); 3425 return 1; 3426 } 3427} 3428 3429static u32 mlxsw_sp_nexthop_group_hash_obj(const void *data, u32 len, u32 seed) 3430{ 3431 const struct mlxsw_sp_nexthop_group *nh_grp = data; 3432 const struct mlxsw_sp_nexthop *nh; 3433 struct fib_info *fi; 3434 unsigned int val; 3435 int i; 3436 3437 switch (nh_grp->type) { 3438 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4: 3439 fi = nh_grp->ipv4.fi; 3440 return jhash(&fi, sizeof(fi), seed); 3441 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6: 3442 val = nh_grp->nhgi->count; 3443 for (i = 0; i < nh_grp->nhgi->count; i++) { 3444 nh = &nh_grp->nhgi->nexthops[i]; 3445 val ^= jhash(&nh->ifindex, sizeof(nh->ifindex), seed); 3446 val ^= jhash(&nh->gw_addr, sizeof(nh->gw_addr), seed); 3447 } 3448 return jhash(&val, sizeof(val), seed); 3449 case MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ: 3450 return jhash(&nh_grp->obj.id, sizeof(nh_grp->obj.id), seed); 3451 default: 3452 WARN_ON(1); 3453 return 0; 3454 } 3455} 3456 3457static u32 3458mlxsw_sp_nexthop6_group_hash(struct mlxsw_sp_fib6_entry *fib6_entry, u32 seed) 3459{ 3460 unsigned int val = fib6_entry->nrt6; 3461 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 3462 3463 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) { 3464 struct fib6_nh *fib6_nh = mlxsw_sp_rt6->rt->fib6_nh; 3465 struct net_device *dev = fib6_nh->fib_nh_dev; 3466 struct in6_addr *gw = &fib6_nh->fib_nh_gw6; 3467 3468 val ^= jhash(&dev->ifindex, sizeof(dev->ifindex), seed); 3469 val ^= jhash(gw, sizeof(*gw), seed); 3470 } 3471 3472 return jhash(&val, sizeof(val), seed); 3473} 3474 3475static u32 3476mlxsw_sp_nexthop_group_hash(const void *data, u32 len, u32 seed) 3477{ 3478 const struct mlxsw_sp_nexthop_group_cmp_arg *cmp_arg = data; 3479 3480 switch (cmp_arg->type) { 3481 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4: 3482 return jhash(&cmp_arg->fi, sizeof(cmp_arg->fi), seed); 3483 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6: 3484 return mlxsw_sp_nexthop6_group_hash(cmp_arg->fib6_entry, seed); 3485 case MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ: 3486 return jhash(&cmp_arg->id, sizeof(cmp_arg->id), seed); 3487 default: 3488 WARN_ON(1); 3489 return 0; 3490 } 3491} 3492 3493static const struct rhashtable_params mlxsw_sp_nexthop_group_ht_params = { 3494 .head_offset = offsetof(struct mlxsw_sp_nexthop_group, ht_node), 3495 .hashfn = mlxsw_sp_nexthop_group_hash, 3496 .obj_hashfn = mlxsw_sp_nexthop_group_hash_obj, 3497 .obj_cmpfn = mlxsw_sp_nexthop_group_cmp, 3498}; 3499 3500static int mlxsw_sp_nexthop_group_insert(struct mlxsw_sp *mlxsw_sp, 3501 struct mlxsw_sp_nexthop_group *nh_grp) 3502{ 3503 if (nh_grp->type == MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6 && 3504 !nh_grp->nhgi->gateway) 3505 return 0; 3506 3507 return rhashtable_insert_fast(&mlxsw_sp->router->nexthop_group_ht, 3508 &nh_grp->ht_node, 3509 mlxsw_sp_nexthop_group_ht_params); 3510} 3511 3512static void mlxsw_sp_nexthop_group_remove(struct mlxsw_sp *mlxsw_sp, 3513 struct mlxsw_sp_nexthop_group *nh_grp) 3514{ 3515 if (nh_grp->type == MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6 && 3516 !nh_grp->nhgi->gateway) 3517 return; 3518 3519 rhashtable_remove_fast(&mlxsw_sp->router->nexthop_group_ht, 3520 &nh_grp->ht_node, 3521 mlxsw_sp_nexthop_group_ht_params); 3522} 3523 3524static struct mlxsw_sp_nexthop_group * 3525mlxsw_sp_nexthop4_group_lookup(struct mlxsw_sp *mlxsw_sp, 3526 struct fib_info *fi) 3527{ 3528 struct mlxsw_sp_nexthop_group_cmp_arg cmp_arg; 3529 3530 cmp_arg.type = MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4; 3531 cmp_arg.fi = fi; 3532 return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_group_ht, 3533 &cmp_arg, 3534 mlxsw_sp_nexthop_group_ht_params); 3535} 3536 3537static struct mlxsw_sp_nexthop_group * 3538mlxsw_sp_nexthop6_group_lookup(struct mlxsw_sp *mlxsw_sp, 3539 struct mlxsw_sp_fib6_entry *fib6_entry) 3540{ 3541 struct mlxsw_sp_nexthop_group_cmp_arg cmp_arg; 3542 3543 cmp_arg.type = MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6; 3544 cmp_arg.fib6_entry = fib6_entry; 3545 return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_group_ht, 3546 &cmp_arg, 3547 mlxsw_sp_nexthop_group_ht_params); 3548} 3549 3550static const struct rhashtable_params mlxsw_sp_nexthop_ht_params = { 3551 .key_offset = offsetof(struct mlxsw_sp_nexthop, key), 3552 .head_offset = offsetof(struct mlxsw_sp_nexthop, ht_node), 3553 .key_len = sizeof(struct mlxsw_sp_nexthop_key), 3554}; 3555 3556static int mlxsw_sp_nexthop_insert(struct mlxsw_sp *mlxsw_sp, 3557 struct mlxsw_sp_nexthop *nh) 3558{ 3559 return rhashtable_insert_fast(&mlxsw_sp->router->nexthop_ht, 3560 &nh->ht_node, mlxsw_sp_nexthop_ht_params); 3561} 3562 3563static void mlxsw_sp_nexthop_remove(struct mlxsw_sp *mlxsw_sp, 3564 struct mlxsw_sp_nexthop *nh) 3565{ 3566 rhashtable_remove_fast(&mlxsw_sp->router->nexthop_ht, &nh->ht_node, 3567 mlxsw_sp_nexthop_ht_params); 3568} 3569 3570static struct mlxsw_sp_nexthop * 3571mlxsw_sp_nexthop_lookup(struct mlxsw_sp *mlxsw_sp, 3572 struct mlxsw_sp_nexthop_key key) 3573{ 3574 return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_ht, &key, 3575 mlxsw_sp_nexthop_ht_params); 3576} 3577 3578static int mlxsw_sp_adj_index_mass_update_vr(struct mlxsw_sp *mlxsw_sp, 3579 enum mlxsw_sp_l3proto proto, 3580 u16 vr_id, 3581 u32 adj_index, u16 ecmp_size, 3582 u32 new_adj_index, 3583 u16 new_ecmp_size) 3584{ 3585 char raleu_pl[MLXSW_REG_RALEU_LEN]; 3586 3587 mlxsw_reg_raleu_pack(raleu_pl, 3588 (enum mlxsw_reg_ralxx_protocol) proto, vr_id, 3589 adj_index, ecmp_size, new_adj_index, 3590 new_ecmp_size); 3591 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raleu), raleu_pl); 3592} 3593 3594static int mlxsw_sp_adj_index_mass_update(struct mlxsw_sp *mlxsw_sp, 3595 struct mlxsw_sp_nexthop_group *nh_grp, 3596 u32 old_adj_index, u16 old_ecmp_size) 3597{ 3598 struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi; 3599 struct mlxsw_sp_nexthop_group_vr_entry *vr_entry; 3600 int err; 3601 3602 list_for_each_entry(vr_entry, &nh_grp->vr_list, list) { 3603 err = mlxsw_sp_adj_index_mass_update_vr(mlxsw_sp, 3604 vr_entry->key.proto, 3605 vr_entry->key.vr_id, 3606 old_adj_index, 3607 old_ecmp_size, 3608 nhgi->adj_index, 3609 nhgi->ecmp_size); 3610 if (err) 3611 goto err_mass_update_vr; 3612 } 3613 return 0; 3614 3615err_mass_update_vr: 3616 list_for_each_entry_continue_reverse(vr_entry, &nh_grp->vr_list, list) 3617 mlxsw_sp_adj_index_mass_update_vr(mlxsw_sp, vr_entry->key.proto, 3618 vr_entry->key.vr_id, 3619 nhgi->adj_index, 3620 nhgi->ecmp_size, 3621 old_adj_index, old_ecmp_size); 3622 return err; 3623} 3624 3625static int __mlxsw_sp_nexthop_eth_update(struct mlxsw_sp *mlxsw_sp, 3626 u32 adj_index, 3627 struct mlxsw_sp_nexthop *nh, 3628 bool force, char *ratr_pl) 3629{ 3630 struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry; 3631 struct mlxsw_sp_rif *rif = mlxsw_sp_nexthop_rif(nh); 3632 enum mlxsw_reg_ratr_op op; 3633 u16 rif_index; 3634 3635 rif_index = rif ? rif->rif_index : 3636 mlxsw_sp->router->lb_crif->rif->rif_index; 3637 op = force ? MLXSW_REG_RATR_OP_WRITE_WRITE_ENTRY : 3638 MLXSW_REG_RATR_OP_WRITE_WRITE_ENTRY_ON_ACTIVITY; 3639 mlxsw_reg_ratr_pack(ratr_pl, op, true, MLXSW_REG_RATR_TYPE_ETHERNET, 3640 adj_index, rif_index); 3641 switch (nh->action) { 3642 case MLXSW_SP_NEXTHOP_ACTION_FORWARD: 3643 mlxsw_reg_ratr_eth_entry_pack(ratr_pl, neigh_entry->ha); 3644 break; 3645 case MLXSW_SP_NEXTHOP_ACTION_DISCARD: 3646 mlxsw_reg_ratr_trap_action_set(ratr_pl, 3647 MLXSW_REG_RATR_TRAP_ACTION_DISCARD_ERRORS); 3648 break; 3649 case MLXSW_SP_NEXTHOP_ACTION_TRAP: 3650 mlxsw_reg_ratr_trap_action_set(ratr_pl, 3651 MLXSW_REG_RATR_TRAP_ACTION_TRAP); 3652 mlxsw_reg_ratr_trap_id_set(ratr_pl, MLXSW_TRAP_ID_RTR_EGRESS0); 3653 break; 3654 default: 3655 WARN_ON_ONCE(1); 3656 return -EINVAL; 3657 } 3658 if (nh->counter_valid) 3659 mlxsw_reg_ratr_counter_pack(ratr_pl, nh->counter_index, true); 3660 else 3661 mlxsw_reg_ratr_counter_pack(ratr_pl, 0, false); 3662 3663 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ratr), ratr_pl); 3664} 3665 3666int mlxsw_sp_nexthop_eth_update(struct mlxsw_sp *mlxsw_sp, u32 adj_index, 3667 struct mlxsw_sp_nexthop *nh, bool force, 3668 char *ratr_pl) 3669{ 3670 int i; 3671 3672 for (i = 0; i < nh->num_adj_entries; i++) { 3673 int err; 3674 3675 err = __mlxsw_sp_nexthop_eth_update(mlxsw_sp, adj_index + i, 3676 nh, force, ratr_pl); 3677 if (err) 3678 return err; 3679 } 3680 3681 return 0; 3682} 3683 3684static int __mlxsw_sp_nexthop_ipip_update(struct mlxsw_sp *mlxsw_sp, 3685 u32 adj_index, 3686 struct mlxsw_sp_nexthop *nh, 3687 bool force, char *ratr_pl) 3688{ 3689 const struct mlxsw_sp_ipip_ops *ipip_ops; 3690 3691 ipip_ops = mlxsw_sp->router->ipip_ops_arr[nh->ipip_entry->ipipt]; 3692 return ipip_ops->nexthop_update(mlxsw_sp, adj_index, nh->ipip_entry, 3693 force, ratr_pl); 3694} 3695 3696static int mlxsw_sp_nexthop_ipip_update(struct mlxsw_sp *mlxsw_sp, 3697 u32 adj_index, 3698 struct mlxsw_sp_nexthop *nh, bool force, 3699 char *ratr_pl) 3700{ 3701 int i; 3702 3703 for (i = 0; i < nh->num_adj_entries; i++) { 3704 int err; 3705 3706 err = __mlxsw_sp_nexthop_ipip_update(mlxsw_sp, adj_index + i, 3707 nh, force, ratr_pl); 3708 if (err) 3709 return err; 3710 } 3711 3712 return 0; 3713} 3714 3715static int mlxsw_sp_nexthop_update(struct mlxsw_sp *mlxsw_sp, u32 adj_index, 3716 struct mlxsw_sp_nexthop *nh, bool force, 3717 char *ratr_pl) 3718{ 3719 /* When action is discard or trap, the nexthop must be 3720 * programmed as an Ethernet nexthop. 3721 */ 3722 if (nh->type == MLXSW_SP_NEXTHOP_TYPE_ETH || 3723 nh->action == MLXSW_SP_NEXTHOP_ACTION_DISCARD || 3724 nh->action == MLXSW_SP_NEXTHOP_ACTION_TRAP) 3725 return mlxsw_sp_nexthop_eth_update(mlxsw_sp, adj_index, nh, 3726 force, ratr_pl); 3727 else 3728 return mlxsw_sp_nexthop_ipip_update(mlxsw_sp, adj_index, nh, 3729 force, ratr_pl); 3730} 3731 3732static int 3733mlxsw_sp_nexthop_group_update(struct mlxsw_sp *mlxsw_sp, 3734 struct mlxsw_sp_nexthop_group_info *nhgi, 3735 bool reallocate) 3736{ 3737 char ratr_pl[MLXSW_REG_RATR_LEN]; 3738 u32 adj_index = nhgi->adj_index; /* base */ 3739 struct mlxsw_sp_nexthop *nh; 3740 int i; 3741 3742 for (i = 0; i < nhgi->count; i++) { 3743 nh = &nhgi->nexthops[i]; 3744 3745 if (!nh->should_offload) { 3746 nh->offloaded = 0; 3747 continue; 3748 } 3749 3750 if (nh->update || reallocate) { 3751 int err = 0; 3752 3753 err = mlxsw_sp_nexthop_update(mlxsw_sp, adj_index, nh, 3754 true, ratr_pl); 3755 if (err) 3756 return err; 3757 nh->update = 0; 3758 nh->offloaded = 1; 3759 } 3760 adj_index += nh->num_adj_entries; 3761 } 3762 return 0; 3763} 3764 3765static int 3766mlxsw_sp_nexthop_fib_entries_update(struct mlxsw_sp *mlxsw_sp, 3767 struct mlxsw_sp_nexthop_group *nh_grp) 3768{ 3769 struct mlxsw_sp_fib_entry *fib_entry; 3770 int err; 3771 3772 list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) { 3773 err = mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry); 3774 if (err) 3775 return err; 3776 } 3777 return 0; 3778} 3779 3780struct mlxsw_sp_adj_grp_size_range { 3781 u16 start; /* Inclusive */ 3782 u16 end; /* Inclusive */ 3783}; 3784 3785/* Ordered by range start value */ 3786static const struct mlxsw_sp_adj_grp_size_range 3787mlxsw_sp1_adj_grp_size_ranges[] = { 3788 { .start = 1, .end = 64 }, 3789 { .start = 512, .end = 512 }, 3790 { .start = 1024, .end = 1024 }, 3791 { .start = 2048, .end = 2048 }, 3792 { .start = 4096, .end = 4096 }, 3793}; 3794 3795/* Ordered by range start value */ 3796static const struct mlxsw_sp_adj_grp_size_range 3797mlxsw_sp2_adj_grp_size_ranges[] = { 3798 { .start = 1, .end = 128 }, 3799 { .start = 256, .end = 256 }, 3800 { .start = 512, .end = 512 }, 3801 { .start = 1024, .end = 1024 }, 3802 { .start = 2048, .end = 2048 }, 3803 { .start = 4096, .end = 4096 }, 3804}; 3805 3806static void mlxsw_sp_adj_grp_size_round_up(const struct mlxsw_sp *mlxsw_sp, 3807 u16 *p_adj_grp_size) 3808{ 3809 int i; 3810 3811 for (i = 0; i < mlxsw_sp->router->adj_grp_size_ranges_count; i++) { 3812 const struct mlxsw_sp_adj_grp_size_range *size_range; 3813 3814 size_range = &mlxsw_sp->router->adj_grp_size_ranges[i]; 3815 3816 if (*p_adj_grp_size >= size_range->start && 3817 *p_adj_grp_size <= size_range->end) 3818 return; 3819 3820 if (*p_adj_grp_size <= size_range->end) { 3821 *p_adj_grp_size = size_range->end; 3822 return; 3823 } 3824 } 3825} 3826 3827static void mlxsw_sp_adj_grp_size_round_down(const struct mlxsw_sp *mlxsw_sp, 3828 u16 *p_adj_grp_size, 3829 unsigned int alloc_size) 3830{ 3831 int i; 3832 3833 for (i = mlxsw_sp->router->adj_grp_size_ranges_count - 1; i >= 0; i--) { 3834 const struct mlxsw_sp_adj_grp_size_range *size_range; 3835 3836 size_range = &mlxsw_sp->router->adj_grp_size_ranges[i]; 3837 3838 if (alloc_size >= size_range->end) { 3839 *p_adj_grp_size = size_range->end; 3840 return; 3841 } 3842 } 3843} 3844 3845static int mlxsw_sp_fix_adj_grp_size(struct mlxsw_sp *mlxsw_sp, 3846 u16 *p_adj_grp_size) 3847{ 3848 unsigned int alloc_size; 3849 int err; 3850 3851 /* Round up the requested group size to the next size supported 3852 * by the device and make sure the request can be satisfied. 3853 */ 3854 mlxsw_sp_adj_grp_size_round_up(mlxsw_sp, p_adj_grp_size); 3855 err = mlxsw_sp_kvdl_alloc_count_query(mlxsw_sp, 3856 MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 3857 *p_adj_grp_size, &alloc_size); 3858 if (err) 3859 return err; 3860 /* It is possible the allocation results in more allocated 3861 * entries than requested. Try to use as much of them as 3862 * possible. 3863 */ 3864 mlxsw_sp_adj_grp_size_round_down(mlxsw_sp, p_adj_grp_size, alloc_size); 3865 3866 return 0; 3867} 3868 3869static void 3870mlxsw_sp_nexthop_group_normalize(struct mlxsw_sp_nexthop_group_info *nhgi) 3871{ 3872 int i, g = 0, sum_norm_weight = 0; 3873 struct mlxsw_sp_nexthop *nh; 3874 3875 for (i = 0; i < nhgi->count; i++) { 3876 nh = &nhgi->nexthops[i]; 3877 3878 if (!nh->should_offload) 3879 continue; 3880 if (g > 0) 3881 g = gcd(nh->nh_weight, g); 3882 else 3883 g = nh->nh_weight; 3884 } 3885 3886 for (i = 0; i < nhgi->count; i++) { 3887 nh = &nhgi->nexthops[i]; 3888 3889 if (!nh->should_offload) 3890 continue; 3891 nh->norm_nh_weight = nh->nh_weight / g; 3892 sum_norm_weight += nh->norm_nh_weight; 3893 } 3894 3895 nhgi->sum_norm_weight = sum_norm_weight; 3896} 3897 3898static void 3899mlxsw_sp_nexthop_group_rebalance(struct mlxsw_sp_nexthop_group_info *nhgi) 3900{ 3901 int i, weight = 0, lower_bound = 0; 3902 int total = nhgi->sum_norm_weight; 3903 u16 ecmp_size = nhgi->ecmp_size; 3904 3905 for (i = 0; i < nhgi->count; i++) { 3906 struct mlxsw_sp_nexthop *nh = &nhgi->nexthops[i]; 3907 int upper_bound; 3908 3909 if (!nh->should_offload) 3910 continue; 3911 weight += nh->norm_nh_weight; 3912 upper_bound = DIV_ROUND_CLOSEST(ecmp_size * weight, total); 3913 nh->num_adj_entries = upper_bound - lower_bound; 3914 lower_bound = upper_bound; 3915 } 3916} 3917 3918static struct mlxsw_sp_nexthop * 3919mlxsw_sp_rt6_nexthop(struct mlxsw_sp_nexthop_group *nh_grp, 3920 const struct mlxsw_sp_rt6 *mlxsw_sp_rt6); 3921 3922static void 3923mlxsw_sp_nexthop4_group_offload_refresh(struct mlxsw_sp *mlxsw_sp, 3924 struct mlxsw_sp_nexthop_group *nh_grp) 3925{ 3926 int i; 3927 3928 for (i = 0; i < nh_grp->nhgi->count; i++) { 3929 struct mlxsw_sp_nexthop *nh = &nh_grp->nhgi->nexthops[i]; 3930 3931 if (nh->offloaded) 3932 nh->key.fib_nh->fib_nh_flags |= RTNH_F_OFFLOAD; 3933 else 3934 nh->key.fib_nh->fib_nh_flags &= ~RTNH_F_OFFLOAD; 3935 } 3936} 3937 3938static void 3939__mlxsw_sp_nexthop6_group_offload_refresh(struct mlxsw_sp_nexthop_group *nh_grp, 3940 struct mlxsw_sp_fib6_entry *fib6_entry) 3941{ 3942 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 3943 3944 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) { 3945 struct fib6_nh *fib6_nh = mlxsw_sp_rt6->rt->fib6_nh; 3946 struct mlxsw_sp_nexthop *nh; 3947 3948 nh = mlxsw_sp_rt6_nexthop(nh_grp, mlxsw_sp_rt6); 3949 if (nh && nh->offloaded) 3950 fib6_nh->fib_nh_flags |= RTNH_F_OFFLOAD; 3951 else 3952 fib6_nh->fib_nh_flags &= ~RTNH_F_OFFLOAD; 3953 } 3954} 3955 3956static void 3957mlxsw_sp_nexthop6_group_offload_refresh(struct mlxsw_sp *mlxsw_sp, 3958 struct mlxsw_sp_nexthop_group *nh_grp) 3959{ 3960 struct mlxsw_sp_fib6_entry *fib6_entry; 3961 3962 /* Unfortunately, in IPv6 the route and the nexthop are described by 3963 * the same struct, so we need to iterate over all the routes using the 3964 * nexthop group and set / clear the offload indication for them. 3965 */ 3966 list_for_each_entry(fib6_entry, &nh_grp->fib_list, 3967 common.nexthop_group_node) 3968 __mlxsw_sp_nexthop6_group_offload_refresh(nh_grp, fib6_entry); 3969} 3970 3971static void 3972mlxsw_sp_nexthop_bucket_offload_refresh(struct mlxsw_sp *mlxsw_sp, 3973 const struct mlxsw_sp_nexthop *nh, 3974 u16 bucket_index) 3975{ 3976 struct mlxsw_sp_nexthop_group *nh_grp = nh->nhgi->nh_grp; 3977 bool offload = false, trap = false; 3978 3979 if (nh->offloaded) { 3980 if (nh->action == MLXSW_SP_NEXTHOP_ACTION_TRAP) 3981 trap = true; 3982 else 3983 offload = true; 3984 } 3985 nexthop_bucket_set_hw_flags(mlxsw_sp_net(mlxsw_sp), nh_grp->obj.id, 3986 bucket_index, offload, trap); 3987} 3988 3989static void 3990mlxsw_sp_nexthop_obj_group_offload_refresh(struct mlxsw_sp *mlxsw_sp, 3991 struct mlxsw_sp_nexthop_group *nh_grp) 3992{ 3993 int i; 3994 3995 /* Do not update the flags if the nexthop group is being destroyed 3996 * since: 3997 * 1. The nexthop objects is being deleted, in which case the flags are 3998 * irrelevant. 3999 * 2. The nexthop group was replaced by a newer group, in which case 4000 * the flags of the nexthop object were already updated based on the 4001 * new group. 4002 */ 4003 if (nh_grp->can_destroy) 4004 return; 4005 4006 nexthop_set_hw_flags(mlxsw_sp_net(mlxsw_sp), nh_grp->obj.id, 4007 nh_grp->nhgi->adj_index_valid, false); 4008 4009 /* Update flags of individual nexthop buckets in case of a resilient 4010 * nexthop group. 4011 */ 4012 if (!nh_grp->nhgi->is_resilient) 4013 return; 4014 4015 for (i = 0; i < nh_grp->nhgi->count; i++) { 4016 struct mlxsw_sp_nexthop *nh = &nh_grp->nhgi->nexthops[i]; 4017 4018 mlxsw_sp_nexthop_bucket_offload_refresh(mlxsw_sp, nh, i); 4019 } 4020} 4021 4022static void 4023mlxsw_sp_nexthop_group_offload_refresh(struct mlxsw_sp *mlxsw_sp, 4024 struct mlxsw_sp_nexthop_group *nh_grp) 4025{ 4026 switch (nh_grp->type) { 4027 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4: 4028 mlxsw_sp_nexthop4_group_offload_refresh(mlxsw_sp, nh_grp); 4029 break; 4030 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6: 4031 mlxsw_sp_nexthop6_group_offload_refresh(mlxsw_sp, nh_grp); 4032 break; 4033 case MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ: 4034 mlxsw_sp_nexthop_obj_group_offload_refresh(mlxsw_sp, nh_grp); 4035 break; 4036 } 4037} 4038 4039static int 4040mlxsw_sp_nexthop_group_refresh(struct mlxsw_sp *mlxsw_sp, 4041 struct mlxsw_sp_nexthop_group *nh_grp) 4042{ 4043 struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi; 4044 u16 ecmp_size, old_ecmp_size; 4045 struct mlxsw_sp_nexthop *nh; 4046 bool offload_change = false; 4047 u32 adj_index; 4048 bool old_adj_index_valid; 4049 u32 old_adj_index; 4050 int i, err2, err; 4051 4052 if (!nhgi->gateway) 4053 return mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp); 4054 4055 for (i = 0; i < nhgi->count; i++) { 4056 nh = &nhgi->nexthops[i]; 4057 4058 if (nh->should_offload != nh->offloaded) { 4059 offload_change = true; 4060 if (nh->should_offload) 4061 nh->update = 1; 4062 } 4063 } 4064 if (!offload_change) { 4065 /* Nothing was added or removed, so no need to reallocate. Just 4066 * update MAC on existing adjacency indexes. 4067 */ 4068 err = mlxsw_sp_nexthop_group_update(mlxsw_sp, nhgi, false); 4069 if (err) { 4070 dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n"); 4071 goto set_trap; 4072 } 4073 /* Flags of individual nexthop buckets might need to be 4074 * updated. 4075 */ 4076 mlxsw_sp_nexthop_group_offload_refresh(mlxsw_sp, nh_grp); 4077 return 0; 4078 } 4079 mlxsw_sp_nexthop_group_normalize(nhgi); 4080 if (!nhgi->sum_norm_weight) { 4081 /* No neigh of this group is connected so we just set 4082 * the trap and let everthing flow through kernel. 4083 */ 4084 err = 0; 4085 goto set_trap; 4086 } 4087 4088 ecmp_size = nhgi->sum_norm_weight; 4089 err = mlxsw_sp_fix_adj_grp_size(mlxsw_sp, &ecmp_size); 4090 if (err) 4091 /* No valid allocation size available. */ 4092 goto set_trap; 4093 4094 err = mlxsw_sp_kvdl_alloc(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 4095 ecmp_size, &adj_index); 4096 if (err) { 4097 /* We ran out of KVD linear space, just set the 4098 * trap and let everything flow through kernel. 4099 */ 4100 dev_warn(mlxsw_sp->bus_info->dev, "Failed to allocate KVD linear area for nexthop group.\n"); 4101 goto set_trap; 4102 } 4103 old_adj_index_valid = nhgi->adj_index_valid; 4104 old_adj_index = nhgi->adj_index; 4105 old_ecmp_size = nhgi->ecmp_size; 4106 nhgi->adj_index_valid = 1; 4107 nhgi->adj_index = adj_index; 4108 nhgi->ecmp_size = ecmp_size; 4109 mlxsw_sp_nexthop_group_rebalance(nhgi); 4110 err = mlxsw_sp_nexthop_group_update(mlxsw_sp, nhgi, true); 4111 if (err) { 4112 dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n"); 4113 goto set_trap; 4114 } 4115 4116 mlxsw_sp_nexthop_group_offload_refresh(mlxsw_sp, nh_grp); 4117 4118 if (!old_adj_index_valid) { 4119 /* The trap was set for fib entries, so we have to call 4120 * fib entry update to unset it and use adjacency index. 4121 */ 4122 err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp); 4123 if (err) { 4124 dev_warn(mlxsw_sp->bus_info->dev, "Failed to add adjacency index to fib entries.\n"); 4125 goto set_trap; 4126 } 4127 return 0; 4128 } 4129 4130 err = mlxsw_sp_adj_index_mass_update(mlxsw_sp, nh_grp, 4131 old_adj_index, old_ecmp_size); 4132 mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 4133 old_ecmp_size, old_adj_index); 4134 if (err) { 4135 dev_warn(mlxsw_sp->bus_info->dev, "Failed to mass-update adjacency index for nexthop group.\n"); 4136 goto set_trap; 4137 } 4138 4139 return 0; 4140 4141set_trap: 4142 old_adj_index_valid = nhgi->adj_index_valid; 4143 nhgi->adj_index_valid = 0; 4144 for (i = 0; i < nhgi->count; i++) { 4145 nh = &nhgi->nexthops[i]; 4146 nh->offloaded = 0; 4147 } 4148 err2 = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp); 4149 if (err2) 4150 dev_warn(mlxsw_sp->bus_info->dev, "Failed to set traps for fib entries.\n"); 4151 mlxsw_sp_nexthop_group_offload_refresh(mlxsw_sp, nh_grp); 4152 if (old_adj_index_valid) 4153 mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 4154 nhgi->ecmp_size, nhgi->adj_index); 4155 return err; 4156} 4157 4158static void __mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp_nexthop *nh, 4159 bool removing) 4160{ 4161 if (!removing) { 4162 nh->action = MLXSW_SP_NEXTHOP_ACTION_FORWARD; 4163 nh->should_offload = 1; 4164 } else if (nh->nhgi->is_resilient) { 4165 nh->action = MLXSW_SP_NEXTHOP_ACTION_TRAP; 4166 nh->should_offload = 1; 4167 } else { 4168 nh->should_offload = 0; 4169 } 4170 nh->update = 1; 4171} 4172 4173static int 4174mlxsw_sp_nexthop_dead_neigh_replace(struct mlxsw_sp *mlxsw_sp, 4175 struct mlxsw_sp_neigh_entry *neigh_entry) 4176{ 4177 struct neighbour *n, *old_n = neigh_entry->key.n; 4178 struct mlxsw_sp_nexthop *nh; 4179 struct net_device *dev; 4180 bool entry_connected; 4181 u8 nud_state, dead; 4182 int err; 4183 4184 nh = list_first_entry(&neigh_entry->nexthop_list, 4185 struct mlxsw_sp_nexthop, neigh_list_node); 4186 dev = mlxsw_sp_nexthop_dev(nh); 4187 4188 n = neigh_lookup(nh->neigh_tbl, &nh->gw_addr, dev); 4189 if (!n) { 4190 n = neigh_create(nh->neigh_tbl, &nh->gw_addr, dev); 4191 if (IS_ERR(n)) 4192 return PTR_ERR(n); 4193 neigh_event_send(n, NULL); 4194 } 4195 4196 mlxsw_sp_neigh_entry_remove(mlxsw_sp, neigh_entry); 4197 neigh_entry->key.n = n; 4198 err = mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry); 4199 if (err) 4200 goto err_neigh_entry_insert; 4201 4202 read_lock_bh(&n->lock); 4203 nud_state = n->nud_state; 4204 dead = n->dead; 4205 read_unlock_bh(&n->lock); 4206 entry_connected = nud_state & NUD_VALID && !dead; 4207 4208 list_for_each_entry(nh, &neigh_entry->nexthop_list, 4209 neigh_list_node) { 4210 neigh_release(old_n); 4211 neigh_clone(n); 4212 __mlxsw_sp_nexthop_neigh_update(nh, !entry_connected); 4213 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp); 4214 } 4215 4216 neigh_release(n); 4217 4218 return 0; 4219 4220err_neigh_entry_insert: 4221 neigh_entry->key.n = old_n; 4222 mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry); 4223 neigh_release(n); 4224 return err; 4225} 4226 4227static void 4228mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp, 4229 struct mlxsw_sp_neigh_entry *neigh_entry, 4230 bool removing, bool dead) 4231{ 4232 struct mlxsw_sp_nexthop *nh; 4233 4234 if (list_empty(&neigh_entry->nexthop_list)) 4235 return; 4236 4237 if (dead) { 4238 int err; 4239 4240 err = mlxsw_sp_nexthop_dead_neigh_replace(mlxsw_sp, 4241 neigh_entry); 4242 if (err) 4243 dev_err(mlxsw_sp->bus_info->dev, "Failed to replace dead neigh\n"); 4244 return; 4245 } 4246 4247 list_for_each_entry(nh, &neigh_entry->nexthop_list, 4248 neigh_list_node) { 4249 __mlxsw_sp_nexthop_neigh_update(nh, removing); 4250 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp); 4251 } 4252} 4253 4254static void mlxsw_sp_nexthop_crif_init(struct mlxsw_sp_nexthop *nh, 4255 struct mlxsw_sp_crif *crif) 4256{ 4257 if (nh->crif) 4258 return; 4259 4260 nh->crif = crif; 4261 list_add(&nh->crif_list_node, &crif->nexthop_list); 4262} 4263 4264static void mlxsw_sp_nexthop_crif_fini(struct mlxsw_sp_nexthop *nh) 4265{ 4266 if (!nh->crif) 4267 return; 4268 4269 list_del(&nh->crif_list_node); 4270 nh->crif = NULL; 4271} 4272 4273static int mlxsw_sp_nexthop_neigh_init(struct mlxsw_sp *mlxsw_sp, 4274 struct mlxsw_sp_nexthop *nh) 4275{ 4276 struct mlxsw_sp_neigh_entry *neigh_entry; 4277 struct net_device *dev; 4278 struct neighbour *n; 4279 u8 nud_state, dead; 4280 int err; 4281 4282 if (WARN_ON(!nh->crif->rif)) 4283 return 0; 4284 4285 if (!nh->nhgi->gateway || nh->neigh_entry) 4286 return 0; 4287 dev = mlxsw_sp_nexthop_dev(nh); 4288 4289 /* Take a reference of neigh here ensuring that neigh would 4290 * not be destructed before the nexthop entry is finished. 4291 * The reference is taken either in neigh_lookup() or 4292 * in neigh_create() in case n is not found. 4293 */ 4294 n = neigh_lookup(nh->neigh_tbl, &nh->gw_addr, dev); 4295 if (!n) { 4296 n = neigh_create(nh->neigh_tbl, &nh->gw_addr, dev); 4297 if (IS_ERR(n)) 4298 return PTR_ERR(n); 4299 neigh_event_send(n, NULL); 4300 } 4301 neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n); 4302 if (!neigh_entry) { 4303 neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n); 4304 if (IS_ERR(neigh_entry)) { 4305 err = -EINVAL; 4306 goto err_neigh_entry_create; 4307 } 4308 } 4309 4310 /* If that is the first nexthop connected to that neigh, add to 4311 * nexthop_neighs_list 4312 */ 4313 if (list_empty(&neigh_entry->nexthop_list)) 4314 list_add_tail(&neigh_entry->nexthop_neighs_list_node, 4315 &mlxsw_sp->router->nexthop_neighs_list); 4316 4317 nh->neigh_entry = neigh_entry; 4318 list_add_tail(&nh->neigh_list_node, &neigh_entry->nexthop_list); 4319 read_lock_bh(&n->lock); 4320 nud_state = n->nud_state; 4321 dead = n->dead; 4322 read_unlock_bh(&n->lock); 4323 __mlxsw_sp_nexthop_neigh_update(nh, !(nud_state & NUD_VALID && !dead)); 4324 4325 return 0; 4326 4327err_neigh_entry_create: 4328 neigh_release(n); 4329 return err; 4330} 4331 4332static void mlxsw_sp_nexthop_neigh_fini(struct mlxsw_sp *mlxsw_sp, 4333 struct mlxsw_sp_nexthop *nh) 4334{ 4335 struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry; 4336 struct neighbour *n; 4337 4338 if (!neigh_entry) 4339 return; 4340 n = neigh_entry->key.n; 4341 4342 __mlxsw_sp_nexthop_neigh_update(nh, true); 4343 list_del(&nh->neigh_list_node); 4344 nh->neigh_entry = NULL; 4345 4346 /* If that is the last nexthop connected to that neigh, remove from 4347 * nexthop_neighs_list 4348 */ 4349 if (list_empty(&neigh_entry->nexthop_list)) 4350 list_del(&neigh_entry->nexthop_neighs_list_node); 4351 4352 if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list)) 4353 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry); 4354 4355 neigh_release(n); 4356} 4357 4358static bool mlxsw_sp_ipip_netdev_ul_up(struct net_device *ol_dev) 4359{ 4360 struct net_device *ul_dev; 4361 bool is_up; 4362 4363 rcu_read_lock(); 4364 ul_dev = mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev); 4365 is_up = ul_dev ? (ul_dev->flags & IFF_UP) : true; 4366 rcu_read_unlock(); 4367 4368 return is_up; 4369} 4370 4371static void mlxsw_sp_nexthop_ipip_init(struct mlxsw_sp *mlxsw_sp, 4372 struct mlxsw_sp_nexthop *nh, 4373 struct mlxsw_sp_ipip_entry *ipip_entry) 4374{ 4375 struct mlxsw_sp_crif *crif; 4376 bool removing; 4377 4378 if (!nh->nhgi->gateway || nh->ipip_entry) 4379 return; 4380 4381 crif = mlxsw_sp_crif_lookup(mlxsw_sp->router, ipip_entry->ol_dev); 4382 if (WARN_ON(!crif)) 4383 return; 4384 4385 nh->ipip_entry = ipip_entry; 4386 removing = !mlxsw_sp_ipip_netdev_ul_up(ipip_entry->ol_dev); 4387 __mlxsw_sp_nexthop_neigh_update(nh, removing); 4388 mlxsw_sp_nexthop_crif_init(nh, crif); 4389} 4390 4391static void mlxsw_sp_nexthop_ipip_fini(struct mlxsw_sp *mlxsw_sp, 4392 struct mlxsw_sp_nexthop *nh) 4393{ 4394 struct mlxsw_sp_ipip_entry *ipip_entry = nh->ipip_entry; 4395 4396 if (!ipip_entry) 4397 return; 4398 4399 __mlxsw_sp_nexthop_neigh_update(nh, true); 4400 nh->ipip_entry = NULL; 4401} 4402 4403static bool mlxsw_sp_nexthop4_ipip_type(const struct mlxsw_sp *mlxsw_sp, 4404 const struct fib_nh *fib_nh, 4405 enum mlxsw_sp_ipip_type *p_ipipt) 4406{ 4407 struct net_device *dev = fib_nh->fib_nh_dev; 4408 4409 return dev && 4410 fib_nh->nh_parent->fib_type == RTN_UNICAST && 4411 mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, p_ipipt); 4412} 4413 4414static int mlxsw_sp_nexthop_type_init(struct mlxsw_sp *mlxsw_sp, 4415 struct mlxsw_sp_nexthop *nh, 4416 const struct net_device *dev) 4417{ 4418 const struct mlxsw_sp_ipip_ops *ipip_ops; 4419 struct mlxsw_sp_ipip_entry *ipip_entry; 4420 struct mlxsw_sp_crif *crif; 4421 int err; 4422 4423 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, dev); 4424 if (ipip_entry) { 4425 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt]; 4426 if (ipip_ops->can_offload(mlxsw_sp, dev)) { 4427 nh->type = MLXSW_SP_NEXTHOP_TYPE_IPIP; 4428 mlxsw_sp_nexthop_ipip_init(mlxsw_sp, nh, ipip_entry); 4429 return 0; 4430 } 4431 } 4432 4433 nh->type = MLXSW_SP_NEXTHOP_TYPE_ETH; 4434 crif = mlxsw_sp_crif_lookup(mlxsw_sp->router, dev); 4435 if (!crif) 4436 return 0; 4437 4438 mlxsw_sp_nexthop_crif_init(nh, crif); 4439 4440 if (!crif->rif) 4441 return 0; 4442 4443 err = mlxsw_sp_nexthop_neigh_init(mlxsw_sp, nh); 4444 if (err) 4445 goto err_neigh_init; 4446 4447 return 0; 4448 4449err_neigh_init: 4450 mlxsw_sp_nexthop_crif_fini(nh); 4451 return err; 4452} 4453 4454static int mlxsw_sp_nexthop_type_rif_made(struct mlxsw_sp *mlxsw_sp, 4455 struct mlxsw_sp_nexthop *nh) 4456{ 4457 switch (nh->type) { 4458 case MLXSW_SP_NEXTHOP_TYPE_ETH: 4459 return mlxsw_sp_nexthop_neigh_init(mlxsw_sp, nh); 4460 case MLXSW_SP_NEXTHOP_TYPE_IPIP: 4461 break; 4462 } 4463 4464 return 0; 4465} 4466 4467static void mlxsw_sp_nexthop_type_rif_gone(struct mlxsw_sp *mlxsw_sp, 4468 struct mlxsw_sp_nexthop *nh) 4469{ 4470 switch (nh->type) { 4471 case MLXSW_SP_NEXTHOP_TYPE_ETH: 4472 mlxsw_sp_nexthop_neigh_fini(mlxsw_sp, nh); 4473 break; 4474 case MLXSW_SP_NEXTHOP_TYPE_IPIP: 4475 mlxsw_sp_nexthop_ipip_fini(mlxsw_sp, nh); 4476 break; 4477 } 4478} 4479 4480static void mlxsw_sp_nexthop_type_fini(struct mlxsw_sp *mlxsw_sp, 4481 struct mlxsw_sp_nexthop *nh) 4482{ 4483 mlxsw_sp_nexthop_type_rif_gone(mlxsw_sp, nh); 4484 mlxsw_sp_nexthop_crif_fini(nh); 4485} 4486 4487static int mlxsw_sp_nexthop4_init(struct mlxsw_sp *mlxsw_sp, 4488 struct mlxsw_sp_nexthop_group *nh_grp, 4489 struct mlxsw_sp_nexthop *nh, 4490 struct fib_nh *fib_nh) 4491{ 4492 struct net_device *dev = fib_nh->fib_nh_dev; 4493 struct in_device *in_dev; 4494 int err; 4495 4496 nh->nhgi = nh_grp->nhgi; 4497 nh->key.fib_nh = fib_nh; 4498#ifdef CONFIG_IP_ROUTE_MULTIPATH 4499 nh->nh_weight = fib_nh->fib_nh_weight; 4500#else 4501 nh->nh_weight = 1; 4502#endif 4503 memcpy(&nh->gw_addr, &fib_nh->fib_nh_gw4, sizeof(fib_nh->fib_nh_gw4)); 4504 nh->neigh_tbl = &arp_tbl; 4505 err = mlxsw_sp_nexthop_insert(mlxsw_sp, nh); 4506 if (err) 4507 return err; 4508 4509 mlxsw_sp_nexthop_counter_alloc(mlxsw_sp, nh); 4510 list_add_tail(&nh->router_list_node, &mlxsw_sp->router->nexthop_list); 4511 4512 if (!dev) 4513 return 0; 4514 nh->ifindex = dev->ifindex; 4515 4516 rcu_read_lock(); 4517 in_dev = __in_dev_get_rcu(dev); 4518 if (in_dev && IN_DEV_IGNORE_ROUTES_WITH_LINKDOWN(in_dev) && 4519 fib_nh->fib_nh_flags & RTNH_F_LINKDOWN) { 4520 rcu_read_unlock(); 4521 return 0; 4522 } 4523 rcu_read_unlock(); 4524 4525 err = mlxsw_sp_nexthop_type_init(mlxsw_sp, nh, dev); 4526 if (err) 4527 goto err_nexthop_neigh_init; 4528 4529 return 0; 4530 4531err_nexthop_neigh_init: 4532 list_del(&nh->router_list_node); 4533 mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh); 4534 mlxsw_sp_nexthop_remove(mlxsw_sp, nh); 4535 return err; 4536} 4537 4538static void mlxsw_sp_nexthop4_fini(struct mlxsw_sp *mlxsw_sp, 4539 struct mlxsw_sp_nexthop *nh) 4540{ 4541 mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh); 4542 list_del(&nh->router_list_node); 4543 mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh); 4544 mlxsw_sp_nexthop_remove(mlxsw_sp, nh); 4545} 4546 4547static void mlxsw_sp_nexthop4_event(struct mlxsw_sp *mlxsw_sp, 4548 unsigned long event, struct fib_nh *fib_nh) 4549{ 4550 struct mlxsw_sp_nexthop_key key; 4551 struct mlxsw_sp_nexthop *nh; 4552 4553 key.fib_nh = fib_nh; 4554 nh = mlxsw_sp_nexthop_lookup(mlxsw_sp, key); 4555 if (!nh) 4556 return; 4557 4558 switch (event) { 4559 case FIB_EVENT_NH_ADD: 4560 mlxsw_sp_nexthop_type_init(mlxsw_sp, nh, fib_nh->fib_nh_dev); 4561 break; 4562 case FIB_EVENT_NH_DEL: 4563 mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh); 4564 break; 4565 } 4566 4567 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp); 4568} 4569 4570static void mlxsw_sp_nexthop_rif_update(struct mlxsw_sp *mlxsw_sp, 4571 struct mlxsw_sp_rif *rif) 4572{ 4573 struct net_device *dev = mlxsw_sp_rif_dev(rif); 4574 struct mlxsw_sp_nexthop *nh; 4575 bool removing; 4576 4577 list_for_each_entry(nh, &rif->crif->nexthop_list, crif_list_node) { 4578 switch (nh->type) { 4579 case MLXSW_SP_NEXTHOP_TYPE_ETH: 4580 removing = false; 4581 break; 4582 case MLXSW_SP_NEXTHOP_TYPE_IPIP: 4583 removing = !mlxsw_sp_ipip_netdev_ul_up(dev); 4584 break; 4585 default: 4586 WARN_ON(1); 4587 continue; 4588 } 4589 4590 __mlxsw_sp_nexthop_neigh_update(nh, removing); 4591 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp); 4592 } 4593} 4594 4595static int mlxsw_sp_nexthop_rif_made_sync(struct mlxsw_sp *mlxsw_sp, 4596 struct mlxsw_sp_rif *rif) 4597{ 4598 struct mlxsw_sp_nexthop *nh, *tmp; 4599 unsigned int n = 0; 4600 int err; 4601 4602 list_for_each_entry_safe(nh, tmp, &rif->crif->nexthop_list, 4603 crif_list_node) { 4604 err = mlxsw_sp_nexthop_type_rif_made(mlxsw_sp, nh); 4605 if (err) 4606 goto err_nexthop_type_rif; 4607 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp); 4608 n++; 4609 } 4610 4611 return 0; 4612 4613err_nexthop_type_rif: 4614 list_for_each_entry_safe(nh, tmp, &rif->crif->nexthop_list, 4615 crif_list_node) { 4616 if (!n--) 4617 break; 4618 mlxsw_sp_nexthop_type_rif_gone(mlxsw_sp, nh); 4619 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp); 4620 } 4621 return err; 4622} 4623 4624static void mlxsw_sp_nexthop_rif_gone_sync(struct mlxsw_sp *mlxsw_sp, 4625 struct mlxsw_sp_rif *rif) 4626{ 4627 struct mlxsw_sp_nexthop *nh, *tmp; 4628 4629 list_for_each_entry_safe(nh, tmp, &rif->crif->nexthop_list, 4630 crif_list_node) { 4631 mlxsw_sp_nexthop_type_rif_gone(mlxsw_sp, nh); 4632 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp); 4633 } 4634} 4635 4636static int mlxsw_sp_adj_trap_entry_init(struct mlxsw_sp *mlxsw_sp) 4637{ 4638 enum mlxsw_reg_ratr_trap_action trap_action; 4639 char ratr_pl[MLXSW_REG_RATR_LEN]; 4640 int err; 4641 4642 err = mlxsw_sp_kvdl_alloc(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1, 4643 &mlxsw_sp->router->adj_trap_index); 4644 if (err) 4645 return err; 4646 4647 trap_action = MLXSW_REG_RATR_TRAP_ACTION_TRAP; 4648 mlxsw_reg_ratr_pack(ratr_pl, MLXSW_REG_RATR_OP_WRITE_WRITE_ENTRY, true, 4649 MLXSW_REG_RATR_TYPE_ETHERNET, 4650 mlxsw_sp->router->adj_trap_index, 4651 mlxsw_sp->router->lb_crif->rif->rif_index); 4652 mlxsw_reg_ratr_trap_action_set(ratr_pl, trap_action); 4653 mlxsw_reg_ratr_trap_id_set(ratr_pl, MLXSW_TRAP_ID_RTR_EGRESS0); 4654 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ratr), ratr_pl); 4655 if (err) 4656 goto err_ratr_write; 4657 4658 return 0; 4659 4660err_ratr_write: 4661 mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1, 4662 mlxsw_sp->router->adj_trap_index); 4663 return err; 4664} 4665 4666static void mlxsw_sp_adj_trap_entry_fini(struct mlxsw_sp *mlxsw_sp) 4667{ 4668 mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1, 4669 mlxsw_sp->router->adj_trap_index); 4670} 4671 4672static int mlxsw_sp_nexthop_group_inc(struct mlxsw_sp *mlxsw_sp) 4673{ 4674 int err; 4675 4676 if (refcount_inc_not_zero(&mlxsw_sp->router->num_groups)) 4677 return 0; 4678 4679 err = mlxsw_sp_adj_trap_entry_init(mlxsw_sp); 4680 if (err) 4681 return err; 4682 4683 refcount_set(&mlxsw_sp->router->num_groups, 1); 4684 4685 return 0; 4686} 4687 4688static void mlxsw_sp_nexthop_group_dec(struct mlxsw_sp *mlxsw_sp) 4689{ 4690 if (!refcount_dec_and_test(&mlxsw_sp->router->num_groups)) 4691 return; 4692 4693 mlxsw_sp_adj_trap_entry_fini(mlxsw_sp); 4694} 4695 4696static void 4697mlxsw_sp_nh_grp_activity_get(struct mlxsw_sp *mlxsw_sp, 4698 const struct mlxsw_sp_nexthop_group *nh_grp, 4699 unsigned long *activity) 4700{ 4701 char *ratrad_pl; 4702 int i, err; 4703 4704 ratrad_pl = kmalloc(MLXSW_REG_RATRAD_LEN, GFP_KERNEL); 4705 if (!ratrad_pl) 4706 return; 4707 4708 mlxsw_reg_ratrad_pack(ratrad_pl, nh_grp->nhgi->adj_index, 4709 nh_grp->nhgi->count); 4710 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ratrad), ratrad_pl); 4711 if (err) 4712 goto out; 4713 4714 for (i = 0; i < nh_grp->nhgi->count; i++) { 4715 if (!mlxsw_reg_ratrad_activity_vector_get(ratrad_pl, i)) 4716 continue; 4717 bitmap_set(activity, i, 1); 4718 } 4719 4720out: 4721 kfree(ratrad_pl); 4722} 4723 4724#define MLXSW_SP_NH_GRP_ACTIVITY_UPDATE_INTERVAL 1000 /* ms */ 4725 4726static void 4727mlxsw_sp_nh_grp_activity_update(struct mlxsw_sp *mlxsw_sp, 4728 const struct mlxsw_sp_nexthop_group *nh_grp) 4729{ 4730 unsigned long *activity; 4731 4732 activity = bitmap_zalloc(nh_grp->nhgi->count, GFP_KERNEL); 4733 if (!activity) 4734 return; 4735 4736 mlxsw_sp_nh_grp_activity_get(mlxsw_sp, nh_grp, activity); 4737 nexthop_res_grp_activity_update(mlxsw_sp_net(mlxsw_sp), nh_grp->obj.id, 4738 nh_grp->nhgi->count, activity); 4739 4740 bitmap_free(activity); 4741} 4742 4743static void 4744mlxsw_sp_nh_grp_activity_work_schedule(struct mlxsw_sp *mlxsw_sp) 4745{ 4746 unsigned int interval = MLXSW_SP_NH_GRP_ACTIVITY_UPDATE_INTERVAL; 4747 4748 mlxsw_core_schedule_dw(&mlxsw_sp->router->nh_grp_activity_dw, 4749 msecs_to_jiffies(interval)); 4750} 4751 4752static void mlxsw_sp_nh_grp_activity_work(struct work_struct *work) 4753{ 4754 struct mlxsw_sp_nexthop_group_info *nhgi; 4755 struct mlxsw_sp_router *router; 4756 bool reschedule = false; 4757 4758 router = container_of(work, struct mlxsw_sp_router, 4759 nh_grp_activity_dw.work); 4760 4761 mutex_lock(&router->lock); 4762 4763 list_for_each_entry(nhgi, &router->nh_res_grp_list, list) { 4764 mlxsw_sp_nh_grp_activity_update(router->mlxsw_sp, nhgi->nh_grp); 4765 reschedule = true; 4766 } 4767 4768 mutex_unlock(&router->lock); 4769 4770 if (!reschedule) 4771 return; 4772 mlxsw_sp_nh_grp_activity_work_schedule(router->mlxsw_sp); 4773} 4774 4775static int 4776mlxsw_sp_nexthop_obj_single_validate(struct mlxsw_sp *mlxsw_sp, 4777 const struct nh_notifier_single_info *nh, 4778 struct netlink_ext_ack *extack) 4779{ 4780 int err = -EINVAL; 4781 4782 if (nh->is_fdb) 4783 NL_SET_ERR_MSG_MOD(extack, "FDB nexthops are not supported"); 4784 else if (nh->has_encap) 4785 NL_SET_ERR_MSG_MOD(extack, "Encapsulating nexthops are not supported"); 4786 else 4787 err = 0; 4788 4789 return err; 4790} 4791 4792static int 4793mlxsw_sp_nexthop_obj_group_entry_validate(struct mlxsw_sp *mlxsw_sp, 4794 const struct nh_notifier_single_info *nh, 4795 struct netlink_ext_ack *extack) 4796{ 4797 int err; 4798 4799 err = mlxsw_sp_nexthop_obj_single_validate(mlxsw_sp, nh, extack); 4800 if (err) 4801 return err; 4802 4803 /* Device only nexthops with an IPIP device are programmed as 4804 * encapsulating adjacency entries. 4805 */ 4806 if (!nh->gw_family && !nh->is_reject && 4807 !mlxsw_sp_netdev_ipip_type(mlxsw_sp, nh->dev, NULL)) { 4808 NL_SET_ERR_MSG_MOD(extack, "Nexthop group entry does not have a gateway"); 4809 return -EINVAL; 4810 } 4811 4812 return 0; 4813} 4814 4815static int 4816mlxsw_sp_nexthop_obj_group_validate(struct mlxsw_sp *mlxsw_sp, 4817 const struct nh_notifier_grp_info *nh_grp, 4818 struct netlink_ext_ack *extack) 4819{ 4820 int i; 4821 4822 if (nh_grp->is_fdb) { 4823 NL_SET_ERR_MSG_MOD(extack, "FDB nexthop groups are not supported"); 4824 return -EINVAL; 4825 } 4826 4827 for (i = 0; i < nh_grp->num_nh; i++) { 4828 const struct nh_notifier_single_info *nh; 4829 int err; 4830 4831 nh = &nh_grp->nh_entries[i].nh; 4832 err = mlxsw_sp_nexthop_obj_group_entry_validate(mlxsw_sp, nh, 4833 extack); 4834 if (err) 4835 return err; 4836 } 4837 4838 return 0; 4839} 4840 4841static int 4842mlxsw_sp_nexthop_obj_res_group_size_validate(struct mlxsw_sp *mlxsw_sp, 4843 const struct nh_notifier_res_table_info *nh_res_table, 4844 struct netlink_ext_ack *extack) 4845{ 4846 unsigned int alloc_size; 4847 bool valid_size = false; 4848 int err, i; 4849 4850 if (nh_res_table->num_nh_buckets < 32) { 4851 NL_SET_ERR_MSG_MOD(extack, "Minimum number of buckets is 32"); 4852 return -EINVAL; 4853 } 4854 4855 for (i = 0; i < mlxsw_sp->router->adj_grp_size_ranges_count; i++) { 4856 const struct mlxsw_sp_adj_grp_size_range *size_range; 4857 4858 size_range = &mlxsw_sp->router->adj_grp_size_ranges[i]; 4859 4860 if (nh_res_table->num_nh_buckets >= size_range->start && 4861 nh_res_table->num_nh_buckets <= size_range->end) { 4862 valid_size = true; 4863 break; 4864 } 4865 } 4866 4867 if (!valid_size) { 4868 NL_SET_ERR_MSG_MOD(extack, "Invalid number of buckets"); 4869 return -EINVAL; 4870 } 4871 4872 err = mlxsw_sp_kvdl_alloc_count_query(mlxsw_sp, 4873 MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 4874 nh_res_table->num_nh_buckets, 4875 &alloc_size); 4876 if (err || nh_res_table->num_nh_buckets != alloc_size) { 4877 NL_SET_ERR_MSG_MOD(extack, "Number of buckets does not fit allocation size of any KVDL partition"); 4878 return -EINVAL; 4879 } 4880 4881 return 0; 4882} 4883 4884static int 4885mlxsw_sp_nexthop_obj_res_group_validate(struct mlxsw_sp *mlxsw_sp, 4886 const struct nh_notifier_res_table_info *nh_res_table, 4887 struct netlink_ext_ack *extack) 4888{ 4889 int err; 4890 u16 i; 4891 4892 err = mlxsw_sp_nexthop_obj_res_group_size_validate(mlxsw_sp, 4893 nh_res_table, 4894 extack); 4895 if (err) 4896 return err; 4897 4898 for (i = 0; i < nh_res_table->num_nh_buckets; i++) { 4899 const struct nh_notifier_single_info *nh; 4900 int err; 4901 4902 nh = &nh_res_table->nhs[i]; 4903 err = mlxsw_sp_nexthop_obj_group_entry_validate(mlxsw_sp, nh, 4904 extack); 4905 if (err) 4906 return err; 4907 } 4908 4909 return 0; 4910} 4911 4912static int mlxsw_sp_nexthop_obj_validate(struct mlxsw_sp *mlxsw_sp, 4913 unsigned long event, 4914 struct nh_notifier_info *info) 4915{ 4916 struct nh_notifier_single_info *nh; 4917 4918 if (event != NEXTHOP_EVENT_REPLACE && 4919 event != NEXTHOP_EVENT_RES_TABLE_PRE_REPLACE && 4920 event != NEXTHOP_EVENT_BUCKET_REPLACE) 4921 return 0; 4922 4923 switch (info->type) { 4924 case NH_NOTIFIER_INFO_TYPE_SINGLE: 4925 return mlxsw_sp_nexthop_obj_single_validate(mlxsw_sp, info->nh, 4926 info->extack); 4927 case NH_NOTIFIER_INFO_TYPE_GRP: 4928 return mlxsw_sp_nexthop_obj_group_validate(mlxsw_sp, 4929 info->nh_grp, 4930 info->extack); 4931 case NH_NOTIFIER_INFO_TYPE_RES_TABLE: 4932 return mlxsw_sp_nexthop_obj_res_group_validate(mlxsw_sp, 4933 info->nh_res_table, 4934 info->extack); 4935 case NH_NOTIFIER_INFO_TYPE_RES_BUCKET: 4936 nh = &info->nh_res_bucket->new_nh; 4937 return mlxsw_sp_nexthop_obj_group_entry_validate(mlxsw_sp, nh, 4938 info->extack); 4939 default: 4940 NL_SET_ERR_MSG_MOD(info->extack, "Unsupported nexthop type"); 4941 return -EOPNOTSUPP; 4942 } 4943} 4944 4945static bool mlxsw_sp_nexthop_obj_is_gateway(struct mlxsw_sp *mlxsw_sp, 4946 const struct nh_notifier_info *info) 4947{ 4948 const struct net_device *dev; 4949 4950 switch (info->type) { 4951 case NH_NOTIFIER_INFO_TYPE_SINGLE: 4952 dev = info->nh->dev; 4953 return info->nh->gw_family || info->nh->is_reject || 4954 mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, NULL); 4955 case NH_NOTIFIER_INFO_TYPE_GRP: 4956 case NH_NOTIFIER_INFO_TYPE_RES_TABLE: 4957 /* Already validated earlier. */ 4958 return true; 4959 default: 4960 return false; 4961 } 4962} 4963 4964static void mlxsw_sp_nexthop_obj_blackhole_init(struct mlxsw_sp *mlxsw_sp, 4965 struct mlxsw_sp_nexthop *nh) 4966{ 4967 nh->action = MLXSW_SP_NEXTHOP_ACTION_DISCARD; 4968 nh->should_offload = 1; 4969 /* While nexthops that discard packets do not forward packets 4970 * via an egress RIF, they still need to be programmed using a 4971 * valid RIF, so use the loopback RIF created during init. 4972 */ 4973 nh->crif = mlxsw_sp->router->lb_crif; 4974} 4975 4976static void mlxsw_sp_nexthop_obj_blackhole_fini(struct mlxsw_sp *mlxsw_sp, 4977 struct mlxsw_sp_nexthop *nh) 4978{ 4979 nh->crif = NULL; 4980 nh->should_offload = 0; 4981} 4982 4983static int 4984mlxsw_sp_nexthop_obj_init(struct mlxsw_sp *mlxsw_sp, 4985 struct mlxsw_sp_nexthop_group *nh_grp, 4986 struct mlxsw_sp_nexthop *nh, 4987 struct nh_notifier_single_info *nh_obj, int weight) 4988{ 4989 struct net_device *dev = nh_obj->dev; 4990 int err; 4991 4992 nh->nhgi = nh_grp->nhgi; 4993 nh->nh_weight = weight; 4994 4995 switch (nh_obj->gw_family) { 4996 case AF_INET: 4997 memcpy(&nh->gw_addr, &nh_obj->ipv4, sizeof(nh_obj->ipv4)); 4998 nh->neigh_tbl = &arp_tbl; 4999 break; 5000 case AF_INET6: 5001 memcpy(&nh->gw_addr, &nh_obj->ipv6, sizeof(nh_obj->ipv6)); 5002#if IS_ENABLED(CONFIG_IPV6) 5003 nh->neigh_tbl = &nd_tbl; 5004#endif 5005 break; 5006 } 5007 5008 mlxsw_sp_nexthop_counter_alloc(mlxsw_sp, nh); 5009 list_add_tail(&nh->router_list_node, &mlxsw_sp->router->nexthop_list); 5010 nh->ifindex = dev->ifindex; 5011 5012 err = mlxsw_sp_nexthop_type_init(mlxsw_sp, nh, dev); 5013 if (err) 5014 goto err_type_init; 5015 5016 if (nh_obj->is_reject) 5017 mlxsw_sp_nexthop_obj_blackhole_init(mlxsw_sp, nh); 5018 5019 /* In a resilient nexthop group, all the nexthops must be written to 5020 * the adjacency table. Even if they do not have a valid neighbour or 5021 * RIF. 5022 */ 5023 if (nh_grp->nhgi->is_resilient && !nh->should_offload) { 5024 nh->action = MLXSW_SP_NEXTHOP_ACTION_TRAP; 5025 nh->should_offload = 1; 5026 } 5027 5028 return 0; 5029 5030err_type_init: 5031 list_del(&nh->router_list_node); 5032 mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh); 5033 return err; 5034} 5035 5036static void mlxsw_sp_nexthop_obj_fini(struct mlxsw_sp *mlxsw_sp, 5037 struct mlxsw_sp_nexthop *nh) 5038{ 5039 if (nh->action == MLXSW_SP_NEXTHOP_ACTION_DISCARD) 5040 mlxsw_sp_nexthop_obj_blackhole_fini(mlxsw_sp, nh); 5041 mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh); 5042 list_del(&nh->router_list_node); 5043 mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh); 5044 nh->should_offload = 0; 5045} 5046 5047static int 5048mlxsw_sp_nexthop_obj_group_info_init(struct mlxsw_sp *mlxsw_sp, 5049 struct mlxsw_sp_nexthop_group *nh_grp, 5050 struct nh_notifier_info *info) 5051{ 5052 struct mlxsw_sp_nexthop_group_info *nhgi; 5053 struct mlxsw_sp_nexthop *nh; 5054 bool is_resilient = false; 5055 unsigned int nhs; 5056 int err, i; 5057 5058 switch (info->type) { 5059 case NH_NOTIFIER_INFO_TYPE_SINGLE: 5060 nhs = 1; 5061 break; 5062 case NH_NOTIFIER_INFO_TYPE_GRP: 5063 nhs = info->nh_grp->num_nh; 5064 break; 5065 case NH_NOTIFIER_INFO_TYPE_RES_TABLE: 5066 nhs = info->nh_res_table->num_nh_buckets; 5067 is_resilient = true; 5068 break; 5069 default: 5070 return -EINVAL; 5071 } 5072 5073 nhgi = kzalloc(struct_size(nhgi, nexthops, nhs), GFP_KERNEL); 5074 if (!nhgi) 5075 return -ENOMEM; 5076 nh_grp->nhgi = nhgi; 5077 nhgi->nh_grp = nh_grp; 5078 nhgi->gateway = mlxsw_sp_nexthop_obj_is_gateway(mlxsw_sp, info); 5079 nhgi->is_resilient = is_resilient; 5080 nhgi->count = nhs; 5081 for (i = 0; i < nhgi->count; i++) { 5082 struct nh_notifier_single_info *nh_obj; 5083 int weight; 5084 5085 nh = &nhgi->nexthops[i]; 5086 switch (info->type) { 5087 case NH_NOTIFIER_INFO_TYPE_SINGLE: 5088 nh_obj = info->nh; 5089 weight = 1; 5090 break; 5091 case NH_NOTIFIER_INFO_TYPE_GRP: 5092 nh_obj = &info->nh_grp->nh_entries[i].nh; 5093 weight = info->nh_grp->nh_entries[i].weight; 5094 break; 5095 case NH_NOTIFIER_INFO_TYPE_RES_TABLE: 5096 nh_obj = &info->nh_res_table->nhs[i]; 5097 weight = 1; 5098 break; 5099 default: 5100 err = -EINVAL; 5101 goto err_nexthop_obj_init; 5102 } 5103 err = mlxsw_sp_nexthop_obj_init(mlxsw_sp, nh_grp, nh, nh_obj, 5104 weight); 5105 if (err) 5106 goto err_nexthop_obj_init; 5107 } 5108 err = mlxsw_sp_nexthop_group_inc(mlxsw_sp); 5109 if (err) 5110 goto err_group_inc; 5111 err = mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp); 5112 if (err) { 5113 NL_SET_ERR_MSG_MOD(info->extack, "Failed to write adjacency entries to the device"); 5114 goto err_group_refresh; 5115 } 5116 5117 /* Add resilient nexthop groups to a list so that the activity of their 5118 * nexthop buckets will be periodically queried and cleared. 5119 */ 5120 if (nhgi->is_resilient) { 5121 if (list_empty(&mlxsw_sp->router->nh_res_grp_list)) 5122 mlxsw_sp_nh_grp_activity_work_schedule(mlxsw_sp); 5123 list_add(&nhgi->list, &mlxsw_sp->router->nh_res_grp_list); 5124 } 5125 5126 return 0; 5127 5128err_group_refresh: 5129 mlxsw_sp_nexthop_group_dec(mlxsw_sp); 5130err_group_inc: 5131 i = nhgi->count; 5132err_nexthop_obj_init: 5133 for (i--; i >= 0; i--) { 5134 nh = &nhgi->nexthops[i]; 5135 mlxsw_sp_nexthop_obj_fini(mlxsw_sp, nh); 5136 } 5137 kfree(nhgi); 5138 return err; 5139} 5140 5141static void 5142mlxsw_sp_nexthop_obj_group_info_fini(struct mlxsw_sp *mlxsw_sp, 5143 struct mlxsw_sp_nexthop_group *nh_grp) 5144{ 5145 struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi; 5146 struct mlxsw_sp_router *router = mlxsw_sp->router; 5147 int i; 5148 5149 if (nhgi->is_resilient) { 5150 list_del(&nhgi->list); 5151 if (list_empty(&mlxsw_sp->router->nh_res_grp_list)) 5152 cancel_delayed_work(&router->nh_grp_activity_dw); 5153 } 5154 5155 mlxsw_sp_nexthop_group_dec(mlxsw_sp); 5156 for (i = nhgi->count - 1; i >= 0; i--) { 5157 struct mlxsw_sp_nexthop *nh = &nhgi->nexthops[i]; 5158 5159 mlxsw_sp_nexthop_obj_fini(mlxsw_sp, nh); 5160 } 5161 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp); 5162 WARN_ON_ONCE(nhgi->adj_index_valid); 5163 kfree(nhgi); 5164} 5165 5166static struct mlxsw_sp_nexthop_group * 5167mlxsw_sp_nexthop_obj_group_create(struct mlxsw_sp *mlxsw_sp, 5168 struct nh_notifier_info *info) 5169{ 5170 struct mlxsw_sp_nexthop_group *nh_grp; 5171 int err; 5172 5173 nh_grp = kzalloc(sizeof(*nh_grp), GFP_KERNEL); 5174 if (!nh_grp) 5175 return ERR_PTR(-ENOMEM); 5176 INIT_LIST_HEAD(&nh_grp->vr_list); 5177 err = rhashtable_init(&nh_grp->vr_ht, 5178 &mlxsw_sp_nexthop_group_vr_ht_params); 5179 if (err) 5180 goto err_nexthop_group_vr_ht_init; 5181 INIT_LIST_HEAD(&nh_grp->fib_list); 5182 nh_grp->type = MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ; 5183 nh_grp->obj.id = info->id; 5184 5185 err = mlxsw_sp_nexthop_obj_group_info_init(mlxsw_sp, nh_grp, info); 5186 if (err) 5187 goto err_nexthop_group_info_init; 5188 5189 nh_grp->can_destroy = false; 5190 5191 return nh_grp; 5192 5193err_nexthop_group_info_init: 5194 rhashtable_destroy(&nh_grp->vr_ht); 5195err_nexthop_group_vr_ht_init: 5196 kfree(nh_grp); 5197 return ERR_PTR(err); 5198} 5199 5200static void 5201mlxsw_sp_nexthop_obj_group_destroy(struct mlxsw_sp *mlxsw_sp, 5202 struct mlxsw_sp_nexthop_group *nh_grp) 5203{ 5204 if (!nh_grp->can_destroy) 5205 return; 5206 mlxsw_sp_nexthop_obj_group_info_fini(mlxsw_sp, nh_grp); 5207 WARN_ON_ONCE(!list_empty(&nh_grp->fib_list)); 5208 WARN_ON_ONCE(!list_empty(&nh_grp->vr_list)); 5209 rhashtable_destroy(&nh_grp->vr_ht); 5210 kfree(nh_grp); 5211} 5212 5213static struct mlxsw_sp_nexthop_group * 5214mlxsw_sp_nexthop_obj_group_lookup(struct mlxsw_sp *mlxsw_sp, u32 id) 5215{ 5216 struct mlxsw_sp_nexthop_group_cmp_arg cmp_arg; 5217 5218 cmp_arg.type = MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ; 5219 cmp_arg.id = id; 5220 return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_group_ht, 5221 &cmp_arg, 5222 mlxsw_sp_nexthop_group_ht_params); 5223} 5224 5225static int mlxsw_sp_nexthop_obj_group_add(struct mlxsw_sp *mlxsw_sp, 5226 struct mlxsw_sp_nexthop_group *nh_grp) 5227{ 5228 return mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp); 5229} 5230 5231static int 5232mlxsw_sp_nexthop_obj_group_replace(struct mlxsw_sp *mlxsw_sp, 5233 struct mlxsw_sp_nexthop_group *nh_grp, 5234 struct mlxsw_sp_nexthop_group *old_nh_grp, 5235 struct netlink_ext_ack *extack) 5236{ 5237 struct mlxsw_sp_nexthop_group_info *old_nhgi = old_nh_grp->nhgi; 5238 struct mlxsw_sp_nexthop_group_info *new_nhgi = nh_grp->nhgi; 5239 int err; 5240 5241 old_nh_grp->nhgi = new_nhgi; 5242 new_nhgi->nh_grp = old_nh_grp; 5243 nh_grp->nhgi = old_nhgi; 5244 old_nhgi->nh_grp = nh_grp; 5245 5246 if (old_nhgi->adj_index_valid && new_nhgi->adj_index_valid) { 5247 /* Both the old adjacency index and the new one are valid. 5248 * Routes are currently using the old one. Tell the device to 5249 * replace the old adjacency index with the new one. 5250 */ 5251 err = mlxsw_sp_adj_index_mass_update(mlxsw_sp, old_nh_grp, 5252 old_nhgi->adj_index, 5253 old_nhgi->ecmp_size); 5254 if (err) { 5255 NL_SET_ERR_MSG_MOD(extack, "Failed to replace old adjacency index with new one"); 5256 goto err_out; 5257 } 5258 } else if (old_nhgi->adj_index_valid && !new_nhgi->adj_index_valid) { 5259 /* The old adjacency index is valid, while the new one is not. 5260 * Iterate over all the routes using the group and change them 5261 * to trap packets to the CPU. 5262 */ 5263 err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, old_nh_grp); 5264 if (err) { 5265 NL_SET_ERR_MSG_MOD(extack, "Failed to update routes to trap packets"); 5266 goto err_out; 5267 } 5268 } else if (!old_nhgi->adj_index_valid && new_nhgi->adj_index_valid) { 5269 /* The old adjacency index is invalid, while the new one is. 5270 * Iterate over all the routes using the group and change them 5271 * to forward packets using the new valid index. 5272 */ 5273 err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, old_nh_grp); 5274 if (err) { 5275 NL_SET_ERR_MSG_MOD(extack, "Failed to update routes to forward packets"); 5276 goto err_out; 5277 } 5278 } 5279 5280 /* Make sure the flags are set / cleared based on the new nexthop group 5281 * information. 5282 */ 5283 mlxsw_sp_nexthop_obj_group_offload_refresh(mlxsw_sp, old_nh_grp); 5284 5285 /* At this point 'nh_grp' is just a shell that is not used by anyone 5286 * and its nexthop group info is the old info that was just replaced 5287 * with the new one. Remove it. 5288 */ 5289 nh_grp->can_destroy = true; 5290 mlxsw_sp_nexthop_obj_group_destroy(mlxsw_sp, nh_grp); 5291 5292 return 0; 5293 5294err_out: 5295 old_nhgi->nh_grp = old_nh_grp; 5296 nh_grp->nhgi = new_nhgi; 5297 new_nhgi->nh_grp = nh_grp; 5298 old_nh_grp->nhgi = old_nhgi; 5299 return err; 5300} 5301 5302static int mlxsw_sp_nexthop_obj_new(struct mlxsw_sp *mlxsw_sp, 5303 struct nh_notifier_info *info) 5304{ 5305 struct mlxsw_sp_nexthop_group *nh_grp, *old_nh_grp; 5306 struct netlink_ext_ack *extack = info->extack; 5307 int err; 5308 5309 nh_grp = mlxsw_sp_nexthop_obj_group_create(mlxsw_sp, info); 5310 if (IS_ERR(nh_grp)) 5311 return PTR_ERR(nh_grp); 5312 5313 old_nh_grp = mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, info->id); 5314 if (!old_nh_grp) 5315 err = mlxsw_sp_nexthop_obj_group_add(mlxsw_sp, nh_grp); 5316 else 5317 err = mlxsw_sp_nexthop_obj_group_replace(mlxsw_sp, nh_grp, 5318 old_nh_grp, extack); 5319 5320 if (err) { 5321 nh_grp->can_destroy = true; 5322 mlxsw_sp_nexthop_obj_group_destroy(mlxsw_sp, nh_grp); 5323 } 5324 5325 return err; 5326} 5327 5328static void mlxsw_sp_nexthop_obj_del(struct mlxsw_sp *mlxsw_sp, 5329 struct nh_notifier_info *info) 5330{ 5331 struct mlxsw_sp_nexthop_group *nh_grp; 5332 5333 nh_grp = mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, info->id); 5334 if (!nh_grp) 5335 return; 5336 5337 nh_grp->can_destroy = true; 5338 mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp); 5339 5340 /* If the group still has routes using it, then defer the delete 5341 * operation until the last route using it is deleted. 5342 */ 5343 if (!list_empty(&nh_grp->fib_list)) 5344 return; 5345 mlxsw_sp_nexthop_obj_group_destroy(mlxsw_sp, nh_grp); 5346} 5347 5348static int mlxsw_sp_nexthop_obj_bucket_query(struct mlxsw_sp *mlxsw_sp, 5349 u32 adj_index, char *ratr_pl) 5350{ 5351 MLXSW_REG_ZERO(ratr, ratr_pl); 5352 mlxsw_reg_ratr_op_set(ratr_pl, MLXSW_REG_RATR_OP_QUERY_READ); 5353 mlxsw_reg_ratr_adjacency_index_low_set(ratr_pl, adj_index); 5354 mlxsw_reg_ratr_adjacency_index_high_set(ratr_pl, adj_index >> 16); 5355 5356 return mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ratr), ratr_pl); 5357} 5358 5359static int mlxsw_sp_nexthop_obj_bucket_compare(char *ratr_pl, char *ratr_pl_new) 5360{ 5361 /* Clear the opcode and activity on both the old and new payload as 5362 * they are irrelevant for the comparison. 5363 */ 5364 mlxsw_reg_ratr_op_set(ratr_pl, MLXSW_REG_RATR_OP_QUERY_READ); 5365 mlxsw_reg_ratr_a_set(ratr_pl, 0); 5366 mlxsw_reg_ratr_op_set(ratr_pl_new, MLXSW_REG_RATR_OP_QUERY_READ); 5367 mlxsw_reg_ratr_a_set(ratr_pl_new, 0); 5368 5369 /* If the contents of the adjacency entry are consistent with the 5370 * replacement request, then replacement was successful. 5371 */ 5372 if (!memcmp(ratr_pl, ratr_pl_new, MLXSW_REG_RATR_LEN)) 5373 return 0; 5374 5375 return -EINVAL; 5376} 5377 5378static int 5379mlxsw_sp_nexthop_obj_bucket_adj_update(struct mlxsw_sp *mlxsw_sp, 5380 struct mlxsw_sp_nexthop *nh, 5381 struct nh_notifier_info *info) 5382{ 5383 u16 bucket_index = info->nh_res_bucket->bucket_index; 5384 struct netlink_ext_ack *extack = info->extack; 5385 bool force = info->nh_res_bucket->force; 5386 char ratr_pl_new[MLXSW_REG_RATR_LEN]; 5387 char ratr_pl[MLXSW_REG_RATR_LEN]; 5388 u32 adj_index; 5389 int err; 5390 5391 /* No point in trying an atomic replacement if the idle timer interval 5392 * is smaller than the interval in which we query and clear activity. 5393 */ 5394 if (!force && info->nh_res_bucket->idle_timer_ms < 5395 MLXSW_SP_NH_GRP_ACTIVITY_UPDATE_INTERVAL) 5396 force = true; 5397 5398 adj_index = nh->nhgi->adj_index + bucket_index; 5399 err = mlxsw_sp_nexthop_update(mlxsw_sp, adj_index, nh, force, ratr_pl); 5400 if (err) { 5401 NL_SET_ERR_MSG_MOD(extack, "Failed to overwrite nexthop bucket"); 5402 return err; 5403 } 5404 5405 if (!force) { 5406 err = mlxsw_sp_nexthop_obj_bucket_query(mlxsw_sp, adj_index, 5407 ratr_pl_new); 5408 if (err) { 5409 NL_SET_ERR_MSG_MOD(extack, "Failed to query nexthop bucket state after replacement. State might be inconsistent"); 5410 return err; 5411 } 5412 5413 err = mlxsw_sp_nexthop_obj_bucket_compare(ratr_pl, ratr_pl_new); 5414 if (err) { 5415 NL_SET_ERR_MSG_MOD(extack, "Nexthop bucket was not replaced because it was active during replacement"); 5416 return err; 5417 } 5418 } 5419 5420 nh->update = 0; 5421 nh->offloaded = 1; 5422 mlxsw_sp_nexthop_bucket_offload_refresh(mlxsw_sp, nh, bucket_index); 5423 5424 return 0; 5425} 5426 5427static int mlxsw_sp_nexthop_obj_bucket_replace(struct mlxsw_sp *mlxsw_sp, 5428 struct nh_notifier_info *info) 5429{ 5430 u16 bucket_index = info->nh_res_bucket->bucket_index; 5431 struct netlink_ext_ack *extack = info->extack; 5432 struct mlxsw_sp_nexthop_group_info *nhgi; 5433 struct nh_notifier_single_info *nh_obj; 5434 struct mlxsw_sp_nexthop_group *nh_grp; 5435 struct mlxsw_sp_nexthop *nh; 5436 int err; 5437 5438 nh_grp = mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, info->id); 5439 if (!nh_grp) { 5440 NL_SET_ERR_MSG_MOD(extack, "Nexthop group was not found"); 5441 return -EINVAL; 5442 } 5443 5444 nhgi = nh_grp->nhgi; 5445 5446 if (bucket_index >= nhgi->count) { 5447 NL_SET_ERR_MSG_MOD(extack, "Nexthop bucket index out of range"); 5448 return -EINVAL; 5449 } 5450 5451 nh = &nhgi->nexthops[bucket_index]; 5452 mlxsw_sp_nexthop_obj_fini(mlxsw_sp, nh); 5453 5454 nh_obj = &info->nh_res_bucket->new_nh; 5455 err = mlxsw_sp_nexthop_obj_init(mlxsw_sp, nh_grp, nh, nh_obj, 1); 5456 if (err) { 5457 NL_SET_ERR_MSG_MOD(extack, "Failed to initialize nexthop object for nexthop bucket replacement"); 5458 goto err_nexthop_obj_init; 5459 } 5460 5461 err = mlxsw_sp_nexthop_obj_bucket_adj_update(mlxsw_sp, nh, info); 5462 if (err) 5463 goto err_nexthop_obj_bucket_adj_update; 5464 5465 return 0; 5466 5467err_nexthop_obj_bucket_adj_update: 5468 mlxsw_sp_nexthop_obj_fini(mlxsw_sp, nh); 5469err_nexthop_obj_init: 5470 nh_obj = &info->nh_res_bucket->old_nh; 5471 mlxsw_sp_nexthop_obj_init(mlxsw_sp, nh_grp, nh, nh_obj, 1); 5472 /* The old adjacency entry was not overwritten */ 5473 nh->update = 0; 5474 nh->offloaded = 1; 5475 return err; 5476} 5477 5478static int mlxsw_sp_nexthop_obj_event(struct notifier_block *nb, 5479 unsigned long event, void *ptr) 5480{ 5481 struct nh_notifier_info *info = ptr; 5482 struct mlxsw_sp_router *router; 5483 int err = 0; 5484 5485 router = container_of(nb, struct mlxsw_sp_router, nexthop_nb); 5486 err = mlxsw_sp_nexthop_obj_validate(router->mlxsw_sp, event, info); 5487 if (err) 5488 goto out; 5489 5490 mutex_lock(&router->lock); 5491 5492 switch (event) { 5493 case NEXTHOP_EVENT_REPLACE: 5494 err = mlxsw_sp_nexthop_obj_new(router->mlxsw_sp, info); 5495 break; 5496 case NEXTHOP_EVENT_DEL: 5497 mlxsw_sp_nexthop_obj_del(router->mlxsw_sp, info); 5498 break; 5499 case NEXTHOP_EVENT_BUCKET_REPLACE: 5500 err = mlxsw_sp_nexthop_obj_bucket_replace(router->mlxsw_sp, 5501 info); 5502 break; 5503 default: 5504 break; 5505 } 5506 5507 mutex_unlock(&router->lock); 5508 5509out: 5510 return notifier_from_errno(err); 5511} 5512 5513static bool mlxsw_sp_fi_is_gateway(const struct mlxsw_sp *mlxsw_sp, 5514 struct fib_info *fi) 5515{ 5516 const struct fib_nh *nh = fib_info_nh(fi, 0); 5517 5518 return nh->fib_nh_gw_family || 5519 mlxsw_sp_nexthop4_ipip_type(mlxsw_sp, nh, NULL); 5520} 5521 5522static int 5523mlxsw_sp_nexthop4_group_info_init(struct mlxsw_sp *mlxsw_sp, 5524 struct mlxsw_sp_nexthop_group *nh_grp) 5525{ 5526 unsigned int nhs = fib_info_num_path(nh_grp->ipv4.fi); 5527 struct mlxsw_sp_nexthop_group_info *nhgi; 5528 struct mlxsw_sp_nexthop *nh; 5529 int err, i; 5530 5531 nhgi = kzalloc(struct_size(nhgi, nexthops, nhs), GFP_KERNEL); 5532 if (!nhgi) 5533 return -ENOMEM; 5534 nh_grp->nhgi = nhgi; 5535 nhgi->nh_grp = nh_grp; 5536 nhgi->gateway = mlxsw_sp_fi_is_gateway(mlxsw_sp, nh_grp->ipv4.fi); 5537 nhgi->count = nhs; 5538 for (i = 0; i < nhgi->count; i++) { 5539 struct fib_nh *fib_nh; 5540 5541 nh = &nhgi->nexthops[i]; 5542 fib_nh = fib_info_nh(nh_grp->ipv4.fi, i); 5543 err = mlxsw_sp_nexthop4_init(mlxsw_sp, nh_grp, nh, fib_nh); 5544 if (err) 5545 goto err_nexthop4_init; 5546 } 5547 err = mlxsw_sp_nexthop_group_inc(mlxsw_sp); 5548 if (err) 5549 goto err_group_inc; 5550 err = mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp); 5551 if (err) 5552 goto err_group_refresh; 5553 5554 return 0; 5555 5556err_group_refresh: 5557 mlxsw_sp_nexthop_group_dec(mlxsw_sp); 5558err_group_inc: 5559 i = nhgi->count; 5560err_nexthop4_init: 5561 for (i--; i >= 0; i--) { 5562 nh = &nhgi->nexthops[i]; 5563 mlxsw_sp_nexthop4_fini(mlxsw_sp, nh); 5564 } 5565 kfree(nhgi); 5566 return err; 5567} 5568 5569static void 5570mlxsw_sp_nexthop4_group_info_fini(struct mlxsw_sp *mlxsw_sp, 5571 struct mlxsw_sp_nexthop_group *nh_grp) 5572{ 5573 struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi; 5574 int i; 5575 5576 mlxsw_sp_nexthop_group_dec(mlxsw_sp); 5577 for (i = nhgi->count - 1; i >= 0; i--) { 5578 struct mlxsw_sp_nexthop *nh = &nhgi->nexthops[i]; 5579 5580 mlxsw_sp_nexthop4_fini(mlxsw_sp, nh); 5581 } 5582 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp); 5583 WARN_ON_ONCE(nhgi->adj_index_valid); 5584 kfree(nhgi); 5585} 5586 5587static struct mlxsw_sp_nexthop_group * 5588mlxsw_sp_nexthop4_group_create(struct mlxsw_sp *mlxsw_sp, struct fib_info *fi) 5589{ 5590 struct mlxsw_sp_nexthop_group *nh_grp; 5591 int err; 5592 5593 nh_grp = kzalloc(sizeof(*nh_grp), GFP_KERNEL); 5594 if (!nh_grp) 5595 return ERR_PTR(-ENOMEM); 5596 INIT_LIST_HEAD(&nh_grp->vr_list); 5597 err = rhashtable_init(&nh_grp->vr_ht, 5598 &mlxsw_sp_nexthop_group_vr_ht_params); 5599 if (err) 5600 goto err_nexthop_group_vr_ht_init; 5601 INIT_LIST_HEAD(&nh_grp->fib_list); 5602 nh_grp->type = MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4; 5603 nh_grp->ipv4.fi = fi; 5604 fib_info_hold(fi); 5605 5606 err = mlxsw_sp_nexthop4_group_info_init(mlxsw_sp, nh_grp); 5607 if (err) 5608 goto err_nexthop_group_info_init; 5609 5610 err = mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp); 5611 if (err) 5612 goto err_nexthop_group_insert; 5613 5614 nh_grp->can_destroy = true; 5615 5616 return nh_grp; 5617 5618err_nexthop_group_insert: 5619 mlxsw_sp_nexthop4_group_info_fini(mlxsw_sp, nh_grp); 5620err_nexthop_group_info_init: 5621 fib_info_put(fi); 5622 rhashtable_destroy(&nh_grp->vr_ht); 5623err_nexthop_group_vr_ht_init: 5624 kfree(nh_grp); 5625 return ERR_PTR(err); 5626} 5627 5628static void 5629mlxsw_sp_nexthop4_group_destroy(struct mlxsw_sp *mlxsw_sp, 5630 struct mlxsw_sp_nexthop_group *nh_grp) 5631{ 5632 if (!nh_grp->can_destroy) 5633 return; 5634 mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp); 5635 mlxsw_sp_nexthop4_group_info_fini(mlxsw_sp, nh_grp); 5636 fib_info_put(nh_grp->ipv4.fi); 5637 WARN_ON_ONCE(!list_empty(&nh_grp->vr_list)); 5638 rhashtable_destroy(&nh_grp->vr_ht); 5639 kfree(nh_grp); 5640} 5641 5642static int mlxsw_sp_nexthop4_group_get(struct mlxsw_sp *mlxsw_sp, 5643 struct mlxsw_sp_fib_entry *fib_entry, 5644 struct fib_info *fi) 5645{ 5646 struct mlxsw_sp_nexthop_group *nh_grp; 5647 5648 if (fi->nh) { 5649 nh_grp = mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, 5650 fi->nh->id); 5651 if (WARN_ON_ONCE(!nh_grp)) 5652 return -EINVAL; 5653 goto out; 5654 } 5655 5656 nh_grp = mlxsw_sp_nexthop4_group_lookup(mlxsw_sp, fi); 5657 if (!nh_grp) { 5658 nh_grp = mlxsw_sp_nexthop4_group_create(mlxsw_sp, fi); 5659 if (IS_ERR(nh_grp)) 5660 return PTR_ERR(nh_grp); 5661 } 5662out: 5663 list_add_tail(&fib_entry->nexthop_group_node, &nh_grp->fib_list); 5664 fib_entry->nh_group = nh_grp; 5665 return 0; 5666} 5667 5668static void mlxsw_sp_nexthop4_group_put(struct mlxsw_sp *mlxsw_sp, 5669 struct mlxsw_sp_fib_entry *fib_entry) 5670{ 5671 struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group; 5672 5673 list_del(&fib_entry->nexthop_group_node); 5674 if (!list_empty(&nh_grp->fib_list)) 5675 return; 5676 5677 if (nh_grp->type == MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ) { 5678 mlxsw_sp_nexthop_obj_group_destroy(mlxsw_sp, nh_grp); 5679 return; 5680 } 5681 5682 mlxsw_sp_nexthop4_group_destroy(mlxsw_sp, nh_grp); 5683} 5684 5685static bool 5686mlxsw_sp_fib4_entry_should_offload(const struct mlxsw_sp_fib_entry *fib_entry) 5687{ 5688 struct mlxsw_sp_fib4_entry *fib4_entry; 5689 5690 fib4_entry = container_of(fib_entry, struct mlxsw_sp_fib4_entry, 5691 common); 5692 return !fib4_entry->dscp; 5693} 5694 5695static bool 5696mlxsw_sp_fib_entry_should_offload(const struct mlxsw_sp_fib_entry *fib_entry) 5697{ 5698 struct mlxsw_sp_nexthop_group *nh_group = fib_entry->nh_group; 5699 5700 switch (fib_entry->fib_node->fib->proto) { 5701 case MLXSW_SP_L3_PROTO_IPV4: 5702 if (!mlxsw_sp_fib4_entry_should_offload(fib_entry)) 5703 return false; 5704 break; 5705 case MLXSW_SP_L3_PROTO_IPV6: 5706 break; 5707 } 5708 5709 switch (fib_entry->type) { 5710 case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE: 5711 return !!nh_group->nhgi->adj_index_valid; 5712 case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL: 5713 return !!mlxsw_sp_nhgi_rif(nh_group->nhgi); 5714 case MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE: 5715 case MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP: 5716 case MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP: 5717 return true; 5718 default: 5719 return false; 5720 } 5721} 5722 5723static struct mlxsw_sp_nexthop * 5724mlxsw_sp_rt6_nexthop(struct mlxsw_sp_nexthop_group *nh_grp, 5725 const struct mlxsw_sp_rt6 *mlxsw_sp_rt6) 5726{ 5727 int i; 5728 5729 for (i = 0; i < nh_grp->nhgi->count; i++) { 5730 struct mlxsw_sp_nexthop *nh = &nh_grp->nhgi->nexthops[i]; 5731 struct net_device *dev = mlxsw_sp_nexthop_dev(nh); 5732 struct fib6_info *rt = mlxsw_sp_rt6->rt; 5733 5734 if (dev && dev == rt->fib6_nh->fib_nh_dev && 5735 ipv6_addr_equal((const struct in6_addr *) &nh->gw_addr, 5736 &rt->fib6_nh->fib_nh_gw6)) 5737 return nh; 5738 } 5739 5740 return NULL; 5741} 5742 5743static void 5744mlxsw_sp_fib4_offload_failed_flag_set(struct mlxsw_sp *mlxsw_sp, 5745 struct fib_entry_notifier_info *fen_info) 5746{ 5747 u32 *p_dst = (u32 *) &fen_info->dst; 5748 struct fib_rt_info fri; 5749 5750 fri.fi = fen_info->fi; 5751 fri.tb_id = fen_info->tb_id; 5752 fri.dst = cpu_to_be32(*p_dst); 5753 fri.dst_len = fen_info->dst_len; 5754 fri.dscp = fen_info->dscp; 5755 fri.type = fen_info->type; 5756 fri.offload = false; 5757 fri.trap = false; 5758 fri.offload_failed = true; 5759 fib_alias_hw_flags_set(mlxsw_sp_net(mlxsw_sp), &fri); 5760} 5761 5762static void 5763mlxsw_sp_fib4_entry_hw_flags_set(struct mlxsw_sp *mlxsw_sp, 5764 struct mlxsw_sp_fib_entry *fib_entry) 5765{ 5766 u32 *p_dst = (u32 *) fib_entry->fib_node->key.addr; 5767 int dst_len = fib_entry->fib_node->key.prefix_len; 5768 struct mlxsw_sp_fib4_entry *fib4_entry; 5769 struct fib_rt_info fri; 5770 bool should_offload; 5771 5772 should_offload = mlxsw_sp_fib_entry_should_offload(fib_entry); 5773 fib4_entry = container_of(fib_entry, struct mlxsw_sp_fib4_entry, 5774 common); 5775 fri.fi = fib4_entry->fi; 5776 fri.tb_id = fib4_entry->tb_id; 5777 fri.dst = cpu_to_be32(*p_dst); 5778 fri.dst_len = dst_len; 5779 fri.dscp = fib4_entry->dscp; 5780 fri.type = fib4_entry->type; 5781 fri.offload = should_offload; 5782 fri.trap = !should_offload; 5783 fri.offload_failed = false; 5784 fib_alias_hw_flags_set(mlxsw_sp_net(mlxsw_sp), &fri); 5785} 5786 5787static void 5788mlxsw_sp_fib4_entry_hw_flags_clear(struct mlxsw_sp *mlxsw_sp, 5789 struct mlxsw_sp_fib_entry *fib_entry) 5790{ 5791 u32 *p_dst = (u32 *) fib_entry->fib_node->key.addr; 5792 int dst_len = fib_entry->fib_node->key.prefix_len; 5793 struct mlxsw_sp_fib4_entry *fib4_entry; 5794 struct fib_rt_info fri; 5795 5796 fib4_entry = container_of(fib_entry, struct mlxsw_sp_fib4_entry, 5797 common); 5798 fri.fi = fib4_entry->fi; 5799 fri.tb_id = fib4_entry->tb_id; 5800 fri.dst = cpu_to_be32(*p_dst); 5801 fri.dst_len = dst_len; 5802 fri.dscp = fib4_entry->dscp; 5803 fri.type = fib4_entry->type; 5804 fri.offload = false; 5805 fri.trap = false; 5806 fri.offload_failed = false; 5807 fib_alias_hw_flags_set(mlxsw_sp_net(mlxsw_sp), &fri); 5808} 5809 5810#if IS_ENABLED(CONFIG_IPV6) 5811static void 5812mlxsw_sp_fib6_offload_failed_flag_set(struct mlxsw_sp *mlxsw_sp, 5813 struct fib6_info **rt_arr, 5814 unsigned int nrt6) 5815{ 5816 int i; 5817 5818 /* In IPv6 a multipath route is represented using multiple routes, so 5819 * we need to set the flags on all of them. 5820 */ 5821 for (i = 0; i < nrt6; i++) 5822 fib6_info_hw_flags_set(mlxsw_sp_net(mlxsw_sp), rt_arr[i], 5823 false, false, true); 5824} 5825#else 5826static void 5827mlxsw_sp_fib6_offload_failed_flag_set(struct mlxsw_sp *mlxsw_sp, 5828 struct fib6_info **rt_arr, 5829 unsigned int nrt6) 5830{ 5831} 5832#endif 5833 5834#if IS_ENABLED(CONFIG_IPV6) 5835static void 5836mlxsw_sp_fib6_entry_hw_flags_set(struct mlxsw_sp *mlxsw_sp, 5837 struct mlxsw_sp_fib_entry *fib_entry) 5838{ 5839 struct mlxsw_sp_fib6_entry *fib6_entry; 5840 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 5841 bool should_offload; 5842 5843 should_offload = mlxsw_sp_fib_entry_should_offload(fib_entry); 5844 5845 /* In IPv6 a multipath route is represented using multiple routes, so 5846 * we need to set the flags on all of them. 5847 */ 5848 fib6_entry = container_of(fib_entry, struct mlxsw_sp_fib6_entry, 5849 common); 5850 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) 5851 fib6_info_hw_flags_set(mlxsw_sp_net(mlxsw_sp), mlxsw_sp_rt6->rt, 5852 should_offload, !should_offload, false); 5853} 5854#else 5855static void 5856mlxsw_sp_fib6_entry_hw_flags_set(struct mlxsw_sp *mlxsw_sp, 5857 struct mlxsw_sp_fib_entry *fib_entry) 5858{ 5859} 5860#endif 5861 5862#if IS_ENABLED(CONFIG_IPV6) 5863static void 5864mlxsw_sp_fib6_entry_hw_flags_clear(struct mlxsw_sp *mlxsw_sp, 5865 struct mlxsw_sp_fib_entry *fib_entry) 5866{ 5867 struct mlxsw_sp_fib6_entry *fib6_entry; 5868 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 5869 5870 fib6_entry = container_of(fib_entry, struct mlxsw_sp_fib6_entry, 5871 common); 5872 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) 5873 fib6_info_hw_flags_set(mlxsw_sp_net(mlxsw_sp), mlxsw_sp_rt6->rt, 5874 false, false, false); 5875} 5876#else 5877static void 5878mlxsw_sp_fib6_entry_hw_flags_clear(struct mlxsw_sp *mlxsw_sp, 5879 struct mlxsw_sp_fib_entry *fib_entry) 5880{ 5881} 5882#endif 5883 5884static void 5885mlxsw_sp_fib_entry_hw_flags_set(struct mlxsw_sp *mlxsw_sp, 5886 struct mlxsw_sp_fib_entry *fib_entry) 5887{ 5888 switch (fib_entry->fib_node->fib->proto) { 5889 case MLXSW_SP_L3_PROTO_IPV4: 5890 mlxsw_sp_fib4_entry_hw_flags_set(mlxsw_sp, fib_entry); 5891 break; 5892 case MLXSW_SP_L3_PROTO_IPV6: 5893 mlxsw_sp_fib6_entry_hw_flags_set(mlxsw_sp, fib_entry); 5894 break; 5895 } 5896} 5897 5898static void 5899mlxsw_sp_fib_entry_hw_flags_clear(struct mlxsw_sp *mlxsw_sp, 5900 struct mlxsw_sp_fib_entry *fib_entry) 5901{ 5902 switch (fib_entry->fib_node->fib->proto) { 5903 case MLXSW_SP_L3_PROTO_IPV4: 5904 mlxsw_sp_fib4_entry_hw_flags_clear(mlxsw_sp, fib_entry); 5905 break; 5906 case MLXSW_SP_L3_PROTO_IPV6: 5907 mlxsw_sp_fib6_entry_hw_flags_clear(mlxsw_sp, fib_entry); 5908 break; 5909 } 5910} 5911 5912static void 5913mlxsw_sp_fib_entry_hw_flags_refresh(struct mlxsw_sp *mlxsw_sp, 5914 struct mlxsw_sp_fib_entry *fib_entry, 5915 enum mlxsw_reg_ralue_op op) 5916{ 5917 switch (op) { 5918 case MLXSW_REG_RALUE_OP_WRITE_WRITE: 5919 mlxsw_sp_fib_entry_hw_flags_set(mlxsw_sp, fib_entry); 5920 break; 5921 case MLXSW_REG_RALUE_OP_WRITE_DELETE: 5922 mlxsw_sp_fib_entry_hw_flags_clear(mlxsw_sp, fib_entry); 5923 break; 5924 default: 5925 break; 5926 } 5927} 5928 5929static void 5930mlxsw_sp_fib_entry_ralue_pack(char *ralue_pl, 5931 const struct mlxsw_sp_fib_entry *fib_entry, 5932 enum mlxsw_reg_ralue_op op) 5933{ 5934 struct mlxsw_sp_fib *fib = fib_entry->fib_node->fib; 5935 enum mlxsw_reg_ralxx_protocol proto; 5936 u32 *p_dip; 5937 5938 proto = (enum mlxsw_reg_ralxx_protocol) fib->proto; 5939 5940 switch (fib->proto) { 5941 case MLXSW_SP_L3_PROTO_IPV4: 5942 p_dip = (u32 *) fib_entry->fib_node->key.addr; 5943 mlxsw_reg_ralue_pack4(ralue_pl, proto, op, fib->vr->id, 5944 fib_entry->fib_node->key.prefix_len, 5945 *p_dip); 5946 break; 5947 case MLXSW_SP_L3_PROTO_IPV6: 5948 mlxsw_reg_ralue_pack6(ralue_pl, proto, op, fib->vr->id, 5949 fib_entry->fib_node->key.prefix_len, 5950 fib_entry->fib_node->key.addr); 5951 break; 5952 } 5953} 5954 5955static int mlxsw_sp_fib_entry_op_remote(struct mlxsw_sp *mlxsw_sp, 5956 struct mlxsw_sp_fib_entry *fib_entry, 5957 enum mlxsw_reg_ralue_op op) 5958{ 5959 struct mlxsw_sp_nexthop_group *nh_group = fib_entry->nh_group; 5960 struct mlxsw_sp_nexthop_group_info *nhgi = nh_group->nhgi; 5961 char ralue_pl[MLXSW_REG_RALUE_LEN]; 5962 enum mlxsw_reg_ralue_trap_action trap_action; 5963 u16 trap_id = 0; 5964 u32 adjacency_index = 0; 5965 u16 ecmp_size = 0; 5966 5967 /* In case the nexthop group adjacency index is valid, use it 5968 * with provided ECMP size. Otherwise, setup trap and pass 5969 * traffic to kernel. 5970 */ 5971 if (mlxsw_sp_fib_entry_should_offload(fib_entry)) { 5972 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP; 5973 adjacency_index = nhgi->adj_index; 5974 ecmp_size = nhgi->ecmp_size; 5975 } else if (!nhgi->adj_index_valid && nhgi->count && 5976 mlxsw_sp_nhgi_rif(nhgi)) { 5977 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP; 5978 adjacency_index = mlxsw_sp->router->adj_trap_index; 5979 ecmp_size = 1; 5980 } else { 5981 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP; 5982 trap_id = MLXSW_TRAP_ID_RTR_INGRESS0; 5983 } 5984 5985 mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op); 5986 mlxsw_reg_ralue_act_remote_pack(ralue_pl, trap_action, trap_id, 5987 adjacency_index, ecmp_size); 5988 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl); 5989} 5990 5991static int mlxsw_sp_fib_entry_op_local(struct mlxsw_sp *mlxsw_sp, 5992 struct mlxsw_sp_fib_entry *fib_entry, 5993 enum mlxsw_reg_ralue_op op) 5994{ 5995 struct mlxsw_sp_rif *rif = mlxsw_sp_nhgi_rif(fib_entry->nh_group->nhgi); 5996 enum mlxsw_reg_ralue_trap_action trap_action; 5997 char ralue_pl[MLXSW_REG_RALUE_LEN]; 5998 u16 trap_id = 0; 5999 u16 rif_index = 0; 6000 6001 if (mlxsw_sp_fib_entry_should_offload(fib_entry)) { 6002 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP; 6003 rif_index = rif->rif_index; 6004 } else { 6005 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP; 6006 trap_id = MLXSW_TRAP_ID_RTR_INGRESS0; 6007 } 6008 6009 mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op); 6010 mlxsw_reg_ralue_act_local_pack(ralue_pl, trap_action, trap_id, 6011 rif_index); 6012 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl); 6013} 6014 6015static int mlxsw_sp_fib_entry_op_trap(struct mlxsw_sp *mlxsw_sp, 6016 struct mlxsw_sp_fib_entry *fib_entry, 6017 enum mlxsw_reg_ralue_op op) 6018{ 6019 char ralue_pl[MLXSW_REG_RALUE_LEN]; 6020 6021 mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op); 6022 mlxsw_reg_ralue_act_ip2me_pack(ralue_pl); 6023 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl); 6024} 6025 6026static int mlxsw_sp_fib_entry_op_blackhole(struct mlxsw_sp *mlxsw_sp, 6027 struct mlxsw_sp_fib_entry *fib_entry, 6028 enum mlxsw_reg_ralue_op op) 6029{ 6030 enum mlxsw_reg_ralue_trap_action trap_action; 6031 char ralue_pl[MLXSW_REG_RALUE_LEN]; 6032 6033 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_DISCARD_ERROR; 6034 mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op); 6035 mlxsw_reg_ralue_act_local_pack(ralue_pl, trap_action, 0, 0); 6036 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl); 6037} 6038 6039static int 6040mlxsw_sp_fib_entry_op_unreachable(struct mlxsw_sp *mlxsw_sp, 6041 struct mlxsw_sp_fib_entry *fib_entry, 6042 enum mlxsw_reg_ralue_op op) 6043{ 6044 enum mlxsw_reg_ralue_trap_action trap_action; 6045 char ralue_pl[MLXSW_REG_RALUE_LEN]; 6046 u16 trap_id; 6047 6048 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP; 6049 trap_id = MLXSW_TRAP_ID_RTR_INGRESS1; 6050 6051 mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op); 6052 mlxsw_reg_ralue_act_local_pack(ralue_pl, trap_action, trap_id, 0); 6053 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl); 6054} 6055 6056static int 6057mlxsw_sp_fib_entry_op_ipip_decap(struct mlxsw_sp *mlxsw_sp, 6058 struct mlxsw_sp_fib_entry *fib_entry, 6059 enum mlxsw_reg_ralue_op op) 6060{ 6061 struct mlxsw_sp_ipip_entry *ipip_entry = fib_entry->decap.ipip_entry; 6062 const struct mlxsw_sp_ipip_ops *ipip_ops; 6063 char ralue_pl[MLXSW_REG_RALUE_LEN]; 6064 int err; 6065 6066 if (WARN_ON(!ipip_entry)) 6067 return -EINVAL; 6068 6069 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt]; 6070 err = ipip_ops->decap_config(mlxsw_sp, ipip_entry, 6071 fib_entry->decap.tunnel_index); 6072 if (err) 6073 return err; 6074 6075 mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op); 6076 mlxsw_reg_ralue_act_ip2me_tun_pack(ralue_pl, 6077 fib_entry->decap.tunnel_index); 6078 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl); 6079} 6080 6081static int mlxsw_sp_fib_entry_op_nve_decap(struct mlxsw_sp *mlxsw_sp, 6082 struct mlxsw_sp_fib_entry *fib_entry, 6083 enum mlxsw_reg_ralue_op op) 6084{ 6085 char ralue_pl[MLXSW_REG_RALUE_LEN]; 6086 6087 mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op); 6088 mlxsw_reg_ralue_act_ip2me_tun_pack(ralue_pl, 6089 fib_entry->decap.tunnel_index); 6090 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl); 6091} 6092 6093static int __mlxsw_sp_fib_entry_op(struct mlxsw_sp *mlxsw_sp, 6094 struct mlxsw_sp_fib_entry *fib_entry, 6095 enum mlxsw_reg_ralue_op op) 6096{ 6097 switch (fib_entry->type) { 6098 case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE: 6099 return mlxsw_sp_fib_entry_op_remote(mlxsw_sp, fib_entry, op); 6100 case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL: 6101 return mlxsw_sp_fib_entry_op_local(mlxsw_sp, fib_entry, op); 6102 case MLXSW_SP_FIB_ENTRY_TYPE_TRAP: 6103 return mlxsw_sp_fib_entry_op_trap(mlxsw_sp, fib_entry, op); 6104 case MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE: 6105 return mlxsw_sp_fib_entry_op_blackhole(mlxsw_sp, fib_entry, op); 6106 case MLXSW_SP_FIB_ENTRY_TYPE_UNREACHABLE: 6107 return mlxsw_sp_fib_entry_op_unreachable(mlxsw_sp, fib_entry, 6108 op); 6109 case MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP: 6110 return mlxsw_sp_fib_entry_op_ipip_decap(mlxsw_sp, 6111 fib_entry, op); 6112 case MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP: 6113 return mlxsw_sp_fib_entry_op_nve_decap(mlxsw_sp, fib_entry, op); 6114 } 6115 return -EINVAL; 6116} 6117 6118static int mlxsw_sp_fib_entry_op(struct mlxsw_sp *mlxsw_sp, 6119 struct mlxsw_sp_fib_entry *fib_entry, 6120 enum mlxsw_reg_ralue_op op) 6121{ 6122 int err = __mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry, op); 6123 6124 if (err) 6125 return err; 6126 6127 mlxsw_sp_fib_entry_hw_flags_refresh(mlxsw_sp, fib_entry, op); 6128 6129 return err; 6130} 6131 6132static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp, 6133 struct mlxsw_sp_fib_entry *fib_entry) 6134{ 6135 return mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry, 6136 MLXSW_REG_RALUE_OP_WRITE_WRITE); 6137} 6138 6139static int mlxsw_sp_fib_entry_del(struct mlxsw_sp *mlxsw_sp, 6140 struct mlxsw_sp_fib_entry *fib_entry) 6141{ 6142 return mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry, 6143 MLXSW_REG_RALUE_OP_WRITE_DELETE); 6144} 6145 6146static int 6147mlxsw_sp_fib4_entry_type_set(struct mlxsw_sp *mlxsw_sp, 6148 const struct fib_entry_notifier_info *fen_info, 6149 struct mlxsw_sp_fib_entry *fib_entry) 6150{ 6151 struct mlxsw_sp_nexthop_group_info *nhgi = fib_entry->nh_group->nhgi; 6152 union mlxsw_sp_l3addr dip = { .addr4 = htonl(fen_info->dst) }; 6153 struct mlxsw_sp_router *router = mlxsw_sp->router; 6154 u32 tb_id = mlxsw_sp_fix_tb_id(fen_info->tb_id); 6155 int ifindex = nhgi->nexthops[0].ifindex; 6156 struct mlxsw_sp_ipip_entry *ipip_entry; 6157 6158 switch (fen_info->type) { 6159 case RTN_LOCAL: 6160 ipip_entry = mlxsw_sp_ipip_entry_find_by_decap(mlxsw_sp, ifindex, 6161 MLXSW_SP_L3_PROTO_IPV4, dip); 6162 if (ipip_entry && ipip_entry->ol_dev->flags & IFF_UP) { 6163 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP; 6164 return mlxsw_sp_fib_entry_decap_init(mlxsw_sp, 6165 fib_entry, 6166 ipip_entry); 6167 } 6168 if (mlxsw_sp_router_nve_is_decap(mlxsw_sp, tb_id, 6169 MLXSW_SP_L3_PROTO_IPV4, 6170 &dip)) { 6171 u32 tunnel_index; 6172 6173 tunnel_index = router->nve_decap_config.tunnel_index; 6174 fib_entry->decap.tunnel_index = tunnel_index; 6175 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP; 6176 return 0; 6177 } 6178 fallthrough; 6179 case RTN_BROADCAST: 6180 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP; 6181 return 0; 6182 case RTN_BLACKHOLE: 6183 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE; 6184 return 0; 6185 case RTN_UNREACHABLE: 6186 case RTN_PROHIBIT: 6187 /* Packets hitting these routes need to be trapped, but 6188 * can do so with a lower priority than packets directed 6189 * at the host, so use action type local instead of trap. 6190 */ 6191 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_UNREACHABLE; 6192 return 0; 6193 case RTN_UNICAST: 6194 if (nhgi->gateway) 6195 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_REMOTE; 6196 else 6197 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL; 6198 return 0; 6199 default: 6200 return -EINVAL; 6201 } 6202} 6203 6204static void 6205mlxsw_sp_fib_entry_type_unset(struct mlxsw_sp *mlxsw_sp, 6206 struct mlxsw_sp_fib_entry *fib_entry) 6207{ 6208 switch (fib_entry->type) { 6209 case MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP: 6210 mlxsw_sp_fib_entry_decap_fini(mlxsw_sp, fib_entry); 6211 break; 6212 default: 6213 break; 6214 } 6215} 6216 6217static void 6218mlxsw_sp_fib4_entry_type_unset(struct mlxsw_sp *mlxsw_sp, 6219 struct mlxsw_sp_fib4_entry *fib4_entry) 6220{ 6221 mlxsw_sp_fib_entry_type_unset(mlxsw_sp, &fib4_entry->common); 6222} 6223 6224static struct mlxsw_sp_fib4_entry * 6225mlxsw_sp_fib4_entry_create(struct mlxsw_sp *mlxsw_sp, 6226 struct mlxsw_sp_fib_node *fib_node, 6227 const struct fib_entry_notifier_info *fen_info) 6228{ 6229 struct mlxsw_sp_fib4_entry *fib4_entry; 6230 struct mlxsw_sp_fib_entry *fib_entry; 6231 int err; 6232 6233 fib4_entry = kzalloc(sizeof(*fib4_entry), GFP_KERNEL); 6234 if (!fib4_entry) 6235 return ERR_PTR(-ENOMEM); 6236 fib_entry = &fib4_entry->common; 6237 6238 err = mlxsw_sp_nexthop4_group_get(mlxsw_sp, fib_entry, fen_info->fi); 6239 if (err) 6240 goto err_nexthop4_group_get; 6241 6242 err = mlxsw_sp_nexthop_group_vr_link(fib_entry->nh_group, 6243 fib_node->fib); 6244 if (err) 6245 goto err_nexthop_group_vr_link; 6246 6247 err = mlxsw_sp_fib4_entry_type_set(mlxsw_sp, fen_info, fib_entry); 6248 if (err) 6249 goto err_fib4_entry_type_set; 6250 6251 fib4_entry->fi = fen_info->fi; 6252 fib_info_hold(fib4_entry->fi); 6253 fib4_entry->tb_id = fen_info->tb_id; 6254 fib4_entry->type = fen_info->type; 6255 fib4_entry->dscp = fen_info->dscp; 6256 6257 fib_entry->fib_node = fib_node; 6258 6259 return fib4_entry; 6260 6261err_fib4_entry_type_set: 6262 mlxsw_sp_nexthop_group_vr_unlink(fib_entry->nh_group, fib_node->fib); 6263err_nexthop_group_vr_link: 6264 mlxsw_sp_nexthop4_group_put(mlxsw_sp, &fib4_entry->common); 6265err_nexthop4_group_get: 6266 kfree(fib4_entry); 6267 return ERR_PTR(err); 6268} 6269 6270static void mlxsw_sp_fib4_entry_destroy(struct mlxsw_sp *mlxsw_sp, 6271 struct mlxsw_sp_fib4_entry *fib4_entry) 6272{ 6273 struct mlxsw_sp_fib_node *fib_node = fib4_entry->common.fib_node; 6274 6275 fib_info_put(fib4_entry->fi); 6276 mlxsw_sp_fib4_entry_type_unset(mlxsw_sp, fib4_entry); 6277 mlxsw_sp_nexthop_group_vr_unlink(fib4_entry->common.nh_group, 6278 fib_node->fib); 6279 mlxsw_sp_nexthop4_group_put(mlxsw_sp, &fib4_entry->common); 6280 kfree(fib4_entry); 6281} 6282 6283static struct mlxsw_sp_fib4_entry * 6284mlxsw_sp_fib4_entry_lookup(struct mlxsw_sp *mlxsw_sp, 6285 const struct fib_entry_notifier_info *fen_info) 6286{ 6287 struct mlxsw_sp_fib4_entry *fib4_entry; 6288 struct mlxsw_sp_fib_node *fib_node; 6289 struct mlxsw_sp_fib *fib; 6290 struct mlxsw_sp_vr *vr; 6291 6292 vr = mlxsw_sp_vr_find(mlxsw_sp, fen_info->tb_id); 6293 if (!vr) 6294 return NULL; 6295 fib = mlxsw_sp_vr_fib(vr, MLXSW_SP_L3_PROTO_IPV4); 6296 6297 fib_node = mlxsw_sp_fib_node_lookup(fib, &fen_info->dst, 6298 sizeof(fen_info->dst), 6299 fen_info->dst_len); 6300 if (!fib_node) 6301 return NULL; 6302 6303 fib4_entry = container_of(fib_node->fib_entry, 6304 struct mlxsw_sp_fib4_entry, common); 6305 if (fib4_entry->tb_id == fen_info->tb_id && 6306 fib4_entry->dscp == fen_info->dscp && 6307 fib4_entry->type == fen_info->type && 6308 fib4_entry->fi == fen_info->fi) 6309 return fib4_entry; 6310 6311 return NULL; 6312} 6313 6314static const struct rhashtable_params mlxsw_sp_fib_ht_params = { 6315 .key_offset = offsetof(struct mlxsw_sp_fib_node, key), 6316 .head_offset = offsetof(struct mlxsw_sp_fib_node, ht_node), 6317 .key_len = sizeof(struct mlxsw_sp_fib_key), 6318 .automatic_shrinking = true, 6319}; 6320 6321static int mlxsw_sp_fib_node_insert(struct mlxsw_sp_fib *fib, 6322 struct mlxsw_sp_fib_node *fib_node) 6323{ 6324 return rhashtable_insert_fast(&fib->ht, &fib_node->ht_node, 6325 mlxsw_sp_fib_ht_params); 6326} 6327 6328static void mlxsw_sp_fib_node_remove(struct mlxsw_sp_fib *fib, 6329 struct mlxsw_sp_fib_node *fib_node) 6330{ 6331 rhashtable_remove_fast(&fib->ht, &fib_node->ht_node, 6332 mlxsw_sp_fib_ht_params); 6333} 6334 6335static struct mlxsw_sp_fib_node * 6336mlxsw_sp_fib_node_lookup(struct mlxsw_sp_fib *fib, const void *addr, 6337 size_t addr_len, unsigned char prefix_len) 6338{ 6339 struct mlxsw_sp_fib_key key; 6340 6341 memset(&key, 0, sizeof(key)); 6342 memcpy(key.addr, addr, addr_len); 6343 key.prefix_len = prefix_len; 6344 return rhashtable_lookup_fast(&fib->ht, &key, mlxsw_sp_fib_ht_params); 6345} 6346 6347static struct mlxsw_sp_fib_node * 6348mlxsw_sp_fib_node_create(struct mlxsw_sp_fib *fib, const void *addr, 6349 size_t addr_len, unsigned char prefix_len) 6350{ 6351 struct mlxsw_sp_fib_node *fib_node; 6352 6353 fib_node = kzalloc(sizeof(*fib_node), GFP_KERNEL); 6354 if (!fib_node) 6355 return NULL; 6356 6357 list_add(&fib_node->list, &fib->node_list); 6358 memcpy(fib_node->key.addr, addr, addr_len); 6359 fib_node->key.prefix_len = prefix_len; 6360 6361 return fib_node; 6362} 6363 6364static void mlxsw_sp_fib_node_destroy(struct mlxsw_sp_fib_node *fib_node) 6365{ 6366 list_del(&fib_node->list); 6367 kfree(fib_node); 6368} 6369 6370static int mlxsw_sp_fib_lpm_tree_link(struct mlxsw_sp *mlxsw_sp, 6371 struct mlxsw_sp_fib_node *fib_node) 6372{ 6373 struct mlxsw_sp_prefix_usage req_prefix_usage; 6374 struct mlxsw_sp_fib *fib = fib_node->fib; 6375 struct mlxsw_sp_lpm_tree *lpm_tree; 6376 int err; 6377 6378 lpm_tree = mlxsw_sp->router->lpm.proto_trees[fib->proto]; 6379 if (lpm_tree->prefix_ref_count[fib_node->key.prefix_len] != 0) 6380 goto out; 6381 6382 mlxsw_sp_prefix_usage_cpy(&req_prefix_usage, &lpm_tree->prefix_usage); 6383 mlxsw_sp_prefix_usage_set(&req_prefix_usage, fib_node->key.prefix_len); 6384 lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage, 6385 fib->proto); 6386 if (IS_ERR(lpm_tree)) 6387 return PTR_ERR(lpm_tree); 6388 6389 err = mlxsw_sp_vrs_lpm_tree_replace(mlxsw_sp, fib, lpm_tree); 6390 if (err) 6391 goto err_lpm_tree_replace; 6392 6393out: 6394 lpm_tree->prefix_ref_count[fib_node->key.prefix_len]++; 6395 return 0; 6396 6397err_lpm_tree_replace: 6398 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree); 6399 return err; 6400} 6401 6402static void mlxsw_sp_fib_lpm_tree_unlink(struct mlxsw_sp *mlxsw_sp, 6403 struct mlxsw_sp_fib_node *fib_node) 6404{ 6405 struct mlxsw_sp_lpm_tree *lpm_tree = fib_node->fib->lpm_tree; 6406 struct mlxsw_sp_prefix_usage req_prefix_usage; 6407 struct mlxsw_sp_fib *fib = fib_node->fib; 6408 int err; 6409 6410 if (--lpm_tree->prefix_ref_count[fib_node->key.prefix_len] != 0) 6411 return; 6412 /* Try to construct a new LPM tree from the current prefix usage 6413 * minus the unused one. If we fail, continue using the old one. 6414 */ 6415 mlxsw_sp_prefix_usage_cpy(&req_prefix_usage, &lpm_tree->prefix_usage); 6416 mlxsw_sp_prefix_usage_clear(&req_prefix_usage, 6417 fib_node->key.prefix_len); 6418 lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage, 6419 fib->proto); 6420 if (IS_ERR(lpm_tree)) 6421 return; 6422 6423 err = mlxsw_sp_vrs_lpm_tree_replace(mlxsw_sp, fib, lpm_tree); 6424 if (err) 6425 goto err_lpm_tree_replace; 6426 6427 return; 6428 6429err_lpm_tree_replace: 6430 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree); 6431} 6432 6433static int mlxsw_sp_fib_node_init(struct mlxsw_sp *mlxsw_sp, 6434 struct mlxsw_sp_fib_node *fib_node, 6435 struct mlxsw_sp_fib *fib) 6436{ 6437 int err; 6438 6439 err = mlxsw_sp_fib_node_insert(fib, fib_node); 6440 if (err) 6441 return err; 6442 fib_node->fib = fib; 6443 6444 err = mlxsw_sp_fib_lpm_tree_link(mlxsw_sp, fib_node); 6445 if (err) 6446 goto err_fib_lpm_tree_link; 6447 6448 return 0; 6449 6450err_fib_lpm_tree_link: 6451 fib_node->fib = NULL; 6452 mlxsw_sp_fib_node_remove(fib, fib_node); 6453 return err; 6454} 6455 6456static void mlxsw_sp_fib_node_fini(struct mlxsw_sp *mlxsw_sp, 6457 struct mlxsw_sp_fib_node *fib_node) 6458{ 6459 struct mlxsw_sp_fib *fib = fib_node->fib; 6460 6461 mlxsw_sp_fib_lpm_tree_unlink(mlxsw_sp, fib_node); 6462 fib_node->fib = NULL; 6463 mlxsw_sp_fib_node_remove(fib, fib_node); 6464} 6465 6466static struct mlxsw_sp_fib_node * 6467mlxsw_sp_fib_node_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id, const void *addr, 6468 size_t addr_len, unsigned char prefix_len, 6469 enum mlxsw_sp_l3proto proto) 6470{ 6471 struct mlxsw_sp_fib_node *fib_node; 6472 struct mlxsw_sp_fib *fib; 6473 struct mlxsw_sp_vr *vr; 6474 int err; 6475 6476 vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id, NULL); 6477 if (IS_ERR(vr)) 6478 return ERR_CAST(vr); 6479 fib = mlxsw_sp_vr_fib(vr, proto); 6480 6481 fib_node = mlxsw_sp_fib_node_lookup(fib, addr, addr_len, prefix_len); 6482 if (fib_node) 6483 return fib_node; 6484 6485 fib_node = mlxsw_sp_fib_node_create(fib, addr, addr_len, prefix_len); 6486 if (!fib_node) { 6487 err = -ENOMEM; 6488 goto err_fib_node_create; 6489 } 6490 6491 err = mlxsw_sp_fib_node_init(mlxsw_sp, fib_node, fib); 6492 if (err) 6493 goto err_fib_node_init; 6494 6495 return fib_node; 6496 6497err_fib_node_init: 6498 mlxsw_sp_fib_node_destroy(fib_node); 6499err_fib_node_create: 6500 mlxsw_sp_vr_put(mlxsw_sp, vr); 6501 return ERR_PTR(err); 6502} 6503 6504static void mlxsw_sp_fib_node_put(struct mlxsw_sp *mlxsw_sp, 6505 struct mlxsw_sp_fib_node *fib_node) 6506{ 6507 struct mlxsw_sp_vr *vr = fib_node->fib->vr; 6508 6509 if (fib_node->fib_entry) 6510 return; 6511 mlxsw_sp_fib_node_fini(mlxsw_sp, fib_node); 6512 mlxsw_sp_fib_node_destroy(fib_node); 6513 mlxsw_sp_vr_put(mlxsw_sp, vr); 6514} 6515 6516static int mlxsw_sp_fib_node_entry_link(struct mlxsw_sp *mlxsw_sp, 6517 struct mlxsw_sp_fib_entry *fib_entry) 6518{ 6519 struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node; 6520 int err; 6521 6522 fib_node->fib_entry = fib_entry; 6523 6524 err = mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry); 6525 if (err) 6526 goto err_fib_entry_update; 6527 6528 return 0; 6529 6530err_fib_entry_update: 6531 fib_node->fib_entry = NULL; 6532 return err; 6533} 6534 6535static void 6536mlxsw_sp_fib_node_entry_unlink(struct mlxsw_sp *mlxsw_sp, 6537 struct mlxsw_sp_fib_entry *fib_entry) 6538{ 6539 struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node; 6540 6541 mlxsw_sp_fib_entry_del(mlxsw_sp, fib_entry); 6542 fib_node->fib_entry = NULL; 6543} 6544 6545static bool mlxsw_sp_fib4_allow_replace(struct mlxsw_sp_fib4_entry *fib4_entry) 6546{ 6547 struct mlxsw_sp_fib_node *fib_node = fib4_entry->common.fib_node; 6548 struct mlxsw_sp_fib4_entry *fib4_replaced; 6549 6550 if (!fib_node->fib_entry) 6551 return true; 6552 6553 fib4_replaced = container_of(fib_node->fib_entry, 6554 struct mlxsw_sp_fib4_entry, common); 6555 if (fib4_entry->tb_id == RT_TABLE_MAIN && 6556 fib4_replaced->tb_id == RT_TABLE_LOCAL) 6557 return false; 6558 6559 return true; 6560} 6561 6562static int 6563mlxsw_sp_router_fib4_replace(struct mlxsw_sp *mlxsw_sp, 6564 const struct fib_entry_notifier_info *fen_info) 6565{ 6566 struct mlxsw_sp_fib4_entry *fib4_entry, *fib4_replaced; 6567 struct mlxsw_sp_fib_entry *replaced; 6568 struct mlxsw_sp_fib_node *fib_node; 6569 int err; 6570 6571 if (fen_info->fi->nh && 6572 !mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, fen_info->fi->nh->id)) 6573 return 0; 6574 6575 fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, fen_info->tb_id, 6576 &fen_info->dst, sizeof(fen_info->dst), 6577 fen_info->dst_len, 6578 MLXSW_SP_L3_PROTO_IPV4); 6579 if (IS_ERR(fib_node)) { 6580 dev_warn(mlxsw_sp->bus_info->dev, "Failed to get FIB node\n"); 6581 return PTR_ERR(fib_node); 6582 } 6583 6584 fib4_entry = mlxsw_sp_fib4_entry_create(mlxsw_sp, fib_node, fen_info); 6585 if (IS_ERR(fib4_entry)) { 6586 dev_warn(mlxsw_sp->bus_info->dev, "Failed to create FIB entry\n"); 6587 err = PTR_ERR(fib4_entry); 6588 goto err_fib4_entry_create; 6589 } 6590 6591 if (!mlxsw_sp_fib4_allow_replace(fib4_entry)) { 6592 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry); 6593 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 6594 return 0; 6595 } 6596 6597 replaced = fib_node->fib_entry; 6598 err = mlxsw_sp_fib_node_entry_link(mlxsw_sp, &fib4_entry->common); 6599 if (err) { 6600 dev_warn(mlxsw_sp->bus_info->dev, "Failed to link FIB entry to node\n"); 6601 goto err_fib_node_entry_link; 6602 } 6603 6604 /* Nothing to replace */ 6605 if (!replaced) 6606 return 0; 6607 6608 mlxsw_sp_fib_entry_hw_flags_clear(mlxsw_sp, replaced); 6609 fib4_replaced = container_of(replaced, struct mlxsw_sp_fib4_entry, 6610 common); 6611 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_replaced); 6612 6613 return 0; 6614 6615err_fib_node_entry_link: 6616 fib_node->fib_entry = replaced; 6617 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry); 6618err_fib4_entry_create: 6619 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 6620 return err; 6621} 6622 6623static void mlxsw_sp_router_fib4_del(struct mlxsw_sp *mlxsw_sp, 6624 struct fib_entry_notifier_info *fen_info) 6625{ 6626 struct mlxsw_sp_fib4_entry *fib4_entry; 6627 struct mlxsw_sp_fib_node *fib_node; 6628 6629 fib4_entry = mlxsw_sp_fib4_entry_lookup(mlxsw_sp, fen_info); 6630 if (!fib4_entry) 6631 return; 6632 fib_node = fib4_entry->common.fib_node; 6633 6634 mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, &fib4_entry->common); 6635 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry); 6636 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 6637} 6638 6639static bool mlxsw_sp_fib6_rt_should_ignore(const struct fib6_info *rt) 6640{ 6641 /* Multicast routes aren't supported, so ignore them. Neighbour 6642 * Discovery packets are specifically trapped. 6643 */ 6644 if (ipv6_addr_type(&rt->fib6_dst.addr) & IPV6_ADDR_MULTICAST) 6645 return true; 6646 6647 /* Cloned routes are irrelevant in the forwarding path. */ 6648 if (rt->fib6_flags & RTF_CACHE) 6649 return true; 6650 6651 return false; 6652} 6653 6654static struct mlxsw_sp_rt6 *mlxsw_sp_rt6_create(struct fib6_info *rt) 6655{ 6656 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 6657 6658 mlxsw_sp_rt6 = kzalloc(sizeof(*mlxsw_sp_rt6), GFP_KERNEL); 6659 if (!mlxsw_sp_rt6) 6660 return ERR_PTR(-ENOMEM); 6661 6662 /* In case of route replace, replaced route is deleted with 6663 * no notification. Take reference to prevent accessing freed 6664 * memory. 6665 */ 6666 mlxsw_sp_rt6->rt = rt; 6667 fib6_info_hold(rt); 6668 6669 return mlxsw_sp_rt6; 6670} 6671 6672#if IS_ENABLED(CONFIG_IPV6) 6673static void mlxsw_sp_rt6_release(struct fib6_info *rt) 6674{ 6675 fib6_info_release(rt); 6676} 6677#else 6678static void mlxsw_sp_rt6_release(struct fib6_info *rt) 6679{ 6680} 6681#endif 6682 6683static void mlxsw_sp_rt6_destroy(struct mlxsw_sp_rt6 *mlxsw_sp_rt6) 6684{ 6685 struct fib6_nh *fib6_nh = mlxsw_sp_rt6->rt->fib6_nh; 6686 6687 if (!mlxsw_sp_rt6->rt->nh) 6688 fib6_nh->fib_nh_flags &= ~RTNH_F_OFFLOAD; 6689 mlxsw_sp_rt6_release(mlxsw_sp_rt6->rt); 6690 kfree(mlxsw_sp_rt6); 6691} 6692 6693static struct fib6_info * 6694mlxsw_sp_fib6_entry_rt(const struct mlxsw_sp_fib6_entry *fib6_entry) 6695{ 6696 return list_first_entry(&fib6_entry->rt6_list, struct mlxsw_sp_rt6, 6697 list)->rt; 6698} 6699 6700static struct mlxsw_sp_rt6 * 6701mlxsw_sp_fib6_entry_rt_find(const struct mlxsw_sp_fib6_entry *fib6_entry, 6702 const struct fib6_info *rt) 6703{ 6704 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 6705 6706 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) { 6707 if (mlxsw_sp_rt6->rt == rt) 6708 return mlxsw_sp_rt6; 6709 } 6710 6711 return NULL; 6712} 6713 6714static bool mlxsw_sp_nexthop6_ipip_type(const struct mlxsw_sp *mlxsw_sp, 6715 const struct fib6_info *rt, 6716 enum mlxsw_sp_ipip_type *ret) 6717{ 6718 return rt->fib6_nh->fib_nh_dev && 6719 mlxsw_sp_netdev_ipip_type(mlxsw_sp, rt->fib6_nh->fib_nh_dev, ret); 6720} 6721 6722static int mlxsw_sp_nexthop6_init(struct mlxsw_sp *mlxsw_sp, 6723 struct mlxsw_sp_nexthop_group *nh_grp, 6724 struct mlxsw_sp_nexthop *nh, 6725 const struct fib6_info *rt) 6726{ 6727 struct net_device *dev = rt->fib6_nh->fib_nh_dev; 6728 int err; 6729 6730 nh->nhgi = nh_grp->nhgi; 6731 nh->nh_weight = rt->fib6_nh->fib_nh_weight; 6732 memcpy(&nh->gw_addr, &rt->fib6_nh->fib_nh_gw6, sizeof(nh->gw_addr)); 6733#if IS_ENABLED(CONFIG_IPV6) 6734 nh->neigh_tbl = &nd_tbl; 6735#endif 6736 mlxsw_sp_nexthop_counter_alloc(mlxsw_sp, nh); 6737 6738 list_add_tail(&nh->router_list_node, &mlxsw_sp->router->nexthop_list); 6739 6740 if (!dev) 6741 return 0; 6742 nh->ifindex = dev->ifindex; 6743 6744 err = mlxsw_sp_nexthop_type_init(mlxsw_sp, nh, dev); 6745 if (err) 6746 goto err_nexthop_type_init; 6747 6748 return 0; 6749 6750err_nexthop_type_init: 6751 list_del(&nh->router_list_node); 6752 mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh); 6753 return err; 6754} 6755 6756static void mlxsw_sp_nexthop6_fini(struct mlxsw_sp *mlxsw_sp, 6757 struct mlxsw_sp_nexthop *nh) 6758{ 6759 mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh); 6760 list_del(&nh->router_list_node); 6761 mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh); 6762} 6763 6764static bool mlxsw_sp_rt6_is_gateway(const struct mlxsw_sp *mlxsw_sp, 6765 const struct fib6_info *rt) 6766{ 6767 return rt->fib6_nh->fib_nh_gw_family || 6768 mlxsw_sp_nexthop6_ipip_type(mlxsw_sp, rt, NULL); 6769} 6770 6771static int 6772mlxsw_sp_nexthop6_group_info_init(struct mlxsw_sp *mlxsw_sp, 6773 struct mlxsw_sp_nexthop_group *nh_grp, 6774 struct mlxsw_sp_fib6_entry *fib6_entry) 6775{ 6776 struct mlxsw_sp_nexthop_group_info *nhgi; 6777 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 6778 struct mlxsw_sp_nexthop *nh; 6779 int err, i; 6780 6781 nhgi = kzalloc(struct_size(nhgi, nexthops, fib6_entry->nrt6), 6782 GFP_KERNEL); 6783 if (!nhgi) 6784 return -ENOMEM; 6785 nh_grp->nhgi = nhgi; 6786 nhgi->nh_grp = nh_grp; 6787 mlxsw_sp_rt6 = list_first_entry(&fib6_entry->rt6_list, 6788 struct mlxsw_sp_rt6, list); 6789 nhgi->gateway = mlxsw_sp_rt6_is_gateway(mlxsw_sp, mlxsw_sp_rt6->rt); 6790 nhgi->count = fib6_entry->nrt6; 6791 for (i = 0; i < nhgi->count; i++) { 6792 struct fib6_info *rt = mlxsw_sp_rt6->rt; 6793 6794 nh = &nhgi->nexthops[i]; 6795 err = mlxsw_sp_nexthop6_init(mlxsw_sp, nh_grp, nh, rt); 6796 if (err) 6797 goto err_nexthop6_init; 6798 mlxsw_sp_rt6 = list_next_entry(mlxsw_sp_rt6, list); 6799 } 6800 nh_grp->nhgi = nhgi; 6801 err = mlxsw_sp_nexthop_group_inc(mlxsw_sp); 6802 if (err) 6803 goto err_group_inc; 6804 err = mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp); 6805 if (err) 6806 goto err_group_refresh; 6807 6808 return 0; 6809 6810err_group_refresh: 6811 mlxsw_sp_nexthop_group_dec(mlxsw_sp); 6812err_group_inc: 6813 i = nhgi->count; 6814err_nexthop6_init: 6815 for (i--; i >= 0; i--) { 6816 nh = &nhgi->nexthops[i]; 6817 mlxsw_sp_nexthop6_fini(mlxsw_sp, nh); 6818 } 6819 kfree(nhgi); 6820 return err; 6821} 6822 6823static void 6824mlxsw_sp_nexthop6_group_info_fini(struct mlxsw_sp *mlxsw_sp, 6825 struct mlxsw_sp_nexthop_group *nh_grp) 6826{ 6827 struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi; 6828 int i; 6829 6830 mlxsw_sp_nexthop_group_dec(mlxsw_sp); 6831 for (i = nhgi->count - 1; i >= 0; i--) { 6832 struct mlxsw_sp_nexthop *nh = &nhgi->nexthops[i]; 6833 6834 mlxsw_sp_nexthop6_fini(mlxsw_sp, nh); 6835 } 6836 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp); 6837 WARN_ON_ONCE(nhgi->adj_index_valid); 6838 kfree(nhgi); 6839} 6840 6841static struct mlxsw_sp_nexthop_group * 6842mlxsw_sp_nexthop6_group_create(struct mlxsw_sp *mlxsw_sp, 6843 struct mlxsw_sp_fib6_entry *fib6_entry) 6844{ 6845 struct mlxsw_sp_nexthop_group *nh_grp; 6846 int err; 6847 6848 nh_grp = kzalloc(sizeof(*nh_grp), GFP_KERNEL); 6849 if (!nh_grp) 6850 return ERR_PTR(-ENOMEM); 6851 INIT_LIST_HEAD(&nh_grp->vr_list); 6852 err = rhashtable_init(&nh_grp->vr_ht, 6853 &mlxsw_sp_nexthop_group_vr_ht_params); 6854 if (err) 6855 goto err_nexthop_group_vr_ht_init; 6856 INIT_LIST_HEAD(&nh_grp->fib_list); 6857 nh_grp->type = MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6; 6858 6859 err = mlxsw_sp_nexthop6_group_info_init(mlxsw_sp, nh_grp, fib6_entry); 6860 if (err) 6861 goto err_nexthop_group_info_init; 6862 6863 err = mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp); 6864 if (err) 6865 goto err_nexthop_group_insert; 6866 6867 nh_grp->can_destroy = true; 6868 6869 return nh_grp; 6870 6871err_nexthop_group_insert: 6872 mlxsw_sp_nexthop6_group_info_fini(mlxsw_sp, nh_grp); 6873err_nexthop_group_info_init: 6874 rhashtable_destroy(&nh_grp->vr_ht); 6875err_nexthop_group_vr_ht_init: 6876 kfree(nh_grp); 6877 return ERR_PTR(err); 6878} 6879 6880static void 6881mlxsw_sp_nexthop6_group_destroy(struct mlxsw_sp *mlxsw_sp, 6882 struct mlxsw_sp_nexthop_group *nh_grp) 6883{ 6884 if (!nh_grp->can_destroy) 6885 return; 6886 mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp); 6887 mlxsw_sp_nexthop6_group_info_fini(mlxsw_sp, nh_grp); 6888 WARN_ON_ONCE(!list_empty(&nh_grp->vr_list)); 6889 rhashtable_destroy(&nh_grp->vr_ht); 6890 kfree(nh_grp); 6891} 6892 6893static int mlxsw_sp_nexthop6_group_get(struct mlxsw_sp *mlxsw_sp, 6894 struct mlxsw_sp_fib6_entry *fib6_entry) 6895{ 6896 struct fib6_info *rt = mlxsw_sp_fib6_entry_rt(fib6_entry); 6897 struct mlxsw_sp_nexthop_group *nh_grp; 6898 6899 if (rt->nh) { 6900 nh_grp = mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, 6901 rt->nh->id); 6902 if (WARN_ON_ONCE(!nh_grp)) 6903 return -EINVAL; 6904 goto out; 6905 } 6906 6907 nh_grp = mlxsw_sp_nexthop6_group_lookup(mlxsw_sp, fib6_entry); 6908 if (!nh_grp) { 6909 nh_grp = mlxsw_sp_nexthop6_group_create(mlxsw_sp, fib6_entry); 6910 if (IS_ERR(nh_grp)) 6911 return PTR_ERR(nh_grp); 6912 } 6913 6914 /* The route and the nexthop are described by the same struct, so we 6915 * need to the update the nexthop offload indication for the new route. 6916 */ 6917 __mlxsw_sp_nexthop6_group_offload_refresh(nh_grp, fib6_entry); 6918 6919out: 6920 list_add_tail(&fib6_entry->common.nexthop_group_node, 6921 &nh_grp->fib_list); 6922 fib6_entry->common.nh_group = nh_grp; 6923 6924 return 0; 6925} 6926 6927static void mlxsw_sp_nexthop6_group_put(struct mlxsw_sp *mlxsw_sp, 6928 struct mlxsw_sp_fib_entry *fib_entry) 6929{ 6930 struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group; 6931 6932 list_del(&fib_entry->nexthop_group_node); 6933 if (!list_empty(&nh_grp->fib_list)) 6934 return; 6935 6936 if (nh_grp->type == MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ) { 6937 mlxsw_sp_nexthop_obj_group_destroy(mlxsw_sp, nh_grp); 6938 return; 6939 } 6940 6941 mlxsw_sp_nexthop6_group_destroy(mlxsw_sp, nh_grp); 6942} 6943 6944static int 6945mlxsw_sp_nexthop6_group_update(struct mlxsw_sp *mlxsw_sp, 6946 struct mlxsw_sp_fib6_entry *fib6_entry) 6947{ 6948 struct mlxsw_sp_nexthop_group *old_nh_grp = fib6_entry->common.nh_group; 6949 struct mlxsw_sp_fib_node *fib_node = fib6_entry->common.fib_node; 6950 int err; 6951 6952 mlxsw_sp_nexthop_group_vr_unlink(old_nh_grp, fib_node->fib); 6953 fib6_entry->common.nh_group = NULL; 6954 list_del(&fib6_entry->common.nexthop_group_node); 6955 6956 err = mlxsw_sp_nexthop6_group_get(mlxsw_sp, fib6_entry); 6957 if (err) 6958 goto err_nexthop6_group_get; 6959 6960 err = mlxsw_sp_nexthop_group_vr_link(fib6_entry->common.nh_group, 6961 fib_node->fib); 6962 if (err) 6963 goto err_nexthop_group_vr_link; 6964 6965 /* In case this entry is offloaded, then the adjacency index 6966 * currently associated with it in the device's table is that 6967 * of the old group. Start using the new one instead. 6968 */ 6969 err = mlxsw_sp_fib_entry_update(mlxsw_sp, &fib6_entry->common); 6970 if (err) 6971 goto err_fib_entry_update; 6972 6973 if (list_empty(&old_nh_grp->fib_list)) 6974 mlxsw_sp_nexthop6_group_destroy(mlxsw_sp, old_nh_grp); 6975 6976 return 0; 6977 6978err_fib_entry_update: 6979 mlxsw_sp_nexthop_group_vr_unlink(fib6_entry->common.nh_group, 6980 fib_node->fib); 6981err_nexthop_group_vr_link: 6982 mlxsw_sp_nexthop6_group_put(mlxsw_sp, &fib6_entry->common); 6983err_nexthop6_group_get: 6984 list_add_tail(&fib6_entry->common.nexthop_group_node, 6985 &old_nh_grp->fib_list); 6986 fib6_entry->common.nh_group = old_nh_grp; 6987 mlxsw_sp_nexthop_group_vr_link(old_nh_grp, fib_node->fib); 6988 return err; 6989} 6990 6991static int 6992mlxsw_sp_fib6_entry_nexthop_add(struct mlxsw_sp *mlxsw_sp, 6993 struct mlxsw_sp_fib6_entry *fib6_entry, 6994 struct fib6_info **rt_arr, unsigned int nrt6) 6995{ 6996 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 6997 int err, i; 6998 6999 for (i = 0; i < nrt6; i++) { 7000 mlxsw_sp_rt6 = mlxsw_sp_rt6_create(rt_arr[i]); 7001 if (IS_ERR(mlxsw_sp_rt6)) { 7002 err = PTR_ERR(mlxsw_sp_rt6); 7003 goto err_rt6_unwind; 7004 } 7005 7006 list_add_tail(&mlxsw_sp_rt6->list, &fib6_entry->rt6_list); 7007 fib6_entry->nrt6++; 7008 } 7009 7010 err = mlxsw_sp_nexthop6_group_update(mlxsw_sp, fib6_entry); 7011 if (err) 7012 goto err_rt6_unwind; 7013 7014 return 0; 7015 7016err_rt6_unwind: 7017 for (; i > 0; i--) { 7018 fib6_entry->nrt6--; 7019 mlxsw_sp_rt6 = list_last_entry(&fib6_entry->rt6_list, 7020 struct mlxsw_sp_rt6, list); 7021 list_del(&mlxsw_sp_rt6->list); 7022 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6); 7023 } 7024 return err; 7025} 7026 7027static void 7028mlxsw_sp_fib6_entry_nexthop_del(struct mlxsw_sp *mlxsw_sp, 7029 struct mlxsw_sp_fib6_entry *fib6_entry, 7030 struct fib6_info **rt_arr, unsigned int nrt6) 7031{ 7032 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 7033 int i; 7034 7035 for (i = 0; i < nrt6; i++) { 7036 mlxsw_sp_rt6 = mlxsw_sp_fib6_entry_rt_find(fib6_entry, 7037 rt_arr[i]); 7038 if (WARN_ON_ONCE(!mlxsw_sp_rt6)) 7039 continue; 7040 7041 fib6_entry->nrt6--; 7042 list_del(&mlxsw_sp_rt6->list); 7043 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6); 7044 } 7045 7046 mlxsw_sp_nexthop6_group_update(mlxsw_sp, fib6_entry); 7047} 7048 7049static int 7050mlxsw_sp_fib6_entry_type_set_local(struct mlxsw_sp *mlxsw_sp, 7051 struct mlxsw_sp_fib_entry *fib_entry, 7052 const struct fib6_info *rt) 7053{ 7054 struct mlxsw_sp_nexthop_group_info *nhgi = fib_entry->nh_group->nhgi; 7055 union mlxsw_sp_l3addr dip = { .addr6 = rt->fib6_dst.addr }; 7056 u32 tb_id = mlxsw_sp_fix_tb_id(rt->fib6_table->tb6_id); 7057 struct mlxsw_sp_router *router = mlxsw_sp->router; 7058 int ifindex = nhgi->nexthops[0].ifindex; 7059 struct mlxsw_sp_ipip_entry *ipip_entry; 7060 7061 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP; 7062 ipip_entry = mlxsw_sp_ipip_entry_find_by_decap(mlxsw_sp, ifindex, 7063 MLXSW_SP_L3_PROTO_IPV6, 7064 dip); 7065 7066 if (ipip_entry && ipip_entry->ol_dev->flags & IFF_UP) { 7067 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP; 7068 return mlxsw_sp_fib_entry_decap_init(mlxsw_sp, fib_entry, 7069 ipip_entry); 7070 } 7071 if (mlxsw_sp_router_nve_is_decap(mlxsw_sp, tb_id, 7072 MLXSW_SP_L3_PROTO_IPV6, &dip)) { 7073 u32 tunnel_index; 7074 7075 tunnel_index = router->nve_decap_config.tunnel_index; 7076 fib_entry->decap.tunnel_index = tunnel_index; 7077 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP; 7078 } 7079 7080 return 0; 7081} 7082 7083static int mlxsw_sp_fib6_entry_type_set(struct mlxsw_sp *mlxsw_sp, 7084 struct mlxsw_sp_fib_entry *fib_entry, 7085 const struct fib6_info *rt) 7086{ 7087 if (rt->fib6_flags & RTF_LOCAL) 7088 return mlxsw_sp_fib6_entry_type_set_local(mlxsw_sp, fib_entry, 7089 rt); 7090 if (rt->fib6_flags & RTF_ANYCAST) 7091 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP; 7092 else if (rt->fib6_type == RTN_BLACKHOLE) 7093 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE; 7094 else if (rt->fib6_flags & RTF_REJECT) 7095 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_UNREACHABLE; 7096 else if (fib_entry->nh_group->nhgi->gateway) 7097 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_REMOTE; 7098 else 7099 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL; 7100 7101 return 0; 7102} 7103 7104static void 7105mlxsw_sp_fib6_entry_rt_destroy_all(struct mlxsw_sp_fib6_entry *fib6_entry) 7106{ 7107 struct mlxsw_sp_rt6 *mlxsw_sp_rt6, *tmp; 7108 7109 list_for_each_entry_safe(mlxsw_sp_rt6, tmp, &fib6_entry->rt6_list, 7110 list) { 7111 fib6_entry->nrt6--; 7112 list_del(&mlxsw_sp_rt6->list); 7113 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6); 7114 } 7115} 7116 7117static struct mlxsw_sp_fib6_entry * 7118mlxsw_sp_fib6_entry_create(struct mlxsw_sp *mlxsw_sp, 7119 struct mlxsw_sp_fib_node *fib_node, 7120 struct fib6_info **rt_arr, unsigned int nrt6) 7121{ 7122 struct mlxsw_sp_fib6_entry *fib6_entry; 7123 struct mlxsw_sp_fib_entry *fib_entry; 7124 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 7125 int err, i; 7126 7127 fib6_entry = kzalloc(sizeof(*fib6_entry), GFP_KERNEL); 7128 if (!fib6_entry) 7129 return ERR_PTR(-ENOMEM); 7130 fib_entry = &fib6_entry->common; 7131 7132 INIT_LIST_HEAD(&fib6_entry->rt6_list); 7133 7134 for (i = 0; i < nrt6; i++) { 7135 mlxsw_sp_rt6 = mlxsw_sp_rt6_create(rt_arr[i]); 7136 if (IS_ERR(mlxsw_sp_rt6)) { 7137 err = PTR_ERR(mlxsw_sp_rt6); 7138 goto err_rt6_unwind; 7139 } 7140 list_add_tail(&mlxsw_sp_rt6->list, &fib6_entry->rt6_list); 7141 fib6_entry->nrt6++; 7142 } 7143 7144 err = mlxsw_sp_nexthop6_group_get(mlxsw_sp, fib6_entry); 7145 if (err) 7146 goto err_rt6_unwind; 7147 7148 err = mlxsw_sp_nexthop_group_vr_link(fib_entry->nh_group, 7149 fib_node->fib); 7150 if (err) 7151 goto err_nexthop_group_vr_link; 7152 7153 err = mlxsw_sp_fib6_entry_type_set(mlxsw_sp, fib_entry, rt_arr[0]); 7154 if (err) 7155 goto err_fib6_entry_type_set; 7156 7157 fib_entry->fib_node = fib_node; 7158 7159 return fib6_entry; 7160 7161err_fib6_entry_type_set: 7162 mlxsw_sp_nexthop_group_vr_unlink(fib_entry->nh_group, fib_node->fib); 7163err_nexthop_group_vr_link: 7164 mlxsw_sp_nexthop6_group_put(mlxsw_sp, fib_entry); 7165err_rt6_unwind: 7166 for (; i > 0; i--) { 7167 fib6_entry->nrt6--; 7168 mlxsw_sp_rt6 = list_last_entry(&fib6_entry->rt6_list, 7169 struct mlxsw_sp_rt6, list); 7170 list_del(&mlxsw_sp_rt6->list); 7171 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6); 7172 } 7173 kfree(fib6_entry); 7174 return ERR_PTR(err); 7175} 7176 7177static void 7178mlxsw_sp_fib6_entry_type_unset(struct mlxsw_sp *mlxsw_sp, 7179 struct mlxsw_sp_fib6_entry *fib6_entry) 7180{ 7181 mlxsw_sp_fib_entry_type_unset(mlxsw_sp, &fib6_entry->common); 7182} 7183 7184static void mlxsw_sp_fib6_entry_destroy(struct mlxsw_sp *mlxsw_sp, 7185 struct mlxsw_sp_fib6_entry *fib6_entry) 7186{ 7187 struct mlxsw_sp_fib_node *fib_node = fib6_entry->common.fib_node; 7188 7189 mlxsw_sp_fib6_entry_type_unset(mlxsw_sp, fib6_entry); 7190 mlxsw_sp_nexthop_group_vr_unlink(fib6_entry->common.nh_group, 7191 fib_node->fib); 7192 mlxsw_sp_nexthop6_group_put(mlxsw_sp, &fib6_entry->common); 7193 mlxsw_sp_fib6_entry_rt_destroy_all(fib6_entry); 7194 WARN_ON(fib6_entry->nrt6); 7195 kfree(fib6_entry); 7196} 7197 7198static struct mlxsw_sp_fib6_entry * 7199mlxsw_sp_fib6_entry_lookup(struct mlxsw_sp *mlxsw_sp, 7200 const struct fib6_info *rt) 7201{ 7202 struct mlxsw_sp_fib6_entry *fib6_entry; 7203 struct mlxsw_sp_fib_node *fib_node; 7204 struct mlxsw_sp_fib *fib; 7205 struct fib6_info *cmp_rt; 7206 struct mlxsw_sp_vr *vr; 7207 7208 vr = mlxsw_sp_vr_find(mlxsw_sp, rt->fib6_table->tb6_id); 7209 if (!vr) 7210 return NULL; 7211 fib = mlxsw_sp_vr_fib(vr, MLXSW_SP_L3_PROTO_IPV6); 7212 7213 fib_node = mlxsw_sp_fib_node_lookup(fib, &rt->fib6_dst.addr, 7214 sizeof(rt->fib6_dst.addr), 7215 rt->fib6_dst.plen); 7216 if (!fib_node) 7217 return NULL; 7218 7219 fib6_entry = container_of(fib_node->fib_entry, 7220 struct mlxsw_sp_fib6_entry, common); 7221 cmp_rt = mlxsw_sp_fib6_entry_rt(fib6_entry); 7222 if (rt->fib6_table->tb6_id == cmp_rt->fib6_table->tb6_id && 7223 rt->fib6_metric == cmp_rt->fib6_metric && 7224 mlxsw_sp_fib6_entry_rt_find(fib6_entry, rt)) 7225 return fib6_entry; 7226 7227 return NULL; 7228} 7229 7230static bool mlxsw_sp_fib6_allow_replace(struct mlxsw_sp_fib6_entry *fib6_entry) 7231{ 7232 struct mlxsw_sp_fib_node *fib_node = fib6_entry->common.fib_node; 7233 struct mlxsw_sp_fib6_entry *fib6_replaced; 7234 struct fib6_info *rt, *rt_replaced; 7235 7236 if (!fib_node->fib_entry) 7237 return true; 7238 7239 fib6_replaced = container_of(fib_node->fib_entry, 7240 struct mlxsw_sp_fib6_entry, 7241 common); 7242 rt = mlxsw_sp_fib6_entry_rt(fib6_entry); 7243 rt_replaced = mlxsw_sp_fib6_entry_rt(fib6_replaced); 7244 if (rt->fib6_table->tb6_id == RT_TABLE_MAIN && 7245 rt_replaced->fib6_table->tb6_id == RT_TABLE_LOCAL) 7246 return false; 7247 7248 return true; 7249} 7250 7251static int mlxsw_sp_router_fib6_replace(struct mlxsw_sp *mlxsw_sp, 7252 struct fib6_info **rt_arr, 7253 unsigned int nrt6) 7254{ 7255 struct mlxsw_sp_fib6_entry *fib6_entry, *fib6_replaced; 7256 struct mlxsw_sp_fib_entry *replaced; 7257 struct mlxsw_sp_fib_node *fib_node; 7258 struct fib6_info *rt = rt_arr[0]; 7259 int err; 7260 7261 if (rt->fib6_src.plen) 7262 return -EINVAL; 7263 7264 if (mlxsw_sp_fib6_rt_should_ignore(rt)) 7265 return 0; 7266 7267 if (rt->nh && !mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, rt->nh->id)) 7268 return 0; 7269 7270 fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, rt->fib6_table->tb6_id, 7271 &rt->fib6_dst.addr, 7272 sizeof(rt->fib6_dst.addr), 7273 rt->fib6_dst.plen, 7274 MLXSW_SP_L3_PROTO_IPV6); 7275 if (IS_ERR(fib_node)) 7276 return PTR_ERR(fib_node); 7277 7278 fib6_entry = mlxsw_sp_fib6_entry_create(mlxsw_sp, fib_node, rt_arr, 7279 nrt6); 7280 if (IS_ERR(fib6_entry)) { 7281 err = PTR_ERR(fib6_entry); 7282 goto err_fib6_entry_create; 7283 } 7284 7285 if (!mlxsw_sp_fib6_allow_replace(fib6_entry)) { 7286 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry); 7287 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 7288 return 0; 7289 } 7290 7291 replaced = fib_node->fib_entry; 7292 err = mlxsw_sp_fib_node_entry_link(mlxsw_sp, &fib6_entry->common); 7293 if (err) 7294 goto err_fib_node_entry_link; 7295 7296 /* Nothing to replace */ 7297 if (!replaced) 7298 return 0; 7299 7300 mlxsw_sp_fib_entry_hw_flags_clear(mlxsw_sp, replaced); 7301 fib6_replaced = container_of(replaced, struct mlxsw_sp_fib6_entry, 7302 common); 7303 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_replaced); 7304 7305 return 0; 7306 7307err_fib_node_entry_link: 7308 fib_node->fib_entry = replaced; 7309 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry); 7310err_fib6_entry_create: 7311 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 7312 return err; 7313} 7314 7315static int mlxsw_sp_router_fib6_append(struct mlxsw_sp *mlxsw_sp, 7316 struct fib6_info **rt_arr, 7317 unsigned int nrt6) 7318{ 7319 struct mlxsw_sp_fib6_entry *fib6_entry; 7320 struct mlxsw_sp_fib_node *fib_node; 7321 struct fib6_info *rt = rt_arr[0]; 7322 int err; 7323 7324 if (rt->fib6_src.plen) 7325 return -EINVAL; 7326 7327 if (mlxsw_sp_fib6_rt_should_ignore(rt)) 7328 return 0; 7329 7330 fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, rt->fib6_table->tb6_id, 7331 &rt->fib6_dst.addr, 7332 sizeof(rt->fib6_dst.addr), 7333 rt->fib6_dst.plen, 7334 MLXSW_SP_L3_PROTO_IPV6); 7335 if (IS_ERR(fib_node)) 7336 return PTR_ERR(fib_node); 7337 7338 if (WARN_ON_ONCE(!fib_node->fib_entry)) { 7339 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 7340 return -EINVAL; 7341 } 7342 7343 fib6_entry = container_of(fib_node->fib_entry, 7344 struct mlxsw_sp_fib6_entry, common); 7345 err = mlxsw_sp_fib6_entry_nexthop_add(mlxsw_sp, fib6_entry, rt_arr, 7346 nrt6); 7347 if (err) 7348 goto err_fib6_entry_nexthop_add; 7349 7350 return 0; 7351 7352err_fib6_entry_nexthop_add: 7353 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 7354 return err; 7355} 7356 7357static void mlxsw_sp_router_fib6_del(struct mlxsw_sp *mlxsw_sp, 7358 struct fib6_info **rt_arr, 7359 unsigned int nrt6) 7360{ 7361 struct mlxsw_sp_fib6_entry *fib6_entry; 7362 struct mlxsw_sp_fib_node *fib_node; 7363 struct fib6_info *rt = rt_arr[0]; 7364 7365 if (mlxsw_sp_fib6_rt_should_ignore(rt)) 7366 return; 7367 7368 /* Multipath routes are first added to the FIB trie and only then 7369 * notified. If we vetoed the addition, we will get a delete 7370 * notification for a route we do not have. Therefore, do not warn if 7371 * route was not found. 7372 */ 7373 fib6_entry = mlxsw_sp_fib6_entry_lookup(mlxsw_sp, rt); 7374 if (!fib6_entry) 7375 return; 7376 7377 /* If not all the nexthops are deleted, then only reduce the nexthop 7378 * group. 7379 */ 7380 if (nrt6 != fib6_entry->nrt6) { 7381 mlxsw_sp_fib6_entry_nexthop_del(mlxsw_sp, fib6_entry, rt_arr, 7382 nrt6); 7383 return; 7384 } 7385 7386 fib_node = fib6_entry->common.fib_node; 7387 7388 mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, &fib6_entry->common); 7389 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry); 7390 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 7391} 7392 7393static struct mlxsw_sp_mr_table * 7394mlxsw_sp_router_fibmr_family_to_table(struct mlxsw_sp_vr *vr, int family) 7395{ 7396 if (family == RTNL_FAMILY_IPMR) 7397 return vr->mr_table[MLXSW_SP_L3_PROTO_IPV4]; 7398 else 7399 return vr->mr_table[MLXSW_SP_L3_PROTO_IPV6]; 7400} 7401 7402static int mlxsw_sp_router_fibmr_add(struct mlxsw_sp *mlxsw_sp, 7403 struct mfc_entry_notifier_info *men_info, 7404 bool replace) 7405{ 7406 struct mlxsw_sp_mr_table *mrt; 7407 struct mlxsw_sp_vr *vr; 7408 7409 vr = mlxsw_sp_vr_get(mlxsw_sp, men_info->tb_id, NULL); 7410 if (IS_ERR(vr)) 7411 return PTR_ERR(vr); 7412 7413 mrt = mlxsw_sp_router_fibmr_family_to_table(vr, men_info->info.family); 7414 return mlxsw_sp_mr_route_add(mrt, men_info->mfc, replace); 7415} 7416 7417static void mlxsw_sp_router_fibmr_del(struct mlxsw_sp *mlxsw_sp, 7418 struct mfc_entry_notifier_info *men_info) 7419{ 7420 struct mlxsw_sp_mr_table *mrt; 7421 struct mlxsw_sp_vr *vr; 7422 7423 vr = mlxsw_sp_vr_find(mlxsw_sp, men_info->tb_id); 7424 if (WARN_ON(!vr)) 7425 return; 7426 7427 mrt = mlxsw_sp_router_fibmr_family_to_table(vr, men_info->info.family); 7428 mlxsw_sp_mr_route_del(mrt, men_info->mfc); 7429 mlxsw_sp_vr_put(mlxsw_sp, vr); 7430} 7431 7432static int 7433mlxsw_sp_router_fibmr_vif_add(struct mlxsw_sp *mlxsw_sp, 7434 struct vif_entry_notifier_info *ven_info) 7435{ 7436 struct mlxsw_sp_mr_table *mrt; 7437 struct mlxsw_sp_rif *rif; 7438 struct mlxsw_sp_vr *vr; 7439 7440 vr = mlxsw_sp_vr_get(mlxsw_sp, ven_info->tb_id, NULL); 7441 if (IS_ERR(vr)) 7442 return PTR_ERR(vr); 7443 7444 mrt = mlxsw_sp_router_fibmr_family_to_table(vr, ven_info->info.family); 7445 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, ven_info->dev); 7446 return mlxsw_sp_mr_vif_add(mrt, ven_info->dev, 7447 ven_info->vif_index, 7448 ven_info->vif_flags, rif); 7449} 7450 7451static void 7452mlxsw_sp_router_fibmr_vif_del(struct mlxsw_sp *mlxsw_sp, 7453 struct vif_entry_notifier_info *ven_info) 7454{ 7455 struct mlxsw_sp_mr_table *mrt; 7456 struct mlxsw_sp_vr *vr; 7457 7458 vr = mlxsw_sp_vr_find(mlxsw_sp, ven_info->tb_id); 7459 if (WARN_ON(!vr)) 7460 return; 7461 7462 mrt = mlxsw_sp_router_fibmr_family_to_table(vr, ven_info->info.family); 7463 mlxsw_sp_mr_vif_del(mrt, ven_info->vif_index); 7464 mlxsw_sp_vr_put(mlxsw_sp, vr); 7465} 7466 7467static void mlxsw_sp_fib4_node_flush(struct mlxsw_sp *mlxsw_sp, 7468 struct mlxsw_sp_fib_node *fib_node) 7469{ 7470 struct mlxsw_sp_fib4_entry *fib4_entry; 7471 7472 fib4_entry = container_of(fib_node->fib_entry, 7473 struct mlxsw_sp_fib4_entry, common); 7474 mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, fib_node->fib_entry); 7475 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry); 7476 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 7477} 7478 7479static void mlxsw_sp_fib6_node_flush(struct mlxsw_sp *mlxsw_sp, 7480 struct mlxsw_sp_fib_node *fib_node) 7481{ 7482 struct mlxsw_sp_fib6_entry *fib6_entry; 7483 7484 fib6_entry = container_of(fib_node->fib_entry, 7485 struct mlxsw_sp_fib6_entry, common); 7486 mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, fib_node->fib_entry); 7487 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry); 7488 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 7489} 7490 7491static void mlxsw_sp_fib_node_flush(struct mlxsw_sp *mlxsw_sp, 7492 struct mlxsw_sp_fib_node *fib_node) 7493{ 7494 switch (fib_node->fib->proto) { 7495 case MLXSW_SP_L3_PROTO_IPV4: 7496 mlxsw_sp_fib4_node_flush(mlxsw_sp, fib_node); 7497 break; 7498 case MLXSW_SP_L3_PROTO_IPV6: 7499 mlxsw_sp_fib6_node_flush(mlxsw_sp, fib_node); 7500 break; 7501 } 7502} 7503 7504static void mlxsw_sp_vr_fib_flush(struct mlxsw_sp *mlxsw_sp, 7505 struct mlxsw_sp_vr *vr, 7506 enum mlxsw_sp_l3proto proto) 7507{ 7508 struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto); 7509 struct mlxsw_sp_fib_node *fib_node, *tmp; 7510 7511 list_for_each_entry_safe(fib_node, tmp, &fib->node_list, list) { 7512 bool do_break = &tmp->list == &fib->node_list; 7513 7514 mlxsw_sp_fib_node_flush(mlxsw_sp, fib_node); 7515 if (do_break) 7516 break; 7517 } 7518} 7519 7520static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp) 7521{ 7522 int max_vrs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); 7523 int i, j; 7524 7525 for (i = 0; i < max_vrs; i++) { 7526 struct mlxsw_sp_vr *vr = &mlxsw_sp->router->vrs[i]; 7527 7528 if (!mlxsw_sp_vr_is_used(vr)) 7529 continue; 7530 7531 for (j = 0; j < MLXSW_SP_L3_PROTO_MAX; j++) 7532 mlxsw_sp_mr_table_flush(vr->mr_table[j]); 7533 mlxsw_sp_vr_fib_flush(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV4); 7534 7535 /* If virtual router was only used for IPv4, then it's no 7536 * longer used. 7537 */ 7538 if (!mlxsw_sp_vr_is_used(vr)) 7539 continue; 7540 mlxsw_sp_vr_fib_flush(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV6); 7541 } 7542} 7543 7544struct mlxsw_sp_fib6_event_work { 7545 struct fib6_info **rt_arr; 7546 unsigned int nrt6; 7547}; 7548 7549struct mlxsw_sp_fib_event_work { 7550 struct work_struct work; 7551 netdevice_tracker dev_tracker; 7552 union { 7553 struct mlxsw_sp_fib6_event_work fib6_work; 7554 struct fib_entry_notifier_info fen_info; 7555 struct fib_rule_notifier_info fr_info; 7556 struct fib_nh_notifier_info fnh_info; 7557 struct mfc_entry_notifier_info men_info; 7558 struct vif_entry_notifier_info ven_info; 7559 }; 7560 struct mlxsw_sp *mlxsw_sp; 7561 unsigned long event; 7562}; 7563 7564static int 7565mlxsw_sp_router_fib6_work_init(struct mlxsw_sp_fib6_event_work *fib6_work, 7566 struct fib6_entry_notifier_info *fen6_info) 7567{ 7568 struct fib6_info *rt = fen6_info->rt; 7569 struct fib6_info **rt_arr; 7570 struct fib6_info *iter; 7571 unsigned int nrt6; 7572 int i = 0; 7573 7574 nrt6 = fen6_info->nsiblings + 1; 7575 7576 rt_arr = kcalloc(nrt6, sizeof(struct fib6_info *), GFP_ATOMIC); 7577 if (!rt_arr) 7578 return -ENOMEM; 7579 7580 fib6_work->rt_arr = rt_arr; 7581 fib6_work->nrt6 = nrt6; 7582 7583 rt_arr[0] = rt; 7584 fib6_info_hold(rt); 7585 7586 if (!fen6_info->nsiblings) 7587 return 0; 7588 7589 list_for_each_entry(iter, &rt->fib6_siblings, fib6_siblings) { 7590 if (i == fen6_info->nsiblings) 7591 break; 7592 7593 rt_arr[i + 1] = iter; 7594 fib6_info_hold(iter); 7595 i++; 7596 } 7597 WARN_ON_ONCE(i != fen6_info->nsiblings); 7598 7599 return 0; 7600} 7601 7602static void 7603mlxsw_sp_router_fib6_work_fini(struct mlxsw_sp_fib6_event_work *fib6_work) 7604{ 7605 int i; 7606 7607 for (i = 0; i < fib6_work->nrt6; i++) 7608 mlxsw_sp_rt6_release(fib6_work->rt_arr[i]); 7609 kfree(fib6_work->rt_arr); 7610} 7611 7612static void mlxsw_sp_router_fib4_event_work(struct work_struct *work) 7613{ 7614 struct mlxsw_sp_fib_event_work *fib_work = 7615 container_of(work, struct mlxsw_sp_fib_event_work, work); 7616 struct mlxsw_sp *mlxsw_sp = fib_work->mlxsw_sp; 7617 int err; 7618 7619 mutex_lock(&mlxsw_sp->router->lock); 7620 mlxsw_sp_span_respin(mlxsw_sp); 7621 7622 switch (fib_work->event) { 7623 case FIB_EVENT_ENTRY_REPLACE: 7624 err = mlxsw_sp_router_fib4_replace(mlxsw_sp, 7625 &fib_work->fen_info); 7626 if (err) { 7627 dev_warn(mlxsw_sp->bus_info->dev, "FIB replace failed.\n"); 7628 mlxsw_sp_fib4_offload_failed_flag_set(mlxsw_sp, 7629 &fib_work->fen_info); 7630 } 7631 fib_info_put(fib_work->fen_info.fi); 7632 break; 7633 case FIB_EVENT_ENTRY_DEL: 7634 mlxsw_sp_router_fib4_del(mlxsw_sp, &fib_work->fen_info); 7635 fib_info_put(fib_work->fen_info.fi); 7636 break; 7637 case FIB_EVENT_NH_ADD: 7638 case FIB_EVENT_NH_DEL: 7639 mlxsw_sp_nexthop4_event(mlxsw_sp, fib_work->event, 7640 fib_work->fnh_info.fib_nh); 7641 fib_info_put(fib_work->fnh_info.fib_nh->nh_parent); 7642 break; 7643 } 7644 mutex_unlock(&mlxsw_sp->router->lock); 7645 kfree(fib_work); 7646} 7647 7648static void mlxsw_sp_router_fib6_event_work(struct work_struct *work) 7649{ 7650 struct mlxsw_sp_fib_event_work *fib_work = 7651 container_of(work, struct mlxsw_sp_fib_event_work, work); 7652 struct mlxsw_sp_fib6_event_work *fib6_work = &fib_work->fib6_work; 7653 struct mlxsw_sp *mlxsw_sp = fib_work->mlxsw_sp; 7654 int err; 7655 7656 mutex_lock(&mlxsw_sp->router->lock); 7657 mlxsw_sp_span_respin(mlxsw_sp); 7658 7659 switch (fib_work->event) { 7660 case FIB_EVENT_ENTRY_REPLACE: 7661 err = mlxsw_sp_router_fib6_replace(mlxsw_sp, 7662 fib6_work->rt_arr, 7663 fib6_work->nrt6); 7664 if (err) { 7665 dev_warn(mlxsw_sp->bus_info->dev, "FIB replace failed.\n"); 7666 mlxsw_sp_fib6_offload_failed_flag_set(mlxsw_sp, 7667 fib6_work->rt_arr, 7668 fib6_work->nrt6); 7669 } 7670 mlxsw_sp_router_fib6_work_fini(fib6_work); 7671 break; 7672 case FIB_EVENT_ENTRY_APPEND: 7673 err = mlxsw_sp_router_fib6_append(mlxsw_sp, 7674 fib6_work->rt_arr, 7675 fib6_work->nrt6); 7676 if (err) { 7677 dev_warn(mlxsw_sp->bus_info->dev, "FIB append failed.\n"); 7678 mlxsw_sp_fib6_offload_failed_flag_set(mlxsw_sp, 7679 fib6_work->rt_arr, 7680 fib6_work->nrt6); 7681 } 7682 mlxsw_sp_router_fib6_work_fini(fib6_work); 7683 break; 7684 case FIB_EVENT_ENTRY_DEL: 7685 mlxsw_sp_router_fib6_del(mlxsw_sp, 7686 fib6_work->rt_arr, 7687 fib6_work->nrt6); 7688 mlxsw_sp_router_fib6_work_fini(fib6_work); 7689 break; 7690 } 7691 mutex_unlock(&mlxsw_sp->router->lock); 7692 kfree(fib_work); 7693} 7694 7695static void mlxsw_sp_router_fibmr_event_work(struct work_struct *work) 7696{ 7697 struct mlxsw_sp_fib_event_work *fib_work = 7698 container_of(work, struct mlxsw_sp_fib_event_work, work); 7699 struct mlxsw_sp *mlxsw_sp = fib_work->mlxsw_sp; 7700 bool replace; 7701 int err; 7702 7703 rtnl_lock(); 7704 mutex_lock(&mlxsw_sp->router->lock); 7705 switch (fib_work->event) { 7706 case FIB_EVENT_ENTRY_REPLACE: 7707 case FIB_EVENT_ENTRY_ADD: 7708 replace = fib_work->event == FIB_EVENT_ENTRY_REPLACE; 7709 7710 err = mlxsw_sp_router_fibmr_add(mlxsw_sp, &fib_work->men_info, 7711 replace); 7712 if (err) 7713 dev_warn(mlxsw_sp->bus_info->dev, "MR entry add failed.\n"); 7714 mr_cache_put(fib_work->men_info.mfc); 7715 break; 7716 case FIB_EVENT_ENTRY_DEL: 7717 mlxsw_sp_router_fibmr_del(mlxsw_sp, &fib_work->men_info); 7718 mr_cache_put(fib_work->men_info.mfc); 7719 break; 7720 case FIB_EVENT_VIF_ADD: 7721 err = mlxsw_sp_router_fibmr_vif_add(mlxsw_sp, 7722 &fib_work->ven_info); 7723 if (err) 7724 dev_warn(mlxsw_sp->bus_info->dev, "MR VIF add failed.\n"); 7725 netdev_put(fib_work->ven_info.dev, &fib_work->dev_tracker); 7726 break; 7727 case FIB_EVENT_VIF_DEL: 7728 mlxsw_sp_router_fibmr_vif_del(mlxsw_sp, 7729 &fib_work->ven_info); 7730 netdev_put(fib_work->ven_info.dev, &fib_work->dev_tracker); 7731 break; 7732 } 7733 mutex_unlock(&mlxsw_sp->router->lock); 7734 rtnl_unlock(); 7735 kfree(fib_work); 7736} 7737 7738static void mlxsw_sp_router_fib4_event(struct mlxsw_sp_fib_event_work *fib_work, 7739 struct fib_notifier_info *info) 7740{ 7741 struct fib_entry_notifier_info *fen_info; 7742 struct fib_nh_notifier_info *fnh_info; 7743 7744 switch (fib_work->event) { 7745 case FIB_EVENT_ENTRY_REPLACE: 7746 case FIB_EVENT_ENTRY_DEL: 7747 fen_info = container_of(info, struct fib_entry_notifier_info, 7748 info); 7749 fib_work->fen_info = *fen_info; 7750 /* Take reference on fib_info to prevent it from being 7751 * freed while work is queued. Release it afterwards. 7752 */ 7753 fib_info_hold(fib_work->fen_info.fi); 7754 break; 7755 case FIB_EVENT_NH_ADD: 7756 case FIB_EVENT_NH_DEL: 7757 fnh_info = container_of(info, struct fib_nh_notifier_info, 7758 info); 7759 fib_work->fnh_info = *fnh_info; 7760 fib_info_hold(fib_work->fnh_info.fib_nh->nh_parent); 7761 break; 7762 } 7763} 7764 7765static int mlxsw_sp_router_fib6_event(struct mlxsw_sp_fib_event_work *fib_work, 7766 struct fib_notifier_info *info) 7767{ 7768 struct fib6_entry_notifier_info *fen6_info; 7769 int err; 7770 7771 switch (fib_work->event) { 7772 case FIB_EVENT_ENTRY_REPLACE: 7773 case FIB_EVENT_ENTRY_APPEND: 7774 case FIB_EVENT_ENTRY_DEL: 7775 fen6_info = container_of(info, struct fib6_entry_notifier_info, 7776 info); 7777 err = mlxsw_sp_router_fib6_work_init(&fib_work->fib6_work, 7778 fen6_info); 7779 if (err) 7780 return err; 7781 break; 7782 } 7783 7784 return 0; 7785} 7786 7787static void 7788mlxsw_sp_router_fibmr_event(struct mlxsw_sp_fib_event_work *fib_work, 7789 struct fib_notifier_info *info) 7790{ 7791 switch (fib_work->event) { 7792 case FIB_EVENT_ENTRY_REPLACE: 7793 case FIB_EVENT_ENTRY_ADD: 7794 case FIB_EVENT_ENTRY_DEL: 7795 memcpy(&fib_work->men_info, info, sizeof(fib_work->men_info)); 7796 mr_cache_hold(fib_work->men_info.mfc); 7797 break; 7798 case FIB_EVENT_VIF_ADD: 7799 case FIB_EVENT_VIF_DEL: 7800 memcpy(&fib_work->ven_info, info, sizeof(fib_work->ven_info)); 7801 netdev_hold(fib_work->ven_info.dev, &fib_work->dev_tracker, 7802 GFP_ATOMIC); 7803 break; 7804 } 7805} 7806 7807static int mlxsw_sp_router_fib_rule_event(unsigned long event, 7808 struct fib_notifier_info *info, 7809 struct mlxsw_sp *mlxsw_sp) 7810{ 7811 struct netlink_ext_ack *extack = info->extack; 7812 struct fib_rule_notifier_info *fr_info; 7813 struct fib_rule *rule; 7814 int err = 0; 7815 7816 /* nothing to do at the moment */ 7817 if (event == FIB_EVENT_RULE_DEL) 7818 return 0; 7819 7820 fr_info = container_of(info, struct fib_rule_notifier_info, info); 7821 rule = fr_info->rule; 7822 7823 /* Rule only affects locally generated traffic */ 7824 if (rule->iifindex == mlxsw_sp_net(mlxsw_sp)->loopback_dev->ifindex) 7825 return 0; 7826 7827 switch (info->family) { 7828 case AF_INET: 7829 if (!fib4_rule_default(rule) && !rule->l3mdev) 7830 err = -EOPNOTSUPP; 7831 break; 7832 case AF_INET6: 7833 if (!fib6_rule_default(rule) && !rule->l3mdev) 7834 err = -EOPNOTSUPP; 7835 break; 7836 case RTNL_FAMILY_IPMR: 7837 if (!ipmr_rule_default(rule) && !rule->l3mdev) 7838 err = -EOPNOTSUPP; 7839 break; 7840 case RTNL_FAMILY_IP6MR: 7841 if (!ip6mr_rule_default(rule) && !rule->l3mdev) 7842 err = -EOPNOTSUPP; 7843 break; 7844 } 7845 7846 if (err < 0) 7847 NL_SET_ERR_MSG_MOD(extack, "FIB rules not supported"); 7848 7849 return err; 7850} 7851 7852/* Called with rcu_read_lock() */ 7853static int mlxsw_sp_router_fib_event(struct notifier_block *nb, 7854 unsigned long event, void *ptr) 7855{ 7856 struct mlxsw_sp_fib_event_work *fib_work; 7857 struct fib_notifier_info *info = ptr; 7858 struct mlxsw_sp_router *router; 7859 int err; 7860 7861 if ((info->family != AF_INET && info->family != AF_INET6 && 7862 info->family != RTNL_FAMILY_IPMR && 7863 info->family != RTNL_FAMILY_IP6MR)) 7864 return NOTIFY_DONE; 7865 7866 router = container_of(nb, struct mlxsw_sp_router, fib_nb); 7867 7868 switch (event) { 7869 case FIB_EVENT_RULE_ADD: 7870 case FIB_EVENT_RULE_DEL: 7871 err = mlxsw_sp_router_fib_rule_event(event, info, 7872 router->mlxsw_sp); 7873 return notifier_from_errno(err); 7874 case FIB_EVENT_ENTRY_ADD: 7875 case FIB_EVENT_ENTRY_REPLACE: 7876 case FIB_EVENT_ENTRY_APPEND: 7877 if (info->family == AF_INET) { 7878 struct fib_entry_notifier_info *fen_info = ptr; 7879 7880 if (fen_info->fi->fib_nh_is_v6) { 7881 NL_SET_ERR_MSG_MOD(info->extack, "IPv6 gateway with IPv4 route is not supported"); 7882 return notifier_from_errno(-EINVAL); 7883 } 7884 } 7885 break; 7886 } 7887 7888 fib_work = kzalloc(sizeof(*fib_work), GFP_ATOMIC); 7889 if (!fib_work) 7890 return NOTIFY_BAD; 7891 7892 fib_work->mlxsw_sp = router->mlxsw_sp; 7893 fib_work->event = event; 7894 7895 switch (info->family) { 7896 case AF_INET: 7897 INIT_WORK(&fib_work->work, mlxsw_sp_router_fib4_event_work); 7898 mlxsw_sp_router_fib4_event(fib_work, info); 7899 break; 7900 case AF_INET6: 7901 INIT_WORK(&fib_work->work, mlxsw_sp_router_fib6_event_work); 7902 err = mlxsw_sp_router_fib6_event(fib_work, info); 7903 if (err) 7904 goto err_fib_event; 7905 break; 7906 case RTNL_FAMILY_IP6MR: 7907 case RTNL_FAMILY_IPMR: 7908 INIT_WORK(&fib_work->work, mlxsw_sp_router_fibmr_event_work); 7909 mlxsw_sp_router_fibmr_event(fib_work, info); 7910 break; 7911 } 7912 7913 mlxsw_core_schedule_work(&fib_work->work); 7914 7915 return NOTIFY_DONE; 7916 7917err_fib_event: 7918 kfree(fib_work); 7919 return NOTIFY_BAD; 7920} 7921 7922static struct mlxsw_sp_rif * 7923mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp, 7924 const struct net_device *dev) 7925{ 7926 int max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); 7927 int i; 7928 7929 for (i = 0; i < max_rifs; i++) 7930 if (mlxsw_sp->router->rifs[i] && 7931 mlxsw_sp_rif_dev_is(mlxsw_sp->router->rifs[i], dev)) 7932 return mlxsw_sp->router->rifs[i]; 7933 7934 return NULL; 7935} 7936 7937bool mlxsw_sp_rif_exists(struct mlxsw_sp *mlxsw_sp, 7938 const struct net_device *dev) 7939{ 7940 struct mlxsw_sp_rif *rif; 7941 7942 mutex_lock(&mlxsw_sp->router->lock); 7943 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 7944 mutex_unlock(&mlxsw_sp->router->lock); 7945 7946 return rif; 7947} 7948 7949u16 mlxsw_sp_rif_vid(struct mlxsw_sp *mlxsw_sp, const struct net_device *dev) 7950{ 7951 struct mlxsw_sp_rif *rif; 7952 u16 vid = 0; 7953 7954 mutex_lock(&mlxsw_sp->router->lock); 7955 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 7956 if (!rif) 7957 goto out; 7958 7959 /* We only return the VID for VLAN RIFs. Otherwise we return an 7960 * invalid value (0). 7961 */ 7962 if (rif->ops->type != MLXSW_SP_RIF_TYPE_VLAN) 7963 goto out; 7964 7965 vid = mlxsw_sp_fid_8021q_vid(rif->fid); 7966 7967out: 7968 mutex_unlock(&mlxsw_sp->router->lock); 7969 return vid; 7970} 7971 7972static int mlxsw_sp_router_rif_disable(struct mlxsw_sp *mlxsw_sp, u16 rif) 7973{ 7974 char ritr_pl[MLXSW_REG_RITR_LEN]; 7975 int err; 7976 7977 mlxsw_reg_ritr_rif_pack(ritr_pl, rif); 7978 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 7979 if (err) 7980 return err; 7981 7982 mlxsw_reg_ritr_enable_set(ritr_pl, false); 7983 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 7984} 7985 7986static int mlxsw_sp_router_rif_made_sync(struct mlxsw_sp *mlxsw_sp, 7987 struct mlxsw_sp_rif *rif) 7988{ 7989 int err; 7990 7991 err = mlxsw_sp_neigh_rif_made_sync(mlxsw_sp, rif); 7992 if (err) 7993 return err; 7994 7995 err = mlxsw_sp_nexthop_rif_made_sync(mlxsw_sp, rif); 7996 if (err) 7997 goto err_nexthop; 7998 7999 return 0; 8000 8001err_nexthop: 8002 mlxsw_sp_neigh_rif_gone_sync(mlxsw_sp, rif); 8003 return err; 8004} 8005 8006static void mlxsw_sp_router_rif_gone_sync(struct mlxsw_sp *mlxsw_sp, 8007 struct mlxsw_sp_rif *rif) 8008{ 8009 /* Signal to nexthop cleanup that the RIF is going away. */ 8010 rif->crif->rif = NULL; 8011 8012 mlxsw_sp_router_rif_disable(mlxsw_sp, rif->rif_index); 8013 mlxsw_sp_nexthop_rif_gone_sync(mlxsw_sp, rif); 8014 mlxsw_sp_neigh_rif_gone_sync(mlxsw_sp, rif); 8015} 8016 8017static bool __mlxsw_sp_dev_addr_list_empty(const struct net_device *dev) 8018{ 8019 struct inet6_dev *inet6_dev; 8020 struct in_device *idev; 8021 8022 idev = __in_dev_get_rcu(dev); 8023 if (idev && idev->ifa_list) 8024 return false; 8025 8026 inet6_dev = __in6_dev_get(dev); 8027 if (inet6_dev && !list_empty(&inet6_dev->addr_list)) 8028 return false; 8029 8030 return true; 8031} 8032 8033static bool mlxsw_sp_dev_addr_list_empty(const struct net_device *dev) 8034{ 8035 bool addr_list_empty; 8036 8037 rcu_read_lock(); 8038 addr_list_empty = __mlxsw_sp_dev_addr_list_empty(dev); 8039 rcu_read_unlock(); 8040 8041 return addr_list_empty; 8042} 8043 8044static bool 8045mlxsw_sp_rif_should_config(struct mlxsw_sp_rif *rif, struct net_device *dev, 8046 unsigned long event) 8047{ 8048 bool addr_list_empty; 8049 8050 switch (event) { 8051 case NETDEV_UP: 8052 return rif == NULL; 8053 case NETDEV_DOWN: 8054 addr_list_empty = mlxsw_sp_dev_addr_list_empty(dev); 8055 8056 /* macvlans do not have a RIF, but rather piggy back on the 8057 * RIF of their lower device. 8058 */ 8059 if (netif_is_macvlan(dev) && addr_list_empty) 8060 return true; 8061 8062 if (rif && addr_list_empty && 8063 !netif_is_l3_slave(mlxsw_sp_rif_dev(rif))) 8064 return true; 8065 /* It is possible we already removed the RIF ourselves 8066 * if it was assigned to a netdev that is now a bridge 8067 * or LAG slave. 8068 */ 8069 return false; 8070 } 8071 8072 return false; 8073} 8074 8075static enum mlxsw_sp_rif_type 8076mlxsw_sp_dev_rif_type(const struct mlxsw_sp *mlxsw_sp, 8077 const struct net_device *dev) 8078{ 8079 enum mlxsw_sp_fid_type type; 8080 8081 if (mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, NULL)) 8082 return MLXSW_SP_RIF_TYPE_IPIP_LB; 8083 8084 /* Otherwise RIF type is derived from the type of the underlying FID. */ 8085 if (is_vlan_dev(dev) && netif_is_bridge_master(vlan_dev_real_dev(dev))) 8086 type = MLXSW_SP_FID_TYPE_8021Q; 8087 else if (netif_is_bridge_master(dev) && br_vlan_enabled(dev)) 8088 type = MLXSW_SP_FID_TYPE_8021Q; 8089 else if (netif_is_bridge_master(dev)) 8090 type = MLXSW_SP_FID_TYPE_8021D; 8091 else 8092 type = MLXSW_SP_FID_TYPE_RFID; 8093 8094 return mlxsw_sp_fid_type_rif_type(mlxsw_sp, type); 8095} 8096 8097static int mlxsw_sp_rif_index_alloc(struct mlxsw_sp *mlxsw_sp, u16 *p_rif_index, 8098 u8 rif_entries) 8099{ 8100 *p_rif_index = gen_pool_alloc(mlxsw_sp->router->rifs_table, 8101 rif_entries); 8102 if (*p_rif_index == 0) 8103 return -ENOBUFS; 8104 *p_rif_index -= MLXSW_SP_ROUTER_GENALLOC_OFFSET; 8105 8106 /* RIF indexes must be aligned to the allocation size. */ 8107 WARN_ON_ONCE(*p_rif_index % rif_entries); 8108 8109 return 0; 8110} 8111 8112static void mlxsw_sp_rif_index_free(struct mlxsw_sp *mlxsw_sp, u16 rif_index, 8113 u8 rif_entries) 8114{ 8115 gen_pool_free(mlxsw_sp->router->rifs_table, 8116 MLXSW_SP_ROUTER_GENALLOC_OFFSET + rif_index, rif_entries); 8117} 8118 8119static struct mlxsw_sp_rif *mlxsw_sp_rif_alloc(size_t rif_size, u16 rif_index, 8120 u16 vr_id, 8121 struct mlxsw_sp_crif *crif) 8122{ 8123 struct net_device *l3_dev = crif ? crif->key.dev : NULL; 8124 struct mlxsw_sp_rif *rif; 8125 8126 rif = kzalloc(rif_size, GFP_KERNEL); 8127 if (!rif) 8128 return NULL; 8129 8130 INIT_LIST_HEAD(&rif->neigh_list); 8131 if (l3_dev) { 8132 ether_addr_copy(rif->addr, l3_dev->dev_addr); 8133 rif->mtu = l3_dev->mtu; 8134 } 8135 rif->vr_id = vr_id; 8136 rif->rif_index = rif_index; 8137 if (crif) { 8138 rif->crif = crif; 8139 crif->rif = rif; 8140 } 8141 8142 return rif; 8143} 8144 8145static void mlxsw_sp_rif_free(struct mlxsw_sp_rif *rif) 8146{ 8147 WARN_ON(!list_empty(&rif->neigh_list)); 8148 8149 if (rif->crif) 8150 rif->crif->rif = NULL; 8151 kfree(rif); 8152} 8153 8154struct mlxsw_sp_rif *mlxsw_sp_rif_by_index(const struct mlxsw_sp *mlxsw_sp, 8155 u16 rif_index) 8156{ 8157 return mlxsw_sp->router->rifs[rif_index]; 8158} 8159 8160u16 mlxsw_sp_rif_index(const struct mlxsw_sp_rif *rif) 8161{ 8162 return rif->rif_index; 8163} 8164 8165u16 mlxsw_sp_ipip_lb_rif_index(const struct mlxsw_sp_rif_ipip_lb *lb_rif) 8166{ 8167 return lb_rif->common.rif_index; 8168} 8169 8170u16 mlxsw_sp_ipip_lb_ul_vr_id(const struct mlxsw_sp_rif_ipip_lb *lb_rif) 8171{ 8172 struct net_device *dev = mlxsw_sp_rif_dev(&lb_rif->common); 8173 u32 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(dev); 8174 struct mlxsw_sp_vr *ul_vr; 8175 8176 ul_vr = mlxsw_sp_vr_get(lb_rif->common.mlxsw_sp, ul_tb_id, NULL); 8177 if (WARN_ON(IS_ERR(ul_vr))) 8178 return 0; 8179 8180 return ul_vr->id; 8181} 8182 8183u16 mlxsw_sp_ipip_lb_ul_rif_id(const struct mlxsw_sp_rif_ipip_lb *lb_rif) 8184{ 8185 return lb_rif->ul_rif_id; 8186} 8187 8188static bool 8189mlxsw_sp_router_port_l3_stats_enabled(struct mlxsw_sp_rif *rif) 8190{ 8191 return mlxsw_sp_rif_counter_valid_get(rif, 8192 MLXSW_SP_RIF_COUNTER_EGRESS) && 8193 mlxsw_sp_rif_counter_valid_get(rif, 8194 MLXSW_SP_RIF_COUNTER_INGRESS); 8195} 8196 8197static int 8198mlxsw_sp_router_port_l3_stats_enable(struct mlxsw_sp_rif *rif) 8199{ 8200 int err; 8201 8202 err = mlxsw_sp_rif_counter_alloc(rif, MLXSW_SP_RIF_COUNTER_INGRESS); 8203 if (err) 8204 return err; 8205 8206 /* Clear stale data. */ 8207 err = mlxsw_sp_rif_counter_fetch_clear(rif, 8208 MLXSW_SP_RIF_COUNTER_INGRESS, 8209 NULL); 8210 if (err) 8211 goto err_clear_ingress; 8212 8213 err = mlxsw_sp_rif_counter_alloc(rif, MLXSW_SP_RIF_COUNTER_EGRESS); 8214 if (err) 8215 goto err_alloc_egress; 8216 8217 /* Clear stale data. */ 8218 err = mlxsw_sp_rif_counter_fetch_clear(rif, 8219 MLXSW_SP_RIF_COUNTER_EGRESS, 8220 NULL); 8221 if (err) 8222 goto err_clear_egress; 8223 8224 return 0; 8225 8226err_clear_egress: 8227 mlxsw_sp_rif_counter_free(rif, MLXSW_SP_RIF_COUNTER_EGRESS); 8228err_alloc_egress: 8229err_clear_ingress: 8230 mlxsw_sp_rif_counter_free(rif, MLXSW_SP_RIF_COUNTER_INGRESS); 8231 return err; 8232} 8233 8234static void 8235mlxsw_sp_router_port_l3_stats_disable(struct mlxsw_sp_rif *rif) 8236{ 8237 mlxsw_sp_rif_counter_free(rif, MLXSW_SP_RIF_COUNTER_EGRESS); 8238 mlxsw_sp_rif_counter_free(rif, MLXSW_SP_RIF_COUNTER_INGRESS); 8239} 8240 8241static void 8242mlxsw_sp_router_port_l3_stats_report_used(struct mlxsw_sp_rif *rif, 8243 struct netdev_notifier_offload_xstats_info *info) 8244{ 8245 if (!mlxsw_sp_router_port_l3_stats_enabled(rif)) 8246 return; 8247 netdev_offload_xstats_report_used(info->report_used); 8248} 8249 8250static int 8251mlxsw_sp_router_port_l3_stats_fetch(struct mlxsw_sp_rif *rif, 8252 struct rtnl_hw_stats64 *p_stats) 8253{ 8254 struct mlxsw_sp_rif_counter_set_basic ingress; 8255 struct mlxsw_sp_rif_counter_set_basic egress; 8256 int err; 8257 8258 err = mlxsw_sp_rif_counter_fetch_clear(rif, 8259 MLXSW_SP_RIF_COUNTER_INGRESS, 8260 &ingress); 8261 if (err) 8262 return err; 8263 8264 err = mlxsw_sp_rif_counter_fetch_clear(rif, 8265 MLXSW_SP_RIF_COUNTER_EGRESS, 8266 &egress); 8267 if (err) 8268 return err; 8269 8270#define MLXSW_SP_ROUTER_ALL_GOOD(SET, SFX) \ 8271 ((SET.good_unicast_ ## SFX) + \ 8272 (SET.good_multicast_ ## SFX) + \ 8273 (SET.good_broadcast_ ## SFX)) 8274 8275 p_stats->rx_packets = MLXSW_SP_ROUTER_ALL_GOOD(ingress, packets); 8276 p_stats->tx_packets = MLXSW_SP_ROUTER_ALL_GOOD(egress, packets); 8277 p_stats->rx_bytes = MLXSW_SP_ROUTER_ALL_GOOD(ingress, bytes); 8278 p_stats->tx_bytes = MLXSW_SP_ROUTER_ALL_GOOD(egress, bytes); 8279 p_stats->rx_errors = ingress.error_packets; 8280 p_stats->tx_errors = egress.error_packets; 8281 p_stats->rx_dropped = ingress.discard_packets; 8282 p_stats->tx_dropped = egress.discard_packets; 8283 p_stats->multicast = ingress.good_multicast_packets + 8284 ingress.good_broadcast_packets; 8285 8286#undef MLXSW_SP_ROUTER_ALL_GOOD 8287 8288 return 0; 8289} 8290 8291static int 8292mlxsw_sp_router_port_l3_stats_report_delta(struct mlxsw_sp_rif *rif, 8293 struct netdev_notifier_offload_xstats_info *info) 8294{ 8295 struct rtnl_hw_stats64 stats = {}; 8296 int err; 8297 8298 if (!mlxsw_sp_router_port_l3_stats_enabled(rif)) 8299 return 0; 8300 8301 err = mlxsw_sp_router_port_l3_stats_fetch(rif, &stats); 8302 if (err) 8303 return err; 8304 8305 netdev_offload_xstats_report_delta(info->report_delta, &stats); 8306 return 0; 8307} 8308 8309struct mlxsw_sp_router_hwstats_notify_work { 8310 struct work_struct work; 8311 struct net_device *dev; 8312 netdevice_tracker dev_tracker; 8313}; 8314 8315static void mlxsw_sp_router_hwstats_notify_work(struct work_struct *work) 8316{ 8317 struct mlxsw_sp_router_hwstats_notify_work *hws_work = 8318 container_of(work, struct mlxsw_sp_router_hwstats_notify_work, 8319 work); 8320 8321 rtnl_lock(); 8322 rtnl_offload_xstats_notify(hws_work->dev); 8323 rtnl_unlock(); 8324 netdev_put(hws_work->dev, &hws_work->dev_tracker); 8325 kfree(hws_work); 8326} 8327 8328static void 8329mlxsw_sp_router_hwstats_notify_schedule(struct net_device *dev) 8330{ 8331 struct mlxsw_sp_router_hwstats_notify_work *hws_work; 8332 8333 /* To collect notification payload, the core ends up sending another 8334 * notifier block message, which would deadlock on the attempt to 8335 * acquire the router lock again. Just postpone the notification until 8336 * later. 8337 */ 8338 8339 hws_work = kzalloc(sizeof(*hws_work), GFP_KERNEL); 8340 if (!hws_work) 8341 return; 8342 8343 INIT_WORK(&hws_work->work, mlxsw_sp_router_hwstats_notify_work); 8344 netdev_hold(dev, &hws_work->dev_tracker, GFP_KERNEL); 8345 hws_work->dev = dev; 8346 mlxsw_core_schedule_work(&hws_work->work); 8347} 8348 8349int mlxsw_sp_rif_dev_ifindex(const struct mlxsw_sp_rif *rif) 8350{ 8351 return mlxsw_sp_rif_dev(rif)->ifindex; 8352} 8353 8354bool mlxsw_sp_rif_has_dev(const struct mlxsw_sp_rif *rif) 8355{ 8356 return !!mlxsw_sp_rif_dev(rif); 8357} 8358 8359bool mlxsw_sp_rif_dev_is(const struct mlxsw_sp_rif *rif, 8360 const struct net_device *dev) 8361{ 8362 return mlxsw_sp_rif_dev(rif) == dev; 8363} 8364 8365static void mlxsw_sp_rif_push_l3_stats(struct mlxsw_sp_rif *rif) 8366{ 8367 struct rtnl_hw_stats64 stats = {}; 8368 8369 if (!mlxsw_sp_router_port_l3_stats_fetch(rif, &stats)) 8370 netdev_offload_xstats_push_delta(mlxsw_sp_rif_dev(rif), 8371 NETDEV_OFFLOAD_XSTATS_TYPE_L3, 8372 &stats); 8373} 8374 8375static struct mlxsw_sp_rif * 8376mlxsw_sp_rif_create(struct mlxsw_sp *mlxsw_sp, 8377 const struct mlxsw_sp_rif_params *params, 8378 struct netlink_ext_ack *extack) 8379{ 8380 u8 rif_entries = params->double_entry ? 2 : 1; 8381 u32 tb_id = l3mdev_fib_table(params->dev); 8382 const struct mlxsw_sp_rif_ops *ops; 8383 struct mlxsw_sp_fid *fid = NULL; 8384 enum mlxsw_sp_rif_type type; 8385 struct mlxsw_sp_crif *crif; 8386 struct mlxsw_sp_rif *rif; 8387 struct mlxsw_sp_vr *vr; 8388 u16 rif_index; 8389 int i, err; 8390 8391 type = mlxsw_sp_dev_rif_type(mlxsw_sp, params->dev); 8392 ops = mlxsw_sp->router->rif_ops_arr[type]; 8393 8394 vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id ? : RT_TABLE_MAIN, extack); 8395 if (IS_ERR(vr)) 8396 return ERR_CAST(vr); 8397 vr->rif_count++; 8398 8399 err = mlxsw_sp_rif_index_alloc(mlxsw_sp, &rif_index, rif_entries); 8400 if (err) { 8401 NL_SET_ERR_MSG_MOD(extack, "Exceeded number of supported router interfaces"); 8402 goto err_rif_index_alloc; 8403 } 8404 8405 crif = mlxsw_sp_crif_lookup(mlxsw_sp->router, params->dev); 8406 if (WARN_ON(!crif)) { 8407 err = -ENOENT; 8408 goto err_crif_lookup; 8409 } 8410 8411 rif = mlxsw_sp_rif_alloc(ops->rif_size, rif_index, vr->id, crif); 8412 if (!rif) { 8413 err = -ENOMEM; 8414 goto err_rif_alloc; 8415 } 8416 netdev_hold(params->dev, &rif->dev_tracker, GFP_KERNEL); 8417 mlxsw_sp->router->rifs[rif_index] = rif; 8418 rif->mlxsw_sp = mlxsw_sp; 8419 rif->ops = ops; 8420 rif->rif_entries = rif_entries; 8421 8422 if (ops->fid_get) { 8423 fid = ops->fid_get(rif, params, extack); 8424 if (IS_ERR(fid)) { 8425 err = PTR_ERR(fid); 8426 goto err_fid_get; 8427 } 8428 rif->fid = fid; 8429 } 8430 8431 if (ops->setup) 8432 ops->setup(rif, params); 8433 8434 err = ops->configure(rif, extack); 8435 if (err) 8436 goto err_configure; 8437 8438 for (i = 0; i < MLXSW_SP_L3_PROTO_MAX; i++) { 8439 err = mlxsw_sp_mr_rif_add(vr->mr_table[i], rif); 8440 if (err) 8441 goto err_mr_rif_add; 8442 } 8443 8444 err = mlxsw_sp_router_rif_made_sync(mlxsw_sp, rif); 8445 if (err) 8446 goto err_rif_made_sync; 8447 8448 if (netdev_offload_xstats_enabled(params->dev, 8449 NETDEV_OFFLOAD_XSTATS_TYPE_L3)) { 8450 err = mlxsw_sp_router_port_l3_stats_enable(rif); 8451 if (err) 8452 goto err_stats_enable; 8453 mlxsw_sp_router_hwstats_notify_schedule(params->dev); 8454 } else { 8455 mlxsw_sp_rif_counters_alloc(rif); 8456 } 8457 8458 atomic_add(rif_entries, &mlxsw_sp->router->rifs_count); 8459 return rif; 8460 8461err_stats_enable: 8462 mlxsw_sp_router_rif_gone_sync(mlxsw_sp, rif); 8463err_rif_made_sync: 8464err_mr_rif_add: 8465 for (i--; i >= 0; i--) 8466 mlxsw_sp_mr_rif_del(vr->mr_table[i], rif); 8467 ops->deconfigure(rif); 8468err_configure: 8469 if (fid) 8470 mlxsw_sp_fid_put(fid); 8471err_fid_get: 8472 mlxsw_sp->router->rifs[rif_index] = NULL; 8473 netdev_put(params->dev, &rif->dev_tracker); 8474 mlxsw_sp_rif_free(rif); 8475err_rif_alloc: 8476err_crif_lookup: 8477 mlxsw_sp_rif_index_free(mlxsw_sp, rif_index, rif_entries); 8478err_rif_index_alloc: 8479 vr->rif_count--; 8480 mlxsw_sp_vr_put(mlxsw_sp, vr); 8481 return ERR_PTR(err); 8482} 8483 8484static void mlxsw_sp_rif_destroy(struct mlxsw_sp_rif *rif) 8485{ 8486 struct net_device *dev = mlxsw_sp_rif_dev(rif); 8487 const struct mlxsw_sp_rif_ops *ops = rif->ops; 8488 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 8489 struct mlxsw_sp_crif *crif = rif->crif; 8490 struct mlxsw_sp_fid *fid = rif->fid; 8491 u8 rif_entries = rif->rif_entries; 8492 u16 rif_index = rif->rif_index; 8493 struct mlxsw_sp_vr *vr; 8494 int i; 8495 8496 atomic_sub(rif_entries, &mlxsw_sp->router->rifs_count); 8497 mlxsw_sp_router_rif_gone_sync(mlxsw_sp, rif); 8498 vr = &mlxsw_sp->router->vrs[rif->vr_id]; 8499 8500 if (netdev_offload_xstats_enabled(dev, NETDEV_OFFLOAD_XSTATS_TYPE_L3)) { 8501 mlxsw_sp_rif_push_l3_stats(rif); 8502 mlxsw_sp_router_port_l3_stats_disable(rif); 8503 mlxsw_sp_router_hwstats_notify_schedule(dev); 8504 } else { 8505 mlxsw_sp_rif_counters_free(rif); 8506 } 8507 8508 for (i = 0; i < MLXSW_SP_L3_PROTO_MAX; i++) 8509 mlxsw_sp_mr_rif_del(vr->mr_table[i], rif); 8510 ops->deconfigure(rif); 8511 if (fid) 8512 /* Loopback RIFs are not associated with a FID. */ 8513 mlxsw_sp_fid_put(fid); 8514 mlxsw_sp->router->rifs[rif->rif_index] = NULL; 8515 netdev_put(dev, &rif->dev_tracker); 8516 mlxsw_sp_rif_free(rif); 8517 mlxsw_sp_rif_index_free(mlxsw_sp, rif_index, rif_entries); 8518 vr->rif_count--; 8519 mlxsw_sp_vr_put(mlxsw_sp, vr); 8520 8521 if (crif->can_destroy) 8522 mlxsw_sp_crif_free(crif); 8523} 8524 8525void mlxsw_sp_rif_destroy_by_dev(struct mlxsw_sp *mlxsw_sp, 8526 struct net_device *dev) 8527{ 8528 struct mlxsw_sp_rif *rif; 8529 8530 mutex_lock(&mlxsw_sp->router->lock); 8531 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 8532 if (!rif) 8533 goto out; 8534 mlxsw_sp_rif_destroy(rif); 8535out: 8536 mutex_unlock(&mlxsw_sp->router->lock); 8537} 8538 8539static void mlxsw_sp_rif_destroy_vlan_upper(struct mlxsw_sp *mlxsw_sp, 8540 struct net_device *br_dev, 8541 u16 vid) 8542{ 8543 struct net_device *upper_dev; 8544 struct mlxsw_sp_crif *crif; 8545 8546 rcu_read_lock(); 8547 upper_dev = __vlan_find_dev_deep_rcu(br_dev, htons(ETH_P_8021Q), vid); 8548 rcu_read_unlock(); 8549 8550 if (!upper_dev) 8551 return; 8552 8553 crif = mlxsw_sp_crif_lookup(mlxsw_sp->router, upper_dev); 8554 if (!crif || !crif->rif) 8555 return; 8556 8557 mlxsw_sp_rif_destroy(crif->rif); 8558} 8559 8560static int mlxsw_sp_inetaddr_bridge_event(struct mlxsw_sp *mlxsw_sp, 8561 struct net_device *l3_dev, 8562 int lower_pvid, 8563 unsigned long event, 8564 struct netlink_ext_ack *extack); 8565 8566int mlxsw_sp_router_bridge_vlan_add(struct mlxsw_sp *mlxsw_sp, 8567 struct net_device *br_dev, 8568 u16 new_vid, bool is_pvid, 8569 struct netlink_ext_ack *extack) 8570{ 8571 struct mlxsw_sp_rif *old_rif; 8572 struct mlxsw_sp_rif *new_rif; 8573 struct net_device *upper_dev; 8574 u16 old_pvid = 0; 8575 u16 new_pvid; 8576 int err = 0; 8577 8578 mutex_lock(&mlxsw_sp->router->lock); 8579 old_rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, br_dev); 8580 if (old_rif) { 8581 /* If the RIF on the bridge is not a VLAN RIF, we shouldn't have 8582 * gotten a PVID notification. 8583 */ 8584 if (WARN_ON(old_rif->ops->type != MLXSW_SP_RIF_TYPE_VLAN)) 8585 old_rif = NULL; 8586 else 8587 old_pvid = mlxsw_sp_fid_8021q_vid(old_rif->fid); 8588 } 8589 8590 if (is_pvid) 8591 new_pvid = new_vid; 8592 else if (old_pvid == new_vid) 8593 new_pvid = 0; 8594 else 8595 goto out; 8596 8597 if (old_pvid == new_pvid) 8598 goto out; 8599 8600 if (new_pvid) { 8601 struct mlxsw_sp_rif_params params = { 8602 .dev = br_dev, 8603 .vid = new_pvid, 8604 }; 8605 8606 /* If there is a VLAN upper with the same VID as the new PVID, 8607 * kill its RIF, if there is one. 8608 */ 8609 mlxsw_sp_rif_destroy_vlan_upper(mlxsw_sp, br_dev, new_pvid); 8610 8611 if (mlxsw_sp_dev_addr_list_empty(br_dev)) 8612 goto out; 8613 new_rif = mlxsw_sp_rif_create(mlxsw_sp, ¶ms, extack); 8614 if (IS_ERR(new_rif)) { 8615 err = PTR_ERR(new_rif); 8616 goto out; 8617 } 8618 8619 if (old_pvid) 8620 mlxsw_sp_rif_migrate_destroy(mlxsw_sp, old_rif, new_rif, 8621 true); 8622 } else { 8623 mlxsw_sp_rif_destroy(old_rif); 8624 } 8625 8626 if (old_pvid) { 8627 rcu_read_lock(); 8628 upper_dev = __vlan_find_dev_deep_rcu(br_dev, htons(ETH_P_8021Q), 8629 old_pvid); 8630 rcu_read_unlock(); 8631 if (upper_dev) 8632 err = mlxsw_sp_inetaddr_bridge_event(mlxsw_sp, 8633 upper_dev, 8634 new_pvid, 8635 NETDEV_UP, extack); 8636 } 8637 8638out: 8639 mutex_unlock(&mlxsw_sp->router->lock); 8640 return err; 8641} 8642 8643static void 8644mlxsw_sp_rif_subport_params_init(struct mlxsw_sp_rif_params *params, 8645 struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan) 8646{ 8647 struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port; 8648 8649 params->vid = mlxsw_sp_port_vlan->vid; 8650 params->lag = mlxsw_sp_port->lagged; 8651 if (params->lag) 8652 params->lag_id = mlxsw_sp_port->lag_id; 8653 else 8654 params->system_port = mlxsw_sp_port->local_port; 8655} 8656 8657static struct mlxsw_sp_rif_subport * 8658mlxsw_sp_rif_subport_rif(const struct mlxsw_sp_rif *rif) 8659{ 8660 return container_of(rif, struct mlxsw_sp_rif_subport, common); 8661} 8662 8663static struct mlxsw_sp_rif * 8664mlxsw_sp_rif_subport_get(struct mlxsw_sp *mlxsw_sp, 8665 const struct mlxsw_sp_rif_params *params, 8666 struct netlink_ext_ack *extack) 8667{ 8668 struct mlxsw_sp_rif_subport *rif_subport; 8669 struct mlxsw_sp_rif *rif; 8670 8671 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, params->dev); 8672 if (!rif) 8673 return mlxsw_sp_rif_create(mlxsw_sp, params, extack); 8674 8675 rif_subport = mlxsw_sp_rif_subport_rif(rif); 8676 refcount_inc(&rif_subport->ref_count); 8677 return rif; 8678} 8679 8680static void mlxsw_sp_rif_subport_put(struct mlxsw_sp_rif *rif) 8681{ 8682 struct mlxsw_sp_rif_subport *rif_subport; 8683 8684 rif_subport = mlxsw_sp_rif_subport_rif(rif); 8685 if (!refcount_dec_and_test(&rif_subport->ref_count)) 8686 return; 8687 8688 mlxsw_sp_rif_destroy(rif); 8689} 8690 8691static int mlxsw_sp_rif_mac_profile_index_alloc(struct mlxsw_sp *mlxsw_sp, 8692 struct mlxsw_sp_rif_mac_profile *profile, 8693 struct netlink_ext_ack *extack) 8694{ 8695 u8 max_rif_mac_profiles = mlxsw_sp->router->max_rif_mac_profile; 8696 struct mlxsw_sp_router *router = mlxsw_sp->router; 8697 int id; 8698 8699 id = idr_alloc(&router->rif_mac_profiles_idr, profile, 0, 8700 max_rif_mac_profiles, GFP_KERNEL); 8701 8702 if (id >= 0) { 8703 profile->id = id; 8704 return 0; 8705 } 8706 8707 if (id == -ENOSPC) 8708 NL_SET_ERR_MSG_MOD(extack, 8709 "Exceeded number of supported router interface MAC profiles"); 8710 8711 return id; 8712} 8713 8714static struct mlxsw_sp_rif_mac_profile * 8715mlxsw_sp_rif_mac_profile_index_free(struct mlxsw_sp *mlxsw_sp, u8 mac_profile) 8716{ 8717 struct mlxsw_sp_rif_mac_profile *profile; 8718 8719 profile = idr_remove(&mlxsw_sp->router->rif_mac_profiles_idr, 8720 mac_profile); 8721 WARN_ON(!profile); 8722 return profile; 8723} 8724 8725static struct mlxsw_sp_rif_mac_profile * 8726mlxsw_sp_rif_mac_profile_alloc(const char *mac) 8727{ 8728 struct mlxsw_sp_rif_mac_profile *profile; 8729 8730 profile = kzalloc(sizeof(*profile), GFP_KERNEL); 8731 if (!profile) 8732 return NULL; 8733 8734 ether_addr_copy(profile->mac_prefix, mac); 8735 refcount_set(&profile->ref_count, 1); 8736 return profile; 8737} 8738 8739static struct mlxsw_sp_rif_mac_profile * 8740mlxsw_sp_rif_mac_profile_find(const struct mlxsw_sp *mlxsw_sp, const char *mac) 8741{ 8742 struct mlxsw_sp_router *router = mlxsw_sp->router; 8743 struct mlxsw_sp_rif_mac_profile *profile; 8744 int id; 8745 8746 idr_for_each_entry(&router->rif_mac_profiles_idr, profile, id) { 8747 if (ether_addr_equal_masked(profile->mac_prefix, mac, 8748 mlxsw_sp->mac_mask)) 8749 return profile; 8750 } 8751 8752 return NULL; 8753} 8754 8755static u64 mlxsw_sp_rif_mac_profiles_occ_get(void *priv) 8756{ 8757 const struct mlxsw_sp *mlxsw_sp = priv; 8758 8759 return atomic_read(&mlxsw_sp->router->rif_mac_profiles_count); 8760} 8761 8762static u64 mlxsw_sp_rifs_occ_get(void *priv) 8763{ 8764 const struct mlxsw_sp *mlxsw_sp = priv; 8765 8766 return atomic_read(&mlxsw_sp->router->rifs_count); 8767} 8768 8769static struct mlxsw_sp_rif_mac_profile * 8770mlxsw_sp_rif_mac_profile_create(struct mlxsw_sp *mlxsw_sp, const char *mac, 8771 struct netlink_ext_ack *extack) 8772{ 8773 struct mlxsw_sp_rif_mac_profile *profile; 8774 int err; 8775 8776 profile = mlxsw_sp_rif_mac_profile_alloc(mac); 8777 if (!profile) 8778 return ERR_PTR(-ENOMEM); 8779 8780 err = mlxsw_sp_rif_mac_profile_index_alloc(mlxsw_sp, profile, extack); 8781 if (err) 8782 goto profile_index_alloc_err; 8783 8784 atomic_inc(&mlxsw_sp->router->rif_mac_profiles_count); 8785 return profile; 8786 8787profile_index_alloc_err: 8788 kfree(profile); 8789 return ERR_PTR(err); 8790} 8791 8792static void mlxsw_sp_rif_mac_profile_destroy(struct mlxsw_sp *mlxsw_sp, 8793 u8 mac_profile) 8794{ 8795 struct mlxsw_sp_rif_mac_profile *profile; 8796 8797 atomic_dec(&mlxsw_sp->router->rif_mac_profiles_count); 8798 profile = mlxsw_sp_rif_mac_profile_index_free(mlxsw_sp, mac_profile); 8799 kfree(profile); 8800} 8801 8802static int mlxsw_sp_rif_mac_profile_get(struct mlxsw_sp *mlxsw_sp, 8803 const char *mac, u8 *p_mac_profile, 8804 struct netlink_ext_ack *extack) 8805{ 8806 struct mlxsw_sp_rif_mac_profile *profile; 8807 8808 profile = mlxsw_sp_rif_mac_profile_find(mlxsw_sp, mac); 8809 if (profile) { 8810 refcount_inc(&profile->ref_count); 8811 goto out; 8812 } 8813 8814 profile = mlxsw_sp_rif_mac_profile_create(mlxsw_sp, mac, extack); 8815 if (IS_ERR(profile)) 8816 return PTR_ERR(profile); 8817 8818out: 8819 *p_mac_profile = profile->id; 8820 return 0; 8821} 8822 8823static void mlxsw_sp_rif_mac_profile_put(struct mlxsw_sp *mlxsw_sp, 8824 u8 mac_profile) 8825{ 8826 struct mlxsw_sp_rif_mac_profile *profile; 8827 8828 profile = idr_find(&mlxsw_sp->router->rif_mac_profiles_idr, 8829 mac_profile); 8830 if (WARN_ON(!profile)) 8831 return; 8832 8833 if (!refcount_dec_and_test(&profile->ref_count)) 8834 return; 8835 8836 mlxsw_sp_rif_mac_profile_destroy(mlxsw_sp, mac_profile); 8837} 8838 8839static bool mlxsw_sp_rif_mac_profile_is_shared(const struct mlxsw_sp_rif *rif) 8840{ 8841 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 8842 struct mlxsw_sp_rif_mac_profile *profile; 8843 8844 profile = idr_find(&mlxsw_sp->router->rif_mac_profiles_idr, 8845 rif->mac_profile_id); 8846 if (WARN_ON(!profile)) 8847 return false; 8848 8849 return refcount_read(&profile->ref_count) > 1; 8850} 8851 8852static int mlxsw_sp_rif_mac_profile_edit(struct mlxsw_sp_rif *rif, 8853 const char *new_mac) 8854{ 8855 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 8856 struct mlxsw_sp_rif_mac_profile *profile; 8857 8858 profile = idr_find(&mlxsw_sp->router->rif_mac_profiles_idr, 8859 rif->mac_profile_id); 8860 if (WARN_ON(!profile)) 8861 return -EINVAL; 8862 8863 ether_addr_copy(profile->mac_prefix, new_mac); 8864 return 0; 8865} 8866 8867static int 8868mlxsw_sp_rif_mac_profile_replace(struct mlxsw_sp *mlxsw_sp, 8869 struct mlxsw_sp_rif *rif, 8870 const char *new_mac, 8871 struct netlink_ext_ack *extack) 8872{ 8873 u8 mac_profile; 8874 int err; 8875 8876 if (!mlxsw_sp_rif_mac_profile_is_shared(rif) && 8877 !mlxsw_sp_rif_mac_profile_find(mlxsw_sp, new_mac)) 8878 return mlxsw_sp_rif_mac_profile_edit(rif, new_mac); 8879 8880 err = mlxsw_sp_rif_mac_profile_get(mlxsw_sp, new_mac, 8881 &mac_profile, extack); 8882 if (err) 8883 return err; 8884 8885 mlxsw_sp_rif_mac_profile_put(mlxsw_sp, rif->mac_profile_id); 8886 rif->mac_profile_id = mac_profile; 8887 return 0; 8888} 8889 8890static int 8891__mlxsw_sp_port_vlan_router_join(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan, 8892 struct net_device *l3_dev, 8893 struct netlink_ext_ack *extack) 8894{ 8895 struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port; 8896 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp; 8897 struct mlxsw_sp_rif_params params; 8898 u16 vid = mlxsw_sp_port_vlan->vid; 8899 struct mlxsw_sp_rif *rif; 8900 struct mlxsw_sp_fid *fid; 8901 int err; 8902 8903 params = (struct mlxsw_sp_rif_params) { 8904 .dev = l3_dev, 8905 .vid = vid, 8906 }; 8907 8908 mlxsw_sp_rif_subport_params_init(¶ms, mlxsw_sp_port_vlan); 8909 rif = mlxsw_sp_rif_subport_get(mlxsw_sp, ¶ms, extack); 8910 if (IS_ERR(rif)) 8911 return PTR_ERR(rif); 8912 8913 /* FID was already created, just take a reference */ 8914 fid = rif->ops->fid_get(rif, ¶ms, extack); 8915 err = mlxsw_sp_fid_port_vid_map(fid, mlxsw_sp_port, vid); 8916 if (err) 8917 goto err_fid_port_vid_map; 8918 8919 err = mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, false); 8920 if (err) 8921 goto err_port_vid_learning_set; 8922 8923 err = mlxsw_sp_port_vid_stp_set(mlxsw_sp_port, vid, 8924 BR_STATE_FORWARDING); 8925 if (err) 8926 goto err_port_vid_stp_set; 8927 8928 mlxsw_sp_port_vlan->fid = fid; 8929 8930 return 0; 8931 8932err_port_vid_stp_set: 8933 mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, true); 8934err_port_vid_learning_set: 8935 mlxsw_sp_fid_port_vid_unmap(fid, mlxsw_sp_port, vid); 8936err_fid_port_vid_map: 8937 mlxsw_sp_fid_put(fid); 8938 mlxsw_sp_rif_subport_put(rif); 8939 return err; 8940} 8941 8942static void 8943__mlxsw_sp_port_vlan_router_leave(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan) 8944{ 8945 struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port; 8946 struct mlxsw_sp_fid *fid = mlxsw_sp_port_vlan->fid; 8947 struct mlxsw_sp_rif *rif = mlxsw_sp_fid_rif(fid); 8948 u16 vid = mlxsw_sp_port_vlan->vid; 8949 8950 if (WARN_ON(mlxsw_sp_fid_type(fid) != MLXSW_SP_FID_TYPE_RFID)) 8951 return; 8952 8953 mlxsw_sp_port_vlan->fid = NULL; 8954 mlxsw_sp_port_vid_stp_set(mlxsw_sp_port, vid, BR_STATE_BLOCKING); 8955 mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, true); 8956 mlxsw_sp_fid_port_vid_unmap(fid, mlxsw_sp_port, vid); 8957 mlxsw_sp_fid_put(fid); 8958 mlxsw_sp_rif_subport_put(rif); 8959} 8960 8961static int 8962mlxsw_sp_port_vlan_router_join_existing(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan, 8963 struct net_device *l3_dev, 8964 struct netlink_ext_ack *extack) 8965{ 8966 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port_vlan->mlxsw_sp_port->mlxsw_sp; 8967 8968 lockdep_assert_held(&mlxsw_sp->router->lock); 8969 8970 if (!mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev)) 8971 return 0; 8972 8973 return __mlxsw_sp_port_vlan_router_join(mlxsw_sp_port_vlan, l3_dev, 8974 extack); 8975} 8976 8977void 8978mlxsw_sp_port_vlan_router_leave(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan) 8979{ 8980 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port_vlan->mlxsw_sp_port->mlxsw_sp; 8981 8982 mutex_lock(&mlxsw_sp->router->lock); 8983 __mlxsw_sp_port_vlan_router_leave(mlxsw_sp_port_vlan); 8984 mutex_unlock(&mlxsw_sp->router->lock); 8985} 8986 8987static int mlxsw_sp_inetaddr_port_vlan_event(struct net_device *l3_dev, 8988 struct net_device *port_dev, 8989 unsigned long event, u16 vid, 8990 struct netlink_ext_ack *extack) 8991{ 8992 struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(port_dev); 8993 struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan; 8994 8995 mlxsw_sp_port_vlan = mlxsw_sp_port_vlan_find_by_vid(mlxsw_sp_port, vid); 8996 if (WARN_ON(!mlxsw_sp_port_vlan)) 8997 return -EINVAL; 8998 8999 switch (event) { 9000 case NETDEV_UP: 9001 return __mlxsw_sp_port_vlan_router_join(mlxsw_sp_port_vlan, 9002 l3_dev, extack); 9003 case NETDEV_DOWN: 9004 __mlxsw_sp_port_vlan_router_leave(mlxsw_sp_port_vlan); 9005 break; 9006 } 9007 9008 return 0; 9009} 9010 9011static int mlxsw_sp_inetaddr_port_event(struct net_device *port_dev, 9012 unsigned long event, bool nomaster, 9013 struct netlink_ext_ack *extack) 9014{ 9015 if (!nomaster && (netif_is_any_bridge_port(port_dev) || 9016 netif_is_lag_port(port_dev))) 9017 return 0; 9018 9019 return mlxsw_sp_inetaddr_port_vlan_event(port_dev, port_dev, event, 9020 MLXSW_SP_DEFAULT_VID, extack); 9021} 9022 9023static int __mlxsw_sp_inetaddr_lag_event(struct net_device *l3_dev, 9024 struct net_device *lag_dev, 9025 unsigned long event, u16 vid, 9026 struct netlink_ext_ack *extack) 9027{ 9028 struct net_device *port_dev; 9029 struct list_head *iter; 9030 int err; 9031 9032 netdev_for_each_lower_dev(lag_dev, port_dev, iter) { 9033 if (mlxsw_sp_port_dev_check(port_dev)) { 9034 err = mlxsw_sp_inetaddr_port_vlan_event(l3_dev, 9035 port_dev, 9036 event, vid, 9037 extack); 9038 if (err) 9039 return err; 9040 } 9041 } 9042 9043 return 0; 9044} 9045 9046static int mlxsw_sp_inetaddr_lag_event(struct net_device *lag_dev, 9047 unsigned long event, bool nomaster, 9048 struct netlink_ext_ack *extack) 9049{ 9050 if (!nomaster && netif_is_bridge_port(lag_dev)) 9051 return 0; 9052 9053 return __mlxsw_sp_inetaddr_lag_event(lag_dev, lag_dev, event, 9054 MLXSW_SP_DEFAULT_VID, extack); 9055} 9056 9057static int mlxsw_sp_inetaddr_bridge_event(struct mlxsw_sp *mlxsw_sp, 9058 struct net_device *l3_dev, 9059 int lower_pvid, 9060 unsigned long event, 9061 struct netlink_ext_ack *extack) 9062{ 9063 struct mlxsw_sp_rif_params params = { 9064 .dev = l3_dev, 9065 }; 9066 struct mlxsw_sp_rif *rif; 9067 int err; 9068 9069 switch (event) { 9070 case NETDEV_UP: 9071 if (netif_is_bridge_master(l3_dev) && br_vlan_enabled(l3_dev)) { 9072 u16 proto; 9073 9074 br_vlan_get_proto(l3_dev, &proto); 9075 if (proto == ETH_P_8021AD) { 9076 NL_SET_ERR_MSG_MOD(extack, "Adding an IP address to 802.1ad bridge is not supported"); 9077 return -EOPNOTSUPP; 9078 } 9079 err = br_vlan_get_pvid(l3_dev, ¶ms.vid); 9080 if (err) 9081 return err; 9082 if (!params.vid) 9083 return 0; 9084 } else if (is_vlan_dev(l3_dev)) { 9085 params.vid = vlan_dev_vlan_id(l3_dev); 9086 9087 /* If the VID matches PVID of the bridge below, the 9088 * bridge owns the RIF for this VLAN. Don't do anything. 9089 */ 9090 if ((int)params.vid == lower_pvid) 9091 return 0; 9092 } 9093 9094 rif = mlxsw_sp_rif_create(mlxsw_sp, ¶ms, extack); 9095 if (IS_ERR(rif)) 9096 return PTR_ERR(rif); 9097 break; 9098 case NETDEV_DOWN: 9099 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev); 9100 mlxsw_sp_rif_destroy(rif); 9101 break; 9102 } 9103 9104 return 0; 9105} 9106 9107static int mlxsw_sp_inetaddr_vlan_event(struct mlxsw_sp *mlxsw_sp, 9108 struct net_device *vlan_dev, 9109 unsigned long event, bool nomaster, 9110 struct netlink_ext_ack *extack) 9111{ 9112 struct net_device *real_dev = vlan_dev_real_dev(vlan_dev); 9113 u16 vid = vlan_dev_vlan_id(vlan_dev); 9114 u16 lower_pvid; 9115 int err; 9116 9117 if (!nomaster && netif_is_bridge_port(vlan_dev)) 9118 return 0; 9119 9120 if (mlxsw_sp_port_dev_check(real_dev)) { 9121 return mlxsw_sp_inetaddr_port_vlan_event(vlan_dev, real_dev, 9122 event, vid, extack); 9123 } else if (netif_is_lag_master(real_dev)) { 9124 return __mlxsw_sp_inetaddr_lag_event(vlan_dev, real_dev, event, 9125 vid, extack); 9126 } else if (netif_is_bridge_master(real_dev) && 9127 br_vlan_enabled(real_dev)) { 9128 err = br_vlan_get_pvid(real_dev, &lower_pvid); 9129 if (err) 9130 return err; 9131 return mlxsw_sp_inetaddr_bridge_event(mlxsw_sp, vlan_dev, 9132 lower_pvid, event, 9133 extack); 9134 } 9135 9136 return 0; 9137} 9138 9139static bool mlxsw_sp_rif_macvlan_is_vrrp4(const u8 *mac) 9140{ 9141 u8 vrrp4[ETH_ALEN] = { 0x00, 0x00, 0x5e, 0x00, 0x01, 0x00 }; 9142 u8 mask[ETH_ALEN] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0x00 }; 9143 9144 return ether_addr_equal_masked(mac, vrrp4, mask); 9145} 9146 9147static bool mlxsw_sp_rif_macvlan_is_vrrp6(const u8 *mac) 9148{ 9149 u8 vrrp6[ETH_ALEN] = { 0x00, 0x00, 0x5e, 0x00, 0x02, 0x00 }; 9150 u8 mask[ETH_ALEN] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0x00 }; 9151 9152 return ether_addr_equal_masked(mac, vrrp6, mask); 9153} 9154 9155static int mlxsw_sp_rif_vrrp_op(struct mlxsw_sp *mlxsw_sp, u16 rif_index, 9156 const u8 *mac, bool adding) 9157{ 9158 char ritr_pl[MLXSW_REG_RITR_LEN]; 9159 u8 vrrp_id = adding ? mac[5] : 0; 9160 int err; 9161 9162 if (!mlxsw_sp_rif_macvlan_is_vrrp4(mac) && 9163 !mlxsw_sp_rif_macvlan_is_vrrp6(mac)) 9164 return 0; 9165 9166 mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index); 9167 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 9168 if (err) 9169 return err; 9170 9171 if (mlxsw_sp_rif_macvlan_is_vrrp4(mac)) 9172 mlxsw_reg_ritr_if_vrrp_id_ipv4_set(ritr_pl, vrrp_id); 9173 else 9174 mlxsw_reg_ritr_if_vrrp_id_ipv6_set(ritr_pl, vrrp_id); 9175 9176 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 9177} 9178 9179static int mlxsw_sp_rif_macvlan_add(struct mlxsw_sp *mlxsw_sp, 9180 const struct net_device *macvlan_dev, 9181 struct netlink_ext_ack *extack) 9182{ 9183 struct macvlan_dev *vlan = netdev_priv(macvlan_dev); 9184 struct mlxsw_sp_rif *rif; 9185 int err; 9186 9187 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, vlan->lowerdev); 9188 if (!rif) 9189 return 0; 9190 9191 err = mlxsw_sp_rif_fdb_op(mlxsw_sp, macvlan_dev->dev_addr, 9192 mlxsw_sp_fid_index(rif->fid), true); 9193 if (err) 9194 return err; 9195 9196 err = mlxsw_sp_rif_vrrp_op(mlxsw_sp, rif->rif_index, 9197 macvlan_dev->dev_addr, true); 9198 if (err) 9199 goto err_rif_vrrp_add; 9200 9201 /* Make sure the bridge driver does not have this MAC pointing at 9202 * some other port. 9203 */ 9204 if (rif->ops->fdb_del) 9205 rif->ops->fdb_del(rif, macvlan_dev->dev_addr); 9206 9207 return 0; 9208 9209err_rif_vrrp_add: 9210 mlxsw_sp_rif_fdb_op(mlxsw_sp, macvlan_dev->dev_addr, 9211 mlxsw_sp_fid_index(rif->fid), false); 9212 return err; 9213} 9214 9215static void __mlxsw_sp_rif_macvlan_del(struct mlxsw_sp *mlxsw_sp, 9216 const struct net_device *macvlan_dev) 9217{ 9218 struct macvlan_dev *vlan = netdev_priv(macvlan_dev); 9219 struct mlxsw_sp_rif *rif; 9220 9221 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, vlan->lowerdev); 9222 /* If we do not have a RIF, then we already took care of 9223 * removing the macvlan's MAC during RIF deletion. 9224 */ 9225 if (!rif) 9226 return; 9227 mlxsw_sp_rif_vrrp_op(mlxsw_sp, rif->rif_index, macvlan_dev->dev_addr, 9228 false); 9229 mlxsw_sp_rif_fdb_op(mlxsw_sp, macvlan_dev->dev_addr, 9230 mlxsw_sp_fid_index(rif->fid), false); 9231} 9232 9233void mlxsw_sp_rif_macvlan_del(struct mlxsw_sp *mlxsw_sp, 9234 const struct net_device *macvlan_dev) 9235{ 9236 mutex_lock(&mlxsw_sp->router->lock); 9237 __mlxsw_sp_rif_macvlan_del(mlxsw_sp, macvlan_dev); 9238 mutex_unlock(&mlxsw_sp->router->lock); 9239} 9240 9241static int mlxsw_sp_inetaddr_macvlan_event(struct mlxsw_sp *mlxsw_sp, 9242 struct net_device *macvlan_dev, 9243 unsigned long event, 9244 struct netlink_ext_ack *extack) 9245{ 9246 switch (event) { 9247 case NETDEV_UP: 9248 return mlxsw_sp_rif_macvlan_add(mlxsw_sp, macvlan_dev, extack); 9249 case NETDEV_DOWN: 9250 __mlxsw_sp_rif_macvlan_del(mlxsw_sp, macvlan_dev); 9251 break; 9252 } 9253 9254 return 0; 9255} 9256 9257static int __mlxsw_sp_inetaddr_event(struct mlxsw_sp *mlxsw_sp, 9258 struct net_device *dev, 9259 unsigned long event, bool nomaster, 9260 struct netlink_ext_ack *extack) 9261{ 9262 if (mlxsw_sp_port_dev_check(dev)) 9263 return mlxsw_sp_inetaddr_port_event(dev, event, nomaster, 9264 extack); 9265 else if (netif_is_lag_master(dev)) 9266 return mlxsw_sp_inetaddr_lag_event(dev, event, nomaster, 9267 extack); 9268 else if (netif_is_bridge_master(dev)) 9269 return mlxsw_sp_inetaddr_bridge_event(mlxsw_sp, dev, -1, event, 9270 extack); 9271 else if (is_vlan_dev(dev)) 9272 return mlxsw_sp_inetaddr_vlan_event(mlxsw_sp, dev, event, 9273 nomaster, extack); 9274 else if (netif_is_macvlan(dev)) 9275 return mlxsw_sp_inetaddr_macvlan_event(mlxsw_sp, dev, event, 9276 extack); 9277 else 9278 return 0; 9279} 9280 9281static int mlxsw_sp_inetaddr_event(struct notifier_block *nb, 9282 unsigned long event, void *ptr) 9283{ 9284 struct in_ifaddr *ifa = (struct in_ifaddr *) ptr; 9285 struct net_device *dev = ifa->ifa_dev->dev; 9286 struct mlxsw_sp_router *router; 9287 struct mlxsw_sp_rif *rif; 9288 int err = 0; 9289 9290 /* NETDEV_UP event is handled by mlxsw_sp_inetaddr_valid_event */ 9291 if (event == NETDEV_UP) 9292 return NOTIFY_DONE; 9293 9294 router = container_of(nb, struct mlxsw_sp_router, inetaddr_nb); 9295 mutex_lock(&router->lock); 9296 rif = mlxsw_sp_rif_find_by_dev(router->mlxsw_sp, dev); 9297 if (!mlxsw_sp_rif_should_config(rif, dev, event)) 9298 goto out; 9299 9300 err = __mlxsw_sp_inetaddr_event(router->mlxsw_sp, dev, event, false, 9301 NULL); 9302out: 9303 mutex_unlock(&router->lock); 9304 return notifier_from_errno(err); 9305} 9306 9307static int mlxsw_sp_inetaddr_valid_event(struct notifier_block *unused, 9308 unsigned long event, void *ptr) 9309{ 9310 struct in_validator_info *ivi = (struct in_validator_info *) ptr; 9311 struct net_device *dev = ivi->ivi_dev->dev; 9312 struct mlxsw_sp *mlxsw_sp; 9313 struct mlxsw_sp_rif *rif; 9314 int err = 0; 9315 9316 mlxsw_sp = mlxsw_sp_lower_get(dev); 9317 if (!mlxsw_sp) 9318 return NOTIFY_DONE; 9319 9320 mutex_lock(&mlxsw_sp->router->lock); 9321 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 9322 if (!mlxsw_sp_rif_should_config(rif, dev, event)) 9323 goto out; 9324 9325 err = __mlxsw_sp_inetaddr_event(mlxsw_sp, dev, event, false, 9326 ivi->extack); 9327out: 9328 mutex_unlock(&mlxsw_sp->router->lock); 9329 return notifier_from_errno(err); 9330} 9331 9332struct mlxsw_sp_inet6addr_event_work { 9333 struct work_struct work; 9334 struct mlxsw_sp *mlxsw_sp; 9335 struct net_device *dev; 9336 netdevice_tracker dev_tracker; 9337 unsigned long event; 9338}; 9339 9340static void mlxsw_sp_inet6addr_event_work(struct work_struct *work) 9341{ 9342 struct mlxsw_sp_inet6addr_event_work *inet6addr_work = 9343 container_of(work, struct mlxsw_sp_inet6addr_event_work, work); 9344 struct mlxsw_sp *mlxsw_sp = inet6addr_work->mlxsw_sp; 9345 struct net_device *dev = inet6addr_work->dev; 9346 unsigned long event = inet6addr_work->event; 9347 struct mlxsw_sp_rif *rif; 9348 9349 rtnl_lock(); 9350 mutex_lock(&mlxsw_sp->router->lock); 9351 9352 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 9353 if (!mlxsw_sp_rif_should_config(rif, dev, event)) 9354 goto out; 9355 9356 __mlxsw_sp_inetaddr_event(mlxsw_sp, dev, event, false, NULL); 9357out: 9358 mutex_unlock(&mlxsw_sp->router->lock); 9359 rtnl_unlock(); 9360 netdev_put(dev, &inet6addr_work->dev_tracker); 9361 kfree(inet6addr_work); 9362} 9363 9364/* Called with rcu_read_lock() */ 9365static int mlxsw_sp_inet6addr_event(struct notifier_block *nb, 9366 unsigned long event, void *ptr) 9367{ 9368 struct inet6_ifaddr *if6 = (struct inet6_ifaddr *) ptr; 9369 struct mlxsw_sp_inet6addr_event_work *inet6addr_work; 9370 struct net_device *dev = if6->idev->dev; 9371 struct mlxsw_sp_router *router; 9372 9373 /* NETDEV_UP event is handled by mlxsw_sp_inet6addr_valid_event */ 9374 if (event == NETDEV_UP) 9375 return NOTIFY_DONE; 9376 9377 inet6addr_work = kzalloc(sizeof(*inet6addr_work), GFP_ATOMIC); 9378 if (!inet6addr_work) 9379 return NOTIFY_BAD; 9380 9381 router = container_of(nb, struct mlxsw_sp_router, inet6addr_nb); 9382 INIT_WORK(&inet6addr_work->work, mlxsw_sp_inet6addr_event_work); 9383 inet6addr_work->mlxsw_sp = router->mlxsw_sp; 9384 inet6addr_work->dev = dev; 9385 inet6addr_work->event = event; 9386 netdev_hold(dev, &inet6addr_work->dev_tracker, GFP_ATOMIC); 9387 mlxsw_core_schedule_work(&inet6addr_work->work); 9388 9389 return NOTIFY_DONE; 9390} 9391 9392static int mlxsw_sp_inet6addr_valid_event(struct notifier_block *unused, 9393 unsigned long event, void *ptr) 9394{ 9395 struct in6_validator_info *i6vi = (struct in6_validator_info *) ptr; 9396 struct net_device *dev = i6vi->i6vi_dev->dev; 9397 struct mlxsw_sp *mlxsw_sp; 9398 struct mlxsw_sp_rif *rif; 9399 int err = 0; 9400 9401 mlxsw_sp = mlxsw_sp_lower_get(dev); 9402 if (!mlxsw_sp) 9403 return NOTIFY_DONE; 9404 9405 mutex_lock(&mlxsw_sp->router->lock); 9406 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 9407 if (!mlxsw_sp_rif_should_config(rif, dev, event)) 9408 goto out; 9409 9410 err = __mlxsw_sp_inetaddr_event(mlxsw_sp, dev, event, false, 9411 i6vi->extack); 9412out: 9413 mutex_unlock(&mlxsw_sp->router->lock); 9414 return notifier_from_errno(err); 9415} 9416 9417static int mlxsw_sp_rif_edit(struct mlxsw_sp *mlxsw_sp, u16 rif_index, 9418 const char *mac, int mtu, u8 mac_profile) 9419{ 9420 char ritr_pl[MLXSW_REG_RITR_LEN]; 9421 int err; 9422 9423 mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index); 9424 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 9425 if (err) 9426 return err; 9427 9428 mlxsw_reg_ritr_mtu_set(ritr_pl, mtu); 9429 mlxsw_reg_ritr_if_mac_memcpy_to(ritr_pl, mac); 9430 mlxsw_reg_ritr_if_mac_profile_id_set(ritr_pl, mac_profile); 9431 mlxsw_reg_ritr_op_set(ritr_pl, MLXSW_REG_RITR_RIF_CREATE); 9432 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 9433} 9434 9435static int 9436mlxsw_sp_router_port_change_event(struct mlxsw_sp *mlxsw_sp, 9437 struct mlxsw_sp_rif *rif, 9438 struct netlink_ext_ack *extack) 9439{ 9440 struct net_device *dev = mlxsw_sp_rif_dev(rif); 9441 u8 old_mac_profile; 9442 u16 fid_index; 9443 int err; 9444 9445 fid_index = mlxsw_sp_fid_index(rif->fid); 9446 9447 err = mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->addr, fid_index, false); 9448 if (err) 9449 return err; 9450 9451 old_mac_profile = rif->mac_profile_id; 9452 err = mlxsw_sp_rif_mac_profile_replace(mlxsw_sp, rif, dev->dev_addr, 9453 extack); 9454 if (err) 9455 goto err_rif_mac_profile_replace; 9456 9457 err = mlxsw_sp_rif_edit(mlxsw_sp, rif->rif_index, dev->dev_addr, 9458 dev->mtu, rif->mac_profile_id); 9459 if (err) 9460 goto err_rif_edit; 9461 9462 err = mlxsw_sp_rif_fdb_op(mlxsw_sp, dev->dev_addr, fid_index, true); 9463 if (err) 9464 goto err_rif_fdb_op; 9465 9466 if (rif->mtu != dev->mtu) { 9467 struct mlxsw_sp_vr *vr; 9468 int i; 9469 9470 /* The RIF is relevant only to its mr_table instance, as unlike 9471 * unicast routing, in multicast routing a RIF cannot be shared 9472 * between several multicast routing tables. 9473 */ 9474 vr = &mlxsw_sp->router->vrs[rif->vr_id]; 9475 for (i = 0; i < MLXSW_SP_L3_PROTO_MAX; i++) 9476 mlxsw_sp_mr_rif_mtu_update(vr->mr_table[i], 9477 rif, dev->mtu); 9478 } 9479 9480 ether_addr_copy(rif->addr, dev->dev_addr); 9481 rif->mtu = dev->mtu; 9482 9483 netdev_dbg(dev, "Updated RIF=%d\n", rif->rif_index); 9484 9485 return 0; 9486 9487err_rif_fdb_op: 9488 mlxsw_sp_rif_edit(mlxsw_sp, rif->rif_index, rif->addr, rif->mtu, 9489 old_mac_profile); 9490err_rif_edit: 9491 mlxsw_sp_rif_mac_profile_replace(mlxsw_sp, rif, rif->addr, extack); 9492err_rif_mac_profile_replace: 9493 mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->addr, fid_index, true); 9494 return err; 9495} 9496 9497static int mlxsw_sp_router_port_pre_changeaddr_event(struct mlxsw_sp_rif *rif, 9498 struct netdev_notifier_pre_changeaddr_info *info) 9499{ 9500 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 9501 struct mlxsw_sp_rif_mac_profile *profile; 9502 struct netlink_ext_ack *extack; 9503 u8 max_rif_mac_profiles; 9504 u64 occ; 9505 9506 extack = netdev_notifier_info_to_extack(&info->info); 9507 9508 profile = mlxsw_sp_rif_mac_profile_find(mlxsw_sp, info->dev_addr); 9509 if (profile) 9510 return 0; 9511 9512 max_rif_mac_profiles = mlxsw_sp->router->max_rif_mac_profile; 9513 occ = mlxsw_sp_rif_mac_profiles_occ_get(mlxsw_sp); 9514 if (occ < max_rif_mac_profiles) 9515 return 0; 9516 9517 if (!mlxsw_sp_rif_mac_profile_is_shared(rif)) 9518 return 0; 9519 9520 NL_SET_ERR_MSG_MOD(extack, "Exceeded number of supported router interface MAC profiles"); 9521 return -ENOBUFS; 9522} 9523 9524static bool mlxsw_sp_router_netdevice_interesting(struct mlxsw_sp *mlxsw_sp, 9525 struct net_device *dev) 9526{ 9527 struct vlan_dev_priv *vlan; 9528 9529 if (netif_is_lag_master(dev) || 9530 netif_is_bridge_master(dev) || 9531 mlxsw_sp_port_dev_check(dev) || 9532 mlxsw_sp_netdev_is_ipip_ol(mlxsw_sp, dev) || 9533 netif_is_l3_master(dev)) 9534 return true; 9535 9536 if (!is_vlan_dev(dev)) 9537 return false; 9538 9539 vlan = vlan_dev_priv(dev); 9540 return netif_is_lag_master(vlan->real_dev) || 9541 netif_is_bridge_master(vlan->real_dev) || 9542 mlxsw_sp_port_dev_check(vlan->real_dev); 9543} 9544 9545static struct mlxsw_sp_crif * 9546mlxsw_sp_crif_register(struct mlxsw_sp_router *router, struct net_device *dev) 9547{ 9548 struct mlxsw_sp_crif *crif; 9549 int err; 9550 9551 if (WARN_ON(mlxsw_sp_crif_lookup(router, dev))) 9552 return NULL; 9553 9554 crif = mlxsw_sp_crif_alloc(dev); 9555 if (!crif) 9556 return ERR_PTR(-ENOMEM); 9557 9558 err = mlxsw_sp_crif_insert(router, crif); 9559 if (err) 9560 goto err_netdev_insert; 9561 9562 return crif; 9563 9564err_netdev_insert: 9565 mlxsw_sp_crif_free(crif); 9566 return ERR_PTR(err); 9567} 9568 9569static void mlxsw_sp_crif_unregister(struct mlxsw_sp_router *router, 9570 struct mlxsw_sp_crif *crif) 9571{ 9572 struct mlxsw_sp_nexthop *nh, *tmp; 9573 9574 mlxsw_sp_crif_remove(router, crif); 9575 9576 list_for_each_entry_safe(nh, tmp, &crif->nexthop_list, crif_list_node) 9577 mlxsw_sp_nexthop_type_fini(router->mlxsw_sp, nh); 9578 9579 if (crif->rif) 9580 crif->can_destroy = true; 9581 else 9582 mlxsw_sp_crif_free(crif); 9583} 9584 9585static int mlxsw_sp_netdevice_register(struct mlxsw_sp_router *router, 9586 struct net_device *dev) 9587{ 9588 struct mlxsw_sp_crif *crif; 9589 9590 if (!mlxsw_sp_router_netdevice_interesting(router->mlxsw_sp, dev)) 9591 return 0; 9592 9593 crif = mlxsw_sp_crif_register(router, dev); 9594 return PTR_ERR_OR_ZERO(crif); 9595} 9596 9597static void mlxsw_sp_netdevice_unregister(struct mlxsw_sp_router *router, 9598 struct net_device *dev) 9599{ 9600 struct mlxsw_sp_crif *crif; 9601 9602 if (!mlxsw_sp_router_netdevice_interesting(router->mlxsw_sp, dev)) 9603 return; 9604 9605 /* netdev_run_todo(), by way of netdev_wait_allrefs_any(), rebroadcasts 9606 * the NETDEV_UNREGISTER message, so we can get here twice. If that's 9607 * what happened, the netdevice state is NETREG_UNREGISTERED. In that 9608 * case, we expect to have collected the CRIF already, and warn if it 9609 * still exists. Otherwise we expect the CRIF to exist. 9610 */ 9611 crif = mlxsw_sp_crif_lookup(router, dev); 9612 if (dev->reg_state == NETREG_UNREGISTERED) { 9613 if (!WARN_ON(crif)) 9614 return; 9615 } 9616 if (WARN_ON(!crif)) 9617 return; 9618 9619 mlxsw_sp_crif_unregister(router, crif); 9620} 9621 9622static bool mlxsw_sp_is_offload_xstats_event(unsigned long event) 9623{ 9624 switch (event) { 9625 case NETDEV_OFFLOAD_XSTATS_ENABLE: 9626 case NETDEV_OFFLOAD_XSTATS_DISABLE: 9627 case NETDEV_OFFLOAD_XSTATS_REPORT_USED: 9628 case NETDEV_OFFLOAD_XSTATS_REPORT_DELTA: 9629 return true; 9630 } 9631 9632 return false; 9633} 9634 9635static int 9636mlxsw_sp_router_port_offload_xstats_cmd(struct mlxsw_sp_rif *rif, 9637 unsigned long event, 9638 struct netdev_notifier_offload_xstats_info *info) 9639{ 9640 switch (info->type) { 9641 case NETDEV_OFFLOAD_XSTATS_TYPE_L3: 9642 break; 9643 default: 9644 return 0; 9645 } 9646 9647 switch (event) { 9648 case NETDEV_OFFLOAD_XSTATS_ENABLE: 9649 return mlxsw_sp_router_port_l3_stats_enable(rif); 9650 case NETDEV_OFFLOAD_XSTATS_DISABLE: 9651 mlxsw_sp_router_port_l3_stats_disable(rif); 9652 return 0; 9653 case NETDEV_OFFLOAD_XSTATS_REPORT_USED: 9654 mlxsw_sp_router_port_l3_stats_report_used(rif, info); 9655 return 0; 9656 case NETDEV_OFFLOAD_XSTATS_REPORT_DELTA: 9657 return mlxsw_sp_router_port_l3_stats_report_delta(rif, info); 9658 } 9659 9660 WARN_ON_ONCE(1); 9661 return 0; 9662} 9663 9664static int 9665mlxsw_sp_netdevice_offload_xstats_cmd(struct mlxsw_sp *mlxsw_sp, 9666 struct net_device *dev, 9667 unsigned long event, 9668 struct netdev_notifier_offload_xstats_info *info) 9669{ 9670 struct mlxsw_sp_rif *rif; 9671 9672 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 9673 if (!rif) 9674 return 0; 9675 9676 return mlxsw_sp_router_port_offload_xstats_cmd(rif, event, info); 9677} 9678 9679static bool mlxsw_sp_is_router_event(unsigned long event) 9680{ 9681 switch (event) { 9682 case NETDEV_PRE_CHANGEADDR: 9683 case NETDEV_CHANGEADDR: 9684 case NETDEV_CHANGEMTU: 9685 return true; 9686 default: 9687 return false; 9688 } 9689} 9690 9691static int mlxsw_sp_netdevice_router_port_event(struct net_device *dev, 9692 unsigned long event, void *ptr) 9693{ 9694 struct netlink_ext_ack *extack = netdev_notifier_info_to_extack(ptr); 9695 struct mlxsw_sp *mlxsw_sp; 9696 struct mlxsw_sp_rif *rif; 9697 9698 mlxsw_sp = mlxsw_sp_lower_get(dev); 9699 if (!mlxsw_sp) 9700 return 0; 9701 9702 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 9703 if (!rif) 9704 return 0; 9705 9706 switch (event) { 9707 case NETDEV_CHANGEMTU: 9708 case NETDEV_CHANGEADDR: 9709 return mlxsw_sp_router_port_change_event(mlxsw_sp, rif, extack); 9710 case NETDEV_PRE_CHANGEADDR: 9711 return mlxsw_sp_router_port_pre_changeaddr_event(rif, ptr); 9712 default: 9713 WARN_ON_ONCE(1); 9714 break; 9715 } 9716 9717 return 0; 9718} 9719 9720static int mlxsw_sp_port_vrf_join(struct mlxsw_sp *mlxsw_sp, 9721 struct net_device *l3_dev, 9722 struct netlink_ext_ack *extack) 9723{ 9724 struct mlxsw_sp_rif *rif; 9725 9726 /* If netdev is already associated with a RIF, then we need to 9727 * destroy it and create a new one with the new virtual router ID. 9728 */ 9729 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev); 9730 if (rif) 9731 __mlxsw_sp_inetaddr_event(mlxsw_sp, l3_dev, NETDEV_DOWN, false, 9732 extack); 9733 9734 return __mlxsw_sp_inetaddr_event(mlxsw_sp, l3_dev, NETDEV_UP, false, 9735 extack); 9736} 9737 9738static void mlxsw_sp_port_vrf_leave(struct mlxsw_sp *mlxsw_sp, 9739 struct net_device *l3_dev) 9740{ 9741 struct mlxsw_sp_rif *rif; 9742 9743 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev); 9744 if (!rif) 9745 return; 9746 __mlxsw_sp_inetaddr_event(mlxsw_sp, l3_dev, NETDEV_DOWN, false, NULL); 9747} 9748 9749static bool mlxsw_sp_is_vrf_event(unsigned long event, void *ptr) 9750{ 9751 struct netdev_notifier_changeupper_info *info = ptr; 9752 9753 if (event != NETDEV_PRECHANGEUPPER && event != NETDEV_CHANGEUPPER) 9754 return false; 9755 return netif_is_l3_master(info->upper_dev); 9756} 9757 9758static int 9759mlxsw_sp_netdevice_vrf_event(struct net_device *l3_dev, unsigned long event, 9760 struct netdev_notifier_changeupper_info *info) 9761{ 9762 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_lower_get(l3_dev); 9763 int err = 0; 9764 9765 /* We do not create a RIF for a macvlan, but only use it to 9766 * direct more MAC addresses to the router. 9767 */ 9768 if (!mlxsw_sp || netif_is_macvlan(l3_dev)) 9769 return 0; 9770 9771 switch (event) { 9772 case NETDEV_PRECHANGEUPPER: 9773 break; 9774 case NETDEV_CHANGEUPPER: 9775 if (info->linking) { 9776 struct netlink_ext_ack *extack; 9777 9778 extack = netdev_notifier_info_to_extack(&info->info); 9779 err = mlxsw_sp_port_vrf_join(mlxsw_sp, l3_dev, extack); 9780 } else { 9781 mlxsw_sp_port_vrf_leave(mlxsw_sp, l3_dev); 9782 } 9783 break; 9784 } 9785 9786 return err; 9787} 9788 9789struct mlxsw_sp_router_replay_inetaddr_up { 9790 struct mlxsw_sp *mlxsw_sp; 9791 struct netlink_ext_ack *extack; 9792 unsigned int done; 9793 bool deslavement; 9794}; 9795 9796static int mlxsw_sp_router_replay_inetaddr_up(struct net_device *dev, 9797 struct netdev_nested_priv *priv) 9798{ 9799 struct mlxsw_sp_router_replay_inetaddr_up *ctx = priv->data; 9800 bool nomaster = ctx->deslavement; 9801 struct mlxsw_sp_crif *crif; 9802 int err; 9803 9804 if (mlxsw_sp_dev_addr_list_empty(dev)) 9805 return 0; 9806 9807 crif = mlxsw_sp_crif_lookup(ctx->mlxsw_sp->router, dev); 9808 if (!crif || crif->rif) 9809 return 0; 9810 9811 if (!mlxsw_sp_rif_should_config(crif->rif, dev, NETDEV_UP)) 9812 return 0; 9813 9814 err = __mlxsw_sp_inetaddr_event(ctx->mlxsw_sp, dev, NETDEV_UP, 9815 nomaster, ctx->extack); 9816 if (err) 9817 return err; 9818 9819 ctx->done++; 9820 return 0; 9821} 9822 9823static int mlxsw_sp_router_unreplay_inetaddr_up(struct net_device *dev, 9824 struct netdev_nested_priv *priv) 9825{ 9826 struct mlxsw_sp_router_replay_inetaddr_up *ctx = priv->data; 9827 bool nomaster = ctx->deslavement; 9828 struct mlxsw_sp_crif *crif; 9829 9830 if (!ctx->done) 9831 return 0; 9832 9833 if (mlxsw_sp_dev_addr_list_empty(dev)) 9834 return 0; 9835 9836 crif = mlxsw_sp_crif_lookup(ctx->mlxsw_sp->router, dev); 9837 if (!crif || !crif->rif) 9838 return 0; 9839 9840 /* We are rolling back NETDEV_UP, so ask for that. */ 9841 if (!mlxsw_sp_rif_should_config(crif->rif, dev, NETDEV_UP)) 9842 return 0; 9843 9844 __mlxsw_sp_inetaddr_event(ctx->mlxsw_sp, dev, NETDEV_DOWN, nomaster, 9845 NULL); 9846 9847 ctx->done--; 9848 return 0; 9849} 9850 9851int mlxsw_sp_netdevice_enslavement_replay(struct mlxsw_sp *mlxsw_sp, 9852 struct net_device *upper_dev, 9853 struct netlink_ext_ack *extack) 9854{ 9855 struct mlxsw_sp_router_replay_inetaddr_up ctx = { 9856 .mlxsw_sp = mlxsw_sp, 9857 .extack = extack, 9858 .deslavement = false, 9859 }; 9860 struct netdev_nested_priv priv = { 9861 .data = &ctx, 9862 }; 9863 int err; 9864 9865 err = mlxsw_sp_router_replay_inetaddr_up(upper_dev, &priv); 9866 if (err) 9867 return err; 9868 9869 err = netdev_walk_all_upper_dev_rcu(upper_dev, 9870 mlxsw_sp_router_replay_inetaddr_up, 9871 &priv); 9872 if (err) 9873 goto err_replay_up; 9874 9875 return 0; 9876 9877err_replay_up: 9878 netdev_walk_all_upper_dev_rcu(upper_dev, 9879 mlxsw_sp_router_unreplay_inetaddr_up, 9880 &priv); 9881 mlxsw_sp_router_unreplay_inetaddr_up(upper_dev, &priv); 9882 return err; 9883} 9884 9885void mlxsw_sp_netdevice_deslavement_replay(struct mlxsw_sp *mlxsw_sp, 9886 struct net_device *dev) 9887{ 9888 struct mlxsw_sp_router_replay_inetaddr_up ctx = { 9889 .mlxsw_sp = mlxsw_sp, 9890 .deslavement = true, 9891 }; 9892 struct netdev_nested_priv priv = { 9893 .data = &ctx, 9894 }; 9895 9896 mlxsw_sp_router_replay_inetaddr_up(dev, &priv); 9897} 9898 9899static int 9900mlxsw_sp_port_vid_router_join_existing(struct mlxsw_sp_port *mlxsw_sp_port, 9901 u16 vid, struct net_device *dev, 9902 struct netlink_ext_ack *extack) 9903{ 9904 struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan; 9905 9906 mlxsw_sp_port_vlan = mlxsw_sp_port_vlan_find_by_vid(mlxsw_sp_port, 9907 vid); 9908 if (WARN_ON(!mlxsw_sp_port_vlan)) 9909 return -EINVAL; 9910 9911 return mlxsw_sp_port_vlan_router_join_existing(mlxsw_sp_port_vlan, 9912 dev, extack); 9913} 9914 9915static void 9916mlxsw_sp_port_vid_router_leave(struct mlxsw_sp_port *mlxsw_sp_port, u16 vid, 9917 struct net_device *dev) 9918{ 9919 struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan; 9920 9921 mlxsw_sp_port_vlan = mlxsw_sp_port_vlan_find_by_vid(mlxsw_sp_port, 9922 vid); 9923 if (WARN_ON(!mlxsw_sp_port_vlan)) 9924 return; 9925 9926 __mlxsw_sp_port_vlan_router_leave(mlxsw_sp_port_vlan); 9927} 9928 9929static int __mlxsw_sp_router_port_join_lag(struct mlxsw_sp_port *mlxsw_sp_port, 9930 struct net_device *lag_dev, 9931 struct netlink_ext_ack *extack) 9932{ 9933 u16 default_vid = MLXSW_SP_DEFAULT_VID; 9934 struct net_device *upper_dev; 9935 struct list_head *iter; 9936 int done = 0; 9937 u16 vid; 9938 int err; 9939 9940 err = mlxsw_sp_port_vid_router_join_existing(mlxsw_sp_port, default_vid, 9941 lag_dev, extack); 9942 if (err) 9943 return err; 9944 9945 netdev_for_each_upper_dev_rcu(lag_dev, upper_dev, iter) { 9946 if (!is_vlan_dev(upper_dev)) 9947 continue; 9948 9949 vid = vlan_dev_vlan_id(upper_dev); 9950 err = mlxsw_sp_port_vid_router_join_existing(mlxsw_sp_port, vid, 9951 upper_dev, extack); 9952 if (err) 9953 goto err_router_join_dev; 9954 9955 ++done; 9956 } 9957 9958 return 0; 9959 9960err_router_join_dev: 9961 netdev_for_each_upper_dev_rcu(lag_dev, upper_dev, iter) { 9962 if (!is_vlan_dev(upper_dev)) 9963 continue; 9964 if (!done--) 9965 break; 9966 9967 vid = vlan_dev_vlan_id(upper_dev); 9968 mlxsw_sp_port_vid_router_leave(mlxsw_sp_port, vid, upper_dev); 9969 } 9970 9971 mlxsw_sp_port_vid_router_leave(mlxsw_sp_port, default_vid, lag_dev); 9972 return err; 9973} 9974 9975static void 9976__mlxsw_sp_router_port_leave_lag(struct mlxsw_sp_port *mlxsw_sp_port, 9977 struct net_device *lag_dev) 9978{ 9979 u16 default_vid = MLXSW_SP_DEFAULT_VID; 9980 struct net_device *upper_dev; 9981 struct list_head *iter; 9982 u16 vid; 9983 9984 netdev_for_each_upper_dev_rcu(lag_dev, upper_dev, iter) { 9985 if (!is_vlan_dev(upper_dev)) 9986 continue; 9987 9988 vid = vlan_dev_vlan_id(upper_dev); 9989 mlxsw_sp_port_vid_router_leave(mlxsw_sp_port, vid, upper_dev); 9990 } 9991 9992 mlxsw_sp_port_vid_router_leave(mlxsw_sp_port, default_vid, lag_dev); 9993} 9994 9995int mlxsw_sp_router_port_join_lag(struct mlxsw_sp_port *mlxsw_sp_port, 9996 struct net_device *lag_dev, 9997 struct netlink_ext_ack *extack) 9998{ 9999 int err; 10000 10001 mutex_lock(&mlxsw_sp_port->mlxsw_sp->router->lock); 10002 err = __mlxsw_sp_router_port_join_lag(mlxsw_sp_port, lag_dev, extack); 10003 mutex_unlock(&mlxsw_sp_port->mlxsw_sp->router->lock); 10004 10005 return err; 10006} 10007 10008void mlxsw_sp_router_port_leave_lag(struct mlxsw_sp_port *mlxsw_sp_port, 10009 struct net_device *lag_dev) 10010{ 10011 mutex_lock(&mlxsw_sp_port->mlxsw_sp->router->lock); 10012 __mlxsw_sp_router_port_leave_lag(mlxsw_sp_port, lag_dev); 10013 mutex_unlock(&mlxsw_sp_port->mlxsw_sp->router->lock); 10014} 10015 10016static int mlxsw_sp_router_netdevice_event(struct notifier_block *nb, 10017 unsigned long event, void *ptr) 10018{ 10019 struct net_device *dev = netdev_notifier_info_to_dev(ptr); 10020 struct mlxsw_sp_router *router; 10021 struct mlxsw_sp *mlxsw_sp; 10022 int err = 0; 10023 10024 router = container_of(nb, struct mlxsw_sp_router, netdevice_nb); 10025 mlxsw_sp = router->mlxsw_sp; 10026 10027 mutex_lock(&mlxsw_sp->router->lock); 10028 10029 if (event == NETDEV_REGISTER) { 10030 err = mlxsw_sp_netdevice_register(router, dev); 10031 if (err) 10032 /* No need to roll this back, UNREGISTER will collect it 10033 * anyhow. 10034 */ 10035 goto out; 10036 } 10037 10038 if (mlxsw_sp_is_offload_xstats_event(event)) 10039 err = mlxsw_sp_netdevice_offload_xstats_cmd(mlxsw_sp, dev, 10040 event, ptr); 10041 else if (mlxsw_sp_netdev_is_ipip_ol(mlxsw_sp, dev)) 10042 err = mlxsw_sp_netdevice_ipip_ol_event(mlxsw_sp, dev, 10043 event, ptr); 10044 else if (mlxsw_sp_netdev_is_ipip_ul(mlxsw_sp, dev)) 10045 err = mlxsw_sp_netdevice_ipip_ul_event(mlxsw_sp, dev, 10046 event, ptr); 10047 else if (mlxsw_sp_is_router_event(event)) 10048 err = mlxsw_sp_netdevice_router_port_event(dev, event, ptr); 10049 else if (mlxsw_sp_is_vrf_event(event, ptr)) 10050 err = mlxsw_sp_netdevice_vrf_event(dev, event, ptr); 10051 10052 if (event == NETDEV_UNREGISTER) 10053 mlxsw_sp_netdevice_unregister(router, dev); 10054 10055out: 10056 mutex_unlock(&mlxsw_sp->router->lock); 10057 10058 return notifier_from_errno(err); 10059} 10060 10061struct mlxsw_sp_macvlan_replay { 10062 struct mlxsw_sp *mlxsw_sp; 10063 struct netlink_ext_ack *extack; 10064}; 10065 10066static int mlxsw_sp_macvlan_replay_upper(struct net_device *dev, 10067 struct netdev_nested_priv *priv) 10068{ 10069 const struct mlxsw_sp_macvlan_replay *rms = priv->data; 10070 struct netlink_ext_ack *extack = rms->extack; 10071 struct mlxsw_sp *mlxsw_sp = rms->mlxsw_sp; 10072 10073 if (!netif_is_macvlan(dev)) 10074 return 0; 10075 10076 return mlxsw_sp_rif_macvlan_add(mlxsw_sp, dev, extack); 10077} 10078 10079static int mlxsw_sp_macvlan_replay(struct mlxsw_sp_rif *rif, 10080 struct netlink_ext_ack *extack) 10081{ 10082 struct mlxsw_sp_macvlan_replay rms = { 10083 .mlxsw_sp = rif->mlxsw_sp, 10084 .extack = extack, 10085 }; 10086 struct netdev_nested_priv priv = { 10087 .data = &rms, 10088 }; 10089 10090 return netdev_walk_all_upper_dev_rcu(mlxsw_sp_rif_dev(rif), 10091 mlxsw_sp_macvlan_replay_upper, 10092 &priv); 10093} 10094 10095static int __mlxsw_sp_rif_macvlan_flush(struct net_device *dev, 10096 struct netdev_nested_priv *priv) 10097{ 10098 struct mlxsw_sp_rif *rif = (struct mlxsw_sp_rif *)priv->data; 10099 10100 if (!netif_is_macvlan(dev)) 10101 return 0; 10102 10103 return mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, dev->dev_addr, 10104 mlxsw_sp_fid_index(rif->fid), false); 10105} 10106 10107static int mlxsw_sp_rif_macvlan_flush(struct mlxsw_sp_rif *rif) 10108{ 10109 struct net_device *dev = mlxsw_sp_rif_dev(rif); 10110 struct netdev_nested_priv priv = { 10111 .data = (void *)rif, 10112 }; 10113 10114 if (!netif_is_macvlan_port(dev)) 10115 return 0; 10116 10117 return netdev_walk_all_upper_dev_rcu(dev, 10118 __mlxsw_sp_rif_macvlan_flush, &priv); 10119} 10120 10121static void mlxsw_sp_rif_subport_setup(struct mlxsw_sp_rif *rif, 10122 const struct mlxsw_sp_rif_params *params) 10123{ 10124 struct mlxsw_sp_rif_subport *rif_subport; 10125 10126 rif_subport = mlxsw_sp_rif_subport_rif(rif); 10127 refcount_set(&rif_subport->ref_count, 1); 10128 rif_subport->vid = params->vid; 10129 rif_subport->lag = params->lag; 10130 if (params->lag) 10131 rif_subport->lag_id = params->lag_id; 10132 else 10133 rif_subport->system_port = params->system_port; 10134} 10135 10136static int mlxsw_sp_rif_subport_op(struct mlxsw_sp_rif *rif, bool enable) 10137{ 10138 struct net_device *dev = mlxsw_sp_rif_dev(rif); 10139 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 10140 struct mlxsw_sp_rif_subport *rif_subport; 10141 char ritr_pl[MLXSW_REG_RITR_LEN]; 10142 u16 efid; 10143 10144 rif_subport = mlxsw_sp_rif_subport_rif(rif); 10145 mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_SP_IF, 10146 rif->rif_index, rif->vr_id, dev->mtu); 10147 mlxsw_reg_ritr_mac_pack(ritr_pl, dev->dev_addr); 10148 mlxsw_reg_ritr_if_mac_profile_id_set(ritr_pl, rif->mac_profile_id); 10149 efid = mlxsw_sp_fid_index(rif->fid); 10150 mlxsw_reg_ritr_sp_if_pack(ritr_pl, rif_subport->lag, 10151 rif_subport->lag ? rif_subport->lag_id : 10152 rif_subport->system_port, 10153 efid, 0); 10154 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 10155} 10156 10157static int mlxsw_sp_rif_subport_configure(struct mlxsw_sp_rif *rif, 10158 struct netlink_ext_ack *extack) 10159{ 10160 struct net_device *dev = mlxsw_sp_rif_dev(rif); 10161 u8 mac_profile; 10162 int err; 10163 10164 err = mlxsw_sp_rif_mac_profile_get(rif->mlxsw_sp, rif->addr, 10165 &mac_profile, extack); 10166 if (err) 10167 return err; 10168 rif->mac_profile_id = mac_profile; 10169 10170 err = mlxsw_sp_rif_subport_op(rif, true); 10171 if (err) 10172 goto err_rif_subport_op; 10173 10174 err = mlxsw_sp_macvlan_replay(rif, extack); 10175 if (err) 10176 goto err_macvlan_replay; 10177 10178 err = mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, dev->dev_addr, 10179 mlxsw_sp_fid_index(rif->fid), true); 10180 if (err) 10181 goto err_rif_fdb_op; 10182 10183 err = mlxsw_sp_fid_rif_set(rif->fid, rif); 10184 if (err) 10185 goto err_fid_rif_set; 10186 10187 return 0; 10188 10189err_fid_rif_set: 10190 mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, dev->dev_addr, 10191 mlxsw_sp_fid_index(rif->fid), false); 10192err_rif_fdb_op: 10193 mlxsw_sp_rif_macvlan_flush(rif); 10194err_macvlan_replay: 10195 mlxsw_sp_rif_subport_op(rif, false); 10196err_rif_subport_op: 10197 mlxsw_sp_rif_mac_profile_put(rif->mlxsw_sp, mac_profile); 10198 return err; 10199} 10200 10201static void mlxsw_sp_rif_subport_deconfigure(struct mlxsw_sp_rif *rif) 10202{ 10203 struct net_device *dev = mlxsw_sp_rif_dev(rif); 10204 struct mlxsw_sp_fid *fid = rif->fid; 10205 10206 mlxsw_sp_fid_rif_unset(fid); 10207 mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, dev->dev_addr, 10208 mlxsw_sp_fid_index(fid), false); 10209 mlxsw_sp_rif_macvlan_flush(rif); 10210 mlxsw_sp_rif_subport_op(rif, false); 10211 mlxsw_sp_rif_mac_profile_put(rif->mlxsw_sp, rif->mac_profile_id); 10212} 10213 10214static struct mlxsw_sp_fid * 10215mlxsw_sp_rif_subport_fid_get(struct mlxsw_sp_rif *rif, 10216 const struct mlxsw_sp_rif_params *params, 10217 struct netlink_ext_ack *extack) 10218{ 10219 return mlxsw_sp_fid_rfid_get(rif->mlxsw_sp, rif->rif_index); 10220} 10221 10222static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_subport_ops = { 10223 .type = MLXSW_SP_RIF_TYPE_SUBPORT, 10224 .rif_size = sizeof(struct mlxsw_sp_rif_subport), 10225 .setup = mlxsw_sp_rif_subport_setup, 10226 .configure = mlxsw_sp_rif_subport_configure, 10227 .deconfigure = mlxsw_sp_rif_subport_deconfigure, 10228 .fid_get = mlxsw_sp_rif_subport_fid_get, 10229}; 10230 10231static int mlxsw_sp_rif_fid_op(struct mlxsw_sp_rif *rif, u16 fid, bool enable) 10232{ 10233 enum mlxsw_reg_ritr_if_type type = MLXSW_REG_RITR_FID_IF; 10234 struct net_device *dev = mlxsw_sp_rif_dev(rif); 10235 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 10236 char ritr_pl[MLXSW_REG_RITR_LEN]; 10237 10238 mlxsw_reg_ritr_pack(ritr_pl, enable, type, rif->rif_index, rif->vr_id, 10239 dev->mtu); 10240 mlxsw_reg_ritr_mac_pack(ritr_pl, dev->dev_addr); 10241 mlxsw_reg_ritr_if_mac_profile_id_set(ritr_pl, rif->mac_profile_id); 10242 mlxsw_reg_ritr_fid_if_fid_set(ritr_pl, fid); 10243 10244 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 10245} 10246 10247u16 mlxsw_sp_router_port(const struct mlxsw_sp *mlxsw_sp) 10248{ 10249 return mlxsw_core_max_ports(mlxsw_sp->core) + 1; 10250} 10251 10252static int mlxsw_sp_rif_fid_configure(struct mlxsw_sp_rif *rif, 10253 struct netlink_ext_ack *extack) 10254{ 10255 struct net_device *dev = mlxsw_sp_rif_dev(rif); 10256 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 10257 u16 fid_index = mlxsw_sp_fid_index(rif->fid); 10258 u8 mac_profile; 10259 int err; 10260 10261 err = mlxsw_sp_rif_mac_profile_get(mlxsw_sp, rif->addr, 10262 &mac_profile, extack); 10263 if (err) 10264 return err; 10265 rif->mac_profile_id = mac_profile; 10266 10267 err = mlxsw_sp_rif_fid_op(rif, fid_index, true); 10268 if (err) 10269 goto err_rif_fid_op; 10270 10271 err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC, 10272 mlxsw_sp_router_port(mlxsw_sp), true); 10273 if (err) 10274 goto err_fid_mc_flood_set; 10275 10276 err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC, 10277 mlxsw_sp_router_port(mlxsw_sp), true); 10278 if (err) 10279 goto err_fid_bc_flood_set; 10280 10281 err = mlxsw_sp_macvlan_replay(rif, extack); 10282 if (err) 10283 goto err_macvlan_replay; 10284 10285 err = mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, dev->dev_addr, 10286 mlxsw_sp_fid_index(rif->fid), true); 10287 if (err) 10288 goto err_rif_fdb_op; 10289 10290 err = mlxsw_sp_fid_rif_set(rif->fid, rif); 10291 if (err) 10292 goto err_fid_rif_set; 10293 10294 return 0; 10295 10296err_fid_rif_set: 10297 mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, dev->dev_addr, 10298 mlxsw_sp_fid_index(rif->fid), false); 10299err_rif_fdb_op: 10300 mlxsw_sp_rif_macvlan_flush(rif); 10301err_macvlan_replay: 10302 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC, 10303 mlxsw_sp_router_port(mlxsw_sp), false); 10304err_fid_bc_flood_set: 10305 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC, 10306 mlxsw_sp_router_port(mlxsw_sp), false); 10307err_fid_mc_flood_set: 10308 mlxsw_sp_rif_fid_op(rif, fid_index, false); 10309err_rif_fid_op: 10310 mlxsw_sp_rif_mac_profile_put(mlxsw_sp, mac_profile); 10311 return err; 10312} 10313 10314static void mlxsw_sp_rif_fid_deconfigure(struct mlxsw_sp_rif *rif) 10315{ 10316 struct net_device *dev = mlxsw_sp_rif_dev(rif); 10317 u16 fid_index = mlxsw_sp_fid_index(rif->fid); 10318 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 10319 struct mlxsw_sp_fid *fid = rif->fid; 10320 10321 mlxsw_sp_fid_rif_unset(fid); 10322 mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, dev->dev_addr, 10323 mlxsw_sp_fid_index(fid), false); 10324 mlxsw_sp_rif_macvlan_flush(rif); 10325 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC, 10326 mlxsw_sp_router_port(mlxsw_sp), false); 10327 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC, 10328 mlxsw_sp_router_port(mlxsw_sp), false); 10329 mlxsw_sp_rif_fid_op(rif, fid_index, false); 10330 mlxsw_sp_rif_mac_profile_put(rif->mlxsw_sp, rif->mac_profile_id); 10331} 10332 10333static struct mlxsw_sp_fid * 10334mlxsw_sp_rif_fid_fid_get(struct mlxsw_sp_rif *rif, 10335 const struct mlxsw_sp_rif_params *params, 10336 struct netlink_ext_ack *extack) 10337{ 10338 int rif_ifindex = mlxsw_sp_rif_dev_ifindex(rif); 10339 10340 return mlxsw_sp_fid_8021d_get(rif->mlxsw_sp, rif_ifindex); 10341} 10342 10343static void mlxsw_sp_rif_fid_fdb_del(struct mlxsw_sp_rif *rif, const char *mac) 10344{ 10345 struct switchdev_notifier_fdb_info info = {}; 10346 struct net_device *dev; 10347 10348 dev = br_fdb_find_port(mlxsw_sp_rif_dev(rif), mac, 0); 10349 if (!dev) 10350 return; 10351 10352 info.addr = mac; 10353 info.vid = 0; 10354 call_switchdev_notifiers(SWITCHDEV_FDB_DEL_TO_BRIDGE, dev, &info.info, 10355 NULL); 10356} 10357 10358static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_fid_ops = { 10359 .type = MLXSW_SP_RIF_TYPE_FID, 10360 .rif_size = sizeof(struct mlxsw_sp_rif), 10361 .configure = mlxsw_sp_rif_fid_configure, 10362 .deconfigure = mlxsw_sp_rif_fid_deconfigure, 10363 .fid_get = mlxsw_sp_rif_fid_fid_get, 10364 .fdb_del = mlxsw_sp_rif_fid_fdb_del, 10365}; 10366 10367static struct mlxsw_sp_fid * 10368mlxsw_sp_rif_vlan_fid_get(struct mlxsw_sp_rif *rif, 10369 const struct mlxsw_sp_rif_params *params, 10370 struct netlink_ext_ack *extack) 10371{ 10372 struct net_device *dev = mlxsw_sp_rif_dev(rif); 10373 struct net_device *br_dev; 10374 10375 if (WARN_ON(!params->vid)) 10376 return ERR_PTR(-EINVAL); 10377 10378 if (is_vlan_dev(dev)) { 10379 br_dev = vlan_dev_real_dev(dev); 10380 if (WARN_ON(!netif_is_bridge_master(br_dev))) 10381 return ERR_PTR(-EINVAL); 10382 } 10383 10384 return mlxsw_sp_fid_8021q_get(rif->mlxsw_sp, params->vid); 10385} 10386 10387static void mlxsw_sp_rif_vlan_fdb_del(struct mlxsw_sp_rif *rif, const char *mac) 10388{ 10389 struct net_device *rif_dev = mlxsw_sp_rif_dev(rif); 10390 struct switchdev_notifier_fdb_info info = {}; 10391 u16 vid = mlxsw_sp_fid_8021q_vid(rif->fid); 10392 struct net_device *br_dev; 10393 struct net_device *dev; 10394 10395 br_dev = is_vlan_dev(rif_dev) ? vlan_dev_real_dev(rif_dev) : rif_dev; 10396 dev = br_fdb_find_port(br_dev, mac, vid); 10397 if (!dev) 10398 return; 10399 10400 info.addr = mac; 10401 info.vid = vid; 10402 call_switchdev_notifiers(SWITCHDEV_FDB_DEL_TO_BRIDGE, dev, &info.info, 10403 NULL); 10404} 10405 10406static int mlxsw_sp_rif_vlan_op(struct mlxsw_sp_rif *rif, u16 vid, u16 efid, 10407 bool enable) 10408{ 10409 struct net_device *dev = mlxsw_sp_rif_dev(rif); 10410 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 10411 char ritr_pl[MLXSW_REG_RITR_LEN]; 10412 10413 mlxsw_reg_ritr_vlan_if_pack(ritr_pl, enable, rif->rif_index, rif->vr_id, 10414 dev->mtu, dev->dev_addr, 10415 rif->mac_profile_id, vid, efid); 10416 10417 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 10418} 10419 10420static int mlxsw_sp_rif_vlan_configure(struct mlxsw_sp_rif *rif, u16 efid, 10421 struct netlink_ext_ack *extack) 10422{ 10423 struct net_device *dev = mlxsw_sp_rif_dev(rif); 10424 u16 vid = mlxsw_sp_fid_8021q_vid(rif->fid); 10425 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 10426 u8 mac_profile; 10427 int err; 10428 10429 err = mlxsw_sp_rif_mac_profile_get(mlxsw_sp, rif->addr, 10430 &mac_profile, extack); 10431 if (err) 10432 return err; 10433 rif->mac_profile_id = mac_profile; 10434 10435 err = mlxsw_sp_rif_vlan_op(rif, vid, efid, true); 10436 if (err) 10437 goto err_rif_vlan_fid_op; 10438 10439 err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC, 10440 mlxsw_sp_router_port(mlxsw_sp), true); 10441 if (err) 10442 goto err_fid_mc_flood_set; 10443 10444 err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC, 10445 mlxsw_sp_router_port(mlxsw_sp), true); 10446 if (err) 10447 goto err_fid_bc_flood_set; 10448 10449 err = mlxsw_sp_macvlan_replay(rif, extack); 10450 if (err) 10451 goto err_macvlan_replay; 10452 10453 err = mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, dev->dev_addr, 10454 mlxsw_sp_fid_index(rif->fid), true); 10455 if (err) 10456 goto err_rif_fdb_op; 10457 10458 err = mlxsw_sp_fid_rif_set(rif->fid, rif); 10459 if (err) 10460 goto err_fid_rif_set; 10461 10462 return 0; 10463 10464err_fid_rif_set: 10465 mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, dev->dev_addr, 10466 mlxsw_sp_fid_index(rif->fid), false); 10467err_rif_fdb_op: 10468 mlxsw_sp_rif_macvlan_flush(rif); 10469err_macvlan_replay: 10470 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC, 10471 mlxsw_sp_router_port(mlxsw_sp), false); 10472err_fid_bc_flood_set: 10473 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC, 10474 mlxsw_sp_router_port(mlxsw_sp), false); 10475err_fid_mc_flood_set: 10476 mlxsw_sp_rif_vlan_op(rif, vid, 0, false); 10477err_rif_vlan_fid_op: 10478 mlxsw_sp_rif_mac_profile_put(mlxsw_sp, mac_profile); 10479 return err; 10480} 10481 10482static void mlxsw_sp_rif_vlan_deconfigure(struct mlxsw_sp_rif *rif) 10483{ 10484 struct net_device *dev = mlxsw_sp_rif_dev(rif); 10485 u16 vid = mlxsw_sp_fid_8021q_vid(rif->fid); 10486 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 10487 10488 mlxsw_sp_fid_rif_unset(rif->fid); 10489 mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, dev->dev_addr, 10490 mlxsw_sp_fid_index(rif->fid), false); 10491 mlxsw_sp_rif_macvlan_flush(rif); 10492 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC, 10493 mlxsw_sp_router_port(mlxsw_sp), false); 10494 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC, 10495 mlxsw_sp_router_port(mlxsw_sp), false); 10496 mlxsw_sp_rif_vlan_op(rif, vid, 0, false); 10497 mlxsw_sp_rif_mac_profile_put(rif->mlxsw_sp, rif->mac_profile_id); 10498} 10499 10500static int mlxsw_sp1_rif_vlan_configure(struct mlxsw_sp_rif *rif, 10501 struct netlink_ext_ack *extack) 10502{ 10503 return mlxsw_sp_rif_vlan_configure(rif, 0, extack); 10504} 10505 10506static const struct mlxsw_sp_rif_ops mlxsw_sp1_rif_vlan_ops = { 10507 .type = MLXSW_SP_RIF_TYPE_VLAN, 10508 .rif_size = sizeof(struct mlxsw_sp_rif), 10509 .configure = mlxsw_sp1_rif_vlan_configure, 10510 .deconfigure = mlxsw_sp_rif_vlan_deconfigure, 10511 .fid_get = mlxsw_sp_rif_vlan_fid_get, 10512 .fdb_del = mlxsw_sp_rif_vlan_fdb_del, 10513}; 10514 10515static int mlxsw_sp2_rif_vlan_configure(struct mlxsw_sp_rif *rif, 10516 struct netlink_ext_ack *extack) 10517{ 10518 u16 efid = mlxsw_sp_fid_index(rif->fid); 10519 10520 return mlxsw_sp_rif_vlan_configure(rif, efid, extack); 10521} 10522 10523static const struct mlxsw_sp_rif_ops mlxsw_sp2_rif_vlan_ops = { 10524 .type = MLXSW_SP_RIF_TYPE_VLAN, 10525 .rif_size = sizeof(struct mlxsw_sp_rif), 10526 .configure = mlxsw_sp2_rif_vlan_configure, 10527 .deconfigure = mlxsw_sp_rif_vlan_deconfigure, 10528 .fid_get = mlxsw_sp_rif_vlan_fid_get, 10529 .fdb_del = mlxsw_sp_rif_vlan_fdb_del, 10530}; 10531 10532static struct mlxsw_sp_rif_ipip_lb * 10533mlxsw_sp_rif_ipip_lb_rif(struct mlxsw_sp_rif *rif) 10534{ 10535 return container_of(rif, struct mlxsw_sp_rif_ipip_lb, common); 10536} 10537 10538static void 10539mlxsw_sp_rif_ipip_lb_setup(struct mlxsw_sp_rif *rif, 10540 const struct mlxsw_sp_rif_params *params) 10541{ 10542 struct mlxsw_sp_rif_params_ipip_lb *params_lb; 10543 struct mlxsw_sp_rif_ipip_lb *rif_lb; 10544 10545 params_lb = container_of(params, struct mlxsw_sp_rif_params_ipip_lb, 10546 common); 10547 rif_lb = mlxsw_sp_rif_ipip_lb_rif(rif); 10548 rif_lb->lb_config = params_lb->lb_config; 10549} 10550 10551static int 10552mlxsw_sp1_rif_ipip_lb_configure(struct mlxsw_sp_rif *rif, 10553 struct netlink_ext_ack *extack) 10554{ 10555 struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif); 10556 struct net_device *dev = mlxsw_sp_rif_dev(rif); 10557 u32 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(dev); 10558 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 10559 struct mlxsw_sp_vr *ul_vr; 10560 int err; 10561 10562 ul_vr = mlxsw_sp_vr_get(mlxsw_sp, ul_tb_id, extack); 10563 if (IS_ERR(ul_vr)) 10564 return PTR_ERR(ul_vr); 10565 10566 err = mlxsw_sp_rif_ipip_lb_op(lb_rif, ul_vr->id, 0, true); 10567 if (err) 10568 goto err_loopback_op; 10569 10570 lb_rif->ul_vr_id = ul_vr->id; 10571 lb_rif->ul_rif_id = 0; 10572 ++ul_vr->rif_count; 10573 return 0; 10574 10575err_loopback_op: 10576 mlxsw_sp_vr_put(mlxsw_sp, ul_vr); 10577 return err; 10578} 10579 10580static void mlxsw_sp1_rif_ipip_lb_deconfigure(struct mlxsw_sp_rif *rif) 10581{ 10582 struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif); 10583 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 10584 struct mlxsw_sp_vr *ul_vr; 10585 10586 ul_vr = &mlxsw_sp->router->vrs[lb_rif->ul_vr_id]; 10587 mlxsw_sp_rif_ipip_lb_op(lb_rif, ul_vr->id, 0, false); 10588 10589 --ul_vr->rif_count; 10590 mlxsw_sp_vr_put(mlxsw_sp, ul_vr); 10591} 10592 10593static const struct mlxsw_sp_rif_ops mlxsw_sp1_rif_ipip_lb_ops = { 10594 .type = MLXSW_SP_RIF_TYPE_IPIP_LB, 10595 .rif_size = sizeof(struct mlxsw_sp_rif_ipip_lb), 10596 .setup = mlxsw_sp_rif_ipip_lb_setup, 10597 .configure = mlxsw_sp1_rif_ipip_lb_configure, 10598 .deconfigure = mlxsw_sp1_rif_ipip_lb_deconfigure, 10599}; 10600 10601static const struct mlxsw_sp_rif_ops *mlxsw_sp1_rif_ops_arr[] = { 10602 [MLXSW_SP_RIF_TYPE_SUBPORT] = &mlxsw_sp_rif_subport_ops, 10603 [MLXSW_SP_RIF_TYPE_VLAN] = &mlxsw_sp1_rif_vlan_ops, 10604 [MLXSW_SP_RIF_TYPE_FID] = &mlxsw_sp_rif_fid_ops, 10605 [MLXSW_SP_RIF_TYPE_IPIP_LB] = &mlxsw_sp1_rif_ipip_lb_ops, 10606}; 10607 10608static int 10609mlxsw_sp_rif_ipip_lb_ul_rif_op(struct mlxsw_sp_rif *ul_rif, bool enable) 10610{ 10611 struct mlxsw_sp *mlxsw_sp = ul_rif->mlxsw_sp; 10612 char ritr_pl[MLXSW_REG_RITR_LEN]; 10613 10614 mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_LOOPBACK_IF, 10615 ul_rif->rif_index, ul_rif->vr_id, IP_MAX_MTU); 10616 mlxsw_reg_ritr_loopback_protocol_set(ritr_pl, 10617 MLXSW_REG_RITR_LOOPBACK_GENERIC); 10618 10619 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 10620} 10621 10622static struct mlxsw_sp_rif * 10623mlxsw_sp_ul_rif_create(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_vr *vr, 10624 struct mlxsw_sp_crif *ul_crif, 10625 struct netlink_ext_ack *extack) 10626{ 10627 struct mlxsw_sp_rif *ul_rif; 10628 u8 rif_entries = 1; 10629 u16 rif_index; 10630 int err; 10631 10632 err = mlxsw_sp_rif_index_alloc(mlxsw_sp, &rif_index, rif_entries); 10633 if (err) { 10634 NL_SET_ERR_MSG_MOD(extack, "Exceeded number of supported router interfaces"); 10635 return ERR_PTR(err); 10636 } 10637 10638 ul_rif = mlxsw_sp_rif_alloc(sizeof(*ul_rif), rif_index, vr->id, 10639 ul_crif); 10640 if (!ul_rif) { 10641 err = -ENOMEM; 10642 goto err_rif_alloc; 10643 } 10644 10645 mlxsw_sp->router->rifs[rif_index] = ul_rif; 10646 ul_rif->mlxsw_sp = mlxsw_sp; 10647 ul_rif->rif_entries = rif_entries; 10648 err = mlxsw_sp_rif_ipip_lb_ul_rif_op(ul_rif, true); 10649 if (err) 10650 goto ul_rif_op_err; 10651 10652 atomic_add(rif_entries, &mlxsw_sp->router->rifs_count); 10653 return ul_rif; 10654 10655ul_rif_op_err: 10656 mlxsw_sp->router->rifs[rif_index] = NULL; 10657 mlxsw_sp_rif_free(ul_rif); 10658err_rif_alloc: 10659 mlxsw_sp_rif_index_free(mlxsw_sp, rif_index, rif_entries); 10660 return ERR_PTR(err); 10661} 10662 10663static void mlxsw_sp_ul_rif_destroy(struct mlxsw_sp_rif *ul_rif) 10664{ 10665 struct mlxsw_sp *mlxsw_sp = ul_rif->mlxsw_sp; 10666 u8 rif_entries = ul_rif->rif_entries; 10667 u16 rif_index = ul_rif->rif_index; 10668 10669 atomic_sub(rif_entries, &mlxsw_sp->router->rifs_count); 10670 mlxsw_sp_rif_ipip_lb_ul_rif_op(ul_rif, false); 10671 mlxsw_sp->router->rifs[ul_rif->rif_index] = NULL; 10672 mlxsw_sp_rif_free(ul_rif); 10673 mlxsw_sp_rif_index_free(mlxsw_sp, rif_index, rif_entries); 10674} 10675 10676static struct mlxsw_sp_rif * 10677mlxsw_sp_ul_rif_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id, 10678 struct mlxsw_sp_crif *ul_crif, 10679 struct netlink_ext_ack *extack) 10680{ 10681 struct mlxsw_sp_vr *vr; 10682 int err; 10683 10684 vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id, extack); 10685 if (IS_ERR(vr)) 10686 return ERR_CAST(vr); 10687 10688 if (refcount_inc_not_zero(&vr->ul_rif_refcnt)) 10689 return vr->ul_rif; 10690 10691 vr->ul_rif = mlxsw_sp_ul_rif_create(mlxsw_sp, vr, ul_crif, extack); 10692 if (IS_ERR(vr->ul_rif)) { 10693 err = PTR_ERR(vr->ul_rif); 10694 goto err_ul_rif_create; 10695 } 10696 10697 vr->rif_count++; 10698 refcount_set(&vr->ul_rif_refcnt, 1); 10699 10700 return vr->ul_rif; 10701 10702err_ul_rif_create: 10703 mlxsw_sp_vr_put(mlxsw_sp, vr); 10704 return ERR_PTR(err); 10705} 10706 10707static void mlxsw_sp_ul_rif_put(struct mlxsw_sp_rif *ul_rif) 10708{ 10709 struct mlxsw_sp *mlxsw_sp = ul_rif->mlxsw_sp; 10710 struct mlxsw_sp_vr *vr; 10711 10712 vr = &mlxsw_sp->router->vrs[ul_rif->vr_id]; 10713 10714 if (!refcount_dec_and_test(&vr->ul_rif_refcnt)) 10715 return; 10716 10717 vr->rif_count--; 10718 mlxsw_sp_ul_rif_destroy(ul_rif); 10719 mlxsw_sp_vr_put(mlxsw_sp, vr); 10720} 10721 10722int mlxsw_sp_router_ul_rif_get(struct mlxsw_sp *mlxsw_sp, u32 ul_tb_id, 10723 u16 *ul_rif_index) 10724{ 10725 struct mlxsw_sp_rif *ul_rif; 10726 int err = 0; 10727 10728 mutex_lock(&mlxsw_sp->router->lock); 10729 ul_rif = mlxsw_sp_ul_rif_get(mlxsw_sp, ul_tb_id, NULL, NULL); 10730 if (IS_ERR(ul_rif)) { 10731 err = PTR_ERR(ul_rif); 10732 goto out; 10733 } 10734 *ul_rif_index = ul_rif->rif_index; 10735out: 10736 mutex_unlock(&mlxsw_sp->router->lock); 10737 return err; 10738} 10739 10740void mlxsw_sp_router_ul_rif_put(struct mlxsw_sp *mlxsw_sp, u16 ul_rif_index) 10741{ 10742 struct mlxsw_sp_rif *ul_rif; 10743 10744 mutex_lock(&mlxsw_sp->router->lock); 10745 ul_rif = mlxsw_sp->router->rifs[ul_rif_index]; 10746 if (WARN_ON(!ul_rif)) 10747 goto out; 10748 10749 mlxsw_sp_ul_rif_put(ul_rif); 10750out: 10751 mutex_unlock(&mlxsw_sp->router->lock); 10752} 10753 10754static int 10755mlxsw_sp2_rif_ipip_lb_configure(struct mlxsw_sp_rif *rif, 10756 struct netlink_ext_ack *extack) 10757{ 10758 struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif); 10759 struct net_device *dev = mlxsw_sp_rif_dev(rif); 10760 u32 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(dev); 10761 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 10762 struct mlxsw_sp_rif *ul_rif; 10763 int err; 10764 10765 ul_rif = mlxsw_sp_ul_rif_get(mlxsw_sp, ul_tb_id, NULL, extack); 10766 if (IS_ERR(ul_rif)) 10767 return PTR_ERR(ul_rif); 10768 10769 err = mlxsw_sp_rif_ipip_lb_op(lb_rif, 0, ul_rif->rif_index, true); 10770 if (err) 10771 goto err_loopback_op; 10772 10773 lb_rif->ul_vr_id = 0; 10774 lb_rif->ul_rif_id = ul_rif->rif_index; 10775 10776 return 0; 10777 10778err_loopback_op: 10779 mlxsw_sp_ul_rif_put(ul_rif); 10780 return err; 10781} 10782 10783static void mlxsw_sp2_rif_ipip_lb_deconfigure(struct mlxsw_sp_rif *rif) 10784{ 10785 struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif); 10786 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 10787 struct mlxsw_sp_rif *ul_rif; 10788 10789 ul_rif = mlxsw_sp_rif_by_index(mlxsw_sp, lb_rif->ul_rif_id); 10790 mlxsw_sp_rif_ipip_lb_op(lb_rif, 0, lb_rif->ul_rif_id, false); 10791 mlxsw_sp_ul_rif_put(ul_rif); 10792} 10793 10794static const struct mlxsw_sp_rif_ops mlxsw_sp2_rif_ipip_lb_ops = { 10795 .type = MLXSW_SP_RIF_TYPE_IPIP_LB, 10796 .rif_size = sizeof(struct mlxsw_sp_rif_ipip_lb), 10797 .setup = mlxsw_sp_rif_ipip_lb_setup, 10798 .configure = mlxsw_sp2_rif_ipip_lb_configure, 10799 .deconfigure = mlxsw_sp2_rif_ipip_lb_deconfigure, 10800}; 10801 10802static const struct mlxsw_sp_rif_ops *mlxsw_sp2_rif_ops_arr[] = { 10803 [MLXSW_SP_RIF_TYPE_SUBPORT] = &mlxsw_sp_rif_subport_ops, 10804 [MLXSW_SP_RIF_TYPE_VLAN] = &mlxsw_sp2_rif_vlan_ops, 10805 [MLXSW_SP_RIF_TYPE_FID] = &mlxsw_sp_rif_fid_ops, 10806 [MLXSW_SP_RIF_TYPE_IPIP_LB] = &mlxsw_sp2_rif_ipip_lb_ops, 10807}; 10808 10809static int mlxsw_sp_rifs_table_init(struct mlxsw_sp *mlxsw_sp) 10810{ 10811 struct gen_pool *rifs_table; 10812 int err; 10813 10814 rifs_table = gen_pool_create(0, -1); 10815 if (!rifs_table) 10816 return -ENOMEM; 10817 10818 gen_pool_set_algo(rifs_table, gen_pool_first_fit_order_align, 10819 NULL); 10820 10821 err = gen_pool_add(rifs_table, MLXSW_SP_ROUTER_GENALLOC_OFFSET, 10822 MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS), -1); 10823 if (err) 10824 goto err_gen_pool_add; 10825 10826 mlxsw_sp->router->rifs_table = rifs_table; 10827 10828 return 0; 10829 10830err_gen_pool_add: 10831 gen_pool_destroy(rifs_table); 10832 return err; 10833} 10834 10835static void mlxsw_sp_rifs_table_fini(struct mlxsw_sp *mlxsw_sp) 10836{ 10837 gen_pool_destroy(mlxsw_sp->router->rifs_table); 10838} 10839 10840static int mlxsw_sp_rifs_init(struct mlxsw_sp *mlxsw_sp) 10841{ 10842 u64 max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); 10843 struct devlink *devlink = priv_to_devlink(mlxsw_sp->core); 10844 struct mlxsw_core *core = mlxsw_sp->core; 10845 int err; 10846 10847 if (!MLXSW_CORE_RES_VALID(core, MAX_RIF_MAC_PROFILES)) 10848 return -EIO; 10849 mlxsw_sp->router->max_rif_mac_profile = 10850 MLXSW_CORE_RES_GET(core, MAX_RIF_MAC_PROFILES); 10851 10852 mlxsw_sp->router->rifs = kcalloc(max_rifs, 10853 sizeof(struct mlxsw_sp_rif *), 10854 GFP_KERNEL); 10855 if (!mlxsw_sp->router->rifs) 10856 return -ENOMEM; 10857 10858 err = mlxsw_sp_rifs_table_init(mlxsw_sp); 10859 if (err) 10860 goto err_rifs_table_init; 10861 10862 idr_init(&mlxsw_sp->router->rif_mac_profiles_idr); 10863 atomic_set(&mlxsw_sp->router->rif_mac_profiles_count, 0); 10864 atomic_set(&mlxsw_sp->router->rifs_count, 0); 10865 devl_resource_occ_get_register(devlink, 10866 MLXSW_SP_RESOURCE_RIF_MAC_PROFILES, 10867 mlxsw_sp_rif_mac_profiles_occ_get, 10868 mlxsw_sp); 10869 devl_resource_occ_get_register(devlink, 10870 MLXSW_SP_RESOURCE_RIFS, 10871 mlxsw_sp_rifs_occ_get, 10872 mlxsw_sp); 10873 10874 return 0; 10875 10876err_rifs_table_init: 10877 kfree(mlxsw_sp->router->rifs); 10878 return err; 10879} 10880 10881static void mlxsw_sp_rifs_fini(struct mlxsw_sp *mlxsw_sp) 10882{ 10883 int max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); 10884 struct devlink *devlink = priv_to_devlink(mlxsw_sp->core); 10885 int i; 10886 10887 WARN_ON_ONCE(atomic_read(&mlxsw_sp->router->rifs_count)); 10888 for (i = 0; i < max_rifs; i++) 10889 WARN_ON_ONCE(mlxsw_sp->router->rifs[i]); 10890 10891 devl_resource_occ_get_unregister(devlink, MLXSW_SP_RESOURCE_RIFS); 10892 devl_resource_occ_get_unregister(devlink, 10893 MLXSW_SP_RESOURCE_RIF_MAC_PROFILES); 10894 WARN_ON(!idr_is_empty(&mlxsw_sp->router->rif_mac_profiles_idr)); 10895 idr_destroy(&mlxsw_sp->router->rif_mac_profiles_idr); 10896 mlxsw_sp_rifs_table_fini(mlxsw_sp); 10897 kfree(mlxsw_sp->router->rifs); 10898} 10899 10900static int 10901mlxsw_sp_ipip_config_tigcr(struct mlxsw_sp *mlxsw_sp) 10902{ 10903 char tigcr_pl[MLXSW_REG_TIGCR_LEN]; 10904 10905 mlxsw_reg_tigcr_pack(tigcr_pl, true, 0); 10906 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(tigcr), tigcr_pl); 10907} 10908 10909static int mlxsw_sp_ipips_init(struct mlxsw_sp *mlxsw_sp) 10910{ 10911 int err; 10912 10913 INIT_LIST_HEAD(&mlxsw_sp->router->ipip_list); 10914 10915 err = mlxsw_sp_ipip_ecn_encap_init(mlxsw_sp); 10916 if (err) 10917 return err; 10918 err = mlxsw_sp_ipip_ecn_decap_init(mlxsw_sp); 10919 if (err) 10920 return err; 10921 10922 return mlxsw_sp_ipip_config_tigcr(mlxsw_sp); 10923} 10924 10925static int mlxsw_sp1_ipips_init(struct mlxsw_sp *mlxsw_sp) 10926{ 10927 mlxsw_sp->router->ipip_ops_arr = mlxsw_sp1_ipip_ops_arr; 10928 return mlxsw_sp_ipips_init(mlxsw_sp); 10929} 10930 10931static int mlxsw_sp2_ipips_init(struct mlxsw_sp *mlxsw_sp) 10932{ 10933 mlxsw_sp->router->ipip_ops_arr = mlxsw_sp2_ipip_ops_arr; 10934 return mlxsw_sp_ipips_init(mlxsw_sp); 10935} 10936 10937static void mlxsw_sp_ipips_fini(struct mlxsw_sp *mlxsw_sp) 10938{ 10939 WARN_ON(!list_empty(&mlxsw_sp->router->ipip_list)); 10940} 10941 10942static void mlxsw_sp_router_fib_dump_flush(struct notifier_block *nb) 10943{ 10944 struct mlxsw_sp_router *router; 10945 10946 /* Flush pending FIB notifications and then flush the device's 10947 * table before requesting another dump. The FIB notification 10948 * block is unregistered, so no need to take RTNL. 10949 */ 10950 mlxsw_core_flush_owq(); 10951 router = container_of(nb, struct mlxsw_sp_router, fib_nb); 10952 mlxsw_sp_router_fib_flush(router->mlxsw_sp); 10953} 10954 10955#ifdef CONFIG_IP_ROUTE_MULTIPATH 10956struct mlxsw_sp_mp_hash_config { 10957 DECLARE_BITMAP(headers, __MLXSW_REG_RECR2_HEADER_CNT); 10958 DECLARE_BITMAP(fields, __MLXSW_REG_RECR2_FIELD_CNT); 10959 DECLARE_BITMAP(inner_headers, __MLXSW_REG_RECR2_HEADER_CNT); 10960 DECLARE_BITMAP(inner_fields, __MLXSW_REG_RECR2_INNER_FIELD_CNT); 10961 bool inc_parsing_depth; 10962}; 10963 10964#define MLXSW_SP_MP_HASH_HEADER_SET(_headers, _header) \ 10965 bitmap_set(_headers, MLXSW_REG_RECR2_##_header, 1) 10966 10967#define MLXSW_SP_MP_HASH_FIELD_SET(_fields, _field) \ 10968 bitmap_set(_fields, MLXSW_REG_RECR2_##_field, 1) 10969 10970#define MLXSW_SP_MP_HASH_FIELD_RANGE_SET(_fields, _field, _nr) \ 10971 bitmap_set(_fields, MLXSW_REG_RECR2_##_field, _nr) 10972 10973static void mlxsw_sp_mp_hash_inner_l3(struct mlxsw_sp_mp_hash_config *config) 10974{ 10975 unsigned long *inner_headers = config->inner_headers; 10976 unsigned long *inner_fields = config->inner_fields; 10977 10978 /* IPv4 inner */ 10979 MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV4_EN_NOT_TCP_NOT_UDP); 10980 MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV4_EN_TCP_UDP); 10981 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV4_SIP0, 4); 10982 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV4_DIP0, 4); 10983 /* IPv6 inner */ 10984 MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV6_EN_NOT_TCP_NOT_UDP); 10985 MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV6_EN_TCP_UDP); 10986 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_SIP0_7); 10987 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV6_SIP8, 8); 10988 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_DIP0_7); 10989 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV6_DIP8, 8); 10990 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_NEXT_HEADER); 10991 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_FLOW_LABEL); 10992} 10993 10994static void mlxsw_sp_mp4_hash_outer_addr(struct mlxsw_sp_mp_hash_config *config) 10995{ 10996 unsigned long *headers = config->headers; 10997 unsigned long *fields = config->fields; 10998 10999 MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV4_EN_NOT_TCP_NOT_UDP); 11000 MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV4_EN_TCP_UDP); 11001 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV4_SIP0, 4); 11002 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV4_DIP0, 4); 11003} 11004 11005static void 11006mlxsw_sp_mp_hash_inner_custom(struct mlxsw_sp_mp_hash_config *config, 11007 u32 hash_fields) 11008{ 11009 unsigned long *inner_headers = config->inner_headers; 11010 unsigned long *inner_fields = config->inner_fields; 11011 11012 /* IPv4 Inner */ 11013 MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV4_EN_NOT_TCP_NOT_UDP); 11014 MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV4_EN_TCP_UDP); 11015 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_SRC_IP) 11016 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV4_SIP0, 4); 11017 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_DST_IP) 11018 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV4_DIP0, 4); 11019 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_IP_PROTO) 11020 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV4_PROTOCOL); 11021 /* IPv6 inner */ 11022 MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV6_EN_NOT_TCP_NOT_UDP); 11023 MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV6_EN_TCP_UDP); 11024 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_SRC_IP) { 11025 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_SIP0_7); 11026 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV6_SIP8, 8); 11027 } 11028 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_DST_IP) { 11029 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_DIP0_7); 11030 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV6_DIP8, 8); 11031 } 11032 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_IP_PROTO) 11033 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_NEXT_HEADER); 11034 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_FLOWLABEL) 11035 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_FLOW_LABEL); 11036 /* L4 inner */ 11037 MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, TCP_UDP_EN_IPV4); 11038 MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, TCP_UDP_EN_IPV6); 11039 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_SRC_PORT) 11040 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_TCP_UDP_SPORT); 11041 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_DST_PORT) 11042 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_TCP_UDP_DPORT); 11043} 11044 11045static void mlxsw_sp_mp4_hash_init(struct mlxsw_sp *mlxsw_sp, 11046 struct mlxsw_sp_mp_hash_config *config) 11047{ 11048 struct net *net = mlxsw_sp_net(mlxsw_sp); 11049 unsigned long *headers = config->headers; 11050 unsigned long *fields = config->fields; 11051 u32 hash_fields; 11052 11053 switch (READ_ONCE(net->ipv4.sysctl_fib_multipath_hash_policy)) { 11054 case 0: 11055 mlxsw_sp_mp4_hash_outer_addr(config); 11056 break; 11057 case 1: 11058 mlxsw_sp_mp4_hash_outer_addr(config); 11059 MLXSW_SP_MP_HASH_HEADER_SET(headers, TCP_UDP_EN_IPV4); 11060 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV4_PROTOCOL); 11061 MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_SPORT); 11062 MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_DPORT); 11063 break; 11064 case 2: 11065 /* Outer */ 11066 mlxsw_sp_mp4_hash_outer_addr(config); 11067 /* Inner */ 11068 mlxsw_sp_mp_hash_inner_l3(config); 11069 break; 11070 case 3: 11071 hash_fields = READ_ONCE(net->ipv4.sysctl_fib_multipath_hash_fields); 11072 /* Outer */ 11073 MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV4_EN_NOT_TCP_NOT_UDP); 11074 MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV4_EN_TCP_UDP); 11075 MLXSW_SP_MP_HASH_HEADER_SET(headers, TCP_UDP_EN_IPV4); 11076 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_SRC_IP) 11077 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV4_SIP0, 4); 11078 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_DST_IP) 11079 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV4_DIP0, 4); 11080 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_IP_PROTO) 11081 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV4_PROTOCOL); 11082 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_SRC_PORT) 11083 MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_SPORT); 11084 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_DST_PORT) 11085 MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_DPORT); 11086 /* Inner */ 11087 mlxsw_sp_mp_hash_inner_custom(config, hash_fields); 11088 break; 11089 } 11090} 11091 11092static void mlxsw_sp_mp6_hash_outer_addr(struct mlxsw_sp_mp_hash_config *config) 11093{ 11094 unsigned long *headers = config->headers; 11095 unsigned long *fields = config->fields; 11096 11097 MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV6_EN_NOT_TCP_NOT_UDP); 11098 MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV6_EN_TCP_UDP); 11099 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_SIP0_7); 11100 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV6_SIP8, 8); 11101 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_DIP0_7); 11102 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV6_DIP8, 8); 11103} 11104 11105static void mlxsw_sp_mp6_hash_init(struct mlxsw_sp *mlxsw_sp, 11106 struct mlxsw_sp_mp_hash_config *config) 11107{ 11108 u32 hash_fields = ip6_multipath_hash_fields(mlxsw_sp_net(mlxsw_sp)); 11109 unsigned long *headers = config->headers; 11110 unsigned long *fields = config->fields; 11111 11112 switch (ip6_multipath_hash_policy(mlxsw_sp_net(mlxsw_sp))) { 11113 case 0: 11114 mlxsw_sp_mp6_hash_outer_addr(config); 11115 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_NEXT_HEADER); 11116 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_FLOW_LABEL); 11117 break; 11118 case 1: 11119 mlxsw_sp_mp6_hash_outer_addr(config); 11120 MLXSW_SP_MP_HASH_HEADER_SET(headers, TCP_UDP_EN_IPV6); 11121 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_NEXT_HEADER); 11122 MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_SPORT); 11123 MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_DPORT); 11124 break; 11125 case 2: 11126 /* Outer */ 11127 mlxsw_sp_mp6_hash_outer_addr(config); 11128 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_NEXT_HEADER); 11129 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_FLOW_LABEL); 11130 /* Inner */ 11131 mlxsw_sp_mp_hash_inner_l3(config); 11132 config->inc_parsing_depth = true; 11133 break; 11134 case 3: 11135 /* Outer */ 11136 MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV6_EN_NOT_TCP_NOT_UDP); 11137 MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV6_EN_TCP_UDP); 11138 MLXSW_SP_MP_HASH_HEADER_SET(headers, TCP_UDP_EN_IPV6); 11139 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_SRC_IP) { 11140 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_SIP0_7); 11141 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV6_SIP8, 8); 11142 } 11143 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_DST_IP) { 11144 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_DIP0_7); 11145 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV6_DIP8, 8); 11146 } 11147 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_IP_PROTO) 11148 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_NEXT_HEADER); 11149 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_FLOWLABEL) 11150 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_FLOW_LABEL); 11151 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_SRC_PORT) 11152 MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_SPORT); 11153 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_DST_PORT) 11154 MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_DPORT); 11155 /* Inner */ 11156 mlxsw_sp_mp_hash_inner_custom(config, hash_fields); 11157 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_MASK) 11158 config->inc_parsing_depth = true; 11159 break; 11160 } 11161} 11162 11163static int mlxsw_sp_mp_hash_parsing_depth_adjust(struct mlxsw_sp *mlxsw_sp, 11164 bool old_inc_parsing_depth, 11165 bool new_inc_parsing_depth) 11166{ 11167 int err; 11168 11169 if (!old_inc_parsing_depth && new_inc_parsing_depth) { 11170 err = mlxsw_sp_parsing_depth_inc(mlxsw_sp); 11171 if (err) 11172 return err; 11173 mlxsw_sp->router->inc_parsing_depth = true; 11174 } else if (old_inc_parsing_depth && !new_inc_parsing_depth) { 11175 mlxsw_sp_parsing_depth_dec(mlxsw_sp); 11176 mlxsw_sp->router->inc_parsing_depth = false; 11177 } 11178 11179 return 0; 11180} 11181 11182static int mlxsw_sp_mp_hash_init(struct mlxsw_sp *mlxsw_sp) 11183{ 11184 bool old_inc_parsing_depth, new_inc_parsing_depth; 11185 struct mlxsw_sp_mp_hash_config config = {}; 11186 char recr2_pl[MLXSW_REG_RECR2_LEN]; 11187 unsigned long bit; 11188 u32 seed; 11189 int err; 11190 11191 seed = jhash(mlxsw_sp->base_mac, sizeof(mlxsw_sp->base_mac), 0); 11192 mlxsw_reg_recr2_pack(recr2_pl, seed); 11193 mlxsw_sp_mp4_hash_init(mlxsw_sp, &config); 11194 mlxsw_sp_mp6_hash_init(mlxsw_sp, &config); 11195 11196 old_inc_parsing_depth = mlxsw_sp->router->inc_parsing_depth; 11197 new_inc_parsing_depth = config.inc_parsing_depth; 11198 err = mlxsw_sp_mp_hash_parsing_depth_adjust(mlxsw_sp, 11199 old_inc_parsing_depth, 11200 new_inc_parsing_depth); 11201 if (err) 11202 return err; 11203 11204 for_each_set_bit(bit, config.headers, __MLXSW_REG_RECR2_HEADER_CNT) 11205 mlxsw_reg_recr2_outer_header_enables_set(recr2_pl, bit, 1); 11206 for_each_set_bit(bit, config.fields, __MLXSW_REG_RECR2_FIELD_CNT) 11207 mlxsw_reg_recr2_outer_header_fields_enable_set(recr2_pl, bit, 1); 11208 for_each_set_bit(bit, config.inner_headers, __MLXSW_REG_RECR2_HEADER_CNT) 11209 mlxsw_reg_recr2_inner_header_enables_set(recr2_pl, bit, 1); 11210 for_each_set_bit(bit, config.inner_fields, __MLXSW_REG_RECR2_INNER_FIELD_CNT) 11211 mlxsw_reg_recr2_inner_header_fields_enable_set(recr2_pl, bit, 1); 11212 11213 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(recr2), recr2_pl); 11214 if (err) 11215 goto err_reg_write; 11216 11217 return 0; 11218 11219err_reg_write: 11220 mlxsw_sp_mp_hash_parsing_depth_adjust(mlxsw_sp, new_inc_parsing_depth, 11221 old_inc_parsing_depth); 11222 return err; 11223} 11224 11225static void mlxsw_sp_mp_hash_fini(struct mlxsw_sp *mlxsw_sp) 11226{ 11227 bool old_inc_parsing_depth = mlxsw_sp->router->inc_parsing_depth; 11228 11229 mlxsw_sp_mp_hash_parsing_depth_adjust(mlxsw_sp, old_inc_parsing_depth, 11230 false); 11231} 11232#else 11233static int mlxsw_sp_mp_hash_init(struct mlxsw_sp *mlxsw_sp) 11234{ 11235 return 0; 11236} 11237 11238static void mlxsw_sp_mp_hash_fini(struct mlxsw_sp *mlxsw_sp) 11239{ 11240} 11241#endif 11242 11243static int mlxsw_sp_dscp_init(struct mlxsw_sp *mlxsw_sp) 11244{ 11245 char rdpm_pl[MLXSW_REG_RDPM_LEN]; 11246 unsigned int i; 11247 11248 MLXSW_REG_ZERO(rdpm, rdpm_pl); 11249 11250 /* HW is determining switch priority based on DSCP-bits, but the 11251 * kernel is still doing that based on the ToS. Since there's a 11252 * mismatch in bits we need to make sure to translate the right 11253 * value ToS would observe, skipping the 2 least-significant ECN bits. 11254 */ 11255 for (i = 0; i < MLXSW_REG_RDPM_DSCP_ENTRY_REC_MAX_COUNT; i++) 11256 mlxsw_reg_rdpm_pack(rdpm_pl, i, rt_tos2priority(i << 2)); 11257 11258 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rdpm), rdpm_pl); 11259} 11260 11261static int __mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp) 11262{ 11263 struct net *net = mlxsw_sp_net(mlxsw_sp); 11264 char rgcr_pl[MLXSW_REG_RGCR_LEN]; 11265 u64 max_rifs; 11266 bool usp; 11267 11268 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_RIFS)) 11269 return -EIO; 11270 max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); 11271 usp = READ_ONCE(net->ipv4.sysctl_ip_fwd_update_priority); 11272 11273 mlxsw_reg_rgcr_pack(rgcr_pl, true, true); 11274 mlxsw_reg_rgcr_max_router_interfaces_set(rgcr_pl, max_rifs); 11275 mlxsw_reg_rgcr_usp_set(rgcr_pl, usp); 11276 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl); 11277} 11278 11279static void __mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp) 11280{ 11281 char rgcr_pl[MLXSW_REG_RGCR_LEN]; 11282 11283 mlxsw_reg_rgcr_pack(rgcr_pl, false, false); 11284 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl); 11285} 11286 11287static int mlxsw_sp_lb_rif_init(struct mlxsw_sp *mlxsw_sp, 11288 struct netlink_ext_ack *extack) 11289{ 11290 struct mlxsw_sp_router *router = mlxsw_sp->router; 11291 struct mlxsw_sp_rif *lb_rif; 11292 int err; 11293 11294 router->lb_crif = mlxsw_sp_crif_alloc(NULL); 11295 if (!router->lb_crif) 11296 return -ENOMEM; 11297 11298 /* Create a generic loopback RIF associated with the main table 11299 * (default VRF). Any table can be used, but the main table exists 11300 * anyway, so we do not waste resources. Loopback RIFs are usually 11301 * created with a NULL CRIF, but this RIF is used as a fallback RIF 11302 * for blackhole nexthops, and nexthops expect to have a valid CRIF. 11303 */ 11304 lb_rif = mlxsw_sp_ul_rif_get(mlxsw_sp, RT_TABLE_MAIN, router->lb_crif, 11305 extack); 11306 if (IS_ERR(lb_rif)) { 11307 err = PTR_ERR(lb_rif); 11308 goto err_ul_rif_get; 11309 } 11310 11311 return 0; 11312 11313err_ul_rif_get: 11314 mlxsw_sp_crif_free(router->lb_crif); 11315 return err; 11316} 11317 11318static void mlxsw_sp_lb_rif_fini(struct mlxsw_sp *mlxsw_sp) 11319{ 11320 mlxsw_sp_ul_rif_put(mlxsw_sp->router->lb_crif->rif); 11321 mlxsw_sp_crif_free(mlxsw_sp->router->lb_crif); 11322} 11323 11324static int mlxsw_sp1_router_init(struct mlxsw_sp *mlxsw_sp) 11325{ 11326 size_t size_ranges_count = ARRAY_SIZE(mlxsw_sp1_adj_grp_size_ranges); 11327 11328 mlxsw_sp->router->rif_ops_arr = mlxsw_sp1_rif_ops_arr; 11329 mlxsw_sp->router->adj_grp_size_ranges = mlxsw_sp1_adj_grp_size_ranges; 11330 mlxsw_sp->router->adj_grp_size_ranges_count = size_ranges_count; 11331 11332 return 0; 11333} 11334 11335const struct mlxsw_sp_router_ops mlxsw_sp1_router_ops = { 11336 .init = mlxsw_sp1_router_init, 11337 .ipips_init = mlxsw_sp1_ipips_init, 11338}; 11339 11340static int mlxsw_sp2_router_init(struct mlxsw_sp *mlxsw_sp) 11341{ 11342 size_t size_ranges_count = ARRAY_SIZE(mlxsw_sp2_adj_grp_size_ranges); 11343 11344 mlxsw_sp->router->rif_ops_arr = mlxsw_sp2_rif_ops_arr; 11345 mlxsw_sp->router->adj_grp_size_ranges = mlxsw_sp2_adj_grp_size_ranges; 11346 mlxsw_sp->router->adj_grp_size_ranges_count = size_ranges_count; 11347 11348 return 0; 11349} 11350 11351const struct mlxsw_sp_router_ops mlxsw_sp2_router_ops = { 11352 .init = mlxsw_sp2_router_init, 11353 .ipips_init = mlxsw_sp2_ipips_init, 11354}; 11355 11356int mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp, 11357 struct netlink_ext_ack *extack) 11358{ 11359 struct mlxsw_sp_router *router; 11360 struct notifier_block *nb; 11361 int err; 11362 11363 router = kzalloc(sizeof(*mlxsw_sp->router), GFP_KERNEL); 11364 if (!router) 11365 return -ENOMEM; 11366 mutex_init(&router->lock); 11367 mlxsw_sp->router = router; 11368 router->mlxsw_sp = mlxsw_sp; 11369 11370 err = mlxsw_sp->router_ops->init(mlxsw_sp); 11371 if (err) 11372 goto err_router_ops_init; 11373 11374 INIT_LIST_HEAD(&mlxsw_sp->router->nh_res_grp_list); 11375 INIT_DELAYED_WORK(&mlxsw_sp->router->nh_grp_activity_dw, 11376 mlxsw_sp_nh_grp_activity_work); 11377 INIT_LIST_HEAD(&mlxsw_sp->router->nexthop_neighs_list); 11378 err = __mlxsw_sp_router_init(mlxsw_sp); 11379 if (err) 11380 goto err_router_init; 11381 11382 err = mlxsw_sp->router_ops->ipips_init(mlxsw_sp); 11383 if (err) 11384 goto err_ipips_init; 11385 11386 err = rhashtable_init(&mlxsw_sp->router->crif_ht, 11387 &mlxsw_sp_crif_ht_params); 11388 if (err) 11389 goto err_crif_ht_init; 11390 11391 err = mlxsw_sp_rifs_init(mlxsw_sp); 11392 if (err) 11393 goto err_rifs_init; 11394 11395 err = rhashtable_init(&mlxsw_sp->router->nexthop_ht, 11396 &mlxsw_sp_nexthop_ht_params); 11397 if (err) 11398 goto err_nexthop_ht_init; 11399 11400 err = rhashtable_init(&mlxsw_sp->router->nexthop_group_ht, 11401 &mlxsw_sp_nexthop_group_ht_params); 11402 if (err) 11403 goto err_nexthop_group_ht_init; 11404 11405 INIT_LIST_HEAD(&mlxsw_sp->router->nexthop_list); 11406 err = mlxsw_sp_lpm_init(mlxsw_sp); 11407 if (err) 11408 goto err_lpm_init; 11409 11410 err = mlxsw_sp_mr_init(mlxsw_sp, &mlxsw_sp_mr_tcam_ops); 11411 if (err) 11412 goto err_mr_init; 11413 11414 err = mlxsw_sp_vrs_init(mlxsw_sp); 11415 if (err) 11416 goto err_vrs_init; 11417 11418 err = mlxsw_sp_lb_rif_init(mlxsw_sp, extack); 11419 if (err) 11420 goto err_lb_rif_init; 11421 11422 err = mlxsw_sp_neigh_init(mlxsw_sp); 11423 if (err) 11424 goto err_neigh_init; 11425 11426 err = mlxsw_sp_mp_hash_init(mlxsw_sp); 11427 if (err) 11428 goto err_mp_hash_init; 11429 11430 err = mlxsw_sp_dscp_init(mlxsw_sp); 11431 if (err) 11432 goto err_dscp_init; 11433 11434 router->inetaddr_nb.notifier_call = mlxsw_sp_inetaddr_event; 11435 err = register_inetaddr_notifier(&router->inetaddr_nb); 11436 if (err) 11437 goto err_register_inetaddr_notifier; 11438 11439 router->inet6addr_nb.notifier_call = mlxsw_sp_inet6addr_event; 11440 err = register_inet6addr_notifier(&router->inet6addr_nb); 11441 if (err) 11442 goto err_register_inet6addr_notifier; 11443 11444 router->inetaddr_valid_nb.notifier_call = mlxsw_sp_inetaddr_valid_event; 11445 err = register_inetaddr_validator_notifier(&router->inetaddr_valid_nb); 11446 if (err) 11447 goto err_register_inetaddr_valid_notifier; 11448 11449 nb = &router->inet6addr_valid_nb; 11450 nb->notifier_call = mlxsw_sp_inet6addr_valid_event; 11451 err = register_inet6addr_validator_notifier(nb); 11452 if (err) 11453 goto err_register_inet6addr_valid_notifier; 11454 11455 mlxsw_sp->router->netevent_nb.notifier_call = 11456 mlxsw_sp_router_netevent_event; 11457 err = register_netevent_notifier(&mlxsw_sp->router->netevent_nb); 11458 if (err) 11459 goto err_register_netevent_notifier; 11460 11461 mlxsw_sp->router->netdevice_nb.notifier_call = 11462 mlxsw_sp_router_netdevice_event; 11463 err = register_netdevice_notifier_net(mlxsw_sp_net(mlxsw_sp), 11464 &mlxsw_sp->router->netdevice_nb); 11465 if (err) 11466 goto err_register_netdev_notifier; 11467 11468 mlxsw_sp->router->nexthop_nb.notifier_call = 11469 mlxsw_sp_nexthop_obj_event; 11470 err = register_nexthop_notifier(mlxsw_sp_net(mlxsw_sp), 11471 &mlxsw_sp->router->nexthop_nb, 11472 extack); 11473 if (err) 11474 goto err_register_nexthop_notifier; 11475 11476 mlxsw_sp->router->fib_nb.notifier_call = mlxsw_sp_router_fib_event; 11477 err = register_fib_notifier(mlxsw_sp_net(mlxsw_sp), 11478 &mlxsw_sp->router->fib_nb, 11479 mlxsw_sp_router_fib_dump_flush, extack); 11480 if (err) 11481 goto err_register_fib_notifier; 11482 11483 return 0; 11484 11485err_register_fib_notifier: 11486 unregister_nexthop_notifier(mlxsw_sp_net(mlxsw_sp), 11487 &mlxsw_sp->router->nexthop_nb); 11488err_register_nexthop_notifier: 11489 unregister_netdevice_notifier_net(mlxsw_sp_net(mlxsw_sp), 11490 &router->netdevice_nb); 11491err_register_netdev_notifier: 11492 unregister_netevent_notifier(&mlxsw_sp->router->netevent_nb); 11493err_register_netevent_notifier: 11494 unregister_inet6addr_validator_notifier(&router->inet6addr_valid_nb); 11495err_register_inet6addr_valid_notifier: 11496 unregister_inetaddr_validator_notifier(&router->inetaddr_valid_nb); 11497err_register_inetaddr_valid_notifier: 11498 unregister_inet6addr_notifier(&router->inet6addr_nb); 11499err_register_inet6addr_notifier: 11500 unregister_inetaddr_notifier(&router->inetaddr_nb); 11501err_register_inetaddr_notifier: 11502 mlxsw_core_flush_owq(); 11503err_dscp_init: 11504 mlxsw_sp_mp_hash_fini(mlxsw_sp); 11505err_mp_hash_init: 11506 mlxsw_sp_neigh_fini(mlxsw_sp); 11507err_neigh_init: 11508 mlxsw_sp_lb_rif_fini(mlxsw_sp); 11509err_lb_rif_init: 11510 mlxsw_sp_vrs_fini(mlxsw_sp); 11511err_vrs_init: 11512 mlxsw_sp_mr_fini(mlxsw_sp); 11513err_mr_init: 11514 mlxsw_sp_lpm_fini(mlxsw_sp); 11515err_lpm_init: 11516 rhashtable_destroy(&mlxsw_sp->router->nexthop_group_ht); 11517err_nexthop_group_ht_init: 11518 rhashtable_destroy(&mlxsw_sp->router->nexthop_ht); 11519err_nexthop_ht_init: 11520 mlxsw_sp_rifs_fini(mlxsw_sp); 11521err_rifs_init: 11522 rhashtable_destroy(&mlxsw_sp->router->crif_ht); 11523err_crif_ht_init: 11524 mlxsw_sp_ipips_fini(mlxsw_sp); 11525err_ipips_init: 11526 __mlxsw_sp_router_fini(mlxsw_sp); 11527err_router_init: 11528 cancel_delayed_work_sync(&mlxsw_sp->router->nh_grp_activity_dw); 11529err_router_ops_init: 11530 mutex_destroy(&mlxsw_sp->router->lock); 11531 kfree(mlxsw_sp->router); 11532 return err; 11533} 11534 11535void mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp) 11536{ 11537 struct mlxsw_sp_router *router = mlxsw_sp->router; 11538 11539 unregister_fib_notifier(mlxsw_sp_net(mlxsw_sp), &router->fib_nb); 11540 unregister_nexthop_notifier(mlxsw_sp_net(mlxsw_sp), 11541 &router->nexthop_nb); 11542 unregister_netdevice_notifier_net(mlxsw_sp_net(mlxsw_sp), 11543 &router->netdevice_nb); 11544 unregister_netevent_notifier(&router->netevent_nb); 11545 unregister_inet6addr_validator_notifier(&router->inet6addr_valid_nb); 11546 unregister_inetaddr_validator_notifier(&router->inetaddr_valid_nb); 11547 unregister_inet6addr_notifier(&router->inet6addr_nb); 11548 unregister_inetaddr_notifier(&router->inetaddr_nb); 11549 mlxsw_core_flush_owq(); 11550 mlxsw_sp_mp_hash_fini(mlxsw_sp); 11551 mlxsw_sp_neigh_fini(mlxsw_sp); 11552 mlxsw_sp_lb_rif_fini(mlxsw_sp); 11553 mlxsw_sp_vrs_fini(mlxsw_sp); 11554 mlxsw_sp_mr_fini(mlxsw_sp); 11555 mlxsw_sp_lpm_fini(mlxsw_sp); 11556 rhashtable_destroy(&router->nexthop_group_ht); 11557 rhashtable_destroy(&router->nexthop_ht); 11558 mlxsw_sp_rifs_fini(mlxsw_sp); 11559 rhashtable_destroy(&mlxsw_sp->router->crif_ht); 11560 mlxsw_sp_ipips_fini(mlxsw_sp); 11561 __mlxsw_sp_router_fini(mlxsw_sp); 11562 cancel_delayed_work_sync(&router->nh_grp_activity_dw); 11563 mutex_destroy(&router->lock); 11564 kfree(router); 11565} 11566