1// SPDX-License-Identifier: GPL-2.0 2/* 3 * Interconnect framework core driver 4 * 5 * Copyright (c) 2017-2019, Linaro Ltd. 6 * Author: Georgi Djakov <georgi.djakov@linaro.org> 7 */ 8 9#include <linux/debugfs.h> 10#include <linux/device.h> 11#include <linux/idr.h> 12#include <linux/init.h> 13#include <linux/interconnect.h> 14#include <linux/interconnect-provider.h> 15#include <linux/list.h> 16#include <linux/mutex.h> 17#include <linux/slab.h> 18#include <linux/of.h> 19#include <linux/overflow.h> 20 21#include "internal.h" 22 23#define CREATE_TRACE_POINTS 24#include "trace.h" 25 26static DEFINE_IDR(icc_idr); 27static LIST_HEAD(icc_providers); 28static int providers_count; 29static bool synced_state; 30static DEFINE_MUTEX(icc_lock); 31static DEFINE_MUTEX(icc_bw_lock); 32static struct dentry *icc_debugfs_dir; 33 34static void icc_summary_show_one(struct seq_file *s, struct icc_node *n) 35{ 36 if (!n) 37 return; 38 39 seq_printf(s, "%-42s %12u %12u\n", 40 n->name, n->avg_bw, n->peak_bw); 41} 42 43static int icc_summary_show(struct seq_file *s, void *data) 44{ 45 struct icc_provider *provider; 46 47 seq_puts(s, " node tag avg peak\n"); 48 seq_puts(s, "--------------------------------------------------------------------\n"); 49 50 mutex_lock(&icc_lock); 51 52 list_for_each_entry(provider, &icc_providers, provider_list) { 53 struct icc_node *n; 54 55 list_for_each_entry(n, &provider->nodes, node_list) { 56 struct icc_req *r; 57 58 icc_summary_show_one(s, n); 59 hlist_for_each_entry(r, &n->req_list, req_node) { 60 u32 avg_bw = 0, peak_bw = 0; 61 62 if (!r->dev) 63 continue; 64 65 if (r->enabled) { 66 avg_bw = r->avg_bw; 67 peak_bw = r->peak_bw; 68 } 69 70 seq_printf(s, " %-27s %12u %12u %12u\n", 71 dev_name(r->dev), r->tag, avg_bw, peak_bw); 72 } 73 } 74 } 75 76 mutex_unlock(&icc_lock); 77 78 return 0; 79} 80DEFINE_SHOW_ATTRIBUTE(icc_summary); 81 82static void icc_graph_show_link(struct seq_file *s, int level, 83 struct icc_node *n, struct icc_node *m) 84{ 85 seq_printf(s, "%s\"%d:%s\" -> \"%d:%s\"\n", 86 level == 2 ? "\t\t" : "\t", 87 n->id, n->name, m->id, m->name); 88} 89 90static void icc_graph_show_node(struct seq_file *s, struct icc_node *n) 91{ 92 seq_printf(s, "\t\t\"%d:%s\" [label=\"%d:%s", 93 n->id, n->name, n->id, n->name); 94 seq_printf(s, "\n\t\t\t|avg_bw=%ukBps", n->avg_bw); 95 seq_printf(s, "\n\t\t\t|peak_bw=%ukBps", n->peak_bw); 96 seq_puts(s, "\"]\n"); 97} 98 99static int icc_graph_show(struct seq_file *s, void *data) 100{ 101 struct icc_provider *provider; 102 struct icc_node *n; 103 int cluster_index = 0; 104 int i; 105 106 seq_puts(s, "digraph {\n\trankdir = LR\n\tnode [shape = record]\n"); 107 mutex_lock(&icc_lock); 108 109 /* draw providers as cluster subgraphs */ 110 cluster_index = 0; 111 list_for_each_entry(provider, &icc_providers, provider_list) { 112 seq_printf(s, "\tsubgraph cluster_%d {\n", ++cluster_index); 113 if (provider->dev) 114 seq_printf(s, "\t\tlabel = \"%s\"\n", 115 dev_name(provider->dev)); 116 117 /* draw nodes */ 118 list_for_each_entry(n, &provider->nodes, node_list) 119 icc_graph_show_node(s, n); 120 121 /* draw internal links */ 122 list_for_each_entry(n, &provider->nodes, node_list) 123 for (i = 0; i < n->num_links; ++i) 124 if (n->provider == n->links[i]->provider) 125 icc_graph_show_link(s, 2, n, 126 n->links[i]); 127 128 seq_puts(s, "\t}\n"); 129 } 130 131 /* draw external links */ 132 list_for_each_entry(provider, &icc_providers, provider_list) 133 list_for_each_entry(n, &provider->nodes, node_list) 134 for (i = 0; i < n->num_links; ++i) 135 if (n->provider != n->links[i]->provider) 136 icc_graph_show_link(s, 1, n, 137 n->links[i]); 138 139 mutex_unlock(&icc_lock); 140 seq_puts(s, "}"); 141 142 return 0; 143} 144DEFINE_SHOW_ATTRIBUTE(icc_graph); 145 146static struct icc_node *node_find(const int id) 147{ 148 return idr_find(&icc_idr, id); 149} 150 151static struct icc_node *node_find_by_name(const char *name) 152{ 153 struct icc_provider *provider; 154 struct icc_node *n; 155 156 list_for_each_entry(provider, &icc_providers, provider_list) { 157 list_for_each_entry(n, &provider->nodes, node_list) { 158 if (!strcmp(n->name, name)) 159 return n; 160 } 161 } 162 163 return NULL; 164} 165 166static struct icc_path *path_init(struct device *dev, struct icc_node *dst, 167 ssize_t num_nodes) 168{ 169 struct icc_node *node = dst; 170 struct icc_path *path; 171 int i; 172 173 path = kzalloc(struct_size(path, reqs, num_nodes), GFP_KERNEL); 174 if (!path) 175 return ERR_PTR(-ENOMEM); 176 177 path->num_nodes = num_nodes; 178 179 for (i = num_nodes - 1; i >= 0; i--) { 180 node->provider->users++; 181 hlist_add_head(&path->reqs[i].req_node, &node->req_list); 182 path->reqs[i].node = node; 183 path->reqs[i].dev = dev; 184 path->reqs[i].enabled = true; 185 /* reference to previous node was saved during path traversal */ 186 node = node->reverse; 187 } 188 189 return path; 190} 191 192static struct icc_path *path_find(struct device *dev, struct icc_node *src, 193 struct icc_node *dst) 194{ 195 struct icc_path *path = ERR_PTR(-EPROBE_DEFER); 196 struct icc_node *n, *node = NULL; 197 struct list_head traverse_list; 198 struct list_head edge_list; 199 struct list_head visited_list; 200 size_t i, depth = 1; 201 bool found = false; 202 203 INIT_LIST_HEAD(&traverse_list); 204 INIT_LIST_HEAD(&edge_list); 205 INIT_LIST_HEAD(&visited_list); 206 207 list_add(&src->search_list, &traverse_list); 208 src->reverse = NULL; 209 210 do { 211 list_for_each_entry_safe(node, n, &traverse_list, search_list) { 212 if (node == dst) { 213 found = true; 214 list_splice_init(&edge_list, &visited_list); 215 list_splice_init(&traverse_list, &visited_list); 216 break; 217 } 218 for (i = 0; i < node->num_links; i++) { 219 struct icc_node *tmp = node->links[i]; 220 221 if (!tmp) { 222 path = ERR_PTR(-ENOENT); 223 goto out; 224 } 225 226 if (tmp->is_traversed) 227 continue; 228 229 tmp->is_traversed = true; 230 tmp->reverse = node; 231 list_add_tail(&tmp->search_list, &edge_list); 232 } 233 } 234 235 if (found) 236 break; 237 238 list_splice_init(&traverse_list, &visited_list); 239 list_splice_init(&edge_list, &traverse_list); 240 241 /* count the hops including the source */ 242 depth++; 243 244 } while (!list_empty(&traverse_list)); 245 246out: 247 248 /* reset the traversed state */ 249 list_for_each_entry_reverse(n, &visited_list, search_list) 250 n->is_traversed = false; 251 252 if (found) 253 path = path_init(dev, dst, depth); 254 255 return path; 256} 257 258/* 259 * We want the path to honor all bandwidth requests, so the average and peak 260 * bandwidth requirements from each consumer are aggregated at each node. 261 * The aggregation is platform specific, so each platform can customize it by 262 * implementing its own aggregate() function. 263 */ 264 265static int aggregate_requests(struct icc_node *node) 266{ 267 struct icc_provider *p = node->provider; 268 struct icc_req *r; 269 u32 avg_bw, peak_bw; 270 271 node->avg_bw = 0; 272 node->peak_bw = 0; 273 274 if (p->pre_aggregate) 275 p->pre_aggregate(node); 276 277 hlist_for_each_entry(r, &node->req_list, req_node) { 278 if (r->enabled) { 279 avg_bw = r->avg_bw; 280 peak_bw = r->peak_bw; 281 } else { 282 avg_bw = 0; 283 peak_bw = 0; 284 } 285 p->aggregate(node, r->tag, avg_bw, peak_bw, 286 &node->avg_bw, &node->peak_bw); 287 288 /* during boot use the initial bandwidth as a floor value */ 289 if (!synced_state) { 290 node->avg_bw = max(node->avg_bw, node->init_avg); 291 node->peak_bw = max(node->peak_bw, node->init_peak); 292 } 293 } 294 295 return 0; 296} 297 298static int apply_constraints(struct icc_path *path) 299{ 300 struct icc_node *next, *prev = NULL; 301 struct icc_provider *p; 302 int ret = -EINVAL; 303 int i; 304 305 for (i = 0; i < path->num_nodes; i++) { 306 next = path->reqs[i].node; 307 p = next->provider; 308 309 /* both endpoints should be valid master-slave pairs */ 310 if (!prev || (p != prev->provider && !p->inter_set)) { 311 prev = next; 312 continue; 313 } 314 315 /* set the constraints */ 316 ret = p->set(prev, next); 317 if (ret) 318 goto out; 319 320 prev = next; 321 } 322out: 323 return ret; 324} 325 326int icc_std_aggregate(struct icc_node *node, u32 tag, u32 avg_bw, 327 u32 peak_bw, u32 *agg_avg, u32 *agg_peak) 328{ 329 *agg_avg += avg_bw; 330 *agg_peak = max(*agg_peak, peak_bw); 331 332 return 0; 333} 334EXPORT_SYMBOL_GPL(icc_std_aggregate); 335 336/* of_icc_xlate_onecell() - Translate function using a single index. 337 * @spec: OF phandle args to map into an interconnect node. 338 * @data: private data (pointer to struct icc_onecell_data) 339 * 340 * This is a generic translate function that can be used to model simple 341 * interconnect providers that have one device tree node and provide 342 * multiple interconnect nodes. A single cell is used as an index into 343 * an array of icc nodes specified in the icc_onecell_data struct when 344 * registering the provider. 345 */ 346struct icc_node *of_icc_xlate_onecell(struct of_phandle_args *spec, 347 void *data) 348{ 349 struct icc_onecell_data *icc_data = data; 350 unsigned int idx = spec->args[0]; 351 352 if (idx >= icc_data->num_nodes) { 353 pr_err("%s: invalid index %u\n", __func__, idx); 354 return ERR_PTR(-EINVAL); 355 } 356 357 return icc_data->nodes[idx]; 358} 359EXPORT_SYMBOL_GPL(of_icc_xlate_onecell); 360 361/** 362 * of_icc_get_from_provider() - Look-up interconnect node 363 * @spec: OF phandle args to use for look-up 364 * 365 * Looks for interconnect provider under the node specified by @spec and if 366 * found, uses xlate function of the provider to map phandle args to node. 367 * 368 * Returns a valid pointer to struct icc_node_data on success or ERR_PTR() 369 * on failure. 370 */ 371struct icc_node_data *of_icc_get_from_provider(struct of_phandle_args *spec) 372{ 373 struct icc_node *node = ERR_PTR(-EPROBE_DEFER); 374 struct icc_node_data *data = NULL; 375 struct icc_provider *provider; 376 377 if (!spec) 378 return ERR_PTR(-EINVAL); 379 380 mutex_lock(&icc_lock); 381 list_for_each_entry(provider, &icc_providers, provider_list) { 382 if (provider->dev->of_node == spec->np) { 383 if (provider->xlate_extended) { 384 data = provider->xlate_extended(spec, provider->data); 385 if (!IS_ERR(data)) { 386 node = data->node; 387 break; 388 } 389 } else { 390 node = provider->xlate(spec, provider->data); 391 if (!IS_ERR(node)) 392 break; 393 } 394 } 395 } 396 mutex_unlock(&icc_lock); 397 398 if (!node) 399 return ERR_PTR(-EINVAL); 400 401 if (IS_ERR(node)) 402 return ERR_CAST(node); 403 404 if (!data) { 405 data = kzalloc(sizeof(*data), GFP_KERNEL); 406 if (!data) 407 return ERR_PTR(-ENOMEM); 408 data->node = node; 409 } 410 411 return data; 412} 413EXPORT_SYMBOL_GPL(of_icc_get_from_provider); 414 415static void devm_icc_release(struct device *dev, void *res) 416{ 417 icc_put(*(struct icc_path **)res); 418} 419 420struct icc_path *devm_of_icc_get(struct device *dev, const char *name) 421{ 422 struct icc_path **ptr, *path; 423 424 ptr = devres_alloc(devm_icc_release, sizeof(*ptr), GFP_KERNEL); 425 if (!ptr) 426 return ERR_PTR(-ENOMEM); 427 428 path = of_icc_get(dev, name); 429 if (!IS_ERR(path)) { 430 *ptr = path; 431 devres_add(dev, ptr); 432 } else { 433 devres_free(ptr); 434 } 435 436 return path; 437} 438EXPORT_SYMBOL_GPL(devm_of_icc_get); 439 440/** 441 * of_icc_get_by_index() - get a path handle from a DT node based on index 442 * @dev: device pointer for the consumer device 443 * @idx: interconnect path index 444 * 445 * This function will search for a path between two endpoints and return an 446 * icc_path handle on success. Use icc_put() to release constraints when they 447 * are not needed anymore. 448 * If the interconnect API is disabled, NULL is returned and the consumer 449 * drivers will still build. Drivers are free to handle this specifically, 450 * but they don't have to. 451 * 452 * Return: icc_path pointer on success or ERR_PTR() on error. NULL is returned 453 * when the API is disabled or the "interconnects" DT property is missing. 454 */ 455struct icc_path *of_icc_get_by_index(struct device *dev, int idx) 456{ 457 struct icc_path *path; 458 struct icc_node_data *src_data, *dst_data; 459 struct device_node *np; 460 struct of_phandle_args src_args, dst_args; 461 int ret; 462 463 if (!dev || !dev->of_node) 464 return ERR_PTR(-ENODEV); 465 466 np = dev->of_node; 467 468 /* 469 * When the consumer DT node do not have "interconnects" property 470 * return a NULL path to skip setting constraints. 471 */ 472 if (!of_property_present(np, "interconnects")) 473 return NULL; 474 475 /* 476 * We use a combination of phandle and specifier for endpoint. For now 477 * lets support only global ids and extend this in the future if needed 478 * without breaking DT compatibility. 479 */ 480 ret = of_parse_phandle_with_args(np, "interconnects", 481 "#interconnect-cells", idx * 2, 482 &src_args); 483 if (ret) 484 return ERR_PTR(ret); 485 486 of_node_put(src_args.np); 487 488 ret = of_parse_phandle_with_args(np, "interconnects", 489 "#interconnect-cells", idx * 2 + 1, 490 &dst_args); 491 if (ret) 492 return ERR_PTR(ret); 493 494 of_node_put(dst_args.np); 495 496 src_data = of_icc_get_from_provider(&src_args); 497 498 if (IS_ERR(src_data)) { 499 dev_err_probe(dev, PTR_ERR(src_data), "error finding src node\n"); 500 return ERR_CAST(src_data); 501 } 502 503 dst_data = of_icc_get_from_provider(&dst_args); 504 505 if (IS_ERR(dst_data)) { 506 dev_err_probe(dev, PTR_ERR(dst_data), "error finding dst node\n"); 507 kfree(src_data); 508 return ERR_CAST(dst_data); 509 } 510 511 mutex_lock(&icc_lock); 512 path = path_find(dev, src_data->node, dst_data->node); 513 mutex_unlock(&icc_lock); 514 if (IS_ERR(path)) { 515 dev_err(dev, "%s: invalid path=%ld\n", __func__, PTR_ERR(path)); 516 goto free_icc_data; 517 } 518 519 if (src_data->tag && src_data->tag == dst_data->tag) 520 icc_set_tag(path, src_data->tag); 521 522 path->name = kasprintf(GFP_KERNEL, "%s-%s", 523 src_data->node->name, dst_data->node->name); 524 if (!path->name) { 525 kfree(path); 526 path = ERR_PTR(-ENOMEM); 527 } 528 529free_icc_data: 530 kfree(src_data); 531 kfree(dst_data); 532 return path; 533} 534EXPORT_SYMBOL_GPL(of_icc_get_by_index); 535 536/** 537 * of_icc_get() - get a path handle from a DT node based on name 538 * @dev: device pointer for the consumer device 539 * @name: interconnect path name 540 * 541 * This function will search for a path between two endpoints and return an 542 * icc_path handle on success. Use icc_put() to release constraints when they 543 * are not needed anymore. 544 * If the interconnect API is disabled, NULL is returned and the consumer 545 * drivers will still build. Drivers are free to handle this specifically, 546 * but they don't have to. 547 * 548 * Return: icc_path pointer on success or ERR_PTR() on error. NULL is returned 549 * when the API is disabled or the "interconnects" DT property is missing. 550 */ 551struct icc_path *of_icc_get(struct device *dev, const char *name) 552{ 553 struct device_node *np; 554 int idx = 0; 555 556 if (!dev || !dev->of_node) 557 return ERR_PTR(-ENODEV); 558 559 np = dev->of_node; 560 561 /* 562 * When the consumer DT node do not have "interconnects" property 563 * return a NULL path to skip setting constraints. 564 */ 565 if (!of_property_present(np, "interconnects")) 566 return NULL; 567 568 /* 569 * We use a combination of phandle and specifier for endpoint. For now 570 * lets support only global ids and extend this in the future if needed 571 * without breaking DT compatibility. 572 */ 573 if (name) { 574 idx = of_property_match_string(np, "interconnect-names", name); 575 if (idx < 0) 576 return ERR_PTR(idx); 577 } 578 579 return of_icc_get_by_index(dev, idx); 580} 581EXPORT_SYMBOL_GPL(of_icc_get); 582 583/** 584 * icc_get() - get a path handle between two endpoints 585 * @dev: device pointer for the consumer device 586 * @src: source node name 587 * @dst: destination node name 588 * 589 * This function will search for a path between two endpoints and return an 590 * icc_path handle on success. Use icc_put() to release constraints when they 591 * are not needed anymore. 592 * 593 * Return: icc_path pointer on success or ERR_PTR() on error. NULL is returned 594 * when the API is disabled. 595 */ 596struct icc_path *icc_get(struct device *dev, const char *src, const char *dst) 597{ 598 struct icc_node *src_node, *dst_node; 599 struct icc_path *path = ERR_PTR(-EPROBE_DEFER); 600 601 mutex_lock(&icc_lock); 602 603 src_node = node_find_by_name(src); 604 if (!src_node) { 605 dev_err(dev, "%s: invalid src=%s\n", __func__, src); 606 goto out; 607 } 608 609 dst_node = node_find_by_name(dst); 610 if (!dst_node) { 611 dev_err(dev, "%s: invalid dst=%s\n", __func__, dst); 612 goto out; 613 } 614 615 path = path_find(dev, src_node, dst_node); 616 if (IS_ERR(path)) { 617 dev_err(dev, "%s: invalid path=%ld\n", __func__, PTR_ERR(path)); 618 goto out; 619 } 620 621 path->name = kasprintf(GFP_KERNEL, "%s-%s", src_node->name, dst_node->name); 622 if (!path->name) { 623 kfree(path); 624 path = ERR_PTR(-ENOMEM); 625 } 626out: 627 mutex_unlock(&icc_lock); 628 return path; 629} 630 631/** 632 * icc_set_tag() - set an optional tag on a path 633 * @path: the path we want to tag 634 * @tag: the tag value 635 * 636 * This function allows consumers to append a tag to the requests associated 637 * with a path, so that a different aggregation could be done based on this tag. 638 */ 639void icc_set_tag(struct icc_path *path, u32 tag) 640{ 641 int i; 642 643 if (!path) 644 return; 645 646 mutex_lock(&icc_lock); 647 648 for (i = 0; i < path->num_nodes; i++) 649 path->reqs[i].tag = tag; 650 651 mutex_unlock(&icc_lock); 652} 653EXPORT_SYMBOL_GPL(icc_set_tag); 654 655/** 656 * icc_get_name() - Get name of the icc path 657 * @path: interconnect path 658 * 659 * This function is used by an interconnect consumer to get the name of the icc 660 * path. 661 * 662 * Returns a valid pointer on success, or NULL otherwise. 663 */ 664const char *icc_get_name(struct icc_path *path) 665{ 666 if (!path) 667 return NULL; 668 669 return path->name; 670} 671EXPORT_SYMBOL_GPL(icc_get_name); 672 673/** 674 * icc_set_bw() - set bandwidth constraints on an interconnect path 675 * @path: interconnect path 676 * @avg_bw: average bandwidth in kilobytes per second 677 * @peak_bw: peak bandwidth in kilobytes per second 678 * 679 * This function is used by an interconnect consumer to express its own needs 680 * in terms of bandwidth for a previously requested path between two endpoints. 681 * The requests are aggregated and each node is updated accordingly. The entire 682 * path is locked by a mutex to ensure that the set() is completed. 683 * The @path can be NULL when the "interconnects" DT properties is missing, 684 * which will mean that no constraints will be set. 685 * 686 * Returns 0 on success, or an appropriate error code otherwise. 687 */ 688int icc_set_bw(struct icc_path *path, u32 avg_bw, u32 peak_bw) 689{ 690 struct icc_node *node; 691 u32 old_avg, old_peak; 692 size_t i; 693 int ret; 694 695 if (!path) 696 return 0; 697 698 if (WARN_ON(IS_ERR(path) || !path->num_nodes)) 699 return -EINVAL; 700 701 mutex_lock(&icc_bw_lock); 702 703 old_avg = path->reqs[0].avg_bw; 704 old_peak = path->reqs[0].peak_bw; 705 706 for (i = 0; i < path->num_nodes; i++) { 707 node = path->reqs[i].node; 708 709 /* update the consumer request for this path */ 710 path->reqs[i].avg_bw = avg_bw; 711 path->reqs[i].peak_bw = peak_bw; 712 713 /* aggregate requests for this node */ 714 aggregate_requests(node); 715 716 trace_icc_set_bw(path, node, i, avg_bw, peak_bw); 717 } 718 719 ret = apply_constraints(path); 720 if (ret) { 721 pr_debug("interconnect: error applying constraints (%d)\n", 722 ret); 723 724 for (i = 0; i < path->num_nodes; i++) { 725 node = path->reqs[i].node; 726 path->reqs[i].avg_bw = old_avg; 727 path->reqs[i].peak_bw = old_peak; 728 aggregate_requests(node); 729 } 730 apply_constraints(path); 731 } 732 733 mutex_unlock(&icc_bw_lock); 734 735 trace_icc_set_bw_end(path, ret); 736 737 return ret; 738} 739EXPORT_SYMBOL_GPL(icc_set_bw); 740 741static int __icc_enable(struct icc_path *path, bool enable) 742{ 743 int i; 744 745 if (!path) 746 return 0; 747 748 if (WARN_ON(IS_ERR(path) || !path->num_nodes)) 749 return -EINVAL; 750 751 mutex_lock(&icc_lock); 752 753 for (i = 0; i < path->num_nodes; i++) 754 path->reqs[i].enabled = enable; 755 756 mutex_unlock(&icc_lock); 757 758 return icc_set_bw(path, path->reqs[0].avg_bw, 759 path->reqs[0].peak_bw); 760} 761 762int icc_enable(struct icc_path *path) 763{ 764 return __icc_enable(path, true); 765} 766EXPORT_SYMBOL_GPL(icc_enable); 767 768int icc_disable(struct icc_path *path) 769{ 770 return __icc_enable(path, false); 771} 772EXPORT_SYMBOL_GPL(icc_disable); 773 774/** 775 * icc_put() - release the reference to the icc_path 776 * @path: interconnect path 777 * 778 * Use this function to release the constraints on a path when the path is 779 * no longer needed. The constraints will be re-aggregated. 780 */ 781void icc_put(struct icc_path *path) 782{ 783 struct icc_node *node; 784 size_t i; 785 int ret; 786 787 if (!path || WARN_ON(IS_ERR(path))) 788 return; 789 790 ret = icc_set_bw(path, 0, 0); 791 if (ret) 792 pr_err("%s: error (%d)\n", __func__, ret); 793 794 mutex_lock(&icc_lock); 795 for (i = 0; i < path->num_nodes; i++) { 796 node = path->reqs[i].node; 797 hlist_del(&path->reqs[i].req_node); 798 if (!WARN_ON(!node->provider->users)) 799 node->provider->users--; 800 } 801 mutex_unlock(&icc_lock); 802 803 kfree_const(path->name); 804 kfree(path); 805} 806EXPORT_SYMBOL_GPL(icc_put); 807 808static struct icc_node *icc_node_create_nolock(int id) 809{ 810 struct icc_node *node; 811 812 /* check if node already exists */ 813 node = node_find(id); 814 if (node) 815 return node; 816 817 node = kzalloc(sizeof(*node), GFP_KERNEL); 818 if (!node) 819 return ERR_PTR(-ENOMEM); 820 821 id = idr_alloc(&icc_idr, node, id, id + 1, GFP_KERNEL); 822 if (id < 0) { 823 WARN(1, "%s: couldn't get idr\n", __func__); 824 kfree(node); 825 return ERR_PTR(id); 826 } 827 828 node->id = id; 829 830 return node; 831} 832 833/** 834 * icc_node_create() - create a node 835 * @id: node id 836 * 837 * Return: icc_node pointer on success, or ERR_PTR() on error 838 */ 839struct icc_node *icc_node_create(int id) 840{ 841 struct icc_node *node; 842 843 mutex_lock(&icc_lock); 844 845 node = icc_node_create_nolock(id); 846 847 mutex_unlock(&icc_lock); 848 849 return node; 850} 851EXPORT_SYMBOL_GPL(icc_node_create); 852 853/** 854 * icc_node_destroy() - destroy a node 855 * @id: node id 856 */ 857void icc_node_destroy(int id) 858{ 859 struct icc_node *node; 860 861 mutex_lock(&icc_lock); 862 863 node = node_find(id); 864 if (node) { 865 idr_remove(&icc_idr, node->id); 866 WARN_ON(!hlist_empty(&node->req_list)); 867 } 868 869 mutex_unlock(&icc_lock); 870 871 if (!node) 872 return; 873 874 kfree(node->links); 875 kfree(node); 876} 877EXPORT_SYMBOL_GPL(icc_node_destroy); 878 879/** 880 * icc_link_create() - create a link between two nodes 881 * @node: source node id 882 * @dst_id: destination node id 883 * 884 * Create a link between two nodes. The nodes might belong to different 885 * interconnect providers and the @dst_id node might not exist (if the 886 * provider driver has not probed yet). So just create the @dst_id node 887 * and when the actual provider driver is probed, the rest of the node 888 * data is filled. 889 * 890 * Return: 0 on success, or an error code otherwise 891 */ 892int icc_link_create(struct icc_node *node, const int dst_id) 893{ 894 struct icc_node *dst; 895 struct icc_node **new; 896 int ret = 0; 897 898 if (!node->provider) 899 return -EINVAL; 900 901 mutex_lock(&icc_lock); 902 903 dst = node_find(dst_id); 904 if (!dst) { 905 dst = icc_node_create_nolock(dst_id); 906 907 if (IS_ERR(dst)) { 908 ret = PTR_ERR(dst); 909 goto out; 910 } 911 } 912 913 new = krealloc(node->links, 914 (node->num_links + 1) * sizeof(*node->links), 915 GFP_KERNEL); 916 if (!new) { 917 ret = -ENOMEM; 918 goto out; 919 } 920 921 node->links = new; 922 node->links[node->num_links++] = dst; 923 924out: 925 mutex_unlock(&icc_lock); 926 927 return ret; 928} 929EXPORT_SYMBOL_GPL(icc_link_create); 930 931/** 932 * icc_node_add() - add interconnect node to interconnect provider 933 * @node: pointer to the interconnect node 934 * @provider: pointer to the interconnect provider 935 */ 936void icc_node_add(struct icc_node *node, struct icc_provider *provider) 937{ 938 if (WARN_ON(node->provider)) 939 return; 940 941 mutex_lock(&icc_lock); 942 mutex_lock(&icc_bw_lock); 943 944 node->provider = provider; 945 list_add_tail(&node->node_list, &provider->nodes); 946 947 /* get the initial bandwidth values and sync them with hardware */ 948 if (provider->get_bw) { 949 provider->get_bw(node, &node->init_avg, &node->init_peak); 950 } else { 951 node->init_avg = INT_MAX; 952 node->init_peak = INT_MAX; 953 } 954 node->avg_bw = node->init_avg; 955 node->peak_bw = node->init_peak; 956 957 if (node->avg_bw || node->peak_bw) { 958 if (provider->pre_aggregate) 959 provider->pre_aggregate(node); 960 961 if (provider->aggregate) 962 provider->aggregate(node, 0, node->init_avg, node->init_peak, 963 &node->avg_bw, &node->peak_bw); 964 if (provider->set) 965 provider->set(node, node); 966 } 967 968 node->avg_bw = 0; 969 node->peak_bw = 0; 970 971 mutex_unlock(&icc_bw_lock); 972 mutex_unlock(&icc_lock); 973} 974EXPORT_SYMBOL_GPL(icc_node_add); 975 976/** 977 * icc_node_del() - delete interconnect node from interconnect provider 978 * @node: pointer to the interconnect node 979 */ 980void icc_node_del(struct icc_node *node) 981{ 982 mutex_lock(&icc_lock); 983 984 list_del(&node->node_list); 985 986 mutex_unlock(&icc_lock); 987} 988EXPORT_SYMBOL_GPL(icc_node_del); 989 990/** 991 * icc_nodes_remove() - remove all previously added nodes from provider 992 * @provider: the interconnect provider we are removing nodes from 993 * 994 * Return: 0 on success, or an error code otherwise 995 */ 996int icc_nodes_remove(struct icc_provider *provider) 997{ 998 struct icc_node *n, *tmp; 999 1000 if (WARN_ON(IS_ERR_OR_NULL(provider))) 1001 return -EINVAL; 1002 1003 list_for_each_entry_safe_reverse(n, tmp, &provider->nodes, node_list) { 1004 icc_node_del(n); 1005 icc_node_destroy(n->id); 1006 } 1007 1008 return 0; 1009} 1010EXPORT_SYMBOL_GPL(icc_nodes_remove); 1011 1012/** 1013 * icc_provider_init() - initialize a new interconnect provider 1014 * @provider: the interconnect provider to initialize 1015 * 1016 * Must be called before adding nodes to the provider. 1017 */ 1018void icc_provider_init(struct icc_provider *provider) 1019{ 1020 WARN_ON(!provider->set); 1021 1022 INIT_LIST_HEAD(&provider->nodes); 1023} 1024EXPORT_SYMBOL_GPL(icc_provider_init); 1025 1026/** 1027 * icc_provider_register() - register a new interconnect provider 1028 * @provider: the interconnect provider to register 1029 * 1030 * Return: 0 on success, or an error code otherwise 1031 */ 1032int icc_provider_register(struct icc_provider *provider) 1033{ 1034 if (WARN_ON(!provider->xlate && !provider->xlate_extended)) 1035 return -EINVAL; 1036 1037 mutex_lock(&icc_lock); 1038 list_add_tail(&provider->provider_list, &icc_providers); 1039 mutex_unlock(&icc_lock); 1040 1041 dev_dbg(provider->dev, "interconnect provider registered\n"); 1042 1043 return 0; 1044} 1045EXPORT_SYMBOL_GPL(icc_provider_register); 1046 1047/** 1048 * icc_provider_deregister() - deregister an interconnect provider 1049 * @provider: the interconnect provider to deregister 1050 */ 1051void icc_provider_deregister(struct icc_provider *provider) 1052{ 1053 mutex_lock(&icc_lock); 1054 WARN_ON(provider->users); 1055 1056 list_del(&provider->provider_list); 1057 mutex_unlock(&icc_lock); 1058} 1059EXPORT_SYMBOL_GPL(icc_provider_deregister); 1060 1061static const struct of_device_id __maybe_unused ignore_list[] = { 1062 { .compatible = "qcom,sc7180-ipa-virt" }, 1063 { .compatible = "qcom,sc8180x-ipa-virt" }, 1064 { .compatible = "qcom,sdx55-ipa-virt" }, 1065 { .compatible = "qcom,sm8150-ipa-virt" }, 1066 { .compatible = "qcom,sm8250-ipa-virt" }, 1067 {} 1068}; 1069 1070static int of_count_icc_providers(struct device_node *np) 1071{ 1072 struct device_node *child; 1073 int count = 0; 1074 1075 for_each_available_child_of_node(np, child) { 1076 if (of_property_read_bool(child, "#interconnect-cells") && 1077 likely(!of_match_node(ignore_list, child))) 1078 count++; 1079 count += of_count_icc_providers(child); 1080 } 1081 1082 return count; 1083} 1084 1085void icc_sync_state(struct device *dev) 1086{ 1087 struct icc_provider *p; 1088 struct icc_node *n; 1089 static int count; 1090 1091 count++; 1092 1093 if (count < providers_count) 1094 return; 1095 1096 mutex_lock(&icc_lock); 1097 mutex_lock(&icc_bw_lock); 1098 synced_state = true; 1099 list_for_each_entry(p, &icc_providers, provider_list) { 1100 dev_dbg(p->dev, "interconnect provider is in synced state\n"); 1101 list_for_each_entry(n, &p->nodes, node_list) { 1102 if (n->init_avg || n->init_peak) { 1103 n->init_avg = 0; 1104 n->init_peak = 0; 1105 aggregate_requests(n); 1106 p->set(n, n); 1107 } 1108 } 1109 } 1110 mutex_unlock(&icc_bw_lock); 1111 mutex_unlock(&icc_lock); 1112} 1113EXPORT_SYMBOL_GPL(icc_sync_state); 1114 1115static int __init icc_init(void) 1116{ 1117 struct device_node *root; 1118 1119 /* Teach lockdep about lock ordering wrt. shrinker: */ 1120 fs_reclaim_acquire(GFP_KERNEL); 1121 might_lock(&icc_bw_lock); 1122 fs_reclaim_release(GFP_KERNEL); 1123 1124 root = of_find_node_by_path("/"); 1125 1126 providers_count = of_count_icc_providers(root); 1127 of_node_put(root); 1128 1129 icc_debugfs_dir = debugfs_create_dir("interconnect", NULL); 1130 debugfs_create_file("interconnect_summary", 0444, 1131 icc_debugfs_dir, NULL, &icc_summary_fops); 1132 debugfs_create_file("interconnect_graph", 0444, 1133 icc_debugfs_dir, NULL, &icc_graph_fops); 1134 1135 icc_debugfs_client_init(icc_debugfs_dir); 1136 1137 return 0; 1138} 1139 1140device_initcall(icc_init); 1141