1// SPDX-License-Identifier: GPL-2.0 2/* 3 * drivers/auth_ctl/auth_ctrl.c 4 * 5 * Copyright (c) 2022 Huawei Device Co., Ltd. 6 * 7 */ 8#include <linux/sched.h> 9#include <linux/list.h> 10#include <linux/mutex.h> 11#include <linux/stop_machine.h> 12#include <linux/sched/auth_ctrl.h> 13#include <linux/sched/rtg_auth.h> 14#include <linux/sched/qos_ctrl.h> 15#include <linux/sched/qos_auth.h> 16#include <uapi/linux/sched/types.h> 17 18#include "auth_ctrl.h" 19#include "qos_ctrl.h" 20 21typedef long (*qos_ctrl_func)(int abi, void __user *uarg); 22 23static long ctrl_qos_operation(int abi, void __user *uarg); 24static long ctrl_qos_policy(int abi, void __user *uarg); 25 26#define QOS_LEVEL_SET_MAX 5 27 28static qos_ctrl_func g_func_array[QOS_CTRL_MAX_NR] = { 29 NULL, /* reserved */ 30 ctrl_qos_operation, 31 ctrl_qos_policy, 32}; 33 34static struct qos_policy_map qos_policy_array[QOS_POLICY_MAX_NR]; 35 36void remove_qos_tasks(struct auth_struct *auth) 37{ 38 int i; 39 struct qos_task_struct *tmp, *next; 40 struct task_struct *p; 41 42 mutex_lock(&auth->mutex); 43 for (i = QOS_POLICY_MIN_LEVEL; i < NR_QOS; ++i) { 44 list_for_each_entry_safe(tmp, next, &auth->tasks[i], qos_list) { 45 p = container_of(tmp, struct task_struct, qts); 46 if (!list_empty(&tmp->qos_list)) { 47 list_del_init(&tmp->qos_list); 48 tmp->in_qos = NO_QOS; 49 put_task_struct(p); 50 } 51 } 52 } 53 mutex_unlock(&auth->mutex); 54} 55 56static void init_sched_attr(struct sched_attr *attr) 57{ 58 memset(attr, 0, sizeof(struct sched_attr)); 59} 60 61static inline bool is_system(unsigned int uid) 62{ 63 return uid == SYSTEM_UID; 64} 65 66/* This function must be called when p is valid. That means the p's refcount must exist */ 67static int sched_set_task_qos_attr(struct task_struct *p, int level, int status) 68{ 69 struct qos_policy_item *item; 70 struct qos_policy_map *policy_map; 71 struct sched_attr attr; 72 73 read_lock(&qos_policy_array[status].lock); 74 if (!qos_policy_array[status].initialized) { 75 pr_err("[QOS_CTRL] dirty qos policy, pid=%d, uid=%d, status=%d\n", 76 p->pid, p->cred->uid.val, status); 77 read_unlock(&qos_policy_array[status].lock); 78 return -DIRTY_QOS_POLICY; 79 } 80 81 policy_map = &qos_policy_array[status]; 82 item = &policy_map->levels[level]; 83 84 init_sched_attr(&attr); 85 attr.size = sizeof(struct sched_attr); 86 attr.sched_policy = SCHED_NORMAL; 87 88 if (policy_map->policy_flag & QOS_FLAG_NICE) 89 attr.sched_nice = item->nice; 90 91 if (policy_map->policy_flag & QOS_FLAG_LATENCY_NICE) { 92 attr.sched_flags |= SCHED_FLAG_LATENCY_NICE; 93 attr.sched_latency_nice = item->latency_nice; 94 } 95 96 if ((policy_map->policy_flag & QOS_FLAG_RT) && item->rt_sched_priority) { 97 attr.sched_policy = SCHED_FIFO; 98 attr.sched_flags |= SCHED_FLAG_RESET_ON_FORK; 99 attr.sched_priority = item->rt_sched_priority; 100 } 101 102 read_unlock(&qos_policy_array[status].lock); 103 104 if (unlikely(p->flags & PF_EXITING)) { 105 pr_info("[QOS_CTRL] dying task, no need to set qos\n"); 106 return -THREAD_EXITING; 107 } 108 109 return sched_setattr_nocheck(p, &attr); 110} 111 112/* 113 * Switch qos mode when status changed. 114 * Lock auth before calling this function 115 */ 116void qos_switch(struct auth_struct *auth, int target_status) 117{ 118 int i; 119 int ret; 120 struct task_struct *task; 121 struct qos_task_struct *qts; 122 123 if (!auth) { 124 pr_err("[QOS_CTRL] auth no exist, qos switch failed\n"); 125 return; 126 } 127 128 lockdep_assert_held(&auth->mutex); 129 130 if (auth->status == target_status) { 131 pr_info("[QOS_CTRL] same status, no need to switch qos\n"); 132 return; 133 } 134 135 for (i = QOS_POLICY_MIN_LEVEL; i < NR_QOS; ++i) { 136 list_for_each_entry(qts, &auth->tasks[i], qos_list) { 137 task = container_of(qts, struct task_struct, qts); 138 ret = sched_set_task_qos_attr(task, i, target_status); 139 if (ret) 140 pr_err("[QOS_CTRL] set qos attr failed, qos switch failed\n"); 141 } 142 } 143} 144 145static int qos_insert_task(struct task_struct *p, struct list_head *head, unsigned int level) 146{ 147 struct qos_task_struct *qts = &p->qts; 148 149 if (qts->in_qos > NO_QOS) { 150 pr_err("[QOS_CTRL] qos apply still active, no duplicate add\n"); 151 return -PID_DUPLICATE; 152 } 153 154 if (likely(list_empty(&qts->qos_list))) { 155 get_task_struct(p); 156 list_add(&qts->qos_list, head); 157 qts->in_qos = level; 158 } 159 160 return 0; 161} 162 163static int qos_remove_task(struct task_struct *p) 164{ 165 struct qos_task_struct *qts = (struct qos_task_struct *) &p->qts; 166 167 if (qts->in_qos == NO_QOS) { 168 pr_err("[QOS_CTRL] task not in qos, no need to remove\n"); 169 return -PID_NOT_EXIST; 170 } 171 172 if (likely(!list_empty(&qts->qos_list))) { 173 list_del_init(&qts->qos_list); 174 qts->in_qos = NO_QOS; 175 put_task_struct(p); 176 } 177 178 return 0; 179} 180 181static inline bool super_user(struct task_struct *p) 182{ 183 return super_uid(task_uid(p).val); 184} 185 186/* 187 * judge permission for changing tasks' qos 188 */ 189static bool can_change_qos(struct task_struct *p, unsigned int qos_level) 190{ 191 struct auth_struct *auth; 192 auth = get_authority(p); 193 /* just system & root user can set(be setted) high qos level */ 194 if (!auth || (auth && !super_user(p) && qos_level > QOS_LEVEL_SET_MAX)) { 195 pr_err("[QOS_CTRL] %d have no permission to change qos\n", p->pid); 196 return false; 197 } 198 199 return true; 200} 201 202int qos_apply(struct qos_ctrl_data *data) 203{ 204 unsigned int level = data->level; 205 struct auth_struct *auth; 206 struct task_struct *p; 207 struct qos_task_struct *qts; 208 int pid = data->pid; 209 int ret; 210 211 if (level >= NR_QOS || level == NO_QOS) { 212 pr_err("[QOS_CTRL] no this qos level, qos apply failed\n"); 213 ret = -ARG_INVALID; 214 goto out; 215 } 216 217 p = find_get_task_by_vpid((pid_t)pid); 218 if (unlikely(!p)) { 219 pr_err("[QOS_CTRL] no matching task for this pid, qos apply failed\n"); 220 ret = -ESRCH; 221 goto out; 222 } 223 224 if (unlikely(p->flags & PF_EXITING)) { 225 pr_info("[QOS_CTRL] dying task, no need to set qos\n"); 226 ret = -THREAD_EXITING; 227 goto out_put_task; 228 } 229 230 if (!can_change_qos(current, level)) { 231 pr_err("[QOS_CTRL] QOS apply not permit\n"); 232 ret = -ARG_INVALID; 233 goto out_put_task; 234 } 235 236 auth = get_authority(p); 237 if (!auth) { 238 pr_err("[QOS_CTRL] no auth data for pid=%d(%s), qos apply failed\n", 239 p->tgid, p->comm); 240 ret = -PID_NOT_FOUND; 241 goto out_put_task; 242 } 243 244 mutex_lock(&auth->mutex); 245 if (auth->status == AUTH_STATUS_DEAD) { 246 pr_err("[QOS_CTRL] this auth data has been deleted\n"); 247 ret = -INVALID_AUTH; 248 goto out_unlock; 249 } 250 251 if (auth->num[level] >= QOS_NUM_MAX) { 252 pr_err("[QOS_CTRL] qos num exceeds limit, cached only\n"); 253 ret = -QOS_THREAD_NUM_EXCEED_LIMIT; 254 goto out_unlock; 255 } 256 257 qts = (struct qos_task_struct *) &p->qts; 258 259 /* effective qos must in range [NO_QOS, NR_QOS) */ 260 if (qts->in_qos != NO_QOS) { 261 if (qts->in_qos == level) { 262 ret = 0; 263 goto out_unlock; 264 } 265 266 --auth->num[qts->in_qos]; 267 qos_remove_task(p); 268 } 269 270 ret = qos_insert_task(p, &auth->tasks[level], level); 271 if (ret < 0) { 272 pr_err("[QOS_CTRL] insert task to qos list %d failed\n", level); 273 goto out_unlock; 274 } 275 276 ++auth->num[level]; 277 278 ret = sched_set_task_qos_attr(p, level, auth->status); 279 if (ret) { 280 pr_err("[QOS_CTRL] set qos_level %d for thread %d on status %d failed\n", 281 level, p->pid, auth->status); 282 --auth->num[level]; 283 qos_remove_task(p); 284 } 285 286out_unlock: 287 mutex_unlock(&auth->mutex); 288 put_auth_struct(auth); 289out_put_task: 290 put_task_struct(p); 291out: 292 return ret; 293} 294 295int qos_leave(struct qos_ctrl_data *data) 296{ 297 unsigned int level; 298 struct auth_struct *auth; 299 struct task_struct *p; 300 struct qos_task_struct *qts; 301 int pid = data->pid; 302 int ret; 303 304 p = find_get_task_by_vpid((pid_t)pid); 305 if (!p) { 306 pr_err("[QOS_CTRL] no matching task for this pid, qos apply failed\n"); 307 ret = -ESRCH; 308 goto out; 309 } 310 311 if (unlikely(p->flags & PF_EXITING)) { 312 pr_info("[QOS_CTRL] dying task, no need to set qos\n"); 313 ret = -THREAD_EXITING; 314 goto out_put_task; 315 } 316 317 auth = get_authority(p); 318 if (!auth) { 319 pr_err("[QOS_CTRL] no auth data for pid=%d(%s), qos stop failed\n", 320 p->tgid, p->comm); 321 ret = -PID_NOT_FOUND; 322 goto out_put_task; 323 } 324 325 mutex_lock(&auth->mutex); 326 327 qts = (struct qos_task_struct *) &p->qts; 328 329 level = qts->in_qos; 330 if (level == NO_QOS) { 331 pr_err("[QOS_CTRL] task not in qos list, qos stop failed\n"); 332 ret = -ARG_INVALID; 333 goto out_unlock; 334 } 335 336 if (!can_change_qos(current, 0)) { 337 pr_err("[QOS_CTRL] apply for others not permit\n"); 338 ret = -ARG_INVALID; 339 goto out_unlock; 340 } 341 342 if (auth->status == AUTH_STATUS_DEAD) { 343 pr_err("[QOS_CTRL] this auth data has been deleted\n"); 344 ret = -INVALID_AUTH; 345 goto out_unlock; 346 } 347 348 ret = qos_remove_task(p); 349 if (ret < 0) { 350 pr_err("[QOS_CTRL] remove task from qos list %d failed\n", level); 351 goto out_unlock; 352 } 353 354 --auth->num[level]; 355 356 /* 357 * NO NEED to judge whether current status is AUTH_STATUS_DISABLE. 358 * In the auth destoring context, the removing of thread's sched attr was protected by 359 * auth->mutex, AUTH_STATUS_DISABLED will never appear here. 360 * 361 * The second param 3 means nothing, actually you can use any valid level here, cause the 362 * policy matching AUTH_STATUS_DISABLED has default parameters for all qos level, which can 363 * keep a powerful thread to behave like a ordinary thread. 364 */ 365 ret = sched_set_task_qos_attr(p, 3, AUTH_STATUS_DISABLED); 366 if (ret) 367 pr_err("[QOS_CTRL] set qos_level %d for thread %d on status %d to default failed\n", 368 level, p->pid, auth->status); 369 370out_unlock: 371 mutex_unlock(&auth->mutex); 372 put_auth_struct(auth); 373out_put_task: 374 put_task_struct(p); 375out: 376 return ret; 377} 378 379int qos_get(struct qos_ctrl_data *data) 380{ 381 struct task_struct *p; 382 struct qos_task_struct *qts; 383 int pid = data->pid; 384 int ret = 0; 385 386 p = find_get_task_by_vpid((pid_t)pid); 387 if (unlikely(!p)) { 388 pr_err("[QOS_CTRL] no matching task for this pid, qos get failed\n"); 389 ret = -ESRCH; 390 goto out; 391 } 392 393 if (unlikely(p->flags & PF_EXITING)) { 394 pr_info("[QOS_CTRL] dying task, no need to set qos\n"); 395 ret = -THREAD_EXITING; 396 goto out_put_task; 397 } 398 399 qts = (struct qos_task_struct *) &p->qts; 400 data->qos = qts->in_qos; 401 402out_put_task: 403 put_task_struct(p); 404out: 405 return ret; 406} 407 408void init_task_qos(struct task_struct *p) 409{ 410 struct qos_task_struct *qts = (struct qos_task_struct *) &p->qts; 411 412 INIT_LIST_HEAD(&qts->qos_list); 413 qts->in_qos = NO_QOS; 414} 415 416/* 417 * Remove statistic info in auth when task exit 418 */ 419void sched_exit_qos_list(struct task_struct *p) 420{ 421 struct auth_struct *auth; 422 struct qos_task_struct *qts = (struct qos_task_struct *) &p->qts; 423 424 /* 425 * For common tasks(the vast majority): 426 * skip get authority, fast return here. 427 * 428 * For qos tasks: 429 * If contend with auth_delete() happens, 430 * 1. function return here, auth_delete() will do the clean up 431 * 2. function go on, either no auth return, either do clean up here 432 * Both cases guarantee data synchronization 433 */ 434 if (likely(qts->in_qos == NO_QOS)) 435 return; 436 437 auth = get_authority(p); 438 if (!auth) 439 goto out; 440 441 mutex_lock(&auth->mutex); 442 if (qts->in_qos == NO_QOS) { 443 mutex_unlock(&auth->mutex); 444 goto out_put_auth; 445 } 446 --auth->num[qts->in_qos]; 447 list_del_init(&qts->qos_list); 448 qts->in_qos = NO_QOS; 449 put_task_struct(p); 450 mutex_unlock(&auth->mutex); 451 452out_put_auth: 453 put_auth_struct(auth); 454out: 455 return; 456} 457 458typedef int (*qos_manipulate_func)(struct qos_ctrl_data *data); 459 460static qos_manipulate_func qos_func_array[QOS_OPERATION_CMD_MAX_NR] = { 461 NULL, 462 qos_apply, //1 463 qos_leave, 464 qos_get, 465}; 466 467static long do_qos_manipulate(struct qos_ctrl_data *data) 468{ 469 long ret = 0; 470 unsigned int type = data->type; 471 472 if (type <= 0 || type >= QOS_OPERATION_CMD_MAX_NR) { 473 pr_err("[QOS_CTRL] CMD_ID_QOS_MANIPULATE type not valid\n"); 474 return -ARG_INVALID; 475 } 476 477 if (qos_func_array[type]) 478 ret = (long)(*qos_func_array[type])(data); 479 480 return ret; 481} 482 483static long ctrl_qos_operation(int abi, void __user *uarg) 484{ 485 struct qos_ctrl_data qos_data; 486 int ret = -1; 487 488#pragma GCC diagnostic push 489#pragma GCC diagonstic ignored "-Wpointer-to-int-cast" 490 491 switch (abi) { 492 case QOS_IOCTL_ABI_ARM32: 493 ret = copy_from_user(&qos_data, 494 (void __user *)compat_ptr((compat_uptr_t)uarg), 495 sizeof(struct qos_ctrl_data)); 496 break; 497 case QOS_IOCTL_ABI_AARCH64: 498 ret = copy_from_user(&qos_data, uarg, sizeof(struct qos_ctrl_data)); 499 break; 500 default: 501 pr_err("[QOS_CTRL] abi format error\n"); 502 break; 503 } 504 505#pragma GCC diagnostic pop 506 507 if (ret) { 508 pr_err("[QOS_CTRL] %s copy user data failed\n", __func__); 509 return ret; 510 } 511 512 ret = do_qos_manipulate(&qos_data); 513 if (ret < 0) { 514 pr_err("[QOS_CTRL] CMD_ID_QOS_MANIPULATE failed\n"); 515 return ret; 516 } 517 518#pragma GCC diagnostic push 519#pragma GCC diagonstic ignored "-Wpointer-to-int-cast" 520 521 switch (abi) { 522 case QOS_IOCTL_ABI_ARM32: 523 ret = copy_to_user((void __user *)compat_ptr((compat_uptr_t)uarg), 524 &qos_data, sizeof(struct qos_ctrl_data)); 525 break; 526 case QOS_IOCTL_ABI_AARCH64: 527 ret = copy_to_user(uarg, &qos_data, sizeof(struct qos_ctrl_data)); 528 break; 529 default: 530 pr_err("[QOS_CTRL] abi format error\n"); 531 break; 532 } 533 534#pragma GCC diagnostic pop 535 536 if (ret) { 537 pr_err("[QOS_CTRL] %s copy to user failed\n", __func__); 538 return ret; 539 } 540 return 0; 541} 542 543#define MAX_LATENCY_NICE 19 544#define MIN_LATENCY_NICE -20 545 546static inline bool valid_nice(int nice) 547{ 548 return nice >= MIN_NICE && nice <= MAX_NICE; 549} 550 551static inline bool valid_latency_nice(int latency_nice) 552{ 553 return latency_nice >= MIN_LATENCY_NICE && latency_nice <= MAX_LATENCY_NICE; 554} 555 556static inline bool valid_uclamp(int uclamp_min, int uclamp_max) 557{ 558 if (uclamp_min > uclamp_max) 559 return false; 560 if (uclamp_max > SCHED_CAPACITY_SCALE) 561 return false; 562 563 return true; 564} 565 566static inline bool valid_rt(int sched_priority) 567{ 568 if (sched_priority > MAX_USER_RT_PRIO - 1 || sched_priority < 0) 569 return false; 570 571 return true; 572} 573 574static bool valid_qos_flag(unsigned int qos_flag) 575{ 576 if (qos_flag & ~QOS_FLAG_ALL) 577 return false; 578 579 return true; 580} 581 582static inline bool valid_qos_item(struct qos_policy_datas *datas) 583{ 584 int i; 585 int type = datas->policy_type; 586 struct qos_policy_data *data; 587 588 if (type <= 0 || type >= QOS_POLICY_MAX_NR) { 589 pr_err("[QOS_CTRL] not valid qos policy type, policy change failed\n"); 590 goto out_failed; 591 } 592 593 if (!valid_qos_flag(datas->policy_flag)) { 594 pr_err("[QOS_CTRL] not valid qos flag, policy change failed\n"); 595 goto out_failed; 596 } 597 598 /* check user space qos polcicy data, level 0 reserved */ 599 for (i = 0; i < NR_QOS; ++i) { 600 data = &datas->policys[i]; 601 602 if (!valid_nice(data->nice)) { 603 pr_err("[QOS_CTRL] invalid nice, policy change failed\n"); 604 goto out_failed; 605 } 606 607 if (!valid_latency_nice(data->latency_nice)) { 608 pr_err("[QOS_CTRL] invalid latency_nice, policy change failed\n"); 609 goto out_failed; 610 } 611 612 if (!valid_uclamp(data->uclamp_min, data->uclamp_max)) { 613 pr_err("[QOS_CTRL] invalid uclamp, policy change failed\n"); 614 goto out_failed; 615 } 616 617 if (!valid_rt(data->rt_sched_priority)) { 618 pr_err("[QOS_CTRL] invalid rt, policy change failed\n"); 619 goto out_failed; 620 } 621 } 622 623 return true; 624 625out_failed: 626 pr_err("[QOS_CTRL] not valid qos policy params\n"); 627 return false; 628} 629 630static long do_qos_policy_change(struct qos_policy_datas *datas) 631{ 632 long ret = 0; 633 int i; 634 struct qos_policy_item *item; 635 struct qos_policy_data *data; 636 int type = datas->policy_type; 637 638 if (type >= QOS_POLICY_MAX_NR) { 639 pr_err("[QOS_CTRL] not valid policy type\n"); 640 goto out_failed; 641 } 642 643 if (!valid_qos_item(datas)) 644 goto out_failed; 645 646 write_lock(&qos_policy_array[type].lock); 647 for (i = QOS_POLICY_MIN_LEVEL; i < NR_QOS; ++i) { 648 item = &qos_policy_array[type].levels[i]; 649 650 /* user space policy params */ 651 data = &datas->policys[i]; 652 653 item->nice = data->nice; 654 item->latency_nice = data->latency_nice; 655 item->uclamp_min = data->uclamp_min; 656 item->uclamp_max = data->uclamp_max; 657 /* only specific qos level could use SCHED_FIFO */ 658 item->rt_sched_priority = (i < MIN_RT_QOS_LEVEL) ? 0 : 659 data->rt_sched_priority; 660 } 661 qos_policy_array[type].policy_flag = datas->policy_flag; 662 qos_policy_array[type].initialized = true; 663 write_unlock(&qos_policy_array[type].lock); 664 665 return ret; 666 667out_failed: 668 return -ARG_INVALID; 669} 670 671static long ctrl_qos_policy(int abi, void __user *uarg) 672{ 673 struct qos_policy_datas policy_datas; 674 long ret = -1; 675 676#pragma GCC diagnostic push 677#pragma GCC diagnostic ignored "-Wpointer-to-int-cast" 678 679 switch (abi) { 680 case QOS_IOCTL_ABI_ARM32: 681 ret = copy_from_user(&policy_datas, 682 (void __user *)compat_ptr((compat_uptr_t)uarg), 683 sizeof(struct qos_policy_datas)); 684 break; 685 case QOS_IOCTL_ABI_AARCH64: 686 ret = copy_from_user(&policy_datas, uarg, sizeof(struct qos_policy_datas)); 687 break; 688 default: 689 pr_err("[QOS_CTRL] abi format error\n"); 690 break; 691 } 692 693#pragma GCC diagnostic pop 694 695 if (ret) { 696 pr_err("[QOS_RTG] %s copy user data failed\n", __func__); 697 return ret; 698 } 699 700 return do_qos_policy_change(&policy_datas); 701} 702 703long do_qos_ctrl_ioctl(int abi, struct file *file, unsigned int cmd, unsigned long arg) 704{ 705 void __user *uarg = (void __user *)arg; 706 unsigned int func_cmd = _IOC_NR(cmd); 707 708 if (uarg == NULL) { 709 pr_err("%s: invalid user uarg\n", __func__); 710 return -EINVAL; 711 } 712 713 if (_IOC_TYPE(cmd) != QOS_CTRL_IPC_MAGIG) { 714 pr_err("%s: qos ctrl magic fail, TYPE=%d\n", 715 __func__, _IOC_TYPE(cmd)); 716 return -EINVAL; 717 } 718 719 if (func_cmd >= QOS_CTRL_MAX_NR) { 720 pr_err("%s: qos ctrl cmd error, cmd:%d\n", 721 __func__, _IOC_TYPE(cmd)); 722 return -EINVAL; 723 } 724 725#ifdef CONFIG_QOS_AUTHORITY 726 if (!check_authorized(func_cmd, QOS_AUTH_FLAG)) { 727 pr_err("[QOS_CTRL] %s: pid not authorized\n", __func__); 728 return -PID_NOT_AUTHORIZED; 729 } 730#endif 731 732 if (g_func_array[func_cmd]) 733 return (*g_func_array[func_cmd])(abi, uarg); 734 735 return -EINVAL; 736} 737 738static void init_qos_policy_array(void) 739{ 740 int i; 741 742 /* index 0 reserved */ 743 for (i = 1; i < QOS_POLICY_MAX_NR; ++i) 744 rwlock_init(&qos_policy_array[i].lock); 745 746 pr_info("[QOS_CTRL] lock in qos policy initialized\n"); 747} 748 749int __init init_qos_ctrl(void) 750{ 751 init_qos_policy_array(); 752 753 return 0; 754} 755 756