birq.c 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620
  1. /*
  2. * birq
  3. *
  4. * Balance IRQ
  5. *
  6. */
  7. #ifdef HAVE_CONFIG_H
  8. #include "config.h"
  9. #endif /* HAVE_CONFIG_H */
  10. #include <stdio.h>
  11. #include <stdlib.h>
  12. #include <unistd.h>
  13. #include <sys/types.h>
  14. #include <errno.h>
  15. #include <assert.h>
  16. #include <string.h>
  17. #include <signal.h>
  18. #include <syslog.h>
  19. #include <fcntl.h>
  20. #include <time.h>
  21. #ifdef HAVE_GETOPT_H
  22. #include <getopt.h>
  23. #endif
  24. #include "birq.h"
  25. #include "lub/log.h"
  26. #include "lub/list.h"
  27. #include "lub/ini.h"
  28. #include "irq.h"
  29. #include "numa.h"
  30. #include "cpu.h"
  31. #include "statistics.h"
  32. #include "balance.h"
  33. #include "pxm.h"
  34. #ifndef VERSION
  35. #define VERSION "1.2.0"
  36. #endif
  37. /* Signal handlers */
  38. static volatile int sigterm = 0; /* Exit if 1 */
  39. static void sighandler(int signo);
  40. static volatile int sighup = 0; /* Re-read config file */
  41. static void sighup_handler(int signo);
  42. static void help(int status, const char *argv0);
  43. static struct options *opts_init(void);
  44. static void opts_free(struct options *opts);
  45. static int opts_parse(int argc, char *argv[], struct options *opts);
  46. static int parse_config(const char *fname, struct options *opts);
  47. /* Command line options */
  48. struct options {
  49. char *pidfile;
  50. char *cfgfile;
  51. int cfgfile_userdefined;
  52. char *pxm; /* Proximity config file */
  53. int debug; /* Don't daemonize in debug mode */
  54. int log_facility;
  55. float threshold;
  56. float load_limit;
  57. int verbose;
  58. int ht;
  59. int non_local_cpus;
  60. unsigned int long_interval;
  61. unsigned int short_interval;
  62. birq_choose_strategy_e strategy;
  63. cpumask_t exclude_cpus;
  64. };
  65. /*--------------------------------------------------------- */
  66. int main(int argc, char **argv)
  67. {
  68. int retval = -1;
  69. struct options *opts = NULL;
  70. int pidfd = -1;
  71. unsigned int interval;
  72. /* Signal vars */
  73. struct sigaction sig_act;
  74. sigset_t sig_set;
  75. /* IRQ list. It contain all found IRQs. */
  76. lub_list_t *irqs;
  77. /* IRQs need to be balanced */
  78. lub_list_t *balance_irqs;
  79. /* CPU list. It contain all found CPUs. */
  80. lub_list_t *cpus;
  81. /* NUMA list. It contain all found NUMA nodes. */
  82. lub_list_t *numas;
  83. /* Proximity list. */
  84. lub_list_t *pxms;
  85. /* Parse command line options */
  86. opts = opts_init();
  87. if (opts_parse(argc, argv, opts))
  88. goto err;
  89. /* Parse config file */
  90. if (!access(opts->cfgfile, R_OK)) {
  91. if (parse_config(opts->cfgfile, opts))
  92. goto err;
  93. } else if (opts->cfgfile_userdefined) {
  94. fprintf(stderr, "Error: Can't find config file %s\n",
  95. opts->cfgfile);
  96. goto err;
  97. }
  98. /* Initialize syslog */
  99. openlog(argv[0], LOG_CONS, opts->log_facility);
  100. syslog(LOG_INFO, "Start daemon\n");
  101. /* Fork the daemon */
  102. if (!opts->debug) {
  103. /* Daemonize */
  104. if (daemon(0, 0) < 0) {
  105. syslog(LOG_ERR, "Can't daemonize\n");
  106. goto err;
  107. }
  108. /* Write pidfile */
  109. if ((pidfd = open(opts->pidfile,
  110. O_WRONLY | O_CREAT | O_EXCL | O_TRUNC,
  111. S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH)) < 0) {
  112. syslog(LOG_WARNING, "Can't open pidfile %s: %s\n",
  113. opts->pidfile, strerror(errno));
  114. } else {
  115. char str[20];
  116. snprintf(str, sizeof(str), "%u\n", getpid());
  117. str[sizeof(str) - 1] = '\0';
  118. if (write(pidfd, str, strlen(str)) < 0)
  119. syslog(LOG_WARNING, "Can't write to %s: %s\n",
  120. opts->pidfile, strerror(errno));
  121. close(pidfd);
  122. }
  123. }
  124. /* Set signal handler */
  125. sigemptyset(&sig_set);
  126. sigaddset(&sig_set, SIGTERM);
  127. sigaddset(&sig_set, SIGINT);
  128. sigaddset(&sig_set, SIGQUIT);
  129. sig_act.sa_flags = 0;
  130. sig_act.sa_mask = sig_set;
  131. sig_act.sa_handler = &sighandler;
  132. sigaction(SIGTERM, &sig_act, NULL);
  133. sigaction(SIGINT, &sig_act, NULL);
  134. sigaction(SIGQUIT, &sig_act, NULL);
  135. /* SIGHUP handler */
  136. sigemptyset(&sig_set);
  137. sigaddset(&sig_set, SIGHUP);
  138. sig_act.sa_flags = 0;
  139. sig_act.sa_mask = sig_set;
  140. sig_act.sa_handler = &sighup_handler;
  141. sigaction(SIGHUP, &sig_act, NULL);
  142. /* Randomize */
  143. srand(time(NULL));
  144. /* Scan NUMA nodes */
  145. numas = lub_list_new(numa_list_compare);
  146. scan_numas(numas);
  147. if (opts->verbose)
  148. show_numas(numas);
  149. /* Scan CPUs */
  150. cpus = lub_list_new(cpu_list_compare);
  151. scan_cpus(cpus, opts->ht);
  152. if (opts->verbose)
  153. show_cpus(cpus);
  154. /* Prepare data structures */
  155. irqs = lub_list_new(irq_list_compare);
  156. balance_irqs = lub_list_new(irq_list_compare);
  157. /* Parse proximity file */
  158. pxms = lub_list_new(NULL);
  159. if (opts->pxm)
  160. parse_pxm_config(opts->pxm, pxms, numas);
  161. if (opts->verbose)
  162. show_pxms(pxms);
  163. /* Main loop */
  164. while (!sigterm) {
  165. lub_list_node_t *node;
  166. char outstr[10];
  167. time_t t;
  168. struct tm *tmp;
  169. t = time(NULL);
  170. tmp = localtime(&t);
  171. if (tmp) {
  172. strftime(outstr, sizeof(outstr), "%H:%M:%S", tmp);
  173. printf("----[ %s ]----------------------------------------------------------------\n", outstr);
  174. }
  175. /* Re-read config file on SIGHUP */
  176. if (sighup) {
  177. if (!access(opts->cfgfile, R_OK)) {
  178. syslog(LOG_INFO, "Re-reading config file\n");
  179. if (parse_config(opts->cfgfile, opts))
  180. syslog(LOG_ERR, "Error while config file parsing\n");
  181. } else if (opts->cfgfile_userdefined)
  182. syslog(LOG_ERR, "Can't find config file\n");
  183. sighup = 0;
  184. }
  185. /* Rescan PCI devices for new IRQs. */
  186. scan_irqs(irqs, balance_irqs, pxms);
  187. if (opts->verbose)
  188. irq_list_show(irqs);
  189. /* Link IRQs to CPUs due to real current smp affinity. */
  190. link_irqs_to_cpus(cpus, irqs);
  191. /* Gather statistics on CPU load and number of interrupts. */
  192. gather_statistics(cpus, irqs);
  193. show_statistics(cpus, opts->verbose);
  194. /* Choose IRQ to move to another CPU. */
  195. choose_irqs_to_move(cpus, balance_irqs,
  196. opts->threshold, opts->strategy, &opts->exclude_cpus);
  197. /* Balance IRQs */
  198. if (lub_list_len(balance_irqs) != 0) {
  199. /* Set short interval to make balancing faster. */
  200. interval = opts->short_interval;
  201. /* Choose new CPU for IRQs need to be balanced. */
  202. balance(cpus, balance_irqs, opts->load_limit,
  203. &opts->exclude_cpus, opts->non_local_cpus);
  204. /* Write new values to /proc/irq/<IRQ>/smp_affinity */
  205. apply_affinity(balance_irqs);
  206. /* Free list of balanced IRQs */
  207. while ((node = lub_list__get_tail(balance_irqs))) {
  208. lub_list_del(balance_irqs, node);
  209. lub_list_node_free(node);
  210. }
  211. } else {
  212. /* If nothing to balance */
  213. interval = opts->long_interval;
  214. }
  215. /* Wait before next iteration */
  216. sleep(interval);
  217. }
  218. /* Free data structures */
  219. irq_list_free(irqs);
  220. lub_list_free(balance_irqs);
  221. cpu_list_free(cpus);
  222. numa_list_free(numas);
  223. pxm_list_free(pxms);
  224. retval = 0;
  225. err:
  226. /* Remove pidfile */
  227. if (pidfd >= 0) {
  228. if (unlink(opts->pidfile) < 0) {
  229. syslog(LOG_WARNING, "Can't remove pid-file %s: %s\n",
  230. opts->pidfile, strerror(errno));
  231. }
  232. }
  233. /* Free command line options */
  234. opts_free(opts);
  235. syslog(LOG_INFO, "Stop daemon\n");
  236. return retval;
  237. }
  238. /*--------------------------------------------------------- */
  239. /* Signal handler for temination signals (like SIGTERM, SIGINT, ...) */
  240. static void sighandler(int signo)
  241. {
  242. sigterm = 1;
  243. signo = signo; /* Happy compiler */
  244. }
  245. /*--------------------------------------------------------- */
  246. /* Re-read config file on SIGHUP */
  247. static void sighup_handler(int signo)
  248. {
  249. sighup = 1;
  250. signo = signo; /* Happy compiler */
  251. }
  252. /*--------------------------------------------------------- */
  253. /* Set defaults for options from config file (not command line) */
  254. static void opts_default_config(struct options *opts)
  255. {
  256. assert(opts);
  257. opts->threshold = BIRQ_DEFAULT_THRESHOLD;
  258. opts->load_limit = BIRQ_DEFAULT_LOAD_LIMIT;
  259. opts->ht = 1; /* It's 1 since 1.5.0 */
  260. opts->non_local_cpus = 0;
  261. opts->long_interval = BIRQ_LONG_INTERVAL;
  262. opts->short_interval = BIRQ_SHORT_INTERVAL;
  263. opts->strategy = BIRQ_CHOOSE_RND;
  264. cpus_clear(opts->exclude_cpus);
  265. }
  266. /*--------------------------------------------------------- */
  267. /* Initialize option structure by defaults */
  268. static struct options *opts_init(void)
  269. {
  270. struct options *opts = NULL;
  271. // Allocate structures
  272. opts = malloc(sizeof(*opts));
  273. assert(opts);
  274. cpus_init(opts->exclude_cpus);
  275. // Set command line options defaults.
  276. opts->debug = 0; /* daemonize by default */
  277. opts->pidfile = strdup(BIRQ_PIDFILE);
  278. opts->cfgfile = strdup(BIRQ_CFGFILE);
  279. opts->cfgfile_userdefined = 0;
  280. opts->pxm = NULL;
  281. opts->log_facility = LOG_DAEMON;
  282. opts->verbose = 0;
  283. // The daemon can be used without config file. So set defaults for
  284. // options from config file.
  285. opts_default_config(opts);
  286. return opts;
  287. }
  288. /*--------------------------------------------------------- */
  289. /* Free option structure */
  290. static void opts_free(struct options *opts)
  291. {
  292. if (opts->pidfile)
  293. free(opts->pidfile);
  294. if (opts->cfgfile)
  295. free(opts->cfgfile);
  296. if (opts->pxm)
  297. free(opts->pxm);
  298. cpus_free(opts->exclude_cpus);
  299. free(opts);
  300. }
  301. /* Parse y/n options */
  302. static int opt_parse_y_n(const char *optarg, int *flag)
  303. {
  304. assert(optarg);
  305. assert(flag);
  306. if (!strcmp(optarg, "y"))
  307. *flag = 1;
  308. else if (!strcmp(optarg, "yes"))
  309. *flag = 1;
  310. else if (!strcmp(optarg, "n"))
  311. *flag = 0;
  312. else if (!strcmp(optarg, "no"))
  313. *flag = 0;
  314. else {
  315. fprintf(stderr, "Error: Illegal flag value %s.\n", optarg);
  316. return -1;
  317. }
  318. return 0;
  319. }
  320. /* Parse 'strategy' option */
  321. static int opt_parse_strategy(const char *optarg, birq_choose_strategy_e *strategy)
  322. {
  323. assert(optarg);
  324. assert(strategy);
  325. if (!strcmp(optarg, "max"))
  326. *strategy = BIRQ_CHOOSE_MAX;
  327. else if (!strcmp(optarg, "min"))
  328. *strategy = BIRQ_CHOOSE_MIN;
  329. else if (!strcmp(optarg, "rnd"))
  330. *strategy = BIRQ_CHOOSE_RND;
  331. else {
  332. fprintf(stderr, "Error: Illegal strategy value %s.\n", optarg);
  333. return -1;
  334. }
  335. return 0;
  336. }
  337. /* Parse 'threshold' and 'load-limit' options */
  338. static int opt_parse_threshold(const char *optarg, float *threshold)
  339. {
  340. char *endptr;
  341. float thresh;
  342. assert(optarg);
  343. assert(threshold);
  344. thresh = strtof(optarg, &endptr);
  345. if (endptr == optarg) {
  346. fprintf(stderr, "Error: Illegal threshold/load-limit value %s.\n", optarg);
  347. return -1;
  348. }
  349. if (thresh > 100.00) {
  350. fprintf(stderr, "Error: The threshold/load-limit value %s > 100.\n", optarg);
  351. return -1;
  352. }
  353. *threshold = thresh;
  354. return 0;
  355. }
  356. /* Parse 'short-interval' and 'long-interval' options */
  357. static int opt_parse_interval(const char *optarg, unsigned int *interval)
  358. {
  359. char *endptr;
  360. unsigned long int val;
  361. assert(optarg);
  362. assert(interval);
  363. val = strtoul(optarg, &endptr, 10);
  364. if (endptr == optarg) {
  365. fprintf(stderr, "Error: Illegal interval value %s.\n", optarg);
  366. return -1;
  367. }
  368. *interval = val;
  369. return 0;
  370. }
  371. /*--------------------------------------------------------- */
  372. /* Parse command line options */
  373. static int opts_parse(int argc, char *argv[], struct options *opts)
  374. {
  375. static const char *shortopts = "hp:c:dO:vx:";
  376. #ifdef HAVE_GETOPT_H
  377. static const struct option longopts[] = {
  378. {"help", 0, NULL, 'h'},
  379. {"pid", 1, NULL, 'p'},
  380. {"conf", 1, NULL, 'c'},
  381. {"debug", 0, NULL, 'd'},
  382. {"facility", 1, NULL, 'O'},
  383. {"verbose", 0, NULL, 'v'},
  384. {"pxm", 1, NULL, 'x'},
  385. {NULL, 0, NULL, 0}
  386. };
  387. #endif
  388. optind = 1;
  389. while(1) {
  390. int opt;
  391. #ifdef HAVE_GETOPT_H
  392. opt = getopt_long(argc, argv, shortopts, longopts, NULL);
  393. #else
  394. opt = getopt(argc, argv, shortopts);
  395. #endif
  396. if (-1 == opt)
  397. break;
  398. switch (opt) {
  399. case 'p':
  400. if (opts->pidfile)
  401. free(opts->pidfile);
  402. opts->pidfile = strdup(optarg);
  403. break;
  404. case 'c':
  405. if (opts->cfgfile)
  406. free(opts->cfgfile);
  407. opts->cfgfile = strdup(optarg);
  408. opts->cfgfile_userdefined = 1;
  409. break;
  410. case 'x':
  411. if (opts->pxm)
  412. free(opts->pxm);
  413. opts->pxm = strdup(optarg);
  414. break;
  415. case 'd':
  416. opts->debug = 1;
  417. break;
  418. case 'v':
  419. opts->verbose = 1;
  420. break;
  421. case 'O':
  422. if (lub_log_facility(optarg, &(opts->log_facility))) {
  423. fprintf(stderr, "Error: Illegal syslog facility %s.\n", optarg);
  424. exit(-1);
  425. }
  426. break;
  427. case 'h':
  428. help(0, argv[0]);
  429. exit(0);
  430. break;
  431. default:
  432. help(-1, argv[0]);
  433. exit(-1);
  434. break;
  435. }
  436. }
  437. return 0;
  438. }
  439. /*--------------------------------------------------------- */
  440. /* Print help message */
  441. static void help(int status, const char *argv0)
  442. {
  443. const char *name = NULL;
  444. if (!argv0)
  445. return;
  446. /* Find the basename */
  447. name = strrchr(argv0, '/');
  448. if (name)
  449. name++;
  450. else
  451. name = argv0;
  452. if (status != 0) {
  453. fprintf(stderr, "Try `%s -h' for more information.\n",
  454. name);
  455. } else {
  456. printf("Version : %s\n", VERSION);
  457. printf("Usage : %s [options]\n", name);
  458. printf("Daemon to balance IRQs.\n");
  459. printf("Options :\n");
  460. printf("\t-h, --help Print this help.\n");
  461. printf("\t-d, --debug Debug mode. Don't daemonize.\n");
  462. printf("\t-v, --verbose Be verbose.\n");
  463. printf("\t-r, --ht This option is obsoleted. The Hyper Threading is enabled by default.\n");
  464. printf("\t-p <path>, --pid=<path> File to save daemon's PID to (" BIRQ_PIDFILE ").\n");
  465. printf("\t-c <path>, --conf=<path> Config file (" BIRQ_CFGFILE ").\n");
  466. printf("\t-x <path>, --pxm=<path> Proximity config file.\n");
  467. printf("\t-O, --facility Syslog facility (DAEMON).\n");
  468. printf("\t-t <float>, --threshold=<float> Threshold to consider CPU is overloaded, in percents. Default threhold is %.2f.\n",
  469. BIRQ_DEFAULT_THRESHOLD);
  470. printf("\t-l <float>, --load-limit=<float> Don't move IRQs to CPUs loaded more than this limit, in percents. Default limit is %.2f.\n",
  471. BIRQ_DEFAULT_LOAD_LIMIT);
  472. printf("\t-i <sec>, --short-interval=<sec> Short iteration interval.\n");
  473. printf("\t-I <sec>, --long-interval=<sec> Long iteration interval.\n");
  474. printf("\t-s <strategy>, --strategy=<strategy> Strategy to choose IRQ to move (min/max/rnd).\n");
  475. }
  476. }
  477. /*--------------------------------------------------------- */
  478. /* Parse config file */
  479. static int parse_config(const char *fname, struct options *opts)
  480. {
  481. int ret = -1; /* Pessimistic retval */
  482. lub_ini_t *ini = NULL;
  483. const char *tmp = NULL;
  484. cpumask_t use_cpus;
  485. // Set options defaults. It's necessary because the config file can be
  486. // re-read by SIGHUP. So we need to drop old values to default state.
  487. opts_default_config(opts);
  488. ini = lub_ini_new();
  489. if (lub_ini_parse_file(ini, fname)) {
  490. lub_ini_free(ini);
  491. return -1;
  492. }
  493. if ((tmp = lub_ini_find(ini, "strategy")))
  494. if (opt_parse_strategy(tmp, &opts->strategy) < 0)
  495. goto err;
  496. if ((tmp = lub_ini_find(ini, "threshold")))
  497. if (opt_parse_threshold(tmp, &opts->threshold))
  498. goto err;
  499. if ((tmp = lub_ini_find(ini, "load-limit")))
  500. if (opt_parse_threshold(tmp, &opts->load_limit))
  501. goto err;
  502. if ((tmp = lub_ini_find(ini, "short-interval")))
  503. if (opt_parse_interval(tmp, &opts->short_interval))
  504. goto err;
  505. if ((tmp = lub_ini_find(ini, "long-interval")))
  506. if (opt_parse_interval(tmp, &opts->long_interval))
  507. goto err;
  508. if ((tmp = lub_ini_find(ini, "exclude-cpus"))) {
  509. if (cpumask_parse_user(tmp, strlen(tmp), opts->exclude_cpus)) {
  510. fprintf(stderr, "Error: Can't parse exclude-cpus option \"%s\".\n", tmp);
  511. goto err;
  512. }
  513. }
  514. cpus_init(use_cpus);
  515. if ((tmp = lub_ini_find(ini, "use-cpus"))) {
  516. if (cpumask_parse_user(tmp, strlen(tmp), use_cpus)) {
  517. fprintf(stderr, "Error: Can't parse use-cpus option \"%s\".\n", tmp);
  518. goto err;
  519. }
  520. } else {
  521. cpus_setall(use_cpus);
  522. }
  523. /* The exclude-cpus option was implemented first. So the
  524. * programm is based on it. The use-cpus options really
  525. * says to exclude all the cpus that is not within bitmask.
  526. * So invert use-cpus and we'll get exclude-cpus mask.
  527. */
  528. cpus_complement(use_cpus, use_cpus);
  529. /* Calculate real exclude-cpu mask (consider use-cpus option).
  530. * real-exclude-cpus = exclude-cpus | ~use-cpus
  531. */
  532. cpus_or(opts->exclude_cpus, opts->exclude_cpus, use_cpus);
  533. cpus_free(use_cpus);
  534. if ((tmp = lub_ini_find(ini, "ht")))
  535. if (opt_parse_y_n(tmp, &opts->ht))
  536. goto err;
  537. if ((tmp = lub_ini_find(ini, "non-local-cpus")))
  538. if (opt_parse_y_n(tmp, &opts->non_local_cpus))
  539. goto err;
  540. ret = 0;
  541. err:
  542. lub_ini_free(ini);
  543. return ret;
  544. }