1 /*
2 * Copyright © 2015 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 /* A collection of unit tests for cache.c */
25
26 #include <gtest/gtest.h>
27
28 #include <stdio.h>
29 #include <stdlib.h>
30 #include <stdbool.h>
31 #include <string.h>
32 #include <ftw.h>
33 #include <errno.h>
34 #include <stdarg.h>
35 #include <inttypes.h>
36 #include <limits.h>
37 #include <time.h>
38 #include <unistd.h>
39
40 #include "util/mesa-sha1.h"
41 #include "util/disk_cache.h"
42 #include "util/ralloc.h"
43
44 #ifdef ENABLE_SHADER_CACHE
45
46 /* Callback for nftw used in rmrf_local below.
47 */
48 static int
remove_entry(const char *path, const struct stat *sb, int typeflag, struct FTW *ftwbuf)49 remove_entry(const char *path,
50 const struct stat *sb,
51 int typeflag,
52 struct FTW *ftwbuf)
53 {
54 int err = remove(path);
55
56 if (err)
57 fprintf(stderr, "Error removing %s: %s\n", path, strerror(errno));
58
59 return err;
60 }
61
62 /* Recursively remove a directory.
63 *
64 * This is equivalent to "rm -rf <dir>" with one bit of protection
65 * that the directory name must begin with "." to ensure we don't
66 * wander around deleting more than intended.
67 *
68 * Returns 0 on success, -1 on any error.
69 */
70 static int
rmrf_local(const char *path)71 rmrf_local(const char *path)
72 {
73 if (path == NULL || *path == '\0' || *path != '.')
74 return -1;
75
76 return nftw(path, remove_entry, 64, FTW_DEPTH | FTW_PHYS);
77 }
78
79 static void
check_directories_created(void *mem_ctx, const char *cache_dir)80 check_directories_created(void *mem_ctx, const char *cache_dir)
81 {
82 bool sub_dirs_created = false;
83
84 char buf[PATH_MAX];
85 if (getcwd(buf, PATH_MAX)) {
86 char *full_path = ralloc_asprintf(mem_ctx, "%s%s", buf, ++cache_dir);
87 struct stat sb;
88 if (stat(full_path, &sb) != -1 && S_ISDIR(sb.st_mode))
89 sub_dirs_created = true;
90 }
91
92 EXPECT_TRUE(sub_dirs_created) << "create sub dirs";
93 }
94
95 static bool
does_cache_contain(struct disk_cache *cache, const cache_key key)96 does_cache_contain(struct disk_cache *cache, const cache_key key)
97 {
98 void *result;
99
100 result = disk_cache_get(cache, key, NULL);
101
102 if (result) {
103 free(result);
104 return true;
105 }
106
107 return false;
108 }
109
110 static bool
cache_exists(struct disk_cache *cache)111 cache_exists(struct disk_cache *cache)
112 {
113 uint8_t key[20];
114 char data[] = "some test data";
115
116 if (!cache)
117 return false;
118
119 disk_cache_compute_key(cache, data, sizeof(data), key);
120 disk_cache_put(cache, key, data, sizeof(data), NULL);
121 disk_cache_wait_for_idle(cache);
122 void *result = disk_cache_get(cache, key, NULL);
123
124 free(result);
125 return result != NULL;
126 }
127
128 #define CACHE_TEST_TMP "./cache-test-tmp"
129
130 static void
test_disk_cache_create(void *mem_ctx, const char *cache_dir_name)131 test_disk_cache_create(void *mem_ctx, const char *cache_dir_name)
132 {
133 struct disk_cache *cache;
134 int err;
135
136 /* Before doing anything else, ensure that with
137 * MESA_SHADER_CACHE_DISABLE set to true, that disk_cache_create returns NULL.
138 */
139 setenv("MESA_SHADER_CACHE_DISABLE", "true", 1);
140 cache = disk_cache_create("test", "make_check", 0);
141 EXPECT_EQ(cache, nullptr) << "disk_cache_create with MESA_SHADER_CACHE_DISABLE set";
142
143 unsetenv("MESA_SHADER_CACHE_DISABLE");
144
145 #ifdef SHADER_CACHE_DISABLE_BY_DEFAULT
146 /* With SHADER_CACHE_DISABLE_BY_DEFAULT, ensure that with
147 * MESA_SHADER_CACHE_DISABLE set to nothing, disk_cache_create returns NULL.
148 */
149 unsetenv("MESA_SHADER_CACHE_DISABLE");
150 cache = disk_cache_create("test", "make_check", 0);
151 EXPECT_EQ(cache, nullptr)
152 << "disk_cache_create with MESA_SHADER_CACHE_DISABLE unset "
153 "and SHADER_CACHE_DISABLE_BY_DEFAULT build option";
154
155 /* For remaining tests, ensure that the cache is enabled. */
156 setenv("MESA_SHADER_CACHE_DISABLE", "false", 1);
157 #endif /* SHADER_CACHE_DISABLE_BY_DEFAULT */
158
159 /* For the first real disk_cache_create() clear these environment
160 * variables to test creation of cache in home directory.
161 */
162 unsetenv("MESA_SHADER_CACHE_DIR");
163 unsetenv("XDG_CACHE_HOME");
164
165 cache = disk_cache_create("test", "make_check", 0);
166 EXPECT_NE(cache, nullptr) << "disk_cache_create with no environment variables";
167
168 disk_cache_destroy(cache);
169
170 #ifdef ANDROID
171 /* Android doesn't try writing to disk (just calls the cache callbacks), so
172 * the directory tests below don't apply.
173 */
174 return;
175 #endif
176
177 /* Test with XDG_CACHE_HOME set */
178 setenv("XDG_CACHE_HOME", CACHE_TEST_TMP "/xdg-cache-home", 1);
179 cache = disk_cache_create("test", "make_check", 0);
180 EXPECT_FALSE(cache_exists(cache))
181 << "disk_cache_create with XDG_CACHE_HOME set with a non-existing parent directory";
182
183 err = mkdir(CACHE_TEST_TMP, 0755);
184 if (err != 0) {
185 fprintf(stderr, "Error creating %s: %s\n", CACHE_TEST_TMP, strerror(errno));
186 GTEST_FAIL();
187 }
188 disk_cache_destroy(cache);
189
190 cache = disk_cache_create("test", "make_check", 0);
191 EXPECT_TRUE(cache_exists(cache))
192 << "disk_cache_create with XDG_CACHE_HOME set";
193
194 char *path = ralloc_asprintf(
195 mem_ctx, "%s%s", CACHE_TEST_TMP "/xdg-cache-home/", cache_dir_name);
196 check_directories_created(mem_ctx, path);
197
198 disk_cache_destroy(cache);
199
200 /* Test with MESA_SHADER_CACHE_DIR set */
201 err = rmrf_local(CACHE_TEST_TMP);
202 EXPECT_EQ(err, 0) << "Removing " CACHE_TEST_TMP;
203
204 setenv("MESA_SHADER_CACHE_DIR", CACHE_TEST_TMP "/mesa-shader-cache-dir", 1);
205 cache = disk_cache_create("test", "make_check", 0);
206 EXPECT_FALSE(cache_exists(cache))
207 << "disk_cache_create with MESA_SHADER_CACHE_DIR set with a non-existing parent directory";
208
209 err = mkdir(CACHE_TEST_TMP, 0755);
210 if (err != 0) {
211 fprintf(stderr, "Error creating %s: %s\n", CACHE_TEST_TMP, strerror(errno));
212 GTEST_FAIL();
213 }
214 disk_cache_destroy(cache);
215
216 cache = disk_cache_create("test", "make_check", 0);
217 EXPECT_TRUE(cache_exists(cache)) << "disk_cache_create with MESA_SHADER_CACHE_DIR set";
218
219 path = ralloc_asprintf(
220 mem_ctx, "%s%s", CACHE_TEST_TMP "/mesa-shader-cache-dir/", cache_dir_name);
221 check_directories_created(mem_ctx, path);
222
223 disk_cache_destroy(cache);
224 }
225
226 static void
test_put_and_get(bool test_cache_size_limit)227 test_put_and_get(bool test_cache_size_limit)
228 {
229 struct disk_cache *cache;
230 char blob[] = "This is a blob of thirty-seven bytes";
231 uint8_t blob_key[20];
232 char string[] = "While this string has thirty-four";
233 uint8_t string_key[20];
234 char *result;
235 size_t size;
236 uint8_t *one_KB, *one_MB;
237 uint8_t one_KB_key[20], one_MB_key[20];
238 int count;
239
240 #ifdef SHADER_CACHE_DISABLE_BY_DEFAULT
241 setenv("MESA_SHADER_CACHE_DISABLE", "false", 1);
242 #endif /* SHADER_CACHE_DISABLE_BY_DEFAULT */
243
244 cache = disk_cache_create("test", "make_check", 0);
245
246 disk_cache_compute_key(cache, blob, sizeof(blob), blob_key);
247
248 /* Ensure that disk_cache_get returns nothing before anything is added. */
249 result = (char *) disk_cache_get(cache, blob_key, &size);
250 EXPECT_EQ(result, nullptr) << "disk_cache_get with non-existent item (pointer)";
251 EXPECT_EQ(size, 0) << "disk_cache_get with non-existent item (size)";
252
253 /* Simple test of put and get. */
254 disk_cache_put(cache, blob_key, blob, sizeof(blob), NULL);
255
256 /* disk_cache_put() hands things off to a thread so wait for it. */
257 disk_cache_wait_for_idle(cache);
258
259 result = (char *) disk_cache_get(cache, blob_key, &size);
260 EXPECT_STREQ(blob, result) << "disk_cache_get of existing item (pointer)";
261 EXPECT_EQ(size, sizeof(blob)) << "disk_cache_get of existing item (size)";
262
263 free(result);
264
265 /* Test put and get of a second item. */
266 disk_cache_compute_key(cache, string, sizeof(string), string_key);
267 disk_cache_put(cache, string_key, string, sizeof(string), NULL);
268
269 /* disk_cache_put() hands things off to a thread so wait for it. */
270 disk_cache_wait_for_idle(cache);
271
272 result = (char *) disk_cache_get(cache, string_key, &size);
273 EXPECT_STREQ(result, string) << "2nd disk_cache_get of existing item (pointer)";
274 EXPECT_EQ(size, sizeof(string)) << "2nd disk_cache_get of existing item (size)";
275
276 free(result);
277
278 /* Set the cache size to 1KB and add a 1KB item to force an eviction. */
279 disk_cache_destroy(cache);
280
281 if (!test_cache_size_limit)
282 return;
283
284 setenv("MESA_SHADER_CACHE_MAX_SIZE", "1K", 1);
285 cache = disk_cache_create("test", "make_check", 0);
286
287 one_KB = (uint8_t *) calloc(1, 1024);
288
289 /* Obviously the SHA-1 hash of 1024 zero bytes isn't particularly
290 * interesting. But we do have want to take some special care with
291 * the hash we use here. The issue is that in this artificial case,
292 * (with only three files in the cache), the probability is good
293 * that each of the three files will end up in their own
294 * directory. Then, if the directory containing the .tmp file for
295 * the new item being added for disk_cache_put() is the chosen victim
296 * directory for eviction, then no suitable file will be found and
297 * nothing will be evicted.
298 *
299 * That's actually expected given how the eviction code is
300 * implemented, (which expects to only evict once things are more
301 * interestingly full than that).
302 *
303 * For this test, we force this signature to land in the same
304 * directory as the original blob first written to the cache.
305 */
306 disk_cache_compute_key(cache, one_KB, 1024, one_KB_key);
307 one_KB_key[0] = blob_key[0];
308
309 disk_cache_put(cache, one_KB_key, one_KB, 1024, NULL);
310
311 free(one_KB);
312
313 /* disk_cache_put() hands things off to a thread so wait for it. */
314 disk_cache_wait_for_idle(cache);
315
316 result = (char *) disk_cache_get(cache, one_KB_key, &size);
317 EXPECT_NE(result, nullptr) << "3rd disk_cache_get of existing item (pointer)";
318 EXPECT_EQ(size, 1024) << "3rd disk_cache_get of existing item (size)";
319
320 free(result);
321
322 /* Ensure eviction happened by checking that both of the previous
323 * cache itesm were evicted.
324 */
325 bool contains_1KB_file = false;
326 count = 0;
327 if (does_cache_contain(cache, blob_key))
328 count++;
329
330 if (does_cache_contain(cache, string_key))
331 count++;
332
333 if (does_cache_contain(cache, one_KB_key)) {
334 count++;
335 contains_1KB_file = true;
336 }
337
338 EXPECT_TRUE(contains_1KB_file)
339 << "disk_cache_put eviction last file == MAX_SIZE (1KB)";
340 EXPECT_EQ(count, 1) << "disk_cache_put eviction with MAX_SIZE=1K";
341
342 /* Now increase the size to 1M, add back both items, and ensure all
343 * three that have been added are available via disk_cache_get.
344 */
345 disk_cache_destroy(cache);
346
347 setenv("MESA_SHADER_CACHE_MAX_SIZE", "1M", 1);
348 cache = disk_cache_create("test", "make_check", 0);
349
350 disk_cache_put(cache, blob_key, blob, sizeof(blob), NULL);
351 disk_cache_put(cache, string_key, string, sizeof(string), NULL);
352
353 /* disk_cache_put() hands things off to a thread so wait for it. */
354 disk_cache_wait_for_idle(cache);
355
356 count = 0;
357 if (does_cache_contain(cache, blob_key))
358 count++;
359
360 if (does_cache_contain(cache, string_key))
361 count++;
362
363 if (does_cache_contain(cache, one_KB_key))
364 count++;
365
366 EXPECT_EQ(count, 3) << "no eviction before overflow with MAX_SIZE=1M";
367
368 /* Finally, check eviction again after adding an object of size 1M. */
369 one_MB = (uint8_t *) calloc(1024, 1024);
370
371 disk_cache_compute_key(cache, one_MB, 1024 * 1024, one_MB_key);
372 one_MB_key[0] = blob_key[0];
373
374 disk_cache_put(cache, one_MB_key, one_MB, 1024 * 1024, NULL);
375
376 free(one_MB);
377
378 /* disk_cache_put() hands things off to a thread so wait for it. */
379 disk_cache_wait_for_idle(cache);
380
381 bool contains_1MB_file = false;
382 count = 0;
383 if (does_cache_contain(cache, blob_key))
384 count++;
385
386 if (does_cache_contain(cache, string_key))
387 count++;
388
389 if (does_cache_contain(cache, one_KB_key))
390 count++;
391
392 if (does_cache_contain(cache, one_MB_key)) {
393 count++;
394 contains_1MB_file = true;
395 }
396
397 EXPECT_TRUE(contains_1MB_file)
398 << "disk_cache_put eviction last file == MAX_SIZE (1MB)";
399 EXPECT_EQ(count, 1) << "eviction after overflow with MAX_SIZE=1M";
400
401 disk_cache_destroy(cache);
402 }
403
404 static void
test_put_key_and_get_key(void)405 test_put_key_and_get_key(void)
406 {
407 struct disk_cache *cache;
408 bool result;
409
410 uint8_t key_a[20] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,
411 10, 11, 12, 13, 14, 15, 16, 17, 18, 19};
412 uint8_t key_b[20] = { 20, 21, 22, 23, 24, 25, 26, 27, 28, 29,
413 30, 33, 32, 33, 34, 35, 36, 37, 38, 39};
414 uint8_t key_a_collide[20] =
415 { 0, 1, 42, 43, 44, 45, 46, 47, 48, 49,
416 50, 55, 52, 53, 54, 55, 56, 57, 58, 59};
417
418 #ifdef SHADER_CACHE_DISABLE_BY_DEFAULT
419 setenv("MESA_SHADER_CACHE_DISABLE", "false", 1);
420 #endif /* SHADER_CACHE_DISABLE_BY_DEFAULT */
421
422 cache = disk_cache_create("test", "make_check", 0);
423
424 /* First test that disk_cache_has_key returns false before disk_cache_put_key */
425 result = disk_cache_has_key(cache, key_a);
426 EXPECT_EQ(result, 0) << "disk_cache_has_key before key added";
427
428 /* Then a couple of tests of disk_cache_put_key followed by disk_cache_has_key */
429 disk_cache_put_key(cache, key_a);
430 result = disk_cache_has_key(cache, key_a);
431 EXPECT_EQ(result, 1) << "disk_cache_has_key after key added";
432
433 disk_cache_put_key(cache, key_b);
434 result = disk_cache_has_key(cache, key_b);
435 EXPECT_EQ(result, 1) << "2nd disk_cache_has_key after key added";
436
437 /* Test that a key with the same two bytes as an existing key
438 * forces an eviction.
439 */
440 disk_cache_put_key(cache, key_a_collide);
441 result = disk_cache_has_key(cache, key_a_collide);
442 EXPECT_EQ(result, 1) << "put_key of a colliding key lands in the cache";
443
444 result = disk_cache_has_key(cache, key_a);
445 EXPECT_EQ(result, 0) << "put_key of a colliding key evicts from the cache";
446
447 /* And finally test that we can re-add the original key to re-evict
448 * the colliding key.
449 */
450 disk_cache_put_key(cache, key_a);
451 result = disk_cache_has_key(cache, key_a);
452 EXPECT_EQ(result, 1) << "put_key of original key lands again";
453
454 result = disk_cache_has_key(cache, key_a_collide);
455 EXPECT_EQ(result, 0) << "put_key of orginal key evicts the colliding key";
456
457 disk_cache_destroy(cache);
458 }
459
460 /* To make sure we are not just using the inmemory cache index for the single
461 * file cache we test adding and retriving cache items between two different
462 * cache instances.
463 */
464 static void
test_put_and_get_between_instances()465 test_put_and_get_between_instances()
466 {
467 char blob[] = "This is a blob of thirty-seven bytes";
468 uint8_t blob_key[20];
469 char string[] = "While this string has thirty-four";
470 uint8_t string_key[20];
471 char *result;
472 size_t size;
473
474 #ifdef SHADER_CACHE_DISABLE_BY_DEFAULT
475 setenv("MESA_SHADER_CACHE_DISABLE", "false", 1);
476 #endif /* SHADER_CACHE_DISABLE_BY_DEFAULT */
477
478 struct disk_cache *cache1 = disk_cache_create("test_between_instances",
479 "make_check", 0);
480 struct disk_cache *cache2 = disk_cache_create("test_between_instances",
481 "make_check", 0);
482
483 disk_cache_compute_key(cache1, blob, sizeof(blob), blob_key);
484
485 /* Ensure that disk_cache_get returns nothing before anything is added. */
486 result = (char *) disk_cache_get(cache1, blob_key, &size);
487 EXPECT_EQ(result, nullptr) << "disk_cache_get(cache1) with non-existent item (pointer)";
488 EXPECT_EQ(size, 0) << "disk_cache_get(cach1) with non-existent item (size)";
489
490 result = (char *) disk_cache_get(cache2, blob_key, &size);
491 EXPECT_EQ(result, nullptr) << "disk_cache_get(cache2) with non-existent item (pointer)";
492 EXPECT_EQ(size, 0) << "disk_cache_get(cache2) with non-existent item (size)";
493
494 /* Simple test of put and get. */
495 disk_cache_put(cache1, blob_key, blob, sizeof(blob), NULL);
496
497 /* disk_cache_put() hands things off to a thread so wait for it. */
498 disk_cache_wait_for_idle(cache1);
499
500 result = (char *) disk_cache_get(cache2, blob_key, &size);
501 EXPECT_STREQ(blob, result) << "disk_cache_get(cache2) of existing item (pointer)";
502 EXPECT_EQ(size, sizeof(blob)) << "disk_cache_get of(cache2) existing item (size)";
503
504 free(result);
505
506 /* Test put and get of a second item, via the opposite instances */
507 disk_cache_compute_key(cache2, string, sizeof(string), string_key);
508 disk_cache_put(cache2, string_key, string, sizeof(string), NULL);
509
510 /* disk_cache_put() hands things off to a thread so wait for it. */
511 disk_cache_wait_for_idle(cache2);
512
513 result = (char *) disk_cache_get(cache1, string_key, &size);
514 EXPECT_STREQ(result, string) << "2nd disk_cache_get(cache1) of existing item (pointer)";
515 EXPECT_EQ(size, sizeof(string)) << "2nd disk_cache_get(cache1) of existing item (size)";
516
517 free(result);
518
519 disk_cache_destroy(cache1);
520 disk_cache_destroy(cache2);
521 }
522 #endif /* ENABLE_SHADER_CACHE */
523
524 class Cache : public ::testing::Test {
525 protected:
526 void *mem_ctx;
527
Cache()528 Cache() {
529 mem_ctx = ralloc_context(NULL);
530 }
~Cache()531 ~Cache() {
532 ralloc_free(mem_ctx);
533 }
534 };
535
TEST_F(Cache, MultiFile)536 TEST_F(Cache, MultiFile)
537 {
538 #ifndef ENABLE_SHADER_CACHE
539 GTEST_SKIP() << "ENABLE_SHADER_CACHE not defined.";
540 #else
541 test_disk_cache_create(mem_ctx, CACHE_DIR_NAME);
542
543 test_put_and_get(true);
544
545 test_put_key_and_get_key();
546
547 int err = rmrf_local(CACHE_TEST_TMP);
548 EXPECT_EQ(err, 0) << "Removing " CACHE_TEST_TMP " again";
549 #endif
550 }
551
TEST_F(Cache, SingleFile)552 TEST_F(Cache, SingleFile)
553 {
554 #ifndef ENABLE_SHADER_CACHE
555 GTEST_SKIP() << "ENABLE_SHADER_CACHE not defined.";
556 #else
557 setenv("MESA_DISK_CACHE_SINGLE_FILE", "true", 1);
558
559 test_disk_cache_create(mem_ctx, CACHE_DIR_NAME_SF);
560
561 /* We skip testing cache size limit as the single file cache currently
562 * doesn't have any functionality to enforce cache size limits.
563 */
564 test_put_and_get(false);
565
566 test_put_key_and_get_key();
567
568 test_put_and_get_between_instances();
569
570 setenv("MESA_DISK_CACHE_SINGLE_FILE", "false", 1);
571
572 int err = rmrf_local(CACHE_TEST_TMP);
573 EXPECT_EQ(err, 0) << "Removing " CACHE_TEST_TMP " again";
574 #endif
575 }
576