1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Generic userspace implementations of gettimeofday() and similar.
4 */
5#include <vdso/datapage.h>
6#include <vdso/helpers.h>
7
8#ifndef vdso_calc_delta
9/*
10 * Default implementation which works for all sane clocksources. That
11 * obviously excludes x86/TSC.
12 */
13static __always_inline
14u64 vdso_calc_delta(u64 cycles, u64 last, u64 mask, u32 mult)
15{
16	return ((cycles - last) & mask) * mult;
17}
18#endif
19
20#ifndef vdso_shift_ns
21static __always_inline u64 vdso_shift_ns(u64 ns, u32 shift)
22{
23	return ns >> shift;
24}
25#endif
26
27#ifndef __arch_vdso_hres_capable
28static inline bool __arch_vdso_hres_capable(void)
29{
30	return true;
31}
32#endif
33
34#ifndef vdso_clocksource_ok
35static inline bool vdso_clocksource_ok(const struct vdso_data *vd)
36{
37	return vd->clock_mode != VDSO_CLOCKMODE_NONE;
38}
39#endif
40
41#ifndef vdso_cycles_ok
42static inline bool vdso_cycles_ok(u64 cycles)
43{
44	return true;
45}
46#endif
47
48#ifdef CONFIG_TIME_NS
49static __always_inline int do_hres_timens(const struct vdso_data *vdns, clockid_t clk,
50					  struct __kernel_timespec *ts)
51{
52	const struct vdso_data *vd = __arch_get_timens_vdso_data();
53	const struct timens_offset *offs = &vdns->offset[clk];
54	const struct vdso_timestamp *vdso_ts;
55	u64 cycles, last, ns;
56	u32 seq;
57	s64 sec;
58
59	if (clk != CLOCK_MONOTONIC_RAW)
60		vd = &vd[CS_HRES_COARSE];
61	else
62		vd = &vd[CS_RAW];
63	vdso_ts = &vd->basetime[clk];
64
65	do {
66		seq = vdso_read_begin(vd);
67
68		if (unlikely(!vdso_clocksource_ok(vd)))
69			return -1;
70
71		cycles = __arch_get_hw_counter(vd->clock_mode, vd);
72		if (unlikely(!vdso_cycles_ok(cycles)))
73			return -1;
74		ns = vdso_ts->nsec;
75		last = vd->cycle_last;
76		ns += vdso_calc_delta(cycles, last, vd->mask, vd->mult);
77		ns = vdso_shift_ns(ns, vd->shift);
78		sec = vdso_ts->sec;
79	} while (unlikely(vdso_read_retry(vd, seq)));
80
81	/* Add the namespace offset */
82	sec += offs->sec;
83	ns += offs->nsec;
84
85	/*
86	 * Do this outside the loop: a race inside the loop could result
87	 * in __iter_div_u64_rem() being extremely slow.
88	 */
89	ts->tv_sec = sec + __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
90	ts->tv_nsec = ns;
91
92	return 0;
93}
94#else
95static __always_inline const struct vdso_data *__arch_get_timens_vdso_data(void)
96{
97	return NULL;
98}
99
100static __always_inline int do_hres_timens(const struct vdso_data *vdns, clockid_t clk,
101					  struct __kernel_timespec *ts)
102{
103	return -EINVAL;
104}
105#endif
106
107static __always_inline int do_hres(const struct vdso_data *vd, clockid_t clk,
108				   struct __kernel_timespec *ts)
109{
110	const struct vdso_timestamp *vdso_ts = &vd->basetime[clk];
111	u64 cycles, last, sec, ns;
112	u32 seq;
113
114	/* Allows to compile the high resolution parts out */
115	if (!__arch_vdso_hres_capable())
116		return -1;
117
118	do {
119		/*
120		 * Open coded to handle VDSO_CLOCKMODE_TIMENS. Time namespace
121		 * enabled tasks have a special VVAR page installed which
122		 * has vd->seq set to 1 and vd->clock_mode set to
123		 * VDSO_CLOCKMODE_TIMENS. For non time namespace affected tasks
124		 * this does not affect performance because if vd->seq is
125		 * odd, i.e. a concurrent update is in progress the extra
126		 * check for vd->clock_mode is just a few extra
127		 * instructions while spin waiting for vd->seq to become
128		 * even again.
129		 */
130		while (unlikely((seq = READ_ONCE(vd->seq)) & 1)) {
131			if (IS_ENABLED(CONFIG_TIME_NS) &&
132			    vd->clock_mode == VDSO_CLOCKMODE_TIMENS)
133				return do_hres_timens(vd, clk, ts);
134			cpu_relax();
135		}
136		smp_rmb();
137
138		if (unlikely(!vdso_clocksource_ok(vd)))
139			return -1;
140
141		cycles = __arch_get_hw_counter(vd->clock_mode, vd);
142		if (unlikely(!vdso_cycles_ok(cycles)))
143			return -1;
144		ns = vdso_ts->nsec;
145		last = vd->cycle_last;
146		ns += vdso_calc_delta(cycles, last, vd->mask, vd->mult);
147		ns = vdso_shift_ns(ns, vd->shift);
148		sec = vdso_ts->sec;
149	} while (unlikely(vdso_read_retry(vd, seq)));
150
151	/*
152	 * Do this outside the loop: a race inside the loop could result
153	 * in __iter_div_u64_rem() being extremely slow.
154	 */
155	ts->tv_sec = sec + __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
156	ts->tv_nsec = ns;
157
158	return 0;
159}
160
161#ifdef CONFIG_TIME_NS
162static __always_inline int do_coarse_timens(const struct vdso_data *vdns, clockid_t clk,
163					    struct __kernel_timespec *ts)
164{
165	const struct vdso_data *vd = __arch_get_timens_vdso_data();
166	const struct vdso_timestamp *vdso_ts = &vd->basetime[clk];
167	const struct timens_offset *offs = &vdns->offset[clk];
168	u64 nsec;
169	s64 sec;
170	s32 seq;
171
172	do {
173		seq = vdso_read_begin(vd);
174		sec = vdso_ts->sec;
175		nsec = vdso_ts->nsec;
176	} while (unlikely(vdso_read_retry(vd, seq)));
177
178	/* Add the namespace offset */
179	sec += offs->sec;
180	nsec += offs->nsec;
181
182	/*
183	 * Do this outside the loop: a race inside the loop could result
184	 * in __iter_div_u64_rem() being extremely slow.
185	 */
186	ts->tv_sec = sec + __iter_div_u64_rem(nsec, NSEC_PER_SEC, &nsec);
187	ts->tv_nsec = nsec;
188	return 0;
189}
190#else
191static __always_inline int do_coarse_timens(const struct vdso_data *vdns, clockid_t clk,
192					    struct __kernel_timespec *ts)
193{
194	return -1;
195}
196#endif
197
198static __always_inline int do_coarse(const struct vdso_data *vd, clockid_t clk,
199				     struct __kernel_timespec *ts)
200{
201	const struct vdso_timestamp *vdso_ts = &vd->basetime[clk];
202	u32 seq;
203
204	do {
205		/*
206		 * Open coded to handle VDSO_CLOCK_TIMENS. See comment in
207		 * do_hres().
208		 */
209		while ((seq = READ_ONCE(vd->seq)) & 1) {
210			if (IS_ENABLED(CONFIG_TIME_NS) &&
211			    vd->clock_mode == VDSO_CLOCKMODE_TIMENS)
212				return do_coarse_timens(vd, clk, ts);
213			cpu_relax();
214		}
215		smp_rmb();
216
217		ts->tv_sec = vdso_ts->sec;
218		ts->tv_nsec = vdso_ts->nsec;
219	} while (unlikely(vdso_read_retry(vd, seq)));
220
221	return 0;
222}
223
224static __always_inline int
225__cvdso_clock_gettime_common(const struct vdso_data *vd, clockid_t clock,
226			     struct __kernel_timespec *ts)
227{
228	u32 msk;
229
230	/* Check for negative values or invalid clocks */
231	if (unlikely((u32) clock >= MAX_CLOCKS))
232		return -1;
233
234	/*
235	 * Convert the clockid to a bitmask and use it to check which
236	 * clocks are handled in the VDSO directly.
237	 */
238	msk = 1U << clock;
239	if (likely(msk & VDSO_HRES))
240		vd = &vd[CS_HRES_COARSE];
241	else if (msk & VDSO_COARSE)
242		return do_coarse(&vd[CS_HRES_COARSE], clock, ts);
243	else if (msk & VDSO_RAW)
244		vd = &vd[CS_RAW];
245	else
246		return -1;
247
248	return do_hres(vd, clock, ts);
249}
250
251static __maybe_unused int
252__cvdso_clock_gettime_data(const struct vdso_data *vd, clockid_t clock,
253			   struct __kernel_timespec *ts)
254{
255	int ret = __cvdso_clock_gettime_common(vd, clock, ts);
256
257	if (unlikely(ret))
258		return clock_gettime_fallback(clock, ts);
259	return 0;
260}
261
262static __maybe_unused int
263__cvdso_clock_gettime(clockid_t clock, struct __kernel_timespec *ts)
264{
265	return __cvdso_clock_gettime_data(__arch_get_vdso_data(), clock, ts);
266}
267
268#ifdef BUILD_VDSO32
269static __maybe_unused int
270__cvdso_clock_gettime32_data(const struct vdso_data *vd, clockid_t clock,
271			     struct old_timespec32 *res)
272{
273	struct __kernel_timespec ts;
274	int ret;
275
276	ret = __cvdso_clock_gettime_common(vd, clock, &ts);
277
278	if (unlikely(ret))
279		return clock_gettime32_fallback(clock, res);
280
281	/* For ret == 0 */
282	res->tv_sec = ts.tv_sec;
283	res->tv_nsec = ts.tv_nsec;
284
285	return ret;
286}
287
288static __maybe_unused int
289__cvdso_clock_gettime32(clockid_t clock, struct old_timespec32 *res)
290{
291	return __cvdso_clock_gettime32_data(__arch_get_vdso_data(), clock, res);
292}
293#endif /* BUILD_VDSO32 */
294
295static __maybe_unused int
296__cvdso_gettimeofday_data(const struct vdso_data *vd,
297			  struct __kernel_old_timeval *tv, struct timezone *tz)
298{
299
300	if (likely(tv != NULL)) {
301		struct __kernel_timespec ts;
302
303		if (do_hres(&vd[CS_HRES_COARSE], CLOCK_REALTIME, &ts))
304			return gettimeofday_fallback(tv, tz);
305
306		tv->tv_sec = ts.tv_sec;
307		tv->tv_usec = (u32)ts.tv_nsec / NSEC_PER_USEC;
308	}
309
310	if (unlikely(tz != NULL)) {
311		if (IS_ENABLED(CONFIG_TIME_NS) &&
312		    vd->clock_mode == VDSO_CLOCKMODE_TIMENS)
313			vd = __arch_get_timens_vdso_data();
314
315		tz->tz_minuteswest = vd[CS_HRES_COARSE].tz_minuteswest;
316		tz->tz_dsttime = vd[CS_HRES_COARSE].tz_dsttime;
317	}
318
319	return 0;
320}
321
322static __maybe_unused int
323__cvdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz)
324{
325	return __cvdso_gettimeofday_data(__arch_get_vdso_data(), tv, tz);
326}
327
328#ifdef VDSO_HAS_TIME
329static __maybe_unused __kernel_old_time_t
330__cvdso_time_data(const struct vdso_data *vd, __kernel_old_time_t *time)
331{
332	__kernel_old_time_t t;
333
334	if (IS_ENABLED(CONFIG_TIME_NS) &&
335	    vd->clock_mode == VDSO_CLOCKMODE_TIMENS)
336		vd = __arch_get_timens_vdso_data();
337
338	t = READ_ONCE(vd[CS_HRES_COARSE].basetime[CLOCK_REALTIME].sec);
339
340	if (time)
341		*time = t;
342
343	return t;
344}
345
346static __maybe_unused __kernel_old_time_t __cvdso_time(__kernel_old_time_t *time)
347{
348	return __cvdso_time_data(__arch_get_vdso_data(), time);
349}
350#endif /* VDSO_HAS_TIME */
351
352#ifdef VDSO_HAS_CLOCK_GETRES
353static __maybe_unused
354int __cvdso_clock_getres_common(const struct vdso_data *vd, clockid_t clock,
355				struct __kernel_timespec *res)
356{
357	u32 msk;
358	u64 ns;
359
360	/* Check for negative values or invalid clocks */
361	if (unlikely((u32) clock >= MAX_CLOCKS))
362		return -1;
363
364	if (IS_ENABLED(CONFIG_TIME_NS) &&
365	    vd->clock_mode == VDSO_CLOCKMODE_TIMENS)
366		vd = __arch_get_timens_vdso_data();
367
368	/*
369	 * Convert the clockid to a bitmask and use it to check which
370	 * clocks are handled in the VDSO directly.
371	 */
372	msk = 1U << clock;
373	if (msk & (VDSO_HRES | VDSO_RAW)) {
374		/*
375		 * Preserves the behaviour of posix_get_hrtimer_res().
376		 */
377		ns = READ_ONCE(vd[CS_HRES_COARSE].hrtimer_res);
378	} else if (msk & VDSO_COARSE) {
379		/*
380		 * Preserves the behaviour of posix_get_coarse_res().
381		 */
382		ns = LOW_RES_NSEC;
383	} else {
384		return -1;
385	}
386
387	if (likely(res)) {
388		res->tv_sec = 0;
389		res->tv_nsec = ns;
390	}
391	return 0;
392}
393
394static __maybe_unused
395int __cvdso_clock_getres_data(const struct vdso_data *vd, clockid_t clock,
396			      struct __kernel_timespec *res)
397{
398	int ret = __cvdso_clock_getres_common(vd, clock, res);
399
400	if (unlikely(ret))
401		return clock_getres_fallback(clock, res);
402	return 0;
403}
404
405static __maybe_unused
406int __cvdso_clock_getres(clockid_t clock, struct __kernel_timespec *res)
407{
408	return __cvdso_clock_getres_data(__arch_get_vdso_data(), clock, res);
409}
410
411#ifdef BUILD_VDSO32
412static __maybe_unused int
413__cvdso_clock_getres_time32_data(const struct vdso_data *vd, clockid_t clock,
414				 struct old_timespec32 *res)
415{
416	struct __kernel_timespec ts;
417	int ret;
418
419	ret = __cvdso_clock_getres_common(vd, clock, &ts);
420
421	if (unlikely(ret))
422		return clock_getres32_fallback(clock, res);
423
424	if (likely(res)) {
425		res->tv_sec = ts.tv_sec;
426		res->tv_nsec = ts.tv_nsec;
427	}
428	return ret;
429}
430
431static __maybe_unused int
432__cvdso_clock_getres_time32(clockid_t clock, struct old_timespec32 *res)
433{
434	return __cvdso_clock_getres_time32_data(__arch_get_vdso_data(),
435						clock, res);
436}
437#endif /* BUILD_VDSO32 */
438#endif /* VDSO_HAS_CLOCK_GETRES */
439