Home Home > GIT Browse
summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorThomas Gleixner <tglx@linutronix.de>2009-11-06 16:39:32 +0100
committerThomas Gleixner <tglx@linutronix.de>2009-11-06 16:39:32 +0100
commitf846ed7a8e32b6358ba968f8a0d9e428d9d36699 (patch)
treeeea97f4c7987d69660f9c5dbd8a12fe7d195418a
parent7b724e6c828448ea82ba809204a53307cc235b0f (diff)
parentd2ac742de047029bb44d0f1b7d4b01c35678a19a (diff)
Merge branch 'rt/head' into rt/2.6.31
-rw-r--r--Documentation/trace/ftrace.txt9
-rw-r--r--Documentation/trace/histograms.txt112
-rw-r--r--drivers/gpu/drm/r128/r128_cce.c18
-rw-r--r--drivers/gpu/drm/r128/r128_drv.h8
-rw-r--r--drivers/gpu/drm/r128/r128_state.c36
-rw-r--r--drivers/misc/Kconfig2
-rw-r--r--drivers/misc/hwlat_detector.c12
-rw-r--r--fs/pipe.c41
-rw-r--r--include/linux/sched.h3
-rw-r--r--include/trace/events/hist.h29
-rw-r--r--init/main.c13
-rw-r--r--kernel/hrtimer.c10
-rw-r--r--kernel/trace/Kconfig65
-rw-r--r--kernel/trace/Makefile1
-rw-r--r--kernel/trace/latency_hist.c460
-rw-r--r--net/sched/cls_api.c2
-rw-r--r--net/unix/af_unix.c2
-rw-r--r--security/keys/keyctl.c2
18 files changed, 589 insertions, 236 deletions
diff --git a/Documentation/trace/ftrace.txt b/Documentation/trace/ftrace.txt
index 355d0f1f8c50..a4fcad7b423c 100644
--- a/Documentation/trace/ftrace.txt
+++ b/Documentation/trace/ftrace.txt
@@ -111,9 +111,14 @@ of ftrace. Here is a list of some of the key files:
For example, the time interrupts are disabled.
This time is saved in this file. The max trace
will also be stored, and displayed by "trace".
- A new max trace will only be recorded if the
+ A new max trace will only be recorded, if the
latency is greater than the value in this
- file. (in microseconds)
+ file (in microseconds). Note that the max latency
+ recorded by the wakeup and the wakeup_rt tracer
+ do not necessarily reflect the worst-case latency
+ of the system, but may be erroneously high in
+ case two or more processes share the maximum
+ priority of the system.
buffer_size_kb:
diff --git a/Documentation/trace/histograms.txt b/Documentation/trace/histograms.txt
index 664505717219..2f179673710a 100644
--- a/Documentation/trace/histograms.txt
+++ b/Documentation/trace/histograms.txt
@@ -24,26 +24,43 @@ histograms of potential sources of latency, the kernel stores the time
stamp at the start of a critical section, determines the time elapsed
when the end of the section is reached, and increments the frequency
counter of that latency value - irrespective of whether any concurrently
-running process is affected by latency or not.
+running process is affected by this latency or not.
- Configuration items (in the Kernel hacking/Tracers submenu)
- CONFIG_INTERRUPT_OFF_LATENCY
- CONFIG_PREEMPT_OFF_LATENCY
+ CONFIG_INTERRUPT_OFF_HIST
+ CONFIG_PREEMPT_OFF_HIST
* Effective latencies
-
-Effective latencies are actually occuring during wakeup of a process. To
-determine effective latencies, the kernel stores the time stamp when a
-process is scheduled to be woken up, and determines the duration of the
-wakeup time shortly before control is passed over to this process. Note
-that the apparent latency in user space may be considerably longer,
-since
-i) interrupts may be disabled preventing the scheduler from initiating
-the wakeup mechanism, and
+There are two types of effective latencies, wakeup latencies and missed
+timer latencies
+
+* Wakeup latencies
+Wakeup latencies may occur during wakeup of a process. To determine
+wakeup latencies, the kernel stores the time stamp when a process is
+scheduled to be woken up, and determines the duration of the wakeup time
+shortly before control is passed over to this process. Note that the
+apparent latency in user space may be considerably longer, since
+i) interrupts may be disabled preventing the timer from waking up a process
+in time
ii) the process may be interrupted after control is passed over to it
but before user space execution takes place.
+If a particular wakeup latency is highest so far, details of the task
+that is suffering from this latency are stored as well (see below).
+- Configuration item (in the Kernel hacking/Tracers submenu)
+ CONFIG_WAKEUP_LATENCY_HIST
+
+* Missed timer latencies
+
+Missed timer latencies occur when a timer interrupt is serviced later
+than it should. This is mainly due to disabled interrupts. To determine
+the missed timer latency, the expected and the real execution time of a
+timer are compared. If the former precedes the latter, the difference is
+entered into the missed timer offsets histogram. If the timer is
+responsible to wakeup a sleeping process and the latency is highest so
+far among previous wakeup timers, details of the related task are stored
+as well (see below).
- Configuration item (in the Kernel hacking/Tracers submenu)
- CONFIG_WAKEUP_LATENCY
+ CONFIG_MISSED_TIMER_OFFSETS_HIST
* Usage
@@ -59,30 +76,36 @@ from shell command line level, or add
nodev /sys sysfs defaults 0 0
nodev /sys/kernel/debug debugfs defaults 0 0
-to the file /etc/fstab. All latency histogram related files are
+to the file /etc/fstab in order to implicitly mount the debug file
+system at every reboot. All latency histogram related files are
available in the directory /sys/kernel/debug/tracing/latency_hist. A
particular histogram type is enabled by writing non-zero to the related
variable in the /sys/kernel/debug/tracing/latency_hist/enable directory.
-Select "preemptirqsoff" for the histograms of potential sources of
-latencies and "wakeup" for histograms of effective latencies. The
-histogram data - one per CPU - are available in the files
+Select "preemptirqsoff" for histograms of potential sources of
+latencies, "wakeup" for histograms of wakeup latencies and
+"missed_timer_offsets" for histograms of missed timer offsets,
+respectively.
+The histogram data - one per CPU - are available in the files
/sys/kernel/debug/tracing/latency_hist/preemptoff/CPUx
/sys/kernel/debug/tracing/latency_hist/irqsoff/CPUx
/sys/kernel/debug/tracing/latency_hist/preemptirqsoff/CPUx
/sys/kernel/debug/tracing/latency_hist/wakeup/CPUx.
+/sys/kernel/debug/tracing/latency_hist/wakeup/sharedprio/CPUx.
+/sys/kernel/debug/tracing/latency_hist/missed_timer_offsets/CPUx.
The histograms are reset by writing non-zero to the file "reset" in a
particular latency directory. To reset all latency data, use
-#!/bin/sh
+#!/bin/bash
-HISTDIR=/sys/kernel/debug/tracing/latency_hist
+TRACINGDIR=/sys/kernel/debug/tracing
+HISTDIR=$TRACINGDIR/latency_hist
if test -d $HISTDIR
then
cd $HISTDIR
- for i in */reset
+ for i in `find . | grep /reset$`
do
echo 1 >$i
done
@@ -92,19 +115,19 @@ fi
* Data format
Latency data are stored with a resolution of one microsecond. The
-maximum latency is 10,240 microseconds. The data are only valid, if the
-overflow register is empty. Every output line contains the latency in
-microseconds in the first row and the number of samples in the second
-row. To display only lines with a positive latency count, use, for
-example,
+maximum latency is 10,240 microseconds. Every output line contains the
+latency in microseconds in the first row and the number of samples in
+the second row. To display only lines with a positive latency count,
+use, for example,
grep -v " 0$" /sys/kernel/debug/tracing/latency_hist/preemptoff/CPU0
-#Minimum latency: 0 microseconds.
-#Average latency: 0 microseconds.
-#Maximum latency: 25 microseconds.
+#Minimum latency: 0 microseconds
+#Average latency: 0 microseconds
+#Maximum latency: 25 microseconds
#Total samples: 3104770694
-#There are 0 samples greater or equal than 10240 microseconds
+#There are 0 samples lower than 0 microseconds.
+#There are 0 samples greater or equal than 10240 microseconds.
#usecs samples
0 2984486876
1 49843506
@@ -133,6 +156,23 @@ grep -v " 0$" /sys/kernel/debug/tracing/latency_hist/preemptoff/CPU0
25 1
+* Two types of wakeup latency histograms
+
+Two different algorithms are used to determine the wakeup latency of a
+process. One of them only considers processes that exclusively use the
+highest priority of the system, the other one records the wakeup latency
+of a process even if it shares the highest system latency with other
+processes. The former is used to determine the worst-case latency of a
+system; if higher than expected, the hardware and or system software
+(e.g. the Linux kernel) may need to be debugged and fixed. The latter
+reflects the priority design of a given system; if higher than expected,
+the system design may need to be re-evaluated - the hardware
+manufacturer or the kernel developers must not be blamed for such
+latencies. The exclusive-priority wakeup latency histograms are located
+in the "wakeup" subdirectory, the shared-priority histograms are located
+in the "wakeup/sharedprio" subdirectory.
+
+
* Wakeup latency of a selected process
To only collect wakeup latency data of a particular process, write the
@@ -143,14 +183,18 @@ PID of the requested process to
PIDs are not considered, if this variable is set to 0.
-* Details of the process with the highest wakeup latency so far
+* Details of processes with the highest wakeup or missed timer
+latency so far
-Selected data of the process that suffered from the highest wakeup
-latency that occurred in a particular CPU are available in the file
+Selected data of processes that suffered from the highest wakeup or
+missed timer latency that occurred on a particular CPU are available in
+the files
-/sys/kernel/debug/tracing/latency_hist/wakeup/max_latency-CPUx.
+/sys/kernel/debug/tracing/latency_hist/wakeup/max_latency-CPUx
+/sys/kernel/debug/tracing/latency_hist/wakeup/sharedprio/max_latency-CPUx
+/sys/kernel/debug/tracing/latency_hist/missed_timer_offsets/max_latency-CPUx
The format of the data is
<PID> <Priority> <Latency> <Command>
-These data are also reset when the wakeup histogram ist reset.
+These data are also reset when the related histograms are reset.
diff --git a/drivers/gpu/drm/r128/r128_cce.c b/drivers/gpu/drm/r128/r128_cce.c
index c75fd3564040..ebf9f63e5d49 100644
--- a/drivers/gpu/drm/r128/r128_cce.c
+++ b/drivers/gpu/drm/r128/r128_cce.c
@@ -353,6 +353,11 @@ static int r128_do_init_cce(struct drm_device * dev, drm_r128_init_t * init)
DRM_DEBUG("\n");
+ if (dev->dev_private) {
+ DRM_DEBUG("called when already initialized\n");
+ return -EINVAL;
+ }
+
dev_priv = kzalloc(sizeof(drm_r128_private_t), GFP_KERNEL);
if (dev_priv == NULL)
return -ENOMEM;
@@ -649,6 +654,8 @@ int r128_cce_start(struct drm_device *dev, void *data, struct drm_file *file_pri
LOCK_TEST_WITH_RETURN(dev, file_priv);
+ DEV_INIT_TEST_WITH_RETURN(dev_priv);
+
if (dev_priv->cce_running || dev_priv->cce_mode == R128_PM4_NONPM4) {
DRM_DEBUG("while CCE running\n");
return 0;
@@ -671,6 +678,8 @@ int r128_cce_stop(struct drm_device *dev, void *data, struct drm_file *file_priv
LOCK_TEST_WITH_RETURN(dev, file_priv);
+ DEV_INIT_TEST_WITH_RETURN(dev_priv);
+
/* Flush any pending CCE commands. This ensures any outstanding
* commands are exectuted by the engine before we turn it off.
*/
@@ -708,10 +717,7 @@ int r128_cce_reset(struct drm_device *dev, void *data, struct drm_file *file_pri
LOCK_TEST_WITH_RETURN(dev, file_priv);
- if (!dev_priv) {
- DRM_DEBUG("called before init done\n");
- return -EINVAL;
- }
+ DEV_INIT_TEST_WITH_RETURN(dev_priv);
r128_do_cce_reset(dev_priv);
@@ -728,6 +734,8 @@ int r128_cce_idle(struct drm_device *dev, void *data, struct drm_file *file_priv
LOCK_TEST_WITH_RETURN(dev, file_priv);
+ DEV_INIT_TEST_WITH_RETURN(dev_priv);
+
if (dev_priv->cce_running) {
r128_do_cce_flush(dev_priv);
}
@@ -741,6 +749,8 @@ int r128_engine_reset(struct drm_device *dev, void *data, struct drm_file *file_
LOCK_TEST_WITH_RETURN(dev, file_priv);
+ DEV_INIT_TEST_WITH_RETURN(dev->dev_private);
+
return r128_do_engine_reset(dev);
}
diff --git a/drivers/gpu/drm/r128/r128_drv.h b/drivers/gpu/drm/r128/r128_drv.h
index 797a26c42dab..3c60829d82e9 100644
--- a/drivers/gpu/drm/r128/r128_drv.h
+++ b/drivers/gpu/drm/r128/r128_drv.h
@@ -422,6 +422,14 @@ static __inline__ void r128_update_ring_snapshot(drm_r128_private_t * dev_priv)
* Misc helper macros
*/
+#define DEV_INIT_TEST_WITH_RETURN(_dev_priv) \
+do { \
+ if (!_dev_priv) { \
+ DRM_ERROR("called with no initialization\n"); \
+ return -EINVAL; \
+ } \
+} while (0)
+
#define RING_SPACE_TEST_WITH_RETURN( dev_priv ) \
do { \
drm_r128_ring_buffer_t *ring = &dev_priv->ring; int i; \
diff --git a/drivers/gpu/drm/r128/r128_state.c b/drivers/gpu/drm/r128/r128_state.c
index 026a48c95c8f..af2665cf4718 100644
--- a/drivers/gpu/drm/r128/r128_state.c
+++ b/drivers/gpu/drm/r128/r128_state.c
@@ -1244,14 +1244,18 @@ static void r128_cce_dispatch_stipple(struct drm_device * dev, u32 * stipple)
static int r128_cce_clear(struct drm_device *dev, void *data, struct drm_file *file_priv)
{
drm_r128_private_t *dev_priv = dev->dev_private;
- drm_r128_sarea_t *sarea_priv = dev_priv->sarea_priv;
+ drm_r128_sarea_t *sarea_priv;
drm_r128_clear_t *clear = data;
DRM_DEBUG("\n");
LOCK_TEST_WITH_RETURN(dev, file_priv);
+ DEV_INIT_TEST_WITH_RETURN(dev_priv);
+
RING_SPACE_TEST_WITH_RETURN(dev_priv);
+ sarea_priv = dev_priv->sarea_priv;
+
if (sarea_priv->nbox > R128_NR_SAREA_CLIPRECTS)
sarea_priv->nbox = R128_NR_SAREA_CLIPRECTS;
@@ -1312,6 +1316,8 @@ static int r128_cce_flip(struct drm_device *dev, void *data, struct drm_file *fi
LOCK_TEST_WITH_RETURN(dev, file_priv);
+ DEV_INIT_TEST_WITH_RETURN(dev_priv);
+
RING_SPACE_TEST_WITH_RETURN(dev_priv);
if (!dev_priv->page_flipping)
@@ -1331,6 +1337,8 @@ static int r128_cce_swap(struct drm_device *dev, void *data, struct drm_file *fi
LOCK_TEST_WITH_RETURN(dev, file_priv);
+ DEV_INIT_TEST_WITH_RETURN(dev_priv);
+
RING_SPACE_TEST_WITH_RETURN(dev_priv);
if (sarea_priv->nbox > R128_NR_SAREA_CLIPRECTS)
@@ -1354,10 +1362,7 @@ static int r128_cce_vertex(struct drm_device *dev, void *data, struct drm_file *
LOCK_TEST_WITH_RETURN(dev, file_priv);
- if (!dev_priv) {
- DRM_ERROR("called with no initialization\n");
- return -EINVAL;
- }
+ DEV_INIT_TEST_WITH_RETURN(dev_priv);
DRM_DEBUG("pid=%d index=%d count=%d discard=%d\n",
DRM_CURRENTPID, vertex->idx, vertex->count, vertex->discard);
@@ -1410,10 +1415,7 @@ static int r128_cce_indices(struct drm_device *dev, void *data, struct drm_file
LOCK_TEST_WITH_RETURN(dev, file_priv);
- if (!dev_priv) {
- DRM_ERROR("called with no initialization\n");
- return -EINVAL;
- }
+ DEV_INIT_TEST_WITH_RETURN(dev_priv);
DRM_DEBUG("pid=%d buf=%d s=%d e=%d d=%d\n", DRM_CURRENTPID,
elts->idx, elts->start, elts->end, elts->discard);
@@ -1476,6 +1478,8 @@ static int r128_cce_blit(struct drm_device *dev, void *data, struct drm_file *fi
LOCK_TEST_WITH_RETURN(dev, file_priv);
+ DEV_INIT_TEST_WITH_RETURN(dev_priv);
+
DRM_DEBUG("pid=%d index=%d\n", DRM_CURRENTPID, blit->idx);
if (blit->idx < 0 || blit->idx >= dma->buf_count) {
@@ -1501,6 +1505,8 @@ static int r128_cce_depth(struct drm_device *dev, void *data, struct drm_file *f
LOCK_TEST_WITH_RETURN(dev, file_priv);
+ DEV_INIT_TEST_WITH_RETURN(dev_priv);
+
RING_SPACE_TEST_WITH_RETURN(dev_priv);
ret = -EINVAL;
@@ -1531,6 +1537,8 @@ static int r128_cce_stipple(struct drm_device *dev, void *data, struct drm_file
LOCK_TEST_WITH_RETURN(dev, file_priv);
+ DEV_INIT_TEST_WITH_RETURN(dev_priv);
+
if (DRM_COPY_FROM_USER(&mask, stipple->mask, 32 * sizeof(u32)))
return -EFAULT;
@@ -1555,10 +1563,7 @@ static int r128_cce_indirect(struct drm_device *dev, void *data, struct drm_file
LOCK_TEST_WITH_RETURN(dev, file_priv);
- if (!dev_priv) {
- DRM_ERROR("called with no initialization\n");
- return -EINVAL;
- }
+ DEV_INIT_TEST_WITH_RETURN(dev_priv);
DRM_DEBUG("idx=%d s=%d e=%d d=%d\n",
indirect->idx, indirect->start, indirect->end,
@@ -1620,10 +1625,7 @@ static int r128_getparam(struct drm_device *dev, void *data, struct drm_file *fi
drm_r128_getparam_t *param = data;
int value;
- if (!dev_priv) {
- DRM_ERROR("called with no initialization\n");
- return -EINVAL;
- }
+ DEV_INIT_TEST_WITH_RETURN(dev_priv);
DRM_DEBUG("pid=%d\n", DRM_CURRENTPID);
diff --git a/drivers/misc/Kconfig b/drivers/misc/Kconfig
index f51ba7b08f9b..d16b79fcc8fd 100644
--- a/drivers/misc/Kconfig
+++ b/drivers/misc/Kconfig
@@ -79,7 +79,7 @@ config IBM_ASM
config HWLAT_DETECTOR
tristate "Testing module to detect hardware-induced latencies"
depends on DEBUG_FS
- select RING_BUFFER
+ depends on RING_BUFFER
default m
---help---
A simple hardware latency detector. Use this module to detect
diff --git a/drivers/misc/hwlat_detector.c b/drivers/misc/hwlat_detector.c
index be6553f2ddfc..d9549e9b58f4 100644
--- a/drivers/misc/hwlat_detector.c
+++ b/drivers/misc/hwlat_detector.c
@@ -607,7 +607,11 @@ static ssize_t debug_enable_fwrite(struct file *filp,
if (!enabled)
goto unlock;
enabled = 0;
- stop_kthread();
+ err = stop_kthread();
+ if (err) {
+ printk(KERN_ERR BANNER "cannot stop kthread\n");
+ return -EFAULT;
+ }
wake_up(&data.wq); /* reader(s) should return */
}
unlock:
@@ -1194,9 +1198,13 @@ out:
*/
static void detector_exit(void)
{
+ int err;
+
if (enabled) {
enabled = 0;
- stop_kthread();
+ err = stop_kthread();
+ if (err)
+ printk(KERN_ERR BANNER "cannot stop kthread\n");
}
free_debugfs();
diff --git a/fs/pipe.c b/fs/pipe.c
index 7f30ed2a148e..c085c8f87050 100644
--- a/fs/pipe.c
+++ b/fs/pipe.c
@@ -789,36 +789,55 @@ pipe_rdwr_release(struct inode *inode, struct file *filp)
static int
pipe_read_open(struct inode *inode, struct file *filp)
{
- /* We could have perhaps used atomic_t, but this and friends
- below are the only places. So it doesn't seem worthwhile. */
+ int ret = -ENOENT;
+
mutex_lock(&inode->i_mutex);
- inode->i_pipe->readers++;
+
+ if (inode->i_pipe) {
+ ret = 0;
+ inode->i_pipe->readers++;
+ }
+
mutex_unlock(&inode->i_mutex);
- return 0;
+ return ret;
}
static int
pipe_write_open(struct inode *inode, struct file *filp)
{
+ int ret = -ENOENT;
+
mutex_lock(&inode->i_mutex);
- inode->i_pipe->writers++;
+
+ if (inode->i_pipe) {
+ ret = 0;
+ inode->i_pipe->writers++;
+ }
+
mutex_unlock(&inode->i_mutex);
- return 0;
+ return ret;
}
static int
pipe_rdwr_open(struct inode *inode, struct file *filp)
{
+ int ret = -ENOENT;
+
mutex_lock(&inode->i_mutex);
- if (filp->f_mode & FMODE_READ)
- inode->i_pipe->readers++;
- if (filp->f_mode & FMODE_WRITE)
- inode->i_pipe->writers++;
+
+ if (inode->i_pipe) {
+ ret = 0;
+ if (filp->f_mode & FMODE_READ)
+ inode->i_pipe->readers++;
+ if (filp->f_mode & FMODE_WRITE)
+ inode->i_pipe->writers++;
+ }
+
mutex_unlock(&inode->i_mutex);
- return 0;
+ return ret;
}
/*
diff --git a/include/linux/sched.h b/include/linux/sched.h
index c231a2467a83..676126d56894 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -1548,6 +1548,9 @@ struct task_struct {
unsigned long trace;
/* bitmask of trace recursion */
unsigned long trace_recursion;
+#ifdef CONFIG_WAKEUP_LATENCY_HIST
+ u64 preempt_timestamp_hist;
+#endif
#endif /* CONFIG_TRACING */
#ifdef CONFIG_PREEMPT_RT
/*
diff --git a/include/trace/events/hist.h b/include/trace/events/hist.h
index 73b0454eb90f..2dc4c70760e1 100644
--- a/include/trace/events/hist.h
+++ b/include/trace/events/hist.h
@@ -17,8 +17,8 @@ TRACE_EVENT(preemptirqsoff_hist,
TP_ARGS(reason, starthist),
TP_STRUCT__entry(
- __field( int, reason )
- __field( int, starthist )
+ __field(int, reason )
+ __field(int, starthist )
),
TP_fast_assign(
@@ -31,6 +31,31 @@ TRACE_EVENT(preemptirqsoff_hist,
);
#endif
+#ifndef CONFIG_MISSED_TIMER_OFFSETS_HIST
+#define trace_hrtimer_interrupt(a,b,c)
+#else
+TRACE_EVENT(hrtimer_interrupt,
+
+ TP_PROTO(int cpu, long long offset, struct task_struct *task),
+
+ TP_ARGS(cpu, offset, task),
+
+ TP_STRUCT__entry(
+ __array(char, comm, TASK_COMM_LEN)
+ __field(int, cpu )
+ __field(long long, offset )
+ ),
+
+ TP_fast_assign(
+ strncpy(__entry->comm, task != NULL ? task->comm : "", TASK_COMM_LEN);
+ __entry->cpu = cpu;
+ __entry->offset = offset;
+ ),
+
+ TP_printk("cpu=%d offset=%lld thread=%s", __entry->cpu, __entry->offset, __entry->comm)
+);
+#endif
+
#endif /* _TRACE_HIST_H */
/* This part must be outside protection */
diff --git a/init/main.c b/init/main.c
index bdbdaab0469e..ff438b198fe3 100644
--- a/init/main.c
+++ b/init/main.c
@@ -936,7 +936,15 @@ static int __init kernel_init(void * unused)
WARN_ON(irqs_disabled());
#endif
-#define DEBUG_COUNT (defined(CONFIG_DEBUG_RT_MUTEXES) + defined(CONFIG_IRQSOFF_TRACER) + defined(CONFIG_PREEMPT_TRACER) + defined(CONFIG_STACK_TRACER) + defined(CONFIG_INTERRUPT_OFF_HIST) + defined(CONFIG_PREEMPT_OFF_HIST) + defined(CONFIG_WAKEUP_LATENCY_HIST) + defined(CONFIG_DEBUG_SLAB) + defined(CONFIG_DEBUG_PAGEALLOC) + defined(CONFIG_LOCKDEP) + (defined(CONFIG_FTRACE) - defined(CONFIG_FTRACE_MCOUNT_RECORD)))
+#define DEBUG_COUNT (defined(CONFIG_DEBUG_RT_MUTEXES) + \
+ defined(CONFIG_IRQSOFF_TRACER) + defined(CONFIG_PREEMPT_TRACER) + \
+ defined(CONFIG_STACK_TRACER) + defined(CONFIG_INTERRUPT_OFF_HIST) + \
+ defined(CONFIG_PREEMPT_OFF_HIST) + \
+ defined(CONFIG_WAKEUP_LATENCY_HIST) + \
+ defined(CONFIG_MISSED_TIMER_OFFSETS_HIST) + \
+ defined(CONFIG_DEBUG_SLAB) + defined(CONFIG_DEBUG_PAGEALLOC) + \
+ defined(CONFIG_LOCKDEP) + \
+ (defined(CONFIG_FTRACE) - defined(CONFIG_FTRACE_MCOUNT_RECORD)))
#if DEBUG_COUNT > 0
printk(KERN_ERR "*****************************************************************************\n");
@@ -968,6 +976,9 @@ static int __init kernel_init(void * unused)
#ifdef CONFIG_WAKEUP_LATENCY_HIST
printk(KERN_ERR "* CONFIG_WAKEUP_LATENCY_HIST *\n");
#endif
+#ifdef CONFIG_MISSED_TIMER_OFFSETS_HIST
+ printk(KERN_ERR "* CONFIG_MISSED_TIMER_OFFSETS_HIST *\n");
+#endif
#ifdef CONFIG_DEBUG_SLAB
printk(KERN_ERR "* CONFIG_DEBUG_SLAB *\n");
#endif
diff --git a/kernel/hrtimer.c b/kernel/hrtimer.c
index 875327a43ad4..1840ba8c8283 100644
--- a/kernel/hrtimer.c
+++ b/kernel/hrtimer.c
@@ -48,6 +48,8 @@
#include <asm/uaccess.h>
+#include <trace/events/hist.h>
+
/*
* The timer bases:
*
@@ -1349,6 +1351,7 @@ static inline int hrtimer_rt_defer(struct hrtimer *timer) { return 0; }
#ifdef CONFIG_HIGH_RES_TIMERS
static int force_clock_reprogram;
+static enum hrtimer_restart hrtimer_wakeup(struct hrtimer *timer);
/*
* After 5 iteration's attempts, we consider that hrtimer_interrupt()
@@ -1419,6 +1422,13 @@ void hrtimer_interrupt(struct clock_event_device *dev)
timer = rb_entry(node, struct hrtimer, node);
+ trace_hrtimer_interrupt(raw_smp_processor_id(),
+ ktime_to_ns(ktime_sub(
+ hrtimer_get_expires(timer), basenow)),
+ timer->function == hrtimer_wakeup ?
+ container_of(timer, struct hrtimer_sleeper,
+ timer)->task : NULL);
+
/*
* The immediate goal for using the softexpires is
* minimizing wakeups, not running timers at the
diff --git a/kernel/trace/Kconfig b/kernel/trace/Kconfig
index fea2f14a5d62..471c5f19f573 100644
--- a/kernel/trace/Kconfig
+++ b/kernel/trace/Kconfig
@@ -143,7 +143,6 @@ config FUNCTION_GRAPH_TRACER
the return value. This is done by setting the current return
address on the current task structure into a stack of calls.
-
config IRQSOFF_TRACER
bool "Interrupts-off Latency Tracer"
default n
@@ -171,16 +170,19 @@ config INTERRUPT_OFF_HIST
bool "Interrupts-off Latency Histogram"
depends on IRQSOFF_TRACER
help
- This option generates a continuously updated histogram (one per cpu)
+ This option generates continuously updated histograms (one per cpu)
of the duration of time periods with interrupts disabled. The
- histogram is disabled by default. To enable it, write a non-zero
- number to the related file in
+ histograms are disabled by default. To enable them, write a non-zero
+ number to
/sys/kernel/debug/tracing/latency_hist/enable/preemptirqsoff
- If PREEMPT_OFF_HIST is also selected, an additional histogram (one
- per cpu) is generated that accumulates the duration of time periods
- when both interrupts and preemption are disabled.
+ If PREEMPT_OFF_HIST is also selected, additional histograms (one
+ per cpu) are generated that accumulate the duration of time periods
+ when both interrupts and preemption are disabled. The histogram data
+ will be located in the debug file system at
+
+ /sys/kernel/debug/tracing/latency_hist/irqsoff
config PREEMPT_TRACER
bool "Preemption-off Latency Tracer"
@@ -208,16 +210,19 @@ config PREEMPT_OFF_HIST
bool "Preemption-off Latency Histogram"
depends on PREEMPT_TRACER
help
- This option generates a continuously updated histogram (one per cpu)
+ This option generates continuously updated histograms (one per cpu)
of the duration of time periods with preemption disabled. The
- histogram is disabled by default. To enable it, write a non-zero
+ histograms are disabled by default. To enable them, write a non-zero
number to
/sys/kernel/debug/tracing/latency_hist/enable/preemptirqsoff
- If INTERRUPT_OFF_HIST is also selected, an additional histogram (one
- per cpu) is generated that accumulates the duration of time periods
- when both interrupts and preemption are disabled.
+ If INTERRUPT_OFF_HIST is also selected, additional histograms (one
+ per cpu) are generated that accumulate the duration of time periods
+ when both interrupts and preemption are disabled. The histogram data
+ will be located in the debug file system at
+
+ /sys/kernel/debug/tracing/latency_hist/preemptoff
config SCHED_TRACER
bool "Scheduling Latency Tracer"
@@ -232,12 +237,42 @@ config WAKEUP_LATENCY_HIST
bool "Scheduling Latency Histogram"
depends on SCHED_TRACER
help
- This option generates a continuously updated histogram (one per cpu)
- of the scheduling latency of the highest priority task. The histogram
- is disabled by default. To enable it, write a non-zero number to
+ This option generates continuously updated histograms (one per cpu)
+ of the scheduling latency of the highest priority task.
+ The histograms are disabled by default. To enable them, write a
+ non-zero number to
/sys/kernel/debug/tracing/latency_hist/enable/wakeup
+ Two different algorithms are used, one to determine the latency of
+ processes that exclusively use the highest priority of the system and
+ another one to determine the latency of processes that share the
+ highest system priority with other processes. The former is used to
+ improve hardware and system software, the latter to optimize the
+ priority design of a given system. The histogram data will be
+ located in the debug file system at
+
+ /sys/kernel/debug/tracing/latency_hist/wakeup
+
+ and
+
+ /sys/kernel/debug/tracing/latency_hist/wakeup/sharedprio
+
+config MISSED_TIMER_OFFSETS_HIST
+ depends on GENERIC_TIME
+ select GENERIC_TRACER
+ bool "Missed timer offsets histogram"
+ help
+ Generate a histogram of missed timer offsets in microseconds. The
+ histograms are disabled by default. To enable them, write a non-zero
+ number to
+
+ /sys/kernel/debug/tracing/latency_hist/enable/missed_timer_offsets
+
+ The histogram data will be located in the debug file system at
+
+ /sys/kernel/debug/tracing/latency_hist/missed_timer_offsets
+
config SYSPROF_TRACER
bool "Sysprof Tracer"
depends on X86
diff --git a/kernel/trace/Makefile b/kernel/trace/Makefile
index c71519bb516e..8136881c59ac 100644
--- a/kernel/trace/Makefile
+++ b/kernel/trace/Makefile
@@ -38,6 +38,7 @@ obj-$(CONFIG_SCHED_TRACER) += trace_sched_wakeup.o
obj-$(CONFIG_INTERRUPT_OFF_HIST) += latency_hist.o
obj-$(CONFIG_PREEMPT_OFF_HIST) += latency_hist.o
obj-$(CONFIG_WAKEUP_LATENCY_HIST) += latency_hist.o
+obj-$(CONFIG_MISSED_TIMER_OFFSETS_HIST) += latency_hist.o
obj-$(CONFIG_NOP_TRACER) += trace_nop.o
obj-$(CONFIG_STACK_TRACER) += trace_stack.o
obj-$(CONFIG_MMIOTRACE) += trace_mmiotrace.o
diff --git a/kernel/trace/latency_hist.c b/kernel/trace/latency_hist.c
index c83463e9b37e..8705501ed408 100644
--- a/kernel/trace/latency_hist.c
+++ b/kernel/trace/latency_hist.c
@@ -34,6 +34,8 @@ enum {
PREEMPTOFF_LATENCY,
PREEMPTIRQSOFF_LATENCY,
WAKEUP_LATENCY,
+ WAKEUP_LATENCY_SHAREDPRIO,
+ MISSED_TIMER_OFFSETS,
MAX_LATENCY_TYPE,
};
@@ -41,9 +43,11 @@ enum {
struct hist_data {
atomic_t hist_mode; /* 0 log, 1 don't log */
+ long offset; /* set it to MAX_ENTRY_NUM/2 for a bipolar scale */
unsigned long min_lat;
unsigned long max_lat;
- unsigned long long beyond_hist_bound_samples;
+ unsigned long long below_hist_bound_samples;
+ unsigned long long above_hist_bound_samples;
unsigned long long accumulate_lat;
unsigned long long total_samples;
unsigned long long hist_array[MAX_ENTRY_NUM];
@@ -59,16 +63,22 @@ static char *latency_hist_dir_root = "latency_hist";
#ifdef CONFIG_INTERRUPT_OFF_HIST
static DEFINE_PER_CPU(struct hist_data, irqsoff_hist);
static char *irqsoff_hist_dir = "irqsoff";
+static DEFINE_PER_CPU(cycles_t, hist_irqsoff_start);
+static DEFINE_PER_CPU(int, hist_irqsoff_counting);
#endif
#ifdef CONFIG_PREEMPT_OFF_HIST
static DEFINE_PER_CPU(struct hist_data, preemptoff_hist);
static char *preemptoff_hist_dir = "preemptoff";
+static DEFINE_PER_CPU(cycles_t, hist_preemptoff_start);
+static DEFINE_PER_CPU(int, hist_preemptoff_counting);
#endif
#if defined(CONFIG_PREEMPT_OFF_HIST) && defined(CONFIG_INTERRUPT_OFF_HIST)
static DEFINE_PER_CPU(struct hist_data, preemptirqsoff_hist);
static char *preemptirqsoff_hist_dir = "preemptirqsoff";
+static DEFINE_PER_CPU(cycles_t, hist_preemptirqsoff_start);
+static DEFINE_PER_CPU(int, hist_preemptirqsoff_counting);
#endif
#if defined(CONFIG_PREEMPT_OFF_HIST) || defined(CONFIG_INTERRUPT_OFF_HIST)
@@ -79,9 +89,21 @@ static struct enable_data preemptirqsoff_enabled_data = {
};
#endif
+#if defined(CONFIG_WAKEUP_LATENCY_HIST) || \
+ defined(CONFIG_MISSED_TIMER_OFFSETS_HIST)
+struct maxlatproc_data {
+ char comm[FIELD_SIZEOF(struct task_struct, comm)];
+ unsigned int pid;
+ unsigned int prio;
+ unsigned long latency;
+};
+#endif
+
#ifdef CONFIG_WAKEUP_LATENCY_HIST
static DEFINE_PER_CPU(struct hist_data, wakeup_latency_hist);
+static DEFINE_PER_CPU(struct hist_data, wakeup_latency_hist_sharedprio);
static char *wakeup_latency_hist_dir = "wakeup";
+static char *wakeup_latency_hist_dir_sharedprio = "sharedprio";
static notrace void probe_wakeup_latency_hist_start(struct rq *rq,
struct task_struct *p, int success);
static notrace void probe_wakeup_latency_hist_stop(struct rq *rq,
@@ -90,23 +112,34 @@ static struct enable_data wakeup_latency_enabled_data = {
.latency_type = WAKEUP_LATENCY,
.enabled = 0,
};
-static struct task_struct *ts;
-struct maxlatproc_data {
- char comm[sizeof(ts->comm)];
- unsigned int pid;
- unsigned int prio;
- unsigned long latency;
-};
static DEFINE_PER_CPU(struct maxlatproc_data, wakeup_maxlatproc);
-static unsigned wakeup_prio = (unsigned)-1;
+static DEFINE_PER_CPU(struct maxlatproc_data, wakeup_maxlatproc_sharedprio);
static struct task_struct *wakeup_task;
+static int wakeup_sharedprio;
static int wakeup_pid;
#endif
+#ifdef CONFIG_MISSED_TIMER_OFFSETS_HIST
+static DEFINE_PER_CPU(struct hist_data, missed_timer_offsets);
+static char *missed_timer_offsets_dir = "missed_timer_offsets";
+static notrace void probe_hrtimer_interrupt(int cpu,
+ long long offset, struct task_struct *task);
+static struct enable_data missed_timer_offsets_enabled_data = {
+ .latency_type = MISSED_TIMER_OFFSETS,
+ .enabled = 0,
+};
+static DEFINE_PER_CPU(struct maxlatproc_data,
+ missed_timer_offsets_maxlatproc);
+#endif
+
void notrace latency_hist(int latency_type, int cpu, unsigned long latency,
struct task_struct *p)
{
struct hist_data *my_hist;
+#if defined(CONFIG_WAKEUP_LATENCY_HIST) || \
+ defined(CONFIG_MISSED_TIMER_OFFSETS_HIST)
+ struct maxlatproc_data *mp = NULL;
+#endif
if (cpu < 0 || cpu >= NR_CPUS || latency_type < 0 ||
latency_type >= MAX_LATENCY_TYPE)
@@ -118,22 +151,30 @@ void notrace latency_hist(int latency_type, int cpu, unsigned long latency,
my_hist = &per_cpu(irqsoff_hist, cpu);
break;
#endif
-
#ifdef CONFIG_PREEMPT_OFF_HIST
case PREEMPTOFF_LATENCY:
my_hist = &per_cpu(preemptoff_hist, cpu);
break;
#endif
-
#if defined(CONFIG_PREEMPT_OFF_HIST) && defined(CONFIG_INTERRUPT_OFF_HIST)
case PREEMPTIRQSOFF_LATENCY:
my_hist = &per_cpu(preemptirqsoff_hist, cpu);
break;
#endif
-
#ifdef CONFIG_WAKEUP_LATENCY_HIST
case WAKEUP_LATENCY:
my_hist = &per_cpu(wakeup_latency_hist, cpu);
+ mp = &per_cpu(wakeup_maxlatproc, cpu);
+ break;
+ case WAKEUP_LATENCY_SHAREDPRIO:
+ my_hist = &per_cpu(wakeup_latency_hist_sharedprio, cpu);
+ mp = &per_cpu(wakeup_maxlatproc_sharedprio, cpu);
+ break;
+#endif
+#ifdef CONFIG_MISSED_TIMER_OFFSETS_HIST
+ case MISSED_TIMER_OFFSETS:
+ my_hist = &per_cpu(missed_timer_offsets, cpu);
+ mp = &per_cpu(missed_timer_offsets_maxlatproc, cpu);
break;
#endif
default:
@@ -143,18 +184,23 @@ void notrace latency_hist(int latency_type, int cpu, unsigned long latency,
if (atomic_read(&my_hist->hist_mode) == 0)
return;
- if (latency >= MAX_ENTRY_NUM)
- my_hist->beyond_hist_bound_samples++;
- else
+ latency += my_hist->offset;
+
+ if (latency < 0 || latency >= MAX_ENTRY_NUM) {
+ if (latency < 0)
+ my_hist->below_hist_bound_samples++;
+ else
+ my_hist->above_hist_bound_samples++;
+ } else
my_hist->hist_array[latency]++;
if (latency < my_hist->min_lat)
my_hist->min_lat = latency;
else if (latency > my_hist->max_lat) {
-#ifdef CONFIG_WAKEUP_LATENCY_HIST
- if (latency_type == WAKEUP_LATENCY) {
- struct maxlatproc_data *mp =
- &per_cpu(wakeup_maxlatproc, cpu);
+#if defined (CONFIG_WAKEUP_LATENCY_HIST) || \
+ defined(CONFIG_MISSED_TIMER_OFFSETS_HIST)
+ if (latency_type == WAKEUP_LATENCY ||
+ latency_type == WAKEUP_LATENCY_SHAREDPRIO) {
strncpy(mp->comm, p->comm, sizeof(mp->comm));
mp->pid = task_pid_nr(p);
mp->prio = p->prio;
@@ -163,7 +209,15 @@ void notrace latency_hist(int latency_type, int cpu, unsigned long latency,
#endif
my_hist->max_lat = latency;
}
-
+#ifdef CONFIG_MISSED_TIMER_OFFSETS_HIST
+ if (latency_type == MISSED_TIMER_OFFSETS && p != NULL &&
+ latency > mp->latency) {
+ strncpy(mp->comm, p->comm, sizeof(mp->comm));
+ mp->pid = task_pid_nr(p);
+ mp->prio = p->prio;
+ mp->latency = latency;
+ }
+#endif
my_hist->total_samples++;
my_hist->accumulate_lat += latency;
return;
@@ -179,30 +233,42 @@ static void *l_start(struct seq_file *m, loff_t *pos)
return NULL;
if (index == 0) {
- char avgstr[32];
+ char minstr[32], avgstr[32], maxstr[32];
atomic_dec(&my_hist->hist_mode);
+
if (likely(my_hist->total_samples)) {
unsigned long avg = (unsigned long)
div64_u64(my_hist->accumulate_lat,
my_hist->total_samples);
- sprintf(avgstr, "%lu", avg);
- } else
- strcpy(avgstr, "<undef>");
+ snprintf(minstr, sizeof(minstr), "%ld",
+ (long) my_hist->min_lat - my_hist->offset);
+ snprintf(avgstr, sizeof(avgstr), "%ld",
+ (long) avg - my_hist->offset);
+ snprintf(maxstr, sizeof(minstr), "%ld",
+ (long) my_hist->max_lat - my_hist->offset);
+ } else {
+ strcpy(minstr, "<undef>");
+ strcpy(avgstr, minstr);
+ strcpy(maxstr, minstr);
+ }
- seq_printf(m, "#Minimum latency: %lu microseconds.\n"
- "#Average latency: %s microseconds.\n"
- "#Maximum latency: %lu microseconds.\n"
+ seq_printf(m, "#Minimum latency: %s microseconds\n"
+ "#Average latency: %s microseconds\n"
+ "#Maximum latency: %s microseconds\n"
"#Total samples: %llu\n"
+ "#There are %llu samples lower than %ld"
+ " microseconds.\n"
"#There are %llu samples greater or equal"
- " than %d microseconds\n"
- "#usecs\t%16s\n"
- , my_hist->min_lat
- , avgstr
- , my_hist->max_lat
- , my_hist->total_samples
- , my_hist->beyond_hist_bound_samples
- , MAX_ENTRY_NUM, "samples");
+ " than %ld microseconds.\n"
+ "#usecs\t%16s\n",
+ minstr, avgstr, maxstr,
+ my_hist->total_samples,
+ my_hist->below_hist_bound_samples,
+ -my_hist->offset,
+ my_hist->above_hist_bound_samples,
+ MAX_ENTRY_NUM - my_hist->offset,
+ "samples");
}
if (index >= MAX_ENTRY_NUM)
return NULL;
@@ -233,8 +299,15 @@ static int l_show(struct seq_file *m, void *p)
{
int index = *(loff_t *) p;
struct hist_data *my_hist = m->private;
+ char *fmt;
- seq_printf(m, "%5d\t%16llu\n", index, my_hist->hist_array[index]);
+ if (my_hist->offset)
+ fmt = "%6d\t%16llu\n";
+ else
+ fmt = "%5d\t%16llu\n";
+
+ seq_printf(m, fmt, index - my_hist->offset,
+ my_hist->hist_array[index]);
return 0;
}
@@ -269,7 +342,8 @@ static void hist_reset(struct hist_data *hist)
atomic_dec(&hist->hist_mode);
memset(hist->hist_array, 0, sizeof(hist->hist_array));
- hist->beyond_hist_bound_samples = 0ULL;
+ hist->below_hist_bound_samples = 0ULL;
+ hist->above_hist_bound_samples = 0ULL;
hist->min_lat = 0xFFFFFFFFUL;
hist->max_lat = 0UL;
hist->total_samples = 0ULL;
@@ -284,10 +358,13 @@ latency_hist_reset(struct file *file, const char __user *a,
{
int cpu;
struct hist_data *hist;
+#if defined(CONFIG_WAKEUP_LATENCY_HIST) || \
+ defined(CONFIG_MISSED_TIMER_OFFSETS_HIST)
+ struct maxlatproc_data *mp = NULL;
+#endif
int latency_type = (int) file->private_data;
switch (latency_type) {
-
#ifdef CONFIG_PREEMPT_OFF_HIST
case PREEMPTOFF_LATENCY:
for_each_online_cpu(cpu) {
@@ -296,7 +373,6 @@ latency_hist_reset(struct file *file, const char __user *a,
}
break;
#endif
-
#ifdef CONFIG_INTERRUPT_OFF_HIST
case IRQSOFF_LATENCY:
for_each_online_cpu(cpu) {
@@ -305,7 +381,6 @@ latency_hist_reset(struct file *file, const char __user *a,
}
break;
#endif
-
#if defined(CONFIG_INTERRUPT_OFF_HIST) && defined(CONFIG_PREEMPT_OFF_HIST)
case PREEMPTIRQSOFF_LATENCY:
for_each_online_cpu(cpu) {
@@ -314,16 +389,34 @@ latency_hist_reset(struct file *file, const char __user *a,
}
break;
#endif
-
#ifdef CONFIG_WAKEUP_LATENCY_HIST
case WAKEUP_LATENCY:
for_each_online_cpu(cpu) {
- struct maxlatproc_data *mp =
- &per_cpu(wakeup_maxlatproc, cpu);
+ hist = &per_cpu(wakeup_latency_hist, cpu);
+ hist_reset(hist);
+ mp = &per_cpu(wakeup_maxlatproc, cpu);
mp->comm[0] = '\0';
mp->prio = mp->pid = mp->latency = 0;
- hist = &per_cpu(wakeup_latency_hist, cpu);
+ }
+ break;
+ case WAKEUP_LATENCY_SHAREDPRIO:
+ for_each_online_cpu(cpu) {
+ hist = &per_cpu(wakeup_latency_hist_sharedprio, cpu);
hist_reset(hist);
+ mp = &per_cpu(wakeup_maxlatproc_sharedprio, cpu);
+ mp->comm[0] = '\0';
+ mp->prio = mp->pid = mp->latency = 0;
+ }
+ break;
+#endif
+#ifdef CONFIG_MISSED_TIMER_OFFSETS_HIST
+ case MISSED_TIMER_OFFSETS:
+ for_each_online_cpu(cpu) {
+ hist = &per_cpu(missed_timer_offsets, cpu);
+ hist_reset(hist);
+ mp = &per_cpu(missed_timer_offsets_maxlatproc, cpu);
+ mp->comm[0] = '\0';
+ mp->prio = mp->pid = mp->latency = 0;
}
break;
#endif
@@ -341,8 +434,6 @@ latency_hist_show_pid(struct file *filp, char __user *ubuf,
int r;
r = snprintf(buf, sizeof(buf), "%u\n", wakeup_pid);
- if (r > sizeof(buf))
- r = sizeof(buf);
return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
}
@@ -367,7 +458,10 @@ latency_hist_pid(struct file *filp, const char __user *ubuf,
wakeup_pid = pid;
return cnt;
}
+#endif
+#if defined(CONFIG_WAKEUP_LATENCY_HIST) || \
+ defined(CONFIG_MISSED_TIMER_OFFSETS_HIST)
static ssize_t
latency_hist_show_maxlatproc(struct file *filp, char __user *ubuf,
size_t cnt, loff_t *ppos)
@@ -379,27 +473,11 @@ latency_hist_show_maxlatproc(struct file *filp, char __user *ubuf,
r = snprintf(buf, sizeof(buf), "%5d %3d %ld %s\n",
mp->pid, mp->prio, mp->latency, mp->comm);
- if (r > sizeof(buf))
- r = sizeof(buf);
return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
}
#endif
-#if defined(CONFIG_INTERRUPT_OFF_HIST) || defined(CONFIG_PREEMPT_OFF_HIST)
-#ifdef CONFIG_INTERRUPT_OFF_HIST
-static DEFINE_PER_CPU(cycles_t, hist_irqsoff_start);
-static DEFINE_PER_CPU(int, hist_irqsoff_counting);
-#endif
-#ifdef CONFIG_PREEMPT_OFF_HIST
-static DEFINE_PER_CPU(cycles_t, hist_preemptoff_start);
-static DEFINE_PER_CPU(int, hist_preemptoff_counting);
-#endif
-#if defined(CONFIG_INTERRUPT_OFF_HIST) && defined(CONFIG_PREEMPT_OFF_HIST)
-static DEFINE_PER_CPU(cycles_t, hist_preemptirqsoff_start);
-static DEFINE_PER_CPU(int, hist_preemptirqsoff_counting);
-#endif
-
static ssize_t
latency_hist_show_enable(struct file *filp, char __user *ubuf,
size_t cnt, loff_t *ppos)
@@ -409,8 +487,6 @@ latency_hist_show_enable(struct file *filp, char __user *ubuf,
int r;
r = snprintf(buf, sizeof(buf), "%d\n", ed->enabled);
- if (r > sizeof(buf))
- r = sizeof(buf);
return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
}
@@ -440,6 +516,19 @@ latency_hist_enable(struct file *filp, const char __user *ubuf,
int ret;
switch (ed->latency_type) {
+#if defined(CONFIG_INTERRUPT_OFF_HIST) || defined(CONFIG_PREEMPT_OFF_HIST)
+ case PREEMPTIRQSOFF_LATENCY:
+ ret = register_trace_preemptirqsoff_hist(
+ probe_preemptirqsoff_hist);
+ if (ret) {
+ pr_info("wakeup trace: Couldn't assign "
+ "probe_preemptirqsoff_hist "
+ "to trace_preemptirqsoff_hist\n");
+ return ret;
+ }
+ break;
+#endif
+#ifdef CONFIG_WAKEUP_LATENCY_HIST
case WAKEUP_LATENCY:
ret = register_trace_sched_wakeup(
probe_wakeup_latency_hist_start);
@@ -472,23 +561,49 @@ latency_hist_enable(struct file *filp, const char __user *ubuf,
return ret;
}
break;
- case PREEMPTIRQSOFF_LATENCY:
- ret = register_trace_preemptirqsoff_hist(
- probe_preemptirqsoff_hist);
+#endif
+#ifdef CONFIG_MISSED_TIMER_OFFSETS_HIST
+ case MISSED_TIMER_OFFSETS:
+ ret = register_trace_hrtimer_interrupt(
+ probe_hrtimer_interrupt);
if (ret) {
pr_info("wakeup trace: Couldn't assign "
- "probe_preemptirqsoff_hist "
- "to trace_preemptirqsoff_hist\n");
+ "probe_hrtimer_interrupt "
+ "to trace_hrtimer_interrupt\n");
return ret;
}
break;
+#endif
default:
break;
}
} else {
- int cpu;
-
switch (ed->latency_type) {
+#if defined(CONFIG_INTERRUPT_OFF_HIST) || defined(CONFIG_PREEMPT_OFF_HIST)
+ case PREEMPTIRQSOFF_LATENCY:
+ {
+ int cpu;
+
+ unregister_trace_preemptirqsoff_hist(
+ probe_preemptirqsoff_hist);
+ for_each_online_cpu(cpu) {
+#ifdef CONFIG_INTERRUPT_OFF_HIST
+ per_cpu(hist_irqsoff_counting,
+ cpu) = 0;
+#endif
+#ifdef CONFIG_PREEMPT_OFF_HIST
+ per_cpu(hist_preemptoff_counting,
+ cpu) = 0;
+#endif
+#if defined(CONFIG_INTERRUPT_OFF_HIST) && defined(CONFIG_PREEMPT_OFF_HIST)
+ per_cpu(hist_preemptirqsoff_counting,
+ cpu) = 0;
+#endif
+ }
+ }
+ break;
+#endif
+#ifdef CONFIG_WAKEUP_LATEHCY_HIST
case WAKEUP_LATENCY:
unregister_trace_sched_wakeup(
probe_wakeup_latency_hist_start);
@@ -497,23 +612,15 @@ latency_hist_enable(struct file *filp, const char __user *ubuf,
unregister_trace_sched_switch(
probe_wakeup_latency_hist_stop);
wakeup_task = NULL;
- wakeup_prio = (unsigned)-1;
+ wakeup_sharedprio = 0;
break;
- case PREEMPTIRQSOFF_LATENCY:
- unregister_trace_preemptirqsoff_hist(
- probe_preemptirqsoff_hist);
- for_each_online_cpu(cpu) {
-#ifdef CONFIG_INTERRUPT_OFF_HIST
- per_cpu(hist_irqsoff_counting, cpu) = 0;
-#endif
-#ifdef CONFIG_PREEMPT_OFF_HIST
- per_cpu(hist_preemptoff_counting, cpu) = 0;
-#endif
-#if defined(CONFIG_INTERRUPT_OFF_HIST) && defined(CONFIG_PREEMPT_OFF_HIST)
- per_cpu(hist_preemptirqsoff_counting, cpu) = 0;
#endif
- }
+#ifdef CONFIG_MISSED_TIMER_OFFSETS_HIST
+ case MISSED_TIMER_OFFSETS:
+ unregister_trace_hrtimer_interrupt(
+ probe_hrtimer_interrupt);
break;
+#endif
default:
break;
}
@@ -522,28 +629,34 @@ latency_hist_enable(struct file *filp, const char __user *ubuf,
return cnt;
}
-static struct file_operations latency_hist_reset_fops = {
+static const struct file_operations latency_hist_reset_fops = {
.open = tracing_open_generic,
.write = latency_hist_reset,
};
-static struct file_operations latency_hist_pid_fops = {
+static const struct file_operations latency_hist_enable_fops = {
.open = tracing_open_generic,
- .read = latency_hist_show_pid,
- .write = latency_hist_pid,
+ .read = latency_hist_show_enable,
+ .write = latency_hist_enable,
};
-static struct file_operations latency_hist_maxlatproc_fops = {
+#ifdef CONFIG_WAKEUP_LATENCY_HIST
+static const struct file_operations latency_hist_pid_fops = {
.open = tracing_open_generic,
- .read = latency_hist_show_maxlatproc,
+ .read = latency_hist_show_pid,
+ .write = latency_hist_pid,
};
+#endif
-static struct file_operations latency_hist_enable_fops = {
+#if defined(CONFIG_WAKEUP_LATENCY_HIST) || \
+ defined(CONFIG_MISSED_TIMER_OFFSETS_HIST)
+static const struct file_operations latency_hist_maxlatproc_fops = {
.open = tracing_open_generic,
- .read = latency_hist_show_enable,
- .write = latency_hist_enable,
+ .read = latency_hist_show_maxlatproc,
};
+#endif
+#if defined(CONFIG_INTERRUPT_OFF_HIST) || defined(CONFIG_PREEMPT_OFF_HIST)
notrace void probe_preemptirqsoff_hist(int reason, int starthist)
{
int cpu = raw_smp_processor_id();
@@ -643,11 +756,9 @@ notrace void probe_preemptirqsoff_hist(int reason, int starthist)
#endif
}
}
-
#endif
#ifdef CONFIG_WAKEUP_LATENCY_HIST
-static cycle_t wakeup_start;
static DEFINE_ATOMIC_SPINLOCK(wakeup_lock);
notrace void probe_wakeup_latency_hist_start(struct rq *rq,
@@ -657,23 +768,28 @@ notrace void probe_wakeup_latency_hist_start(struct rq *rq,
struct task_struct *curr = rq_curr(rq);
if (wakeup_pid) {
+ if ((wakeup_task && p->prio == wakeup_task->prio) ||
+ p->prio == curr->prio)
+ wakeup_sharedprio = 1;
if (likely(wakeup_pid != task_pid_nr(p)))
return;
} else {
if (likely(!rt_task(p)) ||
- p->prio >= wakeup_prio ||
- p->prio >= curr->prio)
+ (wakeup_task && p->prio > wakeup_task->prio) ||
+ p->prio > curr->prio)
return;
+ if ((wakeup_task && p->prio == wakeup_task->prio) ||
+ p->prio == curr->prio)
+ wakeup_sharedprio = 1;
}
atomic_spin_lock_irqsave(&wakeup_lock, flags);
if (wakeup_task)
put_task_struct(wakeup_task);
-
get_task_struct(p);
wakeup_task = p;
- wakeup_prio = p->prio;
- wakeup_start = ftrace_now(raw_smp_processor_id());
+ wakeup_task->preempt_timestamp_hist =
+ ftrace_now(raw_smp_processor_id());
atomic_spin_unlock_irqrestore(&wakeup_lock, flags);
}
@@ -692,31 +808,59 @@ notrace void probe_wakeup_latency_hist_stop(struct rq *rq,
stop = ftrace_now(cpu);
atomic_spin_lock_irqsave(&wakeup_lock, flags);
- if (next != wakeup_task)
+
+ latency = nsecs_to_usecs(stop - next->preempt_timestamp_hist);
+ if (next != wakeup_task) {
+ if (wakeup_task && next->prio == wakeup_task->prio)
+ latency_hist(WAKEUP_LATENCY_SHAREDPRIO, cpu, latency,
+ next);
goto out;
+ }
- latency = nsecs_to_usecs(stop - wakeup_start);
- latency_hist(WAKEUP_LATENCY, cpu, latency, next);
+ if (wakeup_sharedprio) {
+ latency_hist(WAKEUP_LATENCY_SHAREDPRIO, cpu, latency, next);
+ wakeup_sharedprio = 0;
+ } else
+ latency_hist(WAKEUP_LATENCY, cpu, latency, next);
put_task_struct(wakeup_task);
wakeup_task = NULL;
- wakeup_prio = (unsigned)-1;
out:
atomic_spin_unlock_irqrestore(&wakeup_lock, flags);
}
+#endif
+
+#ifdef CONFIG_MISSED_TIMER_OFFSETS_HIST
+notrace void probe_hrtimer_interrupt(int cpu, long long latency_ns,
+ struct task_struct *task)
+{
+ if (latency_ns <= 0) {
+ unsigned long latency;
+
+ latency = (unsigned long) div_s64(-latency_ns, 1000);
+ latency_hist(MISSED_TIMER_OFFSETS, cpu, latency, task);
+ }
+}
#endif
static __init int latency_hist_init(void)
{
struct dentry *latency_hist_root = NULL;
struct dentry *dentry;
+#ifdef CONFIG_WAKEUP_LATENCY_HIST
+ struct dentry *dentry_sharedprio;
+#endif
struct dentry *entry;
struct dentry *latency_hist_enable_root;
- int i = 0, len = 0;
+ int i = 0;
struct hist_data *my_hist;
char name[64];
char *cpufmt = "CPU%d";
+#if defined(CONFIG_WAKEUP_LATENCY_HIST) || \
+ defined(CONFIG_MISSED_TIMER_OFFSETS_HIST)
+ char *cpufmt_maxlatproc = "max_latency-CPU%d";
+#endif
dentry = tracing_init_dentry();
@@ -729,92 +873,118 @@ static __init int latency_hist_init(void)
#ifdef CONFIG_INTERRUPT_OFF_HIST
dentry = debugfs_create_dir(irqsoff_hist_dir, latency_hist_root);
for_each_possible_cpu(i) {
- len = sprintf(name, cpufmt, i);
- name[len] = '\0';
+ sprintf(name, cpufmt, i);
entry = debugfs_create_file(name, 0444, dentry,
- &per_cpu(irqsoff_hist, i),
- &latency_hist_fops);
+ &per_cpu(irqsoff_hist, i), &latency_hist_fops);
my_hist = &per_cpu(irqsoff_hist, i);
atomic_set(&my_hist->hist_mode, 1);
my_hist->min_lat = 0xFFFFFFFFUL;
}
entry = debugfs_create_file("reset", 0644, dentry,
- (void *)IRQSOFF_LATENCY,
- &latency_hist_reset_fops);
+ (void *)IRQSOFF_LATENCY, &latency_hist_reset_fops);
#endif
#ifdef CONFIG_PREEMPT_OFF_HIST
dentry = debugfs_create_dir(preemptoff_hist_dir,
- latency_hist_root);
+ latency_hist_root);
for_each_possible_cpu(i) {
- len = sprintf(name, cpufmt, i);
- name[len] = '\0';
+ sprintf(name, cpufmt, i);
entry = debugfs_create_file(name, 0444, dentry,
- &per_cpu(preemptoff_hist, i),
- &latency_hist_fops);
+ &per_cpu(preemptoff_hist, i), &latency_hist_fops);
my_hist = &per_cpu(preemptoff_hist, i);
atomic_set(&my_hist->hist_mode, 1);
my_hist->min_lat = 0xFFFFFFFFUL;
}
entry = debugfs_create_file("reset", 0644, dentry,
- (void *)PREEMPTOFF_LATENCY,
- &latency_hist_reset_fops);
+ (void *)PREEMPTOFF_LATENCY, &latency_hist_reset_fops);
#endif
#if defined(CONFIG_INTERRUPT_OFF_HIST) && defined(CONFIG_PREEMPT_OFF_HIST)
dentry = debugfs_create_dir(preemptirqsoff_hist_dir,
- latency_hist_root);
+ latency_hist_root);
for_each_possible_cpu(i) {
- len = sprintf(name, cpufmt, i);
- name[len] = '\0';
+ sprintf(name, cpufmt, i);
entry = debugfs_create_file(name, 0444, dentry,
- &per_cpu(preemptirqsoff_hist, i),
- &latency_hist_fops);
+ &per_cpu(preemptirqsoff_hist, i), &latency_hist_fops);
my_hist = &per_cpu(preemptirqsoff_hist, i);
atomic_set(&my_hist->hist_mode, 1);
my_hist->min_lat = 0xFFFFFFFFUL;
}
entry = debugfs_create_file("reset", 0644, dentry,
- (void *)PREEMPTIRQSOFF_LATENCY,
- &latency_hist_reset_fops);
+ (void *)PREEMPTIRQSOFF_LATENCY, &latency_hist_reset_fops);
#endif
#if defined(CONFIG_INTERRUPT_OFF_HIST) || defined(CONFIG_PREEMPT_OFF_HIST)
entry = debugfs_create_file("preemptirqsoff", 0644,
- latency_hist_enable_root,
- (void *)&preemptirqsoff_enabled_data,
- &latency_hist_enable_fops);
+ latency_hist_enable_root, (void *)&preemptirqsoff_enabled_data,
+ &latency_hist_enable_fops);
#endif
#ifdef CONFIG_WAKEUP_LATENCY_HIST
dentry = debugfs_create_dir(wakeup_latency_hist_dir,
- latency_hist_root);
+ latency_hist_root);
+ dentry_sharedprio = debugfs_create_dir(
+ wakeup_latency_hist_dir_sharedprio, dentry);
for_each_possible_cpu(i) {
- len = sprintf(name, cpufmt, i);
- name[len] = '\0';
+ sprintf(name, cpufmt, i);
entry = debugfs_create_file(name, 0444, dentry,
- &per_cpu(wakeup_latency_hist, i),
- &latency_hist_fops);
+ &per_cpu(wakeup_latency_hist, i),
+ &latency_hist_fops);
my_hist = &per_cpu(wakeup_latency_hist, i);
atomic_set(&my_hist->hist_mode, 1);
my_hist->min_lat = 0xFFFFFFFFUL;
- len = sprintf(name, "max_latency-CPU%d", i);
- name[len] = '\0';
+ sprintf(name, cpufmt, i);
+ entry = debugfs_create_file(name, 0444, dentry_sharedprio,
+ &per_cpu(wakeup_latency_hist_sharedprio, i),
+ &latency_hist_fops);
+ my_hist = &per_cpu(wakeup_latency_hist_sharedprio, i);
+ atomic_set(&my_hist->hist_mode, 1);
+ my_hist->min_lat = 0xFFFFFFFFUL;
+
+ sprintf(name, cpufmt_maxlatproc, i);
entry = debugfs_create_file(name, 0444, dentry,
- &per_cpu(wakeup_maxlatproc, i),
- &latency_hist_maxlatproc_fops);
+ &per_cpu(wakeup_maxlatproc, i),
+ &latency_hist_maxlatproc_fops);
+
+ sprintf(name, cpufmt_maxlatproc, i);
+ entry = debugfs_create_file(name, 0444, dentry_sharedprio,
+ &per_cpu(wakeup_maxlatproc_sharedprio, i),
+ &latency_hist_maxlatproc_fops);
}
entry = debugfs_create_file("pid", 0644, dentry,
- (void *)&wakeup_pid,
- &latency_hist_pid_fops);
+ (void *)&wakeup_pid, &latency_hist_pid_fops);
entry = debugfs_create_file("reset", 0644, dentry,
- (void *)WAKEUP_LATENCY,
- &latency_hist_reset_fops);
+ (void *)WAKEUP_LATENCY, &latency_hist_reset_fops);
+ entry = debugfs_create_file("reset", 0644, dentry_sharedprio,
+ (void *)WAKEUP_LATENCY_SHAREDPRIO, &latency_hist_reset_fops);
entry = debugfs_create_file("wakeup", 0644,
- latency_hist_enable_root,
- (void *)&wakeup_latency_enabled_data,
- &latency_hist_enable_fops);
+ latency_hist_enable_root, (void *)&wakeup_latency_enabled_data,
+ &latency_hist_enable_fops);
+#endif
+
+#ifdef CONFIG_MISSED_TIMER_OFFSETS_HIST
+ dentry = debugfs_create_dir(missed_timer_offsets_dir,
+ latency_hist_root);
+ for_each_possible_cpu(i) {
+ sprintf(name, cpufmt, i);
+ entry = debugfs_create_file(name, 0444, dentry,
+ &per_cpu(missed_timer_offsets, i), &latency_hist_fops);
+ my_hist = &per_cpu(missed_timer_offsets, i);
+ atomic_set(&my_hist->hist_mode, 1);
+ my_hist->min_lat = 0xFFFFFFFFUL;
+
+ sprintf(name, cpufmt_maxlatproc, i);
+ entry = debugfs_create_file(name, 0444, dentry,
+ &per_cpu(missed_timer_offsets_maxlatproc, i),
+ &latency_hist_maxlatproc_fops);
+ }
+ entry = debugfs_create_file("reset", 0644, dentry,
+ (void *)MISSED_TIMER_OFFSETS, &latency_hist_reset_fops);
+ entry = debugfs_create_file("missed_timer_offsets", 0644,
+ latency_hist_enable_root,
+ (void *)&missed_timer_offsets_enabled_data,
+ &latency_hist_enable_fops);
#endif
return 0;
}
diff --git a/net/sched/cls_api.c b/net/sched/cls_api.c
index 09cdcdfe7e91..b83ac88a1448 100644
--- a/net/sched/cls_api.c
+++ b/net/sched/cls_api.c
@@ -348,7 +348,7 @@ static int tcf_fill_node(struct sk_buff *skb, struct tcf_proto *tp,
tcm = NLMSG_DATA(nlh);
tcm->tcm_family = AF_UNSPEC;
tcm->tcm__pad1 = 0;
- tcm->tcm__pad1 = 0;
+ tcm->tcm__pad2 = 0;
tcm->tcm_ifindex = qdisc_dev(tp->q)->ifindex;
tcm->tcm_parent = tp->classid;
tcm->tcm_info = TC_H_MAKE(tp->prio, tp->protocol);
diff --git a/net/unix/af_unix.c b/net/unix/af_unix.c
index 51ab497115eb..fc820cd75453 100644
--- a/net/unix/af_unix.c
+++ b/net/unix/af_unix.c
@@ -1074,6 +1074,8 @@ restart:
err = -ECONNREFUSED;
if (other->sk_state != TCP_LISTEN)
goto out_unlock;
+ if (other->sk_shutdown & RCV_SHUTDOWN)
+ goto out_unlock;
if (unix_recvq_full(other)) {
err = -EAGAIN;
diff --git a/security/keys/keyctl.c b/security/keys/keyctl.c
index 7f09fb897d2b..667aecd0fe52 100644
--- a/security/keys/keyctl.c
+++ b/security/keys/keyctl.c
@@ -860,7 +860,7 @@ static long get_instantiation_keyring(key_serial_t ringid,
/* otherwise specify the destination keyring recorded in the
* authorisation key (any KEY_SPEC_*_KEYRING) */
if (ringid >= KEY_SPEC_REQUESTOR_KEYRING) {
- *_dest_keyring = rka->dest_keyring;
+ *_dest_keyring = key_get(rka->dest_keyring);
return 0;
}