From a2c3dfc422ab282c21232e63f975c73c3bbb70e3 Mon Sep 17 00:00:00 2001
From: "Justin Clark-Casey (justincc)" <jjustincc@googlemail.com>
Date: Thu, 5 Jul 2012 23:15:59 +0100
Subject: [PATCH] Log warning if time between invocations of the watchdog
 thread is twice the timer setting.

This is to help detect situations where thread timeout warnings are being generated because of general machine issues rather than deadlock, network or other problems.
---
 OpenSim/Framework/Watchdog.cs | 24 ++++++++++++++++++++++--
 1 file changed, 22 insertions(+), 2 deletions(-)

diff --git a/OpenSim/Framework/Watchdog.cs b/OpenSim/Framework/Watchdog.cs
index 2b3a719daf..8a74f538f7 100644
--- a/OpenSim/Framework/Watchdog.cs
+++ b/OpenSim/Framework/Watchdog.cs
@@ -101,12 +101,24 @@ namespace OpenSim.Framework
         private static Dictionary<int, ThreadWatchdogInfo> m_threads;
         private static System.Timers.Timer m_watchdogTimer;
 
+        /// <summary>
+        /// Last time the watchdog thread ran.
+        /// </summary>
+        /// <remarks>
+        /// Should run every WATCHDOG_INTERVAL_MS
+        /// </remarks>
+        public static int LastWatchdogThreadTick { get; private set; }
+
         static Watchdog()
         {
             m_threads = new Dictionary<int, ThreadWatchdogInfo>();
             m_watchdogTimer = new System.Timers.Timer(WATCHDOG_INTERVAL_MS);
             m_watchdogTimer.AutoReset = false;
             m_watchdogTimer.Elapsed += WatchdogTimerElapsed;
+
+            // Set now so we don't get alerted on the first run
+            LastWatchdogThreadTick = Environment.TickCount & Int32.MaxValue;
+
             m_watchdogTimer.Start();
         }
 
@@ -264,6 +276,16 @@ namespace OpenSim.Framework
         /// <param name="e"></param>
         private static void WatchdogTimerElapsed(object sender, System.Timers.ElapsedEventArgs e)
         {
+            int now = Environment.TickCount & Int32.MaxValue;
+            int msElapsed = now - LastWatchdogThreadTick;
+
+            if (msElapsed > WATCHDOG_INTERVAL_MS * 2)
+                m_log.WarnFormat(
+                    "[WATCHDOG]: {0} ms since Watchdog last ran.  Interval should be approximately {1} ms",
+                    msElapsed, WATCHDOG_INTERVAL_MS);
+
+            LastWatchdogThreadTick = Environment.TickCount & Int32.MaxValue;
+
             Action<ThreadWatchdogInfo> callback = OnWatchdogTimeout;
 
             if (callback != null)
@@ -272,8 +294,6 @@ namespace OpenSim.Framework
 
                 lock (m_threads)
                 {
-                    int now = Environment.TickCount & Int32.MaxValue;
-
                     foreach (ThreadWatchdogInfo threadInfo in m_threads.Values)
                     {
                         if (threadInfo.Thread.ThreadState == ThreadState.Stopped)