* Changed the watchdog timer to improve the speed of UpdateThread(), only track threads once the first call to UpdateThread() has been made, and allow re-tracking of threads that timed out but revived later

* Added a commented out call to Watchdog.UpdateThread() in OdeScene. If it turns out that loading a large OAR file or some other operation is timing out the heartbeat thread, we'll need to uncomment it

OpenSim/Framework/Watchdog.cs

@@ -28,6 +28,7 @@
 using System;
 using System.Collections.Generic;
 using System.Threading;
+using log4net;
 
 namespace OpenSim.Framework
 {
@@ -66,6 +67,7 @@ namespace OpenSim.Framework
         /// stopped or has not called UpdateThread() in time</summary>
         public static event WatchdogTimeout OnWatchdogTimeout;
 
+        private static readonly ILog m_log = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType);
         private static Dictionary<int, ThreadWatchdogInfo> m_threads;
         private static System.Timers.Timer m_watchdogTimer;
 
@@ -95,9 +97,6 @@ namespace OpenSim.Framework
             thread.IsBackground = isBackground;
             thread.Start();
 
-            lock (m_threads)
-                m_threads.Add(thread.ManagedThreadId, new ThreadWatchdogInfo(thread));
-
             return thread;
         }
 
@@ -109,24 +108,6 @@ namespace OpenSim.Framework
             UpdateThread(Thread.CurrentThread.ManagedThreadId);
         }
 
-        /// <summary>
-        /// Marks a thread as alive
-        /// </summary>
-        /// <param name="threadID">The ManagedThreadId of the thread to mark as
-        /// alive</param>
-        public static void UpdateThread(int threadID)
-        {
-            ThreadWatchdogInfo threadInfo;
-
-            lock (m_threads)
-            {
-                if (m_threads.TryGetValue(threadID, out threadInfo))
-                {
-                    threadInfo.LastTick = Environment.TickCount & Int32.MaxValue;
-                }
-            }
-        }
-
         /// <summary>
         /// Stops watchdog tracking on the current thread
         /// </summary>
@@ -137,19 +118,38 @@ namespace OpenSim.Framework
             return RemoveThread(Thread.CurrentThread.ManagedThreadId);
         }
 
-        /// <summary>
+        private static void AddThread(ThreadWatchdogInfo threadInfo)
-        /// Stops watchdog tracking on a thread
+        {
-        /// </summary>
+            m_log.Debug("[WATCHDOG]: Started tracking thread \"" + threadInfo.Thread.Name + "\" (ID " + threadInfo.Thread.ManagedThreadId + ")");
-        /// <param name="threadID">The ManagedThreadId of the thread to stop
+
-        /// tracking</param>
+            lock (m_threads)
-        /// <returns>True if the thread was removed from the list of tracked
+                m_threads.Add(threadInfo.Thread.ManagedThreadId, threadInfo);
-        /// threads, otherwise false</returns>
+        }
-        public static bool RemoveThread(int threadID)
+
+        private static bool RemoveThread(int threadID)
         {
             lock (m_threads)
                 return m_threads.Remove(threadID);
         }
 
+        private static void UpdateThread(int threadID)
+        {
+            ThreadWatchdogInfo threadInfo;
+
+            // Although TryGetValue is not a thread safe operation, we use a try/catch here instead
+            // of a lock for speed. Adding/removing threads is a very rare operation compared to
+            // UpdateThread(), and a single UpdateThread() failure here and there won't break
+            // anything
+            try
+            {
+                if (m_threads.TryGetValue(threadID, out threadInfo))
+                    threadInfo.LastTick = Environment.TickCount & Int32.MaxValue;
+                else
+                    AddThread(new ThreadWatchdogInfo(Thread.CurrentThread));
+            }
+            catch { }
+        }
+
         private static void WatchdogTimerElapsed(object sender, System.Timers.ElapsedEventArgs e)
         {
             WatchdogTimeout callback = OnWatchdogTimeout;
@@ -160,7 +160,7 @@ namespace OpenSim.Framework
 
                 lock (m_threads)
                 {
-                    int now = Environment.TickCount;
+                    int now = Environment.TickCount & Int32.MaxValue;
 
                     foreach (ThreadWatchdogInfo threadInfo in m_threads.Values)
                     {

OpenSim/Region/Framework/Scenes/Scene.cs

@@ -1025,6 +1025,7 @@ namespace OpenSim.Region.Framework.Scenes
                 float physicsFPS = 0;
 
                 frameMS = Environment.TickCount;
+
                 try
                 {
                     // Increment the frame counter
@@ -1152,6 +1153,7 @@ namespace OpenSim.Region.Framework.Scenes
                 if ((maintc < (m_timespan * 1000)) && maintc > 0)
                     Thread.Sleep(maintc);
 
+                // Tell the watchdog that this thread is still alive
                 Watchdog.UpdateThread();
             }
         }

OpenSim/Region/Physics/OdePlugin/OdePlugin.cs

@@ -2705,8 +2705,6 @@ namespace OpenSim.Region.Physics.OdePlugin
                                 {
                                     foreach (OdePrim prim in _taintedPrimL)
                                     {
-       
-                                    
                                         if (prim.m_taintremove)
                                         {
 //Console.WriteLine("Simulate calls RemovePrimThreadLocked");                                          
@@ -2719,6 +2717,12 @@ namespace OpenSim.Region.Physics.OdePlugin
                                         }
                                         processedtaints = true;
                                         prim.m_collisionscore = 0;
+
+                                        // This loop can block up the Heartbeat for a very long time on large regions.
+                                        // We need to let the Watchdog know that the Heartbeat is not dead
+                                        // NOTE: This is currently commented out, but if things like OAR loading are 
+                                        // timing the heartbeat out we will need to uncomment it
+                                        //Watchdog.UpdateThread();
                                     }
 
                                     if (SupportsNINJAJoints)