Add a scene maintenance thread in parallel to the heartbeat thread. The maintenance thread will end up running regular jobs that don't need to be in the main scene loop.

The idea is to make the critical main scene loop as skinny as possible - it doesn't need to run things that aren't time critical and don't depend on update ordering.
This will be done gradually over time to try and uncover any issues.  Many non-criticial scene loop activities are being launched on separate threadpool threads anyway.
This may also allow modules to register their own maintenance jobs without having to maintain their own timers and threads.
Currently the maintenance loop runs once a second, as opposed to the 89ms scene loop.

OpenSim/Region/Framework/Scenes/Scene.cs

@@ -168,6 +168,11 @@ namespace OpenSim.Region.Framework.Scenes
             protected set;
         }
 
+        /// <summary>
+        /// Current maintenance run number
+        /// </summary>
+        public uint MaintenanceRun { get; private set; }
+
         /// <summary>
         /// The minimum length of time in seconds that will be taken for a scene frame.  If the frame takes less time then we
         /// will sleep for the remaining period.
@@ -178,6 +183,11 @@ namespace OpenSim.Region.Framework.Scenes
         /// </remarks>
         public float MinFrameTime { get; private set; }
 
+        /// <summary>
+        /// The minimum length of time in seconds that will be taken for a maintenance run.
+        /// </summary>
+        public float MinMaintenanceTime { get; private set; }
+
         private int m_update_physics = 1;
         private int m_update_entitymovement = 1;
         private int m_update_objects = 1;
@@ -205,6 +215,11 @@ namespace OpenSim.Region.Framework.Scenes
         /// </summary>
         private int m_lastFrameTick;
 
+        /// <summary>
+        /// Tick at which the last maintenance run occurred.
+        /// </summary>
+        private int m_lastMaintenanceTick;
+
         /// <summary>
         /// Signals whether temporary objects are currently being cleaned up.  Needed because this is launched
         /// asynchronously from the update loop.
@@ -560,6 +575,7 @@ namespace OpenSim.Region.Framework.Scenes
         {
             m_config = config;
             MinFrameTime = 0.089f;
+            MinMaintenanceTime = 1;
 
             Random random = new Random();
 
@@ -1231,6 +1247,10 @@ namespace OpenSim.Region.Framework.Scenes
             // don't turn on the watchdog alarm for this thread until the second frame, in order to prevent false
             // alarms for scenes with many objects.
             Update(1);
+
+            Watchdog.StartThread(
+                    Maintenance, string.Format("Maintenance ({0})", RegionInfo.RegionName), ThreadPriority.Normal, false, true);
+
             Watchdog.GetCurrentThreadInfo().AlarmIfTimeout = true;
             Update(-1);
 
@@ -1246,6 +1266,63 @@ namespace OpenSim.Region.Framework.Scenes
             Watchdog.RemoveThread();
         }
 
+        private void Maintenance()
+        {
+            DoMaintenance(-1);
+
+            Watchdog.RemoveThread();
+        }
+
+        public void DoMaintenance(int runs)
+        {
+            long? endRun = null;
+            int runtc;
+            int previousMaintenanceTick;
+
+            if (runs >= 0)
+                endRun = MaintenanceRun + runs;
+
+            List<Vector3> coarseLocations;
+            List<UUID> avatarUUIDs;
+
+            while (!m_shuttingDown && (endRun == null || MaintenanceRun < endRun))
+            {
+                runtc = Util.EnvironmentTickCount();
+                ++MaintenanceRun;
+
+                // Coarse locations relate to positions of green dots on the mini-map (on a SecondLife client)
+                if (MaintenanceRun % (m_update_coarse_locations / 10) == 0)
+                {
+                    SceneGraph.GetCoarseLocations(out coarseLocations, out avatarUUIDs, 60);
+                    // Send coarse locations to clients
+                    ForEachScenePresence(delegate(ScenePresence presence)
+                    {
+                        presence.SendCoarseLocations(coarseLocations, avatarUUIDs);
+                    });
+                }
+
+                Watchdog.UpdateThread();
+
+                previousMaintenanceTick = m_lastMaintenanceTick;
+                m_lastMaintenanceTick = Util.EnvironmentTickCount();
+                runtc = Util.EnvironmentTickCountSubtract(m_lastMaintenanceTick, runtc);
+                runtc = (int)(MinMaintenanceTime * 1000) - runtc;
+    
+                if (runtc > 0)
+                    Thread.Sleep(runtc);
+    
+                // Optionally warn if a frame takes double the amount of time that it should.
+                if (DebugUpdates
+                    && Util.EnvironmentTickCountSubtract(
+                        m_lastMaintenanceTick, previousMaintenanceTick) > (int)(MinMaintenanceTime * 1000 * 2))
+                    m_log.WarnFormat(
+                        "[SCENE]: Maintenance took {0} ms (desired max {1} ms) in {2}",
+                        Util.EnvironmentTickCountSubtract(m_lastMaintenanceTick, previousMaintenanceTick),
+                        MinMaintenanceTime * 1000,
+                        RegionInfo.RegionName);
+            }
+        }
+
         public override void Update(int frames)
         {
             long? endFrame = null;
@@ -1257,8 +1334,6 @@ namespace OpenSim.Region.Framework.Scenes
             int tmpPhysicsMS, tmpPhysicsMS2, tmpAgentMS, tmpTempOnRezMS, evMS, backMS, terMS;
             int previousFrameTick;
             int maintc;
-            List<Vector3> coarseLocations;
-            List<UUID> avatarUUIDs;
 
             while (!m_shuttingDown && (endFrame == null || Frame < endFrame))
             {
@@ -1310,17 +1385,6 @@ namespace OpenSim.Region.Framework.Scenes
                     if (Frame % m_update_presences == 0)
                         m_sceneGraph.UpdatePresences();
     
-                    // Coarse locations relate to positions of green dots on the mini-map (on a SecondLife client)
-                    if (Frame % m_update_coarse_locations == 0)
-                    {
-                        SceneGraph.GetCoarseLocations(out coarseLocations, out avatarUUIDs, 60);
-                        // Send coarse locations to clients 
-                        ForEachScenePresence(delegate(ScenePresence presence)
-                        {
-                            presence.SendCoarseLocations(coarseLocations, avatarUUIDs);
-                        });
-                    }
-    
                     agentMS += Util.EnvironmentTickCountSubtract(tmpAgentMS);
     
                     // Delete temp-on-rez stuff
@@ -1428,7 +1492,6 @@ namespace OpenSim.Region.Framework.Scenes
     
                 EventManager.TriggerRegionHeartbeatEnd(this);
 
-                // Tell the watchdog that this thread is still alive
                 Watchdog.UpdateThread();
 
                 previousFrameTick = m_lastFrameTick;