diff --git a/OpenSim/Framework/Servers/HttpServer/PollServiceRequestManager.cs b/OpenSim/Framework/Servers/HttpServer/PollServiceRequestManager.cs index 2206febdb1..0062d4ef15 100644 --- a/OpenSim/Framework/Servers/HttpServer/PollServiceRequestManager.cs +++ b/OpenSim/Framework/Servers/HttpServer/PollServiceRequestManager.cs @@ -65,6 +65,7 @@ namespace OpenSim.Framework.Servers.HttpServer String.Format("PollServiceWorkerThread{0}", i), ThreadPriority.Normal, false, + true, int.MaxValue); } @@ -73,6 +74,7 @@ namespace OpenSim.Framework.Servers.HttpServer "PollServiceWatcherThread", ThreadPriority.Normal, false, + true, 1000 * 60 * 10); } diff --git a/OpenSim/Framework/Watchdog.cs b/OpenSim/Framework/Watchdog.cs index 2dd6ebe49a..e443f0aaf5 100644 --- a/OpenSim/Framework/Watchdog.cs +++ b/OpenSim/Framework/Watchdog.cs @@ -72,6 +72,11 @@ namespace OpenSim.Framework /// public bool IsTimedOut { get; set; } + /// + /// Will this thread trigger the alarm function if it has timed out? + /// + public bool AlarmIfTimeout { get; set; } + public ThreadWatchdogInfo(Thread thread, int timeout) { Thread = thread; @@ -112,12 +117,13 @@ namespace OpenSim.Framework /// The method that will be executed in a new thread /// A name to give to the new thread /// Priority to run the thread at - /// True to run this thread as a background - /// thread, otherwise false + /// True to run this thread as a background thread, otherwise false + /// Trigger an alarm function is we have timed out /// The newly created Thread object - public static Thread StartThread(ThreadStart start, string name, ThreadPriority priority, bool isBackground) + public static Thread StartThread( + ThreadStart start, string name, ThreadPriority priority, bool isBackground, bool alarmIfTimeout) { - return StartThread(start, name, priority, isBackground, WATCHDOG_TIMEOUT_MS); + return StartThread(start, name, priority, isBackground, alarmIfTimeout, WATCHDOG_TIMEOUT_MS); } /// @@ -128,21 +134,21 @@ namespace OpenSim.Framework /// Priority to run the thread at /// True to run this thread as a background /// thread, otherwise false - /// - /// Number of milliseconds to wait until we issue a warning about timeout. - /// + /// Trigger an alarm function is we have timed out + /// Number of milliseconds to wait until we issue a warning about timeout. /// The newly created Thread object public static Thread StartThread( - ThreadStart start, string name, ThreadPriority priority, bool isBackground, int timeout) + ThreadStart start, string name, ThreadPriority priority, bool isBackground, bool alarmIfTimeout, int timeout) { Thread thread = new Thread(start); thread.Name = name; thread.Priority = priority; thread.IsBackground = isBackground; - ThreadWatchdogInfo twi = new ThreadWatchdogInfo(thread, timeout); + ThreadWatchdogInfo twi = new ThreadWatchdogInfo(thread, timeout) { AlarmIfTimeout = alarmIfTimeout }; - m_log.Debug("[WATCHDOG]: Started tracking thread \"" + twi.Thread.Name + "\" (ID " + twi.Thread.ManagedThreadId + ")"); + m_log.DebugFormat( + "[WATCHDOG]: Started tracking thread {0}, ID {1}", twi.Thread.Name, twi.Thread.ManagedThreadId); lock (m_threads) m_threads.Add(twi.Thread.ManagedThreadId, twi); @@ -230,6 +236,26 @@ namespace OpenSim.Framework return m_threads.Values.ToArray(); } + /// + /// Return the current thread's watchdog info. + /// + /// The watchdog info. null if the thread isn't being monitored. + public static ThreadWatchdogInfo GetCurrentThreadInfo() + { + lock (m_threads) + { + if (m_threads.ContainsKey(Thread.CurrentThread.ManagedThreadId)) + return m_threads[Thread.CurrentThread.ManagedThreadId]; + } + + return null; + } + + /// + /// Check watched threads. Fire alarm if appropriate. + /// + /// + /// private static void WatchdogTimerElapsed(object sender, System.Timers.ElapsedEventArgs e) { WatchdogTimeout callback = OnWatchdogTimeout; @@ -246,21 +272,18 @@ namespace OpenSim.Framework { if (threadInfo.Thread.ThreadState == ThreadState.Stopped) { - timedOut = threadInfo; RemoveThread(threadInfo.Thread.ManagedThreadId); - break; + callback(threadInfo.Thread, threadInfo.LastTick); } else if (!threadInfo.IsTimedOut && now - threadInfo.LastTick >= threadInfo.Timeout) { threadInfo.IsTimedOut = true; - timedOut = threadInfo; - break; + + if (threadInfo.AlarmIfTimeout) + callback(threadInfo.Thread, threadInfo.LastTick); } } } - - if (timedOut != null) - callback(timedOut.Thread, timedOut.LastTick); } m_watchdogTimer.Start(); diff --git a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs index 1e22fcc395..fb6b11e209 100644 --- a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs +++ b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs @@ -244,8 +244,11 @@ namespace OpenSim.Region.ClientStack.LindenUDP base.Start(m_recvBufferSize, m_asyncPacketHandling); // Start the packet processing threads - Watchdog.StartThread(IncomingPacketHandler, "Incoming Packets (" + m_scene.RegionInfo.RegionName + ")", ThreadPriority.Normal, false); - Watchdog.StartThread(OutgoingPacketHandler, "Outgoing Packets (" + m_scene.RegionInfo.RegionName + ")", ThreadPriority.Normal, false); + Watchdog.StartThread( + IncomingPacketHandler, "Incoming Packets (" + m_scene.RegionInfo.RegionName + ")", ThreadPriority.Normal, false, true); + Watchdog.StartThread( + OutgoingPacketHandler, "Outgoing Packets (" + m_scene.RegionInfo.RegionName + ")", ThreadPriority.Normal, false, true); + m_elapsedMSSinceLastStatReport = Environment.TickCount; } diff --git a/OpenSim/Region/CoreModules/InterGrid/OpenGridProtocolModule.cs b/OpenSim/Region/CoreModules/InterGrid/OpenGridProtocolModule.cs index f3677390a9..a6e2548406 100644 --- a/OpenSim/Region/CoreModules/InterGrid/OpenGridProtocolModule.cs +++ b/OpenSim/Region/CoreModules/InterGrid/OpenGridProtocolModule.cs @@ -1210,7 +1210,7 @@ namespace OpenSim.Region.CoreModules.InterGrid if (homeScene.TryGetScenePresence(avatarId,out avatar)) { KillAUser ku = new KillAUser(avatar,mod); - Watchdog.StartThread(ku.ShutdownNoLogout, "OGPShutdown", ThreadPriority.Normal, true); + Watchdog.StartThread(ku.ShutdownNoLogout, "OGPShutdown", ThreadPriority.Normal, true, true); } } diff --git a/OpenSim/Region/CoreModules/World/WorldMap/WorldMapModule.cs b/OpenSim/Region/CoreModules/World/WorldMap/WorldMapModule.cs index b315d2c003..74b047be14 100644 --- a/OpenSim/Region/CoreModules/World/WorldMap/WorldMapModule.cs +++ b/OpenSim/Region/CoreModules/World/WorldMap/WorldMapModule.cs @@ -351,6 +351,7 @@ namespace OpenSim.Region.CoreModules.World.WorldMap process, string.Format("MapItemRequestThread ({0})", m_scene.RegionInfo.RegionName), ThreadPriority.BelowNormal, + true, true); } diff --git a/OpenSim/Region/Framework/Scenes/Scene.cs b/OpenSim/Region/Framework/Scenes/Scene.cs index cf6e6af727..19d4bad9e8 100644 --- a/OpenSim/Region/Framework/Scenes/Scene.cs +++ b/OpenSim/Region/Framework/Scenes/Scene.cs @@ -1140,7 +1140,7 @@ namespace OpenSim.Region.Framework.Scenes HeartbeatThread = Watchdog.StartThread( - Heartbeat, string.Format("Heartbeat ({0})", RegionInfo.RegionName), ThreadPriority.Normal, false); + Heartbeat, string.Format("Heartbeat ({0})", RegionInfo.RegionName), ThreadPriority.Normal, false, false); } /// @@ -1178,6 +1178,13 @@ namespace OpenSim.Region.Framework.Scenes try { m_eventManager.TriggerOnRegionStarted(this); + + // The first frame can take a very long time due to physics actors being added on startup. Therefore, + // don't turn on the watchdog alarm for this thread until the second frame, in order to prevent false + // alarms for scenes with many objects. + Update(); + Watchdog.GetCurrentThreadInfo().AlarmIfTimeout = true; + while (!shuttingdown) Update(); @@ -1206,7 +1213,7 @@ namespace OpenSim.Region.Framework.Scenes ++Frame; -// m_log.DebugFormat("[SCENE]: Processing frame {0}", Frame); +// m_log.DebugFormat("[SCENE]: Processing frame {0} in {1}", Frame, RegionInfo.RegionName); try { @@ -1418,7 +1425,6 @@ namespace OpenSim.Region.Framework.Scenes entry.checkAtTargets(); } - /// /// Send out simstats data to all clients /// diff --git a/OpenSim/Region/OptionalModules/Agent/InternetRelayClientView/Server/IRCClientView.cs b/OpenSim/Region/OptionalModules/Agent/InternetRelayClientView/Server/IRCClientView.cs index c928af7892..d3c96e27ae 100644 --- a/OpenSim/Region/OptionalModules/Agent/InternetRelayClientView/Server/IRCClientView.cs +++ b/OpenSim/Region/OptionalModules/Agent/InternetRelayClientView/Server/IRCClientView.cs @@ -70,7 +70,7 @@ namespace OpenSim.Region.OptionalModules.Agent.InternetRelayClientView.Server m_client = client; m_scene = scene; - Watchdog.StartThread(InternalLoop, "IRCClientView", ThreadPriority.Normal, false); + Watchdog.StartThread(InternalLoop, "IRCClientView", ThreadPriority.Normal, false, true); } private void SendServerCommand(string command) diff --git a/OpenSim/Region/OptionalModules/Agent/InternetRelayClientView/Server/IRCServer.cs b/OpenSim/Region/OptionalModules/Agent/InternetRelayClientView/Server/IRCServer.cs index eb390268d7..a7c5020eb6 100644 --- a/OpenSim/Region/OptionalModules/Agent/InternetRelayClientView/Server/IRCServer.cs +++ b/OpenSim/Region/OptionalModules/Agent/InternetRelayClientView/Server/IRCServer.cs @@ -57,7 +57,7 @@ namespace OpenSim.Region.OptionalModules.Agent.InternetRelayClientView.Server m_listener.Start(50); - Watchdog.StartThread(ListenLoop, "IRCServer", ThreadPriority.Normal, false); + Watchdog.StartThread(ListenLoop, "IRCServer", ThreadPriority.Normal, false, true); m_baseScene = baseScene; } diff --git a/OpenSim/Region/Physics/OdePlugin/ODEPrim.cs b/OpenSim/Region/Physics/OdePlugin/ODEPrim.cs index 75364b73fc..97890ee398 100644 --- a/OpenSim/Region/Physics/OdePlugin/ODEPrim.cs +++ b/OpenSim/Region/Physics/OdePlugin/ODEPrim.cs @@ -1474,6 +1474,8 @@ Console.WriteLine("CreateGeom:"); /// private void changeadd() { +// m_log.DebugFormat("[ODE PRIM]: Adding prim {0}", Name); + int[] iprimspaceArrItem = _parent_scene.calculateSpaceArrayItemFromPos(_position); IntPtr targetspace = _parent_scene.calculateSpaceForGeom(_position); diff --git a/OpenSim/Region/ScriptEngine/Shared/Api/Implementation/AsyncCommandManager.cs b/OpenSim/Region/ScriptEngine/Shared/Api/Implementation/AsyncCommandManager.cs index ee32755a0b..14edde41d4 100644 --- a/OpenSim/Region/ScriptEngine/Shared/Api/Implementation/AsyncCommandManager.cs +++ b/OpenSim/Region/ScriptEngine/Shared/Api/Implementation/AsyncCommandManager.cs @@ -137,7 +137,9 @@ namespace OpenSim.Region.ScriptEngine.Shared.Api if (cmdHandlerThread == null) { // Start the thread that will be doing the work - cmdHandlerThread = Watchdog.StartThread(CmdHandlerThreadLoop, "AsyncLSLCmdHandlerThread", ThreadPriority.Normal, true); + cmdHandlerThread + = Watchdog.StartThread( + CmdHandlerThreadLoop, "AsyncLSLCmdHandlerThread", ThreadPriority.Normal, true, true); } }