OpenSim
/
OpenSimMirror
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Stop spurious scene loop startup timeout alarms for scenes with many prims. 

On the first frame, all startup scene objects are added to the physics scene.
This can cause a considerable delay, so we don't start raising the alarm on scene loop timeouts until the second frame.
This commit also slightly changes the behaviour of timeout reporting.
Previously, a report was made for the very first timed out thread, ignoring all others until the next watchdog check.
Instead, we now report every timed out thread, though we still only do this once no matter how long the timeout.
xassetservice
Justin Clark-Casey (justincc) 2012-02-24 05:02:33 +00:00
parent fe229f10e6
commit f67f37074f
10 changed files with 65 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
OpenSim/Framework/Servers/HttpServer/PollServiceRequestManager.cs
View File

@ -65,6 +65,7 @@ namespace OpenSim.Framework.Servers.HttpServer
String.Format("PollServiceWorkerThread{0}", i),
ThreadPriority.Normal,
false,
true,
int.MaxValue);
}
@ -73,6 +74,7 @@ namespace OpenSim.Framework.Servers.HttpServer
"PollServiceWatcherThread",
ThreadPriority.Normal,
false,
true,
1000 * 60 * 10);
}

57
OpenSim/Framework/Watchdog.cs
View File

@ -72,6 +72,11 @@ namespace OpenSim.Framework
/// </summary>
public bool IsTimedOut { get; set; }
/// <summary>
/// Will this thread trigger the alarm function if it has timed out?
/// </summary>
public bool AlarmIfTimeout { get; set; }
public ThreadWatchdogInfo(Thread thread, int timeout)
{
Thread = thread;
@ -112,12 +117,13 @@ namespace OpenSim.Framework
/// <param name="start">The method that will be executed in a new thread</param>
/// <param name="name">A name to give to the new thread</param>
/// <param name="priority">Priority to run the thread at</param>
/// <param name="isBackground">True to run this thread as a background
/// thread, otherwise false</param>
/// <param name="isBackground">True to run this thread as a background thread, otherwise false</param>
/// <param name="alarmIfTimeout">Trigger an alarm function is we have timed out</param>
/// <returns>The newly created Thread object</returns>
public static Thread StartThread(ThreadStart start, string name, ThreadPriority priority, bool isBackground)
public static Thread StartThread(
ThreadStart start, string name, ThreadPriority priority, bool isBackground, bool alarmIfTimeout)
{
return StartThread(start, name, priority, isBackground, WATCHDOG_TIMEOUT_MS);
return StartThread(start, name, priority, isBackground, alarmIfTimeout, WATCHDOG_TIMEOUT_MS);
}
/// <summary>
@ -128,21 +134,21 @@ namespace OpenSim.Framework
/// <param name="priority">Priority to run the thread at</param>
/// <param name="isBackground">True to run this thread as a background
/// thread, otherwise false</param>
/// <param name="timeout">
/// Number of milliseconds to wait until we issue a warning about timeout.
/// </para>
/// <param name="alarmIfTimeout">Trigger an alarm function is we have timed out</param>
/// <param name="timeout">Number of milliseconds to wait until we issue a warning about timeout.</param>
/// <returns>The newly created Thread object</returns>
public static Thread StartThread(
ThreadStart start, string name, ThreadPriority priority, bool isBackground, int timeout)
ThreadStart start, string name, ThreadPriority priority, bool isBackground, bool alarmIfTimeout, int timeout)
{
Thread thread = new Thread(start);
thread.Name = name;
thread.Priority = priority;
thread.IsBackground = isBackground;
ThreadWatchdogInfo twi = new ThreadWatchdogInfo(thread, timeout);
ThreadWatchdogInfo twi = new ThreadWatchdogInfo(thread, timeout) { AlarmIfTimeout = alarmIfTimeout };
m_log.Debug("[WATCHDOG]: Started tracking thread \"" + twi.Thread.Name + "\" (ID " + twi.Thread.ManagedThreadId + ")");
m_log.DebugFormat(
"[WATCHDOG]: Started tracking thread {0}, ID {1}", twi.Thread.Name, twi.Thread.ManagedThreadId);
lock (m_threads)
m_threads.Add(twi.Thread.ManagedThreadId, twi);
@ -230,6 +236,26 @@ namespace OpenSim.Framework
return m_threads.Values.ToArray();
}
/// <summary>
/// Return the current thread's watchdog info.
/// </summary>
/// <returns>The watchdog info. null if the thread isn't being monitored.</returns>
public static ThreadWatchdogInfo GetCurrentThreadInfo()
{
lock (m_threads)
{
if (m_threads.ContainsKey(Thread.CurrentThread.ManagedThreadId))
return m_threads[Thread.CurrentThread.ManagedThreadId];
}
return null;
}
/// <summary>
/// Check watched threads. Fire alarm if appropriate.
/// </summary>
/// <param name="sender"></param>
/// <param name="e"></param>
private static void WatchdogTimerElapsed(object sender, System.Timers.ElapsedEventArgs e)
{
WatchdogTimeout callback = OnWatchdogTimeout;
@ -246,21 +272,18 @@ namespace OpenSim.Framework
{
if (threadInfo.Thread.ThreadState == ThreadState.Stopped)
{
timedOut = threadInfo;
RemoveThread(threadInfo.Thread.ManagedThreadId);
break;
callback(threadInfo.Thread, threadInfo.LastTick);
}
else if (!threadInfo.IsTimedOut && now - threadInfo.LastTick >= threadInfo.Timeout)
{
threadInfo.IsTimedOut = true;
timedOut = threadInfo;
break;
if (threadInfo.AlarmIfTimeout)
callback(threadInfo.Thread, threadInfo.LastTick);
}
}
}
if (timedOut != null)
callback(timedOut.Thread, timedOut.LastTick);
}
m_watchdogTimer.Start();

7
OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs
View File

@ -244,8 +244,11 @@ namespace OpenSim.Region.ClientStack.LindenUDP
base.Start(m_recvBufferSize, m_asyncPacketHandling);
// Start the packet processing threads
Watchdog.StartThread(IncomingPacketHandler, "Incoming Packets (" + m_scene.RegionInfo.RegionName + ")", ThreadPriority.Normal, false);
Watchdog.StartThread(OutgoingPacketHandler, "Outgoing Packets (" + m_scene.RegionInfo.RegionName + ")", ThreadPriority.Normal, false);
Watchdog.StartThread(
IncomingPacketHandler, "Incoming Packets (" + m_scene.RegionInfo.RegionName + ")", ThreadPriority.Normal, false, true);
Watchdog.StartThread(
OutgoingPacketHandler, "Outgoing Packets (" + m_scene.RegionInfo.RegionName + ")", ThreadPriority.Normal, false, true);
m_elapsedMSSinceLastStatReport = Environment.TickCount;
}

2
OpenSim/Region/CoreModules/InterGrid/OpenGridProtocolModule.cs
View File

@ -1210,7 +1210,7 @@ namespace OpenSim.Region.CoreModules.InterGrid
if (homeScene.TryGetScenePresence(avatarId,out avatar))
{
KillAUser ku = new KillAUser(avatar,mod);
Watchdog.StartThread(ku.ShutdownNoLogout, "OGPShutdown", ThreadPriority.Normal, true);
Watchdog.StartThread(ku.ShutdownNoLogout, "OGPShutdown", ThreadPriority.Normal, true, true);
}
}

1
OpenSim/Region/CoreModules/World/WorldMap/WorldMapModule.cs
View File

@ -351,6 +351,7 @@ namespace OpenSim.Region.CoreModules.World.WorldMap
process,
string.Format("MapItemRequestThread ({0})", m_scene.RegionInfo.RegionName),
ThreadPriority.BelowNormal,
true,
true);
}

12
OpenSim/Region/Framework/Scenes/Scene.cs
View File

@ -1140,7 +1140,7 @@ namespace OpenSim.Region.Framework.Scenes
HeartbeatThread
= Watchdog.StartThread(
Heartbeat, string.Format("Heartbeat ({0})", RegionInfo.RegionName), ThreadPriority.Normal, false);
Heartbeat, string.Format("Heartbeat ({0})", RegionInfo.RegionName), ThreadPriority.Normal, false, false);
}
/// <summary>
@ -1178,6 +1178,13 @@ namespace OpenSim.Region.Framework.Scenes
try
{
m_eventManager.TriggerOnRegionStarted(this);
// The first frame can take a very long time due to physics actors being added on startup. Therefore,
// don't turn on the watchdog alarm for this thread until the second frame, in order to prevent false
// alarms for scenes with many objects.
Update();
Watchdog.GetCurrentThreadInfo().AlarmIfTimeout = true;
while (!shuttingdown)
Update();
@ -1206,7 +1213,7 @@ namespace OpenSim.Region.Framework.Scenes
++Frame;
// m_log.DebugFormat("[SCENE]: Processing frame {0}", Frame);
// m_log.DebugFormat("[SCENE]: Processing frame {0} in {1}", Frame, RegionInfo.RegionName);
try
{
@ -1418,7 +1425,6 @@ namespace OpenSim.Region.Framework.Scenes
entry.checkAtTargets();
}
/// <summary>
/// Send out simstats data to all clients
/// </summary>

2
OpenSim/Region/OptionalModules/Agent/InternetRelayClientView/Server/IRCClientView.cs
View File

@ -70,7 +70,7 @@ namespace OpenSim.Region.OptionalModules.Agent.InternetRelayClientView.Server
m_client = client;
m_scene = scene;
Watchdog.StartThread(InternalLoop, "IRCClientView", ThreadPriority.Normal, false);
Watchdog.StartThread(InternalLoop, "IRCClientView", ThreadPriority.Normal, false, true);
}
private void SendServerCommand(string command)

2
OpenSim/Region/OptionalModules/Agent/InternetRelayClientView/Server/IRCServer.cs
View File

@ -57,7 +57,7 @@ namespace OpenSim.Region.OptionalModules.Agent.InternetRelayClientView.Server
m_listener.Start(50);
Watchdog.StartThread(ListenLoop, "IRCServer", ThreadPriority.Normal, false);
Watchdog.StartThread(ListenLoop, "IRCServer", ThreadPriority.Normal, false, true);
m_baseScene = baseScene;
}

2
OpenSim/Region/Physics/OdePlugin/ODEPrim.cs
View File

@ -1474,6 +1474,8 @@ Console.WriteLine("CreateGeom:");
/// </summary>
private void changeadd()
{
// m_log.DebugFormat("[ODE PRIM]: Adding prim {0}", Name);
int[] iprimspaceArrItem = _parent_scene.calculateSpaceArrayItemFromPos(_position);
IntPtr targetspace = _parent_scene.calculateSpaceForGeom(_position);

4
OpenSim/Region/ScriptEngine/Shared/Api/Implementation/AsyncCommandManager.cs
View File

@ -137,7 +137,9 @@ namespace OpenSim.Region.ScriptEngine.Shared.Api
if (cmdHandlerThread == null)
{
// Start the thread that will be doing the work
cmdHandlerThread = Watchdog.StartThread(CmdHandlerThreadLoop, "AsyncLSLCmdHandlerThread", ThreadPriority.Normal, true);
cmdHandlerThread
= Watchdog.StartThread(
CmdHandlerThreadLoop, "AsyncLSLCmdHandlerThread", ThreadPriority.Normal, true, true);
}
}