In Watchdog, add ability to specific timeout for a thread.
This also changes the point of registration to the StartThread() call rather than the first Update()remove-scene-viewer
parent
e549f2f443
commit
182908d216
|
@ -40,18 +40,30 @@ namespace OpenSim.Framework
|
||||||
{
|
{
|
||||||
/// <summary>Timer interval in milliseconds for the watchdog timer</summary>
|
/// <summary>Timer interval in milliseconds for the watchdog timer</summary>
|
||||||
const double WATCHDOG_INTERVAL_MS = 2500.0d;
|
const double WATCHDOG_INTERVAL_MS = 2500.0d;
|
||||||
|
|
||||||
/// <summary>Maximum timeout in milliseconds before a thread is considered dead</summary>
|
/// <summary>Maximum timeout in milliseconds before a thread is considered dead</summary>
|
||||||
const int WATCHDOG_TIMEOUT_MS = 5000;
|
const int WATCHDOG_TIMEOUT_MS = 5000;
|
||||||
|
|
||||||
[System.Diagnostics.DebuggerDisplay("{Thread.Name}")]
|
[System.Diagnostics.DebuggerDisplay("{Thread.Name}")]
|
||||||
public class ThreadWatchdogInfo
|
public class ThreadWatchdogInfo
|
||||||
{
|
{
|
||||||
public Thread Thread;
|
public Thread Thread { get; private set; }
|
||||||
public int LastTick;
|
public int LastTick { get; set; }
|
||||||
|
|
||||||
public ThreadWatchdogInfo(Thread thread)
|
/// <summary>
|
||||||
|
/// Number of seconds before we notify that the thread is having a problem.
|
||||||
|
/// </summary>
|
||||||
|
public int Timeout { get; set; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Is this thread considered timed out?
|
||||||
|
/// </summary>
|
||||||
|
public bool IsTimedOut { get; set; }
|
||||||
|
|
||||||
|
public ThreadWatchdogInfo(Thread thread, int timeout)
|
||||||
{
|
{
|
||||||
Thread = thread;
|
Thread = thread;
|
||||||
|
Timeout = timeout;
|
||||||
LastTick = Environment.TickCount & Int32.MaxValue;
|
LastTick = Environment.TickCount & Int32.MaxValue;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -82,7 +94,7 @@ namespace OpenSim.Framework
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Start a new thread that is tracked by the watchdog timer
|
/// Start a new thread that is tracked by the watchdog timer.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="start">The method that will be executed in a new thread</param>
|
/// <param name="start">The method that will be executed in a new thread</param>
|
||||||
/// <param name="name">A name to give to the new thread</param>
|
/// <param name="name">A name to give to the new thread</param>
|
||||||
|
@ -91,6 +103,24 @@ namespace OpenSim.Framework
|
||||||
/// thread, otherwise false</param>
|
/// thread, otherwise false</param>
|
||||||
/// <returns>The newly created Thread object</returns>
|
/// <returns>The newly created Thread object</returns>
|
||||||
public static Thread StartThread(ThreadStart start, string name, ThreadPriority priority, bool isBackground)
|
public static Thread StartThread(ThreadStart start, string name, ThreadPriority priority, bool isBackground)
|
||||||
|
{
|
||||||
|
return StartThread(start, name, priority, isBackground, WATCHDOG_TIMEOUT_MS);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Start a new thread that is tracked by the watchdog timer
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="start">The method that will be executed in a new thread</param>
|
||||||
|
/// <param name="name">A name to give to the new thread</param>
|
||||||
|
/// <param name="priority">Priority to run the thread at</param>
|
||||||
|
/// <param name="isBackground">True to run this thread as a background
|
||||||
|
/// thread, otherwise false</param>
|
||||||
|
/// <param name="timeout">
|
||||||
|
/// Number of seconds to wait until we issue a warning about timeout.
|
||||||
|
/// </para>
|
||||||
|
/// <returns>The newly created Thread object</returns>
|
||||||
|
public static Thread StartThread(
|
||||||
|
ThreadStart start, string name, ThreadPriority priority, bool isBackground, int timeout)
|
||||||
{
|
{
|
||||||
Thread thread = new Thread(start);
|
Thread thread = new Thread(start);
|
||||||
thread.Name = name;
|
thread.Name = name;
|
||||||
|
@ -98,6 +128,13 @@ namespace OpenSim.Framework
|
||||||
thread.IsBackground = isBackground;
|
thread.IsBackground = isBackground;
|
||||||
thread.Start();
|
thread.Start();
|
||||||
|
|
||||||
|
ThreadWatchdogInfo twi = new ThreadWatchdogInfo(thread, timeout);
|
||||||
|
|
||||||
|
m_log.Debug("[WATCHDOG]: Started tracking thread \"" + twi.Thread.Name + "\" (ID " + twi.Thread.ManagedThreadId + ")");
|
||||||
|
|
||||||
|
lock (m_threads)
|
||||||
|
m_threads.Add(twi.Thread.ManagedThreadId, twi);
|
||||||
|
|
||||||
return thread;
|
return thread;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -121,14 +158,6 @@ namespace OpenSim.Framework
|
||||||
return RemoveThread(Thread.CurrentThread.ManagedThreadId);
|
return RemoveThread(Thread.CurrentThread.ManagedThreadId);
|
||||||
}
|
}
|
||||||
|
|
||||||
private static void AddThread(ThreadWatchdogInfo threadInfo)
|
|
||||||
{
|
|
||||||
m_log.Debug("[WATCHDOG]: Started tracking thread \"" + threadInfo.Thread.Name + "\" (ID " + threadInfo.Thread.ManagedThreadId + ")");
|
|
||||||
|
|
||||||
lock (m_threads)
|
|
||||||
m_threads.Add(threadInfo.Thread.ManagedThreadId, threadInfo);
|
|
||||||
}
|
|
||||||
|
|
||||||
private static bool RemoveThread(int threadID)
|
private static bool RemoveThread(int threadID)
|
||||||
{
|
{
|
||||||
lock (m_threads)
|
lock (m_threads)
|
||||||
|
@ -165,9 +194,14 @@ namespace OpenSim.Framework
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
if (m_threads.TryGetValue(threadID, out threadInfo))
|
if (m_threads.TryGetValue(threadID, out threadInfo))
|
||||||
|
{
|
||||||
threadInfo.LastTick = Environment.TickCount & Int32.MaxValue;
|
threadInfo.LastTick = Environment.TickCount & Int32.MaxValue;
|
||||||
|
threadInfo.IsTimedOut = false;
|
||||||
|
}
|
||||||
else
|
else
|
||||||
AddThread(new ThreadWatchdogInfo(Thread.CurrentThread));
|
{
|
||||||
|
m_log.WarnFormat("[WATCHDOG]: Asked to update thread {0} which is not being monitored", threadID);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
catch { }
|
catch { }
|
||||||
}
|
}
|
||||||
|
@ -196,10 +230,16 @@ namespace OpenSim.Framework
|
||||||
|
|
||||||
foreach (ThreadWatchdogInfo threadInfo in m_threads.Values)
|
foreach (ThreadWatchdogInfo threadInfo in m_threads.Values)
|
||||||
{
|
{
|
||||||
if (threadInfo.Thread.ThreadState == ThreadState.Stopped || now - threadInfo.LastTick >= WATCHDOG_TIMEOUT_MS)
|
if (threadInfo.Thread.ThreadState == ThreadState.Stopped)
|
||||||
{
|
{
|
||||||
timedOut = threadInfo;
|
timedOut = threadInfo;
|
||||||
m_threads.Remove(threadInfo.Thread.ManagedThreadId);
|
RemoveThread(threadInfo.Thread.ManagedThreadId);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
else if (!threadInfo.IsTimedOut && now - threadInfo.LastTick >= threadInfo.Timeout)
|
||||||
|
{
|
||||||
|
threadInfo.IsTimedOut = true;
|
||||||
|
timedOut = threadInfo;
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue