diff --git a/OpenSim/Framework/Servers/HttpServer/PollServiceRequestManager.cs b/OpenSim/Framework/Servers/HttpServer/PollServiceRequestManager.cs
index 0062d4ef15..f96fd1fa20 100644
--- a/OpenSim/Framework/Servers/HttpServer/PollServiceRequestManager.cs
+++ b/OpenSim/Framework/Servers/HttpServer/PollServiceRequestManager.cs
@@ -66,6 +66,7 @@ namespace OpenSim.Framework.Servers.HttpServer
ThreadPriority.Normal,
false,
true,
+ null,
int.MaxValue);
}
@@ -75,6 +76,7 @@ namespace OpenSim.Framework.Servers.HttpServer
ThreadPriority.Normal,
false,
true,
+ null,
1000 * 60 * 10);
}
diff --git a/OpenSim/Framework/Watchdog.cs b/OpenSim/Framework/Watchdog.cs
index e93e50e2ca..7552cd15b6 100644
--- a/OpenSim/Framework/Watchdog.cs
+++ b/OpenSim/Framework/Watchdog.cs
@@ -42,7 +42,7 @@ namespace OpenSim.Framework
const double WATCHDOG_INTERVAL_MS = 2500.0d;
/// Maximum timeout in milliseconds before a thread is considered dead
- const int WATCHDOG_TIMEOUT_MS = 5000;
+ public const int WATCHDOG_TIMEOUT_MS = 5000;
[System.Diagnostics.DebuggerDisplay("{Thread.Name}")]
public class ThreadWatchdogInfo
@@ -58,7 +58,7 @@ namespace OpenSim.Framework
public int FirstTick { get; private set; }
///
- /// First time this heartbeat update was invoked
+ /// Last time this heartbeat update was invoked
///
public int LastTick { get; set; }
@@ -77,6 +77,11 @@ namespace OpenSim.Framework
///
public bool AlarmIfTimeout { get; set; }
+ ///
+ /// Method execute if alarm goes off. If null then no alarm method is fired.
+ ///
+ public Func AlarmMethod { get; set; }
+
public ThreadWatchdogInfo(Thread thread, int timeout)
{
Thread = thread;
@@ -87,16 +92,10 @@ namespace OpenSim.Framework
}
///
- /// This event is called whenever a tracked thread is stopped or
- /// has not called UpdateThread() in time
- ///
- /// The thread that has been identified as dead
- /// The last time this thread called UpdateThread()
- public delegate void WatchdogTimeout(Thread thread, int lastTick);
-
- /// This event is called whenever a tracked thread is
- /// stopped or has not called UpdateThread() in time
- public static event WatchdogTimeout OnWatchdogTimeout;
+ /// This event is called whenever a tracked thread is
+ /// stopped or has not called UpdateThread() in time<
+ /// /summary>
+ public static event Action OnWatchdogTimeout;
private static readonly ILog m_log = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType);
private static Dictionary m_threads;
@@ -123,7 +122,7 @@ namespace OpenSim.Framework
public static Thread StartThread(
ThreadStart start, string name, ThreadPriority priority, bool isBackground, bool alarmIfTimeout)
{
- return StartThread(start, name, priority, isBackground, alarmIfTimeout, WATCHDOG_TIMEOUT_MS);
+ return StartThread(start, name, priority, isBackground, alarmIfTimeout, null, WATCHDOG_TIMEOUT_MS);
}
///
@@ -135,17 +134,24 @@ namespace OpenSim.Framework
/// True to run this thread as a background
/// thread, otherwise false
/// Trigger an alarm function is we have timed out
+ ///
+ /// Alarm method to call if alarmIfTimeout is true and there is a timeout.
+ /// Normally, this will just return some useful debugging information.
+ ///
/// Number of milliseconds to wait until we issue a warning about timeout.
/// The newly created Thread object
public static Thread StartThread(
- ThreadStart start, string name, ThreadPriority priority, bool isBackground, bool alarmIfTimeout, int timeout)
+ ThreadStart start, string name, ThreadPriority priority, bool isBackground,
+ bool alarmIfTimeout, Func alarmMethod, int timeout)
{
Thread thread = new Thread(start);
thread.Name = name;
thread.Priority = priority;
thread.IsBackground = isBackground;
- ThreadWatchdogInfo twi = new ThreadWatchdogInfo(thread, timeout) { AlarmIfTimeout = alarmIfTimeout };
+ ThreadWatchdogInfo twi
+ = new ThreadWatchdogInfo(thread, timeout)
+ { AlarmIfTimeout = alarmIfTimeout, AlarmMethod = alarmMethod };
m_log.DebugFormat(
"[WATCHDOG]: Started tracking thread {0}, ID {1}", twi.Thread.Name, twi.Thread.ManagedThreadId);
@@ -258,7 +264,7 @@ namespace OpenSim.Framework
///
private static void WatchdogTimerElapsed(object sender, System.Timers.ElapsedEventArgs e)
{
- WatchdogTimeout callback = OnWatchdogTimeout;
+ Action callback = OnWatchdogTimeout;
if (callback != null)
{
@@ -296,7 +302,7 @@ namespace OpenSim.Framework
if (callbackInfos != null)
foreach (ThreadWatchdogInfo callbackInfo in callbackInfos)
- callback(callbackInfo.Thread, callbackInfo.LastTick);
+ callback(callbackInfo);
}
m_watchdogTimer.Start();
diff --git a/OpenSim/Region/Application/OpenSim.cs b/OpenSim/Region/Application/OpenSim.cs
index ca0432fb6c..c3590abba0 100644
--- a/OpenSim/Region/Application/OpenSim.cs
+++ b/OpenSim/Region/Application/OpenSim.cs
@@ -438,12 +438,16 @@ namespace OpenSim
}
}
- private void WatchdogTimeoutHandler(System.Threading.Thread thread, int lastTick)
+ private void WatchdogTimeoutHandler(Watchdog.ThreadWatchdogInfo twi)
{
int now = Environment.TickCount & Int32.MaxValue;
- m_log.ErrorFormat("[WATCHDOG]: Timeout detected for thread \"{0}\". ThreadState={1}. Last tick was {2}ms ago",
- thread.Name, thread.ThreadState, now - lastTick);
+ m_log.ErrorFormat(
+ "[WATCHDOG]: Timeout detected for thread \"{0}\". ThreadState={1}. Last tick was {2}ms ago. {3}",
+ twi.Thread.Name,
+ twi.Thread.ThreadState,
+ now - twi.LastTick,
+ twi.AlarmMethod != null ? string.Format("Data: {0}", twi.AlarmMethod()) : "");
}
#region Console Commands
diff --git a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs
index 32ba590718..e37adb86b8 100644
--- a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs
+++ b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs
@@ -163,6 +163,16 @@ namespace OpenSim.Region.ClientStack.LindenUDP
private int m_malformedCount = 0; // Guard against a spamming attack
+ ///
+ /// Record current outgoing client for monitoring purposes.
+ ///
+ private IClientAPI m_currentOutgoingClient;
+
+ ///
+ /// Recording current incoming client for monitoring purposes.
+ ///
+ private IClientAPI m_currentIncomingClient;
+
public LLUDPServer(IPAddress listenIP, ref uint port, int proxyPortOffsetParm, bool allow_alternate_port, IConfigSource configSource, AgentCircuitManager circuitManager)
: base(listenIP, (int)port)
{
@@ -244,19 +254,56 @@ namespace OpenSim.Region.ClientStack.LindenUDP
if (m_scene == null)
throw new InvalidOperationException("[LLUDPSERVER]: Cannot LLUDPServer.Start() without an IScene reference");
- m_log.Info("[LLUDPSERVER]: Starting the LLUDP server in " + (m_asyncPacketHandling ? "asynchronous" : "synchronous") + " mode");
+ m_log.InfoFormat(
+ "[LLUDPSERVER]: Starting the LLUDP server in {0} mode",
+ m_asyncPacketHandling ? "asynchronous" : "synchronous");
base.Start(m_recvBufferSize, m_asyncPacketHandling);
// Start the packet processing threads
Watchdog.StartThread(
- IncomingPacketHandler, "Incoming Packets (" + m_scene.RegionInfo.RegionName + ")", ThreadPriority.Normal, false, true);
+ IncomingPacketHandler,
+ string.Format("Incoming Packets ({0})", m_scene.RegionInfo.RegionName),
+ ThreadPriority.Normal,
+ false,
+ true,
+ GetWatchdogIncomingAlarmData,
+ Watchdog.WATCHDOG_TIMEOUT_MS);
+
Watchdog.StartThread(
- OutgoingPacketHandler, "Outgoing Packets (" + m_scene.RegionInfo.RegionName + ")", ThreadPriority.Normal, false, true);
+ OutgoingPacketHandler,
+ string.Format("Outgoing Packets ({0})", m_scene.RegionInfo.RegionName),
+ ThreadPriority.Normal,
+ false,
+ true,
+ GetWatchdogOutgoingAlarmData,
+ Watchdog.WATCHDOG_TIMEOUT_MS);
m_elapsedMSSinceLastStatReport = Environment.TickCount;
}
+ ///
+ /// If the outgoing UDP thread times out, then return client that was being processed to help with debugging.
+ ///
+ ///
+ private string GetWatchdogIncomingAlarmData()
+ {
+ return string.Format(
+ "Client is {0}",
+ m_currentIncomingClient != null ? m_currentIncomingClient.Name : "none");
+ }
+
+ ///
+ /// If the outgoing UDP thread times out, then return client that was being processed to help with debugging.
+ ///
+ ///
+ private string GetWatchdogOutgoingAlarmData()
+ {
+ return string.Format(
+ "Client is {0}",
+ m_currentOutgoingClient != null ? m_currentOutgoingClient.Name : "none");
+ }
+
public new void Stop()
{
m_log.Info("[LLUDPSERVER]: Shutting down the LLUDP server for " + m_scene.RegionInfo.RegionName);
@@ -1173,6 +1220,8 @@ namespace OpenSim.Region.ClientStack.LindenUDP
// client. m_packetSent will be set to true if a packet is sent
m_scene.ForEachClient(clientPacketHandler);
+ m_currentOutgoingClient = null;
+
// If nothing was sent, sleep for the minimum amount of time before a
// token bucket could get more tokens
if (!m_packetSent)
@@ -1191,6 +1240,8 @@ namespace OpenSim.Region.ClientStack.LindenUDP
private void ClientOutgoingPacketHandler(IClientAPI client)
{
+ m_currentOutgoingClient = client;
+
try
{
if (client is LLClientView)
@@ -1216,8 +1267,8 @@ namespace OpenSim.Region.ClientStack.LindenUDP
}
catch (Exception ex)
{
- m_log.Error("[LLUDPSERVER]: OutgoingPacketHandler iteration for " + client.Name +
- " threw an exception: " + ex.Message, ex);
+ m_log.Error(
+ string.Format("[LLUDPSERVER]: OutgoingPacketHandler iteration for {0} threw ", client.Name), ex);
}
}
@@ -1243,6 +1294,8 @@ namespace OpenSim.Region.ClientStack.LindenUDP
{
nticks++;
watch1.Start();
+ m_currentOutgoingClient = client;
+
try
{
if (client is LLClientView)
@@ -1344,6 +1397,8 @@ namespace OpenSim.Region.ClientStack.LindenUDP
// Make sure this client is still alive
if (m_scene.TryGetClient(udpClient.AgentID, out client))
{
+ m_currentIncomingClient = client;
+
try
{
// Process this packet
@@ -1361,6 +1416,10 @@ namespace OpenSim.Region.ClientStack.LindenUDP
m_log.ErrorFormat("[LLUDPSERVER]: Client packet handler for {0} for packet {1} threw an exception", udpClient.AgentID, packet.Type);
m_log.Error(e.Message, e);
}
+ finally
+ {
+ m_currentIncomingClient = null;
+ }
}
else
{