diff --git a/OpenSim/Addons/Groups/Hypergrid/GroupsServiceHGConnectorModule.cs b/OpenSim/Addons/Groups/Hypergrid/GroupsServiceHGConnectorModule.cs
index a750d8d870..9fbc1b3501 100644
--- a/OpenSim/Addons/Groups/Hypergrid/GroupsServiceHGConnectorModule.cs
+++ b/OpenSim/Addons/Groups/Hypergrid/GroupsServiceHGConnectorModule.cs
@@ -32,6 +32,7 @@ using System.Reflection;
using System.Text;
using OpenSim.Framework;
+using OpenSim.Framework.Monitoring;
using OpenSim.Framework.Servers;
using OpenSim.Region.Framework.Scenes;
using OpenSim.Region.Framework.Interfaces;
@@ -560,7 +561,7 @@ namespace OpenSim.Groups
// so we have the list of urls to send the notice to
// this may take a long time...
- Util.RunThreadNoTimeout(delegate
+ Watchdog.RunInThread(delegate
{
foreach (string u in urls)
{
@@ -571,7 +572,7 @@ namespace OpenSim.Groups
hasAttachment, attType, attName, attItemID, AgentUUIForOutside(attOwnerID));
}
}
- }, "AddGroupNotice", null);
+ }, string.Format("AddGroupNotice (agent {0}, group {1})", RequestingAgentID, groupID) , null);
return true;
}
diff --git a/OpenSim/Framework/Monitoring/Watchdog.cs b/OpenSim/Framework/Monitoring/Watchdog.cs
index 427ed7bb0d..e9e7bd21c0 100644
--- a/OpenSim/Framework/Monitoring/Watchdog.cs
+++ b/OpenSim/Framework/Monitoring/Watchdog.cs
@@ -187,15 +187,16 @@ namespace OpenSim.Framework.Monitoring
/// Priority to run the thread at
/// True to run this thread as a background thread, otherwise false
/// Trigger an alarm function is we have timed out
+ /// If true then creation of thread is logged.
/// The newly created Thread object
public static Thread StartThread(
- ThreadStart start, string name, ThreadPriority priority, bool isBackground, bool alarmIfTimeout)
+ ThreadStart start, string name, ThreadPriority priority, bool isBackground, bool alarmIfTimeout, bool log = true)
{
- return StartThread(start, name, priority, isBackground, alarmIfTimeout, null, DEFAULT_WATCHDOG_TIMEOUT_MS);
+ return StartThread(start, name, priority, isBackground, alarmIfTimeout, null, DEFAULT_WATCHDOG_TIMEOUT_MS, log);
}
///
- /// Start a new thread that is tracked by the watchdog timer
+ /// Start a new thread that is tracked by the watchdog
///
/// The method that will be executed in a new thread
/// A name to give to the new thread
@@ -208,10 +209,11 @@ namespace OpenSim.Framework.Monitoring
/// Normally, this will just return some useful debugging information.
///
/// Number of milliseconds to wait until we issue a warning about timeout.
+ /// If true then creation of thread is logged.
/// The newly created Thread object
public static Thread StartThread(
ThreadStart start, string name, ThreadPriority priority, bool isBackground,
- bool alarmIfTimeout, Func alarmMethod, int timeout)
+ bool alarmIfTimeout, Func alarmMethod, int timeout, bool log = true)
{
Thread thread = new Thread(start);
thread.Name = name;
@@ -222,8 +224,9 @@ namespace OpenSim.Framework.Monitoring
= new ThreadWatchdogInfo(thread, timeout)
{ AlarmIfTimeout = alarmIfTimeout, AlarmMethod = alarmMethod };
- m_log.DebugFormat(
- "[WATCHDOG]: Started tracking thread {0}, ID {1}", twi.Thread.Name, twi.Thread.ManagedThreadId);
+ if (log)
+ m_log.DebugFormat(
+ "[WATCHDOG]: Started tracking thread {0}, ID {1}", twi.Thread.Name, twi.Thread.ManagedThreadId);
lock (m_threads)
m_threads.Add(twi.Thread.ManagedThreadId, twi);
@@ -233,6 +236,39 @@ namespace OpenSim.Framework.Monitoring
return thread;
}
+ ///
+ /// Run the callback in a new thread immediately. If the thread exits with an exception log it but do
+ /// not propogate it.
+ ///
+ /// Code for the thread to execute.
+ /// Name of the thread
+ /// Object to pass to the thread.
+ public static void RunInThread(WaitCallback callback, string name, object obj, bool log = false)
+ {
+ if (Util.FireAndForgetMethod == FireAndForgetMethod.RegressionTest)
+ {
+ Culture.SetCurrentCulture();
+ callback(obj);
+ return;
+ }
+
+ ThreadStart ts = new ThreadStart(delegate()
+ {
+ try
+ {
+ Culture.SetCurrentCulture();
+ callback(obj);
+ Watchdog.RemoveThread(log:false);
+ }
+ catch (Exception e)
+ {
+ m_log.Error(string.Format("[WATCHDOG]: Exception in thread {0}.", name), e);
+ }
+ });
+
+ StartThread(ts, name, ThreadPriority.Normal, true, false, log:log);
+ }
+
///
/// Marks the current thread as alive
///
@@ -244,24 +280,26 @@ namespace OpenSim.Framework.Monitoring
///
/// Stops watchdog tracking on the current thread
///
+ /// If true then normal events in thread removal are not logged.
///
/// True if the thread was removed from the list of tracked
/// threads, otherwise false
///
- public static bool RemoveThread()
+ public static bool RemoveThread(bool log = true)
{
- return RemoveThread(Thread.CurrentThread.ManagedThreadId);
+ return RemoveThread(Thread.CurrentThread.ManagedThreadId, log);
}
- private static bool RemoveThread(int threadID)
+ private static bool RemoveThread(int threadID, bool log = true)
{
lock (m_threads)
{
ThreadWatchdogInfo twi;
if (m_threads.TryGetValue(threadID, out twi))
{
- m_log.DebugFormat(
- "[WATCHDOG]: Removing thread {0}, ID {1}", twi.Thread.Name, twi.Thread.ManagedThreadId);
+ if (log)
+ m_log.DebugFormat(
+ "[WATCHDOG]: Removing thread {0}, ID {1}", twi.Thread.Name, twi.Thread.ManagedThreadId);
twi.Cleanup();
m_threads.Remove(threadID);
diff --git a/OpenSim/Framework/Util.cs b/OpenSim/Framework/Util.cs
index 2c38571147..fefa05039a 100644
--- a/OpenSim/Framework/Util.cs
+++ b/OpenSim/Framework/Util.cs
@@ -2399,36 +2399,6 @@ namespace OpenSim.Framework
#endregion FireAndForget Threading Pattern
- ///
- /// Run the callback on a different thread, outside the thread pool. This is used for tasks
- /// that may take a long time.
- ///
- public static void RunThreadNoTimeout(WaitCallback callback, string name, object obj)
- {
- if (FireAndForgetMethod == FireAndForgetMethod.RegressionTest)
- {
- Culture.SetCurrentCulture();
- callback(obj);
- return;
- }
-
- Thread t = new Thread(delegate()
- {
- try
- {
- Culture.SetCurrentCulture();
- callback(obj);
- }
- catch (Exception e)
- {
- m_log.Error("Exception in thread " + name, e);
- }
- });
-
- t.Name = name;
- t.Start();
- }
-
///
/// Environment.TickCount is an int but it counts all 32 bits so it goes positive
/// and negative every 24.9 days. This trims down TickCount so it doesn't wrap
diff --git a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs
index 87aa37dad1..e879e7636f 100644
--- a/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs
+++ b/OpenSim/Region/ClientStack/Linden/UDP/LLUDPServer.cs
@@ -447,6 +447,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP
{
StartInbound();
StartOutbound();
+ OqrEngine.Start();
m_elapsedMSSinceLastStatReport = Environment.TickCount;
}
@@ -491,6 +492,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP
m_log.Info("[LLUDPSERVER]: Shutting down the LLUDP server for " + Scene.Name);
base.StopOutbound();
base.StopInbound();
+ OqrEngine.Stop();
}
protected override bool EnablePools()
diff --git a/OpenSim/Region/ClientStack/Linden/UDP/OutgoingQueueRefillEngine.cs b/OpenSim/Region/ClientStack/Linden/UDP/OutgoingQueueRefillEngine.cs
index 0659d8e47e..069c9c8162 100644
--- a/OpenSim/Region/ClientStack/Linden/UDP/OutgoingQueueRefillEngine.cs
+++ b/OpenSim/Region/ClientStack/Linden/UDP/OutgoingQueueRefillEngine.cs
@@ -92,7 +92,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP
"debug lludp oqre",
"debug lludp oqre ",
"Start, stop or get status of OutgoingQueueRefillEngine.",
- "Experimental.",
+ "If stopped then refill requests are processed directly via the threadpool.",
HandleOqreCommand);
}
diff --git a/OpenSim/Region/CoreModules/Asset/FlotsamAssetCache.cs b/OpenSim/Region/CoreModules/Asset/FlotsamAssetCache.cs
index f06d70d9e0..9d6870f1b5 100644
--- a/OpenSim/Region/CoreModules/Asset/FlotsamAssetCache.cs
+++ b/OpenSim/Region/CoreModules/Asset/FlotsamAssetCache.cs
@@ -43,6 +43,7 @@ using Mono.Addins;
using OpenMetaverse;
using OpenSim.Framework;
using OpenSim.Framework.Console;
+using OpenSim.Framework.Monitoring;
using OpenSim.Region.Framework.Interfaces;
using OpenSim.Region.Framework.Scenes;
using OpenSim.Services.Interfaces;
@@ -963,7 +964,8 @@ namespace OpenSim.Region.CoreModules.Asset
case "assets":
con.Output("Ensuring assets are cached for all scenes.");
- Util.RunThreadNoTimeout(delegate {
+ Watchdog.RunInThread(delegate
+ {
int assetReferenceTotal = TouchAllSceneAssets(true);
con.OutputFormat("Completed check with {0} assets.", assetReferenceTotal);
}, "TouchAllSceneAssets", null);
diff --git a/OpenSim/Region/CoreModules/Avatar/Inventory/Archiver/InventoryArchiveWriteRequest.cs b/OpenSim/Region/CoreModules/Avatar/Inventory/Archiver/InventoryArchiveWriteRequest.cs
index d15703eb73..a5b5aa24c7 100644
--- a/OpenSim/Region/CoreModules/Avatar/Inventory/Archiver/InventoryArchiveWriteRequest.cs
+++ b/OpenSim/Region/CoreModules/Avatar/Inventory/Archiver/InventoryArchiveWriteRequest.cs
@@ -34,6 +34,7 @@ using System.Xml;
using log4net;
using OpenMetaverse;
using OpenSim.Framework;
+using OpenSim.Framework.Monitoring;
using OpenSim.Framework.Serialization;
using OpenSim.Framework.Serialization.External;
using OpenSim.Region.CoreModules.World.Archiver;
@@ -356,7 +357,7 @@ namespace OpenSim.Region.CoreModules.Avatar.Inventory.Archiver
m_scene.UserAccountService, m_scene.RegionInfo.ScopeID,
options, ReceivedAllAssets);
- Util.RunThreadNoTimeout(o => ar.Execute(), "AssetsRequest", null);
+ Watchdog.RunInThread(o => ar.Execute(), string.Format("AssetsRequest ({0})", m_scene.Name), null);
}
else
{
diff --git a/OpenSim/Region/CoreModules/ServiceConnectorsOut/Inventory/LocalInventoryServiceConnector.cs b/OpenSim/Region/CoreModules/ServiceConnectorsOut/Inventory/LocalInventoryServiceConnector.cs
index f80fab3e1d..470ef0274d 100644
--- a/OpenSim/Region/CoreModules/ServiceConnectorsOut/Inventory/LocalInventoryServiceConnector.cs
+++ b/OpenSim/Region/CoreModules/ServiceConnectorsOut/Inventory/LocalInventoryServiceConnector.cs
@@ -33,6 +33,7 @@ using System;
using System.Collections.Generic;
using System.Reflection;
using OpenSim.Framework;
+using OpenSim.Framework.Monitoring;
using OpenSim.Data;
using OpenSim.Server.Base;
using OpenSim.Region.Framework.Interfaces;
@@ -183,12 +184,12 @@ namespace OpenSim.Region.CoreModules.ServiceConnectorsOut.Inventory
// Protect ourselves against the caller subsequently modifying the items list
List items = new List(invCol.Items);
- Util.RunThreadNoTimeout(delegate
+ Watchdog.RunInThread(delegate
{
foreach (InventoryItemBase item in items)
if (!string.IsNullOrEmpty(item.CreatorData))
UserManager.AddUser(item.CreatorIdAsUuid, item.CreatorData);
- }, "GetFolderContent", null);
+ }, string.Format("GetFolderContent (user {0}, folder {1})", userID, folderID), null);
}
return invCol;
diff --git a/OpenSim/Region/CoreModules/World/Archiver/ArchiveReadRequest.cs b/OpenSim/Region/CoreModules/World/Archiver/ArchiveReadRequest.cs
index c8056d425d..3e0c9f30b0 100644
--- a/OpenSim/Region/CoreModules/World/Archiver/ArchiveReadRequest.cs
+++ b/OpenSim/Region/CoreModules/World/Archiver/ArchiveReadRequest.cs
@@ -36,6 +36,7 @@ using System.Xml;
using log4net;
using OpenMetaverse;
using OpenSim.Framework;
+using OpenSim.Framework.Monitoring;
using OpenSim.Framework.Serialization;
using OpenSim.Framework.Serialization.External;
using OpenSim.Region.CoreModules.World.Terrain;
@@ -371,7 +372,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver
// Start the scripts. We delayed this because we want the OAR to finish loading ASAP, so
// that users can enter the scene. If we allow the scripts to start in the loop above
// then they significantly increase the time until the OAR finishes loading.
- Util.RunThreadNoTimeout(delegate(object o)
+ Watchdog.RunInThread(o =>
{
Thread.Sleep(15000);
m_log.Info("[ARCHIVER]: Starting scripts in scene objects");
@@ -386,7 +387,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver
sceneContext.SceneObjects.Clear();
}
- }, "ReadArchiveStartScripts", null);
+ }, string.Format("ReadArchiveStartScripts (request {0})", m_requestId), null);
m_log.InfoFormat("[ARCHIVER]: Successfully loaded archive");
diff --git a/OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequest.cs b/OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequest.cs
index 448147ac7b..803f24e2a9 100644
--- a/OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequest.cs
+++ b/OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequest.cs
@@ -36,6 +36,7 @@ using System.Xml;
using log4net;
using OpenMetaverse;
using OpenSim.Framework;
+using OpenSim.Framework.Monitoring;
using OpenSim.Framework.Serialization;
using OpenSim.Region.CoreModules.World.Terrain;
using OpenSim.Region.Framework.Interfaces;
@@ -199,7 +200,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver
m_rootScene.AssetService, m_rootScene.UserAccountService,
m_rootScene.RegionInfo.ScopeID, options, ReceivedAllAssets);
- Util.RunThreadNoTimeout(o => ar.Execute(), "AssetsRequest", null);
+ Watchdog.RunInThread(o => ar.Execute(), "Archive Assets Request", null);
// CloseArchive() will be called from ReceivedAllAssets()
}
diff --git a/OpenSim/Region/CoreModules/World/Archiver/AssetsRequest.cs b/OpenSim/Region/CoreModules/World/Archiver/AssetsRequest.cs
index 4f428c35a6..ec39bc02af 100644
--- a/OpenSim/Region/CoreModules/World/Archiver/AssetsRequest.cs
+++ b/OpenSim/Region/CoreModules/World/Archiver/AssetsRequest.cs
@@ -33,6 +33,7 @@ using System.Timers;
using log4net;
using OpenMetaverse;
using OpenSim.Framework;
+using OpenSim.Framework.Monitoring;
using OpenSim.Framework.Serialization;
using OpenSim.Framework.Serialization.External;
using OpenSim.Services.Interfaces;
@@ -226,7 +227,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver
finally
{
if (timedOut)
- Util.RunThreadNoTimeout(PerformAssetsRequestCallback, "AssetsRequestCallback", true);
+ Watchdog.RunInThread(PerformAssetsRequestCallback, "Archive Assets Request Callback", true);
}
}
@@ -295,7 +296,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver
// We want to stop using the asset cache thread asap
// as we now need to do the work of producing the rest of the archive
- Util.RunThreadNoTimeout(PerformAssetsRequestCallback, "AssetsRequestCallback", false);
+ Watchdog.RunInThread(PerformAssetsRequestCallback, "Archive Assets Request Callback", false);
}
else
{
diff --git a/OpenSim/Region/Framework/Scenes/Scene.cs b/OpenSim/Region/Framework/Scenes/Scene.cs
index 5f0dbd7497..28dbccb95f 100644
--- a/OpenSim/Region/Framework/Scenes/Scene.cs
+++ b/OpenSim/Region/Framework/Scenes/Scene.cs
@@ -1623,7 +1623,12 @@ namespace OpenSim.Region.Framework.Scenes
{
tmpMS = Util.EnvironmentTickCount();
m_cleaningTemps = true;
- Util.RunThreadNoTimeout(delegate { CleanTempObjects(); m_cleaningTemps = false; }, "CleanTempObjects", null);
+
+ Watchdog.RunInThread(
+ delegate { CleanTempObjects(); m_cleaningTemps = false; },
+ string.Format("CleanTempObjects ({0})", Name),
+ null);
+
tempOnRezMS = Util.EnvironmentTickCountSubtract(tmpMS);
}
@@ -1803,7 +1808,7 @@ namespace OpenSim.Region.Framework.Scenes
if (!m_backingup)
{
m_backingup = true;
- Util.RunThreadNoTimeout(BackupWaitCallback, "BackupWaitCallback", null);
+ Watchdog.RunInThread(o => Backup(false), string.Format("BackupWaitCallback ({0})", Name), null);
}
}
@@ -1814,14 +1819,6 @@ namespace OpenSim.Region.Framework.Scenes
{
m_eventManager.TriggerOnFrame();
}
-
- ///
- /// Wrapper for Backup() that can be called with Util.RunThreadNoTimeout()
- ///
- private void BackupWaitCallback(object o)
- {
- Backup(false);
- }
///
/// Backup the scene.
diff --git a/OpenSim/Region/Framework/Scenes/ScenePresence.cs b/OpenSim/Region/Framework/Scenes/ScenePresence.cs
index 3c37de84ec..59b521db5f 100644
--- a/OpenSim/Region/Framework/Scenes/ScenePresence.cs
+++ b/OpenSim/Region/Framework/Scenes/ScenePresence.cs
@@ -37,6 +37,7 @@ using log4net;
using Nini.Config;
using OpenSim.Framework;
using OpenSim.Framework.Client;
+using OpenSim.Framework.Monitoring;
using OpenSim.Region.Framework.Interfaces;
using OpenSim.Region.Framework.Scenes.Animation;
using OpenSim.Region.Framework.Scenes.Types;
@@ -3362,7 +3363,7 @@ namespace OpenSim.Region.Framework.Scenes
SentInitialDataToClient = true;
// Send all scene object to the new client
- Util.RunThreadNoTimeout(delegate
+ Watchdog.RunInThread(delegate
{
// m_log.DebugFormat(
// "[SCENE PRESENCE]: Sending initial data to {0} agent {1} in {2}, tp flags {3}",
@@ -3380,8 +3381,7 @@ namespace OpenSim.Region.Framework.Scenes
if (e != null && e is SceneObjectGroup)
((SceneObjectGroup)e).SendFullUpdateToClient(ControllingClient);
}
-
- }, "SendInitialDataToClient", null);
+ }, string.Format("SendInitialDataToClient ({0} in {1})", Name, Scene.Name), null);
}
///
diff --git a/prebuild.xml b/prebuild.xml
index ba18a2c50f..a10d824053 100644
--- a/prebuild.xml
+++ b/prebuild.xml
@@ -2489,6 +2489,7 @@
+