OpenSim
/
OpenSimMirror
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

For monitoring purposes, start non-timeout tasks (which do not currently use a threadpool) via Watchdog.RunInThread() rather than Util.RunThreadNoTimeout() 

The functionality is the same but this allow us to monitor such tasks via "show threads" and abort them for test purposes, etc.
Also extends thread names to provide more info (e.g. SendInitialDataToClient says what client the task is for).
ghosts
Justin Clark-Casey (justincc) 2014-09-05 23:20:59 +01:00
parent e4d16e6795
commit 41f2f3132b
12 changed files with 79 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
OpenSim/Addons/Groups/Hypergrid/GroupsServiceHGConnectorModule.cs
View File

@ -32,6 +32,7 @@ using System.Reflection;
using System.Text;
using OpenSim.Framework;
using OpenSim.Framework.Monitoring;
using OpenSim.Framework.Servers;
using OpenSim.Region.Framework.Scenes;
using OpenSim.Region.Framework.Interfaces;
@ -560,7 +561,7 @@ namespace OpenSim.Groups
// so we have the list of urls to send the notice to
// this may take a long time...
Util.RunThreadNoTimeout(delegate
Watchdog.RunInThread(delegate
{
foreach (string u in urls)
{
@ -571,7 +572,7 @@ namespace OpenSim.Groups
hasAttachment, attType, attName, attItemID, AgentUUIForOutside(attOwnerID));
}
}
}, "AddGroupNotice", null);
}, string.Format("AddGroupNotice (agent {0}, group {1})", RequestingAgentID, groupID) , null);
return true;
}

60
OpenSim/Framework/Monitoring/Watchdog.cs
View File

@ -187,15 +187,16 @@ namespace OpenSim.Framework.Monitoring
/// <param name="priority">Priority to run the thread at</param>
/// <param name="isBackground">True to run this thread as a background thread, otherwise false</param>
/// <param name="alarmIfTimeout">Trigger an alarm function is we have timed out</param>
/// <param name="log">If true then creation of thread is logged.</param>
/// <returns>The newly created Thread object</returns>
public static Thread StartThread(
ThreadStart start, string name, ThreadPriority priority, bool isBackground, bool alarmIfTimeout)
ThreadStart start, string name, ThreadPriority priority, bool isBackground, bool alarmIfTimeout, bool log = true)
{
return StartThread(start, name, priority, isBackground, alarmIfTimeout, null, DEFAULT_WATCHDOG_TIMEOUT_MS);
return StartThread(start, name, priority, isBackground, alarmIfTimeout, null, DEFAULT_WATCHDOG_TIMEOUT_MS, log);
}
/// <summary>
/// Start a new thread that is tracked by the watchdog timer
/// Start a new thread that is tracked by the watchdog
/// </summary>
/// <param name="start">The method that will be executed in a new thread</param>
/// <param name="name">A name to give to the new thread</param>
@ -208,10 +209,11 @@ namespace OpenSim.Framework.Monitoring
/// Normally, this will just return some useful debugging information.
/// </param>
/// <param name="timeout">Number of milliseconds to wait until we issue a warning about timeout.</param>
/// <param name="log">If true then creation of thread is logged.</param>
/// <returns>The newly created Thread object</returns>
public static Thread StartThread(
ThreadStart start, string name, ThreadPriority priority, bool isBackground,
bool alarmIfTimeout, Func<string> alarmMethod, int timeout)
bool alarmIfTimeout, Func<string> alarmMethod, int timeout, bool log = true)
{
Thread thread = new Thread(start);
thread.Name = name;
@ -222,8 +224,9 @@ namespace OpenSim.Framework.Monitoring
= new ThreadWatchdogInfo(thread, timeout)
{ AlarmIfTimeout = alarmIfTimeout, AlarmMethod = alarmMethod };
m_log.DebugFormat(
"[WATCHDOG]: Started tracking thread {0}, ID {1}", twi.Thread.Name, twi.Thread.ManagedThreadId);
if (log)
m_log.DebugFormat(
"[WATCHDOG]: Started tracking thread {0}, ID {1}", twi.Thread.Name, twi.Thread.ManagedThreadId);
lock (m_threads)
m_threads.Add(twi.Thread.ManagedThreadId, twi);
@ -233,6 +236,39 @@ namespace OpenSim.Framework.Monitoring
return thread;
}
/// <summary>
/// Run the callback in a new thread immediately. If the thread exits with an exception log it but do
/// not propogate it.
/// </summary>
/// <param name="callback">Code for the thread to execute.</param>
/// <param name="name">Name of the thread</param>
/// <param name="obj">Object to pass to the thread.</param>
public static void RunInThread(WaitCallback callback, string name, object obj, bool log = false)
{
if (Util.FireAndForgetMethod == FireAndForgetMethod.RegressionTest)
{
Culture.SetCurrentCulture();
callback(obj);
return;
}
ThreadStart ts = new ThreadStart(delegate()
{
try
{
Culture.SetCurrentCulture();
callback(obj);
Watchdog.RemoveThread(log:false);
}
catch (Exception e)
{
m_log.Error(string.Format("[WATCHDOG]: Exception in thread {0}.", name), e);
}
});
StartThread(ts, name, ThreadPriority.Normal, true, false, log:log);
}
/// <summary>
/// Marks the current thread as alive
/// </summary>
@ -244,24 +280,26 @@ namespace OpenSim.Framework.Monitoring
/// <summary>
/// Stops watchdog tracking on the current thread
/// </summary>
/// <param name="log">If true then normal events in thread removal are not logged.</param>
/// <returns>
/// True if the thread was removed from the list of tracked
/// threads, otherwise false
/// </returns>
public static bool RemoveThread()
public static bool RemoveThread(bool log = true)
{
return RemoveThread(Thread.CurrentThread.ManagedThreadId);
return RemoveThread(Thread.CurrentThread.ManagedThreadId, log);
}
private static bool RemoveThread(int threadID)
private static bool RemoveThread(int threadID, bool log = true)
{
lock (m_threads)
{
ThreadWatchdogInfo twi;
if (m_threads.TryGetValue(threadID, out twi))
{
m_log.DebugFormat(
"[WATCHDOG]: Removing thread {0}, ID {1}", twi.Thread.Name, twi.Thread.ManagedThreadId);
if (log)
m_log.DebugFormat(
"[WATCHDOG]: Removing thread {0}, ID {1}", twi.Thread.Name, twi.Thread.ManagedThreadId);
twi.Cleanup();
m_threads.Remove(threadID);

30
OpenSim/Framework/Util.cs
View File

@ -2399,36 +2399,6 @@ namespace OpenSim.Framework
#endregion FireAndForget Threading Pattern
/// <summary>
/// Run the callback on a different thread, outside the thread pool. This is used for tasks
/// that may take a long time.
/// </summary>
public static void RunThreadNoTimeout(WaitCallback callback, string name, object obj)
{
if (FireAndForgetMethod == FireAndForgetMethod.RegressionTest)
{
Culture.SetCurrentCulture();
callback(obj);
return;
}
Thread t = new Thread(delegate()
{
try
{
Culture.SetCurrentCulture();
callback(obj);
}
catch (Exception e)
{
m_log.Error("Exception in thread " + name, e);
}
});
t.Name = name;
t.Start();
}
/// <summary>
/// Environment.TickCount is an int but it counts all 32 bits so it goes positive
/// and negative every 24.9 days. This trims down TickCount so it doesn't wrap

4
OpenSim/Region/CoreModules/Asset/FlotsamAssetCache.cs
View File

@ -43,6 +43,7 @@ using Mono.Addins;
using OpenMetaverse;
using OpenSim.Framework;
using OpenSim.Framework.Console;
using OpenSim.Framework.Monitoring;
using OpenSim.Region.Framework.Interfaces;
using OpenSim.Region.Framework.Scenes;
using OpenSim.Services.Interfaces;
@ -963,7 +964,8 @@ namespace OpenSim.Region.CoreModules.Asset
case "assets":
con.Output("Ensuring assets are cached for all scenes.");
Util.RunThreadNoTimeout(delegate {
Watchdog.RunInThread(delegate
{
int assetReferenceTotal = TouchAllSceneAssets(true);
con.OutputFormat("Completed check with {0} assets.", assetReferenceTotal);
}, "TouchAllSceneAssets", null);

3
OpenSim/Region/CoreModules/Avatar/Inventory/Archiver/InventoryArchiveWriteRequest.cs
View File

@ -34,6 +34,7 @@ using System.Xml;
using log4net;
using OpenMetaverse;
using OpenSim.Framework;
using OpenSim.Framework.Monitoring;
using OpenSim.Framework.Serialization;
using OpenSim.Framework.Serialization.External;
using OpenSim.Region.CoreModules.World.Archiver;
@ -356,7 +357,7 @@ namespace OpenSim.Region.CoreModules.Avatar.Inventory.Archiver
m_scene.UserAccountService, m_scene.RegionInfo.ScopeID,
options, ReceivedAllAssets);
Util.RunThreadNoTimeout(o => ar.Execute(), "AssetsRequest", null);
Watchdog.RunInThread(o => ar.Execute(), string.Format("AssetsRequest ({0})", m_scene.Name), null);
}
else
{

5
OpenSim/Region/CoreModules/ServiceConnectorsOut/Inventory/LocalInventoryServiceConnector.cs
View File

@ -33,6 +33,7 @@ using System;
using System.Collections.Generic;
using System.Reflection;
using OpenSim.Framework;
using OpenSim.Framework.Monitoring;
using OpenSim.Data;
using OpenSim.Server.Base;
using OpenSim.Region.Framework.Interfaces;
@ -183,12 +184,12 @@ namespace OpenSim.Region.CoreModules.ServiceConnectorsOut.Inventory
// Protect ourselves against the caller subsequently modifying the items list
List<InventoryItemBase> items = new List<InventoryItemBase>(invCol.Items);
Util.RunThreadNoTimeout(delegate
Watchdog.RunInThread(delegate
{
foreach (InventoryItemBase item in items)
if (!string.IsNullOrEmpty(item.CreatorData))
UserManager.AddUser(item.CreatorIdAsUuid, item.CreatorData);
}, "GetFolderContent", null);
}, string.Format("GetFolderContent (user {0}, folder {1})", userID, folderID), null);
}
return invCol;

5
OpenSim/Region/CoreModules/World/Archiver/ArchiveReadRequest.cs
View File

@ -36,6 +36,7 @@ using System.Xml;
using log4net;
using OpenMetaverse;
using OpenSim.Framework;
using OpenSim.Framework.Monitoring;
using OpenSim.Framework.Serialization;
using OpenSim.Framework.Serialization.External;
using OpenSim.Region.CoreModules.World.Terrain;
@ -371,7 +372,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver
// Start the scripts. We delayed this because we want the OAR to finish loading ASAP, so
// that users can enter the scene. If we allow the scripts to start in the loop above
// then they significantly increase the time until the OAR finishes loading.
Util.RunThreadNoTimeout(delegate(object o)
Watchdog.RunInThread(o =>
{
Thread.Sleep(15000);
m_log.Info("[ARCHIVER]: Starting scripts in scene objects");
@ -386,7 +387,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver
sceneContext.SceneObjects.Clear();
}
}, "ReadArchiveStartScripts", null);
}, string.Format("ReadArchiveStartScripts (request {0})", m_requestId), null);
m_log.InfoFormat("[ARCHIVER]: Successfully loaded archive");

3
OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequest.cs
View File

@ -36,6 +36,7 @@ using System.Xml;
using log4net;
using OpenMetaverse;
using OpenSim.Framework;
using OpenSim.Framework.Monitoring;
using OpenSim.Framework.Serialization;
using OpenSim.Region.CoreModules.World.Terrain;
using OpenSim.Region.Framework.Interfaces;
@ -199,7 +200,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver
m_rootScene.AssetService, m_rootScene.UserAccountService,
m_rootScene.RegionInfo.ScopeID, options, ReceivedAllAssets);
Util.RunThreadNoTimeout(o => ar.Execute(), "AssetsRequest", null);
Watchdog.RunInThread(o => ar.Execute(), "Archive Assets Request", null);
// CloseArchive() will be called from ReceivedAllAssets()
}

5
OpenSim/Region/CoreModules/World/Archiver/AssetsRequest.cs
View File

@ -33,6 +33,7 @@ using System.Timers;
using log4net;
using OpenMetaverse;
using OpenSim.Framework;
using OpenSim.Framework.Monitoring;
using OpenSim.Framework.Serialization;
using OpenSim.Framework.Serialization.External;
using OpenSim.Services.Interfaces;
@ -226,7 +227,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver
finally
{
if (timedOut)
Util.RunThreadNoTimeout(PerformAssetsRequestCallback, "AssetsRequestCallback", true);
Watchdog.RunInThread(PerformAssetsRequestCallback, "Archive Assets Request Callback", true);
}
}
@ -295,7 +296,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver
// We want to stop using the asset cache thread asap
// as we now need to do the work of producing the rest of the archive
Util.RunThreadNoTimeout(PerformAssetsRequestCallback, "AssetsRequestCallback", false);
Watchdog.RunInThread(PerformAssetsRequestCallback, "Archive Assets Request Callback", false);
}
else
{

17
OpenSim/Region/Framework/Scenes/Scene.cs
View File

@ -1623,7 +1623,12 @@ namespace OpenSim.Region.Framework.Scenes
{
tmpMS = Util.EnvironmentTickCount();
m_cleaningTemps = true;
Util.RunThreadNoTimeout(delegate { CleanTempObjects(); m_cleaningTemps = false; }, "CleanTempObjects", null);
Watchdog.RunInThread(
delegate { CleanTempObjects(); m_cleaningTemps = false; },
string.Format("CleanTempObjects ({0})", Name),
null);
tempOnRezMS = Util.EnvironmentTickCountSubtract(tmpMS);
}
@ -1803,7 +1808,7 @@ namespace OpenSim.Region.Framework.Scenes
if (!m_backingup)
{
m_backingup = true;
Util.RunThreadNoTimeout(BackupWaitCallback, "BackupWaitCallback", null);
Watchdog.RunInThread(o => Backup(false), string.Format("BackupWaitCallback ({0})", Name), null);
}
}
@ -1814,14 +1819,6 @@ namespace OpenSim.Region.Framework.Scenes
{
m_eventManager.TriggerOnFrame();
}
/// <summary>
/// Wrapper for Backup() that can be called with Util.RunThreadNoTimeout()
/// </summary>
private void BackupWaitCallback(object o)
{
Backup(false);
}
/// <summary>
/// Backup the scene.

6
OpenSim/Region/Framework/Scenes/ScenePresence.cs
View File

@ -37,6 +37,7 @@ using log4net;
using Nini.Config;
using OpenSim.Framework;
using OpenSim.Framework.Client;
using OpenSim.Framework.Monitoring;
using OpenSim.Region.Framework.Interfaces;
using OpenSim.Region.Framework.Scenes.Animation;
using OpenSim.Region.Framework.Scenes.Types;
@ -3362,7 +3363,7 @@ namespace OpenSim.Region.Framework.Scenes
SentInitialDataToClient = true;
// Send all scene object to the new client
Util.RunThreadNoTimeout(delegate
Watchdog.RunInThread(delegate
{
// m_log.DebugFormat(
// "[SCENE PRESENCE]: Sending initial data to {0} agent {1} in {2}, tp flags {3}",
@ -3380,8 +3381,7 @@ namespace OpenSim.Region.Framework.Scenes
if (e != null && e is SceneObjectGroup)
((SceneObjectGroup)e).SendFullUpdateToClient(ControllingClient);
}
}, "SendInitialDataToClient", null);
}, string.Format("SendInitialDataToClient ({0} in {1})", Name, Scene.Name), null);
}
/// <summary>

1
prebuild.xml
View File

@ -2489,6 +2489,7 @@
<Reference name="OpenMetaverse.StructuredData" path="../../../bin/"/>
<Reference name="OpenSim.Framework"/>
<Reference name="OpenSim.Framework.Monitoring"/>
<Reference name="OpenSim.Data"/>
<Reference name="OpenSim.Framework.Servers"/>
<Reference name="OpenSim.Framework.Servers.HttpServer"/>