From 41f2f3132bdcbfb8020c7fd6e5f3b7e48c75b1cf Mon Sep 17 00:00:00 2001 From: "Justin Clark-Casey (justincc)" Date: Fri, 5 Sep 2014 23:20:59 +0100 Subject: [PATCH] For monitoring purposes, start non-timeout tasks (which do not currently use a threadpool) via Watchdog.RunInThread() rather than Util.RunThreadNoTimeout() The functionality is the same but this allow us to monitor such tasks via "show threads" and abort them for test purposes, etc. Also extends thread names to provide more info (e.g. SendInitialDataToClient says what client the task is for). --- .../GroupsServiceHGConnectorModule.cs | 5 +- OpenSim/Framework/Monitoring/Watchdog.cs | 60 +++++++++++++++---- OpenSim/Framework/Util.cs | 30 ---------- .../CoreModules/Asset/FlotsamAssetCache.cs | 4 +- .../Archiver/InventoryArchiveWriteRequest.cs | 3 +- .../LocalInventoryServiceConnector.cs | 5 +- .../World/Archiver/ArchiveReadRequest.cs | 5 +- .../World/Archiver/ArchiveWriteRequest.cs | 3 +- .../World/Archiver/AssetsRequest.cs | 5 +- OpenSim/Region/Framework/Scenes/Scene.cs | 17 +++--- .../Region/Framework/Scenes/ScenePresence.cs | 6 +- prebuild.xml | 1 + 12 files changed, 79 insertions(+), 65 deletions(-) diff --git a/OpenSim/Addons/Groups/Hypergrid/GroupsServiceHGConnectorModule.cs b/OpenSim/Addons/Groups/Hypergrid/GroupsServiceHGConnectorModule.cs index a750d8d870..9fbc1b3501 100644 --- a/OpenSim/Addons/Groups/Hypergrid/GroupsServiceHGConnectorModule.cs +++ b/OpenSim/Addons/Groups/Hypergrid/GroupsServiceHGConnectorModule.cs @@ -32,6 +32,7 @@ using System.Reflection; using System.Text; using OpenSim.Framework; +using OpenSim.Framework.Monitoring; using OpenSim.Framework.Servers; using OpenSim.Region.Framework.Scenes; using OpenSim.Region.Framework.Interfaces; @@ -560,7 +561,7 @@ namespace OpenSim.Groups // so we have the list of urls to send the notice to // this may take a long time... - Util.RunThreadNoTimeout(delegate + Watchdog.RunInThread(delegate { foreach (string u in urls) { @@ -571,7 +572,7 @@ namespace OpenSim.Groups hasAttachment, attType, attName, attItemID, AgentUUIForOutside(attOwnerID)); } } - }, "AddGroupNotice", null); + }, string.Format("AddGroupNotice (agent {0}, group {1})", RequestingAgentID, groupID) , null); return true; } diff --git a/OpenSim/Framework/Monitoring/Watchdog.cs b/OpenSim/Framework/Monitoring/Watchdog.cs index 427ed7bb0d..e9e7bd21c0 100644 --- a/OpenSim/Framework/Monitoring/Watchdog.cs +++ b/OpenSim/Framework/Monitoring/Watchdog.cs @@ -187,15 +187,16 @@ namespace OpenSim.Framework.Monitoring /// Priority to run the thread at /// True to run this thread as a background thread, otherwise false /// Trigger an alarm function is we have timed out + /// If true then creation of thread is logged. /// The newly created Thread object public static Thread StartThread( - ThreadStart start, string name, ThreadPriority priority, bool isBackground, bool alarmIfTimeout) + ThreadStart start, string name, ThreadPriority priority, bool isBackground, bool alarmIfTimeout, bool log = true) { - return StartThread(start, name, priority, isBackground, alarmIfTimeout, null, DEFAULT_WATCHDOG_TIMEOUT_MS); + return StartThread(start, name, priority, isBackground, alarmIfTimeout, null, DEFAULT_WATCHDOG_TIMEOUT_MS, log); } /// - /// Start a new thread that is tracked by the watchdog timer + /// Start a new thread that is tracked by the watchdog /// /// The method that will be executed in a new thread /// A name to give to the new thread @@ -208,10 +209,11 @@ namespace OpenSim.Framework.Monitoring /// Normally, this will just return some useful debugging information. /// /// Number of milliseconds to wait until we issue a warning about timeout. + /// If true then creation of thread is logged. /// The newly created Thread object public static Thread StartThread( ThreadStart start, string name, ThreadPriority priority, bool isBackground, - bool alarmIfTimeout, Func alarmMethod, int timeout) + bool alarmIfTimeout, Func alarmMethod, int timeout, bool log = true) { Thread thread = new Thread(start); thread.Name = name; @@ -222,8 +224,9 @@ namespace OpenSim.Framework.Monitoring = new ThreadWatchdogInfo(thread, timeout) { AlarmIfTimeout = alarmIfTimeout, AlarmMethod = alarmMethod }; - m_log.DebugFormat( - "[WATCHDOG]: Started tracking thread {0}, ID {1}", twi.Thread.Name, twi.Thread.ManagedThreadId); + if (log) + m_log.DebugFormat( + "[WATCHDOG]: Started tracking thread {0}, ID {1}", twi.Thread.Name, twi.Thread.ManagedThreadId); lock (m_threads) m_threads.Add(twi.Thread.ManagedThreadId, twi); @@ -233,6 +236,39 @@ namespace OpenSim.Framework.Monitoring return thread; } + /// + /// Run the callback in a new thread immediately. If the thread exits with an exception log it but do + /// not propogate it. + /// + /// Code for the thread to execute. + /// Name of the thread + /// Object to pass to the thread. + public static void RunInThread(WaitCallback callback, string name, object obj, bool log = false) + { + if (Util.FireAndForgetMethod == FireAndForgetMethod.RegressionTest) + { + Culture.SetCurrentCulture(); + callback(obj); + return; + } + + ThreadStart ts = new ThreadStart(delegate() + { + try + { + Culture.SetCurrentCulture(); + callback(obj); + Watchdog.RemoveThread(log:false); + } + catch (Exception e) + { + m_log.Error(string.Format("[WATCHDOG]: Exception in thread {0}.", name), e); + } + }); + + StartThread(ts, name, ThreadPriority.Normal, true, false, log:log); + } + /// /// Marks the current thread as alive /// @@ -244,24 +280,26 @@ namespace OpenSim.Framework.Monitoring /// /// Stops watchdog tracking on the current thread /// + /// If true then normal events in thread removal are not logged. /// /// True if the thread was removed from the list of tracked /// threads, otherwise false /// - public static bool RemoveThread() + public static bool RemoveThread(bool log = true) { - return RemoveThread(Thread.CurrentThread.ManagedThreadId); + return RemoveThread(Thread.CurrentThread.ManagedThreadId, log); } - private static bool RemoveThread(int threadID) + private static bool RemoveThread(int threadID, bool log = true) { lock (m_threads) { ThreadWatchdogInfo twi; if (m_threads.TryGetValue(threadID, out twi)) { - m_log.DebugFormat( - "[WATCHDOG]: Removing thread {0}, ID {1}", twi.Thread.Name, twi.Thread.ManagedThreadId); + if (log) + m_log.DebugFormat( + "[WATCHDOG]: Removing thread {0}, ID {1}", twi.Thread.Name, twi.Thread.ManagedThreadId); twi.Cleanup(); m_threads.Remove(threadID); diff --git a/OpenSim/Framework/Util.cs b/OpenSim/Framework/Util.cs index 2c38571147..fefa05039a 100644 --- a/OpenSim/Framework/Util.cs +++ b/OpenSim/Framework/Util.cs @@ -2399,36 +2399,6 @@ namespace OpenSim.Framework #endregion FireAndForget Threading Pattern - /// - /// Run the callback on a different thread, outside the thread pool. This is used for tasks - /// that may take a long time. - /// - public static void RunThreadNoTimeout(WaitCallback callback, string name, object obj) - { - if (FireAndForgetMethod == FireAndForgetMethod.RegressionTest) - { - Culture.SetCurrentCulture(); - callback(obj); - return; - } - - Thread t = new Thread(delegate() - { - try - { - Culture.SetCurrentCulture(); - callback(obj); - } - catch (Exception e) - { - m_log.Error("Exception in thread " + name, e); - } - }); - - t.Name = name; - t.Start(); - } - /// /// Environment.TickCount is an int but it counts all 32 bits so it goes positive /// and negative every 24.9 days. This trims down TickCount so it doesn't wrap diff --git a/OpenSim/Region/CoreModules/Asset/FlotsamAssetCache.cs b/OpenSim/Region/CoreModules/Asset/FlotsamAssetCache.cs index f06d70d9e0..9d6870f1b5 100644 --- a/OpenSim/Region/CoreModules/Asset/FlotsamAssetCache.cs +++ b/OpenSim/Region/CoreModules/Asset/FlotsamAssetCache.cs @@ -43,6 +43,7 @@ using Mono.Addins; using OpenMetaverse; using OpenSim.Framework; using OpenSim.Framework.Console; +using OpenSim.Framework.Monitoring; using OpenSim.Region.Framework.Interfaces; using OpenSim.Region.Framework.Scenes; using OpenSim.Services.Interfaces; @@ -963,7 +964,8 @@ namespace OpenSim.Region.CoreModules.Asset case "assets": con.Output("Ensuring assets are cached for all scenes."); - Util.RunThreadNoTimeout(delegate { + Watchdog.RunInThread(delegate + { int assetReferenceTotal = TouchAllSceneAssets(true); con.OutputFormat("Completed check with {0} assets.", assetReferenceTotal); }, "TouchAllSceneAssets", null); diff --git a/OpenSim/Region/CoreModules/Avatar/Inventory/Archiver/InventoryArchiveWriteRequest.cs b/OpenSim/Region/CoreModules/Avatar/Inventory/Archiver/InventoryArchiveWriteRequest.cs index d15703eb73..a5b5aa24c7 100644 --- a/OpenSim/Region/CoreModules/Avatar/Inventory/Archiver/InventoryArchiveWriteRequest.cs +++ b/OpenSim/Region/CoreModules/Avatar/Inventory/Archiver/InventoryArchiveWriteRequest.cs @@ -34,6 +34,7 @@ using System.Xml; using log4net; using OpenMetaverse; using OpenSim.Framework; +using OpenSim.Framework.Monitoring; using OpenSim.Framework.Serialization; using OpenSim.Framework.Serialization.External; using OpenSim.Region.CoreModules.World.Archiver; @@ -356,7 +357,7 @@ namespace OpenSim.Region.CoreModules.Avatar.Inventory.Archiver m_scene.UserAccountService, m_scene.RegionInfo.ScopeID, options, ReceivedAllAssets); - Util.RunThreadNoTimeout(o => ar.Execute(), "AssetsRequest", null); + Watchdog.RunInThread(o => ar.Execute(), string.Format("AssetsRequest ({0})", m_scene.Name), null); } else { diff --git a/OpenSim/Region/CoreModules/ServiceConnectorsOut/Inventory/LocalInventoryServiceConnector.cs b/OpenSim/Region/CoreModules/ServiceConnectorsOut/Inventory/LocalInventoryServiceConnector.cs index f80fab3e1d..470ef0274d 100644 --- a/OpenSim/Region/CoreModules/ServiceConnectorsOut/Inventory/LocalInventoryServiceConnector.cs +++ b/OpenSim/Region/CoreModules/ServiceConnectorsOut/Inventory/LocalInventoryServiceConnector.cs @@ -33,6 +33,7 @@ using System; using System.Collections.Generic; using System.Reflection; using OpenSim.Framework; +using OpenSim.Framework.Monitoring; using OpenSim.Data; using OpenSim.Server.Base; using OpenSim.Region.Framework.Interfaces; @@ -183,12 +184,12 @@ namespace OpenSim.Region.CoreModules.ServiceConnectorsOut.Inventory // Protect ourselves against the caller subsequently modifying the items list List items = new List(invCol.Items); - Util.RunThreadNoTimeout(delegate + Watchdog.RunInThread(delegate { foreach (InventoryItemBase item in items) if (!string.IsNullOrEmpty(item.CreatorData)) UserManager.AddUser(item.CreatorIdAsUuid, item.CreatorData); - }, "GetFolderContent", null); + }, string.Format("GetFolderContent (user {0}, folder {1})", userID, folderID), null); } return invCol; diff --git a/OpenSim/Region/CoreModules/World/Archiver/ArchiveReadRequest.cs b/OpenSim/Region/CoreModules/World/Archiver/ArchiveReadRequest.cs index c8056d425d..3e0c9f30b0 100644 --- a/OpenSim/Region/CoreModules/World/Archiver/ArchiveReadRequest.cs +++ b/OpenSim/Region/CoreModules/World/Archiver/ArchiveReadRequest.cs @@ -36,6 +36,7 @@ using System.Xml; using log4net; using OpenMetaverse; using OpenSim.Framework; +using OpenSim.Framework.Monitoring; using OpenSim.Framework.Serialization; using OpenSim.Framework.Serialization.External; using OpenSim.Region.CoreModules.World.Terrain; @@ -371,7 +372,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver // Start the scripts. We delayed this because we want the OAR to finish loading ASAP, so // that users can enter the scene. If we allow the scripts to start in the loop above // then they significantly increase the time until the OAR finishes loading. - Util.RunThreadNoTimeout(delegate(object o) + Watchdog.RunInThread(o => { Thread.Sleep(15000); m_log.Info("[ARCHIVER]: Starting scripts in scene objects"); @@ -386,7 +387,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver sceneContext.SceneObjects.Clear(); } - }, "ReadArchiveStartScripts", null); + }, string.Format("ReadArchiveStartScripts (request {0})", m_requestId), null); m_log.InfoFormat("[ARCHIVER]: Successfully loaded archive"); diff --git a/OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequest.cs b/OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequest.cs index 448147ac7b..803f24e2a9 100644 --- a/OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequest.cs +++ b/OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequest.cs @@ -36,6 +36,7 @@ using System.Xml; using log4net; using OpenMetaverse; using OpenSim.Framework; +using OpenSim.Framework.Monitoring; using OpenSim.Framework.Serialization; using OpenSim.Region.CoreModules.World.Terrain; using OpenSim.Region.Framework.Interfaces; @@ -199,7 +200,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver m_rootScene.AssetService, m_rootScene.UserAccountService, m_rootScene.RegionInfo.ScopeID, options, ReceivedAllAssets); - Util.RunThreadNoTimeout(o => ar.Execute(), "AssetsRequest", null); + Watchdog.RunInThread(o => ar.Execute(), "Archive Assets Request", null); // CloseArchive() will be called from ReceivedAllAssets() } diff --git a/OpenSim/Region/CoreModules/World/Archiver/AssetsRequest.cs b/OpenSim/Region/CoreModules/World/Archiver/AssetsRequest.cs index 4f428c35a6..ec39bc02af 100644 --- a/OpenSim/Region/CoreModules/World/Archiver/AssetsRequest.cs +++ b/OpenSim/Region/CoreModules/World/Archiver/AssetsRequest.cs @@ -33,6 +33,7 @@ using System.Timers; using log4net; using OpenMetaverse; using OpenSim.Framework; +using OpenSim.Framework.Monitoring; using OpenSim.Framework.Serialization; using OpenSim.Framework.Serialization.External; using OpenSim.Services.Interfaces; @@ -226,7 +227,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver finally { if (timedOut) - Util.RunThreadNoTimeout(PerformAssetsRequestCallback, "AssetsRequestCallback", true); + Watchdog.RunInThread(PerformAssetsRequestCallback, "Archive Assets Request Callback", true); } } @@ -295,7 +296,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver // We want to stop using the asset cache thread asap // as we now need to do the work of producing the rest of the archive - Util.RunThreadNoTimeout(PerformAssetsRequestCallback, "AssetsRequestCallback", false); + Watchdog.RunInThread(PerformAssetsRequestCallback, "Archive Assets Request Callback", false); } else { diff --git a/OpenSim/Region/Framework/Scenes/Scene.cs b/OpenSim/Region/Framework/Scenes/Scene.cs index 5f0dbd7497..28dbccb95f 100644 --- a/OpenSim/Region/Framework/Scenes/Scene.cs +++ b/OpenSim/Region/Framework/Scenes/Scene.cs @@ -1623,7 +1623,12 @@ namespace OpenSim.Region.Framework.Scenes { tmpMS = Util.EnvironmentTickCount(); m_cleaningTemps = true; - Util.RunThreadNoTimeout(delegate { CleanTempObjects(); m_cleaningTemps = false; }, "CleanTempObjects", null); + + Watchdog.RunInThread( + delegate { CleanTempObjects(); m_cleaningTemps = false; }, + string.Format("CleanTempObjects ({0})", Name), + null); + tempOnRezMS = Util.EnvironmentTickCountSubtract(tmpMS); } @@ -1803,7 +1808,7 @@ namespace OpenSim.Region.Framework.Scenes if (!m_backingup) { m_backingup = true; - Util.RunThreadNoTimeout(BackupWaitCallback, "BackupWaitCallback", null); + Watchdog.RunInThread(o => Backup(false), string.Format("BackupWaitCallback ({0})", Name), null); } } @@ -1814,14 +1819,6 @@ namespace OpenSim.Region.Framework.Scenes { m_eventManager.TriggerOnFrame(); } - - /// - /// Wrapper for Backup() that can be called with Util.RunThreadNoTimeout() - /// - private void BackupWaitCallback(object o) - { - Backup(false); - } /// /// Backup the scene. diff --git a/OpenSim/Region/Framework/Scenes/ScenePresence.cs b/OpenSim/Region/Framework/Scenes/ScenePresence.cs index 3c37de84ec..59b521db5f 100644 --- a/OpenSim/Region/Framework/Scenes/ScenePresence.cs +++ b/OpenSim/Region/Framework/Scenes/ScenePresence.cs @@ -37,6 +37,7 @@ using log4net; using Nini.Config; using OpenSim.Framework; using OpenSim.Framework.Client; +using OpenSim.Framework.Monitoring; using OpenSim.Region.Framework.Interfaces; using OpenSim.Region.Framework.Scenes.Animation; using OpenSim.Region.Framework.Scenes.Types; @@ -3362,7 +3363,7 @@ namespace OpenSim.Region.Framework.Scenes SentInitialDataToClient = true; // Send all scene object to the new client - Util.RunThreadNoTimeout(delegate + Watchdog.RunInThread(delegate { // m_log.DebugFormat( // "[SCENE PRESENCE]: Sending initial data to {0} agent {1} in {2}, tp flags {3}", @@ -3380,8 +3381,7 @@ namespace OpenSim.Region.Framework.Scenes if (e != null && e is SceneObjectGroup) ((SceneObjectGroup)e).SendFullUpdateToClient(ControllingClient); } - - }, "SendInitialDataToClient", null); + }, string.Format("SendInitialDataToClient ({0} in {1})", Name, Scene.Name), null); } /// diff --git a/prebuild.xml b/prebuild.xml index ba18a2c50f..a10d824053 100644 --- a/prebuild.xml +++ b/prebuild.xml @@ -2489,6 +2489,7 @@ +