OpenSim
/
OpenSimMirror
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

SCRIPT SUPPORT IS STILL BROKEN. 

Bugfix: Scripts exceeding max and set to be killed were not killed, only removed.
Added ability to re-read configuration while OpenSim is running
All regions now sharing one MaintenanceThread
New MaintenanceThread:
- checks for script execution timeout
- re-reads config
- starts/stops threads if thread active count becomes too high/low compared to config
Speed increase on event execution:
- Reuse of try{}catch{} blocks
- Time calculation on event execution
ThreadPoolClientBranch
Tedd Hansen 2008-02-01 22:18:55 +00:00
parent 8a4e8a8e31
commit a6726b0c9d
5 changed files with 424 additions and 198 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

232
OpenSim/Region/ScriptEngine/Common/ScriptEngineBase/EventQueueManager.cs
View File

@ -65,12 +65,26 @@ namespace OpenSim.Region.ScriptEngine.Common.ScriptEngineBase
// increase number of threads to allow more concurrent script executions in OpenSim.
//
public ScriptEngine m_ScriptEngine;
/// <summary>
/// List of threads processing event queue
/// List of threads (classes) processing event queue
/// </summary>
private List<EventQueueThreadClass> eventQueueThreads;// = new List<EventQueueThreadClass>();
private object eventQueueThreadsLock;// = new object();
internal List<EventQueueThreadClass> eventQueueThreads;
/// <summary>
/// Global static list of threads (classes) processing event queue -- used by max enforcment thread
/// </summary>
private List<EventQueueThreadClass> staticGlobalEventQueueThreads;
/// <summary>
/// Locking access to eventQueueThreads AND staticGlobalEventQueueThreads. Note that this may or may not be static depending on PrivateRegionThreads config setting.
/// </summary>
private object eventQueueThreadsLock;
/// <summary>
/// Used internally to specify how many threads should exit gracefully
/// </summary>
public int ThreadsToExit;
public object ThreadsToExitLock = new object();
private static List<EventQueueThreadClass> staticEventQueueThreads;// = new List<EventQueueThreadClass>();
private static object staticEventQueueThreadsLock;// = new object();
@ -80,22 +94,43 @@ namespace OpenSim.Region.ScriptEngine.Common.ScriptEngineBase
/// <summary>
/// How many threads to process queue with
/// </summary>
private int numberOfThreads;
internal int numberOfThreads;
/// <summary>
/// Maximum time one function can use for execution before we perform a thread kill
/// Maximum time one function can use for execution before we perform a thread kill.
/// </summary>
private int maxFunctionExecutionTimems
{
get { return (int)(maxFunctionExecutionTimens / 10000); }
set { maxFunctionExecutionTimens = value * 10000; }
}
/// <summary>
/// Contains nanoseconds version of maxFunctionExecutionTimems so that it matches time calculations better (performance reasons).
/// WARNING! ONLY UPDATE maxFunctionExecutionTimems, NEVER THIS DIRECTLY.
/// </summary>
public long maxFunctionExecutionTimens;
/// <summary>
/// Enforce max execution time
/// </summary>
public bool EnforceMaxExecutionTime;
/// <summary>
/// Kill script (unload) when it exceeds execution time
/// </summary>
private int maxFunctionExecutionTimems;
private bool EnforceMaxExecutionTime;
private bool KillScriptOnMaxFunctionExecutionTime;
/// <summary>
/// List of localID locks for mutex processing of script events
/// </summary>
private List<uint> objectLocks = new List<uint>();
private object tryLockLock = new object(); // Mutex lock object
/// <summary>
/// Queue containing events waiting to be executed
/// </summary>
public Queue<QueueItemStruct> eventQueue = new Queue<QueueItemStruct>();
#region " Queue structures "
/// <summary>
/// Queue item structure
/// </summary>
@ -128,18 +163,9 @@ namespace OpenSim.Region.ScriptEngine.Common.ScriptEngineBase
public int[] _int;
public string[] _string;
}
#endregion
/// <summary>
/// List of localID locks for mutex processing of script events
/// </summary>
private List<uint> objectLocks = new List<uint>();
private object tryLockLock = new object(); // Mutex lock object
public ScriptEngine m_ScriptEngine;
public Thread ExecutionTimeoutEnforcingThread;
#region " Initialization / Startup "
public EventQueueManager(ScriptEngine _ScriptEngine)
{
m_ScriptEngine = _ScriptEngine;
@ -167,66 +193,79 @@ namespace OpenSim.Region.ScriptEngine.Common.ScriptEngineBase
eventQueueThreadsLock = staticEventQueueThreadsLock;
}
numberOfThreads = m_ScriptEngine.ScriptConfigSource.GetInt("NumberOfScriptThreads", 2);
ReadConfig();
}
private void ReadConfig()
{
numberOfThreads = m_ScriptEngine.ScriptConfigSource.GetInt("NumberOfScriptThreads", 2);
maxFunctionExecutionTimems = m_ScriptEngine.ScriptConfigSource.GetInt("MaxEventExecutionTimeMs", 5000);
EnforceMaxExecutionTime = m_ScriptEngine.ScriptConfigSource.GetBoolean("EnforceMaxEventExecutionTime", false);
KillScriptOnMaxFunctionExecutionTime = m_ScriptEngine.ScriptConfigSource.GetBoolean("DeactivateScriptOnTimeout", false);
// Start function max exec time enforcement thread
if (EnforceMaxExecutionTime)
{
ExecutionTimeoutEnforcingThread = new Thread(ExecutionTimeoutEnforcingLoop);
ExecutionTimeoutEnforcingThread.Name = "ExecutionTimeoutEnforcingThread";
ExecutionTimeoutEnforcingThread.IsBackground = true;
ExecutionTimeoutEnforcingThread.Start();
}
//
// Start event queue processing threads (worker threads)
//
lock (eventQueueThreadsLock)
{
for (int ThreadCount = eventQueueThreads.Count; ThreadCount < numberOfThreads; ThreadCount++)
{
StartNewThreadClass();
}
}
}
#endregion
#region " Shutdown all threads "
~EventQueueManager()
{
try
{
if (ExecutionTimeoutEnforcingThread != null)
{
if (ExecutionTimeoutEnforcingThread.IsAlive)
{
ExecutionTimeoutEnforcingThread.Abort();
}
}
}
catch (Exception ex)
{
Stop();
}
private void Stop()
{
// Kill worker threads
lock (eventQueueThreadsLock)
{
foreach (EventQueueThreadClass EventQueueThread in new ArrayList(eventQueueThreads))
foreach (EventQueueThreadClass EventQueueThread in eventQueueThreads)
{
EventQueueThread.Shutdown();
}
eventQueueThreads.Clear();
staticGlobalEventQueueThreads.Clear();
}
// Todo: Clean up our queues
// Remove all entries from our event queue
lock (queueLock)
{
eventQueue.Clear();
}
}
#endregion
#region " Start / stop script execution threads (ThreadClasses) "
private void StartNewThreadClass()
{
EventQueueThreadClass eqtc = new EventQueueThreadClass(this);
eventQueueThreads.Add(eqtc);
staticGlobalEventQueueThreads.Add(eqtc);
m_ScriptEngine.Log.Debug("DotNetEngine", "Started new script execution thread. Current thread count: " + eventQueueThreads.Count);
}
private void AbortThreadClass(EventQueueThreadClass threadClass)
{
if (eventQueueThreads.Contains(threadClass))
eventQueueThreads.Remove(threadClass);
if (staticGlobalEventQueueThreads.Contains(threadClass))
staticGlobalEventQueueThreads.Remove(threadClass);
try
{
threadClass.Shutdown();
}
catch (Exception ex)
{
m_ScriptEngine.Log.Error("EventQueueManager", "If you see this, could you please report it to Tedd:");
m_ScriptEngine.Log.Error("EventQueueManager", "Script thread execution timeout kill ended in exception: " + ex.ToString());
}
m_ScriptEngine.Log.Debug("DotNetEngine", "Killed script execution thread. Remaining thread count: " + eventQueueThreads.Count);
}
#endregion
#region " Mutex locks for queue access "
/// <summary>
/// Try to get a mutex lock on localID
/// </summary>
@ -262,8 +301,9 @@ namespace OpenSim.Region.ScriptEngine.Common.ScriptEngineBase
}
}
}
#endregion
#region " Add events to execution queue "
/// <summary>
/// Add event to event execution queue
/// </summary>
@ -317,62 +357,72 @@ namespace OpenSim.Region.ScriptEngine.Common.ScriptEngineBase
eventQueue.Enqueue(QIS);
}
}
#endregion
#region " Maintenance thread "
/// <summary>
/// A thread should run in this loop and check all running scripts
/// Adjust number of script thread classes. It can start new, but if it needs to stop it will just set number of threads in "ThreadsToExit" and threads will have to exit themselves.
/// Called from MaintenanceThread
/// </summary>
public void ExecutionTimeoutEnforcingLoop()
public void AdjustNumberOfScriptThreads()
{
try
{
while (true)
{
System.Threading.Thread.Sleep(maxFunctionExecutionTimems);
lock (eventQueueThreadsLock)
{
foreach (EventQueueThreadClass EventQueueThread in new ArrayList(eventQueueThreads))
int diff = numberOfThreads - eventQueueThreads.Count;
// Positive number: Start
// Negative number: too many are running
if (diff > 0)
{
if (EventQueueThread.InExecution)
// We need to add more threads
for (int ThreadCount = eventQueueThreads.Count; ThreadCount < numberOfThreads; ThreadCount++)
{
if (DateTime.Now.Subtract(EventQueueThread.LastExecutionStarted).Milliseconds >
maxFunctionExecutionTimems)
{
// We need to kill this thread!
EventQueueThread.KillCurrentScript = KillScriptOnMaxFunctionExecutionTime;
AbortThreadClass(EventQueueThread);
// Then start another
StartNewThreadClass();
}
}
}
}
}
}
catch (ThreadAbortException tae)
if (diff < 0)
{
// We need to kill some threads
lock (ThreadsToExitLock)
{
ThreadsToExit = Math.Abs(diff);
}
}
}
}
private void AbortThreadClass(EventQueueThreadClass threadClass)
/// <summary>
/// Check if any thread class has been executing an event too long
/// </summary>
public void CheckScriptMaxExecTime()
{
try
// Iterate through all ScriptThreadClasses and check how long their current function has been executing
lock (eventQueueThreadsLock)
{
threadClass.Shutdown();
}
catch (Exception ex)
foreach (EventQueueThreadClass EventQueueThread in staticGlobalEventQueueThreads)
{
Console.WriteLine("Could you please report this to Tedd:");
Console.WriteLine("Script thread execution timeout kill ended in exception: " + ex.ToString());
}
m_ScriptEngine.Log.Debug("DotNetEngine", "Killed script execution thread, count: " + eventQueueThreads.Count);
}
// Is thread currently executing anything?
if (EventQueueThread.InExecution)
{
// Has execution time expired?
if (DateTime.Now.Ticks - EventQueueThread.LastExecutionStarted >
maxFunctionExecutionTimens)
{
// Yes! We need to kill this thread!
private void StartNewThreadClass()
{
EventQueueThreadClass eqtc = new EventQueueThreadClass(this);
eventQueueThreads.Add(eqtc);
m_ScriptEngine.Log.Debug("DotNetEngine", "Started new script execution thread, count: " + eventQueueThreads.Count);
// Set flag if script should be removed or not
EventQueueThread.KillCurrentScript = KillScriptOnMaxFunctionExecutionTime;
// Abort this thread
AbortThreadClass(EventQueueThread);
// We do not need to start another, MaintenenceThread will do that for us
//StartNewThreadClass();
}
}
}
}
}
#endregion
}
}

60
OpenSim/Region/ScriptEngine/Common/ScriptEngineBase/EventQueueThreadClass.cs
View File

@ -19,7 +19,7 @@ namespace OpenSim.Region.ScriptEngine.Common.ScriptEngineBase
/// </summary>
private int nothingToDoSleepms;// = 50;
public DateTime LastExecutionStarted;
public long LastExecutionStarted;
public bool InExecution = false;
public bool KillCurrentScript = false;
@ -109,11 +109,33 @@ namespace OpenSim.Region.ScriptEngine.Common.ScriptEngineBase
//myScriptEngine.m_logger.Verbose("ScriptEngine", "EventQueueManager Worker thread spawned");
try
{
EventQueueManager.QueueItemStruct BlankQIS = new EventQueueManager.QueueItemStruct();
while (true)
{
try
{
EventQueueManager.QueueItemStruct BlankQIS = new EventQueueManager.QueueItemStruct();
while (true)
{
// Every now and then check if we should shut down
if (eventQueueManager.ThreadsToExit > 0)
{
// Someone should shut down, lets get exclusive lock
lock (eventQueueManager.ThreadsToExitLock)
{
// Lets re-check in case someone grabbed it
if (eventQueueManager.ThreadsToExit > 0)
{
// We are go for shutdown
eventQueueManager.ThreadsToExit--;
Shutdown();
return;
}
}
}
//try
// {
EventQueueManager.QueueItemStruct QIS = BlankQIS;
bool GotItem = false;
@ -157,23 +179,29 @@ namespace OpenSim.Region.ScriptEngine.Common.ScriptEngineBase
try
{
#if DEBUG
eventQueueManager.m_ScriptEngine.Log.Debug("ScriptEngine", "Executing event:\r\n"
eventQueueManager.m_ScriptEngine.Log.Debug("ScriptEngine",
"Executing event:\r\n"
+ "QIS.localID: " + QIS.localID
+ ", QIS.itemID: " + QIS.itemID
+ ", QIS.functionName: " + QIS.functionName);
+ ", QIS.functionName: " +
QIS.functionName);
#endif
LastExecutionStarted = DateTime.Now;
LastExecutionStarted = DateTime.Now.Ticks;
KillCurrentScript = false;
InExecution = true;
eventQueueManager.m_ScriptEngine.m_ScriptManager.ExecuteEvent(QIS.localID, QIS.itemID,
QIS.functionName, QIS.llDetectParams, QIS.param);
eventQueueManager.m_ScriptEngine.m_ScriptManager.ExecuteEvent(QIS.localID,
QIS.itemID,
QIS.functionName,
QIS.llDetectParams,
QIS.param);
InExecution = false;
}
catch (Exception e)
{
InExecution = false;
// DISPLAY ERROR INWORLD
string text = "Error executing script function \"" + QIS.functionName + "\":\r\n";
string text = "Error executing script function \"" + QIS.functionName +
"\":\r\n";
if (e.InnerException != null)
{
// Send inner exception
@ -192,11 +220,14 @@ namespace OpenSim.Region.ScriptEngine.Common.ScriptEngineBase
{
if (text.Length > 1500)
text = text.Substring(0, 1500);
IScriptHost m_host = eventQueueManager.m_ScriptEngine.World.GetSceneObjectPart(QIS.localID);
IScriptHost m_host =
eventQueueManager.m_ScriptEngine.World.GetSceneObjectPart(QIS.localID);
//if (m_host != null)
//{
eventQueueManager.m_ScriptEngine.World.SimChat(Helpers.StringToField(text), ChatTypeEnum.Say, 0,
m_host.AbsolutePosition, m_host.Name, m_host.UUID);
eventQueueManager.m_ScriptEngine.World.SimChat(Helpers.StringToField(text),
ChatTypeEnum.Say, 0,
m_host.AbsolutePosition,
m_host.Name, m_host.UUID);
}
catch
{
@ -205,14 +236,16 @@ namespace OpenSim.Region.ScriptEngine.Common.ScriptEngineBase
//{
// T oconsole
eventQueueManager.m_ScriptEngine.Log.Error("ScriptEngine",
"Unable to send text in-world:\r\n" + text);
"Unable to send text in-world:\r\n" +
text);
}
finally
{
// So we are done sending message in-world
if (KillCurrentScript)
{
eventQueueManager.m_ScriptEngine.m_ScriptManager.RemoveScript(QIS.localID, QIS.itemID);
eventQueueManager.m_ScriptEngine.m_ScriptManager.StopScript(
QIS.localID, QIS.itemID);
}
}
}
@ -224,6 +257,7 @@ namespace OpenSim.Region.ScriptEngine.Common.ScriptEngineBase
}
} // Something in queue
}
}
catch (ThreadAbortException tae)
{
eventQueueManager.m_ScriptEngine.Log.Notice("ScriptEngine", "ThreadAbortException while executing function.");

127
OpenSim/Region/ScriptEngine/Common/ScriptEngineBase/MaintenanceThread.cs Normal file
View File

@ -0,0 +1,127 @@
using System;
using System.Collections.Generic;
using System.Text;
using System.Threading;
namespace OpenSim.Region.ScriptEngine.Common.ScriptEngineBase
{
/// <summary>
/// This class does maintenance on script engine.
/// </summary>
public class MaintenanceThread
{
public ScriptEngine m_ScriptEngine;
private int MaintenanceLoopms;
public MaintenanceThread(ScriptEngine _ScriptEngine)
{
m_ScriptEngine = _ScriptEngine;
ReadConfig();
// Start maintenance thread
StartMaintenanceThread();
}
~MaintenanceThread()
{
StopMaintenanceThread();
}
private void ReadConfig()
{
MaintenanceLoopms = m_ScriptEngine.ScriptConfigSource.GetInt("MaintenanceLoopms", 50);
}
#region " Maintenance thread "
/// <summary>
/// Maintenance thread. Enforcing max execution time for example.
/// </summary>
public static Thread MaintenanceThreadThread;
/// <summary>
/// Starts maintenance thread
/// </summary>
private void StartMaintenanceThread()
{
StopMaintenanceThread();
MaintenanceThreadThread = new Thread(MaintenanceLoop);
MaintenanceThreadThread.Name = "ScriptMaintenanceThread";
MaintenanceThreadThread.IsBackground = true;
MaintenanceThreadThread.Start();
}
/// <summary>
/// Stops maintenance thread
/// </summary>
private void StopMaintenanceThread()
{
try
{
if (MaintenanceThreadThread != null)
{
if (MaintenanceThreadThread.IsAlive)
{
MaintenanceThreadThread.Abort();
}
}
}
catch (Exception ex)
{
m_ScriptEngine.Log.Error("EventQueueManager", "Exception stopping maintenence thread: " + ex.ToString());
}
}
/// <summary>
/// A thread should run in this loop and check all running scripts
/// </summary>
public void MaintenanceLoop()
{
try
{
long Last_maxFunctionExecutionTimens = 0;// DateTime.Now.Ticks;
long Last_ReReadConfigFilens = DateTime.Now.Ticks;
while (true)
{
System.Threading.Thread.Sleep(MaintenanceLoopms); // Sleep
// Re-reading config every x seconds?
if (m_ScriptEngine.ReReadConfigFileSeconds > 0)
{
// Check if its time to re-read config
if (DateTime.Now.Ticks - Last_ReReadConfigFilens > m_ScriptEngine.ReReadConfigFilens)
{
// Its time to re-read config file
m_ScriptEngine.ConfigSource.Reload(); // Re-read config
Last_ReReadConfigFilens = DateTime.Now.Ticks; // Reset time
}
}
// Adjust number of running script threads if not correct
if (m_ScriptEngine.m_EventQueueManager.eventQueueThreads.Count != m_ScriptEngine.m_EventQueueManager.numberOfThreads)
{
m_ScriptEngine.m_EventQueueManager.AdjustNumberOfScriptThreads();
}
// Check if any script has exceeded its max execution time
if (m_ScriptEngine.m_EventQueueManager.EnforceMaxExecutionTime)
{
if (DateTime.Now.Ticks - Last_maxFunctionExecutionTimens > m_ScriptEngine.m_EventQueueManager.maxFunctionExecutionTimens)
{
m_ScriptEngine.m_EventQueueManager.CheckScriptMaxExecTime(); // Do check
Last_maxFunctionExecutionTimens = DateTime.Now.Ticks; // Reset time
}
}
}
}
catch (ThreadAbortException tae)
{
}
}
#endregion
}
}

12
OpenSim/Region/ScriptEngine/Common/ScriptEngineBase/ScriptEngine.cs
View File

@ -55,6 +55,15 @@ namespace OpenSim.Region.ScriptEngine.Common.ScriptEngineBase
public IConfig ScriptConfigSource;
public abstract string ScriptConfigSourceName { get; }
/// <summary>
/// How many seconds between re-reading config-file. 0 = never. ScriptEngine will try to adjust to new config changes.
/// </summary>
public int ReReadConfigFileSeconds {
get { return (int)(ReReadConfigFilens / 10000); }
set { ReReadConfigFilens = value * 10000; }
}
public long ReReadConfigFilens = 0;
public ScriptManager GetScriptManager()
{
return _GetScriptManager();
@ -93,6 +102,9 @@ namespace OpenSim.Region.ScriptEngine.Common.ScriptEngineBase
m_AppDomainManager = new AppDomainManager(ScriptConfigSource.GetInt("ScriptsPerAppDomain", 1));
m_LSLLongCmdHandler = new LSLLongCmdHandler(this);
ReReadConfigFileSeconds = ScriptConfigSource.GetInt("ReReadConfig", 0);
// Should we iterate the region for scripts that needs starting?
// Or can we assume we are loaded before anything else so we can use proper events?
}

3
bin/OpenSim.ini.example
View File

@ -135,6 +135,7 @@ ScriptThreadPriority=BelowNormal
; true: Each region will get <NumberOfScriptThreads> dedicated to scripts within that region
; Number of threads will be <NumberOfScriptThreads>*<NumberOfRegions>
; false: All regions share <NumberOfScriptThreads> for all their scripts
; Note! If you run multiple script engines based on "OpenSim.Region.ScriptEngine.Common" then all of them will share the same threads.
PrivateRegionThreads=false
; How long MAX should a script event be allowed to run (per event execution)?
@ -163,3 +164,5 @@ SleepTimeIfNoScriptExecutionMs=50
; Each AppDomain has some memory overhead. But leaving dead scripts in memory also has memory overhead.
ScriptsPerAppDomain=1
; ReRead ScriptEngine config options how often?
ReReadConfig=0