Highly experimental
A separate thread is used to enforce max function (event) execution time for scripts.ThreadPoolClientBranch
Tedd Hansen
parent
43ecb1b4d2
commit
5d6e89eaf9
|
|
@ -69,29 +69,31 @@ namespace OpenSim.Region.ScriptEngine.Common.ScriptEngineBase
|
|||
/// <summary>
|
||||
/// List of threads processing event queue
|
||||
/// </summary>
|
||||
private List<Thread> eventQueueThreads = new List<Thread>();
|
||||
private List<EventQueueThreadClass> eventQueueThreads = new List<EventQueueThreadClass>();
|
||||
private object eventQueueThreadsLock = new object();
|
||||
|
||||
private object queueLock = new object(); // Mutex lock object
|
||||
|
||||
/// <summary>
|
||||
/// How many ms to sleep if queue is empty
|
||||
/// </summary>
|
||||
private int nothingToDoSleepms = 50;
|
||||
public object queueLock = new object(); // Mutex lock object
|
||||
|
||||
/// <summary>
|
||||
/// How many threads to process queue with
|
||||
/// </summary>
|
||||
private int numberOfThreads = 2;
|
||||
|
||||
/// <summary>
|
||||
/// Maximum time one function can use for execution before we perform a thread kill
|
||||
/// </summary>
|
||||
private int maxFunctionExecutionTimems = 50;
|
||||
private bool EnforceMaxExecutionTime = true;
|
||||
|
||||
/// <summary>
|
||||
/// Queue containing events waiting to be executed
|
||||
/// </summary>
|
||||
private Queue<QueueItemStruct> eventQueue = new Queue<QueueItemStruct>();
|
||||
public Queue<QueueItemStruct> eventQueue = new Queue<QueueItemStruct>();
|
||||
|
||||
/// <summary>
|
||||
/// Queue item structure
|
||||
/// </summary>
|
||||
private struct QueueItemStruct
|
||||
public struct QueueItemStruct
|
||||
{
|
||||
public uint localID;
|
||||
public LLUUID itemID;
|
||||
|
|
@ -128,177 +130,72 @@ namespace OpenSim.Region.ScriptEngine.Common.ScriptEngineBase
|
|||
|
||||
private object tryLockLock = new object(); // Mutex lock object
|
||||
|
||||
private ScriptEngine m_ScriptEngine;
|
||||
public ScriptEngine m_ScriptEngine;
|
||||
|
||||
public Thread ExecutionTimeoutEnforcingThread;
|
||||
|
||||
public EventQueueManager(ScriptEngine _ScriptEngine)
|
||||
{
|
||||
m_ScriptEngine = _ScriptEngine;
|
||||
|
||||
// Start function max exec time enforcement thread
|
||||
if (EnforceMaxExecutionTime)
|
||||
{
|
||||
ExecutionTimeoutEnforcingThread = new Thread(ExecutionTimeoutEnforcingLoop);
|
||||
ExecutionTimeoutEnforcingThread.Name = "ExecutionTimeoutEnforcingThread";
|
||||
ExecutionTimeoutEnforcingThread.IsBackground = true;
|
||||
ExecutionTimeoutEnforcingThread.Start();
|
||||
}
|
||||
|
||||
//
|
||||
// Start event queue processing threads (worker threads)
|
||||
//
|
||||
lock (eventQueueThreadsLock)
|
||||
{
|
||||
for (int ThreadCount = 0; ThreadCount <= numberOfThreads; ThreadCount++)
|
||||
{
|
||||
Thread EventQueueThread = new Thread(EventQueueThreadLoop);
|
||||
eventQueueThreads.Add(EventQueueThread);
|
||||
EventQueueThread.IsBackground = true;
|
||||
EventQueueThread.Priority = ThreadPriority.BelowNormal;
|
||||
EventQueueThread.Name = "EventQueueManagerThread_" + ThreadCount;
|
||||
EventQueueThread.Start();
|
||||
StartNewThreadClass();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
~EventQueueManager()
|
||||
{
|
||||
// Kill worker threads
|
||||
foreach (Thread EventQueueThread in new ArrayList(eventQueueThreads))
|
||||
{
|
||||
if (EventQueueThread != null && EventQueueThread.IsAlive == true)
|
||||
{
|
||||
try
|
||||
{
|
||||
EventQueueThread.Abort();
|
||||
EventQueueThread.Join();
|
||||
}
|
||||
catch (Exception)
|
||||
if (ExecutionTimeoutEnforcingThread != null)
|
||||
{
|
||||
//myScriptEngine.Log.Verbose("ScriptEngine", "EventQueueManager Exception killing worker thread: " + e.ToString());
|
||||
if (ExecutionTimeoutEnforcingThread.IsAlive)
|
||||
{
|
||||
ExecutionTimeoutEnforcingThread.Abort();
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
// Kill worker threads
|
||||
lock (eventQueueThreadsLock)
|
||||
{
|
||||
foreach (EventQueueThreadClass EventQueueThread in new ArrayList(eventQueueThreads))
|
||||
{
|
||||
EventQueueThread.Shutdown();
|
||||
}
|
||||
eventQueueThreads.Clear();
|
||||
}
|
||||
// Todo: Clean up our queues
|
||||
eventQueue.Clear();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Queue processing thread loop
|
||||
/// </summary>
|
||||
private void EventQueueThreadLoop()
|
||||
{
|
||||
//myScriptEngine.m_logger.Verbose("ScriptEngine", "EventQueueManager Worker thread spawned");
|
||||
try
|
||||
{
|
||||
QueueItemStruct BlankQIS = new QueueItemStruct();
|
||||
while (true)
|
||||
{
|
||||
try
|
||||
{
|
||||
QueueItemStruct QIS = BlankQIS;
|
||||
bool GotItem = false;
|
||||
|
||||
if (eventQueue.Count == 0)
|
||||
{
|
||||
// Nothing to do? Sleep a bit waiting for something to do
|
||||
Thread.Sleep(nothingToDoSleepms);
|
||||
}
|
||||
else
|
||||
{
|
||||
// Something in queue, process
|
||||
//myScriptEngine.m_logger.Verbose("ScriptEngine", "Processing event for localID: " + QIS.localID + ", itemID: " + QIS.itemID + ", FunctionName: " + QIS.FunctionName);
|
||||
|
||||
// OBJECT BASED LOCK - TWO THREADS WORKING ON SAME OBJECT IS NOT GOOD
|
||||
lock (queueLock)
|
||||
{
|
||||
GotItem = false;
|
||||
for (int qc = 0; qc < eventQueue.Count; qc++)
|
||||
{
|
||||
// Get queue item
|
||||
QIS = eventQueue.Dequeue();
|
||||
|
||||
// Check if object is being processed by someone else
|
||||
if (TryLock(QIS.localID) == false)
|
||||
{
|
||||
// Object is already being processed, requeue it
|
||||
eventQueue.Enqueue(QIS);
|
||||
}
|
||||
else
|
||||
{
|
||||
// We have lock on an object and can process it
|
||||
GotItem = true;
|
||||
break;
|
||||
}
|
||||
} // go through queue
|
||||
} // lock
|
||||
|
||||
if (GotItem == true)
|
||||
{
|
||||
// Execute function
|
||||
try
|
||||
{
|
||||
#if DEBUG
|
||||
m_ScriptEngine.Log.Debug("ScriptEngine", "Executing event:\r\n"
|
||||
+ "QIS.localID: " + QIS.localID
|
||||
+ ", QIS.itemID: " + QIS.itemID
|
||||
+ ", QIS.functionName: " + QIS.functionName);
|
||||
#endif
|
||||
m_ScriptEngine.m_ScriptManager.ExecuteEvent(QIS.localID, QIS.itemID,
|
||||
QIS.functionName, QIS.llDetectParams, QIS.param);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
// DISPLAY ERROR INWORLD
|
||||
string text = "Error executing script function \"" + QIS.functionName + "\":\r\n";
|
||||
if (e.InnerException != null)
|
||||
{
|
||||
// Send inner exception
|
||||
text += e.InnerException.Message.ToString();
|
||||
}
|
||||
else
|
||||
{
|
||||
text += "\r\n";
|
||||
// Send normal
|
||||
text += e.Message.ToString();
|
||||
}
|
||||
try
|
||||
{
|
||||
if (text.Length > 1500)
|
||||
text = text.Substring(0, 1500);
|
||||
IScriptHost m_host = m_ScriptEngine.World.GetSceneObjectPart(QIS.localID);
|
||||
//if (m_host != null)
|
||||
//{
|
||||
m_ScriptEngine.World.SimChat(Helpers.StringToField(text), ChatTypeEnum.Say, 0,
|
||||
m_host.AbsolutePosition, m_host.Name, m_host.UUID);
|
||||
}
|
||||
catch
|
||||
{
|
||||
//}
|
||||
//else
|
||||
//{
|
||||
// T oconsole
|
||||
m_ScriptEngine.Log.Error("ScriptEngine",
|
||||
"Unable to send text in-world:\r\n" + text);
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseLock(QIS.localID);
|
||||
}
|
||||
}
|
||||
} // Something in queue
|
||||
}
|
||||
catch (ThreadAbortException tae)
|
||||
{
|
||||
throw tae;
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
m_ScriptEngine.Log.Error("ScriptEngine", "Exception in EventQueueThreadLoop: " + e.ToString());
|
||||
}
|
||||
} // while
|
||||
} // try
|
||||
catch (ThreadAbortException)
|
||||
{
|
||||
//myScriptEngine.Log.Verbose("ScriptEngine", "EventQueueManager Worker thread killed: " + tae.Message);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Try to get a mutex lock on localID
|
||||
/// </summary>
|
||||
/// <param name="localID"></param>
|
||||
/// <returns></returns>
|
||||
private bool TryLock(uint localID)
|
||||
public bool TryLock(uint localID)
|
||||
{
|
||||
lock (tryLockLock)
|
||||
{
|
||||
|
|
@ -318,7 +215,7 @@ namespace OpenSim.Region.ScriptEngine.Common.ScriptEngineBase
|
|||
/// Release mutex lock on localID
|
||||
/// </summary>
|
||||
/// <param name="localID"></param>
|
||||
private void ReleaseLock(uint localID)
|
||||
public void ReleaseLock(uint localID)
|
||||
{
|
||||
lock (tryLockLock)
|
||||
{
|
||||
|
|
@ -383,5 +280,58 @@ namespace OpenSim.Region.ScriptEngine.Common.ScriptEngineBase
|
|||
eventQueue.Enqueue(QIS);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A thread should run in this loop and check all running scripts
|
||||
/// </summary>
|
||||
public void ExecutionTimeoutEnforcingLoop()
|
||||
{
|
||||
try
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
System.Threading.Thread.Sleep(maxFunctionExecutionTimems);
|
||||
lock (eventQueueThreadsLock)
|
||||
{
|
||||
foreach (EventQueueThreadClass EventQueueThread in new ArrayList(eventQueueThreads))
|
||||
{
|
||||
if (EventQueueThread.InExecution)
|
||||
{
|
||||
if (DateTime.Now.Subtract(EventQueueThread.LastExecutionStarted).Milliseconds >
|
||||
maxFunctionExecutionTimems)
|
||||
{
|
||||
// We need to kill this thread!
|
||||
AbortThreadClass(EventQueueThread);
|
||||
// Then start another
|
||||
StartNewThreadClass();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (ThreadAbortException tae)
|
||||
{
|
||||
}
|
||||
}
|
||||
|
||||
private static void AbortThreadClass(EventQueueThreadClass threadClass)
|
||||
{
|
||||
try
|
||||
{
|
||||
threadClass.Shutdown();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Console.WriteLine("Could you please report this to Tedd:");
|
||||
Console.WriteLine("Script thread execution timeout kill ended in exception: " + ex.ToString());
|
||||
}
|
||||
}
|
||||
|
||||
private void StartNewThreadClass()
|
||||
{
|
||||
EventQueueThreadClass eqtc = new EventQueueThreadClass(this);
|
||||
eventQueueThreads.Add(eqtc);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,201 @@
|
|||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Text;
|
||||
using System.Threading;
|
||||
using libsecondlife;
|
||||
using OpenSim.Framework;
|
||||
using OpenSim.Region.Environment.Scenes.Scripting;
|
||||
|
||||
namespace OpenSim.Region.ScriptEngine.Common.ScriptEngineBase
|
||||
{
|
||||
/// <summary>
|
||||
/// Because every thread needs some data set for it (time started to execute current function), it will do its work within a class
|
||||
/// </summary>
|
||||
public class EventQueueThreadClass
|
||||
{
|
||||
/// <summary>
|
||||
/// How many ms to sleep if queue is empty
|
||||
/// </summary>
|
||||
private int nothingToDoSleepms = 50;
|
||||
|
||||
public DateTime LastExecutionStarted;
|
||||
public bool InExecution = false;
|
||||
|
||||
private EventQueueManager eventQueueManager;
|
||||
public Thread EventQueueThread;
|
||||
private static int ThreadCount = 0;
|
||||
|
||||
public EventQueueThreadClass(EventQueueManager eqm)
|
||||
{
|
||||
eventQueueManager = eqm;
|
||||
Start();
|
||||
}
|
||||
|
||||
~EventQueueThreadClass()
|
||||
{
|
||||
Shutdown();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Start thread
|
||||
/// </summary>
|
||||
private void Start()
|
||||
{
|
||||
EventQueueThread = new Thread(EventQueueThreadLoop);
|
||||
EventQueueThread.IsBackground = true;
|
||||
EventQueueThread.Priority = ThreadPriority.BelowNormal;
|
||||
EventQueueThread.Name = "EventQueueManagerThread_" + ThreadCount;
|
||||
EventQueueThread.Start();
|
||||
|
||||
// Look at this... Don't you wish everyone did that solid coding everywhere? :P
|
||||
if (ThreadCount == int.MaxValue)
|
||||
ThreadCount = 0;
|
||||
ThreadCount++;
|
||||
}
|
||||
|
||||
public void Shutdown()
|
||||
{
|
||||
if (EventQueueThread != null && EventQueueThread.IsAlive == true)
|
||||
{
|
||||
try
|
||||
{
|
||||
EventQueueThread.Abort();
|
||||
EventQueueThread.Join();
|
||||
}
|
||||
catch (Exception)
|
||||
{
|
||||
//myScriptEngine.Log.Verbose("ScriptEngine", "EventQueueManager Exception killing worker thread: " + e.ToString());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/// <summary>
|
||||
/// Queue processing thread loop
|
||||
/// </summary>
|
||||
private void EventQueueThreadLoop()
|
||||
{
|
||||
//myScriptEngine.m_logger.Verbose("ScriptEngine", "EventQueueManager Worker thread spawned");
|
||||
try
|
||||
{
|
||||
EventQueueManager.QueueItemStruct BlankQIS = new EventQueueManager.QueueItemStruct();
|
||||
while (true)
|
||||
{
|
||||
try
|
||||
{
|
||||
EventQueueManager.QueueItemStruct QIS = BlankQIS;
|
||||
bool GotItem = false;
|
||||
|
||||
if (eventQueueManager.eventQueue.Count == 0)
|
||||
{
|
||||
// Nothing to do? Sleep a bit waiting for something to do
|
||||
Thread.Sleep(nothingToDoSleepms);
|
||||
}
|
||||
else
|
||||
{
|
||||
// Something in queue, process
|
||||
//myScriptEngine.m_logger.Verbose("ScriptEngine", "Processing event for localID: " + QIS.localID + ", itemID: " + QIS.itemID + ", FunctionName: " + QIS.FunctionName);
|
||||
|
||||
// OBJECT BASED LOCK - TWO THREADS WORKING ON SAME OBJECT IS NOT GOOD
|
||||
lock (eventQueueManager.queueLock)
|
||||
{
|
||||
GotItem = false;
|
||||
for (int qc = 0; qc < eventQueueManager.eventQueue.Count; qc++)
|
||||
{
|
||||
// Get queue item
|
||||
QIS = eventQueueManager.eventQueue.Dequeue();
|
||||
|
||||
// Check if object is being processed by someone else
|
||||
if (eventQueueManager.TryLock(QIS.localID) == false)
|
||||
{
|
||||
// Object is already being processed, requeue it
|
||||
eventQueueManager.eventQueue.Enqueue(QIS);
|
||||
}
|
||||
else
|
||||
{
|
||||
// We have lock on an object and can process it
|
||||
GotItem = true;
|
||||
break;
|
||||
}
|
||||
} // go through queue
|
||||
} // lock
|
||||
|
||||
if (GotItem == true)
|
||||
{
|
||||
// Execute function
|
||||
try
|
||||
{
|
||||
#if DEBUG
|
||||
eventQueueManager.m_ScriptEngine.Log.Debug("ScriptEngine", "Executing event:\r\n"
|
||||
+ "QIS.localID: " + QIS.localID
|
||||
+ ", QIS.itemID: " + QIS.itemID
|
||||
+ ", QIS.functionName: " + QIS.functionName);
|
||||
#endif
|
||||
LastExecutionStarted = DateTime.Now;
|
||||
InExecution = true;
|
||||
eventQueueManager.m_ScriptEngine.m_ScriptManager.ExecuteEvent(QIS.localID, QIS.itemID,
|
||||
QIS.functionName, QIS.llDetectParams, QIS.param);
|
||||
InExecution = false;
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
InExecution = false;
|
||||
// DISPLAY ERROR INWORLD
|
||||
string text = "Error executing script function \"" + QIS.functionName + "\":\r\n";
|
||||
if (e.InnerException != null)
|
||||
{
|
||||
// Send inner exception
|
||||
text += e.InnerException.Message.ToString();
|
||||
}
|
||||
else
|
||||
{
|
||||
text += "\r\n";
|
||||
// Send normal
|
||||
text += e.Message.ToString();
|
||||
}
|
||||
try
|
||||
{
|
||||
if (text.Length > 1500)
|
||||
text = text.Substring(0, 1500);
|
||||
IScriptHost m_host = eventQueueManager.m_ScriptEngine.World.GetSceneObjectPart(QIS.localID);
|
||||
//if (m_host != null)
|
||||
//{
|
||||
eventQueueManager.m_ScriptEngine.World.SimChat(Helpers.StringToField(text), ChatTypeEnum.Say, 0,
|
||||
m_host.AbsolutePosition, m_host.Name, m_host.UUID);
|
||||
}
|
||||
catch
|
||||
{
|
||||
//}
|
||||
//else
|
||||
//{
|
||||
// T oconsole
|
||||
eventQueueManager.m_ScriptEngine.Log.Error("ScriptEngine",
|
||||
"Unable to send text in-world:\r\n" + text);
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
InExecution = false;
|
||||
eventQueueManager.ReleaseLock(QIS.localID);
|
||||
}
|
||||
}
|
||||
} // Something in queue
|
||||
}
|
||||
catch (ThreadAbortException tae)
|
||||
{
|
||||
throw tae;
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
eventQueueManager.m_ScriptEngine.Log.Error("ScriptEngine", "Exception in EventQueueThreadLoop: " + e.ToString());
|
||||
}
|
||||
} // while
|
||||
} // try
|
||||
catch (ThreadAbortException)
|
||||
{
|
||||
//myScriptEngine.Log.Verbose("ScriptEngine", "EventQueueManager Worker thread killed: " + tae.Message);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue