From 8728d4ce6ae42cd60937836776d0b13289c73f72 Mon Sep 17 00:00:00 2001 From: "Justin Clark-Casey (justincc)" Date: Wed, 16 Jan 2013 00:12:40 +0000 Subject: [PATCH] Implement co-operative script termination if termination comes during a script wait event (llSleep(), etc.) This makes use of EventWaitHandles since various web references indicate that Thread.Interrupt() can also cause runtime instability. If co-op termination is enabled, then termination sets the wait handle instead of waiting for a timeout before possibly aborting the thread. This allows the script to cleanly terminate if it's in a llSleep/LL function delay or the next time it enters such a wait without any timeout period. Co-op termination is not yet testable since checking for termination request within loops that never trigger a wait is not yet implemented. --- .../Framework/Scenes/Scene.Inventory.cs | 26 ++- .../Interfaces/IScriptInstance.cs | 13 ++ .../Shared/Api/Implementation/LSL_Api.cs | 31 +++- OpenSim/Region/ScriptEngine/Shared/Helpers.cs | 18 ++ .../Shared/Instance/ScriptInstance.cs | 52 +++++- .../Instance/Tests/CoopTerminationTests.cs | 157 ++++++++++++++++++ .../Region/ScriptEngine/XEngine/XEngine.cs | 5 + prebuild.xml | 5 +- 8 files changed, 293 insertions(+), 14 deletions(-) create mode 100644 OpenSim/Region/ScriptEngine/Shared/Instance/Tests/CoopTerminationTests.cs diff --git a/OpenSim/Region/Framework/Scenes/Scene.Inventory.cs b/OpenSim/Region/Framework/Scenes/Scene.Inventory.cs index 1e56bb70a9..12d70130ad 100644 --- a/OpenSim/Region/Framework/Scenes/Scene.Inventory.cs +++ b/OpenSim/Region/Framework/Scenes/Scene.Inventory.cs @@ -1739,6 +1739,21 @@ namespace OpenSim.Region.Framework.Scenes /// /// The part where the script was rezzed if successful. False otherwise. public SceneObjectPart RezNewScript(UUID agentID, InventoryItemBase itemBase) + { + return RezNewScript( + agentID, + itemBase, + "default\n{\n state_entry()\n {\n llSay(0, \"Script running\");\n }\n}"); + } + + /// + /// Rez a new script from nothing with given script text. + /// + /// + /// Template item. + /// + /// The part where the script was rezzed if successful. False otherwise. + public SceneObjectPart RezNewScript(UUID agentID, InventoryItemBase itemBase, string scriptText) { // The part ID is the folder ID! SceneObjectPart part = GetSceneObjectPart(itemBase.Folder); @@ -1759,9 +1774,14 @@ namespace OpenSim.Region.Framework.Scenes return null; } - AssetBase asset = CreateAsset(itemBase.Name, itemBase.Description, (sbyte)itemBase.AssetType, - Encoding.ASCII.GetBytes("default\n{\n state_entry()\n {\n llSay(0, \"Script running\");\n }\n}"), - agentID); + AssetBase asset + = CreateAsset( + itemBase.Name, + itemBase.Description, + (sbyte)itemBase.AssetType, + Encoding.ASCII.GetBytes(scriptText), + agentID); + AssetService.Store(asset); TaskInventoryItem taskItem = new TaskInventoryItem(); diff --git a/OpenSim/Region/ScriptEngine/Interfaces/IScriptInstance.cs b/OpenSim/Region/ScriptEngine/Interfaces/IScriptInstance.cs index 9de2d725f2..38fff521a8 100644 --- a/OpenSim/Region/ScriptEngine/Interfaces/IScriptInstance.cs +++ b/OpenSim/Region/ScriptEngine/Interfaces/IScriptInstance.cs @@ -28,6 +28,7 @@ using System; using System.Collections; using System.Collections.Generic; +using System.Threading; using OpenMetaverse; using log4net; using OpenSim.Framework; @@ -180,6 +181,18 @@ namespace OpenSim.Region.ScriptEngine.Interfaces void Suspend(); void Resume(); + /// + /// If true then scripts should look to terminate their threads in co-operation with the script engine rather + /// than through Thread.Abort() + /// + bool CoopTermination { get; } + + /// + /// Used for script sleeps when we are using co-operative script termination. + /// + /// null if CoopTermination is not active + EventWaitHandle CoopSleepHandle { get; } + /// /// Process the next event queued for this script instance. /// diff --git a/OpenSim/Region/ScriptEngine/Shared/Api/Implementation/LSL_Api.cs b/OpenSim/Region/ScriptEngine/Shared/Api/Implementation/LSL_Api.cs index a1f28f5422..5b54293962 100644 --- a/OpenSim/Region/ScriptEngine/Shared/Api/Implementation/LSL_Api.cs +++ b/OpenSim/Region/ScriptEngine/Shared/Api/Implementation/LSL_Api.cs @@ -82,6 +82,12 @@ namespace OpenSim.Region.ScriptEngine.Shared.Api public class LSL_Api : MarshalByRefObject, ILSL_Api, IScriptApi { private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); + + /// + /// Instance of this script. + /// + protected IScriptInstance m_scriptInstance; + protected IScriptEngine m_ScriptEngine; protected SceneObjectPart m_host; @@ -109,11 +115,12 @@ namespace OpenSim.Region.ScriptEngine.Shared.Api public void Initialize(IScriptInstance scriptInstance) { - m_ScriptEngine = scriptInstance.Engine; - m_host = scriptInstance.Part; - m_item = scriptInstance.ScriptTask; + m_scriptInstance = scriptInstance; + m_ScriptEngine = m_scriptInstance.Engine; + m_host = m_scriptInstance.Part; + m_item = m_scriptInstance.ScriptTask; - LoadLimits(); // read script limits from config. + LoadConfig(); m_TransferModule = m_ScriptEngine.World.RequestModuleInterface(); @@ -125,7 +132,7 @@ namespace OpenSim.Region.ScriptEngine.Shared.Api /// /// Load configuration items that affect script, object and run-time behavior. */ /// - private void LoadLimits() + private void LoadConfig() { m_ScriptDelayFactor = m_ScriptEngine.Config.GetFloat("ScriptDelayFactor", 1.0f); @@ -166,7 +173,16 @@ namespace OpenSim.Region.ScriptEngine.Shared.Api delay = (int)((float)delay * m_ScriptDelayFactor); if (delay == 0) return; - System.Threading.Thread.Sleep(delay); + + Sleep(delay); + } + + protected virtual void Sleep(int delay) + { + if (!m_scriptInstance.CoopTermination) + System.Threading.Thread.Sleep(delay); + else if (m_scriptInstance.CoopSleepHandle.WaitOne(delay)) + throw new ScriptCoopStopException(); } public Scene World @@ -2944,7 +2960,8 @@ namespace OpenSim.Region.ScriptEngine.Shared.Api { // m_log.Info("llSleep snoozing " + sec + "s."); m_host.AddScriptLPS(1); - Thread.Sleep((int)(sec * 1000)); + + Sleep((int)(sec * 1000)); } public LSL_Float llGetMass() diff --git a/OpenSim/Region/ScriptEngine/Shared/Helpers.cs b/OpenSim/Region/ScriptEngine/Shared/Helpers.cs index 0108f447f3..a3c2a47645 100644 --- a/OpenSim/Region/ScriptEngine/Shared/Helpers.cs +++ b/OpenSim/Region/ScriptEngine/Shared/Helpers.cs @@ -81,6 +81,24 @@ namespace OpenSim.Region.ScriptEngine.Shared } } + /// + /// Used to signal when the script is stopping in co-operation with the script engine + /// (instead of through Thread.Abort()). + /// + [Serializable] + public class ScriptCoopStopException : Exception + { + public ScriptCoopStopException() + { + } + + protected ScriptCoopStopException( + SerializationInfo info, + StreamingContext context) + { + } + } + public class DetectParams { public const int AGENT = 1; diff --git a/OpenSim/Region/ScriptEngine/Shared/Instance/ScriptInstance.cs b/OpenSim/Region/ScriptEngine/Shared/Instance/ScriptInstance.cs index a2ff51b59f..00048a1dfc 100644 --- a/OpenSim/Region/ScriptEngine/Shared/Instance/ScriptInstance.cs +++ b/OpenSim/Region/ScriptEngine/Shared/Instance/ScriptInstance.cs @@ -200,6 +200,10 @@ namespace OpenSim.Region.ScriptEngine.Shared.Instance public static readonly long MaxMeasurementPeriod = 30 * TimeSpan.TicksPerMinute; + public bool CoopTermination { get; private set; } + + public EventWaitHandle CoopSleepHandle { get; private set; } + public void ClearQueue() { m_TimerQueued = false; @@ -233,6 +237,12 @@ namespace OpenSim.Region.ScriptEngine.Shared.Instance m_postOnRez = postOnRez; m_AttachedAvatar = Part.ParentGroup.AttachedAvatar; m_RegionID = Part.ParentGroup.Scene.RegionInfo.RegionID; + + if (Engine.Config.GetString("ScriptStopStrategy", "abort") == "co-op") + { + CoopTermination = true; + CoopSleepHandle = new AutoResetEvent(false); + } } /// @@ -532,9 +542,34 @@ namespace OpenSim.Region.ScriptEngine.Shared.Instance } // Wait for the current event to complete. - if (!m_InSelfDelete && workItem.Wait(new TimeSpan((long)timeout * 100000))) + if (!m_InSelfDelete) { - return true; + if (!CoopTermination) + { + // If we're not co-operative terminating then try and wait for the event to complete before stopping + if (workItem.Wait(new TimeSpan((long)timeout * 100000))) + return true; + } + else + { + m_log.DebugFormat( + "[SCRIPT INSTANCE]: Co-operatively stopping script {0} {1} in {2} {3}", + ScriptName, ItemID, PrimName, ObjectID); + + // This will terminate the event on next handle check by the script. + CoopSleepHandle.Set(); + + // For now, we will wait forever since the event should always cleanly terminate once LSL loop + // checking is implemented. May want to allow a shorter timeout option later. + if (workItem.Wait(TimeSpan.MaxValue)) + { + m_log.DebugFormat( + "[SCRIPT INSTANCE]: Co-operatively stopped script {0} {1} in {2} {3}", + ScriptName, ItemID, PrimName, ObjectID); + + return true; + } + } } lock (EventQueue) @@ -547,6 +582,7 @@ namespace OpenSim.Region.ScriptEngine.Shared.Instance // If the event still hasn't stopped and we the stop isn't the result of script or object removal, then // forcibly abort the work item (this aborts the underlying thread). + // Co-operative termination should never reach this point. if (!m_InSelfDelete) { m_log.DebugFormat( @@ -786,7 +822,11 @@ namespace OpenSim.Region.ScriptEngine.Shared.Instance m_InEvent = false; m_CurrentEvent = String.Empty; - if ((!(e is TargetInvocationException) || (!(e.InnerException is SelfDeleteException) && !(e.InnerException is ScriptDeleteException))) && !(e is ThreadAbortException)) + if ((!(e is TargetInvocationException) + || (!(e.InnerException is SelfDeleteException) + && !(e.InnerException is ScriptDeleteException) + && !(e.InnerException is ScriptCoopStopException))) + && !(e is ThreadAbortException)) { try { @@ -834,6 +874,12 @@ namespace OpenSim.Region.ScriptEngine.Shared.Instance m_InSelfDelete = true; Part.Inventory.RemoveInventoryItem(ItemID); } + else if ((e is TargetInvocationException) && (e.InnerException is ScriptCoopStopException)) + { + m_log.DebugFormat( + "[SCRIPT INSTANCE]: Script {0}.{1} in event {2}, state {3} stopped co-operatively.", + PrimName, ScriptName, data.EventName, State); + } } } } diff --git a/OpenSim/Region/ScriptEngine/Shared/Instance/Tests/CoopTerminationTests.cs b/OpenSim/Region/ScriptEngine/Shared/Instance/Tests/CoopTerminationTests.cs new file mode 100644 index 0000000000..f3a6cc9503 --- /dev/null +++ b/OpenSim/Region/ScriptEngine/Shared/Instance/Tests/CoopTerminationTests.cs @@ -0,0 +1,157 @@ +/* + * Copyright (c) Contributors, http://opensimulator.org/ + * See CONTRIBUTORS.TXT for a full list of copyright holders. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the OpenSimulator Project nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +using System; +using System.Collections.Generic; +using System.Threading; +using Nini.Config; +using NUnit.Framework; +using OpenMetaverse; +using OpenSim.Framework; +using OpenSim.Region.CoreModules.Scripting.WorldComm; +using OpenSim.Region.Framework.Scenes; +using OpenSim.Region.Framework.Interfaces; +using OpenSim.Region.ScriptEngine.XEngine; +using OpenSim.Tests.Common; +using OpenSim.Tests.Common.Mock; + +namespace OpenSim.Region.ScriptEngine.Shared.Instance.Tests +{ + /// + /// Test that co-operative script thread termination is working correctly. + /// + [TestFixture] + public class CoopTerminationTests : OpenSimTestCase + { + private TestScene m_scene; + private OpenSim.Region.ScriptEngine.XEngine.XEngine m_xEngine; + + private AutoResetEvent m_chatEvent = new AutoResetEvent(false); + private AutoResetEvent m_stoppedEvent = new AutoResetEvent(false); + + private OSChatMessage m_osChatMessageReceived; + + [TestFixtureSetUp] + public void Init() + { + //AppDomain.CurrentDomain.SetData("APPBASE", Environment.CurrentDirectory + "/bin"); +// Console.WriteLine(AppDomain.CurrentDomain.BaseDirectory); + m_xEngine = new OpenSim.Region.ScriptEngine.XEngine.XEngine(); + + IniConfigSource configSource = new IniConfigSource(); + + IConfig startupConfig = configSource.AddConfig("Startup"); + startupConfig.Set("DefaultScriptEngine", "XEngine"); + + IConfig xEngineConfig = configSource.AddConfig("XEngine"); + xEngineConfig.Set("Enabled", "true"); + xEngineConfig.Set("StartDelay", "0"); + + // These tests will not run with AppDomainLoading = true, at least on mono. For unknown reasons, the call + // to AssemblyResolver.OnAssemblyResolve fails. + xEngineConfig.Set("AppDomainLoading", "false"); + + xEngineConfig.Set("ScriptStopStrategy", "co-op"); + + m_scene = new SceneHelpers().SetupScene("My Test", UUID.Random(), 1000, 1000, configSource); + SceneHelpers.SetupSceneModules(m_scene, configSource, m_xEngine); + m_scene.StartScripts(); + } + + /// + /// Test co-operative termination on derez of an object containing a script with a long-running event. + /// + /// + /// TODO: Actually compiling the script is incidental to this test. Really want a way to compile test scripts + /// within the build itself. + /// + [Test] + public void TestStopOnLongSleep() + { + TestHelpers.InMethod(); + TestHelpers.EnableLogging(); + + UUID userId = TestHelpers.ParseTail(0x1); +// UUID objectId = TestHelpers.ParseTail(0x100); +// UUID itemId = TestHelpers.ParseTail(0x3); + string itemName = "TestStopOnObjectDerezLongSleep() Item"; + + SceneObjectGroup so = SceneHelpers.CreateSceneObject(1, userId, "TestStopOnObjectDerezLongSleep", 0x100); + m_scene.AddNewSceneObject(so, true); + + InventoryItemBase itemTemplate = new InventoryItemBase(); +// itemTemplate.ID = itemId; + itemTemplate.Name = itemName; + itemTemplate.Folder = so.UUID; + itemTemplate.InvType = (int)InventoryType.LSL; + + m_scene.EventManager.OnChatFromWorld += OnChatFromWorld; + + SceneObjectPart partWhereRezzed = m_scene.RezNewScript(userId, itemTemplate, +@"default +{ + state_entry() + { + llSay(0, ""Thin Lizzy""); + llSleep(60); + } +}"); + + TaskInventoryItem rezzedItem = partWhereRezzed.Inventory.GetInventoryItem(itemName); + + // Wait for the script to start the event before we try stopping it. + m_chatEvent.WaitOne(60000); + + Console.WriteLine("Script started with message [{0}]", m_osChatMessageReceived.Message); + + // FIXME: This is a very poor way of trying to avoid a low-probability race condition where the script + // executes llSay() but has not started the sleep before we try to stop it. + Thread.Sleep(1000); + + // We need a way of carrying on if StopScript() fail, since it won't return if the script isn't actually + // stopped. This kind of multi-threading is far from ideal in a regression test. + new Thread(() => { m_xEngine.StopScript(rezzedItem.ItemID); m_stoppedEvent.Set(); }).Start(); + + if (!m_stoppedEvent.WaitOne(30000)) + Assert.Fail("Script did not co-operatively stop."); + + bool running; + TaskInventoryItem scriptItem = partWhereRezzed.Inventory.GetInventoryItem(itemName); + Assert.That( + SceneObjectPartInventory.TryGetScriptInstanceRunning(m_scene, scriptItem, out running), Is.True); + Assert.That(running, Is.False); + } + + private void OnChatFromWorld(object sender, OSChatMessage oscm) + { +// Console.WriteLine("Got chat [{0}]", oscm.Message); + + m_osChatMessageReceived = oscm; + m_chatEvent.Set(); + } + } +} \ No newline at end of file diff --git a/OpenSim/Region/ScriptEngine/XEngine/XEngine.cs b/OpenSim/Region/ScriptEngine/XEngine/XEngine.cs index c2be37e435..82ddbec761 100644 --- a/OpenSim/Region/ScriptEngine/XEngine/XEngine.cs +++ b/OpenSim/Region/ScriptEngine/XEngine/XEngine.cs @@ -1716,9 +1716,14 @@ namespace OpenSim.Region.ScriptEngine.XEngine IScriptInstance instance = GetInstance(itemID); if (instance != null) + { instance.Stop(m_WaitForEventCompletionOnScriptStop); + } else + { +// m_log.DebugFormat("[XENGINE]: Could not find script with ID {0} to stop in {1}", itemID, World.Name); m_runFlags.AddOrUpdate(itemID, false, 240); + } } public DetectParams GetDetectParams(UUID itemID, int idx) diff --git a/prebuild.xml b/prebuild.xml index 078df6355f..529b494d96 100644 --- a/prebuild.xml +++ b/prebuild.xml @@ -2372,7 +2372,9 @@ - + + + @@ -3262,6 +3264,7 @@ +