* Switched all operations on the list of clients that could be either sync or async to use Scene.ForEachClient() instead of referencing ClientManager directly

* Added a new [Startup] config option called use_async_when_possible to signal how to run operations that could be either sync or async
* Changed Scene.ForEachClient to respect use_async_when_possible
* Fixing a potential deadlock in Parallel.ForEach by locking on a temporary object instead of the enumerator (which may be shared across multiple invocations on ForEach). Thank you diva

OpenSim/Framework/Parallel.cs

@@ -118,6 +118,7 @@ namespace OpenSim.Framework
             int counter = threadCount;
             AutoResetEvent threadFinishEvent = new AutoResetEvent(false);
             IEnumerator<T> enumerator = enumerable.GetEnumerator();
+            object syncRoot = new object();
             Exception exception = null;
 
             for (int i = 0; i < threadCount; i++)
@@ -131,7 +132,7 @@ namespace OpenSim.Framework
                         {
                             T entry;
 
-                            lock (enumerator)
+                            lock (syncRoot)
                             {
                                 if (!enumerator.MoveNext())
                                     break;

OpenSim/Region/ClientStack/LindenUDP/LLUDPServer.cs

@@ -262,7 +262,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP
                 for (int i = 0; i < packetCount; i++)
                 {
                     byte[] data = datas[i];
-                    m_scene.ClientManager.ForEach(
+                    m_scene.ForEachClient(
                         delegate(IClientAPI client)
                         {
                             if (client is LLClientView)
@@ -274,7 +274,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP
             else
             {
                 byte[] data = packet.ToBytes();
-                m_scene.ClientManager.ForEach(
+                m_scene.ForEachClient(
                     delegate(IClientAPI client)
                     {
                         if (client is LLClientView)

OpenSim/Region/CoreModules/Avatar/Gods/GodsModule.cs

@@ -114,7 +114,7 @@ namespace OpenSim.Region.CoreModules.Avatar.Gods
                     {
                         string reasonStr = Utils.BytesToString(reason);
 
-                        m_scene.ClientManager.ForEach(
+                        m_scene.ForEachClient(
                             delegate(IClientAPI controller)
                             {
                                 if (controller.AgentId != godID)

OpenSim/Region/Framework/Scenes/Scene.PacketHandlers.cs

@@ -408,7 +408,7 @@ namespace OpenSim.Region.Framework.Scenes
             }
             ViewerEffectPacket.EffectBlock[] effectBlockArray = effectBlock.ToArray();
 
-            ClientManager.ForEach(
+            ForEachClient(
                 delegate(IClientAPI client)
                 {
                     if (client.AgentId != remoteClient.AgentId)

OpenSim/Region/Framework/Scenes/Scene.cs

@@ -106,11 +106,11 @@ namespace OpenSim.Region.Framework.Scenes
         public bool m_physicalPrim;
         public float m_maxNonphys = 256;
         public float m_maxPhys = 10;
-        public bool m_clampPrimSize = false;
+        public bool m_clampPrimSize;
-        public bool m_trustBinaries = false;
+        public bool m_trustBinaries;
-        public bool m_allowScriptCrossings = false;
+        public bool m_allowScriptCrossings;
-        public bool m_useFlySlow = false;
+        public bool m_useFlySlow;
-        public bool m_usePreJump = false;
+        public bool m_usePreJump;
         public bool m_seeIntoRegionFromNeighbor;
         // TODO: need to figure out how allow client agents but deny
         // root agents when ACL denies access to root agent
@@ -118,11 +118,11 @@ namespace OpenSim.Region.Framework.Scenes
         public int MaxUndoCount = 5;
         private int m_RestartTimerCounter;
         private readonly Timer m_restartTimer = new Timer(15000); // Wait before firing
-        private int m_incrementsof15seconds = 0;
+        private int m_incrementsof15seconds;
-        private volatile bool m_backingup = false;
+        private volatile bool m_backingup;
+        private bool m_useAsyncWhenPossible = true;
 
         private Dictionary<UUID, ReturnInfo> m_returns = new Dictionary<UUID, ReturnInfo>();
-        
         private Dictionary<UUID, SceneObjectGroup> m_groupsWithTargets = new Dictionary<UUID, SceneObjectGroup>();
 
         protected string m_simulatorVersion = "OpenSimulator Server";
@@ -142,8 +142,8 @@ namespace OpenSim.Region.Framework.Scenes
 
         public IXfer XferManager;
 
-        protected IAssetService m_AssetService = null;
+        protected IAssetService m_AssetService;
-        protected IAuthorizationService m_AuthorizationService = null;
+        protected IAuthorizationService m_AuthorizationService;
 
         private Object m_heartbeatLock = new Object();
 
@@ -184,7 +184,7 @@ namespace OpenSim.Region.Framework.Scenes
             }
         }
 
-        protected IInventoryService m_InventoryService = null;
+        protected IInventoryService m_InventoryService;
 
         public IInventoryService InventoryService
         {
@@ -204,7 +204,7 @@ namespace OpenSim.Region.Framework.Scenes
             }
         }
 
-        protected IGridService m_GridService = null;
+        protected IGridService m_GridService;
 
         public IGridService GridService
         {
@@ -252,7 +252,7 @@ namespace OpenSim.Region.Framework.Scenes
         // Central Update Loop
 
         protected int m_fps = 10;
-        protected int m_frame = 0;
+        protected int m_frame;
         protected float m_timespan = 0.089f;
         protected DateTime m_lastupdate = DateTime.UtcNow;
 
@@ -265,17 +265,17 @@ namespace OpenSim.Region.Framework.Scenes
         private int m_update_terrain = 50;
         private int m_update_land = 1;
 
-        private int frameMS = 0;
+        private int frameMS;
-        private int physicsMS2 = 0;
+        private int physicsMS2;
-        private int physicsMS = 0;
+        private int physicsMS;
-        private int otherMS = 0;
+        private int otherMS;
 
         private bool m_physics_enabled = true;
         private bool m_scripts_enabled = true;
         private string m_defaultScriptEngine;
-        private int m_LastLogin = 0;
+        private int m_LastLogin;
-        private Thread HeartbeatThread = null;
+        private Thread HeartbeatThread;
-        private volatile bool shuttingdown = false;
+        private volatile bool shuttingdown;
 
         private int m_lastUpdate = Environment.TickCount;
         private bool m_firstHeartbeat = true;
@@ -479,6 +479,9 @@ namespace OpenSim.Region.Framework.Scenes
                 //
                 IConfig startupConfig = m_config.Configs["Startup"];
 
+                // Should we try to run loops synchronously or asynchronously?
+                m_useAsyncWhenPossible = startupConfig.GetBoolean("use_async_when_possible", true);
+
                 //Animation states
                 m_useFlySlow = startupConfig.GetBoolean("enableflyslow", false);
                 // TODO: Change default to true once the feature is supported
@@ -4253,7 +4256,10 @@ namespace OpenSim.Region.Framework.Scenes
 
         public void ForEachClient(Action<IClientAPI> action)
         {
-            ClientManager.ForEachSync(action);
+            if (m_useAsyncWhenPossible)
+                ClientManager.ForEach(action);
+            else
+                ClientManager.ForEachSync(action);
         }
 
         public void ForEachSOG(Action<SceneObjectGroup> action)

OpenSim/Region/OptionalModules/ContentManagementSystem/MetaEntity.cs

@@ -183,10 +183,12 @@ namespace OpenSim.Region.OptionalModules.ContentManagement
         public virtual void HideFromAll()
         {
             foreach (SceneObjectPart part in m_Entity.Children.Values)
-                m_Entity.Scene.ClientManager.ForEach(
+            {
+                m_Entity.Scene.ForEachClient(
                     delegate(IClientAPI controller)
                     { controller.SendKillObject(m_Entity.RegionHandle, part.LocalId); }
                 );
+            }
         }
 
         public void SendFullUpdate(IClientAPI client)
@@ -202,7 +204,7 @@ namespace OpenSim.Region.OptionalModules.ContentManagement
 
         public void SendFullUpdateToAll()
         {
-            m_Entity.Scene.ClientManager.ForEach(
+            m_Entity.Scene.ForEachClient(
                 delegate(IClientAPI controller)
                 { m_Entity.SendFullUpdateToClient(controller); }
             );

bin/OpenSim.ini.example

@@ -44,6 +44,13 @@
     ; performance on .NET/Windows
     ;async_call_method = SmartThreadPool
     
+    ; There are several operations on large collections (such as
+    ; the current avatar list) that can be run synchronously or
+    ; in parallel. Running in parallel should increase performance
+    ; on a multi-core system, but will make debugging more
+    ; difficult if something deadlocks or times out
+    use_async_when_possible = true
+    
     ; Max threads to allocate on the FireAndForget thread pool
     ; when running with the SmartThreadPool option above
     MaxPoolThreads = 15