Add "show threadpool calls active" console debug command.

This shows named threadpool calls (excluding timer and network calls) that are currently queued or running.
Also shows total of labelled and any anonymous calls.

OpenSim/Framework/Servers/ServerBase.cs

@@ -293,10 +293,16 @@ namespace OpenSim.Framework.Servers
                 HandleDebugThreadpoolLevel);
 
             m_console.Commands.AddCommand(
-                "Debug", false, "show threadpool calls",
+                "Debug", false, "show threadpool calls active",
-                "show threadpool calls",
+                "show threadpool calls active",
-                "Show the number of labelled threadpool calls.",
+                "Show details about threadpool calls that are still active (currently waiting or in progress)",
-                HandleShowThreadpoolCalls);
+                HandleShowThreadpoolCallsActive);
+
+            m_console.Commands.AddCommand(
+                "Debug", false, "show threadpool calls complete",
+                "show threadpool calls complete",
+                "Show details about threadpool calls that have been completed.",
+                HandleShowThreadpoolCallsComplete);
 
             m_console.Commands.AddCommand(
                 "Debug", false, "force gc",
@@ -361,7 +367,36 @@ namespace OpenSim.Framework.Servers
             Notice("serialosdreq is now {0}", setSerializeOsdRequests);
         }
 
-        private void HandleShowThreadpoolCalls(string module, string[] args)
+        private void HandleShowThreadpoolCallsActive(string module, string[] args)
+        {
+            List<KeyValuePair<string, int>> calls = Util.GetFireAndForgetCallsInProgress().ToList();
+            calls.Sort((kvp1, kvp2) => kvp2.Value.CompareTo(kvp1.Value));
+            int namedCalls = 0;
+
+            ConsoleDisplayList cdl = new ConsoleDisplayList();
+            foreach (KeyValuePair<string, int> kvp in calls)
+            {
+                if (kvp.Value > 0)
+                {
+                    cdl.AddRow(kvp.Key, kvp.Value);
+                    namedCalls += kvp.Value;
+                }
+            }
+
+            cdl.AddRow("TOTAL NAMED", namedCalls);
+
+            long allQueuedCalls = Util.TotalQueuedFireAndForgetCalls;
+            long allRunningCalls = Util.TotalRunningFireAndForgetCalls;
+
+            cdl.AddRow("TOTAL QUEUED", allQueuedCalls);
+            cdl.AddRow("TOTAL RUNNING", allRunningCalls);
+            cdl.AddRow("TOTAL ANONYMOUS", allQueuedCalls + allRunningCalls - namedCalls);
+            cdl.AddRow("TOTAL ALL", allQueuedCalls + allRunningCalls);
+
+            MainConsole.Instance.Output(cdl.ToString());
+        }
+
+        private void HandleShowThreadpoolCallsComplete(string module, string[] args)
         {
             List<KeyValuePair<string, int>> calls = Util.GetFireAndForgetCallsMade().ToList();
             calls.Sort((kvp1, kvp2) => kvp2.Value.CompareTo(kvp1.Value));

OpenSim/Framework/Util.cs

@@ -2052,6 +2052,9 @@ namespace OpenSim.Framework
         private static long numTotalThreadFuncsCalled = 0;
         private static Int32 threadFuncOverloadMode = 0;
 
+        public static long TotalQueuedFireAndForgetCalls { get { return numQueuedThreadFuncs; } }
+        public static long TotalRunningFireAndForgetCalls { get { return numRunningThreadFuncs; } }
+
         // Maps (ThreadFunc number -> Thread)
         private static ConcurrentDictionary<long, ThreadInfo> activeThreads = new ConcurrentDictionary<long, ThreadInfo>();
 
@@ -2089,6 +2092,13 @@ namespace OpenSim.Framework
 
         private static Dictionary<string, int> m_fireAndForgetCallsMade = new Dictionary<string, int>();
 
+        public static Dictionary<string, int> GetFireAndForgetCallsInProgress()
+        {
+            return new Dictionary<string, int>(m_fireAndForgetCallsInProgress);
+        }
+
+        private static Dictionary<string, int> m_fireAndForgetCallsInProgress = new Dictionary<string, int>();
+
         public static void FireAndForget(System.Threading.WaitCallback callback)
         {
             FireAndForget(callback, null, null);
@@ -2109,6 +2119,11 @@ namespace OpenSim.Framework
                     m_fireAndForgetCallsMade[context] = 1;
                 else
                     m_fireAndForgetCallsMade[context]++;
+
+                if (!m_fireAndForgetCallsInProgress.ContainsKey(context))
+                    m_fireAndForgetCallsInProgress[context] = 1;
+                else
+                    m_fireAndForgetCallsInProgress[context]++;
             }
 
             WaitCallback realCallback;
@@ -2121,7 +2136,15 @@ namespace OpenSim.Framework
             if (FireAndForgetMethod == FireAndForgetMethod.RegressionTest)
             {
                 // If we're running regression tests, then we want any exceptions to rise up to the test code.
-                realCallback = o => { Culture.SetCurrentCulture(); callback(o); };
+                realCallback = 
+                    o => 
+                    { 
+                        Culture.SetCurrentCulture(); 
+                        callback(o); 
+                        
+                        if (context != null)
+                            m_fireAndForgetCallsInProgress[context]--;
+                    };
             }
             else
             {
@@ -2160,6 +2183,9 @@ namespace OpenSim.Framework
                         activeThreads.TryRemove(threadFuncNum, out dummy);
                         if ((loggingEnabled || (threadFuncOverloadMode == 1)) && threadInfo.LogThread)
                             m_log.DebugFormat("Exit threadfunc {0} ({1})", threadFuncNum, FormatDuration(threadInfo.Elapsed()));
+
+                        if (context != null)
+                            m_fireAndForgetCallsInProgress[context]--;
                     }
                 };
             }

OpenSim/Region/CoreModules/Framework/EntityTransfer/HGEntityTransferModule.cs

@@ -110,6 +110,11 @@ namespace OpenSim.Region.CoreModules.Framework.EntityTransfer
             }
         }
 
+        /// <summary>
+        /// Used for processing analysis of incoming attachments in a controlled fashion.
+        /// </summary>
+        private HGIncomingSceneObjectEngine m_incomingSceneObjectEngine;
+
         #region ISharedRegionModule
 
         public override string Name
@@ -155,6 +160,7 @@ namespace OpenSim.Region.CoreModules.Framework.EntityTransfer
                 scene.RegisterModuleInterface<IUserAgentVerificationModule>(this);
                 //scene.EventManager.OnIncomingSceneObject += OnIncomingSceneObject;
 
+                m_incomingSceneObjectEngine = new HGIncomingSceneObjectEngine(scene.Name);
                 m_incomingSceneObjectEngine.Start();
             }
         }
@@ -540,8 +546,6 @@ namespace OpenSim.Region.CoreModules.Framework.EntityTransfer
             }
         }
 
-        private HGIncomingSceneObjectEngine m_incomingSceneObjectEngine = new HGIncomingSceneObjectEngine();
-
         public override bool HandleIncomingSceneObject(SceneObjectGroup so, Vector3 newPosition)
         {
             // FIXME: We must make it so that we can use SOG.IsAttachment here.  At the moment it is always null!