Added debug flag: LogThreadPool. It makes us log every use of the main threadpool.

Resolves http://opensimulator.org/mantis/view.php?id=6945

OpenSim/Framework/Util.cs

@@ -116,6 +116,16 @@ namespace OpenSim.Framework
     {
         private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
 
+        /// <summary>
+        /// Log every invocation of a thread using the threadpool.
+        /// </summary>
+        public static bool LogThreadPool { get; set; }
+
+        static Util()
+        {
+            LogThreadPool = false;
+        }
+
         private static uint nextXferID = 5000;
         private static Random randomClass = new Random();
 
@@ -1887,10 +1897,17 @@ namespace OpenSim.Framework
             }
         }
 
+        
+        private static long nextThreadFuncNum = 0;
+        private static long numQueuedThreadFuncs = 0;
+        private static long numRunningThreadFuncs = 0;
+
         public static void FireAndForget(System.Threading.WaitCallback callback, object obj)
         {
             WaitCallback realCallback;
 
+            long threadFuncNum = Interlocked.Increment(ref nextThreadFuncNum);
+
             if (FireAndForgetMethod == FireAndForgetMethod.RegressionTest)
             {
                 // If we're running regression tests, then we want any exceptions to rise up to the test code.
@@ -1903,49 +1920,117 @@ namespace OpenSim.Framework
                 // for decimals places but is read by a culture that treats commas as number seperators.
                 realCallback = o =>
                 {
-                    Culture.SetCurrentCulture();
+                    long numQueued1 = Interlocked.Decrement(ref numQueuedThreadFuncs);
+                    long numRunning1 = Interlocked.Increment(ref numRunningThreadFuncs);
 
                     try
                     {
+                        if (LogThreadPool)
+                            m_log.DebugFormat("Run threadfunc {0} (Queued {1}, Running {2})", threadFuncNum, numQueued1, numRunning1);
+
+                        Culture.SetCurrentCulture();
+
                         callback(o);
                     }
                     catch (Exception e)
                     {
-                        m_log.ErrorFormat(
+                        m_log.Error("[UTIL]: FireAndForget thread terminated with error ", e);
-                            "[UTIL]: Continuing after async_call_method thread terminated with exception {0}{1}",
+                    }
-                            e.Message, e.StackTrace);
+                    finally
+                    {
+                        Interlocked.Decrement(ref numRunningThreadFuncs);
+                        if (LogThreadPool)
+                            m_log.Debug("Exit threadfunc " + threadFuncNum);
                     }
                 };
             }
 
-            switch (FireAndForgetMethod)
+            long numQueued = Interlocked.Increment(ref numQueuedThreadFuncs);
+            try
             {
-                case FireAndForgetMethod.RegressionTest:
+                if (LogThreadPool)
-                case FireAndForgetMethod.None:
+                    m_log.DebugFormat("Queue threadfunc {0} (Queued {1}, Running {2}) {3}",
-                    realCallback.Invoke(obj);
+                        threadFuncNum, numQueued, numRunningThreadFuncs, GetFireAndForgetStackTrace(true));
-                    break;
+
-                case FireAndForgetMethod.UnsafeQueueUserWorkItem:
+                switch (FireAndForgetMethod)
-                    ThreadPool.UnsafeQueueUserWorkItem(realCallback, obj);
+                {
-                    break;
+                    case FireAndForgetMethod.RegressionTest:
-                case FireAndForgetMethod.QueueUserWorkItem:
+                    case FireAndForgetMethod.None:
-                    ThreadPool.QueueUserWorkItem(realCallback, obj);
+                        realCallback.Invoke(obj);
-                    break;
+                        break;
-                case FireAndForgetMethod.BeginInvoke:
+                    case FireAndForgetMethod.UnsafeQueueUserWorkItem:
-                    FireAndForgetWrapper wrapper = FireAndForgetWrapper.Instance;
+                        ThreadPool.UnsafeQueueUserWorkItem(realCallback, obj);
-                    wrapper.FireAndForget(realCallback, obj);
+                        break;
-                    break;
+                    case FireAndForgetMethod.QueueUserWorkItem:
-                case FireAndForgetMethod.SmartThreadPool:
+                        ThreadPool.QueueUserWorkItem(realCallback, obj);
-                    if (m_ThreadPool == null)
+                        break;
-                        InitThreadPool(2, 15); 
+                    case FireAndForgetMethod.BeginInvoke:
-                    m_ThreadPool.QueueWorkItem((cb, o) => cb(o), realCallback, obj);
+                        FireAndForgetWrapper wrapper = FireAndForgetWrapper.Instance;
-                    break;
+                        wrapper.FireAndForget(realCallback, obj);
-                case FireAndForgetMethod.Thread:
+                        break;
-                    Thread thread = new Thread(delegate(object o) { realCallback(o); });
+                    case FireAndForgetMethod.SmartThreadPool:
-                    thread.Start(obj);
+                        if (m_ThreadPool == null)
-                    break;
+                            InitThreadPool(2, 15);
-                default:
+                        m_ThreadPool.QueueWorkItem((cb, o) => cb(o), realCallback, obj);
-                    throw new NotImplementedException();
+                        break;
+                    case FireAndForgetMethod.Thread:
+                        Thread thread = new Thread(delegate(object o) { realCallback(o); });
+                        thread.Start(obj);
+                        break;
+                    default:
+                        throw new NotImplementedException();
+                }
             }
+            catch (Exception)
+            {
+                Interlocked.Decrement(ref numQueuedThreadFuncs);
+                throw;
+            }
+        }
+
+        /// <summary>
+        /// Returns a stack trace for a thread added using FireAndForget().
+        /// </summary>
+        /// <param name="full">True: return full stack trace; False: return only the first frame</param>
+        private static string GetFireAndForgetStackTrace(bool full)
+        {
+            string src = Environment.StackTrace;
+            string[] lines = src.Split(new string[] { Environment.NewLine }, StringSplitOptions.None);
+            
+            StringBuilder dest = new StringBuilder(src.Length);
+
+            bool started = false;
+            bool first = true;
+
+            for (int i = 0; i < lines.Length; i++)
+            {
+                string line = lines[i];
+
+                if (!started)
+                {
+                    // Skip the initial stack frames, because they're of no interest for debugging
+                    if (line.Contains("StackTrace") || line.Contains("FireAndForget"))
+                        continue;
+                    started = true;
+                }
+
+                if (first)
+                {
+                    line = line.TrimStart();
+                    first = false;
+                }
+
+                bool last = (i == lines.Length - 1) || !full;
+                if (last)
+                    dest.Append(line);
+                else
+                    dest.AppendLine(line);
+
+                if (!full)
+                    break;
+            }
+
+            return dest.ToString();
         }
 
         /// <summary>