From a9e8bd59a377e7c7ce81e3693875467926dd7d4b Mon Sep 17 00:00:00 2001
From: Mic Bowman <cmickeyb@gmail.com>
Date: Mon, 13 Feb 2012 19:38:22 -0800
Subject: [PATCH] Fix a race condition in the simian groups connector. When
 requests were too slow they would circumvent the cache (piling up on the
 network service and making the problem even worse). This condition happens
 frequently during permission checks.

---
 .../SimianGroupsServicesConnectorModule.cs    | 73 ++++++++++++++++---
 1 file changed, 64 insertions(+), 9 deletions(-)

diff --git a/OpenSim/Region/OptionalModules/Avatar/XmlRpcGroups/SimianGroupsServicesConnectorModule.cs b/OpenSim/Region/OptionalModules/Avatar/XmlRpcGroups/SimianGroupsServicesConnectorModule.cs
index 42008dae43..130513d4c0 100644
--- a/OpenSim/Region/OptionalModules/Avatar/XmlRpcGroups/SimianGroupsServicesConnectorModule.cs
+++ b/OpenSim/Region/OptionalModules/Avatar/XmlRpcGroups/SimianGroupsServicesConnectorModule.cs
@@ -30,6 +30,7 @@ using System.Collections;
 using System.Collections.Generic;
 using System.Collections.Specialized;
 using System.Reflection;
+using System.Threading;
 
 using Nwc.XmlRpc;
 
@@ -167,6 +168,8 @@ namespace OpenSim.Region.OptionalModules.Avatar.XmlRpcGroups
 
         private bool m_debugEnabled = false;
 
+        private Dictionary<string, bool> m_pendingRequests = new Dictionary<string,bool>();
+        
         private ExpiringCache<string, OSDMap> m_memoryCache;
         private int m_cacheTimeout = 30;
 
@@ -1348,6 +1351,7 @@ namespace OpenSim.Region.OptionalModules.Avatar.XmlRpcGroups
             // Immediately forward the request if the cache is disabled.
             if (m_cacheTimeout == 0)
             {
+                m_log.WarnFormat("[SIMIAN GROUPS CONNECTOR]: cache is disabled");
                 return WebUtil.PostToService(m_groupsServerURI, requestArgs);
             }
 
@@ -1355,6 +1359,8 @@ namespace OpenSim.Region.OptionalModules.Avatar.XmlRpcGroups
             if (requestArgs["RequestMethod"] == "RemoveGeneric"
                 || requestArgs["RequestMethod"] == "AddGeneric")
             {
+                m_log.WarnFormat("[SIMIAN GROUPS CONNECTOR]: clearing generics cache");
+                
                 // Any and all updates cause the cache to clear
                 m_memoryCache.Clear();
 
@@ -1366,18 +1372,67 @@ namespace OpenSim.Region.OptionalModules.Avatar.XmlRpcGroups
 
             // Create the cache key for the request and see if we have it cached
             string CacheKey = WebUtil.BuildQueryString(requestArgs);
-            OSDMap response = null;
-            if (!m_memoryCache.TryGetValue(CacheKey, out response))
-            {
-                // if it wasn't in the cache, pass the request to the Simian Grid Services
-                response = WebUtil.PostToService(m_groupsServerURI, requestArgs);
 
-                // and cache the response
-                m_memoryCache.AddOrUpdate(CacheKey, response, TimeSpan.FromSeconds(m_cacheTimeout));
+            // This code uses a leader/follower pattern. On a cache miss, the request is added
+            // to a queue; the first thread to add it to the queue completes the request while
+            // follow on threads busy wait for the results, this situation seems to happen
+            // often when checking permissions
+            while (true)
+            {
+                OSDMap response = null;
+                bool firstRequest = false;
+
+                lock (m_memoryCache)
+                {
+                    if (m_memoryCache.TryGetValue(CacheKey, out response))
+                        return response;
+                    
+                    if (! m_pendingRequests.ContainsKey(CacheKey))
+                    {
+                        m_pendingRequests.Add(CacheKey,true);
+                        firstRequest = true;
+                    }
+                }
+                
+                if (firstRequest)
+                {
+                    // if it wasn't in the cache, pass the request to the Simian Grid Services
+                    try
+                    {
+                        response = WebUtil.PostToService(m_groupsServerURI, requestArgs);
+                    }
+                    catch (Exception e)
+                    {
+                        m_log.InfoFormat("[SIMIAN GROUPS CONNECTOR] request failed {0}",CacheKey);
+                    }
+                    
+                    // and cache the response
+                    lock (m_memoryCache)
+                    {
+                        m_memoryCache.AddOrUpdate(CacheKey, response, TimeSpan.FromSeconds(m_cacheTimeout));
+                        m_pendingRequests.Remove(CacheKey);
+                    }
+
+                    return response;
+                }
+
+                Thread.Sleep(50); // waiting for a web request to complete, 50msecs is reasonable
             }
 
-            // return cached response
-            return response;
+            // if (!m_memoryCache.TryGetValue(CacheKey, out response))
+            // {
+            //     m_log.WarnFormat("[SIMIAN GROUPS CONNECTOR]: query not in the cache");
+            //     Util.PrintCallStack();
+                
+            //     // if it wasn't in the cache, pass the request to the Simian Grid Services
+            //     response = WebUtil.PostToService(m_groupsServerURI, requestArgs);
+
+            //     // and cache the response
+            //     m_memoryCache.AddOrUpdate(CacheKey, response, TimeSpan.FromSeconds(m_cacheTimeout));
+            // }
+
+            // // return cached response
+            // return response;
         }
         #endregion