several changes to llHttpRequest processing: options section renamed ScriptsHttpRequestModule; throttle by prim (PrimRequestsBurst = 3, PrimRequestsPerSec = 1) and max concurrent connections per instance (MaxPoolThreads = 5), llhttprequest returns if error, nullkey if throttled, reqid otherwise

OpenSim/Region/CoreModules/Scripting/HttpRequest/ScriptsHttpRequests.cs

@@ -27,6 +27,7 @@
 
 using System;
 using System.Collections.Generic;
+using System.Collections.Concurrent;
 using System.IO;
 using System.Net;
 using System.Net.Mail;
@@ -101,11 +102,22 @@ namespace OpenSim.Region.CoreModules.Scripting.HttpRequest
         private OutboundUrlFilter m_outboundUrlFilter;
         private string m_proxyurl = "";
         private string m_proxyexcepts = "";
+        
+        private float m_primpersec = 1.0f;
+        private float m_primburst = 3f;
+
+        private struct ThrottleData
+        {
+            public double lastTime;
+            public float count;
+        }
 
         // <request id, HttpRequestClass>
         private Dictionary<UUID, HttpRequestClass> m_pendingRequests;
+        private ConcurrentQueue<HttpRequestClass> m_CompletedRequests;
+        private ConcurrentDictionary<uint, ThrottleData> m_RequestsThrottle;
+
         private Scene m_scene;
-        // private Queue<HttpRequestClass> rpcQueue = new Queue<HttpRequestClass>();
         public static SmartThreadPool ThreadPool = null;
 
         public HttpRequestModule()
@@ -119,6 +131,41 @@ namespace OpenSim.Region.CoreModules.Scripting.HttpRequest
             return UUID.Zero;
         }
 
+        public bool CheckThrottle(uint localID)
+        {
+            ThrottleData th;
+            double now = Util.GetTimeStamp();
+            bool ret = false;
+
+            if (m_RequestsThrottle.TryGetValue(localID, out th))
+            {
+                double delta = now - th.lastTime;
+                th.lastTime = now;
+
+                float add = (float)(m_primpersec * delta);
+                th.count += add;
+                if (th.count > m_primburst)
+                    th.count = m_primburst;
+
+                ret = th.count > 0;
+            }
+            else
+            {
+                th = new ThrottleData()
+                {
+                    lastTime = now,
+                    count = m_primburst
+                };
+                ret = true;
+            }
+
+            if (ret)
+                th.count--;
+
+            m_RequestsThrottle[localID] = th;
+            return ret;
+        }
+
         public UUID StartHttpRequest(
             uint localID, UUID itemID, string url, List<string> parameters, Dictionary<string, string> headers, string body,
             out HttpInitialRequestStatus status)
@@ -243,9 +290,7 @@ namespace OpenSim.Region.CoreModules.Scripting.HttpRequest
                 return false;
 
             lock (HttpListLock)
-            {
                 m_pendingRequests.Add(req.ReqID, req);
-            }
 
             req.Process();
 
@@ -256,14 +301,19 @@ namespace OpenSim.Region.CoreModules.Scripting.HttpRequest
         {
             if (m_pendingRequests != null)
             {
+                List<UUID> toremove = new List<UUID>();
                 lock (HttpListLock)
                 {
-                    HttpRequestClass tmpReq;
+                    foreach (HttpRequestClass tmpReq in m_pendingRequests.Values)
-                    if (m_pendingRequests.TryGetValue(m_itemID, out tmpReq))
                     {
-                        tmpReq.Stop();
+                        if(tmpReq.ItemID == m_itemID)
-                        m_pendingRequests.Remove(m_itemID);
+                        {
+                            tmpReq.Stop();
+                            toremove.Add(tmpReq.ReqID);
+                        }
                     }
+                    foreach(UUID id in toremove)
+                        m_pendingRequests.Remove(id);
                 }
             }
         }
@@ -276,37 +326,37 @@ namespace OpenSim.Region.CoreModules.Scripting.HttpRequest
         * finished.  I thought about setting up a queue for this, but
         * it will need some refactoring and this works 'enough' right now
         */
+        public void GotCompletedRequest(HttpRequestClass req)
+        {
+            if(req.Removed)
+                return;
+            lock (HttpListLock)
+            {
+                m_pendingRequests.Remove(req.ReqID);
+                m_CompletedRequests.Enqueue(req);
+            }
+        }
 
         public IServiceRequest GetNextCompletedRequest()
         {
-            lock (HttpListLock)
+            HttpRequestClass req;
+            while(m_CompletedRequests.TryDequeue(out req))
             {
-                foreach (UUID luid in m_pendingRequests.Keys)
+                if(!req.Removed)
-                {
+                    return req;
-                    HttpRequestClass tmpReq;
-
-                    if (m_pendingRequests.TryGetValue(luid, out tmpReq))
-                    {
-                        if (tmpReq.Finished)
-                        {
-                            return tmpReq;
-                        }
-                    }
-                }
             }
             return null;
         }
 
-        public void RemoveCompletedRequest(UUID id)
+        public void RemoveCompletedRequest(UUID reqId)
         {
             lock (HttpListLock)
             {
                 HttpRequestClass tmpReq;
-                if (m_pendingRequests.TryGetValue(id, out tmpReq))
+                if (m_pendingRequests.TryGetValue(reqId, out tmpReq))
                 {
                     tmpReq.Stop();
-                    tmpReq = null;
+                    m_pendingRequests.Remove(reqId);
-                    m_pendingRequests.Remove(id);
                 }
             }
         }
@@ -322,17 +372,20 @@ namespace OpenSim.Region.CoreModules.Scripting.HttpRequest
 
             HttpRequestClass.HttpBodyMaxLenMAX = config.Configs["Network"].GetInt("HttpBodyMaxLenMAX", 16384);
 
-
             m_outboundUrlFilter = new OutboundUrlFilter("Script HTTP request module", config);
-            int maxThreads = 15;
 
-            IConfig httpConfig = config.Configs["HttpRequestModule"];
+            int maxThreads = 5;
+            IConfig httpConfig = config.Configs["ScriptsHttpRequestModule"];
             if (httpConfig != null)
             {
                 maxThreads = httpConfig.GetInt("MaxPoolThreads", maxThreads);
+                m_primburst = httpConfig.GetFloat("PrimRequestsBurst", m_primburst);
+                m_primpersec = httpConfig.GetFloat("PrimRequestsPerSec", m_primpersec);
             }
 
             m_pendingRequests = new Dictionary<UUID, HttpRequestClass>();
+            m_CompletedRequests = new ConcurrentQueue<HttpRequestClass>();
+            m_RequestsThrottle = new ConcurrentDictionary<uint, ThrottleData>();
 
             // First instance sets this up for all sims
             if (ThreadPool == null)
@@ -406,11 +459,8 @@ namespace OpenSim.Region.CoreModules.Scripting.HttpRequest
         /// </summary>
         public HttpRequestModule RequestModule { get; set; }
 
-        private bool _finished;
+        public bool Finished { get; private set;}
-        public bool Finished
+        public bool Removed{ get; set;}
-        {
-            get { return _finished; }
-        }
 
         public static int HttpBodyMaxLenMAX = 16384;
 
@@ -427,19 +477,10 @@ namespace OpenSim.Region.CoreModules.Scripting.HttpRequest
         public bool HttpPragmaNoCache = true;
 
         // Request info
-        private UUID _itemID;
+        public UUID ReqID { get; set; }
-        public UUID ItemID
+        public UUID ItemID {  get; set;}
-        {
+        public uint LocalID { get; set;}
-            get { return _itemID; }
+
-            set { _itemID = value; }
-        }
-        private uint _localID;
-        public uint LocalID
-        {
-            get { return _localID; }
-            set { _localID = value; }
-        }
-        public DateTime Next;
         public string proxyurl;
         public string proxyexcepts;
 
@@ -454,12 +495,7 @@ namespace OpenSim.Region.CoreModules.Scripting.HttpRequest
         public int MaxRedirects { get; set; }
 
         public string OutboundBody;
-        private UUID _reqID;
+
-        public UUID ReqID
-        {
-            get { return _reqID; }
-            set { _reqID = value; }
-        }
         public HttpWebRequest Request;
         public string ResponseBody;
         public List<string> ResponseMetadata;
@@ -469,10 +505,7 @@ namespace OpenSim.Region.CoreModules.Scripting.HttpRequest
 
         public void Process()
         {
-            _finished = false;
+            WorkItem = HttpRequestModule.ThreadPool.QueueWorkItem(new WorkItemCallback(StpSendWrapper), null);
-
-            lock (HttpRequestModule.ThreadPool)
-                WorkItem = HttpRequestModule.ThreadPool.QueueWorkItem(new WorkItemCallback(StpSendWrapper), null);
         }
 
         private object StpSendWrapper(object o)
@@ -521,6 +554,9 @@ namespace OpenSim.Region.CoreModules.Scripting.HttpRequest
 
         public void SendRequest()
         {
+            if(Removed)
+                 return;
+
             HttpWebResponse response = null;
             Stream resStream = null;
             byte[] buf = new byte[HttpBodyMaxLenMAX + 16];
@@ -672,7 +708,6 @@ namespace OpenSim.Region.CoreModules.Scripting.HttpRequest
                 if (response != null)
                     response.Close();
 
-
                 // We need to resubmit
                 if (
                     (Status == (int)HttpStatusCode.MovedPermanently
@@ -684,7 +719,8 @@ namespace OpenSim.Region.CoreModules.Scripting.HttpRequest
                     {
                         Status = (int)OSHttpStatusCode.ClientErrorJoker;
                         ResponseBody = "Number of redirects exceeded max redirects";
-                        _finished = true;
+                        WorkItem = null;
+                        RequestModule.GotCompletedRequest(this);
                     }
                     else
                     {
@@ -694,13 +730,15 @@ namespace OpenSim.Region.CoreModules.Scripting.HttpRequest
                         {
                             Status = (int)OSHttpStatusCode.ClientErrorJoker;
                             ResponseBody = "HTTP redirect code but no location header";
-                            _finished = true;
+                            WorkItem = null;
+                            RequestModule.GotCompletedRequest(this);
                         }
                         else if (!RequestModule.CheckAllowed(new Uri(location)))
                         {
                             Status = (int)OSHttpStatusCode.ClientErrorJoker;
                             ResponseBody = "URL from HTTP redirect blocked: " + location;
-                            _finished = true;
+                            WorkItem = null;
+                            RequestModule.GotCompletedRequest(this);
                         }
                         else
                         {
@@ -717,9 +755,10 @@ namespace OpenSim.Region.CoreModules.Scripting.HttpRequest
                 }
                 else
                 {
-                    _finished = true;
+                    WorkItem = null;
                     if (ResponseBody == null)
                         ResponseBody = String.Empty;
+                    RequestModule.GotCompletedRequest(this);
                 }
             }
         }
@@ -728,10 +767,12 @@ namespace OpenSim.Region.CoreModules.Scripting.HttpRequest
         {
             try
             {
+                Removed = true;
+                if(WorkItem == null)
+                    return;
+
                 if (!WorkItem.Cancel())
-                {
                     WorkItem.Cancel(true);
-                }
             }
             catch (Exception)
             {

OpenSim/Region/Framework/Interfaces/IHttpRequests.cs

@@ -87,5 +87,6 @@ namespace OpenSim.Region.Framework.Interfaces
         void StopHttpRequest(uint m_localID, UUID m_itemID);
         IServiceRequest GetNextCompletedRequest();
         void RemoveCompletedRequest(UUID id);
+        bool CheckThrottle(uint localID);
     }
 }

OpenSim/Region/ScriptEngine/Shared/Api/Implementation/LSL_Api.cs

@@ -13945,14 +13945,14 @@ namespace OpenSim.Region.ScriptEngine.Shared.Api
 
         public LSL_Key llHTTPRequest(string url, LSL_List parameters, string body)
         {
-            // Partial implementation: support for parameter flags needed
-            //   see http://wiki.secondlife.com/wiki/LlHTTPRequest
-            // parameter flags support are implemented in ScriptsHttpRequests.cs
-            //   in StartHttpRequest
-
             m_host.AddScriptLPS(1);
-            IHttpRequestModule httpScriptMod =
+            IHttpRequestModule httpScriptMod = m_ScriptEngine.World.RequestModuleInterface<IHttpRequestModule>();
-                m_ScriptEngine.World.RequestModuleInterface<IHttpRequestModule>();
+            if(httpScriptMod == null)
+                return "";
+
+            if(!httpScriptMod.CheckThrottle(m_host.LocalId))
+                return UUID.Zero.ToString();
+
             List<string> param = new List<string>();
             bool  ok;
             Int32 flag;
@@ -14123,8 +14123,7 @@ namespace OpenSim.Region.ScriptEngine.Shared.Api
             }
 
             HttpInitialRequestStatus status;
-            UUID reqID
+            UUID reqID = httpScriptMod.StartHttpRequest(m_host.LocalId, m_item.ItemID, url, param, httpHeaders, body, out status);
-                = httpScriptMod.StartHttpRequest(m_host.LocalId, m_item.ItemID, url, param, httpHeaders, body, out status);
 
             if (status == HttpInitialRequestStatus.DISALLOWED_BY_FILTER)
                 Error("llHttpRequest", string.Format("Request to {0} disallowed by filter", url));
@@ -14132,7 +14131,7 @@ namespace OpenSim.Region.ScriptEngine.Shared.Api
             if (reqID != UUID.Zero)
                 return reqID.ToString();
             else
-                return null;
+                return "";
         }
 
 

OpenSim/Region/ScriptEngine/Shared/Api/Implementation/Plugins/HttpRequest.cs

@@ -48,14 +48,11 @@ namespace OpenSim.Region.ScriptEngine.Shared.Api.Plugins
             if (m_CmdManager.m_ScriptEngine.World == null)
                 return;
 
-            IHttpRequestModule iHttpReq =
+            IHttpRequestModule iHttpReq = m_CmdManager.m_ScriptEngine.World.RequestModuleInterface<IHttpRequestModule>();
-                m_CmdManager.m_ScriptEngine.World.RequestModuleInterface<IHttpRequestModule>();
+            if(iHttpReq == null)
-
+                return;
-            HttpRequestClass httpInfo = null;
-
-            if (iHttpReq != null)
-                httpInfo = (HttpRequestClass)iHttpReq.GetNextCompletedRequest();
 
+            HttpRequestClass httpInfo = (HttpRequestClass)iHttpReq.GetNextCompletedRequest();
             while (httpInfo != null)
             {
                 //m_log.Debug("[AsyncLSL]:" + httpInfo.response_body + httpInfo.status);
@@ -67,8 +64,6 @@ namespace OpenSim.Region.ScriptEngine.Shared.Api.Plugins
                 // implemented here yet anyway.  Should be fixed if/when maxsize
                 // is supported
 
-                iHttpReq.RemoveCompletedRequest(httpInfo.ReqID);
-
                 object[] resobj = new object[]
                 {
                     new LSL_Types.LSLString(httpInfo.ReqID.ToString()),