Revamp the HTTP textures handler to allow a maximum of four fetches

at any time and to drop requests for avatars n longer in the scene

OpenSim/Capabilities/Handlers/GetTexture/GetTextureHandler.cs

@@ -47,36 +47,36 @@ using Caps = OpenSim.Framework.Capabilities.Caps;
 
 namespace OpenSim.Capabilities.Handlers
 {
-    public class GetTextureHandler : BaseStreamHandler
+    public class GetTextureHandler
     {
         private static readonly ILog m_log =
             LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
+
         private IAssetService m_assetService;
 
         public const string DefaultFormat = "x-j2c";
 
-        // TODO: Change this to a config option
+        public GetTextureHandler(IAssetService assService)
-        const string REDIRECT_URL = null;
-
-        public GetTextureHandler(string path, IAssetService assService, string name, string description)
-            : base("GET", path, name, description)
         {
             m_assetService = assService;
         }
 
-        public override byte[] Handle(string path, Stream request, IOSHttpRequest httpRequest, IOSHttpResponse httpResponse)
+        public Hashtable Handle(Hashtable request)
         {
-            // Try to parse the texture ID from the request URL
+            Hashtable ret = new Hashtable();
-            NameValueCollection query = HttpUtility.ParseQueryString(httpRequest.Url.Query);
+            ret["int_response_code"] = (int)System.Net.HttpStatusCode.NotFound;
-            string textureStr = query.GetOne("texture_id");
+            ret["content_type"] = "text/plain";
-            string format = query.GetOne("format");
+            ret["keepalive"] = false;
+            ret["reusecontext"] = false;
+
+            string textureStr = (string)request["texture_id"];
+            string format = (string)request["format"];
 
             //m_log.DebugFormat("[GETTEXTURE]: called {0}", textureStr);
 
             if (m_assetService == null)
             {
                 m_log.Error("[GETTEXTURE]: Cannot fetch texture " + textureStr + " without an asset service");
-                httpResponse.StatusCode = (int)System.Net.HttpStatusCode.NotFound;
             }
 
             UUID textureID;
@@ -91,30 +91,30 @@ namespace OpenSim.Capabilities.Handlers
                 }
                 else
                 {
-                    formats = WebUtil.GetPreferredImageTypes(httpRequest.Headers.Get("Accept"));
+                    formats = new string[1] { DefaultFormat }; // default
+                    if (((Hashtable)request["headers"])["Accept"] != null)
+                        formats = WebUtil.GetPreferredImageTypes((string)((Hashtable)request["headers"])["Accept"]);
                     if (formats.Length == 0)
                         formats = new string[1] { DefaultFormat }; // default
 
                 }
                 // OK, we have an array with preferred formats, possibly with only one entry
 
-                httpResponse.StatusCode = (int)System.Net.HttpStatusCode.NotFound;
                 foreach (string f in formats)
                 {
-                    if (FetchTexture(httpRequest, httpResponse, textureID, f))
+                    if (FetchTexture(request, ret, textureID, f))
                         break;
                 }
             }
             else
             {
-                m_log.Warn("[GETTEXTURE]: Failed to parse a texture_id from GetTexture request: " + httpRequest.Url);
+                m_log.Warn("[GETTEXTURE]: Failed to parse a texture_id from GetTexture request: " + (string)request["uri"]);
             }
 
 //            m_log.DebugFormat(
 //                "[GETTEXTURE]: For texture {0} sending back response {1}, data length {2}",
 //                textureID, httpResponse.StatusCode, httpResponse.ContentLength);
-
+            return ret;
-            return null;
         }
 
         /// <summary>
@@ -125,7 +125,7 @@ namespace OpenSim.Capabilities.Handlers
         /// <param name="textureID"></param>
         /// <param name="format"></param>
         /// <returns>False for "caller try another codec"; true otherwise</returns>
-        private bool FetchTexture(IOSHttpRequest httpRequest, IOSHttpResponse httpResponse, UUID textureID, string format)
+        private bool FetchTexture(Hashtable request, Hashtable response, UUID textureID, string format)
         {
 //            m_log.DebugFormat("[GETTEXTURE]: {0} with requested format {1}", textureID, format);
             AssetBase texture;
@@ -134,30 +134,6 @@ namespace OpenSim.Capabilities.Handlers
             if (format != DefaultFormat)
                 fullID = fullID + "-" + format;
 
-            if (!String.IsNullOrEmpty(REDIRECT_URL))
-            {
-                // Only try to fetch locally cached textures. Misses are redirected
-                texture = m_assetService.GetCached(fullID);
-
-                if (texture != null)
-                {
-                    if (texture.Type != (sbyte)AssetType.Texture)
-                    {
-                        httpResponse.StatusCode = (int)System.Net.HttpStatusCode.NotFound;
-                        return true;
-                    }
-                    WriteTextureData(httpRequest, httpResponse, texture, format);
-                }
-                else
-                {
-                    string textureUrl = REDIRECT_URL + textureID.ToString();
-                    m_log.Debug("[GETTEXTURE]: Redirecting texture request to " + textureUrl);
-                    httpResponse.RedirectLocation = textureUrl;
-                    return true;
-                }
-            }
-            else // no redirect
-            {
             // try the cache
             texture = m_assetService.GetCached(fullID);
 
@@ -171,13 +147,11 @@ namespace OpenSim.Capabilities.Handlers
                 if (texture != null)
                 {
                     if (texture.Type != (sbyte)AssetType.Texture)
-                        {
-                            httpResponse.StatusCode = (int)System.Net.HttpStatusCode.NotFound;
                         return true;
-                        }
+
                     if (format == DefaultFormat)
                     {
-                            WriteTextureData(httpRequest, httpResponse, texture, format);
+                        WriteTextureData(request, response, texture, format);
                         return true;
                     }
                     else
@@ -190,7 +164,7 @@ namespace OpenSim.Capabilities.Handlers
                         newTexture.Flags = AssetFlags.Collectable;
                         newTexture.Temporary = true;
                         m_assetService.Store(newTexture);
-                            WriteTextureData(httpRequest, httpResponse, newTexture, format);
+                        WriteTextureData(request, response, newTexture, format);
                         return true;
                     }
                 }
@@ -198,20 +172,23 @@ namespace OpenSim.Capabilities.Handlers
            else // it was on the cache
            {
                //m_log.DebugFormat("[GETTEXTURE]: texture was in the cache");
-                   WriteTextureData(httpRequest, httpResponse, texture, format);
+               WriteTextureData(request, response, texture, format);
                return true;
            }
-            }
 
             // not found
 //            m_log.Warn("[GETTEXTURE]: Texture " + textureID + " not found");
-            httpResponse.StatusCode = (int)System.Net.HttpStatusCode.NotFound;
             return true;
         }
 
-        private void WriteTextureData(IOSHttpRequest request, IOSHttpResponse response, AssetBase texture, string format)
+        private void WriteTextureData(Hashtable request, Hashtable response, AssetBase texture, string format)
         {
-            string range = request.Headers.GetOne("Range");
+            Hashtable headers = new Hashtable();
+            response["headers"] = headers;
+
+            string range = String.Empty;
+            if (((Hashtable)request["headers"])["Range"] != null)
+                range = (string)((Hashtable)request["headers"])["Range"];
 
             if (!String.IsNullOrEmpty(range)) // JP2's only
             {
@@ -226,7 +203,7 @@ namespace OpenSim.Capabilities.Handlers
                     {
 //                        response.StatusCode = (int)System.Net.HttpStatusCode.RequestedRangeNotSatisfiable;
                         // viewers don't seem to handle RequestedRangeNotSatisfiable and keep retrying with same parameters
-                        response.StatusCode = (int)System.Net.HttpStatusCode.NotFound;
+                        response["int_response_code"] = (int)System.Net.HttpStatusCode.NotFound;
                     }
                     else
                     {
@@ -240,31 +217,33 @@ namespace OpenSim.Capabilities.Handlers
                         // We were accidentally sending back 404 before in this situation
                         // https://issues.apache.org/bugzilla/show_bug.cgi?id=51878 supports sending 206 even if the
                         // entire range is requested, and viewer 3.2.2 (and very probably earlier) seems fine with this.
-                        response.StatusCode = (int)System.Net.HttpStatusCode.PartialContent;
+                        response["int_response_code"] = (int)System.Net.HttpStatusCode.PartialContent;
+                        response["content-type"] = texture.Metadata.ContentType;
+                        headers["Content-Range"] = String.Format("bytes {0}-{1}/{2}", start, end, texture.Data.Length);
     
-                        response.ContentLength = len;
+                        byte[] d = new byte[len];
-                        response.ContentType = texture.Metadata.ContentType;
+                        Array.Copy(texture.Data, start, d, 0, len);
-                        response.AddHeader("Content-Range", String.Format("bytes {0}-{1}/{2}", start, end, texture.Data.Length));
+                        response["bin_response_data"] = d;
-    
+//                        response.Body.Write(texture.Data, start, len);
-                        response.Body.Write(texture.Data, start, len);
                     }
                 }
                 else
                 {
                     m_log.Warn("[GETTEXTURE]: Malformed Range header: " + range);
-                    response.StatusCode = (int)System.Net.HttpStatusCode.BadRequest;
+                    response["int_response_code"] = (int)System.Net.HttpStatusCode.BadRequest;
                 }
             }
             else // JP2's or other formats
             {
                 // Full content request
-                response.StatusCode = (int)System.Net.HttpStatusCode.OK;
+                response["int_response_code"] = (int)System.Net.HttpStatusCode.OK;
-                response.ContentLength = texture.Data.Length;
                 if (format == DefaultFormat)
-                    response.ContentType = texture.Metadata.ContentType;
+                    response["content_type"] = texture.Metadata.ContentType;
                 else
-                    response.ContentType = "image/" + format;
+                    response["content_type"] = "image/" + format;
-                response.Body.Write(texture.Data, 0, texture.Data.Length);
+                
+                response["bin_response_data"] = texture.Data;
+//                response.Body.Write(texture.Data, 0, texture.Data.Length);
             }
 
 //            if (response.StatusCode < 200 || response.StatusCode > 299)

OpenSim/Capabilities/Handlers/GetTexture/GetTextureServerConnector.cs

@@ -33,6 +33,7 @@ using OpenSim.Framework.Servers.HttpServer;
 using OpenSim.Server.Handlers.Base;
 using OpenMetaverse;
 
+/*
 namespace OpenSim.Capabilities.Handlers
 {
     public class GetTextureServerConnector : ServiceConnector
@@ -63,7 +64,8 @@ namespace OpenSim.Capabilities.Handlers
                 throw new Exception(String.Format("Failed to load AssetService from {0}; config is {1}", assetService, m_ConfigName));
 
             server.AddStreamHandler(
-                new GetTextureHandler("/CAPS/GetTexture/" /*+ UUID.Random() */, m_AssetService, "GetTexture", null));
+                new GetTextureHandler("/CAPS/GetTexture/", m_AssetService, "GetTexture", null));
         }
     }
 }
+*/

OpenSim/Capabilities/Handlers/GetTexture/Tests/GetTextureHandlerTests.cs

@@ -39,6 +39,7 @@ using OpenSim.Region.Framework.Scenes;
 using OpenSim.Tests.Common;
 using OpenSim.Tests.Common.Mock;
 
+/*
 namespace OpenSim.Capabilities.Handlers.GetTexture.Tests
 {
     [TestFixture]
@@ -61,3 +62,4 @@ namespace OpenSim.Capabilities.Handlers.GetTexture.Tests
         }
     }
 }
+*/

OpenSim/Framework/Servers/HttpServer/BaseHttpServer.cs

@@ -1449,7 +1449,8 @@ namespace OpenSim.Framework.Servers.HttpServer
         internal byte[] DoHTTPGruntWork(Hashtable responsedata, OSHttpResponse response)
         {
             int responsecode;
-            string responseString;
+            string responseString = String.Empty;
+            byte[] responseData = null;
             string contentType;
 
             if (responsedata == null)
@@ -1465,6 +1466,9 @@ namespace OpenSim.Framework.Servers.HttpServer
                 {
                     //m_log.Info("[BASE HTTP SERVER]: Doing HTTP Grunt work with response");
                     responsecode = (int)responsedata["int_response_code"];
+                    if (responsedata["bin_response_data"] != null)
+                        responseData = (byte[])responsedata["bin_response_data"];
+                    else
                         responseString = (string)responsedata["str_response_string"];
                     contentType = (string)responsedata["content_type"];
                 }
@@ -1520,8 +1524,22 @@ namespace OpenSim.Framework.Servers.HttpServer
 
             response.AddHeader("Content-Type", contentType);
 
+            if (responsedata.ContainsKey("headers"))
+            {
+                Hashtable headerdata = (Hashtable)responsedata["headers"];
+
+                foreach (string header in headerdata.Keys)
+                    response.AddHeader(header, (string)headerdata[header]);
+            }
+
             byte[] buffer;
 
+            if (responseData != null)
+            {
+                buffer = responseData;
+            }
+            else
+            {
                 if (!(contentType.Contains("image")
                     || contentType.Contains("x-shockwave-flash")
                     || contentType.Contains("application/x-oar")
@@ -1539,6 +1557,7 @@ namespace OpenSim.Framework.Servers.HttpServer
                 response.SendChunked = false;
                 response.ContentLength64 = buffer.Length;
                 response.ContentEncoding = Encoding.UTF8;
+            }
 
             return buffer;
         }

OpenSim/Framework/Servers/HttpServer/PollServiceEventArgs.cs

@@ -52,7 +52,8 @@ namespace OpenSim.Framework.Servers.HttpServer
         {
             Normal = 0,
             LslHttp = 1,
-            Inventory = 2
+            Inventory = 2,
+            Texture = 3
         }
 
         public PollServiceEventArgs(

OpenSim/Framework/Servers/HttpServer/PollServiceRequestManager.cs

@@ -231,8 +231,7 @@ namespace OpenSim.Framework.Servers.HttpServer
         {
             if (m_running)
             {
-                if (req.PollServiceArgs.Type == PollServiceEventArgs.EventType.LslHttp ||
+                if (req.PollServiceArgs.Type != PollServiceEventArgs.EventType.Normal)
-                    req.PollServiceArgs.Type == PollServiceEventArgs.EventType.Inventory)
                 {
                     m_requests.Enqueue(req);
                 }

OpenSim/Region/ClientStack/Linden/Caps/GetTextureModule.cs

@@ -27,18 +27,13 @@
 
 using System;
 using System.Collections;
-using System.Collections.Specialized;
+using System.Collections.Generic;
-using System.Drawing;
-using System.Drawing.Imaging;
 using System.Reflection;
-using System.IO;
+using System.Threading;
-using System.Web;
 using log4net;
 using Nini.Config;
 using Mono.Addins;
 using OpenMetaverse;
-using OpenMetaverse.StructuredData;
-using OpenMetaverse.Imaging;
 using OpenSim.Framework;
 using OpenSim.Framework.Servers;
 using OpenSim.Framework.Servers.HttpServer;
@@ -47,64 +42,73 @@ using OpenSim.Region.Framework.Scenes;
 using OpenSim.Services.Interfaces;
 using Caps = OpenSim.Framework.Capabilities.Caps;
 using OpenSim.Capabilities.Handlers;
+using OpenSim.Framework.Monitoring;
 
 namespace OpenSim.Region.ClientStack.Linden
 {
 
-    [Extension(Path = "/OpenSim/RegionModules", NodeName = "RegionModule")]
+    /// <summary>
+    /// This module implements both WebFetchTextureDescendents and FetchTextureDescendents2 capabilities.
+    /// </summary>
+    [Extension(Path = "/OpenSim/RegionModules", NodeName = "RegionModule", Id = "GetTextureModule")]
     public class GetTextureModule : INonSharedRegionModule
     {
-//        private static readonly ILog m_log =
+        private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
-//            LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
 
         private Scene m_scene;
-        private IAssetService m_assetService;
 
-        private bool m_Enabled = false;
+        private static GetTextureHandler m_getTextureHandler;
 
-        // TODO: Change this to a config option
+        private IAssetService m_assetService = null;
-        const string REDIRECT_URL = null;
 
-        private string m_URL;
+        private Dictionary<UUID, string> m_capsDict = new Dictionary<UUID, string>();
+        private static Thread[] m_workerThreads = null;
+
+        private static OpenMetaverse.BlockingQueue<PollServiceTextureEventArgs> m_queue =
+                new OpenMetaverse.BlockingQueue<PollServiceTextureEventArgs>();
 
         #region ISharedRegionModule Members
 
         public void Initialise(IConfigSource source)
         {
-            IConfig config = source.Configs["ClientStack.LindenCaps"];
-            if (config == null)
-                return;
-
-            m_URL = config.GetString("Cap_GetTexture", string.Empty);
-            // Cap doesn't exist
-            if (m_URL != string.Empty)
-                m_Enabled = true;
         }
 
         public void AddRegion(Scene s)
         {
-            if (!m_Enabled)
-                return;
-
             m_scene = s;
+            m_assetService = s.AssetService;
         }
 
         public void RemoveRegion(Scene s)
         {
-            if (!m_Enabled)
-                return;
-
             m_scene.EventManager.OnRegisterCaps -= RegisterCaps;
+            m_scene.EventManager.OnDeregisterCaps -= DeregisterCaps;
             m_scene = null;
         }
 
         public void RegionLoaded(Scene s)
         {
-            if (!m_Enabled)
+            // We'll reuse the same handler for all requests.
-                return;
+            m_getTextureHandler = new GetTextureHandler(m_assetService);
 
-            m_assetService = m_scene.RequestModuleInterface<IAssetService>();
             m_scene.EventManager.OnRegisterCaps += RegisterCaps;
+            m_scene.EventManager.OnDeregisterCaps += DeregisterCaps;
+
+            if (m_workerThreads == null)
+            {
+                m_workerThreads = new Thread[4];
+
+                for (uint i = 0; i < 4; i++)
+                {
+                    m_workerThreads[i] = Watchdog.StartThread(DoTextureRequests,
+                            String.Format("TextureWorkerThread{0}", i),
+                            ThreadPriority.Normal,
+                            false,
+                            true,
+                            null,
+                            int.MaxValue);
+                }
+            }
         }
 
         public void PostInitialise()
@@ -122,24 +126,155 @@ namespace OpenSim.Region.ClientStack.Linden
 
         #endregion
 
-        public void RegisterCaps(UUID agentID, Caps caps)
+        ~GetTextureModule()
         {
-            UUID capID = UUID.Random();
+            foreach (Thread t in m_workerThreads)
-
+                t.Abort();
-            //caps.RegisterHandler("GetTexture", new StreamHandler("GET", "/CAPS/" + capID, ProcessGetTexture));
-            if (m_URL == "localhost")
-            {
-//                m_log.DebugFormat("[GETTEXTURE]: /CAPS/{0} in region {1}", capID, m_scene.RegionInfo.RegionName);
-                caps.RegisterHandler(
-                    "GetTexture",
-                    new GetTextureHandler("/CAPS/" + capID + "/", m_assetService, "GetTexture", agentID.ToString()));
         }
-            else
+
+        private class PollServiceTextureEventArgs : PollServiceEventArgs
         {
-//                m_log.DebugFormat("[GETTEXTURE]: {0} in region {1}", m_URL, m_scene.RegionInfo.RegionName);
+            private List<Hashtable> requests =
-                caps.RegisterHandler("GetTexture", m_URL);
+                    new List<Hashtable>();
+            private Dictionary<UUID, Hashtable> responses =
+                    new Dictionary<UUID, Hashtable>();
+
+            private Scene m_scene;
+
+            public PollServiceTextureEventArgs(UUID pId, Scene scene) :
+                    base(null, null, null, null, pId, 30000)
+            {
+                m_scene = scene;
+
+                HasEvents = (x, y) => { return this.responses.ContainsKey(x); };
+                GetEvents = (x, y, s) =>
+                {
+                    try
+                    {
+                        return this.responses[x];
+                    }
+                    finally
+                    {
+                        responses.Remove(x);
+                    }
+                };
+
+                Request = (x, y) =>
+                {
+                    y["RequestID"] = x.ToString();
+                    lock (this.requests)
+                        this.requests.Add(y);
+
+                    m_queue.Enqueue(this);
+                };
+
+                NoEvents = (x, y) =>
+                {
+                    lock (this.requests)
+                    {
+                        Hashtable request = requests.Find(id => id["RequestID"].ToString() == x.ToString());
+                        requests.Remove(request);
+                    }
+
+                    Hashtable response = new Hashtable();
+
+                    response["int_response_code"] = 500;
+                    response["str_response_string"] = "Script timeout";
+                    response["content_type"] = "text/plain";
+                    response["keepalive"] = false;
+                    response["reusecontext"] = false;
+
+                    return response;
+                };
+            }
+
+            public void Process()
+            {
+                Hashtable response;
+                Hashtable request = null;
+
+                try
+                {
+                    lock (this.requests)
+                    {
+                        request = requests[0];
+                        requests.RemoveAt(0);
+                    }
+                }
+                catch
+                {
+                    return;
+                }
+
+                UUID requestID = new UUID(request["RequestID"].ToString());
+
+                // If the avatar is gone, don't bother to get the texture
+                if (m_scene.GetScenePresence(Id) == null)
+                {
+                    response = new Hashtable();
+
+                    response["int_response_code"] = 500;
+                    response["str_response_string"] = "Script timeout";
+                    response["content_type"] = "text/plain";
+                    response["keepalive"] = false;
+                    response["reusecontext"] = false;
+
+                    responses[requestID] = response;
+                    return;
+                }
+
+                response = m_getTextureHandler.Handle(request);
+
+                responses[requestID] = response; 
             }
         }
 
+        private void RegisterCaps(UUID agentID, Caps caps)
+        {
+            string capUrl = "/CAPS/" + UUID.Random() + "/";
+
+            // Register this as a poll service
+            // absurd large timeout to tune later to make a bit less than viewer
+            PollServiceTextureEventArgs args = new PollServiceTextureEventArgs(agentID, m_scene);
+            
+            args.Type = PollServiceEventArgs.EventType.Texture;
+            MainServer.Instance.AddPollServiceHTTPHandler(capUrl, args);
+
+            string hostName = m_scene.RegionInfo.ExternalHostName;
+            uint port = (MainServer.Instance == null) ? 0 : MainServer.Instance.Port;
+            string protocol = "http";
+            
+            if (MainServer.Instance.UseSSL)
+            {
+                hostName = MainServer.Instance.SSLCommonName;
+                port = MainServer.Instance.SSLPort;
+                protocol = "https";
             }
+            caps.RegisterHandler("GetTexture", String.Format("{0}://{1}:{2}{3}", protocol, hostName, port, capUrl));
+
+            m_capsDict[agentID] = capUrl;
+        }
+
+        private void DeregisterCaps(UUID agentID, Caps caps)
+        {
+            string capUrl;
+
+            if (m_capsDict.TryGetValue(agentID, out capUrl))
+            {
+                MainServer.Instance.RemoveHTTPHandler("", capUrl);
+                m_capsDict.Remove(agentID);
+            }
+        }
+
+        private void DoTextureRequests()
+        {
+            while (true)
+            {
+                PollServiceTextureEventArgs args = m_queue.Dequeue();
+
+                args.Process();
+            }
+        }
+    }
+
 }