OpenSim
/
OpenSimMirror
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Revamp the HTTP textures handler to allow a maximum of four fetches 

at any time and to drop requests for avatars n longer in the scene
avinationmerge
Melanie 2012-09-14 21:24:25 +02:00
parent 9f93bef111
commit 387e59ff7f
7 changed files with 294 additions and 157 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

163
OpenSim/Capabilities/Handlers/GetTexture/GetTextureHandler.cs
View File

@ -47,36 +47,36 @@ using Caps = OpenSim.Framework.Capabilities.Caps;
namespace OpenSim.Capabilities.Handlers
{
public class GetTextureHandler : BaseStreamHandler
public class GetTextureHandler
{
private static readonly ILog m_log =
LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
private IAssetService m_assetService;
public const string DefaultFormat = "x-j2c";
// TODO: Change this to a config option
const string REDIRECT_URL = null;
public GetTextureHandler(string path, IAssetService assService, string name, string description)
: base("GET", path, name, description)
public GetTextureHandler(IAssetService assService)
{
m_assetService = assService;
}
public override byte[] Handle(string path, Stream request, IOSHttpRequest httpRequest, IOSHttpResponse httpResponse)
public Hashtable Handle(Hashtable request)
{
// Try to parse the texture ID from the request URL
NameValueCollection query = HttpUtility.ParseQueryString(httpRequest.Url.Query);
string textureStr = query.GetOne("texture_id");
string format = query.GetOne("format");
Hashtable ret = new Hashtable();
ret["int_response_code"] = (int)System.Net.HttpStatusCode.NotFound;
ret["content_type"] = "text/plain";
ret["keepalive"] = false;
ret["reusecontext"] = false;
string textureStr = (string)request["texture_id"];
string format = (string)request["format"];
//m_log.DebugFormat("[GETTEXTURE]: called {0}", textureStr);
if (m_assetService == null)
{
m_log.Error("[GETTEXTURE]: Cannot fetch texture " + textureStr + " without an asset service");
httpResponse.StatusCode = (int)System.Net.HttpStatusCode.NotFound;
}
UUID textureID;
@ -91,30 +91,30 @@ namespace OpenSim.Capabilities.Handlers
}
else
{
formats = WebUtil.GetPreferredImageTypes(httpRequest.Headers.Get("Accept"));
formats = new string[1] { DefaultFormat }; // default
if (((Hashtable)request["headers"])["Accept"] != null)
formats = WebUtil.GetPreferredImageTypes((string)((Hashtable)request["headers"])["Accept"]);
if (formats.Length == 0)
formats = new string[1] { DefaultFormat }; // default
}
// OK, we have an array with preferred formats, possibly with only one entry
httpResponse.StatusCode = (int)System.Net.HttpStatusCode.NotFound;
foreach (string f in formats)
{
if (FetchTexture(httpRequest, httpResponse, textureID, f))
if (FetchTexture(request, ret, textureID, f))
break;
}
}
else
{
m_log.Warn("[GETTEXTURE]: Failed to parse a texture_id from GetTexture request: " + httpRequest.Url);
m_log.Warn("[GETTEXTURE]: Failed to parse a texture_id from GetTexture request: " + (string)request["uri"]);
}
// m_log.DebugFormat(
// "[GETTEXTURE]: For texture {0} sending back response {1}, data length {2}",
// textureID, httpResponse.StatusCode, httpResponse.ContentLength);
return null;
return ret;
}
/// <summary>
@ -125,7 +125,7 @@ namespace OpenSim.Capabilities.Handlers
/// <param name="textureID"></param>
/// <param name="format"></param>
/// <returns>False for "caller try another codec"; true otherwise</returns>
private bool FetchTexture(IOSHttpRequest httpRequest, IOSHttpResponse httpResponse, UUID textureID, string format)
private bool FetchTexture(Hashtable request, Hashtable response, UUID textureID, string format)
{
// m_log.DebugFormat("[GETTEXTURE]: {0} with requested format {1}", textureID, format);
AssetBase texture;
@ -134,84 +134,61 @@ namespace OpenSim.Capabilities.Handlers
if (format != DefaultFormat)
fullID = fullID + "-" + format;
if (!String.IsNullOrEmpty(REDIRECT_URL))
// try the cache
texture = m_assetService.GetCached(fullID);
if (texture == null)
{
// Only try to fetch locally cached textures. Misses are redirected
texture = m_assetService.GetCached(fullID);
//m_log.DebugFormat("[GETTEXTURE]: texture was not in the cache");
// Fetch locally or remotely. Misses return a 404
texture = m_assetService.Get(textureID.ToString());
if (texture != null)
{
if (texture.Type != (sbyte)AssetType.Texture)
return true;
if (format == DefaultFormat)
{
httpResponse.StatusCode = (int)System.Net.HttpStatusCode.NotFound;
WriteTextureData(request, response, texture, format);
return true;
}
WriteTextureData(httpRequest, httpResponse, texture, format);
}
else
{
string textureUrl = REDIRECT_URL + textureID.ToString();
m_log.Debug("[GETTEXTURE]: Redirecting texture request to " + textureUrl);
httpResponse.RedirectLocation = textureUrl;
return true;
}
}
else // no redirect
{
// try the cache
texture = m_assetService.GetCached(fullID);
if (texture == null)
{
//m_log.DebugFormat("[GETTEXTURE]: texture was not in the cache");
// Fetch locally or remotely. Misses return a 404
texture = m_assetService.Get(textureID.ToString());
if (texture != null)
else
{
if (texture.Type != (sbyte)AssetType.Texture)
{
httpResponse.StatusCode = (int)System.Net.HttpStatusCode.NotFound;
return true;
}
if (format == DefaultFormat)
{
WriteTextureData(httpRequest, httpResponse, texture, format);
return true;
}
else
{
AssetBase newTexture = new AssetBase(texture.ID + "-" + format, texture.Name, (sbyte)AssetType.Texture, texture.Metadata.CreatorID);
newTexture.Data = ConvertTextureData(texture, format);
if (newTexture.Data.Length == 0)
return false; // !!! Caller try another codec, please!
AssetBase newTexture = new AssetBase(texture.ID + "-" + format, texture.Name, (sbyte)AssetType.Texture, texture.Metadata.CreatorID);
newTexture.Data = ConvertTextureData(texture, format);
if (newTexture.Data.Length == 0)
return false; // !!! Caller try another codec, please!
newTexture.Flags = AssetFlags.Collectable;
newTexture.Temporary = true;
m_assetService.Store(newTexture);
WriteTextureData(httpRequest, httpResponse, newTexture, format);
return true;
}
newTexture.Flags = AssetFlags.Collectable;
newTexture.Temporary = true;
m_assetService.Store(newTexture);
WriteTextureData(request, response, newTexture, format);
return true;
}
}
else // it was on the cache
{
//m_log.DebugFormat("[GETTEXTURE]: texture was in the cache");
WriteTextureData(httpRequest, httpResponse, texture, format);
return true;
}
}
}
}
else // it was on the cache
{
//m_log.DebugFormat("[GETTEXTURE]: texture was in the cache");
WriteTextureData(request, response, texture, format);
return true;
}
// not found
// m_log.Warn("[GETTEXTURE]: Texture " + textureID + " not found");
httpResponse.StatusCode = (int)System.Net.HttpStatusCode.NotFound;
return true;
}
private void WriteTextureData(IOSHttpRequest request, IOSHttpResponse response, AssetBase texture, string format)
private void WriteTextureData(Hashtable request, Hashtable response, AssetBase texture, string format)
{
string range = request.Headers.GetOne("Range");
Hashtable headers = new Hashtable();
response["headers"] = headers;
string range = String.Empty;
if (((Hashtable)request["headers"])["Range"] != null)
range = (string)((Hashtable)request["headers"])["Range"];
if (!String.IsNullOrEmpty(range)) // JP2's only
{
@ -226,7 +203,7 @@ namespace OpenSim.Capabilities.Handlers
{
// response.StatusCode = (int)System.Net.HttpStatusCode.RequestedRangeNotSatisfiable;
// viewers don't seem to handle RequestedRangeNotSatisfiable and keep retrying with same parameters
response.StatusCode = (int)System.Net.HttpStatusCode.NotFound;
response["int_response_code"] = (int)System.Net.HttpStatusCode.NotFound;
}
else
{
@ -240,31 +217,33 @@ namespace OpenSim.Capabilities.Handlers
// We were accidentally sending back 404 before in this situation
// https://issues.apache.org/bugzilla/show_bug.cgi?id=51878 supports sending 206 even if the
// entire range is requested, and viewer 3.2.2 (and very probably earlier) seems fine with this.
response.StatusCode = (int)System.Net.HttpStatusCode.PartialContent;
response["int_response_code"] = (int)System.Net.HttpStatusCode.PartialContent;
response["content-type"] = texture.Metadata.ContentType;
headers["Content-Range"] = String.Format("bytes {0}-{1}/{2}", start, end, texture.Data.Length);
response.ContentLength = len;
response.ContentType = texture.Metadata.ContentType;
response.AddHeader("Content-Range", String.Format("bytes {0}-{1}/{2}", start, end, texture.Data.Length));
response.Body.Write(texture.Data, start, len);
byte[] d = new byte[len];
Array.Copy(texture.Data, start, d, 0, len);
response["bin_response_data"] = d;
// response.Body.Write(texture.Data, start, len);
}
}
else
{
m_log.Warn("[GETTEXTURE]: Malformed Range header: " + range);
response.StatusCode = (int)System.Net.HttpStatusCode.BadRequest;
response["int_response_code"] = (int)System.Net.HttpStatusCode.BadRequest;
}
}
else // JP2's or other formats
{
// Full content request
response.StatusCode = (int)System.Net.HttpStatusCode.OK;
response.ContentLength = texture.Data.Length;
response["int_response_code"] = (int)System.Net.HttpStatusCode.OK;
if (format == DefaultFormat)
response.ContentType = texture.Metadata.ContentType;
response["content_type"] = texture.Metadata.ContentType;
else
response.ContentType = "image/" + format;
response.Body.Write(texture.Data, 0, texture.Data.Length);
response["content_type"] = "image/" + format;
response["bin_response_data"] = texture.Data;
// response.Body.Write(texture.Data, 0, texture.Data.Length);
}
// if (response.StatusCode < 200 || response.StatusCode > 299)

4
OpenSim/Capabilities/Handlers/GetTexture/GetTextureServerConnector.cs
View File

@ -33,6 +33,7 @@ using OpenSim.Framework.Servers.HttpServer;
using OpenSim.Server.Handlers.Base;
using OpenMetaverse;
/*
namespace OpenSim.Capabilities.Handlers
{
public class GetTextureServerConnector : ServiceConnector
@ -63,7 +64,8 @@ namespace OpenSim.Capabilities.Handlers
throw new Exception(String.Format("Failed to load AssetService from {0}; config is {1}", assetService, m_ConfigName));
server.AddStreamHandler(
new GetTextureHandler("/CAPS/GetTexture/" /*+ UUID.Random() */, m_AssetService, "GetTexture", null));
new GetTextureHandler("/CAPS/GetTexture/", m_AssetService, "GetTexture", null));
}
}
}
*/

2
OpenSim/Capabilities/Handlers/GetTexture/Tests/GetTextureHandlerTests.cs
View File

@ -39,6 +39,7 @@ using OpenSim.Region.Framework.Scenes;
using OpenSim.Tests.Common;
using OpenSim.Tests.Common.Mock;
/*
namespace OpenSim.Capabilities.Handlers.GetTexture.Tests
{
[TestFixture]
@ -61,3 +62,4 @@ namespace OpenSim.Capabilities.Handlers.GetTexture.Tests
}
}
}
*/

47
OpenSim/Framework/Servers/HttpServer/BaseHttpServer.cs
View File

@ -1449,7 +1449,8 @@ namespace OpenSim.Framework.Servers.HttpServer
internal byte[] DoHTTPGruntWork(Hashtable responsedata, OSHttpResponse response)
{
int responsecode;
string responseString;
string responseString = String.Empty;
byte[] responseData = null;
string contentType;
if (responsedata == null)
@ -1465,7 +1466,10 @@ namespace OpenSim.Framework.Servers.HttpServer
{
//m_log.Info("[BASE HTTP SERVER]: Doing HTTP Grunt work with response");
responsecode = (int)responsedata["int_response_code"];
responseString = (string)responsedata["str_response_string"];
if (responsedata["bin_response_data"] != null)
responseData = (byte[])responsedata["bin_response_data"];
else
responseString = (string)responsedata["str_response_string"];
contentType = (string)responsedata["content_type"];
}
catch
@ -1520,25 +1524,40 @@ namespace OpenSim.Framework.Servers.HttpServer
response.AddHeader("Content-Type", contentType);
if (responsedata.ContainsKey("headers"))
{
Hashtable headerdata = (Hashtable)responsedata["headers"];
foreach (string header in headerdata.Keys)
response.AddHeader(header, (string)headerdata[header]);
}
byte[] buffer;
if (!(contentType.Contains("image")
|| contentType.Contains("x-shockwave-flash")
|| contentType.Contains("application/x-oar")
|| contentType.Contains("application/vnd.ll.mesh")))
if (responseData != null)
{
// Text
buffer = Encoding.UTF8.GetBytes(responseString);
buffer = responseData;
}
else
{
// Binary!
buffer = Convert.FromBase64String(responseString);
}
if (!(contentType.Contains("image")
|| contentType.Contains("x-shockwave-flash")
|| contentType.Contains("application/x-oar")
|| contentType.Contains("application/vnd.ll.mesh")))
{
// Text
buffer = Encoding.UTF8.GetBytes(responseString);
}
else
{
// Binary!
buffer = Convert.FromBase64String(responseString);
}
response.SendChunked = false;
response.ContentLength64 = buffer.Length;
response.ContentEncoding = Encoding.UTF8;
response.SendChunked = false;
response.ContentLength64 = buffer.Length;
response.ContentEncoding = Encoding.UTF8;
}
return buffer;
}

3
OpenSim/Framework/Servers/HttpServer/PollServiceEventArgs.cs
View File

@ -52,7 +52,8 @@ namespace OpenSim.Framework.Servers.HttpServer
{
Normal = 0,
LslHttp = 1,
Inventory = 2
Inventory = 2,
Texture = 3
}
public PollServiceEventArgs(

3
OpenSim/Framework/Servers/HttpServer/PollServiceRequestManager.cs
View File

@ -231,8 +231,7 @@ namespace OpenSim.Framework.Servers.HttpServer
{
if (m_running)
{
if (req.PollServiceArgs.Type == PollServiceEventArgs.EventType.LslHttp ||
req.PollServiceArgs.Type == PollServiceEventArgs.EventType.Inventory)
if (req.PollServiceArgs.Type != PollServiceEventArgs.EventType.Normal)
{
m_requests.Enqueue(req);
}

221
OpenSim/Region/ClientStack/Linden/Caps/GetTextureModule.cs
View File

@ -27,18 +27,13 @@
using System;
using System.Collections;
using System.Collections.Specialized;
using System.Drawing;
using System.Drawing.Imaging;
using System.Collections.Generic;
using System.Reflection;
using System.IO;
using System.Web;
using System.Threading;
using log4net;
using Nini.Config;
using Mono.Addins;
using OpenMetaverse;
using OpenMetaverse.StructuredData;
using OpenMetaverse.Imaging;
using OpenSim.Framework;
using OpenSim.Framework.Servers;
using OpenSim.Framework.Servers.HttpServer;
@ -47,64 +42,73 @@ using OpenSim.Region.Framework.Scenes;
using OpenSim.Services.Interfaces;
using Caps = OpenSim.Framework.Capabilities.Caps;
using OpenSim.Capabilities.Handlers;
using OpenSim.Framework.Monitoring;
namespace OpenSim.Region.ClientStack.Linden
{
[Extension(Path = "/OpenSim/RegionModules", NodeName = "RegionModule")]
/// <summary>
/// This module implements both WebFetchTextureDescendents and FetchTextureDescendents2 capabilities.
/// </summary>
[Extension(Path = "/OpenSim/RegionModules", NodeName = "RegionModule", Id = "GetTextureModule")]
public class GetTextureModule : INonSharedRegionModule
{
// private static readonly ILog m_log =
// LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
private Scene m_scene;
private IAssetService m_assetService;
private bool m_Enabled = false;
private static GetTextureHandler m_getTextureHandler;
// TODO: Change this to a config option
const string REDIRECT_URL = null;
private IAssetService m_assetService = null;
private string m_URL;
private Dictionary<UUID, string> m_capsDict = new Dictionary<UUID, string>();
private static Thread[] m_workerThreads = null;
private static OpenMetaverse.BlockingQueue<PollServiceTextureEventArgs> m_queue =
new OpenMetaverse.BlockingQueue<PollServiceTextureEventArgs>();
#region ISharedRegionModule Members
public void Initialise(IConfigSource source)
{
IConfig config = source.Configs["ClientStack.LindenCaps"];
if (config == null)
return;
m_URL = config.GetString("Cap_GetTexture", string.Empty);
// Cap doesn't exist
if (m_URL != string.Empty)
m_Enabled = true;
}
public void AddRegion(Scene s)
{
if (!m_Enabled)
return;
m_scene = s;
m_assetService = s.AssetService;
}
public void RemoveRegion(Scene s)
{
if (!m_Enabled)
return;
m_scene.EventManager.OnRegisterCaps -= RegisterCaps;
m_scene.EventManager.OnDeregisterCaps -= DeregisterCaps;
m_scene = null;
}
public void RegionLoaded(Scene s)
{
if (!m_Enabled)
return;
// We'll reuse the same handler for all requests.
m_getTextureHandler = new GetTextureHandler(m_assetService);
m_assetService = m_scene.RequestModuleInterface<IAssetService>();
m_scene.EventManager.OnRegisterCaps += RegisterCaps;
m_scene.EventManager.OnDeregisterCaps += DeregisterCaps;
if (m_workerThreads == null)
{
m_workerThreads = new Thread[4];
for (uint i = 0; i < 4; i++)
{
m_workerThreads[i] = Watchdog.StartThread(DoTextureRequests,
String.Format("TextureWorkerThread{0}", i),
ThreadPriority.Normal,
false,
true,
null,
int.MaxValue);
}
}
}
public void PostInitialise()
@ -122,24 +126,155 @@ namespace OpenSim.Region.ClientStack.Linden
#endregion
public void RegisterCaps(UUID agentID, Caps caps)
~GetTextureModule()
{
UUID capID = UUID.Random();
foreach (Thread t in m_workerThreads)
t.Abort();
}
//caps.RegisterHandler("GetTexture", new StreamHandler("GET", "/CAPS/" + capID, ProcessGetTexture));
if (m_URL == "localhost")
private class PollServiceTextureEventArgs : PollServiceEventArgs
{
private List<Hashtable> requests =
new List<Hashtable>();
private Dictionary<UUID, Hashtable> responses =
new Dictionary<UUID, Hashtable>();
private Scene m_scene;
public PollServiceTextureEventArgs(UUID pId, Scene scene) :
base(null, null, null, null, pId, 30000)
{
// m_log.DebugFormat("[GETTEXTURE]: /CAPS/{0} in region {1}", capID, m_scene.RegionInfo.RegionName);
caps.RegisterHandler(
"GetTexture",
new GetTextureHandler("/CAPS/" + capID + "/", m_assetService, "GetTexture", agentID.ToString()));
m_scene = scene;
HasEvents = (x, y) => { return this.responses.ContainsKey(x); };
GetEvents = (x, y, s) =>
{
try
{
return this.responses[x];
}
finally
{
responses.Remove(x);
}
};
Request = (x, y) =>
{
y["RequestID"] = x.ToString();
lock (this.requests)
this.requests.Add(y);
m_queue.Enqueue(this);
};
NoEvents = (x, y) =>
{
lock (this.requests)
{
Hashtable request = requests.Find(id => id["RequestID"].ToString() == x.ToString());
requests.Remove(request);
}
Hashtable response = new Hashtable();
response["int_response_code"] = 500;
response["str_response_string"] = "Script timeout";
response["content_type"] = "text/plain";
response["keepalive"] = false;
response["reusecontext"] = false;
return response;
};
}
else
public void Process()
{
// m_log.DebugFormat("[GETTEXTURE]: {0} in region {1}", m_URL, m_scene.RegionInfo.RegionName);
caps.RegisterHandler("GetTexture", m_URL);
Hashtable response;
Hashtable request = null;
try
{
lock (this.requests)
{
request = requests[0];
requests.RemoveAt(0);
}
}
catch
{
return;
}
UUID requestID = new UUID(request["RequestID"].ToString());
// If the avatar is gone, don't bother to get the texture
if (m_scene.GetScenePresence(Id) == null)
{
response = new Hashtable();
response["int_response_code"] = 500;
response["str_response_string"] = "Script timeout";
response["content_type"] = "text/plain";
response["keepalive"] = false;
response["reusecontext"] = false;
responses[requestID] = response;
return;
}
response = m_getTextureHandler.Handle(request);
responses[requestID] = response;
}
}
private void RegisterCaps(UUID agentID, Caps caps)
{
string capUrl = "/CAPS/" + UUID.Random() + "/";
// Register this as a poll service
// absurd large timeout to tune later to make a bit less than viewer
PollServiceTextureEventArgs args = new PollServiceTextureEventArgs(agentID, m_scene);
args.Type = PollServiceEventArgs.EventType.Texture;
MainServer.Instance.AddPollServiceHTTPHandler(capUrl, args);
string hostName = m_scene.RegionInfo.ExternalHostName;
uint port = (MainServer.Instance == null) ? 0 : MainServer.Instance.Port;
string protocol = "http";
if (MainServer.Instance.UseSSL)
{
hostName = MainServer.Instance.SSLCommonName;
port = MainServer.Instance.SSLPort;
protocol = "https";
}
caps.RegisterHandler("GetTexture", String.Format("{0}://{1}:{2}{3}", protocol, hostName, port, capUrl));
m_capsDict[agentID] = capUrl;
}
private void DeregisterCaps(UUID agentID, Caps caps)
{
string capUrl;
if (m_capsDict.TryGetValue(agentID, out capUrl))
{
MainServer.Instance.RemoveHTTPHandler("", capUrl);
m_capsDict.Remove(agentID);
}
}
private void DoTextureRequests()
{
while (true)
{
PollServiceTextureEventArgs args = m_queue.Dequeue();
args.Process();
}
}
}
}