* Fix problem where known missing assets would stop save oar ever completing

* Issue was that region server was silently dropping an XmlException caused by trying to deserialize the blank asset service response
* So make asset service return http status NOT FOUND rather than OK in accordance with REST
* and interpret this correctly in the async response so that a null object is sent back
* This means that this fix won't be active until both region simulator and server reach this revision

OpenSim/Framework/Servers/BaseGetAssetStreamHandler.cs

@@ -64,7 +64,7 @@ namespace OpenSim.Framework.Servers
                 
                 if (!UUID.TryParse(p[0], out assetID))
                 {
-                    m_log.InfoFormat(
+                    m_log.DebugFormat(
                         "[REST]: GET:/asset ignoring request with malformed UUID {0}", p[0]);
                     return result;
                 }
@@ -91,12 +91,14 @@ namespace OpenSim.Framework.Servers
                 }
                 else
                 {
+                    m_log.DebugFormat("[REST]: GET:/asset failed to find {0}", assetID);
+                    
+                    httpResponse.StatusCode = (int)HttpStatusCode.NotFound;
+                    
                     if (StatsManager.AssetStats != null)
                     {
                         StatsManager.AssetStats.AddNotFoundRequest();
                     }
-
-                    m_log.InfoFormat("[REST]: GET:/asset failed to find {0}", assetID);
                 }
             }
 

OpenSim/Framework/Servers/HttpServer/AsynchronousRestObjectRequester.cs

@@ -38,7 +38,7 @@ namespace OpenSim.Framework.Servers.HttpServer
 {
     public class AsynchronousRestObjectRequester
     {
-        //private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
+        private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
         
         /// <summary>
         /// Perform an asynchronous REST request.
@@ -56,7 +56,7 @@ namespace OpenSim.Framework.Servers.HttpServer
         public static void MakeRequest<TRequest, TResponse>(string verb,
                 string requestUrl, TRequest obj, Action<TResponse> action)
         {
-            //m_log.DebugFormat("[ASYNC REQUEST]: Starting {0} on {1}", verb, requestUrl);
+//            m_log.DebugFormat("[ASYNC REQUEST]: Starting {0} {1}", verb, requestUrl);
             
             Type type = typeof (TRequest);
 
@@ -114,20 +114,60 @@ namespace OpenSim.Framework.Servers.HttpServer
 
             request.BeginGetResponse(delegate(IAsyncResult res2)
             {
-                response = request.EndGetResponse(res2);
+                try
+                {
+                    // If the server returns a 404, this appears to trigger a System.Net.WebException even though that isn't
+                    // documented in MSDN
+                    response = request.EndGetResponse(res2);
+              
+                    try
+                    {
+                        deserial = (TResponse)deserializer.Deserialize(response.GetResponseStream());
+                    }
+                    catch (System.InvalidOperationException)
+                    {
+                    }
+                }
+                catch (WebException e)
+                {
+                    if (e.Status == WebExceptionStatus.ProtocolError)
+                    {
+                        if (e.Response is HttpWebResponse)
+                        {
+                            HttpWebResponse httpResponse = (HttpWebResponse)e.Response;
+                        
+                            if (httpResponse.StatusCode != HttpStatusCode.NotFound)
+                            {
+                                // We don't appear to be handling any other status codes, so log these feailures to that
+                                // people don't spend unnecessary hours hunting phantom bugs.
+                                m_log.DebugFormat(
+                                    "[ASYNC REQUEST]: Request {0} {1} failed with unexpected status code {2}", 
+                                    verb, requestUrl, httpResponse.StatusCode);
+                            }
+                        }
+                    }
+                    else
+                    {
+                        m_log.ErrorFormat("[ASYNC REQUEST]: Request {0} {1} failed with exception {2}", verb, requestUrl, e);
+                    }
+                }
+                catch (Exception e)
+                {
+                    m_log.ErrorFormat("[ASYNC REQUEST]: Request {0} {1} failed with exception {2}", verb, requestUrl, e);
+                }
+
+                //  m_log.DebugFormat("[ASYNC REQUEST]: Received {0}", deserial.ToString());
 
                 try
                 {
-                    deserial = (TResponse) deserializer.Deserialize(
+                    action(deserial);
-                            response.GetResponseStream());
                 }
-                catch (System.InvalidOperationException)
+                catch (Exception e)
                 {
+                    m_log.ErrorFormat(
+                        "[ASYNC REQUEST]: Request {0} {1} callback failed with exception {2}", verb, requestUrl, e);
                 }
                     
-              //  m_log.DebugFormat("[ASYNC REQUEST]: Received {0}", deserial.ToString());
-
-                action(deserial);
             }, null);         
         }
     }

OpenSim/Region/CoreModules/World/Archiver/AssetsRequest.cs

@@ -229,11 +229,13 @@ namespace OpenSim.Region.CoreModules.World.Archiver
                                                            
                     if (asset != null)
                     {
+//                        m_log.DebugFormat("[ARCHIVER]: Recording asset {0} as found", id);
                         m_foundAssetUuids.Add(asset.FullID);
                         m_assetsArchiver.WriteAsset(asset);
                     }
                     else
                     {
+//                        m_log.DebugFormat("[ARCHIVER]: Recording asset {0} as not found", id);
                         m_notFoundAssetUuids.Add(new UUID(id));
                     }