Implemented FetchAssetMetadataSet in DB backends.

This method fetches metadata for a subset of the entries in the assets
database. This functionality is used in the ForEach calls in the asset
storage providers in AssetInventoryServer. With this implemented,
frontends such as the BrowseFrontend should now work.

- MySQL: implemented, sanity tested
- SQLite: implemented, sanity tested
- MSSQL: implemented, not tested
- NHibernate: not implemented

OpenSim/Data/AssetDataBase.cs

@@ -25,6 +25,7 @@
  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
+using System.Collections.Generic;
 using OpenMetaverse;
 using OpenSim.Framework;
 
@@ -37,6 +38,8 @@ namespace OpenSim.Data
         public abstract void UpdateAsset(AssetBase asset);
         public abstract bool ExistsAsset(UUID uuid);
 
+        public abstract List<AssetMetadata> FetchAssetMetadataSet(int start, int count);
+
         public abstract string Version { get; }
         public abstract string Name { get; }
         public abstract void Initialise(string connect);

OpenSim/Data/IAssetData.cs

@@ -25,6 +25,7 @@
  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
+using System.Collections.Generic;
 using OpenMetaverse;
 using OpenSim.Framework;
 
@@ -36,6 +37,7 @@ namespace OpenSim.Data
         void CreateAsset(AssetBase asset);
         void UpdateAsset(AssetBase asset);
         bool ExistsAsset(UUID uuid);
+        List<AssetMetadata> FetchAssetMetadataSet(int start, int count);
         void Initialise(string connect);
     }
 

OpenSim/Data/MSSQL/MSSQLAssetData.cs

@@ -28,6 +28,7 @@
 using System;
 using System.Data;
 using System.Reflection;
+using System.Collections.Generic;
 using OpenMetaverse;
 using log4net;
 using OpenSim.Framework;
@@ -245,6 +246,41 @@ namespace OpenSim.Data.MSSQL
             return false;
         }
 
+        /// <summary>
+        /// Returns a list of AssetMetadata objects. The list is a subset of
+        /// the entire data set offset by <paramref name="start" /> containing
+        /// <paramref name="count" /> elements.
+        /// </summary>
+        /// <param name="start">The number of results to discard from the total data set.</param>
+        /// <param name="count">The number of rows the returned list should contain.</param>
+        /// <returns>A list of AssetMetadata objects.</returns>
+        public override List<AssetMetadata> FetchAssetMetadataSet(int start, int count)
+        {
+            List<AssetMetadata> retList = new List<AssetMetadata>(count);
+
+            using (AutoClosingSqlCommand command = database.Query("SELECT name,description,assetType,temporary,id FROM assets LIMIT @start, @count"))
+            {
+                command.Parameters.Add(database.CreateParameter("start", start));
+                command.Parameters.Add(database.CreateParameter("count", count));
+
+                using (IDataReader reader = command.ExecuteReader())
+                {
+                    while (reader.Read())
+                    {
+                        AssetMetadata metadata = new AssetMetadata();
+                        // Region Main
+                        metadata.FullID = new UUID((Guid)reader["id"]);
+                        metadata.Name = (string)reader["name"];
+                        metadata.Description = (string)reader["description"];
+                        metadata.Type = Convert.ToSByte(reader["assetType"]);
+                        metadata.Temporary = Convert.ToBoolean(reader["temporary"]);
+                    }
+                }
+            }
+
+            return retList;
+        }
+
         #endregion
     }
 }

OpenSim/Data/MySQL/MySQLAssetData.cs

@@ -28,6 +28,7 @@
 using System;
 using System.Data;
 using System.Reflection;
+using System.Collections.Generic;
 using log4net;
 using MySql.Data.MySqlClient;
 using OpenMetaverse;
@@ -311,6 +312,56 @@ namespace OpenSim.Data.MySQL
             return assetExists;
         }
 
+        /// <summary>
+        /// Returns a list of AssetMetadata objects. The list is a subset of
+        /// the entire data set offset by <paramref name="start" /> containing
+        /// <paramref name="count" /> elements.
+        /// </summary>
+        /// <param name="start">The number of results to discard from the total data set.</param>
+        /// <param name="count">The number of rows the returned list should contain.</param>
+        /// <returns>A list of AssetMetadata objects.</returns>
+        public override List<AssetMetadata> FetchAssetMetadataSet(int start, int count)
+        {
+            List<AssetMetadata> retList = new List<AssetMetadata>(count);
+
+            lock (_dbConnection)
+            {
+                _dbConnection.CheckConnection();
+
+                MySqlCommand cmd = new MySqlCommand("SELECT name,description,assetType,temporary,id FROM assets LIMIT ?start, ?count", _dbConnection.Connection);
+                cmd.Parameters.AddWithValue("?start", start);
+                cmd.Parameters.AddWithValue("?count", count);
+
+                try
+                {
+                    using (MySqlDataReader dbReader = cmd.ExecuteReader())
+                    {
+                        while (dbReader.Read())
+                        {
+                            AssetMetadata metadata = new AssetMetadata();
+                            metadata.Name = (string) dbReader["name"];
+                            metadata.Description = (string) dbReader["description"];
+                            metadata.Type = (sbyte) dbReader["assetType"];
+                            metadata.Temporary = (bool) dbReader["temporary"]; // Not sure if this is correct.
+                            metadata.FullID = new UUID((string) dbReader["id"]);
+
+                            // Current SHA1s are not stored/computed.
+                            metadata.SHA1 = new byte[] {};
+
+                            retList.Add(metadata);
+                        }
+                    }
+                }
+                catch (Exception e)
+                {
+                    m_log.Error("[ASSETS DB]: MySql failure fetching asset set" + Environment.NewLine + e.ToString() + Environment.NewLine + "Attempting reconnection");
+                    _dbConnection.Reconnect();
+                }
+            }
+
+            return retList;
+        }
+
         #endregion
 
         /// <summary>

OpenSim/Data/NHibernate/NHibernateAssetData.cs

@@ -26,6 +26,7 @@
  */
 
 using System.Reflection;
+using System.Collections.Generic;
 using log4net;
 using OpenMetaverse;
 using OpenSim.Framework;
@@ -109,6 +110,20 @@ namespace OpenSim.Data.NHibernate
             return (FetchAsset(uuid) != null);
         }
 
+        /// <summary>
+        /// Returns a list of AssetMetadata objects. The list is a subset of
+        /// the entire data set offset by <paramref name="start" /> containing
+        /// <paramref name="count" /> elements.
+        /// </summary>
+        /// <param name="start">The number of results to discard from the total data set.</param>
+        /// <param name="count">The number of rows the returned list should contain.</param>
+        /// <returns>A list of AssetMetadata objects.</returns>
+        public override List<AssetMetadata> FetchAssetMetadataSet(int start, int count)
+        {
+            List<AssetMetadata> retList = new List<AssetMetadata>(count);
+            return retList;
+        }
+
         public void DeleteAsset(UUID uuid)
         {
 

OpenSim/Data/SQLite/SQLiteAssetData.cs

@@ -28,6 +28,7 @@
 using System;
 using System.Data;
 using System.Reflection;
+using System.Collections.Generic;
 using log4net;
 using Mono.Data.SqliteClient;
 using OpenMetaverse;
@@ -49,6 +50,7 @@ namespace OpenSim.Data.SQLite
         /// Artificial constructor called upon plugin load
         /// </summary>
         private const string SelectAssetSQL = "select * from assets where UUID=:UUID";
+        private const string SelectAssetMetadataSQL = "select Name, Description, Type, Temporary, UUID from assets limit :start, :count";
         private const string DeleteAssetSQL = "delete from assets where UUID=:UUID";
         private const string InsertAssetSQL = "insert into assets(UUID, Name, Description, Type, Local, Temporary, Data) values(:UUID, :Name, :Description, :Type, :Local, :Temporary, :Data)";
         private const string UpdateAssetSQL = "update assets set Name=:Name, Description=:Description, Type=:Type, Local=:Local, Temporary=:Temporary, Data=:Data where UUID=:UUID";
@@ -256,6 +258,54 @@ namespace OpenSim.Data.SQLite
             return asset;
         }
 
+        private static AssetMetadata buildAssetMetadata(IDataReader row)
+        {
+            AssetMetadata metadata = new AssetMetadata();
+
+            metadata.FullID = new UUID((string) row["UUID"]);
+            metadata.Name = (string) row["Name"];
+            metadata.Description = (string) row["Description"];
+            metadata.Type = Convert.ToSByte(row["Type"]);
+            metadata.Temporary = Convert.ToBoolean(row["Temporary"]); // Not sure if this is correct.
+
+            // Current SHA1s are not stored/computed.
+            metadata.SHA1 = new byte[] {};
+
+            return metadata;
+        }
+
+        /// <summary>
+        /// Returns a list of AssetMetadata objects. The list is a subset of
+        /// the entire data set offset by <paramref name="start" /> containing
+        /// <paramref name="count" /> elements.
+        /// </summary>
+        /// <param name="start">The number of results to discard from the total data set.</param>
+        /// <param name="count">The number of rows the returned list should contain.</param>
+        /// <returns>A list of AssetMetadata objects.</returns>
+        public override List<AssetMetadata> FetchAssetMetadataSet(int start, int count)
+        {
+            List<AssetMetadata> retList = new List<AssetMetadata>(count);
+
+            lock (this)
+            {
+                using (SqliteCommand cmd = new SqliteCommand(SelectAssetMetadataSQL, m_conn))
+                {
+                    cmd.Parameters.Add(new SqliteParameter(":start", start));
+                    cmd.Parameters.Add(new SqliteParameter(":count", count));
+
+                    using (IDataReader reader = cmd.ExecuteReader())
+                    {
+                        while (reader.Read())
+                        {
+                            AssetMetadata metadata = buildAssetMetadata(reader);
+                            retList.Add(metadata);
+                        }
+                    }
+                }
+            }
+
+            return retList;
+        }
 
         /***********************************************************************
          *

OpenSim/Grid/AssetInventoryServer/Plugins/BrowseFrontendPlugin.cs

@@ -42,7 +42,7 @@ namespace OpenSim.Grid.AssetInventoryServer.Plugins
     public class BrowseFrontendPlugin : IAssetInventoryServerPlugin
     {
         private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
-        //private AssetInventoryServer m_server;
+        private AssetInventoryServer m_server;
 
         public BrowseFrontendPlugin()
         {
@@ -52,10 +52,10 @@ namespace OpenSim.Grid.AssetInventoryServer.Plugins
 
         public void Initialise(AssetInventoryServer server)
         {
-            //m_server = server;
+            m_server = server;
 
             // Request for / or /?...
-            //m_server.HttpServer.AddStreamHandler(new BrowseRequestHandler(server));
+            m_server.HttpServer.AddStreamHandler(new BrowseRequestHandler(server));
 
             m_log.Info("[BROWSEFRONTEND]: Browser Frontend loaded.");
         }
@@ -86,102 +86,87 @@ namespace OpenSim.Grid.AssetInventoryServer.Plugins
 
         #endregion IPlugin implementation
 
-        //public class BrowseRequestHandler : IStreamedRequestHandler
+        public class BrowseRequestHandler : BaseStreamHandler
-        //{
+        {
-        //    AssetInventoryServer m_server;
+            AssetInventoryServer m_server;
-        //    string m_contentType;
-        //    string m_httpMethod;
-        //    string m_path;
 
-        //    public BrowseRequestHandler(AssetInventoryServer server)
+            //public BrowseRequestHandler(AssetInventoryServer server) : base("GET", "(^/$|(^/\?.*)")
-        //    {
+            public BrowseRequestHandler(AssetInventoryServer server) : base("GET", "/")
-        //        m_server = server;
+            {
-        //        m_contentType = null;
+                m_server = server;
-        //        m_httpMethod = "GET";
+            }
-        //        m_path = @"(^/$)|(^/\?.*)";
-        //    }
 
-        //    #region IStreamedRequestHandler implementation
+            public override string ContentType
+            {
+                get { return "text/html"; }
+            }
 
-        //    public string ContentType
+            #region IStreamedRequestHandler implementation
-        //    {
-        //        get { return m_contentType; }
-        //    }
 
-        //    public string HttpMethod
+            public override byte[] Handle(string path, Stream request, OSHttpRequest httpRequest, OSHttpResponse httpResponse)
-        //    {
+            {
-        //        get { return m_httpMethod; }
+                const int ASSETS_PER_PAGE = 25;
-        //    }
+                const string HEADER = "<html><head><title>Asset Server</title></head><body>";
+                const string TABLE_HEADER =
+                    "<table><tr><th>Name</th><th>Description</th><th>Type</th><th>ID</th><th>Temporary</th><th>SHA-1</th></tr>";
+                const string TABLE_FOOTER = "</table>";
+                const string FOOTER = "</body></html>";
 
-        //    public string Path
+                UUID authToken = Utils.GetAuthToken(httpRequest);
-        //    {
-        //        get { return m_path; }
-        //    }
 
-        //    public byte[] Handle(string path, Stream request, OSHttpRequest httpRequest, OSHttpResponse httpResponse)
+                StringBuilder html = new StringBuilder();
-        //    {
+                int start = 0;
-        //        const int ASSETS_PER_PAGE = 25;
+                uint page = 0;
-        //        const string HEADER = "<html><head><title>Asset Server</title></head><body>";
-        //        const string TABLE_HEADER =
-        //            "<table><tr><th>Name</th><th>Description</th><th>Type</th><th>ID</th><th>Temporary</th><th>SHA-1</th></tr>";
-        //        const string TABLE_FOOTER = "</table>";
-        //        const string FOOTER = "</body></html>";
 
-        //        UUID authToken = Utils.GetAuthToken(httpRequest);
+                if (!String.IsNullOrEmpty(httpRequest.Url.Query))
+                {
+                    NameValueCollection query = HttpUtility.ParseQueryString(httpRequest.Url.Query);
+                    if (!String.IsNullOrEmpty(query["page"]) && UInt32.TryParse(query["page"], out page))
+                        start = (int)page * ASSETS_PER_PAGE;
+                }
 
-        //        StringBuilder html = new StringBuilder();
+                html.AppendLine(HEADER);
-        //        int start = 0;
-        //        uint page = 0;
 
-        //        if (!String.IsNullOrEmpty(httpRequest.Url.Query))
+                html.AppendLine("<p>");
-        //        {
+                if (page > 0)
-        //            NameValueCollection query = HttpUtility.ParseQueryString(httpRequest.Url.Query);
+                    html.AppendFormat("<a href=\"{0}?page={1}\">&lt; Previous Page</a> | ", httpRequest.RawUrl, page - 1);
-        //            if (!String.IsNullOrEmpty(query["page"]) && UInt32.TryParse(query["page"], out page))
+                html.AppendFormat("<a href=\"{0}?page={1}\">Next Page &gt;</a>", httpRequest.RawUrl, page + 1);
-        //                start = (int)page * ASSETS_PER_PAGE;
+                html.AppendLine("</p>");
-        //        }
 
-        //        html.AppendLine(HEADER);
+                html.AppendLine(TABLE_HEADER);
 
-        //        html.AppendLine("<p>");
+                m_server.StorageProvider.ForEach(
-        //        if (page > 0)
+                    delegate(AssetMetadata data)
-        //            html.AppendFormat("<a href=\"{0}?page={1}\">&lt; Previous Page</a> | ", httpRequest.RawUrl, page - 1);
+                    {
-        //        html.AppendFormat("<a href=\"{0}?page={1}\">Next Page &gt;</a>", httpRequest.RawUrl, page + 1);
+                        if (m_server.AuthorizationProvider.IsMetadataAuthorized(authToken, data.FullID))
-        //        html.AppendLine("</p>");
+                        {
+                            html.AppendLine(String.Format(
+                                "<tr><td>{0}</td><td>{1}</td><td>{2}</td><td>{3}</td><td>{4}</td><td>{5}</td></tr>",
+                                data.Name, data.Description, data.ContentType, data.ID, data.Temporary,
+                                BitConverter.ToString(data.SHA1).Replace("-", String.Empty)));
+                        }
+                        else
+                        {
+                            html.AppendLine(String.Format(
+                                "<tr><td>[Protected Asset]</td><td>&nbsp;</td><td>&nbsp;</td><td>{0}</td><td>{1}</td><td>&nbsp;</td></tr>",
+                                data.ID, data.Temporary));
+                        }
+                    }, start, ASSETS_PER_PAGE
+                );
 
-        //        html.AppendLine(TABLE_HEADER);
+                html.AppendLine(TABLE_FOOTER);
 
-        //        m_server.StorageProvider.ForEach(
+                html.AppendLine(FOOTER);
-        //            delegate(Metadata data)
-        //            {
-        //                if (m_server.AuthorizationProvider.IsMetadataAuthorized(authToken, data.ID))
-        //                {
-        //                    html.AppendLine(String.Format(
-        //                        "<tr><td>{0}</td><td>{1}</td><td>{2}</td><td>{3}</td><td>{4}</td><td>{5}</td></tr>",
-        //                        data.Name, data.Description, data.ContentType, data.ID, data.Temporary,
-        //                        BitConverter.ToString(data.SHA1).Replace("-", String.Empty)));
-        //                }
-        //                else
-        //                {
-        //                    html.AppendLine(String.Format(
-        //                        "<tr><td>[Protected Asset]</td><td>&nbsp;</td><td>&nbsp;</td><td>{0}</td><td>{1}</td><td>&nbsp;</td></tr>",
-        //                        data.ID, data.Temporary));
-        //                }
-        //            }, start, ASSETS_PER_PAGE
-        //        );
 
-        //        html.AppendLine(TABLE_FOOTER);
+                byte[] responseData = System.Text.Encoding.UTF8.GetBytes(html.ToString());
 
-        //        html.AppendLine(FOOTER);
+                httpResponse.StatusCode = (int) HttpStatusCode.OK;
-
-        //        byte[] responseData = System.Text.Encoding.UTF8.GetBytes(html.ToString());
-
-        //        httpResponse.StatusCode = (int) HttpStatusCode.OK;
                 //httpResponse.Body.Write(responseData, 0, responseData.Length);
                 //httpResponse.Body.Flush();
-        //        return responseData;
+                return responseData;
-        //    }
+            }
 
-        //    #endregion IStreamedRequestHandler implementation
+            #endregion IStreamedRequestHandler implementation
-        //}
+        }
     }
 }

OpenSim/Grid/AssetInventoryServer/Plugins/OpenSim/OpenSimAssetStoragePlugin.cs

@@ -28,6 +28,7 @@
 using System;
 using System.Reflection;
 using System.Data;
+using System.Collections.Generic;
 using OpenMetaverse;
 using OpenSim.Framework;
 using OpenSim.Data;
@@ -117,42 +118,17 @@ namespace OpenSim.Grid.AssetInventoryServer.Plugins.OpenSim
         {
             int rowCount = 0;
 
-            //using (MySqlConnection dbConnection = new MySqlConnection(m_openSimConfig.GetString("asset_database_connect")))
+            foreach (AssetMetadata metadata in m_assetProvider.FetchAssetMetadataSet(start, count))
-            //{
+            {
-            //    MySqlDataReader reader;
+                // We set the ContentType here because Utils is only in
+                // AssetInventoryServer. This should be moved to the DB
+                // backends when the equivalent of SLAssetTypeToContentType is
+                // in OpenSim.Framework or similar.
+                metadata.ContentType = Utils.SLAssetTypeToContentType(metadata.Type);
 
-            //    try
+                action(metadata);
-            //    {
+                ++rowCount;
-            //        dbConnection.Open();
+            }
-
-            //        MySqlCommand command = dbConnection.CreateCommand();
-            //        command.CommandText = String.Format("SELECT name,description,assetType,temporary,data,id FROM assets LIMIT {0}, {1}",
-            //            start, count);
-            //        reader = command.ExecuteReader();
-            //    }
-            //    catch (MySqlException ex)
-            //    {
-            //        m_log.Error("[OPENSIMASSETSTORAGE]: Connection to MySQL backend failed: " + ex.Message);
-            //        return 0;
-            //    }
-
-            //    while (reader.Read())
-            //    {
-            //        Metadata metadata = new Metadata();
-            //        metadata.CreationDate = OpenMetaverse.Utils.Epoch;
-            //        metadata.Description = reader.GetString(1);
-            //        metadata.ID = UUID.Parse(reader.GetString(5));
-            //        metadata.Name = reader.GetString(0);
-            //        metadata.SHA1 = OpenMetaverse.Utils.SHA1((byte[])reader.GetValue(4));
-            //        metadata.Temporary = reader.GetBoolean(3);
-            //        metadata.ContentType = Utils.SLAssetTypeToContentType(reader.GetInt32(2));
-
-            //        action(metadata);
-            //        ++rowCount;
-            //    }
-
-            //    reader.Close();
-            //}
 
             return rowCount;
         }

OpenSim/Tests/Common/Mock/TestAssetDataPlugin.cs

@@ -25,6 +25,7 @@
  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
+using System.Collections.Generic;
 using OpenMetaverse;
 using OpenSim.Framework;
 using OpenSim.Data;
@@ -48,6 +49,7 @@ namespace OpenSim.Tests.Common.Mock
         public void CreateAsset(AssetBase asset) {}
         public void UpdateAsset(AssetBase asset) {}
         public bool ExistsAsset(UUID uuid) { return false; }
+        public List<AssetMetadata> FetchAssetMetadataSet(int start, int count) { return new List<AssetMetadata>(count); }
         public void Initialise(string connect) {}
     }
 }