diff --git a/OpenSim/Data/AssetDataBase.cs b/OpenSim/Data/AssetDataBase.cs index e1a810c9af..1bb432cc56 100644 --- a/OpenSim/Data/AssetDataBase.cs +++ b/OpenSim/Data/AssetDataBase.cs @@ -37,9 +37,8 @@ namespace OpenSim.Data public abstract class AssetDataBase : IAssetDataPlugin { public abstract AssetBase GetAsset(UUID uuid); - public abstract void StoreAsset(AssetBase asset); - public abstract bool ExistsAsset(UUID uuid); + public abstract bool[] AssetsExist(UUID[] uuids); public abstract List FetchAssetMetadataSet(int start, int count); diff --git a/OpenSim/Data/IAssetData.cs b/OpenSim/Data/IAssetData.cs index f31b215c0f..a41e310891 100644 --- a/OpenSim/Data/IAssetData.cs +++ b/OpenSim/Data/IAssetData.cs @@ -35,7 +35,7 @@ namespace OpenSim.Data { AssetBase GetAsset(UUID uuid); void StoreAsset(AssetBase asset); - bool ExistsAsset(UUID uuid); + bool[] AssetsExist(UUID[] uuids); List FetchAssetMetadataSet(int start, int count); void Initialise(string connect); bool Delete(string id); diff --git a/OpenSim/Data/IXAssetDataPlugin.cs b/OpenSim/Data/IXAssetDataPlugin.cs index 74ad6f497d..2d247974a2 100644 --- a/OpenSim/Data/IXAssetDataPlugin.cs +++ b/OpenSim/Data/IXAssetDataPlugin.cs @@ -39,7 +39,7 @@ namespace OpenSim.Data { AssetBase GetAsset(UUID uuid); void StoreAsset(AssetBase asset); - bool ExistsAsset(UUID uuid); + bool[] AssetsExist(UUID[] uuids); List FetchAssetMetadataSet(int start, int count); void Initialise(string connect); bool Delete(string id); diff --git a/OpenSim/Data/MSSQL/MSSQLAssetData.cs b/OpenSim/Data/MSSQL/MSSQLAssetData.cs index f3e008dc4b..ce7039616c 100644 --- a/OpenSim/Data/MSSQL/MSSQLAssetData.cs +++ b/OpenSim/Data/MSSQL/MSSQLAssetData.cs @@ -225,17 +225,38 @@ namespace OpenSim.Data.MSSQL // } /// - /// Check if asset exist in m_database + /// Check if the assets exist in the database. /// - /// - /// true if exist. - override public bool ExistsAsset(UUID uuid) + /// The assets' IDs + /// For each asset: true if it exists, false otherwise + public override bool[] AssetsExist(UUID[] uuids) { - if (GetAsset(uuid) != null) + if (uuids.Length == 0) + return new bool[0]; + + HashSet exist = new HashSet(); + + string ids = "'" + string.Join("','", uuids) + "'"; + string sql = string.Format("SELECT id FROM assets WHERE id IN ({0})", ids); + + using (SqlConnection conn = new SqlConnection(m_connectionString)) + using (SqlCommand cmd = new SqlCommand(sql, conn)) { - return true; + conn.Open(); + using (SqlDataReader reader = cmd.ExecuteReader()) + { + while (reader.Read()) + { + UUID id = DBGuid.FromDB(reader["id"]); + exist.Add(id); + } + } } - return false; + + bool[] results = new bool[uuids.Length]; + for (int i = 0; i < uuids.Length; i++) + results[i] = exist.Contains(uuids[i]); + return results; } /// diff --git a/OpenSim/Data/MySQL/MySQLAssetData.cs b/OpenSim/Data/MySQL/MySQLAssetData.cs index 21362b907f..c96139df75 100644 --- a/OpenSim/Data/MySQL/MySQLAssetData.cs +++ b/OpenSim/Data/MySQL/MySQLAssetData.cs @@ -257,46 +257,44 @@ namespace OpenSim.Data.MySQL } /// - /// Check if the asset exists in the database + /// Check if the assets exist in the database. /// - /// The asset UUID - /// true if it exists, false otherwise. - override public bool ExistsAsset(UUID uuid) + /// The assets' IDs + /// For each asset: true if it exists, false otherwise + public override bool[] AssetsExist(UUID[] uuids) { -// m_log.DebugFormat("[ASSETS DB]: Checking for asset {0}", uuid); + if (uuids.Length == 0) + return new bool[0]; - bool assetExists = false; + HashSet exist = new HashSet(); + + string ids = "'" + string.Join("','", uuids) + "'"; + string sql = string.Format("SELECT id FROM assets WHERE id IN ({0})", ids); lock (m_dbLock) { using (MySqlConnection dbcon = new MySqlConnection(m_connectionString)) { dbcon.Open(); - using (MySqlCommand cmd = new MySqlCommand("SELECT id FROM assets WHERE id=?id", dbcon)) + using (MySqlCommand cmd = new MySqlCommand(sql, dbcon)) { - cmd.Parameters.AddWithValue("?id", uuid.ToString()); - - try + using (MySqlDataReader dbReader = cmd.ExecuteReader()) { - using (MySqlDataReader dbReader = cmd.ExecuteReader(CommandBehavior.SingleRow)) + while (dbReader.Read()) { - if (dbReader.Read()) - { -// m_log.DebugFormat("[ASSETS DB]: Found asset {0}", uuid); - assetExists = true; - } + UUID id = DBGuid.FromDB(dbReader["id"]); + exist.Add(id); } } - catch (Exception e) - { - m_log.Error( - string.Format("[ASSETS DB]: MySql failure fetching asset {0}. Exception ", uuid), e); - } } } } - return assetExists; + bool[] results = new bool[uuids.Length]; + for (int i = 0; i < uuids.Length; i++) + results[i] = exist.Contains(uuids[i]); + + return results; } /// diff --git a/OpenSim/Data/MySQL/MySQLXAssetData.cs b/OpenSim/Data/MySQL/MySQLXAssetData.cs index 91389cef7d..1bf6a9a1db 100644 --- a/OpenSim/Data/MySQL/MySQLXAssetData.cs +++ b/OpenSim/Data/MySQL/MySQLXAssetData.cs @@ -397,45 +397,43 @@ namespace OpenSim.Data.MySQL } /// - /// Check if the asset exists in the database + /// Check if the assets exist in the database. /// - /// The asset UUID - /// true if it exists, false otherwise. - public bool ExistsAsset(UUID uuid) + /// The asset UUID's + /// For each asset: true if it exists, false otherwise + public bool[] AssetsExist(UUID[] uuids) { -// m_log.DebugFormat("[ASSETS DB]: Checking for asset {0}", uuid); + if (uuids.Length == 0) + return new bool[0]; - bool assetExists = false; + HashSet exists = new HashSet(); + + string ids = "'" + string.Join("','", uuids) + "'"; + string sql = string.Format("SELECT ID FROM assets WHERE ID IN ({0})", ids); lock (m_dbLock) { using (MySqlConnection dbcon = new MySqlConnection(m_connectionString)) { dbcon.Open(); - using (MySqlCommand cmd = new MySqlCommand("SELECT ID FROM XAssetsMeta WHERE ID=?ID", dbcon)) + using (MySqlCommand cmd = new MySqlCommand(sql, dbcon)) { - cmd.Parameters.AddWithValue("?ID", uuid.ToString()); - - try + using (MySqlDataReader dbReader = cmd.ExecuteReader()) { - using (MySqlDataReader dbReader = cmd.ExecuteReader(CommandBehavior.SingleRow)) + while (dbReader.Read()) { - if (dbReader.Read()) - { -// m_log.DebugFormat("[ASSETS DB]: Found asset {0}", uuid); - assetExists = true; - } + UUID id = DBGuid.FromDB(dbReader["ID"]); + exists.Add(id); } } - catch (Exception e) - { - m_log.Error(string.Format("[XASSETS DB]: MySql failure fetching asset {0}", uuid), e); - } } } } - return assetExists; + bool[] results = new bool[uuids.Length]; + for (int i = 0; i < uuids.Length; i++) + results[i] = exists.Contains(uuids[i]); + return results; } diff --git a/OpenSim/Data/PGSQL/PGSQLAssetData.cs b/OpenSim/Data/PGSQL/PGSQLAssetData.cs index ab748566b9..ca18dc9604 100644 --- a/OpenSim/Data/PGSQL/PGSQLAssetData.cs +++ b/OpenSim/Data/PGSQL/PGSQLAssetData.cs @@ -231,17 +231,38 @@ namespace OpenSim.Data.PGSQL // } /// - /// Check if asset exist in m_database + /// Check if the assets exist in the database. /// - /// - /// true if exist. - override public bool ExistsAsset(UUID uuid) + /// The assets' IDs + /// For each asset: true if it exists, false otherwise + public override bool[] AssetsExist(UUID[] uuids) { - if (GetAsset(uuid) != null) + if (uuids.Length == 0) + return new bool[0]; + + HashSet exist = new HashSet(); + + string ids = "'" + string.Join("','", uuids) + "'"; + string sql = string.Format("SELECT id FROM assets WHERE id IN ({0})", ids); + + using (NpgsqlConnection conn = new NpgsqlConnection(m_connectionString)) + using (NpgsqlCommand cmd = new NpgsqlCommand(sql, conn)) { - return true; + conn.Open(); + using (NpgsqlDataReader reader = cmd.ExecuteReader()) + { + while (reader.Read()) + { + UUID id = DBGuid.FromDB(reader["id"]); + exist.Add(id); + } + } } - return false; + + bool[] results = new bool[uuids.Length]; + for (int i = 0; i < uuids.Length; i++) + results[i] = exist.Contains(uuids[i]); + return results; } /// diff --git a/OpenSim/Data/PGSQL/PGSQLXAssetData.cs b/OpenSim/Data/PGSQL/PGSQLXAssetData.cs index e959619ed0..c6cebff982 100644 --- a/OpenSim/Data/PGSQL/PGSQLXAssetData.cs +++ b/OpenSim/Data/PGSQL/PGSQLXAssetData.cs @@ -406,6 +406,43 @@ namespace OpenSim.Data.PGSQL return exists; } + /// + /// Check if the assets exist in the database. + /// + /// The assets' IDs + /// For each asset: true if it exists, false otherwise + public bool[] AssetsExist(UUID[] uuids) + { + if (uuids.Length == 0) + return new bool[0]; + + HashSet exist = new HashSet(); + + string ids = "'" + string.Join("','", uuids) + "'"; + string sql = string.Format(@"SELECT ""ID"" FROM XAssetsMeta WHERE ""ID"" IN ({0})", ids); + + using (NpgsqlConnection conn = new NpgsqlConnection(m_connectionString)) + { + conn.Open(); + using (NpgsqlCommand cmd = new NpgsqlCommand(sql, conn)) + { + using (NpgsqlDataReader reader = cmd.ExecuteReader()) + { + while (reader.Read()) + { + UUID id = DBGuid.FromDB(reader["id"]); + exist.Add(id); + } + } + } + } + + bool[] results = new bool[uuids.Length]; + for (int i = 0; i < uuids.Length; i++) + results[i] = exist.Contains(uuids[i]); + return results; + } + /// /// Check if the asset exists in the database /// diff --git a/OpenSim/Data/SQLite/SQLiteAssetData.cs b/OpenSim/Data/SQLite/SQLiteAssetData.cs index c32982e1e3..1f3237619c 100644 --- a/OpenSim/Data/SQLite/SQLiteAssetData.cs +++ b/OpenSim/Data/SQLite/SQLiteAssetData.cs @@ -152,7 +152,7 @@ namespace OpenSim.Data.SQLite } //m_log.Info("[ASSET DB]: Creating Asset " + asset.FullID.ToString()); - if (ExistsAsset(asset.FullID)) + if (AssetsExist(new[] { asset.FullID })[0]) { //LogAssetLoad(asset); @@ -214,32 +214,39 @@ namespace OpenSim.Data.SQLite // } /// - /// Check if an asset exist in database + /// Check if the assets exist in the database. /// - /// The asset UUID - /// True if exist, or false. - override public bool ExistsAsset(UUID uuid) + /// The assets' IDs + /// For each asset: true if it exists, false otherwise + public override bool[] AssetsExist(UUID[] uuids) { - lock (this) + if (uuids.Length == 0) + return new bool[0]; + + HashSet exist = new HashSet(); + + string ids = "'" + string.Join("','", uuids) + "'"; + string sql = string.Format("SELECT id FROM assets WHERE id IN ({0})", ids); + + lock (this) { using (SqliteCommand cmd = new SqliteCommand(SelectAssetSQL, m_conn)) { - cmd.Parameters.Add(new SqliteParameter(":UUID", uuid.ToString())); using (IDataReader reader = cmd.ExecuteReader()) { - if (reader.Read()) + while (reader.Read()) { - reader.Close(); - return true; - } - else - { - reader.Close(); - return false; + UUID id = new UUID((string)reader["UUID"]); + exist.Add(id); } } } } + + bool[] results = new bool[uuids.Length]; + for (int i = 0; i < uuids.Length; i++) + results[i] = exist.Contains(uuids[i]); + return results; } /// diff --git a/OpenSim/Data/Tests/AssetTests.cs b/OpenSim/Data/Tests/AssetTests.cs index 8cb2ee083d..d778d1c046 100644 --- a/OpenSim/Data/Tests/AssetTests.cs +++ b/OpenSim/Data/Tests/AssetTests.cs @@ -107,10 +107,11 @@ namespace OpenSim.Data.Tests public void T001_LoadEmpty() { TestHelpers.InMethod(); - - Assert.That(m_db.ExistsAsset(uuid1), Is.False); - Assert.That(m_db.ExistsAsset(uuid2), Is.False); - Assert.That(m_db.ExistsAsset(uuid3), Is.False); + + bool[] exist = m_db.AssetsExist(new[] { uuid1, uuid2, uuid3 }); + Assert.IsFalse(exist[0]); + Assert.IsFalse(exist[1]); + Assert.IsFalse(exist[2]); } [Test] @@ -159,9 +160,10 @@ namespace OpenSim.Data.Tests AssetBase a3b = m_db.GetAsset(uuid3); Assert.That(a3b, Constraints.PropertyCompareConstraint(a3a)); - Assert.That(m_db.ExistsAsset(uuid1), Is.True); - Assert.That(m_db.ExistsAsset(uuid2), Is.True); - Assert.That(m_db.ExistsAsset(uuid3), Is.True); + bool[] exist = m_db.AssetsExist(new[] { uuid1, uuid2, uuid3 }); + Assert.IsTrue(exist[0]); + Assert.IsTrue(exist[1]); + Assert.IsTrue(exist[2]); List metadatas = m_db.FetchAssetMetadataSet(0, 1000); diff --git a/OpenSim/Framework/Util.cs b/OpenSim/Framework/Util.cs index c7a7341159..b133ff37b0 100644 --- a/OpenSim/Framework/Util.cs +++ b/OpenSim/Framework/Util.cs @@ -52,6 +52,8 @@ using OpenMetaverse; using OpenMetaverse.StructuredData; using Amib.Threading; using System.Collections.Concurrent; +using System.Collections.Specialized; +using System.Web; namespace OpenSim.Framework { @@ -865,6 +867,54 @@ namespace OpenSim.Framework return null; } + /// + /// Parses a foreign asset ID. + /// + /// A possibly-foreign asset ID: http://grid.example.com:8002/00000000-0000-0000-0000-000000000000 + /// The URL: http://grid.example.com:8002 + /// The asset ID: 00000000-0000-0000-0000-000000000000. Returned even if 'id' isn't foreign. + /// True: this is a foreign asset ID; False: it isn't + public static bool ParseForeignAssetID(string id, out string url, out string assetID) + { + url = String.Empty; + assetID = String.Empty; + + UUID uuid; + if (UUID.TryParse(id, out uuid)) + { + assetID = uuid.ToString(); + return false; + } + + if ((id.Length == 0) || (id[0] != 'h' && id[0] != 'H')) + return false; + + Uri assetUri; + if (!Uri.TryCreate(id, UriKind.Absolute, out assetUri) || assetUri.Scheme != Uri.UriSchemeHttp) + return false; + + // Simian + if (assetUri.Query != string.Empty) + { + NameValueCollection qscoll = HttpUtility.ParseQueryString(assetUri.Query); + assetID = qscoll["id"]; + if (assetID != null) + url = id.Replace(assetID, ""); // Malformed again, as simian expects + else + url = id; // !!! best effort + } + else // robust + { + url = "http://" + assetUri.Authority; + assetID = assetUri.LocalPath.Trim(new char[] { '/' }); + } + + if (!UUID.TryParse(assetID, out uuid)) + return false; + + return true; + } + /// /// Removes all invalid path chars (OS dependent) /// diff --git a/OpenSim/Region/CoreModules/Asset/FlotsamAssetCache.cs b/OpenSim/Region/CoreModules/Asset/FlotsamAssetCache.cs index c7bd3d0c73..f06d70d9e0 100644 --- a/OpenSim/Region/CoreModules/Asset/FlotsamAssetCache.cs +++ b/OpenSim/Region/CoreModules/Asset/FlotsamAssetCache.cs @@ -1038,6 +1038,18 @@ namespace OpenSim.Region.CoreModules.Asset return true; } + public bool[] AssetsExist(string[] ids) + { + bool[] exist = new bool[ids.Length]; + + for (int i = 0; i < ids.Length; i++) + { + exist[i] = Check(ids[i]); + } + + return exist; + } + public string Store(AssetBase asset) { if (asset.FullID == UUID.Zero) diff --git a/OpenSim/Region/CoreModules/Framework/InventoryAccess/HGAssetMapper.cs b/OpenSim/Region/CoreModules/Framework/InventoryAccess/HGAssetMapper.cs index d4fb1baf48..532bc74a76 100644 --- a/OpenSim/Region/CoreModules/Framework/InventoryAccess/HGAssetMapper.cs +++ b/OpenSim/Region/CoreModules/Framework/InventoryAccess/HGAssetMapper.cs @@ -145,11 +145,11 @@ namespace OpenSim.Region.CoreModules.Framework.InventoryAccess string id = m_scene.AssetService.Store(asset1); if (id == string.Empty) { - m_log.DebugFormat("[HG ASSET MAPPER]: Asset server {0} did not accept {1}", url, asset.ID); + m_log.DebugFormat("[HG ASSET MAPPER]: Failed to post asset {0} to asset server {1}: the server did not accept the asset", asset.ID, url); success = false; } else - m_log.DebugFormat("[HG ASSET MAPPER]: Posted copy of asset {0} from local asset server to {1}", asset1.ID, url); + m_log.DebugFormat("[HG ASSET MAPPER]: Posted asset {0} to asset server {1}", asset1.ID, url); } return success; } @@ -279,36 +279,65 @@ namespace OpenSim.Region.CoreModules.Framework.InventoryAccess public void Post(UUID assetID, UUID ownerID, string userAssetURL) { - // Post the item from the local AssetCache onto the remote asset server - // and place an entry in m_assetMap + m_log.DebugFormat("[HG ASSET MAPPER]: Starting to send asset {0} with children to asset server {1}", assetID, userAssetURL); + + // Find all the embedded assets - m_log.Debug("[HG ASSET MAPPER]: Posting object " + assetID + " to asset server " + userAssetURL); AssetBase asset = m_scene.AssetService.Get(assetID.ToString()); - if (asset != null) + if (asset == null) { - Dictionary ids = new Dictionary(); - HGUuidGatherer uuidGatherer = new HGUuidGatherer(m_scene.AssetService, string.Empty); - uuidGatherer.GatherAssetUuids(asset.FullID, asset.Type, ids); - bool success = false; - foreach (UUID uuid in ids.Keys) + m_log.DebugFormat("[HG ASSET MAPPER]: Something wrong with asset {0}, it could not be found", assetID); + return; + } + + Dictionary ids = new Dictionary(); + HGUuidGatherer uuidGatherer = new HGUuidGatherer(m_scene.AssetService, string.Empty); + uuidGatherer.GatherAssetUuids(asset.FullID, asset.Type, ids); + + // Check which assets already exist in the destination server + + string url = userAssetURL; + if (!url.EndsWith("/") && !url.EndsWith("=")) + url = url + "/"; + + string[] remoteAssetIDs = new string[ids.Count]; + int i = 0; + foreach (UUID id in ids.Keys) + remoteAssetIDs[i++] = url + id.ToString(); + + bool[] exist = m_scene.AssetService.AssetsExist(remoteAssetIDs); + + var existSet = new HashSet(); + i = 0; + foreach (UUID id in ids.Keys) + { + if (exist[i]) + existSet.Add(id.ToString()); + ++i; + } + + // Send only those assets which don't already exist in the destination server + + bool success = true; + + foreach (UUID uuid in ids.Keys) + { + if (!existSet.Contains(uuid.ToString())) { asset = m_scene.AssetService.Get(uuid.ToString()); if (asset == null) m_log.DebugFormat("[HG ASSET MAPPER]: Could not find asset {0}", uuid); else - success = PostAsset(userAssetURL, asset); + success &= PostAsset(userAssetURL, asset); } - - // maybe all pieces got there... - if (!success) - m_log.DebugFormat("[HG ASSET MAPPER]: Problems posting item {0} to asset server {1}", assetID, userAssetURL); else - m_log.DebugFormat("[HG ASSET MAPPER]: Successfully posted item {0} to asset server {1}", assetID, userAssetURL); - + m_log.DebugFormat("[HG ASSET MAPPER]: Didn't post asset {0} because it already exists in asset server {1}", uuid, userAssetURL); } - else - m_log.DebugFormat("[HG ASSET MAPPER]: Something wrong with asset {0}, it could not be found", assetID); + if (!success) + m_log.DebugFormat("[HG ASSET MAPPER]: Problems sending asset {0} with children to asset server {1}", assetID, userAssetURL); + else + m_log.DebugFormat("[HG ASSET MAPPER]: Successfully sent asset {0} with children to asset server {1}", assetID, userAssetURL); } #endregion diff --git a/OpenSim/Region/CoreModules/ServiceConnectorsOut/Asset/HGAssetBroker.cs b/OpenSim/Region/CoreModules/ServiceConnectorsOut/Asset/HGAssetBroker.cs index 9f58175722..ff8b051098 100644 --- a/OpenSim/Region/CoreModules/ServiceConnectorsOut/Asset/HGAssetBroker.cs +++ b/OpenSim/Region/CoreModules/ServiceConnectorsOut/Asset/HGAssetBroker.cs @@ -312,6 +312,23 @@ namespace OpenSim.Region.CoreModules.ServiceConnectorsOut.Asset } } + public virtual bool[] AssetsExist(string[] ids) + { + int numHG = 0; + foreach (string id in ids) + { + if (IsHG(id)) + ++numHG; + } + + if (numHG == 0) + return m_GridService.AssetsExist(ids); + else if (numHG == ids.Length) + return m_HGService.AssetsExist(ids); + else + throw new Exception("[HG ASSET CONNECTOR]: AssetsExist: all the assets must be either local or foreign"); + } + public string Store(AssetBase asset) { bool isHG = IsHG(asset.ID); diff --git a/OpenSim/Region/CoreModules/ServiceConnectorsOut/Asset/LocalAssetServiceConnector.cs b/OpenSim/Region/CoreModules/ServiceConnectorsOut/Asset/LocalAssetServiceConnector.cs index 52b103978e..97b7559ed7 100644 --- a/OpenSim/Region/CoreModules/ServiceConnectorsOut/Asset/LocalAssetServiceConnector.cs +++ b/OpenSim/Region/CoreModules/ServiceConnectorsOut/Asset/LocalAssetServiceConnector.cs @@ -253,6 +253,11 @@ namespace OpenSim.Region.CoreModules.ServiceConnectorsOut.Asset }); } + public bool[] AssetsExist(string[] ids) + { + return m_AssetService.AssetsExist(ids); + } + public string Store(AssetBase asset) { if (m_Cache != null) diff --git a/OpenSim/Server/Handlers/Asset/AssetServerConnector.cs b/OpenSim/Server/Handlers/Asset/AssetServerConnector.cs index cc4325ac4c..9b86986c55 100644 --- a/OpenSim/Server/Handlers/Asset/AssetServerConnector.cs +++ b/OpenSim/Server/Handlers/Asset/AssetServerConnector.cs @@ -86,6 +86,7 @@ namespace OpenSim.Server.Handlers.Asset server.AddStreamHandler(new AssetServerGetHandler(m_AssetService)); server.AddStreamHandler(new AssetServerPostHandler(m_AssetService)); server.AddStreamHandler(new AssetServerDeleteHandler(m_AssetService, allowedRemoteDeleteTypes)); + server.AddStreamHandler(new AssetsExistHandler(m_AssetService)); MainConsole.Instance.Commands.AddCommand("Assets", false, "show asset", @@ -212,4 +213,4 @@ namespace OpenSim.Server.Handlers.Asset } } } -} \ No newline at end of file +} diff --git a/OpenSim/Server/Handlers/Asset/AssetServerGetHandler.cs b/OpenSim/Server/Handlers/Asset/AssetServerGetHandler.cs index 8b23a832ac..ed3b4af62e 100644 --- a/OpenSim/Server/Handlers/Asset/AssetServerGetHandler.cs +++ b/OpenSim/Server/Handlers/Asset/AssetServerGetHandler.cs @@ -64,45 +64,61 @@ namespace OpenSim.Server.Handlers.Asset if (p.Length == 0) return result; - if (p.Length > 1 && p[1] == "data") + if (p.Length > 1) { - result = m_AssetService.GetData(p[0]); - if (result == null) + string id = p[0]; + string cmd = p[1]; + + if (cmd == "data") { - httpResponse.StatusCode = (int)HttpStatusCode.NotFound; - httpResponse.ContentType = "text/plain"; - result = new byte[0]; + result = m_AssetService.GetData(id); + if (result == null) + { + httpResponse.StatusCode = (int)HttpStatusCode.NotFound; + httpResponse.ContentType = "text/plain"; + result = new byte[0]; + } + else + { + httpResponse.StatusCode = (int)HttpStatusCode.OK; + httpResponse.ContentType = "application/octet-stream"; + } + } + else if (cmd == "metadata") + { + AssetMetadata metadata = m_AssetService.GetMetadata(id); + + if (metadata != null) + { + XmlSerializer xs = + new XmlSerializer(typeof(AssetMetadata)); + result = ServerUtils.SerializeResult(xs, metadata); + + httpResponse.StatusCode = (int)HttpStatusCode.OK; + httpResponse.ContentType = + SLUtil.SLAssetTypeToContentType(metadata.Type); + } + else + { + httpResponse.StatusCode = (int)HttpStatusCode.NotFound; + httpResponse.ContentType = "text/plain"; + result = new byte[0]; + } } else { - httpResponse.StatusCode = (int)HttpStatusCode.OK; - httpResponse.ContentType = "application/octet-stream"; - } - } - else if (p.Length > 1 && p[1] == "metadata") - { - AssetMetadata metadata = m_AssetService.GetMetadata(p[0]); - - if (metadata != null) - { - XmlSerializer xs = - new XmlSerializer(typeof(AssetMetadata)); - result = ServerUtils.SerializeResult(xs, metadata); - - httpResponse.StatusCode = (int)HttpStatusCode.OK; - httpResponse.ContentType = - SLUtil.SLAssetTypeToContentType(metadata.Type); - } - else - { - httpResponse.StatusCode = (int)HttpStatusCode.NotFound; + // Unknown request + httpResponse.StatusCode = (int)HttpStatusCode.BadRequest; httpResponse.ContentType = "text/plain"; result = new byte[0]; } } - else + else if (p.Length == 1) { - AssetBase asset = m_AssetService.Get(p[0]); + // Get the entire asset (metadata + data) + + string id = p[0]; + AssetBase asset = m_AssetService.Get(id); if (asset != null) { @@ -120,6 +136,14 @@ namespace OpenSim.Server.Handlers.Asset result = new byte[0]; } } + else + { + // Unknown request + httpResponse.StatusCode = (int)HttpStatusCode.BadRequest; + httpResponse.ContentType = "text/plain"; + result = new byte[0]; + } + return result; } } diff --git a/OpenSim/Server/Handlers/Asset/AssetsExistHandler.cs b/OpenSim/Server/Handlers/Asset/AssetsExistHandler.cs new file mode 100644 index 0000000000..6d01f86d6c --- /dev/null +++ b/OpenSim/Server/Handlers/Asset/AssetsExistHandler.cs @@ -0,0 +1,80 @@ +/* + * Copyright (c) Contributors, http://opensimulator.org/ + * See CONTRIBUTORS.TXT for a full list of copyright holders. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the OpenSimulator Project nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +using Nini.Config; +using log4net; +using System; +using System.Reflection; +using System.IO; +using System.Net; +using System.Text; +using System.Text.RegularExpressions; +using System.Xml; +using System.Xml.Serialization; +using OpenSim.Server.Base; +using OpenSim.Services.Interfaces; +using OpenSim.Framework; +using OpenSim.Framework.Servers.HttpServer; +using OpenMetaverse; + +namespace OpenSim.Server.Handlers.Asset +{ + public class AssetsExistHandler : BaseStreamHandler + { + //private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); + + private IAssetService m_AssetService; + + public AssetsExistHandler(IAssetService service) : + base("POST", "/get_assets_exist") + { + m_AssetService = service; + } + + protected override byte[] ProcessRequest(string path, Stream request, IOSHttpRequest httpRequest, IOSHttpResponse httpResponse) + { + XmlSerializer xs; + + string[] ids; + try + { + xs = new XmlSerializer(typeof(string[])); + ids = (string[])xs.Deserialize(request); + } + catch (Exception) + { + httpResponse.StatusCode = (int)HttpStatusCode.BadRequest; + return null; + } + + bool[] exist = m_AssetService.AssetsExist(ids); + + xs = new XmlSerializer(typeof(bool[])); + return ServerUtils.SerializeResult(xs, exist); + } + } +} diff --git a/OpenSim/Services/AssetService/AssetService.cs b/OpenSim/Services/AssetService/AssetService.cs index 08fd3f858a..0aefa16fd2 100644 --- a/OpenSim/Services/AssetService/AssetService.cs +++ b/OpenSim/Services/AssetService/AssetService.cs @@ -153,9 +153,24 @@ namespace OpenSim.Services.AssetService return true; } + public virtual bool[] AssetsExist(string[] ids) + { + try + { + UUID[] uuid = Array.ConvertAll(ids, id => UUID.Parse(id)); + return m_Database.AssetsExist(uuid); + } + catch (Exception e) + { + m_log.Error("[ASSET SERVICE]: Exception getting assets ", e); + return new bool[ids.Length]; + } + } + public virtual string Store(AssetBase asset) { - if (!m_Database.ExistsAsset(asset.FullID)) + bool exists = m_Database.AssetsExist(new[] { asset.FullID })[0]; + if (!exists) { // m_log.DebugFormat( // "[ASSET SERVICE]: Storing asset {0} {1}, bytes {2}", asset.Name, asset.FullID, asset.Data.Length); @@ -186,4 +201,4 @@ namespace OpenSim.Services.AssetService return m_Database.Delete(id); } } -} \ No newline at end of file +} diff --git a/OpenSim/Services/AssetService/XAssetService.cs b/OpenSim/Services/AssetService/XAssetService.cs index 6047616f65..f58b769d25 100644 --- a/OpenSim/Services/AssetService/XAssetService.cs +++ b/OpenSim/Services/AssetService/XAssetService.cs @@ -175,9 +175,16 @@ namespace OpenSim.Services.AssetService return true; } + public virtual bool[] AssetsExist(string[] ids) + { + UUID[] uuid = Array.ConvertAll(ids, id => UUID.Parse(id)); + return m_Database.AssetsExist(uuid); + } + public virtual string Store(AssetBase asset) { - if (!m_Database.ExistsAsset(asset.FullID)) + bool exists = m_Database.AssetsExist(new[] { asset.FullID })[0]; + if (!exists) { // m_log.DebugFormat( // "[XASSET SERVICE]: Storing asset {0} {1}, bytes {2}", asset.Name, asset.FullID, asset.Data.Length); @@ -217,4 +224,4 @@ namespace OpenSim.Services.AssetService m_ChainedAssetService.Delete(asset.ID); } } -} \ No newline at end of file +} diff --git a/OpenSim/Services/Connectors/Asset/AssetServicesConnector.cs b/OpenSim/Services/Connectors/Asset/AssetServicesConnector.cs index 8b04d7f4f3..32415e9e3a 100644 --- a/OpenSim/Services/Connectors/Asset/AssetServicesConnector.cs +++ b/OpenSim/Services/Connectors/Asset/AssetServicesConnector.cs @@ -254,6 +254,27 @@ namespace OpenSim.Services.Connectors return true; } + public virtual bool[] AssetsExist(string[] ids) + { + string uri = m_ServerURI + "/get_assets_exist"; + + bool[] exist = null; + try + { + exist = SynchronousRestObjectRequester.MakeRequest("POST", uri, ids); + } + catch (Exception) + { + // This is most likely to happen because the server doesn't support this function, + // so just silently return "doesn't exist" for all the assets. + } + + if (exist == null) + exist = new bool[ids.Length]; + + return exist; + } + public string Store(AssetBase asset) { if (asset.Local) diff --git a/OpenSim/Services/Connectors/Asset/HGAssetServiceConnector.cs b/OpenSim/Services/Connectors/Asset/HGAssetServiceConnector.cs index c395178740..3710c86d22 100644 --- a/OpenSim/Services/Connectors/Asset/HGAssetServiceConnector.cs +++ b/OpenSim/Services/Connectors/Asset/HGAssetServiceConnector.cs @@ -36,6 +36,7 @@ using OpenSim.Framework; using OpenSim.Services.Interfaces; using OpenSim.Services.Connectors.Hypergrid; using OpenSim.Services.Connectors.SimianGrid; +using OpenMetaverse; namespace OpenSim.Services.Connectors { @@ -83,39 +84,6 @@ namespace OpenSim.Services.Connectors } } - private bool StringToUrlAndAssetID(string id, out string url, out string assetID) - { - url = String.Empty; - assetID = String.Empty; - - Uri assetUri; - - if (Uri.TryCreate(id, UriKind.Absolute, out assetUri) && - assetUri.Scheme == Uri.UriSchemeHttp) - { - // Simian - if (assetUri.Query != string.Empty) - { - NameValueCollection qscoll = HttpUtility.ParseQueryString(assetUri.Query); - assetID = qscoll["id"]; - if (assetID != null) - url = id.Replace(assetID, ""); // Malformed again, as simian expects - else - url = id; // !!! best effort - } - else // robust - { - url = "http://" + assetUri.Authority; - assetID = assetUri.LocalPath.Trim(new char[] { '/' }); - } - - return true; - } - - m_log.DebugFormat("[HG ASSET SERVICE]: Malformed URL {0}", id); - return false; - } - private IAssetService GetConnector(string url) { IAssetService connector = null; @@ -149,7 +117,7 @@ namespace OpenSim.Services.Connectors string url = string.Empty; string assetID = string.Empty; - if (StringToUrlAndAssetID(id, out url, out assetID)) + if (Util.ParseForeignAssetID(id, out url, out assetID)) { IAssetService connector = GetConnector(url); return connector.Get(assetID); @@ -163,7 +131,7 @@ namespace OpenSim.Services.Connectors string url = string.Empty; string assetID = string.Empty; - if (StringToUrlAndAssetID(id, out url, out assetID)) + if (Util.ParseForeignAssetID(id, out url, out assetID)) { IAssetService connector = GetConnector(url); return connector.GetCached(assetID); @@ -177,7 +145,7 @@ namespace OpenSim.Services.Connectors string url = string.Empty; string assetID = string.Empty; - if (StringToUrlAndAssetID(id, out url, out assetID)) + if (Util.ParseForeignAssetID(id, out url, out assetID)) { IAssetService connector = GetConnector(url); return connector.GetMetadata(assetID); @@ -196,7 +164,7 @@ namespace OpenSim.Services.Connectors string url = string.Empty; string assetID = string.Empty; - if (StringToUrlAndAssetID(id, out url, out assetID)) + if (Util.ParseForeignAssetID(id, out url, out assetID)) { IAssetService connector = GetConnector(url); return connector.Get(assetID, sender, handler); @@ -205,12 +173,72 @@ namespace OpenSim.Services.Connectors return false; } + + private struct AssetAndIndex + { + public UUID assetID; + public int index; + + public AssetAndIndex(UUID assetID, int index) + { + this.assetID = assetID; + this.index = index; + } + } + + public virtual bool[] AssetsExist(string[] ids) + { + // This method is a bit complicated because it works even if the assets belong to different + // servers; that requires sending separate requests to each server. + + // Group the assets by the server they belong to + + var url2assets = new Dictionary>(); + + for (int i = 0; i < ids.Length; i++) + { + string url = string.Empty; + string assetID = string.Empty; + + if (Util.ParseForeignAssetID(ids[i], out url, out assetID)) + { + if (!url2assets.ContainsKey(url)) + url2assets.Add(url, new List()); + url2assets[url].Add(new AssetAndIndex(UUID.Parse(assetID), i)); + } + } + + // Query each of the servers in turn + + bool[] exist = new bool[ids.Length]; + + foreach (string url in url2assets.Keys) + { + IAssetService connector = GetConnector(url); + lock (EndPointLock(connector)) + { + List curAssets = url2assets[url]; + string[] assetIDs = curAssets.ConvertAll(a => a.assetID.ToString()).ToArray(); + bool[] curExist = connector.AssetsExist(assetIDs); + + int i = 0; + foreach (AssetAndIndex ai in curAssets) + { + exist[ai.index] = curExist[i]; + ++i; + } + } + } + + return exist; + } + public string Store(AssetBase asset) { string url = string.Empty; string assetID = string.Empty; - if (StringToUrlAndAssetID(asset.ID, out url, out assetID)) + if (Util.ParseForeignAssetID(asset.ID, out url, out assetID)) { IAssetService connector = GetConnector(url); // Restore the assetID to a simple UUID diff --git a/OpenSim/Services/Connectors/SimianGrid/SimianAssetServiceConnector.cs b/OpenSim/Services/Connectors/SimianGrid/SimianAssetServiceConnector.cs index 6f8d9ed2e4..01cbf91e3e 100644 --- a/OpenSim/Services/Connectors/SimianGrid/SimianAssetServiceConnector.cs +++ b/OpenSim/Services/Connectors/SimianGrid/SimianAssetServiceConnector.cs @@ -231,6 +231,26 @@ namespace OpenSim.Services.Connectors.SimianGrid return true; } + public bool[] AssetsExist(string[] ids) + { + if (String.IsNullOrEmpty(m_serverUrl)) + { + m_log.Error("[SIMIAN ASSET CONNECTOR]: No AssetServerURI configured"); + throw new InvalidOperationException(); + } + + bool[] exist = new bool[ids.Length]; + + for (int i = 0; i < ids.Length; i++) + { + AssetMetadata metadata = GetMetadata(ids[i]); + if (metadata != null) + exist[i] = true; + } + + return exist; + } + /// /// Creates a new asset /// diff --git a/OpenSim/Services/Interfaces/IAssetService.cs b/OpenSim/Services/Interfaces/IAssetService.cs index 3c469c6f43..8f1e039f8d 100644 --- a/OpenSim/Services/Interfaces/IAssetService.cs +++ b/OpenSim/Services/Interfaces/IAssetService.cs @@ -75,6 +75,13 @@ namespace OpenSim.Services.Interfaces /// /// True if the id was parseable, false otherwise bool Get(string id, Object sender, AssetRetrieved handler); + + /// + /// Check if assets exist in the database. + /// + /// The assets' IDs + /// For each asset: true if it exists, false otherwise + bool[] AssetsExist(string[] ids); /// /// Creates a new asset diff --git a/OpenSim/Tests/Common/Mock/BaseAssetRepository.cs b/OpenSim/Tests/Common/Mock/BaseAssetRepository.cs index cfefd38651..e6e08cd107 100644 --- a/OpenSim/Tests/Common/Mock/BaseAssetRepository.cs +++ b/OpenSim/Tests/Common/Mock/BaseAssetRepository.cs @@ -25,6 +25,7 @@ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +using System; using System.Collections.Generic; using OpenMetaverse; using OpenSim.Framework; @@ -37,7 +38,7 @@ namespace OpenSim.Tests.Common.Mock public AssetBase FetchAsset(UUID uuid) { - if (ExistsAsset(uuid)) + if (AssetsExist(new[] { uuid })[0]) return Assets[uuid]; else return null; @@ -53,9 +54,9 @@ namespace OpenSim.Tests.Common.Mock CreateAsset(asset); } - public bool ExistsAsset(UUID uuid) - { - return Assets.ContainsKey(uuid); + public bool[] AssetsExist(UUID[] uuids) + { + return Array.ConvertAll(uuids, id => Assets.ContainsKey(id)); } } } \ No newline at end of file