From 0bf1209f908bb9a384ddb3a4255a75bf2317c478 Mon Sep 17 00:00:00 2001 From: Diva Canto Date: Fri, 8 May 2015 20:53:28 -0700 Subject: [PATCH] Another major overhaul of inventory downloading, this time pertaining to inventory links. Added yet another function to IInventoryService to get multiple items at once, so that fetching collections of linked items is done once per folder instead of once per item. --- .../FetchInvDescHandler.cs | 270 ++++++++++-------- .../Library/LocalInventoryService.cs | 16 ++ .../Inventory/HGInventoryBroker.cs | 17 ++ .../LocalInventoryServiceConnector.cs | 5 + .../RemoteXInventoryServiceConnector.cs | 8 + .../Inventory/XInventoryInConnector.cs | 36 +++ .../Inventory/XInventoryServicesConnector.cs | 37 +++ .../SimianInventoryServiceConnector.cs | 15 + .../Services/Interfaces/IInventoryService.cs | 9 +- .../InventoryService/XInventoryService.cs | 15 + 10 files changed, 311 insertions(+), 117 deletions(-) diff --git a/OpenSim/Capabilities/Handlers/FetchInventoryDescendents/FetchInvDescHandler.cs b/OpenSim/Capabilities/Handlers/FetchInventoryDescendents/FetchInvDescHandler.cs index b7f332d1d5..8dd6235461 100644 --- a/OpenSim/Capabilities/Handlers/FetchInventoryDescendents/FetchInvDescHandler.cs +++ b/OpenSim/Capabilities/Handlers/FetchInventoryDescendents/FetchInvDescHandler.cs @@ -603,127 +603,17 @@ namespace OpenSim.Capabilities.Handlers // Do some post-processing. May need to fetch more from inv server for links foreach (InventoryCollection contents in fetchedContents) { - InventoryCollectionWithDescendents coll = new InventoryCollectionWithDescendents(); - coll.Collection = contents; - - if (contents == null) - { - bad_folders.Add(fids[i++]); - continue; - } - - // Find the original request LLSDFetchInventoryDescendents freq = fetchFolders[i++]; - // The inventory server isn't sending FolderID in the collection... - // Must fetch it individually - if (contents.FolderID == UUID.Zero) - { - InventoryFolderBase containingFolder = new InventoryFolderBase(); - containingFolder.ID = freq.folder_id; - containingFolder.Owner = freq.owner_id; - containingFolder = m_InventoryService.GetFolder(containingFolder); + InventoryCollectionWithDescendents coll = new InventoryCollectionWithDescendents(); + coll.Collection = contents; - if (containingFolder != null) - { - contents.FolderID = containingFolder.ID; - contents.OwnerID = containingFolder.Owner; - contents.Version = containingFolder.Version; - } - else - { - // Was it really a request for folder Zero? - // This is an overkill, but Firestorm really asks for folder Zero. - // I'm leaving the code here for the time being, but commented. - if (fetchFolders[i - 1].folder_id == UUID.Zero) - { - //coll.Collection.OwnerID = freq.owner_id; - //coll.Collection.FolderID = contents.FolderID; - //containingFolder = m_InventoryService.GetRootFolder(freq.owner_id); - //if (containingFolder != null) - //{ - // m_log.WarnFormat("[WEB FETCH INV DESC HANDLER]: Request for parent of folder {0}", containingFolder.ID); - // coll.Collection.Folders.Clear(); - // coll.Collection.Folders.Add(containingFolder); - // if (m_LibraryService != null && m_LibraryService.LibraryRootFolder != null) - // { - // InventoryFolderBase lib = new InventoryFolderBase(m_LibraryService.LibraryRootFolder.ID, m_LibraryService.LibraryRootFolder.Owner); - // lib.Name = m_LibraryService.LibraryRootFolder.Name; - // lib.Type = m_LibraryService.LibraryRootFolder.Type; - // lib.Version = m_LibraryService.LibraryRootFolder.Version; - // coll.Collection.Folders.Add(lib); - // } - // coll.Collection.Items.Clear(); - //} - } - else - { - m_log.WarnFormat("[WEB FETCH INV DESC HANDLER]: Unable to fetch folder {0}", freq.folder_id); - bad_folders.Add(freq.folder_id); - continue; - } - } - } + if (BadFolder(freq, contents, bad_folders)) + continue; - if (freq.fetch_items && contents.Items != null) - { - List itemsToReturn = contents.Items; - List originalItems = new List(itemsToReturn); - - // descendents must only include the links, not the linked items we add - coll.Descendents = originalItems.Count; - - // Add target items for links in this folder before the links themselves. - foreach (InventoryItemBase item in originalItems) - { - if (item.AssetType == (int)AssetType.Link) - { - InventoryItemBase linkedItem = m_InventoryService.GetItem(new InventoryItemBase(item.AssetID)); - - // Take care of genuinely broken links where the target doesn't exist - // HACK: Also, don't follow up links that just point to other links. In theory this is legitimate, - // but no viewer has been observed to set these up and this is the lazy way of avoiding cycles - // rather than having to keep track of every folder requested in the recursion. - if (linkedItem != null && linkedItem.AssetType != (int)AssetType.Link) - { - itemsToReturn.Insert(0, linkedItem); - } - } - } - - // Now scan for folder links and insert the items they target and those links at the head of the return data - foreach (InventoryItemBase item in originalItems) - { - if (item.AssetType == (int)AssetType.LinkFolder) - { - InventoryCollection linkedFolderContents = m_InventoryService.GetFolderContent(coll.Collection.OwnerID, item.AssetID); - List links = linkedFolderContents.Items; - - itemsToReturn.InsertRange(0, links); - - foreach (InventoryItemBase link in linkedFolderContents.Items) - { - // Take care of genuinely broken links where the target doesn't exist - // HACK: Also, don't follow up links that just point to other links. In theory this is legitimate, - // but no viewer has been observed to set these up and this is the lazy way of avoiding cycles - // rather than having to keep track of every folder requested in the recursion. - if (link != null) - { - //m_log.DebugFormat( - // "[WEB FETCH INV DESC HANDLER]: Adding item {0} {1} from folder {2} linked from {3}", - // link.Name, (AssetType)link.AssetType, item.AssetID, contents.FolderID); - - InventoryItemBase linkedItem - = m_InventoryService.GetItem(new InventoryItemBase(link.AssetID)); - - if (linkedItem != null) - itemsToReturn.Insert(0, linkedItem); - } - } - } - } - } + // Next: link management + ProcessLinks(freq, coll); result.Add(coll); } @@ -732,6 +622,154 @@ namespace OpenSim.Capabilities.Handlers return result; } + private bool BadFolder(LLSDFetchInventoryDescendents freq, InventoryCollection contents, List bad_folders) + { + bool bad = false; + if (contents == null) + { + bad_folders.Add(freq.folder_id); + bad = true; + } + + // The inventory server isn't sending FolderID in the collection... + // Must fetch it individually + if (contents.FolderID == UUID.Zero) + { + InventoryFolderBase containingFolder = new InventoryFolderBase(); + containingFolder.ID = freq.folder_id; + containingFolder.Owner = freq.owner_id; + containingFolder = m_InventoryService.GetFolder(containingFolder); + + if (containingFolder != null) + { + contents.FolderID = containingFolder.ID; + contents.OwnerID = containingFolder.Owner; + contents.Version = containingFolder.Version; + } + else + { + // Was it really a request for folder Zero? + // This is an overkill, but Firestorm really asks for folder Zero. + // I'm leaving the code here for the time being, but commented. + if (freq.folder_id == UUID.Zero) + { + //coll.Collection.OwnerID = freq.owner_id; + //coll.Collection.FolderID = contents.FolderID; + //containingFolder = m_InventoryService.GetRootFolder(freq.owner_id); + //if (containingFolder != null) + //{ + // m_log.WarnFormat("[WEB FETCH INV DESC HANDLER]: Request for parent of folder {0}", containingFolder.ID); + // coll.Collection.Folders.Clear(); + // coll.Collection.Folders.Add(containingFolder); + // if (m_LibraryService != null && m_LibraryService.LibraryRootFolder != null) + // { + // InventoryFolderBase lib = new InventoryFolderBase(m_LibraryService.LibraryRootFolder.ID, m_LibraryService.LibraryRootFolder.Owner); + // lib.Name = m_LibraryService.LibraryRootFolder.Name; + // lib.Type = m_LibraryService.LibraryRootFolder.Type; + // lib.Version = m_LibraryService.LibraryRootFolder.Version; + // coll.Collection.Folders.Add(lib); + // } + // coll.Collection.Items.Clear(); + //} + } + else + { + m_log.WarnFormat("[WEB FETCH INV DESC HANDLER]: Unable to fetch folder {0}", freq.folder_id); + bad_folders.Add(freq.folder_id); + bad = true; + } + } + } + + return bad; + } + + private void ProcessLinks(LLSDFetchInventoryDescendents freq, InventoryCollectionWithDescendents coll) + { + InventoryCollection contents = coll.Collection; + + if (freq.fetch_items && contents.Items != null) + { + List itemsToReturn = contents.Items; + + // descendents must only include the links, not the linked items we add + coll.Descendents = itemsToReturn.Count; + + // Add target items for links in this folder before the links themselves. + List itemIDs = new List(); + List folderIDs = new List(); + foreach (InventoryItemBase item in itemsToReturn) + { + //m_log.DebugFormat("[XXX]: {0} {1}", item.Name, item.AssetType); + if (item.AssetType == (int)AssetType.Link) + itemIDs.Add(item.AssetID); + + else if (item.AssetType == (int)AssetType.LinkFolder) + folderIDs.Add(item.AssetID); + } + + //m_log.DebugFormat("[XXX]: folder {0} has {1} links and {2} linkfolders", contents.FolderID, itemIDs.Count, folderIDs.Count); + + // Scan for folder links and insert the items they target and those links at the head of the return data + if (folderIDs.Count > 0) + { + InventoryCollection[] linkedFolders = m_InventoryService.GetMultipleFoldersContent(coll.Collection.OwnerID, folderIDs.ToArray()); + foreach (InventoryCollection linkedFolderContents in linkedFolders) + { + List links = linkedFolderContents.Items; + + itemsToReturn.InsertRange(0, links); + + foreach (InventoryItemBase link in linkedFolderContents.Items) + { + // Take care of genuinely broken links where the target doesn't exist + // HACK: Also, don't follow up links that just point to other links. In theory this is legitimate, + // but no viewer has been observed to set these up and this is the lazy way of avoiding cycles + // rather than having to keep track of every folder requested in the recursion. + if (link != null) + { + //m_log.DebugFormat( + // "[WEB FETCH INV DESC HANDLER]: Adding item {0} {1} from folder {2} linked from {3} ({4} {5})", + // link.Name, (AssetType)link.AssetType, linkedFolderContents.FolderID, contents.FolderID, link.ID, link.AssetID); + itemIDs.Add(link.ID); + } + } + } + } + + if (itemIDs.Count > 0) + { + InventoryItemBase[] linked = m_InventoryService.GetMultipleItems(freq.owner_id, itemIDs.ToArray()); + if (linked == null) + { + // OMG!!! One by one!!! This is fallback code, in case the backend isn't updated + m_log.WarnFormat("[WEB FETCH INV DESC HANDLER]: GetMultipleItems failed. Falling back to fetching inventory items one by one."); + linked = new InventoryItemBase[itemIDs.Count]; + int i = 0; + InventoryItemBase item = new InventoryItemBase(); + item.Owner = freq.owner_id; + foreach (UUID id in itemIDs) + { + item.ID = id; + linked[i++] = m_InventoryService.GetItem(item); + } + } + + foreach (InventoryItemBase linkedItem in linked) + { + // Take care of genuinely broken links where the target doesn't exist + // HACK: Also, don't follow up links that just point to other links. In theory this is legitimate, + // but no viewer has been observed to set these up and this is the lazy way of avoiding cycles + // rather than having to keep track of every folder requested in the recursion. + if (linkedItem != null && linkedItem.AssetType != (int)AssetType.Link) + { + itemsToReturn.Insert(0, linkedItem); + } + } + } + } + + } /// /// Convert an internal inventory folder object into an LLSD object. diff --git a/OpenSim/Region/CoreModules/Framework/Library/LocalInventoryService.cs b/OpenSim/Region/CoreModules/Framework/Library/LocalInventoryService.cs index eb7d3a94b7..e657f53be2 100644 --- a/OpenSim/Region/CoreModules/Framework/Library/LocalInventoryService.cs +++ b/OpenSim/Region/CoreModules/Framework/Library/LocalInventoryService.cs @@ -99,6 +99,22 @@ namespace OpenSim.Region.CoreModules.Framework.Library return invColl; } + public virtual InventoryItemBase[] GetMultipleItems(UUID principalID, UUID[] itemIDs) + { + InventoryItemBase[] itemColl = new InventoryItemBase[itemIDs.Length]; + int i = 0; + InventoryItemBase item = new InventoryItemBase(); + item.Owner = principalID; + foreach (UUID fid in itemIDs) + { + item.ID = fid; + itemColl[i++] = GetItem(item); + } + + return itemColl; + } + + /// /// Add a new folder to the user's inventory /// diff --git a/OpenSim/Region/CoreModules/ServiceConnectorsOut/Inventory/HGInventoryBroker.cs b/OpenSim/Region/CoreModules/ServiceConnectorsOut/Inventory/HGInventoryBroker.cs index 232cfdf7a9..6a83b428ed 100644 --- a/OpenSim/Region/CoreModules/ServiceConnectorsOut/Inventory/HGInventoryBroker.cs +++ b/OpenSim/Region/CoreModules/ServiceConnectorsOut/Inventory/HGInventoryBroker.cs @@ -615,6 +615,23 @@ namespace OpenSim.Region.CoreModules.ServiceConnectorsOut.Inventory return connector.GetItem(item); } + public InventoryItemBase[] GetMultipleItems(UUID userID, UUID[] itemIDs) + { + if (itemIDs == null) + return new InventoryItemBase[0]; + //m_log.Debug("[HG INVENTORY CONNECTOR]: GetItem " + item.ID); + + string invURL = GetInventoryServiceURL(userID); + + if (invURL == null) // not there, forward to local inventory connector to resolve + lock (m_Lock) + return m_LocalGridInventoryService.GetMultipleItems(userID, itemIDs); + + IInventoryService connector = GetConnector(invURL); + + return connector.GetMultipleItems(userID, itemIDs); + } + public InventoryFolderBase GetFolder(InventoryFolderBase folder) { if (folder == null) diff --git a/OpenSim/Region/CoreModules/ServiceConnectorsOut/Inventory/LocalInventoryServiceConnector.cs b/OpenSim/Region/CoreModules/ServiceConnectorsOut/Inventory/LocalInventoryServiceConnector.cs index 75dd200a18..2f29a7cd53 100644 --- a/OpenSim/Region/CoreModules/ServiceConnectorsOut/Inventory/LocalInventoryServiceConnector.cs +++ b/OpenSim/Region/CoreModules/ServiceConnectorsOut/Inventory/LocalInventoryServiceConnector.cs @@ -307,6 +307,11 @@ namespace OpenSim.Region.CoreModules.ServiceConnectorsOut.Inventory return item; } + public InventoryItemBase[] GetMultipleItems(UUID userID, UUID[] itemIDs) + { + return m_InventoryService.GetMultipleItems(userID, itemIDs); + } + public InventoryFolderBase GetFolder(InventoryFolderBase folder) { return m_InventoryService.GetFolder(folder); diff --git a/OpenSim/Region/CoreModules/ServiceConnectorsOut/Inventory/RemoteXInventoryServiceConnector.cs b/OpenSim/Region/CoreModules/ServiceConnectorsOut/Inventory/RemoteXInventoryServiceConnector.cs index 9beb382b4c..e0cc1e8e64 100644 --- a/OpenSim/Region/CoreModules/ServiceConnectorsOut/Inventory/RemoteXInventoryServiceConnector.cs +++ b/OpenSim/Region/CoreModules/ServiceConnectorsOut/Inventory/RemoteXInventoryServiceConnector.cs @@ -303,6 +303,14 @@ namespace OpenSim.Region.CoreModules.ServiceConnectorsOut.Inventory return m_RemoteConnector.GetItem(item); } + public InventoryItemBase[] GetMultipleItems(UUID userID, UUID[] itemIDs) + { + if (itemIDs == null) + return new InventoryItemBase[0]; + + return m_RemoteConnector.GetMultipleItems(userID, itemIDs); + } + public InventoryFolderBase GetFolder(InventoryFolderBase folder) { //m_log.DebugFormat("[XINVENTORY CONNECTOR]: GetFolder {0}", folder.ID); diff --git a/OpenSim/Server/Handlers/Inventory/XInventoryInConnector.cs b/OpenSim/Server/Handlers/Inventory/XInventoryInConnector.cs index 4f01e36903..cf0762ba0b 100644 --- a/OpenSim/Server/Handlers/Inventory/XInventoryInConnector.cs +++ b/OpenSim/Server/Handlers/Inventory/XInventoryInConnector.cs @@ -149,6 +149,8 @@ namespace OpenSim.Server.Handlers.Inventory return HandleDeleteItems(request); case "GETITEM": return HandleGetItem(request); + case "GETMULTIPLEITEMS": + return HandleGetMultipleItems(request); case "GETFOLDER": return HandleGetFolder(request); case "GETACTIVEGESTURES": @@ -576,6 +578,40 @@ namespace OpenSim.Server.Handlers.Inventory return Util.UTF8NoBomEncoding.GetBytes(xmlString); } + byte[] HandleGetMultipleItems(Dictionary request) + { + Dictionary resultSet = new Dictionary(); + UUID principal = UUID.Zero; + UUID.TryParse(request["PRINCIPAL"].ToString(), out principal); + string itemIDstr = request["ITEMS"].ToString(); + int count = 0; + Int32.TryParse(request["COUNT"].ToString(), out count); + + UUID[] fids = new UUID[count]; + string[] uuids = itemIDstr.Split(','); + int i = 0; + foreach (string id in uuids) + { + UUID fid = UUID.Zero; + if (UUID.TryParse(id, out fid)) + fids[i] = fid; + i += 1; + } + + InventoryItemBase[] itemsList = m_InventoryService.GetMultipleItems(principal, fids); + if (itemsList != null && itemsList.Length > 0) + { + count = 0; + foreach (InventoryItemBase item in itemsList) + resultSet["item_" + count++] = (item == null) ? (object)"NULL" : EncodeItem(item); + } + + string xmlString = ServerUtils.BuildXmlResponse(resultSet); + + //m_log.DebugFormat("[XXX]: resp string: {0}", xmlString); + return Util.UTF8NoBomEncoding.GetBytes(xmlString); + } + byte[] HandleGetFolder(Dictionary request) { Dictionary result = new Dictionary(); diff --git a/OpenSim/Services/Connectors/Inventory/XInventoryServicesConnector.cs b/OpenSim/Services/Connectors/Inventory/XInventoryServicesConnector.cs index 56dece3fd1..0cea4a1e47 100644 --- a/OpenSim/Services/Connectors/Inventory/XInventoryServicesConnector.cs +++ b/OpenSim/Services/Connectors/Inventory/XInventoryServicesConnector.cs @@ -316,6 +316,7 @@ namespace OpenSim.Services.Connectors return inventoryArr; } + public List GetFolderItems(UUID principalID, UUID folderID) { Dictionary ret = MakeRequest("GETFOLDERITEMS", @@ -526,6 +527,42 @@ namespace OpenSim.Services.Connectors return null; } + public virtual InventoryItemBase[] GetMultipleItems(UUID principalID, UUID[] itemIDs) + { + InventoryItemBase[] itemArr = new InventoryItemBase[itemIDs.Length]; + try + { + Dictionary resultSet = MakeRequest("GETMULTIPLEITEMS", + new Dictionary { + { "PRINCIPAL", principalID.ToString() }, + { "ITEMS", String.Join(",", itemIDs) }, + { "COUNT", itemIDs.Length.ToString() } + }); + + if (!CheckReturn(resultSet)) + return null; + + int i = 0; + foreach (KeyValuePair kvp in resultSet) + { + InventoryCollection inventory = new InventoryCollection(); + if (kvp.Key.StartsWith("item_")) + { + if (kvp.Value is Dictionary) + itemArr[i++] = BuildItem((Dictionary)kvp.Value); + else + itemArr[i++] = null; + } + } + } + catch (Exception e) + { + m_log.WarnFormat("[XINVENTORY SERVICES CONNECTOR]: Exception in GetMultipleItems: {0}", e.Message); + } + + return itemArr; + } + public InventoryFolderBase GetFolder(InventoryFolderBase folder) { try diff --git a/OpenSim/Services/Connectors/SimianGrid/SimianInventoryServiceConnector.cs b/OpenSim/Services/Connectors/SimianGrid/SimianInventoryServiceConnector.cs index 0331c66d2e..fdeea1852d 100644 --- a/OpenSim/Services/Connectors/SimianGrid/SimianInventoryServiceConnector.cs +++ b/OpenSim/Services/Connectors/SimianGrid/SimianInventoryServiceConnector.cs @@ -301,6 +301,21 @@ namespace OpenSim.Services.Connectors.SimianGrid return null; } + public InventoryItemBase[] GetMultipleItems(UUID principalID, UUID[] itemIDs) + { + InventoryItemBase[] result = new InventoryItemBase[itemIDs.Length]; + int i = 0; + InventoryItemBase item = new InventoryItemBase(); + item.Owner = principalID; + foreach (UUID id in itemIDs) + { + item.ID = id; + result[i++] = GetItem(item); + } + + return result; + } + /// /// Get a folder, given by its UUID /// diff --git a/OpenSim/Services/Interfaces/IInventoryService.cs b/OpenSim/Services/Interfaces/IInventoryService.cs index 829f1699bd..1e8652c2c6 100644 --- a/OpenSim/Services/Interfaces/IInventoryService.cs +++ b/OpenSim/Services/Interfaces/IInventoryService.cs @@ -82,7 +82,7 @@ namespace OpenSim.Services.Interfaces /// /// /// - /// Inventory content. null if the request failed. + /// Inventory content. InventoryCollection[] GetMultipleFoldersContent(UUID userID, UUID[] folderIDs); /// @@ -163,6 +163,13 @@ namespace OpenSim.Services.Interfaces /// null if no item was found, otherwise the found item InventoryItemBase GetItem(InventoryItemBase item); + /// + /// Get multiple items, given by their UUIDs + /// + /// + /// null if no item was found, otherwise the found item + InventoryItemBase[] GetMultipleItems(UUID userID, UUID[] ids); + /// /// Get a folder, given by its UUID /// diff --git a/OpenSim/Services/InventoryService/XInventoryService.cs b/OpenSim/Services/InventoryService/XInventoryService.cs index 6582b75cf0..50cadab600 100644 --- a/OpenSim/Services/InventoryService/XInventoryService.cs +++ b/OpenSim/Services/InventoryService/XInventoryService.cs @@ -610,6 +610,21 @@ namespace OpenSim.Services.InventoryService return ConvertToOpenSim(items[0]); } + public virtual InventoryItemBase[] GetMultipleItems(UUID userID, UUID[] ids) + { + InventoryItemBase[] items = new InventoryItemBase[ids.Length]; + int i = 0; + InventoryItemBase item = new InventoryItemBase(); + item.Owner = userID; + foreach (UUID id in ids) + { + item.ID = id; + items[i++] = GetItem(item); + } + + return items; + } + public virtual InventoryFolderBase GetFolder(InventoryFolderBase folder) { XInventoryFolder[] folders = m_Database.GetFolders(