* Improve memory usage when writing OARs

* This should make saving large OARs a somewhat better experience
* However, the problem where saving an archive pulls large numbers of assets into the asset cache isn't yet resolved
* This patch also removes lots of archive writing spam that crept in

OpenSim/Region/CoreModules/Avatar/Inventory/Archiver/InventoryArchiveWriteRequest.cs

@@ -45,7 +45,7 @@ namespace OpenSim.Region.CoreModules.Avatar.Inventory.Archiver
     {
         private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);      
         
-        protected TarArchiveWriter archive = new TarArchiveWriter();
+        protected TarArchiveWriter m_archive;
         protected UuidGatherer m_assetGatherer;
         protected Dictionary<UUID, int> assetUuids = new Dictionary<UUID, int>();
         
@@ -87,14 +87,14 @@ namespace OpenSim.Region.CoreModules.Avatar.Inventory.Archiver
         protected void ReceivedAllAssets(IDictionary<UUID, AssetBase> assetsFound, ICollection<UUID> assetsNotFoundUuids)
         {
             AssetsArchiver assetsArchiver = new AssetsArchiver(assetsFound);
-            assetsArchiver.Archive(archive);
+            assetsArchiver.Archive(m_archive);
             
             Exception reportedException = null;
             bool succeeded = true;
             
             try
             {
-                archive.WriteTar(m_saveStream);
+                m_archive.Close();
             } 
             catch (IOException e)
             {
@@ -172,7 +172,7 @@ namespace OpenSim.Region.CoreModules.Avatar.Inventory.Archiver
             
             writer.WriteEndElement();
 
-            archive.AddFile(filename, sw.ToString());
+            m_archive.WriteFile(filename, sw.ToString());
 
             m_assetGatherer.GatherAssetUuids(inventoryItem.AssetID, (AssetType)inventoryItem.AssetType, assetUuids);
         }
@@ -185,7 +185,7 @@ namespace OpenSim.Region.CoreModules.Avatar.Inventory.Archiver
                     inventoryFolder.Name,
                     InventoryArchiveConstants.INVENTORY_NODE_NAME_COMPONENT_SEPARATOR, 
                     inventoryFolder.ID);
-            archive.AddDir(path);
+            m_archive.WriteDir(path);
             
             List<InventoryFolderImpl> childFolders = inventoryFolder.RequestListOfFolderImpls();
             List<InventoryItemBase> items = inventoryFolder.RequestListOfItems();           
@@ -279,6 +279,8 @@ namespace OpenSim.Region.CoreModules.Avatar.Inventory.Archiver
             {
                 inventoryItem = m_userInfo.RootFolder.FindItemByPath(m_invPath);
             }
+            
+            m_archive = new TarArchiveWriter(m_saveStream);            
 
             if (null == inventoryFolder)
             {

OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequestExecution.cs

@@ -85,17 +85,17 @@ namespace OpenSim.Region.CoreModules.World.Archiver
             
             m_log.InfoFormat("[ARCHIVER]: Creating archive file.  This may take some time.");
 
-            TarArchiveWriter archive = new TarArchiveWriter();
+            TarArchiveWriter archive = new TarArchiveWriter(m_saveStream);
 
             // Write out control file
-            archive.AddFile(ArchiveConstants.CONTROL_FILE_PATH, Create0p2ControlFile());
+            archive.WriteFile(ArchiveConstants.CONTROL_FILE_PATH, Create0p2ControlFile());
             
             m_log.InfoFormat("[ARCHIVER]: Added control file to archive.");
             
             // Write out region settings
             string settingsPath 
                 = String.Format("{0}{1}.xml", ArchiveConstants.SETTINGS_PATH, m_scene.RegionInfo.RegionName);
-            archive.AddFile(settingsPath, RegionSettingsSerializer.Serialize(m_scene.RegionInfo.RegionSettings));
+            archive.WriteFile(settingsPath, RegionSettingsSerializer.Serialize(m_scene.RegionInfo.RegionSettings));
             
             m_log.InfoFormat("[ARCHIVER]: Added region settings to archive.");
 
@@ -105,7 +105,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver
             
             MemoryStream ms = new MemoryStream();
             m_terrainModule.SaveToStream(terrainPath, ms);
-            archive.AddFile(terrainPath, ms.ToArray());
+            archive.WriteFile(terrainPath, ms.ToArray());
             ms.Close();
             
             m_log.InfoFormat("[ARCHIVER]: Added terrain information to archive.");
@@ -125,7 +125,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver
                         Math.Round(position.X), Math.Round(position.Y), Math.Round(position.Z),
                         sceneObject.UUID);
 
-                archive.AddFile(filename, serializedObject);
+                archive.WriteFile(filename, serializedObject);
             }
             
             m_log.InfoFormat("[ARCHIVER]: Added scene objects to archive.");
@@ -134,7 +134,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver
             AssetsArchiver assetsArchiver = new AssetsArchiver(assetsFound);
             assetsArchiver.Archive(archive);
 
-            archive.WriteTar(m_saveStream);
+            archive.Close();
 
             m_log.InfoFormat("[ARCHIVER]: Wrote out OpenSimulator archive for {0}", m_scene.RegionInfo.RegionName);
             

OpenSim/Region/CoreModules/World/Archiver/AssetsArchiver.cs

@@ -110,7 +110,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver
 
             xtw.WriteEndDocument();
 
-            archive.AddFile("assets.xml", sw.ToString());
+            archive.WriteFile("assets.xml", sw.ToString());
         }
 
         /// <summary>
@@ -141,7 +141,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver
                         asset.Type, asset.ID);
                 }
 
-                archive.AddFile(
+                archive.WriteFile(
                     ArchiveConstants.ASSETS_PATH + uuid.ToString() + extension,
                     asset.Data);
                 

OpenSim/Region/CoreModules/World/Archiver/TarArchiveReader.cs

@@ -38,7 +38,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver
     /// </summary>
     public class TarArchiveReader
     {
-        private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
+        //private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
         
         public enum TarEntryType 
         {
@@ -113,14 +113,14 @@ namespace OpenSim.Region.CoreModules.World.Archiver
             {
                 int longNameLength = ConvertOctalBytesToDecimal(header, 124, 11);
                 tarHeader.FilePath = m_asciiEncoding.GetString(ReadData(longNameLength));
-                m_log.DebugFormat("[TAR ARCHIVE READER]: Got long file name {0}", tarHeader.FilePath);
+                //m_log.DebugFormat("[TAR ARCHIVE READER]: Got long file name {0}", tarHeader.FilePath);
                 header = m_br.ReadBytes(512);
             }
             else
             {               
                 tarHeader.FilePath = m_asciiEncoding.GetString(header, 0, 100);
                 tarHeader.FilePath = tarHeader.FilePath.Trim(m_nullCharArray);
-                m_log.DebugFormat("[TAR ARCHIVE READER]: Got short file name {0}", tarHeader.FilePath);
+                //m_log.DebugFormat("[TAR ARCHIVE READER]: Got short file name {0}", tarHeader.FilePath);
             }
                         
             tarHeader.FileSize = ConvertOctalBytesToDecimal(header, 124, 11);
@@ -168,14 +168,14 @@ namespace OpenSim.Region.CoreModules.World.Archiver
         {
             byte[] data = m_br.ReadBytes(fileSize);
 
-            m_log.DebugFormat("[TAR ARCHIVE READER]: fileSize {0}", fileSize);
+            //m_log.DebugFormat("[TAR ARCHIVE READER]: fileSize {0}", fileSize);
 
             // Read the rest of the empty padding in the 512 byte block
             if (fileSize % 512 != 0)
             {
                 int paddingLeft = 512 - (fileSize % 512);
 
-                m_log.DebugFormat("[TAR ARCHIVE READER]: Reading {0} padding bytes", paddingLeft);
+                //m_log.DebugFormat("[TAR ARCHIVE READER]: Reading {0} padding bytes", paddingLeft);
 
                 m_br.ReadBytes(paddingLeft);
             }

OpenSim/Region/CoreModules/World/Archiver/TarArchiveWriter.cs

@@ -39,65 +39,80 @@ namespace OpenSim.Region.CoreModules.World.Archiver
     {
         //private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
 
-        protected Dictionary<string, byte[]> m_files = new Dictionary<string, byte[]>();
-
         protected static ASCIIEncoding m_asciiEncoding = new ASCIIEncoding();
 
         /// <summary>
-        /// Add a directory to the tar archive.  We can only handle one path level right now!
+        /// Binary writer for the underlying stream
+        /// </summary>
+        protected BinaryWriter m_bw;
+        
+        public TarArchiveWriter(Stream s)
+        {
+            m_bw = new BinaryWriter(s);
+        }
+        
+        /// <summary>
+        /// Write a directory entry to the tar archive.  We can only handle one path level right now!
         /// </summary>
         /// <param name="dirName"></param>
-        public void AddDir(string dirName)
+        public void WriteDir(string dirName)
         {
             // Directories are signalled by a final /
             if (!dirName.EndsWith("/"))
                 dirName += "/";
 
-            AddFile(dirName, new byte[0]);
+            WriteFile(dirName, new byte[0]);
-        }
-
-        /// <summary>
-        /// Add a file to the tar archive
-        /// </summary>
-        /// <param name="filePath"></param>
-        /// <param name="data"></param>
-        public void AddFile(string filePath, string data)
-        {
-            AddFile(filePath, m_asciiEncoding.GetBytes(data));
-        }
-
-        /// <summary>
-        /// Add a file to the tar archive
-        /// </summary>
-        /// <param name="filePath"></param>
-        /// <param name="data"></param>
-        public void AddFile(string filePath, byte[] data)
-        {
-            m_files[filePath] = data;
         }
         
         /// <summary>
-        /// Write the raw tar archive data to a stream.  The stream will be closed on completion.
+        /// Write a file to the tar archive
+        /// </summary>
+        /// <param name="filePath"></param>
+        /// <param name="data"></param>
+        public void WriteFile(string filePath, string data)
+        {
+            WriteFile(filePath, m_asciiEncoding.GetBytes(data));
+        }
+
+        /// <summary>
+        /// Write a file to the tar archive
+        /// </summary>
+        /// <param name="filePath"></param>
+        /// <param name="data"></param>
+        public void WriteFile(string filePath, byte[] data)
+        {
+            if (filePath.Length > 100)
+                WriteEntry("././@LongLink", m_asciiEncoding.GetBytes(filePath), 'L');
+            
+            char fileType;
+            
+            if (filePath.EndsWith("/"))
+            {
+                fileType = '5';
+            }
+            else
+            {
+                fileType = '0';
+            }     
+         
+            WriteEntry(filePath, data, fileType);
+        }
+        
+        /// <summary>
+        /// Finish writing the raw tar archive data to a stream.  The stream will be closed on completion.
         /// </summary>
         /// <param name="s">Stream to which to write the data</param>
         /// <returns></returns>
-        public void WriteTar(Stream s)
+        public void Close()
         {
-            BinaryWriter bw = new BinaryWriter(s);
-
-            foreach (string filePath in m_files.Keys)
-            {
-                WriteFile(bw, filePath, m_files[filePath]);
-            }
-
             //m_log.Debug("[TAR ARCHIVE WRITER]: Writing final consecutive 0 blocks");
             
             // Write two consecutive 0 blocks to end the archive
             byte[] finalZeroPadding = new byte[1024];
-            bw.Write(finalZeroPadding);
+            m_bw.Write(finalZeroPadding);
 
-            bw.Flush();
+            m_bw.Flush();
-            bw.Close();
+            m_bw.Close();
         }
 
         public static byte[] ConvertDecimalToPaddedOctalBytes(int d, int padding)
@@ -119,39 +134,14 @@ namespace OpenSim.Region.CoreModules.World.Archiver
 
             return oBytes;
         }    
-        
-        /// <summary>
-        /// Write a particular file of data
-        /// </summary>
-        /// <param name="filePath"></param>
-        /// <param name="data"></param>
-        protected void WriteFile(BinaryWriter bw, string filePath, byte[] data)
-        {
-            if (filePath.Length > 100)
-                WriteEntry(bw, "././@LongLink", m_asciiEncoding.GetBytes(filePath), 'L');
-            
-            char fileType;
-            
-            if (filePath.EndsWith("/"))
-            {
-                fileType = '5';
-            }
-            else
-            {
-                fileType = '0';
-            }     
-         
-            WriteEntry(bw, filePath, data, fileType);
-        }
 
         /// <summary>
-        /// Write a particular file of data
+        /// Write a particular entry
         /// </summary>
-        /// <param name="bw"></param>
         /// <param name="filePath"></param>
         /// <param name="data"></param>
         /// <param name="fileType"></param>
-        protected void WriteEntry(BinaryWriter bw, string filePath, byte[] data, char fileType)
+        protected void WriteEntry(string filePath, byte[] data, char fileType)
         {
             byte[] header = new byte[512];
 
@@ -208,10 +198,10 @@ namespace OpenSim.Region.CoreModules.World.Archiver
             header[154] = 0;
 
             // Write out header
-            bw.Write(header);
+            m_bw.Write(header);
 
             // Write out data
-            bw.Write(data);
+            m_bw.Write(data);
 
             if (data.Length % 512 != 0)
             {
@@ -220,7 +210,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver
                 //m_log.DebugFormat("[TAR ARCHIVE WRITER]: Padding data with {0} bytes", paddingRequired);
 
                 byte[] padding = new byte[paddingRequired];
-                bw.Write(padding);
+                m_bw.Write(padding);
             }            
         }   
     }

OpenSim/Region/CoreModules/World/Archiver/Tests/ArchiverTests.cs

@@ -173,9 +173,9 @@ namespace OpenSim.Region.CoreModules.World.Archiver.Tests
             //log4net.Config.XmlConfigurator.Configure();
             
             MemoryStream archiveWriteStream = new MemoryStream();
-            TarArchiveWriter tar = new TarArchiveWriter();
+            TarArchiveWriter tar = new TarArchiveWriter(archiveWriteStream);
             
-            tar.AddFile(ArchiveConstants.CONTROL_FILE_PATH, ArchiveWriteRequestExecution.Create0p2ControlFile());
+            tar.WriteFile(ArchiveConstants.CONTROL_FILE_PATH, ArchiveWriteRequestExecution.Create0p2ControlFile());
             
             string part1Name = "object1";
             PrimitiveBaseShape shape = PrimitiveBaseShape.CreateCylinder();
@@ -194,9 +194,9 @@ namespace OpenSim.Region.CoreModules.World.Archiver.Tests
                 part1Name,
                 Math.Round(groupPosition.X), Math.Round(groupPosition.Y), Math.Round(groupPosition.Z),
                 part1.UUID);            
-            tar.AddFile(ArchiveConstants.OBJECTS_PATH + object1FileName, object1.ToXmlString2());
+            tar.WriteFile(ArchiveConstants.OBJECTS_PATH + object1FileName, object1.ToXmlString2());
             
-            tar.WriteTar(archiveWriteStream);
+            tar.Close();
             
             MemoryStream archiveReadStream = new MemoryStream(archiveWriteStream.ToArray());