OpenSim
/
OpenSimMirror
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

* Improve memory usage when writing OARs 

* This should make saving large OARs a somewhat better experience
* However, the problem where saving an archive pulls large numbers of assets into the asset cache isn't yet resolved
* This patch also removes lots of archive writing spam that crept in
0.6.4-rc1
Justin Clarke Casey 2009-03-06 20:12:08 +00:00
parent 08509d5cf2
commit 85774de231
6 changed files with 81 additions and 89 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
OpenSim/Region/CoreModules/Avatar/Inventory/Archiver/InventoryArchiveWriteRequest.cs
View File

@ -45,7 +45,7 @@ namespace OpenSim.Region.CoreModules.Avatar.Inventory.Archiver
{
private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
protected TarArchiveWriter archive = new TarArchiveWriter();
protected TarArchiveWriter m_archive;
protected UuidGatherer m_assetGatherer;
protected Dictionary<UUID, int> assetUuids = new Dictionary<UUID, int>();
@ -87,14 +87,14 @@ namespace OpenSim.Region.CoreModules.Avatar.Inventory.Archiver
protected void ReceivedAllAssets(IDictionary<UUID, AssetBase> assetsFound, ICollection<UUID> assetsNotFoundUuids)
{
AssetsArchiver assetsArchiver = new AssetsArchiver(assetsFound);
assetsArchiver.Archive(archive);
assetsArchiver.Archive(m_archive);
Exception reportedException = null;
bool succeeded = true;
try
{
archive.WriteTar(m_saveStream);
m_archive.Close();
}
catch (IOException e)
{
@ -172,7 +172,7 @@ namespace OpenSim.Region.CoreModules.Avatar.Inventory.Archiver
writer.WriteEndElement();
archive.AddFile(filename, sw.ToString());
m_archive.WriteFile(filename, sw.ToString());
m_assetGatherer.GatherAssetUuids(inventoryItem.AssetID, (AssetType)inventoryItem.AssetType, assetUuids);
}
@ -185,7 +185,7 @@ namespace OpenSim.Region.CoreModules.Avatar.Inventory.Archiver
inventoryFolder.Name,
InventoryArchiveConstants.INVENTORY_NODE_NAME_COMPONENT_SEPARATOR,
inventoryFolder.ID);
archive.AddDir(path);
m_archive.WriteDir(path);
List<InventoryFolderImpl> childFolders = inventoryFolder.RequestListOfFolderImpls();
List<InventoryItemBase> items = inventoryFolder.RequestListOfItems();
@ -280,6 +280,8 @@ namespace OpenSim.Region.CoreModules.Avatar.Inventory.Archiver
inventoryItem = m_userInfo.RootFolder.FindItemByPath(m_invPath);
}
m_archive = new TarArchiveWriter(m_saveStream);
if (null == inventoryFolder)
{
if (null == inventoryItem)

12
OpenSim/Region/CoreModules/World/Archiver/ArchiveWriteRequestExecution.cs
View File

@ -85,17 +85,17 @@ namespace OpenSim.Region.CoreModules.World.Archiver
m_log.InfoFormat("[ARCHIVER]: Creating archive file. This may take some time.");
TarArchiveWriter archive = new TarArchiveWriter();
TarArchiveWriter archive = new TarArchiveWriter(m_saveStream);
// Write out control file
archive.AddFile(ArchiveConstants.CONTROL_FILE_PATH, Create0p2ControlFile());
archive.WriteFile(ArchiveConstants.CONTROL_FILE_PATH, Create0p2ControlFile());
m_log.InfoFormat("[ARCHIVER]: Added control file to archive.");
// Write out region settings
string settingsPath
= String.Format("{0}{1}.xml", ArchiveConstants.SETTINGS_PATH, m_scene.RegionInfo.RegionName);
archive.AddFile(settingsPath, RegionSettingsSerializer.Serialize(m_scene.RegionInfo.RegionSettings));
archive.WriteFile(settingsPath, RegionSettingsSerializer.Serialize(m_scene.RegionInfo.RegionSettings));
m_log.InfoFormat("[ARCHIVER]: Added region settings to archive.");
@ -105,7 +105,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver
MemoryStream ms = new MemoryStream();
m_terrainModule.SaveToStream(terrainPath, ms);
archive.AddFile(terrainPath, ms.ToArray());
archive.WriteFile(terrainPath, ms.ToArray());
ms.Close();
m_log.InfoFormat("[ARCHIVER]: Added terrain information to archive.");
@ -125,7 +125,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver
Math.Round(position.X), Math.Round(position.Y), Math.Round(position.Z),
sceneObject.UUID);
archive.AddFile(filename, serializedObject);
archive.WriteFile(filename, serializedObject);
}
m_log.InfoFormat("[ARCHIVER]: Added scene objects to archive.");
@ -134,7 +134,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver
AssetsArchiver assetsArchiver = new AssetsArchiver(assetsFound);
assetsArchiver.Archive(archive);
archive.WriteTar(m_saveStream);
archive.Close();
m_log.InfoFormat("[ARCHIVER]: Wrote out OpenSimulator archive for {0}", m_scene.RegionInfo.RegionName);

4
OpenSim/Region/CoreModules/World/Archiver/AssetsArchiver.cs
View File

@ -110,7 +110,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver
xtw.WriteEndDocument();
archive.AddFile("assets.xml", sw.ToString());
archive.WriteFile("assets.xml", sw.ToString());
}
/// <summary>
@ -141,7 +141,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver
asset.Type, asset.ID);
}
archive.AddFile(
archive.WriteFile(
ArchiveConstants.ASSETS_PATH + uuid.ToString() + extension,
asset.Data);

10
OpenSim/Region/CoreModules/World/Archiver/TarArchiveReader.cs
View File

@ -38,7 +38,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver
/// </summary>
public class TarArchiveReader
{
private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
//private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
public enum TarEntryType
{
@ -113,14 +113,14 @@ namespace OpenSim.Region.CoreModules.World.Archiver
{
int longNameLength = ConvertOctalBytesToDecimal(header, 124, 11);
tarHeader.FilePath = m_asciiEncoding.GetString(ReadData(longNameLength));
m_log.DebugFormat("[TAR ARCHIVE READER]: Got long file name {0}", tarHeader.FilePath);
//m_log.DebugFormat("[TAR ARCHIVE READER]: Got long file name {0}", tarHeader.FilePath);
header = m_br.ReadBytes(512);
}
else
{
tarHeader.FilePath = m_asciiEncoding.GetString(header, 0, 100);
tarHeader.FilePath = tarHeader.FilePath.Trim(m_nullCharArray);
m_log.DebugFormat("[TAR ARCHIVE READER]: Got short file name {0}", tarHeader.FilePath);
//m_log.DebugFormat("[TAR ARCHIVE READER]: Got short file name {0}", tarHeader.FilePath);
}
tarHeader.FileSize = ConvertOctalBytesToDecimal(header, 124, 11);
@ -168,14 +168,14 @@ namespace OpenSim.Region.CoreModules.World.Archiver
{
byte[] data = m_br.ReadBytes(fileSize);
m_log.DebugFormat("[TAR ARCHIVE READER]: fileSize {0}", fileSize);
//m_log.DebugFormat("[TAR ARCHIVE READER]: fileSize {0}", fileSize);
// Read the rest of the empty padding in the 512 byte block
if (fileSize % 512 != 0)
{
int paddingLeft = 512 - (fileSize % 512);
m_log.DebugFormat("[TAR ARCHIVE READER]: Reading {0} padding bytes", paddingLeft);
//m_log.DebugFormat("[TAR ARCHIVE READER]: Reading {0} padding bytes", paddingLeft);
m_br.ReadBytes(paddingLeft);
}

96
OpenSim/Region/CoreModules/World/Archiver/TarArchiveWriter.cs
View File

@ -39,65 +39,80 @@ namespace OpenSim.Region.CoreModules.World.Archiver
{
//private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
protected Dictionary<string, byte[]> m_files = new Dictionary<string, byte[]>();
protected static ASCIIEncoding m_asciiEncoding = new ASCIIEncoding();
/// <summary>
/// Add a directory to the tar archive. We can only handle one path level right now!
/// Binary writer for the underlying stream
/// </summary>
protected BinaryWriter m_bw;
public TarArchiveWriter(Stream s)
{
m_bw = new BinaryWriter(s);
}
/// <summary>
/// Write a directory entry to the tar archive. We can only handle one path level right now!
/// </summary>
/// <param name="dirName"></param>
public void AddDir(string dirName)
public void WriteDir(string dirName)
{
// Directories are signalled by a final /
if (!dirName.EndsWith("/"))
dirName += "/";
AddFile(dirName, new byte[0]);
WriteFile(dirName, new byte[0]);
}
/// <summary>
/// Add a file to the tar archive
/// Write a file to the tar archive
/// </summary>
/// <param name="filePath"></param>
/// <param name="data"></param>
public void AddFile(string filePath, string data)
public void WriteFile(string filePath, string data)
{
AddFile(filePath, m_asciiEncoding.GetBytes(data));
WriteFile(filePath, m_asciiEncoding.GetBytes(data));
}
/// <summary>
/// Add a file to the tar archive
/// Write a file to the tar archive
/// </summary>
/// <param name="filePath"></param>
/// <param name="data"></param>
public void AddFile(string filePath, byte[] data)
public void WriteFile(string filePath, byte[] data)
{
m_files[filePath] = data;
if (filePath.Length > 100)
WriteEntry("././@LongLink", m_asciiEncoding.GetBytes(filePath), 'L');
char fileType;
if (filePath.EndsWith("/"))
{
fileType = '5';
}
else
{
fileType = '0';
}
WriteEntry(filePath, data, fileType);
}
/// <summary>
/// Write the raw tar archive data to a stream. The stream will be closed on completion.
/// Finish writing the raw tar archive data to a stream. The stream will be closed on completion.
/// </summary>
/// <param name="s">Stream to which to write the data</param>
/// <returns></returns>
public void WriteTar(Stream s)
public void Close()
{
BinaryWriter bw = new BinaryWriter(s);
foreach (string filePath in m_files.Keys)
{
WriteFile(bw, filePath, m_files[filePath]);
}
//m_log.Debug("[TAR ARCHIVE WRITER]: Writing final consecutive 0 blocks");
// Write two consecutive 0 blocks to end the archive
byte[] finalZeroPadding = new byte[1024];
bw.Write(finalZeroPadding);
m_bw.Write(finalZeroPadding);
bw.Flush();
bw.Close();
m_bw.Flush();
m_bw.Close();
}
public static byte[] ConvertDecimalToPaddedOctalBytes(int d, int padding)
@ -121,37 +136,12 @@ namespace OpenSim.Region.CoreModules.World.Archiver
}
/// <summary>
/// Write a particular file of data
/// Write a particular entry
/// </summary>
/// <param name="filePath"></param>
/// <param name="data"></param>
protected void WriteFile(BinaryWriter bw, string filePath, byte[] data)
{
if (filePath.Length > 100)
WriteEntry(bw, "././@LongLink", m_asciiEncoding.GetBytes(filePath), 'L');
char fileType;
if (filePath.EndsWith("/"))
{
fileType = '5';
}
else
{
fileType = '0';
}
WriteEntry(bw, filePath, data, fileType);
}
/// <summary>
/// Write a particular file of data
/// </summary>
/// <param name="bw"></param>
/// <param name="filePath"></param>
/// <param name="data"></param>
/// <param name="fileType"></param>
protected void WriteEntry(BinaryWriter bw, string filePath, byte[] data, char fileType)
protected void WriteEntry(string filePath, byte[] data, char fileType)
{
byte[] header = new byte[512];
@ -208,10 +198,10 @@ namespace OpenSim.Region.CoreModules.World.Archiver
header[154] = 0;
// Write out header
bw.Write(header);
m_bw.Write(header);
// Write out data
bw.Write(data);
m_bw.Write(data);
if (data.Length % 512 != 0)
{
@ -220,7 +210,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver
//m_log.DebugFormat("[TAR ARCHIVE WRITER]: Padding data with {0} bytes", paddingRequired);
byte[] padding = new byte[paddingRequired];
bw.Write(padding);
m_bw.Write(padding);
}
}
}

8
OpenSim/Region/CoreModules/World/Archiver/Tests/ArchiverTests.cs
View File

@ -173,9 +173,9 @@ namespace OpenSim.Region.CoreModules.World.Archiver.Tests
//log4net.Config.XmlConfigurator.Configure();
MemoryStream archiveWriteStream = new MemoryStream();
TarArchiveWriter tar = new TarArchiveWriter();
TarArchiveWriter tar = new TarArchiveWriter(archiveWriteStream);
tar.AddFile(ArchiveConstants.CONTROL_FILE_PATH, ArchiveWriteRequestExecution.Create0p2ControlFile());
tar.WriteFile(ArchiveConstants.CONTROL_FILE_PATH, ArchiveWriteRequestExecution.Create0p2ControlFile());
string part1Name = "object1";
PrimitiveBaseShape shape = PrimitiveBaseShape.CreateCylinder();
@ -194,9 +194,9 @@ namespace OpenSim.Region.CoreModules.World.Archiver.Tests
part1Name,
Math.Round(groupPosition.X), Math.Round(groupPosition.Y), Math.Round(groupPosition.Z),
part1.UUID);
tar.AddFile(ArchiveConstants.OBJECTS_PATH + object1FileName, object1.ToXmlString2());
tar.WriteFile(ArchiveConstants.OBJECTS_PATH + object1FileName, object1.ToXmlString2());
tar.WriteTar(archiveWriteStream);
tar.Close();
MemoryStream archiveReadStream = new MemoryStream(archiveWriteStream.ToArray());