OpenSim
/
OpenSimMirror
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

add own version of utf8 getbytes. More recent .net versions (core?) do have similar, but not want got there now, besides only similar..

master
UbitUmarov 2020-05-21 22:41:15 +01:00
parent 6e0d82f584
commit 8d8ead9776
2 changed files with 177 additions and 121 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

203
OpenSim/Framework/Util.cs
View File

@ -2526,25 +2526,10 @@ namespace OpenSim.Framework
if (String.IsNullOrEmpty(str)) if (String.IsNullOrEmpty(str))
return Utils.EmptyBytes; return Utils.EmptyBytes;
if (!str.EndsWith("\0")) byte[] data = new byte[256];
str += "\0"; int r = osUTF8Getbytes(str, data, 255, true); // real use limit is 255 not 256
if (r != 255)
// Because this is UTF-8 encoding and not ASCII, it's possible we Array.Resize<byte>(ref data, r);
// might have gotten an oversized array even after the string trim
byte[] data = UTF8.GetBytes(str);
if (data.Length > 255) //play safe
{
int cut = 254;
if((data[cut] & 0x80 ) != 0 )
{
while(cut > 0 && (data[cut] & 0xc0) != 0xc0)
cut--;
}
Array.Resize<byte>(ref data, cut + 1);
data[cut] = 0;
}
return data; return data;
} }
@ -2577,25 +2562,10 @@ namespace OpenSim.Framework
if (String.IsNullOrEmpty(str)) if (String.IsNullOrEmpty(str))
return Utils.EmptyBytes; return Utils.EmptyBytes;
if (!str.EndsWith("\0")) byte[] data = new byte[1024];
str += "\0"; int r = osUTF8Getbytes(str, data, 1024, true);
if (r != 1024)
// Because this is UTF-8 encoding and not ASCII, it's possible we Array.Resize<byte>(ref data, r);
// might have gotten an oversized array even after the string trim
byte[] data = UTF8.GetBytes(str);
if (data.Length > 1024)
{
int cut = 1023;
if((data[cut] & 0x80 ) != 0 )
{
while(cut > 0 && (data[cut] & 0xc0) != 0xc0)
cut--;
}
Array.Resize<byte>(ref data, cut + 1);
data[cut] = 0;
}
return data; return data;
} }
@ -2628,25 +2598,10 @@ namespace OpenSim.Framework
if (String.IsNullOrEmpty(str)) if (String.IsNullOrEmpty(str))
return Utils.EmptyBytes; return Utils.EmptyBytes;
if (!str.EndsWith("\0")) byte[] data = new byte[MaxLength];
str += "\0"; int r = osUTF8Getbytes(str, data, MaxLength, true);
if (r != MaxLength)
// Because this is UTF-8 encoding and not ASCII, it's possible we Array.Resize<byte>(ref data, r);
// might have gotten an oversized array even after the string trim
byte[] data = UTF8.GetBytes(str);
if (data.Length > MaxLength)
{
int cut = MaxLength - 1;
if ((data[cut] & 0x80) != 0)
{
while (cut > 0 && (data[cut] & 0xc0) != 0xc0)
cut--;
}
Array.Resize<byte>(ref data, cut + 1);
data[cut] = 0;
}
return data; return data;
} }
@ -2655,23 +2610,129 @@ namespace OpenSim.Framework
if (String.IsNullOrEmpty(str)) if (String.IsNullOrEmpty(str))
return Utils.EmptyBytes; return Utils.EmptyBytes;
// Because this is UTF-8 encoding and not ASCII, it's possible we byte[] data = new byte[MaxLength];
// might have gotten an oversized array even after the string trim int r = osUTF8Getbytes(str, data, MaxLength, false);
byte[] data = UTF8.GetBytes(str); if (r != MaxLength)
Array.Resize<byte>(ref data, r);
if (data.Length > MaxLength)
{
int cut = MaxLength - 1;
if ((data[cut] & 0x80) != 0)
{
while (cut > 0 && (data[cut] & 0xc0) != 0xc0)
cut--;
}
Array.Resize<byte>(ref data, cut);
}
return data; return data;
} }
[System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.AggressiveInlining)]
public static int osUTF8Getbytes(string srcstr, byte[] dstarray, int maxdstlen, bool NullTerm = false)
{
return osUTF8Getbytes(srcstr, dstarray, 0, maxdstlen, NullTerm);
}
public static unsafe int osUTF8Getbytes(string srcstr, byte* dstarray, int maxdstlen, bool NullTerm = false)
{
if (string.IsNullOrEmpty(srcstr))
return 0;
fixed (char* srcbase = srcstr)
{
return osUTF8Getbytes(srcbase, srcstr.Length, dstarray, maxdstlen, NullTerm);
}
}
public static unsafe int osUTF8Getbytes(string srcstr, byte[] dstarray, int pos, int maxdstlen, bool NullTerm = false)
{
if (string.IsNullOrEmpty(srcstr))
return 0;
if (pos + maxdstlen > dstarray.Length)
return 0;
fixed (char* srcbase = srcstr)
{
fixed (byte* dstbase = &dstarray[pos])
{
return osUTF8Getbytes(srcbase, srcstr.Length, dstbase, maxdstlen, NullTerm);
}
}
}
public static unsafe int osUTF8Getbytes(char* srcarray, int srclenght, byte* dstarray, int maxdstlen, bool NullTerm = false)
{
int dstlen = NullTerm ? maxdstlen - 1 : maxdstlen;
int srclen = srclenght >= dstlen ? dstlen : srclenght;
char c;
char* src = srcarray;
char* srcend = src + srclen;
byte* dst = dstarray;
byte* dstend = dst + dstlen;
while (src < srcend && dst < dstend)
{
c = *src;
++src;
if (c <= 0x7f)
{
*dst = (byte)c;
++dst;
continue;
}
if (c < 0x800)
{
if (dst + 1 >= dstend)
break;
*dst = (byte)(0xC0 | (c >> 6));
++dst;
*dst = (byte)(0x80 | (c & 0x3F));
++dst;
continue;
}
if (c >= 0xD800 && c < 0xE000)
{
if (c >= 0xDC00)
continue; // ignore invalid
if (src + 1 >= srcend || dst + 3 >= dstend)
break;
int a = c;
c = *src;
++src;
if (c < 0xDC00 || c > 0xDFFF)
continue; // ignore invalid
a = (a << 10) + c - 0x35fdc00;
*dst = (byte)(0xF0 | (a >> 18));
++dst;
*dst = (byte)(0x80 | ((a >> 12) & 0x3f));
++dst;
*dst = (byte)(0x80 | ((a >> 6) & 0x3f));
++dst;
*dst = (byte)(0x80 | (a & 0x3f));
++dst;
continue;
}
if (dst + 2 >= dstend)
break;
*dst = (byte)(0xE0 | (c >> 12));
++dst;
*dst = (byte)(0x80 | ((c >> 6) & 0x3f));
++dst;
*dst = (byte)(0x80 | (c & 0x3f));
++dst;
}
int ret = (int)(dst - dstarray);
if (NullTerm && ret > 0 && *(dst - 1) != 0)
{
*dst = 0;
++ret;
}
return ret;
}
/// <summary> /// <summary>
/// Pretty format the hashtable contents to a single line. /// Pretty format the hashtable contents to a single line.
/// </summary> /// </summary>

91
OpenSim/Region/ClientStack/Linden/UDP/LLUDPZeroEncoder.cs
View File

@ -135,6 +135,34 @@ namespace OpenSim.Region.ClientStack.LindenUDP
} }
} }
public unsafe void AddBytes(byte* src, int srclen)
{
for (int i = 0; i < srclen; ++i)
{
if (src[i] == 0x00)
{
zerocount++;
if (zerocount == 0)
{
m_dest[pos++] = 0x00;
m_dest[pos++] = 0xff;
zerocount++;
}
}
else
{
if (zerocount != 0)
{
m_dest[pos++] = 0x00;
m_dest[pos++] = (byte)zerocount;
zerocount = 0;
}
m_dest[pos++] = src[i];
}
}
}
public unsafe void AddByte(byte v) public unsafe void AddByte(byte v)
{ {
if (v == 0x00) if (v == 0x00)
@ -278,7 +306,7 @@ namespace OpenSim.Region.ClientStack.LindenUDP
} }
// maxlen <= 255 and includes null termination byte // maxlen <= 255 and includes null termination byte
public void AddShortString(string str, int maxlen) public unsafe void AddShortString(string str, int maxlen)
{ {
if (String.IsNullOrEmpty(str)) if (String.IsNullOrEmpty(str))
{ {
@ -286,33 +314,21 @@ namespace OpenSim.Region.ClientStack.LindenUDP
return; return;
} }
--maxlen; // account for null term byte* data = stackalloc byte[maxlen];
bool NullTerm = str.EndsWith("\0"); int len = Util.osUTF8Getbytes(str, data, maxlen, true);
byte[] data = Util.UTF8.GetBytes(str); if (len == 0)
int len = data.Length;
if(NullTerm)
--len;
if(len <= maxlen)
{ {
AddByte((byte)(len + 1));
AddBytes(data, len);
AddZeros(1); AddZeros(1);
return; return;
} }
if ((data[maxlen] & 0x80) != 0) AddByte((byte)(len));
{ AddBytes(data, len);
while (maxlen > 0 && (data[maxlen] & 0xc0) != 0xc0)
maxlen--;
}
AddByte((byte)(maxlen + 1));
AddBytes(data, maxlen);
AddZeros(1);
} }
// maxlen <= 255 and includes null termination byte, maxchars == max len of utf8 source
public void AddShortString(string str, int maxchars, int maxlen) // maxlen <= 255 and includes null termination byte, maxchars == max len of utf16 source
public unsafe void AddShortString(string str, int maxchars, int maxlen)
{ {
if (String.IsNullOrEmpty(str)) if (String.IsNullOrEmpty(str))
{ {
@ -320,41 +336,20 @@ namespace OpenSim.Region.ClientStack.LindenUDP
return; return;
} }
--maxlen; // account for null term
bool NullTerm = false;
byte[] data;
if (str.Length > maxchars) if (str.Length > maxchars)
{ str = str.Substring(0, maxchars);
data = Util.UTF8.GetBytes(str.Substring(0,maxchars));
}
else
{
NullTerm = str.EndsWith("\0");
data = Util.UTF8.GetBytes(str);
}
int len = data.Length; byte* data = stackalloc byte[maxlen];
if (NullTerm) int len = Util.osUTF8Getbytes(str, data, maxlen, true);
--len;
if (len <= maxlen) if (len == 0)
{ {
AddByte((byte)(len + 1));
AddBytes(data, len);
AddZeros(1); AddZeros(1);
return; return;
} }
if ((data[maxlen] & 0x80) != 0) AddByte((byte)(len));
{ AddBytes(data, len);
while (maxlen > 0 && (data[maxlen] & 0xc0) != 0xc0)
maxlen--;
}
AddByte((byte)(maxlen + 1));
AddBytes(data, maxlen);
AddZeros(1);
} }
} }