diff --git a/src/CharsetDetector.cs b/src/CharsetDetector.cs
index 5fea071..8c45438 100644
--- a/src/CharsetDetector.cs
+++ b/src/CharsetDetector.cs
@@ -117,6 +117,20 @@ private CharsetDetector()
_lastChar = 0x00;
}
+ ///
+ /// Detect the character encoding of these bytes.
+ /// It searches for BOM from the start of the span.
+ /// Slice the span before passing it if only a range should be inspected.
+ ///
+ /// The bytes containing the text
+ ///
+ public static DetectionResult DetectFromBytes(ReadOnlySpan bytes)
+ {
+ var detector = new CharsetDetector();
+ detector.Feed(bytes);
+ return detector.DataEnd();
+ }
+
///
/// Detect the character encoding form this byte array.
/// It searches for BOM from bytes[0].
@@ -130,9 +144,7 @@ public static DetectionResult DetectFromBytes(byte[] bytes)
throw new ArgumentNullException(nameof(bytes));
}
- var detector = new CharsetDetector();
- detector.Feed(bytes, 0, bytes.Length);
- return detector.DataEnd();
+ return DetectFromBytes(bytes.AsSpan());
}
///
@@ -161,10 +173,8 @@ public static DetectionResult DetectFromBytes(byte[] bytes, int offset, int len)
{
throw new ArgumentException($"{nameof(len)} is greater than the number of bytes from {nameof(offset)} to the end of the array.");
}
-
- var detector = new CharsetDetector();
- detector.Feed(bytes, offset, len);
- return detector.DataEnd();
+
+ return DetectFromBytes(bytes.AsSpan(offset, len));
}
///
@@ -291,7 +301,7 @@ private static async Task ReadStreamAsync(Stream stream, long? maxBytes, Charset
private static bool FeedDetector(CharsetDetector detector, long? maxBytes, byte[] buff, int read, ref long readTotal, ref int toRead)
{
- detector.Feed(buff, 0, read);
+ detector.Feed(buff.AsSpan(0, read));
if (maxBytes == null)
{
@@ -401,37 +411,39 @@ private static FileStream OpenFile(string filePath)
FileShare.ReadWrite);
}
- protected virtual void Feed(byte[] buf, int offset, int len)
+ protected virtual void Feed(ReadOnlySpan buf)
{
if (_done)
{
return;
}
- if (len > 0)
- _gotData = true;
+ if (buf.Length == 0)
+ return;
+
+ _gotData = true;
// If the data starts with BOM, we know it is UTF
if (_start)
{
_start = false;
- _done = IsStartsWithBom(buf, offset, len);
+ _done = IsStartsWithBom(buf);
if (_done)
return;
}
- FindInputState(buf, offset, len);
+ FindInputState(buf);
foreach (var prober in CharsetProbers)
{
- _done = RunProber(buf, offset, len, prober);
+ _done = RunProber(buf, prober);
if (_done)
return;
}
}
- private bool IsStartsWithBom(byte[] buf, int offset, int len)
+ private bool IsStartsWithBom(ReadOnlySpan buf)
{
- var bomSet = FindCharSetByBom(buf, offset, len);
+ var bomSet = FindCharSetByBom(buf);
if (bomSet != null)
{
_detectionDetail = new DetectionDetail(bomSet, 1.0f)
@@ -443,9 +455,9 @@ private bool IsStartsWithBom(byte[] buf, int offset, int len)
return false;
}
- private bool RunProber(byte[] buf, int offset, int len, CharsetProber charsetProber)
+ private bool RunProber(ReadOnlySpan buf, CharsetProber charsetProber)
{
- var probingState = charsetProber.HandleData(buf, offset, len);
+ var probingState = charsetProber.HandleData(buf);
if (probingState == ProbingState.FoundIt)
{
_detectionDetail = new DetectionDetail(charsetProber);
@@ -454,9 +466,9 @@ private bool RunProber(byte[] buf, int offset, int len, CharsetProber charsetPro
return false;
}
- private void FindInputState(byte[] buf, int offset, int len)
+ private void FindInputState(ReadOnlySpan buf)
{
- for (int i = offset; i < len; i++)
+ for (int i = 0; i < buf.Length; i++)
{
// other than 0xa0, if every other character is ascii, the page is ascii
if ((buf[i] & 0x80) != 0 && buf[i] != 0xA0)
@@ -485,59 +497,59 @@ private void FindInputState(byte[] buf, int offset, int len)
}
}
- private static string FindCharSetByBom(byte[] buf, int offset, int len)
+ private static string FindCharSetByBom(ReadOnlySpan buf)
{
- if (len < 2)
+ if (buf.Length < 2)
return null;
- var buf0 = buf[offset + 0];
- var buf1 = buf[offset + 1];
+ var buf0 = buf[0];
+ var buf1 = buf[1];
if (buf0 == 0xFE && buf1 == 0xFF)
{
// FE FF 00 00 UCS-4, unusual octet order BOM (3412)
- return len > 3
- && buf[offset + 2] == 0x00 && buf[offset + 3] == 0x00
+ return buf.Length > 3
+ && buf[2] == 0x00 && buf[3] == 0x00
? CodepageName.X_ISO_10646_UCS_4_3412
: CodepageName.UTF16_BE;
}
if (buf0 == 0xFF && buf1 == 0xFE)
{
- return len > 3
- && buf[offset + 2] == 0x00 && buf[offset + 3] == 0x00
+ return buf.Length > 3
+ && buf[2] == 0x00 && buf[3] == 0x00
? CodepageName.UTF32_LE
: CodepageName.UTF16_LE;
}
- if (len < 3)
+ if (buf.Length < 3)
return null;
- if (buf0 == 0xEF && buf1 == 0xBB && buf[offset + 2] == 0xBF)
+ if (buf0 == 0xEF && buf1 == 0xBB && buf[2] == 0xBF)
return CodepageName.UTF8;
- if (len < 4)
+ if (buf.Length < 4)
return null;
//Here, because anyway further more than 3 positions are checked.
if (buf0 == 0x00 && buf1 == 0x00)
{
- if (buf[offset + 2] == 0xFE && buf[offset + 3] == 0xFF)
+ if (buf[2] == 0xFE && buf[3] == 0xFF)
return CodepageName.UTF32_BE;
// 00 00 FF FE UCS-4, unusual octet order BOM (2143)
- if (buf[offset + 2] == 0xFF && buf[offset + 3] == 0xFE)
+ if (buf[2] == 0xFF && buf[3] == 0xFE)
return CodepageName.X_ISO_10646_UCS_4_2143;
}
// Detect utf-7 with bom (see table in https://en.wikipedia.org/wiki/Byte_order_mark)
- if (buf0 == 0x2B && buf1 == 0x2F && buf[offset + 2] == 0x76)
- if (buf[offset + 3] == 0x38 || buf[offset + 3] == 0x39 || buf[offset + 3] == 0x2B || buf[offset + 3] == 0x2F)
+ if (buf0 == 0x2B && buf1 == 0x2F && buf[2] == 0x76)
+ if (buf[3] == 0x38 || buf[3] == 0x39 || buf[3] == 0x2B || buf[3] == 0x2F)
return CodepageName.UTF7;
// Detect GB18030 with bom (see table in https://en.wikipedia.org/wiki/Byte_order_mark)
// TODO: If you remove this check, GB18030Prober will still be defined as GB18030 -- It's feature or bug?
- if (buf0 == 0x84 && buf1 == 0x31 && buf[offset + 2] == 0x95 && buf[offset + 3] == 0x33)
+ if (buf0 == 0x84 && buf1 == 0x31 && buf[2] == 0x95 && buf[3] == 0x33)
return CodepageName.GB18030;
return null;
diff --git a/src/Core/Analyzers/CharDistributionAnalyser.cs b/src/Core/Analyzers/CharDistributionAnalyser.cs
index b353eb2..3ce5eae 100644
--- a/src/Core/Analyzers/CharDistributionAnalyser.cs
+++ b/src/Core/Analyzers/CharDistributionAnalyser.cs
@@ -35,6 +35,8 @@
*
* ***** END LICENSE BLOCK ***** */
+using System;
+
namespace UtfUnknown.Core.Analyzers;
///
@@ -77,20 +79,18 @@ public CharDistributionAnalyser()
/// This allow multiple encoding of a language to share one frequency table
///
/// A
- ///
///
- public abstract int GetOrder(byte[] buf, int offset);
+ public abstract int GetOrder(ReadOnlySpan buf);
///
/// Feed a character with known length
///
/// A
- /// buf offset
/// 1 of 2 char length?
- public void HandleOneChar(byte[] buf, int offset, int charLen)
+ public void HandleOneChar(ReadOnlySpan buf, int charLen)
{
//we only care about 2-bytes character in our distribution analysis
- int order = (charLen == 2) ? GetOrder(buf, offset) : -1;
+ int order = (charLen == 2) ? GetOrder(buf) : -1;
if (order >= 0)
{
totalChars++;
@@ -136,4 +136,4 @@ public bool GotEnoughData()
{
return totalChars > ENOUGH_DATA_THRESHOLD;
}
-}
\ No newline at end of file
+}
diff --git a/src/Core/Analyzers/MultiByte/Chinese/BIG5DistributionAnalyser.cs b/src/Core/Analyzers/MultiByte/Chinese/BIG5DistributionAnalyser.cs
index 636bfa7..62733a5 100644
--- a/src/Core/Analyzers/MultiByte/Chinese/BIG5DistributionAnalyser.cs
+++ b/src/Core/Analyzers/MultiByte/Chinese/BIG5DistributionAnalyser.cs
@@ -1,3 +1,5 @@
+using System;
+
namespace UtfUnknown.Core.Analyzers.Chinese;
public class BIG5DistributionAnalyser : CharDistributionAnalyser
@@ -914,13 +916,13 @@ public BIG5DistributionAnalyser()
/// second byte range: 0x40 -- 0x7e , 0xa1 -- 0xfe
/// no validation needed here. State machine has done that
///
- public override int GetOrder(byte[] buf, int offset)
+ public override int GetOrder(ReadOnlySpan buf)
{
- if (buf[offset] >= 0xA4) {
- if (buf[offset+1] >= 0xA1)
- return 157 * (buf[offset] - 0xA4) + buf[offset+1] - 0xA1 + 63;
+ if (buf[0] >= 0xA4) {
+ if (buf[1] >= 0xA1)
+ return 157 * (buf[0] - 0xA4) + buf[1] - 0xA1 + 63;
else
- return 157 * (buf[offset] - 0xA4) + buf[offset+1] - 0x40;
+ return 157 * (buf[0] - 0xA4) + buf[1] - 0x40;
} else {
return -1;
}
diff --git a/src/Core/Analyzers/MultiByte/Chinese/EUCTWDistributionAnalyser.cs b/src/Core/Analyzers/MultiByte/Chinese/EUCTWDistributionAnalyser.cs
index c840ad0..1648bf7 100644
--- a/src/Core/Analyzers/MultiByte/Chinese/EUCTWDistributionAnalyser.cs
+++ b/src/Core/Analyzers/MultiByte/Chinese/EUCTWDistributionAnalyser.cs
@@ -1,3 +1,5 @@
+using System;
+
namespace UtfUnknown.Core.Analyzers.Chinese;
public class EUCTWDistributionAnalyser : CharDistributionAnalyser
@@ -417,11 +419,11 @@ public EUCTWDistributionAnalyser()
/// second byte range: 0xa1 -- 0xfe
/// no validation needed here. State machine has done that
///
- public override int GetOrder(byte[] buf, int offset)
+ public override int GetOrder(ReadOnlySpan buf)
{
- if (buf[offset] >= 0xC4)
- return 94 * (buf[offset] - 0xC4) + buf[offset+1] - 0xA1;
+ if (buf[0] >= 0xC4)
+ return 94 * (buf[0] - 0xC4) + buf[1] - 0xA1;
else
return -1;
}
-}
\ No newline at end of file
+}
diff --git a/src/Core/Analyzers/MultiByte/Chinese/GB18030DistributionAnalyser.cs b/src/Core/Analyzers/MultiByte/Chinese/GB18030DistributionAnalyser.cs
index ae49f3f..9ffc1fe 100644
--- a/src/Core/Analyzers/MultiByte/Chinese/GB18030DistributionAnalyser.cs
+++ b/src/Core/Analyzers/MultiByte/Chinese/GB18030DistributionAnalyser.cs
@@ -1,3 +1,5 @@
+using System;
+
namespace UtfUnknown.Core.Analyzers.Chinese;
public class GB18030DistributionAnalyser : CharDistributionAnalyser
@@ -463,10 +465,10 @@ public GB18030DistributionAnalyser() : base()
/// no validation needed here. State machine has done that
///
///
- public override int GetOrder(byte[] buf, int offset)
+ public override int GetOrder(ReadOnlySpan buf)
{
- if (buf[offset] >= 0xB0 && buf[offset+1] >= 0xA1)
- return 94 * (buf[offset] - 0xb0) + buf[offset+1] - 0xA1;
+ if (buf[0] >= 0xB0 && buf[1] >= 0xA1)
+ return 94 * (buf[0] - 0xb0) + buf[1] - 0xA1;
else
return -1;
}
diff --git a/src/Core/Analyzers/MultiByte/Japanese/EUCJPContextAnalyser.cs b/src/Core/Analyzers/MultiByte/Japanese/EUCJPContextAnalyser.cs
index 83bdaa7..fbd636e 100644
--- a/src/Core/Analyzers/MultiByte/Japanese/EUCJPContextAnalyser.cs
+++ b/src/Core/Analyzers/MultiByte/Japanese/EUCJPContextAnalyser.cs
@@ -1,12 +1,14 @@
+using System;
+
namespace UtfUnknown.Core.Analyzers.Japanese;
public class EUCJPContextAnalyser : JapaneseContextAnalyser
{
private const byte HIRAGANA_FIRST_BYTE = 0xA4;
- protected override int GetOrder(byte[] buf, int offset, out int charLen)
+ protected override int GetOrder(ReadOnlySpan buf, out int charLen)
{
- byte high = buf[offset];
+ byte high = buf[0];
//find out current char's byte length
if (high == 0x8E || high >= 0xA1 && high <= 0xFE)
@@ -18,21 +20,21 @@ protected override int GetOrder(byte[] buf, int offset, out int charLen)
// return its order if it is hiragana
if (high == HIRAGANA_FIRST_BYTE) {
- byte low = buf[offset+1];
+ byte low = buf[1];
if (low >= 0xA1 && low <= 0xF3)
return low - 0xA1;
}
return -1;
}
- protected override int GetOrder(byte[] buf, int offset)
+ protected override int GetOrder(ReadOnlySpan buf)
{
// We are only interested in Hiragana
- if (buf[offset] == HIRAGANA_FIRST_BYTE) {
- byte low = buf[offset+1];
+ if (buf[0] == HIRAGANA_FIRST_BYTE) {
+ byte low = buf[1];
if (low >= 0xA1 && low <= 0xF3)
return low - 0xA1;
}
return -1;
}
-}
\ No newline at end of file
+}
diff --git a/src/Core/Analyzers/MultiByte/Japanese/EUCJPDistributionAnalyser.cs b/src/Core/Analyzers/MultiByte/Japanese/EUCJPDistributionAnalyser.cs
index fc624f9..8700f2c 100644
--- a/src/Core/Analyzers/MultiByte/Japanese/EUCJPDistributionAnalyser.cs
+++ b/src/Core/Analyzers/MultiByte/Japanese/EUCJPDistributionAnalyser.cs
@@ -1,3 +1,5 @@
+using System;
+
namespace UtfUnknown.Core.Analyzers.Japanese;
public class EUCJPDistributionAnalyser : SJISDistributionAnalyser
@@ -7,10 +9,10 @@ public class EUCJPDistributionAnalyser : SJISDistributionAnalyser
/// second byte range: 0xa1 -- 0xfe
/// no validation needed here. State machine has done that
///
- public override int GetOrder(byte[] buf, int offset)
+ public override int GetOrder(ReadOnlySpan buf)
{
- if (buf[offset] >= 0xA0)
- return 94 * (buf[offset] - 0xA1) + buf[offset+1] - 0xA1;
+ if (buf[0] >= 0xA0)
+ return 94 * (buf[0] - 0xA1) + buf[1] - 0xA1;
else
return -1;
}
diff --git a/src/Core/Analyzers/MultiByte/Japanese/JapaneseContextAnalyser.cs b/src/Core/Analyzers/MultiByte/Japanese/JapaneseContextAnalyser.cs
index dd47bad..6ffc83b 100644
--- a/src/Core/Analyzers/MultiByte/Japanese/JapaneseContextAnalyser.cs
+++ b/src/Core/Analyzers/MultiByte/Japanese/JapaneseContextAnalyser.cs
@@ -36,6 +36,8 @@
*
* ***** END LICENSE BLOCK ***** */
+using System;
+
namespace UtfUnknown.Core.Analyzers.Japanese;
public abstract class JapaneseContextAnalyser
@@ -165,9 +167,9 @@ public float GetConfidence()
return DONT_KNOW;
}
- public void HandleData(byte[] buf, int offset, int len)
+ public void HandleData(ReadOnlySpan buf)
{
- int max = offset + len;
+ int max = buf.Length;
if (done)
return;
@@ -179,8 +181,8 @@ public void HandleData(byte[] buf, int offset, int len)
// to record those bytes as well and analyse the character once it
// is complete, but since a character will not make much difference,
// skipping it will simplify our logic and improve performance.
- for (int i = needToSkipCharNum+offset; i < max; ) {
- int order = GetOrder(buf, i, out var charLen);
+ for (int i = needToSkipCharNum; i < max; ) {
+ int order = GetOrder(buf.Slice(i), out var charLen);
i += charLen;
if (i > max) {
needToSkipCharNum = i - max;
@@ -199,7 +201,7 @@ public void HandleData(byte[] buf, int offset, int len)
}
}
- public void HandleOneChar(byte[] buf, int offset, int charLen)
+ public void HandleOneChar(ReadOnlySpan buf, int charLen)
{
if (totalRel > MAX_REL_THRESHOLD)
done = true;
@@ -207,7 +209,7 @@ public void HandleOneChar(byte[] buf, int offset, int charLen)
return;
// Only 2-bytes characters are of our interest
- int order = (charLen == 2) ? GetOrder(buf, offset) : -1;
+ int order = (charLen == 2) ? GetOrder(buf) : -1;
if (order != -1 && lastCharOrder != -1) {
totalRel++;
// count this sequence to its category counter
@@ -227,9 +229,9 @@ public void Reset()
}
}
- protected abstract int GetOrder(byte[] buf, int offset, out int charLen);
+ protected abstract int GetOrder(ReadOnlySpan buf, out int charLen);
- protected abstract int GetOrder(byte[] buf, int offset);
+ protected abstract int GetOrder(ReadOnlySpan buf);
public bool GotEnoughData()
{
diff --git a/src/Core/Analyzers/MultiByte/Japanese/SJISContextAnalyser.cs b/src/Core/Analyzers/MultiByte/Japanese/SJISContextAnalyser.cs
index df96134..b559d31 100644
--- a/src/Core/Analyzers/MultiByte/Japanese/SJISContextAnalyser.cs
+++ b/src/Core/Analyzers/MultiByte/Japanese/SJISContextAnalyser.cs
@@ -1,35 +1,37 @@
-namespace UtfUnknown.Core.Analyzers.Japanese;
+using System;
+
+namespace UtfUnknown.Core.Analyzers.Japanese;
public class SJISContextAnalyser : JapaneseContextAnalyser
{
private const byte HIRAGANA_FIRST_BYTE = 0x82;
- protected override int GetOrder(byte[] buf, int offset, out int charLen)
+ protected override int GetOrder(ReadOnlySpan buf, out int charLen)
{
//find out current char's byte length
- if (buf[offset] >= 0x81 && buf[offset] <= 0x9F
- || buf[offset] >= 0xe0 && buf[offset] <= 0xFC)
+ if (buf[0] >= 0x81 && buf[0] <= 0x9F
+ || buf[0] >= 0xe0 && buf[0] <= 0xFC)
charLen = 2;
else
charLen = 1;
// return its order if it is hiragana
- if (buf[offset] == HIRAGANA_FIRST_BYTE) {
- byte low = buf[offset+1];
+ if (buf[0] == HIRAGANA_FIRST_BYTE) {
+ byte low = buf[1];
if (low >= 0x9F && low <= 0xF1)
return low - 0x9F;
}
return -1;
}
- protected override int GetOrder(byte[] buf, int offset)
+ protected override int GetOrder(ReadOnlySpan buf)
{
// We are only interested in Hiragana
- if (buf[offset] == HIRAGANA_FIRST_BYTE) {
- byte low = buf[offset+1];
+ if (buf[0] == HIRAGANA_FIRST_BYTE) {
+ byte low = buf[1];
if (low >= 0x9F && low <= 0xF1)
return low - 0x9F;
}
return -1;
}
-}
\ No newline at end of file
+}
diff --git a/src/Core/Analyzers/MultiByte/Japanese/SJISDistributionAnalyser.cs b/src/Core/Analyzers/MultiByte/Japanese/SJISDistributionAnalyser.cs
index 83f4068..b1f2f41 100644
--- a/src/Core/Analyzers/MultiByte/Japanese/SJISDistributionAnalyser.cs
+++ b/src/Core/Analyzers/MultiByte/Japanese/SJISDistributionAnalyser.cs
@@ -1,3 +1,5 @@
+using System;
+
namespace UtfUnknown.Core.Analyzers.Japanese;
public class SJISDistributionAnalyser : CharDistributionAnalyser
@@ -558,19 +560,19 @@ public SJISDistributionAnalyser()
/// second byte range: 0x40 -- 0x7e, 0x81 -- oxfe
/// no validation needed here. State machine has done that
///
- public override int GetOrder(byte[] buf, int offset)
+ public override int GetOrder(ReadOnlySpan buf)
{
int order;
- if (buf[offset] >= 0x81 && buf[offset] <= 0x9F)
- order = 188 * (buf[offset] - 0x81);
- else if (buf[offset] >= 0xE0 && buf[offset] <= 0xEF)
- order = 188 * (buf[offset] - 0xE0 + 31);
+ if (buf[0] >= 0x81 && buf[0] <= 0x9F)
+ order = 188 * (buf[0] - 0x81);
+ else if (buf[0] >= 0xE0 && buf[0] <= 0xEF)
+ order = 188 * (buf[0] - 0xE0 + 31);
else
return -1;
- order += buf[offset+1] - 0x40;
+ order += buf[1] - 0x40;
- if (buf[offset+1] > 0x7F)
+ if (buf[1] > 0x7F)
order--;
return order;
}
diff --git a/src/Core/Analyzers/MultiByte/Korean/EUCKRDistributionAnalyser.cs b/src/Core/Analyzers/MultiByte/Korean/EUCKRDistributionAnalyser.cs
index 736c8b4..2146be5 100644
--- a/src/Core/Analyzers/MultiByte/Korean/EUCKRDistributionAnalyser.cs
+++ b/src/Core/Analyzers/MultiByte/Korean/EUCKRDistributionAnalyser.cs
@@ -1,4 +1,6 @@
-namespace UtfUnknown.Core.Analyzers.Korean;
+using System;
+
+namespace UtfUnknown.Core.Analyzers.Korean;
public class EUCKRDistributionAnalyser : CharDistributionAnalyser
{
@@ -583,10 +585,10 @@ public EUCKRDistributionAnalyser()
/// second byte range: 0xa1 -- 0xfe
/// no validation needed here. State machine has done that
///
- public override int GetOrder(byte[] buf, int offset)
+ public override int GetOrder(ReadOnlySpan buf)
{
- if (buf[offset] >= 0xB0)
- return 94 * (buf[offset] - 0xB0) + buf[offset+1] - 0xA1;
+ if (buf[0] >= 0xB0)
+ return 94 * (buf[0] - 0xB0) + buf[1] - 0xA1;
else
return -1;
}
diff --git a/src/Core/Probers/CharsetProber.cs b/src/Core/Probers/CharsetProber.cs
index f30d2c8..593ef53 100644
--- a/src/Core/Probers/CharsetProber.cs
+++ b/src/Core/Probers/CharsetProber.cs
@@ -36,8 +36,10 @@
*
* ***** END LICENSE BLOCK ***** */
+using System;
using System.IO;
using System.Text;
+using System.Buffers;
namespace UtfUnknown.Core.Probers;
@@ -60,12 +62,10 @@ public abstract class CharsetProber
/// Feed data to the prober
///
/// a buffer
- /// offset into buffer
- /// number of bytes available into buffer
///
/// A
///
- public abstract ProbingState HandleData(byte[] buf, int offset, int len);
+ public abstract ProbingState HandleData(ReadOnlySpan buf);
///
/// Reset prober state
@@ -98,16 +98,16 @@ public virtual string DumpStatus()
///
///
/// filtered buffer
- protected static byte[] FilterWithoutEnglishLetters(byte[] buf, int offset, int len)
+ protected static byte[] FilterWithoutEnglishLetters(ReadOnlySpan buf)
{
byte[] result;
using (MemoryStream ms = new MemoryStream(buf.Length))
{
bool meetMSB = false;
- int max = offset + len;
- int prev = offset;
- int cur = offset;
+ int max = buf.Length;
+ int prev = 0;
+ int cur = 0;
while (cur < max)
{
@@ -121,7 +121,7 @@ protected static byte[] FilterWithoutEnglishLetters(byte[] buf, int offset, int
{
if (meetMSB && cur > prev)
{
- ms.Write(buf, prev, cur - prev);
+ WriteSpanToStream(ms, buf.Slice(prev, cur - prev));
ms.WriteByte(SPACE);
meetMSB = false;
}
@@ -131,7 +131,7 @@ protected static byte[] FilterWithoutEnglishLetters(byte[] buf, int offset, int
}
if (meetMSB && cur > prev)
- ms.Write(buf, prev, cur - prev);
+ WriteSpanToStream(ms, buf.Slice(prev, cur - prev));
ms.SetLength(ms.Position);
result = ms.ToArray();
}
@@ -144,21 +144,19 @@ protected static byte[] FilterWithoutEnglishLetters(byte[] buf, int offset, int
/// both English characters and upper ASCII characters.
///
/// a filtered copy of the input buffer
- protected static byte[] FilterWithEnglishLetters(byte[] buf, int offset, int len)
+ protected static byte[] FilterWithEnglishLetters(ReadOnlySpan buf)
{
byte[] result;
using (MemoryStream ms = new MemoryStream(buf.Length))
{
-
bool inTag = false;
- int max = offset + len;
- int prev = offset;
- int cur = offset;
+ int max = buf.Length;
+ int prev = 0;
+ int cur = 0;
while (cur < max)
{
-
byte b = buf[cur];
if (b == GREATER_THAN)
@@ -172,7 +170,7 @@ protected static byte[] FilterWithEnglishLetters(byte[] buf, int offset, int len
{
if (cur > prev && !inTag)
{
- ms.Write(buf, prev, cur - prev);
+ WriteSpanToStream(ms, buf.Slice(prev, cur - prev));
ms.WriteByte(SPACE);
}
prev = cur + 1;
@@ -183,10 +181,30 @@ protected static byte[] FilterWithEnglishLetters(byte[] buf, int offset, int len
// If the current segment contains more than just a symbol
// and it is not inside a tag then keep it.
if (!inTag && cur > prev)
- ms.Write(buf, prev, cur - prev);
+ WriteSpanToStream(ms, buf.Slice(prev, cur - prev));
ms.SetLength(ms.Position);
result = ms.ToArray();
}
return result;
}
-}
\ No newline at end of file
+
+ private static void WriteSpanToStream(MemoryStream stream, ReadOnlySpan buffer)
+ {
+#if NET8_0_OR_GREATER
+ stream.Write(buffer);
+#else
+ byte[] rent = ArrayPool.Shared.Rent(buffer.Length);
+
+ try
+ {
+ buffer.CopyTo(rent);
+
+ stream.Write(rent, 0, buffer.Length);
+ }
+ finally
+ {
+ ArrayPool.Shared.Return(rent);
+ }
+#endif
+ }
+}
diff --git a/src/Core/Probers/EscCharsetProber.cs b/src/Core/Probers/EscCharsetProber.cs
index 51796b1..2c5dda6 100644
--- a/src/Core/Probers/EscCharsetProber.cs
+++ b/src/Core/Probers/EscCharsetProber.cs
@@ -35,6 +35,7 @@
*
* ***** END LICENSE BLOCK ***** */
+using System;
using System.Text;
using UtfUnknown.Core.Models;
@@ -70,11 +71,11 @@ public override void Reset()
detectedCharset = null;
}
- public override ProbingState HandleData(byte[] buf, int offset, int len)
+ public override ProbingState HandleData(ReadOnlySpan buf)
{
- int max = offset + len;
+ int max = buf.Length;
- for (int i = offset; i < max && state == ProbingState.Detecting; i++) {
+ for (int i = 0; i < max && state == ProbingState.Detecting; i++) {
for (int j = activeSM - 1; j >= 0; j--) {
// byte is feed to all active state machine
int codingState = codingSM[j].NextState(buf[i]);
diff --git a/src/Core/Probers/HebrewProber.cs b/src/Core/Probers/HebrewProber.cs
index d6c4da1..646f345 100644
--- a/src/Core/Probers/HebrewProber.cs
+++ b/src/Core/Probers/HebrewProber.cs
@@ -36,6 +36,7 @@
*
* ***** END LICENSE BLOCK ***** */
+using System;
using System.Text;
/*
@@ -209,15 +210,15 @@ public void SetModelProbers(CharsetProber logical, CharsetProber visual)
* The input buffer should not contain any white spaces that are not (' ')
* or any low-ascii punctuation marks.
*/
- public override ProbingState HandleData(byte[] buf, int offset, int len)
+ public override ProbingState HandleData(ReadOnlySpan buf)
{
// Both model probers say it's not them. No reason to continue.
if (GetState() == ProbingState.NotMe)
return ProbingState.NotMe;
- int max = offset + len;
+ int max = buf.Length;
- for (int i = offset; i < max; i++) {
+ for (int i = 0; i < max; i++) {
byte b = buf[i];
diff --git a/src/Core/Probers/Latin1Prober.cs b/src/Core/Probers/Latin1Prober.cs
index 267a2c3..180327e 100644
--- a/src/Core/Probers/Latin1Prober.cs
+++ b/src/Core/Probers/Latin1Prober.cs
@@ -36,6 +36,7 @@
*
* ***** END LICENSE BLOCK ***** */
+using System;
using System.Text;
namespace UtfUnknown.Core.Probers;
@@ -130,9 +131,9 @@ public override void Reset()
freqCounter[i] = 0;
}
- public override ProbingState HandleData(byte[] buf, int offset, int len)
+ public override ProbingState HandleData(ReadOnlySpan buf)
{
- byte[] newbuf = FilterWithEnglishLetters(buf, offset, len);
+ byte[] newbuf = FilterWithEnglishLetters(buf);
byte charClass, freq;
for (int i = 0; i < newbuf.Length; i++)
diff --git a/src/Core/Probers/MBCSGroupProber.cs b/src/Core/Probers/MBCSGroupProber.cs
index 1e5c5f9..7668e71 100644
--- a/src/Core/Probers/MBCSGroupProber.cs
+++ b/src/Core/Probers/MBCSGroupProber.cs
@@ -36,6 +36,7 @@
*
* ***** END LICENSE BLOCK ***** */
+using System;
using System.Text;
using UtfUnknown.Core.Probers.MultiByte;
@@ -106,16 +107,16 @@ public override void Reset()
state = ProbingState.Detecting;
}
- public override ProbingState HandleData(byte[] buf, int offset, int len)
+ public override ProbingState HandleData(ReadOnlySpan buf)
{
// do filtering to reduce load to probers
- byte[] highbyteBuf = new byte[len];
+ byte[] highbyteBuf = new byte[buf.Length];
int hptr = 0;
//assume previous is not ascii, it will do no harm except add some noise
bool keepNext = true;
- int max = offset + len;
+ int max = buf.Length;
- for (int i = offset; i < max; i++)
+ for (int i = 0; i < max; i++)
{
if ((buf[i] & 0x80) != 0)
{
@@ -137,7 +138,7 @@ public override ProbingState HandleData(byte[] buf, int offset, int len)
{
if (isActive[i])
{
- var st = probers[i].HandleData(highbyteBuf, 0, hptr);
+ var st = probers[i].HandleData(highbyteBuf.AsSpan(0, hptr));
if (st == ProbingState.FoundIt)
{
bestGuess = i;
diff --git a/src/Core/Probers/MultiByte/Chinese/Big5Prober.cs b/src/Core/Probers/MultiByte/Chinese/Big5Prober.cs
index 22de251..b4dcad9 100644
--- a/src/Core/Probers/MultiByte/Chinese/Big5Prober.cs
+++ b/src/Core/Probers/MultiByte/Chinese/Big5Prober.cs
@@ -36,6 +36,7 @@
*
* ***** END LICENSE BLOCK ***** */
+using System;
using System.Text;
using UtfUnknown.Core.Analyzers.Chinese;
@@ -58,11 +59,11 @@ public Big5Prober()
Reset();
}
- public override ProbingState HandleData(byte[] buf, int offset, int len)
+ public override ProbingState HandleData(ReadOnlySpan buf)
{
- int max = offset + len;
+ int max = buf.Length;
- for (int i = offset; i < max; i++)
+ for (int i = 0; i < max; i++)
{
var codingState = codingSM.NextState(buf[i]);
if (codingState == StateMachineModel.ERROR)
@@ -78,14 +79,14 @@ public override ProbingState HandleData(byte[] buf, int offset, int len)
if (codingState == StateMachineModel.START)
{
int charLen = codingSM.CurrentCharLen;
- if (i == offset)
+ if (i == 0)
{
- lastChar[1] = buf[offset];
- distributionAnalyser.HandleOneChar(lastChar, 0, charLen);
+ lastChar[1] = buf[0];
+ distributionAnalyser.HandleOneChar(lastChar, charLen);
}
else
{
- distributionAnalyser.HandleOneChar(buf, i - 1, charLen);
+ distributionAnalyser.HandleOneChar(buf.Slice(i - 1), charLen);
}
}
}
diff --git a/src/Core/Probers/MultiByte/Chinese/EUCTWProber.cs b/src/Core/Probers/MultiByte/Chinese/EUCTWProber.cs
index 9e10dea..df17787 100644
--- a/src/Core/Probers/MultiByte/Chinese/EUCTWProber.cs
+++ b/src/Core/Probers/MultiByte/Chinese/EUCTWProber.cs
@@ -36,6 +36,7 @@
*
* ***** END LICENSE BLOCK ***** */
+using System;
using System.Text;
using UtfUnknown.Core.Analyzers.Chinese;
@@ -57,10 +58,10 @@ public EUCTWProber()
Reset();
}
- public override ProbingState HandleData(byte[] buf, int offset, int len)
+ public override ProbingState HandleData(ReadOnlySpan buf)
{
int codingState;
- int max = offset + len;
+ int max = buf.Length;
for (int i = 0; i < max; i++)
{
@@ -80,14 +81,14 @@ public override ProbingState HandleData(byte[] buf, int offset, int len)
if (codingState == StateMachineModel.START)
{
int charLen = codingSM.CurrentCharLen;
- if (i == offset)
+ if (i == 0)
{
- lastChar[1] = buf[offset];
- distributionAnalyser.HandleOneChar(lastChar, 0, charLen);
+ lastChar[1] = buf[0];
+ distributionAnalyser.HandleOneChar(lastChar, charLen);
}
else
{
- distributionAnalyser.HandleOneChar(buf, i - 1, charLen);
+ distributionAnalyser.HandleOneChar(buf.Slice(i - 1), charLen);
}
}
}
diff --git a/src/Core/Probers/MultiByte/Chinese/GB18030Prober.cs b/src/Core/Probers/MultiByte/Chinese/GB18030Prober.cs
index 3a4c79d..c76ae08 100644
--- a/src/Core/Probers/MultiByte/Chinese/GB18030Prober.cs
+++ b/src/Core/Probers/MultiByte/Chinese/GB18030Prober.cs
@@ -36,6 +36,7 @@
*
* ***** END LICENSE BLOCK ***** */
+using System;
using System.Text;
using UtfUnknown.Core.Analyzers.Chinese;
@@ -64,11 +65,11 @@ public override string GetCharsetName()
return CodepageName.GB18030;
}
- public override ProbingState HandleData(byte[] buf, int offset, int len)
+ public override ProbingState HandleData(ReadOnlySpan buf)
{
- int max = offset + len;
+ int max = buf.Length;
- for (int i = offset; i < max; i++)
+ for (int i = 0; i < max; i++)
{
var codingState = codingSM.NextState(buf[i]);
@@ -87,14 +88,14 @@ public override ProbingState HandleData(byte[] buf, int offset, int len)
if (codingState == StateMachineModel.START)
{
int charLen = codingSM.CurrentCharLen;
- if (i == offset)
+ if (i == 0)
{
- lastChar[1] = buf[offset];
- analyser.HandleOneChar(lastChar, 0, charLen);
+ lastChar[1] = buf[0];
+ analyser.HandleOneChar(lastChar, charLen);
}
else
{
- analyser.HandleOneChar(buf, i - 1, charLen);
+ analyser.HandleOneChar(buf.Slice(i - 1), charLen);
}
}
}
diff --git a/src/Core/Probers/MultiByte/Japanese/EUCJPProber.cs b/src/Core/Probers/MultiByte/Japanese/EUCJPProber.cs
index 7a21740..2cfd10e 100644
--- a/src/Core/Probers/MultiByte/Japanese/EUCJPProber.cs
+++ b/src/Core/Probers/MultiByte/Japanese/EUCJPProber.cs
@@ -35,6 +35,7 @@
*
* ***** END LICENSE BLOCK ***** */
+using System;
using System.Text;
using UtfUnknown.Core.Analyzers.Japanese;
@@ -63,12 +64,12 @@ public override string GetCharsetName()
return CodepageName.EUC_JP;
}
- public override ProbingState HandleData(byte[] buf, int offset, int len)
+ public override ProbingState HandleData(ReadOnlySpan buf)
{
int codingState;
- int max = offset + len;
+ int max = buf.Length;
- for (int i = offset; i < max; i++)
+ for (int i = 0; i < max; i++)
{
codingState = codingSM.NextState(buf[i]);
if (codingState == StateMachineModel.ERROR)
@@ -84,16 +85,16 @@ public override ProbingState HandleData(byte[] buf, int offset, int len)
if (codingState == StateMachineModel.START)
{
int charLen = codingSM.CurrentCharLen;
- if (i == offset)
+ if (i == 0)
{
- lastChar[1] = buf[offset];
- contextAnalyser.HandleOneChar(lastChar, 0, charLen);
- distributionAnalyser.HandleOneChar(lastChar, 0, charLen);
+ lastChar[1] = buf[0];
+ contextAnalyser.HandleOneChar(lastChar, charLen);
+ distributionAnalyser.HandleOneChar(lastChar, charLen);
}
else
{
- contextAnalyser.HandleOneChar(buf, i - 1, charLen);
- distributionAnalyser.HandleOneChar(buf, i - 1, charLen);
+ contextAnalyser.HandleOneChar(buf.Slice(i - 1), charLen);
+ distributionAnalyser.HandleOneChar(buf.Slice(i - 1), charLen);
}
}
}
diff --git a/src/Core/Probers/MultiByte/Japanese/SJISProber.cs b/src/Core/Probers/MultiByte/Japanese/SJISProber.cs
index a8ece9e..e1eeeaf 100644
--- a/src/Core/Probers/MultiByte/Japanese/SJISProber.cs
+++ b/src/Core/Probers/MultiByte/Japanese/SJISProber.cs
@@ -36,6 +36,7 @@
*
* ***** END LICENSE BLOCK ***** */
+using System;
using System.Text;
using UtfUnknown.Core.Analyzers.Japanese;
@@ -70,12 +71,12 @@ public override string GetCharsetName()
return CodepageName.SHIFT_JIS;
}
- public override ProbingState HandleData(byte[] buf, int offset, int len)
+ public override ProbingState HandleData(ReadOnlySpan buf)
{
int codingState;
- int max = offset + len;
+ int max = buf.Length;
- for (int i = offset; i < max; i++)
+ for (int i = 0; i < max; i++)
{
codingState = codingSM.NextState(buf[i]);
if (codingState == StateMachineModel.ERROR)
@@ -91,16 +92,16 @@ public override ProbingState HandleData(byte[] buf, int offset, int len)
if (codingState == StateMachineModel.START)
{
int charLen = codingSM.CurrentCharLen;
- if (i == offset)
+ if (i == 0)
{
- lastChar[1] = buf[offset];
- contextAnalyser.HandleOneChar(lastChar, 2 - charLen, charLen);
- distributionAnalyser.HandleOneChar(lastChar, 0, charLen);
+ lastChar[1] = buf[0];
+ contextAnalyser.HandleOneChar(lastChar.AsSpan(2 - charLen), charLen);
+ distributionAnalyser.HandleOneChar(lastChar, charLen);
}
else
{
- contextAnalyser.HandleOneChar(buf, i + 1 - charLen, charLen);
- distributionAnalyser.HandleOneChar(buf, i - 1, charLen);
+ contextAnalyser.HandleOneChar(buf.Slice(i + 1 - charLen), charLen);
+ distributionAnalyser.HandleOneChar(buf.Slice(i - 1), charLen);
}
}
}
@@ -128,4 +129,4 @@ public override float GetConfidence(StringBuilder status = null)
float distribCf = distributionAnalyser.GetConfidence();
return (contxtCf > distribCf ? contxtCf : distribCf);
}
-}
\ No newline at end of file
+}
diff --git a/src/Core/Probers/MultiByte/Korean/CP949Prober.cs b/src/Core/Probers/MultiByte/Korean/CP949Prober.cs
index 307cdee..dae7c4e 100644
--- a/src/Core/Probers/MultiByte/Korean/CP949Prober.cs
+++ b/src/Core/Probers/MultiByte/Korean/CP949Prober.cs
@@ -35,6 +35,7 @@
*
* ***** END LICENSE BLOCK ***** */
+using System;
using System.Text;
using UtfUnknown.Core.Analyzers.Korean;
@@ -63,12 +64,12 @@ public override string GetCharsetName()
return CodepageName.CP949;
}
- public override ProbingState HandleData(byte[] buf, int offset, int len)
+ public override ProbingState HandleData(ReadOnlySpan buf)
{
int codingState;
- int max = offset + len;
+ int max = buf.Length;
- for (int i = offset; i < max; i++)
+ for (int i = 0; i < max; i++)
{
codingState = codingSM.NextState(buf[i]);
if (codingState == StateMachineModel.ERROR)
@@ -86,14 +87,14 @@ public override ProbingState HandleData(byte[] buf, int offset, int len)
if (codingState == StateMachineModel.START)
{
int charLen = codingSM.CurrentCharLen;
- if (i == offset)
+ if (i == 0)
{
- lastChar[1] = buf[offset];
- distributionAnalyser.HandleOneChar(lastChar, 0, charLen);
+ lastChar[1] = buf[0];
+ distributionAnalyser.HandleOneChar(lastChar, charLen);
}
else
{
- distributionAnalyser.HandleOneChar(buf, i - 1, charLen);
+ distributionAnalyser.HandleOneChar(buf.Slice(i - 1), charLen);
}
}
}
diff --git a/src/Core/Probers/MultiByte/Korean/EUCKRProber.cs b/src/Core/Probers/MultiByte/Korean/EUCKRProber.cs
index 87028f8..15941aa 100644
--- a/src/Core/Probers/MultiByte/Korean/EUCKRProber.cs
+++ b/src/Core/Probers/MultiByte/Korean/EUCKRProber.cs
@@ -35,6 +35,7 @@
*
* ***** END LICENSE BLOCK ***** */
+using System;
using System.Text;
using UtfUnknown.Core.Analyzers.Korean;
@@ -61,12 +62,12 @@ public override string GetCharsetName()
return CodepageName.EUC_KR;
}
- public override ProbingState HandleData(byte[] buf, int offset, int len)
+ public override ProbingState HandleData(ReadOnlySpan buf)
{
int codingState;
- int max = offset + len;
+ int max = buf.Length;
- for (int i = offset; i < max; i++)
+ for (int i = 0; i < max; i++)
{
codingState = codingSM.NextState(buf[i]);
if (codingState == StateMachineModel.ERROR)
@@ -84,14 +85,14 @@ public override ProbingState HandleData(byte[] buf, int offset, int len)
if (codingState == StateMachineModel.START)
{
int charLen = codingSM.CurrentCharLen;
- if (i == offset)
+ if (i == 0)
{
- lastChar[1] = buf[offset];
- distributionAnalyser.HandleOneChar(lastChar, 0, charLen);
+ lastChar[1] = buf[0];
+ distributionAnalyser.HandleOneChar(lastChar, charLen);
}
else
{
- distributionAnalyser.HandleOneChar(buf, i - 1, charLen);
+ distributionAnalyser.HandleOneChar(buf.Slice(i - 1), charLen);
}
}
}
diff --git a/src/Core/Probers/MultiByte/UTF8Prober.cs b/src/Core/Probers/MultiByte/UTF8Prober.cs
index 907bdc8..bd42b47 100644
--- a/src/Core/Probers/MultiByte/UTF8Prober.cs
+++ b/src/Core/Probers/MultiByte/UTF8Prober.cs
@@ -36,6 +36,7 @@
*
* ***** END LICENSE BLOCK ***** */
+using System;
using System.Text;
using UtfUnknown.Core.Models;
@@ -68,11 +69,11 @@ public override void Reset()
state = ProbingState.Detecting;
}
- public override ProbingState HandleData(byte[] buf, int offset, int len)
+ public override ProbingState HandleData(ReadOnlySpan buf)
{
- int max = offset + len;
+ int max = buf.Length;
- for (int i = offset; i < max; i++)
+ for (int i = 0; i < max; i++)
{
var codingState = codingSM.NextState(buf[i]);
diff --git a/src/Core/Probers/SBCSGroupProber.cs b/src/Core/Probers/SBCSGroupProber.cs
index 0284860..83c3770 100644
--- a/src/Core/Probers/SBCSGroupProber.cs
+++ b/src/Core/Probers/SBCSGroupProber.cs
@@ -36,6 +36,7 @@
*
* ***** END LICENSE BLOCK ***** */
+using System;
using System.Text;
#region using languages
@@ -253,7 +254,7 @@ public SBCSGroupProber()
Reset();
}
- public override ProbingState HandleData(byte[] buf, int offset, int len)
+ public override ProbingState HandleData(ReadOnlySpan buf)
{
// apply filter to original buffer, and we got new buffer back
// depend on what script it is, we will feed them the new buffer
@@ -262,7 +263,7 @@ public override ProbingState HandleData(byte[] buf, int offset, int len)
// of each prober since as of now, there are no probers here which
// recognize languages with English characters.
- byte[] newBuf = FilterWithoutEnglishLetters(buf, offset, len);
+ byte[] newBuf = FilterWithoutEnglishLetters(buf);
if (newBuf.Length == 0)
return state; // Nothing to see here, move on.
@@ -271,7 +272,7 @@ public override ProbingState HandleData(byte[] buf, int offset, int len)
{
if (isActive[i])
{
- ProbingState st = probers[i].HandleData(newBuf, 0, newBuf.Length);
+ ProbingState st = probers[i].HandleData(newBuf);
if (st == ProbingState.FoundIt)
{
diff --git a/src/Core/Probers/SingleByteCharSetProber.cs b/src/Core/Probers/SingleByteCharSetProber.cs
index 5b74d42..af9a08a 100644
--- a/src/Core/Probers/SingleByteCharSetProber.cs
+++ b/src/Core/Probers/SingleByteCharSetProber.cs
@@ -90,11 +90,11 @@ public SingleByteCharSetProber(SequenceModel model, bool reversed,
Reset();
}
- public override ProbingState HandleData(byte[] buf, int offset, int len)
+ public override ProbingState HandleData(ReadOnlySpan buf)
{
- int max = offset + len;
+ int max = buf.Length;
- for (int i = offset; i < max; i++)
+ for (int i = 0; i < max; i++)
{
byte order = model.GetOrder(buf[i]);
diff --git a/src/UTF-unknown.csproj b/src/UTF-unknown.csproj
index 1b0bece..07b3510 100644
--- a/src/UTF-unknown.csproj
+++ b/src/UTF-unknown.csproj
@@ -18,6 +18,7 @@
+
diff --git a/tests/CharsetDetectorTest.cs b/tests/CharsetDetectorTest.cs
index 43006f1..64152dc 100644
--- a/tests/CharsetDetectorTest.cs
+++ b/tests/CharsetDetectorTest.cs
@@ -4,6 +4,7 @@
// Rudi Pettazzi
//
+using System;
using System.IO;
using System.Text;
using System.Threading.Tasks;
@@ -193,6 +194,129 @@ public void TestBomUtf8()
Assert.That(result.Detected.HasBOM, Is.True);
}
+ [Test]
+ public void DetectFromReadOnlySpan()
+ {
+ ReadOnlySpan buf = new byte[] { 0xEF, 0xBB, 0xBF, 0x68, 0x65, 0x6C, 0x6C, 0x6F, 0x21 };
+ var result = CharsetDetector.DetectFromBytes(buf);
+ Assert.That(result.Detected.EncodingName, Is.EqualTo(CodepageName.UTF8));
+ Assert.That(result.Detected.Confidence, Is.EqualTo(1.0f));
+ Assert.That(result.Detected.HasBOM, Is.True);
+ }
+
+ [Test]
+ public void DetectFromReadOnlySpan_Empty()
+ {
+ var result = CharsetDetector.DetectFromBytes(ReadOnlySpan.Empty);
+ Assert.That(result.Detected, Is.Null);
+ }
+
+ [Test]
+ public void DetectFromReadOnlySpan_ASCII()
+ {
+ ReadOnlySpan buf = Encoding.ASCII.GetBytes("Hello, World!");
+ var result = CharsetDetector.DetectFromBytes(buf);
+ Assert.That(result.Detected.EncodingName, Is.EqualTo(CodepageName.ASCII));
+ Assert.That(result.Detected.Confidence, Is.EqualTo(1.0f));
+ Assert.That(result.Detected.HasBOM, Is.False);
+ }
+
+ [Test]
+ public void DetectFromReadOnlySpan_UTF8()
+ {
+ string s = "ウィキペディアはオープンコンテントの百科事典です。";
+ ReadOnlySpan buf = Encoding.UTF8.GetBytes(s);
+ var result = CharsetDetector.DetectFromBytes(buf);
+ Assert.That(result.Detected.EncodingName, Is.EqualTo(CodepageName.UTF8));
+ Assert.That(result.Detected.Confidence, Is.EqualTo(1.0f));
+ Assert.That(result.Detected.HasBOM, Is.False);
+ }
+
+ [Test]
+ [TestCase(0, 10, CodepageName.ASCII)]
+ [TestCase(0, 100, CodepageName.UTF8)]
+ [TestCase(10, 100, CodepageName.UTF8)]
+ public void DetectFromReadOnlySpan_Sliced(int offset, int len, string detectedCodepage)
+ {
+ // Arrange - demonstrates the key use case: detecting encoding from a slice without copying
+ string s = "UTF-Unknown은 파일, 스트림, 그 외 바이트 배열의 캐릭터 셋을 탐지하는 라이브러리입니다." +
+ "대한민국 (大韓民國, Republic of Korea)";
+ byte[] bytes = Encoding.UTF8.GetBytes(s);
+
+ // Act
+ var result = CharsetDetector.DetectFromBytes(bytes.AsSpan(offset, len));
+
+ // Assert
+ Assert.That(result.Detected.EncodingName, Is.EqualTo(detectedCodepage));
+ Assert.That(result.Detected.Confidence, Is.EqualTo(1.0f));
+ Assert.That(result.Detected.HasBOM, Is.False);
+ }
+
+ [Test]
+ [TestCase(new byte[] { 0x2B, 0x2F, 0x76, 0x38 })]
+ [TestCase(new byte[] { 0x2B, 0x2F, 0x76, 0x39 })]
+ [TestCase(new byte[] { 0x2B, 0x2F, 0x76, 0x2B })]
+ [TestCase(new byte[] { 0x2B, 0x2F, 0x76, 0x2F })]
+ [TestCase(new byte[] { 0x2B, 0x2F, 0x76, 0x38, 0x2D })]
+ public void DetectFromReadOnlySpan_BomUtf7(byte[] bufferBytes)
+ {
+ ReadOnlySpan buf = bufferBytes;
+ var result = CharsetDetector.DetectFromBytes(buf).Detected;
+ Assert.That(result.EncodingName, Is.EqualTo(CodepageName.UTF7));
+ Assert.That(result.Confidence, Is.EqualTo(1.0f));
+ Assert.That(result.HasBOM, Is.True);
+ }
+
+ [Test]
+ public void DetectFromReadOnlySpan_BomGb18030()
+ {
+ ReadOnlySpan buf = new byte[] { 0x84, 0x31, 0x95, 0x33 };
+ var result = CharsetDetector.DetectFromBytes(buf).Detected;
+ Assert.That(result.EncodingName, Is.EqualTo(CodepageName.GB18030));
+ Assert.That(result.Confidence, Is.EqualTo(1.0f));
+ Assert.That(result.HasBOM, Is.True);
+ }
+
+ [Test]
+ public void DetectFromReadOnlySpan_BomUTF16_BE()
+ {
+ ReadOnlySpan buf = new byte[] { 0xFE, 0xFF, 0x00, 0x68, 0x00, 0x65 };
+ var result = CharsetDetector.DetectFromBytes(buf);
+ Assert.That(result.Detected.EncodingName, Is.EqualTo(CodepageName.UTF16_BE));
+ Assert.That(result.Detected.Confidence, Is.EqualTo(1.0f));
+ Assert.That(result.Detected.HasBOM, Is.True);
+ }
+
+ [Test]
+ public void DetectFromReadOnlySpan_BomUTF16_LE()
+ {
+ ReadOnlySpan buf = new byte[] { 0xFF, 0xFE, 0x68, 0x00, 0x65, 0x00 };
+ var result = CharsetDetector.DetectFromBytes(buf);
+ Assert.That(result.Detected.EncodingName, Is.EqualTo(CodepageName.UTF16_LE));
+ Assert.That(result.Detected.Confidence, Is.EqualTo(1.0f));
+ Assert.That(result.Detected.HasBOM, Is.True);
+ }
+
+ [Test]
+ public void DetectFromReadOnlySpan_BomUTF32_BE()
+ {
+ ReadOnlySpan buf = new byte[] { 0x00, 0x00, 0xFE, 0xFF, 0x00, 0x00, 0x00, 0x68 };
+ var result = CharsetDetector.DetectFromBytes(buf);
+ Assert.That(result.Detected.EncodingName, Is.EqualTo(CodepageName.UTF32_BE));
+ Assert.That(result.Detected.Confidence, Is.EqualTo(1.0f));
+ Assert.That(result.Detected.HasBOM, Is.True);
+ }
+
+ [Test]
+ public void DetectFromReadOnlySpan_BomUTF32_LE()
+ {
+ ReadOnlySpan buf = new byte[] { 0xFF, 0xFE, 0x00, 0x00, 0x68, 0x00, 0x00, 0x00 };
+ var result = CharsetDetector.DetectFromBytes(buf);
+ Assert.That(result.Detected.EncodingName, Is.EqualTo(CodepageName.UTF32_LE));
+ Assert.That(result.Detected.Confidence, Is.EqualTo(1.0f));
+ Assert.That(result.Detected.HasBOM, Is.True);
+ }
+
[Test]
public void Test2byteArrayBomUTF16_BE()
{