How Can I Determine a String's Encoding in C#?

Determining a String's Encoding in C#

Is there a way to determine a string's encoding in C#? For instance, if you have a filename string but are unsure whether it is encoded in UTF-16 or the default encoding of the system, this guide demonstrates how to determine its encoding.

Solution:

The code below has the following capabilities:

• Detects UTF-7, UTF-8/16/32 (bom, no bom, little & big endian)
• Falls back to the local default codepage if no Unicode encoding is found
• Detects (with high probability) unicode files with the BOM/signature missing
• Searches for charset=xyz and encoding=xyz inside file to help determine encoding
• Customizable 'taster' parameter to control the number of bytes to check
• Returns the encoding and decoded text file

public Encoding detectTextEncoding(string filename, out String text, int taster = 1000)
{
    byte[] b = File.ReadAllBytes(filename);

    // Check for BOM/signature
    if (b.Length >= 4 && b[0] == 0x00 && b[1] == 0x00 && b[2] == 0xFE && b[3] == 0xFF)
    {
        text = Encoding.GetEncoding("utf-32BE").GetString(b, 4, b.Length - 4);
        return Encoding.GetEncoding("utf-32BE");
    }
    else if (b.Length >= 4 && b[0] == 0xFF && b[1] == 0xFE && b[2] == 0x00 && b[3] == 0x00)
    {
        text = Encoding.UTF32.GetString(b, 4, b.Length - 4); 
        return Encoding.UTF32;
    }
    else if (b.Length >= 2 && b[0] == 0xFE && b[1] == 0xFF)
    {
        text = Encoding.BigEndianUnicode.GetString(b, 2, b.Length - 2);
        return Encoding.BigEndianUnicode;
    }
    else if (b.Length >= 2 && b[0] == 0xFF && b[1] == 0xFE)
    {
        text = Encoding.Unicode.GetString(b, 2, b.Length - 2); 
        return Encoding.Unicode;
    }
    else if (b.Length >= 3 && b[0] == 0xEF && b[1] == 0xBB && b[2] == 0xBF)
    {
        text = Encoding.UTF8.GetString(b, 3, b.Length - 3); 
        return Encoding.UTF8;
    }
    else if (b.Length >= 3 && b[0] == 0x2b && b[1] == 0x2f && b[2] == 0x76)
    {
        text = Encoding.UTF7.GetString(b, 3, b.Length - 3); 
        return Encoding.UTF7;
    }

    // Check for UTF8 without a BOM/signature
    bool utf8 = false;
    int i = 0;
    while (i < taster - 4)
    {
        if (b[i] <= 0x7F) { i += 1; continue; }
        if (b[i] >= 0xC2 && b[i] < 0xE0 && b[i + 1] >= 0x80 && b[i + 1] < 0xC0) { i += 2; utf8 = true; continue; }
        if (b[i] >= 0xE0 && b[i] < 0xF0 && b[i + 1] >= 0x80 && b[i + 1] < 0xC0 && b[i + 2] >= 0x80 && b[i + 2] < 0xC0) { i += 3; utf8 = true; continue; }
        if (b[i] >= 0xF0 && b[i] < 0xF5 && b[i + 1] >= 0x80 && b[i + 1] < 0xC0 && b[i + 2] >= 0x80 && b[i + 2] < 0xC0 && b[i + 3] >= 0x80 && b[i + 3] < 0xC0) { i += 4; utf8 = true; continue; }
        utf8 = false; break;
    }
    if (utf8 == true)
    {
        text = Encoding.UTF8.GetString(b);
        return Encoding.UTF8;
    }

    // Check for UTF-16 without BOM/signature
    double threshold = 0.1;
    int count = 0;
    for (int n = 0; n < taster; n += 2) if (b[n] == 0) count++;
    if (((double)count) / taster > threshold)
    {
        text = Encoding.BigEndianUnicode.GetString(b);
        return Encoding.BigEndianUnicode; 
    }
    count = 0;
    for (int n = 1; n < taster; n += 2) if (b[n] == 0) count++;
    if (((double)count) / taster > threshold)
    {
        text = Encoding.Unicode.GetString(b);
        return Encoding.Unicode; 
    }

    // Check for "charset=xyz" or "encoding=xyz"
    for (int n = 0; n < taster - 9; n++)
    {
        if ((b[n + 0] == 'c' || b[n + 0] == 'C') && (b[n + 1] == 'h' || b[n + 1] == 'H') && (b[n + 2] == 'a' || b[n + 2] == 'A') && (b[n + 3] == 'r' || b[n + 3] == 'R') && (b[n + 4] == 's' || b[n + 4] == 'S') && (b[n + 5] == 'e' || b[n + 5] == 'E') && (b[n + 6] == 't' || b[n + 6] == 'T') && (b[n + 7] == '='))
        {
            n += 8;
            if (b[n] == '&quot;' || b[n] == '\'') n++;
            int oldn = n;
            while (n < taster &amp;&amp; (b[n] == '_' || b[n] == '-' || (b[n] >= '0' && b[n] <= '9') || (b[n] >= 'a' && b[n] <= 'z') || (b[n] >= 'A' && b[n] <= 'Z'))) n++;
            byte[] nb = new byte[n - oldn];
            Array.Copy(b, oldn, nb, 0, n - oldn);
            try
            {
                string internalEnc = Encoding.ASCII.GetString(nb);
                text = Encoding.GetEncoding(internalEnc).GetString(b);
                return Encoding.GetEncoding(internalEnc);
            }
            catch { break; }
        }
    }

    // Fallback to local default codepage
    text = Encoding.Default.GetString(b);
    return Encoding.Default;
}

The above is the detailed content of How Can I Determine a String's Encoding in C#?. For more information, please follow other related articles on the PHP Chinese website!