golang中的编解码操作

最近在写一个用go来做的爬虫程序，需要对抓取到的网站进行解码，然后对符合要求的网站内容保存到硬盘中。这就需要能够识别不同的编码方式，然后统一编解码为utf-8再保存。期间查了一下go对不同编码方式的编解码操作，记录一下以方便大家的使用

基本知识

我们知道，go是用的utf-8编码，utf-8是一种变长的编码，也就是说，对于不同的字符，占据的内存空间是不同的。采用utf-8编码的单个字符，具体占用的内存空间从1字节到4字节之间。比如：

package main

import (
   "fmt"
)

func main() {
   str := "欧足联sb"
   for i := 0; i < len(str); i+=1 {
      fmt.Printf("(%v, %c)   ", str[i], str[i])
   }
}

// (230, æ)   (172, ¬)   (167, §)   (232, è)   (182, ¶)   (179, ³)   (232, è)   (129, )   (148, )   (115, s)   (98, b)

可以发现，"欧足联"这三个简体汉字共占用了9个字符，而"sb"这两个字母仅占用了2个字符。这就是所谓的变长编码。utf-8是当今最常用的编码方式，但是并不是唯一一种编码方式，对于非utf-8编码的内容如gbk，如果不对gbk编码的内容解码为utf-8，则这部分内容不可读。

编解码方式

对非utf-8内容解码，或者对utf-8编码的内容用其它方式编码，在go中可以用两个库搞定：

"golang.org/x/text/transform"
"golang.org/x/net/html/charset"

具体的操作方式，直接上代码：

package main

import (
   "fmt"
   "golang.org/x/text/transform"
   "golang.org/x/net/html/charset"
   "io/ioutil"
   "os"
   "strings"
)

type testCase struct {
   utf8, other, otherEncoding string
}

// testCases for encoding and decoding.
var testCases = []testCase{
   {"Résumé", "Résumé", "utf8"},
   {"Résumé", "R\xe9sum\xe9", "latin1"},
   {"これは漢字です。", "S0\x8c0o0"oW[g0Y0\x020", "UTF-16LE"},
   {"これは漢字です。", "0S0\x8c0oo"[W0g0Y0\x02", "UTF-16BE"},
   {"Hello, world", "Hello, world", "ASCII"},
   {"Gdańsk", "Gda\xf1sk", "ISO-8859-2"},
   {"Ââ Čč Đđ Ŋŋ Õõ Šš Žž Åå Ää", "\xc2\xe2 \xc8\xe8 \xa9\xb9 \xaf\xbf \xd5\xf5 \xaa\xba \xac\xbc \xc5\xe5 \xc4\xe4", "ISO-8859-10"},
   {"สำหรับ", "\xca\xd3\xcb\xc3\u047a", "ISO-8859-11"},
   {"latviešu", "latvie\xf0u", "ISO-8859-13"},
   {"Seònaid", "Se\xf2naid", "ISO-8859-14"},
   {"€1 is cheap", "\xa41 is cheap", "ISO-8859-15"},
   {"românește", "rom\xe2ne\xbate", "ISO-8859-16"},
   {"nutraĵo", "nutra\xbco", "ISO-8859-3"},
   {"Kalâdlit", "Kal\xe2dlit", "ISO-8859-4"},
   {"русский", "\xe0\xe3\xe1\xe1\xda\xd8\xd9", "ISO-8859-5"},
   {"ελληνικά", "\xe5\xeb\xeb\xe7\xed\xe9\xea\xdc", "ISO-8859-7"},
   {"Kağan", "Ka\xf0an", "ISO-8859-9"},
   {"Résumé", "R\x8esum\x8e", "macintosh"},
   {"Gdańsk", "Gda\xf1sk", "windows-1250"},
   {"русский", "\xf0\xf3\xf1\xf1\xea\xe8\xe9", "windows-1251"},
   {"Résumé", "R\xe9sum\xe9", "windows-1252"},
   {"ελληνικά", "\xe5\xeb\xeb\xe7\xed\xe9\xea\xdc", "windows-1253"},
   {"Kağan", "Ka\xf0an", "windows-1254"},
   {"עִבְרִית", "\xf2\xc4\xe1\xc0\xf8\xc4\xe9\xfa", "windows-1255"},
   {"العربية", "\xc7\xe1\xda\xd1\xc8\xed\xc9", "windows-1256"},
   {"latviešu", "latvie\xf0u", "windows-1257"},
   {"Việt", "Vi\xea\xf2t", "windows-1258"},
   {"สำหรับ", "\xca\xd3\xcb\xc3\u047a", "windows-874"},
   {"русский", "\xd2\xd5\xd3\xd3\xcb\xc9\xca", "KOI8-R"},
   {"українська", "\xd5\xcb\xd2\xc1\xa7\xce\xd3\xd8\xcb\xc1", "KOI8-U"},
   {"Hello 常用國字標準字體表", "Hello \xb1`\xa5\u03b0\xea\xa6r\xbc\u0437\u01e6r\xc5\xe9\xaa\xed", "big5"},
   {"Hello 常用國字標準字體表", "Hello \xb3\xa3\xd3\xc3\x87\xf8\xd7\xd6\x98\xcb\x9c\xca\xd7\xd6\xf3\x77\xb1\xed", "gbk"},
   {"Hello 常用國字標準字體表", "Hello \xb3\xa3\xd3\xc3\x87\xf8\xd7\xd6\x98\xcb\x9c\xca\xd7\xd6\xf3\x77\xb1\xed", "gb18030"},
   {"עִבְרִית", "\x81\x30\xfb\x30\x81\x30\xf6\x34\x81\x30\xf9\x33\x81\x30\xf6\x30\x81\x30\xfb\x36\x81\x30\xf6\x34\x81\x30\xfa\x31\x81\x30\xfb\x38", "gb18030"},
   {"㧯", "\x82\x31\x89\x38", "gb18030"},
   {"これは漢字です。", "\x82\xb1\x82\xea\x82\xcd\x8a\xbf\x8e\x9a\x82\xc5\x82\xb7\x81B", "SJIS"},
   {"Hello, 世界!", "Hello, \x90\xa2\x8aE!", "SJIS"},
   {"ｲｳｴｵｶ", "\xb2\xb3\xb4\xb5\xb6", "SJIS"},
   {"これは漢字です。", "\xa4\xb3\xa4\xec\xa4\u03f4\xc1\xbb\xfa\xa4\u01e4\xb9\xa1\xa3", "EUC-JP"},
   {"Hello, 世界!", "Hello, \x1b$B@$3&\x1b(B!", "ISO-2022-JP"},
   {"다음과 같은 조건을 따라야 합니다: 저작자표시", "\xb4\xd9\xc0\xbd\xb0\xfa \xb0\xb0\xc0\xba \xc1\xb6\xb0\xc7\xc0\xbb \xb5\xfb\xb6\xf3\xbe\xdf \xc7մϴ\xd9: \xc0\xfa\xc0\xdb\xc0\xdaǥ\xbd\xc3", "EUC-KR"},
}

func transformString(t transform.Transformer, s string) (string, error) {
   r := transform.NewReader(strings.NewReader(s), t)
   b, err := ioutil.ReadAll(r)
   return string(b), err
}

func Decode(otherEncodeStr, encodeType string) (string, error) {
   e, _ := charset.Lookup(encodeType)
   if e == nil {
      return "", fmt.Errorf("%s: not found", encodeType)
   }
   decodeStr, err := transformString(e.NewDecoder(), otherEncodeStr)
   if err != nil {
      return "", err
   }
   return decodeStr, nil
}

func Encode(utf8EncodeStr, encodeType string) (string, error) {
   e, _ := charset.Lookup(encodeType)
   if e == nil {
      return "", fmt.Errorf("%s: not found", encodeType)
   }
   encodeStr, err := transformString(e.NewEncoder(), utf8EncodeStr)
   if err != nil {
      return "", err
   }
   return encodeStr, nil
}

func main() {
   for _, tc := range testCases {
      decodeStr, err := Decode(tc.other, tc.otherEncoding)
      if err != nil {
         fmt.Fprintln(os.Stderr, err)
         continue
      }
      if decodeStr != tc.utf8 {
         fmt.Fprintf(os.Stderr, "%v decode error\n", tc.other)
      }
   }

   fmt.Println("==========================================")

   for _, tc := range testCases {
      encodeStr, err := Encode(tc.utf8, tc.otherEncoding)
      if err != nil {
         fmt.Fprintln(os.Stderr, err)
         continue
      }
      if encodeStr != tc.other {
         fmt.Fprintf(os.Stderr, "%v encode error\n", tc.utf8)
      }
   }
}

在testCase中，utf8为用utf-8编码的可读字符串；other为其它编码的字符强行用utf-8解码后的字符，基本不可读。otherEncoding就是对应的编码方式。在进行编解码之前，需要调用charset.Lookup()来获取对应的编解码器，然后利用transform模块进行编解码操作。

本文由创作，采用知识共享署名4.0 国际许可协议进行许可。本站文章除注明转载/出处外，均为本站原创或翻译，转载前请务必署名。最后编辑时间为: 2020/08/09 03:52