node.js - nodejs 用superagent抓取网页gbk编码乱码的问题

Question

网上找了太多了。什么比如iconv-lite什么的，都试过了。就是不行。

href = 'http://www.qq.com/';
            console.log(href);
            superagent.get(href).end(function (err, res) {
                var str = res.text;
                var buf = new Buffer(str);
                str = iconv.decode(buf, 'GBK');
                console.log(str);


            });  
        
        折腾了一夜了，可有解决方案？
        
        
        
        

Answer 1

http://web-engineer.cn/article/29

Answer 2

var charset = require('superagent-charset');
var superagent = charset(require('superagent'));

var href = 'http://www.qq.com/';
superagent.get(href).charset('gbk').end(function (err, res) {
  console.log(res.text);
});

用 superagent-charset 模块

---

刚刚抽时间改了下这个模块，https://github.com/52cik/superagent-charset
不知道作者什么时候合并我的代码，
你可以直接 npm i 52cik/superagent-charset 来安装使用。

使用方法：

var charset = require('superagent-charset');
var superagent = charset(require('superagent'));

var href = 'http://www.qq.com/';
superagent
  .get(href)
  .charset() // 不写会自动检测编码
  .end(function (err, res) {
    console.log(res.text);
  });

Answer 3

不是superagent，使用nodejs提供的http模块
解码后最终的文件内容是UTF-8的

new Buffer(string[,encoding])本身就会有一个转码的过程，默认为UTF-8，也就是认为
如下,
先编码，然后toString后,再new Buffer，最后用GBK解码，得到的结果就有问题啦

var buffer2=iconv.encode("不是superagent，使用nodejs提供的http模块","GBK");
var str=iconv.decode(new Buffer(buffer2.toString()), "GBK");
console.log(str);

贴个使用原生的例子:

var http=require('http');
var iconv = require('iconv-lite');
var zlib=require('zlib');

var clientRequest = http.request('http://www.qq.com', function(response) {
    var dataReceived=false,responseBuffer, responseLength, responseLengthReceived = 0;

    responseLength = parseInt(response.headers["content-length"], 10);
    responseLength = !isNaN(responseLength) ? responseLength : 0;

    responseBuffer = new Buffer(responseLength);

    response.on("data", receiveData);
    response.on("end", processReceivedData);

    function receiveData(chunk){
        if (!chunk.length || dataReceived) {
            return;
        }

        if (responseLengthReceived + chunk.length > responseBuffer.length) {
            if (responseLengthReceived + chunk.length <= 1024 * 1024 * 16) {
                var tmpNewBuffer = new Buffer(responseLengthReceived + chunk.length);
                responseBuffer.copy(tmpNewBuffer, 0, 0, responseBuffer.length);
                chunk.copy(tmpNewBuffer, responseBuffer.length, 0, chunk.length);
                responseBuffer = tmpNewBuffer;
            }
        } else {
            chunk.copy(responseBuffer, responseLengthReceived, 0, chunk.length);
        }

        responseLengthReceived += chunk.length;
    }

    function processReceivedData() {
        responseBuffer = responseBuffer.slice(0, responseLengthReceived);
        var decodeAndReturnResponse = function (error, responseBuffer) {
            var responseBody =iconv.decode(responseBuffer, "GBK");
            console.log(responseBody);
        };
        
        if (contentEncoding && /(gzip|deflate)/.test(contentEncoding)) {
            console.log("zlib.unzip");
            zlib.unzip(responseBuffer, decodeAndReturnResponse);
        } else {
            decodeAndReturnResponse(undefined, responseBuffer);
        }
    }
});

clientRequest.end();