前言

最近在学习go，学习一门语言最好的方式就是实践，之前学习python也是从爬虫入手，现在使用go语言写一个网易云音乐的爬虫，下面会简单介绍开发的过程，代码是初学者的水平，欢迎吐槽。

本项目github地址https://github.com/zhujiajunup/yunyinyue

开发工具

• go1.11.2 windows/amd64
• Google Chrome 71.0.3578.98
• Fiddler v5.0.20182.28034

获取数据

不管用什么语言写爬虫，但步骤总是一致的，只是实现使用不应的语言而已，第一步当然是确认你想要什么，本次的目标是网易云音乐，我是想获取用户首页的听歌排行榜。

最先需要弄明白的是这些数据是怎么获取的，即云音乐是如何向服务器请求数据的，打开chrome的调试工具（F12）,点到Network，搜索“钟无艳”

可以看到数据是https://music.163.com/weapi/v1/play/record?csrf_token=的POST请求来获取的，再看看该请求发送了什么数据

可以看到提交了一个表单，参数为params和encSecKey。发送的数据应该是加了密的，下一步就需要知道云音乐是如何进行加密传输的。

从调试窗口可以看到，该请求是由https://s3.music.126.net/web/s/core_86994123ce247287ad52aafce6acdf9b.js?86994123ce247287ad52aafce6acdf9b发出的，加密逻辑应该就是在该js中处理的，将该js保存到本地并格式化后，并搜索encSecKey在哪里赋值的

v9m.bl9c = function(Y9P, e8e) { var i8a = {}, e8e = NEJ.X({}, e8e), mp3x = Y9P.indexOf("?"); if (window.GEnc && /(^|\.com)\/api/.test(Y9P) && !(e8e.headers && e8e.headers[eq1x.Bx8p] == eq1x.Iy0x) && !e8e.noEnc) { if (mp3x != -1) { i8a = k8c.hc2x(Y9P.substring(mp3x + 1)); Y9P = Y9P.substring(0, mp3x) } if (e8e.query) { i8a = NEJ.X(i8a, k8c.fQ1x(e8e.query) ? k8c.hc2x(e8e.query) : e8e.query) } if (e8e.data) { i8a = NEJ.X(i8a, k8c.fQ1x(e8e.data) ? k8c.hc2x(e8e.data) : e8e.data) } i8a["csrf_token"] = v9m.gO2x("__csrf"); Y9P = Y9P.replace("api", "weapi"); e8e.method = "post"; delete e8e.query; var bUK2x = window.asrsea(JSON.stringify(i8a), brA4E(["流泪", "强"]), brA4E(WU5Z.md), brA4E(["爱心", "女孩", "惊恐", "大笑"])); e8e.data = k8c.cz9q({ params: bUK2x.encText, encSecKey: bUK2x.encSecKey }) } cwC9t(Y9P, e8e) };

可以看到是通过window.asrsea函数来获取的，接下来看window.asrsea是如何定义的

function() { function a(a) { var d, e, b = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789", c = ""; for (d = 0; a > d; d += 1) e = Math.random() * b.length, e = Math.floor(e), c += b.charAt(e); return c } function b(a, b) { var c = CryptoJS.enc.Utf8.parse(b), d = CryptoJS.enc.Utf8.parse("0102030405060708"), e = CryptoJS.enc.Utf8.parse(a), f = CryptoJS.AES.encrypt(e, c, { iv: d, mode: CryptoJS.mode.CBC }); return f.toString() } function c(a, b, c) { var d, e; return setMaxDigits(131), d = new RSAKeyPair(b, "", c), e = encryptedString(d, a) } function d(d, e, f, g) { var h = {}, i = a(16); return h.encText = b(d, g), h.encText = b(h.encText, i), h.encSecKey = c(i, e, f), h } function e(a, b, d, e) { var f = {}; return f.encText = c(a + e, b, d), f } window.asrsea = d, window.ecnonasr = e } ();

加密算法就是这段代码，函数接收四个参数，进行了aes和ras加密，具体逻辑这里不进行详解，知道了处理逻辑，现在得获取这四个参数分别是什么，接下来使用Fiddler来将js替换为本地js,将参数打印出来即可。

Fiddler调试获取参数

• 配置代理
  
• 配置Https

• 修改core.js
  将https://s3.music.126.net/web/s/core_86994123ce247287ad52aafce6acdf9b.js?86994123ce247287ad52aafce6acdf9b保存到本地，比如命名为core.js，编辑器打开编辑，在指定位置添加如下打印信息

v9m.bl9c = function(Y9P, e8e) { var i8a = {}, e8e = NEJ.X({}, e8e), mp3x = Y9P.indexOf("?"); if (window.GEnc && /(^|\.com)\/api/.test(Y9P) && !(e8e.headers && e8e.headers[eq1x.Bx8p] == eq1x.Iy0x) && !e8e.noEnc) { if (mp3x != -1) { i8a = k8c.hc2x(Y9P.substring(mp3x + 1)); Y9P = Y9P.substring(0, mp3x) } if (e8e.query) { i8a = NEJ.X(i8a, k8c.fQ1x(e8e.query) ? k8c.hc2x(e8e.query) : e8e.query) } if (e8e.data) { i8a = NEJ.X(i8a, k8c.fQ1x(e8e.data) ? k8c.hc2x(e8e.data) : e8e.data) } i8a["csrf_token"] = v9m.gO2x("__csrf"); Y9P = Y9P.replace("api", "weapi"); e8e.method = "post"; delete e8e.query; var bUK2x = window.asrsea(JSON.stringify(i8a), brA4E(["流泪", "强"]), brA4E(WU5Z.md), brA4E(["爱心", "女孩", "惊恐", "大笑"])); window.console.info(Y9P); window.console.info(JSON.stringify(i8a)); window.console.info(JSON.stringify( brA4E(["流泪", "强"]))); window.console.info(JSON.stringify(brA4E(WU5Z.md))); window.console.info(JSON.stringify(brA4E(["爱心", "女孩", "惊恐", "大笑"]))); e8e.data = k8c.cz9q({ params: bUK2x.encText, encSecKey: bUK2x.encSecKey }) } cwC9t(Y9P, e8e) };

• 配置Fiddler Rule
  

一切准备就绪后，再打开浏览器，输入https://music.163.com/#/user/songs/rank?id=62947535，打开调试窗口得Console，就可以看到本地js中添加的输出日志了

不难发现除了第一个参数外，其他三个参数都是固定的，因此在后面的处理中，只需要处理第一个参数即可。

第一个参数就是加密前的请求参数

{ "uid": "62947535", "type": "-1", "limit": "1000", "offset": "0", "total": "true", "csrf_token": "" }

那么已经弄清楚了数据获取的逻辑，接下来就是按照这个逻辑用go语言实现一遍了，最关键的就是加密算法了

go实现

项目结构

加密算法

来自https://studygolang.com/topics/5815

/* Package encrypt provides encrypt algorithm such as rsa & aes */ package encrypt import ( "bytes" "crypto/aes" "crypto/cipher" "encoding/base64" "fmt" "math/big" "math/rand" "time" ) // generate string for given size func RandomStr(size int) (result []byte) { s := "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" strBytes := []byte(s) r := rand.New(rand.NewSource(time.Now().UnixNano())) for i := 0; i < size; i++ { result = append(result, strBytes[r.Intn(len(strBytes))]) } return } func AesEncrypt(sSrc string, sKey string, aseKey string) (string, error) { iv := []byte(aseKey) block, err := aes.NewCipher([]byte(sKey)) if err != nil { return "", err } padding := block.BlockSize() - len([]byte(sSrc))%block.BlockSize() src := append([]byte(sSrc), bytes.Repeat([]byte{byte(padding)}, padding)...) model := cipher.NewCBCEncrypter(block, iv) cipherText := make([]byte, len(src)) model.CryptBlocks(cipherText, src) return base64.StdEncoding.EncodeToString(cipherText), nil } func RsaEncrypt(key string, pubKey string, modulus string) string { rKey := "" for i := len(key) - 1; i >= 0; i-- { // reserve key rKey += key[i : i+1] } hexRKey := "" for _, char := range []rune(rKey) { hexRKey += fmt.Sprintf("%x", int(char)) } bigRKey, _ := big.NewInt(0).SetString(hexRKey, 16) bigPubKey, _ := big.NewInt(0).SetString(pubKey, 16) bigModulus, _ := big.NewInt(0).SetString(modulus, 16) bigRs := bigRKey.Exp(bigRKey, bigPubKey, bigModulus) hexRs := fmt.Sprintf("%x", bigRs) return addPadding(hexRs, modulus) } func addPadding(encText string, modulus string) string { ml := len(modulus) for i := 0; ml > 0 && modulus[i:i+1] == "0"; i++ { ml-- } num := ml - len(encText) prefix := "" for i := 0; i < num; i++ { prefix += "0" } return prefix + encText }

• Music163Spider

type Music163Spider struct { // send request client *http.Client // request's header headers map[string]string } func NewMusic164Spider() (spider Music163Spider) { headers := make(map[string]string) headers["Accept"] = "ext/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8" // empty here headers["Accept-Encoding"] = "" headers["Content-Type"] = "application/x-www-form-urlencoded" headers["User-Agent"] = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36" headers["Host"] = constants.Music163Host headers["Cache-Control"] = "no-cache" headers["Connection"] = "keep-alive" headers["Pragma"] = "no-cache" headers["Origin"] = fmt.Sprintf("%s%s", constants.HttpsPrefix, constants.Music163Host) headers["Accept"] = "ext/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8" return Music163Spider{ client: &http.Client{}, headers: headers, } }

加密函数，实现window.asrsea的加密功能

func (spider Music163Spider) dataEncrypt(dataBytes []byte) (content map[string]string) { content = make(map[string]string) randomBytes := encrypt.RandomStr(16) params, err := encrypt.AesEncrypt(string(dataBytes), constants.SrcretKey, constants.AseKey) if err != nil { fmt.Println(err) } params, err = encrypt.AesEncrypt(params, string(randomBytes), constants.AseKey) if err != nil { fmt.Println(err) } encSecKey := encrypt.RsaEncrypt(string(randomBytes), constants.PubKey, constants.Modulus) if err != nil { fmt.Println(err) } content["params"] = string(params) content["encSecKey"] = string(encSecKey) return content }

定义了发送post请求的方法

 func (spider Music163Spider) httpPost(url string, headers map[string]string, params interface{}) (result []byte, err error) { body := make(url2.Values) jsonParams, err := json.Marshal(params) if err != nil { return nil, err } encryptResultMap := spider.dataEncrypt(jsonParams) body["params"] = []string{encryptResultMap["params"]} body["encSecKey"] = []string{encryptResultMap["encSecKey"]} req, err := http.NewRequest("POST", url, strings.NewReader(body.Encode())) for key, value := range headers { req.Header.Add(key, value) } if err != nil { return nil, err } resp, err := spider.client.Do(req) defer resp.Body.Close() data, err := ioutil.ReadAll(resp.Body) if err != nil { return nil, err } return data, nil }

发送参数

type BaseRequestBody struct { Offset string `json:"offset"` Total string `json:"totail"` Limit string `json:"limit"` CsrfToken string `json:"csrf_token"` } type PlayRecordRequestBody struct { BaseRequestBody Type string `json:"type"` Uid string `json:"uid"` }

根据获取排行榜请求返回数据来创建相应的对象

type SongDetail struct { Song common.Song `json:"song"` // ignore other } type PlayRecord struct { PlayCount int `json:"playCount"` Score int `json:"score"` SongDetail SongDetail `json:"song"` } type PlayRecordResp struct { Code int `json:"code"` AllData []PlayRecord `json:"allData"` WeekData []PlayRecord `json:"weekData"` }

定义好对象后，就可以使用模拟发送请求了, spider的GetPlayRecord方法

func (spider Music163Spider) GetPlayRecord(userId string) (record response.PlayRecordResp, err error) { playRecordReqBody := request.PlayRecordRequestBody{ Uid: userId, Type: "-1", BaseRequestBody: request.BaseRequestBody{ Offset: "0", Total: "true", Limit: "1000", CsrfToken: "", }, } playRecordUrl := fmt.Sprintf("%s%s%s?csrf_token=", constants.HttpsPrefix, constants.Music163Host, constants.PlayRecord) result, err := spider.httpPost(playRecordUrl, spider.headers, playRecordReqBody) if err != nil { return } playRecordResp := response.PlayRecordResp{} json.Unmarshal([]byte(result), &playRecordResp) return playRecordResp, nil }

测试一下

func main() { musicSpider := spider.NewMusic164Spider() record, _ := musicSpider.GetPlayRecord("62947535") jsonData, _ := json.MarshalIndent(record, "", "\t") fmt.Println(string(jsonData)) }

结果输出

Reference

• Go in action
• golang +es 爬取网易云音乐评论(整理版本1)
• Python 从零开始爬虫(七)——实战：网易云音乐评论爬取（附加密算法）