Skip to content

Commit

Permalink
Bilibili dash&4K (#732)
Browse files Browse the repository at this point in the history 
* Fix(bilibili): Change bilibili extract method to DASH API

Apply the latest DASH API from webpage;
4K format now is supported;

Closes #670, #684, #716

* Fix(bilibili): some url can not get 4K format information.

* Remove DEPRECATED codes&comments

* fix(bilibili): Update bangumi API

Update bangumi API (typo);
Removed seasonType param(unparam);
Removed unreached codes;

* extractors/bilibili: Update tests

* extractors/bilibili: avid&cid extract method is invalid in some cases. #734

* refactor(bilibili): Move the process of audio part to earlier stage
  • Loading branch information
@Half9000
Half9000 authored Jul 8, 2020
1 parent 5070a77 commit c48882c
Show file tree
Hide file tree
Showing 4 changed files with 129 additions and 74 deletions.
2 changes: 1 addition & 1 deletion downloader/downloader.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -634,7 +634,7 @@ func (downloader *Downloader) Download(data *types.Data) error {
}

fmt.Printf("Merging video parts into %s\n", mergedFilePath)
if stream.Ext != "mp4" || data.Site == "YouTube youtube.com" {
if stream.Ext != "mp4" || data.Site == "YouTube youtube.com" || data.Site == "哔哩哔哩 bilibili.com" {
return utils.MergeFilesWithSameExtension(parts, mergedFilePath)
}
return utils.MergeToMP4(parts, mergedFilePath, title)
Expand Down
150 changes: 98 additions & 52 deletions extractors/bilibili/bilibili.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,23 +14,16 @@ import (
)

const (
bilibiliAPI = "https://interface.bilibili.com/v2/playurl?"
bilibiliBangumiAPI = "https://bangumi.bilibili.com/player/web_api/v2/playurl?"
bilibiliAPI = "https://api.bilibili.com/x/player/playurl?"
bilibiliBangumiAPI = "https://api.bilibili.com/pgc/player/web/playurl?"
bilibiliTokenAPI = "https://api.bilibili.com/x/player/playurl/token?"
)

const (
// BiliBili blocks keys from time to time.
// You can extract from the Android client or bilibiliPlayer.min.js
appKey = "iVGUTjsxvpLeuDCf"
secKey = "aHRmhWMLkdeMuILqORnYZocwMBpMEOdt"
)

const referer = "https://www.bilibili.com"

var utoken string

func genAPI(aid, cid int, bangumi bool, quality, seasonType, cookie string) (string, error) {
func genAPI(aid, cid, quality int, bangumi bool, cookie string) (string, error) {
var (
err error
baseAPIURL string
Expand All @@ -55,44 +48,53 @@ func genAPI(aid, cid int, bangumi bool, quality, seasonType, cookie string) (str
}
utoken = t.Data.Token
}
var api string
if bangumi {
// The parameters need to be sorted by name
// qn=0 flag makes the CDN address different every time
// quality=116(1080P 60) is the highest quality so far
// quality=120(4k) is the highest quality so far
params = fmt.Sprintf(
"appkey=%s&cid=%d&module=bangumi&otype=json&qn=%s&quality=%s&season_type=%s&type=",
appKey, cid, quality, quality, seasonType,
"cid=%d&bvid=&qn=%d&type=&otype=json&fourk=1&fnver=0&fnval=16",
cid, quality,
)
baseAPIURL = bilibiliBangumiAPI
} else {
params = fmt.Sprintf(
"appkey=%s&cid=%d&otype=json&qn=%s&quality=%s&type=",
appKey, cid, quality, quality,
"avid=%d&cid=%d&bvid=&qn=%d&type=&otype=json&fourk=1&fnver=0&fnval=16",
aid, cid, quality,
)
baseAPIURL = bilibiliAPI
}
api = baseAPIURL + params
// bangumi utoken also need to put in params to sign, but the ordinary video doesn't need
api := fmt.Sprintf(
"%s%s&sign=%s", baseAPIURL, params, utils.Md5(params+secKey),
)
if !bangumi && utoken != "" {
api = fmt.Sprintf("%s&utoken=%s", api, utoken)
}
return api, nil
}

func genParts(durl []dURLData) ([]*types.Part, int64) {
var size int64
parts := make([]*types.Part, len(durl))
for index, data := range durl {
size += data.Size
parts[index] = &types.Part{
URL: data.URL,
Size: data.Size,
Ext: "flv",
func genParts(dashData *dashInfo, quality int, referer string) ([]*types.Part, error) {
parts := make([]*types.Part, 2)
checked := false
for _, stream := range dashData.Streams.Video {
if stream.ID == quality {
s, err := request.Size(stream.BaseURL, referer)
if err != nil {
return nil, err
}
parts[0] = &types.Part{
URL: stream.BaseURL,
Size: s,
Ext: "mp4",
}
checked = true
break
}
}
return parts, size
if !checked {
return nil, nil
}
return parts, nil
}

type bilibiliOptions struct {
Expand All @@ -113,12 +115,18 @@ func extractBangumi(url, html string, extractOption types.Options) ([]*types.Dat
return nil, err
}
if !extractOption.Playlist {
aid := data.EpInfo.Aid
cid := data.EpInfo.Cid
if aid <= 0 || cid <= 0 {
aid = data.EpList[0].Aid
cid = data.EpList[0].Cid
}
options := bilibiliOptions{
url: url,
html: html,
bangumi: true,
aid: data.EpInfo.Aid,
cid: data.EpInfo.Cid,
aid: aid,
cid: cid,
}
return []*types.Data{bilibiliDownload(options, extractOption)}, nil
}
Expand Down Expand Up @@ -263,9 +271,8 @@ func (e *extractor) Extract(url string, option types.Options) ([]*types.Data, er
// bilibiliDownload is the download function for a single URL
func bilibiliDownload(options bilibiliOptions, extractOption types.Options) *types.Data {
var (
err error
html string
seasonType string
err error
html string
)
if options.html != "" {
// reuse html string, but this can't be reused in case of playlist
Expand All @@ -276,53 +283,92 @@ func bilibiliDownload(options bilibiliOptions, extractOption types.Options) *typ
return types.EmptyData(options.url, err)
}
}
if options.bangumi {
seasonType = utils.MatchOneOf(html, `"season_type":(\d+)`, `"ssType":(\d+)`)[1]
}

// Get "accept_quality" and "accept_description"
// "accept_description":["高清 1080P","高清 720P","清晰 480P","流畅 360P"],
// "accept_quality":[80,48,32,16],
api, err := genAPI(options.aid, options.cid, options.bangumi, "15", seasonType, extractOption.Cookie)
// "accept_quality":[120,112,80,48,32,16],
api, err := genAPI(options.aid, options.cid, 120, options.bangumi, extractOption.Cookie)
if err != nil {
return types.EmptyData(options.url, err)
}
jsonString, err := request.Get(api, referer, nil)
if err != nil {
return types.EmptyData(options.url, err)
}
var quality qualityInfo
err = json.Unmarshal([]byte(jsonString), &quality)

var data dash
err = json.Unmarshal([]byte(jsonString), &data)
if err != nil {
return types.EmptyData(options.url, err)
}
var dashData dashInfo
if data.Data.Description == nil {
dashData = data.Result
} else {
dashData = data.Data
}

streams := make(map[string]*types.Stream, len(quality.Quality))
for _, q := range quality.Quality {
apiURL, err := genAPI(options.aid, options.cid, options.bangumi, strconv.Itoa(q), seasonType, extractOption.Cookie)
// Get audio part
var audioID int
audios := map[int]string{}
bandwidth := 0
for _, stream := range dashData.Streams.Audio {
if stream.Bandwidth > bandwidth {
audioID = stream.ID
}
audios[stream.ID] = stream.BaseURL
bandwidth = stream.Bandwidth
}
s, err := request.Size(audios[audioID], referer)
if err != nil {
return types.EmptyData(options.url, err)
}
audioPart := &types.Part{
URL: audios[audioID],
Size: s,
Ext: "m4a",
}

streams := make(map[string]*types.Stream, len(dashData.Quality))
for _, q := range dashData.Quality {
// Avoid duplicate streams
if _, ok := streams[strconv.Itoa(q)]; ok {
continue
}
api, err := genAPI(options.aid, options.cid, q, options.bangumi, extractOption.Cookie)
if err != nil {
return types.EmptyData(options.url, err)
}
jsonString, err := request.Get(apiURL, referer, nil)
jsonString, err := request.Get(api, referer, nil)
if err != nil {
return types.EmptyData(options.url, err)
}
var data bilibiliData

err = json.Unmarshal([]byte(jsonString), &data)
if err != nil {
return types.EmptyData(options.url, err)
}

// Avoid duplicate streams
if _, ok := streams[strconv.Itoa(data.Quality)]; ok {
if data.Data.Description == nil {
dashData = data.Result
} else {
dashData = data.Data
}
parts, err := genParts(&dashData, q, options.url)
if parts == nil {
continue
}

parts, size := genParts(data.DURL)
streams[strconv.Itoa(data.Quality)] = &types.Stream{
if err != nil {
return types.EmptyData(options.url, err)
}
parts[1] = audioPart
var size int64
for _, part := range parts {
size += part.Size
}
streams[strconv.Itoa(q)] = &types.Stream{
Parts: parts,
Size: size,
Quality: qualityString[data.Quality],
Quality: qualityString[q],
}
}

Expand Down
8 changes: 4 additions & 4 deletions extractors/bilibili/bilibili_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,17 +20,17 @@ func TestBilibili(t *testing.T) {
Title: "【2018拜年祭单品】相遇day by day",
Quality: "高清 1080P",
},
playlist: true,
playlist: false,
},
{
name: "normal test 2",
args: test.Args{
URL: "https://www.bilibili.com/video/av41301960",
Title: "【英雄联盟】2019赛季CG 《觉醒》",
Size: 85790602,
Size: 65774670,
Quality: "高清 1080P",
},
playlist: true,
playlist: false,
},
{
name: "bangumi test",
Expand All @@ -45,7 +45,7 @@ func TestBilibili(t *testing.T) {
args: test.Args{
URL: "https://www.bilibili.com/bangumi/play/ss5050",
Title: "一人之下：第1话 异人刀兵起，道炁携阴阳",
Quality: "高清 1080P",
Quality: "高清 720P",
},
playlist: true,
},
Expand Down
43 changes: 26 additions & 17 deletions extractors/bilibili/types.go
View file
Original file line number Diff line number Diff line change
@@ -1,22 +1,5 @@
package bilibili

type qualityInfo struct {
Description []string `json:"accept_description"`
Quality []int `json:"accept_quality"`
}

type dURLData struct {
Size int64 `json:"size"`
URL string `json:"url"`
Order int `json:"order"`
}

type bilibiliData struct {
DURL []dURLData `json:"durl"`
Format string `json:"format"`
Quality int `json:"quality"`
}

// {"code":0,"message":"0","ttl":1,"data":{"token":"aaa"}}
// {"code":-101,"message":"账号未登录","ttl":1}
type tokenData struct {
Expand Down Expand Up @@ -57,7 +40,33 @@ type multiPage struct {
VideoData multiPageVideoData `json:"videoData"`
}

type dashStream struct {
ID int `json:"id"`
BaseURL string `json:"baseUrl"`
Bandwidth int `json:"bandwidth"`
}

type dashStreams struct {
Video []dashStream `json:"video"`
Audio []dashStream `json:"audio"`
}

type dashInfo struct {
CurQuality int `json:"quality"`
Description []string `json:"accept_description"`
Quality []int `json:"accept_quality"`
Streams dashStreams `json:"dash"`
}

type dash struct {
Code int `json:"code"`
Message string `json:"message"`
Data dashInfo `json:"data"`
Result dashInfo `json:"result"`
}

var qualityString = map[int]string{
120: "超清 4K",
116: "高清 1080P60",
74: "高清 720P60",
112: "高清 1080P+",
Expand Down

0 comments on commit c48882c

Please sign in to comment.