nicksxs
/
graph-tutorial
1
0
Fork 0
Code Issues 0 Pull Requests 0 Projects 0 Releases 0 Wiki Activity
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
73 Commits
4 Branches
21 MiB
 
 
 
Tree: ec2425c125
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'ec2425c125'
${ noResults }
graph-tutorial/app/Services/BilibiliServiceV2.php

1132 lines
52 KiB
Raw Blame History

<?php
namespace App\Services;
use App\BilibiliCollections;
use App\BilibiliUpVideos;
use App\BilibiliVideoParts;
use App\BilibiliVideos;
use App\Repositories\BilibiliVideoRepository;
use Exception;
use GuzzleHttp\Client;
use Illuminate\Support\Arr;
use Illuminate\Support\Facades\App;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\Facades\Redis;
class BilibiliServiceV2
{
private $collectionNumberListUrl = "https://api.bilibili.com/medialist/gateway/base/created?ps=100&up_mid=279025&is_space=0&jsonp=jsonp";
private $collectionListUrl = "https://api.bilibili.com/medialist/gateway/base/spaceDetail?ps=20&keyword=&order=mtime&type=0&tid=0&jsonp=jsonp";
private $videoPartsUrl = "https://api.bilibili.com/x/player/pagelist?jsonp=jsonp&aid=";
private $invalidTitle = "已失效视频";
// private $baseDir = "/Volumes/intel660p/video/mv/";
private $baseDir = "/Volumes/Crucial X6/Video/";
// private $remoteDir = "/data/";
private $remoteDir = "/Volumes/Crucial X6/Video/";
protected $repository;
public function __construct(BilibiliVideoRepository $repository)
{
$this->repository = $repository;
}
public function queryPlayList()
{
$pageNo = 1;
$url = "https://space.bilibili.com/ajax/member/getSubmitVideos?mid=391073761&pagesize=30&tid=0&keyword=&order=pubdate&page=";
for ($i = 1; $i < 30; $i++) {
# code...
$result = json_decode(file_get_contents($url . $i), true);
foreach ($result['data']['vlist'] as $item) {
echo "av" . $item['aid'] . " ";
}
// echo $result['data']['vlist'][0]['aid'] . "\t";
// $pageNo++;
}
}
/**
* 获取各个 up 主的视频 id
* @param int $mediaId
* @throws Exception
*/
public function queryUpVideoList($mediaId = 320491072)
{
Log::info("schedule queryUpVideoList inner started at: " . date("Y-m-d H:i:s"));
// 475250 kyokyo
// 3489535 熊小颖
// 320491072 绯绯
// 10139490 短短
// 116683 咬人猫
// 16539048 小仙若
// 10278125 香草猫饼
// 391073761 女团直拍
// 267781236 韩国女团饭拍直拍
// 23400436 小雪_juvia
$list = BilibiliUpVideos::all();
foreach ($list as $item) {
Log::info("schedule queryUpVideoList current up is {$item['up_name']}, started at: " . date("Y-m-d H:i:s"));
// $files = scandir($this->baseDir . "bilibili/" . $item['up_name']);
$mediaId = $item['mid'];
$videos = [];
$pageNo = 1;
$url = "https://api.bilibili.com/x/space/arc/search?mid={$mediaId}&ps=30&tid=0&keyword=&order=pubdate&jsonp=jsonp&pn=";
// https://space.bilibili.com/475250/video
for ($i = 1; $i < 5; $i++) {
$curl = curl_init();
curl_setopt_array($curl, array(
CURLOPT_URL => "https://api.bilibili.com/x/space/arc/search?mid={$mediaId}&ps=30&tid=0&keyword=&order=pubdate&jsonp=jsonp&pn={$i}",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "GET",
CURLOPT_HTTPHEADER => array(
"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3",
"Accept-Encoding: gzip, deflate, br",
"Accept-Language: zh-CN,zh;q=0.9,en-US;q=0.8,en;q=0.7,zh-TW;q=0.6,ja;q=0.5",
"Cache-Control: no-cache",
"Connection: keep-alive",
"Referer: https://www.bilibili.com",
"Pragma: no-cache",
"Sec-Fetch-Mode: navigate",
"Sec-Fetch-Site: none",
"Sec-Fetch-User: ?1",
"Upgrade-Insecure-Requests: 1",
"User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.97 Safari/537.36",
"cache-control: no-cache"
),
));
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
// echo $response;exit;
$result = json_decode($response, true);
if (!array_key_exists("data", $result) || !array_key_exists("list", $result["data"])) {
Log::info("result is null " . json_encode($response));
continue;
}
$result = Arr::get($result["data"]["list"], "vlist", []);
if (count($result) > 0) {
foreach ($result as $vItem) {
$bVideo = BilibiliVideos::firstOrCreate(["aid" => $vItem["aid"]],
[
"title" => $vItem["title"],
"from_type" => 2,
"from_up_name" => $item["up_name"],
"bv_id" => $vItem["bvid"],
"up_mid" => $mediaId,
"collection_mid" => 0
]);
if ($bVideo->from_type != 2) {
$bVideo->from_type = 3;
}
$bVideo->bv_id = $vItem["bvid"];
$bVideo->up_mid = $mediaId;
$bVideo->from_up_name = $item["up_name"];
$bVideo->save();
$videos[] = $vItem["aid"];
}
} else {
break;
}
}
usleep(random_int(1000, 10000) * 1000);
}
$item['videos'] = json_encode($videos);
// $item['downloaded_videos'] = json_encode([]);
$item->save();
}
}
/**
* 手动执行查询
* @param int $mediaId
* @throws Exception
*/
public function queryLocalUpVideoList($mediaId = 320491072)
{
Log::info("schedule queryUpVideoList inner started at: " . date("Y-m-d H:i:s"));
// 475250 kyokyo
// 3489535 熊小颖
// 320491072 绯绯
// 10139490 短短
// 116683 咬人猫
// 16539048 小仙若
// 10278125 香草猫饼
// 391073761 女团直拍
// $list = BilibiliUpVideos::all();
$list = [];
$up = ["mid" => 391073761, "up_name" => "女团直拍"];
$up = ["mid" => 254624554, "up_name" => "女团能量站"];
$list[] = $up;
foreach ($list as $item) {
Log::info("schedule queryUpVideoList current up is {$item['up_name']}, started at: " . date("Y-m-d H:i:s"));
$files = scandir($this->baseDir . "bilibili/" . $item['up_name']);
$mediaId = $item['mid'];
$videos = [];
$pageNo = 1;
$url = "https://api.bilibili.com/x/space/arc/search?mid={$mediaId}&ps=30&tid=0&keyword=&order=pubdate&jsonp=jsonp&pn=";
// https://space.bilibili.com/475250/video
for ($i = 1; $i < 50; $i++) {
$curl = curl_init();
curl_setopt_array($curl, array(
CURLOPT_URL => "https://api.bilibili.com/x/space/arc/search?mid={$mediaId}&ps=30&tid=0&keyword=&order=pubdate&jsonp=jsonp&pn={$i}",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "GET",
CURLOPT_HTTPHEADER => array(
"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3",
"Accept-Encoding: gzip, deflate, br",
"Accept-Language: zh-CN,zh;q=0.9,en-US;q=0.8,en;q=0.7,zh-TW;q=0.6,ja;q=0.5",
"Cache-Control: no-cache",
"Connection: keep-alive",
"Referer: https://www.bilibili.com",
"Pragma: no-cache",
"Sec-Fetch-Mode: navigate",
"Sec-Fetch-Site: none",
"Sec-Fetch-User: ?1",
"Upgrade-Insecure-Requests: 1",
"User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.97 Safari/537.36",
"cache-control: no-cache"
),
));
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
// echo $response;
$result = json_decode($response, true);
$result = Arr::get($result["data"]["list"], "vlist", []);
if (count($result) > 0) {
foreach ($result as $vItem) {
echo "av" . $vItem["aid"] . " ";
// $videos[] = $vItem["aid"];
}
} else {
break;
}
}
usleep(random_int(1000, 10000) * 1000);
}
// $item['videos'] = json_encode($videos);
// $item['downloaded_videos'] = json_encode([]);
// $item->save();
}
}
public function compareAndDownloadUpVideos($isAll = false)
{
$env = App::environment();
$list = BilibiliUpVideos::orderBy("created_at", "desc")->get();
foreach ($list as $item) {
dump("当前 up名称是: " . $item["up_name"] . "\n");
if ($item["is_downloaded"] == 1) {
continue;
}
if (!$this->checkDiskSpace()) {
Log::info('磁盘空间不足');
exit;
}
Log::info("schedule compareAndDownloadUpVideos current up is {$item['up_name']}, started at: " . date("Y-m-d H:i:s"));
if ($isAll) {
$vItems = BilibiliVideos::where("up_mid", $item["mid"])->get();
} else {
$vItems = BilibiliVideos::where('up_mid', $item["mid"])
->where('created_at', '>=', date("Y-m-d H:i:s", strtotime("-1 week")))->get();
}
$videoList = [];
$videoPartsMap = [];
if (count($vItems) > 0) {
foreach ($vItems as $vItem) {
if ($vItem["is_download"] == 1 && $vItem["download_status"] != 1) {
// $videoList[] = "av" . $vItem["aid"];
if (!$this->repository->softLockUpdate(["id" => $vItem["id"]], $vItem["version"], ["download_status" => 1])) {
Log::warning("soft Lock failed id : " . $vItem["id"]);
continue;
}
$videoPartsMap = [];
$videoPartsMap[$vItem["aid"]] = $vItem["total_parts"];
echo "当前 up名称是: " . $item["up_name"] . " 当前下载的视频 title: " . $vItem["title"] . " 当前下载的视频 aid 是:" . $vItem["aid"] . "\n";
Log::info("当前 up名称是: " . $item["up_name"] . " 当前下载的视频 title: " . $vItem["title"] . " 当前下载的视频 aid 是:" . $vItem["aid"]);
if ($env == "local") {
$result = false;
if ($item['mid'] == 27174777) {
$result = $this->partDownloadBSitePlaylist($videoPartsMap, "/Volumes/WD/tmp/bilibili/", "女团");
} else if ($item["mid"] == 391316322) {
$result = $this->partDownloadBSitePlaylist($videoPartsMap, "/Volumes/WD/tmp/bilibili/", "娜娜");
} else if ($item["mid"] == 396501206) {
$result = $this->partDownloadBSitePlaylist($videoPartsMap, "/Volumes/WD/tmp/bilibili/", "佳佳");
} else {
$result = $this->partDownloadBSitePlaylist($videoPartsMap, $this->baseDir . "bilibili/", $item['up_name']);
}
if ($result) {
if (!$this->repository->softLockUpdate(
["id" => $vItem["id"]],
$vItem["version"] + 1,
["download_status" => 0,
"is_downloaded" => 1
])) {
Log::warning("soft Lock failed id : " . $vItem["id"]);
}
}
} else {
$result = $this->partDownloadBSitePlaylist($videoPartsMap, $this->remoteDir . "bilibili/", $item["up_name"]);
if ($result) {
if (!$this->repository->softLockUpdate(
["id" => $vItem["id"]],
$vItem["version"] + 1,
["download_status" => 0,
"is_downloaded" => 1
])) {
Log::warning("soft Lock failed id : " . $vItem["id"]);
}
}
}
}
}
}
}
}
public function compareAndDownloadCollectionVideos($isAll = false)
{
$env = App::environment();
$list = BilibiliCollections::all();
foreach ($list as $item) {
dump("收藏夹名称是: " . $item["title"] . "\n");
if ($item["is_downloaded"] == 1) {
continue;
}
if (!$this->checkDiskSpace()) {
Log::info('磁盘空间不足');
exit;
}
Log::info("schedule compareAndDownloadCollectionVideos current collection is {$item['title']}, started at: " . date("Y-m-d H:i:s"));
$videoList = [];
$videoPartsMap = [];
if ($isAll) {
$vItems = BilibiliVideos::where("collection_mid", $item["media_id"])->get();
} else {
$vItems = BilibiliVideos::where('collection_mid', $item["media_id"])
->where('created_at', '>=', date("Y-m-d H:i:s", strtotime("-1 week")))->get();
}
if (count($vItems) > 0) {
foreach ($vItems as $vItem) {
if ($vItem["is_download"] == 1 && $vItem["is_downloaded"] == 0 && $vItem["download_status"] == 0) {
// $videoList[] = "av" . $vItem["aid"];
if (!$this->repository->softLockUpdate(["id" => $vItem["id"]], $vItem["version"], ["download_status" => 1])) {
Log::warning("soft Lock failed id : " . $vItem["id"]);
continue;
}
$videoPartsMap = [];
$videoPartsMap[$vItem["aid"]] = $vItem["total_parts"];
echo "收藏夹名称是: " . $item["title"] . " 当前下载的视频 title: " . $vItem["title"] . " 当前下载的视频 aid 是:" . $vItem["aid"] . "\n";
Log::info("收藏夹名称是: " . $item["title"] . " 当前下载的视频 title: " . $vItem["title"] . " 当前下载的视频 aid 是:" . $vItem["aid"]);
if ($env == "local") {
$result = false;
if ($item['title'] == "默认收藏夹") {
dump("xxxxxxxxxxxx----------------");
$result = $this->partDownloadBSitePlaylist($videoPartsMap, "/Volumes/WD/tmp/", "bilibili");
} else if ($item['title'] == '少女时代') {
$result = $this->partDownloadBSitePlaylist($videoPartsMap, "/Volumes/WD/tmp/bilibili/少女时代", "");
} else if ($item['title'] == 'aoa') {
$result = $this->partDownloadBSitePlaylist($videoPartsMap, "/Volumes/WD/tmp/bilibili/aoa", "");
} else if ($item['title'] == 'blackpink') {
$result = $this->partDownloadBSitePlaylist($videoPartsMap, "/Volumes/WD/tmp/bilibili/blackpink", "");
} else if ($item['title'] == 'wjsn') {
$result = $this->partDownloadBSitePlaylist($videoPartsMap, "/Volumes/WD/tmp/bilibili/wjsn", "");
} else {
$result = $this->partDownloadBSitePlaylist($videoPartsMap, $this->baseDir . "bilibili/", $item['title']);
}
if ($result) {
if (!$this->repository->softLockUpdate(
["id" => $vItem["id"]],
$vItem["version"] + 1,
["download_status" => 0,
"is_downloaded" => 1
])) {
Log::warning("soft Lock failed id : " . $vItem["id"]);
}
}
} else {
$result = $this->partDownloadBSitePlaylist($videoPartsMap, $this->remoteDir . "bilibili", $item["title"]);
if ($result) {
if (!$this->repository->softLockUpdate(
["id" => $vItem["id"]],
$vItem["version"] + 1,
["download_status" => 0,
"is_downloaded" => 1
])) {
Log::warning("soft Lock failed id : " . $vItem["id"]);
}
}
}
}
}
}
}
}
public function queryFavList()
{
// aoa 50076725
// default 50069625
// 美腿 156009025
// 允儿 154442125
// 少女时代 153944525
$invalidTitle = "已失效视频";
$pageNo = 1;
for ($i = 1; $i < 65; $i++) {
$curl = curl_init();
$url = "https://api.bilibili.com/medialist/gateway/base/spaceDetail?media_id=154442125&pn={$i}&ps=20&keyword=&order=mtime&type=0&tid=0&jsonp=jsonp";
curl_setopt_array($curl, array(
CURLOPT_URL => $url,
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "GET",
CURLOPT_HTTPHEADER => array(
"Accept: application/json, text/plain, */*",
"Accept-Encoding: gzip, deflate, br",
"Accept-Language: en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7,zh-TW;q=0.6,ja;q=0.5",
"Cache-Control: no-cache",
"Connection: keep-alive",
// "Cookie: buvid3=5566647C-DDE5-4AFF-8711-89C9DB2B7061110244infoc; LIVE_BUVID=AUTO2315591376644842; sid=kbyrjw72; stardustvideo=1; CURRENT_FNVAL=16; rpdid=|(J~|JluJmmk0J\'ullmuJ~~kJ; CURRENT_QUALITY=80; UM_distinctid=16b11bb4099eb-033fd8b69a435d-37647e03-13c680-16b11bb40bf97; fts=1562149769; im_notify_type_279025=0; _uuid=69A60E94-E8FE-54B8-DCCD-0F519F0DEF7480343infoc; DedeUserID=279025; DedeUserID__ckMd5=9a79e15294e6b8bb; SESSDATA=b114a39d%2C1573361994%2C2ed0a2a1; bili_jct=b35f8e9780e4a80dd07f316f781f179b; bp_t_offset_279025=315386152995357818",
"Cookie: SESSDATA=64a15917%2C1578628130%2Ceb05cdc1",
"Host: api.bilibili.com",
"Origin: https://space.bilibili.com",
"Postman-Token: c7f849ec-aaf9-4c71-851d-4ca061e725d0,f8f582bc-4872-4aa7-b209-b926364d4d1a",
"Referer: https://space.bilibili.com/279025/favlist",
"Sec-Fetch-Mode: cors",
"Sec-Fetch-Site: same-site",
"User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.70 Safari/537.36",
"cache-control: no-cache"
),
));
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
$responseArr = json_decode($response, true);
if ($responseArr['code'] == 0) {
$data = $responseArr['data'];
$medias = $data['medias'];
if (count($medias) > 0) {
foreach ($medias as $item) {
if ($item['title'] != $invalidTitle) {
echo "av" . $item['id'] . " ";
}
}
}
}
}
usleep(random_int(1000, 10000) * 1000);
}
}
public function getAoaFavList($id = 50076725)
{
$invalidTitle = "已失效视频";
$pageNo = 1;
for ($i = 1; $i < 65; $i++) {
$curl = curl_init();
$url = "https://api.bilibili.com/medialist/gateway/base/spaceDetail?media_id=50076725&pn={$i}&ps=20&keyword=&order=mtime&type=0&tid=0&jsonp=jsonp";
curl_setopt_array($curl, array(
CURLOPT_URL => $url,
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "GET",
CURLOPT_HTTPHEADER => array(
"Accept: application/json, text/plain, */*",
"Accept-Encoding: gzip, deflate, br",
"Accept-Language: en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7,zh-TW;q=0.6,ja;q=0.5",
"Cache-Control: no-cache",
"Connection: keep-alive",
"Cookie: buvid3=5566647C-DDE5-4AFF-8711-89C9DB2B7061110244infoc; LIVE_BUVID=AUTO2315591376644842; sid=kbyrjw72; stardustvideo=1; CURRENT_FNVAL=16; rpdid=|(J~|JluJmmk0J'ullmuJ~~kJ; CURRENT_QUALITY=80; UM_distinctid=16b11bb4099eb-033fd8b69a435d-37647e03-13c680-16b11bb40bf97; fts=1562149769; im_notify_type_279025=0; _uuid=69A60E94-E8FE-54B8-DCCD-0F519F0DEF7480343infoc; DedeUserID=279025; DedeUserID__ckMd5=9a79e15294e6b8bb; SESSDATA=b114a39d%2C1573361994%2C2ed0a2a1; bili_jct=b35f8e9780e4a80dd07f316f781f179b; bp_t_offset_279025=317592473398470273",
"Host: api.bilibili.com",
"Origin: https://space.bilibili.com",
"Postman-Token: c7f849ec-aaf9-4c71-851d-4ca061e725d0,f8f582bc-4872-4aa7-b209-b926364d4d1a",
"Referer: https://space.bilibili.com/279025/favlist",
"Sec-Fetch-Mode: cors",
"Sec-Fetch-Site: same-site",
"User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.70 Safari/537.36",
"cache-control: no-cache"
),
));
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
$responseArr = json_decode($response, true);
if ($responseArr['code'] == 0) {
$data = $responseArr['data'];
$medias = $data['medias'];
if (count($medias) > 0) {
foreach ($medias as $item) {
if ($item['title'] != $invalidTitle) {
echo "av" . $item['id'] . " ";
}
}
}
}
}
usleep(random_int(1000, 10000) * 1000);
}
}
public function queryCollectionList()
{
// $curl = curl_init();
//
// curl_setopt_array($curl, array(
// CURLOPT_URL => "https://api.bilibili.com/medialist/gateway/base/created?pn=1&ps=100&up_mid=279025&is_space=0&jsonp=jsonp",
// CURLOPT_RETURNTRANSFER => true,
// CURLOPT_ENCODING => "",
// CURLOPT_MAXREDIRS => 10,
// CURLOPT_TIMEOUT => 0,
// CURLOPT_FOLLOWLOCATION => true,
// CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
// CURLOPT_CUSTOMREQUEST => "GET",
// CURLOPT_HTTPHEADER => array(
// "Connection: keep-alive",
// "Pragma: no-cache",
// "Cache-Control: no-cache",
// "Accept: application/json, text/plain, */*",
// "Origin: https://space.bilibili.com",
// "User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36",
// "Sec-Fetch-Site: same-site",
// "Sec-Fetch-Mode: cors",
// "Referer: https://space.bilibili.com/279025/favlist?spm_id_from=333.851.b_696e7465726e6174696f6e616c486561646572.15",
// "Accept-Encoding: gzip, deflate, br",
// "Accept-Language: zh-CN,zh;q=0.9,en-US;q=0.8,en;q=0.7,zh-TW;q=0.6,ja;q=0.5",
// "Cookie: _uuid=D6E5438B-5A95-439F-7512-FC3509457A9A28409infoc; buvid3=5566647C-DDE5-4AFF-8711-89C9DB2B7061110244infoc; LIVE_BUVID=AUTO3415734420289108; UM_distinctid=16e62db8b9934b-0f76f63ae51bee-1c3c6a5a-13c680-16e62db8b9acaf; CURRENT_FNVAL=16; stardustvideo=1; rpdid=|(ku|l|lRYlJ0J'ul~JYuYY|u; finger=0539dad4; im_notify_type_279025=0; sid=llgc5h9q; DedeUserID=279025; DedeUserID__ckMd5=9a79e15294e6b8bb; bili_jct=07b8811aa5ca68020eb0c11e0e01c22f; laboratory=1-1; CURRENT_QUALITY=80; SESSDATA=64a15917%2C1578628130%2Ceb05cdc1; bp_t_offset_279025=336855788264916597; INTVER=1"
// ),
// ));
//
// $response = curl_exec($curl);
$response = $this->requestCollectionNumberList($this->collectionNumberListUrl, 1);
$res = json_decode($response, true);
if ($res['code'] == 0) {
$list = $res['data']['list'];
foreach ($list as $item) {
$collection = new BilibiliCollections();
$collection->title = $item["title"];
$collection->is_downloaded = 0;
$collection->media_id = $item["id"];
$currentList = [];
for ($pageNo = 1; $pageNo < 100; $pageNo++) {
$listResponse = $this->requestCollectionList($this->collectionListUrl, $item['id'], $pageNo);
dump("pageNo: " . $pageNo);
dump($listResponse);
$responseArr = json_decode($listResponse, true);
if ($responseArr['code'] == 0) {
$data = $responseArr["data"];
$medias = Arr::get($data, "medias", []);
if (count($medias) > 0) {
foreach ($medias as $media) {
if ($media['title'] != $this->invalidTitle) {
$currentList[] = $media["id"];
}
}
} else {
break;
}
}
}
$collection->videos = json_encode($currentList);
$collection->save();
}
}
// curl_close($curl);
echo $response;
}
public function queryDBCollectionList()
{
$list = BilibiliCollections::all();
foreach ($list as $item) {
Log::info("schedule queryDBCollectionList current collection is: " . $item['title'] . " started at :" . date("Y-m-d H:i:s"));
$currentList = [];
for ($pageNo = 1; $pageNo < 10; $pageNo++) {
$listResponse = $this->requestCollectionList($this->collectionListUrl, $item['media_id'], $pageNo);
// dump("pageNo: " . $pageNo);
// dump($listResponse);
$responseArr = json_decode($listResponse, true);
if ($responseArr['code'] == 0) {
$data = $responseArr["data"];
$medias = Arr::get($data, "medias", []);
if (count($medias) > 0) {
foreach ($medias as $media) {
if ($media['title'] != $this->invalidTitle) {
$bVideo = BilibiliVideos::firstOrCreate(["aid" => $media["id"]],
[
"title" => $media["title"],
"from_type" => 1,
"from_collection_name" => $item['title'],
"bv_id" => $media["bv_id"],
"collection_mid" => $item["media_id"],
"up_mid" => 0
]);
if ($bVideo->from_type != 1) {
$bVideo->from_type = 3;
$bVideo->from_collection_name = $item["title"];
}
$bVideo->bv_id = $media["bv_id"];
$bVideo->collection_mid = $item["media_id"];
$bVideo->save();
$currentList[] = $media["id"];
}
}
} else {
break;
}
}
}
$item->videos = json_encode($currentList);
$item->save();
}
}
public function queryForVideoParts()
{
$i = 0;
$list = BilibiliVideos::orderBy('id', 'desc')->simplePaginate(2000, null, 'page', $i);
// dump($list->items()[0]->aid);
while ($list->isNotEmpty()) {
foreach ($list->items() as $item) {
// dump("current item", [$item->getAttributes()]);
if ($item->is_download == 0) {
// 不需要下载
continue;
}
$response = $this->requestVideoParts($item->aid);
dump($response);
$responseJson = json_decode($response, true);
if ($responseJson["code"] !== 0) {
Log::error("response is error, aid: " . $item->aid . " response: " . $response);
if ($responseJson["code"] == -404 || $responseJson["code"] == "-404" || $responseJson["message"] == "啥都木有") {
Log::error("response is error and save is_download = 0, aid: " . $item->aid . " response: " . $response);
$item->is_download = 0;
$item->save();
}
continue;
}
$partList = Arr::get($responseJson, "data");
if (count($partList) > 0) {
foreach ($partList as $part) {
if ($item["from_type"] == 1) {
if ($item["from_collection_name"] == "默认收藏夹") {
$downloadPath = $this->baseDir . "bilibili";
} else {
$downloadPath = $this->baseDir . "bilibili/" . $item["from_collection_name"];
}
} else {
$downloadPath = $this->baseDir . "bilibili/" . $item["from_up_name"];
}
$pItem = BilibiliVideoParts::firstOrCreate([
"aid" => $item->aid,
"page" => $part["page"]
], [
"title" => $item["title"],
"part" => $part["part"],
"duration" => $part["duration"],
"to_download_path" => $downloadPath
]);
$pItem["title"] = $item["title"];
$pItem["to_download_path"] = $downloadPath;
$pItem->save();
}
$item->total_parts = count($partList);
$item->save();
}
try {
usleep(random_int(10, 100) * 1000);
} catch (Exception $e) {
}
}
break;
$i++;
$list = BilibiliVideos::simplePaginate(2000, null, 'page', $i);
try {
usleep(random_int(10, 1000) * 1000);
} catch (Exception $e) {
}
}
// while ($list->hasPages()) {
// foreach ($list as $item) {
// $item->
// }
// $list = BilibiliVideos::simplePaginate()
// }
}
/**
* 检查是否已下载
*/
public function checkVideoHasDownload()
{
$i = 1;
$list = BilibiliVideos::simplePaginate(50, null, 'page', $i);
$dirFiles = [];
$dirFiles["/Volumes/WD/tmp/bilibili"] = scandir("/Volumes/WD/tmp/bilibili");
$dirFiles["/Volumes/WD/tmp/bilibili/aoa"] = scandir("/Volumes/WD/tmp/bilibili/aoa");
$dirFiles["/Volumes/WD/tmp/bilibili/女团"] = scandir("/Volumes/WD/tmp/bilibili/女团");
$dirFiles["/Volumes/WD/tmp/bilibili/少女时代"] = scandir("/Volumes/WD/tmp/bilibili/少女时代");
while ($list->isNotEmpty()) {
foreach ($list->items() as $item) {
// dump("current item", [$item->getAttributes()]);
if ($item['is_download'] == 0) {
continue;
}
$list = BilibiliVideoParts::where(['aid' => $item['aid']])->get();
if (count($list) > 0) {
foreach ($list as $part) {
// dump("here is parts", [$part->getAttributes()]);
if (is_dir($part["to_download_path"])) {
// dump("this is path ".$part["to_download_path"]);
if (array_key_exists($part["to_download_path"], $dirFiles)) {
$files = Arr::get($dirFiles, $part["to_download_path"]);
} else {
$files = scandir($part["to_download_path"]);
$dirFiles[$part["to_download_path"]] = $files;
}
// dump($files);
$itemCount = 0;
$toRemove = [];
foreach ($files as $file) {
if ($file == "." || $file == "..") {
continue;
} else {
// dump("current file: ". $file ." ".$part["title"]);
$pos = strpos($file, $part["title"]);
// dump("current compare result " . $pos);
if ($pos !== false) {
// dump($file);
$itemCount++;
if (strpos($file, $part["title"] . " P" . $part["page"]) !== false) {
Log::info("file matched " . $file);
$toRemove[] = $file;
$part["is_downloaded"] = 1;
$part->save();
}
}
}
}
$dirFiles[$part["to_download_path"]] = array_diff($files, $toRemove);
if ($itemCount == $item["total_parts"]) {
$item["is_downloaded"] = 1;
$item->save();
}
}
}
}
// exit;
}
$i++;
$list = BilibiliVideos::simplePaginate(50, null, 'page', $i);
// exit;
}
}
public function checkVideoHasDownloadV2()
{
$i = 1;
$list = BilibiliVideos::simplePaginate(50, null, 'page', $i);
$dirFiles = [];
$dirFiles["/Volumes/WD/tmp/bilibili"] = scandir("/Volumes/WD/tmp/bilibili");
$dirFiles["/Volumes/WD/tmp/bilibili/aoa"] = scandir("/Volumes/WD/tmp/bilibili/aoa");
$dirFiles["/Volumes/WD/tmp/bilibili/女团"] = scandir("/Volumes/WD/tmp/bilibili/女团");
$dirFiles["/Volumes/WD/tmp/bilibili/少女时代"] = scandir("/Volumes/WD/tmp/bilibili/少女时代");
while ($list->isNotEmpty()) {
foreach ($list->items() as $item) {
dump("current item", [$item->getAttributes()]);
if ($item['is_download'] == 0) {
continue;
}
$list = BilibiliVideoParts::where(['aid' => $item['aid']])->get();
if (count($list) > 0) {
foreach ($list as $part) {
// dump("here is parts", [$part->getAttributes()]);
if (is_dir($part["to_download_path"])) {
// dump("this is path ".$part["to_download_path"]);
if (array_key_exists($part["to_download_path"], $dirFiles)) {
$files = Arr::get($dirFiles, $part["to_download_path"]);
} else {
$files = scandir($part["to_download_path"]);
$dirFiles[$part["to_download_path"]] = $files;
}
// dump($files);
$itemCount = 0;
$toRemove = [];
foreach ($files as $file) {
if ($file == "." || $file == "..") {
continue;
} else {
dump("current file: " . $file . " " . $part["title"]);
$pos = strpos($file, $part["title"]);
// dump("current compare result " . $pos);
if ($pos !== false) {
dump($file);
$itemCount++;
if (strpos($file, $part["title"] . " P" . $part["page"]) !== false) {
Log::info("file matched " . $file);
$toRemove[] = $file;
$part["is_downloaded"] = 1;
$part->save();
}
}
}
}
$dirFiles[$part["to_download_path"]] = array_diff($files, $toRemove);
if ($itemCount == $item["total_parts"]) {
$item["is_downloaded"] = 1;
$item->save();
}
}
}
}
// exit;
}
$i++;
$list = BilibiliVideos::simplePaginate(50, null, 'page', $i);
// exit;
}
}
public function requestVideoParts($aid)
{
return $this->request($this->videoPartsUrl . $aid);
}
public function requestCollectionList($url, $mediaId, $pn)
{
return $this->request($url . "&media_id=" . $mediaId . "&pn=" . $pn, true);
}
public function requestCollectionNumberList($url, $pn)
{
return $this->request($url . "&pn=" . $pn);
}
public function request($url, $userCookie = false)
{
$curl = curl_init();
$headerArray = array(
"Connection: keep-alive",
"Pragma: no-cache",
"Cache-Control: no-cache",
"Accept: application/json, text/plain, */*",
"Origin: https://space.bilibili.com",
"User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36",
"Sec-Fetch-Site: same-site",
"Sec-Fetch-Mode: cors",
"Referer: https://space.bilibili.com/279025/favlist?spm_id_from=333.851.b_696e7465726e6174696f6e616c486561646572.15",
"Accept-Encoding: gzip, deflate, br",
"Accept-Language: zh-CN,zh;q=0.9,en-US;q=0.8,en;q=0.7,zh-TW;q=0.6,ja;q=0.5",
// "Cookie: SESSDATA=64a15917%2C1578628130%2Ceb05cdc1"
);
if ($userCookie) {
$cookie = Redis::connection()->get("bilibili_cookie");
$headerArray[] = "Cookie: $cookie";
// $headerArray[] = "Cookie: SESSDATA=94247a4e%2C1651981649%2C1dba1%2Ab1;";
// $headerArray[] = "Cookie: _uuid=D6E5438B-5A95-439F-7512-FC3509457A9A28409infoc; buvid3=5566647C-DDE5-4AFF-8711-89C9DB2B7061110244infoc; LIVE_BUVID=AUTO3415734420289108; UM_distinctid=16e62db8b9934b-0f76f63ae51bee-1c3c6a5a-13c680-16e62db8b9acaf; CURRENT_FNVAL=16; stardustvideo=1; rpdid=|(ku|l|lRYlJ0J'ul~JYuYY|u; im_notify_type_279025=0; sid=llgc5h9q; laboratory=1-1; CURRENT_QUALITY=80; INTVER=1; DedeUserID=279025; DedeUserID__ckMd5=9a79e15294e6b8bb; SESSDATA=b169300a%2C1581262828%2C3654f611; bili_jct=597a5b9adb6170698e396fb053bc4aba; bp_t_offset_279025=343238964368145599";
}
curl_setopt_array($curl, array(
CURLOPT_URL => $url,
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 0,
CURLOPT_FOLLOWLOCATION => true,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "GET",
CURLOPT_HTTPHEADER => $headerArray
));
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
\Log::error($err);
return null;
}
return $response;
}
public function downloadBSitePlaylist($aidList, $dir = "/Volumes/intel660p/video/mv/mp4", $subDir)
{
dump("downloadBSitePlaylist : xxxxxxxxxxxxxxxxxxxxxxxxxxxx");
$dirExists = is_dir($dir);
if ($dirExists) {
$innerDir = $dir . "/" . $subDir;
if (!is_dir($innerDir)) {
mkdir($innerDir);
}
foreach ($aidList as $aid) {
Log::info("current download command is : cd '{$innerDir}' && annie -r https://www.bilibili.com/video/av80815149 -p " . $aid);
$downloadResult = shell_exec('cd "' . $innerDir . '" && annie -r https://www.bilibili.com/video/av80815149 -p ' . $aid);
Log::info($downloadResult);
Log::info("current download result: " . $downloadResult);
try {
usleep(random_int(1000, 10000) * 1000);
} catch (Exception $e) {
}
}
}
}
/**
* 分 p 下载视频
* @param $aidMap
* @param string $dir
* @param $subDir
*/
public function partDownloadBSitePlaylist($aidMap, $dir = "/Volumes/intel660p/video/mv/mp4", $subDir)
{
$env = App::environment();
$cookie = Redis::connection()->get("bilibili_cookie");
$dirExists = is_dir($dir);
dump($dirExists);
echo "dir is $dir, and $dirExists";
if ($dirExists) {
$innerDir = $dir . "/" . $subDir;
dump($innerDir);
if (!is_dir($innerDir)) {
mkdir($innerDir);
}
if (!$this->checkDiskSpace($innerDir)) {
Log::info('磁盘空间不足');
exit;
}
echo "当前视频的下载路径是: " . $innerDir . "\n";
Log::info("当前视频的下载路径是: " . $innerDir);
foreach ($aidMap as $aid => $parts) {
dump($aid);
Log::info("current download command is : cd '{$innerDir}' && annie -r https://www.bilibili.com/video/av80815149 -p " . $aid);
if ($env == "local") {
$downloadResult = shell_exec('export http_proxy=http://127.0.0.1:1087; export https_proxy=http://127.0.0.1:1087; cd "' . $innerDir . '" && url="https://www.bilibili.com/video/av' . $aid . '?p="
for i in $(seq 1 ' . $parts . ')
do
annie -f 64 $url$i
done && echo "ok"');
} else {
// 修改单个获取信息，否则会把合集列表都拉过来
$formatArray = [];
for ($i = 1; $i <= $parts; $i++) {
$formatInfoShell = "lux -j -i -c \"{$cookie}\" https://www.bilibili.com/video/av{$aid}?p=$i";
$formatInfoShellResult = shell_exec($formatInfoShell);
$formatInfoList = json_decode($formatInfoShellResult, true);
if (!is_array($formatInfoList)) {
Log::error("$aid format info error " . $formatInfoShellResult);
break;
}
$formatArray[] = $formatInfoList[0];
usleep(500 * 1000);
}
foreach ($formatArray as $formatInfo) {
$keys = array_keys($formatInfo["streams"]);
$largeHev1FormatCode = -1;
$largeFormatCode = -1;
$largeFormatStr = "";
foreach ($keys as $key) {
if (str_contains($key, "-")) {
$formats = explode("-", $key);
// 127-12, 后面的 12 代表 hev1 编码的
// 这边是想要获取到 hev1 编码的最高清晰度格式
if ($formats[1] == "12" && (int)$formats[0] > $largeHev1FormatCode) {
$largeHev1FormatCode = (int)$formats[0];
}
} else {
if ((int)$key > $largeFormatCode) {
$largeFormatCode = $key;
$largeFormatStr = $key;
}
}
}
if ($largeHev1FormatCode == -1) {
$formatArray[] = $largeFormatStr;
} else {
$formatArray[] = $largeHev1FormatCode . "-12";
}
}
$downloadShell = "cd $innerDir";
for ($i = 1; $i <= count($formatArray); $i++) {
$downloadShell .= " && lux -f " . $formatArray[$i - 1] . " -c \"" . $cookie . "\" https://www.bilibili.com/video/av" . $aid . "?p=" . $i;
}
$downloadShell .= " && echo \"ok\"";
$downloadResult = shell_exec($downloadShell);
// $downloadResult = shell_exec('cd "' . $innerDir . '" && url="https://www.bilibili.com/video/av' . $aid . '?p="
//for i in $(seq 1 ' . $parts . ')
//do
//lux -c "'. $cookie.'" $url$i
//done && echo "ok"');
//
}
Log::info($downloadResult);
Log::info("$aid current download result: " . $downloadResult);
$lastLine = explode("\n", $downloadResult)[count(explode("\n", $downloadResult)) - 2];
Log::info("lastLine is $lastLine");
try {
usleep(random_int(1000, 10000) * 1000);
} catch (Exception $e) {
}
if (trim($lastLine) == 'ok') {
return true;
} else {
return false;
}
}
}
}
public function deleteFileNotContainP()
{
$dir = "/Volumes/intel660p/video/mv/mp4";
$files = scandir($dir);
foreach ($files as $file) {
if ($file == "." || $file == "..") {
continue;
}
if (preg_match("#P\d{1,2}#", $file)) {
echo $file . "\n";
} else {
echo "to delete file " . $file . "\n";
unlink($dir . "/" . $file);
}
}
}
public function compareAndDownloadNewVideos()
{
$list = BilibiliCollections::all();
foreach ($list as $item) {
if ($item['is_downloaded'] == 1) {
continue;
}
$undownloaded = array_diff($item['videos'], $item['downloaded_videos']);
}
dump($list);
}
public function commonRequest($url, $cookie, $media_id, $pageNo)
{
$client = new Client([
// Base URI is used with relative requests
'base_uri' => 'https://api.bilibili.com',
// You can set any number of default request options.
'timeout' => 2.0,
]);
$jar = new \GuzzleHttp\Cookie\CookieJar;
$r = $client->request('GET', 'http://httpbin.org/cookies', [
'cookies' => $jar
]);
}
public function insertDBTest()
{
$aNo = 484525;
for ($i = 0; $i < 5000000; $i++) {
$tempANo = $aNo + $i;
$sql = "insert into `bilibili_video_temps`( `aid`, `title`, `from_type`, `from_collection_name`, `from_up_name`, `is_download`, `is_downloaded`, `total_parts`, `created_at`, `updated_at`) VALUES ( " . $tempANo . ", '【钢琴】《鬼灭之刃》OP《红莲华》by LiSA', 2, '', '绯绯', 1, 1, 1, '2020-01-08 10:05:40', '2020-01-13 23:35:17');";
$result = DB::insert($sql);
if ($i % 100 == 0) {
print_r($result);
}
usleep(10);
}
}
public function searchVideoFiles($filename)
{
$list = BilibiliVideos::where("title", "like", "%" . trim($filename) . "%")->get();
dump($list);
}
public function checkDiskSpace($dir = "/Volumes/Crucial X6/Video/bilibili/")
{
if (disk_free_space($dir) > 5 * 1024 * 1024 * 1024) {
return true;
}
return false;
}
}