diff --git a/app/Console/Commands/BiliVideoCode.php b/app/Console/Commands/BiliVideoCode.php index eeecaa7..6053511 100644 --- a/app/Console/Commands/BiliVideoCode.php +++ b/app/Console/Commands/BiliVideoCode.php @@ -55,9 +55,9 @@ class BiliVideoCode extends Command // $bilibili->queryDBCollectionList(); // exit; // $bilibili->queryForVideoParts(); - $bilibili->compareAndDownloadUpVideos(true); + $bilibili->compareAndDownloadUpVideos(); // exit; - $bilibili->compareAndDownloadCollectionVideos(true); + $bilibili->compareAndDownloadCollectionVideos(); // $bilibili->queryCollectionList(); // $bilibili->compareAndDownloadNewVideos(); // $bilibili->queryLocalUpVideoList(); diff --git a/app/Console/Commands/ReplaceBlog.php b/app/Console/Commands/ReplaceBlog.php index a55242f..203730c 100644 --- a/app/Console/Commands/ReplaceBlog.php +++ b/app/Console/Commands/ReplaceBlog.php @@ -40,6 +40,6 @@ class ReplaceBlog extends Command { // $service = new ReplaceBlogService(); - $service->replaceFrontMatter("/Users/shixuesen/workspace/dev/src/myHugoBlog/content/posts"); + $service->replaceFrontMatter("/Users/shixuesen/workspace/dev/src/myHugoBlog/content/temp"); } } diff --git a/app/Console/Commands/XiuGirlScrape.php b/app/Console/Commands/XiuGirlScrape.php index ce24d94..e8fd83e 100644 --- a/app/Console/Commands/XiuGirlScrape.php +++ b/app/Console/Commands/XiuGirlScrape.php @@ -40,6 +40,39 @@ class XiuGirlScrape extends Command public function handle() { // +// $curl = curl_init(); + +// curl_setopt_array($curl, array( +// CURLOPT_URL => 'https://img.xsnvshen.com/album/22162/35724/000.jpg', +// CURLOPT_RETURNTRANSFER => true, +// CURLOPT_ENCODING => '', +// CURLOPT_MAXREDIRS => 10, +// CURLOPT_TIMEOUT => 0, +// CURLOPT_FOLLOWLOCATION => true, +// CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1, +// CURLOPT_CUSTOMREQUEST => 'GET', +// CURLOPT_HTTPHEADER => array( +// 'authority: img.xsnvshen.com', +// 'pragma: no-cache', +// 'cache-control: no-cache', +// 'sec-ch-ua: " Not A;Brand";v="99", "Chromium";v="90", "Google Chrome";v="90"', +// 'sec-ch-ua-mobile: ?0', +// 'upgrade-insecure-requests: 1', +// 'user-agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.212 Safari/537.36', +// 'accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9', +// 'sec-fetch-site: same-site', +// 'sec-fetch-mode: navigate', +// 'sec-fetch-user: ?1', +// 'sec-fetch-dest: document', +// 'referer: https://www.xsnvshen.com/album/35724/', +// 'accept-language: zh-CN,zh;q=0.9', +//// 'cookie: __cfduid=dac5872f65e79a40a5b30229ba97beb6a1619333692' +// ), +// )); +// +// $response = curl_exec($curl); +// file_put_contents("/Users/shixuesen/workspace/dev/src/graph-tutorial/0000000112.jpg", $response); +// exit; $xiu1 = new NewXiuGirlsService(); $xiu1->scrapeXiuGirls();exit; $xiu = new XiuGirlsService(); diff --git a/app/Http/Controllers/WeiboController.php b/app/Http/Controllers/WeiboController.php index 0b6b43b..df94f30 100644 --- a/app/Http/Controllers/WeiboController.php +++ b/app/Http/Controllers/WeiboController.php @@ -32,7 +32,7 @@ class WeiboController extends Controller // for( $i = 341; $i >= 1 ; $i-- ) // $url[] = 'https://m.weibo.cn/api/container/getIndex?containerid=1076035893812490&openApp=0&page='.$i; - for( $i = 20; $i >= 1 ; $i-- ) + for( $i = 1; $i >= 1 ; $i-- ) $url[] = 'https://m.weibo.cn/api/container/getIndex?containerid=230259&openApp=0&page='.$i; // for ($i = 5; $i >= 1; $i--) { // $url[] = 'https://m.weibo.cn/feed/group?gid=4423532052076817&&page=' . $i; diff --git a/app/Services/BilibiliServiceV2.php b/app/Services/BilibiliServiceV2.php index 3820609..bea87ea 100644 --- a/app/Services/BilibiliServiceV2.php +++ b/app/Services/BilibiliServiceV2.php @@ -71,7 +71,7 @@ class BilibiliServiceV2 $url = "https://api.bilibili.com/x/space/arc/search?mid={$mediaId}&ps=30&tid=0&keyword=&order=pubdate&jsonp=jsonp&pn="; // https://space.bilibili.com/475250/video - for ($i = 1; $i < 10; $i++) { + for ($i = 1; $i < 5; $i++) { $curl = curl_init(); curl_setopt_array($curl, array( @@ -533,7 +533,7 @@ class BilibiliServiceV2 foreach ($list as $item) { Log::info("schedule queryDBCollectionList current collection is: " . $item['title'] . " started at :" . date("Y-m-d H:i:s")); $currentList = []; - for ($pageNo = 1; $pageNo < 100; $pageNo++) { + for ($pageNo = 1; $pageNo < 10; $pageNo++) { $listResponse = $this->requestCollectionList($this->collectionListUrl, $item['media_id'], $pageNo); // dump("pageNo: " . $pageNo); // dump($listResponse); @@ -805,7 +805,7 @@ class BilibiliServiceV2 // "Cookie: SESSDATA=64a15917%2C1578628130%2Ceb05cdc1" ); if ($userCookie) { - $headerArray[] = "Cookie: SESSDATA=75ed2a5a%2C1620866280%2Cc0fab*b1;"; + $headerArray[] = "Cookie: SESSDATA=216ce1c5%2C1636429756%2Cb86c3%2A51;"; // $headerArray[] = "Cookie: _uuid=D6E5438B-5A95-439F-7512-FC3509457A9A28409infoc; buvid3=5566647C-DDE5-4AFF-8711-89C9DB2B7061110244infoc; LIVE_BUVID=AUTO3415734420289108; UM_distinctid=16e62db8b9934b-0f76f63ae51bee-1c3c6a5a-13c680-16e62db8b9acaf; CURRENT_FNVAL=16; stardustvideo=1; rpdid=|(ku|l|lRYlJ0J'ul~JYuYY|u; im_notify_type_279025=0; sid=llgc5h9q; laboratory=1-1; CURRENT_QUALITY=80; INTVER=1; DedeUserID=279025; DedeUserID__ckMd5=9a79e15294e6b8bb; SESSDATA=b169300a%2C1581262828%2C3654f611; bili_jct=597a5b9adb6170698e396fb053bc4aba; bp_t_offset_279025=343238964368145599"; } diff --git a/app/Services/CommonService.php b/app/Services/CommonService.php index 96814e3..dba2b47 100644 --- a/app/Services/CommonService.php +++ b/app/Services/CommonService.php @@ -21,8 +21,20 @@ class CommonService // $cookieArr = parseCookieHeader($cookieStr); // $cookieJar = CookieJar::fromArray($cookieArr, ".v2ph.com"); $client = new GuzzleHttp\Client(['headers' => [ - 'User-Agent' => 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.109 Safari/537.36', - 'Referer' => $host, + 'User-Agent' => 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.212 Safari/537.36', +// 'Referer' => $host, + 'authority' => 'img.xsnvshen.com', + 'pragma' => 'no-cache', + 'cache-control' => 'no-cache', + 'sec-ch-ua' => ' Not A;Brand";v="99", "Chromium";v="90", "Google Chrome";v="90"', + 'sec-ch-ua-mobile' => '?0', + 'accept' => 'image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8', + 'sec-fetch-site' => 'same-site', + 'sec-fetch-mode' => 'no-cors', + 'sec-fetch-dest' => 'image', + 'referer' => 'https://www.xsnvshen.com/album/35686', + 'accept-language' => 'zh-CN,zh;q=0.9', + 'cookie' => '__cfduid=dac5872f65e79a40a5b30229ba97beb6a1619333692' // 'cookies' => $cookieJar ]]); $fileParts = pathinfo($imageUrl); diff --git a/app/Services/NewXiuGirlsService.php b/app/Services/NewXiuGirlsService.php index 50c004c..4d8bd2e 100644 --- a/app/Services/NewXiuGirlsService.php +++ b/app/Services/NewXiuGirlsService.php @@ -145,7 +145,7 @@ class NewXiuGirlsService } $opts = array("http"=> ["header" => - "User-Agent:Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.109 Safari/537.36 \r\n + "User-Agent:Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.212 Safari/537.36 \r\n Referer:".$baseUrl.$item."\r\n" ]); $this->processDownloadImage($image, $baseUrl, $item, $albumPath, $username, $albumCode); @@ -279,6 +279,8 @@ class NewXiuGirlsService } echo curl_error($curl_handle); + echo $query;exit; + exit; $fp = fopen($albumPath . "/" . $username . "-" . $albumCode . "-" . pathinfo("http:".$image)["filename"].".jpg", "x"); fwrite($fp, $query); fclose($fp); @@ -465,12 +467,39 @@ class NewXiuGirlsService */ private function processDownloadImage(string $image, string $baseUrl, $item, string $albumPath, string $username, string $albumCode): void { +// CommonService::downloadImage($albumPath, "https:" . $image);exit; $curl_handle = curl_init(); - curl_setopt($curl_handle, CURLOPT_URL, "https:" . $image); - curl_setopt($curl_handle, CURLOPT_CONNECTTIMEOUT, 2000); - curl_setopt($curl_handle, CURLOPT_RETURNTRANSFER, 1); - curl_setopt($curl_handle, CURLOPT_USERAGENT, "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.109 Safari/537.36"); - curl_setopt($curl_handle, CURLOPT_REFERER, $baseUrl . $item); +// curl_setopt($curl_handle, CURLOPT_URL, "https:" . $image); +// curl_setopt($curl_handle, CURLOPT_CONNECTTIMEOUT, 20000); +// curl_setopt($curl_handle, CURLOPT_RETURNTRANSFER, 1); +// curl_setopt($curl_handle, CURLOPT_USERAGENT, "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.212 Safari/537.36"); +// curl_setopt($curl_handle, CURLOPT_REFERER, $baseUrl . $item); + echo "image url is " . $image . "\n"; + curl_setopt_array($curl_handle, array( + CURLOPT_URL => "https:" . $image, + CURLOPT_RETURNTRANSFER => true, + CURLOPT_ENCODING => '', + CURLOPT_MAXREDIRS => 10, + CURLOPT_TIMEOUT => 0, + CURLOPT_FOLLOWLOCATION => true, + CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1, + CURLOPT_CUSTOMREQUEST => 'GET', + CURLOPT_HTTPHEADER => array( + 'authority: img.xsnvshen.com', + 'pragma: no-cache', + 'cache-control: no-cache', + 'sec-ch-ua: " Not A;Brand";v="99", "Chromium";v="90", "Google Chrome";v="90"', + 'sec-ch-ua-mobile: ?0', + 'user-agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.212 Safari/537.36', + 'accept: image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8', + 'sec-fetch-site: same-site', + 'sec-fetch-mode: no-cors', + 'sec-fetch-dest: image', + 'referer: https://www.xsnvshen.com/album/' . $item, + 'accept-language: zh-CN,zh;q=0.9', +// 'cookie: __cfduid=dac5872f65e79a40a5b30229ba97beb6a1619333692' + ), + )); $content = curl_exec($curl_handle); $i = 1; while ($content === false) { @@ -485,8 +514,10 @@ class NewXiuGirlsService break; } } +// $content = file_get_contents("https:" . $image); - echo curl_error($curl_handle); +// echo curl_error($curl_handle); +// echo $content;exit; $fp = fopen($albumPath . "/" . $username . "-" . $albumCode . "-" . pathinfo("http:" . $image)["filename"] . ".jpg", "x"); fwrite($fp, $content); fclose($fp); diff --git a/app/Services/RemoveLargeMediaService.php b/app/Services/RemoveLargeMediaService.php new file mode 100644 index 0000000..6194030 --- /dev/null +++ b/app/Services/RemoveLargeMediaService.php @@ -0,0 +1,226 @@ +bili = new BilibiliServiceV2(); + } + + public function handleVideos($dir = "/Users/shixuesen/Documents/tmp/柚木/2017/泡泡条纹袜/") + { + $files = scandir($dir); + foreach ($files as $file) { + if ($file == "." || $file == "..") { + continue; + } + $subDir = implode("/", [$dir, $file]); + $isDir = is_dir($subDir); + if ($isDir) { + $subFiles = scandir($subDir); + foreach ($subFiles as $subFile) { + $subPathFile = implode("/", [$subDir, $subFile]); + if (is_dir($subPathFile) || $subFile == ".DS_Store") { + continue; + } + $mime = mime_content_type($subPathFile); +// dump("file type", [$mime, $subPathFile]); +// continue; + if (strstr($mime, "video/")) { + if (is_file($subPathFile)) { + $fileInfo = pathinfo($subPathFile); + dump("fileInfo", $fileInfo); + if (ends_with($fileInfo["filename"], "-1")) { + continue; + } + if (is_file($fileInfo["dirname"] . '/' .$fileInfo["filename"] . '-1'. '.' . $fileInfo["extension"])) { + unlink($subPathFile); + continue; + } + $targetFile = $fileInfo["dirname"] . '/' .$fileInfo["filename"] . '-1'. '.' . $fileInfo["extension"]; + dump("targetFile", [$targetFile]); +// $result = shell_exec("handBrakeCli -Z 'Very Fast 720p30' -i '". $subPathFile ."' -o '". $targetFile . " && echo 'success'"); + $result = shell_exec("handBrakeCli -Z 'Very Fast 720p30' -i '". $subPathFile ."' -o '". $targetFile . "'"); + dump($result); + + } + } + } + } + } + print_r($files); + } + +// public function processDir($baseDir = "/Volumes/WD/Video/HuaVid/") + // public function processDir($baseDir = "/Volumes/intel660p/video/mv/bilibili/tara") + public function processDir($baseDir = "/Volumes/WD/tmp/bilibili/aoa") + { + $files = scandir($baseDir); +// $files = array_slice($files, 5670); + $count = count($files); + dump("all count is $count"); + foreach ($files as $index => $file) { + if ($file == "." || $file == ".." || $file == ".DS_Store") { + continue; + } + $subDir = implode("/", [$baseDir, $file]); + $isDir = is_dir($subDir); + if ($isDir) { +// $this->processDir($subDir); + } else { + dump("[all count] is $count, [current index] is $index"); + $this->processVideo($subDir); + } + } + } + + public function processVideo($pathFile) + { + $mime = mime_content_type($pathFile); +// dump("file type", [$mime, $pathFile]);exit; +// continue; + $mediaInfo = new MediaInfo(); + $mediaInfo->setConfig('use_oldxml_mediainfo_output_format', true); + if (strstr($mime, "video/") || strstr($mime, "application/octet-stream")) { + if (is_file($pathFile)) { + $mediaContainer = $mediaInfo->getInfo($pathFile); + foreach ($mediaContainer->getVideos() as $video) { + $height = $video->get('height')->getAbsoluteValue(); + $width = $video->get('width')->getAbsoluteValue(); + if ($height > $width && $width <= 720) { + echo "$pathFile 分辨率小于 720p 跳过\n"; + return; + } + if ($height <= $width && $height <= 720) { + echo "$pathFile 分辨率小于 720p 跳过\n"; + return; + } + + } + $fileInfo = pathinfo($pathFile); + dump("fileInfo", $fileInfo); + if (ends_with($fileInfo["filename"], "-XXXXX#compressed")) { + return; + } + if (starts_with($fileInfo["filename"], "171221 M!CountDown 光州特辑女团现场合集") || starts_with($fileInfo["filename"], "Rainbow - A 现场合集 P14 Rainbow - A + Mach")) { + return; + } + if (Redis::sismember("unneed", $fileInfo["filename"])) { + return; + } + $targetFile = $fileInfo["dirname"] . '/' .$fileInfo["filename"] . '-XXXXX#compressed'. '.' . $fileInfo["extension"]; + if (is_file($targetFile)) { + unlink($pathFile); + rename($targetFile, $pathFile); + return; + } + dump("targetFile", [$targetFile]); +// $result = shell_exec("handBrakeCli -Z 'Very Fast 720p30' -i '". $subPathFile ."' -o '". $targetFile . " && echo 'success'"); + $result = shell_exec("handBrakeCli --json -Z 'Very Fast 720p30' -i '". $pathFile ."' -o '". $targetFile . "'"); +// echo $result; + preg_match_all("#\"State\"\: \"WORKDONE\"#", $result, $match); + if (count($match) > 0) { + echo "compress work done remove the file \n"; + $oldFileSize = filesize($pathFile); + $newFileSize = filesize($targetFile); + if ($newFileSize >= $oldFileSize) { + Redis::sadd("unneed", $fileInfo["filename"]); + echo "old file size is smaller than new one, old is " . file_size($oldFileSize) . " and new is " . file_size($newFileSize) . ", now remove new one"; + unlink($targetFile); + } else { + echo "new file size is smaller than old one, new is " . file_size($newFileSize) . " and old is " . file_size($oldFileSize) . ", now remove old one"; + unlink($pathFile); + rename($targetFile, $pathFile); + } + } + } + } + } + + function human_filesize($bytes, $decimals = 2) { + $sz = 'BKMGTP'; + $factor = floor((strlen($bytes) - 1) / 3); + return sprintf("%.{$decimals}f", $bytes / pow(1024, $factor)) . @$sz[$factor]; + } + + public function processUnCompleteDir($baseDir = "/Volumes/WD/tmp/探花系列【AI高清2K修复】大合集") +// public function processDir($baseDir = "/Volumes/Backup/iPhone nPlayer/") + { + $files = scandir($baseDir); + foreach ($files as $file) { + if ($file == "." || $file == "..") { + continue; + } + $subDir = implode("/", [$baseDir, $file]); + $isDir = is_dir($subDir); + if ($isDir) { + $this->processUnCompleteDir($subDir); + } else { + $this->processUnCompleteVideo($subDir); + } + } + } + + public function processUnCompleteVideo($pathFile) + { + //... + $mediaInfo = new MediaInfo(); + $mediaInfo->setConfig('use_oldxml_mediainfo_output_format', true); + + $mime = mime_content_type($pathFile); +// dump("file type", [$mime, $subPathFile]); +// continue; + if (strstr($mime, "video/")) { + if (is_file($pathFile)) { + $fileInfo = pathinfo($pathFile); +// dump("fileInfo", $fileInfo); + if (ends_with($fileInfo["filename"], "-1")) { + return; + } + if (is_file($fileInfo["dirname"] . '/' .$fileInfo["filename"] . '-1'. '.' . $fileInfo["extension"])) { + $mediaInfoContainer1 = $mediaInfo->getInfo($fileInfo["dirname"] . '/' .$fileInfo["filename"] . '-1'. '.' . $fileInfo["extension"]); + $millSecond1 = $mediaInfoContainer1->getGeneral()->get("duration")->getMilliseconds(); + echo gettype($millSecond1) . "\n"; +// ["duration"] . "\n"; + $mediaInfoContainer = $mediaInfo->getInfo($pathFile); + $millSecond = $mediaInfoContainer->getGeneral()->get("duration")->getMilliseconds(); + echo gettype($millSecond) . "\n"; + if (abs(intval($millSecond) - intval($millSecond1)) > 100) { + echo $pathFile . "\n"; + echo abs(intval($millSecond) - intval($millSecond1)) . "\n"; + } + +// unlink($pathFile); + return; + } +// $targetFile = $fileInfo["dirname"] . '/' .$fileInfo["filename"] . '-1'. '.' . $fileInfo["extension"]; +// dump("targetFile", [$targetFile]); +//// $result = shell_exec("handBrakeCli -Z 'Very Fast 720p30' -i '". $subPathFile ."' -o '". $targetFile . " && echo 'success'"); +// $result = shell_exec("handBrakeCli -Z 'Very Fast 720p30' -i '". $pathFile ."' -o '". $targetFile . "'"); +// dump($result); + + } + } + } + + public function checkFileInfo($pathFile) + { + preg_match_all("#([\S|\s]+)P\d+#", $pathFile, $match); + if (count($match) > 1 && count($match[1]) > 0) { + $this->bili->searchVideoFiles($match[1][0]); + } + + +// dump($match[1][0]); + } + + +} diff --git a/app/Services/ReplaceBlogService.php b/app/Services/ReplaceBlogService.php index 1c2a499..05587c5 100644 --- a/app/Services/ReplaceBlogService.php +++ b/app/Services/ReplaceBlogService.php @@ -28,7 +28,7 @@ class ReplaceBlogService { // dump($yaml); // $html = $document->getContent(); // dump($html); - exit; +// exit; } } diff --git a/app/Services/ReplaceCompressedVideoService.php b/app/Services/ReplaceCompressedVideoService.php index e8eb9e2..72570e1 100644 --- a/app/Services/ReplaceCompressedVideoService.php +++ b/app/Services/ReplaceCompressedVideoService.php @@ -64,7 +64,7 @@ class ReplaceCompressedVideoService public function processDir($baseDir = "/Volumes/WD/tmp/bilibili/aoa") { $files = scandir($baseDir); - $files = array_slice($files, 3035); +// $files = array_slice($files, 5670); $count = count($files); dump("all count is $count"); foreach ($files as $index => $file) { @@ -74,9 +74,9 @@ class ReplaceCompressedVideoService $subDir = implode("/", [$baseDir, $file]); $isDir = is_dir($subDir); if ($isDir) { -// $this->processDir($subDir); + $this->processDir($subDir); } else { - dump("[all count] is $count, [current index] is $index"); +// dump("[all count] is $count, [current index] is $index"); $this->processVideo($subDir); } } diff --git a/app/Services/WeiboService.php b/app/Services/WeiboService.php index 5a10fb7..8d522d9 100644 --- a/app/Services/WeiboService.php +++ b/app/Services/WeiboService.php @@ -16,6 +16,7 @@ class WeiboService // if (!isset($_REQUEST['content'])) die('NoData'); // // $content = trim($_REQUEST['content']); + echo $content;exit; // @mkdir('data'); @@ -84,12 +85,13 @@ class WeiboService # code... $page_info = $mblog["page_info"]; $media_info = $mblog["page_info"]["media_info"]; - $video_url = Arr::get($media_info, "mp4_720p_mp4", ""); + $medis_urls = $mblog["page_info"]["urls"]; + $video_url = Arr::get($medis_urls, "mp4_720p_mp4", ""); if ($video_url == "") { - $video_url = Arr::get($media_info, "mp4_hd_url", ""); + $video_url = Arr::get($medis_urls, "mp4_hd_url", ""); } if ($video_url == "") { - $video_url = Arr::get($media_info, "mp4_sd_url", ""); + $video_url = Arr::get($medis_urls, "mp4_ld_mp4", ""); } if ($video_url != "") { self::process_video($video_url, $video_dir, $page_info["content2"]); diff --git a/app/Services/XiurenjiService.php b/app/Services/XiurenjiService.php index 7d4b0a7..5c81b0c 100644 --- a/app/Services/XiurenjiService.php +++ b/app/Services/XiurenjiService.php @@ -11,8 +11,8 @@ use QL\QueryList; class XiurenjiService { - public $domainUrl = "https://www.xiurenji.com"; - public $xiurenRootUrl = "https://www.xiurenji.com/XiuRen/"; + public $domainUrl = "https://www.xiurenji.cc"; + public $xiurenRootUrl = "https://www.xiurenji.cc/XiuRen/"; // public $rootDir = "/Users/shixuesen/Documents/tmp/xiuren/"; public $rootDir = "/Volumes/Backup/images/xiuren/"; public $queryInstance; @@ -24,7 +24,7 @@ class XiurenjiService public function scrapeAlbum() { $pageSize = 20; - $pageCount = $this->getEncodeHtmlContent("https://www.xiurenji.com/XiuRen/index.html")->find(".page span")->htmls()->get(0); + $pageCount = $this->getEncodeHtmlContent("https://www.xiurenji.cc/XiuRen/index.html")->find(".page span")->htmls()->get(0); print_r($pageCount); if ((int)$pageCount > 0) { for ($i = 0; $i <= ceil($pageCount / $pageSize); $i++) { @@ -112,7 +112,7 @@ class XiurenjiService usleep(random_int(10, 100) * 100); $imageUrl = $image->getAttribute("src"); $trueImageUrl = "https://x1.plmn5.com/U". substr($imageUrl, 2); - $trueImageUrl = "https://www.xiurenji.com" . $imageUrl; + $trueImageUrl = "https://www.xiurenji.cc" . $imageUrl; $fileInfo = pathinfo($trueImageUrl); if (file_exists($dir . "/" .$fileInfo["basename"])) { rename($dir . "/" . $fileInfo["basename"], $dir . "/" . $imageNo . "-" . $fileInfo["basename"]); diff --git a/fail.log b/fail.log index df41a0d..42a507d 100644 --- a/fail.log +++ b/fail.log @@ -1021,3 +1021,13 @@ /Users/shixuesen/OneDrive/Pictures/instagram/Likes/ https://scontent-lax3-1.cdninstagram.com/v/t51.2885-15/e35/173205557_167251781933216_1150698725227016954_n.jpg?se=7&tp=1&_nc_ht=scontent-lax3-1.cdninstagram.com&_nc_cat=1&_nc_ohc=WtORZ2J3Y-UAX_RatNU&edm=APv5SkIAAAAA&ccb=7-4&oh=2b1b884668df7e8e974905edcaf5497f&oe=609CF6BA&_nc_sid=7479f6&ig_cache_key=MjU1MDc4NjM3ODEwNzk3NDI5Nw%3D%3D.2-ccb7-4 /Users/shixuesen/OneDrive/Pictures/instagram/ngoctrinh89/ https://scontent-lax3-1.cdninstagram.com/v/t50.2886-16/98618760_3227004884069355_6044999634251674592_n.mp4?efg=eyJ2ZW5jb2RlX3RhZyI6InZ0c192b2RfdXJsZ2VuLjcyMC5zdG9yeS5kZWZhdWx0In0&_nc_ht=scontent-lax3-1.cdninstagram.com&_nc_cat=103&_nc_ohc=O_mfjDVyUb0AX8STxQP&edm=AOVtZ6oAAAAA&vs=17882073455230920_638136539&_nc_vs=HBksFQAYJEdJak40QVhycTgyZjhYWUxBT0R6NzdvakpfUlRidXFIQUFBQRUAAsgBABUAGCRHRlpSZUFwdVlmS0p1d1FCQUs0T2dWY3R2ZVlIYnBrd0FBQUYVAgLIAQAoABgAGwGIB3VzZV9vaWwBMRUAACaQ3Oj61OnDPxUCKAJDMywXQCuZmZmZmZoYEmRhc2hfYmFzZWxpbmVfMl92MREAdegHAA%3D%3D&_nc_rid=76a0e8f923&ccb=7-4&oe=60834C43&oh=d715adacda8bb3840e0bf87c8f995ad5&_nc_sid=bab638 /Users/shixuesen/OneDrive/Pictures/instagram/bivi_0420/ https://scontent-lax3-1.cdninstagram.com/v/t50.2886-16/74963866_2803615776546370_4938313936994023105_n.mp4?efg=eyJ2ZW5jb2RlX3RhZyI6InZ0c192b2RfdXJsZ2VuLjcyMC5zdG9yeS5kZWZhdWx0In0&_nc_ht=scontent-lax3-1.cdninstagram.com&_nc_cat=110&_nc_ohc=Joh1OfK2nVQAX9xVW53&edm=AOVtZ6oBAAAA&vs=17893525949035078_1378283097&_nc_vs=HBkcFQAYJEdKcmJkd1JDbm5LcTMtVUpBTUcyNDROY2FvaEVidXFIQUFBQRUAAsgBACgAGAAbAYgHdXNlX29pbAExFQAAJoyMwoulhMk%2FFQIoAkMzLBdAFbtkWhysCBgSZGFzaF9iYXNlbGluZV8xX3YxEQB16AcA&_nc_rid=a086d64b7d&ccb=7-4&oe=608B28BB&oh=b8298b616f2d5ccfb99724c1ee61ee8a&_nc_sid=bab638 +/Users/shixuesen/OneDrive/Pictures/instagram/weiman_the_real_one/ https://scontent-lax3-1.cdninstagram.com/v/t50.2886-16/105864276_881068615809681_429462755360194612_n.mp4?efg=eyJ2ZW5jb2RlX3RhZyI6InZ0c192b2RfdXJsZ2VuLjcyMC5zdG9yeS5kZWZhdWx0In0&_nc_ht=scontent-lax3-1.cdninstagram.com&_nc_cat=109&_nc_ohc=ClEGxANuV4gAX8LHX2j&edm=AOVtZ6oBAAAA&vs=17888464805151887_1594195415&_nc_vs=HBksFQAYJEdGUmNUd2FST3Q3RlV5RURBRFJBZ2NFWnd2VUZidXFIQUFBQRUAAsgBABUAGCRHTXZsNEFwY2NSdFBOSFVBQUtWWDI3TmRFdk5tYnBrd0FBQUYVAgLIAQAoABgAGwGIB3VzZV9vaWwBMRUAACaewNWl2N3GPxUCKAJDMywXQC4AAAAAAAAYEmRhc2hfYmFzZWxpbmVfMV92MREAdegHAA%3D%3D&_nc_rid=0321597446&ccb=7-4&oe=609370FE&oh=fb6468d93941af1f109b680c395d0a05&_nc_sid=bab638 +/Users/shixuesen/OneDrive/Pictures/instagram/weiman_the_real_one/ https://scontent-lax3-1.cdninstagram.com/v/t50.2886-16/181075290_128264609354392_7933344908465807688_n.mp4?efg=eyJ2ZW5jb2RlX3RhZyI6InZ0c192b2RfdXJsZ2VuLjcyMC5mZWVkLmRlZmF1bHQifQ&_nc_ht=scontent-lax3-1.cdninstagram.com&_nc_cat=104&_nc_ohc=UIHfK-iP284AX9lknaP&edm=ABmJApABAAAA&vs=17893667830984768_2991277892&_nc_vs=HBksFQAYJEdGcjl5Z3FZVW5ydXAzUUFBRWdod2wwcDV4aHVia1lMQUFBRhUAAsgBABUAGCRHQ0ZpdGdvOXBFbnlZVmdCQUtHQ3k5eXRJSXgxYmtZTEFBQUYVAgLIAQAoABgAGwGIB3VzZV9vaWwBMBUAACaApb%2FzxYzJPxUCKAJDMywXQCWZmZmZmZoYEmRhc2hfYmFzZWxpbmVfMV92MREAdeoHAA%3D%3D&_nc_rid=ce87b51b0f&ccb=7-4&oe=6093B1F8&oh=d89c6aa36976992b1c43124d69e23a97&_nc_sid=6136e7 +/Users/shixuesen/OneDrive/Pictures/instagram/amberchenslife/ https://scontent-lax3-1.cdninstagram.com/v/t50.2886-16/102138368_299639625077541_5560331099598882509_n.mp4?efg=eyJ2ZW5jb2RlX3RhZyI6InZ0c192b2RfdXJsZ2VuLjcyMC5zdG9yeS5kZWZhdWx0In0&_nc_ht=scontent-lax3-1.cdninstagram.com&_nc_cat=109&_nc_ohc=_PtYCk9MayMAX-kaOoM&edm=AOVtZ6oBAAAA&vs=17887276850162485_3780522526&_nc_vs=HBkcFQAYJEdBQ0NGZ1lsMDJsSmhSQUJBTTBHSnRtc1F5cE5idXFIQUFBQRUAAsgBACgAGAAbAYgHdXNlX29pbAExFQAAJuqQnbHFmMY%2FFQIoAkMzLBdAGAAAAAAAABgSZGFzaF9iYXNlbGluZV8xX3YxEQB16AcA&_nc_rid=41b6792dd3&ccb=7-4&oe=6093B623&oh=1ff403f0fe9ce390d2bad1f7ed20d70b&_nc_sid=bab638 +/Users/shixuesen/OneDrive/Pictures/instagram/li_yen_chin/ https://scontent-lax3-1.cdninstagram.com/v/t50.2886-16/181772162_826396801641956_8580539285203080817_n.mp4?efg=eyJ2ZW5jb2RlX3RhZyI6InZ0c192b2RfdXJsZ2VuLjcyMC5mZWVkLmRlZmF1bHQifQ&_nc_ht=scontent-lax3-1.cdninstagram.com&_nc_cat=108&_nc_ohc=OdVYeenYUUEAX8NcBuf&edm=ABmJApABAAAA&vs=17906907298755762_3635376210&_nc_vs=HBksFQAYJEdJS2YxUXJrTGM1LW11OENBSEZXMEtzRU14UjNia1lMQUFBRhUAAsgBABUAGCRHRHl5NGdxMy1rTVU1TWtBQUFwQWhyUVZNUk5UYmtZTEFBQUYVAgLIAQAoABgAGwGIB3VzZV9vaWwBMBUAACbk5aTHl4%2FPPxUCKAJDMywXQEURBiTdLxsYEmRhc2hfYmFzZWxpbmVfMV92MREAdeoHAA%3D%3D&_nc_rid=04f8da9520&ccb=7-4&oe=6093C9B7&oh=15626792ef2f5adaa3d148ef50e1b4d7&_nc_sid=6136e7 +/Users/shixuesen/OneDrive/Pictures/instagram/Likes/ https://scontent-lax3-1.cdninstagram.com/v/t51.2885-15/e35/183146074_211586803843009_3207802134815944041_n.jpg?se=7&tp=1&_nc_ht=scontent-lax3-1.cdninstagram.com&_nc_cat=101&_nc_ohc=A9T-y27nw0IAX-HTT-3&edm=APv5SkIBAAAA&ccb=7-4&oh=848b12687895f16db0e004fe959cf027&oe=60BCE9FC&_nc_sid=7479f6&ig_cache_key=MjU2OTEyMTIxNTY2MzYxNTcxNw%3D%3D.2-ccb7-4 +/Users/shixuesen/OneDrive/Pictures/instagram/Likes/ https://scontent-lax3-1.cdninstagram.com/v/t51.2885-15/e35/182672085_2873338019595533_1153522599175245989_n.jpg?se=7&tp=1&_nc_ht=scontent-lax3-1.cdninstagram.com&_nc_cat=105&_nc_ohc=5QYtpOBeb8UAX_v4t2g&edm=APv5SkIBAAAA&ccb=7-4&oh=8ca94038bb3bb6410604322c03d8fcf1&oe=60BD4EB5&_nc_sid=7479f6&ig_cache_key=MjU2OTEyMTIxNTY5NzMxOTc5Mg%3D%3D.2-ccb7-4 +/Users/shixuesen/OneDrive/Pictures/instagram/yui_xin_/ https://scontent-lax3-1.cdninstagram.com/v/t50.2886-16/88687982_197855032162484_8949640507211987845_n.mp4?efg=eyJ2ZW5jb2RlX3RhZyI6InZ0c192b2RfdXJsZ2VuLjcyMC5zdG9yeS5kZWZhdWx0In0&_nc_ht=scontent-lax3-1.cdninstagram.com&_nc_cat=108&_nc_ohc=0pn5YaWTpgQAX8J3ZAs&edm=AOVtZ6oBAAAA&vs=17904611140826159_2935624126&_nc_vs=HBkcFQAYJEdHNUZTUVcwdU9pMjhyTUFBSVU3MGNPZmdqTjhidXFIQUFBQRUAAsgBACgAGAAbAYgHdXNlX29pbAExFQAAJt7xhonEic4%2FFQIoAkMzLBdAFAAAAAAAABgSZGFzaF9iYXNlbGluZV8yX3YxEQB16AcA&_nc_rid=f51dea0338&ccb=7-4&oe=609C5894&oh=639694261f1886295a17b90d0b507a14&_nc_sid=bab638 +/Users/shixuesen/OneDrive/Pictures/instagram/boram__jj/ https://scontent-lax3-1.cdninstagram.com/v/t51.2885-15/e35/186884661_499567364512012_4467085033471998071_n.jpg?se=7&tp=1&_nc_ht=scontent-lax3-1.cdninstagram.com&_nc_cat=104&_nc_ohc=A2LAd2ZogGYAX-3Zqbk&edm=ABmJApABAAAA&ccb=7-4&oh=7b5448b331d61770b0aebb5b2ab2ff95&oe=60C3C41D&_nc_sid=6136e7&ig_cache_key=MjU3MzE2NDM4Nzg2MDUzMDY3OA%3D%3D.2-ccb7-4 +/Users/shixuesen/OneDrive/Pictures/instagram/loveruby_official/ https://scontent-lax3-2.cdninstagram.com/v/t51.2885-15/e35/187016734_103021585227659_6491538064093475333_n.jpg?se=7&tp=1&_nc_ht=scontent-lax3-2.cdninstagram.com&_nc_cat=111&_nc_ohc=2BKwmDiikq8AX_qB3k2&edm=ABmJApABAAAA&ccb=7-4&oh=12891c602137172deef77150ad708aa5&oe=60CA325D&_nc_sid=6136e7&ig_cache_key=MjU3NTQ3ODExNzM5MzkyMzUxOA%3D%3D.2-ccb7-4 +/Users/shixuesen/OneDrive/Pictures/instagram/amberchenslife/ https://scontent-lax3-1.cdninstagram.com/v/t50.2886-16/98863594_484334979658418_4517059695294025491_n.mp4?efg=eyJ2ZW5jb2RlX3RhZyI6InZ0c192b2RfdXJsZ2VuLjcyMC5zdG9yeS5kZWZhdWx0In0&_nc_ht=scontent-lax3-1.cdninstagram.com&_nc_cat=102&_nc_ohc=esAAPS5D_5kAX-oLPVz&edm=AOVtZ6oBAAAA&vs=17895960235975082_4018238684&_nc_vs=HBkcFQAYJEdPcUo1QVd5TW1vR2dMZ0JBQk1QUEVmYzBhOF9idXFIQUFBQRUAAsgBACgAGAAbAYgHdXNlX29pbAExFQAAJtS6vaX9kco%2FFQIoAkMzLBdAFAAAAAAAABgSZGFzaF9iYXNlbGluZV80X3YxEQB16AcA&_nc_rid=4041964fe5&ccb=7-4&oe=60A54646&oh=e8f7a74ac24e1eca89225eb4b7cc1860&_nc_sid=bab638