Browse Source

add some solution

feature/new_bilibili_and_instagram_sxs20191126
shixuesen 4 years ago
parent
commit
1e85807439
13 changed files with 341 additions and 27 deletions
  1. +2
    -2
      app/Console/Commands/BiliVideoCode.php
  2. +1
    -1
      app/Console/Commands/ReplaceBlog.php
  3. +33
    -0
      app/Console/Commands/XiuGirlScrape.php
  4. +1
    -1
      app/Http/Controllers/WeiboController.php
  5. +3
    -3
      app/Services/BilibiliServiceV2.php
  6. +14
    -2
      app/Services/CommonService.php
  7. +38
    -7
      app/Services/NewXiuGirlsService.php
  8. +226
    -0
      app/Services/RemoveLargeMediaService.php
  9. +1
    -1
      app/Services/ReplaceBlogService.php
  10. +3
    -3
      app/Services/ReplaceCompressedVideoService.php
  11. +5
    -3
      app/Services/WeiboService.php
  12. +4
    -4
      app/Services/XiurenjiService.php
  13. +10
    -0
      fail.log

+ 2
- 2
app/Console/Commands/BiliVideoCode.php View File

@ -55,9 +55,9 @@ class BiliVideoCode extends Command
// $bilibili->queryDBCollectionList();
// exit;
// $bilibili->queryForVideoParts();
$bilibili->compareAndDownloadUpVideos(true);
$bilibili->compareAndDownloadUpVideos();
// exit;
$bilibili->compareAndDownloadCollectionVideos(true);
$bilibili->compareAndDownloadCollectionVideos();
// $bilibili->queryCollectionList();
// $bilibili->compareAndDownloadNewVideos();
// $bilibili->queryLocalUpVideoList();


+ 1
- 1
app/Console/Commands/ReplaceBlog.php View File

@ -40,6 +40,6 @@ class ReplaceBlog extends Command
{
//
$service = new ReplaceBlogService();
$service->replaceFrontMatter("/Users/shixuesen/workspace/dev/src/myHugoBlog/content/posts");
$service->replaceFrontMatter("/Users/shixuesen/workspace/dev/src/myHugoBlog/content/temp");
}
}

+ 33
- 0
app/Console/Commands/XiuGirlScrape.php View File

@ -40,6 +40,39 @@ class XiuGirlScrape extends Command
public function handle()
{
//
// $curl = curl_init();
// curl_setopt_array($curl, array(
// CURLOPT_URL => 'https://img.xsnvshen.com/album/22162/35724/000.jpg',
// CURLOPT_RETURNTRANSFER => true,
// CURLOPT_ENCODING => '',
// CURLOPT_MAXREDIRS => 10,
// CURLOPT_TIMEOUT => 0,
// CURLOPT_FOLLOWLOCATION => true,
// CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
// CURLOPT_CUSTOMREQUEST => 'GET',
// CURLOPT_HTTPHEADER => array(
// 'authority: img.xsnvshen.com',
// 'pragma: no-cache',
// 'cache-control: no-cache',
// 'sec-ch-ua: " Not A;Brand";v="99", "Chromium";v="90", "Google Chrome";v="90"',
// 'sec-ch-ua-mobile: ?0',
// 'upgrade-insecure-requests: 1',
// 'user-agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.212 Safari/537.36',
// 'accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
// 'sec-fetch-site: same-site',
// 'sec-fetch-mode: navigate',
// 'sec-fetch-user: ?1',
// 'sec-fetch-dest: document',
// 'referer: https://www.xsnvshen.com/album/35724/',
// 'accept-language: zh-CN,zh;q=0.9',
//// 'cookie: __cfduid=dac5872f65e79a40a5b30229ba97beb6a1619333692'
// ),
// ));
//
// $response = curl_exec($curl);
// file_put_contents("/Users/shixuesen/workspace/dev/src/graph-tutorial/0000000112.jpg", $response);
// exit;
$xiu1 = new NewXiuGirlsService();
$xiu1->scrapeXiuGirls();exit;
$xiu = new XiuGirlsService();


+ 1
- 1
app/Http/Controllers/WeiboController.php View File

@ -32,7 +32,7 @@ class WeiboController extends Controller
// for( $i = 341; $i >= 1 ; $i-- )
// $url[] = 'https://m.weibo.cn/api/container/getIndex?containerid=1076035893812490&openApp=0&page='.$i;
for( $i = 20; $i >= 1 ; $i-- )
for( $i = 1; $i >= 1 ; $i-- )
$url[] = 'https://m.weibo.cn/api/container/getIndex?containerid=230259&openApp=0&page='.$i;
// for ($i = 5; $i >= 1; $i--) {
// $url[] = 'https://m.weibo.cn/feed/group?gid=4423532052076817&&page=' . $i;


+ 3
- 3
app/Services/BilibiliServiceV2.php View File

@ -71,7 +71,7 @@ class BilibiliServiceV2
$url = "https://api.bilibili.com/x/space/arc/search?mid={$mediaId}&ps=30&tid=0&keyword=&order=pubdate&jsonp=jsonp&pn=";
// https://space.bilibili.com/475250/video
for ($i = 1; $i < 10; $i++) {
for ($i = 1; $i < 5; $i++) {
$curl = curl_init();
curl_setopt_array($curl, array(
@ -533,7 +533,7 @@ class BilibiliServiceV2
foreach ($list as $item) {
Log::info("schedule queryDBCollectionList current collection is: " . $item['title'] . " started at :" . date("Y-m-d H:i:s"));
$currentList = [];
for ($pageNo = 1; $pageNo < 100; $pageNo++) {
for ($pageNo = 1; $pageNo < 10; $pageNo++) {
$listResponse = $this->requestCollectionList($this->collectionListUrl, $item['media_id'], $pageNo);
// dump("pageNo: " . $pageNo);
// dump($listResponse);
@ -805,7 +805,7 @@ class BilibiliServiceV2
// "Cookie: SESSDATA=64a15917%2C1578628130%2Ceb05cdc1"
);
if ($userCookie) {
$headerArray[] = "Cookie: SESSDATA=75ed2a5a%2C1620866280%2Cc0fab*b1;";
$headerArray[] = "Cookie: SESSDATA=216ce1c5%2C1636429756%2Cb86c3%2A51;";
// $headerArray[] = "Cookie: _uuid=D6E5438B-5A95-439F-7512-FC3509457A9A28409infoc; buvid3=5566647C-DDE5-4AFF-8711-89C9DB2B7061110244infoc; LIVE_BUVID=AUTO3415734420289108; UM_distinctid=16e62db8b9934b-0f76f63ae51bee-1c3c6a5a-13c680-16e62db8b9acaf; CURRENT_FNVAL=16; stardustvideo=1; rpdid=|(ku|l|lRYlJ0J'ul~JYuYY|u; im_notify_type_279025=0; sid=llgc5h9q; laboratory=1-1; CURRENT_QUALITY=80; INTVER=1; DedeUserID=279025; DedeUserID__ckMd5=9a79e15294e6b8bb; SESSDATA=b169300a%2C1581262828%2C3654f611; bili_jct=597a5b9adb6170698e396fb053bc4aba; bp_t_offset_279025=343238964368145599";
}


+ 14
- 2
app/Services/CommonService.php View File

@ -21,8 +21,20 @@ class CommonService
// $cookieArr = parseCookieHeader($cookieStr);
// $cookieJar = CookieJar::fromArray($cookieArr, ".v2ph.com");
$client = new GuzzleHttp\Client(['headers' => [
'User-Agent' => 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.109 Safari/537.36',
'Referer' => $host,
'User-Agent' => 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.212 Safari/537.36',
// 'Referer' => $host,
'authority' => 'img.xsnvshen.com',
'pragma' => 'no-cache',
'cache-control' => 'no-cache',
'sec-ch-ua' => ' Not A;Brand";v="99", "Chromium";v="90", "Google Chrome";v="90"',
'sec-ch-ua-mobile' => '?0',
'accept' => 'image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8',
'sec-fetch-site' => 'same-site',
'sec-fetch-mode' => 'no-cors',
'sec-fetch-dest' => 'image',
'referer' => 'https://www.xsnvshen.com/album/35686',
'accept-language' => 'zh-CN,zh;q=0.9',
'cookie' => '__cfduid=dac5872f65e79a40a5b30229ba97beb6a1619333692'
// 'cookies' => $cookieJar
]]);
$fileParts = pathinfo($imageUrl);


+ 38
- 7
app/Services/NewXiuGirlsService.php View File

@ -145,7 +145,7 @@ class NewXiuGirlsService
}
$opts = array("http"=> ["header" =>
"User-Agent:Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.109 Safari/537.36 \r\n
"User-Agent:Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.212 Safari/537.36 \r\n
Referer:".$baseUrl.$item."\r\n"
]);
$this->processDownloadImage($image, $baseUrl, $item, $albumPath, $username, $albumCode);
@ -279,6 +279,8 @@ class NewXiuGirlsService
}
echo curl_error($curl_handle);
echo $query;exit;
exit;
$fp = fopen($albumPath . "/" . $username . "-" . $albumCode . "-" . pathinfo("http:".$image)["filename"].".jpg", "x");
fwrite($fp, $query);
fclose($fp);
@ -465,12 +467,39 @@ class NewXiuGirlsService
*/
private function processDownloadImage(string $image, string $baseUrl, $item, string $albumPath, string $username, string $albumCode): void
{
// CommonService::downloadImage($albumPath, "https:" . $image);exit;
$curl_handle = curl_init();
curl_setopt($curl_handle, CURLOPT_URL, "https:" . $image);
curl_setopt($curl_handle, CURLOPT_CONNECTTIMEOUT, 2000);
curl_setopt($curl_handle, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($curl_handle, CURLOPT_USERAGENT, "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.109 Safari/537.36");
curl_setopt($curl_handle, CURLOPT_REFERER, $baseUrl . $item);
// curl_setopt($curl_handle, CURLOPT_URL, "https:" . $image);
// curl_setopt($curl_handle, CURLOPT_CONNECTTIMEOUT, 20000);
// curl_setopt($curl_handle, CURLOPT_RETURNTRANSFER, 1);
// curl_setopt($curl_handle, CURLOPT_USERAGENT, "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.212 Safari/537.36");
// curl_setopt($curl_handle, CURLOPT_REFERER, $baseUrl . $item);
echo "image url is " . $image . "\n";
curl_setopt_array($curl_handle, array(
CURLOPT_URL => "https:" . $image,
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => '',
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 0,
CURLOPT_FOLLOWLOCATION => true,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => 'GET',
CURLOPT_HTTPHEADER => array(
'authority: img.xsnvshen.com',
'pragma: no-cache',
'cache-control: no-cache',
'sec-ch-ua: " Not A;Brand";v="99", "Chromium";v="90", "Google Chrome";v="90"',
'sec-ch-ua-mobile: ?0',
'user-agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.212 Safari/537.36',
'accept: image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8',
'sec-fetch-site: same-site',
'sec-fetch-mode: no-cors',
'sec-fetch-dest: image',
'referer: https://www.xsnvshen.com/album/' . $item,
'accept-language: zh-CN,zh;q=0.9',
// 'cookie: __cfduid=dac5872f65e79a40a5b30229ba97beb6a1619333692'
),
));
$content = curl_exec($curl_handle);
$i = 1;
while ($content === false) {
@ -485,8 +514,10 @@ class NewXiuGirlsService
break;
}
}
// $content = file_get_contents("https:" . $image);
echo curl_error($curl_handle);
// echo curl_error($curl_handle);
// echo $content;exit;
$fp = fopen($albumPath . "/" . $username . "-" . $albumCode . "-" . pathinfo("http:" . $image)["filename"] . ".jpg", "x");
fwrite($fp, $content);
fclose($fp);


+ 226
- 0
app/Services/RemoveLargeMediaService.php View File

@ -0,0 +1,226 @@
<?php
namespace App\Services;
use Illuminate\Support\Facades\Redis;
use Mhor\MediaInfo\MediaInfo;
class RemoveLargeMediaService
{
private $bili;
public function __construct()
{
$this->bili = new BilibiliServiceV2();
}
public function handleVideos($dir = "/Users/shixuesen/Documents/tmp/柚木/2017/泡泡条纹袜/")
{
$files = scandir($dir);
foreach ($files as $file) {
if ($file == "." || $file == "..") {
continue;
}
$subDir = implode("/", [$dir, $file]);
$isDir = is_dir($subDir);
if ($isDir) {
$subFiles = scandir($subDir);
foreach ($subFiles as $subFile) {
$subPathFile = implode("/", [$subDir, $subFile]);
if (is_dir($subPathFile) || $subFile == ".DS_Store") {
continue;
}
$mime = mime_content_type($subPathFile);
// dump("file type", [$mime, $subPathFile]);
// continue;
if (strstr($mime, "video/")) {
if (is_file($subPathFile)) {
$fileInfo = pathinfo($subPathFile);
dump("fileInfo", $fileInfo);
if (ends_with($fileInfo["filename"], "-1")) {
continue;
}
if (is_file($fileInfo["dirname"] . '/' .$fileInfo["filename"] . '-1'. '.' . $fileInfo["extension"])) {
unlink($subPathFile);
continue;
}
$targetFile = $fileInfo["dirname"] . '/' .$fileInfo["filename"] . '-1'. '.' . $fileInfo["extension"];
dump("targetFile", [$targetFile]);
// $result = shell_exec("handBrakeCli -Z 'Very Fast 720p30' -i '". $subPathFile ."' -o '". $targetFile . " && echo 'success'");
$result = shell_exec("handBrakeCli -Z 'Very Fast 720p30' -i '". $subPathFile ."' -o '". $targetFile . "'");
dump($result);
}
}
}
}
}
print_r($files);
}
// public function processDir($baseDir = "/Volumes/WD/Video/HuaVid/")
// public function processDir($baseDir = "/Volumes/intel660p/video/mv/bilibili/tara")
public function processDir($baseDir = "/Volumes/WD/tmp/bilibili/aoa")
{
$files = scandir($baseDir);
// $files = array_slice($files, 5670);
$count = count($files);
dump("all count is $count");
foreach ($files as $index => $file) {
if ($file == "." || $file == ".." || $file == ".DS_Store") {
continue;
}
$subDir = implode("/", [$baseDir, $file]);
$isDir = is_dir($subDir);
if ($isDir) {
// $this->processDir($subDir);
} else {
dump("[all count] is $count, [current index] is $index");
$this->processVideo($subDir);
}
}
}
public function processVideo($pathFile)
{
$mime = mime_content_type($pathFile);
// dump("file type", [$mime, $pathFile]);exit;
// continue;
$mediaInfo = new MediaInfo();
$mediaInfo->setConfig('use_oldxml_mediainfo_output_format', true);
if (strstr($mime, "video/") || strstr($mime, "application/octet-stream")) {
if (is_file($pathFile)) {
$mediaContainer = $mediaInfo->getInfo($pathFile);
foreach ($mediaContainer->getVideos() as $video) {
$height = $video->get('height')->getAbsoluteValue();
$width = $video->get('width')->getAbsoluteValue();
if ($height > $width && $width <= 720) {
echo "$pathFile 分辨率小于 720p 跳过\n";
return;
}
if ($height <= $width && $height <= 720) {
echo "$pathFile 分辨率小于 720p 跳过\n";
return;
}
}
$fileInfo = pathinfo($pathFile);
dump("fileInfo", $fileInfo);
if (ends_with($fileInfo["filename"], "-XXXXX#compressed")) {
return;
}
if (starts_with($fileInfo["filename"], "171221 M!CountDown 光州特辑女团现场合集") || starts_with($fileInfo["filename"], "Rainbow - A 现场合集 P14 Rainbow - A + Mach")) {
return;
}
if (Redis::sismember("unneed", $fileInfo["filename"])) {
return;
}
$targetFile = $fileInfo["dirname"] . '/' .$fileInfo["filename"] . '-XXXXX#compressed'. '.' . $fileInfo["extension"];
if (is_file($targetFile)) {
unlink($pathFile);
rename($targetFile, $pathFile);
return;
}
dump("targetFile", [$targetFile]);
// $result = shell_exec("handBrakeCli -Z 'Very Fast 720p30' -i '". $subPathFile ."' -o '". $targetFile . " && echo 'success'");
$result = shell_exec("handBrakeCli --json -Z 'Very Fast 720p30' -i '". $pathFile ."' -o '". $targetFile . "'");
// echo $result;
preg_match_all("#\"State\"\: \"WORKDONE\"#", $result, $match);
if (count($match) > 0) {
echo "compress work done remove the file \n";
$oldFileSize = filesize($pathFile);
$newFileSize = filesize($targetFile);
if ($newFileSize >= $oldFileSize) {
Redis::sadd("unneed", $fileInfo["filename"]);
echo "old file size is smaller than new one, old is " . file_size($oldFileSize) . " and new is " . file_size($newFileSize) . ", now remove new one";
unlink($targetFile);
} else {
echo "new file size is smaller than old one, new is " . file_size($newFileSize) . " and old is " . file_size($oldFileSize) . ", now remove old one";
unlink($pathFile);
rename($targetFile, $pathFile);
}
}
}
}
}
function human_filesize($bytes, $decimals = 2) {
$sz = 'BKMGTP';
$factor = floor((strlen($bytes) - 1) / 3);
return sprintf("%.{$decimals}f", $bytes / pow(1024, $factor)) . @$sz[$factor];
}
public function processUnCompleteDir($baseDir = "/Volumes/WD/tmp/探花系列【AI高清2K修复】大合集")
// public function processDir($baseDir = "/Volumes/Backup/iPhone nPlayer/")
{
$files = scandir($baseDir);
foreach ($files as $file) {
if ($file == "." || $file == "..") {
continue;
}
$subDir = implode("/", [$baseDir, $file]);
$isDir = is_dir($subDir);
if ($isDir) {
$this->processUnCompleteDir($subDir);
} else {
$this->processUnCompleteVideo($subDir);
}
}
}
public function processUnCompleteVideo($pathFile)
{
//...
$mediaInfo = new MediaInfo();
$mediaInfo->setConfig('use_oldxml_mediainfo_output_format', true);
$mime = mime_content_type($pathFile);
// dump("file type", [$mime, $subPathFile]);
// continue;
if (strstr($mime, "video/")) {
if (is_file($pathFile)) {
$fileInfo = pathinfo($pathFile);
// dump("fileInfo", $fileInfo);
if (ends_with($fileInfo["filename"], "-1")) {
return;
}
if (is_file($fileInfo["dirname"] . '/' .$fileInfo["filename"] . '-1'. '.' . $fileInfo["extension"])) {
$mediaInfoContainer1 = $mediaInfo->getInfo($fileInfo["dirname"] . '/' .$fileInfo["filename"] . '-1'. '.' . $fileInfo["extension"]);
$millSecond1 = $mediaInfoContainer1->getGeneral()->get("duration")->getMilliseconds();
echo gettype($millSecond1) . "\n";
// ["duration"] . "\n";
$mediaInfoContainer = $mediaInfo->getInfo($pathFile);
$millSecond = $mediaInfoContainer->getGeneral()->get("duration")->getMilliseconds();
echo gettype($millSecond) . "\n";
if (abs(intval($millSecond) - intval($millSecond1)) > 100) {
echo $pathFile . "\n";
echo abs(intval($millSecond) - intval($millSecond1)) . "\n";
}
// unlink($pathFile);
return;
}
// $targetFile = $fileInfo["dirname"] . '/' .$fileInfo["filename"] . '-1'. '.' . $fileInfo["extension"];
// dump("targetFile", [$targetFile]);
//// $result = shell_exec("handBrakeCli -Z 'Very Fast 720p30' -i '". $subPathFile ."' -o '". $targetFile . " && echo 'success'");
// $result = shell_exec("handBrakeCli -Z 'Very Fast 720p30' -i '". $pathFile ."' -o '". $targetFile . "'");
// dump($result);
}
}
}
public function checkFileInfo($pathFile)
{
preg_match_all("#([\S|\s]+)P\d+#", $pathFile, $match);
if (count($match) > 1 && count($match[1]) > 0) {
$this->bili->searchVideoFiles($match[1][0]);
}
// dump($match[1][0]);
}
}

+ 1
- 1
app/Services/ReplaceBlogService.php View File

@ -28,7 +28,7 @@ class ReplaceBlogService {
// dump($yaml);
// $html = $document->getContent();
// dump($html);
exit;
// exit;
}
}


+ 3
- 3
app/Services/ReplaceCompressedVideoService.php View File

@ -64,7 +64,7 @@ class ReplaceCompressedVideoService
public function processDir($baseDir = "/Volumes/WD/tmp/bilibili/aoa")
{
$files = scandir($baseDir);
$files = array_slice($files, 3035);
// $files = array_slice($files, 5670);
$count = count($files);
dump("all count is $count");
foreach ($files as $index => $file) {
@ -74,9 +74,9 @@ class ReplaceCompressedVideoService
$subDir = implode("/", [$baseDir, $file]);
$isDir = is_dir($subDir);
if ($isDir) {
// $this->processDir($subDir);
$this->processDir($subDir);
} else {
dump("[all count] is $count, [current index] is $index");
// dump("[all count] is $count, [current index] is $index");
$this->processVideo($subDir);
}
}


+ 5
- 3
app/Services/WeiboService.php View File

@ -16,6 +16,7 @@ class WeiboService
// if (!isset($_REQUEST['content'])) die('NoData');
//
// $content = trim($_REQUEST['content']);
echo $content;exit;
// @mkdir('data');
@ -84,12 +85,13 @@ class WeiboService
# code...
$page_info = $mblog["page_info"];
$media_info = $mblog["page_info"]["media_info"];
$video_url = Arr::get($media_info, "mp4_720p_mp4", "");
$medis_urls = $mblog["page_info"]["urls"];
$video_url = Arr::get($medis_urls, "mp4_720p_mp4", "");
if ($video_url == "") {
$video_url = Arr::get($media_info, "mp4_hd_url", "");
$video_url = Arr::get($medis_urls, "mp4_hd_url", "");
}
if ($video_url == "") {
$video_url = Arr::get($media_info, "mp4_sd_url", "");
$video_url = Arr::get($medis_urls, "mp4_ld_mp4", "");
}
if ($video_url != "") {
self::process_video($video_url, $video_dir, $page_info["content2"]);


+ 4
- 4
app/Services/XiurenjiService.php View File

@ -11,8 +11,8 @@ use QL\QueryList;
class XiurenjiService
{
public $domainUrl = "https://www.xiurenji.com";
public $xiurenRootUrl = "https://www.xiurenji.com/XiuRen/";
public $domainUrl = "https://www.xiurenji.cc";
public $xiurenRootUrl = "https://www.xiurenji.cc/XiuRen/";
// public $rootDir = "/Users/shixuesen/Documents/tmp/xiuren/";
public $rootDir = "/Volumes/Backup/images/xiuren/";
public $queryInstance;
@ -24,7 +24,7 @@ class XiurenjiService
public function scrapeAlbum()
{
$pageSize = 20;
$pageCount = $this->getEncodeHtmlContent("https://www.xiurenji.com/XiuRen/index.html")->find(".page span")->htmls()->get(0);
$pageCount = $this->getEncodeHtmlContent("https://www.xiurenji.cc/XiuRen/index.html")->find(".page span")->htmls()->get(0);
print_r($pageCount);
if ((int)$pageCount > 0) {
for ($i = 0; $i <= ceil($pageCount / $pageSize); $i++) {
@ -112,7 +112,7 @@ class XiurenjiService
usleep(random_int(10, 100) * 100);
$imageUrl = $image->getAttribute("src");
$trueImageUrl = "https://x1.plmn5.com/U". substr($imageUrl, 2);
$trueImageUrl = "https://www.xiurenji.com" . $imageUrl;
$trueImageUrl = "https://www.xiurenji.cc" . $imageUrl;
$fileInfo = pathinfo($trueImageUrl);
if (file_exists($dir . "/" .$fileInfo["basename"])) {
rename($dir . "/" . $fileInfo["basename"], $dir . "/" . $imageNo . "-" . $fileInfo["basename"]);


+ 10
- 0
fail.log View File

@ -1021,3 +1021,13 @@
/Users/shixuesen/OneDrive/Pictures/instagram/Likes/ https://scontent-lax3-1.cdninstagram.com/v/t51.2885-15/e35/173205557_167251781933216_1150698725227016954_n.jpg?se=7&tp=1&_nc_ht=scontent-lax3-1.cdninstagram.com&_nc_cat=1&_nc_ohc=WtORZ2J3Y-UAX_RatNU&edm=APv5SkIAAAAA&ccb=7-4&oh=2b1b884668df7e8e974905edcaf5497f&oe=609CF6BA&_nc_sid=7479f6&ig_cache_key=MjU1MDc4NjM3ODEwNzk3NDI5Nw%3D%3D.2-ccb7-4
/Users/shixuesen/OneDrive/Pictures/instagram/ngoctrinh89/ https://scontent-lax3-1.cdninstagram.com/v/t50.2886-16/98618760_3227004884069355_6044999634251674592_n.mp4?efg=eyJ2ZW5jb2RlX3RhZyI6InZ0c192b2RfdXJsZ2VuLjcyMC5zdG9yeS5kZWZhdWx0In0&_nc_ht=scontent-lax3-1.cdninstagram.com&_nc_cat=103&_nc_ohc=O_mfjDVyUb0AX8STxQP&edm=AOVtZ6oAAAAA&vs=17882073455230920_638136539&_nc_vs=HBksFQAYJEdJak40QVhycTgyZjhYWUxBT0R6NzdvakpfUlRidXFIQUFBQRUAAsgBABUAGCRHRlpSZUFwdVlmS0p1d1FCQUs0T2dWY3R2ZVlIYnBrd0FBQUYVAgLIAQAoABgAGwGIB3VzZV9vaWwBMRUAACaQ3Oj61OnDPxUCKAJDMywXQCuZmZmZmZoYEmRhc2hfYmFzZWxpbmVfMl92MREAdegHAA%3D%3D&_nc_rid=76a0e8f923&ccb=7-4&oe=60834C43&oh=d715adacda8bb3840e0bf87c8f995ad5&_nc_sid=bab638
/Users/shixuesen/OneDrive/Pictures/instagram/bivi_0420/ https://scontent-lax3-1.cdninstagram.com/v/t50.2886-16/74963866_2803615776546370_4938313936994023105_n.mp4?efg=eyJ2ZW5jb2RlX3RhZyI6InZ0c192b2RfdXJsZ2VuLjcyMC5zdG9yeS5kZWZhdWx0In0&_nc_ht=scontent-lax3-1.cdninstagram.com&_nc_cat=110&_nc_ohc=Joh1OfK2nVQAX9xVW53&edm=AOVtZ6oBAAAA&vs=17893525949035078_1378283097&_nc_vs=HBkcFQAYJEdKcmJkd1JDbm5LcTMtVUpBTUcyNDROY2FvaEVidXFIQUFBQRUAAsgBACgAGAAbAYgHdXNlX29pbAExFQAAJoyMwoulhMk%2FFQIoAkMzLBdAFbtkWhysCBgSZGFzaF9iYXNlbGluZV8xX3YxEQB16AcA&_nc_rid=a086d64b7d&ccb=7-4&oe=608B28BB&oh=b8298b616f2d5ccfb99724c1ee61ee8a&_nc_sid=bab638
/Users/shixuesen/OneDrive/Pictures/instagram/weiman_the_real_one/ https://scontent-lax3-1.cdninstagram.com/v/t50.2886-16/105864276_881068615809681_429462755360194612_n.mp4?efg=eyJ2ZW5jb2RlX3RhZyI6InZ0c192b2RfdXJsZ2VuLjcyMC5zdG9yeS5kZWZhdWx0In0&_nc_ht=scontent-lax3-1.cdninstagram.com&_nc_cat=109&_nc_ohc=ClEGxANuV4gAX8LHX2j&edm=AOVtZ6oBAAAA&vs=17888464805151887_1594195415&_nc_vs=HBksFQAYJEdGUmNUd2FST3Q3RlV5RURBRFJBZ2NFWnd2VUZidXFIQUFBQRUAAsgBABUAGCRHTXZsNEFwY2NSdFBOSFVBQUtWWDI3TmRFdk5tYnBrd0FBQUYVAgLIAQAoABgAGwGIB3VzZV9vaWwBMRUAACaewNWl2N3GPxUCKAJDMywXQC4AAAAAAAAYEmRhc2hfYmFzZWxpbmVfMV92MREAdegHAA%3D%3D&_nc_rid=0321597446&ccb=7-4&oe=609370FE&oh=fb6468d93941af1f109b680c395d0a05&_nc_sid=bab638
/Users/shixuesen/OneDrive/Pictures/instagram/weiman_the_real_one/ https://scontent-lax3-1.cdninstagram.com/v/t50.2886-16/181075290_128264609354392_7933344908465807688_n.mp4?efg=eyJ2ZW5jb2RlX3RhZyI6InZ0c192b2RfdXJsZ2VuLjcyMC5mZWVkLmRlZmF1bHQifQ&_nc_ht=scontent-lax3-1.cdninstagram.com&_nc_cat=104&_nc_ohc=UIHfK-iP284AX9lknaP&edm=ABmJApABAAAA&vs=17893667830984768_2991277892&_nc_vs=HBksFQAYJEdGcjl5Z3FZVW5ydXAzUUFBRWdod2wwcDV4aHVia1lMQUFBRhUAAsgBABUAGCRHQ0ZpdGdvOXBFbnlZVmdCQUtHQ3k5eXRJSXgxYmtZTEFBQUYVAgLIAQAoABgAGwGIB3VzZV9vaWwBMBUAACaApb%2FzxYzJPxUCKAJDMywXQCWZmZmZmZoYEmRhc2hfYmFzZWxpbmVfMV92MREAdeoHAA%3D%3D&_nc_rid=ce87b51b0f&ccb=7-4&oe=6093B1F8&oh=d89c6aa36976992b1c43124d69e23a97&_nc_sid=6136e7
/Users/shixuesen/OneDrive/Pictures/instagram/amberchenslife/ https://scontent-lax3-1.cdninstagram.com/v/t50.2886-16/102138368_299639625077541_5560331099598882509_n.mp4?efg=eyJ2ZW5jb2RlX3RhZyI6InZ0c192b2RfdXJsZ2VuLjcyMC5zdG9yeS5kZWZhdWx0In0&_nc_ht=scontent-lax3-1.cdninstagram.com&_nc_cat=109&_nc_ohc=_PtYCk9MayMAX-kaOoM&edm=AOVtZ6oBAAAA&vs=17887276850162485_3780522526&_nc_vs=HBkcFQAYJEdBQ0NGZ1lsMDJsSmhSQUJBTTBHSnRtc1F5cE5idXFIQUFBQRUAAsgBACgAGAAbAYgHdXNlX29pbAExFQAAJuqQnbHFmMY%2FFQIoAkMzLBdAGAAAAAAAABgSZGFzaF9iYXNlbGluZV8xX3YxEQB16AcA&_nc_rid=41b6792dd3&ccb=7-4&oe=6093B623&oh=1ff403f0fe9ce390d2bad1f7ed20d70b&_nc_sid=bab638
/Users/shixuesen/OneDrive/Pictures/instagram/li_yen_chin/ https://scontent-lax3-1.cdninstagram.com/v/t50.2886-16/181772162_826396801641956_8580539285203080817_n.mp4?efg=eyJ2ZW5jb2RlX3RhZyI6InZ0c192b2RfdXJsZ2VuLjcyMC5mZWVkLmRlZmF1bHQifQ&_nc_ht=scontent-lax3-1.cdninstagram.com&_nc_cat=108&_nc_ohc=OdVYeenYUUEAX8NcBuf&edm=ABmJApABAAAA&vs=17906907298755762_3635376210&_nc_vs=HBksFQAYJEdJS2YxUXJrTGM1LW11OENBSEZXMEtzRU14UjNia1lMQUFBRhUAAsgBABUAGCRHRHl5NGdxMy1rTVU1TWtBQUFwQWhyUVZNUk5UYmtZTEFBQUYVAgLIAQAoABgAGwGIB3VzZV9vaWwBMBUAACbk5aTHl4%2FPPxUCKAJDMywXQEURBiTdLxsYEmRhc2hfYmFzZWxpbmVfMV92MREAdeoHAA%3D%3D&_nc_rid=04f8da9520&ccb=7-4&oe=6093C9B7&oh=15626792ef2f5adaa3d148ef50e1b4d7&_nc_sid=6136e7
/Users/shixuesen/OneDrive/Pictures/instagram/Likes/ https://scontent-lax3-1.cdninstagram.com/v/t51.2885-15/e35/183146074_211586803843009_3207802134815944041_n.jpg?se=7&tp=1&_nc_ht=scontent-lax3-1.cdninstagram.com&_nc_cat=101&_nc_ohc=A9T-y27nw0IAX-HTT-3&edm=APv5SkIBAAAA&ccb=7-4&oh=848b12687895f16db0e004fe959cf027&oe=60BCE9FC&_nc_sid=7479f6&ig_cache_key=MjU2OTEyMTIxNTY2MzYxNTcxNw%3D%3D.2-ccb7-4
/Users/shixuesen/OneDrive/Pictures/instagram/Likes/ https://scontent-lax3-1.cdninstagram.com/v/t51.2885-15/e35/182672085_2873338019595533_1153522599175245989_n.jpg?se=7&tp=1&_nc_ht=scontent-lax3-1.cdninstagram.com&_nc_cat=105&_nc_ohc=5QYtpOBeb8UAX_v4t2g&edm=APv5SkIBAAAA&ccb=7-4&oh=8ca94038bb3bb6410604322c03d8fcf1&oe=60BD4EB5&_nc_sid=7479f6&ig_cache_key=MjU2OTEyMTIxNTY5NzMxOTc5Mg%3D%3D.2-ccb7-4
/Users/shixuesen/OneDrive/Pictures/instagram/yui_xin_/ https://scontent-lax3-1.cdninstagram.com/v/t50.2886-16/88687982_197855032162484_8949640507211987845_n.mp4?efg=eyJ2ZW5jb2RlX3RhZyI6InZ0c192b2RfdXJsZ2VuLjcyMC5zdG9yeS5kZWZhdWx0In0&_nc_ht=scontent-lax3-1.cdninstagram.com&_nc_cat=108&_nc_ohc=0pn5YaWTpgQAX8J3ZAs&edm=AOVtZ6oBAAAA&vs=17904611140826159_2935624126&_nc_vs=HBkcFQAYJEdHNUZTUVcwdU9pMjhyTUFBSVU3MGNPZmdqTjhidXFIQUFBQRUAAsgBACgAGAAbAYgHdXNlX29pbAExFQAAJt7xhonEic4%2FFQIoAkMzLBdAFAAAAAAAABgSZGFzaF9iYXNlbGluZV8yX3YxEQB16AcA&_nc_rid=f51dea0338&ccb=7-4&oe=609C5894&oh=639694261f1886295a17b90d0b507a14&_nc_sid=bab638
/Users/shixuesen/OneDrive/Pictures/instagram/boram__jj/ https://scontent-lax3-1.cdninstagram.com/v/t51.2885-15/e35/186884661_499567364512012_4467085033471998071_n.jpg?se=7&tp=1&_nc_ht=scontent-lax3-1.cdninstagram.com&_nc_cat=104&_nc_ohc=A2LAd2ZogGYAX-3Zqbk&edm=ABmJApABAAAA&ccb=7-4&oh=7b5448b331d61770b0aebb5b2ab2ff95&oe=60C3C41D&_nc_sid=6136e7&ig_cache_key=MjU3MzE2NDM4Nzg2MDUzMDY3OA%3D%3D.2-ccb7-4
/Users/shixuesen/OneDrive/Pictures/instagram/loveruby_official/ https://scontent-lax3-2.cdninstagram.com/v/t51.2885-15/e35/187016734_103021585227659_6491538064093475333_n.jpg?se=7&tp=1&_nc_ht=scontent-lax3-2.cdninstagram.com&_nc_cat=111&_nc_ohc=2BKwmDiikq8AX_qB3k2&edm=ABmJApABAAAA&ccb=7-4&oh=12891c602137172deef77150ad708aa5&oe=60CA325D&_nc_sid=6136e7&ig_cache_key=MjU3NTQ3ODExNzM5MzkyMzUxOA%3D%3D.2-ccb7-4
/Users/shixuesen/OneDrive/Pictures/instagram/amberchenslife/ https://scontent-lax3-1.cdninstagram.com/v/t50.2886-16/98863594_484334979658418_4517059695294025491_n.mp4?efg=eyJ2ZW5jb2RlX3RhZyI6InZ0c192b2RfdXJsZ2VuLjcyMC5zdG9yeS5kZWZhdWx0In0&_nc_ht=scontent-lax3-1.cdninstagram.com&_nc_cat=102&_nc_ohc=esAAPS5D_5kAX-oLPVz&edm=AOVtZ6oBAAAA&vs=17895960235975082_4018238684&_nc_vs=HBkcFQAYJEdPcUo1QVd5TW1vR2dMZ0JBQk1QUEVmYzBhOF9idXFIQUFBQRUAAsgBACgAGAAbAYgHdXNlX29pbAExFQAAJtS6vaX9kco%2FFQIoAkMzLBdAFAAAAAAAABgSZGFzaF9iYXNlbGluZV80X3YxEQB16AcA&_nc_rid=4041964fe5&ccb=7-4&oe=60A54646&oh=e8f7a74ac24e1eca89225eb4b7cc1860&_nc_sid=bab638

Loading…
Cancel
Save