shixuesen
5 years ago
12 changed files with 562 additions and 33 deletions
Unified View
Diff Options
-
+7 -3app/Console/Commands/BiliVideoCode.php
-
+45 -0app/Console/Commands/FullSiteCommand.php
-
+45 -0app/Console/Commands/HandBrakeCommand.php
-
+1 -1app/Console/Commands/RenameTest.php
-
+45 -0app/Console/Commands/TujiguCommand.php
-
+55 -10app/Services/BilibiliService.php
-
+59 -0app/Services/FullSiteService.php
-
+47 -0app/Services/HandBrakeService.php
-
+6 -2app/Services/InstagramService.php
-
+35 -17app/Services/NewNvshenService.php
-
+209 -0app/Services/TujiguService.php
-
+8 -0fail.log
+ 7
- 3
app/Console/Commands/BiliVideoCode.php
View File
+ 45
- 0
app/Console/Commands/FullSiteCommand.php
View File
| @ -0,0 +1,45 @@ | |||||
| <?php | |||||
| namespace App\Console\Commands; | |||||
| use App\Services\FullSiteService; | |||||
| use Illuminate\Console\Command; | |||||
| class FullSiteCommand extends Command | |||||
| { | |||||
| /** | |||||
| * The name and signature of the console command. | |||||
| * | |||||
| * @var string | |||||
| */ | |||||
| protected $signature = 'full:get'; | |||||
| /** | |||||
| * The console command description. | |||||
| * | |||||
| * @var string | |||||
| */ | |||||
| protected $description = 'Command description'; | |||||
| /** | |||||
| * Create a new command instance. | |||||
| * | |||||
| * @return void | |||||
| */ | |||||
| public function __construct() | |||||
| { | |||||
| parent::__construct(); | |||||
| } | |||||
| /** | |||||
| * Execute the console command. | |||||
| * | |||||
| * @return mixed | |||||
| */ | |||||
| public function handle() | |||||
| { | |||||
| $service = new FullSiteService(); | |||||
| $service->fullSiteDownloadImage(); | |||||
| // | |||||
| } | |||||
| } | |||||
+ 45
- 0
app/Console/Commands/HandBrakeCommand.php
View File
| @ -0,0 +1,45 @@ | |||||
| <?php | |||||
| namespace App\Console\Commands; | |||||
| use App\Services\HandBrakeService; | |||||
| use Illuminate\Console\Command; | |||||
| class HandBrakeCommand extends Command | |||||
| { | |||||
| /** | |||||
| * The name and signature of the console command. | |||||
| * | |||||
| * @var string | |||||
| */ | |||||
| protected $signature = 'handle:video'; | |||||
| /** | |||||
| * The console command description. | |||||
| * | |||||
| * @var string | |||||
| */ | |||||
| protected $description = 'Command description'; | |||||
| /** | |||||
| * Create a new command instance. | |||||
| * | |||||
| * @return void | |||||
| */ | |||||
| public function __construct() | |||||
| { | |||||
| parent::__construct(); | |||||
| } | |||||
| /** | |||||
| * Execute the console command. | |||||
| * | |||||
| * @return mixed | |||||
| */ | |||||
| public function handle() | |||||
| { | |||||
| // | |||||
| $service = new HandBrakeService(); | |||||
| $service->handleVideos(); | |||||
| } | |||||
| } | |||||
+ 1
- 1
app/Console/Commands/RenameTest.php
View File
+ 45
- 0
app/Console/Commands/TujiguCommand.php
View File
| @ -0,0 +1,45 @@ | |||||
| <?php | |||||
| namespace App\Console\Commands; | |||||
| use App\Services\TujiguService; | |||||
| use Illuminate\Console\Command; | |||||
| class TujiguCommand extends Command | |||||
| { | |||||
| /** | |||||
| * The name and signature of the console command. | |||||
| * | |||||
| * @var string | |||||
| */ | |||||
| protected $signature = 'tujigu:s'; | |||||
| /** | |||||
| * The console command description. | |||||
| * | |||||
| * @var string | |||||
| */ | |||||
| protected $description = 'Command description'; | |||||
| /** | |||||
| * Create a new command instance. | |||||
| * | |||||
| * @return void | |||||
| */ | |||||
| public function __construct() | |||||
| { | |||||
| parent::__construct(); | |||||
| } | |||||
| /** | |||||
| * Execute the console command. | |||||
| * | |||||
| * @return mixed | |||||
| */ | |||||
| public function handle() | |||||
| { | |||||
| // | |||||
| $service = new TujiguService(); | |||||
| $service->scrapeTujiguGirls(); | |||||
| } | |||||
| } | |||||
+ 55
- 10
app/Services/BilibiliService.php
View File
+ 59
- 0
app/Services/FullSiteService.php
View File
| @ -0,0 +1,59 @@ | |||||
| <?php | |||||
| namespace App\Services; | |||||
| use Log; | |||||
| use QL\QueryList; | |||||
| class FullSiteService { | |||||
| public function fullSiteDownloadImage() | |||||
| { | |||||
| $dir = "/Volumes/WD/tmp/site/mm24.cc/beautyleg/"; | |||||
| $files = scandir($dir); | |||||
| $ql = QueryList::getInstance(); | |||||
| $i = 0; | |||||
| foreach ($files as $file) { | |||||
| $i++; | |||||
| if ($i % 100 == 0) { | |||||
| dump("current counter i is ". $i . "\n"); | |||||
| } | |||||
| if ($file == "." || $file == ".." || $file == ".DS_Store") { | |||||
| continue; | |||||
| } | |||||
| $htmlNum = substr($file, 0, 4); | |||||
| if ((int)$htmlNum < 1072) { | |||||
| continue; | |||||
| } | |||||
| if (str_contains($file, "html")) { | |||||
| dump($file); | |||||
| $ql = $ql->html(file_get_contents($dir. "/" . $file)); | |||||
| $rt = $ql->find('img')->attrs('src'); | |||||
| dump($rt); | |||||
| foreach ($rt as $r) { | |||||
| $filePathInfo = pathinfo($r); | |||||
| // 获取相对路径 | |||||
| $path = parse_url($filePathInfo['dirname'])['path']; | |||||
| if (str_contains($path, "..")) { | |||||
| $localFilePath = $dir . $path . "/"; | |||||
| } else { | |||||
| $localFilePath = $dir . "../" . $path . "/"; | |||||
| } | |||||
| if (is_file($localFilePath . $filePathInfo["basename"])) { | |||||
| continue; | |||||
| } | |||||
| if (!is_dir($localFilePath)) { | |||||
| mkdir($localFilePath, 0777, true); | |||||
| } | |||||
| try { | |||||
| $content = file_get_contents($r); | |||||
| } catch (\Exception $e) { | |||||
| Log::error("current url is " . $r . " trace: " . $e->getTraceAsString()); | |||||
| continue; | |||||
| } | |||||
| file_put_contents($dir . "../" . $path . "/" .$filePathInfo["basename"], $content); | |||||
| usleep(1000 * random_int(1000, 10000)); | |||||
| } | |||||
| // dump($rt);exit; | |||||
| } | |||||
| } | |||||
| } | |||||
| } | |||||
+ 47
- 0
app/Services/HandBrakeService.php
View File
| @ -0,0 +1,47 @@ | |||||
| <?php | |||||
| namespace App\Services; | |||||
| class HandBrakeService | |||||
| { | |||||
| public function handleVideos($dir = "/Users/shixuesen/Documents/tmp/秀人旗下《YouWu尤物馆》套图及视频写真-TLoB/") | |||||
| { | |||||
| $files = scandir($dir); | |||||
| foreach ($files as $file) { | |||||
| if ($file == "." || $file == "..") { | |||||
| continue; | |||||
| } | |||||
| $subDir = implode("/", [$dir, $file]); | |||||
| $isDir = is_dir($subDir); | |||||
| if ($isDir) { | |||||
| $subFiles = scandir($subDir); | |||||
| foreach ($subFiles as $subFile) { | |||||
| $subPathFile = implode("/", [$subDir, $subFile]); | |||||
| if (is_dir($subPathFile) || $subFile == ".DS_Store") { | |||||
| continue; | |||||
| } | |||||
| if (is_file($subPathFile)) { | |||||
| $fileInfo = pathinfo($subPathFile); | |||||
| dump("fileInfo", $fileInfo); | |||||
| if (ends_with($fileInfo["filename"], "-1")) { | |||||
| continue; | |||||
| } | |||||
| if (is_file($fileInfo["dirname"] . '/' .$fileInfo["filename"] . '-1'. '.' . $fileInfo["extension"])) { | |||||
| continue; | |||||
| } | |||||
| $targetFile = $fileInfo["dirname"] . '/' .$fileInfo["filename"] . '-1'. '.' . $fileInfo["extension"]; | |||||
| dump("targetFile", [$targetFile]); | |||||
| // $result = shell_exec("handBrakeCli -Z 'Very Fast 720p30' -i '". $subPathFile ."' -o '". $targetFile . " && echo 'success'"); | |||||
| $result = shell_exec("handBrakeCli -Z 'Very Fast 720p30' -i '". $subPathFile ."' -o '". $targetFile . "'"); | |||||
| dump($result); | |||||
| } | |||||
| } | |||||
| } | |||||
| } | |||||
| print_r($files); | |||||
| } | |||||
| } | |||||
+ 6
- 2
app/Services/InstagramService.php
View File
+ 35
- 17
app/Services/NewNvshenService.php
View File
+ 209
- 0
app/Services/TujiguService.php
View File
| @ -0,0 +1,209 @@ | |||||
| <?php | |||||
| namespace App\Services; | |||||
| use QL\QueryList; | |||||
| class TujiguService | |||||
| { | |||||
| private static $name_dir = [ | |||||
| // "周韦彤" => [ | |||||
| // "dir" => "周韦彤", | |||||
| // "code" => 16274, | |||||
| // "name" => "周韦彤" | |||||
| // ], | |||||
| // "ycc" => [ | |||||
| // "dir" => "ycc", | |||||
| // "code" => 22162, | |||||
| // "name" => "杨晨晨" | |||||
| // ], | |||||
| // "ry" => [ | |||||
| // "dir" => "忍野さら", | |||||
| // "code" => "21250", | |||||
| // "name" => "忍野さら" | |||||
| // ], | |||||
| // "azu" => [ | |||||
| // "dir" => "azu", | |||||
| // "code" => 26002, | |||||
| // "name" => "阿朱" | |||||
| // ], | |||||
| // "xq" => [ | |||||
| // "dir" => "xq", | |||||
| // "code" => 22204, | |||||
| // "name" => "小琪" | |||||
| // ], | |||||
| // "ygh" => [ | |||||
| // "dir" => "ygh", | |||||
| // "code" => 15902, | |||||
| // "name" => "原干惠" | |||||
| // ], | |||||
| // "wyc" => [ | |||||
| // "dir" => "wyc", | |||||
| // "code" => 19702, | |||||
| // "name" => "王语纯" | |||||
| // ], | |||||
| // "zz" => [ | |||||
| // "dir" => "zz", | |||||
| // "code" => 22899, | |||||
| // "name" => "芝芝 booty" | |||||
| // ], | |||||
| // "hlr" => [ | |||||
| // "dir" => "hlr", | |||||
| // "code" => 20015, | |||||
| // "name" => "黄乐然" | |||||
| // ], | |||||
| "jrq" => [ | |||||
| "dir" => "jrq", | |||||
| "code" => 5034, | |||||
| "name" => "姜仁卿" | |||||
| ], | |||||
| // "ny" => [ | |||||
| // "dir" => "ny", | |||||
| // "code" => 26298, | |||||
| // "name" => "奈月" | |||||
| // ], | |||||
| // "杉本有美" => [ | |||||
| // "dir" => "杉本有美", | |||||
| // "code" => 15939, | |||||
| // "name" => "杉本有美" | |||||
| // ] | |||||
| ]; | |||||
| public function scrapeTujiguGirls() | |||||
| { | |||||
| foreach (self::$name_dir as $username => $name) { | |||||
| // $baseDir = "/Users/shixuesen/Documents/tmp/image/xg/" . $name['dir'] . "/"; | |||||
| $baseDir = "/Volumes/intel660p/image/xg/" . $name["dir"] . "/"; | |||||
| // 处理两个网站对应图册名不一致，直接取图册 id 作为唯一性判断的 map | |||||
| if (is_dir($baseDir)) { | |||||
| $albumDirList = scandir($baseDir); | |||||
| foreach ($albumDirList as $albumDir) { | |||||
| if ($albumDir == "." || $albumDir == "..") { | |||||
| continue; | |||||
| } | |||||
| if (is_dir($baseDir . $albumDir) && is_numeric(explode("-", $albumDir)[0])) { | |||||
| $albumCodeMap[explode("-", $albumDir)[0]] = $albumDir; | |||||
| } | |||||
| } | |||||
| } | |||||
| $baseUrl = "https://www.tujigu.com/"; | |||||
| // usleep(random_int(1000, 10000) * 1000); | |||||
| $peopleUrl = "https://www.tujigu.com/t/"; | |||||
| $peopleUrl .= $name['code']; | |||||
| // 获取总的相册数量 | |||||
| $albumNumSelector = "body > div:nth-child(4) > span"; | |||||
| $baseQl = QueryList::get($peopleUrl); | |||||
| $ql = $baseQl->find($albumNumSelector)->htmls(); | |||||
| dump($ql->all()); | |||||
| $onlyOnePage = false; | |||||
| // 相册页数 | |||||
| if (count($ql->all()) == 0) { | |||||
| // 不超过 1 页时，不显示数量 | |||||
| $totalAlbumPage = 1; | |||||
| $onlyOnePage = true; | |||||
| } else { | |||||
| preg_match("#\d+#", $ql->all()[0], $result); | |||||
| dump($result); | |||||
| $totalAlbumNum = 0; | |||||
| if (is_numeric($result[0])) { | |||||
| $totalAlbumNum = $result[0]; | |||||
| } | |||||
| $totalAlbumPage = ceil($totalAlbumNum / 40); | |||||
| } | |||||
| $baseAlbumUrl = "https://www.tujigu.com/t/{$name['code']}/"; | |||||
| for ($i = 1; $i <= $totalAlbumPage; $i++) { | |||||
| if ($onlyOnePage || $totalAlbumPage == 1) { | |||||
| $albumQl = QueryList::get($peopleUrl); | |||||
| } else { | |||||
| $albumQl = QueryList::get($baseAlbumUrl . $i . ".html"); | |||||
| } | |||||
| // dump($albumQl->getHtml()); | |||||
| $albumList = $albumQl->find("body > div.hezi > ul > li > a")->attrs("href"); | |||||
| $pageAlbum = $albumList->all(); | |||||
| dump($pageAlbum); | |||||
| foreach ($pageAlbum as $album) { | |||||
| usleep(1000 * random_int(1000, 10000)); | |||||
| dump("相册：", [$album]); | |||||
| $pageQL = QueryList::get( $album); | |||||
| $page = $pageQL->find("body > div.tuji > p:nth-child(6)")->htmls(); | |||||
| $title = $pageQL->find("body > div.tuji > div.weizhi > h1")->htmls(); | |||||
| dump($title->all()); | |||||
| $titleStr = $title->all()[0]; | |||||
| preg_match("#\d+#", $page->all()[0], $result); | |||||
| $totalImageNum = $result[0]; | |||||
| for ($j = 1; $j <= $totalImageNum; $j++) { | |||||
| $albumCode = substr($album, 25, 5); | |||||
| $baseImageUrl = "https://lns.hywly.com/a/1/{$albumCode}/"; | |||||
| // if ($j == 0) { | |||||
| // $imageName = $j . ".jpg"; | |||||
| // } else { | |||||
| // $suffix = str_pad($j, 3, "0", STR_PAD_LEFT); | |||||
| // $imageName = $suffix . ".jpg"; | |||||
| // } | |||||
| $imageName = $j . ".jpg"; | |||||
| $imageUrl = $baseImageUrl . $imageName; | |||||
| // if ($j == 0) { | |||||
| // $imageName = "000.jpg"; | |||||
| // } | |||||
| $imageName = $username . "-" . $albumCode . "-" . $imageName; | |||||
| $albumPath = ""; | |||||
| if (!file_exists($baseDir . $albumCode . "-" . $titleStr) && !array_key_exists($albumCode, $albumCodeMap)) { | |||||
| dump($baseDir . $albumCode . "-" . $titleStr); | |||||
| mkdir($baseDir . $albumCode . "-" . $titleStr); | |||||
| } | |||||
| if (array_key_exists($albumCode, $albumCodeMap)) { | |||||
| $albumPath = $baseDir . $albumCodeMap[$albumCode]; | |||||
| } else { | |||||
| $albumPath = $baseDir . $albumCode . "-" . $titleStr; | |||||
| } | |||||
| if (file_exists($albumPath . "/" . $imageName)) { | |||||
| dump($albumPath . "/" . $imageName . " exists. skipped!"); | |||||
| continue; | |||||
| } | |||||
| $opts = array('http' => ['header' => | |||||
| "User-Agent:Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.109 Safari/537.36 \r\n | |||||
| Referer:" . $baseUrl . "\r\n" | |||||
| ]); | |||||
| $curl_handle = curl_init(); | |||||
| curl_setopt($curl_handle, CURLOPT_URL, $imageUrl); | |||||
| curl_setopt($curl_handle, CURLOPT_CONNECTTIMEOUT, 2000); | |||||
| curl_setopt($curl_handle, CURLOPT_RETURNTRANSFER, 1); | |||||
| curl_setopt($curl_handle, CURLOPT_USERAGENT, 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.109 Safari/537.36'); | |||||
| curl_setopt($curl_handle, CURLOPT_REFERER, $baseUrl); | |||||
| $query = curl_exec($curl_handle); | |||||
| $i = 1; | |||||
| while ($query === false) { | |||||
| echo 'Curl error: ' . curl_error($curl_handle) . "\n"; | |||||
| echo "retry times: " . $i++ . " times \n"; | |||||
| sleep(1); | |||||
| $sleepTime = 1000 * random_int(1000, 10000); | |||||
| echo "retry sleep {$sleepTime} nano second \n"; | |||||
| usleep($sleepTime); | |||||
| $query = curl_exec($curl_handle); | |||||
| if ($i >= 100) { | |||||
| break; | |||||
| } | |||||
| } | |||||
| echo curl_error($curl_handle); | |||||
| $fp = fopen($albumPath . "/" . $imageName, 'x'); | |||||
| fwrite($fp, $query); | |||||
| fclose($fp); | |||||
| $sleepTime = 1000 * random_int(100, 1000); | |||||
| echo "after write image sleep {$sleepTime} nano second \n"; | |||||
| usleep($sleepTime); | |||||
| echo $imageUrl; | |||||
| } | |||||
| dump($page->all()); | |||||
| // exit; | |||||
| } | |||||
| } | |||||
| } | |||||
| } | |||||
| } | |||||