add feature

.env.local

@@ -15,7 +15,7 @@ DB_PASSWORD=
 
 
 BROADCAST_DRIVER=log
-CACHE_DRIVER=file
+CACHE_DRIVER=array
 QUEUE_CONNECTION=sync
 SESSION_DRIVER=file
 SESSION_LIFETIME=120

app/Console/Commands/BiliVideoCode.php

@@ -49,12 +49,12 @@ class BiliVideoCode extends Command
 //        $bilibili->insertDBTest();exit;
 //        $bilibili->queryPlayList();
 
-//        $bilibili->queryUpVideoList(10278125);
+        $bilibili->queryUpVideoList(10278125);
 //        exit;
 
-//        $bilibili->queryDBCollectionList();
+        $bilibili->queryDBCollectionList();
 //        exit;
-//        $bilibili->queryForVideoParts();
+        $bilibili->queryForVideoParts();
         $bilibili->compareAndDownloadUpVideos();
 //        exit;
         $bilibili->compareAndDownloadCollectionVideos();

app/Console/Commands/CommonTest.php

@@ -6,6 +6,7 @@ use App\Services\CommonScrapeService;
 use App\Services\FileService;
 use App\Utils\FileUtils;
 use Illuminate\Console\Command;
+use Illuminate\Support\Facades\Cache;
 use Illuminate\Support\Facades\Redis;
 
 class CommonTest extends Command
@@ -41,6 +42,10 @@ class CommonTest extends Command
      */
     public function handle()
     {
+        $cache = Cache::driver("redis");
+        $cache->tags(["aaa", "bbb"])->put("key1", "value1");
+        $cache->put("key2", "value2");
+        exit;
         $this->testAnnie();exit;
         $commonScrapeService = new CommonScrapeService();
         $commonScrapeService->scrapeAlbum();exit;

app/Console/Commands/InstagramScrape.php

@@ -45,6 +45,10 @@ class InstagramScrape extends Command
         //
         $start = $this->argument('start');
 //        echo $start;exit;
+//        $userList = file("/Users/shixuesen/OneDrive/Pictures/instagram/user.txt");
+//        $userList = array_slice($userList, $start, 65);
+//        //            $userList = ['1992.ai_'];
+//                        print_r($userList);exit;
         $ins = new InstagramService();
         $ins->scrapeUsers($start);
         $ins->scrapeLikedUsers();

app/Console/Kernel.php

@@ -138,12 +138,12 @@ class Kernel extends ConsoleKernel
             $nvshenService = new NewNvshenService();
             $nvshenService->scrapeNvshenGirls();
 //        })->cron("0 */3 * * *");
-        })->dailyAt('04:00');
+        })->dailyAt('05:00');
         $schedule->call(function () {
             $tuji = new TujiguService();
             $tuji->scrapeTujiguGirls();
 //        })->cron("30 */3 * * *");
-        })->dailyAt('05:00');
+        })->dailyAt('04:00');
         $schedule->call(function () {
             $dailySignService = new DailySignService();
             $dailySignService->signClubHmoe();

app/Services/HandBrakeService.php

@@ -90,14 +90,14 @@ class HandBrakeService
                 }
                 $fileInfo = pathinfo($pathFile);
                 dump("fileInfo", $fileInfo);
-                if (ends_with($fileInfo["filename"], "-720p")) {
+                if (ends_with($fileInfo["filename"], "-1")) {
                     return;
                 }
-                if (is_file($fileInfo["dirname"] . '/' .$fileInfo["filename"] . '-720p'. '.' . $fileInfo["extension"])) {
+                if (is_file($fileInfo["dirname"] . '/' .$fileInfo["filename"] . '-1'. '.' . $fileInfo["extension"])) {
                     unlink($pathFile);
                     return;
                 }
-                $targetFile = $fileInfo["dirname"] . '/' .$fileInfo["filename"] . '-720p'. '.' . $fileInfo["extension"];
+                $targetFile = $fileInfo["dirname"] . '/' .$fileInfo["filename"] . '-1'. '.' . $fileInfo["extension"];
                 dump("targetFile", [$targetFile]);
 //                         $result = shell_exec("handBrakeCli -Z 'Very Fast 720p30' -i '". $subPathFile ."' -o '".  $targetFile  . " &&  echo 'success'");
                 $result = shell_exec("handBrakeCli -Z 'Very Fast 720p30' -i '". $pathFile ."' -o '".  $targetFile . "'");

app/Services/InstagramService.php

@@ -177,6 +177,7 @@ class InstagramService
             curl_setopt($ch, CURLOPT_VERBOSE, 1);
             curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
             curl_setopt($ch, CURLOPT_AUTOREFERER, false);
+            curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
             curl_setopt($ch, CURLOPT_REFERER, "https://scontent-lax3-1.cdninstagram.com");
             curl_setopt($ch, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1);
             curl_setopt($ch, CURLOPT_HEADER, 0);
@@ -338,7 +339,11 @@ class InstagramService
                 //                    $maxId = '1451179182821586881_19900699';
                 //                } else {
 
-                $response = $ig->story->getUserReelMediaFeed($userId);
+                try {
+                    $response = $ig->story->getUserReelMediaFeed($userId);
+                } catch (\Exception $e) {
+                    Log::error("current user has error, $userName, " . $e->getMessage());
+                }
                 foreach ($response->getItems() as $item) {
                     if (Item::VIDEO == $item->getMediaType()) {
                         $videoUrl = $item->getVideoVersions()[0]->getUrl();
@@ -352,7 +357,12 @@ class InstagramService
                 do {
                     // Request the page corresponding to maxId.
                     echo "\n current maxId: " . $maxId;
-                    $response = $ig->timeline->getUserFeed($userId, $maxId);
+                    try {
+                        $response = $ig->timeline->getUserFeed($userId, $maxId);
+                    } catch (\Exception $e) {
+                        Log::error("current user has error, $userName, " . $e->getMessage());
+                        continue 2;
+                    }
 
                     // In this example we're simply printing the IDs of this page's items.
                     foreach ($response->getItems() as $item) {

app/Services/NewNvshenService.php

@@ -114,7 +114,7 @@ class NewNvshenService
 //                dump($albumCodeMap);exit;
             }
             $baseUrl = "https://www.invshen.net";
-            usleep(random_int(1000, 10000) * 1000);
+            usleep(random_int(100, 1000) * 1000);
             $peopleUrl = "https://www.invshen.net/girl/";
             $peopleUrl .= $name['code'];
 
@@ -129,7 +129,7 @@ class NewNvshenService
                     echo 'connection error: ' . $e->getMessage() . "\n";
                     echo "retry times: " . $i . " times \n";
                     sleep(1);
-                    $sleepTime = 1000 * random_int(1000, 10000);
+                    $sleepTime = 1000 * random_int(100, 1000);
                     echo "retry sleep {$sleepTime} nano second \n";
                     usleep($sleepTime);
                 }
@@ -178,7 +178,7 @@ class NewNvshenService
                         dump("相册已超过限制数量，跳出");
                         break 2;
                     }
-                    usleep(10000 * random_int(1000, 10000));
+                    usleep(1000 * random_int(100, 1000));
                     dump("相册：", [$album]);
 //                    $pageQL = $qL0->get($baseUrl . $album);
                     for ($m = 0; $m < 100; $m++) {
@@ -192,7 +192,7 @@ class NewNvshenService
                             echo 'connection error: ' . $e->getMessage() . "\n";
                             echo "retry times: " . $m . " times \n";
                             sleep(1);
-                            $sleepTime = 1000 * random_int(1000, 10000);
+                            $sleepTime = 1000 * random_int(100, 1000);
                             echo "retry sleep {$sleepTime} nano second \n";
                             usleep($sleepTime);
                         }
@@ -246,7 +246,7 @@ class NewNvshenService
                             echo 'Curl error: ' . curl_error($curl_handle) . "\n";
                             echo "retry times: " . $i++ . " times \n";
                             sleep(1);
-                            $sleepTime = 1000 * random_int(1000, 10000);
+                            $sleepTime = 1000 * random_int(100, 1000);
                             echo "retry sleep {$sleepTime} nano second \n";
                             usleep($sleepTime);
                             $query = curl_exec($curl_handle);

app/Services/TujiguService.php

@@ -4,6 +4,7 @@
 namespace App\Services;
 
 
+use Exception;
 use QL\QueryList;
 
 class TujiguService
@@ -104,12 +105,19 @@ class TujiguService
                 }
             }
             $baseUrl = "https://www.tujigu.com/";
-            usleep(random_int(1000, 10000) * 1000);
+            usleep(random_int(100, 1000) * 1000);
             $peopleUrl = "https://www.tujigu.com/t/";
             $peopleUrl .= $name['code'];
             // 获取总的相册数量
             $albumNumSelector = "body > div:nth-child(4) > span";
-            $baseQl = QueryList::get($peopleUrl);
+            echo "111111111";
+            try {
+                $baseQl = QueryList::get($peopleUrl, null, ['timeout' => 5]);
+            } catch (Exception $e) {
+                dump($e->getMessage());
+            }
+            echo "222222222";
+
             $ql = $baseQl->find($albumNumSelector)->htmls();
             dump($ql->all());
             $onlyOnePage = false;
@@ -150,7 +158,7 @@ class TujiguService
                         break 2;
                     }
                     dump("current album page no: " . $i);
-                    usleep(10000 * random_int(1000, 10000));
+                    usleep(1000 * random_int(100, 1000));
                     dump("相册：", [$album]);
                     $pageQL = QueryList::get( $album);
                     $page = $pageQL->find("body > div.tuji > p:nth-child(5)")->htmls();
@@ -211,7 +219,7 @@ class TujiguService
                             echo 'Curl error: ' . curl_error($curl_handle) . "\n";
                             echo "retry times: " . $i++ . " times \n";
                             sleep(1);
-                            $sleepTime = 1000 * random_int(1000, 10000);
+                            $sleepTime = 1000 * random_int(100, 1000);
                             echo "retry sleep {$sleepTime} nano second \n";
                             usleep($sleepTime);
                             $query = curl_exec($curl_handle);

app/Services/XiurenjiService.php

@@ -15,6 +15,7 @@ class XiurenjiService
     public $xiurenRootUrl = "https://www.xiurenji.cc/XiuRen/";
 //    public $rootDir = "/Users/shixuesen/Documents/tmp/xiuren/";
     public $rootDir = "/Volumes/Backup/images/xiuren/";
+//    public $rootDir = "/Volumes/intel660p/image/xiuren/";
     public $queryInstance;
     public function __construct()
     {
@@ -27,6 +28,7 @@ class XiurenjiService
         $pageCount = $this->getEncodeHtmlContent("https://www.xiurenji.cc/XiuRen/index.html")->find(".page span")->htmls()->get(0);
         print_r($pageCount);
         if ((int)$pageCount > 0) {
+            $pageCount = 40;
             for ($i = 0; $i <= ceil($pageCount / $pageSize); $i++) {
                 $urlSuffix = "";
                 if ($i == 0) {

config/cache.php

@@ -17,7 +17,7 @@ return [
     |
     */
 
-    'default' => env('CACHE_DRIVER', 'file'),
+    'default' => env('CACHE_DRIVER', 'array'),
 
     /*
     |--------------------------------------------------------------------------