nicksxs
/
graph-tutorial
1
0
Fork 0
Code Issues 0 Pull Requests 0 Projects 0 Releases 0 Wiki Activity
Browse Source

add feature

feature/new_bilibili_and_instagram_sxs20191126
shixuesen 4 years ago
parent
1e85807439
commit
ba70788521
12 changed files with 1359 additions and 21 deletions
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. +1
    -1
      .env.local
  2. +3
    -3
      app/Console/Commands/BiliVideoCode.php
  3. +5
    -0
      app/Console/Commands/CommonTest.php
  4. +4
    -0
      app/Console/Commands/InstagramScrape.php
  5. +2
    -2
      app/Console/Kernel.php
  6. +3
    -3
      app/Services/HandBrakeService.php
  7. +12
    -2
      app/Services/InstagramService.php
  8. +5
    -5
      app/Services/NewNvshenService.php
  9. +12
    -4
      app/Services/TujiguService.php
  10. +2
    -0
      app/Services/XiurenjiService.php
  11. +1
    -1
      config/cache.php
  12. +1309
    -0
      fail.log

+ 1
- 1
.env.local View File

@ -15,7 +15,7 @@ DB_PASSWORD=
BROADCAST_DRIVER=log
CACHE_DRIVER=file
CACHE_DRIVER=array
QUEUE_CONNECTION=sync
SESSION_DRIVER=file
SESSION_LIFETIME=120


+ 3
- 3
app/Console/Commands/BiliVideoCode.php View File

@ -49,12 +49,12 @@ class BiliVideoCode extends Command
// $bilibili->insertDBTest();exit;
// $bilibili->queryPlayList();
// $bilibili->queryUpVideoList(10278125);
$bilibili->queryUpVideoList(10278125);
// exit;
// $bilibili->queryDBCollectionList();
$bilibili->queryDBCollectionList();
// exit;
// $bilibili->queryForVideoParts();
$bilibili->queryForVideoParts();
$bilibili->compareAndDownloadUpVideos();
// exit;
$bilibili->compareAndDownloadCollectionVideos();


+ 5
- 0
app/Console/Commands/CommonTest.php View File

@ -6,6 +6,7 @@ use App\Services\CommonScrapeService;
use App\Services\FileService;
use App\Utils\FileUtils;
use Illuminate\Console\Command;
use Illuminate\Support\Facades\Cache;
use Illuminate\Support\Facades\Redis;
class CommonTest extends Command
@ -41,6 +42,10 @@ class CommonTest extends Command
*/
public function handle()
{
$cache = Cache::driver("redis");
$cache->tags(["aaa", "bbb"])->put("key1", "value1");
$cache->put("key2", "value2");
exit;
$this->testAnnie();exit;
$commonScrapeService = new CommonScrapeService();
$commonScrapeService->scrapeAlbum();exit;


+ 4
- 0
app/Console/Commands/InstagramScrape.php View File

@ -45,6 +45,10 @@ class InstagramScrape extends Command
//
$start = $this->argument('start');
// echo $start;exit;
// $userList = file("/Users/shixuesen/OneDrive/Pictures/instagram/user.txt");
// $userList = array_slice($userList, $start, 65);
// // $userList = ['1992.ai_'];
// print_r($userList);exit;
$ins = new InstagramService();
$ins->scrapeUsers($start);
$ins->scrapeLikedUsers();


+ 2
- 2
app/Console/Kernel.php View File

@ -138,12 +138,12 @@ class Kernel extends ConsoleKernel
$nvshenService = new NewNvshenService();
$nvshenService->scrapeNvshenGirls();
// })->cron("0 */3 * * *");
})->dailyAt('04:00');
})->dailyAt('05:00');
$schedule->call(function () {
$tuji = new TujiguService();
$tuji->scrapeTujiguGirls();
// })->cron("30 */3 * * *");
})->dailyAt('05:00');
})->dailyAt('04:00');
$schedule->call(function () {
$dailySignService = new DailySignService();
$dailySignService->signClubHmoe();


+ 3
- 3
app/Services/HandBrakeService.php View File

@ -90,14 +90,14 @@ class HandBrakeService
}
$fileInfo = pathinfo($pathFile);
dump("fileInfo", $fileInfo);
if (ends_with($fileInfo["filename"], "-720p")) {
if (ends_with($fileInfo["filename"], "-1")) {
return;
}
if (is_file($fileInfo["dirname"] . '/' .$fileInfo["filename"] . '-720p'. '.' . $fileInfo["extension"])) {
if (is_file($fileInfo["dirname"] . '/' .$fileInfo["filename"] . '-1'. '.' . $fileInfo["extension"])) {
unlink($pathFile);
return;
}
$targetFile = $fileInfo["dirname"] . '/' .$fileInfo["filename"] . '-720p'. '.' . $fileInfo["extension"];
$targetFile = $fileInfo["dirname"] . '/' .$fileInfo["filename"] . '-1'. '.' . $fileInfo["extension"];
dump("targetFile", [$targetFile]);
// $result = shell_exec("handBrakeCli -Z 'Very Fast 720p30' -i '". $subPathFile ."' -o '". $targetFile . " && echo 'success'");
$result = shell_exec("handBrakeCli -Z 'Very Fast 720p30' -i '". $pathFile ."' -o '". $targetFile . "'");


+ 12
- 2
app/Services/InstagramService.php View File

@ -177,6 +177,7 @@ class InstagramService
curl_setopt($ch, CURLOPT_VERBOSE, 1);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_AUTOREFERER, false);
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
curl_setopt($ch, CURLOPT_REFERER, "https://scontent-lax3-1.cdninstagram.com");
curl_setopt($ch, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1);
curl_setopt($ch, CURLOPT_HEADER, 0);
@ -338,7 +339,11 @@ class InstagramService
// $maxId = '1451179182821586881_19900699';
// } else {
$response = $ig->story->getUserReelMediaFeed($userId);
try {
$response = $ig->story->getUserReelMediaFeed($userId);
} catch (\Exception $e) {
Log::error("current user has error, $userName, " . $e->getMessage());
}
foreach ($response->getItems() as $item) {
if (Item::VIDEO == $item->getMediaType()) {
$videoUrl = $item->getVideoVersions()[0]->getUrl();
@ -352,7 +357,12 @@ class InstagramService
do {
// Request the page corresponding to maxId.
echo "\n current maxId: " . $maxId;
$response = $ig->timeline->getUserFeed($userId, $maxId);
try {
$response = $ig->timeline->getUserFeed($userId, $maxId);
} catch (\Exception $e) {
Log::error("current user has error, $userName, " . $e->getMessage());
continue 2;
}
// In this example we're simply printing the IDs of this page's items.
foreach ($response->getItems() as $item) {


+ 5
- 5
app/Services/NewNvshenService.php View File

@ -114,7 +114,7 @@ class NewNvshenService
// dump($albumCodeMap);exit;
}
$baseUrl = "https://www.invshen.net";
usleep(random_int(1000, 10000) * 1000);
usleep(random_int(100, 1000) * 1000);
$peopleUrl = "https://www.invshen.net/girl/";
$peopleUrl .= $name['code'];
@ -129,7 +129,7 @@ class NewNvshenService
echo 'connection error: ' . $e->getMessage() . "\n";
echo "retry times: " . $i . " times \n";
sleep(1);
$sleepTime = 1000 * random_int(1000, 10000);
$sleepTime = 1000 * random_int(100, 1000);
echo "retry sleep {$sleepTime} nano second \n";
usleep($sleepTime);
}
@ -178,7 +178,7 @@ class NewNvshenService
dump("相册已超过限制数量，跳出");
break 2;
}
usleep(10000 * random_int(1000, 10000));
usleep(1000 * random_int(100, 1000));
dump("相册：", [$album]);
// $pageQL = $qL0->get($baseUrl . $album);
for ($m = 0; $m < 100; $m++) {
@ -192,7 +192,7 @@ class NewNvshenService
echo 'connection error: ' . $e->getMessage() . "\n";
echo "retry times: " . $m . " times \n";
sleep(1);
$sleepTime = 1000 * random_int(1000, 10000);
$sleepTime = 1000 * random_int(100, 1000);
echo "retry sleep {$sleepTime} nano second \n";
usleep($sleepTime);
}
@ -246,7 +246,7 @@ class NewNvshenService
echo 'Curl error: ' . curl_error($curl_handle) . "\n";
echo "retry times: " . $i++ . " times \n";
sleep(1);
$sleepTime = 1000 * random_int(1000, 10000);
$sleepTime = 1000 * random_int(100, 1000);
echo "retry sleep {$sleepTime} nano second \n";
usleep($sleepTime);
$query = curl_exec($curl_handle);


+ 12
- 4
app/Services/TujiguService.php View File

@ -4,6 +4,7 @@
namespace App\Services;
use Exception;
use QL\QueryList;
class TujiguService
@ -104,12 +105,19 @@ class TujiguService
}
}
$baseUrl = "https://www.tujigu.com/";
usleep(random_int(1000, 10000) * 1000);
usleep(random_int(100, 1000) * 1000);
$peopleUrl = "https://www.tujigu.com/t/";
$peopleUrl .= $name['code'];
// 获取总的相册数量
$albumNumSelector = "body > div:nth-child(4) > span";
$baseQl = QueryList::get($peopleUrl);
echo "111111111";
try {
$baseQl = QueryList::get($peopleUrl, null, ['timeout' => 5]);
} catch (Exception $e) {
dump($e->getMessage());
}
echo "222222222";
$ql = $baseQl->find($albumNumSelector)->htmls();
dump($ql->all());
$onlyOnePage = false;
@ -150,7 +158,7 @@ class TujiguService
break 2;
}
dump("current album page no: " . $i);
usleep(10000 * random_int(1000, 10000));
usleep(1000 * random_int(100, 1000));
dump("相册：", [$album]);
$pageQL = QueryList::get( $album);
$page = $pageQL->find("body > div.tuji > p:nth-child(5)")->htmls();
@ -211,7 +219,7 @@ class TujiguService
echo 'Curl error: ' . curl_error($curl_handle) . "\n";
echo "retry times: " . $i++ . " times \n";
sleep(1);
$sleepTime = 1000 * random_int(1000, 10000);
$sleepTime = 1000 * random_int(100, 1000);
echo "retry sleep {$sleepTime} nano second \n";
usleep($sleepTime);
$query = curl_exec($curl_handle);


+ 2
- 0
app/Services/XiurenjiService.php View File

@ -15,6 +15,7 @@ class XiurenjiService
public $xiurenRootUrl = "https://www.xiurenji.cc/XiuRen/";
// public $rootDir = "/Users/shixuesen/Documents/tmp/xiuren/";
public $rootDir = "/Volumes/Backup/images/xiuren/";
// public $rootDir = "/Volumes/intel660p/image/xiuren/";
public $queryInstance;
public function __construct()
{
@ -27,6 +28,7 @@ class XiurenjiService
$pageCount = $this->getEncodeHtmlContent("https://www.xiurenji.cc/XiuRen/index.html")->find(".page span")->htmls()->get(0);
print_r($pageCount);
if ((int)$pageCount > 0) {
$pageCount = 40;
for ($i = 0; $i <= ceil($pageCount / $pageSize); $i++) {
$urlSuffix = "";
if ($i == 0) {


+ 1
- 1
config/cache.php View File

@ -17,7 +17,7 @@ return [
|
*/
'default' => env('CACHE_DRIVER', 'file'),
'default' => env('CACHE_DRIVER', 'array'),
/*
|--------------------------------------------------------------------------


+ 1309
- 0
fail.log
File diff suppressed because it is too large
View File


Write Preview
Loading…
Cancel
Save