From 3eb623b0f9d61440e06a255f264110fea0ec0126 Mon Sep 17 00:00:00 2001 From: shixuesen Date: Tue, 26 Nov 2019 23:23:43 +0800 Subject: [PATCH] feature: add bilibili and instagram --- .env.local | 49 ++++ app/BiCookies.php | 28 +++ app/BilibiliCollections.php | 34 +++ app/BilibiliUpVideos.php | 34 +++ app/Http/Controllers/BiCookiesController.php | 93 +++++++ .../BilibiliCollectionsController.php | 90 +++++++ .../BilibiliUpVideosController.php | 85 +++++++ app/Services/BilibiliService.php | 229 ++++++++++++++++++ database/factories/BiCookiesFactory.php | 9 + .../factories/BilibiliCollectionsFactory.php | 9 + .../factories/BilibiliUpVideosFactory.php | 9 + ...1136_create_bilibili_collections_table.php | 37 +++ ...131446_create_bilibili_up_videos_table.php | 37 +++ ...9_11_10_154228_create_bi_cookies_table.php | 32 +++ run.sh | 2 + 15 files changed, 777 insertions(+) create mode 100644 .env.local create mode 100644 app/BiCookies.php create mode 100644 app/BilibiliCollections.php create mode 100644 app/BilibiliUpVideos.php create mode 100644 app/Http/Controllers/BiCookiesController.php create mode 100644 app/Http/Controllers/BilibiliCollectionsController.php create mode 100644 app/Http/Controllers/BilibiliUpVideosController.php create mode 100644 app/Services/BilibiliService.php create mode 100644 database/factories/BiCookiesFactory.php create mode 100644 database/factories/BilibiliCollectionsFactory.php create mode 100644 database/factories/BilibiliUpVideosFactory.php create mode 100644 database/migrations/2019_11_03_131136_create_bilibili_collections_table.php create mode 100644 database/migrations/2019_11_03_131446_create_bilibili_up_videos_table.php create mode 100644 database/migrations/2019_11_10_154228_create_bi_cookies_table.php create mode 100755 run.sh diff --git a/.env.local b/.env.local new file mode 100644 index 0000000..b726151 --- /dev/null +++ b/.env.local @@ -0,0 +1,49 @@ +APP_NAME=Laravel +APP_ENV=local +APP_KEY=base64:BNE3RoAp39H4EB1jWx0c1xJI3arhkasdMAz52FsCc1U= +APP_DEBUG=true +APP_URL=http://localhost + +LOG_CHANNEL=stack + +DB_CONNECTION=mysql +DB_HOST=127.0.0.1 +DB_PORT=3306 +DB_DATABASE=ins +DB_USERNAME=root +DB_PASSWORD= + + +BROADCAST_DRIVER=log +CACHE_DRIVER=file +QUEUE_CONNECTION=sync +SESSION_DRIVER=file +SESSION_LIFETIME=120 + +REDIS_HOST=127.0.0.1 +REDIS_PASSWORD=null +REDIS_PORT=6379 + +MAIL_DRIVER=smtp +MAIL_HOST=smtp.mailtrap.io +MAIL_PORT=2525 +MAIL_USERNAME=null +MAIL_PASSWORD=null +MAIL_ENCRYPTION=null + +PUSHER_APP_ID= +PUSHER_APP_KEY= +PUSHER_APP_SECRET= +PUSHER_APP_CLUSTER=mt1 + +MIX_PUSHER_APP_KEY="${PUSHER_APP_KEY}" +MIX_PUSHER_APP_CLUSTER="${PUSHER_APP_CLUSTER}" + +OAUTH_APP_ID=89b50ce9-fcd4-4d6b-a830-bbfe0fa5703b +OAUTH_APP_PASSWORD=gmwsDOF38_+ycvIDAH885[) +OAUTH_REDIRECT_URI=http://localhost:8000/callback +#OAUTH_SCOPES='openid profile offline_access user.read calendars.read Files.Read.All, Files.ReadWrite.All, Sites.Read.All, Sites.ReadWrite.All' +OAUTH_SCOPES='openid profile offline_access user.read calendars.read' +OAUTH_AUTHORITY=https://login.microsoftonline.com/common +OAUTH_AUTHORIZE_ENDPOINT=/oauth2/v2.0/authorize +OAUTH_TOKEN_ENDPOINT=/oauth2/v2.0/token diff --git a/app/BiCookies.php b/app/BiCookies.php new file mode 100644 index 0000000..02d57a6 --- /dev/null +++ b/app/BiCookies.php @@ -0,0 +1,28 @@ +input())); + \Log::info($request->input("cookies")); + BiCookies::updateOrCreate(['id' => 1], ["cookie" => base64_encode($request->input("cookies"))]); +// echo json_encode($request); + } + /** + * Display a listing of the resource. + * + * @return \Illuminate\Http\Response + */ + public function index() + { + // + } + + /** + * Show the form for creating a new resource. + * + * @return \Illuminate\Http\Response + */ + public function create() + { + // + } + + /** + * Store a newly created resource in storage. + * + * @param \Illuminate\Http\Request $request + * @return \Illuminate\Http\Response + */ + public function store(Request $request) + { + // + } + + /** + * Display the specified resource. + * + * @param \App\BiCookies $biCookies + * @return \Illuminate\Http\Response + */ + public function show(BiCookies $biCookies) + { + // + } + + /** + * Show the form for editing the specified resource. + * + * @param \App\BiCookies $biCookies + * @return \Illuminate\Http\Response + */ + public function edit(BiCookies $biCookies) + { + // + } + + /** + * Update the specified resource in storage. + * + * @param \Illuminate\Http\Request $request + * @param \App\BiCookies $biCookies + * @return \Illuminate\Http\Response + */ + public function update(Request $request, BiCookies $biCookies) + { + // + } + + /** + * Remove the specified resource from storage. + * + * @param \App\BiCookies $biCookies + * @return \Illuminate\Http\Response + */ + public function destroy(BiCookies $biCookies) + { + // + } +} diff --git a/app/Http/Controllers/BilibiliCollectionsController.php b/app/Http/Controllers/BilibiliCollectionsController.php new file mode 100644 index 0000000..8d906f7 --- /dev/null +++ b/app/Http/Controllers/BilibiliCollectionsController.php @@ -0,0 +1,90 @@ +input())); + echo json_encode($request); + } + /** + * Display a listing of the resource. + * + * @return \Illuminate\Http\Response + */ + public function index() + { + // + } + + /** + * Show the form for creating a new resource. + * + * @return \Illuminate\Http\Response + */ + public function create() + { + // + } + + /** + * Store a newly created resource in storage. + * + * @param \Illuminate\Http\Request $request + * @return \Illuminate\Http\Response + */ + public function store(Request $request) + { + // + } + + /** + * Display the specified resource. + * + * @param \App\BilibiliCollections $bilibiliCollections + * @return \Illuminate\Http\Response + */ + public function show(BilibiliCollections $bilibiliCollections) + { + // + } + + /** + * Show the form for editing the specified resource. + * + * @param \App\BilibiliCollections $bilibiliCollections + * @return \Illuminate\Http\Response + */ + public function edit(BilibiliCollections $bilibiliCollections) + { + // + } + + /** + * Update the specified resource in storage. + * + * @param \Illuminate\Http\Request $request + * @param \App\BilibiliCollections $bilibiliCollections + * @return \Illuminate\Http\Response + */ + public function update(Request $request, BilibiliCollections $bilibiliCollections) + { + // + } + + /** + * Remove the specified resource from storage. + * + * @param \App\BilibiliCollections $bilibiliCollections + * @return \Illuminate\Http\Response + */ + public function destroy(BilibiliCollections $bilibiliCollections) + { + // + } +} diff --git a/app/Http/Controllers/BilibiliUpVideosController.php b/app/Http/Controllers/BilibiliUpVideosController.php new file mode 100644 index 0000000..8219b3f --- /dev/null +++ b/app/Http/Controllers/BilibiliUpVideosController.php @@ -0,0 +1,85 @@ + "https://api.bilibili.com/x/space/arc/search?mid=475250&ps=30&tid=0&keyword=&order=pubdate&jsonp=jsonp&pn={$i}", + CURLOPT_RETURNTRANSFER => true, + CURLOPT_ENCODING => "", + CURLOPT_MAXREDIRS => 10, + CURLOPT_TIMEOUT => 30, + CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1, + CURLOPT_CUSTOMREQUEST => "GET", + CURLOPT_HTTPHEADER => array( + "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3", + "Accept-Encoding: gzip, deflate, br", + "Accept-Language: zh-CN,zh;q=0.9,en-US;q=0.8,en;q=0.7,zh-TW;q=0.6,ja;q=0.5", + "Cache-Control: no-cache", + "Connection: keep-alive", + "Referer: https://www.bilibili.com", + "Pragma: no-cache", + "Sec-Fetch-Mode: navigate", + "Sec-Fetch-Site: none", + "Sec-Fetch-User: ?1", + "Upgrade-Insecure-Requests: 1", + "User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.97 Safari/537.36", + "cache-control: no-cache" + ), + )); + + $response = curl_exec($curl); + $err = curl_error($curl); + + curl_close($curl); + + if ($err) { + echo "cURL Error #:" . $err; + } else { + echo $response; + $result = json_decode($response, true); + foreach ($result['data']['list']['vlist'] as $item) { + echo "av" . $item['aid'] . " "; + } + } + exit; + + // echo $result['data']['vlist'][0]['aid'] . "\t"; + // $pageNo++; + } + + } + + public function queryFavList() + { + $invalidTitle = "已失效视频"; + $pageNo = 1; + + + for ($i = 1; $i < 65; $i++) { + $curl = curl_init(); + $url = "https://api.bilibili.com/medialist/gateway/base/spaceDetail?media_id=50069625&pn={$i}&ps=20&keyword=&order=mtime&type=0&tid=0&jsonp=jsonp"; + curl_setopt_array($curl, array( + CURLOPT_URL => $url, + CURLOPT_RETURNTRANSFER => true, + CURLOPT_ENCODING => "", + CURLOPT_MAXREDIRS => 10, + CURLOPT_TIMEOUT => 30, + CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1, + CURLOPT_CUSTOMREQUEST => "GET", + CURLOPT_HTTPHEADER => array( + "Accept: application/json, text/plain, */*", + "Accept-Encoding: gzip, deflate, br", + "Accept-Language: en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7,zh-TW;q=0.6,ja;q=0.5", + "Cache-Control: no-cache", + "Connection: keep-alive", +// "Cookie: buvid3=5566647C-DDE5-4AFF-8711-89C9DB2B7061110244infoc; LIVE_BUVID=AUTO2315591376644842; sid=kbyrjw72; stardustvideo=1; CURRENT_FNVAL=16; rpdid=|(J~|JluJmmk0J\'ullmuJ~~kJ; CURRENT_QUALITY=80; UM_distinctid=16b11bb4099eb-033fd8b69a435d-37647e03-13c680-16b11bb40bf97; fts=1562149769; im_notify_type_279025=0; _uuid=69A60E94-E8FE-54B8-DCCD-0F519F0DEF7480343infoc; DedeUserID=279025; DedeUserID__ckMd5=9a79e15294e6b8bb; SESSDATA=b114a39d%2C1573361994%2C2ed0a2a1; bili_jct=b35f8e9780e4a80dd07f316f781f179b; bp_t_offset_279025=315386152995357818", + "Cookie: SESSDATA=6cf335a1%2C1576033995%2C69d8d2b1", + "Host: api.bilibili.com", + "Origin: https://space.bilibili.com", + "Postman-Token: c7f849ec-aaf9-4c71-851d-4ca061e725d0,f8f582bc-4872-4aa7-b209-b926364d4d1a", + "Referer: https://space.bilibili.com/279025/favlist", + "Sec-Fetch-Mode: cors", + "Sec-Fetch-Site: same-site", + "User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.70 Safari/537.36", + "cache-control: no-cache" + ), + )); + + $response = curl_exec($curl); + $err = curl_error($curl); + + curl_close($curl); + + if ($err) { + echo "cURL Error #:" . $err; + } else { + $responseArr = json_decode($response, true); + if ($responseArr['code'] == 0) { + $data = $responseArr['data']; + $medias = $data['medias']; + if (count($medias) > 0) { + foreach ($medias as $item) { + if ($item['title'] != $invalidTitle) { + echo "av" . $item['id'] . "\t"; + } + } + } + } + } + usleep(random_int(1000, 10000) * 1000); + } + + + } + + public function getAoaFavList($id = 50076725) + { + $invalidTitle = "已失效视频"; + $pageNo = 1; + + + for ($i = 1; $i < 65; $i++) { + $curl = curl_init(); + $url = "https://api.bilibili.com/medialist/gateway/base/spaceDetail?media_id=50076725&pn={$i}&ps=20&keyword=&order=mtime&type=0&tid=0&jsonp=jsonp"; + curl_setopt_array($curl, array( + CURLOPT_URL => $url, + CURLOPT_RETURNTRANSFER => true, + CURLOPT_ENCODING => "", + CURLOPT_MAXREDIRS => 10, + CURLOPT_TIMEOUT => 30, + CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1, + CURLOPT_CUSTOMREQUEST => "GET", + CURLOPT_HTTPHEADER => array( + "Accept: application/json, text/plain, */*", + "Accept-Encoding: gzip, deflate, br", + "Accept-Language: en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7,zh-TW;q=0.6,ja;q=0.5", + "Cache-Control: no-cache", + "Connection: keep-alive", + "Cookie: buvid3=5566647C-DDE5-4AFF-8711-89C9DB2B7061110244infoc; LIVE_BUVID=AUTO2315591376644842; sid=kbyrjw72; stardustvideo=1; CURRENT_FNVAL=16; rpdid=|(J~|JluJmmk0J'ullmuJ~~kJ; CURRENT_QUALITY=80; UM_distinctid=16b11bb4099eb-033fd8b69a435d-37647e03-13c680-16b11bb40bf97; fts=1562149769; im_notify_type_279025=0; _uuid=69A60E94-E8FE-54B8-DCCD-0F519F0DEF7480343infoc; DedeUserID=279025; DedeUserID__ckMd5=9a79e15294e6b8bb; SESSDATA=b114a39d%2C1573361994%2C2ed0a2a1; bili_jct=b35f8e9780e4a80dd07f316f781f179b; bp_t_offset_279025=317592473398470273", + "Host: api.bilibili.com", + "Origin: https://space.bilibili.com", + "Postman-Token: c7f849ec-aaf9-4c71-851d-4ca061e725d0,f8f582bc-4872-4aa7-b209-b926364d4d1a", + "Referer: https://space.bilibili.com/279025/favlist", + "Sec-Fetch-Mode: cors", + "Sec-Fetch-Site: same-site", + "User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.70 Safari/537.36", + "cache-control: no-cache" + ), + )); + + $response = curl_exec($curl); + $err = curl_error($curl); + + curl_close($curl); + + if ($err) { + echo "cURL Error #:" . $err; + } else { + $responseArr = json_decode($response, true); + if ($responseArr['code'] == 0) { + $data = $responseArr['data']; + $medias = $data['medias']; + if (count($medias) > 0) { + foreach ($medias as $item) { + if ($item['title'] != $invalidTitle) { + echo "av" . $item['id'] . " "; + } + } + } + } + } + usleep(random_int(1000, 10000) * 1000); + } + } + + public function downloadBSitePlaylist($aid, $dir = "/Volumes/intel660p/video/mv/mp4") + { + $dirExists = is_dir($dir); + if ($dirExists) { + echo shell_exec("cd {$dir} && annie -n 100 -p " .$aid); + } + } + + public function deleteFileNotContainP() + { + $dir = "/Volumes/intel660p/video/mv/mp4"; + $files = scandir($dir); + foreach ($files as $file) { + if ($file == "." || $file == "..") { + continue; + } + if (preg_match("#P\d{1,2}#", $file)) { + echo $file. "\n"; + } else { + echo "to delete file " . $file ."\n"; + unlink($dir ."/" . $file); + } + } + } +} diff --git a/database/factories/BiCookiesFactory.php b/database/factories/BiCookiesFactory.php new file mode 100644 index 0000000..0e2b93d --- /dev/null +++ b/database/factories/BiCookiesFactory.php @@ -0,0 +1,9 @@ +define(App\BiCookies::class, function (Faker $faker) { + return [ + // + ]; +}); diff --git a/database/factories/BilibiliCollectionsFactory.php b/database/factories/BilibiliCollectionsFactory.php new file mode 100644 index 0000000..78dcf15 --- /dev/null +++ b/database/factories/BilibiliCollectionsFactory.php @@ -0,0 +1,9 @@ +define(App\BilibiliCollections::class, function (Faker $faker) { + return [ + // + ]; +}); diff --git a/database/factories/BilibiliUpVideosFactory.php b/database/factories/BilibiliUpVideosFactory.php new file mode 100644 index 0000000..d3a7146 --- /dev/null +++ b/database/factories/BilibiliUpVideosFactory.php @@ -0,0 +1,9 @@ +define(App\BilibiliUpVideos::class, function (Faker $faker) { + return [ + // + ]; +}); diff --git a/database/migrations/2019_11_03_131136_create_bilibili_collections_table.php b/database/migrations/2019_11_03_131136_create_bilibili_collections_table.php new file mode 100644 index 0000000..eef9071 --- /dev/null +++ b/database/migrations/2019_11_03_131136_create_bilibili_collections_table.php @@ -0,0 +1,37 @@ +increments('id'); + $table->bigInteger("media_id")->comment("收藏夹识别 id"); + $table->string("title")->comment("收藏夹名称"); + $table->bigInteger("video_id")->comment("视频 id"); + $table->unsignedTinyInteger("is_available")->comment("是否有效，0：有效，1：失效"); + $table->unsignedTinyInteger("is_downloaded")->comment("是否已下载, 0: 未下载, 1：已下载"); + $table->timestamps(); + $table->index(["media_id", "video_id"], "idx_mid_aid"); + }); + } + + /** + * Reverse the migrations. + * + * @return void + */ + public function down() + { + Schema::dropIfExists('bilibili_collections'); + } +} diff --git a/database/migrations/2019_11_03_131446_create_bilibili_up_videos_table.php b/database/migrations/2019_11_03_131446_create_bilibili_up_videos_table.php new file mode 100644 index 0000000..e1ab844 --- /dev/null +++ b/database/migrations/2019_11_03_131446_create_bilibili_up_videos_table.php @@ -0,0 +1,37 @@ +increments('id'); + $table->bigInteger("mid")->comment("up的唯一识别 id"); + $table->string("up_name")->comment("up 名"); + $table->bigInteger("video_id")->comment("视频 id"); + $table->unsignedTinyInteger("is_available")->comment("是否有效，0：有效，1：失效"); + $table->unsignedTinyInteger("is_downloaded")->comment("是否已下载, 0: 未下载, 1：已下载"); + $table->timestamps(); + $table->index(["mid", "video_id"], "idx_mid_aid"); + }); + } + + /** + * Reverse the migrations. + * + * @return void + */ + public function down() + { + Schema::dropIfExists('bilibili_up_videos'); + } +} diff --git a/database/migrations/2019_11_10_154228_create_bi_cookies_table.php b/database/migrations/2019_11_10_154228_create_bi_cookies_table.php new file mode 100644 index 0000000..1e61d5b --- /dev/null +++ b/database/migrations/2019_11_10_154228_create_bi_cookies_table.php @@ -0,0 +1,32 @@ +increments('id'); + $table->string("cookie"); + $table->timestamps(); + }); + } + + /** + * Reverse the migrations. + * + * @return void + */ + public function down() + { + Schema::dropIfExists('bi_cookies'); + } +} diff --git a/run.sh b/run.sh new file mode 100755 index 0000000..4770cd7 --- /dev/null +++ b/run.sh @@ -0,0 +1,2 @@ +#!/bin/sh +php /Users/shixuesen/workspace/dev/src/graph-tutorial/artisan serve --env=local