@ -4,13 +4,32 @@
namespace App\Services ; namespace App\Services ;
use App\XiuGirl ;
use Exception ;
use Log ;
use QL\QueryList ; use QL\QueryList ;
use Illuminate\Http\File ; use Illuminate\Http\File ;
use Illuminate\Support\Facades\Storage ; use Illuminate\Support\Facades\Storage ;
use App\Utils\FileUtils ; use App\Utils\FileUtils ;
use App\Utils\CommonUtils ;
class NewXiuGirlsService class NewXiuGirlsService
{ {
private $queryInstance ;
public function __construct ()
{
$this -> queryInstance = QueryList :: getInstance ();
}
/**
* @ return QueryList | null
*/
public function getQueryInstance () : ? QueryList
{
return $this -> queryInstance ;
}
private static $name_dir = [ private static $name_dir = [
" 周韦彤 " => [ " 周韦彤 " => [
" dir " => " 周韦彤 " , " dir " => " 周韦彤 " ,
@ -85,22 +104,15 @@ class NewXiuGirlsService
$NUM_OF_ATTEMPTS = 50 ; $NUM_OF_ATTEMPTS = 50 ;
// self::$name_dir = array_slice(self::$name_dir, 8, 10);
foreach ( self :: $name_dir as $username => $name ) { foreach ( self :: $name_dir as $username => $name ) {
usleep ( random_int ( 1000 , 10000 ) * 1000 );
$peopleUrl = " https://xsnvshen.com/girl/ " ;
$peopleUrl .= $name [ 'code' ];
$albumSelector = " .entryAblum > .star-mod-bd > ul > li > a " ;
$baseQl = QueryList :: get ( $peopleUrl );
$ql = $baseQl -> find ( $albumSelector ) -> attrs ( " href " );
$items = $ql -> all ();
$items = $this -> queryAllAlbum ( $name );
// $items = array_slice($items, 49);
// $items = array_slice($items, 49);
// print_r($items);exit;
// print_r($items);exit;
$baseUrl = " https://www.xsnvshen.com " ; $baseUrl = " https://www.xsnvshen.com " ;
// $baseDir = "/Users/shixuesen/Documents/xg/" . $name['dir ']. "/";
$baseDir = " /Volumes/intel660p/image/xg/ " . $name [ 'dir '] . " / " ;
// $baseDir = "/Users/shixuesen/Documents/xg/" . $name["dir"]. "/";
$baseDir = " /Volumes/intel660p/image/xg/ " . $name [ " dir " ] . " / " ;
if ( ! file_exists ( $baseDir )) { if ( ! file_exists ( $baseDir )) {
mkdir ( $baseDir ); mkdir ( $baseDir );
} }
@ -111,57 +123,15 @@ class NewXiuGirlsService
// $items = array_slice($items, 1);
// $items = array_slice($items, 1);
// print_r($items);continue;
// print_r($items);continue;
foreach ( $items as $item ) { foreach ( $items as $item ) {
usleep ( random_int ( 1000 , 10000 ) * 10 00);
CommonUtils :: randomSleep ( 1 00);
echo " 相册子链接: " . $item . " \n " ; echo " 相册子链接: " . $item . " \n " ;
$queryItemUrlTimes = 1 ;
do {
try {
$html = ( new \QL\QueryList ) -> get ( $baseUrl . $item );
} catch ( \Exception $e ) {
\Log :: error ( " 查询相册子链接失败，将重试， 异常信息: " . $e -> getMessage ());
$sleepTime = 1000 * random_int ( 1000 , 10000 );
echo " 查询相册子链接失败 sleep { $sleepTime } nano second \n " ;
usleep ( $sleepTime );
$queryItemUrlTimes ++ ;
}
break ;
} while ( $queryItemUrlTimes < 10 );
$title = $html -> find ( " h1 > a " ) -> texts ();
dump ( $title );
echo " 相册名: " . $title [ 0 ] . " \n " ;
while ( trim ( $title [ 0 ]) == " 古诗文 " ) {
echo " here error happenned \n " ;
// dump($html->getHtml());
usleep ( random_int ( 1 , 1000 ) * 50000 );
$html = QueryList :: get ( $baseUrl . $item );
$title = $html -> find ( " h1 > a " ) -> texts ();
// continue;
}
$title = $this -> getAlbumTitle ( $baseUrl , $item );
// break;
// break;
$albumPath = " " ; $albumPath = " " ;
$albumCode = explode ( " / " , $item )[ 2 ]; $albumCode = explode ( " / " , $item )[ 2 ];
if ( array_key_exists ( $albumCode , $baseMap )) {
$albumPath = $baseMap [ $albumCode ];
}
if ( $albumPath == " " && ! file_exists ( $baseDir . explode ( " / " , $item )[ 2 ] . " - " . $title [ 0 ])) {
$albumPath = $baseDir . explode ( " / " , $item )[ 2 ] . " - " . $title [ 0 ];
mkdir ( $albumPath );
}
$albumPath = $this -> generateAlbumPath ( $albumPath , $albumCode , $baseMap , $baseDir , $item , $title [ 0 ]);
$attempts = 0 ; $attempts = 0 ;
$images = [];
do {
try {
$images = QueryList :: get ( $baseUrl . $item ) -> find ( " .swi-hd > img " ) -> attrs ( " src " );
} catch ( \Exception $e ) {
echo $e -> getTraceAsString () . " \n " ;
$sleepTime = 10000 * random_int ( 100 , 1000 );
echo " sleep { $sleepTime } nano second \n " ;
usleep ( $sleepTime );
$attempts ++ ;
continue ;
}
break ;
} while ( $attempts <= $NUM_OF_ATTEMPTS );
$images = $this -> queryImages ( $baseUrl , $item , $attempts , $NUM_OF_ATTEMPTS );
foreach ( $images as $image ) { foreach ( $images as $image ) {
if ( strpos ( $image , " thumb_600x900 " ) !== false ) { if ( strpos ( $image , " thumb_600x900 " ) !== false ) {
$image = str_replace ( " thumb_600x900/ " , " " , $image ); $image = str_replace ( " thumb_600x900/ " , " " , $image );
@ -170,147 +140,19 @@ class NewXiuGirlsService
* todo 修改 * todo 修改
* CommonService :: downloadImage ( $albumPath , " https: " . $image ); * CommonService :: downloadImage ( $albumPath , " https: " . $image );
*/ */
if ( file_exists ( $albumPath . " / " . pathinfo ( " http: " . $image )[ 'filename' ] . " .jpg " ) || file_exists ( $albumPath . " / " . $username . " - " . $albumCode . " - " . pathinfo ( " http: " . $image )[ 'filename' ] . " .jpg " )) {
if ( ! $this -> checkIfNeedDownload ( $image , $username , $albumCode , $name , $albumPath )){
continue ; continue ;
} }
$opts = array ( 'http' => [ 'header' =>
$opts = array ( " http " => [ " header " =>
" User-Agent:Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.109 Safari/537.36 \r \n
" User-Agent:Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.109 Safari/537.36 \r \n
Referer : " . $baseUrl . $item . " \r\n "
Referer : " . $baseUrl . $item . " \r\n "
]); ]);
$curl_handle = curl_init ();
curl_setopt ( $curl_handle , CURLOPT_URL , " https: " . $image );
curl_setopt ( $curl_handle , CURLOPT_CONNECTTIMEOUT , 2000 );
curl_setopt ( $curl_handle , CURLOPT_RETURNTRANSFER , 1 );
curl_setopt ( $curl_handle , CURLOPT_USERAGENT , 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.109 Safari/537.36' );
curl_setopt ( $curl_handle , CURLOPT_REFERER , $baseUrl . $item );
$query = curl_exec ( $curl_handle );
$i = 1 ;
while ( $query === false ) {
echo 'Curl error: ' . curl_error ( $curl_handle ) . " \n " ;
echo " retry times: " . $i ++ . " times \n " ;
sleep ( 1 );
$sleepTime = 1000 * random_int ( 1000 , 10000 );
echo " retry sleep { $sleepTime } nano second \n " ;
usleep ( $sleepTime );
$query = curl_exec ( $curl_handle );
if ( $i >= 100 ) {
break ;
}
}
echo curl_error ( $curl_handle );
$fp = fopen ( $albumPath . " / " . $username . " - " . $albumCode . " - " . pathinfo ( " http: " . $image )[ 'filename' ] . " .jpg " , 'x' );
fwrite ( $fp , $query );
fclose ( $fp );
$sleepTime = 1000 * random_int ( 100 , 1000 );
echo " after write image sleep { $sleepTime } nano second \n " ;
usleep ( $sleepTime );
$this -> processDownloadImage ( $image , $baseUrl , $item , $albumPath , $username , $albumCode );
} }
usleep ( 1000 * random_int ( 100 , 1000 )); usleep ( 1000 * random_int ( 100 , 1000 ));
} }
}
exit ;
$baseUrl = " https://www.xsnvshen.com/girl/22162 " ;
$albumSelector = " .entryAblum > .star-mod-bd > ul > li > a " ;
$baseQl = QueryList :: get ( $baseUrl );
$ql = $baseQl -> find ( $albumSelector ) -> attrs ( " href " );
// /*
// new dir ----
$titles = $baseQl -> find ( $albumSelector ) -> attrs ( " * " );
// print_r($titles->all());exit;
$baseDir = " /Users/shixuesen/Documents/xg/ycc/ " ;
foreach ( $titles -> all () as $item ) {
if ( file_exists ( $baseDir . explode ( " / " , $item [ 'href' ])[ 2 ])) {
echo " old name : " . $baseDir . explode ( " / " , $item [ 'href' ])[ 2 ] . " \n " ;
rename ( $baseDir . explode ( " / " , $item [ 'href' ])[ 2 ], $baseDir . explode ( " / " , $item [ 'href' ])[ 2 ] . " - " . $item [ 'title' ]);
}
}
exit ;
// new dir ----
// */
$items = $ql -> all ();
// $items = array_slice($items, 49);
// print_r($items);exit;
$baseUrl = " https://www.xsnvshen.com " ;
$baseDir = " /Users/shixuesen/Documents/xg/ycc/ " ;
// $items = array_slice($items, 1);
// print_r($items);exit;
foreach ( $items as $item ) {
print_r ( $item );
$html = QueryList :: get ( $baseUrl . $item );
$title = $html -> find ( " h1 > a " ) -> texts ();
print_r ( $title );
// print_r(is_dir($baseDir));exit;
$albumPath = " " ;
// if (count($title) >0 ) {
// if (!file_exists($baseDir .$title[0])) {
// mkdir($baseDir .$title[0]);
// }
// $albumPath = $baseDir.$title[0];
// } else {
if ( ! file_exists ( $baseDir . explode ( " / " , $item )[ 2 ])) {
mkdir ( $baseDir . explode ( " / " , $item )[ 2 ]);
}
$albumPath = $baseDir . explode ( " / " , $item )[ 2 ];
// }
$images = QueryList :: get ( $baseUrl . $item ) -> find ( " .swi-hd > img " ) -> attrs ( " src " );
// mkdir("")
// print_r($images);
foreach ( $images as $image ) {
if ( file_exists ( $albumPath . " / " . pathinfo ( " http: " . $image )[ 'filename' ] . " .jpg " )) {
continue ;
// unlink($albumPath."/".pathinfo("http:".$image)['filename'].".jpg");
}
$opts = array ( 'http' => [ 'header' =>
" User-Agent:Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.109 Safari/537.36 \r \n
Referer : " . $baseUrl . $item . " \r\n "
]);
$curl_handle = curl_init ();
curl_setopt ( $curl_handle , CURLOPT_URL , " https: " . $image );
curl_setopt ( $curl_handle , CURLOPT_CONNECTTIMEOUT , 2000 );
curl_setopt ( $curl_handle , CURLOPT_RETURNTRANSFER , 1 );
curl_setopt ( $curl_handle , CURLOPT_USERAGENT , 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.109 Safari/537.36' );
curl_setopt ( $curl_handle , CURLOPT_REFERER , $baseUrl . $item );
$query = curl_exec ( $curl_handle );
$i = 1 ;
while ( $query === false ) {
echo 'Curl error: ' . curl_error ( $curl_handle ) . " \n " ;
echo " retry times: " . $i ++ . " times \n " ;
sleep ( 1 );
$query = curl_exec ( $curl_handle );
if ( $i >= 100 ) {
break ;
}
}
echo curl_error ( $curl_handle );
$fp = fopen ( $albumPath . " / " . pathinfo ( " http: " . $image )[ 'filename' ] . " .jpg " , 'x' );
fwrite ( $fp , $query );
fclose ( $fp );
// file_put_contents("1.jpg", $query);
// curl_close($curl_handle);
// exit;
// $context = stream_context_create($opts);
// $a = file_get_contents("http:".$image, false, $context);exit;
// Storage::put($item.pathinfo($image)["filename"], file_get_contents("http:".$image, false, $context));
}
// exit;
} }
} }
@ -321,10 +163,10 @@ class NewXiuGirlsService
print_r ([ self :: $name_dir [ $iUser ]]); print_r ([ self :: $name_dir [ $iUser ]]);
foreach ([ self :: $name_dir [ $iUser ]] as $username => $name ) { foreach ([ self :: $name_dir [ $iUser ]] as $username => $name ) {
$username = $iUser ; $username = $iUser ;
dump ( " here { $username } , { $name [ 'code '] } " );
dump ( " here { $username } , { $name [ " code ] } " );
usleep ( random_int ( 1000 , 10000 ) * 1000 ); usleep ( random_int ( 1000 , 10000 ) * 1000 );
$peopleUrl = " https://xsnvshen.com/girl/ " ; $peopleUrl = " https://xsnvshen.com/girl/ " ;
$peopleUrl .= $name [ 'code '];
$peopleUrl .= $name [ " code ];
$albumSelector = " .entryAblum > .star-mod-bd > ul > li > a " ; $albumSelector = " .entryAblum > .star-mod-bd > ul > li > a " ;
$baseQl = QueryList :: get ( $peopleUrl ); $baseQl = QueryList :: get ( $peopleUrl );
@ -335,8 +177,8 @@ class NewXiuGirlsService
// print_r($items);exit;
// print_r($items);exit;
$baseUrl = " https://www.xsnvshen.com " ; $baseUrl = " https://www.xsnvshen.com " ;
// $baseDir = "/Users/shixuesen/Documents/xg/" . $name['dir ']. "/";
$baseDir = " /Volumes/intel660p/image/xg/ " . $name [ 'dir '] . " / " ;
// $baseDir = "/Users/shixuesen/Documents/xg/" . $name["dir "]. "/";
$baseDir = " /Volumes/intel660p/image/xg/ " . $name [ " dir ] . " / " ;
if ( ! file_exists ( $baseDir )) { if ( ! file_exists ( $baseDir )) {
mkdir ( $baseDir ); mkdir ( $baseDir );
} }
@ -356,9 +198,9 @@ class NewXiuGirlsService
$queryItemUrlTimes = 1 ; $queryItemUrlTimes = 1 ;
do { do {
try { try {
$html = ( new \QL\ QueryList) -> get ( $baseUrl . $item );
} catch ( \ Exception$e ) {
\ Log:: error ( " 查询相册子链接失败，将重试， 异常信息: " . $e -> getMessage ());
$html = ( new QueryList ) -> get ( $baseUrl . $item );
} catch ( Exception $e ) {
Log :: error ( " 查询相册子链接失败，将重试， 异常信息: " . $e -> getMessage ());
$sleepTime = 1000 * random_int ( 1000 , 10000 ); $sleepTime = 1000 * random_int ( 1000 , 10000 );
echo " 查询相册子链接失败 sleep { $sleepTime } nano second \n " ; echo " 查询相册子链接失败 sleep { $sleepTime } nano second \n " ;
usleep ( $sleepTime ); usleep ( $sleepTime );
@ -389,7 +231,7 @@ class NewXiuGirlsService
do { do {
try { try {
$images = QueryList :: get ( $baseUrl . $item ) -> find ( " .swi-hd > img " ) -> attrs ( " src " ); $images = QueryList :: get ( $baseUrl . $item ) -> find ( " .swi-hd > img " ) -> attrs ( " src " );
} catch ( \ Exception$e ) {
} catch ( Exception $e ) {
echo $e -> getTraceAsString () . " \n " ; echo $e -> getTraceAsString () . " \n " ;
$sleepTime = 10000 * random_int ( 100 , 1000 ); $sleepTime = 10000 * random_int ( 100 , 1000 );
echo " sleep { $sleepTime } nano second \n " ; echo " sleep { $sleepTime } nano second \n " ;
@ -408,10 +250,10 @@ class NewXiuGirlsService
* CommonService :: downloadImage ( $albumPath , " https: " . $image ); * CommonService :: downloadImage ( $albumPath , " https: " . $image );
*/ */
if ( file_exists ( $albumPath . " / " . pathinfo ( " http: " . $image )[ 'filename '] . " .jpg " ) || file_exists ( $albumPath . " / " . $username . " - " . $albumCode . " - " . pathinfo ( " http: " . $image )[ 'filename '] . " .jpg " )) {
if ( file_exists ( $albumPath . " / " . pathinfo ( " http: " . $image )[ " filename ] . " .jpg " ) || file_exists ( $albumPath . " / " . $username . " - " . $albumCode . " - " . pathinfo ( " http: " . $image )[ " filename ] . " .jpg " )) {
continue ; continue ;
} }
$opts = array ( 'http' => [ 'header '=>
$opts = array ( " http " => [ " header =>
" User-Agent:Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.109 Safari/537.36 \r \n
" User-Agent:Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.109 Safari/537.36 \r \n
Referer : " . $baseUrl . $item . " \r\n "
Referer : " . $baseUrl . $item . " \r\n "
]); ]);
@ -419,12 +261,12 @@ class NewXiuGirlsService
curl_setopt ( $curl_handle , CURLOPT_URL , " https: " . $image ); curl_setopt ( $curl_handle , CURLOPT_URL , " https: " . $image );
curl_setopt ( $curl_handle , CURLOPT_CONNECTTIMEOUT , 2000 ); curl_setopt ( $curl_handle , CURLOPT_CONNECTTIMEOUT , 2000 );
curl_setopt ( $curl_handle , CURLOPT_RETURNTRANSFER , 1 ); curl_setopt ( $curl_handle , CURLOPT_RETURNTRANSFER , 1 );
curl_setopt ( $curl_handle , CURLOPT_USERAGENT , ' Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.109 Safari/537.36);
curl_setopt ( $curl_handle , CURLOPT_USERAGENT , " );
curl_setopt ( $curl_handle , CURLOPT_REFERER , $baseUrl . $item ); curl_setopt ( $curl_handle , CURLOPT_REFERER , $baseUrl . $item );
$query = curl_exec ( $curl_handle ); $query = curl_exec ( $curl_handle );
$i = 1 ; $i = 1 ;
while ( $query === false ) { while ( $query === false ) {
echo 'Curl error: '. curl_error ( $curl_handle ) . " \n " ;
echo " Curl error: . curl_error ( $curl_handle ) . " \n " ;
echo " retry times: " . $i ++ . " times \n " ; echo " retry times: " . $i ++ . " times \n " ;
sleep ( 1 ); sleep ( 1 );
$sleepTime = 1000 * random_int ( 1000 , 10000 ); $sleepTime = 1000 * random_int ( 1000 , 10000 );
@ -437,7 +279,7 @@ class NewXiuGirlsService
} }
echo curl_error ( $curl_handle ); echo curl_error ( $curl_handle );
$fp = fopen ( $albumPath . " / " . $username . " - " . $albumCode . " - " . pathinfo ( " http: " . $image )[ 'filename '] . " .jpg " , 'x ');
$fp = fopen ( $albumPath . " / " . $username . " - " . $albumCode . " - " . pathinfo ( " http: " . $image )[ " filename ] . " .jpg " , " x );
fwrite ( $fp , $query ); fwrite ( $fp , $query );
fclose ( $fp ); fclose ( $fp );
$sleepTime = 1000 * random_int ( 100 , 1000 ); $sleepTime = 1000 * random_int ( 100 , 1000 );
@ -451,4 +293,201 @@ class NewXiuGirlsService
} }
exit ; exit ;
} }
/**
* xiugirl网站个人页展示的是全部的相册，不用分页查
* @ param $name
* @ return array
*/
private function queryAllAlbum ( $name ) : array
{
CommonUtils :: randomSleep ( 100 );
$peopleUrl = " https://xsnvshen.com/girl/ " ;
$peopleUrl .= $name [ " code " ];
$albumSelector = " .entryAblum > .star-mod-bd > ul > li > a " ;
$baseQl = $this -> getQueryInstance () -> get ( $peopleUrl );
$ql = $baseQl -> find ( $albumSelector ) -> attrs ( " href " );
$items = $ql -> all ();
return $items ;
}
/**
* 获取相册名
* @ param string $baseUrl
* @ param $item
* @ return array
* @ throws Exception
*/
private function getAlbumTitle ( string $baseUrl , $item )
{
$queryItemUrlTimes = 1 ;
do {
try {
$html = $this -> getQueryInstance () -> get ( $baseUrl . $item );
} catch ( Exception $e ) {
Log :: error ( " 查询相册子链接失败，将重试， 异常信息: " . $e -> getMessage ());
$sleepTime = 1000 * random_int ( 1000 , 10000 );
echo " 查询相册子链接失败 sleep { $sleepTime } nano second \n " ;
usleep ( $sleepTime );
$queryItemUrlTimes ++ ;
continue ;
}
break ;
} while ( $queryItemUrlTimes < 10 );
$title = $html -> find ( " h1 > a " ) -> texts ();
dump ( $title );
echo " 相册名: " . $title [ 0 ] . " \n " ;
while ( trim ( $title [ 0 ]) == " 古诗文 " ) {
echo " here error happenned \n " ;
// dump($html->getHtml());
usleep ( random_int ( 1 , 1000 ) * 50000 );
$html = $this -> getQueryInstance () -> get ( $baseUrl . $item );
$title = $html -> find ( " h1 > a " ) -> texts ();
// continue;
}
return $title ;
}
/**
* @ param string $albumPath
* @ param string $albumCode
* @ param array $baseMap
* @ param string $baseDir
* @ param $item
* @ param $title
* @ return mixed | string
*/
private function generateAlbumPath ( string $albumPath , string $albumCode , array $baseMap , string $baseDir , $item , $title ) : string
{
if ( array_key_exists ( $albumCode , $baseMap )) {
$albumPath = $baseMap [ $albumCode ];
}
if ( $albumPath == " " && ! file_exists ( $baseDir . explode ( " / " , $item )[ 2 ] . " - " . $title )) {
$albumPath = $baseDir . explode ( " / " , $item )[ 2 ] . " - " . $title ;
mkdir ( $albumPath );
}
return $albumPath ;
}
/**
* 获取相册的所有图片链接
* @ param string $baseUrl
* @ param $item
* @ param int $attempts
* @ param int $NUM_OF_ATTEMPTS
* @ return array
* @ throws Exception
*/
private function queryImages ( string $baseUrl , $item , int $attempts , int $NUM_OF_ATTEMPTS ) : array
{
$images = [];
do {
try {
$images = $this -> getQueryInstance () -> get ( $baseUrl . $item ) -> find ( " .swi-hd > img " ) -> attrs ( " src " );
} catch ( Exception $e ) {
echo $e -> getTraceAsString () . " \n " ;
$sleepTime = 10000 * random_int ( 100 , 1000 );
echo " sleep { $sleepTime } nano second \n " ;
usleep ( $sleepTime );
$attempts ++ ;
continue ;
}
break ;
} while ( $attempts <= $NUM_OF_ATTEMPTS );
return $images -> all ();
}
/**
* @ param array $image
* @ param string $username
* @ param string $albumCode
* @ param $name
* @ param string $albumPath
* @ return bool
*/
private function checkIfNeedDownload ( string $image , string $username , string $albumCode , $name , string $albumPath ) : bool
{
$fileInfo = pathinfo ( " http: " . $image );
$imageFileName = $username . " - " . $albumCode . " - " . $fileInfo [ " filename " ] . " .jpg " ;
$images = XiuGirl :: where ([
" user_id " => $name [ " code " ],
" album_id " => $albumCode ,
" image_name " => $imageFileName
]) -> get ();
if ( $images -> count () > 0 ) {
if ( $this -> imageFileExists ( $albumPath , $image , $username , $albumCode )) {
return false ;
} else {
return true ;
}
} else {
if ( $this -> imageFileExists ( $albumPath , $image , $username , $albumCode )) {
$xiuGirl = new XiuGirl ;
$xiuGirl -> user_id = $name [ " code " ];
$xiuGirl -> album_id = $albumCode ;
$xiuGirl -> image_name = $username . " - " . $albumCode . " - " . $fileInfo [ " filename " ] . " .jpg " ;
$xiuGirl -> site_from = " xiugirl " ;
$xiuGirl -> is_downloaded = 1 ;
$xiuGirl -> save ();
return false ;
} else {
return true ;
}
}
}
/**
* @ param string $albumPath
* @ param array $image
* @ param string $username
* @ param string $albumCode
* @ return bool
*/
private function imageFileExists ( string $albumPath , string $image , string $username , string $albumCode ) : bool
{
return file_exists ( $albumPath . " / " . pathinfo ( " http: " . $image )[ " filename " ] . " .jpg " ) || file_exists ( $albumPath . " / " . $username . " - " . $albumCode . " - " . pathinfo ( " http: " . $image )[ " filename " ] . " .jpg " );
}
/**
* @ param array $image
* @ param string $baseUrl
* @ param $item
* @ param string $albumPath
* @ param string $username
* @ param string $albumCode
* @ throws Exception
*/
private function processDownloadImage ( string $image , string $baseUrl , $item , string $albumPath , string $username , string $albumCode ) : void
{
$curl_handle = curl_init ();
curl_setopt ( $curl_handle , CURLOPT_URL , " https: " . $image );
curl_setopt ( $curl_handle , CURLOPT_CONNECTTIMEOUT , 2000 );
curl_setopt ( $curl_handle , CURLOPT_RETURNTRANSFER , 1 );
curl_setopt ( $curl_handle , CURLOPT_USERAGENT , " Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.109 Safari/537.36 " );
curl_setopt ( $curl_handle , CURLOPT_REFERER , $baseUrl . $item );
$content = curl_exec ( $curl_handle );
$i = 1 ;
while ( $content === false ) {
echo " Curl error: " . curl_error ( $curl_handle ) . " \n " ;
echo " retry times: " . $i ++ . " times \n " ;
sleep ( 1 );
$sleepTime = 1000 * random_int ( 1000 , 10000 );
echo " retry sleep { $sleepTime } nano second \n " ;
usleep ( $sleepTime );
$content = curl_exec ( $curl_handle );
if ( $i >= 100 ) {
break ;
}
}
echo curl_error ( $curl_handle );
$fp = fopen ( $albumPath . " / " . $username . " - " . $albumCode . " - " . pathinfo ( " http: " . $image )[ " filename " ] . " .jpg " , " x " );
fwrite ( $fp , $content );
fclose ( $fp );
$sleepTime = 1000 * random_int ( 100 , 1000 );
echo " after write image sleep { $sleepTime } nano second \n " ;
usleep ( $sleepTime );
}
} }
shixuesen
4 years ago