7牛远程抓取图片

七牛远程抓取图片

公司网站之前的用户头像是存储在自己的服务器，后来管理不便，新增用户头像上传到七牛，为了方便管理，数据统一，领导说把本地服务器的头像全部迁移到七牛。

1.梳理下思路

先判断用户的头像是否在七牛，若不存在，本地如果有则抓取到七牛，然后进行批量抓取

2.七牛判断图片是否存在

<span style="color: #008080;"> 1</span>  <span style="color: #008000;">/*</span><span style="color: #008000;">*</span><span style="color: #008080;"> 2</span> <span style="color: #008000;">     * 查看七牛url是否存在</span><span style="color: #008080;"> 3</span> <span style="color: #008000;">     * @param string $url</span><span style="color: #008080;"> 4</span>      <span style="color: #008000;">*/</span><span style="color: #008080;"> 5</span>     <span style="color: #0000ff;">function</span> url_exists(<span style="color: #800080;">$url</span><span style="color: #000000;">) {</span><span style="color: #008080;"> 6</span>         <span style="color: #0000ff;">require_once</span>(COMMON_PATH."qiniu/rs.php"<span style="color: #000000;">);</span><span style="color: #008080;"> 7</span>         <span style="color: #0000ff;">require_once</span>(COMMON_PATH."qiniu/http.php"<span style="color: #000000;">);</span><span style="color: #008080;"> 8</span>         <span style="color: #800080;">$parts</span>=<span style="color: #008080;">parse_url</span>(<span style="color: #800080;">$url</span><span style="color: #000000;">);</span><span style="color: #008080;"> 9</span>         <span style="color: #800080;">$bucket</span> ；<span style="color: #008000;">//</span><span style="color: #008000;">桶</span><span style="color: #008080;">10</span>         <span style="color: #800080;">$key</span>=<span style="color: #008080;">substr</span>(<span style="color: #800080;">$parts</span>['path'], 1);<span style="color: #008000;">//</span><span style="color: #008000;">七牛文件名</span><span style="color: #008080;">11</span> <span style="color: #008000;">       //密钥</span><span style="color: #008080;">12</span>         <span style="color: #800080;">$accessKey</span><span style="color: #000000;"> ; </span><span style="color: #008080;">13</span>         <span style="color: #800080;">$secretKey</span><span style="color: #000000;"> ;</span><span style="color: #008080;">14</span>         Qiniu_SetKeys(<span style="color: #800080;">$accessKey</span>, <span style="color: #800080;">$secretKey</span><span style="color: #000000;">);</span><span style="color: #008080;">15</span>         <span style="color: #800080;">$client</span> = <span style="color: #0000ff;">new</span> Qiniu_MacHttpClient(<span style="color: #0000ff;">null</span><span style="color: #000000;">);</span><span style="color: #008080;">16</span>         <span style="color: #0000ff;">list</span>(<span style="color: #800080;">$ret</span>, <span style="color: #800080;">$err</span>) = Qiniu_RS_Stat(<span style="color: #800080;">$client</span>, <span style="color: #800080;">$bucket</span>, <span style="color: #800080;">$key</span><span style="color: #000000;">);//查看单个文件信息</span><span style="color: #008080;">17</span>         <span style="color: #0000ff;">if</span> (<span style="color: #800080;">$err</span> !== <span style="color: #0000ff;">null</span><span style="color: #000000;">) {</span><span style="color: #008080;">18</span>             <span style="color: #0000ff;">return</span> <span style="color: #0000ff;">false</span><span style="color: #000000;">;</span><span style="color: #008080;">19</span>         } <span style="color: #0000ff;">else</span><span style="color: #000000;"> {</span><span style="color: #008080;">20</span>             <span style="color: #0000ff;">return</span> <span style="color: #0000ff;">true</span><span style="color: #000000;">;</span><span style="color: #008080;">21</span> <span style="color: #000000;">        }</span><span style="color: #008080;">22</span>     }    

3.移动单个文件到七牛

<span style="color: #008080;"> 1</span> <span style="color: #008000;">/*</span><span style="color: #008000;">*</span><span style="color: #008080;"> 2</span> <span style="color: #008000;">     * 移动单个头像到七牛</span><span style="color: #008080;"> 3</span> <span style="color: #008000;">     * @param string  $remotePath 远程文件在服务器上的物理地址</span><span style="color: #008080;"> 4</span> <span style="color: #008000;">     * @param string  $qiniuPath  七牛文件地址</span><span style="color: #008080;"> 5</span>      <span style="color: #008000;">*/</span><span style="color: #008080;"> 6</span>     <span style="color: #0000ff;">function</span> oneFetch(<span style="color: #800080;">$remotePath</span>,<span style="color: #800080;">$qiniuPath</span><span style="color: #000000;">){</span><span style="color: #008080;"> 7</span>         <span style="color: #0000ff;">if</span>(!<span style="color: #800080;">$this</span>->url_exists(<span style="color: #800080;">$qiniuPath</span>) && <span style="color: #008080;">file_exists</span>(<span style="color: #800080;">$remotePath</span><span style="color: #000000;">)){</span><span style="color: #008080;"> 8</span>             <span style="color: #0000ff;">require_once</span>(COMMON_PATH."qiniu/rs.php"<span style="color: #000000;">);</span><span style="color: #008080;"> 9</span>             <span style="color: #0000ff;">require_once</span>(COMMON_PATH."qiniu/http.php"<span style="color: #000000;">);</span><span style="color: #008080;">10</span>             <span style="color: #0000ff;">require_once</span>(COMMON_PATH."qiniu/conf.php"<span style="color: #000000;">);</span><span style="color: #008080;">11</span>             <span style="color: #008080;">12</span>             <span style="color: #800080;">$client</span> = <span style="color: #0000ff;">new</span> Qiniu_MacHttpClient(<span style="color: #0000ff;">NULL</span><span style="color: #000000;">);</span><span style="color: #008080;">13</span>             <span style="color: #008000;">/*</span><span style="color: #008000;">$accessUrl 远程文件访问地址</span><span style="color: #008080;">14</span> <span style="color: #008000;">            *$bucket桶</span><span style="color: #008080;">15</span> <span style="color: #008000;">            *$key 存储在七牛的文件名</span><span style="color: #008080;">16</span>             <span style="color: #008000;">*/</span><span style="color: #008080;">17</span>             <span style="color: #800080;">$ret</span> = Qiniu_RS_Fetch(<span style="color: #800080;">$client</span>,<span style="color: #800080;">$accessUrl</span>, <span style="color: #800080;">$bucket</span>, <span style="color: #800080;">$key</span><span style="color: #000000;">);</span><span style="color: #008080;">18</span>             <span style="color: #008080;">19</span>             <span style="color: #0000ff;">if</span>(<span style="color: #800080;">$ret</span> != <span style="color: #0000ff;">NULL</span><span style="color: #000000;">)</span><span style="color: #008080;">20</span> <span style="color: #000000;">            {</span><span style="color: #008080;">21</span>                 <span style="color: #800080;">$errlog</span> = 'fail.log';<span style="color: #008000;">//</span><span style="color: #008000;">写错误日志</span><span style="color: #008080;">22</span>                 <span style="color: #008080;">file_put_contents</span>(<span style="color: #800080;">$errlog</span>,    "{<span style="color: #800080;">$key</span>}\n",<span style="color: #000000;"> FILE_APPEND);</span><span style="color: #008080;">23</span> <span style="color: #000000;">            }</span><span style="color: #008080;">24</span>             <span style="color: #0000ff;">else</span><span style="color: #008080;">25</span> <span style="color: #000000;">            {</span><span style="color: #008080;">26</span>                 <span style="color: #008080;">var_dump</span>(<span style="color: #800080;">$ret</span><span style="color: #000000;">);</span><span style="color: #008080;">27</span> <span style="color: #000000;">            }</span><span style="color: #008080;">28</span>         }<span style="color: #0000ff;">else</span><span style="color: #000000;">{</span><span style="color: #008080;">29</span>             <span style="color: #008080;">var_dump</span>('已有头像'<span style="color: #000000;">);</span><span style="color: #008080;">30</span> <span style="color: #000000;">        }</span><span style="color: #008080;">31</span>         <span style="color: #008080;">32</span>     }

 

4.批量抓取

<span style="color: #0000ff;">function</span><span style="color: #000000;"> t(){        </span><span style="color: #008080;">header</span>("Content-type: text/html; charset=utf-8"<span style="color: #000000;">);        </span><span style="color: #008080;">set_time_limit</span>(0<span style="color: #000000;">);        </span><span style="color: #800080;">$start</span> = <span style="color: #008080;">intval</span>(<span style="color: #800080;">$_GET</span>['num'<span style="color: #000000;">]);//开始位置        </span><span style="color: #800080;">$str</span> =  ''<span style="color: #000000;">;        </span><span style="color: #008000;">/*</span><span style="color: #008000;">从数据库查询记录</span><span style="color: #008000;">*/</span>        <span style="color: #800080;">$link</span> = <span style="color: #008080;">mysql_connect</span>("localhost", "root", "root") or <span style="color: #0000ff;">die</span>("error"<span style="color: #000000;">);        </span><span style="color: #800080;">$db_selected</span> = <span style="color: #008080;">mysql_select_db</span>("test", <span style="color: #800080;">$link</span><span style="color: #000000;">);        </span><span style="color: #008080;">mysql_query</span> ("SET NAMES UTF8"<span style="color: #000000;">);        </span><span style="color: #800080;">$limit</span> = 10<span style="color: #000000;">;//每次查询数         </span><span style="color: #800080;">$sql</span> = "SELECT * FROM test order by id desc limit {<span style="color: #800080;">$start</span>},{<span style="color: #800080;">$limit</span>}"<span style="color: #000000;">;           </span><span style="color: #800080;">$result</span> = <span style="color: #008080;">mysql_query</span>(<span style="color: #800080;">$sql</span>) or <span style="color: #0000ff;">die</span>(<span style="color: #008080;">mysql_error</span><span style="color: #000000;">());                </span><span style="color: #0000ff;">if</span>(<span style="color: #008080;">mysql_num_rows</span>(<span style="color: #800080;">$result</span><span style="color: #000000;">))        {            </span><span style="color: #0000ff;">while</span>(<span style="color: #800080;">$row</span>=<span style="color: #008080;">mysql_fetch_assoc</span>(<span style="color: #800080;">$result</span><span style="color: #000000;">)){                    oneFetch(</span><span style="color: #800080;">$row</span>['remotePath'], <span style="color: #800080;">$row</span>['qiniuPath'<span style="color: #000000;">]);            }                </span><span style="color: #800080;">$start</span> += 10<span style="color: #000000;">;            </span><span style="color: #800080;">$url</span> = "/t?num={<span style="color: #800080;">$start</span>}"<span style="color: #000000;">;            </span><span style="color: #800080;">$str</span> = '已经更新'.<span style="color: #800080;">$start</span>.'条<br>'<span style="color: #000000;">;            </span><span style="color: #800080;">$str</span> .= '<script>location.href="'.<span style="color: #800080;">$url.'";</script>'<span style="color: #000000;">;//更新$limit条，跳转防止脚本无响应        }        </span><span style="color: #0000ff;">else</span><span style="color: #000000;">        {            </span><span style="color: #0000ff;">echo</span> '结束'<span style="color: #000000;">;        }                 </span><span style="color: #0000ff;">echo</span> <span style="color: #800080;">$str</span><span style="color: #000000;">;        </span><span style="color: #008080;">ob_flush</span><span style="color: #000000;">();        </span><span style="color: #008080;">flush</span><span style="color: #000000;">();        </span><span style="color: #008080;">sleep</span>(2<span style="color: #000000;">);        </span><span style="color: #0000ff;">unset</span>(<span style="color: #800080;">$sql</span>,<span style="color: #800080;">$result</span>,<span style="color: #800080;">$row</span><span style="color: #000000;">);    }</span>