爬取网站图片
运用requests模块处理url并获取数据,etree中xpath方法解析页面标签,urllib模块urlretrieve保存图片,"iso-8859-1"处理中文乱码
#爬取图片并且处理乱码
import requests
from lxml import etree
#urlretrieve可以直接保存图片
from urllib import request
url = "http://pic.netbian.com/4kqiche/"
headers = {
'User-Agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36'
}
response = requests.get(url=url,headers=headers)
page_text = response.text
tree = etree.HTML(page_text)
#xpath得到的data是个list
li_list = tree.xpath("//div[@class='slist']/ul/li")
for li in li_list:
img_src = li.xpath('./a/img/@src')[0]
new_src = url.split("/4")[0] + img_src
print(new_src)
img_name = li.xpath('./a/b/text()')[0]
#通用型,解决中文乱码
img_text = requests.get(url=new_src,headers=headers).text
img_name = img_name.encode('iso-8859-1').decode('gbk') + '.jpg'
request.urlretrieve(url=new_src,filename = "./img/%s"%img_name)
print(f"{img_name}爬取成功")