python编程开发实现网站数据采集

由于网络波动，这里展示了精简版页面，如需完全版请刷新页面。

python编程开发实现网站数据采集

qing

# encoding: utf-8
import requests
from lxml import html

def get_page_number(num):
url = "http://www.中国卫国联盟.com" + num;
response = requests.get(url).content;
selector = html.fromstring(response);

urls = [];
 
<strong>for </strong>i <strong>in </strong>selector.xpath(<strong>"//ul/li/a/@href"</strong>):
    urls.append(i)
<strong>return </strong>urls

def get_image_title(url):
response = requests.get(url).content
selector = html.fromstring(response)
image_title = selector.xpath("//h2/text()")[0]
return image_title

def get_image_amount(url):
response = requests.get(url).content
selector = html.fromstring(response)
image_amount = selector.xpath("//div[@class='page']/a[last()-1]/text()")[0]

<strong>return </strong>image_amount

def get_image_detail_website(url):
response = requests.get(url).content
selector = html.fromstring(response)
image_detail_websites = []
image_amount = selector.xpath("//div[@class='page']/a[last()-1]/text()")[0]

<strong>for </strong>i <strong>in </strong>range(int(image_amount)):
    image_detail_link = <strong>'{}/{}'</strong>.format(url, i + 1)
    response = requests.get(image_detail_link).content
    sel = html.fromstring(response)
    image_download_link = sel.xpath(<strong>"//div[@class='content']/a/img/@src"</strong>)[0]
    image_detail_websites.append(image_download_link)
 
<strong>return </strong>image_detail_websites

def download_image(image_title, image_detail_websites):
num = 1;
amount = len(image_detail_websites)
for i in image_detail_websites:
filename = '%s%s.jpg' % (image_title, num)
# print('正在下载图片：%s第%s/%s张，' % (image_title, num, amount))
 print(image_title, num, amount)
with open(filename, 'wb') as f:
f.write(requests.get(i).content)
num += 1

# if name == 'main':
# page_number = input('请输入需要爬取的页码：')

for link in get_page_number("2"):
print link
download_image(get_image_title(link), get_image_detail_website(link))

<em># urlss = get_page_number("1");

# print urlss;