一条虫虫修炼史
requests+正则表达式爬取猫眼电影100排行榜1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70#!/usr/bin/python3
# -*- coding:utf-8 -*-
import requests
from requests.exceptions import RequestException
import re
import json
#基础网址通过offset偏移值访问页面
base_url='http://maoyan.com/board/4?offset='
#请求头信息
headers={
'Upgrade-Insecure-Requests': '1',
'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/5'
}
def write_to_file(content):
'''
将content追加写入result文件
:param content:
'''
with open('result.txt', 'a', encoding='utf-8') as f:
f.write(json.dumps(content, ensure_ascii=False) + '\n')
f.close()
def parse_page(html):
'''
解析网页HTML将解析的信息保存到文件
:param html:
'''
pattern = re.compile('<dd>.*?board-index.*?>(\d+)</i>.*?data-src="(.*?)".*?name"><a'
+'.*?>(.*?)</a>.*?star">(.*?)</p>.*?releasetime">(.*?)</p>'
+'.*?integer">(.*?)</i>.*?fraction">(.*?)</i>.*?</dd>', re.S)
items = re.findall(pattern, html)
for item in items:
movie_ifo={
'index': item[0],
'image': item[1],
'title': item[2],
'actor': item[3].strip()[3:],
'time': item[4].strip()[5:],
'score': item[5]+item[6]
}
write_to_file(movie_ifo)
def get_one_page(url):
'''
通过网址返回网页的html
:param url:
:return: html
'''
try:
response = requests.get(url,headers=headers)
if response.status_code == 200:
return response.text
return None
except RequestException:
return None
def get_info():
'''
通过不同网址获取信息并保存
:return:
'''
for i in range(0,10):
url = base_url +str(i*10)
html = get_one_page(url)
parse_page(html)
if __name__=='__main__':
get_info()
结果result.txt链接 密码: dham
