爬虫爬取万维钢博客文章

看到万维钢的博客,很多文章有启发,奈何访问很慢,经常报500内部错误,于是爬取文章到本地,便于阅读。

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
#!/usr/bin/python
#-*- coding:utf-8 -*-
import requests
import os
import re
from bs4 import BeautifulSoup

def getHTMLText(url):
try:
r = requests.get(url,timeout=30)
r.raise_for_status()
r.encoding=r.apparent_encoding
return r.text
except:
return "error"

def getArticle(html):
soup = BeautifulSoup(html,"html.parser")
title=soup.find("h2","post-title")
content=soup.find("div","post-content")
try:
print(title.string)
#print(type(content))
for child in content.children:
if child.string is not None:
print(child.string)
except:
return "error"

def main():
#url = "http://www.geekonomics10000.com/"i
for i in range (1,1000):
url = "http://www.geekonomics10000.com/"+str(i)
html = getHTMLText(url)
getArticle(html)

main()

bash下运行python3 craw_wwg.py >>wwg.txt即可保存到本地。

参考:https://www.crummy.com/software/BeautifulSoup/bs4/doc.zh/