下載糗事百科的內容_python版

下載糗事百科的內容_python版,代碼是沒問題的,可以正常運行,但是希望做到以下2方面: 1、多線程下載 2、代碼分離度更高,跟面向對象
關鍵字:下載、python
復制代碼 代碼如下:

#coding:utf-8

import urllib.request
import xml.dom.minidom
import sqlite3
import threading
import time

class logger(object):
def log(self,*msg):
for i in msg:
print(i)

Log = logger()
Log.log('測試下')

class downloader(object):

def __init__(self,url):
self.url = url

def download(self):
Log.log('開始下載',self.url)
try:
content = urllib.request.urlopen(self.url).read()
#req = urllib.request.Request(url)
#response = urllib.request.urlopen(req)
#content = response.read()
Log.log('下載完畢')
return(content)
except:
Log.log('下載出錯')
return(None)


class parser(object):

def __init__(self,content):
#獲得根節點
self.html = xml.dom.minidom.parseString(content)

def parse(self):
Log.log('開始提取數據')
contents = {'content':'','url':[]}
#獲得div節點
divs = self.html.getElementsByTagName('div')
#獲得content節點
for div in divs:
if div.hasAttribute('class') and \
div.getAttribute('class') == 'content':
#獲得糗事百科的內容
textNode = div.childNodes[0]
qContent = textNode.data
#數據填充
contents['content'] = qContent

#獲得上一糗事、下一糗事節點
spans = self.html.getElementsByTagName('span')
for span in spans:
pspan = span.parentNode
if pspan.tagName == 'a':
#pspan為對應的鏈接,此時需要將對應的地址加入數據庫
url = pspan.getAttribute('href')
qid = url[10:][:-4]
#數據填充
contents['url'].append(qid)
Log.log('提取數據完畢')
return(contents)

def downloadPage(qid,db):
url = 'http://www.qiushibaike.com/articles/'+str(qid)+'.htm'
content = downloader(url).download()
if content:
contents = parser(content).parse()
if contents['content']:
db.updateContent(qid,contents['content'])
for i in contents['url']:
db.addQID(i)
if len(contents['url']) == 2:
db.updateStatus(qid,2)

#下載池,表示同時允許下載的鏈接個數
class downloaderPool(object):
def __init__(self,maxLength=15):
self.downloaders = [None]*maxLength
self.downloadList = []
self.db = None

def setDownloadList(self,downloadList):
self.downloadList = list(set(self.downloadList+downloadList))

def setdb(self,db):
self.db = db

def daemon(self):
#每隔一秒查詢線程的狀態,為非活動線程則設置為None
Log.log('設置守護進程')
for index,downloader in enumerate(self.downloaders):
if downloader:
if not downloader.isAlive():
Log.log('將下載器置空',index)
self.downloaders[index] = None

#檢查線程池狀態
for index,downloader in enumerate(self.downloaders):
if not downloader:
qid = self.getQID()
if qid:
#創建線程
t = threading.Thread(target=downloadPage,args=(qid,self.db))
self.downloaders[index] = t
t.start()
t.join()
Log.log('設置下載器',index)
#間隔一秒執行一次
time.sleep(1)

def getQID(self):
try:
tmp = self.downloadList[0]
del self.downloadList[0]
return(tmp)
except:
return(None)

def beginDownload(self):
#創建守護線程
daemon = threading.Thread(target=self.daemon)
daemon.setDaemon(True)
daemon.start()
daemon.join()

def getDownloader(self):
for index,downloader in enumerate(self.downloaders):
if not downloader:
return(index)
return(None)


ADD_Q_ID = 'insert into qiushibaike(id,success) values(?,?)'
UPDATE_Q_CONTENT = 'update qiushibaike set content=? where id=?'
UPDATE_Q_STATUS = 'update qiushibaike set success=? where id=?'
Q_LIST = 'select id from qiushibaike where success=?'
Q_LIST_BY_ID = 'select count(*) from qiushibaike where id=?'
class dbConnect(object):
"""
create table qiushibaike(
id,Integer
content,Varchar
success,Interger
)
#id表示糗事的ID
#content表示糗事的內容
#success表示是否下載成功,當該糗事內容下載完成,且獲得上一頁、下一頁ID時表示下載完成
1表示未完成
2表示完成
"""
def __init__(self,dbpath='db.sqlite'):
self.dbpath = dbpath

def addQID(self,qid):
Log.log('插入糗事百科',qid)
#獲得連接
cn = sqlite3.connect(self.dbpath)
c = cn.cursor()

try:
#添加內容并提交
c.execute(ADD_Q_ID,(qid,1))
cn.commit()
except:
Log.log('添加ID出錯',qid)

#關閉連接
c.close()

cn.close()
Log.log('插入成功')

def updateContent(self,qid,content):
Log.log('更新糗事百科',qid,content)
#獲得連接
cn = sqlite3.connect(self.dbpath)
c = cn.cursor()
#添加內容并提交
c.execute(UPDATE_Q_CONTENT,(content,qid))
cn.commit()
#關閉連接
c.close()
cn.close()
Log.log('更新成功')

def updateStatus(self,qid,flag):
Log.log('更新狀態',qid,flag)
#獲得連接
cn = sqlite3.connect(self.dbpath)
c = cn.cursor()
#添加內容并提交
c.execute(UPDATE_Q_STATUS,(flag,qid))
cn.commit()
#關閉連接
c.close()
cn.close()
Log.log('更新狀態成功')

def getList(self,unDonloaded=1):
Log.log('獲得列表')
l = []
#獲得連接
cn = sqlite3.connect(self.dbpath)
c = cn.cursor()
#獲得數據
c.execute(Q_LIST,(unDonloaded,))
rows = c.fetchall()

for i in rows:
l.append(i[0])
#關閉連接
c.close()
cn.close()

Log.log('獲得列表成功')
return(l)

class singleDownloader(object):
def __init__(self):
self.downloadList = []

def setdb(self,db):
self.db = db

def setDownloadList(self,downloadList):
self.downloadList = list(set(self.downloadList+downloadList))

def beginDownload(self):
for i in self.downloadList:
downloadPage(i,self.db)

def main():
db = dbConnect('db.sqlite')
#dp = downloaderPool()
#dp.setdb(db)
sp = singleDownloader()
sp.setdb(db)

dp=sp

unDownloadedList = db.getList()
#當還有未下載的糗事時就要繼續下載
while(len(unDownloadedList)):
#使用該列表填充下載池
dp.setDownloadList(unDownloadedList)

dp.beginDownload()

time.sleep(1)
#重置參數
unDownloadedList = db.getList()

if __name__ == '__main__':
main()


代碼是沒問題的,可以正常運行,但是希望做到以下2方面:
1、多線程下載
2、代碼分離度更高,跟面向對象
众人帮太赚钱了