科技，始終來自於人性

import requests
from bs4 import BeautifulSoup

###進入每篇文章抓網址
DL = []
def get_dl_link(link):
	global DL
	dl = []
	response = requests.get(link)
	response.encoding = 'UTF-8' 
	soup = BeautifulSoup(response.text, 'lxml')
	articles = soup.find_all('div','box-b')
	for article in articles :
		meta1 = article.find("h1")
		if type(meta1) != type(None):
			title = meta1.getText().replace("[","").replace("]","").replace(" / "," ")
			print(title)
		meta2 = article.find('div','entry')
		if type(meta2) != type(None):
			poster = meta2.find('p','poster').find('img').get("src")
			dl += [title] + [poster]
			for i in meta2.find_all('p','screenshot'): #Screenshot
				dl += [i.find('img').get("src")]
			for i in meta2.find_all('a'): #下載網址
				if "wushare" in str(i) and ("rar" or "zip" ) not in str(i):
					dl += [i.get('href')]
	DL.append(dl)

###Main
page = 1
keyword = "" #關鍵字

while True:
	url = "http://javpop.com/page/" + str(page) +"?s=" + keyword
	response = requests.get(url)
	response.encoding = 'UTF-8' 
	soup = BeautifulSoup(response.text, 'lxml')
	#檢查是不是沒有下一頁
	check = soup.find('h2').getText()
	if check == "Error 404 - Not Found":
		break

	articles = soup.find_all('li')
	for article in articles : #
		meta = article.find('a')
		if type(meta) != type(None) and keyword in str(meta):
			#每篇文章的網址
			link = meta.get("href")
			get_dl_link(link)
	page += 1 #去下一頁

###Export
filename = "javpop_" + keyword +".csv"
with open(filename , "w", encoding = "utf8") as data:
	for i in DL :
		for j in i :
			data.write("%s," % (j))
		data.write("\n")

因為每次有大量抓東西的需求，但如果只靠人力的話，最快的方法就是先查看網頁原始碼，再找出下載網址的部分。這樣的方法十分累人，因此試著用Python搭配requests、BeautifulSoup寫了個直接找出下載網址的程式，沒想到研究如何透過標籤找到我要的內容花了我最多時間(苦笑)

匯出畫面

原本是參考PTT爬蟲，但不同的網站好像不能直接套用，需要重新研究原始碼的結構，之後有空再來處理另外一家網站吧~

另外雖然省去了手動找網址的時間，但還是要手動複製到JDownloader2再存成DLC，正在尋找直接將網址存成DLC的方法

爬蟲新手，若觀念有錯或有更好的寫法，敬請賜教！