import requests
from bs4 import BeautifulSoup
import pandas as pd


#程式欢迎语
print("**欢迎来到UCAR爬虫程式**")
print("\n")

#将资料存入建立好的list
titles = []
date = []
url_list = []
clicks =[]
replys = []

#自定义查询关键字及日期区间
x = str(input("请输入想爬取的关键字："))
print("日期格式输入范例(YYYYMMDD)：20200715")
print("\n")
start_date = int(input("请输入想爬取的起始日期："))		#格式YYYYMMDD:20180101
end_date = int(input("请输入想爬取的结束日期："))		#格式YYYYMMDD:20191231
fake_request_url= "https://forum.u-car.com.tw/forum/list?keywords=" + x

print("\n")
print("爬取中...请稍后")

#抓使用者输入之关键字所有网页
z = 0
while(True):
	z += 1
	real_request_url = fake_request_url + "&page=" + str(z)
	#print(real_request_url)
	response = requests.get(real_request_url)
	response_text = response.text
	soup = BeautifulSoup(response_text, "html.parser")
	#print(soup)

#判断这一页目录有没有文章(有就接下一步，没有就break)
#值得注意的是，这里判断每个目录页是否有文章存在的标签是选择.writer而不是用.title的原因为：在该目录页没有文章时还是有其他的.title标签，若选择.title做判断会导致无穷循环程式无法结束，所以选择.writer为判断依据
#当然也可以选择其他判断标签，这里就以.writer为例
	if soup.select(".writer"):
		pass
	else:
		break

#将所有div class="cell_topic"的内容爬下来储存到变数soup.find1(是个list)
	soup_find1 = soup.find_all('div', 'cell_topic')
#抓发文日期
	#循环soup_find1这个list
	for i in range(len(soup_find1)):
		#第一页且list前两项0,1是置顶广告，所以continue
		if (z == 1 and i <= 1):
			continue
		b = soup_find1[i].find('div', 'postby margin-right-10').find('p').text
		#print(b)
		re_b = b[:10]
		#print(re_b)
		re_b_b = int(re_b.replace('/', ''))
		#print(re_b_b)
		#print(re_b)

#判断发文日期是否符合使用者需求并丢到list
		if (start_date <= re_b_b and re_b_b <= end_date):
			pass
		else:
			continue
		date.append(re_b)
		#print(re_b)

#抓网址
		url = soup_find1[i].find('div', 'title').find('a')
		#print(url)
		a = 'https://forum.u-car.com.tw'
		if url is not None:
			url_list.append(a + url.get('href'))
		else:
			url_list.append("(本文已被删除)")
		#print(a + url.get('href'))

#抓标题
		c = soup_find1[i].find('div', 'title').find('a')
		#print(c)
		if c is not None:
			titles.append(c.text)
		else:
			titles.append('(本文已被删除)')
		print(c.text)

#抓点阅数
		click_count = soup_find1[i].find('div', 'cell_topic_view').find('p')
		if click_count is not None:
			clicks.append(click_count.text)
		else:
			clicks.append("0")
		#print(click_count.text)

#抓回复数
		replys_count = soup_find1[i].find('div', 'cell_topic_chats').find('p')
		if click_count is not None:
			replys.append(replys_count.text)
		else:
			replys.append("0")
		#print(replys_count.text)
	#print('46行循环结束')

print("\n")
print("转档中...请稍后")

#转为DataFrame
df = pd.DataFrame(
    {
        '标题' : titles,
		'点阅数' : clicks,
		'回复数' : replys,
        '发文日期' : date,
        '文章连结' : url_list
    }
)

#另存为csv
df.to_csv( "UCAR_" + x +"回传结果.csv", index = False, encoding = "utf_8_sig")

#程式结束
len_titles = len(titles)
print("本次共爬出 {} 篇文章".format(len_titles))
print("\n")
end = input("请输入任意键结束程式：")
