# -*- coding: utf-8 -*-
"""
Copyright (C) 2015, MuChu Hsu
Contributed by Muchu Hsu (muchu1983@gmail.com)
This file is part of BSD license

<https://opensource.org/licenses/BSD-3-Clause>
"""
import os
import datetime
import re
import json
import logging
from scrapy import Selector
from cameo.utility import Utility
"""
INDIEGOGO 改版 2016-05-27 改寫
從 source_html 的 HTML 檔案解析資料
結果放置於 parsed_result 下
"""
class ParserV2ForINDIEGOGO:
    #建構子
    def __init__(self):
        self.utility = Utility()
        self.dicSubCommandHandler = {
            "explore":[self.parseExplorePage],
            "category":[self.parseCategoryPage],
            "project":[self.parseProjectPage],
            "individuals":[self.parseIndividualsPage],
        }
        self.lstStrCategoryName = [
            "animals", "art", "comic", "community", "dance",
            "design", "education", "environment", "fashion",
            "film", "food", "gaming", "health", "music", "photography",
            "politics", "religion", "small_business", "sports",
            "technology", "theatre", "transmedia", "video_web", "writing"
        ]
        self.SOURCE_HTML_BASE_FOLDER_PATH = u"cameo_res\\source_html"
        self.PARSED_RESULT_BASE_FOLDER_PATH = u"cameo_res\\parsed_result"
        self.dicParsedResultOfProject = {} #project.json 資料
        self.dicParsedResultOfUpdate = {} #update.json 資料
        self.dicParsedResultOfComment = {} #comment.json 資料
        self.dicParsedResultOfReward = {} #reward.json 資料
        self.dicParsedResultOfProfile = {} #profile.json 資料
        
    #取得 parser 使用資訊
    def getUseageMessage(self):
        return (
            "- INDIEGOGO (parser version 2)-\n"
            "useage:\n"
            "explore - parse explore.html then create category_url_list.txt\n"
            "category - parse category.html then create project_url_list.txt\n"
            "project category - parse project's html of given category (or automode) then create .json\n"
            "individuals category - parse individuals's html of given category (or automode) then create .json\n"
        )

    #執行 parser
    def runParser(self, lstSubcommand=None):
        strSubcommand = lstSubcommand[0]
        strArg1 = None
        if len(lstSubcommand) == 2:
            strArg1 = lstSubcommand[1]
        for handler in self.dicSubCommandHandler[strSubcommand]:
            handler(strArg1)
#detect distil networks ######################################################################
    #檢查 是否為 distil networks 頁面
    def detectDistillNetworksInterruptionPage(self, root=None, strUrlMessage=None):
        strPardonText = root.css("body.block-page div.content h1::text").extract_first()
        if strPardonText and strPardonText.strip().startswith("Pardon Our Interruption..."): # is not None
            logging.error("%s is distil networks block-page"%strUrlMessage)
            raise ValueError("%s is distil networks block-page"%strUrlMessage)
#explore #####################################################################################
    #解析 explore.html
    def parseExplorePage(self, uselessArg1=None):
        strExploreHtmlPath = self.SOURCE_HTML_BASE_FOLDER_PATH + u"\\INDIEGOGO\\explore.html"
        strExploreResultFolderPath = self.PARSED_RESULT_BASE_FOLDER_PATH + u"\\INDIEGOGO"
        if not os.path.exists(strExploreResultFolderPath):
            os.mkdir(strExploreResultFolderPath) #mkdir parsed_result/INDIEGOGO/
        with open(strExploreHtmlPath, "r") as expHtmlFile:
            strPageSource = expHtmlFile.read()
        root = Selector(text=strPageSource)
        lstStrCategoryUrls = root.css("explore-category-link-www a.i-uncolored-link::attr(href)").extract()
        if len(lstStrCategoryUrls) == 0:
            lstStrCategoryUrls = root.css("ul.exploreCategories-list li.ng-scope a.ng-binding::attr(href)").extract()
        strCategoryUrlListFilePath = strExploreResultFolderPath + u"\\category_url_list.txt"
        with open(strCategoryUrlListFilePath, "w+") as catUrlListFile:
            for strCategoryUrl in lstStrCategoryUrls:
                strCategoryUrl = re.match("^(https://www.indiegogo.com/explore#/browse/[^\?]*)\?{1}.*$" ,strCategoryUrl).group(1)
                if strCategoryUrl == "https://www.indiegogo.com/explore#/browse/all":
                    continue #skip "All Categories" link
                else:
                    catUrlListFile.write(strCategoryUrl + u"\n")
#category #####################################################################################
    #解析 category.html
    def parseCategoryPage(self, uselessArg1=None):
        strCategoryUrlListFilePath = self.PARSED_RESULT_BASE_FOLDER_PATH + u"\\INDIEGOGO\\category_url_list.txt"
        catUrlListFile = open(strCategoryUrlListFilePath)
        for strCategoryUrl in catUrlListFile:#category loop
            strCategoryName = re.match("^https://www.indiegogo.com/explore#/browse/(.*)$" ,strCategoryUrl).group(1)
            strCategoryHtmlPath = self.SOURCE_HTML_BASE_FOLDER_PATH + u"\\INDIEGOGO\\%s\\category.html"%(strCategoryName)
            if os.path.exists(strCategoryHtmlPath): #檢查 category.html 已下載完成
                strCategoryResultFolderPath = self.PARSED_RESULT_BASE_FOLDER_PATH + u"\\INDIEGOGO\\%s"%(strCategoryName)
                if not os.path.exists(strCategoryResultFolderPath):
                    os.mkdir(strCategoryResultFolderPath) #mkdir parsed_result/INDIEGOGO/category/
                with open(strCategoryHtmlPath, "r") as catHtmlFile: #open category.html
                    strPageSource = catHtmlFile.read()
                    root = Selector(text=strPageSource)
                    #project url
                    lstStrProjUrls = root.css("a.discoveryCard::attr(href)").extract() #parse proj urls
                    #記錄抓取時間
                    strCategoryCrawlTime = self.utility.getCtimeOfFile(strFilePath=strCategoryHtmlPath)
                    #寫入檔案
                    strProjectUrlListFilePath = strCategoryResultFolderPath + u"\\project_url_list.txt"
                    strProjectTimeleftListFilePath = strCategoryResultFolderPath + u"\\project_timeleft_list.txt"
                    with open(strProjectUrlListFilePath, "w+") as projUrlListFile, open(strProjectTimeleftListFilePath, "w+") as projTimeleftListFile:
                        projTimeleftListFile.write(u"Crawl-Time:%s\n"%strCategoryCrawlTime)
                        for strProjUrl in lstStrProjUrls:
                            #write to project_url_list.txt
                            projUrlListFile.write(strProjUrl + u"\n")
                            #記錄剩餘時間
                            strProjTimeleft = root.css("a.discoveryCard[href='%s'] div.discoveryCard-timeleft::text"%strProjUrl).extract_first() #parse proj time left
                            #轉換剩餘時間 (Indemaned 及 No time left 均轉為 0)
                            intRemainDays = self.utility.translateTimeleftTextToPureNum(strTimeleftText=strProjTimeleft, strVer="INDIEGOGO")
                            #專案結束日期
                            strEndDate = None
                            if strProjTimeleft == None: #InDemaned
                                strEndDate = "indemand"
                            elif strProjTimeleft.strip() == "No time left": #No time left
                 
               strEndDate = "closed"
                            elif intRemainDays > 0:
                                dtCategoryCrawlTime = datetime.datetime.strptime(strCategoryCrawlTime, "%Y-%m-%d")
                                dtEndDate = dtCategoryCrawlTime + datetime.timedelta(days=intRemainDays)
                                strEndDate = dtEndDate.strftime("%Y-%m-%d")
                            #write to
 project_timeleft_list.txt
                            projTimeleftListFile.write(u"%s,%s,%s\n"%(strProjUrl, unicode(intRemainDays), unicode(strEndDate)))
#project #####################################################################################
    #解析 project page 進入點
    def parseProjectPage(self, strCategoryName=None):
        lstFuncOfParseProject = [
            self.beforeParseProjectPage,
            self.parseProjectDetailsPage,
      