#! /usr/bin/python

"""
 PCcrawler.py is the producer/consumer version of PF crawler.  This will
 be last the version before transforming into the pure-thread version.

 PCcrawler.py is set up to start a crawl form a given list of frontiers,
 pages already visted, and hash_codes already encountered:

 webcrawl( file_to_dump_to, url_matching_pattern, max_num_page_visited,   \
             links_to_visit, links_already_dispatched, \
             hash_codes_already_visited) :

 ----

 A modular version of webcrawler.py that uses two separate function files
 
 [canonical_URL, page_contents, timestamp] = get_webpage(URL)
 links = scoop_hrefs(page_contents)

 to replace the old get_webinfo() function, which did too many things. 
 The advantage of breaking this up into two parts is that we can see which
 pages causes the program to break.  (Also, retrieving a web page from the
 net is very different than parsing a local html string for href links.)

 So before we would write

 [canonical_URL, sha1_hash, links] = get_webinfo(URL, Permission, RegExp)

 we write

  [canonical_URL, page_contents, timestamp] = get_webpage(ULR)
  print canonical_URL
  links = scoop_hrefs(page_contents)
  for href in links
     print href
     etc.

 
 The function get_webpage() also deals with the proper 'robots.txt'
 permission issues (basically, it politely follows any restrictions)
 so we don't need to worry about it here.
""" 


from datetime import datetime
import sys
import os.path
from urllib.parse import urlparse


#########################
#<<<<< get_webpage module 
#########################

from urllib.robotparser import RobotFileParser
from urllib.parse import urlparse
from urllib.parse import urljoin
import urllib.request, urllib.parse, urllib.error
import urllib.request, urllib.error, urllib.parse
import url_errors
from readwg import process_wg_file

Permissions = {}

#  True, if url is allowed by robots.txt permission.  False, otherwise
#
def can_read(url):

  domain = domain_name(url)
  if domain not in Permissions :
         rp = RobotFileParser()
         rp.set_url(urljoin('http://' + domain, 'robots.txt'))
         try :
            rp.read()
         except:
            return False
         
         Permissions[domain] = rp

  res = False
  try:
    res  = Permissions[domain].can_fetch("*", url)
  except:
    return False

  return res


#  This is the function to put into the producer pool
#
# [timestamp, canonical_url, page_contents] = get_webpage(url)
#
def get_webpage(url):

  timestamp = datetime.now().strftime("%Y-%m-%d:%H:%M:%S") 
  #print "get_webpage(" + url + ")"
  if not can_read(url)  :
      return timestamp, url, url_errors.protected_URL

  #  try to open url, if unsuccessful, return default info and exit
  #

  req = urllib.request.Request(url)
  try :
    #print "Opening: " + url
    f = urllib.request.urlopen(req) 
    #print "Opened: " + url
  #
  # changed IOError to "anything" , since urlopen sometimes throws
  # httplib.BadStatusLine() exception, which apparently is not
  # covered under IOError.
  #
  except IOError as e :
     #print "IOError, e: "
     if hasattr(e, 'code'):
        if e.code == 401 :
          #print "Error 401: " 
          return timestamp, url, url_errors.password_URL
        else :
          return timestamp, url, url_errors.invalid_URL

     else:
          #print "No e-code"
          return timestamp, url, url_errors.invalid_URL
  except: 
        return timestamp, url,  url_errors.invalid_URL
  else:
   
     if (f.info().get_content_type() == "text/html"): #altered 03Oct15 DJS
         #print "Sucess: " + url
         try: 
           page_contents = f.read()
         except:
           page_contents = url_errors.error_reading_URL

         return timestamp, f.geturl(), page_contents
     elif (f.info().get_content_type() == "application/pdf"): #added 22Oct15 DJS
         print ('get_webpage: Found a PDF', url)
         return timestamp, f.geturl(), url_errors.not_text_URL
     else:
         #print "not text/html: " + url
         return timestamp, f.geturl(), url_errors.not_text_URL
       

################################
#>>>> end of get_webpage module
################################


################################
#<<<<  scoop_hrefs module
################################

#from BeautifulSoup import BeautifulSoup
from bs4 import BeautifulSoup # added 03Oct2015 DJS
import re

def scoop_hrefs_beautiful_soup(html_page_contents):
  links = []
  try :
     b = BeautifulSoup(html_page_contents, 'html.parser')
  #except (UnicodeEncodeError, UnicodeDecodeError):
  except :
     pass
  else: 
     for tag in b.findAll('a', href=True):
       links.appe
nd(tag['href'])


  return links


#Alternate method using (precompiled) regular expressions:


href_regexp = re.compile(b'<a\s+href\s*?="\s*?(.*?)"', re.IGNORECASE | re.MULTILINE) #altered 03Oct15 DJS

def scoop_hrefs_regexp(html_page_contents):
  return href_regexp.findall(html_page_contents)


# use both methods and combine the results
#
def scoop_hrefs(html_page_contents):
    
    return set.union(set(scoop_hrefs_beautifu
l_soup(html_page_contents)), \
      set(scoop_hrefs_regexp(html_page_contents)))



################################
#>>>>>>  scoop_hrefs module
################################



################################
#<<<<<<  url_tools module
################################

from urllib.parse import urlparse
from urllib.parse import urljoin
from urllib.parse import urlunsplit

def domain_name(url):
    return urlparse(url)[1]


#  Takes "http://foo.com/page1" and "../page2" to
#  create "http://foo.com/page2"
# 
def href2url(originating_page, href):

    # strip out leading and trailing white space
    # and convert, but do *not* convert to lower-case

    href = href.strip()   
    # href = href.replace("%7","~")
    # href = urllib.unquote(href)

    # Parse out the query and anchor section of url
    # and make relative urls (e.g. "../foo.html") into
    # absolute (complete with "http:")

    try: 
      pieces = urlparse( urljoin(originating_page, href))
    except:
      return ''

    url_scheme = pieces[0]
    url_location = pieces[1]
    url_path = pieces[2]
    return  urlunsplit((url_scheme, url_location, url_path, '',''))



################################
#>>>>>  url_tools module
################################






# returns True if this url is not to be followed
# i.e. is a jpg, gif, pdf, zip, or other document
#

def file_extension(filename) :
    (base, ext) = os.path.splitext(filename)
    if (ext == '.' or ext == ''):
      return ''
    else :
      return ext[1:] 



# Global constants
#


# terminal URL extensions (all lower case) not added to the frontier 
#  (links-to-visit)
#

terminal_extensions = set([ \
  #
  # text file extensions
  #
  'doc', 'docx', 'log', 'msg', 'pages', 'rtf', 'tt', 'wpd', 'wps' , \
  #
  # data file extensions
  #
  'accdb', 'blg', 'dat', 'db', 'efx', 'mdb', 'pdb', 'pps', 'ppt', \
  'pptx', 'sdb', 'sdf', 'sql', 'vcf', 'wks', 'xls', 'xlsx', \
  #
  # image  file extensions
  #
  'bmp', 'gif', 'jpg', 'png', 'psd', 'psp', 'thm', 'tif', 'tiff' ,\
  'ai', 'drw', 'eps', 'ps', 'svg', \
  '3dm', 'dwg', 'dxf', 'pln', \
  'indd', 'pct', 'qxd', 'qxp', 'rels',  \
  #'indd', 'pct', 'pdf', 'qxd', 'qxp', 'rels',  \
  #
  # audio file extensions
  #
  'aac', 'aif', 'iff', 'm3u', 'mid', 'mp3', 'mpa', 'ra', 'wav', 'wma' , \
  #
  # video file extensions
  #
  '3g2', '3gp', 'asf', 'asx', 'avi', 'flv', 'mov', 'mp4', 'mpg', \
  'rm', 'swf', 'vob', 'wmv', \
  #
  # executable file extensions
  #
  'sys', 'dmp', 'app', 'bat', 'cgi', 'exe', 'pif', 'vb', 'ws', \
  #
  # compressed file extensions
  #
  'deb', 'gz', 'pkg', 'rar', 'sit', 'sitx', 'tar', 'gz', 'zip', 'zipx', '7z', \
  #
  # programming file extensions
  #
  'c', 'cc', 'cpp', 'h', 'hpp', 'java', 'pl', 'f', 'for', 'js', \
  #
  # misc file extensions
  #
  'dbx', 'msi', 'part', 'torrent', 'yps', 'dmg', 'iso', 'vcd' , \
  #
  #
  ])




# more_audio_file_extensions = set([\
#    '4mp', 'aa3', 'aac', 'abc', 'adg', 'aif', 'aifc', 'aiiff', \
#    'awb', 'cda', 'cdib', 'dcm', 'dct', 'dfc', 'efa', 'f64', 'flac, \
#    'flp', ,'g7