﻿


import urllib
import urllib2
import re
import sys
import gzip
import StringIO
import copy
import cookielib
import time
import random
import argparse
# from bs4 import BeautifulSoup



class WBinfo():
    def __init__(self, url, header, cookie_str, depth=0, db_set = [None, None]):
        '''
          初始化
          数据库格式：depth varchar(20),name varchar(20), area varchar(20), url varchar(50)
        '''
        self.url = url                        # 爬取的微博url
        self.header = header
        self.cookie_str = cookie_str
        self.header['Cookie'] = self.cookie_str
        self.fans_list = []
        self.depth_set = depth
        self.db = db_set[0]
        self.db_table = db_set[1]

    def wb_sn(self, url, depth):
        '''
          得到微博之间的关系
        '''
        pass

    def sn_info(self, url):
        '''
          得到账号信息，以及粉丝
          返回格式 [[name, area, url], [[name, url], [name, url], ...]]
        '''
        page_num = 2
        info = []
        while True:
            delay = random.uniform(1, 3)
            time.sleep(delay)
            page = self.get_page(url)
            if page:
                break

        # 得到博主的信息
        pat = re.compile(u'(?<=\<a href=")[^<]+?(?=">资料</a>)')
        res = pat.findall(page)
        info_url = u'http://weibo.cn' + res[0]
        self.url = info_url
        area, tag, intr, name = self.account_info()

        info.append([name, area, url])

        # 得到粉丝
        pat = re.compile(u'(?<=粉丝\[).+?(?=\])')
        res = pat.findall(page)
        if res[0] == u'0':    # 没有粉丝
            return info

        # 进入粉丝页，查找粉丝
        pat = re.compile(u'(?<=href=")[^<]+?(?=">粉丝\[)')
        res = pat.findall(page)
        url = u'http://weibo.cn' + res[0]
        fan_list = self.retrive_fans(url, page_num)
        info.append(fan_list)       # 传入主页的信息以及粉丝情况
        return info

    def all_info(self, url=None):
        '''
          返回微博的 关注数，粉丝数，地区
        '''
        page = self.get_page(urlin=url)
        # with open('1.txt', 'w+') as f:
            # f.write(page.encode('utf-8'))
        # with open('1.txt', 'r') as f:
            # page=f.read()
        

        follow = ''
        pat = re.compile(u'(?<=关注\[)\d+?(?=\])')
        res = pat.findall(page)
        # print res
        if len(res) > 0:
            follow = res[0].decode('utf-8').encode('gbk')

        fans = ''
        pat = re.compile(u'(?<=粉丝\[)\d+?(?=\])')
        res = pat.findall(page)
        # print res
        if len(res) > 0:
            fans = res[0].decode('utf-8').encode('gbk')

        pat = re.compile(u'(?<=\<a href=")[^<]+?(?=">资料</a>)')
        res = pat.findall(page)
        info_url = u'http://weibo.cn' + res[0]
        self.url = info_url
        # print info_url
        area, tag, intr, name = self.account_info()

        # print 'f:', follow, 'fans', fans, 'area', area
        return [follow, fans, area]
   
     pass

    def sn_analysis(sel
f, url):
        '''
          社交网络关系分析
          数据来源 : b = a.sn_info(_url)     # [[name, area, url], [[name, url], [name, url], ...]]
        '''
        # 首先得到入口微博以及其粉丝的信息
        all_sn = list()
        main_info = self.sn_info(url)
        all_sn.append(main_info)
        print main_info[0][0], main_info[0][1], main_info[0][2]
        for j in main_info[1][:]:
            print j[0], j[1]
        print ''
        if self.db is not None:         # 写入数据库   name area url
            [follow, fans, area] = self.all_info(main_info[0][2])
            cur = self.db.cursor()
            db_name = main_info[0][0].encode('utf-8')
            db_area = main_info[0][1].encode('utf-8')
            db_url = main_info[0][2].encode('utf-8')
            db_follow = follow.encode('utf-8')
            db_fans = fans.encode('utf-8')
            statement = "insert into %s value('%s', '%s', '%s', '%s', '%s')" % ('all_sn', db_name, db_area, db_url, db_follow, db_fans)
            cur.execute(statement)
            cur.close()
            self.db.commit()

        # 然后再得到粉丝微博的信息以及其各自的粉丝
        for i in main_info[1][:]:
            info = self.sn_info(i[1])
            all_sn.append(info)
            print info[0][0], info[0][1], info[0][2]
            for j in info[1][:]:
                print j[0], j[1]
            # 写入数据库
            [follow, fans, area] = self.all_info(info[0][2])
            cur = self.db.cursor()
            db_name = info[0][0].encode('utf-8')
            db_area = info[0][1].encode('utf-8')
            db_url = info[0][2].encode('utf-8')
            db_follow = follow.encode('utf-8')
            db_fans = fans.encode('utf-8')
            statement = "insert into %s value('%s', '%s', '%s', '%s', '%s')" % ('all_sn', db_name, db_area, db_url, db_follow, db_fans)
            cur.execute(statement)
            cur.close()
            self.db.commit()
            for j in info[1][:]:
                [follow, fans, area] = self.all_info(j[1])
                cur = self.db.cursor()
                db_name = j[0].encode('utf-8')
                db_area = area.encode('utf-8')
                db_url = j[1].encode('utf-8')
                db_follow = follow.encode('utf-8')
                db_fans = fans.encode('utf-8')
                statement = "insert into %s value('%s', '%s', '%s', '%s', '%s')" % ('all_sn', db_name, db_area, db_url, db_follow, db_fans)
                cur.execute(statement)
                cur.close()
                self.db.commit()
            # break
            print ''
        '''
          开始进行关系分析
        '''
        # 先输出已经很明晰的关系   sn_relationship
        # for i in all_sn:
            # for j in i[1][:]:
                # print '*************************************************'
                # print j[0], j[1],
                # print u' 关注了 ',
                # print all_sn[0][0], all_sn[0][2]
                # cur = self.db.cursor()
                # from_name = j[0].encode('utf-8')
                # from_url = j[1].encode('utf-8')
                # to_name = all_sn[0][0].encode('utf-8')
                # to_url = all_sn[0][2].encode('utf-8')
                # statement = "insert into %s value('%s', '%s', '%s', %s)" % ('sn_relationship', from_name, from_url, to_name, to_url)
                # cur.execute(statement)
                # cur.close()
                # self.db.commit()

        # 然后再分析各个之间是否存在关注关系   [[name, area, url], [[name, url], [name, url], ...]]
        # print '-----------------------------'
        no_repeat = []
        for i in all_sn:
            # print i
            obj_list = [[i[0][0], i[0][2]]] + i[1][:]     # 取出一个微博信息
            for object in obj_list:
                for j in all_sn:
                    for k in j[1][:]:       # 在每个微博的粉丝信息里面进行查询
                        if object[0] in k[0] or object[1] in k[1]:
                            # print '********************************************'
                            # print object[0], k[0], object[1], k[1]
                            print '********************************************'
                            print object[0], object[1],
                            print u' 关注了 '
                            print j[0][0], j[0][2]
                            cur = self.db.cursor()
                            from_name = object[0].encode('utf-8')
                            from_url = object[1].encode('utf-8')
                            to_name = j[0][0].encode('utf-8')
                            to_url = j[0][2].encode('utf-8')
                            statement = "insert into %s value('%s', '%s', '%s', '%s')" % ('sn_relationship', from_name, from_url, to_name, to_url)
                            cur.execute(statement)
                            cur.close()
                            self.db.commit()
                            break
        # for

    def retrive_fans(self, url, page_num):
        '''
          返回粉丝页面的粉丝信息
        '''
        fan_list = []
        delay = random.uniform(1, 3)
        time.sleep(delay)
        page = self.get_page(url)

        pat = re.compile(u'(?<=<table>).+?(?=</table>)')
        res = pat.findall(page)
        if len(res) == 0:
            return fan_list

        for i in res:
           