import requests
from  bs4 import BeautifulSoup
from idna import unicode
from urllib.request import urlopen
from urllib.parse import quote
import sys
import time
import imp
import json
from xpinyin import Pinyin#导入拼音模块
import random

def getlnglat(address):
        """根据传入地名参数获取经纬度"""
        url = 'http://api.map.baidu.com/geocoder/v2/'
        output = 'json'#输出结果可以是json也可以是其他类型
        ak = 'hnhLTrxaZPI7jFkIkqA1TuLUWWmHV7Q1'
        add = quote(str(address))#有时候quote会出错KeyError，要先把quote里面转成字符串类型
        uri = url + '?' + 'address=' + add + '&output=' + output + '&ak=' + ak#构建百度地图API查询需要的uri，uri与URL的区别见：https://www.zhihu.com/question/21950864

#下面这块代码是为了防止因为请求次数过多而产生的错误：urllib2.URLError: <urlopen error [Errno 10060] >
#如果出现该错误，就让程序暂停10秒（time.sleep(10)），继续重新获取该小区的坐标。
#timeout=30是防止因为网络原因出现延迟错误

        maxNum=5
        for tries in range(maxNum):
            try:
                req = urlopen(uri,timeout=30)#设置timeout30秒，防止百度屏蔽,如果被屏蔽就等30秒
            except:
                if tries < (maxNum - 1):
                    time.sleep(10)
                    continue
                else:
                    print("Has tried %d times, all failed!", maxNum)
                    break

        res = req.read().decode()
        print(res)
        temp = json.loads(res)
        lat = temp['result']['location']['lat']
        lng = temp['result']['location']['lng']
        return lat, lng
def getagent():
        list1 =['Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.835.163 Safari/535.1',
                    'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:6.0) Gecko/20100101 Firefox/6.0',
                    'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50',
                    'Opera/9.80 (Windows NT 6.1; U; zh-cn) Presto/2.9.168 Version/11.50',
                    'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0; .NET CLR 2.0.50727; SLCC2; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; InfoPath.3; .NET4.0C; Tablet PC 2.0; .NET4.0E)',
                    'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; InfoPath.3)']
        return list1[random.randint(0,5)]
def getheaders():
    headers = {  # 由于安居客网站的反爬虫，这里必须要设置header
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
        'Accept-Language': 'zh-CN,zh;q=0.8',
        'Referer': 'https: // wuhan.anjuke.com / sale /?from=navigation',
        'User-Agent': getagent()
    }
def getagent():
    list1 = [
        'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.835.163 Safari/535.1',
        'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:6.0) Gecko/20100101 Firefox/6.0',
        'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50',
        'Opera/9.80 (Windows NT 6.1; U; zh-cn) Presto/2.9.168 Version/11.50',
        'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36',
        'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; GTB7.0)',
        'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)',
        'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1)',
        'Mozilla/5.0 (Windows; U; Windows NT 6.1; ) AppleWebKit/534.12 (KHTML, like Gecko) Maxthon/3.0 Safari/534.12',
        'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; InfoPath.3; .NET4.0C; .NET4.0E)']
    a=random.randint(0, 9)
    print(a)
    return list1[a]
def getip():
    ip = ['1.197.203.148:9999',
          '163.204.246.188:9999',
          '163.204.246.80:9999',
          '163.204.242.23:9999',
          '182.34.36.23:25995',
          '163.204.242.240	:9999',
          '112.85.164.83:9999',
          '163.204.246.253	:9999',
          '120.83.110.1:9999',
          '182.35.84.183:9999',
          '182.35.81.201:9999',
          '182.35.86.166:9999',
          '182.35.84.154:9999',
          '163.204.240.251:9999',
          '163.204.243.205	:9999',
          '163.204.247.2:9999',
          '36.249.118.58:9999',
          '163.204.241.167:9999',
          '163.204.246.169	:9999']
    b = random.randint(0,18)
    print(b)
    return ip[b]
def gethtml():
    # url = 'http://www.xicidaili.com/nn/'
    # html = requests.get(url=url, headers=getheaders()).text
    # soup = BeautifulSoup(html, 'html.parser')
    # ips = soup.find_all(id='ip_list').find_all('tr')
    # ip_list = []
    # for i in range(1, len(ips)):
    #     ip_info = ips[i]
    #     tds = ip_info.find_all('td')
    #     ip_list.append(tds[1].text + ':' + tds[2].text)
    # # 随机获取列表中一个ip
    # ip = random.choice(ip_list)
    ip = getip()
    response = requests.get('https://www.anjuke.com/sy-city.html', proxies={"http": ip}, timeout=10)
    print(response.text)
def getip():
    ip = ['163.204.241.158	:9999',
          '182.35.87.174:9999',
          '182.35.87.175:9999',
          '182.35.87.176:9999',
          '182.35.87.177:9999',
          '123.163.96.131:9999',
          '1.197.16.130:9999',
          '182.35.85.157:9999',
          '163.204.244.212:9999',
          '163.204.243.252:9999',
          '117.91.132.170:9999',
          '58.253.156.1:9999',
          '1.197.16.129:9999',
          '163.204.246.126:9999',
          '60.13.42.67:9999',
          '49.86.182.174:9999',
          '182.35.80.229:9999',
          '58.253.156.225:9999',
          '163.204.240.126:9999',
          '60.13.42.70:9999',
          '123.163.122.108:9999',
          '175.42.122.46:9999',
          '123.163.97.234:9999',
          '182.35.82.130:9999',
          '163.204.245.72:9999',
          '60.13.42.127:9999',
          '121.233.206.73:9999',
          '163.204.246.149:9999',
          '182.35.83.33:9999',
          '163.204.243.16:9999',
          '163.204.246.174:9999',
          '121.233.251.60:9999',
          '123.169.37.250:38677',
          '120.83.109.29:9999',
          '163.204.241.164:9999',
          '163.204.246.42:9999',
          '163.204.241.228:9999',
          '175.42.68.4:9999',
          '182.35.84.155:9999',
          '112.85.130.205:9999',
          '122.193.247.115:9999',
          '163.204.241.190:9999',
          '163.204.245.237:9999',
          '163.204.242.245:9999',
          '115.53.19.82:9999',
          '112.85.128.146:9999',
          '163.204.244.40:9999',
          '182.35.80.5:9999',
          '163.204.242.130:9999',
          '112.85.129.88:9999',
          '113.121.20.143:9999',
          '182.35.83.136:9999',
          '182.35.80.195:9999',
          '120.83.105.248:9999',
          '112.85.151.152:9999',
          '171.11.178.94:9999',
          '171.12.113.6:9999',
          '112.85.165.194:9999',
          '123.163.122.254:9999',
          '58.253.158.174:9999',
          '120.84.101.164:9999',
          '60.13.42.61:9999',
          '60.13.42.207:9999',
          '1.198.72.219:9999',
          '182.35.80.54:9999',
          '114.230.69.232:9999',
          '163.204.242.126:9999',
          '58.253.154.253:9999',
          '180.108.218.242:9999',
          '112.85.149.238:9999',
          '114.230.69.109:9999',
          '60.13.42.28:9999',
          '163.204.244.39:9999',
          '180.108.218.179:9999',
          '121.233.251.82:9999',
          '113.121.20.252:808',
          '120.83.111.43:9999',
          '182.35.86.234:9999',
          '182.35.83.200:9999',
          '60.13.42.142:9999',
          '120.83.98.106:9999',
          '117.91.130.10:9999',
          '111.226.188.146:8010',
          '180.119.68.222:9999',
          '123.163.96.170:9999',
          '60.13.42.57:9999',
          '113.121.23.248:9999',
          '222.189.144.147:9999',
          '60.13.42.172:9999',
          '183.128.167.248:8118',
          '182.35.86.217:9999',
          '60.13.42.38:9999',
          '222.89.32.141:8070',
          '183.157.84.221:8118',
          '222.189.191.34:9999',
          '123.163.122.129:9999',
          '121.233.227.214:9999',
          '180.119.141.163:9999']
    b = random.randint(0,97)
    print("ip:"+str(b))
    return {'http':'http://'+ip[b]}
def getagent():
    list1 = [
        'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.835.163 Safari/535.1',
        'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:6.0) Gecko/20100101 Firefox/6.0',
        'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50',
        'Opera/9.80 (Windows NT 6.1; U; zh-cn) Presto/2.9.168 Version/11.50',
        'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36',
        'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; GTB7.0)',
        'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)',
        'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1)',
        'Mozilla/5.0 (Windows; U; Windows NT 6.1; ) AppleWebKit/534.12 (KHTML, like Gecko) Maxthon/3.0 Safari/534.12',
        'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; InfoPath.3; .NET4.0C; .NET4.0E)']
    a=random.randint(0, 9)
    print("agent:"+str(a))
    return list1[a]
def get_ip_list(num):

    print(num)
    url = 'http://www.xicidaili.com/nn/%d' %num
    print(url)
    headers = {
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
        'Accept-Language': 'zh-CN,zh;q=0.8',
        'User-Agent': getagent()
    }
    web_data = requests.get(url, headers = headers,timeout=(10))
    soup = BeautifulSoup(web_data.text, 'html.parser')
    ips = soup.find_all('tr')
    ip_list = []
    file = open("西刺ip1.txt", 'a', encoding="utf-8", buffering=1)
    for i in range(1, len(ips)):
        ip_info = ips[i]
        tds = ip_info.find_all('td')
        strip = tds[1].text + ':' + tds[2].text
        ip_list.append(strip)
        print(strip+ ",")
        file.write('\''+strip +'\''+ "," + '\n')
def getip():
    file = open("西刺ip.txt",'a',encoding='utf-8')
    ip = []
    for line in file:
        ip.append(line)
    b = random.randint(0,97)
    print("ip:"+str(b))
    return {'http':'http://'+ip[b]}
if __name__ == '__main__':
    for a in range(1,300):
        get_ip_list(a)
    # print(getip())
    # print('<address>')