【百度地图】获取实时路况地图瓦片

悠悠 2022-10-14 12:51 601阅读 0赞

步骤一：下载地图瓦片

增加了几个反反爬虫策略：

  1. 每请求50次更换ip及header
  2. 每次请求生成1-100的随机数，当为66时暂停0-20s内一个随机时长
  3. 每完成一个时刻瓦片下载，暂停100-300s内一个随机时长

  4. 当请求报错时，更换ip及header重新请求

    s1_getTrafficData.py

    -- coding: utf-8 --

    import http, time, random, os
    import urllib.request

    http://its.map.baidu.com:8002/traffic/TrafficTileService?time=1604900544967&v=016&level=19&x=98760&y=19742

    百度地图切片原点左下角，(x, y)，x为列，y为行

    研究范围：左下(98697, 19700) - 右上(98787, 19766)

  1. def getUserAgent():
  2.     agent_list = [
  3.         "Mozilla/5.0 (Windows NT 6.2; WOW64; rv:21.0) Gecko/20100101 Firefox/21.0",
  4.         "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36",
  5.         'Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:73.0) Gecko/20100101 Firefox/73.0'
  6.     ]
  7.     agent = random.choice(agent_list)
  8.     header = (
  9.         'User-Agent', agent
  10.     )
  12.     return header
  15. def getProxy():
  16.     ip_list = [
  17.         "http://117.93.118.88:3000",
  18.         "http://111.177.192.57:3256",
  19.         "http://112.195.242.60:3256",
  20.         "http://124.205.153.81:80",
  21.         "http://49.85.2.17:3000"
  22.         "http://117.88.208.32:3000",
  23.         "http://114.99.9.117:1133",
  24.         "http://125.72.106.132:3256",
  25.         "http://49.85.188.37:8014",
  26.         "http://124.206.34.66:80",
  27.         "http://117.68.192.93:1133",
  28.         "http://114.233.170.151:8056",
  29.         "http://60.168.207.147:1133",
  30.         "http://114.233.170.48:8056",
  31.         "http://1.70.67.20:9999",
  32.         "http://121.226.215.247:9999",
  33.         "http://114.233.194.202:8086",
  34.         "http://114.112.127.78:80",
  35.         "http://117.66.233.26:9999",
  36.         "http://180.122.38.235:8090",
  37.         "http://117.95.192.119:9999",
  38.         "http://49.85.188.21:8058",
  39.         "http://114.233.168.211:8088",
  40.         "http://180.120.209.130:8888",
  41.     ]
  42.     ip = random.choice(ip_list)
  43.     proxy = urllib.request.ProxyHandler({ 'http': ip})
  44.     return proxy
  47. def requestImg(proxy, header, url, file_name):
  48.     if os.path.exists(file_name):
  49.         return proxy, header
  50.     try:
  52.         opener = urllib.request.build_opener(proxy, urllib.request.HTTPHandler)
  53.         opener = urllib.request.build_opener()
  54.         opener.addheaders=[header]
  55.         urllib.request.install_opener(opener)
  57.         # req=urllib.request.Request(url=url,headers=header)
  58.         # res = urllib.request.urlopen(url, timeout=60)
  59.         req = urllib.request.Request(url)
  60.         res = opener.open(req, timeout=30)
  61.         with open(file_name, "wb") as f:
  62.             content = res.read()
  63.             f.write(content)
  64.             res.close()
  65.     # except urllib.error.HTTPError or urllib.error.URLError as e:
  66.     # print(e.reason)
  67.     # except http.client.IncompleteRead or http.client.RemoteDisconnected as e:
  68.     # if num_retries == 0: # 重连机制
  69.     # return
  70.     # else:
  71.     # requestImg(proxy, header, url, file_name, num_retries - 1)
  72.     except:
  73.         print("exception...")
  74.         proxy = getProxy()
  75.         header = getUserAgent()
  76.         proxy, header = requestImg(proxy, header, url, file_name)
  78.     return proxy, header
  80. if __name__ == "__main__":
  82.     # 3.15-3.21 7:00-20:00
  83.     day_index = 7
  84.     hour_index = 27
  86.     for i in range(1, day_index):
  87.         for j in range(hour_index):
  88.             t = 1615762800000 + i * 86400000 + j * 1800000
  89.             dir_path = "./tiles/images_" + str(i) + "_" + str(j)
  90.             if not os.path.exists(dir_path):
  91.                 os.makedirs(dir_path)
  93.             base_url = "http://its.map.baidu.com:8002/traffic/TrafficTileService?level=17&v=016&time="
  95.             reset_clock = 50
  96.             proxy = getProxy()
  97.             header = getUserAgent()
  99.             for x in range(24670, 24710):
  100.                 for y in range(4920, 4950):
  101.                     url = base_url + str(t) + "&x=" + str(x) + "&y=" + str(y)
  102.                     file_name = dir_path + "/" + str(x) + "-" + str(y) + '.png'
  104.                     # 每50次更换ip、header
  105.                     if reset_clock == 0:
  106.                         proxy = getProxy()
  107.                         header = getUserAgent()
  108.                         reset_clock = 50
  110.                     # 请求
  111.                     proxy1, header1 = requestImg(proxy, header, url, file_name)
  112.                     proxy = proxy1
  113.                     header = header1
  114.                     reset_clock = reset_clock - 1  # 重置时钟-1
  116.                     print(i,j,x,y)
  117.                     time.sleep(random.random())
  119.                     # 每次请求如果遇到随机数66，暂停20s
  120.                     if random.randint(1, 100) == 66:
  121.                         print("sleeping......")
  122.                         time.sleep(20 * random.random())
  124.             # 某天某时所有请求完毕时，停留久一点
  125.             time.sleep(random.uniform(100, 300))
  127.     print("------ok------")

步骤二：去除无数据空图

  1. # s2_replaceEmpty
  2. # -*- coding: utf-8 -*-
  3. #!/usr/bin/env python
  5. import glob, re
  6. from PIL import Image
  8. d = 7
  9. t = 27
  11. for i in range(d):
  12.     for j in range(t):
  13.         p = './tiles/images_'+str(i)+ '_' + str(j)  +'/*.png'
  15.         # 按照x、y顺序对文件名进行排序
  16.         files = glob.glob(p)
  17.         for x in files:
  18.             try:
  19.                 img = Image.open(x)
  20.                 img.close()
  21.             except:
  22.                 img = Image.new(mode='RGBA', size=(256, 256))
  23.                 img.save(x)

步骤三：拼接瓦片

  1. # s3_mergeImages.py
  2. # -*- coding: utf-8 -*-
  3. #!/usr/bin/env python
  4. import glob, re
  5. from PIL import Image
  7. d = 7
  8. t = 27
  10. for i in range(d):
  11.     for j in range(t):
  12.         s = './tiles/images_'+str(i)+ '_' + str(j)  +'/*.png'
  14.         # 按照x、y顺序对文件名进行排序
  15.         files = glob.glob(s)
  16.         files.sort(key=lambda x: tuple(int(i) for i in re.findall(r'\d+', x)[2:4]))
  18.         # 将每一行文件名保存到一个数组中
  19.         imagefiles = { }
  20.         for item in files:
  21.             match = re.findall(r'\d+', item)
  22.             # pre = int(match.group())
  23.             pre = match[2]
  25.             if not imagefiles.get(pre):
  26.                 imagefiles[pre] = []
  28.             imagefiles[pre].append(item)
  30.         # 键值对转排序后的列表
  31.         imagefiles = sorted(zip(imagefiles.keys(), imagefiles.values()))
  33.         # 预先生成合并后大小的空图片
  34.         total_width = len(imagefiles) * 256
  35.         total_height = len(imagefiles[0][1]) * 256
  36.         new_image = Image.new("RGBA", (total_width, total_height))
  38.         # 逐行拼接
  39.         x_offset = 0
  40.         for item in imagefiles:
  41.             y_offset = total_height - 256
  42.             images = list(map(Image.open, item[1])) # 映射函数，返回列表
  43.             for subitem in images:
  44.                 new_image.paste(subitem, (x_offset, y_offset))
  45.                 y_offset -= subitem.size[0]
  46.             x_offset += images[0].size[0]
  48.         f_name = "./merge/merge_"+str(i) + "_"+str(j)+".png"
  49.         new_image.save(f_name, quality = 100)

发表评论

表情：
评论列表 （有 0 条评论，601人围观）

还没有评论，来说两句吧...

相关阅读