万维百科人物

偏执的太偏执、 2022-10-29 09:19 302阅读 0赞

中文维基百科访问不了，还好有个万维百科，可以从中获取名人的出生日期，虽然只有六个字，至少国外的人物出生日期想对比较准确。  
**1 建库脚本**

create database if not exists wiki_person
        default character set utf8
        default collate utf8_general_ci;
    
    show databases;
    
    use wiki_person;
    DROP TABLE IF EXISTS `life`;
    CREATE TABLE `life` (
      `id` char(32) NOT NULL DEFAULT '' COMMENT 'ID',
      `name` varchar(100) NOT NULL DEFAULT '' COMMENT '姓名',
      `cs` varchar(11) NOT NULL DEFAULT '' COMMENT '出生日期',
      `qs` varchar(11) NOT NULL DEFAULT '' COMMENT '去世日期',
      `sex` char(1) NOT NULL DEFAULT '' COMMENT '性别',
      `url` varchar(200) NOT NULL DEFAULT '' COMMENT '链接',
      PRIMARY KEY (`id`)
    ) ENGINE=InnoDB DEFAULT CHARSET=utf8;
    
    -- 修正字段
    ALTER TABLE `life`
    ADD COLUMN `cs_bazi`  char(8) NULL AFTER `url`;
    
    -- 将表life导出sql,其中wiki_person为database，life为table名
    mysqldump -u root -p wiki_person life > /appdata/life20210215.sql

**2 scrapy**

import scrapy
    from pyquery import PyQuery as pq
    import re
    from ..items import LifeItem
    from ..util import str_util
    
    class SpiderQs1900(scrapy.Spider):
        name = 'spider_qs_1900'
        prefix = 'https://www.wanweibaike.com/'
        allowed_domains = ['www.wanweibaike.com']
        start_urls=['https://www.wanweibaike.com']
    
        def parse(self,response):
            for year in range(1900,1910):
                url = "https://www.wanweibaike.com/wiki/Category-{}%E5%B9%B4%E9%80%9D%E4%B8%96".format(year)
                print(url)
                yield scrapy.Request(url,self.parse_cur_page)
    
        def parse_cur_page(self,response):
            soup = pq(response.body_as_unicode())
            person_links = soup('.mw-category a')
            for person_link in person_links:
                person_name = pq(person_link).text()
                href = pq(person_link).attr('href')
                if re.match('^/wiki-\d{4}',href) is None:
                    url = self.prefix+href
                    yield scrapy.Request(url,self.parse_person)
    
        def parse_person(self, response):
            ''' 解析人物 '''
            soup = pq(response.text)
            # id
            url = response.url
            id = str_util.md5(url)
            # 姓名
            name = soup('.content h1').text()
            ######################################## 寿命 ########################################
            content = soup('#mw-content-text p:lt(2)').text()
            if len(re.findall('(她|女性)',content))>0:
                sex = '女'
            else:
                sex = '男'
            cs_qses = re.findall('（(.+?)）',content)
            if cs_qses is None or len(cs_qses)==0:
                pass
            cs_qs = cs_qses[0]
            rqs = re.findall('\d{4}年\d{1,2}月\d{1,2}日',cs_qs)
            if rqs is None or len(rqs)==0:
                pass
            else:
                # 有出生日期、逝世日期才进入一下阶段，没有的话，没有统计的意义
                life_item = self.gen_life_item(id,name,sex,url,rqs[0],rqs[1])
                yield life_item
    
        def gen_life_item(self,id,name,sex,url,cs,qs):
            ''' 寿命item '''
            life_item = LifeItem()
            life_item['id'] = id
            life_item['name'] = name
            life_item['sex'] = sex
            life_item['url']=url
            # 出生日期
            life_item['cs'] = cs
            # 去世日期
            life_item['qs']=qs
            return life_item

scrapyd运行半天，没看到数据有什么更新，因为广电的网络不知道为什么云服务器设置不了白名单，因此也无法通过浏览器查看运行状态，只能通过命令查看

from scrapyd_api import ScrapydAPI
    scrapyd = ScrapydAPI('http://localhost:6800')
    scrapyd.list_jobs('wanwiki')
    
    { 'node_name': 'sp1.test.com.cn', 'pending': [], 'running': [{ 'id': 'b46887fe6b4a11ebaa86fa163e8a50e0', 'spider': 'spider_hy', 'pid': 8895, 'start_time': '2021-02-10 10:50:18.443425'}], 'finished': [{ 'id': '79a4d6da6ac911ebaa86fa163e8a50e0', 'spider': 'spider_qs_1900', 'start_time': '2021-02-09 19:25:13.584024', 'end_time': '2021-02-09 20:35:19.403491'}, { 'id': '9f8a77246ac911ebaa86fa163e8a50e0', 'spider': 'spider_qs_1910', 'start_time': '2021-02-09 19:26:18.444328', 'end_time': '2021-02-09 21:03:53.777995'}]}

发布到scrapyd

cd /appdata/wiki/wanwiki
    scrapyd-deploy -p wanwiki
    # 启动
    curl http://localhost:6800/schedule.json -d project=wanwiki -d spider=spider_hy_2020
    # 取消
    curl http://localhost:6800/cancel.json -d project=wanwiki -d job=b46887fe6b4a11ebaa86fa163e8a50e0

**3 获取八字**  
下面使用的是pandas读取mysql，没有使用分页，下次看能不能整理一个像java那样的miemie分页框架。  
**3.1 获取八字1**  
下面这个八字在2021年2月10日还是可以用的，但是2021年2月11日就不能用了，因为浏览器不支持flash了。

# -*- coding: utf-8 -*-
    # @time : 2021/2/10 11:56
    # @author : dzm
    # @dsec :
    import requests
    import time
    import random
    import urllib
    from pyquery import PyQuery as pq
    import mysql_helper as helper
    import pandas as pd
    from db_engine import engine_wiki
    import re
    import json
    
    def get_cookies():
        sess = requests.session()
        resp = sess.get('https://www.china95.net/paipan/bazi/')
        return sess, resp.cookies.get_dict()
    
    def get_cs_bazi(year,month,day,sex,sess,cookies):
        time.sleep(random.randrange(2,5))
        params= { 'name':'','area':'','sex':sex,'year':year,'month':month,'date':day
            ,'hour':'9','minute':'20','jingdu':120,'jingdufen':0,'taiyang':0,'quanpai':1,'stylebazi':'120','stylebazi':'0'}
        params['submit']=urllib.parse.quote('在线排八字盘'.encode('gb2312'))
        url = 'https://www.china95.net/paipan/bazi/bazi_show.asp'
        r = sess.post(url,data=params,cookies=cookies,verify=False)
        r.encoding = 'gb2312'
        #
        tiangan = pq(r.text)('table.stylebazi tr>td font:eq(22)').text()
        tiangans = tiangan.split('　　　　 ')
        dizhi = pq(r.text)('table.stylebazi tr>td font:eq(24)').text()
        dizhis = dizhi.split('　　　　 ')
        #
        cs_year = tiangans[0]+dizhis[0]
        cs_month = tiangans[1]+dizhis[1]
        cs_day = tiangans[2]+dizhis[2]
        cs_bazi = cs_year + " " + cs_month + " " + cs_day
        #
        start = pq(r.text)('table.stylebazi tr>td b:eq(16)').next().text().replace('\n','')
        dayuns = pq(r.text)('table.stylebazi tr>td b:eq(13)').next().text().split('　　')
        years = pq(r.text)('table.stylebazi tr>td b:eq(15)').next().text().split('　　')
        return cs_year,cs_month,cs_day,cs_bazi,start,dayuns,years
    
    def get_qs_bazi(year,month,day,sex,dayuns,years,sess,cookies):
        time.sleep(random.randrange(2,5))
        params= { 'name':'','area':'','sex':sex,'year':year,'month':month,'date':day
            ,'hour':'9','minute':'20','jingdu':120,'jingdufen':0,'taiyang':0,'quanpai':1,'stylebazi':'120','stylebazi':'0'}
        params['submit']=urllib.parse.quote('在线排八字盘'.encode('gb2312'))
        url = 'https://www.china95.net/paipan/bazi/bazi_show.asp'
        r = sess.post(url,data=params,cookies=cookies,verify=False)
        r.encoding = 'gb2312'
        #
        tiangan = pq(r.text)('table.stylebazi tr>td font:eq(22)').text()
        tiangans = tiangan.split('　　　　 ')
        dizhi = pq(r.text)('table.stylebazi tr>td font:eq(24)').text()
        dizhis = dizhi.split('　　　　 ')
        #
        qs_year = tiangans[0]+dizhis[0]
        qs_month = tiangans[1]+dizhis[1]
        qs_day = tiangans[2]+dizhis[2]
        qs_bazi = qs_year + " " + qs_month + " " + qs_day
        #
        dy = get_dayun(dayuns,years,year)
        #
        return qs_year,qs_month,qs_day,qs_bazi,dy
    
    def get_dayun(dayuns,years,current):
        for i,val in enumerate(years):
            if int(current)>=int(val) and int(current)<int(val)+10:
                return dayuns[i]
    
    def get_datas():
        sess,cookies = get_cookies()
        sql = r'select * from life'
        df = pd.read_sql(sql,engine_wiki())
        results = json.loads(df.to_json(orient='records'))
        year_p = re.compile(r'\d+(?=年)')
        month_p = re.compile(r'(?<=年)\d+(?=月)')
        day_p = re.compile(r'(?<=月)\d+(?=日)')
        #
        for result in results:
            year = year_p.search(result['cs']).group()
            month = month_p.search(result['cs']).group()
            day = day_p.search(result['cs']).group()
            if result['sex']=='男':
                sex = 1
            else:
                sex = 0
            if int(year)<1884:
                continue
            print(result['id'])
            cs_year,cs_month,cs_day,cs_bazi,ln,dayuns,years = get_cs_bazi(year,month,day,sex,sess,cookies)
            result['cs_year'] = cs_year
            result['cs_month'] = cs_month
            result['cs_day'] = cs_day
            result['cs_bazi'] = cs_bazi
            result['ln'] = ln
            #
            year = year_p.search(result['qs']).group()
            month = month_p.search(result['qs']).group()
            day = day_p.search(result['qs']).group()
            qs_year,qs_month,qs_day,qs_bazi,dy = get_qs_bazi(year,month,day,sex,dayuns,years,sess,cookies)
            result['qs_year'] = qs_year
            result['qs_month'] = qs_month
            result['qs_day'] = qs_day
            result['qs_bazi'] = qs_bazi
            result['dy'] = dy
            #
            helper.to_sql('life',engine_wiki(),pd.DataFrame([result]))
    
    
    if __name__ == '__main__':
        get_datas()

**3.2 获取八字2**

# -*- coding: utf-8 -*-
    # @time : 2021/2/11 10:34
    # @author : dzm
    # @dsec : 
    import requests
    import time
    import random
    import urllib
    from pyquery import PyQuery as pq
    import mysql_helper as helper
    import pandas as pd
    from db_engine import engine_wiki
    import re
    import json
    
    def get_cs_bazi(year,month,day,sex):
        params={ 'txtName':'求测者','rdoSex':sex,'zty':0,'pid':'','cid':'请选择城市','data_type':0,'NianXian':1,'cboYear':year,'cboMonth':month
                ,'cboDay':day,'cboHour':'8-辰','cboMinute':'30','YearGan':1,'YearZhi':1,'MoonGan':1,'MoonZhi':1,'DayGan':1,'DayZhi':1
                ,'HourGan':1,'HourZhi':1}
        url = r'https://pp.buyiju.com/bazi/bazi.asp'
        r = requests.post(url,data=params)
        r.encoding = 'utf-8'
        soup = pq(r.text)
        zhu = soup('.content table:eq(0) > tr:eq(1) td:gt(0):lt(4)').text()
        if zhu:
            zhus = zhu.split(' ')
            cs_year = zhus[0]
            cs_month = zhus[1]
            cs_day = zhus[2]
            cs_bazi = cs_year + " " + cs_month + " " + cs_day
            #
            dayuns = []
            dys = soup('.content table:eq(1) > tr:eq(1) td:gt(0)')
            for dy in dys:
                dayuns.append(pq(dy).text())
            years = []
            ys = soup('.content table:eq(1) > tr:eq(4) td:gt(0)')
            for y in ys:
                years.append(pq(y).text())
            #
            start = soup('.content table:eq(1) > tr:eq(5) td:eq(1)').text()
            return cs_year,cs_month,cs_day,cs_bazi,dayuns,years,start
        else:
            return None,None,None,None,None,None,None
    
    def get_qs_bazi(year,month,day,sex,dayuns,years):
        params={ 'txtName':'求测者','rdoSex':sex,'zty':0,'pid':'','cid':'请选择城市','data_type':0,'NianXian':1,'cboYear':year,'cboMonth':month
            ,'cboDay':day,'cboHour':'8-辰','cboMinute':'30','YearGan':1,'YearZhi':1,'MoonGan':1,'MoonZhi':1,'DayGan':1,'DayZhi':1
            ,'HourGan':1,'HourZhi':1}
        url = r'https://pp.buyiju.com/bazi/bazi.asp'
        r = requests.post(url,data=params)
        r.encoding = 'utf-8'
        soup = pq(r.text)
        #
        zhu = soup('.content table:eq(0) > tr:eq(1) td:gt(0):lt(4)').text()
        if zhu:
            zhus = zhu.split(' ')
            qs_year = zhus[0]
            qs_month = zhus[1]
            qs_day = zhus[2]
            qs_bazi = qs_year + " " + qs_month + " " + qs_day
            #
            dy = get_dayun(dayuns,years,year)
            return qs_year,qs_month,qs_day,qs_bazi,dy
        else:
            return None,None,None,None,None
    
    def get_dayun(dayuns,years,current):
        for i,val in enumerate(years):
            if int(current)>=int(val) and int(current)<int(val)+10:
                return dayuns[i]
    
    def get_datas():
        sql = r'select * from life'
        df = pd.read_sql(sql,engine_wiki())
        results = json.loads(df.to_json(orient='records'))
        year_p = re.compile(r'\d+(?=年)')
        month_p = re.compile(r'(?<=年)\d+(?=月)')
        day_p = re.compile(r'(?<=月)\d+(?=日)')
        #
        for result in results:
            year = year_p.search(result['cs']).group()
            month = month_p.search(result['cs']).group()
            day = day_p.search(result['cs']).group()
            if result['sex']=='男':
                sex = 1
            else:
                sex = 0
            # if int(year)<1884:
            # continue
            print(result['id'])
            cs_year,cs_month,cs_day,cs_bazi,dayuns,years,ln = get_cs_bazi(year,month,day,sex)
            if cs_year is None:
                continue
            result['cs_year'] = cs_year
            result['cs_month'] = cs_month
            result['cs_day'] = cs_day
            result['cs_bazi'] = cs_bazi
            result['ln'] = ln
            #
            year = year_p.search(result['qs']).group()
            month = month_p.search(result['qs']).group()
            day = day_p.search(result['qs']).group()
            qs_year,qs_month,qs_day,qs_bazi,dy = get_qs_bazi(year,month,day,sex,dayuns,years)
            if qs_year is None:
                continue
            result['qs_year'] = qs_year
            result['qs_month'] = qs_month
            result['qs_day'] = qs_day
            result['qs_bazi'] = qs_bazi
            result['dy'] = dy
            #
            helper.to_sql('life',engine_wiki(),pd.DataFrame([result]))
    
    if __name__ == '__main__':
        # cs_year,cs_month,cs_day,cs_bazi,dayuns,years,start = get_cs_bazi(1894,2,29,1)
        # print(cs_year,cs_month,cs_day,cs_bazi,dayuns,years,start)
        get_datas()