python多进程卡死

客官°小女子只卖身不卖艺 2024-04-19 11:24 58阅读 0赞

为了提高效率，刚开始用了多线程  threading  和协程  gevent  ，但因为其他原因，后来又取消了线程，改用了多进程，multiprocessing库，代码如下，结果运行时到最后一步卡死了，后来花了三四天才找到原因，原来是用协程时用了猴子补丁monkey patch，但是改用进程时忘记删掉了，导致程序卡死，删掉引用的线程和协程库以及猴子补丁monkey patch就可以正常运行了。

import gevent
    from gevent import monkey, pool
    
    monkey.patch_all()
    import requests, time, queue, json,os
    
    import multiprocessing
    
    
    class Baidu:
        def __init__(self):
            self.headers = {
                "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3",
                "Accept-Encoding": "gzip, deflate, br",
                "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8,ja;q=0.7",
                "Cache-Control": "max-age=0",
                "Connection": "keep-alive",
                "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36",
            }
            self.url = "https://www.baidu.com/s"
            self.hm = "华美顾问"
            self.hm_gg='华美顾问'
    
        def run(self, kw_li,re_dict):
            kw_dict = {}
            # for kw in kw_li:
            #     pm = self.pc(kw)
            #     kw_dict.update({kw: pm})
            # with open('bd{:.0f}.json'.format(time.time()),'w',encoding='utf8')as f:
            #     f.write(json.dumps(kw_dict,indent=4,ensure_ascii=False))
            # print(kw_dict)
            re_dict['百度电脑端']='kw_dict'
            print(re_dict)
            # return kw_dict
    
    
    
    class Baidu_APP:
        def __init__(self):
            self.headers = {
                "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3",
                "Accept-Encoding": "gzip, deflate, br",
                "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8,ja;q=0.7",
                "Cache-Control": "max-age=0",
                "Connection": "keep-alive",
                "User-Agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1",
            }
            self.url = "https://www.baidu.com/from=844b/s"
            self.hm = "华美顾问"
            self.hm_gg='华美顾问'
    
    
        def run(self, kw_li,re_dict):
            kw_dict = {}
            # for kw in kw_li:
            #     pm = self.pc(kw)
            #     kw_dict.update({kw: pm})
            # with open('bd{:.0f}.json'.format(time.time()),'w',encoding='utf8')as f:
            #     f.write(json.dumps(kw_dict,indent=4,ensure_ascii=False))
            # print(kw_dict)
            re_dict['百度手机端']='kw_dict'
            print(re_dict)
            # return kw_dict
    
    
    if __name__ == "__main__":
        # main()
        manager=multiprocessing.Manager()
        re_dict=manager.dict()
        # kw_li, df = read_kw()
        kw_li=['酒店','饭店']
        # t1=time.time()
        baidu = Baidu()
        # baidu_dict=baidu.run(kw_li)
        # t2=time.time()
        # t_b=t2-t1
    
        # t1=time.time()
        baidu_app = Baidu_APP()  
      
        p1=multiprocessing.Process(target=baidu.run,args=(kw_li,re_dict))
        p2=multiprocessing.Process(target=baidu_app.run,args=(kw_li,re_dict))
        # p3=multiprocessing.Process(target=google.run,args=(kw_li,re_dict,))
        p1.start()
        p2.start()
        p1.join()
        print(p1.is_alive())
        print(p2.is_alive())
        p2.join()
        print(p1.is_alive())
        print(p2.is_alive())
        print(re_dict)