爬取菜鸟教程的100例

叁歲伎倆 2022-10-09 00:47 171阅读 0赞

import requests
    from lxml import etree
    
    base_url = 'https://www.runoob.com/python/python-exercise-example%s.html'
    
    
    def get_element(url):
        headers = {
            'cookie': '__gads=Test; Hm_lvt_3eec0b7da6548cf07db3bc477ea905ee=1573454862,1573470948,1573478656,1573713819; Hm_lpvt_3eec0b7da6548cf07db3bc477ea905ee=1573714018; SERVERID=fb669a01438a4693a180d7ad8d474adb|1573713997|1573713863',
            'referer': 'https://www.runoob.com/python/python-100-examples.html',
            'user-agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36'
        }
        response = requests.get(url, headers=headers)
        return etree.HTML(response.text)
    
    
    def write_py(i, text):
        with open('练习实例%s.py' % i, 'w', encoding='utf-8') as file:
            file.write(text)
    
    
    def main():
        for i in range(1, 101):
            html = get_element(base_url % i)
            print(base_url%i)
            print(html)
            # // *[ @ id = "content"] / p[2] / text()
            content = '题目：' + html.xpath('//div[@id="content"]/p[2]/text()')[0] + '\n'
            print(content)
            fenxi = html.xpath('//div[@id="content"]/p[position()>=2]/text()')[0]
            daima = ''.join(html.xpath('//div[@class="hl-main"]/span/text()')) + '\n'
            haha = '"""\n' + content + fenxi + daima + '\n"""'
            # write_py(i, haha)
            # print(fenxi)
    
    if __name__ == '__main__':
        main()

a表示的就是为文本末尾继续写入，w会进行覆盖

import requests
    from lxml import html
    
    url = "http://book.zongheng.com/chapter/952586/60520336.html"
    
    header = {
        "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36"
    }
    page = requests.Session().get(url, headers=header)
    # print(page.text)
    # print(page)
    # page.encoding='utf-8'
    tree = html.fromstring(page.text)
    # print(tree)
    # print(page.text)
    # /表示绝对路径  //表示相对路径  //*[@id="readerFt"]/div/div[5]   //*[@id="readerFt"]/div/div[5]/p[1]
    result = tree.xpath('//*[@id="readerFt"]/div/div[5]/p/text()')
    # //*[@id="monthTicketRankList"]/li[1]/a
    for i in result:
        print(i)
        f=open('小说.txt','a')
        f.write(i+'\n')
    # print(result)

https://blog.csdn.net/qq\_40558166/article/details/102868801?ops\_request\_misc=%257B%2522request%255Fid%2522%253A%2522162435676916780261949018%2522%252C%2522scm%2522%253A%252220140713.130102334%E2%80%A6%2522%257D&request\_id=162435676916780261949018&biz\_id=0&utm\_medium=distribute.pc\_search\_result.none-task-blog-2alltop\_positive~default-2-102868801.first\_rank\_v2\_pc\_rank\_v29&utm\_term=%E7%88%AC%E8%99%AB&spm=1018.2226.3001.4187