[python] Pubmed 검색결과를 csv에 저장하기

Pubmed에 검색해서 나온 논문들의 제목들만 추려서 csv 파일로 출력해주는 python 코드입니다.
BeatifulSoup 부분을 수정하면 snippet도 저장하도록 만들 수 있을 것 같습니다.

python 코드

from bs4 import BeautifulSoup
import requests
import csv
import urllib.parse

def get_titles(url) :

    titleList = []

    web_url = url

    r = requests.get(web_url)
    bs = BeautifulSoup(r.text,'lxml')

    total_result_number = bs.select("div.results-amount-container")
    total_result_number = bs.select("span.value")
    total_result_number = int(bs.select("span.value")[0].text.strip())

    total_page = bs.select("label.of-total-pages")
    total_page = total_page[0].text.strip()
    total_page = int(total_page[3:])

    for i in range(1,total_page+1) :
        web_url = url+"&page="+str(i)
        r = requests.get(web_url)
        bs = BeautifulSoup(r.text,'lxml')
        title = bs.select("a.docsum-title")
        if title :
            for i in title :
                titleList.append([i.text.strip()])

    if total_result_number == len(titleList) :
        print(f"All {len(titleList)} results are succefully retrieved. Now csv will be written.")

    return(titleList)

query = input("Give me query : ")
query = urllib.parse.quote(query)

web_url = f"https://pubmed.ncbi.nlm.nih.gov/?term={query}&sort=date"

titleList = get_titles(web_url)

with open('pubmed_search_result.csv', 'w',newline='', encoding="utf-8") as file:
    writer = csv.writer(file)
    for i in titleList :
        writer.writerow(i)

print("Job is finished.")

코드를 실행하면 "Give me query : " 라는 메시지가 나타나며, 검색어를 입력하면 잠깐 시간이 지난 후 아래와 같은 메시지가 출력되면 완료된 것입니다.

All 21 results are succefully retrieved. Now csv will be written.
Job is finished.

requirement.txt

사용된 패키지들 입니다.

beautifulsoup4==4.12.2
certifi==2023.5.7
charset-normalizer==3.1.0
idna==3.4
lxml==4.9.2
requests==2.30.0
soupsieve==2.4.1
urllib3==2.0.2