diff --git a/.gitignore b/.gitignore old mode 100644 new mode 100755 index 68bc17f..d89d1da --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ +# Jupyter Books +_build/ + # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] diff --git a/notebooks/openalex_preprints.ipynb b/notebooks/openalex_preprints.ipynb deleted file mode 100644 index 2b7ebcc..0000000 --- a/notebooks/openalex_preprints.ipynb +++ /dev/null @@ -1,87 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## How to get all preprints which have not been published in a journal yet?\n", - "\n", - "We use [OpenAlex](https://openalex.org) to retrieve all articles which are a preprint, but have so far not been published by a peer-reviewed journal." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from pyalex import Works, Authors, Sources, Institutions, Concepts, Publishers, Funders\n", - "from itertools import chain\n", - "import pandas as pd\n", - "import pyalex" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Variables reduce the size of the output and the time required for execution\n", - "year = 2020\n", - "n_max = None # when set to None all papers are queried" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "query = Works().filter(type=\"article\", publication_year=year, primary_location={'version': 'submittedVersion'}).sort(cited_by_count=\"desc\")\n", - "\n", - "data = []\n", - "for item in chain(*query.paginate(per_page=200, n_max=n_max)):\n", - " title = item.get('title', None)\n", - " publication_date = item.get('publication_date', None)\n", - " doi = item.get('doi', None)\n", - " cited_by_count = item.get('cited_by_count', None)\n", - " locations_count = item.get('locations_count', None)\n", - " host_organization_source = item.get('primary_location', {}).get('source', {})\n", - " host_organization = None\n", - " if host_organization_source:\n", - " host_organization = host_organization_source.get('display_name')\n", - " \n", - " data.append({'Title': title, 'Publication Date': publication_date, 'DOI': doi, 'Host Organization': host_organization, 'Cited by Count': cited_by_count, 'Locations Count': locations_count})\n", - "\n", - "df = pd.DataFrame.from_dict(data)\n", - "df.to_csv(f'../results/openalex_preprints_{year}.csv')\n", - " " - ] - } - ], - "metadata": { - "interpreter": { - "hash": "dacd09b87f275de77aaa2661dde3428b7c30cfb80959b39c223c1792ae834a1a" - }, - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.4" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/notebooks/openalex_preprints_step_by_step.ipynb b/notebooks/openalex_preprints_step_by_step.ipynb deleted file mode 100644 index 06aa885..0000000 --- a/notebooks/openalex_preprints_step_by_step.ipynb +++ /dev/null @@ -1,360 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Prepint Analysis - How to get all preprints of a topics?\n", - "\n", - "We use [OpenAlex](https://openalex.org) to retrieve all articles which are a preprint, but have so far not been published by a peer-reviewed journal." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 1. Load libraries & define helper functions" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "from pyalex import Works, Authors, Sources, Institutions, Concepts, Publishers, Funders\n", - "from itertools import chain\n", - "import pandas as pd\n", - "import pyalex" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "# Determine if any of the locations (journals) the paper is already published\n", - "def is_any_location_published(locations):\n", - " for location in locations:\n", - " if location['version'] == 'publishedVersion':\n", - " return True\n", - " return False" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "# Combine all authos\n", - "def join_authors(list_of_authors):\n", - " return ', '.join([author['author']['display_name'] for author in list_of_authors])" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "# Extract key information from the locations\n", - "def join_locations(list_of_locations):\n", - " summary = []\n", - " for location in list_of_locations:\n", - " if location['source']:\n", - " summary.append(f\"{location['version']}: {location['source']['host_organization_name']} - {location['landing_page_url']}\")\n", - " else:\n", - " summary.append(f\"{location['version']} - {location['landing_page_url']}\")\n", - " return ', '.join(summary)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 2. Set the Topic & Year\n", - "\n", - "Set the year and the number of papers you want to obtain" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": {}, - "outputs": [], - "source": [ - "# Variables reduce the size of the output and the time required for execution\n", - "topic = 'COVID'\n", - "year = 2023\n", - "n_max = 500 # when set to None all papers are queried" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 3. Get the preprints\n", - "\n", - "Run te following code to get the preprints for the specified parameters" - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "metadata": {}, - "outputs": [], - "source": [ - "query = Works().search(topic).filter(type=\"article\", publication_year=year, primary_location={'version': 'submittedVersion'}, locations={'is_published': False}).sort(cited_by_count=\"desc\")\n", - "\n", - "preprints = []\n", - "\n", - "# Iterate over all query results\n", - "for item in chain(*query.paginate(per_page=200, n_max=n_max)):\n", - " \n", - " # Get key properties\n", - " oa_id = item.get('id', None)\n", - " title = item.get('title', None)\n", - " publication_date = item.get('publication_date', None)\n", - " doi = item.get('doi', None)\n", - " cited_by_count = item.get('cited_by_count', None)\n", - " locations_count = item.get('locations_count', None)\n", - " \n", - " # Join all authors\n", - " authors = join_authors(item['authorships']) \n", - " locations = item.get('locations', None)\n", - " locations_overview = join_locations(item['locations'])\n", - " \n", - " # Only append the paper to the preprints if is not published in any other journal\n", - " if locations_count == 1 or not is_any_location_published(locations):\n", - " preprints.append({'id': oa_id, 'title': title, 'publication_date': publication_date, 'doi': doi,\n", - " 'cited': cited_by_count, 'authors': authors,\n", - " 'locations': locations_overview, 'location_count': locations_count})\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 4. Store the data" - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
idtitlepublication_datedoicitedauthorslocationslocation_count
0https://openalex.org/W4318909870Overcoming Vaccine Skepticism in Pakistan: A C...2023-02-02https://doi.org/10.5281/zenodo.759714112Bibi Aisha SadiqasubmittedVersion: European Organization for Nu...1
1https://openalex.org/W4386199227Clinical Rationale for SARS-CoV-2 Base Spike P...2023-08-25https://doi.org/10.5281/zenodo.82864605Peter A. McCullough, Cade Wynn, Brian C ProctersubmittedVersion: European Organization for Nu...1
2https://openalex.org/W3197911323The Political Economy of a Modern Pandemic: As...2023-07-24https://doi.org/10.32920/23739360.v14John Shields, Zainab Abu AlrobsubmittedVersion - https://doi.org/10.32920/23...2
3https://openalex.org/W4307231237Higher risk of short term COVID-19 vaccine adv...2023-05-02https://doi.org/10.1093/rheumatology/keac6033Mrinalini Dey, R Naveen, Elena Nikiphorou, Par...submittedVersion: National Institutes of Healt...1
4https://openalex.org/W4319655309Navigating the Post-COVID Market: A Prospectiv...2023-02-09https://doi.org/10.5281/zenodo.76251903Peng Sun, Xiaode ZuosubmittedVersion: European Organization for Nu...1
\n", - "
" - ], - "text/plain": [ - " id \\\n", - "0 https://openalex.org/W4318909870 \n", - "1 https://openalex.org/W4386199227 \n", - "2 https://openalex.org/W3197911323 \n", - "3 https://openalex.org/W4307231237 \n", - "4 https://openalex.org/W4319655309 \n", - "\n", - " title publication_date \\\n", - "0 Overcoming Vaccine Skepticism in Pakistan: A C... 2023-02-02 \n", - "1 Clinical Rationale for SARS-CoV-2 Base Spike P... 2023-08-25 \n", - "2 The Political Economy of a Modern Pandemic: As... 2023-07-24 \n", - "3 Higher risk of short term COVID-19 vaccine adv... 2023-05-02 \n", - "4 Navigating the Post-COVID Market: A Prospectiv... 2023-02-09 \n", - "\n", - " doi cited \\\n", - "0 https://doi.org/10.5281/zenodo.7597141 12 \n", - "1 https://doi.org/10.5281/zenodo.8286460 5 \n", - "2 https://doi.org/10.32920/23739360.v1 4 \n", - "3 https://doi.org/10.1093/rheumatology/keac603 3 \n", - "4 https://doi.org/10.5281/zenodo.7625190 3 \n", - "\n", - " authors \\\n", - "0 Bibi Aisha Sadiqa \n", - "1 Peter A. McCullough, Cade Wynn, Brian C Procter \n", - "2 John Shields, Zainab Abu Alrob \n", - "3 Mrinalini Dey, R Naveen, Elena Nikiphorou, Par... \n", - "4 Peng Sun, Xiaode Zuo \n", - "\n", - " locations location_count \n", - "0 submittedVersion: European Organization for Nu... 1 \n", - "1 submittedVersion: European Organization for Nu... 1 \n", - "2 submittedVersion - https://doi.org/10.32920/23... 2 \n", - "3 submittedVersion: National Institutes of Healt... 1 \n", - "4 submittedVersion: European Organization for Nu... 1 " - ] - }, - "execution_count": 20, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "df = pd.DataFrame.from_dict(preprints)\n", - "df.to_csv(f'../results/openalex_preprints_{year}_{n_max}.csv')\n", - "df.to_excel(f'../results/openalex_preprints_{year}_{n_max}.xlsx')\n", - "df.head()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 5. Get a sample paper" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "id https://openalex.org/W1485155423\n", - "title For Hunger-proof Cities: Sustainable Urban Foo...\n", - "publication_date 2023-09-05\n", - "doi https://doi.org/10.32920/24084468.v1\n", - "cited 204\n", - "authors Mustafa Koç, Rod MacRae, L. J. A. Mougeot, J. ...\n", - "locations submittedVersion - https://doi.org/10.32920/24...\n", - "location_count 1\n", - "Name: 0, dtype: object" - ] - }, - "execution_count": 13, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "paper = df.iloc[0]\n", - "paper" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 5. Do basic analysis\n", - "\n", - "TODO:\n", - "- barchart with journals\n", - "- barchart with publication year" - ] - } - ], - "metadata": { - "interpreter": { - "hash": "dacd09b87f275de77aaa2661dde3428b7c30cfb80959b39c223c1792ae834a1a" - }, - "kernelspec": { - "display_name": "research-recipies", - "language": "python", - "name": "venv" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.3" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 515874a..0000000 --- a/requirements.txt +++ /dev/null @@ -1,40 +0,0 @@ -asttokens==2.4.1 -certifi==2024.7.4 -charset-normalizer==3.3.2 -comm==0.2.2 -debugpy==1.8.2 -decorator==5.1.1 -et-xmlfile==1.1.0 -executing==2.0.1 -idna==3.7 -ipykernel==6.29.5 -ipython==8.26.0 -jedi==0.19.1 -jupyter_client==8.6.2 -jupyter_core==5.7.2 -matplotlib-inline==0.1.7 -nest-asyncio==1.6.0 -numpy==2.0.0 -openpyxl==3.1.5 -packaging==24.1 -pandas==2.2.2 -parso==0.8.4 -pexpect==4.9.0 -platformdirs==4.2.2 -prompt_toolkit==3.0.47 -psutil==6.0.0 -ptyprocess==0.7.0 -pure-eval==0.2.2 -pyalex==0.14 -Pygments==2.18.0 -python-dateutil==2.9.0.post0 -pytz==2024.1 -pyzmq==26.0.3 -requests==2.32.3 -six==1.16.0 -stack-data==0.6.3 -tornado==6.4.1 -traitlets==5.14.3 -tzdata==2024.1 -urllib3==2.2.2 -wcwidth==0.2.13