Co-Authorship Network Generator using scraped data from Google Scholar via SerpAPI

Leave a reply

co_author_network_small

  1. main script coauthorship_network.py

     import networkx as nx
 import matplotlib.pyplot as plt
 import bibtexparser
 import re

 #python3 get_articles_with_serpapi.py > articles_Brinkmann.txt
 #grep -o 'title' articles_Brinkmann.txt | wc -l

 # Helper function to clean special characters from strings
 def clean_string(s):
     s = s.replace('.', '')  # Remove periods
     s = re.sub(r'[{}]', '', s)  # Remove curly braces used in BibTeX formatting
     s = re.sub(r'\\[a-zA-Z]{1,2}', '', s)  # Remove LaTeX commands (e.g., \textcopyright)
     return ' '.join([n for n in s.split(' ') if len(n) > 1])  # Remove single-letter names
 # Load the .bib file
 with open('articles_Brinkmann.txt', 'r') as bibtex_file:
     bib_database = bibtexparser.load(bibtex_file)
     print(len(bib_database.entries))

 # Create an empty graph
 G = nx.Graph()

 #internal_authors = ["L Redecke", "T Schulz", "M Brinkmann"]
 #internal_authors = list(set(internal_authors))
 # Initialize an empty set for internal authors
 internal_authors = set()
 # Iterate through each entry in the bibtex file
 for entry in bib_database.entries:

     # Get and clean authors
     authors = entry.get('authors', '').split(', ')
     #cleaned_authors = [clean_string(author) for author in authors]

     # Print authors
     #print(f"Authors: {cleaned_authors}")
     print(f"Authors: {authors}")

     # Add authors to the internal_authors set
     internal_authors.update(authors)

 # Convert the internal_authors set to a list
 #internal_authors = list(internal_authors)
 internal_authors = list(set(internal_authors))
 print(f"Internal Authors: {internal_authors}")

 #print("Nodes:", G.nodes())
 #print("Edges:", G.edges())

 # Iterate through each entry in the bibtex file
 for entry in bib_database.entries:
     title = entry.get('title', '').replace('=', ' ')
     #title = clean_string(f"Paper/{title}")
     #print(title)
     authors = entry.get('authors', '').split(', ')  # Authors are separated by 'and' in BibTeX
     #print(authors)
     for author in authors:
         #author = clean_string(f"Author/{author}")
         if author in internal_authors:
             print(author)
             G.add_edge(author, title)
 ## Try different layout engines if 'sfdp' is problematic
 #try:
 #    pos = nx.nx_pydot.graphviz_layout(G, prog='sfdp')
 #except AssertionError:
 #    print("Error with sfdp layout, switching to 'dot' layout.")
 #    pos = nx.nx_pydot.graphviz_layout(G, prog='dot')  # Fallback to 'dot'
 try:
     pos = nx.nx_pydot.graphviz_layout(G, prog='sfdp')
 except Exception as e:
     print("Error generating layout:", e)
     pos = nx.spring_layout(G)  # Fallback to another layout
 for node in G.nodes():
     if isinstance(node, str):
         G.nodes[node]['label'] = node.replace(' ', '_').replace('/', '_')
 print("Nodes in the graph:", G.nodes())
 # Determine maximum length for author nodes
 max_len = max([len(n) for n in G.nodes() if n in internal_authors])
 print(max_len)
 plt.figure(figsize=(96, 72))
 #plot nodes for authors
 nx.draw_networkx_nodes(G, pos,
     #nodelist=[n for n in G.nodes() if n.startswith('Author/')],
     nodelist=[n for n in G.nodes() if n in internal_authors],
     node_size=max_len*200)
 #plot nodes for publications
 nx.draw_networkx_nodes(G, pos,
     nodelist=[n for n in G.nodes() if n not in internal_authors],
     node_color='y',
     node_size=100)
 nx.draw_networkx_edges(G, pos)
 nx.draw_networkx_labels(G, pos,
     labels={n: n.split('/')[-1].replace(' ', '\n') for n in G.nodes() if n in internal_authors},
     font_color='w', font_size=10, font_weight='bold', font_family='serif') #font_family='sans-serif')
 #'serif': Uses a serif typeface (e.g., Times New Roman).
 #'sans-serif': Uses a sans-serif typeface (e.g., Helvetica, Arial).
 #'monospace': Uses a monospace typeface (e.g., Courier New).
 #'DejaVu Sans': A popular sans-serif typeface available in many environments.
 #'Arial': A widely available sans-serif typeface.
 #'Times New Roman': A classic serif typeface.
 #'Comic Sans MS': A casual sans-serif typeface.
 plt.axis('off')
 # Save the plot as a PNG file
 #, bbox_inches="tight"
 plt.savefig("co_author_network.png", format="png")
 #convert Figure_2.png -crop 1340x750+315+135 co_author_network_cropped_2.png
 plt.show()

 # Count the number of publications and authors
 publications = [n for n in G.nodes() if n not in internal_authors]
 #authors = [n for n in G.nodes() if n in internal_authors]

 # Print the counts
 print(f"Number of Publications: {len(publications)}")
 print(f"Number of Authors: {len(internal_authors)}")
 #68, 476

 ## Optionally, print out the publications and authors themselves
 #print(f"Publications: {publications}")
 print(f"Authors: {internal_authors}")

  2. code of get_articles_with_serpapi.py

     from serpapi import GoogleSearch
 #pip install google-search-results
 #https://github.com/serpapi/google-search-results-python
 #https://serpapi.com/google-scholar-author-co-authors
 #We are able to extract: name, link, author_id, affiliations, email, and thumbnail results.

 params = {
   "engine": "google_scholar_author",
   "author_id": "5AzhtgUAAAAJ",
   "api_key": "ed",
   "num" : 100
 }

 #-- for each publication, view complete citation statistics per year, e.g. for 5AzhtgUAAAAJ:KlAtU1dfN6UC
 #https://scholar.google.com/citations?hl=en&view_op=view_citation&citation_for_view=5AzhtgUAAAAJ:KlAtU1dfN6UC
 #params = {
 #  "engine": "google_scholar_author",
 #  "citation_id": "5AzhtgUAAAAJ:KlAtU1dfN6UC",
 #  "view_op": "view_citation",
 #  "api_key": "ed"
 #}

 search = GoogleSearch(params)
 results = search.get_dict()
 print(results)

 ## Safely get the co_authors
 #co_authors = results.get("co_authors", [])
 #if co_authors:
 #    print("Co-authors:", co_authors)
 #else:
 #    print("No co-authors found.")
 ##co_authors = results["co_authors"]

  3. code of get_coauthors_jiabin.py

     from serpapi import GoogleSearch

 params = {
   "engine": "google_scholar_author",
   "author_id": "P1pS4s0AAAAJ",
   "view_op": "list_colleagues",
   "api_key": "ed"
 }

 search = GoogleSearch(params)
 results = search.get_dict()
 print(results)

 co_authors = results["co_authors"]

  4. get_citations_raw.py

     import requests

 api_key = "60"
 url = "https://api.scrapingdog.com/google_scholar"

 params = {
     "api_key": api_key,
     "query": "Melanie M. Brinkmann",
     "language": "en",
     "page": 10,
     "results": 100
 }

 response = requests.get(url, params=params)

 if response.status_code == 200:
     data = response.json()
     print(data)
 else:
     print(f"Request failed with status code: {response.status_code}")

  5. post-processing of the file generated by get_articles_with_serpapi.py

     #delete all header and ends: namely remove "...'articles': [" and "], cited_by= {"table"= [{citations= {"all"= 5351, "since_2019"= 2358}}, {"h_index"= {"all"= 33, "since_2019"= 25}}, {"i10_index"= {"all"= 47, "since_2019"= 42}}], graph= [{year= 2006, citations= 22}, {year= 2007, citations= 73}, {year= 2008, citations= 93}, {year= 2009, citations= 216}, {year= 2010, citations= 204}, {year= 2011, citations= 293}, {year= 2012, citations= 269}, {year= 2013, citations= 331}, {year= 2014, citations= 282}, {year= 2015, citations= 308}, {year= 2016, citations= 281}, {year= 2017, citations= 301}, {year= 2018, citations= 266}, {year= 2019, citations= 288}, {year= 2020, citations= 369}, {year= 2021, citations= 609}, {year= 2022, citations= 430}, {year= 2023, citations= 366}, {year= 2024, citations= 288}]}, 'public_access': {'link': 'https://scholar.google.com/citations?view_op=list_mandates&hl=en&user=5AzhtgUAAAAJ', 'available': 50, 'not_available': 4}}"

 "}, {'title':" --> "}\n\n@article{1, title="
 replace 1 to actual id (1,...,82)
 "MM Brinkmann" --> "M Brinkmann"
 :-->=
 '-->"
 "link" --> link, "citation_id" --> citation_id, "authors"-->authors, "publication"-->publication, "cited_by"-->cited_by, "serpapi_link"-->serpapi_link, "graph"-->graph, "cites_id"-->cites_id, "year"-->year, "citations"-->citations, "value" --> value

 #manually replace the author complete name by clicking the google scholar links
 #remove the records still with abbreviated name in the links.

 #The end effect as follows:
 @article{1, title= "UNC93B1 delivers nucleotide-sensing toll-like receptors to endolysosomes", link= "https=//scholar.google.com/citations?view_op=view_citation&hl=en&user=5AzhtgUAAAAJ&pagesize=100&citation_for_view=5AzhtgUAAAAJ=W7OEmFMy1HYC", citation_id= "5AzhtgUAAAAJ=W7OEmFMy1HYC", authors= "You-Me Kim, Melanie M Brinkmann, Marie-Eve Paquet, Hidde L Ploegh", publication= "Nature 452 (7184), 234-238, 2008", cited_by= {value= 847, link= "https=//scholar.google.com/scholar?oi=bibs&hl=en&cites=3461748963046634721", serpapi_link= "https=//serpapi.com/search.json?cites=3461748963046634721&engine=google_scholar&hl=en", cites_id= "3461748963046634721"}, year= "2008"}

 @article{2, title= "Proteolytic cleavage in an endolysosomal compartment is required for activation of Toll-like receptor 9", link= "https=//scholar.google.com/citations?view_op=view_citation&hl=en&user=5AzhtgUAAAAJ&pagesize=100&citation_for_view=5AzhtgUAAAAJ=4TOpqqG69KYC", citation_id= "5AzhtgUAAAAJ=4TOpqqG69KYC", authors= "Boyoun Park, Melanie M Brinkmann, Eric Spooner, Clarissa C Lee, You-Me Kim, Hidde L Ploegh", publication= "Nature immunology 9 (12), 1407-1414, 2008", cited_by= {value= 587, link= "https=//scholar.google.com/scholar?oi=bibs&hl=en&cites=8523162291112327960", serpapi_link= "https=//serpapi.com/search.json?cites=8523162291112327960&engine=google_scholar&hl=en", cites_id= "8523162291112327960"}, year= "2008"}

 @article{3, title= "The interaction between the ER membrane protein UNC93B and TLR3, 7, and 9 is crucial for TLR signaling", link= "https=//scholar.google.com/citations?view_op=view_citation&hl=en&user=5AzhtgUAAAAJ&pagesize=100&citation_for_view=5AzhtgUAAAAJ=-f6ydRqryjwC", citation_id= "5AzhtgUAAAAJ=-f6ydRqryjwC", authors= "Melanie M Brinkmann, Eric Spooner, Kasper Hoebe, Bruce Beutler, Hidde L Ploegh, You-Me Kim", publication= "The Journal of cell biology 177 (2), 265-275, 2007", cited_by= {value= 562, link= "https=//scholar.google.com/scholar?oi=bibs&hl=en&cites=13542374013520997852", serpapi_link= "https=//serpapi.com/search.json?cites=13542374013520997852&engine=google_scholar&hl=en", cites_id= "13542374013520997852"}, year= "2007"}

 @article{4, title= "Noncanonical autophagy is required for type I interferon secretion in response to DNA-immune complexes", link= "https=//scholar.google.com/citations?view_op=view_citation&hl=en&user=5AzhtgUAAAAJ&pagesize=100&citation_for_view=5AzhtgUAAAAJ=KlAtU1dfN6UC", citation_id= "5AzhtgUAAAAJ=KlAtU1dfN6UC", authors= "Jill Henault, Jennifer Martinez, Jeffrey M Riggs, Jane Tian, Payal Mehta, Lorraine Clarke, Miwa Sasai, Eicke Latz, Melanie M Brinkmann, Akiko Iwasaki, Anthony J Coyle, Roland Kolbeck, Douglas R Green, Miguel A Sanjuan", publication= "Immunity 37 (6), 986-997, 2012", cited_by= {value= 376, link= "https=//scholar.google.com/scholar?oi=bibs&hl=en&cites=6648645242373278731", serpapi_link= "https=//serpapi.com/search.json?cites=6648645242373278731&engine=google_scholar&hl=en", cites_id= "6648645242373278731"}, year= "2012"}

 @article{5, title= "Granulin is a soluble cofactor for toll-like receptor 9 signaling", link= "https=//scholar.google.com/citations?view_op=view_citation&hl=en&user=5AzhtgUAAAAJ&pagesize=100&citation_for_view=5AzhtgUAAAAJ=M3ejUd6NZC8C", citation_id= "5AzhtgUAAAAJ=M3ejUd6NZC8C", authors= "Boyoun Park, Ludovico Buti, Sungwook Lee, Takashi Matsuwaki, Eric Spooner, Melanie M Brinkmann, Masugi Nishihara, Hidde L Ploegh", publication= "Immunity 34 (4), 505-513, 2011", cited_by= {value= 223, link= "https=//scholar.google.com/scholar?oi=bibs&hl=en&cites=8731573748380815185", serpapi_link= "https=//serpapi.com/search.json?cites=8731573748380815185&engine=google_scholar&hl=en", cites_id= "8731573748380815185"}, year= "2011"}

 #Check how many unique author names in the graphics?
 sed 's/^[ \t]*//;s/[ \t]*$//' author_names.txt | sort -u > author_name_uniq.txt
 sort -uf author_names.txt > author_name_uniq.txt
 cat author_names.txt | tr -cd '\11\12\15\40-\176' | sort -u > author_name_uniq.txt
 sed 's/^[ \t]*//;s/[ \t]*$//' author_names.txt | tr -cd '\11\12\15\40-\176' | sort -uf > author_name_uniq.txt

 sed 's/‐/-/g' author_names.txt | sort -u > author_name_uniq.txt
 cat author_name_uniq.txt | tr '[:upper:]' '[:lower:]' | sort -u > author_name_uniq2.txt

Leave a Reply

Your email address will not be published. Required fields are marked *