Update app.py

app.py

@@ -1,6 +1,9 @@
 import nltk
 import re
 import nltkmodule
+from newspaper import Article
+from newspaper import fulltext
+import requests
 
 from nltk.tokenize import word_tokenize
 from sentence_transformers import SentenceTransformer
@@ -38,13 +41,35 @@ def remove_stopwords(sen):
     sen_new = " ".join([i for i in sen if i not in stop_words])
     return sen_new
 
-def keyphrase_generator(article, model_1, model_2, max_num_keywords):
+def keyphrase_generator(article_link, model_1, model_2, max_num_keywords):
   element=[]
+  final_textrank_list=[]
   document=[]
-  text=[]
+  text_doc=[]
+  score_list=[]
+  sum_list=[]
   model_1 = SentenceTransformer(model_1)
   model_2 = SentenceTransformer(model_2)
+  url = article_link
+  html = requests.get(url).text
+  article = fulltext(html)
   corpus=sent_tokenize(article)
+  indicator_list=['concluded','concludes','in a study', 'concluding','conclude','in sum','in a recent study','therefore','thus','so','hence',
+          'as a result','accordingly','consequently','in short','proves that','shows that','suggests that','demonstrates that','found that','observed that',
+          'indicated that','suggested that','demonstrated that']
+  count_dict={}
+  for l in corpus:
+    c=0
+    for l2 in indicator_list:     
+       if l.find(l2)!=-1:#then it is a substring
+          c=1
+          break           
+    if c:#
+       count_dict[l]=1
+    else:
+       count_dict[l]=0
+  for sent, score in count_dict.items():
+    score_list.append(score)
   clean_sentences_new = pd.Series(corpus).str.replace("[^a-zA-Z]", " ").tolist()
   corpus_embeddings = model_1.encode(clean_sentences_new)
   sim_mat = np.zeros([len(clean_sentences_new), len(clean_sentences_new)])
@@ -54,21 +79,28 @@ def keyphrase_generator(article, model_1, model_2, max_num_keywords):
         sim_mat[i][j] = cosine_similarity(corpus_embeddings[i].reshape(1,768), corpus_embeddings[j].reshape(1,768))[0,0]
   nx_graph = nx.from_numpy_array(sim_mat)
   scores = nx.pagerank(nx_graph)
-  ranked_sentences = sorted(((scores[i],s) for i,s in enumerate(corpus)), reverse=True)
-  for elem in ranked_sentences:
-    element.append(elem[1])
-  a=int((10*len(element))/100.0)
+  sentences=((scores[i],s) for i,s in enumerate(corpus))
+
+  for elem in sentences:
+    element.append(elem[0])
+  for sc, lst in zip(score_list, element):  ########## taking the scores from both the lists 
+    sum1=sc+lst
+    sum_list.append(sum1)
+  x=sorted(((sum_list[i],s) for i,s in enumerate(corpus)), reverse=True)
+  for elem in x:
+    final_textrank_list.append(elem[1]) 
+  a=int((10*len(final_textrank_list))/100.0)
   if(a<5):
     total=5
   else:
     total=int(a)
   for i in range(total):
-    document.append(element[i])
+    document.append(final_textrank_list[i])
   doc=" ".join(document)
   for i in document:
     doc_1=nlp(i)
-    text.append([X.text for X in doc_1.ents])
-  entity_list = [item for sublist in text for item in sublist]
+    text_doc.append([X.text for X in doc_1.ents])
+  entity_list = [item for sublist in text_doc for item in sublist]
   entity_list = [word for word in entity_list if not word in all_stopwords]
   entity_list=list(dict.fromkeys(entity_list))
   doc_embedding = model_2.encode([doc])
@@ -80,9 +112,8 @@ def keyphrase_generator(article, model_1, model_2, max_num_keywords):
   keywords = '\n'.join(keyword_list) 
   return keywords
 
-
 igen=gr.Interface(keyphrase_generator, 
-             inputs=[gr.inputs.Textbox(lines=10, placeholder="Provide article text here",default="", label="article text"),gr.inputs.Textbox(lines=1, placeholder="SBERT model",default="all-mpnet-base-v2", label="Model for TextRank (e.g. all-mpnet-base-v2)"),gr.inputs.Textbox(lines=1, placeholder="SBERT model",default="all-distilroberta-v1",label="Model for keyphrases (e.g. all-distilroberta-v1)"),gr.inputs.Slider(minimum=5, maximum=30, step=1, default=10, label="Max Keywords")], 
+             inputs=[gr.inputs.Textbox(lines=3, placeholder="Provide article link here",default="", label="article link"),gr.inputs.Textbox(lines=1, placeholder="SBERT model",default="all-mpnet-base-v2", label="Model for TextRank (e.g. all-mpnet-base-v2)"),gr.inputs.Textbox(lines=1, placeholder="SBERT model",default="all-distilroberta-v1",label="Model for keyphrases (e.g. all-distilroberta-v1)"),gr.inputs.Slider(minimum=5, maximum=30, step=1, default=10, label="Max Keywords")], 
              outputs="text", theme="huggingface", 
              title="Scientific Article Keyphrase Generator", 
              description="Generates the keyphrases from an article which best describes the article.",