1/31/23, 12:44 PM Twitter_Scraping_Streamlit.

py
1
2
3
4 # importing the libraries
5 import pandas as pd
6 import snscrape.modules.twitter as sntwitter
7 from pymongo import MongoClient
8 import streamlit as st
9 import datetime
10
11
12 # # Scrapping Tweets From Twitter Using Scscrape Twitter Module
13 #
14
15 # In[3]:
16
17
18 def twitter_scraper(hastag, Starting_date, Ending_date, Limit):
19    tweet_list = []
20    for i,tweet in enumerate(sntwitter.TwitterSearchScraper(f'{hastag} since:
{Starting_date} until:{Ending_date}').get_items()):
21        data =
[(tweet.date,tweet.user.username,tweet.rawContent,tweet.lang,tweet.viewCount,tweet.r
eplyCount,tweet.likeCount,tweet.retweetCount,)]
22        tweet_list.append(data)
23        if i > Limit-2:
24            break
25            
26    return tweet_list
27
28
29 # In[4]:
30
31
32 def create_DataFrame(tweet_list):
33    df = pd.DataFrame(tweet_list, columns = [('Date Time','Username','Tweet
Content','Tweet Language','Tweet Views',
34                                                      'Reply Count','like
Count','Retweet Count')])
35    return df
36        
37
38
39 # # Streamlit
40
41 # In[5]:
42
43
44 st.image("https://tse4.mm.bing.net/th?
id=OIP.lRrnOPsaYHpdvGwOiRNBNQHaGP&pid=Api&P=0")
45 st.title("Twitter Scrapping")
46 #Input from user
47 hastag=st.text_input("Please enter your Hastag/Keyword")
48 #starting date
49 Starting_date=st.date_input("Enter your starting date")
50 #ending date
51 Ending_date=st.date_input("Enter your ending date")
52 #limit
53 Limit=st.number_input("Enter the tweet limit")
54
55
56
57 # In[6]:
58
59
60 # Scraping the data and displaying it
61 if st.button("SCRAPE"):
62    Tweets = twitter_scraper(hastag, Starting_date, Ending_date, Limit)
63    st.dataframe(Tweets)
64
65
66 # In[7]:
67
68
69 #Uploading to MongoDB
70 if st.button("Store the scrapped_data in a data_base"):
71   client = MongoClient("mongodb://localhost:27017")
72   db = client["Project"]

localhost:4649/?mode=python 1/2
1/31/23, 12:44 PM Twitter_Scraping_Streamlit.py
73   new_collection = db["scrapped_data"]
74   import json
75   Tweets = twitter_scraper(hastag, Starting_date,Ending_date,Limit)
76   df=create_DataFrame(Tweets)
77   json_file = df.to_json()
78   file = json.loads(json_file)
79   for i in json.loads(json_file):
80     file = json.loads(json_file)
81     new_collection.insert_one(file)
82     st.success("upload Successfully")
83
84 # In[8]:
85
86
87 if st.button("Download as CSV"):
88    Tweets = twitter_scraper(hastag, Starting_date,Ending_date,Limit)
89    df=create_DataFrame(Tweets)
90    csv = df.to_csv()
91    st.download_button("Download data as CSV",
92                      csv,
93                      file_name='large_df.csv'
94                     )
95    
96
97
98 # In[9]:
99
100
101 if st.button("Download as JSON"):
102    Tweets =twitter_scraper(hastag, Starting_date,Ending_date,Limit)
103    df=create_DataFrame(Tweets)
104    csv = df.to_json()
105    st.download_button("Download data as json",
106                      csv,
107                      file_name='large_df.json'
108                     )
109    
110
111
112
113
114
115
116

localhost:4649/?mode=python 2/2