diff --git a/requirements.txt b/requirements.txt index 10aecb66..12bb76b1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -aiohttp +aiohttp==3.7.0 aiodns beautifulsoup4 cchardet diff --git a/twint/run.py b/twint/run.py index f27647c9..e8f91573 100644 --- a/twint/run.py +++ b/twint/run.py @@ -1,5 +1,6 @@ import sys, os, datetime from asyncio import get_event_loop, TimeoutError, ensure_future, new_event_loop, set_event_loop +from tqdm import tqdm from . import datelock, feed, get, output, verbose, storage from .token import TokenExpiryException @@ -29,6 +30,7 @@ def __init__(self, config): self.count = 0 self.user_agent = "" self.config = config + self.progress_bar = tqdm(total=int(config.Limit), desc="Tweets") self.config.Bearer_token = bearer # TODO might have to make some adjustments for it to work with multi-treading # USAGE : to get a new guest token simply do `self.token.refresh()` @@ -46,6 +48,9 @@ def __init__(self, config): logme.debug(__name__ + ':Twint:__init__:pandas_clean') storage.panda.clean() + def __del__(self): + self.progress_bar.close() + def get_resume(self, resumeFile): if not os.path.exists(resumeFile): return '-1' @@ -149,11 +154,14 @@ async def follow(self): await self.Feed() if self.config.User_full: logme.debug(__name__ + ':Twint:follow:userFull') - self.count += await get.Multi(self.feed, self.config, self.conn) + foo = await get.Multi(self.feed, self.config, self.conn) + self.count += foo + self.progress_bar.update(foo) else: logme.debug(__name__ + ':Twint:follow:notUserFull') for user in self.feed: self.count += 1 + self.progress_bar.update(1) username = user.find("a")["name"] await output.Username(username, self.config, self.conn) @@ -164,6 +172,7 @@ async def favorite(self): for tweet in self.feed: tweet_dict = {} self.count += 1 + self.progress_bar.update(1) try: tweet_dict['data-item-id'] = tweet.find("div", {"class": "tweet-text"})['data-id'] t_url = tweet.find("span", {"class": "metadata"}).find("a")["href"] @@ -211,6 +220,7 @@ async def profile(self): logme.debug(__name__ + ':Twint:profile') for tweet in self.feed: self.count += 1 + self.progress_bar.update(1) await output.Tweets(tweet, self.config, self.conn) async def tweets(self): @@ -218,11 +228,14 @@ async def tweets(self): # TODO : need to take care of this later if self.config.Location: logme.debug(__name__ + ':Twint:tweets:location') - self.count += await get.Multi(self.feed, self.config, self.conn) + foo = await get.Multi(self.feed, self.config, self.conn) + self.count += foo + self.progress_bar.update(foo) else: logme.debug(__name__ + ':Twint:tweets:notLocation') for tweet in self.feed: self.count += 1 + self.progress_bar.update(1) await output.Tweets(tweet, self.config, self.conn) async def main(self, callback=None): diff --git a/twint/url.py b/twint/url.py index 97923cef..3dcc268b 100644 --- a/twint/url.py +++ b/twint/url.py @@ -67,7 +67,7 @@ async def MobileProfile(username, init): async def Search(config, init): logme.debug(__name__ + ':Search') url = base - tweet_count = 100 + tweet_count = 100 if not config.Limit else config.Limit q = "" params = [ # ('include_blocking', '1'), @@ -89,7 +89,7 @@ async def Search(config, init): ('send_error_codes', 'true'), ('simple_quoted_tweet', 'true'), ('count', tweet_count), - # ('query_source', 'typed_query'), + ('query_source', 'typed_query'), # ('pc', '1'), ('cursor', str(init)), ('spelling_corrections', '1'),