From 5b3fefdb53604721041700a2738de74f99a1e0c5 Mon Sep 17 00:00:00 2001 From: Julio Biason Date: Sat, 26 Dec 2009 09:37:00 -0200 Subject: [PATCH] added a threshold to reduce the number of requests. --- ...150e2683f92b4a9caab10943885e09338d21a.yaml | 8 +++++-- mitterlib/network/twitter.py | 22 +++++++++++++++++-- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/issues/issue-c8a150e2683f92b4a9caab10943885e09338d21a.yaml b/issues/issue-c8a150e2683f92b4a9caab10943885e09338d21a.yaml index ebdd420..4735a93 100644 --- a/issues/issue-c8a150e2683f92b4a9caab10943885e09338d21a.yaml +++ b/issues/issue-c8a150e2683f92b4a9caab10943885e09338d21a.yaml @@ -8,8 +8,8 @@ type: :feature component: twitter release: 1.0.0 reporter: Julio Biason -status: :unstarted -disposition: +status: :closed +disposition: :fixed creation_time: 2009-12-22 19:07:09.084361 Z references: [] @@ -21,3 +21,7 @@ log_events: - |- I think 5 would be a good default threshold. 10 may also be good (50% of the default page, but we never know.) +- - 2009-12-26 11:36:39.492585 Z + - Julio Biason + - closed with disposition fixed + - "1" diff --git a/mitterlib/network/twitter.py b/mitterlib/network/twitter.py index b80a014..20f89d8 100644 --- a/mitterlib/network/twitter.py +++ b/mitterlib/network/twitter.py @@ -287,6 +287,16 @@ class Connection(NetworkBase): option='secure_server_url', default='https://api.twitter.com/1', is_cmd_option=False) + options.add_option( + group=self.NAMESPACE, + option='page_size', + default=20, + is_cmd_option=False) + options.add_option( + group=self.NAMESPACE, + option='message_threshold', + default=4, + is_cmd_option=False) auth_options(self.NAMESPACE, options, self.AUTH) return @@ -295,7 +305,7 @@ class Connection(NetworkBase): last_id = int(self._options[self.NAMESPACE][config_var]) _log.debug('%s: %d', config_var, last_id) - params = {} + params = {'count': self._options[self.NAMESPACE]['page_size']} if last_id > 0: params['since_id'] = last_id @@ -327,9 +337,17 @@ class Connection(NetworkBase): if top_tweet_id > high_id: high_id = top_tweet_id - result.extend(_make_datetime(response)) + response_data = _make_datetime(response) + result.extend(response_data) page += 1 # Request the next page + _log.debug('%d messages, %d threshold' % (len(response_data, + self._options[self.NAMESPACE]['message_threshold']))) + + if (len(response_data) <= + self._options[self.NAMESPACE]['message_threshold']): + break + if last_id == 0: # do not try to download everything if we don't have a # previous list (or we'll blow the available requests in one