added a threshold to reduce the number of requests.

issues/issue-c8a150e2683f92b4a9caab10943885e09338d21a.yaml

@@ -8,8 +8,8 @@ type: :feature
 component: twitter
 release: 1.0.0
 reporter: Julio Biason <julio.biason@gmail.com>
-status: :unstarted
+status: :closed
-disposition: 
+disposition: :fixed
 creation_time: 2009-12-22 19:07:09.084361 Z
 references: []
 
@@ -21,3 +21,7 @@ log_events:
   - |-
     I think 5 would be a good default threshold. 10 may also be good (50% of the
     default page, but we never know.)
+- - 2009-12-26 11:36:39.492585 Z
+  - Julio Biason <julio.biason@gmail.com>
+  - closed with disposition fixed
+  - "1"

mitterlib/network/twitter.py

@@ -287,6 +287,16 @@ class Connection(NetworkBase):
                 option='secure_server_url',
                 default='https://api.twitter.com/1',
                 is_cmd_option=False)
+        options.add_option(
+                group=self.NAMESPACE,
+                option='page_size',
+                default=20,
+                is_cmd_option=False)
+        options.add_option(
+                group=self.NAMESPACE,
+                option='message_threshold',
+                default=4,
+                is_cmd_option=False)
         auth_options(self.NAMESPACE, options, self.AUTH)
         return
 
@@ -295,7 +305,7 @@ class Connection(NetworkBase):
         last_id = int(self._options[self.NAMESPACE][config_var])
         _log.debug('%s: %d', config_var, last_id)
 
-        params = {}
+        params = {'count': self._options[self.NAMESPACE]['page_size']}
 
         if last_id > 0:
             params['since_id'] = last_id
@@ -327,9 +337,17 @@ class Connection(NetworkBase):
                 if top_tweet_id > high_id:
                     high_id = top_tweet_id
 
-                result.extend(_make_datetime(response))
+                response_data = _make_datetime(response)
+                result.extend(response_data)
                 page += 1   # Request the next page
 
+                _log.debug('%d messages, %d threshold' % (len(response_data,
+                    self._options[self.NAMESPACE]['message_threshold'])))
+
+                if (len(response_data) <=
+                        self._options[self.NAMESPACE]['message_threshold']):
+                    break
+
             if last_id == 0:
                 # do not try to download everything if we don't have a
                 # previous list (or we'll blow the available requests in one