[WIP] Add "page_limit" argument to harvesters

README.md

@@ -214,6 +214,14 @@ Either --start or --end can also be used on their own. Not supplying arguments w
 
 If --end is given with no --start, start will default to the number of days specified in ```settings.DAYS_BACK``` before the given end date.
 
+You can also use the ```page_limit``` (or ```-p```)argument to limit your harvest to a certain number of pages. This is useful for large datasets and for testing locally.
+
+To only harvest 2 pages of data from MIT, run:
+
+```bash
+$ invoke harvester mit --page_limit 2
+```
+
 
 ### Automated OAI PMH Harvester Creation
 Writing a harvester for inclusion with scrAPI?  If the provider makes their metadata available using the OAI-PMH standard, then [autooai](https://github.com/erinspace/autooai) is a utility that will do most of the work for you.

scrapi/base/__init__.py

@@ -165,7 +165,7 @@ def resolve_property(self, dc, ns0):
         ret = dc + ns0
         return ret[0] if len(ret) == 1 else ret
 
-    def harvest(self, start_date=None, end_date=None):
+    def harvest(self, start_date=None, end_date=None, page_limit=None):
 
         start_date = (start_date or date.today() - timedelta(settings.DAYS_BACK)).isoformat()
         end_date = (end_date or date.today()).isoformat()
@@ -180,7 +180,7 @@ def harvest(self, start_date=None, end_date=None):
         url.args['from'] = start_date
         url.args['until'] = end_date
 
-        records = self.get_records(url.url, start_date, end_date)
+        records = self.get_records(url.url, start_date, end_date, page_limit)
 
         rawdoc_list = []
         for record in records:
@@ -196,17 +196,23 @@ def harvest(self, start_date=None, end_date=None):
 
         return rawdoc_list
 
-    def get_records(self, url, start_date, end_date):
+    def get_records(self, url, start_date, end_date, page_limit):
         url = furl(url)
         all_records, token = oai_get_records_and_token(url.url, self.timeout, self.force_request_update, self.namespaces, self.verify)
 
+        pages_harvested = 1
         while token:
-            url.remove('from')
-            url.remove('until')
-            url.remove('metadataPrefix')
-            url.args['resumptionToken'] = token[0]
-            records, token = oai_get_records_and_token(url.url, self.timeout, self.force_request_update, self.namespaces, self.verify)
-            all_records += records
+            print("Page limit is {} and pagees harvested is {}".format(page_limit, pages_harvested))
+            if page_limit and int(page_limit) == int(pages_harvested):
+                break
+            else:
+                url.remove('from')
+                url.remove('until')
+                url.remove('metadataPrefix')
+                url.args['resumptionToken'] = token[0]
+                records, token = oai_get_records_and_token(url.url, self.timeout, self.force_request_update, self.namespaces, self.verify)
+                all_records += records
+                pages_harvested += 1
 
         return all_records
 

scrapi/harvesters/biomedcentral.py

@@ -87,7 +87,7 @@ def schema(self):
             )
         }
 
-    def harvest(self, start_date=None, end_date=None):
+    def harvest(self, start_date=None, end_date=None, page_limit=None):
 
         start_date = start_date or date.today() - timedelta(settings.DAYS_BACK)
 
@@ -96,7 +96,7 @@ def harvest(self, start_date=None, end_date=None):
         date_number = end_date - start_date
 
         search_url = self.URL.format(date_number.days)
-        records = self.get_records(search_url)
+        records = self.get_records(search_url, page_limit)
 
         record_list = []
         for record in records:
@@ -115,7 +115,7 @@ def harvest(self, start_date=None, end_date=None):
 
         return record_list
 
-    def get_records(self, search_url):
+    def get_records(self, search_url, page_limit):
         now = datetime.now()
         records = requests.get(search_url + "#{}".format(date.today()))
         page = 1
@@ -132,8 +132,11 @@ def get_records(self, search_url):
                     continue
                 all_records.append(record)
 
-            page += 1
-            records = requests.get(search_url + '&page={}#{}'.format(str(page), date.today()), throttle=10)
-            current_records = len(records.json()['entries'])
+            if page_limit and int(page_limit) == page:
+                break
+            else:
+                page += 1
+                records = requests.get(search_url + '&page={}#{}'.format(str(page), date.today()), throttle=10)
+                current_records = len(records.json()['entries'])
 
         return all_records

scrapi/harvesters/clinicaltrials.py

@@ -104,7 +104,7 @@ class ClinicalTrialsHarvester(XMLHarvester):
     def namespaces(self):
         return None
 
-    def harvest(self, start_date=None, end_date=None):
+    def harvest(self, start_date=None, end_date=None, page_limit=None):
         """ First, get a list of all recently updated study urls,
         then get the xml one by one and save it into a list
         of docs including other information """
@@ -166,6 +166,8 @@ def harvest(self, start_date=None, end_date=None):
                 official_count += 1
                 count += 1
                 if count % 100 == 0:
+                    if page_limit and page_limit == official_count / 100:
+                        break
                     logger.info("You've requested {} studies, keep going!".format(official_count))
                     count = 0
 

scrapi/harvesters/crossref.py

@@ -106,7 +106,7 @@ def schema(self):
             )
         }
 
-    def harvest(self, start_date=None, end_date=None):
+    def harvest(self, start_date=None, end_date=None, page_limit=None):
         start_date = start_date or date.today() - timedelta(settings.DAYS_BACK)
         end_date = end_date or date.today()
 
@@ -128,4 +128,7 @@ def harvest(self, start_date=None, end_date=None):
                     'filetype': 'json'
                 }))
 
+            if page_limit and int(page_limit) == i / 1000:
+                break
+
         return doc_list

scrapi/harvesters/dailyssrn.py

@@ -27,7 +27,7 @@ class DailyssrnHarvester(XMLHarvester):
         }
     }
 
-    def harvest(self, start_date=None, end_date=None):
+    def harvest(self, start_date=None, end_date=None, page_limit=None):
 
         url = 'http://dailyssrn.com/rss/rss-all-2.0.xml'
 

scrapi/harvesters/dataone.py

@@ -139,12 +139,12 @@ class DataOneHarvester(XMLHarvester):
         'description': ("str[@name='abstract']/node()", single_result)
     }
 
-    def harvest(self, start_date=None, end_date=None):
+    def harvest(self, start_date=None, end_date=None, page_limit=None):
 
         start_date = start_date or date.today() - timedelta(settings.DAYS_BACK)
         end_date = end_date or date.today()
 
-        records = self.get_records(start_date, end_date)
+        records = self.get_records(start_date, end_date, page_limit)
 
         xml_list = []
         for record in records:
@@ -159,7 +159,7 @@ def harvest(self, start_date=None, end_date=None):
 
         return xml_list
 
-    def get_records(self, start_date, end_date):
+    def get_records(self, start_date, end_date, page_limit):
         ''' helper function to get a response from the DataONE
         API, with the specified number of rows.
         Returns an etree element with results '''
@@ -183,4 +183,8 @@ def get_records(self, start_date, end_date):
             docs = etree.XML(data.content).xpath('//doc')
             for doc in docs:
                 yield doc
-            n += 1000
+
+            if page_limit and int(page_limit) == n / 1000:
+                break
+            else:
+                n += 1000

scrapi/harvesters/doepages.py

@@ -25,7 +25,7 @@ class DoepagesHarvester(XMLHarvester):
         'dcq': 'http://purl.org/dc/terms/'
     }
 
-    def harvest(self, start_date=None, end_date=None):
+    def harvest(self, start_date=None, end_date=None, page_limit=None):
 
         start_date = start_date or date.today() - timedelta(settings.DAYS_BACK)
         end_date = end_date or date.today()

scrapi/harvesters/figshare.py

@@ -50,7 +50,7 @@ class FigshareHarvester(JSONHarvester):
         )
     }
 
-    def harvest(self, start_date=None, end_date=None):
+    def harvest(self, start_date=None, end_date=None, page_limit=None):
         """ Figshare should always have a 24 hour delay because they
         manually go through and check for test projects. Most of them
         are removed within 24 hours.
@@ -67,7 +67,7 @@ def harvest(self, start_date=None, end_date=None):
             end_date.isoformat()
         )
 
-        records = self.get_records(search_url)
+        records = self.get_records(search_url, page_limit)
 
         record_list = []
         for record in records:
@@ -86,7 +86,7 @@ def harvest(self, start_date=None, end_date=None):
 
         return record_list
 
-    def get_records(self, search_url):
+    def get_records(self, search_url, page_limit):
         records = requests.get(search_url)
         total_records = records.json()['items_found']
         page = 1
@@ -99,7 +99,10 @@ def get_records(self, search_url):
                 if len(all_records) < total_records:
                     all_records.append(record)
 
-            page += 1
-            records = requests.get(search_url + '&page={}'.format(str(page)), throttle=3)
+            if page_limit and int(page_limit) == page:
+                break
+            else:
+                page += 1
+                records = requests.get(search_url + '&page={}'.format(str(page)), throttle=3)
 
         return all_records

scrapi/harvesters/harvarddataverse.py

@@ -57,7 +57,7 @@ class HarvardDataverseHarvester(JSONHarvester):
         )
     }
 
-    def harvest(self, start_date=None, end_date=None):
+    def harvest(self, start_date=None, end_date=None, page_limit=None):
         start_date = (start_date or date.today() - timedelta(settings.DAYS_BACK)).isoformat()
         end_date = (end_date or date.today()).isoformat()
 
@@ -69,7 +69,7 @@ def harvest(self, start_date=None, end_date=None):
         query.args['order'] = 'asc'
         query.args['fq'] = 'dateSort:[{}T00:00:00Z TO {}T00:00:00Z]'.format(start_date, end_date)
 
-        records = self.get_records(query.url)
+        records = self.get_records(query.url, page_limit)
         record_list = []
         for record in records:
             doc_id = record['global_id']
@@ -87,7 +87,7 @@ def harvest(self, start_date=None, end_date=None):
 
         return record_list
 
-    def get_records(self, search_url):
+    def get_records(self, search_url, page_limit):
         records = requests.get(search_url)
         total_records = records.json()['data']['total_count']
         start = 0
@@ -100,6 +100,9 @@ def get_records(self, search_url):
             for record in record_list:
                 all_records.append(record)
 
-            start += self.MAX_ITEMS_PER_REQUEST
+            if page_limit and int(page_limit) == start / self.MAX_ITEMS_PER_REQUEST:
+                break
+            else:
+                start += self.MAX_ITEMS_PER_REQUEST
 
         return all_records

scrapi/harvesters/osf.py

@@ -73,14 +73,14 @@ def schema(self):
             )
         }
 
-    def harvest(self, start_date=None, end_date=None):
+    def harvest(self, start_date=None, end_date=None, page_limit=None):
         # Always harvest a 2 day period starting 2 days back to honor time given
         # to contributors to cancel a public registration
         start_date = start_date or date.today() - timedelta(4)
         end_date = end_date or date.today() - timedelta(2)
 
         search_url = self.URL.format(start_date.isoformat(), end_date.isoformat())
-        records = self.get_records(search_url)
+        records = self.get_records(search_url, page_limit)
 
         record_list = []
         for record in records:
@@ -99,7 +99,7 @@ def harvest(self, start_date=None, end_date=None):
 
         return record_list
 
-    def get_records(self, search_url):
+    def get_records(self, search_url, page_limit):
         records = requests.get(search_url)
 
         total = int(records.json()['counts']['registration'])
@@ -113,6 +113,10 @@ def get_records(self, search_url):
                 all_records.append(record)
 
             from_arg += 1000
-            records = requests.get(search_url + '&from={}'.format(str(from_arg)), throttle=10)
+
+            if page_limit and int(page_limit) == from_arg / 1000:
+                break
+            else:
+                records = requests.get(search_url + '&from={}'.format(str(from_arg)), throttle=10)
 
         return all_records

scrapi/harvesters/plos.py

@@ -48,7 +48,7 @@ class PlosHarvester(XMLHarvester):
     MAX_ROWS_PER_REQUEST = 999
     BASE_URL = 'http://api.plos.org/search'
 
-    def fetch_rows(self, start_date, end_date):
+    def fetch_rows(self, start_date, end_date, page_limit):
         query = 'publication_date:[{}T00:00:00Z TO {}T00:00:00Z]'.format(start_date, end_date)
 
         resp = requests.get(self.BASE_URL, params={
@@ -72,9 +72,12 @@ def fetch_rows(self, start_date, end_date):
             for doc in etree.XML(response.content).xpath('//doc'):
                 yield doc
 
-            current_row += self.MAX_ROWS_PER_REQUEST
+            if page_limit and int(page_limit) == current_row / self.MAX_ROWS_PER_REQUEST:
+                break
+            else:
+                current_row += self.MAX_ROWS_PER_REQUEST
 
-    def harvest(self, start_date=None, end_date=None):
+    def harvest(self, start_date=None, end_date=None, page_limit=None):
 
         start_date = start_date or date.today() - timedelta(settings.DAYS_BACK)
         end_date = end_date or date.today()
@@ -90,7 +93,7 @@ def harvest(self, start_date=None, end_date=None):
                 'docID': row.xpath("str[@name='id']")[0].text,
             })
             for row in
-            self.fetch_rows(start_date.isoformat(), end_date.isoformat())
+            self.fetch_rows(start_date.isoformat(), end_date.isoformat(), page_limit)
             if row.xpath("arr[@name='abstract']")
             or row.xpath("str[@name='author_display']")
         ]

scrapi/harvesters/scitech.py

@@ -41,7 +41,7 @@ class SciTechHarvester(XMLHarvester):
 
     schema = DOESCHEMA
 
-    def harvest(self, start_date=None, end_date=None):
+    def harvest(self, start_date=None, end_date=None, page_limit=None):
         """A function for querying the SciTech Connect database for raw XML.
         The XML is chunked into smaller pieces, each representing data
         about an article/report. If there are multiple pages of results,
@@ -54,10 +54,10 @@ def harvest(self, start_date=None, end_date=None):
                 'doc': etree.tostring(record),
                 'docID': six.u(record.xpath('dc:ostiId/node()', namespaces=self.namespaces)[0]),
             })
-            for record in self._fetch_records(start_date, end_date)
+            for record in self._fetch_records(start_date, end_date, page_limit)
         ]
 
-    def _fetch_records(self, start_date, end_date):
+    def _fetch_records(self, start_date, end_date, page_limit):
         page = 0
         morepages = True
 
@@ -76,5 +76,8 @@ def _fetch_records(self, start_date, end_date):
             for record in xml.xpath('records/record'):
                 yield record
 
-            page += 1
-            morepages = xml.xpath('//records/@morepages')[0] == 'true'
+            if page_limit and int(page_limit) == page:
+                break
+            else:
+                page += 1
+                morepages = xml.xpath('//records/@morepages')[0] == 'true'