Merge from 'Patty-OFurniture:icrawler_master_patches'

Signed-off-by: Zhiyuan Chen <[email protected]>

Author: ZhiyuanChen

icrawler/builtin/urllist.py

@@ -10,6 +10,11 @@ def worker_exec(self, queue_timeout=2, **kwargs):
             if self.signal.get("reach_max_num"):
                 self.logger.info("downloaded image reached max num, thread %s" " exit", threading.current_thread().name)
                 break
+            if self.signal.get("exceed_storage_space"):
+                self.logger.info(
+                    "downloaded image reached max storage space, thread %s" " exit", threading.current_thread().name
+                )
+                break
             try:
                 url = self.in_queue.get(timeout=queue_timeout)
             except queue.Empty:

icrawler/crawler.py

@@ -82,11 +82,11 @@ def set_logger(self, log_level=logging.INFO):
     def init_signal(self):
         """Init signal
 
-        3 signals are added: ``feeder_exited``, ``parser_exited`` and
-        ``reach_max_num``.
+        4 signals are added: ``feeder_exited``, ``parser_exited``,
+        ``reach_max_num`` and ``exceed_storage_space``.
         """
         self.signal = Signal()
-        self.signal.set(feeder_exited=False, parser_exited=False, reach_max_num=False)
+        self.signal.set(feeder_exited=False, parser_exited=False, reach_max_num=False, exceed_storage_space=False)
 
     def set_storage(self, storage):
         """Set storage backend for downloader

icrawler/downloader.py

@@ -1,3 +1,4 @@
+import errno
 import queue
 import time
 from io import BytesIO
@@ -114,7 +115,7 @@ def download(self, task, default_ext, timeout=5, max_retry=3, overwrite=False, *
                     return
                 self.fetched_num -= 1
 
-        while retry > 0 and not self.signal.get("reach_max_num"):
+        while retry > 0 and not self.signal.get("reach_max_num") and not self.signal.get("exceed_storage_space"):
             try:
                 response = self.session.get(file_url, timeout=timeout)
             except Exception as e:
@@ -136,10 +137,19 @@ def download(self, task, default_ext, timeout=5, max_retry=3, overwrite=False, *
                 with self.lock:
                     self.fetched_num += 1
                     filename = self.get_filename(task, default_ext)
-                self.logger.info("image #%s\t%s", self.fetched_num, file_url)
-                self.storage.write(filename, response.content)
-                task["success"] = True
-                task["filename"] = filename
+                self.logger.info("image #%s\t%s %s", self.fetched_num, filename, file_url)
+
+                task["success"] = False
+                try:
+                    task["filename"] = filename  # may be zero bytes if OSError happened during write()
+                    self.storage.write(filename, response.content)
+                    task["success"] = True
+                except OSError as o:
+                    # errno.EINVAL -- name too long
+                    if o.errno == errno.ENOSPC:
+                        self.signal.set(exceed_storage_space=True)
+                    else:
+                        raise
                 break
             finally:
                 retry -= 1

icrawler/parser.py

@@ -61,6 +61,9 @@ def worker_exec(self, queue_timeout=2, req_timeout=5, max_retry=3, **kwargs):
                     "downloaded image reached max num, thread %s " "is ready to exit", current_thread().name
                 )
                 break
+            if self.signal.get("exceed_storage_space"):
+                self.logger.info("no more storage space, thread %s " "is ready to exit", current_thread().name)
+                break
             # get the page url
             try:
                 url = self.in_queue.get(timeout=queue_timeout)
@@ -90,8 +93,14 @@ def worker_exec(self, queue_timeout=2, req_timeout=5, max_retry=3, **kwargs):
                     )
                 else:
                     self.logger.info(f"parsing result page {url}")
-                    for task in self.parse(response, **kwargs):
-                        while not self.signal.get("reach_max_num"):
+                    task_list = self.parse(response, **kwargs)
+                    if not task_list:
+                        self.logger.debug("self.parse() returned no tasks")
+                        with open("task_list_error.log", "ab") as f:
+                            f.write(response.content)
+
+                    for task in task_list:
+                        while not self.signal.get("reach_max_num") and not self.signal.get("exceed_storage_space"):
                             try:
                                 if isinstance(task, dict):
                                     self.output(task, timeout=1)
@@ -110,6 +119,8 @@ def worker_exec(self, queue_timeout=2, req_timeout=5, max_retry=3, **kwargs):
                                 break
                         if self.signal.get("reach_max_num"):
                             break
+                        if self.signal.get("exceed_storage_space"):
+                            break
                     self.in_queue.task_done()
                     break
                 finally: