diff --git a/swvo/io/dst/wdc.py b/swvo/io/dst/wdc.py index 6e32bbd..b1b7a62 100644 --- a/swvo/io/dst/wdc.py +++ b/swvo/io/dst/wdc.py @@ -88,11 +88,8 @@ def download_and_process(self, start_time: datetime, end_time: datetime, reproce URL = self.URL.replace("YYYYMM", time_interval.strftime("%Y%m")) - if file_path.exists(): - if reprocess_files: - file_path.unlink() - else: - continue + if file_path.exists() and not reprocess_files: + continue logging.debug(f"Downloading file {URL + filename} ...") @@ -192,8 +189,6 @@ def _process_single_file(self, file_path: Path, year, month) -> pd.DataFrame: df.index = df["timestamp"] df.drop(columns=["timestamp"], inplace=True) - file_path.unlink() - return df def read(self, start_time: datetime, end_time: datetime, download: bool = False) -> pd.DataFrame: diff --git a/swvo/io/hp/gfz.py b/swvo/io/hp/gfz.py index 377e210..2734442 100755 --- a/swvo/io/hp/gfz.py +++ b/swvo/io/hp/gfz.py @@ -103,6 +103,9 @@ def download_and_process( f"Hp{self.index_number}/Hp{self.index_number}_ap{self.index_number}_nowcast.txt" ) + if file_path.exists() and not reprocess_files: + continue + for filename_download in filenames_download: logging.debug(f"Downloading file {self.URL + filename_download} ...") @@ -110,12 +113,6 @@ def download_and_process( logging.debug("Processing file ...") - if file_path.exists(): - if reprocess_files: - file_path.unlink() - else: - continue - filenames_download = [x[5:] for x in filenames_download] # strip of folder of filename processed_df = self._process_single_file(temporary_dir, filenames_download) diff --git a/swvo/io/kp/niemegk.py b/swvo/io/kp/niemegk.py index 731030b..c9b3772 100755 --- a/swvo/io/kp/niemegk.py +++ b/swvo/io/kp/niemegk.py @@ -99,11 +99,8 @@ def download_and_process(self, start_time: datetime, end_time: datetime, reproce file_paths, time_intervals = self._get_processed_file_list(start_time, end_time) for file_path, time_interval in zip(file_paths, time_intervals): - if file_path.exists(): - if reprocess_files: - file_path.unlink() - else: - continue + if file_path.exists() and not reprocess_files: + continue data_single_file = processed_df[ (processed_df.index >= time_interval[0]) & (processed_df.index <= time_interval[1]) diff --git a/swvo/io/kp/swpc.py b/swvo/io/kp/swpc.py index aa5a947..096534b 100755 --- a/swvo/io/kp/swpc.py +++ b/swvo/io/kp/swpc.py @@ -83,11 +83,8 @@ def download_and_process(self, target_date: datetime, reprocess_files: bool = Fa file_path = self.data_dir / f"SWPC_KP_FORECAST_{target_date.strftime('%Y%m%d')}.csv" - if file_path.exists(): - if reprocess_files: - file_path.unlink() - else: - return + if file_path.exists() and not reprocess_files: + return temporary_dir = Path("./temp_kp_swpc_wget") temporary_dir.mkdir(exist_ok=True, parents=True) diff --git a/swvo/io/omni/omni_high_res.py b/swvo/io/omni/omni_high_res.py index af836a7..3512f79 100644 --- a/swvo/io/omni/omni_high_res.py +++ b/swvo/io/omni/omni_high_res.py @@ -98,11 +98,8 @@ def download_and_process( file_paths, time_intervals = self._get_processed_file_list(start_time, end_time, cadence_min) for file_path, time_interval in zip(file_paths, time_intervals): - if file_path.exists(): - if reprocess_files: - file_path.unlink() - else: - continue + if file_path.exists() and not reprocess_files: + continue data = self._get_data_from_omni( start=time_interval[0], diff --git a/swvo/io/omni/omni_low_res.py b/swvo/io/omni/omni_low_res.py index 2dc0db2..a4a6cb5 100755 --- a/swvo/io/omni/omni_low_res.py +++ b/swvo/io/omni/omni_low_res.py @@ -141,11 +141,8 @@ def download_and_process(self, start_time: datetime, end_time: datetime, reproce for file_path, time_interval in zip(file_paths, time_intervals): filename = "omni2_" + str(time_interval[0].year) + ".dat" - if file_path.exists(): - if reprocess_files: - file_path.unlink() - else: - continue + if file_path.exists() and not reprocess_files: + continue logging.debug(f"Downloading file {self.URL + filename} ...")