From 14e838adf252c7ab0b50f30d91218a0577fa965c Mon Sep 17 00:00:00 2001 From: Peter Lamberg Date: Mon, 15 Jun 2015 08:19:08 +0300 Subject: [PATCH 01/13] Add documentation and symbol for new planned conveyor strategy. --- bin/podcatcher | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/bin/podcatcher b/bin/podcatcher index 38efb0b..9482708 100755 --- a/bin/podcatcher +++ b/bin/podcatcher @@ -126,7 +126,7 @@ opt.memsize = 1_000 opt.empty = false opt.simulate = false opt.verbose = false -opt.STRATEGIES = [:one, :new, :back_catalog, :all, :chron, :chron_one, :chron_all, :cache] +opt.STRATEGIES = [:one, :new, :back_catalog, :conveyor, :all, :chron, :chron_one, :chron_all, :cache] opt.strategy = opt.STRATEGIES[0] opt.retries = 1 opt.torrent_dir = nil @@ -266,6 +266,12 @@ option_parser = OptionParser.new() do |c| " recent content to older content (may ", " download more than one content file per", " feed),", + "* conveyor: like back_catalog, but downloads", + " only if there are less than feeds parameter", + " content files in the cache per feed.", + " Never deletes files from cache but stops ", + " if cache grows to size specified by size ", + " parameter.", "* one: download one content file (not ", " already downloaded) for each feed, with a ", " preference for recent content,", From e72f15db46c3b31fdb9bad52d0df8e788f6378bd Mon Sep 17 00:00:00 2001 From: Peter Lamberg Date: Mon, 15 Jun 2015 13:35:43 +0300 Subject: [PATCH 02/13] Special handling for deletion with conveyor strategy. --- bin/podcatcher | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/bin/podcatcher b/bin/podcatcher index 9482708..de43726 100755 --- a/bin/podcatcher +++ b/bin/podcatcher @@ -1718,7 +1718,7 @@ class Cache feed.compact! end end - if @opt.strategy == :new or @opt.strategy == :one + if @opt.strategy == :new or @opt.strategy == :one or @opt.strategy == :conveyor feeds.each() do |feed| itemsize = 0 index = nil @@ -1855,6 +1855,11 @@ class Cache itemsize = 0 next end + #special handling for cache full for conveyor strategy + if @opt.strategy == :conveyor and @opt.size and content.size+inc+total > @opt.size + $stderr.puts "Cache has reached maxumum size." if @opt.verbose + break + end #make place in cache while @opt.size and content.size+inc+total > @opt.size break if @opt.simulate From 7fd57fe8d6d8d6f73a788563c4098067274c2092 Mon Sep 17 00:00:00 2001 From: Peter Lamberg Date: Wed, 17 Jun 2015 20:31:15 +0300 Subject: [PATCH 03/13] Add so far unused --files N option. Extract method for mangling file names to be compatible with certain file systems, New method is called restrictName. Fix small typo in a conveyor strategy related log message.j --- bin/podcatcher | 32 ++++++++++++++++++++++++++------ 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/bin/podcatcher b/bin/podcatcher index de43726..a283b98 100755 --- a/bin/podcatcher +++ b/bin/podcatcher @@ -134,6 +134,7 @@ opt.rubytorrent = false opt.upload_rate = nil #10 opt.itemsize = 0 opt.feeds = 1_000 +opt.files = 1_000 opt.FUNCTIONS = [:download, :search] opt.function = opt.FUNCTIONS[0] opt.per_feed = false @@ -267,7 +268,7 @@ option_parser = OptionParser.new() do |c| " download more than one content file per", " feed),", "* conveyor: like back_catalog, but downloads", - " only if there are less than feeds parameter", + " only if there are less than files parameter", " content files in the cache per feed.", " Never deletes files from cache but stops ", " if cache grows to size specified by size ", @@ -404,7 +405,15 @@ option_parser = OptionParser.new() do |c| opt.feeds = e.to_i opt.feeds = nil if opt.feeds<1 end - c.on("-T", "--torrentdir DIR", + c.on("-n", "--files N", + "Do not download more than N content ", + "files per feed. Only works with conveyor", + "strategy. 0 means unbounded.", + "Default value is #{opt.files}.\n") do |e| + opt.files = e.to_i + opt.files = nil if opt.files<1 + end + c.on("-T", "--torrentdir DIR", "Copy torrent files to directory DIR.", "The handling of torrents through an", "external BitTorrent client is left to", @@ -648,6 +657,11 @@ option_parser = OptionParser.new() do |c| opt.feeds = value opt.feeds = nil if opt.feeds<1 end + when 'files' + if value.instance_of?(Fixnum) + opt.files = value + opt.files = nil if opt.files<1 + end when 'horizon' begin date = value.split '.' @@ -1857,7 +1871,7 @@ class Cache end #special handling for cache full for conveyor strategy if @opt.strategy == :conveyor and @opt.size and content.size+inc+total > @opt.size - $stderr.puts "Cache has reached maxumum size." if @opt.verbose + $stderr.puts "Cache has reached maximum size." if @opt.verbose break end #make place in cache @@ -2085,12 +2099,18 @@ private res = nil end res - end + end + def restrictName(dir) + if @opt.restricted_names + return dir.sub(/[\\\/:*?\"<>|!]/, ' ').gsub(/-+/,'-').gsub(/\s+/,' ').strip + else + return dir + end + end def filename(content, dir) #produce filename for content to be downloaded begin #per-feed subfolder if @opt.per_feed and content.feed_title and content.feed_title.size > 0 - newdir = dir+content.feed_title - newdir = dir+content.feed_title.gsub(/[\\\/:*?\"<>|!]/, ' ').gsub(/-+/,'-').gsub(/\s+/,' ').strip if @opt.restricted_names + newdir = restrictName(dir+content.feed_title) if newdir.exist? if newdir.directory? dir = newdir From f71f781f1ec2dfe36023ce0c93026225dcce0496 Mon Sep 17 00:00:00 2001 From: Peter Lamberg Date: Fri, 19 Jun 2015 23:09:50 +0300 Subject: [PATCH 04/13] Fix bug in applying restrictName method. --- bin/podcatcher | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/podcatcher b/bin/podcatcher index a283b98..5bcb06f 100755 --- a/bin/podcatcher +++ b/bin/podcatcher @@ -2110,7 +2110,7 @@ private def filename(content, dir) #produce filename for content to be downloaded begin #per-feed subfolder if @opt.per_feed and content.feed_title and content.feed_title.size > 0 - newdir = restrictName(dir+content.feed_title) + newdir = dir+restrictName(content.feed_title) if newdir.exist? if newdir.directory? dir = newdir From 6854458fb419f90d56ff7ddfff71e5e76339b576 Mon Sep 17 00:00:00 2001 From: Peter Lamberg Date: Fri, 19 Jun 2015 23:09:11 +0300 Subject: [PATCH 05/13] First working implementation of conveyor strategy. --- bin/podcatcher | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/bin/podcatcher b/bin/podcatcher index 5bcb06f..ad643ab 100755 --- a/bin/podcatcher +++ b/bin/podcatcher @@ -1732,6 +1732,27 @@ class Cache feed.compact! end end + if @opt.strategy == :conveyor + feeds.each() do |feed| + existing_files = 0 + if feed.size > 0 && @opt.files && @opt.files > 0 + feeddir = restrictName(feed[0].feed_title) + @cache.each() do |e| + dir = File.split(File.split(e.file)[0])[1] + print "'#{dir}' == '#{feeddir}' #{existing_files}\n" + if dir == feeddir + existing_files += 1 + end + end + files_needed = @opt.files - existing_files + if files_needed < 0 + feed.clear + else + feed.slice! files_needed...feed.size + end + end + end + end if @opt.strategy == :new or @opt.strategy == :one or @opt.strategy == :conveyor feeds.each() do |feed| itemsize = 0 From f5cddbaa6633c9a0f10f2987b41d83b9ad833086 Mon Sep 17 00:00:00 2001 From: Peter Lamberg Date: Sat, 20 Jun 2015 08:24:28 +0300 Subject: [PATCH 06/13] Add support for --itemsize to conveyor strategy. Remove debug print. --- bin/podcatcher | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/bin/podcatcher b/bin/podcatcher index ad643ab..43624a1 100755 --- a/bin/podcatcher +++ b/bin/podcatcher @@ -1739,7 +1739,6 @@ class Cache feeddir = restrictName(feed[0].feed_title) @cache.each() do |e| dir = File.split(File.split(e.file)[0])[1] - print "'#{dir}' == '#{feeddir}' #{existing_files}\n" if dir == feeddir existing_files += 1 end @@ -1748,6 +1747,18 @@ class Cache if files_needed < 0 feed.clear else + if @opt.itemsize && @opt.itemsize > 0 + itemsize = 0 + for i in 0...feed.size + itemsize += feed[i].size + if i+1 > files_needed + files_needed = i+1 + end + if itemsize >= @opt.itemsize + break + end + end + end feed.slice! files_needed...feed.size end end From ac5f03de03bfc5816afd18b79215dc80c4036979 Mon Sep 17 00:00:00 2001 From: Peter Lamberg Date: Sat, 20 Jun 2015 08:25:48 +0300 Subject: [PATCH 07/13] Doc wording on --files. --- bin/podcatcher | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/podcatcher b/bin/podcatcher index 43624a1..aa7fb2b 100755 --- a/bin/podcatcher +++ b/bin/podcatcher @@ -407,8 +407,8 @@ option_parser = OptionParser.new() do |c| end c.on("-n", "--files N", "Do not download more than N content ", - "files per feed. Only works with conveyor", - "strategy. 0 means unbounded.", + "files per feed. This parameter works only", + "with conveyor strategy. 0 means unbounded.", "Default value is #{opt.files}.\n") do |e| opt.files = e.to_i opt.files = nil if opt.files<1 From 7318c4a3e572f408c52012c7603c06252899d211 Mon Sep 17 00:00:00 2001 From: Peter Lamberg Date: Sat, 20 Jun 2015 08:38:39 +0300 Subject: [PATCH 08/13] Force --perfeed to true when conveyor stragey is used. --- bin/podcatcher | 1 + 1 file changed, 1 insertion(+) diff --git a/bin/podcatcher b/bin/podcatcher index aa7fb2b..ed0b76c 100755 --- a/bin/podcatcher +++ b/bin/podcatcher @@ -1733,6 +1733,7 @@ class Cache end end if @opt.strategy == :conveyor + @opt.per_feed = true feeds.each() do |feed| existing_files = 0 if feed.size > 0 && @opt.files && @opt.files > 0 From 61803898a113dffe2d14b6f17c94a69ef250d836 Mon Sep 17 00:00:00 2001 From: Peter Lamberg Date: Sat, 20 Jun 2015 08:36:08 +0300 Subject: [PATCH 09/13] Doc improvement on conveyor strategy. --- bin/podcatcher | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/bin/podcatcher b/bin/podcatcher index ed0b76c..793beeb 100755 --- a/bin/podcatcher +++ b/bin/podcatcher @@ -268,11 +268,12 @@ option_parser = OptionParser.new() do |c| " download more than one content file per", " feed),", "* conveyor: like back_catalog, but downloads", - " only if there are less than files parameter", - " content files in the cache per feed.", - " Never deletes files from cache but stops ", - " if cache grows to size specified by size ", - " parameter.", + " only if feed has less than files parameter", + " content files in the cache.", + " Selecting conveyor strategy forces perfeed", + " mode on. Files are never deleted from", + " cache; conveyor strategy simply stops if", + " the maximum cache size is reached.", "* one: download one content file (not ", " already downloaded) for each feed, with a ", " preference for recent content,", From 2884497db6771df5cbb506171bff3d780b4bd164 Mon Sep 17 00:00:00 2001 From: Peter Lamberg Date: Sat, 20 Jun 2015 14:11:37 +0300 Subject: [PATCH 10/13] Fix bug introduced in commit "Special handling for deletion with conveyor strategy." --- bin/podcatcher | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/podcatcher b/bin/podcatcher index 793beeb..a377075 100755 --- a/bin/podcatcher +++ b/bin/podcatcher @@ -1766,7 +1766,7 @@ class Cache end end end - if @opt.strategy == :new or @opt.strategy == :one or @opt.strategy == :conveyor + if @opt.strategy == :new or @opt.strategy == :one feeds.each() do |feed| itemsize = 0 index = nil From f9f1bdc836596973484d9f5b974e22ce2af78375 Mon Sep 17 00:00:00 2001 From: Peter Lamberg Date: Sat, 20 Jun 2015 14:16:02 +0300 Subject: [PATCH 11/13] Add new conveyor strategy and --files parameter to README.txt. --- README.txt | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/README.txt b/README.txt index ab74af4..17a13f3 100755 --- a/README.txt +++ b/README.txt @@ -55,7 +55,14 @@ Options: recent content to older content (may download more than one content file per feed), - * one: download one content file (not + * conveyor: like back_catalog, but downloads + only if feed has less than files parameter + content files in the cache. + Selecting conveyor strategy forces perfeed + mode on. Files are never deleted from + cache; conveyor strategy simply stops if + the maximum cache size is reached. + * one: download one content file (not already downloaded) for each feed, with a preference for recent content, * all: download all content, with a @@ -139,6 +146,10 @@ Options: or return the first N relevant feeds (when using the search function). 0 means unbounded. Default value is 1000. + -n, --files N Do not download more than N content + files per feed. This parameter works only + with conveyor strategy. 0 means unbounded. + Default value is 1000. -T, --torrentdir DIR Copy torrent files to directory DIR. The handling of torrents through an external BitTorrent client is left to From 00f230bb1f337dc93c03c69a350a482c1221871c Mon Sep 17 00:00:00 2001 From: Peter Lamberg Date: Sat, 20 Jun 2015 14:47:07 +0300 Subject: [PATCH 12/13] Tweak conveyor strategy help. --- README.txt | 10 +++++----- bin/podcatcher | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/README.txt b/README.txt index 17a13f3..505b3db 100755 --- a/README.txt +++ b/README.txt @@ -55,11 +55,11 @@ Options: recent content to older content (may download more than one content file per feed), - * conveyor: like back_catalog, but downloads - only if feed has less than files parameter - content files in the cache. - Selecting conveyor strategy forces perfeed - mode on. Files are never deleted from + * conveyor: like back_catalog, but + downloads up to -n number of files in the + cache per each feed. + Selecting conveyor strategy forces -p + mode on. Files are never deleted from the cache; conveyor strategy simply stops if the maximum cache size is reached. * one: download one content file (not diff --git a/bin/podcatcher b/bin/podcatcher index a377075..ff60e5a 100755 --- a/bin/podcatcher +++ b/bin/podcatcher @@ -267,11 +267,11 @@ option_parser = OptionParser.new() do |c| " recent content to older content (may ", " download more than one content file per", " feed),", - "* conveyor: like back_catalog, but downloads", - " only if feed has less than files parameter", - " content files in the cache.", - " Selecting conveyor strategy forces perfeed", - " mode on. Files are never deleted from", + "* conveyor: like back_catalog, but", + " downloads up to -n number of files in the", + " cache per each feed.", + " Selecting conveyor strategy forces -p", + " mode on. Files are never deleted from the", " cache; conveyor strategy simply stops if", " the maximum cache size is reached.", "* one: download one content file (not ", From 1b0a9bc2f90b6b330513925f28c990093a340e33 Mon Sep 17 00:00:00 2001 From: Peter Lamberg Date: Sat, 20 Jun 2015 14:50:40 +0300 Subject: [PATCH 13/13] Fix tabs --- bin/podcatcher | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/podcatcher b/bin/podcatcher index ff60e5a..0c26d25 100755 --- a/bin/podcatcher +++ b/bin/podcatcher @@ -410,7 +410,7 @@ option_parser = OptionParser.new() do |c| "Do not download more than N content ", "files per feed. This parameter works only", "with conveyor strategy. 0 means unbounded.", - "Default value is #{opt.files}.\n") do |e| + "Default value is #{opt.files}.\n") do |e| opt.files = e.to_i opt.files = nil if opt.files<1 end