Improve handling of paths with invalid Windows characters

lib/rubygems/package.rb

@@ -89,6 +89,18 @@ class TooLongFileName < Error; end
 
   class TarInvalidError < Error; end
 
+  ##
+  # Raised when a filename contains characters that are invalid on Windows
+
+  class InvalidFileNameError < Error
+    def initialize(filename, gem_name = nil)
+      message = "The gem contains a file '#{filename}' with characters in its name that are not allowed on Windows (e.g., colons)."
+      message += " This is a problem with the '#{gem_name}' gem, not Rubygems." if gem_name
+      message += " Please report this issue to the gem author."
+      super message
+    end
+  end
+
   attr_accessor :build_time # :nodoc:
 
   ##
@@ -258,6 +270,10 @@ def add_contents(tar) # :nodoc:
 
   def add_files(tar) # :nodoc:
     @spec.files.each do |file|
+      if invalid_windows_filename?(file)
+        alert_warning "filename '#{file}' contains characters that are invalid on Windows (e.g., colons). This gem may fail to install on Windows."
+      end
+
       stat = File.lstat file
 
       if stat.symlink?
@@ -427,6 +443,11 @@ def extract_tar_gz(io, destination_dir, pattern = "*") # :nodoc:
 
         destination = install_location full_name, destination_dir
 
+        if invalid_windows_filename?(full_name)
+          gem_name = @spec ? @spec.full_name : "unknown"
+          raise Gem::Package::InvalidFileNameError.new(full_name, gem_name)
+        end
+
         if entry.symlink?
           link_target = entry.header.linkname
           real_destination = link_target.start_with?("/") ? link_target : File.expand_path(link_target, File.dirname(destination))
@@ -450,13 +471,18 @@ def extract_tar_gz(io, destination_dir, pattern = "*") # :nodoc:
         end
 
         if entry.file?
-          File.open(destination, "wb") do |out|
-            copy_stream(tar.io, out, entry.size)
-            # Flush needs to happen before chmod because there could be data
-            # in the IO buffer that needs to be written, and that could be
-            # written after the chmod (on close) which would mess up the perms
-            out.flush
-            out.chmod file_mode(entry.header.mode) & ~File.umask
+          begin
+            File.open(destination, "wb") do |out|
+              copy_stream(tar.io, out, entry.size)
+              # Flush needs to happen before chmod because there could be data
+              # in the IO buffer that needs to be written, and that could be
+              # written after the chmod (on close) which would mess up the perms
+              out.flush
+              out.chmod file_mode(entry.header.mode) & ~File.umask
+            end
+          rescue Errno::EINVAL => e
+            gem_name = @spec ? @spec.full_name : "unknown"
+            raise Gem::Package::InvalidFileNameError.new(full_name, gem_name), e.message
           end
         end
 
@@ -529,6 +555,19 @@ def normalize_path(pathname) # :nodoc:
     end
   end
 
+  ##
+  # Checks if a filename contains characters that are invalid on Windows.
+  # Windows doesn't allow: < > : " | ? * \ and control characters (0x00-0x1F).
+  # Colons are the most common issue since they're allowed on Unix.
+  # Note: Colons are only valid as drive letter separators (e.g., C:), not in filenames.
+
+  def invalid_windows_filename?(filename) # :nodoc:
+    return false unless Gem.win_platform?
+
+    basename = File.basename(filename)
+    basename.match?(/[:<>"|?*\\\x00-\x1f]/)
+  end
+
   ##
   # Loads a Gem::Specification from the TarEntry +entry+
 

test/rubygems/test_gem_package.rb

@@ -1303,4 +1303,44 @@ def test_contents_from_io
 
     assert_equal %w[lib/code.rb], package.contents
   end
+
+  def test_invalid_windows_filename
+    package = Gem::Package.new @gem
+
+    if Gem.win_platform?
+      assert package.invalid_windows_filename?("spec/internal/:memory")
+      assert package.invalid_windows_filename?("file:name.rb")
+      assert package.invalid_windows_filename?("file<name.rb")
+      assert package.invalid_windows_filename?('file"name.rb')
+    end
+  end
+
+  def test_invalid_file_name_error_message
+    error = Gem::Package::InvalidFileNameError.new("spec/internal/:memory", "crono-2.0.1")
+    assert_match(/The gem contains a file 'spec\/internal\/:memory'/, error.message)
+    assert_match(/characters in its name that are not allowed on Windows/, error.message)
+    assert_match(/This is a problem with the 'crono-2.0.1' gem, not Rubygems/, error.message)
+    assert_match(/Please report this issue to the gem author/, error.message)
+  end
+
+  def test_extract_tar_gz_invalid_filename
+    pend "Windows filename validation only applies on Windows" unless Gem.win_platform?
+
+    package = Gem::Package.new @gem
+    package.verify
+
+    tgz_io = util_tar_gz do |tar|
+      tar.add_file "spec/internal/:memory", 0o644 do |io|
+        io.write "test content"
+      end
+    end
+
+    e = assert_raise Gem::Package::InvalidFileNameError do
+      package.extract_tar_gz tgz_io, @destination
+    end
+
+    assert_match(/The gem contains a file 'spec\/internal\/:memory'/, e.message)
+    assert_match(/characters in its name that are not allowed on Windows/, e.message)
+    assert_match(/This is a problem with the 'a-2' gem, not Rubygems/, e.message)
+  end
 end