From 4c702227ed7a3394e27007352b282acf73f2f9d3 Mon Sep 17 00:00:00 2001 From: Reini Urban Date: Mon, 11 Jan 2016 16:06:34 +0100 Subject: [PATCH 1/9] fix cperl prerequisite and version checks allow cperl c suffices honor cperl builtin modules --- lib/CPAN.pm | 2 +- lib/CPAN/Distribution.pm | 9 ++++++--- lib/CPAN/Version.pm | 3 ++- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/lib/CPAN.pm b/lib/CPAN.pm index 2d87f47f8..154dcced2 100644 --- a/lib/CPAN.pm +++ b/lib/CPAN.pm @@ -2,7 +2,7 @@ # vim: ts=4 sts=4 sw=4: use strict; package CPAN; -$CPAN::VERSION = '2.27'; +$CPAN::VERSION = '2.27_01'; # with cperl support $CPAN::VERSION =~ s/_//; # we need to run chdir all over and we would get at wrong libraries diff --git a/lib/CPAN/Distribution.pm b/lib/CPAN/Distribution.pm index 341210853..04ba6beef 100644 --- a/lib/CPAN/Distribution.pm +++ b/lib/CPAN/Distribution.pm @@ -9,10 +9,9 @@ use File::Path (); use POSIX ":sys_wait_h"; @CPAN::Distribution::ISA = qw(CPAN::InfoObj); use vars qw($VERSION); -$VERSION = "2.27"; +$VERSION = "2.27_01"; # with cperl support my $run_allow_installing_within_test = 1; # boolean; either in test or in install, there is no third option - # no prepare, because prepare is not a command on the shell command line # TODO: clear instance cache on reload my %instance; @@ -3838,7 +3837,7 @@ sub test { } sub _make_test_illuminate_prereqs { - my($self) = @_; + my ($self) = @_; my @prereq; # local $CPAN::DEBUG = 16; # Distribution @@ -3864,6 +3863,10 @@ sub _make_test_illuminate_prereqs { ) { # lex Class::Accessor::Chained::Fast which has no $VERSION CPAN->debug("m[$m] have available_file[$available_file]") + if $CPAN::DEBUG; + } elsif ($Config::Config{usecperl} + and $m =~ /^(DynaLoader|XSLoader|strict|coretypes)$/) { + CPAN->debug("m[$m] builtin available_version[$available_version]") if $CPAN::DEBUG; } else { push @prereq, $m diff --git a/lib/CPAN/Version.pm b/lib/CPAN/Version.pm index fa75221d9..5acc19c99 100644 --- a/lib/CPAN/Version.pm +++ b/lib/CPAN/Version.pm @@ -2,7 +2,7 @@ package CPAN::Version; use strict; use vars qw($VERSION); -$VERSION = "5.5003"; +$VERSION = "5.5003_01"; # with cperl support # CPAN::Version::vcmp courtesy Jost Krieger sub vcmp { @@ -18,6 +18,7 @@ sub vcmp { for ($l,$r) { s/_//g; + s/c$//; } CPAN->debug("l[$l] r[$r]") if $CPAN::DEBUG; for ($l,$r) { From 6bbc4c4cee27c346b91abf4015966ca62afec478 Mon Sep 17 00:00:00 2001 From: Reini Urban Date: Wed, 23 Mar 2016 15:14:59 +0100 Subject: [PATCH 2/9] in cperl favor YAML::XS and Safe{Load,Dump}File Use the builtin YAML::XS in NonStrict mode. The previous default YAML is actually the worst choice at all. * It's not builtin (compared to CPAN::Meta::YAML), * it's the slowest (and performance matters with CPAN), * it does not understand YAML::XS yaml, * it errors on the simpliest YAML violations, where others continue. YAML::Syck would be a better choice as it does not implement the unsafe YAML 1.1 spec, but cperl put an improved YAML::XS into core. Try Safe{Load,Dump}File before trying {Load,Dump}File. cperl is working on that feature. --- lib/CPAN.pm | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/lib/CPAN.pm b/lib/CPAN.pm index 154dcced2..330262def 100644 --- a/lib/CPAN.pm +++ b/lib/CPAN.pm @@ -63,6 +63,7 @@ use Text::Wrap (); sub find_perl (); sub anycwd (); sub _uniq; +my $CPERL = $Config::Config{usecperl}; no lib "."; @@ -522,9 +523,10 @@ sub _flock { } sub _yaml_module () { - my $yaml_module = $CPAN::Config->{yaml_module} || "YAML"; + my $dflt = $CPERL ? "YAML::XS" : "YAML"; + my $yaml_module = $CPAN::Config->{yaml_module} || $dflt; if ( - $yaml_module ne "YAML" + $yaml_module ne $dflt && !$CPAN::META->has_inst($yaml_module) ) { @@ -559,9 +561,19 @@ sub _yaml_loadfile { # so we do it manually instead my $old_loadcode = ${"$yaml_module\::LoadCode"}; ${ "$yaml_module\::LoadCode" } = $CPAN::Config->{yaml_load_code} || 0; + # CPAN yaml is not strict YAML. Only Ingy YAML is strict, esp. YAML::XS + my $old_nonstrict = ${"$yaml_module\::NonStrict"}; + ${ "$yaml_module\::NonStrict" } = 1 if $yaml_module =~ /^YAML(::XS)?$/; my ($code, @yaml); - if ($code = UNIVERSAL::can($yaml_module, "LoadFile")) { + if ($code = UNIVERSAL::can($yaml_module, "SafeLoadFile")) { + # TODO: allow CPAN::* classes + eval { @yaml = $code->($local_file); }; + if ($@) { + # this shall not be done by the frontend + die CPAN::Exception::yaml_process_error->new($yaml_module,$local_file,"parse",$@); + } + } elsif ($code = UNIVERSAL::can($yaml_module, "LoadFile")) { eval { @yaml = $code->($local_file); }; if ($@) { # this shall not be done by the frontend @@ -582,6 +594,11 @@ sub _yaml_loadfile { } } ${"$yaml_module\::LoadCode"} = $old_loadcode; + if (!defined $old_nonstrict) { + undef ${"$yaml_module\::NonStrict"}; + } else { + ${"$yaml_module\::NonStrict"} = $old_nonstrict; + } return \@yaml; } else { # this shall not be done by the frontend @@ -599,6 +616,8 @@ sub _yaml_dumpfile { if (UNIVERSAL::isa($local_file, "FileHandle")) { $code = UNIVERSAL::can($yaml_module, "Dump"); eval { print $local_file $code->(@what) }; + } elsif ($code = UNIVERSAL::can($yaml_module, "SafeDumpFile")) { + eval { $code->($local_file,@what); }; } elsif ($code = UNIVERSAL::can($yaml_module, "DumpFile")) { eval { $code->($local_file,@what); }; } elsif ($code = UNIVERSAL::can($yaml_module, "Dump")) { From 248170297288d1b356a26019ff617bb3739461d8 Mon Sep 17 00:00:00 2001 From: Reini Urban Date: Tue, 29 Mar 2016 11:07:26 +0200 Subject: [PATCH 3/9] Add PERL_USE_UNSAFE_INC=1 for all CPAN modules, scripts and subprocesses. --- lib/CPAN.pm | 2 ++ lib/CPAN/Distribution.pm | 1 + 2 files changed, 3 insertions(+) diff --git a/lib/CPAN.pm b/lib/CPAN.pm index 330262def..70fd77f21 100644 --- a/lib/CPAN.pm +++ b/lib/CPAN.pm @@ -208,6 +208,8 @@ sub soft_chdir_with_alternatives ($); } $autoload_recursion--; } + # make sure we can install any modules from CPAN without patching them + $ENV{PERL_USE_UNSAFE_INC} = 1; } { diff --git a/lib/CPAN/Distribution.pm b/lib/CPAN/Distribution.pm index 04ba6beef..ce05a1371 100644 --- a/lib/CPAN/Distribution.pm +++ b/lib/CPAN/Distribution.pm @@ -1335,6 +1335,7 @@ Could not determine which directory to use for looking at $dist. # local $ENV{PERL_USE_UNSAFE_INC} = exists $ENV{PERL_USE_UNSAFE_INC} ? $ENV{PERL_USE_UNSAFE_INC} : 1; # look $CPAN::META->set_perl5lib; local $ENV{MAKEFLAGS}; # protect us from outer make calls + local $ENV{PERL_USE_UNSAFE_INC} = 1; unless (system($shell) == 0) { my $code = $? >> 8; From c0246cfe83404448bb2bd48d9e85a150f49a1cdc Mon Sep 17 00:00:00 2001 From: Reini Urban Date: Fri, 10 Jun 2016 13:34:16 +0200 Subject: [PATCH 4/9] extend FirstTime to suggest YAML::XS on cperl, and error on CPAN::Meta::YAML. Work to support CPAN::Meta::YAML features is ongoing See [cperl #155] --- lib/CPAN/FirstTime.pm | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/lib/CPAN/FirstTime.pm b/lib/CPAN/FirstTime.pm index af4a6d775..e6e509c0d 100644 --- a/lib/CPAN/FirstTime.pm +++ b/lib/CPAN/FirstTime.pm @@ -11,7 +11,7 @@ use File::Spec (); use CPAN::Mirrors (); use CPAN::Version (); use vars qw($VERSION $auto_config); -$VERSION = "5.5314"; +$VERSION = "5.5314_01"; =head1 NAME @@ -661,6 +661,9 @@ may be more alternative YAML conforming modules. When I tried two other players, YAML::Tiny and YAML::Perl, they seemed not powerful enough to work with CPAN.pm. This may have changed in the meantime. +The core module CPAN::Meta::YAML cannot load YAML 1.2 !! features, +used for C hashes. + Which YAML implementation would you prefer? =back @@ -984,15 +987,30 @@ sub init { my_yn_prompt(trust_test_report_history => 0, $matcher); # - #= YAML vs. YAML::Syck + #= YAML::Syck, YAML::XS (cperl only), YAML, YAML::Tiny. CPAN::Meta::YAML not yet + # YAML::XS is broken upstream, CPAN::Meta::YAML cannot read spec v2. # if (!$matcher or "yaml_module" =~ /$matcher/) { - my_dflt_prompt(yaml_module => "YAML", $matcher); + my $CPERL = $Config::Config{usecperl}; + my $dflt = $CPERL ? 'YAML::XS' : 'YAML'; + while(1) { + my_dflt_prompt(yaml_module => $dflt, $matcher); + my $given = $CPAN::Config->{yaml_module}; + my $forbidden = $CPERL ? qr/^(CPAN::Meta::YAML)$/ : qr/^(CPAN::Meta::YAML|YAML::XS)$/; + if ($given =~ $forbidden) { + $CPAN::Frontend->mywarn + ("Error: $given cannot be used yet. Try YAML" + . $CPERL ? ", YAML::Syck or YAML::XS\n" : " or YAML::Syck\n"); + $CPAN::Frontend->mysleep(3); + } else { + last; + } + } my $old_v = $CPAN::Config->{load_module_verbosity}; $CPAN::Config->{load_module_verbosity} = q[none]; if (!$auto_config && !$CPAN::META->has_inst($CPAN::Config->{yaml_module})) { $CPAN::Frontend->mywarn - ("Warning (maybe harmless): '$CPAN::Config->{yaml_module}' not installed.\n"); + ("Warning (maybe harmless): '$CPAN::Config->{yaml_module}' not installed. Try $dflt\n"); $CPAN::Frontend->mysleep(3); } $CPAN::Config->{load_module_verbosity} = $old_v; @@ -2181,6 +2199,4 @@ sub prompt_no_strip ($;$) { return _real_prompt(@_); } - - 1; From f3b1e7399256d33f08963b54b38e35d08320689d Mon Sep 17 00:00:00 2001 From: Reini Urban Date: Sun, 16 Oct 2016 18:24:41 +0200 Subject: [PATCH 5/9] skip cperl builtin prereqs --- lib/CPAN/Distribution.pm | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lib/CPAN/Distribution.pm b/lib/CPAN/Distribution.pm index ce05a1371..a0a571ba7 100644 --- a/lib/CPAN/Distribution.pm +++ b/lib/CPAN/Distribution.pm @@ -2704,6 +2704,12 @@ sub follow_prereqs { my(@good_prereq_tuples); for my $p (@prereq_tuples) { # e.g. $p = ['Devel::PartialDump', 'r', 1] + # skip builtins without .pm + if ($Config::Config{usecperl} + and $p->[0] =~ /^(DynaLoader|XSLoader|strict|coretypes)$/) { + CPAN->debug("$p->[0] builtin") if $CPAN::DEBUG; + next; + } # promote if possible if ($p->[1] =~ /^(r|c)$/) { push @good_prereq_tuples, $p; @@ -2718,6 +2724,7 @@ sub follow_prereqs { die "Panic: in follow_prereqs: reqtype[$p->[1]] seen, should never happen"; } } + return unless @good_prereq_tuples; my $pretty_id = $self->pretty_id; my %map = ( b => "build_requires", From 5bd66f3b21bc54928da85a3173a5c8c92b753433 Mon Sep 17 00:00:00 2001 From: Reini Urban Date: Fri, 17 Feb 2017 13:03:01 +0100 Subject: [PATCH 6/9] cperl test updates ignore inc with Test::More because it clashes with the modernized Test::More variant in cperl. add YAML::XS to t/31sessions.t to be ignored. --- t/31sessions.t | 2 ++ t/41distribution.t | 7 +++---- t/44cpanmeta.t | 3 +-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/t/31sessions.t b/t/31sessions.t index 9ef32cb77..baedeb8ee 100644 --- a/t/31sessions.t +++ b/t/31sessions.t @@ -358,6 +358,7 @@ EOF "make CPAN::Test::Dummy::Perl5::Build::Fails" => "Has.already.been.unwrapped", "test CPAN::Test::Dummy::Perl5::Build::Fails" => "(?i:t/00_load.+FAILED)", "o conf dontload_list push YAML" => ".", + "o conf dontload_list push YAML::XS" => ".", "o conf dontload_list push YAML::Syck" => ".", "o conf dontload_list push Parse::CPAN::Meta" => ".", "o conf dontload_list push CPAN::Meta" => ".", @@ -428,6 +429,7 @@ EOF "o conf dontload_list pop" => ".", "o conf dontload_list pop" => ".", "o conf dontload_list pop" => ".", + "o conf dontload_list pop" => ".", "o conf commit" => "commit: wrote", ], }, diff --git a/t/41distribution.t b/t/41distribution.t index e09929df2..f5a87b581 100644 --- a/t/41distribution.t +++ b/t/41distribution.t @@ -1,5 +1,5 @@ # Test CPAN::Distribution objects -# +# # Very, very preliminary API testing, but we have to start somewhere BEGIN { @@ -17,7 +17,7 @@ BEGIN { my $yaml_module = CPAN::_yaml_module(); my $exit_message; if ($CPAN::META->has_inst($yaml_module)) { - # print "# yaml_module[$yaml_module] loadable\n"; + #print STDERR "# yaml_module[$yaml_module] loadable\n"; } else { $exit_message = "No yaml module installed"; } @@ -44,8 +44,7 @@ use File::Temp qw(tempdir); use File::Spec::Functions qw/catdir catfile/; use File::Basename qw/basename/; -use lib "inc"; -use lib "t"; +use lib $Config::Config{usecperl} ? ("t") : ("inc", "t"); use local_utils; use version; diff --git a/t/44cpanmeta.t b/t/44cpanmeta.t index ba572a291..0c20742a2 100644 --- a/t/44cpanmeta.t +++ b/t/44cpanmeta.t @@ -28,8 +28,7 @@ use File::Temp qw(tempdir); use File::Spec::Functions qw/catdir catfile/; use File::Basename qw/basename/; -use lib "inc"; -use lib "t"; +use lib $Config::Config{usecperl} ? ("t") : ("inc", "t"); use local_utils; # prepare local CPAN From 6bd54de196872be292ae1b9109db6f9dc0401e82 Mon Sep 17 00:00:00 2001 From: Reini Urban Date: Fri, 17 Feb 2017 13:19:21 +0100 Subject: [PATCH 7/9] reduce FirstTime init complexity Perl::Critic found these violations in "lib/CPAN/FirstTime.pm": lib/CPAN/FirstTime.pm: Subroutine "init" with high complexity score (154) at line 774, column 1. Consider refactoring. (Severity: 3) --- lib/CPAN/Distribution.pm | 5 ++++ lib/CPAN/FirstTime.pm | 64 ++++++++++++++++++++++------------------ 2 files changed, 41 insertions(+), 28 deletions(-) diff --git a/lib/CPAN/Distribution.pm b/lib/CPAN/Distribution.pm index a0a571ba7..56a802199 100644 --- a/lib/CPAN/Distribution.pm +++ b/lib/CPAN/Distribution.pm @@ -2945,6 +2945,11 @@ sub unsat_prereq { my @merged = sort $merged->required_modules; CPAN->debug("all merged_prereqs[@merged]") if $CPAN::DEBUG; NEED: for my $need_module ( @merged ) { + # skip builtins without .pm + if ($^V =~ /c$/ and $need_module =~ /^(DynaLoader|XSLoader|strict|coretypes)$/) { + CPAN->debug("$need_module builtin") if $CPAN::DEBUG; + next NEED; + } my $need_version = $merged->requirements_for_module($need_module); my($available_version,$inst_file,$available_file,$nmo); if ($need_module eq "perl") { diff --git a/lib/CPAN/FirstTime.pm b/lib/CPAN/FirstTime.pm index e6e509c0d..dd8f6d72e 100644 --- a/lib/CPAN/FirstTime.pm +++ b/lib/CPAN/FirstTime.pm @@ -817,6 +817,39 @@ then restart your command line shell and CPAN before installing modules: } +# +# YAML::Syck, YAML::XS (cperl only), YAML, YAML::Tiny. CPAN::Meta::YAML not yet +# YAML::XS is broken upstream, CPAN::Meta::YAML cannot read spec v2. +# + +sub _yaml_init { + my ($matcher) = @_; + my $CPERL = $Config::Config{usecperl}; + my $dflt = $CPERL ? 'YAML::XS' : 'YAML'; + while(1) { + my_dflt_prompt(yaml_module => $dflt, $matcher); + my $given = $CPAN::Config->{yaml_module}; + my $forbidden = $CPERL ? qr/^(CPAN::Meta::YAML)$/ : qr/^(CPAN::Meta::YAML|YAML::XS)$/; + if ($given =~ $forbidden) { + $CPAN::Frontend->mywarn + ("Error: $given cannot be used yet. Try YAML" + . $CPERL ? ", YAML::Syck or YAML::XS\n" : " or YAML::Syck\n"); + $CPAN::Frontend->mysleep(3); + } else { + last; + } + } + my $old_v = $CPAN::Config->{load_module_verbosity}; + $CPAN::Config->{load_module_verbosity} = q[none]; + if (!$auto_config && !$CPAN::META->has_inst($CPAN::Config->{yaml_module})) { + $CPAN::Frontend->mywarn + ("Warning (maybe harmless): '$CPAN::Config->{yaml_module}' not installed. Try $dflt\n"); + $CPAN::Frontend->mysleep(3); + } + $CPAN::Config->{load_module_verbosity} = $old_v; +} + + sub init { my($configpm, %args) = @_; use Config; @@ -986,38 +1019,13 @@ sub init { my_yn_prompt(trust_test_report_history => 0, $matcher); - # - #= YAML::Syck, YAML::XS (cperl only), YAML, YAML::Tiny. CPAN::Meta::YAML not yet - # YAML::XS is broken upstream, CPAN::Meta::YAML cannot read spec v2. - # + #= YAML module if (!$matcher or "yaml_module" =~ /$matcher/) { - my $CPERL = $Config::Config{usecperl}; - my $dflt = $CPERL ? 'YAML::XS' : 'YAML'; - while(1) { - my_dflt_prompt(yaml_module => $dflt, $matcher); - my $given = $CPAN::Config->{yaml_module}; - my $forbidden = $CPERL ? qr/^(CPAN::Meta::YAML)$/ : qr/^(CPAN::Meta::YAML|YAML::XS)$/; - if ($given =~ $forbidden) { - $CPAN::Frontend->mywarn - ("Error: $given cannot be used yet. Try YAML" - . $CPERL ? ", YAML::Syck or YAML::XS\n" : " or YAML::Syck\n"); - $CPAN::Frontend->mysleep(3); - } else { - last; - } - } - my $old_v = $CPAN::Config->{load_module_verbosity}; - $CPAN::Config->{load_module_verbosity} = q[none]; - if (!$auto_config && !$CPAN::META->has_inst($CPAN::Config->{yaml_module})) { - $CPAN::Frontend->mywarn - ("Warning (maybe harmless): '$CPAN::Config->{yaml_module}' not installed. Try $dflt\n"); - $CPAN::Frontend->mysleep(3); - } - $CPAN::Config->{load_module_verbosity} = $old_v; + _yaml_init($matcher); } # - #= YAML code deserialisation + #= YAML code deserialisation (security problem) # my_yn_prompt(yaml_load_code => 0, $matcher); From b24db1f834038c657e47cf5e398d70a821e5e62a Mon Sep 17 00:00:00 2001 From: Reini Urban Date: Mon, 8 Oct 2018 18:09:16 +0200 Subject: [PATCH 8/9] support PERL_CORE in t/04clean_load.t --- t/04clean_load.t | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/t/04clean_load.t b/t/04clean_load.t index 18a4e6e46..21bdd266f 100644 --- a/t/04clean_load.t +++ b/t/04clean_load.t @@ -11,7 +11,7 @@ my %has_deps = ( my @modules; use File::Find; -find(\&list_modules, 'blib/lib'); +find(\&list_modules, $ENV{PERL_CORE} ? 'lib' : 'blib/lib'); use Test::More; plan(tests => scalar @modules); From 62bf4d2845cf26d6877dac9588897de49fdc6d19 Mon Sep 17 00:00:00 2001 From: Reini Urban Date: Mon, 8 Oct 2018 14:10:57 +0200 Subject: [PATCH 9/9] Fix run_preps_on_packagedir rmtree race with symlinks crossing mountpoints. --- lib/CPAN/Distribution.pm | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/CPAN/Distribution.pm b/lib/CPAN/Distribution.pm index 56a802199..d247f185d 100644 --- a/lib/CPAN/Distribution.pm +++ b/lib/CPAN/Distribution.pm @@ -9,7 +9,7 @@ use File::Path (); use POSIX ":sys_wait_h"; @CPAN::Distribution::ISA = qw(CPAN::InfoObj); use vars qw($VERSION); -$VERSION = "2.27_01"; # with cperl support +$VERSION = "2.27_02"; # with cperl support my $run_allow_installing_within_test = 1; # boolean; either in test or in install, there is no third option # no prepare, because prepare is not a command on the shell command line @@ -612,8 +612,9 @@ See also http://rt.cpan.org/Ticket/Display.html?id=38932\n"); } } $self->{build_dir} = $packagedir; - $self->safe_chdir($builddir); - File::Path::rmtree("tmp-$$"); + $self->safe_chdir(Cwd::abs_path($builddir)); + $self->debug("rmtree $builddir/tmp-$$") if $CPAN::DEBUG; + File::Path::rmtree(File::Spec->catfile(Cwd::abs_path($builddir),"tmp-$$")); $self->safe_chdir($packagedir); $self->_signature_business();