diff --git a/lib/Plack/Util.pm b/lib/Plack/Util.pm
index a0d4e39f2..ebee0b3f7 100644
--- a/lib/Plack/Util.pm
+++ b/lib/Plack/Util.pm
@@ -121,7 +121,9 @@ sub _load_sandbox {
 package Plack::Sandbox::%s;
 {
     my $app = do $_file;
-    if ( !$app && ( my $error = $@ || $! )) { die $error; }
+    if ( !$app && ( my $error = $@ || $! || "returned invalid value.\n")) {
+        die $error;
+    }
     $app;
 }
 END_EVAL
diff --git a/t/Plack-Util/load.t b/t/Plack-Util/load.t
index 0b69d8bf8..3f799958d 100644
--- a/t/Plack-Util/load.t
+++ b/t/Plack-Util/load.t
@@ -45,6 +45,13 @@ use Test::More;
     unlike $@, qr/Died/;
 }
 
+{
+    eval { Plack::Util::load_psgi("t/Plack-Util/") };
+    # Perl 5.20+ gives "Did you try to load a directory",
+    #   <=5.18 "No such file or directory"
+    like $@, qr/Error while loading/;
+}
+
 {
     my $app = Plack::Util::load_psgi("t/Plack-Util/bin/findbin.psgi");
     test_psgi $app, sub {