test: Fix caching issues (#111)

This commit deals with the problems of caching LilyPond versions between
Travis-CI builds reported in #111. Specifically, it turned out that
there is not a separate cache per build-matrix entry (since LilyPond is
not a supported language), but only a dedicated cache per branch.

The issue was then that only one version of LilyPond was actually used
to run the tests, since the installation path was identical for all
versions.

The solution introduced in this commit is to have an explicit cache
management mechanism:

 - Each LilyPond version is installed in its own folder under the
   $HOME/.lilypond cache directory, for instance

     $HOME/.lilypond/linux-64/2.18.2-1

 - Every time a cached LilyPond version is used, the timestamp is
   written to a file in the cache, for later retrieval
 - After all the tests have been run, the cache is cleaned: LilyPond
   versions that have not been used for more than one day (this is
   configurable) are removed from the cache. Note that since this
   cleanup phase is run after the tests have been run, all the LilyPond
   versions that have actually been used in the last test run are kept
   in the cache

Author: Cecca

test/automated_tests.py

@@ -1,16 +1,16 @@
 #!/usr/bin/env python
 
-import subprocess as sp
 import os
 import os.path as osp
 import shutil
 import sys
 import re
+from lilycmd import LilyCmd
 
-from common_functions import print_separator, home_dir, install_root
+from common_functions import print_separator
 
 
-class SimpleTests:
+class SimpleTests(object):
     """Run simple intergration tests. Specifically, this script will look
     for all the files in `usage-examples` directories. All these files
     will be compiled with LilyPond. If the compilation results in a
@@ -56,24 +56,25 @@ def __init__(self, cmd=None):
         # root directory
         self.openlilylib_dir = self.__openlilylib_dir()
 
-        # LilyPond command
-        if self.is_ci_run():
-            try:
-                self.lily_command = osp.join(install_root,
-                                             "bin",
-                                             "lilypond")
-                self.lilypond_version = self.__lilypond_version()
-            except KeyError:
-                sys.exit('Environment variable {} not set. Aborting'.format(self.lily_version_var))
+        if 'CI' in os.environ and bool(os.environ["CI"]):
+            # TODO check definition
+            lily_platform = os.environ["LILY_PLATFORM"]
+            lily_version = os.environ["LILY_VERSION"]
+
+            self.lily_command = LilyCmd.with_version(lily_platform,
+                                                     lily_version)
+            if not self.lily_command.installed:
+                raise Exception('The required lilypond version is not installed')
+            self.lilypond_version = self.lily_command.version
         else:
-            self.lily_command = cmd if cmd else "lilypond"
-            self.lilypond_version = self.__lilypond_version()
+            self.lily_command = LilyCmd.system(cmd if cmd else "lilypond")
+            self.lilypond_version = self.lily_command.version
 
         # Add include path and other options to generated LilyPond command
-        self.lily_command_with_includes = [self.lily_command,
-                "-dno-point-and-click",
-                "-I", self.openlilylib_dir,
-                "-I", os.path.join(self.openlilylib_dir, "ly")]
+        self.lily_command_with_includes_args = [
+            "-dno-point-and-click",
+            "-I", self.openlilylib_dir,
+            "-I", os.path.join(self.openlilylib_dir, "ly")]
         # initialize some lists
         self.test_files = []
         self.included_tests = []
@@ -102,14 +103,6 @@ def __collect_all_in_dir(self, dirname):
                     if os.path.isfile(test_fname) and self.is_lilypond_file(test_fname):
                         self.test_files.append(test_fname)
 
-
-    def __lilypond_version(self):
-        """Determine the LilyPond version actually run by the command self.lily_command"""
-        lily = sp.Popen([self.lily_command, "-v"], stdout=sp.PIPE, stderr=sp.PIPE)
-        version_line = lily.communicate()[0].splitlines()[0]
-        return re.search(r"\d+\.\d+\.\d+", version_line).group(0)
-
-
     def __openlilylib_dir(self):
         """Return the root directory of openLilyLib.
            It's the parent directory of the script."""
@@ -228,7 +221,9 @@ def print_introduction(self):
         print "OpenLilyLib directory: {}".format(self.openlilylib_dir)
 
         print "LilyPond command to be used:"
-        print " ".join(self.lily_command_with_includes + ["-o <output-dir> <test-file>"])
+        print " ".join([self.lily_command.command] +
+                       self.lily_command_with_includes_args +
+                       ["-o <output-dir> <test-file>"])
 
 
     def report(self):
@@ -254,7 +249,8 @@ def report(self):
                 print self.failed_tests[test]
                 print ""
             print_separator()
-            sys.exit(1)
+            return 1
+        return 0
 
 
     def run(self):
@@ -271,13 +267,11 @@ def run(self):
                                     os.path.dirname(self.__relative_path(test)))
             if not os.path.exists(test_result_dir):
                 os.makedirs(test_result_dir)
-            lily = sp.Popen(self.lily_command_with_includes + ['-o',
-                                                               test_result_dir,
-                                                               test],
-                            stdout=sp.PIPE, stderr=sp.PIPE)
-            (out, err) = lily.communicate()
-
-            if lily.returncode == 0:
+            returncode, out, err = self.lily_command.execute(
+                self.lily_command_with_includes_args + ['-o',
+                                                        test_result_dir,
+                                                        test])
+            if returncode == 0:
                 print "------- OK! --------"
             else:
                 # if test failed, add it to the list of failed tests to be reported later
@@ -300,4 +294,9 @@ def run(self):
     tests.clean_results_dir()
     tests.collect_tests()
     tests.run()
-    tests.report()
+    retcode = tests.report()
+
+    # cleanup old version of lilypond
+    LilyCmd.clean_cache()
+
+    sys.exit(retcode)

test/common_functions.py

@@ -32,4 +32,3 @@ def load_lily_versions():
 def print_separator():
     print ""
     print "="*79, "\n"
-

test/install_lilypond.py

@@ -1,14 +1,10 @@
 #!/usr/bin/env python
 
-import subprocess as sp
 import os
-import os.path as osp
-import shutil
 import sys
-import collections
 
-import common_functions
-from common_functions import print_separator, home_dir, install_root
+from lilycmd import LilyCmd
+from common_functions import print_separator
 
 #############################################################
 # Load environment variables
@@ -21,43 +17,6 @@
 except:
     sys.exit('\nScript can only be run in CI mode. Aborting\n')
 
-#########################
-# Configuration constants
-
-# Download site for LilyPond distributions
-binary_site = "http://download.linuxaudio.org/lilypond/binaries/"
-# String template for generating the LilyPond installation command
-lily_install_script = "lilypond-install.sh"
-
-
-#################################
-# Functions doing the actual work
-
-def download_url():
-    """Format a string representing the URL to download the requested LilyPond distribution"""
-    return "{}{}/lilypond-{}.{}.sh".format(
-        binary_site, lily_platform, lily_version, lily_platform)
-
-def install_distribution():
-    """Download and install LilyPond version if not cached"""
-    lilypond_cmd = os.path.join(install_root,
-                                "bin/lilypond")
-    print "\nChecking LilyPond presence with {}\n".format(lilypond_cmd)
-    try:
-        sp.check_call([lilypond_cmd, '--version'])
-        print "LilyPond {} is already installed in cache, continuing with test script.".format(lily_version)
-    except:
-        print "LilyPond {} is not installed yet.".format(lily_version)
-        print "Downloading and installing now"
-        sp.check_call(
-            ["wget", "-O",
-             lily_install_script,
-             download_url()])
-        sp.check_call(["sh", lily_install_script,
-                       "--prefix",
-                       install_root,
-                       "--batch"])
-
 #########################
 # Actual script execution
 
@@ -70,4 +29,4 @@ def install_distribution():
     print "check LilyPond installation."
     print "Requested LilyPond version: {}".format(lily_version)
 
-    install_distribution()
+    LilyCmd.install(lily_platform, lily_version)

test/lilycmd.py

@@ -0,0 +1,187 @@
+import subprocess as sp
+import os
+import os.path as osp
+import shutil
+import sys
+import re
+import datetime
+import dateutil.parser
+from common_functions import print_separator
+
+class LilyCmd(object):
+    """This class represents a lilypond command and provides some
+    facilities to
+
+     - run LilyPond
+     - install LilyPond versions from the internet into a local cache
+     - manage the cache, cleaning old versions
+
+    """
+
+    # Download site for LilyPond distributions
+    binary_site = "http://download.linuxaudio.org/lilypond/binaries/"
+    # String template for generating the LilyPond installation command
+    lily_install_script = "lilypond-install.sh"
+
+    # After this amount of time a LilyPond version will be removed by
+    # a call to LilyCmd.clean_cache()
+    cache_cleanup_interval = datetime.timedelta(days=1)
+
+    # Root directory for the cache
+    cache_root = osp.join(os.getenv('HOME'), '.lilypond')
+
+    def __init__(self, command_path, cached):
+        self.command = command_path
+        self.cached = cached
+        try:
+            self.version = self._lilypond_version()
+            self.installed = True
+        except:
+            self.installed = False
+
+    ####################################################################
+    # Public methods
+
+    @staticmethod
+    def system(cmd_path='lilypond'):
+        """Get a new instance of Lilypond provided by the
+        system. (eg. /usr/bin/lilypond)"""
+        return LilyCmd(cmd_path, cached=False)
+
+    @classmethod
+    def with_version(cls, platform, version):
+        """Get a new version of Lilypond, given the platform and the
+        version.
+
+        **Note**: This command does not install LilyPond in the local
+        cache. To check if the instance returned by this method
+        corresponds to an actually installed LilyPond version, check
+        the `installed` attribute. The reason for not installing
+        LilyPond with this command is that we may be interested in all
+        kind of ancillary information (like the cache directory)
+        without actually installing LilyPond.
+
+        """
+        lily_cmd_path = osp.join(
+            LilyCmd._cache_directory(platform, version),
+            'bin', 'lilypond')
+        lily_cmd = LilyCmd(lily_cmd_path, cached=True)
+        return lily_cmd
+
+    def execute(self, args):
+        """Executes the LilyPond command with the given arguments"""
+        self._mark_cache()
+        lily = sp.Popen([self.command] + args,
+                        stdout=sp.PIPE, stderr=sp.PIPE)
+        (out, err) = lily.communicate()
+        return lily.returncode, out, err
+
+    @classmethod
+    def install(cls, platform, version):
+        """Download and install LilyPond version if not cached"""
+        lilypond_cmd = LilyCmd.with_version(platform, version)
+        print "\nChecking LilyPond presence"
+        if lilypond_cmd.installed:
+            print ("LilyPond {} is already installed in cache," \
+                   +" continuing with test script.").format(
+                       lilypond_cmd.version)
+        else:
+            print "LilyPond {} is not installed yet.".format(version)
+            print "Downloading and installing now"
+            sp.check_call(
+                ["wget", "-O",
+                 cls.lily_install_script,
+                 cls._download_url(platform, version)])
+            sp.check_call(["sh", cls.lily_install_script,
+                           "--prefix",
+                           LilyCmd._cache_directory(platform, version),
+                           "--batch"])
+
+    @classmethod
+    def clean_cache(cls):
+        """Clean the cache from versions of Lilypond older than
+        `cache_cleanup_interval`"""
+        print "Clean cache\n"
+        cached = cls._get_cached_versions()
+        now = datetime.datetime.now()
+        for lily in cached:
+            if lily['last_used'] is None:
+                print 'Removing cached LilyPond', lily['version'],\
+                    lily['platform'], '(never used)'
+                shutil.rmtree(lily['directory'])
+            elif now - lily['last_used'] > cls.cache_cleanup_interval:
+                print 'Removing cached LilyPond', lily['version'],\
+                    lily['platform'], '(last used', \
+                    lily['last_used'].isoformat(), ')'
+                shutil.rmtree(lily['directory'])
+            else:
+                print 'Keeping cached LilyPond', lily['version'],\
+                    lily['platform'], '(last used', \
+                    lily['last_used'].isoformat(), ')'
+
+    ####################################################################
+    # Private members
+
+    @classmethod
+    def _cache_directory(cls, platform, version):
+        """Get the cache directory name for the given platform and version"""
+        return osp.join(cls.cache_root, platform, version)
+
+    def _lilypond_version(self):
+        """Determine the LilyPond version actually run
+        by the command self.lily_command"""
+        lily = sp.Popen([self.command, "--version"],
+                        stdout=sp.PIPE, stderr=sp.PIPE)
+        version_line = lily.communicate()[0].splitlines()[0]
+        return re.search(r"\d+\.\d+\.\d+", version_line).group(0)
+
+    def _mark_cache_file(self):
+        """Get the name of the file that will store the timestamp of the last
+        time this command has been used."""
+        if self.cached:
+            return osp.join(osp.dirname(self.command), '.oll-last-used')
+        else:
+            return None
+
+    def _mark_cache(self):
+        """Write the timestamp of now in the cache file"""
+        if self.cached:
+            with open(self._mark_cache_file(), 'w') as mark_file:
+                mark_file.write(datetime.datetime.now().isoformat())
+
+    def _last_used(self):
+        """Returns the last time this command has been used, or None if it was
+        never used."""
+        if self.cached:
+            if not osp.isfile(self._mark_cache_file()):
+                return None
+            with open(self._mark_cache_file(), 'r') as mark_file:
+                fcontent = mark_file.readline()
+                return dateutil.parser.parse(fcontent)
+
+
+    @classmethod
+    def _download_url(cls, lily_platform, lily_version):
+        """Format a string representing the URL to
+        download the requested LilyPond distribution"""
+        return "{}{}/lilypond-{}.{}.sh".format(
+            cls.binary_site, lily_platform, lily_version, lily_platform)
+
+
+    @classmethod
+    def _get_cached_versions(cls):
+        """Return a list of dictionaries with the attributes of cached
+        versions"""
+        versions = []
+        for platform in os.listdir(cls.cache_root):
+            dname = osp.join(cls.cache_root, platform)
+            if osp.isdir(dname):
+                for version in os.listdir(dname):
+                    if osp.isdir(osp.join(dname, version)):
+                        cmd = LilyCmd.with_version(platform, version)
+                        versions.append(
+                            {'platform': platform,
+                             'version': version,
+                             'last_used': cmd._last_used(),
+                             'directory': osp.abspath(osp.join(dname, version))})
+        return versions