Skip to content

Commit 10825a4

Browse files
committed
Merge pull request #117 from openlilylib/fix-test-cache
test: Fix caching issues (fixes #111) Thank you for picking up this issue! The Arnold font is not yet part of the distribution from fonts.openlilylib.org. So the fail doesn't indicate anything (simply to confirm Matteo's statement). I simply skimmed the code (can't do more, as I'm on a Schubert-only weekend) and looked at the build-test log. So I think this can be merged.
2 parents 60c6c85 + bf3c847 commit 10825a4

File tree

5 files changed

+226
-81
lines changed

5 files changed

+226
-81
lines changed

.travis.yml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ cache:
2020
# Download and install LilyPond (stable and devel)
2121
# when not present already (cached between builds)
2222
install:
23+
- pip install python-dateutil
2324
- python ./test/install_lilypond.py
2425

2526
# This is the script that will actually run the tests

test/automated_tests.py

Lines changed: 35 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,16 @@
11
#!/usr/bin/env python
22

3-
import subprocess as sp
43
import os
54
import os.path as osp
65
import shutil
76
import sys
87
import re
8+
from lilycmd import LilyCmd
99

10-
from common_functions import print_separator, home_dir, install_root
10+
from common_functions import print_separator
1111

1212

13-
class SimpleTests:
13+
class SimpleTests(object):
1414
"""Run simple intergration tests. Specifically, this script will look
1515
for all the files in `usage-examples` directories. All these files
1616
will be compiled with LilyPond. If the compilation results in a
@@ -56,24 +56,25 @@ def __init__(self, cmd=None):
5656
# root directory
5757
self.openlilylib_dir = self.__openlilylib_dir()
5858

59-
# LilyPond command
60-
if self.is_ci_run():
61-
try:
62-
self.lily_command = osp.join(install_root,
63-
"bin",
64-
"lilypond")
65-
self.lilypond_version = self.__lilypond_version()
66-
except KeyError:
67-
sys.exit('Environment variable {} not set. Aborting'.format(self.lily_version_var))
59+
if 'CI' in os.environ and bool(os.environ["CI"]):
60+
# TODO check definition
61+
lily_platform = os.environ["LILY_PLATFORM"]
62+
lily_version = os.environ["LILY_VERSION"]
63+
64+
self.lily_command = LilyCmd.with_version(lily_platform,
65+
lily_version)
66+
if not self.lily_command.installed:
67+
raise Exception('The required lilypond version is not installed')
68+
self.lilypond_version = self.lily_command.version
6869
else:
69-
self.lily_command = cmd if cmd else "lilypond"
70-
self.lilypond_version = self.__lilypond_version()
70+
self.lily_command = LilyCmd.system(cmd if cmd else "lilypond")
71+
self.lilypond_version = self.lily_command.version
7172

7273
# Add include path and other options to generated LilyPond command
73-
self.lily_command_with_includes = [self.lily_command,
74-
"-dno-point-and-click",
75-
"-I", self.openlilylib_dir,
76-
"-I", os.path.join(self.openlilylib_dir, "ly")]
74+
self.lily_command_with_includes_args = [
75+
"-dno-point-and-click",
76+
"-I", self.openlilylib_dir,
77+
"-I", os.path.join(self.openlilylib_dir, "ly")]
7778
# initialize some lists
7879
self.test_files = []
7980
self.included_tests = []
@@ -102,14 +103,6 @@ def __collect_all_in_dir(self, dirname):
102103
if os.path.isfile(test_fname) and self.is_lilypond_file(test_fname):
103104
self.test_files.append(test_fname)
104105

105-
106-
def __lilypond_version(self):
107-
"""Determine the LilyPond version actually run by the command self.lily_command"""
108-
lily = sp.Popen([self.lily_command, "-v"], stdout=sp.PIPE, stderr=sp.PIPE)
109-
version_line = lily.communicate()[0].splitlines()[0]
110-
return re.search(r"\d+\.\d+\.\d+", version_line).group(0)
111-
112-
113106
def __openlilylib_dir(self):
114107
"""Return the root directory of openLilyLib.
115108
It's the parent directory of the script."""
@@ -228,7 +221,9 @@ def print_introduction(self):
228221
print "OpenLilyLib directory: {}".format(self.openlilylib_dir)
229222

230223
print "LilyPond command to be used:"
231-
print " ".join(self.lily_command_with_includes + ["-o <output-dir> <test-file>"])
224+
print " ".join([self.lily_command.command] +
225+
self.lily_command_with_includes_args +
226+
["-o <output-dir> <test-file>"])
232227

233228

234229
def report(self):
@@ -254,7 +249,8 @@ def report(self):
254249
print self.failed_tests[test]
255250
print ""
256251
print_separator()
257-
sys.exit(1)
252+
return 1
253+
return 0
258254

259255

260256
def run(self):
@@ -271,13 +267,11 @@ def run(self):
271267
os.path.dirname(self.__relative_path(test)))
272268
if not os.path.exists(test_result_dir):
273269
os.makedirs(test_result_dir)
274-
lily = sp.Popen(self.lily_command_with_includes + ['-o',
275-
test_result_dir,
276-
test],
277-
stdout=sp.PIPE, stderr=sp.PIPE)
278-
(out, err) = lily.communicate()
279-
280-
if lily.returncode == 0:
270+
returncode, out, err = self.lily_command.execute(
271+
self.lily_command_with_includes_args + ['-o',
272+
test_result_dir,
273+
test])
274+
if returncode == 0:
281275
print "------- OK! --------"
282276
else:
283277
# if test failed, add it to the list of failed tests to be reported later
@@ -300,4 +294,9 @@ def run(self):
300294
tests.clean_results_dir()
301295
tests.collect_tests()
302296
tests.run()
303-
tests.report()
297+
retcode = tests.report()
298+
299+
# cleanup old version of lilypond
300+
LilyCmd.clean_cache()
301+
302+
sys.exit(retcode)

test/common_functions.py

100644100755
Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,4 +32,3 @@ def load_lily_versions():
3232
def print_separator():
3333
print ""
3434
print "="*79, "\n"
35-

test/install_lilypond.py

100644100755
Lines changed: 3 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,10 @@
11
#!/usr/bin/env python
22

3-
import subprocess as sp
43
import os
5-
import os.path as osp
6-
import shutil
74
import sys
8-
import collections
95

10-
import common_functions
11-
from common_functions import print_separator, home_dir, install_root
6+
from lilycmd import LilyCmd
7+
from common_functions import print_separator
128

139
#############################################################
1410
# Load environment variables
@@ -21,43 +17,6 @@
2117
except:
2218
sys.exit('\nScript can only be run in CI mode. Aborting\n')
2319

24-
#########################
25-
# Configuration constants
26-
27-
# Download site for LilyPond distributions
28-
binary_site = "http://download.linuxaudio.org/lilypond/binaries/"
29-
# String template for generating the LilyPond installation command
30-
lily_install_script = "lilypond-install.sh"
31-
32-
33-
#################################
34-
# Functions doing the actual work
35-
36-
def download_url():
37-
"""Format a string representing the URL to download the requested LilyPond distribution"""
38-
return "{}{}/lilypond-{}.{}.sh".format(
39-
binary_site, lily_platform, lily_version, lily_platform)
40-
41-
def install_distribution():
42-
"""Download and install LilyPond version if not cached"""
43-
lilypond_cmd = os.path.join(install_root,
44-
"bin/lilypond")
45-
print "\nChecking LilyPond presence with {}\n".format(lilypond_cmd)
46-
try:
47-
sp.check_call([lilypond_cmd, '--version'])
48-
print "LilyPond {} is already installed in cache, continuing with test script.".format(lily_version)
49-
except:
50-
print "LilyPond {} is not installed yet.".format(lily_version)
51-
print "Downloading and installing now"
52-
sp.check_call(
53-
["wget", "-O",
54-
lily_install_script,
55-
download_url()])
56-
sp.check_call(["sh", lily_install_script,
57-
"--prefix",
58-
install_root,
59-
"--batch"])
60-
6120
#########################
6221
# Actual script execution
6322

@@ -70,4 +29,4 @@ def install_distribution():
7029
print "check LilyPond installation."
7130
print "Requested LilyPond version: {}".format(lily_version)
7231

73-
install_distribution()
32+
LilyCmd.install(lily_platform, lily_version)

test/lilycmd.py

Lines changed: 187 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,187 @@
1+
import subprocess as sp
2+
import os
3+
import os.path as osp
4+
import shutil
5+
import sys
6+
import re
7+
import datetime
8+
import dateutil.parser
9+
from common_functions import print_separator
10+
11+
class LilyCmd(object):
12+
"""This class represents a lilypond command and provides some
13+
facilities to
14+
15+
- run LilyPond
16+
- install LilyPond versions from the internet into a local cache
17+
- manage the cache, cleaning old versions
18+
19+
"""
20+
21+
# Download site for LilyPond distributions
22+
binary_site = "http://download.linuxaudio.org/lilypond/binaries/"
23+
# String template for generating the LilyPond installation command
24+
lily_install_script = "lilypond-install.sh"
25+
26+
# After this amount of time a LilyPond version will be removed by
27+
# a call to LilyCmd.clean_cache()
28+
cache_cleanup_interval = datetime.timedelta(days=1)
29+
30+
# Root directory for the cache
31+
cache_root = osp.join(os.getenv('HOME'), '.lilypond')
32+
33+
def __init__(self, command_path, cached):
34+
self.command = command_path
35+
self.cached = cached
36+
try:
37+
self.version = self._lilypond_version()
38+
self.installed = True
39+
except:
40+
self.installed = False
41+
42+
####################################################################
43+
# Public methods
44+
45+
@staticmethod
46+
def system(cmd_path='lilypond'):
47+
"""Get a new instance of Lilypond provided by the
48+
system. (eg. /usr/bin/lilypond)"""
49+
return LilyCmd(cmd_path, cached=False)
50+
51+
@classmethod
52+
def with_version(cls, platform, version):
53+
"""Get a new version of Lilypond, given the platform and the
54+
version.
55+
56+
**Note**: This command does not install LilyPond in the local
57+
cache. To check if the instance returned by this method
58+
corresponds to an actually installed LilyPond version, check
59+
the `installed` attribute. The reason for not installing
60+
LilyPond with this command is that we may be interested in all
61+
kind of ancillary information (like the cache directory)
62+
without actually installing LilyPond.
63+
64+
"""
65+
lily_cmd_path = osp.join(
66+
LilyCmd._cache_directory(platform, version),
67+
'bin', 'lilypond')
68+
lily_cmd = LilyCmd(lily_cmd_path, cached=True)
69+
return lily_cmd
70+
71+
def execute(self, args):
72+
"""Executes the LilyPond command with the given arguments"""
73+
self._mark_cache()
74+
lily = sp.Popen([self.command] + args,
75+
stdout=sp.PIPE, stderr=sp.PIPE)
76+
(out, err) = lily.communicate()
77+
return lily.returncode, out, err
78+
79+
@classmethod
80+
def install(cls, platform, version):
81+
"""Download and install LilyPond version if not cached"""
82+
lilypond_cmd = LilyCmd.with_version(platform, version)
83+
print "\nChecking LilyPond presence"
84+
if lilypond_cmd.installed:
85+
print ("LilyPond {} is already installed in cache," \
86+
+" continuing with test script.").format(
87+
lilypond_cmd.version)
88+
else:
89+
print "LilyPond {} is not installed yet.".format(version)
90+
print "Downloading and installing now"
91+
sp.check_call(
92+
["wget", "-O",
93+
cls.lily_install_script,
94+
cls._download_url(platform, version)])
95+
sp.check_call(["sh", cls.lily_install_script,
96+
"--prefix",
97+
LilyCmd._cache_directory(platform, version),
98+
"--batch"])
99+
100+
@classmethod
101+
def clean_cache(cls):
102+
"""Clean the cache from versions of Lilypond older than
103+
`cache_cleanup_interval`"""
104+
print "Clean cache\n"
105+
cached = cls._get_cached_versions()
106+
now = datetime.datetime.now()
107+
for lily in cached:
108+
if lily['last_used'] is None:
109+
print 'Removing cached LilyPond', lily['version'],\
110+
lily['platform'], '(never used)'
111+
shutil.rmtree(lily['directory'])
112+
elif now - lily['last_used'] > cls.cache_cleanup_interval:
113+
print 'Removing cached LilyPond', lily['version'],\
114+
lily['platform'], '(last used', \
115+
lily['last_used'].isoformat(), ')'
116+
shutil.rmtree(lily['directory'])
117+
else:
118+
print 'Keeping cached LilyPond', lily['version'],\
119+
lily['platform'], '(last used', \
120+
lily['last_used'].isoformat(), ')'
121+
122+
####################################################################
123+
# Private members
124+
125+
@classmethod
126+
def _cache_directory(cls, platform, version):
127+
"""Get the cache directory name for the given platform and version"""
128+
return osp.join(cls.cache_root, platform, version)
129+
130+
def _lilypond_version(self):
131+
"""Determine the LilyPond version actually run
132+
by the command self.lily_command"""
133+
lily = sp.Popen([self.command, "--version"],
134+
stdout=sp.PIPE, stderr=sp.PIPE)
135+
version_line = lily.communicate()[0].splitlines()[0]
136+
return re.search(r"\d+\.\d+\.\d+", version_line).group(0)
137+
138+
def _mark_cache_file(self):
139+
"""Get the name of the file that will store the timestamp of the last
140+
time this command has been used."""
141+
if self.cached:
142+
return osp.join(osp.dirname(self.command), '.oll-last-used')
143+
else:
144+
return None
145+
146+
def _mark_cache(self):
147+
"""Write the timestamp of now in the cache file"""
148+
if self.cached:
149+
with open(self._mark_cache_file(), 'w') as mark_file:
150+
mark_file.write(datetime.datetime.now().isoformat())
151+
152+
def _last_used(self):
153+
"""Returns the last time this command has been used, or None if it was
154+
never used."""
155+
if self.cached:
156+
if not osp.isfile(self._mark_cache_file()):
157+
return None
158+
with open(self._mark_cache_file(), 'r') as mark_file:
159+
fcontent = mark_file.readline()
160+
return dateutil.parser.parse(fcontent)
161+
162+
163+
@classmethod
164+
def _download_url(cls, lily_platform, lily_version):
165+
"""Format a string representing the URL to
166+
download the requested LilyPond distribution"""
167+
return "{}{}/lilypond-{}.{}.sh".format(
168+
cls.binary_site, lily_platform, lily_version, lily_platform)
169+
170+
171+
@classmethod
172+
def _get_cached_versions(cls):
173+
"""Return a list of dictionaries with the attributes of cached
174+
versions"""
175+
versions = []
176+
for platform in os.listdir(cls.cache_root):
177+
dname = osp.join(cls.cache_root, platform)
178+
if osp.isdir(dname):
179+
for version in os.listdir(dname):
180+
if osp.isdir(osp.join(dname, version)):
181+
cmd = LilyCmd.with_version(platform, version)
182+
versions.append(
183+
{'platform': platform,
184+
'version': version,
185+
'last_used': cmd._last_used(),
186+
'directory': osp.abspath(osp.join(dname, version))})
187+
return versions

0 commit comments

Comments
 (0)