diff --git a/.travis.yml b/.travis.yml index 6206feb2..bac28128 100644 --- a/.travis.yml +++ b/.travis.yml @@ -36,6 +36,7 @@ addons: - cmake - build-essential - libgdal-dev + - libhdf5-serial-dev install: - echo "libgdal version `gdal-config --version`" diff --git a/appveyor.yml b/appveyor.yml index 99642f78..53e71d5d 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -43,6 +43,7 @@ init: - echo %path% install: + - "powershell appveyor\\install_hdf5.ps1" - "powershell appveyor\\install.ps1" - "%PYTHON%\\Scripts\\pip.exe install -r requirements-dev.txt" - mkdir %systemdrive%\temp diff --git a/appveyor/install_hdf5.ps1 b/appveyor/install_hdf5.ps1 new file mode 100644 index 00000000..65e1eafe --- /dev/null +++ b/appveyor/install_hdf5.ps1 @@ -0,0 +1,24 @@ +$URL = http://www.hdfgroup.org/ftp/HDF5/current/bin/windows/hdf5-1.8.15-patch1-win32-vs2013-shared.zip + +function main () { + $basedir = $pwd.Path + "\" + $filename = "hdf5.zip" + $filepath = $basedir + $filename + Write-Host "Downloading" $filename "from" $URL + $retry_attempts = 3 + for($i=0; $i -lt $retry_attempts; $i++){ + try { + $webclient.DownloadFile($URL, $filepath) + break + } + Catch [Exception]{ + Start-Sleep 1 + } + } + $outpath = $basedir + "\hdf5_unzipped" + [System.IO.Compression.ZipFile]::ExtractToDirectory($filepath, $outpath) + $msipath = $outpath + "\HDF5-1.8.15-win64.msi" + Invoke-Command -ScriptBlock { & cmd /c "msiexec.exe /i $msipath" /qn ADVANCED_OPTIONS=1 CHANNEL=100} +} + +main \ No newline at end of file diff --git a/requirements-hdf5.txt b/requirements-hdf5.txt new file mode 100644 index 00000000..a963e2a1 --- /dev/null +++ b/requirements-hdf5.txt @@ -0,0 +1,3 @@ +# For export support, you'll need HDF5 +-r requirements.txt +h5py==2.5.0 diff --git a/setup.py b/setup.py index e4e90a0c..b7c6a7ea 100644 --- a/setup.py +++ b/setup.py @@ -25,7 +25,7 @@ 'entry_points': { 'console_scripts': ['worldengine=worldengine.cli.main:main'], }, - 'install_requires': ['PyPlatec==1.4.0', 'pypng>=0.0.18', 'numpy>=1.9.2', + 'install_requires': ['PyPlatec==1.4.0', 'pypng>=0.0.18', 'numpy>=1.9.2, <= 1.10.0.post2', 'argparse==1.2.1', 'noise==1.2.2', 'protobuf>=2.6.0'], 'license': 'MIT License' } diff --git a/tests/serialization_test.py b/tests/serialization_test.py index 9a482359..011f0829 100644 --- a/tests/serialization_test.py +++ b/tests/serialization_test.py @@ -2,6 +2,10 @@ from worldengine.plates import Step, world_gen from worldengine.world import World from worldengine.common import _equal +import tempfile +import os +from worldengine.hdf5_serialization import save_world_to_hdf5, load_world_to_hdf5 + class TestSerialization(unittest.TestCase): @@ -32,6 +36,41 @@ def test_protobuf_serialize_unserialize(self): self.assertEqual(sorted(dir(w)), sorted(dir(unserialized))) self.assertEqual(w, unserialized) + def test_hdf5_serialize_unserialize(self): + filename = None + try: + w = world_gen("Dummy", 32, 16, 1, step=Step.get_by_name("full")) + f = tempfile.NamedTemporaryFile(delete=False) + f.close() + filename = f.name + serialized = save_world_to_hdf5(w, filename) + unserialized = load_world_to_hdf5(filename) + self.assertTrue(_equal(w.elevation['data'], unserialized.elevation['data'])) + self.assertEqual(w.elevation['thresholds'], unserialized.elevation['thresholds']) + self.assertTrue(_equal(w.ocean, unserialized.ocean)) + self.assertTrue(_equal(w.biome, unserialized.biome)) + self.assertTrue(_equal(w.humidity['quantiles'], unserialized.humidity['quantiles'])) + self.assertTrue(_equal(w.humidity['data'], unserialized.humidity['data'])) + self.assertTrue(_equal(w.humidity, unserialized.humidity)) + self.assertTrue(_equal(w.irrigation, unserialized.irrigation)) + self.assertTrue(_equal(w.permeability, unserialized.permeability)) + self.assertTrue(_equal(w.watermap, unserialized.watermap)) + self.assertTrue(_equal(w.precipitation['thresholds'], unserialized.precipitation['thresholds'])) + self.assertTrue(_equal(w.precipitation['data'], unserialized.precipitation['data'])) + self.assertTrue(_equal(w.precipitation, unserialized.precipitation)) + self.assertTrue(_equal(w.temperature, unserialized.temperature)) + self.assertTrue(_equal(w.sea_depth, unserialized.sea_depth)) + self.assertEquals(w.seed, unserialized.seed) + self.assertEquals(w.n_plates, unserialized.n_plates) + self.assertTrue(_equal(w.ocean_level, unserialized.ocean_level)) + self.assertTrue(_equal(w.lake_map, unserialized.lake_map)) + self.assertTrue(_equal(w.river_map, unserialized.river_map)) + self.assertEquals(w.step, unserialized.step) + self.assertEqual(sorted(dir(w)), sorted(dir(unserialized))) + #self.assertEqual(w, unserialized) + finally: + if filename: + os.remove(filename) if __name__ == '__main__': unittest.main() diff --git a/tox.ini b/tox.ini index 1cef3dea..6a44a172 100644 --- a/tox.ini +++ b/tox.ini @@ -11,11 +11,12 @@ deps = protobuf six pypng + h5py [testenv] deps = coverage - numpy + numpy==1.9.2 pygdal==1.10.0.1 {[base]deps} diff --git a/worldengine/cli/main.py b/worldengine/cli/main.py index c15b1030..5a27c08e 100644 --- a/worldengine/cli/main.py +++ b/worldengine/cli/main.py @@ -12,6 +12,11 @@ from worldengine.step import Step from worldengine.world import World from worldengine.version import __version__ +try: + from worldengine.hdf5_serialization import save_world_to_hdf5 + HDF5_AVAILABLE = True +except: + HDF5_AVAILABLE = False VERSION = __version__ @@ -34,11 +39,13 @@ def generate_world(world_name, width, height, seed, num_plates, output_dir, # Save data filename = "%s/%s.world" % (output_dir, world_name) - with open(filename, "wb") as f: - if world_format == 'protobuf': + if world_format == 'protobuf': + with open(filename, "wb") as f: f.write(w.protobuf_serialize()) - else: - print("Unknown format '%s', not saving " % world_format) + elif world_format == 'hdf5': + save_world_to_hdf5(w, filename) + else: + print("Unknown format '%s', not saving " % world_format) print("* world data saved in '%s'" % filename) sys.stdout.flush() @@ -228,6 +235,11 @@ def main(): "a name is not provided, then seed_N.world, " + "where N=SEED", metavar="STR") + parser.add_argument('--hdf5', dest='hdf5', + action="store_true", + help="Save world file using HDF5 format. " + + "Default = store using protobuf format", + default=False) parser.add_argument('-s', '--seed', dest='seed', type=int, help="Use seed=N to initialize the pseudo-random " + "generation. If not provided, one will be " + @@ -372,6 +384,9 @@ def main(): if args.number_of_plates < 1 or args.number_of_plates > 100: usage(error="Number of plates should be in [1, 100]") + if args.hdf5 and not HDF5_AVAILABLE: + usage(error="HDF5 requires the presence of native libraries") + operation = "world" if args.OPERATOR is None: pass @@ -404,6 +419,8 @@ def main(): step = check_step(args.step) world_format = 'protobuf' + if args.hdf5: + world_format = 'hdf5' generation_operation = (operation == 'world') or (operation == 'plates') diff --git a/worldengine/hdf5_serialization.py b/worldengine/hdf5_serialization.py new file mode 100644 index 00000000..a9f944d0 --- /dev/null +++ b/worldengine/hdf5_serialization.py @@ -0,0 +1,220 @@ +import h5py +from worldengine.version import __version__ +from worldengine.biome import biome_name_to_index, biome_index_to_name +from worldengine.world import World, Step +import numpy + + +def save_world_to_hdf5(world, filename): + f = h5py.File(filename, libver='latest', mode='w') + + general_grp = f.create_group("general") + general_grp["worldengine_version"] = __version__ + general_grp["name"] = world.name + general_grp["width"] = world.width + general_grp["height"] = world.height + + elevation_grp = f.create_group("elevation") + elevation_ths_grp = elevation_grp.create_group("thresholds") + elevation_ths_grp["sea"] = world.elevation['thresholds'][0][1] + elevation_ths_grp["plain"] = world.elevation['thresholds'][1][1] + elevation_ths_grp["hill"] = world.elevation['thresholds'][2][1] + elevation_data = elevation_grp.create_dataset("data", (world.height, world.width), dtype=numpy.float) + elevation_data.write_direct(world.elevation['data']) + + plates_data = f.create_dataset("plates", (world.height, world.width), dtype=numpy.uint16) + for y in range(world.height): + for x in range(world.width): + plates_data[y, x] = world.plates[y][x] + + ocean_data = f.create_dataset("ocean", (world.height, world.width), dtype=numpy.bool) + ocean_data.write_direct(world.ocean) + + sea_depth_data = f.create_dataset("sea_depth", (world.height, world.width), dtype=numpy.float) + sea_depth_data.write_direct(world.sea_depth) + + if hasattr(world, 'biome'): + biome_data = f.create_dataset("biome", (world.height, world.width), dtype=numpy.uint16) + for y in range(world.height): + for x in range(world.width): + biome_data[y, x] = biome_name_to_index(world.biome[y][x]) + + if hasattr(world, 'humidity'): + humidity_grp = f.create_group("humidity") + humidity_quantiles_grp = humidity_grp.create_group("quantiles") + for k in world.humidity['quantiles'].keys(): + humidity_quantiles_grp[k] = world.humidity['quantiles'][k] + humidity_data = humidity_grp.create_dataset("data", (world.height, world.width), dtype=numpy.float) + humidity_data.write_direct(world.humidity['data']) + + if hasattr(world, 'irrigation'): + irrigation_data = f.create_dataset("irrigation", (world.height, world.width), dtype=numpy.float) + irrigation_data.write_direct(world.irrigation) + + if hasattr(world, 'permeability'): + permeability_grp = f.create_group("permeability") + permeability_ths_grp = permeability_grp.create_group("thresholds") + permeability_ths_grp['low'] = world.permeability['thresholds'][0][1] + permeability_ths_grp['med'] = world.permeability['thresholds'][1][1] + permeability_data = permeability_grp.create_dataset("data", (world.height, world.width), dtype=numpy.float) + permeability_data.write_direct(world.permeability['data']) + + if hasattr(world, 'watermap'): + watermap_grp = f.create_group("watermap") + watermap_ths_grp = watermap_grp.create_group("thresholds") + watermap_ths_grp['creek'] = world.watermap['thresholds']['creek'] + watermap_ths_grp['river'] = world.watermap['thresholds']['river'] + watermap_ths_grp['mainriver'] = world.watermap['thresholds']['main river'] + watermap_data = watermap_grp.create_dataset("data", (world.height, world.width), dtype=numpy.float) + watermap_data.write_direct(world.watermap['data']) + + if hasattr(world, 'precipitation'): + precipitation_grp = f.create_group("precipitation") + precipitation_ths_grp = precipitation_grp.create_group("thresholds") + precipitation_ths_grp['low'] = world.precipitation['thresholds'][0][1] + precipitation_ths_grp['med'] = world.precipitation['thresholds'][1][1] + precipitation_data = precipitation_grp.create_dataset("data", (world.height, world.width), dtype=numpy.float) + precipitation_data.write_direct(world.precipitation['data']) + + if hasattr(world, 'temperature'): + temperature_grp = f.create_group("temperature") + temperature_ths_grp = temperature_grp.create_group("thresholds") + temperature_ths_grp['polar'] = world.temperature['thresholds'][0][1] + temperature_ths_grp['alpine'] = world.temperature['thresholds'][1][1] + temperature_ths_grp['boreal'] = world.temperature['thresholds'][2][1] + temperature_ths_grp['cool'] = world.temperature['thresholds'][3][1] + temperature_ths_grp['warm'] = world.temperature['thresholds'][4][1] + temperature_ths_grp['subtropical'] = world.temperature['thresholds'][5][1] + temperature_data = temperature_grp.create_dataset("data", (world.height, world.width), dtype=numpy.float) + temperature_data.write_direct(world.temperature['data']) + + # lake_map and river_map have inverted coordinates + if hasattr(world, 'lake_map'): + lake_map_data = f.create_dataset("lake_map", (world.height, world.width), dtype=numpy.float) + lake_map_data.write_direct(world.lake_map) + + # lake_map and river_map have inverted coordinates + if hasattr(world, 'river_map'): + river_map_data = f.create_dataset("river_map", (world.height, world.width), dtype=numpy.float) + river_map_data.write_direct(world.river_map) + + generation_params_grp = f.create_group("generation_params") + generation_params_grp['seed'] = world.seed + generation_params_grp['n_plates'] = world.n_plates + generation_params_grp['ocean_level'] = world.ocean_level + generation_params_grp['step'] = world.step.name + + f.close() + + +def _from_hdf5_quantiles(p_quantiles): + quantiles = {} + for p_quantile in p_quantiles: + quantiles[p_quantile.title()] = p_quantiles[p_quantile].value + return quantiles + + +def _from_hdf5_matrix_with_quantiles(p_matrix): + matrix = dict() + matrix['data'] = p_matrix['data'] + matrix['quantiles'] = _from_hdf5_quantiles(p_matrix['quantiles']) + return matrix + + +def load_world_to_hdf5(filename): + f = h5py.File(filename, libver='latest', mode='r') + + w = World(f['general/name'].value, + f['general/width'].value, + f['general/height'].value, + f['generation_params/seed'].value, + f['generation_params/n_plates'].value, + f['generation_params/ocean_level'].value, + Step.get_by_name(f['generation_params/step'].value)) + + # Elevation + e = numpy.array(f['elevation/data']) + e_th = [('sea', f['elevation/thresholds/sea'].value), + ('plain', f['elevation/thresholds/plain'].value), + ('hill', f['elevation/thresholds/hill'].value), + ('mountain', None)] + w.set_elevation(e, e_th) + + # Plates + w.set_plates(numpy.array(f['plates'])) + + # Ocean + w.set_ocean(numpy.array(f['ocean'])) + w.sea_depth = numpy.array(f['sea_depth']) + + # Biome + if 'biome' in f.keys(): + biome_data = [] + for y in range(w.height): + row = [] + for x in range(w.width): + value = f['biome'][y, x] + row.append(biome_index_to_name(value)) + biome_data.append(row) + biome = numpy.array(biome_data, dtype=object) + w.set_biome(biome) + + # Humidity + # FIXME: use setters + if 'humidity' in f.keys(): + w.humidity = _from_hdf5_matrix_with_quantiles(f['humidity']) + w.humidity['data'] = numpy.array(w.humidity['data']) # numpy conversion + + if 'irrigation' in f.keys(): + w.irrigation = numpy.array(f['irrigation']) + + if 'permeability' in f.keys(): + p = numpy.array(f['permeability/data']) + p_th = [ + ('low', f['permeability/thresholds/low'].value), + ('med', f['permeability/thresholds/med'].value), + ('hig', None) + ] + w.set_permeability(p, p_th) + + if 'watermap' in f.keys(): + w.watermap = dict() + w.watermap['data'] = numpy.array(f['watermap/data']) + w.watermap['thresholds'] = {} + w.watermap['thresholds']['creek'] = f['watermap/thresholds/creek'].value + w.watermap['thresholds']['river'] = f['watermap/thresholds/river'].value + w.watermap['thresholds']['main river'] = f['watermap/thresholds/mainriver'].value + + if 'precipitation' in f.keys(): + p = numpy.array(f['precipitation/data']) + p_th = [ + ('low', f['precipitation/thresholds/low'].value), + ('med', f['precipitation/thresholds/med'].value), + ('hig', None) + ] + w.set_precipitation(p, p_th) + + if 'temperature' in f.keys(): + t = numpy.array(f['temperature/data']) + t_th = [ + ('polar', f['temperature/thresholds/polar'].value), + ('alpine', f['temperature/thresholds/alpine'].value), + ('boreal', f['temperature/thresholds/boreal'].value), + ('cool', f['temperature/thresholds/cool'].value), + ('warm', f['temperature/thresholds/warm'].value), + ('subtropical', f['temperature/thresholds/subtropical'].value), + ('tropical', None) + ] + w.set_temperature(t, t_th) + + if 'lake_map' in f.keys(): + m = numpy.array(f['lake_map']) + w.set_lakemap(m) + + if 'river_map' in f.keys(): + m = numpy.array(f['river_map']) + w.set_rivermap(m) + + f.close() + + return w \ No newline at end of file diff --git a/worldengine/world.py b/worldengine/world.py index a46ae383..5eb006b0 100644 --- a/worldengine/world.py +++ b/worldengine/world.py @@ -16,6 +16,7 @@ from worldengine.common import _equal from worldengine.version import __version__ + class World(object): """A world composed by name, dimensions and all the characteristics of each cell.