support hdf5 serialization/unserialization of all the world data

.travis.yml

@@ -36,6 +36,7 @@ addons:
     - cmake
     - build-essential
     - libgdal-dev
+    - libhdf5-serial-dev
 
 install:
   - echo "libgdal version `gdal-config --version`"

appveyor.yml

@@ -43,6 +43,7 @@ init:
   - echo %path%
 
 install:
+  - "powershell appveyor\\install_hdf5.ps1"
   - "powershell appveyor\\install.ps1"
   - "%PYTHON%\\Scripts\\pip.exe install -r requirements-dev.txt"
   - mkdir %systemdrive%\temp

appveyor/install_hdf5.ps1

@@ -0,0 +1,24 @@
+$URL = http://www.hdfgroup.org/ftp/HDF5/current/bin/windows/hdf5-1.8.15-patch1-win32-vs2013-shared.zip
+
+function main () {
+    $basedir = $pwd.Path + "\"
+    $filename = "hdf5.zip"
+    $filepath = $basedir + $filename
+    Write-Host "Downloading" $filename "from" $URL
+    $retry_attempts = 3
+    for($i=0; $i -lt $retry_attempts; $i++){
+        try {
+            $webclient.DownloadFile($URL, $filepath)
+            break
+        }
+        Catch [Exception]{
+            Start-Sleep 1
+        }
+    }
+    $outpath = $basedir + "\hdf5_unzipped"
+    [System.IO.Compression.ZipFile]::ExtractToDirectory($filepath, $outpath)
+    $msipath = $outpath + "\HDF5-1.8.15-win64.msi"
+    Invoke-Command -ScriptBlock { & cmd /c "msiexec.exe /i $msipath" /qn ADVANCED_OPTIONS=1 CHANNEL=100}
+}
+
+main

requirements-hdf5.txt

@@ -0,0 +1,3 @@
+# For export support, you'll need HDF5
+-r requirements.txt
+h5py==2.5.0

setup.py

@@ -25,7 +25,7 @@
     'entry_points': {
         'console_scripts': ['worldengine=worldengine.cli.main:main'],
     },
-    'install_requires': ['PyPlatec==1.4.0', 'pypng>=0.0.18', 'numpy>=1.9.2',
+    'install_requires': ['PyPlatec==1.4.0', 'pypng>=0.0.18', 'numpy>=1.9.2, <= 1.10.0.post2',
                          'argparse==1.2.1', 'noise==1.2.2', 'protobuf>=2.6.0'],
     'license': 'MIT License'
 }

tests/serialization_test.py

@@ -2,6 +2,10 @@
 from worldengine.plates import Step, world_gen
 from worldengine.world import World
 from worldengine.common import _equal
+import tempfile
+import os
+from worldengine.hdf5_serialization import save_world_to_hdf5, load_world_to_hdf5
+
 
 class TestSerialization(unittest.TestCase):
 
@@ -32,6 +36,41 @@ def test_protobuf_serialize_unserialize(self):
         self.assertEqual(sorted(dir(w)),            sorted(dir(unserialized)))
         self.assertEqual(w, unserialized)
 
+    def test_hdf5_serialize_unserialize(self):
+        filename = None
+        try:
+            w = world_gen("Dummy", 32, 16, 1, step=Step.get_by_name("full"))
+            f = tempfile.NamedTemporaryFile(delete=False)
+            f.close()
+            filename = f.name
+            serialized = save_world_to_hdf5(w, filename)
+            unserialized = load_world_to_hdf5(filename)
+            self.assertTrue(_equal(w.elevation['data'], unserialized.elevation['data']))
+            self.assertEqual(w.elevation['thresholds'], unserialized.elevation['thresholds'])
+            self.assertTrue(_equal(w.ocean,             unserialized.ocean))
+            self.assertTrue(_equal(w.biome,             unserialized.biome))
+            self.assertTrue(_equal(w.humidity['quantiles'], unserialized.humidity['quantiles']))
+            self.assertTrue(_equal(w.humidity['data'],  unserialized.humidity['data']))
+            self.assertTrue(_equal(w.humidity,          unserialized.humidity))
+            self.assertTrue(_equal(w.irrigation,        unserialized.irrigation))
+            self.assertTrue(_equal(w.permeability,      unserialized.permeability))
+            self.assertTrue(_equal(w.watermap,          unserialized.watermap))
+            self.assertTrue(_equal(w.precipitation['thresholds'], unserialized.precipitation['thresholds']))
+            self.assertTrue(_equal(w.precipitation['data'], unserialized.precipitation['data']))
+            self.assertTrue(_equal(w.precipitation,     unserialized.precipitation))
+            self.assertTrue(_equal(w.temperature,       unserialized.temperature))
+            self.assertTrue(_equal(w.sea_depth,         unserialized.sea_depth))
+            self.assertEquals(w.seed,                   unserialized.seed)
+            self.assertEquals(w.n_plates,               unserialized.n_plates)
+            self.assertTrue(_equal(w.ocean_level,       unserialized.ocean_level))
+            self.assertTrue(_equal(w.lake_map,          unserialized.lake_map))
+            self.assertTrue(_equal(w.river_map,         unserialized.river_map))
+            self.assertEquals(w.step,                   unserialized.step)
+            self.assertEqual(sorted(dir(w)),            sorted(dir(unserialized)))
+            #self.assertEqual(w, unserialized)
+        finally:
+            if filename:
+                os.remove(filename)
 
 if __name__ == '__main__':
     unittest.main()

tox.ini

@@ -11,11 +11,12 @@ deps =
     protobuf
     six
     pypng
+    h5py
 
 [testenv]
 deps =
     coverage
-    numpy
+    numpy==1.9.2
     pygdal==1.10.0.1
     {[base]deps}
 

worldengine/cli/main.py

@@ -12,6 +12,11 @@
 from worldengine.step import Step
 from worldengine.world import World
 from worldengine.version import __version__
+try:
+    from worldengine.hdf5_serialization import save_world_to_hdf5
+    HDF5_AVAILABLE = True
+except:
+    HDF5_AVAILABLE = False
 
 VERSION = __version__
 
@@ -34,11 +39,13 @@ def generate_world(world_name, width, height, seed, num_plates, output_dir,
 
     # Save data
     filename = "%s/%s.world" % (output_dir, world_name)
-    with open(filename, "wb") as f:
-        if world_format == 'protobuf':
+    if world_format == 'protobuf':
+        with open(filename, "wb") as f:
             f.write(w.protobuf_serialize())
-        else:
-            print("Unknown format '%s', not saving " % world_format)
+    elif world_format == 'hdf5':
+        save_world_to_hdf5(w, filename)
+    else:
+        print("Unknown format '%s', not saving " % world_format)
     print("* world data saved in '%s'" % filename)
     sys.stdout.flush()
 
@@ -228,6 +235,11 @@ def main():
              "a name is not provided, then seed_N.world, " +
              "where N=SEED",
         metavar="STR")
+    parser.add_argument('--hdf5', dest='hdf5',
+                        action="store_true",
+                        help="Save world file using HDF5 format. " +
+                             "Default = store using protobuf format",
+                        default=False)
     parser.add_argument('-s', '--seed', dest='seed', type=int,
                         help="Use seed=N to initialize the pseudo-random " +
                              "generation. If not provided, one will be " +
@@ -372,6 +384,9 @@ def main():
     if args.number_of_plates < 1 or args.number_of_plates > 100:
         usage(error="Number of plates should be in [1, 100]")
 
+    if args.hdf5 and not HDF5_AVAILABLE:
+        usage(error="HDF5 requires the presence of native libraries")
+
     operation = "world"
     if args.OPERATOR is None:
         pass
@@ -404,6 +419,8 @@ def main():
     step = check_step(args.step)
 
     world_format = 'protobuf'
+    if args.hdf5:
+        world_format = 'hdf5'
 
     generation_operation = (operation == 'world') or (operation == 'plates')