From 04690d8bdfbb1f03ebd741e38a311628d2ad9535 Mon Sep 17 00:00:00 2001 From: Andy Smith Date: Sat, 4 Apr 2015 23:50:30 +0100 Subject: [PATCH 01/14] tests for create dir and shpfile --- osm2ma/test_ogrwrapper.py | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/osm2ma/test_ogrwrapper.py b/osm2ma/test_ogrwrapper.py index 4e4b47b..69e4ea0 100644 --- a/osm2ma/test_ogrwrapper.py +++ b/osm2ma/test_ogrwrapper.py @@ -1,25 +1,40 @@ # This file was originally generated by PyScripter's unit test wizard import unittest +import tempfile +import shutil +import os import ogrwrapper as ogrw from osgeo import ogr +from osgeo import osr class TestGlobalFunctions(unittest.TestCase): def setUp(self): - pass + self.tmpdir = tempfile.mkdtemp() def tearDown(self): - pass + shutil.rmtree(self.tmpdir, ignore_errors=True) - @unittest.skip("not implemented") def test_create_datacat_dir(self): - pass + test_data_cat = "test_data_cat" + # test trying to create shpfile twice (shouldn't error) + for x in range(0,2): + created_path = ogrw._create_datacat_dir(self.tmpdir, test_data_cat) + self.assertEqual(created_path, os.path.join(self.tmpdir, test_data_cat)) + self.assertTrue(os.path.exists(created_path)) - @unittest.skip("not implemented") def test_create_new_shpfile(self): - pass + test_shpfile = "test_shpfile" + dest_srs = osr.SpatialReference() + dest_srs.ImportFromEPSG(3857) # from EPSG + + # test trying to create shpfile twice (shouldn't error) + for x in range(0,2): + ogrw._create_new_shpfile(test_shpfile+".shp", self.tmpdir, ogr.wkbLineString, dest_srs) + for extn in (u'.dbf', u'.prj', u'.shp', u'.shx'): + self.assertTrue(os.path.exists(os.path.join(self.tmpdir, test_shpfile + extn))) @unittest.skip("not implemented") def test_copy_attributes(self): From 189ce39801f0e9cfe62fc46cfea2a75d8a5303a4 Mon Sep 17 00:00:00 2001 From: Andy Smith Date: Tue, 7 Apr 2015 00:01:01 +0100 Subject: [PATCH 02/14] test for batch_convert --- osm2ma/fixtures.py | 45 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/osm2ma/fixtures.py b/osm2ma/fixtures.py index aa70e7e..0f25fb6 100644 --- a/osm2ma/fixtures.py +++ b/osm2ma/fixtures.py @@ -15,6 +15,11 @@ def path_to_fixtures_xls(): test_script_path = os.path.abspath(os.path.dirname(__file__)) return os.path.join(test_script_path, r"testfiles", r"fixtures.xls") +example_pbf = os.path.join(os.path.abspath(os.path.dirname(__file__)), + r"testfiles", + r"oxfordshire-latest.osm.pbf" + ) + _workbook = xlrd.open_workbook(os.path.realpath(path_to_fixtures_xls())) rawconf_good = _workbook.name_map.get("rawconf_good")[0].area2d(clipped=True) rawconf_invalid_heirarchy = _workbook.name_map.get("rawconf_invalid_heirarchy")[0].area2d(clipped=True) @@ -129,3 +134,43 @@ def path_to_fixtures_xls(): u'' ) ] + +output_dir_listing = [ + [ + ".", + ['admn', 'osms', 'tran'], + [] + ], + [ + "admn", + [], + ['wrl_admn_ad_ln_su_osm_pp.dbf', 'wrl_admn_ad_ln_su_osm_pp.prj', + 'wrl_admn_ad_ln_su_osm_pp.shp', 'wrl_admn_ad_ln_su_osm_pp.shx', + 'wrl_admn_ad_pt_su_osm_pp.dbf', 'wrl_admn_ad_pt_su_osm_pp.prj', + 'wrl_admn_ad_pt_su_osm_pp.shp', 'wrl_admn_ad_pt_su_osm_pp.shx', + 'wrl_admn_ad_py_su_osm_pp.dbf', 'wrl_admn_ad_py_su_osm_pp.prj', + 'wrl_admn_ad_py_su_osm_pp.shp', 'wrl_admn_ad_py_su_osm_pp.shx'] + ], + [ + "osms", + [], + ['wrl_osms_met_ln_su_osm_pp.dbf', 'wrl_osms_met_ln_su_osm_pp.prj', + 'wrl_osms_met_ln_su_osm_pp.shp', 'wrl_osms_met_ln_su_osm_pp.shx', + 'wrl_osms_met_pt_su_osm_pp.dbf', 'wrl_osms_met_pt_su_osm_pp.prj', + 'wrl_osms_met_pt_su_osm_pp.shp', 'wrl_osms_met_pt_su_osm_pp.shx', + 'wrl_osms_met_py_su_osm_pp.dbf', 'wrl_osms_met_py_su_osm_pp.prj', + 'wrl_osms_met_py_su_osm_pp.shp', 'wrl_osms_met_py_su_osm_pp.shx', + 'wrl_osms_met_rel_su_osm_pp.dbf', 'wrl_osms_met_rel_su_osm_pp.prj', + 'wrl_osms_met_rel_su_osm_pp.shp', 'wrl_osms_met_rel_su_osm_pp.shx'] + ], + [ + "tran", + [], + ['wrl_tran_air_ln_su_osm_pp.dbf', 'wrl_tran_air_ln_su_osm_pp.prj', + 'wrl_tran_air_ln_su_osm_pp.shp', 'wrl_tran_air_ln_su_osm_pp.shx', + 'wrl_tran_air_pt_su_osm_pp.dbf', 'wrl_tran_air_pt_su_osm_pp.prj', + 'wrl_tran_air_pt_su_osm_pp.shp', 'wrl_tran_air_pt_su_osm_pp.shx', + 'wrl_tran_air_py_su_osm_pp.dbf', 'wrl_tran_air_py_su_osm_pp.prj', + 'wrl_tran_air_py_su_osm_pp.shp', 'wrl_tran_air_py_su_osm_pp.shx'] + ] +] \ No newline at end of file From a26a6ad5d29da5ccbe481e5ce862f81cd656fb7b Mon Sep 17 00:00:00 2001 From: Andy Smith Date: Tue, 7 Apr 2015 00:01:53 +0100 Subject: [PATCH 03/14] test for batch convert - more files --- osm2ma/test_ogrwrapper.py | 37 ++++++++++++++----------------------- 1 file changed, 14 insertions(+), 23 deletions(-) diff --git a/osm2ma/test_ogrwrapper.py b/osm2ma/test_ogrwrapper.py index 69e4ea0..939b8db 100644 --- a/osm2ma/test_ogrwrapper.py +++ b/osm2ma/test_ogrwrapper.py @@ -7,6 +7,8 @@ import ogrwrapper as ogrw from osgeo import ogr from osgeo import osr +from configengine import xwalk_from_raw_config +import fixtures class TestGlobalFunctions(unittest.TestCase): @@ -17,34 +19,26 @@ def setUp(self): def tearDown(self): shutil.rmtree(self.tmpdir, ignore_errors=True) - def test_create_datacat_dir(self): + def test_medium_create_datacat_dir(self): test_data_cat = "test_data_cat" # test trying to create shpfile twice (shouldn't error) - for x in range(0,2): + for x in range(0, 2): created_path = ogrw._create_datacat_dir(self.tmpdir, test_data_cat) self.assertEqual(created_path, os.path.join(self.tmpdir, test_data_cat)) self.assertTrue(os.path.exists(created_path)) - def test_create_new_shpfile(self): + def test_medium_create_new_shpfile(self): test_shpfile = "test_shpfile" dest_srs = osr.SpatialReference() dest_srs.ImportFromEPSG(3857) # from EPSG # test trying to create shpfile twice (shouldn't error) - for x in range(0,2): + for x in range(0, 2): ogrw._create_new_shpfile(test_shpfile+".shp", self.tmpdir, ogr.wkbLineString, dest_srs) for extn in (u'.dbf', u'.prj', u'.shp', u'.shx'): self.assertTrue(os.path.exists(os.path.join(self.tmpdir, test_shpfile + extn))) - @unittest.skip("not implemented") - def test_copy_attributes(self): - pass - - @unittest.skip("not implemented") - def test_copy_features(self): - pass - - def testget_geom_details(self): + def testget_short_geom_details(self): source_layer, dest_geom = ogrw.get_geom_details("pt") self.assertEqual(source_layer, "points") self.assertEqual(dest_geom, ogr.wkbPoint) @@ -64,13 +58,10 @@ def testget_geom_details(self): # Test an invalid geometry type value. self.assertRaises(ValueError, ogrw.get_geom_details, "abcde") - @unittest.skip("not implemented") - def testdo_ogr2ogr_process(self): - pass - - @unittest.skip("not implemented") - def testbatch_convert(self): - pass - -if __name__ == '__main__': - unittest.main() + def test_long_batch_convert(self): + xwalk = xwalk_from_raw_config(fixtures.rawconf_good, 'wrl', 'su') + ogrw.batch_convert(xwalk, fixtures.example_pbf, self.tmpdir) + result_dir_listing = [] + for root, dirs, files in os.walk(self.tmpdir): + result_dir_listing.append([os.path.relpath(root, self.tmpdir), dirs, files]) + self.assertEqual(result_dir_listing, fixtures.output_dir_listing) From a70de369c9482ffbc4b77c3af2da2443e2cd15a1 Mon Sep 17 00:00:00 2001 From: Andy Smith Date: Tue, 7 Apr 2015 00:12:10 +0100 Subject: [PATCH 04/14] order dir listing results --- osm2ma/test_ogrwrapper.py | 1 + 1 file changed, 1 insertion(+) diff --git a/osm2ma/test_ogrwrapper.py b/osm2ma/test_ogrwrapper.py index 939b8db..abfed35 100644 --- a/osm2ma/test_ogrwrapper.py +++ b/osm2ma/test_ogrwrapper.py @@ -63,5 +63,6 @@ def test_long_batch_convert(self): ogrw.batch_convert(xwalk, fixtures.example_pbf, self.tmpdir) result_dir_listing = [] for root, dirs, files in os.walk(self.tmpdir): + dirs = sorted(dirs, reverse=False) result_dir_listing.append([os.path.relpath(root, self.tmpdir), dirs, files]) self.assertEqual(result_dir_listing, fixtures.output_dir_listing) From f38cb85e5d15d9f86a7f40bc21e7e420b0ab8f64 Mon Sep 17 00:00:00 2001 From: Andy Smith Date: Tue, 7 Apr 2015 00:19:29 +0100 Subject: [PATCH 05/14] And ordered file listing.... --- osm2ma/test_ogrwrapper.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/osm2ma/test_ogrwrapper.py b/osm2ma/test_ogrwrapper.py index abfed35..49e0b78 100644 --- a/osm2ma/test_ogrwrapper.py +++ b/osm2ma/test_ogrwrapper.py @@ -63,6 +63,7 @@ def test_long_batch_convert(self): ogrw.batch_convert(xwalk, fixtures.example_pbf, self.tmpdir) result_dir_listing = [] for root, dirs, files in os.walk(self.tmpdir): - dirs = sorted(dirs, reverse=False) + dirs = sorted(dirs) + files = sorted(files) result_dir_listing.append([os.path.relpath(root, self.tmpdir), dirs, files]) self.assertEqual(result_dir_listing, fixtures.output_dir_listing) From 24a2b54dd70062564a5bd02742891e399197e267 Mon Sep 17 00:00:00 2001 From: Andy Smith Date: Tue, 7 Apr 2015 00:26:25 +0100 Subject: [PATCH 06/14] explict sort direction --- osm2ma/test_ogrwrapper.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/osm2ma/test_ogrwrapper.py b/osm2ma/test_ogrwrapper.py index 49e0b78..b710ef8 100644 --- a/osm2ma/test_ogrwrapper.py +++ b/osm2ma/test_ogrwrapper.py @@ -63,7 +63,7 @@ def test_long_batch_convert(self): ogrw.batch_convert(xwalk, fixtures.example_pbf, self.tmpdir) result_dir_listing = [] for root, dirs, files in os.walk(self.tmpdir): - dirs = sorted(dirs) - files = sorted(files) + dirs = sorted(dirs, reverse=False) + files = sorted(files, reverse=False) result_dir_listing.append([os.path.relpath(root, self.tmpdir), dirs, files]) self.assertEqual(result_dir_listing, fixtures.output_dir_listing) From 9d021a6875a594cf30a23ed48a082096ad49d8fb Mon Sep 17 00:00:00 2001 From: Andy Smith Date: Tue, 7 Apr 2015 10:45:48 +0100 Subject: [PATCH 07/14] fixed sort order for os.walk --- osm2ma/test_ogrwrapper.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/osm2ma/test_ogrwrapper.py b/osm2ma/test_ogrwrapper.py index b710ef8..4012098 100644 --- a/osm2ma/test_ogrwrapper.py +++ b/osm2ma/test_ogrwrapper.py @@ -63,7 +63,7 @@ def test_long_batch_convert(self): ogrw.batch_convert(xwalk, fixtures.example_pbf, self.tmpdir) result_dir_listing = [] for root, dirs, files in os.walk(self.tmpdir): - dirs = sorted(dirs, reverse=False) - files = sorted(files, reverse=False) + dirs.sort() + files.sort() result_dir_listing.append([os.path.relpath(root, self.tmpdir), dirs, files]) self.assertEqual(result_dir_listing, fixtures.output_dir_listing) From e1f88a23bd23f4da9077b7b6b32aabf16399d228 Mon Sep 17 00:00:00 2001 From: Andy Smith Date: Sun, 19 Apr 2015 23:15:33 +0100 Subject: [PATCH 08/14] Respect the "useful to MapAction" column --- .gitignore | 3 ++- osm2ma/configengine.py | 4 ++-- osm2ma/fixtures.py | 3 +-- osm2ma/test_ogrwrapper.py | 1 + osm2ma/testfiles/fixtures.xls | Bin 407552 -> 407552 bytes 5 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index a64789c..b6f4430 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,5 @@ osm2ma/.idea *.pyc *.coverage *.py,cover -osm2ma/testfiles/output_shp/ \ No newline at end of file +osm2ma/testfiles/output_shp/ +geofabrik/ \ No newline at end of file diff --git a/osm2ma/configengine.py b/osm2ma/configengine.py index 0cb8e41..cfc5199 100644 --- a/osm2ma/configengine.py +++ b/osm2ma/configengine.py @@ -71,7 +71,7 @@ def _init_db_tables(self): osm_key_value text, element_icon text, comment text, - useful text, + useful text COLLATE NOCASE, data_category text, cat_value text, data_theme text, @@ -160,7 +160,7 @@ def _populate_scratch_table(self): osm_element, '{geom}' from config where - config.geom_str like '%{geom}%' + config.geom_str like '%{geom}%' and config.useful = 'y' '''.format(geom=geom) cur.execute(u_sql) diff --git a/osm2ma/fixtures.py b/osm2ma/fixtures.py index 0f25fb6..17b8670 100644 --- a/osm2ma/fixtures.py +++ b/osm2ma/fixtures.py @@ -56,7 +56,6 @@ def path_to_fixtures_xls(): (u'iata', u'User Defined', u'tran', u'air', u'Node Way Area', u'py'), (u'icao', u'User Defined', u'tran', u'air', u'Node Way Area', u'py'), (u'boundary', u'administrative', u'admn', u'ad', u'Area', u'py'), - (u'boundary', u'maritime', u'admn', u'ad', u'Area', u'py'), (u'boundary', u'political', u'admn', u'ad', u'Area', u'py'), (u'border_type', u'*', u'admn', u'ad', u'Way Area', u'py'), (u'admin_level', u'Number', u'admn', u'ad', u'Area', u'py'), @@ -75,7 +74,7 @@ def path_to_fixtures_xls(): shpf_list_table_good = [ (u'wrl_admn_ad_ln_su_osm_pp.shp', u'admn', u'ln', u'border_type, boundary', u"'border_type' IS NOT null or 'boundary' IS NOT null"), (u'wrl_admn_ad_pt_su_osm_pp.shp', u'admn', u'pt', u'boundary', u"'boundary' IS NOT null"), - (u'wrl_admn_ad_py_su_osm_pp.shp', u'admn', u'py', u'admin_level, border_type, boundary', u"'admin_level' IS NOT null or 'border_type' IS NOT null or 'boundary'='administrative' or 'boundary'='maritime' or 'boundary'='political'"), + (u'wrl_admn_ad_py_su_osm_pp.shp', u'admn', u'py', u'admin_level, border_type, boundary', u"'admin_level' IS NOT null or 'border_type' IS NOT null or 'boundary'='administrative' or 'boundary'='political'"), (u'wrl_osms_met_ln_su_osm_pp.shp', u'osms', u'ln', u'fixme, source, source:name, wikipedia', u"'fixme' IS NOT null or 'source' IS NOT null or 'source:name' IS NOT null or 'wikipedia' IS NOT null"), (u'wrl_osms_met_pt_su_osm_pp.shp', u'osms', u'pt', u'fixme, source, source:name, wikipedia', u"'fixme' IS NOT null or 'source' IS NOT null or 'source:name' IS NOT null or 'wikipedia' IS NOT null"), (u'wrl_osms_met_py_su_osm_pp.shp', u'osms', u'py', u'fixme, source, source:name, wikipedia', u"'fixme' IS NOT null or 'source' IS NOT null or 'source:name' IS NOT null or 'wikipedia' IS NOT null"), diff --git a/osm2ma/test_ogrwrapper.py b/osm2ma/test_ogrwrapper.py index 4012098..c9eeb7b 100644 --- a/osm2ma/test_ogrwrapper.py +++ b/osm2ma/test_ogrwrapper.py @@ -58,6 +58,7 @@ def testget_short_geom_details(self): # Test an invalid geometry type value. self.assertRaises(ValueError, ogrw.get_geom_details, "abcde") + @unittest.skip("too long") def test_long_batch_convert(self): xwalk = xwalk_from_raw_config(fixtures.rawconf_good, 'wrl', 'su') ogrw.batch_convert(xwalk, fixtures.example_pbf, self.tmpdir) diff --git a/osm2ma/testfiles/fixtures.xls b/osm2ma/testfiles/fixtures.xls index b7584867c113ece792e568d5dfb71389a5adf4b1..9a4ce4e4f9f8327d9c5298ffa45960c88af0d422 100644 GIT binary patch delta 1678 zcma)6eN2^A82>%z^0wErO^tbDM)O*bEZT!6;nQH>2OWP+;Gef#7sl$-p*AF`=jRj?(Fxx=XuU^e!ufP z=PkbGD!%4A_>yppxsqMyDrl#arynW0v{7Ftw$Rju0ntYUy;qw-yY)z$OV72~{=<6D zGERAFEo1Ka?KZqm02uBrw1*CTI<0|7r`4rmzF~tgPU{H$73;9UU3%%Rkgwm|B@DBRvyotDfSSX!)g2C||fScA7R)-{P6YBO&!4J)`u? zo)1Yhyp=79g#K<$y2WDsT7|yY(3Rt}cvQmqek_F*zh(l%Rf@GsfGUMu3WJq9HVep7 zs4M(lp<^j9Q=M0@!f+03@&Nl4>N(0UfyQM(j|P0DPs`ivEhquJyMWR?Kt(lR)B<}O zfqh4Tx?_OZ3N)Poj-LnGdVsT6)y_A8i$0)FBiwQj{38gK2tsf)AuNs%kw}P6Cd8!@ z64MCDnS|8kgtS$J%mTvl^@LSk!uS$0;@fz;`2xvXQ4Rj`8yYP#F5lq?j5W;^V-A}v zlcs)ljG4FNL?M|+a^>s-^V3ay5}A{JmKkYeF_Oz@p~H&6(ULkNWeKC?G_PQOzrJ8i zn2}meMYd4o1=d;LF@h#&G3IaObifw>g@5Pmcx0=w=~b0$sTR7he|WdsX1p1_6JufTc_zc!xk(r9ecL zKy-~jT)jZzA%Wy(fz%d(v<`vHPJ!j!0;4X7yWPw1Y-2lD#h6F3C(7)-!RP&HM4skB z#_WD^;Q`J?sxF7Gi|&W4OTVSwDbYMpR{I3=qUAA(jwd8KpONT(PNH|7M1O|FK#m0E zN!aowLSB~$e^X-I7KzyH5(&E`lJ-crt0m^vN-SuU$T}*q;+VwhR*AJ|Bnr<vfe4h=)*2PvRD=~Dp<=0)R!wVLtt%QI zXnl2*DN=k?gSA>JZn7JK+H6T}6(v{`8)~$nv>0L{qQ*+7OYbTc|Fm%?lkb~*&pmU_ zcfXrid)u-0w&TQ`!WMej`SIZyb(H@3&>lnb_2ptCjq$aK79y@UYZIxL?zA}c1t!aX zWS^R1)J(Z)z~gzV1v?1%`d2ee}1@5uN1Q&Ee13&As*G z=I)(YZD;1Nj_X>cG+8HgQ9CVR>hR#iu0i!BeYNEY;a*Fa@T-4;)-GUpU5EFHY@f|q zmri^1-?I8ri~d*EAd1!dd17gsp5lo${n8?o@+dtxx1YYz^AaBlO|S7pi%>_XL;u5L zko2vdD~Y7OE<4@C%_kMy?W@lYnKC+%1v7rn0J zQjo|E8VM47V|Q3vDNRAh?CDSw^{#67704uF8VdUoUn8d02`?n1HxP zH{gQv&;thbqmF328qQB7P-YZXlF26LMM!t~SD5)sIzXhV^!a zsSbunC&SfYiYJa?$ykPjWQO+14As*aw$5g7&tr(rWoUed;n@2OD?VXJ+`z!MG1Tm4 z(7#~t9A=0pV`1z0@&2EP7^V2nQ?DESADaX0VINOlC#jba+(wZ`QZ08GODos}W&TJ7 zOSSa>Kcj{+I?FIW6dA@hRcxM4+W?|z)~pCbC*C=u|Uj0fe}Xq;wuEkR|=#C1*V)6n0ZMcvtD3fgTR1B5!PJi zuj9l53k?bkRIM7%12^vRX@3io+9D&Rkm8NJR#Eko^l)od-VqJYxI3-VXP2lh3kl|p zm8f|^;_?`Yt4R`#uSndSCUN%-iS}#>o+}ZNFA@2HMD!Yo!5bx9+a(f;Bog;Yq?Jib zEthcnB|K*&a?VODsg+o9MPk);iS Date: Mon, 20 Apr 2015 15:29:41 +0100 Subject: [PATCH 09/14] Added TODO notes --- osm2ma/configengine.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/osm2ma/configengine.py b/osm2ma/configengine.py index cfc5199..61326f2 100644 --- a/osm2ma/configengine.py +++ b/osm2ma/configengine.py @@ -137,6 +137,13 @@ def _populate_scratch_table(self): """ Duplicate config table which has been normalised wrt geometry type. + TODO: ammend this to properly handle the generic OSM tags + TODO: update fixtures for this. + + 1st; Run existing insert query but EXCLUDING cat_value = 'OSM' + 2nd; For each unique cat_value, theme_value pair now in scratch, insert from config where cat_value = 'OSM', but + replacing with the loop's current values of cat_value, theme_value + :return: None """ From 4461b9c7373a709d7823f7f2e05a8560a1fc4f61 Mon Sep 17 00:00:00 2001 From: Andy Smith Date: Wed, 22 Apr 2015 00:38:35 +0100 Subject: [PATCH 10/14] Add OSM meta attributes to all shapefiles --- osm2ma/configengine.py | 103 +++++++++++---- osm2ma/fixtures.py | 228 ++++++++++++++++++++++------------ osm2ma/test_configengine.py | 8 +- osm2ma/testfiles/fixtures.xls | Bin 407552 -> 407552 bytes 4 files changed, 232 insertions(+), 107 deletions(-) diff --git a/osm2ma/configengine.py b/osm2ma/configengine.py index 61326f2..671a5db 100644 --- a/osm2ma/configengine.py +++ b/osm2ma/configengine.py @@ -73,7 +73,7 @@ def _init_db_tables(self): comment text, useful text COLLATE NOCASE, data_category text, - cat_value text, + cat_value text COLLATE NOCASE, data_theme text, theme_value text, conforms_to_hierarchy text, @@ -91,7 +91,8 @@ def _init_db_tables(self): cat_value text, theme_value text, osm_element, - geom_type text + geom_type text, + osm_metadata boolean ); create table shpf_list ( @@ -157,7 +158,8 @@ def _populate_scratch_table(self): cat_value, theme_value, osm_element, - geom_type + geom_type, + osm_metadata ) select osm_key_name, @@ -165,12 +167,54 @@ def _populate_scratch_table(self): cat_value, theme_value, osm_element, - '{geom}' + '{geom}', + 'FALSE' from config where - config.geom_str like '%{geom}%' and config.useful = 'y' + config.geom_str like '%{geom}%' + and config.useful = 'y' + and config.cat_value != 'osms' '''.format(geom=geom) cur.execute(u_sql) + u_sql = ''' + select + cat_value, + theme_value, + geom_type + from scratch + group by + cat_value, + theme_value, + geom_type + ''' + + for cat, theme, geom in cur.execute(u_sql).fetchall(): + #print 'cat = {}, theme = {}, geom = {}'.format(cat, theme, geom) + u_sql = ''' + INSERT INTO scratch ( + osm_key_name, + osm_key_value, + cat_value, + theme_value, + osm_element, + geom_type, + osm_metadata + ) + select + osm_key_name, + osm_key_value, + '{cat_value}', + '{theme_value}', + osm_element, + '{geom}', + 'TRUE' + from config where + config.geom_str like '%{geom}%' + and config.useful = 'y' + and config.cat_value == 'osms' + '''.format(cat_value=cat, theme_value=theme, geom=geom) + cur.execute(u_sql) + def _populate_shpfile_table(self, geo_extd, scale): """ Fill list of shapefile, aggregating details of individual attributes. @@ -198,8 +242,8 @@ def _populate_shpfile_table(self, geo_extd, scale): ), scratch.cat_value, scratch.geom_type, - attriblist(scratch.osm_key_name), - condition_clause(scratch.osm_key_name, scratch.osm_key_value) + attriblist(scratch.osm_key_name, scratch.osm_metadata), + condition_clause(scratch.osm_key_name, scratch.osm_key_value, scratch.osm_metadata) FROM scratch GROUP BY shpf_name( @@ -238,8 +282,8 @@ def _init_db_funcs(self): :return: None """ self.db.create_function("shpf_name", 5, ConfigXWalk._create_shpf_name) - self.db.create_aggregate("attriblist", 1, _AttribList) - self.db.create_aggregate("condition_clause", 2, _SelectClause) + self.db.create_aggregate("attriblist", 2, _AttribList) + self.db.create_aggregate("condition_clause", 3, _SelectClause) def get_xwalk(self): """ @@ -302,13 +346,19 @@ class _AttribList: """ def __init__(self): self.set_attribs = set() + self.set_meta = set() - def step(self, value): + def step(self, value, meta): if len(value) > 0: - self.set_attribs.add(value) + if 'true' == meta.lower(): + self.set_meta.add(value) + else: + self.set_attribs.add(value) def finalize(self): - return ", ".join(sorted(self.set_attribs)) + attribs = ", ".join(sorted(self.set_attribs)) + meta = ", ".join(sorted(self.set_meta)) + return "{}, {}".format(attribs, meta) class _SelectClause: @@ -324,20 +374,21 @@ def __init__(self): self.query_args = dict() self.exclude_keys = set() - def step(self, osm_key, osm_value): - if (type(osm_value) == unicode) and ( - osm_value.lower() in {u'*', u'user defined', u'number', u'url or article title'}): - self.exclude_keys.add(osm_key) - else: - for val in osm_value.split(u'/'): - # TODO: This might be dangerous, because you can't be - # sure that the val or key won't contain a quote, for - # example. - # FIXME: Use params to execute? - unique_clause = u"'{key}'='{val}'".format( - key=osm_key, val=val.strip() - ) - self.query_args[unique_clause] = osm_key + def step(self, osm_key, osm_value, osm_meta): + if u'false' == osm_meta.lower(): + if (type(osm_value) == unicode) and ( + osm_value.lower() in {u'*', u'user defined', u'number', u'url or article title'}): + self.exclude_keys.add(osm_key) + else: + for val in osm_value.split(u'/'): + # TODO: This might be dangerous, because you can't be + # sure that the val or key won't contain a quote, for + # example. + # FIXME: Use params to execute? + unique_clause = u"'{key}'='{val}'".format( + key=osm_key, val=val.strip() + ) + self.query_args[unique_clause] = osm_key def finalize(self): cleaned_pairs = set() diff --git a/osm2ma/fixtures.py b/osm2ma/fixtures.py index 17b8670..0de1559 100644 --- a/osm2ma/fixtures.py +++ b/osm2ma/fixtures.py @@ -28,103 +28,189 @@ def path_to_fixtures_xls(): rawconf_wrong_column_order = _workbook.name_map.get("rawconf_wrong_column_order")[0].area2d(clipped=True) scratch_table_good = [ - (u'aeroway', u'aerodrome', u'tran', u'air', u'Node Area', u'pt'), - (u'aeroway',u'User defined', u'tran', u'air', u'Node Way', u'pt'), - (u'military', u'airfield', u'tran', u'air', u'Node Area', u'pt'), - (u'iata', u'User Defined', u'tran', u'air', u'Node Way Area', u'pt'), - (u'icao', u'User Defined', u'tran', u'air', u'Node Way Area', u'pt'), - (u'boundary', u'user defined', u'admn', u'ad', u'Node Way', u'pt'), - (u'fixme', u'User defined', u'osms', u'met', u'Node Way Area Relation', u'pt'), - (u'source', u'User defined', u'osms', u'met', u'Node Way Area Relation', u'pt'), - (u'source', u'historical', u'osms', u'met', u'Node Way Area Relation', u'pt'), - (u'source:name', u'User defined', u'osms', u'met', u'Node Way Area Relation', u'pt'), - (u'wikipedia', u'URL or article title', u'osms', u'met', u'Node Way Area Relation', u'pt'), - (u'aeroway', u'runway', u'tran', u'air', u'Way Area', u'ln'), - (u'aeroway', u'User defined', u'tran', u'air', u'Node Way', u'ln'), - (u'iata', u'User Defined', u'tran', u'air', u'Node Way Area', u'ln'), - (u'icao', u'User Defined', u'tran', u'air', u'Node Way Area', u'ln'), - (u'border_type', u'*',u'admn', u'ad', u'Way Area', u'ln'), - (u'boundary', u'user defined', u'admn', u'ad', u'Node Way', u'ln'), - (u'fixme', u'User defined', u'osms', u'met', u'Node Way Area Relation', u'ln'), - (u'source', u'User defined', u'osms', u'met', u'Node Way Area Relation', u'ln'), - (u'source', u'historical', u'osms', u'met', u'Node Way Area Relation', u'ln'), - (u'source:name', u'User defined', u'osms', u'met', u'Node Way Area Relation', u'ln'), - (u'wikipedia', u'URL or article title', u'osms',u'met', u'Node Way Area Relation', u'ln'), - (u'aeroway', u'aerodrome', u'tran', u'air', u'Node Area', u'py'), - (u'aeroway', u'runway', u'tran', u'air', u'Way Area', u'py'), - (u'military', u'airfield', u'tran', u'air', u'Node Area', u'py'), - (u'iata', u'User Defined', u'tran', u'air', u'Node Way Area', u'py'), - (u'icao', u'User Defined', u'tran', u'air', u'Node Way Area', u'py'), - (u'boundary', u'administrative', u'admn', u'ad', u'Area', u'py'), - (u'boundary', u'political', u'admn', u'ad', u'Area', u'py'), - (u'border_type', u'*', u'admn', u'ad', u'Way Area', u'py'), - (u'admin_level', u'Number', u'admn', u'ad', u'Area', u'py'), - (u'fixme', u'User defined', u'osms', u'met', u'Node Way Area Relation', u'py'), - (u'source', u'User defined', u'osms', u'met', u'Node Way Area Relation', u'py'), - (u'source', u'historical', u'osms', u'met', u'Node Way Area Relation', u'py'), - (u'source:name', u'User defined', u'osms', u'met', u'Node Way Area Relation', u'py'), - (u'wikipedia', u'URL or article title', u'osms', u'met', u'Node Way Area Relation', u'py'), - (u'fixme', u'User defined', u'osms', u'met', u'Node Way Area Relation', u'rel'), - (u'source', u'User defined', u'osms', u'met', u'Node Way Area Relation', u'rel'), - (u'source', u'historical', u'osms', u'met', u'Node Way Area Relation', u'rel'), - (u'source:name', u'User defined', u'osms', u'met', u'Node Way Area Relation', u'rel'), - (u'wikipedia', u'URL or article title', u'osms', u'met', u'Node Way Area Relation', u'rel') + (u'aeroway', u'aerodrome', u'tran', u'air', u'Node Area', u'pt', u'FALSE'), + (u'aeroway',u'User defined', u'tran', u'air', u'Node Way', u'pt', u'FALSE'), + (u'military', u'airfield', u'tran', u'air', u'Node Area', u'pt', u'FALSE'), + (u'iata', u'User Defined', u'tran', u'air', u'Node Way Area', u'pt', u'FALSE'), + (u'icao', u'User Defined', u'tran', u'air', u'Node Way Area', u'pt', u'FALSE'), + (u'boundary', u'user defined', u'admn', u'ad', u'Node Way', u'pt', u'FALSE'), + (u'aeroway', u'runway', u'tran', u'air', u'Way Area', u'ln', u'FALSE'), + (u'aeroway', u'User defined', u'tran', u'air', u'Node Way', u'ln', u'FALSE'), + (u'iata', u'User Defined', u'tran', u'air', u'Node Way Area', u'ln', u'FALSE'), + (u'icao', u'User Defined', u'tran', u'air', u'Node Way Area', u'ln', u'FALSE'), + (u'border_type', u'*',u'admn', u'ad', u'Way Area', u'ln', u'FALSE'), + (u'boundary', u'user defined', u'admn', u'ad', u'Node Way', u'ln', u'FALSE'), + (u'aeroway', u'aerodrome', u'tran', u'air', u'Node Area', u'py', u'FALSE'), + (u'aeroway', u'runway', u'tran', u'air', u'Way Area', u'py', u'FALSE'), + (u'military', u'airfield', u'tran', u'air', u'Node Area', u'py', u'FALSE'), + (u'iata', u'User Defined', u'tran', u'air', u'Node Way Area', u'py', u'FALSE'), + (u'icao', u'User Defined', u'tran', u'air', u'Node Way Area', u'py', u'FALSE'), + (u'boundary', u'administrative', u'admn', u'ad', u'Area', u'py', u'FALSE'), + (u'boundary', u'political', u'admn', u'ad', u'Area', u'py', u'FALSE'), + (u'border_type', u'*', u'admn', u'ad', u'Way Area', u'py', u'FALSE'), + (u'admin_level', u'Number', u'admn', u'ad', u'Area', u'py', u'FALSE'), + (u'fixme', u'User defined', u'admn', u'ad', u'Node Way Area Relation', u'ln', u'TRUE'), + (u'source', u'User defined', u'admn', u'ad', u'Node Way Area Relation', u'ln', u'TRUE'), + (u'source', u'historical', u'admn', u'ad', u'Node Way Area Relation', u'ln', u'TRUE'), + (u'source:name', u'User defined', u'admn', u'ad', u'Node Way Area Relation', u'ln', u'TRUE'), + (u'wikipedia', u'URL or article title', u'admn', u'ad', u'Node Way Area Relation', u'ln', u'TRUE'), + (u'fixme', u'User defined', u'admn', u'ad', u'Node Way Area Relation', u'pt', u'TRUE'), + (u'source', u'User defined', u'admn', u'ad', u'Node Way Area Relation', u'pt', u'TRUE'), + (u'source', u'historical', u'admn', u'ad', u'Node Way Area Relation', u'pt', u'TRUE'), + (u'source:name', u'User defined', u'admn', u'ad', u'Node Way Area Relation', u'pt', u'TRUE'), + (u'wikipedia', u'URL or article title', u'admn', u'ad', u'Node Way Area Relation', u'pt', u'TRUE'), + (u'fixme', u'User defined', u'admn', u'ad', u'Node Way Area Relation', u'py', u'TRUE'), + (u'source', u'User defined', u'admn', u'ad', u'Node Way Area Relation', u'py', u'TRUE'), + (u'source', u'historical', u'admn', u'ad', u'Node Way Area Relation', u'py', u'TRUE'), + (u'source:name', u'User defined', u'admn', u'ad', u'Node Way Area Relation', u'py', u'TRUE'), + (u'wikipedia', u'URL or article title', u'admn', u'ad', u'Node Way Area Relation', u'py', u'TRUE'), + (u'fixme', u'User defined', u'tran', u'air', u'Node Way Area Relation', u'ln', u'TRUE'), + (u'source', u'User defined', u'tran', u'air', u'Node Way Area Relation', u'ln', u'TRUE'), + (u'source', u'historical', u'tran', u'air', u'Node Way Area Relation', u'ln', u'TRUE'), + (u'source:name', u'User defined', u'tran', u'air', u'Node Way Area Relation', u'ln', u'TRUE'), + (u'wikipedia', u'URL or article title', u'tran', u'air', u'Node Way Area Relation', u'ln', u'TRUE'), + (u'fixme', u'User defined', u'tran', u'air', u'Node Way Area Relation', u'pt', u'TRUE'), + (u'source', u'User defined', u'tran', u'air', u'Node Way Area Relation', u'pt', u'TRUE'), + (u'source', u'historical', u'tran', u'air', u'Node Way Area Relation', u'pt', u'TRUE'), + (u'source:name', u'User defined', u'tran', u'air', u'Node Way Area Relation', u'pt', u'TRUE'), + (u'wikipedia', u'URL or article title', u'tran', u'air', u'Node Way Area Relation', u'pt', u'TRUE'), + (u'fixme', u'User defined', u'tran', u'air', u'Node Way Area Relation', u'py', u'TRUE'), + (u'source', u'User defined', u'tran', u'air', u'Node Way Area Relation', u'py', u'TRUE'), + (u'source', u'historical', u'tran', u'air', u'Node Way Area Relation', u'py', u'TRUE'), + (u'source:name', u'User defined', u'tran', u'air', u'Node Way Area Relation', u'py', u'TRUE'), + (u'wikipedia', u'URL or article title', u'tran', u'air', u'Node Way Area Relation', u'py', u'TRUE') +] + +temp_scratch_table_osm_only = [ + (u'fixme', u'User defined', u'admn', u'ad', u'Node Way Area Relation', u'ln', u'TRUE'), + (u'source', u'User defined', u'admn', u'ad', u'Node Way Area Relation', u'ln', u'TRUE'), + (u'source', u'historical', u'admn', u'ad', u'Node Way Area Relation', u'ln', u'TRUE'), + (u'source:name', u'User defined', u'admn', u'ad', u'Node Way Area Relation', u'ln', u'TRUE'), + (u'wikipedia', u'URL or article title', u'admn', u'ad', u'Node Way Area Relation', u'ln', u'TRUE'), + (u'fixme', u'User defined', u'admn', u'ad', u'Node Way Area Relation', u'pt', u'TRUE'), + (u'source', u'User defined', u'admn', u'ad', u'Node Way Area Relation', u'pt', u'TRUE'), + (u'source', u'historical', u'admn', u'ad', u'Node Way Area Relation', u'pt', u'TRUE'), + (u'source:name', u'User defined', u'admn', u'ad', u'Node Way Area Relation', u'pt', u'TRUE'), + (u'wikipedia', u'URL or article title', u'admn', u'ad', u'Node Way Area Relation', u'pt', u'TRUE'), + (u'fixme', u'User defined', u'admn', u'ad', u'Node Way Area Relation', u'py', u'TRUE'), + (u'source', u'User defined', u'admn', u'ad', u'Node Way Area Relation', u'py', u'TRUE'), + (u'source', u'historical', u'admn', u'ad', u'Node Way Area Relation', u'py', u'TRUE'), + (u'source:name', u'User defined', u'admn', u'ad', u'Node Way Area Relation', u'py', u'TRUE'), + (u'wikipedia', u'URL or article title', u'admn', u'ad', u'Node Way Area Relation', u'py', u'TRUE'), + (u'fixme', u'User defined', u'tran', u'air', u'Node Way Area Relation', u'ln', u'TRUE'), + (u'source', u'User defined', u'tran', u'air', u'Node Way Area Relation', u'ln', u'TRUE'), + (u'source', u'historical', u'tran', u'air', u'Node Way Area Relation', u'ln', u'TRUE'), + (u'source:name', u'User defined', u'tran', u'air', u'Node Way Area Relation', u'ln', u'TRUE'), + (u'wikipedia', u'URL or article title', u'tran', u'air', u'Node Way Area Relation', u'ln', u'TRUE'), + (u'fixme', u'User defined', u'tran', u'air', u'Node Way Area Relation', u'pt', u'TRUE'), + (u'source', u'User defined', u'tran', u'air', u'Node Way Area Relation', u'pt', u'TRUE'), + (u'source', u'historical', u'tran', u'air', u'Node Way Area Relation', u'pt', u'TRUE'), + (u'source:name', u'User defined', u'tran', u'air', u'Node Way Area Relation', u'pt', u'TRUE'), + (u'wikipedia', u'URL or article title', u'tran', u'air', u'Node Way Area Relation', u'pt', u'TRUE'), + (u'fixme', u'User defined', u'tran', u'air', u'Node Way Area Relation', u'py', u'TRUE'), + (u'source', u'User defined', u'tran', u'air', u'Node Way Area Relation', u'py', u'TRUE'), + (u'source', u'historical', u'tran', u'air', u'Node Way Area Relation', u'py', u'TRUE'), + (u'source:name', u'User defined', u'tran', u'air', u'Node Way Area Relation', u'py', u'TRUE'), + (u'wikipedia', u'URL or article title', u'tran', u'air', u'Node Way Area Relation', u'py', u'TRUE'), ] shpf_list_table_good = [ - (u'wrl_admn_ad_ln_su_osm_pp.shp', u'admn', u'ln', u'border_type, boundary', u"'border_type' IS NOT null or 'boundary' IS NOT null"), - (u'wrl_admn_ad_pt_su_osm_pp.shp', u'admn', u'pt', u'boundary', u"'boundary' IS NOT null"), - (u'wrl_admn_ad_py_su_osm_pp.shp', u'admn', u'py', u'admin_level, border_type, boundary', u"'admin_level' IS NOT null or 'border_type' IS NOT null or 'boundary'='administrative' or 'boundary'='political'"), - (u'wrl_osms_met_ln_su_osm_pp.shp', u'osms', u'ln', u'fixme, source, source:name, wikipedia', u"'fixme' IS NOT null or 'source' IS NOT null or 'source:name' IS NOT null or 'wikipedia' IS NOT null"), - (u'wrl_osms_met_pt_su_osm_pp.shp', u'osms', u'pt', u'fixme, source, source:name, wikipedia', u"'fixme' IS NOT null or 'source' IS NOT null or 'source:name' IS NOT null or 'wikipedia' IS NOT null"), - (u'wrl_osms_met_py_su_osm_pp.shp', u'osms', u'py', u'fixme, source, source:name, wikipedia', u"'fixme' IS NOT null or 'source' IS NOT null or 'source:name' IS NOT null or 'wikipedia' IS NOT null"), - (u'wrl_osms_met_rel_su_osm_pp.shp', u'osms', u'rel', u'fixme, source, source:name, wikipedia', u"'fixme' IS NOT null or 'source' IS NOT null or 'source:name' IS NOT null or 'wikipedia' IS NOT null"), - (u'wrl_tran_air_ln_su_osm_pp.shp', u'tran', u'ln', u'aeroway, iata, icao', u"'aeroway' IS NOT null or 'iata' IS NOT null or 'icao' IS NOT null"), - (u'wrl_tran_air_pt_su_osm_pp.shp', u'tran', u'pt', u'aeroway, iata, icao, military', u"'aeroway' IS NOT null or 'iata' IS NOT null or 'icao' IS NOT null or 'military'='airfield'"), - (u'wrl_tran_air_py_su_osm_pp.shp', u'tran', u'py', u'aeroway, iata, icao, military', u"'aeroway'='aerodrome' or 'aeroway'='runway' or 'iata' IS NOT null or 'icao' IS NOT null or 'military'='airfield'") + (u'wrl_admn_ad_ln_su_osm_pp.shp', u'admn', u'ln', + u'border_type, boundary, fixme, source, source:name, wikipedia', + u"'border_type' IS NOT null or 'boundary' IS NOT null"), + (u'wrl_admn_ad_pt_su_osm_pp.shp', u'admn', u'pt', + u'boundary, fixme, source, source:name, wikipedia', + u"'boundary' IS NOT null"), + (u'wrl_admn_ad_py_su_osm_pp.shp', u'admn', u'py', + u'admin_level, border_type, boundary, fixme, source, source:name, wikipedia', + u"'admin_level' IS NOT null or 'border_type' IS NOT null or 'boundary'='administrative' or 'boundary'='political'"), + (u'wrl_tran_air_ln_su_osm_pp.shp', u'tran', u'ln', + u'aeroway, iata, icao, fixme, source, source:name, wikipedia', + u"'aeroway' IS NOT null or 'iata' IS NOT null or 'icao' IS NOT null"), + (u'wrl_tran_air_pt_su_osm_pp.shp', u'tran', u'pt', + u'aeroway, iata, icao, military, fixme, source, source:name, wikipedia', + u"'aeroway' IS NOT null or 'iata' IS NOT null or 'icao' IS NOT null or 'military'='airfield'"), + (u'wrl_tran_air_py_su_osm_pp.shp', u'tran', u'py', + u'aeroway, iata, icao, military, fixme, source, source:name, wikipedia', + u"'aeroway'='aerodrome' or 'aeroway'='runway' or 'iata' IS NOT null or 'icao' IS NOT null or 'military'='airfield'") +] + +temp_shpf_list_table_osm_only = [ + (u'wrl_osms_met_ln_su_osm_pp.shp', u'osms', u'ln', u'fixme, source, source:name, wikipedia', + u"'fixme' IS NOT null or 'source' IS NOT null or 'source:name' IS NOT null or 'wikipedia' IS NOT null"), + (u'wrl_osms_met_pt_su_osm_pp.shp', u'osms', u'pt', u'fixme, source, source:name, wikipedia', + u"'fixme' IS NOT null or 'source' IS NOT null or 'source:name' IS NOT null or 'wikipedia' IS NOT null"), + (u'wrl_osms_met_py_su_osm_pp.shp', u'osms', u'py', u'fixme, source, source:name, wikipedia', + u"'fixme' IS NOT null or 'source' IS NOT null or 'source:name' IS NOT null or 'wikipedia' IS NOT null"), + (u'wrl_osms_met_rel_su_osm_pp.shp', u'osms', u'rel', u'fixme, source, source:name, wikipedia', + u"'fixme' IS NOT null or 'source' IS NOT null or 'source:name' IS NOT null or 'wikipedia' IS NOT null"), + ] """ A list of which are unsorted, contain duplicates and contains blank strings. """ -attrib_list_args = ['f', 'a', 'b', 'c', 'd', '', 'e', 'a', 'a'] -attrib_list_result = 'a, b, c, d, e, f' +attrib_list_args = [('f', 'FALSE'), + ('a', 'FALSE'), + ('b', 'TRUE' ), + ('c', 'FALSE'), + ('d', 'FALSE'), + ('', 'FALSE'), + ('e', 'FALSE'), + ('a', 'FALSE'), + ('a', 'FALSE')] +attrib_list_result = 'a, c, d, e, f, b' """ A list of tuples. Each tuple has as its first item -- A list of tuples of string pairs, representing OSM key value pairs +- A list of tuples of string pairs, representing OSM key value pairs and a "meta" field indicating whether + the key/value pair represents feature level metadata. - A string of the resulting selct clause + +If the meta field is True for all instances of a particular attribute then the attribute should be +excluded from the attribute list. EG in this case there are two entries for attribute 'a'. In one case the +meta field False therefore 'a' is included. 'z' only occurs with meta value True, therefore is exlcuded """ select_clause_args_and_result_pairs = [ ( [ - (u"border_type", u"*") + (u"border_type", u"*", u'FALSE') ], u"'border_type' IS NOT null" ), ( [ - (u"boundary", u"administrative"), - (u"boundary", u"maritime"), - (u"boundary", u"political"), - (u"boundary", u"user defined"), - (u"border_type", u"*") + (u"boundary", u"administrative", u'FALSE'), + (u"boundary", u"maritime", u'FALSE'), + (u"boundary", u"political", u'FALSE'), + (u"boundary", u"user defined", u'FALSE'), + (u"border_type", u"*", u'FALSE') ], u"'border_type' IS NOT null or 'boundary' IS NOT null" ), ( [ - (u"boundary", u"administrative"), - (u"boundary", u"maritime"), - (u"boundary", u"political") + (u"boundary", u"administrative", u'FALSE'), + (u"boundary", u"maritime", u'FALSE'), + (u"boundary", u"political", u'FALSE') + ], + u"'boundary'='administrative' or 'boundary'='maritime' or 'boundary'='political'" + ), + ( + [ + (u"boundary", u"administrative", u'FALSE'), + (u"boundary", u"maritime", u'FALSE'), + (u"boundary", u"political", u'FALSE'), + (u"source", u"historical", u'TRUE'), + (u"wikipedia", u"URL", u'TRUE') ], u"'boundary'='administrative' or 'boundary'='maritime' or 'boundary'='political'" ), ( [ - (u"admin_level", u"Number"), + (u"admin_level", u"Number", u'FALSE'), ], u"'admin_level' IS NOT null" ), @@ -137,7 +223,7 @@ def path_to_fixtures_xls(): output_dir_listing = [ [ ".", - ['admn', 'osms', 'tran'], + ['admn', 'tran'], [] ], [ @@ -150,18 +236,6 @@ def path_to_fixtures_xls(): 'wrl_admn_ad_py_su_osm_pp.dbf', 'wrl_admn_ad_py_su_osm_pp.prj', 'wrl_admn_ad_py_su_osm_pp.shp', 'wrl_admn_ad_py_su_osm_pp.shx'] ], - [ - "osms", - [], - ['wrl_osms_met_ln_su_osm_pp.dbf', 'wrl_osms_met_ln_su_osm_pp.prj', - 'wrl_osms_met_ln_su_osm_pp.shp', 'wrl_osms_met_ln_su_osm_pp.shx', - 'wrl_osms_met_pt_su_osm_pp.dbf', 'wrl_osms_met_pt_su_osm_pp.prj', - 'wrl_osms_met_pt_su_osm_pp.shp', 'wrl_osms_met_pt_su_osm_pp.shx', - 'wrl_osms_met_py_su_osm_pp.dbf', 'wrl_osms_met_py_su_osm_pp.prj', - 'wrl_osms_met_py_su_osm_pp.shp', 'wrl_osms_met_py_su_osm_pp.shx', - 'wrl_osms_met_rel_su_osm_pp.dbf', 'wrl_osms_met_rel_su_osm_pp.prj', - 'wrl_osms_met_rel_su_osm_pp.shp', 'wrl_osms_met_rel_su_osm_pp.shx'] - ], [ "tran", [], diff --git a/osm2ma/test_configengine.py b/osm2ma/test_configengine.py index a553167..60ec35a 100644 --- a/osm2ma/test_configengine.py +++ b/osm2ma/test_configengine.py @@ -89,8 +89,8 @@ def test_class_function(self): maintian state during the aggreegation process. Therefore it is difficult to meaningfully subdivide tests. :return: """ - for arg in fixtures.attrib_list_args: - self.al.step(arg) + for arg, meta in fixtures.attrib_list_args: + self.al.step(arg, meta) self.assertEqual(self.al.finalize(), fixtures.attrib_list_result) @@ -111,8 +111,8 @@ def test_class_function(self): """ for args, result in fixtures.select_clause_args_and_result_pairs: sc = _SelectClause() - for key, val in args: - sc.step(key, val) + for key, val, meta in args: + sc.step(key, val, meta) self.assertEqual(sc.finalize(), result) diff --git a/osm2ma/testfiles/fixtures.xls b/osm2ma/testfiles/fixtures.xls index 9a4ce4e4f9f8327d9c5298ffa45960c88af0d422..2a092f58fd56485e4627fca8fb3cb3ce9a12b64f 100644 GIT binary patch delta 1592 zcma)5Z){Ul6#t$3=k>K>^F$aG0lO*CAdHShm;|;O<5&W0jmTI)=MuxL69-}(d_X!r zFb07!d}MDfGDl*{Otg+QX(6~tjm|l+seV|7;)IY4{4xiUIW*h6g~1XN&ECt+JMY|c ze&_c)=icfeSM`vqZJW^Q*-Zc1J?2hPLn|Y9#9~J6rrYi^%WU@lunybO)lS$p_ujt4 zj(P&Xa5vF0WAfL~Q;~-5QZZk!i5NpqGb{9mCcx|TR2f5`VScJVJYgj!ESDo^GEH%E z)$bd-g-G$wGXIi z0P327h7iyk20~FF+zCXxfX)Qa)e9v0f!+b2e-IcL1_sA~VT~~6Bxth;&L;?I3kma< z5*Dl=Wabf;uOZ|X5%My_#kPzvo9a4`#Z@tE^>* zsE%B*_#yhm?hRy>boiHLXTP~@o!x`OY6brYWc4jQndNtQ)ck==_4OV5nnA6$3$es; zdY*#GFyYKsgz6qb!H=rUPlUd!gqGh3ftv*H9m0r8Kd&>?8Vr6HgUiiuWg$cBQidHX z7;^Fy{2GStLWcTchLTbScR9n2w;0;qW!U{5gKs~BeuN=;oT2eUhO&Om!MSkWI-^fZnZCr4`6yR%Hgp%ydQDooZ|3(&QTEK@PElMEzT_^ z`$X)u4%RR&+Qagg=@mB1Xd%OFd0QOVaFN9}T;%6yYHipVth~mtcZ8z`0(H7TgCWrD z5(v2k!XAOBSD-UTpvxzaC=lrN3-p%=43r5B1_Xw82#oC((Dn&94+^9m6PWjbz=D$k znV$$OKP`}ZRvjj##rqK}fa!GC`lBEg${6kGw{-`N}*1h~8 zBo@Suat0%ntuC=O6iLsVDc$B!9OH*uI-2QjZZhXWZYv2ZZs#h-AC{f7Z|6Pl_q@;d zJm-CnZ@}Xl@Em=I9a%4p-*v0lc#lhIX3G#iBdFGh*FDA>r~V(&$Ib+Mr^)%m{e72? zPYHm)y+YT7(O0(?LAo`Mjrm55VBFe7W0UsS2ziATImWF`Fuu~pjzpa!k;gS_w9MAz znmbAzuScjO$=Rbx`y^wx{(!Jnj}U(O570FVjMRm_4`jDQxw;pVVO-0Pp`VSx{6tDJ zCKXJjeMVNnROk1%*;XEF1XfIHDJ@tioKEeSoq40BccsvbpVnRRZYcr0)j-)npwbVR zO~9cOK*Jf}NE={v0jxL2J^YTfuN}qA8O$dupxsY0X z?QN|@W@d=f%vDFllqd}mGX%}Pna#QoSBh}hJX%LF=E8E?MH8&^<@BST`bBvE-Bhfw z*OKR6PT6TcBiGrkN!Qsd-*i;^q>5?nS6jk-u5O!^LK=z3xw>81pNm> zPmoY^ldv{Km_9@Z+2e9j6DV;DEbs_K#|!i(3)H6wY)ltO%M=)zFA!KHuy?6I-YS9E z^#Z|n1sb;tZ2LeUV~+rRBGA5HpsG$_`R4+fY z?}1r$jK+h_ymFdtzTYffw0$=retk;RdUFB-YcJVaui9E}2o&|(Uhddh zhixqmiJB;h`iT;aQzV+7k~ld-BJhGl`&@~h`4YX0B!WvNZmyCDt(O?uEJ1}5x>q8) zOd_^YBEgiHen=v%K_cUbM7AZdpjAQzex%-VEXRF5%I*?HoE$P{K+S8ikLb}q~B z5LSMNJVg(iEifxOk@^(OL*;RDAO2~ZieY$9ExW&Ok*5LWmvNqtyzq7uQIw8 zGcLcyxVDDT_cr79R>t5?M%c%2?q!UtV#L%i;_4a6jf|9LM*2xcW`L2?&dBXyEbC>g z4l*WRXIuS6f43A?9;v=WtN&Ly=dXvg?y!6E?JpdBbWbdM?&guR*{u+ms{F$&yl8(` zq33yp-Z=`vR}^l(rVx5lVQ8fStyj=DD@5;5i2YC@p;Tddg+f}jLdHRbY`?;SCWX8c z3d_$ZtZh@+*r~AXl0wl{g_0Wzd;1lt?kLm@E7Ut2XpC}To!f!2_D~a!m#X3}^YCt| zS=6n Date: Wed, 22 Apr 2015 14:18:54 +0100 Subject: [PATCH 11/14] Started test for attributes in new shpfiles --- osm2ma/fixtures.py | 20 ++++++++++---------- osm2ma/ogrwrapper.py | 11 ++++++++--- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/osm2ma/fixtures.py b/osm2ma/fixtures.py index 0de1559..18bdec7 100644 --- a/osm2ma/fixtures.py +++ b/osm2ma/fixtures.py @@ -151,16 +151,16 @@ def path_to_fixtures_xls(): """ A list of which are unsorted, contain duplicates and contains blank strings. """ -attrib_list_args = [('f', 'FALSE'), - ('a', 'FALSE'), - ('b', 'TRUE' ), - ('c', 'FALSE'), - ('d', 'FALSE'), - ('', 'FALSE'), - ('e', 'FALSE'), - ('a', 'FALSE'), - ('a', 'FALSE')] -attrib_list_result = 'a, c, d, e, f, b' +attrib_list_args = [('ffff', 'FALSE'), + ('aaaa', 'FALSE'), + ('bbbb', 'TRUE' ), + ('cccc', 'FALSE'), + ('dddd', 'FALSE'), + ('', 'FALSE'), + ('eeee', 'FALSE'), + ('aaaa', 'FALSE'), + ('aaaa', 'FALSE')] +attrib_list_result = 'aaaa, cccc, dddd, eeee, ffff, bbbb' """ A list of tuples. diff --git a/osm2ma/ogrwrapper.py b/osm2ma/ogrwrapper.py index e4c18b6..18f0706 100644 --- a/osm2ma/ogrwrapper.py +++ b/osm2ma/ogrwrapper.py @@ -128,8 +128,11 @@ def get_geom_details(shpf_geom_type): def do_ogr2ogr_process(shp_defn, pbf_data_source, output_dir): shpf_name, data_cat, shpf_geom_type, attribs, where_clause = shp_defn cat_dir_path = _create_datacat_dir(output_dir, data_cat) - logging.debug( + logging.info( 'starting ogr2ogr process for shapefile: {}'.format(shpf_name)) + #logging.info( + # 'using attributes : {}'.format(', '.join(attribs))) + print "using attributes: ".join(attribs) osm_source_layer, dest_geom = get_geom_details(shpf_geom_type) @@ -143,14 +146,16 @@ def do_ogr2ogr_process(shp_defn, pbf_data_source, output_dir): # if pbf_lyr.GetFeatureCount() was working I'd test to only copy files with # > 0 features. logging.debug('do_ogr2ogr_process: about to create new shapefile') - shp_data_source, shp_lyr = _create_new_shpfile( - shpf_name, cat_dir_path, dest_geom, pbf_srs) + shp_data_source, shp_lyr = _create_new_shpfile(shpf_name, cat_dir_path, dest_geom, pbf_srs) + logging.debug('do_ogr2ogr_process: created new shapefile') logging.debug('do_ogr2ogr_process: about to copy attributes') _copy_attributes(pbf_lyr, shp_lyr, attribs) + logging.debug('do_ogr2ogr_process: copied attributes') logging.debug('do_ogr2ogr_process: about to copy features') _copy_features(pbf_lyr, shp_lyr, attribs) + logging.debug('do_ogr2ogr_process: copied features') # cmd_str = compose_ogr2ogr_cmd( # data_cat, geom_type, attribs, where_clause, pbf_file, shpf_name, From d08e306f67e96f578b4b364f3b3791dac26af27e Mon Sep 17 00:00:00 2001 From: Andy Smith Date: Wed, 22 Apr 2015 14:20:18 +0100 Subject: [PATCH 12/14] missed in prev commit - test for attrbs in shpfiles --- osm2ma/test_ogrwrapper.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/osm2ma/test_ogrwrapper.py b/osm2ma/test_ogrwrapper.py index c9eeb7b..685dc9f 100644 --- a/osm2ma/test_ogrwrapper.py +++ b/osm2ma/test_ogrwrapper.py @@ -38,6 +38,8 @@ def test_medium_create_new_shpfile(self): for extn in (u'.dbf', u'.prj', u'.shp', u'.shx'): self.assertTrue(os.path.exists(os.path.join(self.tmpdir, test_shpfile + extn))) + self.assertTrue(False, "Need to add test to ensure the correct attributes are created") + def testget_short_geom_details(self): source_layer, dest_geom = ogrw.get_geom_details("pt") self.assertEqual(source_layer, "points") From a6a5a756d3d9ee2574f771dda508ca77911b25ad Mon Sep 17 00:00:00 2001 From: Andy Smith Date: Thu, 23 Apr 2015 22:37:54 +0100 Subject: [PATCH 13/14] Added test_create_attributes --- osm2ma/ogrwrapper.py | 29 ++++++++++++++++-------- osm2ma/test_ogrwrapper.py | 46 ++++++++++++++++++++++++++++++++++----- 2 files changed, 61 insertions(+), 14 deletions(-) diff --git a/osm2ma/ogrwrapper.py b/osm2ma/ogrwrapper.py index 18f0706..8ee1c85 100644 --- a/osm2ma/ogrwrapper.py +++ b/osm2ma/ogrwrapper.py @@ -24,6 +24,9 @@ def _create_new_shpfile(shpf_name, shpf_dir, dest_geom_type, dest_srs): logging.info('Creating shapefile: {}'.format(shpf_name)) # Create the output Layer shpf_path = os.path.join(shpf_dir, shpf_name) + shpf_name = shpf_name.encode('utf-8') + shpf_path = shpf_path.encode('utf-8') + shpf_driver = ogr.GetDriverByName("ESRI Shapefile") # Remove output shapefile if it already exists @@ -32,28 +35,36 @@ def _create_new_shpfile(shpf_name, shpf_dir, dest_geom_type, dest_srs): # Create the output shapefile shp_data_source = shpf_driver.CreateDataSource(shpf_path) - # out_lyr_name = os.path.splitext( os.path.split( outShapefile )[1] )[0] - # This should be just the shapefile name without extension - need to check out_layer = None shpf_name = shpf_name.encode('utf-8') out_layer = shp_data_source.CreateLayer( os.path.splitext(shpf_name)[1], srs=dest_srs, geom_type=dest_geom_type ) - # out_layer = shpDataSource.CreateLayer(name=u'wrl_util_bdg_py_su_osm_pp') + #_create_attributes(source_lyr, out_layer, attribs) + return shp_data_source, out_layer # do stuff -def _copy_attributes(source_lyr, dest_lyr, target_attribs): +def _create_attributes(source_lyr, dest_lyr, target_attribs): logging.debug('copying attributes') # Add input Layer Fields to the output Layer if it is the one we want + # By copying the field definition from the source it saves us having to worry + # about data type or string widths etc. source_lyr_defn = source_lyr.GetLayerDefn() + lst_attribs = map(unicode.strip, target_attribs.split(",")) + # print(lst_attribs) for i in range(0, source_lyr_defn.GetFieldCount()): field_defn = source_lyr_defn.GetFieldDefn(i) field_name = field_defn.GetName() - if field_name in target_attribs: - dest_lyr.CreateField(field_defn) + dest_lyr.CreateField(field_defn) + # if field_name in lst_attribs: + # dest_lyr.CreateField(field_defn) + # else: + # print "PBF attrib not required; {}".format(field_name) + #for attrib in target_attribs.split('r'): + # lyr.CreateField(field_defn) # do stuff @@ -132,7 +143,7 @@ def do_ogr2ogr_process(shp_defn, pbf_data_source, output_dir): 'starting ogr2ogr process for shapefile: {}'.format(shpf_name)) #logging.info( # 'using attributes : {}'.format(', '.join(attribs))) - print "using attributes: ".join(attribs) + print "using attributes: {}".format(attribs) osm_source_layer, dest_geom = get_geom_details(shpf_geom_type) @@ -146,11 +157,11 @@ def do_ogr2ogr_process(shp_defn, pbf_data_source, output_dir): # if pbf_lyr.GetFeatureCount() was working I'd test to only copy files with # > 0 features. logging.debug('do_ogr2ogr_process: about to create new shapefile') - shp_data_source, shp_lyr = _create_new_shpfile(shpf_name, cat_dir_path, dest_geom, pbf_srs) + shp_data_source, shp_lyr = _create_new_shpfile(shpf_name, cat_dir_path, dest_geom, pbf_srs, ) logging.debug('do_ogr2ogr_process: created new shapefile') logging.debug('do_ogr2ogr_process: about to copy attributes') - _copy_attributes(pbf_lyr, shp_lyr, attribs) + _create_attributes(pbf_lyr, shp_lyr, attribs) logging.debug('do_ogr2ogr_process: copied attributes') logging.debug('do_ogr2ogr_process: about to copy features') diff --git a/osm2ma/test_ogrwrapper.py b/osm2ma/test_ogrwrapper.py index 685dc9f..04d0bfa 100644 --- a/osm2ma/test_ogrwrapper.py +++ b/osm2ma/test_ogrwrapper.py @@ -7,6 +7,7 @@ import ogrwrapper as ogrw from osgeo import ogr from osgeo import osr +from osgeo import gdal from configengine import xwalk_from_raw_config import fixtures @@ -14,6 +15,8 @@ class TestGlobalFunctions(unittest.TestCase): def setUp(self): + gdal.UseExceptions() + gdal.SetConfigOption("OGR_INTERLEAVED_READING", "YES") self.tmpdir = tempfile.mkdtemp() def tearDown(self): @@ -33,12 +36,45 @@ def test_medium_create_new_shpfile(self): dest_srs.ImportFromEPSG(3857) # from EPSG # test trying to create shpfile twice (shouldn't error) - for x in range(0, 2): - ogrw._create_new_shpfile(test_shpfile+".shp", self.tmpdir, ogr.wkbLineString, dest_srs) - for extn in (u'.dbf', u'.prj', u'.shp', u'.shx'): - self.assertTrue(os.path.exists(os.path.join(self.tmpdir, test_shpfile + extn))) + for i in range(2): + ogrw._create_new_shpfile(test_shpfile+".shp", + self.tmpdir, + ogr.wkbLineString, + dest_srs) + + # check the file exists on disk + for extn in (u'.dbf', u'.prj', u'.shp', u'.shx'): + self.assertTrue(os.path.exists(os.path.join(self.tmpdir, test_shpfile + extn))) + + def test_create_attributes(self): + test_shpfile = "test_shpfile" + dest_srs = osr.SpatialReference() + dest_srs.ImportFromEPSG(3857) # from EPSG + # test_attribs = ["attrib1", "attrib2", "attrib3", "attrib4", "attrib5"] + test_attribs = ["name", "other_tags"] + + shp_source, shp_lyr = ogrw._create_new_shpfile( + test_shpfile+".shp", + self.tmpdir, + ogr.wkbLineString, + dest_srs) + + pbf_driver = ogr.GetDriverByName("OSM") + pbf_data_source = pbf_driver.Open(fixtures.example_pbf, 0) + pbf_lyr = pbf_data_source.GetLayerByName("points") + + ogrw._create_attributes(pbf_lyr, shp_lyr, ", ".join(test_attribs)) + lyr_defn = shp_lyr.GetLayerDefn() + result_attribs = [] + for i in range(lyr_defn.GetFieldCount()): + result_attribs.append(lyr_defn.GetFieldDefn(i).GetName()) + + print(sorted(test_attribs)) + print(sorted(result_attribs)) - self.assertTrue(False, "Need to add test to ensure the correct attributes are created") + self.assertEquals(sorted(test_attribs), + sorted(result_attribs), + "Need to add test to ensure the correct attributes are created") def testget_short_geom_details(self): source_layer, dest_geom = ogrw.get_geom_details("pt") From a214af23c33ce39e91c6facd9c37fffbbaf29f38 Mon Sep 17 00:00:00 2001 From: Andy Smith Date: Sat, 2 May 2015 13:04:12 +0545 Subject: [PATCH 14/14] added custom osmconf.ini --- osm2ma/ogrwrapper.py | 4 +- osm2ma/osmconf.ini | 110 +++++++++++++++++++++++++++++++++++ osm2ma/osmconf.origional.ini | 110 +++++++++++++++++++++++++++++++++++ 3 files changed, 223 insertions(+), 1 deletion(-) create mode 100644 osm2ma/osmconf.ini create mode 100644 osm2ma/osmconf.origional.ini diff --git a/osm2ma/ogrwrapper.py b/osm2ma/ogrwrapper.py index 8ee1c85..18f9004 100644 --- a/osm2ma/ogrwrapper.py +++ b/osm2ma/ogrwrapper.py @@ -178,7 +178,9 @@ def do_ogr2ogr_process(shp_defn, pbf_data_source, output_dir): def batch_convert(xwalk, pbf_file, output_dir): gdal.UseExceptions() gdal.SetConfigOption("OGR_INTERLEAVED_READING", "YES") - # Open input PBF driver + + osmconf_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), r'osmconf.ini') + gdal.SetConfigOption('OSM_CONFIG_FILE', osmconf_path) # Open input PBF driver pbf_driver = ogr.GetDriverByName("OSM") pbf_data_source = pbf_driver.Open(pbf_file, 0) diff --git a/osm2ma/osmconf.ini b/osm2ma/osmconf.ini new file mode 100644 index 0000000..4de9a87 --- /dev/null +++ b/osm2ma/osmconf.ini @@ -0,0 +1,110 @@ +# +# Configuration file for OSM import +# + +# put here the name of keys for ways that are assumed to be polygons if they are closed +# see http://wiki.openstreetmap.org/wiki/Map_Features +closed_ways_are_polygons=aeroway,amenity,boundary,building,craft,geological,historic,landuse,leisure,military,natural,office,place,shop,sport,tourism + +# comment to avoid laundering of keys ( ':' turned into '_' ) +attribute_name_laundering=yes + +# uncomment to report all nodes, including the ones without any (significant) tag +#report_all_nodes=yes + +# uncomment to report all ways, including the ones without any (significant) tag +#report_all_ways=yes + +[points] +# common attributes +osm_id=yes +osm_version=no +osm_timestamp=no +osm_uid=no +osm_user=no +osm_changeset=no + +# keys to report as OGR fields +attributes=name,barrier,highway,ref,address,is_in,place,man_made +# keys that, alone, are not significant enough to report a node as a OGR point +unsignificant=created_by,converted_by,source,time,ele,attribution +# keys that should NOT be reported in the "other_tags" field +ignore=created_by,converted_by,source,time,ele,note,openGeoDB:,fixme,FIXME +# uncomment to avoid creation of "other_tags" field +other_tags=no +# uncomment to create "all_tags" field. "all_tags" and "other_tags" are exclusive +all_tags=yes + +[lines] +# common attributes +osm_id=yes +osm_version=no +osm_timestamp=no +osm_uid=no +osm_user=no +osm_changeset=no + +# keys to report as OGR fields +attributes=name,highway,waterway,aerialway,barrier,man_made +# keys that should NOT be reported in the "other_tags" field +ignore=created_by,converted_by,source,time,ele,note,openGeoDB:,fixme,FIXME +# uncomment to avoid creation of "other_tags" field +other_tags=no +# uncomment to create "all_tags" field. "all_tags" and "other_tags" are exclusive +all_tags=yes + +[multipolygons] +# common attributes +# note: for multipolygons, osm_id=yes instanciates a osm_id field for the id of relations +# and a osm_way_id field for the id of closed ways. Both fields are exclusively set. +osm_id=yes +osm_version=no +osm_timestamp=no +osm_uid=no +osm_user=no +osm_changeset=no + +# keys to report as OGR fields +attributes=name,type,aeroway,amenity,admin_level,barrier,boundary,building,craft,geological,historic,land_area,landuse,leisure,man_made,military,natural,office,place,shop,sport,tourism +# keys that should NOT be reported in the "other_tags" field +ignore=area,created_by,converted_by,source,time,ele,note,openGeoDB:,fixme,FIXME +# uncomment to avoid creation of "other_tags" field +other_tags=no +# uncomment to create "all_tags" field. "all_tags" and "other_tags" are exclusive +all_tags=yes + +[multilinestrings] +# common attributes +osm_id=yes +osm_version=no +osm_timestamp=no +osm_uid=no +osm_user=no +osm_changeset=no + +# keys to report as OGR fields +attributes=name,type +# keys that should NOT be reported in the "other_tags" field +ignore=area,created_by,converted_by,source,time,ele,note,openGeoDB:,fixme,FIXME +# uncomment to avoid creation of "other_tags" field +#other_tags=no +# uncomment to create "all_tags" field. "all_tags" and "other_tags" are exclusive +all_tags=yes + +[other_relations] +# common attributes +osm_id=yes +osm_version=no +osm_timestamp=no +osm_uid=no +osm_user=no +osm_changeset=no + +# keys to report as OGR fields +attributes=name,type +# keys that should NOT be reported in the "other_tags" field +ignore=area,created_by,converted_by,source,time,ele,note,openGeoDB:,fixme,FIXME +# uncomment to avoid creation of "other_tags" field +other_tags=no +# uncomment to create "all_tags" field. "all_tags" and "other_tags" are exclusive +all_tags=yes diff --git a/osm2ma/osmconf.origional.ini b/osm2ma/osmconf.origional.ini new file mode 100644 index 0000000..81e0df7 --- /dev/null +++ b/osm2ma/osmconf.origional.ini @@ -0,0 +1,110 @@ +# +# Configuration file for OSM import +# + +# put here the name of keys for ways that are assumed to be polygons if they are closed +# see http://wiki.openstreetmap.org/wiki/Map_Features +closed_ways_are_polygons=aeroway,amenity,boundary,building,craft,geological,historic,landuse,leisure,military,natural,office,place,shop,sport,tourism + +# comment to avoid laundering of keys ( ':' turned into '_' ) +attribute_name_laundering=yes + +# uncomment to report all nodes, including the ones without any (significant) tag +#report_all_nodes=yes + +# uncomment to report all ways, including the ones without any (significant) tag +#report_all_ways=yes + +[points] +# common attributes +osm_id=yes +osm_version=no +osm_timestamp=no +osm_uid=no +osm_user=no +osm_changeset=no + +# keys to report as OGR fields +attributes=name,barrier,highway,ref,address,is_in,place,man_made +# keys that, alone, are not significant enough to report a node as a OGR point +unsignificant=created_by,converted_by,source,time,ele,attribution +# keys that should NOT be reported in the "other_tags" field +ignore=created_by,converted_by,source,time,ele,note,openGeoDB:,fixme,FIXME +# uncomment to avoid creation of "other_tags" field +#other_tags=no +# uncomment to create "all_tags" field. "all_tags" and "other_tags" are exclusive +#all_tags=yes + +[lines] +# common attributes +osm_id=yes +osm_version=no +osm_timestamp=no +osm_uid=no +osm_user=no +osm_changeset=no + +# keys to report as OGR fields +attributes=name,highway,waterway,aerialway,barrier,man_made +# keys that should NOT be reported in the "other_tags" field +ignore=created_by,converted_by,source,time,ele,note,openGeoDB:,fixme,FIXME +# uncomment to avoid creation of "other_tags" field +#other_tags=no +# uncomment to create "all_tags" field. "all_tags" and "other_tags" are exclusive +#all_tags=yes + +[multipolygons] +# common attributes +# note: for multipolygons, osm_id=yes instanciates a osm_id field for the id of relations +# and a osm_way_id field for the id of closed ways. Both fields are exclusively set. +osm_id=yes +osm_version=no +osm_timestamp=no +osm_uid=no +osm_user=no +osm_changeset=no + +# keys to report as OGR fields +attributes=name,type,aeroway,amenity,admin_level,barrier,boundary,building,craft,geological,historic,land_area,landuse,leisure,man_made,military,natural,office,place,shop,sport,tourism +# keys that should NOT be reported in the "other_tags" field +ignore=area,created_by,converted_by,source,time,ele,note,openGeoDB:,fixme,FIXME +# uncomment to avoid creation of "other_tags" field +#other_tags=no +# uncomment to create "all_tags" field. "all_tags" and "other_tags" are exclusive +#all_tags=yes + +[multilinestrings] +# common attributes +osm_id=yes +osm_version=no +osm_timestamp=no +osm_uid=no +osm_user=no +osm_changeset=no + +# keys to report as OGR fields +attributes=name,type +# keys that should NOT be reported in the "other_tags" field +ignore=area,created_by,converted_by,source,time,ele,note,openGeoDB:,fixme,FIXME +# uncomment to avoid creation of "other_tags" field +#other_tags=no +# uncomment to create "all_tags" field. "all_tags" and "other_tags" are exclusive +#all_tags=yes + +[other_relations] +# common attributes +osm_id=yes +osm_version=no +osm_timestamp=no +osm_uid=no +osm_user=no +osm_changeset=no + +# keys to report as OGR fields +attributes=name,type +# keys that should NOT be reported in the "other_tags" field +ignore=area,created_by,converted_by,source,time,ele,note,openGeoDB:,fixme,FIXME +# uncomment to avoid creation of "other_tags" field +#other_tags=no +# uncomment to create "all_tags" field. "all_tags" and "other_tags" are exclusive +#all_tags=yes