From a8192052b7b4aa924fbbf255f0564f92d9a49188 Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Fri, 19 Jan 2024 09:59:13 +0000 Subject: [PATCH 01/22] Gruntfile: Switch to python3.11 --- Gruntfile.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gruntfile.js b/Gruntfile.js index 3c215beff..bbc39cbd2 100755 --- a/Gruntfile.js +++ b/Gruntfile.js @@ -5,7 +5,7 @@ const path = require('path'); partial_install_site = "http://www.onezoom.org"; partial_local_install_site = "http://127.0.0.1:8000"; // if you are running a local installation -preferred_python3 = "python3.7"; // in case you have multiple python3 versions installed +preferred_python3 = "python3.11"; // in case you have multiple python3 versions installed web2py_py = path.join(path.dirname(path.dirname(process.cwd())), 'web2py.py'); /** Generate a function to execute a web2py script, handing over all arguments */ From 691d874b709b4707b2bfb31450b32b4239147bca Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Fri, 19 Jan 2024 10:33:52 +0000 Subject: [PATCH 02/22] package.json: Support Node 18.x To run webpack on node 18 we need to set --openssl-legacy-provider until we can upgrade webpack. --- package.json | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/package.json b/package.json index e7ba30e73..419228a0e 100644 --- a/package.json +++ b/package.json @@ -13,9 +13,10 @@ }, "scripts": { "test": "npx babel-tape-runner OZprivate/rawJS/OZTreeModule/tests/*.js", - "compile_js": "webpack --mode production", - "compile_js_dev:watch": "webpack --watch --config webpack.config.dev.js", - "compile_js_dev": "webpack --mode development" + "comment": "openssl-legacy-provider is needed for node 18 until we upgrade webpack - https://stackoverflow.com/a/69699772", + "compile_js": "NODE_OPTIONS=--openssl-legacy-provider webpack --mode production", + "compile_js_dev:watch": "NODE_OPTIONS=--openssl-legacy-provider webpack --watch --config webpack.config.dev.js", + "compile_js_dev": "NODE_OPTIONS=--openssl-legacy-provider webpack --mode development" }, "repository": { "type": "git", From 433e2ba8e28e9b4235a784422773b004488e21d4 Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Fri, 19 Jan 2024 10:48:22 +0000 Subject: [PATCH 03/22] Gruntfile: web2py_configure for web2py housekeeping Automate housekeeping for web2py in a Grunt task --- Gruntfile.js | 14 ++++++++++++-- README.markdown | 4 +--- _COPY_CONTENTS_TO_WEB2PY_DIR/README.markdown | 9 +-------- 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/Gruntfile.js b/Gruntfile.js index bbc39cbd2..80d26f39c 100755 --- a/Gruntfile.js +++ b/Gruntfile.js @@ -29,6 +29,16 @@ module.exports = function (grunt) { grunt.initConfig({ pkg: grunt.file.readJSON('package.json'), exec: { + web2py_configure: { + cwd: "../../", + command: [ + 'git submodule update --init --recursive', + 'ln -sf applications/OZtree/_COPY_CONTENTS_TO_WEB2PY_DIR/routes.py routes.py', + '( [ -d applications/welcome ] && rm -r -- "applications/welcome" || true )', + '( [ -d applications/examples ] && rm -r -- "applications/examples" || true )', + '( [ -f applications/OZtree/private/appconfig.ini ] || { cp applications/OZtree/private/appconfig.ini.example applications/OZtree/private/appconfig.ini ; echo "****** edit private/appconfig.ini"; exit 1; } )', + ].join(" && "), + }, compile_python: { // compile python to make it faster on the server and hence reduce server load. // should probably be run using the same python version as used to run web2py @@ -220,6 +230,6 @@ module.exports = function (grunt) { grunt.registerTask("compile-js_dev", ["exec:compile_js_dev"]); grunt.registerTask("partial-install", ["compile-js", "css", "copy:dev", "curl-dir:get_minlife", "exec:convert_links_to_local"]); grunt.registerTask("partial-local-install", ["compile-js", "css", "copy:dev", "curl-dir:get_local_minlife", "exec:convert_links_to_local"]); - grunt.registerTask("prod", ["clean", "compile-python", "compile-js", "css", "compress", "copy:prod", "make_release_info", "docs"]); - grunt.registerTask("dev", ["clean", "compile-js_dev", "css", "compress", "copy:dev", "make_release_info", "docs"]); + grunt.registerTask("prod", ["clean", "web2py", "compile-python", "compile-js", "css", "compress", "copy:prod", "make_release_info", "docs"]); + grunt.registerTask("dev", ["clean", "web2py", "compile-js_dev", "css", "compress", "copy:dev", "make_release_info", "docs"]); }; diff --git a/README.markdown b/README.markdown index 19e6ef45d..a745830d9 100755 --- a/README.markdown +++ b/README.markdown @@ -65,7 +65,7 @@ Before anything else, get the OZtree app from [github](https://github.com/OneZoo 2. Install command-line software by running `npm install -g grunt-cli` (you may need to do all this with administrator privileges). 3. Run `npm install` from within the OZtree folder you moved in step 1. then run `grunt dev` (or `grunt prod` if in production mode) - see *"[Building the OneZoom tree viewer](#building-the-onezoom-tree-viewer)"*. 3. [Install](http://dev.mysql.com/downloads/mysql/) & start MySQL, then create a new database (see *"[Setting up the database backend](#setting-up-the-database-backend)"*) -4. Create a appconfig.ini file in `OZtree/private`, with `migrate=1` and which references this database with the appropriate username and password. We also recommend copying the `routes.py` file from `OZtree/_COPY_CONTENTS_TO_WEB2PY_DIR` to the top level of your web2py installation - see *"[Web2py installation](#web2py-installation)"* +4. Edit `private/appconfig.ini` file in `OZtree/private`, with `migrate=1` and with the appropriate database username and password. 5. Fire up a temporary web2py server and visit the main page to create the (empty) database tables - see *"[Starting and shutting down web2py](#starting-and-shutting-down-web2py)"* 6. Load up data into the tables: first create a user and assign it a 'manager' role in the `auth_` tables using the web2py database admin pages, then load the other tables using data from the original OneZoom site (e.g. sent to you via file transfer) - see *"[Filling the database](#filling-the-database)"*. 7. Optimise your installation: @@ -164,8 +164,6 @@ When web2py is run, it will print instructions telling how to shut down the web2 **If this is a new installation** you should now visit `http://127.0.0.1:8000/OZtree/default/` or `https://127.0.0.1:8000/OZtree/default/` to force web2py to create database tables. To load data into the tables, see "Loading Data", below. -Also, if you want to make OneZoom the default application, make a copy of the routes.py file in the folder labelled `_COPY_CONTENTS_TO_WEB2PY_DIR` and place it in the top level web2py directory (see `_COPY_CONTENTS_TO_WEB2PY_DIR/README.markdown`). - Once tables are created, and everything is working, you can set `is_testing = False` in `models/db.py` and `migrate=0` in `private/appconfig.ini`. This will mean that web2py will not make any changes to table structures in the DB, and also that changes to appconfig.ini will require a web2py restart. ### Web2py folder structure diff --git a/_COPY_CONTENTS_TO_WEB2PY_DIR/README.markdown b/_COPY_CONTENTS_TO_WEB2PY_DIR/README.markdown index 7bf3f2c8c..dc513b8d4 100755 --- a/_COPY_CONTENTS_TO_WEB2PY_DIR/README.markdown +++ b/_COPY_CONTENTS_TO_WEB2PY_DIR/README.markdown @@ -1,8 +1 @@ -### Making OneZoom the default application - -To make OneZoom the default web2py app, a copy of the accompanying routes.py file can placed at the root of the web2py folder (at the same level as the `applications` folder). - -### Disabling other applications -You may also wish to delete the `welcome` and `examples` apps (just delete the folders), or disable them from the web2py admin web page (e.g. at http://127.0.0.1:8000/admin). - -You probably shouldn't disable the admin app, as this is the main way of adding users etc to the OneZoom app. It should only be accessible over https (and maybe only locally) anyway, so is not that much of a security risk. \ No newline at end of file +# Python files here will be symlinked by Grunt in web2py_configure From 88a38e2f33deca2160f91a1330a39142ae07ba67 Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Fri, 19 Jan 2024 10:57:02 +0000 Subject: [PATCH 04/22] Gruntfile: venv for web2py #676 We need an up-do-date pymysql, get it by wrapping a venv around web2py Add a web2py-run helper to use when running standalone scripts, that will get the web2py setup right. --- Gruntfile.js | 24 ++++++++++++------- .../Utilities/OneOff/make_usernames.py | 2 +- requirements.txt | 1 + tests/unit/test_modules_embed.py | 6 +++-- tests/unit/test_modules_sponsorship.py | 4 ++-- tests/unit/test_modules_usernames.py | 6 +++-- tests/util.py | 2 +- web2py-run | 8 +++++++ 8 files changed, 37 insertions(+), 16 deletions(-) create mode 100644 requirements.txt create mode 100755 web2py-run diff --git a/Gruntfile.js b/Gruntfile.js index 80d26f39c..0e2925c48 100755 --- a/Gruntfile.js +++ b/Gruntfile.js @@ -7,12 +7,13 @@ partial_install_site = "http://www.onezoom.org"; partial_local_install_site = "http://127.0.0.1:8000"; // if you are running a local installation preferred_python3 = "python3.11"; // in case you have multiple python3 versions installed web2py_py = path.join(path.dirname(path.dirname(process.cwd())), 'web2py.py'); +venv_python = path.join(path.dirname(path.dirname(process.cwd())), 'bin/python'); /** Generate a function to execute a web2py script, handing over all arguments */ function exec_web2py_script(script_name, init_args) { return function () { return [ - preferred_python3, + venv_python, web2py_py, '-S OZtree/default', '-M', @@ -37,6 +38,15 @@ module.exports = function (grunt) { '( [ -d applications/welcome ] && rm -r -- "applications/welcome" || true )', '( [ -d applications/examples ] && rm -r -- "applications/examples" || true )', '( [ -f applications/OZtree/private/appconfig.ini ] || { cp applications/OZtree/private/appconfig.ini.example applications/OZtree/private/appconfig.ini ; echo "****** edit private/appconfig.ini"; exit 1; } )', + preferred_python3 + ' -m venv .', + './bin/pip install -r applications/OZtree/requirements.txt', + ].join(" && "), + }, + web2py_start_dev: { + cwd: "../../", + command: [ + '( [ -f oz.crt ] || openssl req -newkey rsa:2048 -x509 -days 365 -nodes -keyout oz.key -subj "/CN=dev.onezoom/" -out oz.crt; )', + venv_python + ' web2py.py -c oz.crt -k oz.key -p 8000 -a pass', ].join(" && "), }, compile_python: { @@ -44,11 +54,7 @@ module.exports = function (grunt) { // should probably be run using the same python version as used to run web2py cwd: "../../", command: - preferred_python3 + ' -c "import gluon.compileapp; gluon.compileapp.compile_application(\'' - + process.cwd() - + '\', skip_failed_views=True)"' - + ' || ' + // If python 3.7 isn't available, use the system-defined python3 instead - 'python3 -c "import gluon.compileapp; gluon.compileapp.compile_application(\'' + venv_python + ' -c "import gluon.compileapp; gluon.compileapp.compile_application(\'' + process.cwd() + '\', skip_failed_views=True)"' }, @@ -69,7 +75,7 @@ module.exports = function (grunt) { } }, make_release_info: { - command: 'git describe --tags > RELEASE_INFO && python3 OZprivate/ServerScripts/Utilities/get_release_name.py RELEASE_INFO >> RELEASE_INFO' + command: 'git describe --tags > RELEASE_INFO && ' + venv_python + ' OZprivate/ServerScripts/Utilities/get_release_name.py RELEASE_INFO >> RELEASE_INFO' }, test_server: { command: function () { @@ -105,7 +111,7 @@ module.exports = function (grunt) { // Any .html file in /static is fair game. See documentation in // https://github.com/OneZoom/OZtree#onezoom-setup command: - "python3 OZprivate/ServerScripts/Utilities/partial_install.py" + + venv_python + " OZprivate/ServerScripts/Utilities/partial_install.py" + " --search " + partial_local_install_site + // replace local urls, for partial local install " --replace " + partial_install_site + " static/*.html" @@ -230,6 +236,8 @@ module.exports = function (grunt) { grunt.registerTask("compile-js_dev", ["exec:compile_js_dev"]); grunt.registerTask("partial-install", ["compile-js", "css", "copy:dev", "curl-dir:get_minlife", "exec:convert_links_to_local"]); grunt.registerTask("partial-local-install", ["compile-js", "css", "copy:dev", "curl-dir:get_local_minlife", "exec:convert_links_to_local"]); + grunt.registerTask("web2py", ["exec:web2py_configure"]); grunt.registerTask("prod", ["clean", "web2py", "compile-python", "compile-js", "css", "compress", "copy:prod", "make_release_info", "docs"]); grunt.registerTask("dev", ["clean", "web2py", "compile-js_dev", "css", "compress", "copy:dev", "make_release_info", "docs"]); + grunt.registerTask("start-dev", ['exec:web2py_start_dev']); }; diff --git a/OZprivate/ServerScripts/Utilities/OneOff/make_usernames.py b/OZprivate/ServerScripts/Utilities/OneOff/make_usernames.py index aed707ef4..9c53d956f 100644 --- a/OZprivate/ServerScripts/Utilities/OneOff/make_usernames.py +++ b/OZprivate/ServerScripts/Utilities/OneOff/make_usernames.py @@ -1,6 +1,6 @@ """ Run from the OZtree directory as -python3 ../../web2py.py -S OZtree -M -R applications/OZtree/OZprivate/ServerScripts/Utilities/OneOff/make_usernames.py +./web2py-run OZprivate/ServerScripts/Utilities/OneOff/make_usernames.py After 2 passes, will allocate usernames of remaining unallocated reservations using a species name plus year. diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 000000000..77947e063 --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +pymysql==1.1.0 diff --git a/tests/unit/test_modules_embed.py b/tests/unit/test_modules_embed.py index 832a14bf9..6c5558e95 100644 --- a/tests/unit/test_modules_embed.py +++ b/tests/unit/test_modules_embed.py @@ -1,6 +1,8 @@ """ -Run with -python3 web2py.py -S OZtree -M -R applications/OZtree/tests/unit/test_modules_embed.py +Run with:: + + ./web2py-run tests/unit/test_modules_embed.py + """ import re import unittest diff --git a/tests/unit/test_modules_sponsorship.py b/tests/unit/test_modules_sponsorship.py index 63cd4165f..60515d3c8 100644 --- a/tests/unit/test_modules_sponsorship.py +++ b/tests/unit/test_modules_sponsorship.py @@ -1,7 +1,7 @@ """ -Run with +Run with:: -python3 web2py.py -S OZtree -M -R applications/OZtree/tests/unit/test_modules_sponsorship.py + ./web2py-run tests/unit/test_modules_sponsorship.py Note you should make sure prices are set before running tests (manage/SET_PRICES.html) """ diff --git a/tests/unit/test_modules_usernames.py b/tests/unit/test_modules_usernames.py index ccbfc26f9..a72e7a207 100644 --- a/tests/unit/test_modules_usernames.py +++ b/tests/unit/test_modules_usernames.py @@ -1,6 +1,8 @@ """ -Run with -python3 web2py.py -S OZtree -M -R applications/OZtree/tests/unit/test_modules_username.py +Run with:: + + ./web2py-run tests/unit/test_modules_username.py + """ import unittest diff --git a/tests/util.py b/tests/util.py index 800459fe6..30d201aad 100644 --- a/tests/util.py +++ b/tests/util.py @@ -72,7 +72,7 @@ def __init__(self, appconfig_file=None): self.pid = None if self.is_local(): print("> starting web2py") - cmd = ['python3', os.path.join(web2py_app_dir, '..','..','web2py.py'), '-Q', '-i', ip, '-p', port, '-a', 'pass'] + cmd = [os.path.join(web2py_app_dir, '..','..','bin', 'python3'), os.path.join(web2py_app_dir, '..','..','web2py.py'), '-Q', '-i', ip, '-p', port, '-a', 'pass'] if appconfig_file is not None: cmd += ['--args', appconfig_file] self.pid = subprocess.Popen(cmd) diff --git a/web2py-run b/web2py-run new file mode 100755 index 000000000..c17db9934 --- /dev/null +++ b/web2py-run @@ -0,0 +1,8 @@ +#!/bin/sh +set -eu + +SCRIPT="applications/OZtree/$1" +shift + +cd ../../ +exec ./bin/python3 web2py.py -S OZtree -M -R "${SCRIPT}" --args $* From 48a5ac50758d4b968ec0a41c2286d15bbc754d3a Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Fri, 19 Jan 2024 11:21:51 +0000 Subject: [PATCH 05/22] controllers/API: Remove search_log This will impact search performance, by forcing all search queries to go through the unique index before searching. Remove, replace with google analytics later. --- OZprivate/ServerScripts/SQL/create_db_indexes.sql | 4 ---- controllers/API.py | 10 ---------- models/db.py | 6 ------ private/appconfig.ini.example | 1 - 4 files changed, 21 deletions(-) diff --git a/OZprivate/ServerScripts/SQL/create_db_indexes.sql b/OZprivate/ServerScripts/SQL/create_db_indexes.sql index 851b6b815..339716d16 100644 --- a/OZprivate/ServerScripts/SQL/create_db_indexes.sql +++ b/OZprivate/ServerScripts/SQL/create_db_indexes.sql @@ -37,7 +37,6 @@ call MakeFullUnicode('images_by_ott', 'rights'); call MakeFullUnicode('images_by_ott', 'licence'); call MakeFullUnicode('images_by_name', 'rights'); call MakeFullUnicode('images_by_name', 'licence'); -call MakeFullUnicode('search_log', 'search_string'); # note make sure that the name column in vernacular_by_name and the name column in ordered_leaves and ordered_nodes are of the same character set otherwise search can get incredibly slow even with indexes. @@ -213,9 +212,6 @@ CREATE FULLTEXT INDEX ft_user_sponsor_info_index ON reservations (user_more_info DROP INDEX ipni_index ON PoWO; CREATE INDEX ipni_index ON PoWO (ipni_int) USING HASH; -DROP INDEX string_index ON search_log; -CREATE INDEX string_index ON search_log (search_string) USING HASH; - DROP INDEX identifier_index ON partners; CREATE INDEX identifier_index ON partners (partner_identifier) USING HASH; diff --git a/controllers/API.py b/controllers/API.py index f9d32fb62..bc646aede 100755 --- a/controllers/API.py +++ b/controllers/API.py @@ -126,16 +126,6 @@ def search_node(): session.forget(response) response.headers["Access-Control-Allow-Origin"] = '*' searchFor = make_unicode(request.vars.query or '') - try: - if myconf.take('general.log_search_strings'): - if request.vars.no_log: - #'no_log' flag set: this is probably us blatting the search for testing purposes - pass - else: - db.search_log.update_or_insert(db.search_log.search_string==searchFor, search_string=searchFor, search_count=db.search_log.search_count+1) - - except: - pass res1 = search_for_name() if len(res1['leaf_hits']) + len(res1['node_hits']) <15: try: diff --git a/models/db.py b/models/db.py index 55b1e9d78..f63095eb9 100755 --- a/models/db.py +++ b/models/db.py @@ -664,12 +664,6 @@ Field('leaf_click_count', type='integer'), format = '%(ott)s', migrate=is_testing) -# this table collects a list of search terms so we can optimise search -db.define_table('search_log', - Field('search_string', type='string', notnull=True, unique=True, length=name_length_chars), #this should be utf8mb4 - Field('search_count', type='integer', notnull=True), - format = '%(search_string)s', migrate=is_testing) - # This table buffers recently 'visited' EoL taxa (visited through the window popup or via the copyright link) # taxa in this table are stored until at least 1 minute after the taxon is visited, and then read by the EOL update # script (EoLQueryPicsNames.py) to check for updates to the crop location, ratings, etc. Once checked, the taxon diff --git a/private/appconfig.ini.example b/private/appconfig.ini.example index a873376c1..d92c84e1d 100755 --- a/private/appconfig.ini.example +++ b/private/appconfig.ini.example @@ -29,7 +29,6 @@ separator = url = https://www.sandbox.paypal.com [general] -;log_search_strings = 1 [images] ; * url_base: get thumbnail images from this source. If not From aa80e2db2877057f0baa3f92c65be6e69625dab1 Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Fri, 19 Jan 2024 15:01:18 +0000 Subject: [PATCH 06/22] models/db: is_testing iff running under rocket Instead of having to tweak is_testing for production use, check the request environment to see what server we're using. --- README.markdown | 4 ++-- models/db.py | 36 +++++++++++++++--------------------- 2 files changed, 17 insertions(+), 23 deletions(-) diff --git a/README.markdown b/README.markdown index a745830d9..76c7a2712 100755 --- a/README.markdown +++ b/README.markdown @@ -70,7 +70,7 @@ Before anything else, get the OZtree app from [github](https://github.com/OneZoo 6. Load up data into the tables: first create a user and assign it a 'manager' role in the `auth_` tables using the web2py database admin pages, then load the other tables using data from the original OneZoom site (e.g. sent to you via file transfer) - see *"[Filling the database](#filling-the-database)"*. 7. Optimise your installation: * create indexes on the tables by running the SQL script in `OZtree/OZprivate/ServerScripts/SQL/create_db_indexes.sql`. You can do this, for example, by running `SOURCE /path/to/OZtree/OZprivate/ServerScripts/SQL/create_db_indexes.sql` within a mysql client. - * set `is_testing = False` in `models/db.py` and `migrate=0` in appconfig.ini. + * set `migrate=0` in appconfig.ini. ## Downloading the OZtree app @@ -164,7 +164,7 @@ When web2py is run, it will print instructions telling how to shut down the web2 **If this is a new installation** you should now visit `http://127.0.0.1:8000/OZtree/default/` or `https://127.0.0.1:8000/OZtree/default/` to force web2py to create database tables. To load data into the tables, see "Loading Data", below. -Once tables are created, and everything is working, you can set `is_testing = False` in `models/db.py` and `migrate=0` in `private/appconfig.ini`. This will mean that web2py will not make any changes to table structures in the DB, and also that changes to appconfig.ini will require a web2py restart. +Once tables are created, and everything is working, you can set `migrate=0` in `private/appconfig.ini`. This will mean that web2py will not make any changes to table structures in the DB ### Web2py folder structure diff --git a/models/db.py b/models/db.py index f63095eb9..be8754d78 100755 --- a/models/db.py +++ b/models/db.py @@ -11,12 +11,6 @@ ## if SSL/HTTPS is properly configured and you want all HTTP requests to ## be redirected to HTTPS, uncomment the line below: # request.requires_https() -#set the default language -T.set_current_languages('en', 'en-en') -#ALL pages can set ?lang=XXX to override the browser default for translating strings -if request.vars.lang: - T.force(request.vars.lang) - ## app configuration made easy. Look inside private/appconfig.ini from gluon.contrib.appconfig import AppConfig @@ -26,26 +20,26 @@ ## Useful global variables ######################################################################### -## once in production, set is_testing=False to gain optimizations -## this will also set migration=False for all tables, so that the DB table definitions are fixed -is_testing = True +# Running under rocket --> is_testing is true +is_testing = (request.env.server_software or '').lower().startswith('rocket') -## get config params etc -if is_testing: +## Read configuration +if is_testing and len(request.env.cmd_options.args) > 1 and os.path.isfile(request.env.cmd_options.args[-1]): # For unit testing, we might want to load a different appconfig.ini file, which can # be passed in to the rocket server as the last arg on the command-line. # (on the main server this is not used, and we default back to appconfig.ini - try: - if os.path.isfile(request.env.cmd_options.args[-1]): - myconf = AppConfig(request.env.cmd_options.args[-1], reload=True) - else: - raise IOError("No such file") - except (IOError, IndexError, AttributeError): - myconf = AppConfig(reload=True) #changes to appconfig.ini do not require restart - T.is_writable = True #allow translators to add new languages e.g. on the test (beta) site, but not on prod + myconf = AppConfig(request.env.cmd_options.args[-1], reload=is_testing) else: - myconf = AppConfig() #faster to read once and never re-update - T.is_writable = False + # NB: When running under rocket, re-load config every request with is_testing + myconf = AppConfig(reload=is_testing) + +## Configure i18n +T.set_current_languages('en', 'en-en') +# Allow translators to add new languages e.g. on the test (beta) site, but not on prod +T.is_writable = is_testing +#ALL pages can set ?lang=XXX to override the browser default for translating strings +if request.vars.lang: + T.force(request.vars.lang) try: thumb_base_url = myconf.take('images.url_base') From 5b8cb592f46e7f4cbe908be4ca9802606c08f16c Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Fri, 19 Jan 2024 15:09:52 +0000 Subject: [PATCH 07/22] models/db: Database session-handling #82 Not that we should be using sessions much, if at all. --- models/db.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/models/db.py b/models/db.py index be8754d78..b90afe36f 100755 --- a/models/db.py +++ b/models/db.py @@ -153,6 +153,9 @@ auth.settings.reset_password_requires_verification = True auth.settings.allow_basic_login = True +## Configure session handling: http://web2py.com/books/default/chapter/29/13/deployment-recipes#Sessions-in-database +session.connect(request, response, db) + ##restrict site to only logged in users ## https://groups.google.com/forum/#!topic/web2py/0j92-sPp4bc ##NB: useful url to add a guest user programmatically http://stackoverflow.com/questions/35504306/web2py-how-to-programmatically-register-users/35518991 From 2628e9d2b91b84d5fef63a8a83b851528cd8f0bc Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Fri, 19 Jan 2024 16:52:33 +0000 Subject: [PATCH 08/22] install: Install scripts for nginx/supervisord Scripts to configure nginx/supervisord --- Gruntfile.js | 1 + install-nginx.sh | 260 +++++++++++++++++++++++++++++++++++++++++ install-supervisord.sh | 121 +++++++++++++++++++ requirements.txt | 1 + 4 files changed, 383 insertions(+) create mode 100755 install-nginx.sh create mode 100755 install-supervisord.sh diff --git a/Gruntfile.js b/Gruntfile.js index 0e2925c48..920702373 100755 --- a/Gruntfile.js +++ b/Gruntfile.js @@ -35,6 +35,7 @@ module.exports = function (grunt) { command: [ 'git submodule update --init --recursive', 'ln -sf applications/OZtree/_COPY_CONTENTS_TO_WEB2PY_DIR/routes.py routes.py', + 'ln -sf handlers/wsgihandler.py handler.py', '( [ -d applications/welcome ] && rm -r -- "applications/welcome" || true )', '( [ -d applications/examples ] && rm -r -- "applications/examples" || true )', '( [ -f applications/OZtree/private/appconfig.ini ] || { cp applications/OZtree/private/appconfig.ini.example applications/OZtree/private/appconfig.ini ; echo "****** edit private/appconfig.ini"; exit 1; } )', diff --git a/install-nginx.sh b/install-nginx.sh new file mode 100755 index 000000000..fe73320cd --- /dev/null +++ b/install-nginx.sh @@ -0,0 +1,260 @@ +#!/usr/bin/env bash +set -eux +PROJECT_PATH="${PROJECT_PATH-$(dirname "$(readlink -f "$0")")}" # The full project path +WEB2PY_PATH="$(dirname $(dirname "$PROJECT_PATH"))" +WEB2PY_NAME="${WEB2PY_NAME-$(basename ${WEB2PY_PATH})}" # Directory web2py lives in, will be unique per installation +CERT_DIR="/var/lib/dehydrated/certs" + +WWW_SERVER_NAME="${WEB2PY_NAME}" # Assume we checked out web2py in /.../www.onezoom.org +WWW_IMAGES_SERVER_NAME="$(echo ${WWW_SERVER_NAME} | sed 's/^w*/images/')" # images.onezoom.org or imagesdev.onezoom.org + +[ -d "/etc/nginx" ] && NGINX_PATH="/etc/nginx" +[ -d "/usr/local/etc/nginx" ] && NGINX_PATH="/usr/local/etc/nginx" +mkdir -p "${NGINX_PATH}/conf.d/" +NGINX_LOG_PATH="/var/log/nginx" +[ -d "/var/db/acme/live/" ] && NGINX_CERT_PATH="/var/db/acme/live/" +[ -d "/var/lib/dehydrated/certs" ] && NGINX_CERT_PATH="/var/lib/dehydrated/certs" +NGINX_DHPARAM_PATH="${NGINX_PATH}/dhparam.pem" +[ -d "/var/db/acme/live" ] && NGINX_CHALLENGE_PATH="/var/db/acme/live" +[ -d "/var/lib/dehydrated/acme-challenges" ] && NGINX_CHALLENGE_PATH="/var/lib/dehydrated/acme-challenges" + +# Generate NGINX_DHPARAM +[ -e "${NGINX_DHPARAM_PATH}" ] || openssl dhparam -out "${NGINX_DHPARAM_PATH}" 4096 + +# Self-signed bootstrap-cert +for SN in onezoom.org ${WWW_SERVER_NAME} ${WWW_IMAGES_SERVER_NAME}; do + mkdir -p "${NGINX_CERT_PATH}/${SN}" + if [ ! -e "${NGINX_CERT_PATH}/${SN}/privkey.pem" ]; then + openssl req -x509 -newkey rsa:4096 -sha256 -days 365 -nodes \ + -keyout "${NGINX_CERT_PATH}/${SN}/privkey-ss.pem" \ + -out "${NGINX_CERT_PATH}/${SN}/fullchain-ss.pem" \ + -subj "/CN=${SN}" \ + -addext "subjectAltName = DNS:selfsigned.${SN}" + ln -rs ${NGINX_CERT_PATH}/${SN}/privkey-ss.pem ${NGINX_CERT_PATH}/${SN}/privkey.pem + ln -rs ${NGINX_CERT_PATH}/${SN}/fullchain-ss.pem ${NGINX_CERT_PATH}/${SN}/fullchain.pem + ln -rs ${NGINX_CERT_PATH}/${SN}/fullchain-ss.pem ${NGINX_CERT_PATH}/${SN}/chain.pem + fi +done + +# Create NGINX config +cat < ${NGINX_PATH}/nginx.conf +#### Generated by $0 - DO NOT EDIT + +events {} + +http { + include mime.types; + default_type application/octet-stream; + + # Enable Gzip + gzip on; + gzip_http_version 1.0; + gzip_comp_level 2; + gzip_min_length 1100; + gzip_buffers 4 8k; + gzip_proxied any; + gzip_types + # text/html is always compressed by HttpGzipModule + text/css + text/javascript + text/xml + text/plain + text/x-component + text/json + application/javascript + application/json + application/xml + application/rss+xml + font/truetype + font/opentype + application/vnd.ms-fontobject + image/svg+xml; + gzip_proxied expired no-cache no-store private auth; + gzip_disable "MSIE [1-6]\."; + gzip_vary on; + + keepalive_timeout 65; + + server_names_hash_bucket_size 128; + + include /etc/nginx/conf.d/*.conf; +} +EOF + +cat < ${NGINX_PATH}/conf.d/onezoom.org.conf +server { + listen 80; + listen [::]:80; + listen 443 ssl http2; + listen [::]:443 ssl http2; + + server_name server_name onezoom.org default; + + location /.well-known/acme-challenge { + allow all; + default_type text/plain; + alias ${NGINX_CHALLENGE_PATH}; + } + ssl_certificate ${NGINX_CERT_PATH}/onezoom.org/fullchain.pem; + ssl_certificate_key ${NGINX_CERT_PATH}/onezoom.org/privkey.pem; + ssl_trusted_certificate ${NGINX_CERT_PATH}/onezoom.org/fullchain.pem; + ssl_dhparam ${NGINX_DHPARAM_PATH}; + + return 301 \$scheme://www.onezoom.org\$request_uri; +} +EOF + +cat < ${NGINX_PATH}/${WEB2PY_NAME}_static_include.inc +#### Generated by $0 - DO NOT EDIT + +alias ${PROJECT_PATH}/static/; + +#the directives used for static pages on OneZoom +location ~ /FinalOutputs/data/ { + # add_header 'X-static-gzipping' 'on' always; + gzip_static on; + #files in /data/ (e.g. the topology) have timestamps, so never change, and browsers can always use cache + expires max; + add_header Cache-Control "public"; +} + +location ~* \.(?:jpg|jpeg|gif|png|ico|gz|svg)\$ { + #cache images for a little while, even though we also cache them in the js. + #10 mins allows e.g. new crops to show up. + expires 10m; + access_log off; + add_header Cache-Control "public"; +} + +location ~ \.(js|css|html)\$ { + # add_header 'X-static-gzipping' 'on' always; + gzip_static on; + #cache the static js and html, but only for a bit, in case we implement changes + expires 30m; + add_header Cache-Control "public"; +} + +location ~* /(\w+/)?static/trees/[^/]+/ { + # static trees with a trailing slash need to be trimmed so that e.g. + # static/trees/AT/@Homo_sapiens is not seen as a request for a file called '@Homo_sapiens' + # see http://stackoverflow.com/questions/39519355 + rewrite ^(.*/static/trees/[^/]+)/ \$1 last; + return 404; +} +EOF + +cat < ${NGINX_PATH}/conf.d/${WWW_SERVER_NAME}.conf +#### Generated by $0 - DO NOT EDIT + +upstream uwsgi_${WEB2PY_NAME} { + least_conn; + server unix:///var/run/uwsgi/${WEB2PY_NAME}_uwsgi0.sock; + server unix:///var/run/uwsgi/${WEB2PY_NAME}_uwsgi1.sock; + server unix:///var/run/uwsgi/${WEB2PY_NAME}_uwsgi2.sock; + server unix:///var/run/uwsgi/${WEB2PY_NAME}_uwsgi3.sock; + server unix:///var/run/uwsgi/${WEB2PY_NAME}_uwsgi4.sock; +} + +server { + listen 80; + listen [::]:80; + listen 443 ssl http2; + listen [::]:443 ssl http2; + + server_name ${WWW_SERVER_NAME}; + + if (\$scheme != "https") { + return 301 https://\$server_name\$request_uri; + } + location /.well-known/acme-challenge { + allow all; + default_type text/plain; + alias ${NGINX_CHALLENGE_PATH}; + } + ssl_certificate ${NGINX_CERT_PATH}/${WWW_SERVER_NAME}/fullchain.pem; + ssl_certificate_key ${NGINX_CERT_PATH}/${WWW_SERVER_NAME}/privkey.pem; + ssl_trusted_certificate ${NGINX_CERT_PATH}/${WWW_SERVER_NAME}/fullchain.pem; + ssl_dhparam ${NGINX_DHPARAM_PATH}; + + # Generated by https://ssl-config.mozilla.org/ + ssl_session_timeout 1d; + ssl_session_cache shared:MozSSL:10m; # about 40000 sessions + ssl_session_tickets off; + + # intermediate configuration + ssl_protocols TLSv1.2 TLSv1.3; + ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-RSA-CHACHA20-POLY1305; + ssl_prefer_server_ciphers off; + + server_tokens off; + access_log ${NGINX_LOG_PATH}/${WWW_SERVER_NAME}.access.log combined buffer=32k flush=5m; + error_log ${NGINX_LOG_PATH}/${WWW_SERVER_NAME}.error.log; + + #cache filehandles on the server side to max 2000 files, hopefully mostly images + open_file_cache max=2000 inactive=20s; + open_file_cache_valid 60s; + open_file_cache_min_uses 3; + open_file_cache_errors off; + + index index.htm; + location /static/ { + include ${WEB2PY_NAME}_static_include.inc; + } + + location /OZtree/static/ { + include ${WEB2PY_NAME}_static_include.inc; + } + + location / { + location ~ \.json { + #don't log API (json) requests + access_log off; + uwsgi_pass uwsgi_${WEB2PY_NAME}; + } + uwsgi_pass uwsgi_${WEB2PY_NAME}; + include uwsgi_params; + } +} +EOF + +cat < ${NGINX_PATH}/conf.d/${WWW_IMAGES_SERVER_NAME}.conf +#### Generated by $0 - DO NOT EDIT + +server { + listen 80; + listen [::]:80; + listen 443 ssl http2; + listen [::]:443 ssl http2; + + server_name ${WWW_IMAGES_SERVER_NAME}; + server_tokens off; + + location /.well-known/acme-challenge { + allow all; + default_type text/plain; + alias ${NGINX_CHALLENGE_PATH}; + } + ssl_certificate ${NGINX_CERT_PATH}/${WWW_IMAGES_SERVER_NAME}/fullchain.pem; + ssl_certificate_key ${NGINX_CERT_PATH}/${WWW_IMAGES_SERVER_NAME}/privkey.pem; + ssl_trusted_certificate ${NGINX_CERT_PATH}/${WWW_IMAGES_SERVER_NAME}/fullchain.pem; + ssl_dhparam ${NGINX_DHPARAM_PATH}; + + access_log ${NGINX_LOG_PATH}/${WWW_IMAGES_SERVER_NAME}.access.log combined buffer=32k flush=5m; + error_log ${NGINX_LOG_PATH}/${WWW_IMAGES_SERVER_NAME}.error.log; + + #cache filehandles on the server side to max 2000 files, hopefully mostly images + open_file_cache max=2000 inactive=20s; + open_file_cache_valid 60s; + open_file_cache_min_uses 3; + open_file_cache_errors off; + + location / { + access_log off; + expires 10m; + add_header Cache-Control "public"; + root ${PROJECT_PATH}/static/FinalOutputs/img/; + } +} +EOF + +nginx -t diff --git a/install-supervisord.sh b/install-supervisord.sh new file mode 100755 index 000000000..25388ad43 --- /dev/null +++ b/install-supervisord.sh @@ -0,0 +1,121 @@ +#!/usr/bin/env bash +set -eu +PROJECT_PATH="${PROJECT_PATH-$(dirname "$(readlink -f "$0")")}" # The full project path +WEB2PY_PATH="$(dirname $(dirname "$PROJECT_PATH"))" +WEB2PY_NAME="${WEB2PY_NAME-$(basename ${WEB2PY_PATH})}" # Directory web2py lives in, will be unique per installation +DEPLOY_USER="$(stat -c '%U' $0)" + +APP_USER="www" +APP_GROUP="www" +WWW_SERVER_NAME="${WEB2PY_NAME}" + +# NB: Yan says we should write to web2py_path & gluon. Seems risky +for DIR in "${PROJECT_PATH}/errors" \ + "${PROJECT_PATH}/databases" \ + "${PROJECT_PATH}/sessions" \ + "${PROJECT_PATH}/uploads" \ + "${WEB2PY_PATH}/logs" \ + "${WEB2PY_PATH}/deposit" \ + "/var/run/uwsgi"; do + mkdir -p -- "${DIR}" + chown -R ${APP_USER} "${DIR}" + chmod g+w "${DIR}" +done + +[ -d "/etc/supervisor" ] && SUPERVISORD_CONF_PATH="/etc/supervisor" +[ -d "/usr/local/etc/supervisord" ] && SUPERVISORD_CONF_PATH="/usr/local/etc/supervisord" +mkdir -p "${SUPERVISORD_CONF_PATH}/supervisord.conf.d/" +mkdir -p "/var/run/supervisor" + +SUPERVISORD_LOG_PATH="/var/log/supervisord" +mkdir -p "${SUPERVISORD_LOG_PATH}" + +cat < "${SUPERVISORD_CONF_PATH}/supervisord.conf" +; **** Generated by $0 - DO NOT EDIT +; +[unix_http_server] +file=/var/run/supervisor/supervisor.sock ; (the path to the socket file) + +[supervisord] +logfile=${SUPERVISORD_LOG_PATH}/supervisord.log ; (main log file) +logfile_maxbytes=50MB ; (max main logfile bytes b4 rotation;default 50MB) +logfile_backups=10 ; (num of main logfile rotation backups;default 10) +loglevel=info ; (log level;default info; others: debug,warn,trace) +pidfile=/var/run/supervisor/supervisord.pid ; (supervisord pidfile;default supervisord.pid) +nodaemon=false ; (start in foreground if true;default false) +minfds=1024 ; (min. avail startup file descriptors;default 1024) +minprocs=200 ; (min. avail process descriptors;default 200) + +[rpcinterface:supervisor] +supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface + +[supervisorctl] +serverurl=unix:///var/run/supervisor/supervisor.sock ; use a unix:// URL for a unix socket + +[include] +files = supervisord.conf.d/*.conf +EOF + +cat < "${SUPERVISORD_CONF_PATH}/supervisord.conf.d/${WEB2PY_NAME}.conf" +; **** Generated by $0 - DO NOT EDIT +; +[group:${WEB2PY_NAME}] +programs=${WEB2PY_NAME}_uwsgi,${WEB2PY_NAME}_background_tasks,${WEB2PY_NAME}_session_cleanup + +[program:${WEB2PY_NAME}_uwsgi] +directory=${WEB2PY_PATH} +command=${WEB2PY_PATH}/bin/uwsgi + -s /var/run/uwsgi/%(program_name)s%(process_num)d.sock + --chmod-socket=666 + --need-app + --disable-logging + --master + --home=${WEB2PY_PATH} + --wsgi-file handler.py + --processes 1 + --threads 10 + --uid "${APP_USER}" + --gid "${APP_GROUP}" +stdout_logfile=${SUPERVISORD_LOG_PATH}/%(program_name)s_stdout.log +stdout_logfile_maxbytes=10MB +stdout_logfile_backups=10 +stderr_logfile=${SUPERVISORD_LOG_PATH}/%(program_name)s_stderr.log +stderr_logfile_maxbytes=10MB +stderr_logfile_backups=10 +startsecs=10 +stopsignal=QUIT +stopasgroup=true +killasgroup=true +process_name=%(program_name)s%(process_num)d +numprocs=5 + +[program:${WEB2PY_NAME}_background_tasks] +directory=${WEB2PY_PATH} +user=${APP_USER} +group=${APP_GROUP} +command=$(which bash) -c "${WEB2PY_PATH}/bin/python web2py.py -S OZtree/default -M -e -R applications/OZtree/private/background_tasks.py --args ${WWW_SERVER_NAME} verbose && /bin/sleep 86400" +stdout_logfile=${SUPERVISORD_LOG_PATH}/%(program_name)s_stdout.log +stdout_logfile_maxbytes=10MB +stdout_logfile_backups=10 +stderr_logfile=${SUPERVISORD_LOG_PATH}/%(program_name)s_stderr.log +stderr_logfile_maxbytes=10MB +stderr_logfile_backups=10 +startsecs=10 +stopsignal=QUIT +autorestart=true + +[program:${WEB2PY_NAME}_session_cleanup] +directory=${WEB2PY_PATH} +user=${APP_USER} +group=${APP_GROUP} +command=${WEB2PY_PATH}/bin/python web2py.py -S OZtree -M -R scripts/sessions2trash.py +stdout_logfile=${SUPERVISORD_LOG_PATH}/%(program_name)s_stdout.log +stdout_logfile_maxbytes=10MB +stdout_logfile_backups=10 +stderr_logfile=${SUPERVISORD_LOG_PATH}/%(program_name)s_stderr.log +stderr_logfile_maxbytes=10MB +stderr_logfile_backups=10 +startsecs=10 +stopsignal=QUIT +autorestart=true +EOF diff --git a/requirements.txt b/requirements.txt index 77947e063..d9d7db23d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,2 @@ pymysql==1.1.0 +uwsgi==2.0.23 From 48308d2addcecc38e3f31bdd5b70735fb45202ca Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Sat, 20 Jan 2024 17:22:29 +0000 Subject: [PATCH 09/22] Gruntfile: Alter web2py.py to use venv #676 Update shebang at the top of web2py.py so it uses the venv by default. --- Gruntfile.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Gruntfile.js b/Gruntfile.js index 920702373..6376e844f 100755 --- a/Gruntfile.js +++ b/Gruntfile.js @@ -41,6 +41,10 @@ module.exports = function (grunt) { '( [ -f applications/OZtree/private/appconfig.ini ] || { cp applications/OZtree/private/appconfig.ini.example applications/OZtree/private/appconfig.ini ; echo "****** edit private/appconfig.ini"; exit 1; } )', preferred_python3 + ' -m venv .', './bin/pip install -r applications/OZtree/requirements.txt', + // Make web2py.py use the venv + 'mv web2py.py web2py.py.o', + 'echo "#!' + venv_python + '" > web2py.py', + 'tail -n +2 web2py.py.o >> web2py.py', ].join(" && "), }, web2py_start_dev: { From f30b3105cf4b7043bbfb0b2e906bbe508dacc6b7 Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Mon, 5 Feb 2024 12:03:53 +0000 Subject: [PATCH 10/22] README.markdown: Rework installation instructions Remove dependencies that are now part of tree-build, rework instructions to take into account new Grunt rules. Rework based on the --- README.markdown | 199 +++++++++++++++++++++--------------------------- 1 file changed, 85 insertions(+), 114 deletions(-) diff --git a/README.markdown b/README.markdown index 76c7a2712..405803588 100755 --- a/README.markdown +++ b/README.markdown @@ -10,163 +10,134 @@ There are two ways in which you can install OneZoom on a personal computer: full * *Full installation* creates an entire duplicate of the OneZoom website, which is built using the [web2py](http://web2py.com) framework. This creates a fully self-contained local system (apart from the picture files, which can be downloaded separately). This is the most reliable installation method, but requires you to install and run extra software packages, in particular [web2py](http://web2py.com) and a [MySQL](https://www.mysql.com) server. Since this can be quite complicated, the majority of this readme contains instructions for full installation. - ## Requirements and packages -For all installation methods, you will need to install node.js (and npm, the node package manager), and the webpack package. To compile the OneZoom javascript codebase automatically, you will then need to install grunt. To generate documentation or make a partial install, you will also need perl installed on your system. - -For full installation, you will additionally need to install web2py, and ensure that you have the programming language python installed on your system, which is what web2py uses. You will also need access to a database backend (e.g. mySQL running on your own computer, or on a remote server which you can administer). - -To create trees, you will need python and perl, along with a number of libraries, as listed below. - -### Required packages (you will need these even if you're not creating trees) -The OneZoom codebase uses the following software (licenses for each listed in braces). The first three are programming languages which may well already be installed on your computer. - -* [Python](https://www.python.org) (assumed version 3.7) with the following libraries installed: - * mysql-connector-python - * pymysql (needed even if not creating trees) - * piexif - * requests - * Dendropy - * (for functional testing) nose + js2py + selenium + e.g. chromedriver_installer -* [Perl](https://www.perl.org) with the following libraries installed - * File::ReadBackwards - * LWP::Simple - * JSON - * DBI - * Try::Tiny - * Text::CSV - * Image::ExifTool - * DBD::mysql -* [web2py](http://web2py.com) (LGPL license) -* [npm](https://www.npmjs.com/get-npm), part of node.js which, when run will install a large number of other packages including - * [grunt](https://gruntjs.com) (MIT licence): to automate creating the OneZoom website files - * [webpack](https://webpack.js.org) (MIT licence): to package the OneZoom javascript tree viewer into a library - * jsdoc-to-markdown (MIT licence): to produce documentation from source code -* [ImageMagick](https://www.imagemagick.org/script/index.php) (Apache 2.0) for processing thumbnails -* [curl](https://curl.haxx.se) (MIT-like licence) to download partial installs (`curl` is probably already installed on your computer) -* [UIkit 3](https://getuikit.com) (MIT licence) for the User Interface (this code is included in the OneZoom github repo, and does not need downloading) - -## Quick installation steps - -Before anything else, get the OZtree app from [github](https://github.com/OneZoom/OZtree) - see *"Downloading the OZtree app"*. You should also make sure you have node.js and the node package manager (npm), see *"Building the OneZoom tree viewer"* - - -### For a partial installation (less tested): - -1. Install the command-line version of `grunt` using `npm install -g grunt-cli`. You may need to have administrator privileges to do this. -2. From anywhere within the OZtree download, run `npm install` to install all the packages for automation. -3. Create a partial installation by running `grunt partial-install`. This downloads the "minlife" and "minlife_tour" pages from the central OneZoom website, modifies links within them, and places appropriately named html files into the `static` directory of your OZtree distribution. -4. Open e.g. `static/minlife.html` with a web browser of your choice (we recommend Chrome or Safari). Note that this file needs to stay within the static directory to work at all. You may also need to allow your browser to allow local files to be loaded via AJAX (i.e. disabling some local cross-origin checks). Different browsers do this in different ways: for example in Chrome you can start up your browser with the `--allow-file-access-from-files` option, and in Safari, you can choose "Disable Local File Restrictions" from the "Developer" menu. -5. Note that the normal `minlife.html` file will use local versions of data files and the javascript treeviewer, but will get API information form the OneZoom website, *and* also use the OneZoom website as the source for the html page which embeds the viewer. For developers only, who may wish to create a minlife version not only using modified javascript in the treeviewer but also with bespoke html, you can run `grunt partial-local-install`. This is much more effort since it requires you to set up a full installation (as below) before creating the minlife scripts, but once created, the files in `static` will be enough for other users to view (and test) your modifications. - -### For a full installation (recommended): - -1. Install a source code version of [web2py](http://www.web2py.com), placing your [OZtree repository](https://github.com/OneZoom/OZtree) within the web2py `applications` directory. -2. Install command-line software by running `npm install -g grunt-cli` (you may need to do all this with administrator privileges). -3. Run `npm install` from within the OZtree folder you moved in step 1. then run `grunt dev` (or `grunt prod` if in production mode) - see *"[Building the OneZoom tree viewer](#building-the-onezoom-tree-viewer)"*. -3. [Install](http://dev.mysql.com/downloads/mysql/) & start MySQL, then create a new database (see *"[Setting up the database backend](#setting-up-the-database-backend)"*) -4. Edit `private/appconfig.ini` file in `OZtree/private`, with `migrate=1` and with the appropriate database username and password. -5. Fire up a temporary web2py server and visit the main page to create the (empty) database tables - see *"[Starting and shutting down web2py](#starting-and-shutting-down-web2py)"* -6. Load up data into the tables: first create a user and assign it a 'manager' role in the `auth_` tables using the web2py database admin pages, then load the other tables using data from the original OneZoom site (e.g. sent to you via file transfer) - see *"[Filling the database](#filling-the-database)"*. -7. Optimise your installation: - * create indexes on the tables by running the SQL script in `OZtree/OZprivate/ServerScripts/SQL/create_db_indexes.sql`. You can do this, for example, by running `SOURCE /path/to/OZtree/OZprivate/ServerScripts/SQL/create_db_indexes.sql` within a mysql client. - * set `migrate=0` in appconfig.ini. +For any installation, OneZoom requires node & python (3.11). -## Downloading the OZtree app +#### Debian/Ubuntu -Download a copy of the OZtree application from GitHub at [https://github.com/OneZoom/OZtree](https://github.com/OneZoom/OZtree), either as a zip file (not recommended), or probably better (easier to update), by cloning the repository (e.g. using `git clone https://github.com/OneZoom/OZtree` or if you have [GitHub Desktop](https://desktop.github.com) installed, click "Open in Desktop" from the [OZtree repo](https://github.com/OneZoom/OZtree)). Make sure the git folder is called "OZtree" (this is the default when you clone the repo, but not if you download it as a zip file). - -For full installation, you will also need to download the source code version of web2py, either via git (https://github.com/web2py/web2py/) or simply from the download link at http://www.web2py.com/. You can then place the OZtree directory into the `applications` directory of the web2py folder. +``` +apt install nodejs npm +apt install python3 python3-dev python3-venv +``` +### Full installation -## Building the OneZoom tree viewer +In addition, for a full installation you also need nginx, supervisor & MySQL. -Compiling and creating the OneZoom explorer javascript code requires grunt to be installed. This compiles javascript code from multiple sources into a single file. You will need to install the node package manager, npm, then do +#### Debian/Ubuntu ``` -npm install -g grunt-cli -``` +apt install nginx -To install this is likely to require administrator privileges. Other required packages can then be installed from within the OZtree folder by simply typing the following (which may take a while to complete!) +apt install supervisor -``` -npm install +apt install lsb-release +wget https://dev.mysql.com/get/mysql-apt-config_0.8.29-1_all.deb +dpkg -i mysql-apt-config_0.8.29-1_all.deb +apt update && apt install mysql-server +# NB: Select "Use Legacy Authentication Method (Retain MySQL 5.x Compatibility)" ``` -Once these are installed you can run grunt as follows (feel free to examine the configuration options which are stored in `Gruntfile.js` in the main OZtree directory): +#### Windows -#### Compile documentation -`grunt docs`: Use this command to generate a compiled documentation file. This will generate a large compiled markdown file in `OZprivate/rawJS/OZTreeModule/docs/_compiled.markdown`, which is best viewed once you have got web2py running, by pointing your browser to `dev/DOCS` (e.g. at `http://127.0.0.1:8000/dev/DOCS`). Note that viewing this page requires a working internet connection to get various formatting files) +On Windows we recommended downloading the MSI installer as it will make it easier to configure the new server during the installation +Once mysql is installed, you will need to set a root password, and create a database for web2py to use. See http://dev.mysql.com/doc/refman/5.7/en/default-privileges.html. +The mysqld program is responsible for running the new database just created. When this program is running, you can connect to the database. -#### In development mode: -`grunt dev`: This command bundles multiple js files into one. +## Installation -#### In production mode: -`grunt prod`: This command does three things. Firstly, it pre-compiles python code. Then it bundles multiple js files into one. Lastly, it minifies bundled js files. +If performing a full installation, you need to create a database for OneZoom to use: +``` +mysql -p +CREATE DATABASE OneZoom; +CREATE USER 'oz'@'localhost' IDENTIFIED BY 'passwd-you-should-change-this'; +GRANT ALL PRIVILEGES ON OneZoom . * TO 'oz'@'localhost'; +``` -## The server-side database +Firstly, you need to check out the web2py / OneZoom repositories, and run npm: -### Setting up the database backend +``` +# NB: The path should match the eventual hostname for this installation +export WEB2PY_PATH="/srv/.../onezoom.myhostname.org" +mkdir -p ${WEB2PY_PATH} +chown deploy:staff ${WEB2PY_PATH} +git clone https://github.com/web2py/web2py ${WEB2PY_PATH} --branch v2.27.1 +git clone https://github.com/OneZoom/OZtree.git ${WEB2PY_PATH}/applications/OZtree --branch production-next +cd ${WEB2PY_PATH}/applications/OZtree +npm ci --legacy-peer-deps +``` -The web2py instance requires a database to be running. We previously used sqllite, and code for interfacing with sqllite is still present in the codebase, but probably will not work, as we have switched to using mySQL. +Next, ``cp private/appconfig.ini.example private/appconfig.ini`` and edit to match your needs, taking care to: -So a major step when installing OneZoom is: +* Use the same database credentials as configured earlier. -1. Install a locally running copy of mySQL. Make sure the server is installed and not only the client - There are many ways to do this: see http://dev.mysql.com/downloads/mysql/. +Once done, either run - On Windows we recommended downloading the MSI installer as it will make it easier to configure the new server during the installation +* ``./node_modules/.bin/grunt prod`` for a production installation +* ``./node_modules/.bin/grunt dev`` for a development installation - Once mysql is installed, you will need to set a root password, and create a database for web2py to use. See http://dev.mysql.com/doc/refman/5.7/en/default-privileges.html. +### Partial installation - The mysqld program is responsible for running the new database just created. When this program is running, you can connect to the database. +If you'd like your installation to be a partial installation, run: -2. (optional) We find it useful to have a GUI interface to connect to the database and run SQL scripts, this can be used instead of using MySQL command line (similar to Windows command line) that is installed by default with MySQL. On Mac OS X we use the (excellent) http://www.sequelpro.com. On windows you could try http://www.mysql.com/products/workbench/ or https://www.quest.com/products/toad-for-mysql/ +``` +./node_modules/.bin/grunt partial-install +``` -3. Once mysql is installed, you will need to set a root password, and create a database for web2py to use. See http://dev.mysql.com/doc/refman/5.7/en/default-privileges.html. So once mysqld is running, you need to log in to the sql server with the root name and password (if you are using the command line, log in using `mysql -u root -p`), and issue the following SQL commands (the text after the `mysql>` prompt) to create a database for web2py to use: feel free to use a different *'passwd'*. +...and open `static/minlife.html` in a web browser. +Note that this file needs to stay within the static directory to work at all. +You may also need to allow your browser to allow local files to be loaded via AJAX (i.e. disabling some local cross-origin checks). +Different browsers do this in different ways: for example in Chrome you can start up your browser with the `--allow-file-access-from-files` option, and in Safari, you can choose "Disable Local File Restrictions" from the "Developer" menu. - ``` - mysql> create database OneZoom; - Query OK, 1 row affected (0.09 sec) - - mysql> CREATE USER 'oz'@'localhost' IDENTIFIED BY 'passwd'; - Query OK, 0 rows affected (0.19 sec) - - mysql> GRANT ALL PRIVILEGES ON OneZoom . * TO 'oz'@'localhost'; - Query OK, 0 rows affected (0.09 sec) - ``` +Note that the normal `minlife.html` file will use local versions of data files and the javascript treeviewer, +but will get API information form the OneZoom website, *and* also use the OneZoom website as the source for the html page which embeds the viewer. +For developers only, who may wish to create a minlife version not only using modified javascript in the treeviewer but also with bespoke html, you can run `grunt partial-local-install`. +This is much more effort since it requires you to set up a full installation (as below) before creating the minlife scripts, but once created, the files in `static` will be enough for other users to view (and test) your modifications. -The database is now up and running. We recommend that you do *not* load data into it immediately, but first create the tables by installing web2py with `migrate=1` set in the appconfig.ini file. After running an instance of web2py and visiting the new site, the correct table structure should be automatically created (see below). After that you can populate the data into the tables using downloaded files. +## Database setup / migation -## Web2py installation +For full installations, you need to setup your database. -Configuring the OneZoom application to use the database involves creating a file called 'appconfig.ini' in the `private` folder within the OZtree app. A minimal appconfig.ini file to get the site working is in [private/appconfig.ini.example](private/appconfig.ini.example), which can renamed to `appconfig.ini` and modified to use the username and password that you supplied above. +To create required tables, use web2py migrate: -In order to use web2py you need to have a python v3 installed, the latest version can be found at -[https://www.python.org/downloads/](https://www.python.org/downloads/) +* Edit private/appconfig.ini, setting ``migrate=1`` +* ``./web2py-run tests/unit/test_modules_embed.py``, on startup this will create tables as necessary +* Edit private/appconfig.ini, migrate=0 -NB: on Windows, make sure that you add `python` (and ideally `python3`) to the windows path during install, or the commands below will not work +A database dump should be used to add species information: -Assuming you have python version 3 installed, should now try starting web2py as follows. +``` +# NB: OneZoom.dump.sql can be extracted from https://github.com/OneZoom/OZtree-docker +mysql -p +USE OneZoom +SOURCE /OneZoom.dump.sql +``` -### Starting and shutting down web2py +Finally, ensure indexes are created: -On the OneZoom main site, web2py is run using a combination of nginx and uwsgi. This is complete overkill if you just want to run a local copy of OneZoom for testing purposes. You can simply run a [temporary and basic web2py server using Python 3](http://www.web2py.com/books/default/chapter/29/03/overview#Startup). The simplest is to open a command-line prompt in the root web2py folder, and run the following (assuming the command `python3` is linked to something like Python 3.7) +``` +mysql -p +USE OneZoom +SOURCE ${WEB2PY_PATH}/applications/OZtree/OZprivate/ServerScripts/SQL/create_db_indexes.sql +``` -`python3 web2py.py -i 127.0.0.1 -p 8000 -a pass` +## Database explorer -* (NB: it is possible to run a secure OneZoom site over https. To try this using the basic web2py server, create a `.crt` and `.key` file, e.g. by running the following in the web2py root directory: `openssl req -newkey rsa:2048 -x509 -days 365 -nodes -keyout oz.key -out oz.crt`, then use them when running web2py, as in: `python3 web2py.py -c oz.crt -k oz.key -i 127.0.0.1 -p 8000 -a pass`) +(optional) We find it useful to have a GUI interface to connect to the database and run SQL scripts, this can be used instead of using MySQL command line (similar to Windows command line) that is installed by default with MySQL. On Mac OS X we use the (excellent) http://www.sequelpro.com. On windows you could try http://www.mysql.com/products/workbench/ or https://www.quest.com/products/toad-for-mysql/ +## Starting and shutting down web2py -When web2py is run, it will print instructions telling how to shut down the web2py server. For example, on Windows you might use `taskkill /f /pid XXXX`, where `XXXX` is the process id. +On the OneZoom main site, web2py is run using a combination of nginx and uwsgi. This is complete overkill if you just want to run a local copy of OneZoom for testing purposes. You can simply run a [temporary and basic web2py server using Python 3](http://www.web2py.com/books/default/chapter/29/03/overview#Startup). The simplest is to open a command-line prompt in the root web2py folder, and run the following (assuming the command `python3` is linked to something like Python 3.7) -**If this is a new installation** you should now visit `http://127.0.0.1:8000/OZtree/default/` or `https://127.0.0.1:8000/OZtree/default/` to force web2py to create database tables. To load data into the tables, see "Loading Data", below. +``` +./node_modules/.bin/grunt start-dev +``` -Once tables are created, and everything is working, you can set `migrate=0` in `private/appconfig.ini`. This will mean that web2py will not make any changes to table structures in the DB +When web2py is run, it will print instructions telling how to shut down the web2py server. For example, on Windows you might use `taskkill /f /pid XXXX`, where `XXXX` is the process id. -### Web2py folder structure +## Web2py folder structure #### Standard folders `databases` stores all the database structure. From a91ef0301600f5a06afe1e858b833ce9a63a7571 Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Mon, 5 Feb 2024 14:32:26 +0000 Subject: [PATCH 11/22] Gruntfile: Use python3.10, not python3.11 #672 #699 --- Gruntfile.js | 2 +- README.markdown | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/Gruntfile.js b/Gruntfile.js index 6376e844f..12b069569 100755 --- a/Gruntfile.js +++ b/Gruntfile.js @@ -5,7 +5,7 @@ const path = require('path'); partial_install_site = "http://www.onezoom.org"; partial_local_install_site = "http://127.0.0.1:8000"; // if you are running a local installation -preferred_python3 = "python3.11"; // in case you have multiple python3 versions installed +preferred_python3 = "python3.10"; // in case you have multiple python3 versions installed web2py_py = path.join(path.dirname(path.dirname(process.cwd())), 'web2py.py'); venv_python = path.join(path.dirname(path.dirname(process.cwd())), 'bin/python'); diff --git a/README.markdown b/README.markdown index 405803588..2c2f46af5 100755 --- a/README.markdown +++ b/README.markdown @@ -39,6 +39,12 @@ apt update && apt install mysql-server # NB: Select "Use Legacy Authentication Method (Retain MySQL 5.x Compatibility)" ``` +#### FreeBSD + +``` +sudo pkg install nginx py39-supervisor lang/python310 +``` + #### Windows On Windows we recommended downloading the MSI installer as it will make it easier to configure the new server during the installation From ce41a32b48f55b5c641fcd078f4e5b320650f390 Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Mon, 5 Feb 2024 14:34:38 +0000 Subject: [PATCH 12/22] README: Don't mention temporary branch Ideally this would be separated out into instructions for a production instance, but at least not mentioning a temporary branch is good. --- README.markdown | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.markdown b/README.markdown index 2c2f46af5..b19bcd773 100755 --- a/README.markdown +++ b/README.markdown @@ -70,7 +70,7 @@ export WEB2PY_PATH="/srv/.../onezoom.myhostname.org" mkdir -p ${WEB2PY_PATH} chown deploy:staff ${WEB2PY_PATH} git clone https://github.com/web2py/web2py ${WEB2PY_PATH} --branch v2.27.1 -git clone https://github.com/OneZoom/OZtree.git ${WEB2PY_PATH}/applications/OZtree --branch production-next +git clone https://github.com/OneZoom/OZtree.git ${WEB2PY_PATH}/applications/OZtree --branch production cd ${WEB2PY_PATH}/applications/OZtree npm ci --legacy-peer-deps ``` From 2895b3985ea4bcf228280b1f6f04525276f7c579 Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Mon, 5 Feb 2024 15:22:56 +0000 Subject: [PATCH 13/22] install-nginx: /var/db/acme/live/ isn't created OOTB An empty installation doesn't have /var/db/acme/live/, only /var/db/acme/. --- install-nginx.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install-nginx.sh b/install-nginx.sh index fe73320cd..0209769f6 100755 --- a/install-nginx.sh +++ b/install-nginx.sh @@ -12,7 +12,7 @@ WWW_IMAGES_SERVER_NAME="$(echo ${WWW_SERVER_NAME} | sed 's/^w*/images/')" # ima [ -d "/usr/local/etc/nginx" ] && NGINX_PATH="/usr/local/etc/nginx" mkdir -p "${NGINX_PATH}/conf.d/" NGINX_LOG_PATH="/var/log/nginx" -[ -d "/var/db/acme/live/" ] && NGINX_CERT_PATH="/var/db/acme/live/" +[ -d "/var/db/acme/" ] && NGINX_CERT_PATH="/var/db/acme/live/" [ -d "/var/lib/dehydrated/certs" ] && NGINX_CERT_PATH="/var/lib/dehydrated/certs" NGINX_DHPARAM_PATH="${NGINX_PATH}/dhparam.pem" [ -d "/var/db/acme/live" ] && NGINX_CHALLENGE_PATH="/var/db/acme/live" From 950524acf71b1795c4d139096e7ea615dadc8474 Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Mon, 5 Feb 2024 15:23:36 +0000 Subject: [PATCH 14/22] install-nginx: BSD doesn't know about ln -r --- install-nginx.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/install-nginx.sh b/install-nginx.sh index 0209769f6..0b6319a5b 100755 --- a/install-nginx.sh +++ b/install-nginx.sh @@ -30,9 +30,9 @@ for SN in onezoom.org ${WWW_SERVER_NAME} ${WWW_IMAGES_SERVER_NAME}; do -out "${NGINX_CERT_PATH}/${SN}/fullchain-ss.pem" \ -subj "/CN=${SN}" \ -addext "subjectAltName = DNS:selfsigned.${SN}" - ln -rs ${NGINX_CERT_PATH}/${SN}/privkey-ss.pem ${NGINX_CERT_PATH}/${SN}/privkey.pem - ln -rs ${NGINX_CERT_PATH}/${SN}/fullchain-ss.pem ${NGINX_CERT_PATH}/${SN}/fullchain.pem - ln -rs ${NGINX_CERT_PATH}/${SN}/fullchain-ss.pem ${NGINX_CERT_PATH}/${SN}/chain.pem + ln -s ${NGINX_CERT_PATH}/${SN}/privkey-ss.pem ${NGINX_CERT_PATH}/${SN}/privkey.pem + ln -s ${NGINX_CERT_PATH}/${SN}/fullchain-ss.pem ${NGINX_CERT_PATH}/${SN}/fullchain.pem + ln -s ${NGINX_CERT_PATH}/${SN}/fullchain-ss.pem ${NGINX_CERT_PATH}/${SN}/chain.pem fi done From bcd59ea0aa96ee2425f6bdab455ab1dd0f94025b Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Mon, 5 Feb 2024 15:24:42 +0000 Subject: [PATCH 15/22] install-nginx: Use NGINX_PATH when generating include config --- install-nginx.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install-nginx.sh b/install-nginx.sh index 0b6319a5b..27abe894c 100755 --- a/install-nginx.sh +++ b/install-nginx.sh @@ -77,7 +77,7 @@ http { server_names_hash_bucket_size 128; - include /etc/nginx/conf.d/*.conf; + include ${NGINX_PATH}/conf.d/*.conf; } EOF From 814e1031a9513ec2a5e458487d8aeb6f8b855f75 Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Mon, 5 Feb 2024 15:25:14 +0000 Subject: [PATCH 16/22] install-nginx: Remove v6 listeners Nginx falls over if v6 isn't available, which it isn't. --- install-nginx.sh | 6 ------ 1 file changed, 6 deletions(-) diff --git a/install-nginx.sh b/install-nginx.sh index 27abe894c..cbca4e427 100755 --- a/install-nginx.sh +++ b/install-nginx.sh @@ -84,9 +84,7 @@ EOF cat < ${NGINX_PATH}/conf.d/onezoom.org.conf server { listen 80; - listen [::]:80; listen 443 ssl http2; - listen [::]:443 ssl http2; server_name server_name onezoom.org default; @@ -157,9 +155,7 @@ upstream uwsgi_${WEB2PY_NAME} { server { listen 80; - listen [::]:80; listen 443 ssl http2; - listen [::]:443 ssl http2; server_name ${WWW_SERVER_NAME}; @@ -222,9 +218,7 @@ cat < ${NGINX_PATH}/conf.d/${WWW_IMAGES_SERVER_NAME}.conf server { listen 80; - listen [::]:80; listen 443 ssl http2; - listen [::]:443 ssl http2; server_name ${WWW_IMAGES_SERVER_NAME}; server_tokens off; From f9d59ebf345d93eaa7c4d5310565540201372f36 Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Mon, 5 Feb 2024 15:37:30 +0000 Subject: [PATCH 17/22] install-supervisord: BSD-compatible stat syntax --- install-supervisord.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install-supervisord.sh b/install-supervisord.sh index 25388ad43..56c05a79d 100755 --- a/install-supervisord.sh +++ b/install-supervisord.sh @@ -3,7 +3,7 @@ set -eu PROJECT_PATH="${PROJECT_PATH-$(dirname "$(readlink -f "$0")")}" # The full project path WEB2PY_PATH="$(dirname $(dirname "$PROJECT_PATH"))" WEB2PY_NAME="${WEB2PY_NAME-$(basename ${WEB2PY_PATH})}" # Directory web2py lives in, will be unique per installation -DEPLOY_USER="$(stat -c '%U' $0)" +DEPLOY_USER="$(stat -c '%U' install-supervisord.sh 2>/dev/null || stat -f '%Su' install-supervisord.sh 2>/dev/null)" APP_USER="www" APP_GROUP="www" From 5edd180f0aa42ac92712544b20fe8d139bd15d02 Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Mon, 5 Feb 2024 15:38:19 +0000 Subject: [PATCH 18/22] install-supervisord: Correct supervisord conf location on BSD There is no /usr/local/etc/supervisord, it's all in /usr/local/etc. --- install-supervisord.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install-supervisord.sh b/install-supervisord.sh index 56c05a79d..edf6b9552 100755 --- a/install-supervisord.sh +++ b/install-supervisord.sh @@ -23,7 +23,7 @@ for DIR in "${PROJECT_PATH}/errors" \ done [ -d "/etc/supervisor" ] && SUPERVISORD_CONF_PATH="/etc/supervisor" -[ -d "/usr/local/etc/supervisord" ] && SUPERVISORD_CONF_PATH="/usr/local/etc/supervisord" +[ -e "/usr/local/etc/supervisord.conf.sample" ] && SUPERVISORD_CONF_PATH="/usr/local/etc" mkdir -p "${SUPERVISORD_CONF_PATH}/supervisord.conf.d/" mkdir -p "/var/run/supervisor" From 3b5e810eae50d91e26895e99fc8561194a2e0014 Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Mon, 5 Feb 2024 15:39:04 +0000 Subject: [PATCH 19/22] install-supervisord: Put script in debug mode It's comforting to see what it's doing, and makes error messages more intelligable. --- install-supervisord.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install-supervisord.sh b/install-supervisord.sh index edf6b9552..40a404261 100755 --- a/install-supervisord.sh +++ b/install-supervisord.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -set -eu +set -eux PROJECT_PATH="${PROJECT_PATH-$(dirname "$(readlink -f "$0")")}" # The full project path WEB2PY_PATH="$(dirname $(dirname "$PROJECT_PATH"))" WEB2PY_NAME="${WEB2PY_NAME-$(basename ${WEB2PY_PATH})}" # Directory web2py lives in, will be unique per installation From 785c2365ffb51a4f3b945d37465c9a9c4fe9be33 Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Mon, 5 Feb 2024 15:50:17 +0000 Subject: [PATCH 20/22] README.markdown: Separate "production installation" section Move the production installation notes into their own section, mention the install scripts to be run as root. --- README.markdown | 60 ++++++++++++++++++++++++++++++++++++------------- 1 file changed, 44 insertions(+), 16 deletions(-) diff --git a/README.markdown b/README.markdown index b19bcd773..90d724b73 100755 --- a/README.markdown +++ b/README.markdown @@ -12,7 +12,7 @@ There are two ways in which you can install OneZoom on a personal computer: full ## Requirements and packages -For any installation, OneZoom requires node & python (3.11). +For any installation, OneZoom requires node & python (3.10). #### Debian/Ubuntu @@ -21,17 +21,17 @@ apt install nodejs npm apt install python3 python3-dev python3-venv ``` -### Full installation +#### FreeBSD -In addition, for a full installation you also need nginx, supervisor & MySQL. +``` +sudo pkg install lang/python310 node18 npm-node18 +``` + +In addition, for a full installation you also need MySQL: #### Debian/Ubuntu ``` -apt install nginx - -apt install supervisor - apt install lsb-release wget https://dev.mysql.com/get/mysql-apt-config_0.8.29-1_all.deb dpkg -i mysql-apt-config_0.8.29-1_all.deb @@ -39,12 +39,6 @@ apt update && apt install mysql-server # NB: Select "Use Legacy Authentication Method (Retain MySQL 5.x Compatibility)" ``` -#### FreeBSD - -``` -sudo pkg install nginx py39-supervisor lang/python310 -``` - #### Windows On Windows we recommended downloading the MSI installer as it will make it easier to configure the new server during the installation @@ -79,10 +73,9 @@ Next, ``cp private/appconfig.ini.example private/appconfig.ini`` and edit to mat * Use the same database credentials as configured earlier. -Once done, either run +Once done, run ``./node_modules/.bin/grunt dev`` for a development installation. -* ``./node_modules/.bin/grunt prod`` for a production installation -* ``./node_modules/.bin/grunt dev`` for a development installation +For production installation, see later. ### Partial installation @@ -129,6 +122,41 @@ USE OneZoom SOURCE ${WEB2PY_PATH}/applications/OZtree/OZprivate/ServerScripts/SQL/create_db_indexes.sql ``` +## Production installation + +For a production installation of OneZoom, you also need nginx & supervisor: + +#### Debian/Ubuntu + +``` +apt install nginx supervisor +``` + +#### FreeBSD + +``` +sudo pkg install nginx py39-supervisor lang/python310 +``` + +### Installation + +Run grunt to configure onezoom for production use: + +``` +cd ${WEB2PY_PATH}/applications/OZtree +npm ci --legacy-peer-deps +./node_modules/.bin/grunt prod +``` + +Then run the install scripts to set up nginx & supervisord: + +``` +sudo ./install-nginx.sh +sudo ./install-supervisord.sh +``` + +If everything works, restart both. + ## Database explorer (optional) We find it useful to have a GUI interface to connect to the database and run SQL scripts, this can be used instead of using MySQL command line (similar to Windows command line) that is installed by default with MySQL. On Mac OS X we use the (excellent) http://www.sequelpro.com. On windows you could try http://www.mysql.com/products/workbench/ or https://www.quest.com/products/toad-for-mysql/ From 5adc7697c6d8ea319a72a370a7828f49ffff102c Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Mon, 5 Feb 2024 16:32:40 +0000 Subject: [PATCH 21/22] README: Remove mention of README_SERVER #703 There isn't a separate README_SERVER any more, the installation notes are more choose-your-own-adventure and the gory details are mostly automatic now. --- README.markdown | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.markdown b/README.markdown index 90d724b73..029d645ab 100755 --- a/README.markdown +++ b/README.markdown @@ -1,4 +1,6 @@ -This README file contains instructions for installing a private copy of OneZoom, either the tree explorer or the entire website. For gory details on running a *public* OneZoom server, see [README_SERVER.markdown](README_SERVER.markdown). Details of how to customize the OneZoom javascript viewer, along with information about the OneZoom APIs, are available by following the instructions below, then opening the compiled markdown file (at `OZprivate/rawJS/OZTreeModule/docs/_compiled.markdown` or if you are running your own private OneZoom web server, at /dev/DOCS - for example, https://127.0.0.1:8000/dev/DOCS). +This README file contains instructions for installing a private copy of OneZoom, either the tree explorer or the entire website. + +Details of how to customize the OneZoom javascript viewer, along with information about the OneZoom APIs, are available by following the instructions below, then opening the compiled markdown file (at `OZprivate/rawJS/OZTreeModule/docs/_compiled.markdown` or if you are running your own private OneZoom web server, at /dev/DOCS - for example, https://127.0.0.1:8000/dev/DOCS). If you simply want to run a local copy of OneZoom, but not modify the code yourself, we recommend using our [Docker image](https://hub.docker.com/r/onezoom/oztree-complete). The rest of this README provides details for compiling and running a OneZoom instance (something that is done under the hood when [creating the docker image](https://github.com/OneZoom/OZtree-docker)). From c0f28d326e3f59563d1620b20ad21de68d21a0f2 Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Mon, 5 Feb 2024 16:34:09 +0000 Subject: [PATCH 22/22] README: Mysql-under windows notes #703 * Link to MSI installer * Suggest SQL workbench --- README.markdown | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/README.markdown b/README.markdown index 029d645ab..828a0d36c 100755 --- a/README.markdown +++ b/README.markdown @@ -43,7 +43,7 @@ apt update && apt install mysql-server #### Windows -On Windows we recommended downloading the MSI installer as it will make it easier to configure the new server during the installation +On Windows we recommended downloading the [MSI installer](https://dev.mysql.com/downloads/installer/) as it will make it easier to configure the new server during the installation. Once mysql is installed, you will need to set a root password, and create a database for web2py to use. See http://dev.mysql.com/doc/refman/5.7/en/default-privileges.html. The mysqld program is responsible for running the new database just created. When this program is running, you can connect to the database. @@ -97,7 +97,7 @@ but will get API information form the OneZoom website, *and* also use the OneZoo For developers only, who may wish to create a minlife version not only using modified javascript in the treeviewer but also with bespoke html, you can run `grunt partial-local-install`. This is much more effort since it requires you to set up a full installation (as below) before creating the minlife scripts, but once created, the files in `static` will be enough for other users to view (and test) your modifications. -## Database setup / migation +## Database set-up / migation For full installations, you need to setup your database. @@ -161,7 +161,12 @@ If everything works, restart both. ## Database explorer -(optional) We find it useful to have a GUI interface to connect to the database and run SQL scripts, this can be used instead of using MySQL command line (similar to Windows command line) that is installed by default with MySQL. On Mac OS X we use the (excellent) http://www.sequelpro.com. On windows you could try http://www.mysql.com/products/workbench/ or https://www.quest.com/products/toad-for-mysql/ +(optional) We find it useful to have a GUI interface to connect to the database and run SQL scripts, this can be used instead of using MySQL command line (similar to Windows command line) that is installed by default with MySQL. +On Mac OS X we use the (excellent) http://www.sequelpro.com. +On windows you could try http://www.mysql.com/products/workbench/ or https://www.quest.com/products/toad-for-mysql/ +Under windows, [SQL Workbench](https://www.mysql.com/products/workbench/) can also be used, even if your MySQL server is installed under WSL2. + +installing SQL Workbench on Windows works great to connect to the Ubuntu MySQL instance. ## Starting and shutting down web2py