diff --git a/Gruntfile.js b/Gruntfile.js index 3c215beff..12b069569 100755 --- a/Gruntfile.js +++ b/Gruntfile.js @@ -5,14 +5,15 @@ const path = require('path'); partial_install_site = "http://www.onezoom.org"; partial_local_install_site = "http://127.0.0.1:8000"; // if you are running a local installation -preferred_python3 = "python3.7"; // in case you have multiple python3 versions installed +preferred_python3 = "python3.10"; // in case you have multiple python3 versions installed web2py_py = path.join(path.dirname(path.dirname(process.cwd())), 'web2py.py'); +venv_python = path.join(path.dirname(path.dirname(process.cwd())), 'bin/python'); /** Generate a function to execute a web2py script, handing over all arguments */ function exec_web2py_script(script_name, init_args) { return function () { return [ - preferred_python3, + venv_python, web2py_py, '-S OZtree/default', '-M', @@ -29,16 +30,36 @@ module.exports = function (grunt) { grunt.initConfig({ pkg: grunt.file.readJSON('package.json'), exec: { + web2py_configure: { + cwd: "../../", + command: [ + 'git submodule update --init --recursive', + 'ln -sf applications/OZtree/_COPY_CONTENTS_TO_WEB2PY_DIR/routes.py routes.py', + 'ln -sf handlers/wsgihandler.py handler.py', + '( [ -d applications/welcome ] && rm -r -- "applications/welcome" || true )', + '( [ -d applications/examples ] && rm -r -- "applications/examples" || true )', + '( [ -f applications/OZtree/private/appconfig.ini ] || { cp applications/OZtree/private/appconfig.ini.example applications/OZtree/private/appconfig.ini ; echo "****** edit private/appconfig.ini"; exit 1; } )', + preferred_python3 + ' -m venv .', + './bin/pip install -r applications/OZtree/requirements.txt', + // Make web2py.py use the venv + 'mv web2py.py web2py.py.o', + 'echo "#!' + venv_python + '" > web2py.py', + 'tail -n +2 web2py.py.o >> web2py.py', + ].join(" && "), + }, + web2py_start_dev: { + cwd: "../../", + command: [ + '( [ -f oz.crt ] || openssl req -newkey rsa:2048 -x509 -days 365 -nodes -keyout oz.key -subj "/CN=dev.onezoom/" -out oz.crt; )', + venv_python + ' web2py.py -c oz.crt -k oz.key -p 8000 -a pass', + ].join(" && "), + }, compile_python: { // compile python to make it faster on the server and hence reduce server load. // should probably be run using the same python version as used to run web2py cwd: "../../", command: - preferred_python3 + ' -c "import gluon.compileapp; gluon.compileapp.compile_application(\'' - + process.cwd() - + '\', skip_failed_views=True)"' - + ' || ' + // If python 3.7 isn't available, use the system-defined python3 instead - 'python3 -c "import gluon.compileapp; gluon.compileapp.compile_application(\'' + venv_python + ' -c "import gluon.compileapp; gluon.compileapp.compile_application(\'' + process.cwd() + '\', skip_failed_views=True)"' }, @@ -59,7 +80,7 @@ module.exports = function (grunt) { } }, make_release_info: { - command: 'git describe --tags > RELEASE_INFO && python3 OZprivate/ServerScripts/Utilities/get_release_name.py RELEASE_INFO >> RELEASE_INFO' + command: 'git describe --tags > RELEASE_INFO && ' + venv_python + ' OZprivate/ServerScripts/Utilities/get_release_name.py RELEASE_INFO >> RELEASE_INFO' }, test_server: { command: function () { @@ -95,7 +116,7 @@ module.exports = function (grunt) { // Any .html file in /static is fair game. See documentation in // https://github.com/OneZoom/OZtree#onezoom-setup command: - "python3 OZprivate/ServerScripts/Utilities/partial_install.py" + + venv_python + " OZprivate/ServerScripts/Utilities/partial_install.py" + " --search " + partial_local_install_site + // replace local urls, for partial local install " --replace " + partial_install_site + " static/*.html" @@ -220,6 +241,8 @@ module.exports = function (grunt) { grunt.registerTask("compile-js_dev", ["exec:compile_js_dev"]); grunt.registerTask("partial-install", ["compile-js", "css", "copy:dev", "curl-dir:get_minlife", "exec:convert_links_to_local"]); grunt.registerTask("partial-local-install", ["compile-js", "css", "copy:dev", "curl-dir:get_local_minlife", "exec:convert_links_to_local"]); - grunt.registerTask("prod", ["clean", "compile-python", "compile-js", "css", "compress", "copy:prod", "make_release_info", "docs"]); - grunt.registerTask("dev", ["clean", "compile-js_dev", "css", "compress", "copy:dev", "make_release_info", "docs"]); + grunt.registerTask("web2py", ["exec:web2py_configure"]); + grunt.registerTask("prod", ["clean", "web2py", "compile-python", "compile-js", "css", "compress", "copy:prod", "make_release_info", "docs"]); + grunt.registerTask("dev", ["clean", "web2py", "compile-js_dev", "css", "compress", "copy:dev", "make_release_info", "docs"]); + grunt.registerTask("start-dev", ['exec:web2py_start_dev']); }; diff --git a/OZprivate/ServerScripts/SQL/create_db_indexes.sql b/OZprivate/ServerScripts/SQL/create_db_indexes.sql index 851b6b815..339716d16 100644 --- a/OZprivate/ServerScripts/SQL/create_db_indexes.sql +++ b/OZprivate/ServerScripts/SQL/create_db_indexes.sql @@ -37,7 +37,6 @@ call MakeFullUnicode('images_by_ott', 'rights'); call MakeFullUnicode('images_by_ott', 'licence'); call MakeFullUnicode('images_by_name', 'rights'); call MakeFullUnicode('images_by_name', 'licence'); -call MakeFullUnicode('search_log', 'search_string'); # note make sure that the name column in vernacular_by_name and the name column in ordered_leaves and ordered_nodes are of the same character set otherwise search can get incredibly slow even with indexes. @@ -213,9 +212,6 @@ CREATE FULLTEXT INDEX ft_user_sponsor_info_index ON reservations (user_more_info DROP INDEX ipni_index ON PoWO; CREATE INDEX ipni_index ON PoWO (ipni_int) USING HASH; -DROP INDEX string_index ON search_log; -CREATE INDEX string_index ON search_log (search_string) USING HASH; - DROP INDEX identifier_index ON partners; CREATE INDEX identifier_index ON partners (partner_identifier) USING HASH; diff --git a/OZprivate/ServerScripts/Utilities/OneOff/make_usernames.py b/OZprivate/ServerScripts/Utilities/OneOff/make_usernames.py index aed707ef4..9c53d956f 100644 --- a/OZprivate/ServerScripts/Utilities/OneOff/make_usernames.py +++ b/OZprivate/ServerScripts/Utilities/OneOff/make_usernames.py @@ -1,6 +1,6 @@ """ Run from the OZtree directory as -python3 ../../web2py.py -S OZtree -M -R applications/OZtree/OZprivate/ServerScripts/Utilities/OneOff/make_usernames.py +./web2py-run OZprivate/ServerScripts/Utilities/OneOff/make_usernames.py After 2 passes, will allocate usernames of remaining unallocated reservations using a species name plus year. diff --git a/README.markdown b/README.markdown index 19e6ef45d..828a0d36c 100755 --- a/README.markdown +++ b/README.markdown @@ -1,4 +1,6 @@ -This README file contains instructions for installing a private copy of OneZoom, either the tree explorer or the entire website. For gory details on running a *public* OneZoom server, see [README_SERVER.markdown](README_SERVER.markdown). Details of how to customize the OneZoom javascript viewer, along with information about the OneZoom APIs, are available by following the instructions below, then opening the compiled markdown file (at `OZprivate/rawJS/OZTreeModule/docs/_compiled.markdown` or if you are running your own private OneZoom web server, at /dev/DOCS - for example, https://127.0.0.1:8000/dev/DOCS). +This README file contains instructions for installing a private copy of OneZoom, either the tree explorer or the entire website. + +Details of how to customize the OneZoom javascript viewer, along with information about the OneZoom APIs, are available by following the instructions below, then opening the compiled markdown file (at `OZprivate/rawJS/OZTreeModule/docs/_compiled.markdown` or if you are running your own private OneZoom web server, at /dev/DOCS - for example, https://127.0.0.1:8000/dev/DOCS). If you simply want to run a local copy of OneZoom, but not modify the code yourself, we recommend using our [Docker image](https://hub.docker.com/r/onezoom/oztree-complete). The rest of this README provides details for compiling and running a OneZoom instance (something that is done under the hood when [creating the docker image](https://github.com/OneZoom/OZtree-docker)). @@ -10,165 +12,173 @@ There are two ways in which you can install OneZoom on a personal computer: full * *Full installation* creates an entire duplicate of the OneZoom website, which is built using the [web2py](http://web2py.com) framework. This creates a fully self-contained local system (apart from the picture files, which can be downloaded separately). This is the most reliable installation method, but requires you to install and run extra software packages, in particular [web2py](http://web2py.com) and a [MySQL](https://www.mysql.com) server. Since this can be quite complicated, the majority of this readme contains instructions for full installation. - ## Requirements and packages -For all installation methods, you will need to install node.js (and npm, the node package manager), and the webpack package. To compile the OneZoom javascript codebase automatically, you will then need to install grunt. To generate documentation or make a partial install, you will also need perl installed on your system. - -For full installation, you will additionally need to install web2py, and ensure that you have the programming language python installed on your system, which is what web2py uses. You will also need access to a database backend (e.g. mySQL running on your own computer, or on a remote server which you can administer). - -To create trees, you will need python and perl, along with a number of libraries, as listed below. - -### Required packages (you will need these even if you're not creating trees) -The OneZoom codebase uses the following software (licenses for each listed in braces). The first three are programming languages which may well already be installed on your computer. - -* [Python](https://www.python.org) (assumed version 3.7) with the following libraries installed: - * mysql-connector-python - * pymysql (needed even if not creating trees) - * piexif - * requests - * Dendropy - * (for functional testing) nose + js2py + selenium + e.g. chromedriver_installer -* [Perl](https://www.perl.org) with the following libraries installed - * File::ReadBackwards - * LWP::Simple - * JSON - * DBI - * Try::Tiny - * Text::CSV - * Image::ExifTool - * DBD::mysql -* [web2py](http://web2py.com) (LGPL license) -* [npm](https://www.npmjs.com/get-npm), part of node.js which, when run will install a large number of other packages including - * [grunt](https://gruntjs.com) (MIT licence): to automate creating the OneZoom website files - * [webpack](https://webpack.js.org) (MIT licence): to package the OneZoom javascript tree viewer into a library - * jsdoc-to-markdown (MIT licence): to produce documentation from source code -* [ImageMagick](https://www.imagemagick.org/script/index.php) (Apache 2.0) for processing thumbnails -* [curl](https://curl.haxx.se) (MIT-like licence) to download partial installs (`curl` is probably already installed on your computer) -* [UIkit 3](https://getuikit.com) (MIT licence) for the User Interface (this code is included in the OneZoom github repo, and does not need downloading) - -## Quick installation steps - -Before anything else, get the OZtree app from [github](https://github.com/OneZoom/OZtree) - see *"Downloading the OZtree app"*. You should also make sure you have node.js and the node package manager (npm), see *"Building the OneZoom tree viewer"* - - -### For a partial installation (less tested): - -1. Install the command-line version of `grunt` using `npm install -g grunt-cli`. You may need to have administrator privileges to do this. -2. From anywhere within the OZtree download, run `npm install` to install all the packages for automation. -3. Create a partial installation by running `grunt partial-install`. This downloads the "minlife" and "minlife_tour" pages from the central OneZoom website, modifies links within them, and places appropriately named html files into the `static` directory of your OZtree distribution. -4. Open e.g. `static/minlife.html` with a web browser of your choice (we recommend Chrome or Safari). Note that this file needs to stay within the static directory to work at all. You may also need to allow your browser to allow local files to be loaded via AJAX (i.e. disabling some local cross-origin checks). Different browsers do this in different ways: for example in Chrome you can start up your browser with the `--allow-file-access-from-files` option, and in Safari, you can choose "Disable Local File Restrictions" from the "Developer" menu. -5. Note that the normal `minlife.html` file will use local versions of data files and the javascript treeviewer, but will get API information form the OneZoom website, *and* also use the OneZoom website as the source for the html page which embeds the viewer. For developers only, who may wish to create a minlife version not only using modified javascript in the treeviewer but also with bespoke html, you can run `grunt partial-local-install`. This is much more effort since it requires you to set up a full installation (as below) before creating the minlife scripts, but once created, the files in `static` will be enough for other users to view (and test) your modifications. - -### For a full installation (recommended): - -1. Install a source code version of [web2py](http://www.web2py.com), placing your [OZtree repository](https://github.com/OneZoom/OZtree) within the web2py `applications` directory. -2. Install command-line software by running `npm install -g grunt-cli` (you may need to do all this with administrator privileges). -3. Run `npm install` from within the OZtree folder you moved in step 1. then run `grunt dev` (or `grunt prod` if in production mode) - see *"[Building the OneZoom tree viewer](#building-the-onezoom-tree-viewer)"*. -3. [Install](http://dev.mysql.com/downloads/mysql/) & start MySQL, then create a new database (see *"[Setting up the database backend](#setting-up-the-database-backend)"*) -4. Create a appconfig.ini file in `OZtree/private`, with `migrate=1` and which references this database with the appropriate username and password. We also recommend copying the `routes.py` file from `OZtree/_COPY_CONTENTS_TO_WEB2PY_DIR` to the top level of your web2py installation - see *"[Web2py installation](#web2py-installation)"* -5. Fire up a temporary web2py server and visit the main page to create the (empty) database tables - see *"[Starting and shutting down web2py](#starting-and-shutting-down-web2py)"* -6. Load up data into the tables: first create a user and assign it a 'manager' role in the `auth_` tables using the web2py database admin pages, then load the other tables using data from the original OneZoom site (e.g. sent to you via file transfer) - see *"[Filling the database](#filling-the-database)"*. -7. Optimise your installation: - * create indexes on the tables by running the SQL script in `OZtree/OZprivate/ServerScripts/SQL/create_db_indexes.sql`. You can do this, for example, by running `SOURCE /path/to/OZtree/OZprivate/ServerScripts/SQL/create_db_indexes.sql` within a mysql client. - * set `is_testing = False` in `models/db.py` and `migrate=0` in appconfig.ini. +For any installation, OneZoom requires node & python (3.10). -## Downloading the OZtree app +#### Debian/Ubuntu -Download a copy of the OZtree application from GitHub at [https://github.com/OneZoom/OZtree](https://github.com/OneZoom/OZtree), either as a zip file (not recommended), or probably better (easier to update), by cloning the repository (e.g. using `git clone https://github.com/OneZoom/OZtree` or if you have [GitHub Desktop](https://desktop.github.com) installed, click "Open in Desktop" from the [OZtree repo](https://github.com/OneZoom/OZtree)). Make sure the git folder is called "OZtree" (this is the default when you clone the repo, but not if you download it as a zip file). +``` +apt install nodejs npm +apt install python3 python3-dev python3-venv +``` -For full installation, you will also need to download the source code version of web2py, either via git (https://github.com/web2py/web2py/) or simply from the download link at http://www.web2py.com/. You can then place the OZtree directory into the `applications` directory of the web2py folder. +#### FreeBSD +``` +sudo pkg install lang/python310 node18 npm-node18 +``` -## Building the OneZoom tree viewer +In addition, for a full installation you also need MySQL: -Compiling and creating the OneZoom explorer javascript code requires grunt to be installed. This compiles javascript code from multiple sources into a single file. You will need to install the node package manager, npm, then do +#### Debian/Ubuntu ``` -npm install -g grunt-cli +apt install lsb-release +wget https://dev.mysql.com/get/mysql-apt-config_0.8.29-1_all.deb +dpkg -i mysql-apt-config_0.8.29-1_all.deb +apt update && apt install mysql-server +# NB: Select "Use Legacy Authentication Method (Retain MySQL 5.x Compatibility)" ``` -To install this is likely to require administrator privileges. Other required packages can then be installed from within the OZtree folder by simply typing the following (which may take a while to complete!) +#### Windows + +On Windows we recommended downloading the [MSI installer](https://dev.mysql.com/downloads/installer/) as it will make it easier to configure the new server during the installation. +Once mysql is installed, you will need to set a root password, and create a database for web2py to use. See http://dev.mysql.com/doc/refman/5.7/en/default-privileges.html. +The mysqld program is responsible for running the new database just created. When this program is running, you can connect to the database. + +## Installation + +If performing a full installation, you need to create a database for OneZoom to use: ``` -npm install +mysql -p +CREATE DATABASE OneZoom; +CREATE USER 'oz'@'localhost' IDENTIFIED BY 'passwd-you-should-change-this'; +GRANT ALL PRIVILEGES ON OneZoom . * TO 'oz'@'localhost'; ``` -Once these are installed you can run grunt as follows (feel free to examine the configuration options which are stored in `Gruntfile.js` in the main OZtree directory): +Firstly, you need to check out the web2py / OneZoom repositories, and run npm: -#### Compile documentation -`grunt docs`: Use this command to generate a compiled documentation file. This will generate a large compiled markdown file in `OZprivate/rawJS/OZTreeModule/docs/_compiled.markdown`, which is best viewed once you have got web2py running, by pointing your browser to `dev/DOCS` (e.g. at `http://127.0.0.1:8000/dev/DOCS`). Note that viewing this page requires a working internet connection to get various formatting files) +``` +# NB: The path should match the eventual hostname for this installation +export WEB2PY_PATH="/srv/.../onezoom.myhostname.org" +mkdir -p ${WEB2PY_PATH} +chown deploy:staff ${WEB2PY_PATH} +git clone https://github.com/web2py/web2py ${WEB2PY_PATH} --branch v2.27.1 +git clone https://github.com/OneZoom/OZtree.git ${WEB2PY_PATH}/applications/OZtree --branch production +cd ${WEB2PY_PATH}/applications/OZtree +npm ci --legacy-peer-deps +``` -#### In development mode: -`grunt dev`: This command bundles multiple js files into one. +Next, ``cp private/appconfig.ini.example private/appconfig.ini`` and edit to match your needs, taking care to: -#### In production mode: -`grunt prod`: This command does three things. Firstly, it pre-compiles python code. Then it bundles multiple js files into one. Lastly, it minifies bundled js files. +* Use the same database credentials as configured earlier. +Once done, run ``./node_modules/.bin/grunt dev`` for a development installation. -## The server-side database +For production installation, see later. -### Setting up the database backend +### Partial installation -The web2py instance requires a database to be running. We previously used sqllite, and code for interfacing with sqllite is still present in the codebase, but probably will not work, as we have switched to using mySQL. +If you'd like your installation to be a partial installation, run: -So a major step when installing OneZoom is: +``` +./node_modules/.bin/grunt partial-install +``` -1. Install a locally running copy of mySQL. Make sure the server is installed and not only the client - There are many ways to do this: see http://dev.mysql.com/downloads/mysql/. +...and open `static/minlife.html` in a web browser. +Note that this file needs to stay within the static directory to work at all. +You may also need to allow your browser to allow local files to be loaded via AJAX (i.e. disabling some local cross-origin checks). +Different browsers do this in different ways: for example in Chrome you can start up your browser with the `--allow-file-access-from-files` option, and in Safari, you can choose "Disable Local File Restrictions" from the "Developer" menu. - On Windows we recommended downloading the MSI installer as it will make it easier to configure the new server during the installation +Note that the normal `minlife.html` file will use local versions of data files and the javascript treeviewer, +but will get API information form the OneZoom website, *and* also use the OneZoom website as the source for the html page which embeds the viewer. +For developers only, who may wish to create a minlife version not only using modified javascript in the treeviewer but also with bespoke html, you can run `grunt partial-local-install`. +This is much more effort since it requires you to set up a full installation (as below) before creating the minlife scripts, but once created, the files in `static` will be enough for other users to view (and test) your modifications. - Once mysql is installed, you will need to set a root password, and create a database for web2py to use. See http://dev.mysql.com/doc/refman/5.7/en/default-privileges.html. +## Database set-up / migation - The mysqld program is responsible for running the new database just created. When this program is running, you can connect to the database. +For full installations, you need to setup your database. -2. (optional) We find it useful to have a GUI interface to connect to the database and run SQL scripts, this can be used instead of using MySQL command line (similar to Windows command line) that is installed by default with MySQL. On Mac OS X we use the (excellent) http://www.sequelpro.com. On windows you could try http://www.mysql.com/products/workbench/ or https://www.quest.com/products/toad-for-mysql/ +To create required tables, use web2py migrate: -3. Once mysql is installed, you will need to set a root password, and create a database for web2py to use. See http://dev.mysql.com/doc/refman/5.7/en/default-privileges.html. So once mysqld is running, you need to log in to the sql server with the root name and password (if you are using the command line, log in using `mysql -u root -p`), and issue the following SQL commands (the text after the `mysql>` prompt) to create a database for web2py to use: feel free to use a different *'passwd'*. +* Edit private/appconfig.ini, setting ``migrate=1`` +* ``./web2py-run tests/unit/test_modules_embed.py``, on startup this will create tables as necessary +* Edit private/appconfig.ini, migrate=0 - ``` - mysql> create database OneZoom; - Query OK, 1 row affected (0.09 sec) - - mysql> CREATE USER 'oz'@'localhost' IDENTIFIED BY 'passwd'; - Query OK, 0 rows affected (0.19 sec) - - mysql> GRANT ALL PRIVILEGES ON OneZoom . * TO 'oz'@'localhost'; - Query OK, 0 rows affected (0.09 sec) - ``` +A database dump should be used to add species information: -The database is now up and running. We recommend that you do *not* load data into it immediately, but first create the tables by installing web2py with `migrate=1` set in the appconfig.ini file. After running an instance of web2py and visiting the new site, the correct table structure should be automatically created (see below). After that you can populate the data into the tables using downloaded files. +``` +# NB: OneZoom.dump.sql can be extracted from https://github.com/OneZoom/OZtree-docker +mysql -p +USE OneZoom +SOURCE /OneZoom.dump.sql +``` -## Web2py installation +Finally, ensure indexes are created: -Configuring the OneZoom application to use the database involves creating a file called 'appconfig.ini' in the `private` folder within the OZtree app. A minimal appconfig.ini file to get the site working is in [private/appconfig.ini.example](private/appconfig.ini.example), which can renamed to `appconfig.ini` and modified to use the username and password that you supplied above. +``` +mysql -p +USE OneZoom +SOURCE ${WEB2PY_PATH}/applications/OZtree/OZprivate/ServerScripts/SQL/create_db_indexes.sql +``` -In order to use web2py you need to have a python v3 installed, the latest version can be found at -[https://www.python.org/downloads/](https://www.python.org/downloads/) +## Production installation -NB: on Windows, make sure that you add `python` (and ideally `python3`) to the windows path during install, or the commands below will not work +For a production installation of OneZoom, you also need nginx & supervisor: -Assuming you have python version 3 installed, should now try starting web2py as follows. +#### Debian/Ubuntu -### Starting and shutting down web2py +``` +apt install nginx supervisor +``` -On the OneZoom main site, web2py is run using a combination of nginx and uwsgi. This is complete overkill if you just want to run a local copy of OneZoom for testing purposes. You can simply run a [temporary and basic web2py server using Python 3](http://www.web2py.com/books/default/chapter/29/03/overview#Startup). The simplest is to open a command-line prompt in the root web2py folder, and run the following (assuming the command `python3` is linked to something like Python 3.7) +#### FreeBSD -`python3 web2py.py -i 127.0.0.1 -p 8000 -a pass` +``` +sudo pkg install nginx py39-supervisor lang/python310 +``` -* (NB: it is possible to run a secure OneZoom site over https. To try this using the basic web2py server, create a `.crt` and `.key` file, e.g. by running the following in the web2py root directory: `openssl req -newkey rsa:2048 -x509 -days 365 -nodes -keyout oz.key -out oz.crt`, then use them when running web2py, as in: `python3 web2py.py -c oz.crt -k oz.key -i 127.0.0.1 -p 8000 -a pass`) +### Installation +Run grunt to configure onezoom for production use: -When web2py is run, it will print instructions telling how to shut down the web2py server. For example, on Windows you might use `taskkill /f /pid XXXX`, where `XXXX` is the process id. +``` +cd ${WEB2PY_PATH}/applications/OZtree +npm ci --legacy-peer-deps +./node_modules/.bin/grunt prod +``` + +Then run the install scripts to set up nginx & supervisord: -**If this is a new installation** you should now visit `http://127.0.0.1:8000/OZtree/default/` or `https://127.0.0.1:8000/OZtree/default/` to force web2py to create database tables. To load data into the tables, see "Loading Data", below. +``` +sudo ./install-nginx.sh +sudo ./install-supervisord.sh +``` -Also, if you want to make OneZoom the default application, make a copy of the routes.py file in the folder labelled `_COPY_CONTENTS_TO_WEB2PY_DIR` and place it in the top level web2py directory (see `_COPY_CONTENTS_TO_WEB2PY_DIR/README.markdown`). +If everything works, restart both. -Once tables are created, and everything is working, you can set `is_testing = False` in `models/db.py` and `migrate=0` in `private/appconfig.ini`. This will mean that web2py will not make any changes to table structures in the DB, and also that changes to appconfig.ini will require a web2py restart. +## Database explorer + +(optional) We find it useful to have a GUI interface to connect to the database and run SQL scripts, this can be used instead of using MySQL command line (similar to Windows command line) that is installed by default with MySQL. +On Mac OS X we use the (excellent) http://www.sequelpro.com. +On windows you could try http://www.mysql.com/products/workbench/ or https://www.quest.com/products/toad-for-mysql/ +Under windows, [SQL Workbench](https://www.mysql.com/products/workbench/) can also be used, even if your MySQL server is installed under WSL2. + +installing SQL Workbench on Windows works great to connect to the Ubuntu MySQL instance. + +## Starting and shutting down web2py + +On the OneZoom main site, web2py is run using a combination of nginx and uwsgi. This is complete overkill if you just want to run a local copy of OneZoom for testing purposes. You can simply run a [temporary and basic web2py server using Python 3](http://www.web2py.com/books/default/chapter/29/03/overview#Startup). The simplest is to open a command-line prompt in the root web2py folder, and run the following (assuming the command `python3` is linked to something like Python 3.7) + +``` +./node_modules/.bin/grunt start-dev +``` + +When web2py is run, it will print instructions telling how to shut down the web2py server. For example, on Windows you might use `taskkill /f /pid XXXX`, where `XXXX` is the process id. -### Web2py folder structure +## Web2py folder structure #### Standard folders `databases` stores all the database structure. diff --git a/_COPY_CONTENTS_TO_WEB2PY_DIR/README.markdown b/_COPY_CONTENTS_TO_WEB2PY_DIR/README.markdown index 7bf3f2c8c..dc513b8d4 100755 --- a/_COPY_CONTENTS_TO_WEB2PY_DIR/README.markdown +++ b/_COPY_CONTENTS_TO_WEB2PY_DIR/README.markdown @@ -1,8 +1 @@ -### Making OneZoom the default application - -To make OneZoom the default web2py app, a copy of the accompanying routes.py file can placed at the root of the web2py folder (at the same level as the `applications` folder). - -### Disabling other applications -You may also wish to delete the `welcome` and `examples` apps (just delete the folders), or disable them from the web2py admin web page (e.g. at http://127.0.0.1:8000/admin). - -You probably shouldn't disable the admin app, as this is the main way of adding users etc to the OneZoom app. It should only be accessible over https (and maybe only locally) anyway, so is not that much of a security risk. \ No newline at end of file +# Python files here will be symlinked by Grunt in web2py_configure diff --git a/controllers/API.py b/controllers/API.py index f9d32fb62..bc646aede 100755 --- a/controllers/API.py +++ b/controllers/API.py @@ -126,16 +126,6 @@ def search_node(): session.forget(response) response.headers["Access-Control-Allow-Origin"] = '*' searchFor = make_unicode(request.vars.query or '') - try: - if myconf.take('general.log_search_strings'): - if request.vars.no_log: - #'no_log' flag set: this is probably us blatting the search for testing purposes - pass - else: - db.search_log.update_or_insert(db.search_log.search_string==searchFor, search_string=searchFor, search_count=db.search_log.search_count+1) - - except: - pass res1 = search_for_name() if len(res1['leaf_hits']) + len(res1['node_hits']) <15: try: diff --git a/install-nginx.sh b/install-nginx.sh new file mode 100755 index 000000000..cbca4e427 --- /dev/null +++ b/install-nginx.sh @@ -0,0 +1,254 @@ +#!/usr/bin/env bash +set -eux +PROJECT_PATH="${PROJECT_PATH-$(dirname "$(readlink -f "$0")")}" # The full project path +WEB2PY_PATH="$(dirname $(dirname "$PROJECT_PATH"))" +WEB2PY_NAME="${WEB2PY_NAME-$(basename ${WEB2PY_PATH})}" # Directory web2py lives in, will be unique per installation +CERT_DIR="/var/lib/dehydrated/certs" + +WWW_SERVER_NAME="${WEB2PY_NAME}" # Assume we checked out web2py in /.../www.onezoom.org +WWW_IMAGES_SERVER_NAME="$(echo ${WWW_SERVER_NAME} | sed 's/^w*/images/')" # images.onezoom.org or imagesdev.onezoom.org + +[ -d "/etc/nginx" ] && NGINX_PATH="/etc/nginx" +[ -d "/usr/local/etc/nginx" ] && NGINX_PATH="/usr/local/etc/nginx" +mkdir -p "${NGINX_PATH}/conf.d/" +NGINX_LOG_PATH="/var/log/nginx" +[ -d "/var/db/acme/" ] && NGINX_CERT_PATH="/var/db/acme/live/" +[ -d "/var/lib/dehydrated/certs" ] && NGINX_CERT_PATH="/var/lib/dehydrated/certs" +NGINX_DHPARAM_PATH="${NGINX_PATH}/dhparam.pem" +[ -d "/var/db/acme/live" ] && NGINX_CHALLENGE_PATH="/var/db/acme/live" +[ -d "/var/lib/dehydrated/acme-challenges" ] && NGINX_CHALLENGE_PATH="/var/lib/dehydrated/acme-challenges" + +# Generate NGINX_DHPARAM +[ -e "${NGINX_DHPARAM_PATH}" ] || openssl dhparam -out "${NGINX_DHPARAM_PATH}" 4096 + +# Self-signed bootstrap-cert +for SN in onezoom.org ${WWW_SERVER_NAME} ${WWW_IMAGES_SERVER_NAME}; do + mkdir -p "${NGINX_CERT_PATH}/${SN}" + if [ ! -e "${NGINX_CERT_PATH}/${SN}/privkey.pem" ]; then + openssl req -x509 -newkey rsa:4096 -sha256 -days 365 -nodes \ + -keyout "${NGINX_CERT_PATH}/${SN}/privkey-ss.pem" \ + -out "${NGINX_CERT_PATH}/${SN}/fullchain-ss.pem" \ + -subj "/CN=${SN}" \ + -addext "subjectAltName = DNS:selfsigned.${SN}" + ln -s ${NGINX_CERT_PATH}/${SN}/privkey-ss.pem ${NGINX_CERT_PATH}/${SN}/privkey.pem + ln -s ${NGINX_CERT_PATH}/${SN}/fullchain-ss.pem ${NGINX_CERT_PATH}/${SN}/fullchain.pem + ln -s ${NGINX_CERT_PATH}/${SN}/fullchain-ss.pem ${NGINX_CERT_PATH}/${SN}/chain.pem + fi +done + +# Create NGINX config +cat < ${NGINX_PATH}/nginx.conf +#### Generated by $0 - DO NOT EDIT + +events {} + +http { + include mime.types; + default_type application/octet-stream; + + # Enable Gzip + gzip on; + gzip_http_version 1.0; + gzip_comp_level 2; + gzip_min_length 1100; + gzip_buffers 4 8k; + gzip_proxied any; + gzip_types + # text/html is always compressed by HttpGzipModule + text/css + text/javascript + text/xml + text/plain + text/x-component + text/json + application/javascript + application/json + application/xml + application/rss+xml + font/truetype + font/opentype + application/vnd.ms-fontobject + image/svg+xml; + gzip_proxied expired no-cache no-store private auth; + gzip_disable "MSIE [1-6]\."; + gzip_vary on; + + keepalive_timeout 65; + + server_names_hash_bucket_size 128; + + include ${NGINX_PATH}/conf.d/*.conf; +} +EOF + +cat < ${NGINX_PATH}/conf.d/onezoom.org.conf +server { + listen 80; + listen 443 ssl http2; + + server_name server_name onezoom.org default; + + location /.well-known/acme-challenge { + allow all; + default_type text/plain; + alias ${NGINX_CHALLENGE_PATH}; + } + ssl_certificate ${NGINX_CERT_PATH}/onezoom.org/fullchain.pem; + ssl_certificate_key ${NGINX_CERT_PATH}/onezoom.org/privkey.pem; + ssl_trusted_certificate ${NGINX_CERT_PATH}/onezoom.org/fullchain.pem; + ssl_dhparam ${NGINX_DHPARAM_PATH}; + + return 301 \$scheme://www.onezoom.org\$request_uri; +} +EOF + +cat < ${NGINX_PATH}/${WEB2PY_NAME}_static_include.inc +#### Generated by $0 - DO NOT EDIT + +alias ${PROJECT_PATH}/static/; + +#the directives used for static pages on OneZoom +location ~ /FinalOutputs/data/ { + # add_header 'X-static-gzipping' 'on' always; + gzip_static on; + #files in /data/ (e.g. the topology) have timestamps, so never change, and browsers can always use cache + expires max; + add_header Cache-Control "public"; +} + +location ~* \.(?:jpg|jpeg|gif|png|ico|gz|svg)\$ { + #cache images for a little while, even though we also cache them in the js. + #10 mins allows e.g. new crops to show up. + expires 10m; + access_log off; + add_header Cache-Control "public"; +} + +location ~ \.(js|css|html)\$ { + # add_header 'X-static-gzipping' 'on' always; + gzip_static on; + #cache the static js and html, but only for a bit, in case we implement changes + expires 30m; + add_header Cache-Control "public"; +} + +location ~* /(\w+/)?static/trees/[^/]+/ { + # static trees with a trailing slash need to be trimmed so that e.g. + # static/trees/AT/@Homo_sapiens is not seen as a request for a file called '@Homo_sapiens' + # see http://stackoverflow.com/questions/39519355 + rewrite ^(.*/static/trees/[^/]+)/ \$1 last; + return 404; +} +EOF + +cat < ${NGINX_PATH}/conf.d/${WWW_SERVER_NAME}.conf +#### Generated by $0 - DO NOT EDIT + +upstream uwsgi_${WEB2PY_NAME} { + least_conn; + server unix:///var/run/uwsgi/${WEB2PY_NAME}_uwsgi0.sock; + server unix:///var/run/uwsgi/${WEB2PY_NAME}_uwsgi1.sock; + server unix:///var/run/uwsgi/${WEB2PY_NAME}_uwsgi2.sock; + server unix:///var/run/uwsgi/${WEB2PY_NAME}_uwsgi3.sock; + server unix:///var/run/uwsgi/${WEB2PY_NAME}_uwsgi4.sock; +} + +server { + listen 80; + listen 443 ssl http2; + + server_name ${WWW_SERVER_NAME}; + + if (\$scheme != "https") { + return 301 https://\$server_name\$request_uri; + } + location /.well-known/acme-challenge { + allow all; + default_type text/plain; + alias ${NGINX_CHALLENGE_PATH}; + } + ssl_certificate ${NGINX_CERT_PATH}/${WWW_SERVER_NAME}/fullchain.pem; + ssl_certificate_key ${NGINX_CERT_PATH}/${WWW_SERVER_NAME}/privkey.pem; + ssl_trusted_certificate ${NGINX_CERT_PATH}/${WWW_SERVER_NAME}/fullchain.pem; + ssl_dhparam ${NGINX_DHPARAM_PATH}; + + # Generated by https://ssl-config.mozilla.org/ + ssl_session_timeout 1d; + ssl_session_cache shared:MozSSL:10m; # about 40000 sessions + ssl_session_tickets off; + + # intermediate configuration + ssl_protocols TLSv1.2 TLSv1.3; + ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-RSA-CHACHA20-POLY1305; + ssl_prefer_server_ciphers off; + + server_tokens off; + access_log ${NGINX_LOG_PATH}/${WWW_SERVER_NAME}.access.log combined buffer=32k flush=5m; + error_log ${NGINX_LOG_PATH}/${WWW_SERVER_NAME}.error.log; + + #cache filehandles on the server side to max 2000 files, hopefully mostly images + open_file_cache max=2000 inactive=20s; + open_file_cache_valid 60s; + open_file_cache_min_uses 3; + open_file_cache_errors off; + + index index.htm; + location /static/ { + include ${WEB2PY_NAME}_static_include.inc; + } + + location /OZtree/static/ { + include ${WEB2PY_NAME}_static_include.inc; + } + + location / { + location ~ \.json { + #don't log API (json) requests + access_log off; + uwsgi_pass uwsgi_${WEB2PY_NAME}; + } + uwsgi_pass uwsgi_${WEB2PY_NAME}; + include uwsgi_params; + } +} +EOF + +cat < ${NGINX_PATH}/conf.d/${WWW_IMAGES_SERVER_NAME}.conf +#### Generated by $0 - DO NOT EDIT + +server { + listen 80; + listen 443 ssl http2; + + server_name ${WWW_IMAGES_SERVER_NAME}; + server_tokens off; + + location /.well-known/acme-challenge { + allow all; + default_type text/plain; + alias ${NGINX_CHALLENGE_PATH}; + } + ssl_certificate ${NGINX_CERT_PATH}/${WWW_IMAGES_SERVER_NAME}/fullchain.pem; + ssl_certificate_key ${NGINX_CERT_PATH}/${WWW_IMAGES_SERVER_NAME}/privkey.pem; + ssl_trusted_certificate ${NGINX_CERT_PATH}/${WWW_IMAGES_SERVER_NAME}/fullchain.pem; + ssl_dhparam ${NGINX_DHPARAM_PATH}; + + access_log ${NGINX_LOG_PATH}/${WWW_IMAGES_SERVER_NAME}.access.log combined buffer=32k flush=5m; + error_log ${NGINX_LOG_PATH}/${WWW_IMAGES_SERVER_NAME}.error.log; + + #cache filehandles on the server side to max 2000 files, hopefully mostly images + open_file_cache max=2000 inactive=20s; + open_file_cache_valid 60s; + open_file_cache_min_uses 3; + open_file_cache_errors off; + + location / { + access_log off; + expires 10m; + add_header Cache-Control "public"; + root ${PROJECT_PATH}/static/FinalOutputs/img/; + } +} +EOF + +nginx -t diff --git a/install-supervisord.sh b/install-supervisord.sh new file mode 100755 index 000000000..40a404261 --- /dev/null +++ b/install-supervisord.sh @@ -0,0 +1,121 @@ +#!/usr/bin/env bash +set -eux +PROJECT_PATH="${PROJECT_PATH-$(dirname "$(readlink -f "$0")")}" # The full project path +WEB2PY_PATH="$(dirname $(dirname "$PROJECT_PATH"))" +WEB2PY_NAME="${WEB2PY_NAME-$(basename ${WEB2PY_PATH})}" # Directory web2py lives in, will be unique per installation +DEPLOY_USER="$(stat -c '%U' install-supervisord.sh 2>/dev/null || stat -f '%Su' install-supervisord.sh 2>/dev/null)" + +APP_USER="www" +APP_GROUP="www" +WWW_SERVER_NAME="${WEB2PY_NAME}" + +# NB: Yan says we should write to web2py_path & gluon. Seems risky +for DIR in "${PROJECT_PATH}/errors" \ + "${PROJECT_PATH}/databases" \ + "${PROJECT_PATH}/sessions" \ + "${PROJECT_PATH}/uploads" \ + "${WEB2PY_PATH}/logs" \ + "${WEB2PY_PATH}/deposit" \ + "/var/run/uwsgi"; do + mkdir -p -- "${DIR}" + chown -R ${APP_USER} "${DIR}" + chmod g+w "${DIR}" +done + +[ -d "/etc/supervisor" ] && SUPERVISORD_CONF_PATH="/etc/supervisor" +[ -e "/usr/local/etc/supervisord.conf.sample" ] && SUPERVISORD_CONF_PATH="/usr/local/etc" +mkdir -p "${SUPERVISORD_CONF_PATH}/supervisord.conf.d/" +mkdir -p "/var/run/supervisor" + +SUPERVISORD_LOG_PATH="/var/log/supervisord" +mkdir -p "${SUPERVISORD_LOG_PATH}" + +cat < "${SUPERVISORD_CONF_PATH}/supervisord.conf" +; **** Generated by $0 - DO NOT EDIT +; +[unix_http_server] +file=/var/run/supervisor/supervisor.sock ; (the path to the socket file) + +[supervisord] +logfile=${SUPERVISORD_LOG_PATH}/supervisord.log ; (main log file) +logfile_maxbytes=50MB ; (max main logfile bytes b4 rotation;default 50MB) +logfile_backups=10 ; (num of main logfile rotation backups;default 10) +loglevel=info ; (log level;default info; others: debug,warn,trace) +pidfile=/var/run/supervisor/supervisord.pid ; (supervisord pidfile;default supervisord.pid) +nodaemon=false ; (start in foreground if true;default false) +minfds=1024 ; (min. avail startup file descriptors;default 1024) +minprocs=200 ; (min. avail process descriptors;default 200) + +[rpcinterface:supervisor] +supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface + +[supervisorctl] +serverurl=unix:///var/run/supervisor/supervisor.sock ; use a unix:// URL for a unix socket + +[include] +files = supervisord.conf.d/*.conf +EOF + +cat < "${SUPERVISORD_CONF_PATH}/supervisord.conf.d/${WEB2PY_NAME}.conf" +; **** Generated by $0 - DO NOT EDIT +; +[group:${WEB2PY_NAME}] +programs=${WEB2PY_NAME}_uwsgi,${WEB2PY_NAME}_background_tasks,${WEB2PY_NAME}_session_cleanup + +[program:${WEB2PY_NAME}_uwsgi] +directory=${WEB2PY_PATH} +command=${WEB2PY_PATH}/bin/uwsgi + -s /var/run/uwsgi/%(program_name)s%(process_num)d.sock + --chmod-socket=666 + --need-app + --disable-logging + --master + --home=${WEB2PY_PATH} + --wsgi-file handler.py + --processes 1 + --threads 10 + --uid "${APP_USER}" + --gid "${APP_GROUP}" +stdout_logfile=${SUPERVISORD_LOG_PATH}/%(program_name)s_stdout.log +stdout_logfile_maxbytes=10MB +stdout_logfile_backups=10 +stderr_logfile=${SUPERVISORD_LOG_PATH}/%(program_name)s_stderr.log +stderr_logfile_maxbytes=10MB +stderr_logfile_backups=10 +startsecs=10 +stopsignal=QUIT +stopasgroup=true +killasgroup=true +process_name=%(program_name)s%(process_num)d +numprocs=5 + +[program:${WEB2PY_NAME}_background_tasks] +directory=${WEB2PY_PATH} +user=${APP_USER} +group=${APP_GROUP} +command=$(which bash) -c "${WEB2PY_PATH}/bin/python web2py.py -S OZtree/default -M -e -R applications/OZtree/private/background_tasks.py --args ${WWW_SERVER_NAME} verbose && /bin/sleep 86400" +stdout_logfile=${SUPERVISORD_LOG_PATH}/%(program_name)s_stdout.log +stdout_logfile_maxbytes=10MB +stdout_logfile_backups=10 +stderr_logfile=${SUPERVISORD_LOG_PATH}/%(program_name)s_stderr.log +stderr_logfile_maxbytes=10MB +stderr_logfile_backups=10 +startsecs=10 +stopsignal=QUIT +autorestart=true + +[program:${WEB2PY_NAME}_session_cleanup] +directory=${WEB2PY_PATH} +user=${APP_USER} +group=${APP_GROUP} +command=${WEB2PY_PATH}/bin/python web2py.py -S OZtree -M -R scripts/sessions2trash.py +stdout_logfile=${SUPERVISORD_LOG_PATH}/%(program_name)s_stdout.log +stdout_logfile_maxbytes=10MB +stdout_logfile_backups=10 +stderr_logfile=${SUPERVISORD_LOG_PATH}/%(program_name)s_stderr.log +stderr_logfile_maxbytes=10MB +stderr_logfile_backups=10 +startsecs=10 +stopsignal=QUIT +autorestart=true +EOF diff --git a/models/db.py b/models/db.py index 55b1e9d78..b90afe36f 100755 --- a/models/db.py +++ b/models/db.py @@ -11,12 +11,6 @@ ## if SSL/HTTPS is properly configured and you want all HTTP requests to ## be redirected to HTTPS, uncomment the line below: # request.requires_https() -#set the default language -T.set_current_languages('en', 'en-en') -#ALL pages can set ?lang=XXX to override the browser default for translating strings -if request.vars.lang: - T.force(request.vars.lang) - ## app configuration made easy. Look inside private/appconfig.ini from gluon.contrib.appconfig import AppConfig @@ -26,26 +20,26 @@ ## Useful global variables ######################################################################### -## once in production, set is_testing=False to gain optimizations -## this will also set migration=False for all tables, so that the DB table definitions are fixed -is_testing = True +# Running under rocket --> is_testing is true +is_testing = (request.env.server_software or '').lower().startswith('rocket') -## get config params etc -if is_testing: +## Read configuration +if is_testing and len(request.env.cmd_options.args) > 1 and os.path.isfile(request.env.cmd_options.args[-1]): # For unit testing, we might want to load a different appconfig.ini file, which can # be passed in to the rocket server as the last arg on the command-line. # (on the main server this is not used, and we default back to appconfig.ini - try: - if os.path.isfile(request.env.cmd_options.args[-1]): - myconf = AppConfig(request.env.cmd_options.args[-1], reload=True) - else: - raise IOError("No such file") - except (IOError, IndexError, AttributeError): - myconf = AppConfig(reload=True) #changes to appconfig.ini do not require restart - T.is_writable = True #allow translators to add new languages e.g. on the test (beta) site, but not on prod + myconf = AppConfig(request.env.cmd_options.args[-1], reload=is_testing) else: - myconf = AppConfig() #faster to read once and never re-update - T.is_writable = False + # NB: When running under rocket, re-load config every request with is_testing + myconf = AppConfig(reload=is_testing) + +## Configure i18n +T.set_current_languages('en', 'en-en') +# Allow translators to add new languages e.g. on the test (beta) site, but not on prod +T.is_writable = is_testing +#ALL pages can set ?lang=XXX to override the browser default for translating strings +if request.vars.lang: + T.force(request.vars.lang) try: thumb_base_url = myconf.take('images.url_base') @@ -159,6 +153,9 @@ auth.settings.reset_password_requires_verification = True auth.settings.allow_basic_login = True +## Configure session handling: http://web2py.com/books/default/chapter/29/13/deployment-recipes#Sessions-in-database +session.connect(request, response, db) + ##restrict site to only logged in users ## https://groups.google.com/forum/#!topic/web2py/0j92-sPp4bc ##NB: useful url to add a guest user programmatically http://stackoverflow.com/questions/35504306/web2py-how-to-programmatically-register-users/35518991 @@ -664,12 +661,6 @@ Field('leaf_click_count', type='integer'), format = '%(ott)s', migrate=is_testing) -# this table collects a list of search terms so we can optimise search -db.define_table('search_log', - Field('search_string', type='string', notnull=True, unique=True, length=name_length_chars), #this should be utf8mb4 - Field('search_count', type='integer', notnull=True), - format = '%(search_string)s', migrate=is_testing) - # This table buffers recently 'visited' EoL taxa (visited through the window popup or via the copyright link) # taxa in this table are stored until at least 1 minute after the taxon is visited, and then read by the EOL update # script (EoLQueryPicsNames.py) to check for updates to the crop location, ratings, etc. Once checked, the taxon diff --git a/package.json b/package.json index e7ba30e73..419228a0e 100644 --- a/package.json +++ b/package.json @@ -13,9 +13,10 @@ }, "scripts": { "test": "npx babel-tape-runner OZprivate/rawJS/OZTreeModule/tests/*.js", - "compile_js": "webpack --mode production", - "compile_js_dev:watch": "webpack --watch --config webpack.config.dev.js", - "compile_js_dev": "webpack --mode development" + "comment": "openssl-legacy-provider is needed for node 18 until we upgrade webpack - https://stackoverflow.com/a/69699772", + "compile_js": "NODE_OPTIONS=--openssl-legacy-provider webpack --mode production", + "compile_js_dev:watch": "NODE_OPTIONS=--openssl-legacy-provider webpack --watch --config webpack.config.dev.js", + "compile_js_dev": "NODE_OPTIONS=--openssl-legacy-provider webpack --mode development" }, "repository": { "type": "git", diff --git a/private/appconfig.ini.example b/private/appconfig.ini.example index a873376c1..d92c84e1d 100755 --- a/private/appconfig.ini.example +++ b/private/appconfig.ini.example @@ -29,7 +29,6 @@ separator = url = https://www.sandbox.paypal.com [general] -;log_search_strings = 1 [images] ; * url_base: get thumbnail images from this source. If not diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 000000000..d9d7db23d --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +pymysql==1.1.0 +uwsgi==2.0.23 diff --git a/tests/unit/test_modules_embed.py b/tests/unit/test_modules_embed.py index 832a14bf9..6c5558e95 100644 --- a/tests/unit/test_modules_embed.py +++ b/tests/unit/test_modules_embed.py @@ -1,6 +1,8 @@ """ -Run with -python3 web2py.py -S OZtree -M -R applications/OZtree/tests/unit/test_modules_embed.py +Run with:: + + ./web2py-run tests/unit/test_modules_embed.py + """ import re import unittest diff --git a/tests/unit/test_modules_sponsorship.py b/tests/unit/test_modules_sponsorship.py index 63cd4165f..60515d3c8 100644 --- a/tests/unit/test_modules_sponsorship.py +++ b/tests/unit/test_modules_sponsorship.py @@ -1,7 +1,7 @@ """ -Run with +Run with:: -python3 web2py.py -S OZtree -M -R applications/OZtree/tests/unit/test_modules_sponsorship.py + ./web2py-run tests/unit/test_modules_sponsorship.py Note you should make sure prices are set before running tests (manage/SET_PRICES.html) """ diff --git a/tests/unit/test_modules_usernames.py b/tests/unit/test_modules_usernames.py index ccbfc26f9..a72e7a207 100644 --- a/tests/unit/test_modules_usernames.py +++ b/tests/unit/test_modules_usernames.py @@ -1,6 +1,8 @@ """ -Run with -python3 web2py.py -S OZtree -M -R applications/OZtree/tests/unit/test_modules_username.py +Run with:: + + ./web2py-run tests/unit/test_modules_username.py + """ import unittest diff --git a/tests/util.py b/tests/util.py index 800459fe6..30d201aad 100644 --- a/tests/util.py +++ b/tests/util.py @@ -72,7 +72,7 @@ def __init__(self, appconfig_file=None): self.pid = None if self.is_local(): print("> starting web2py") - cmd = ['python3', os.path.join(web2py_app_dir, '..','..','web2py.py'), '-Q', '-i', ip, '-p', port, '-a', 'pass'] + cmd = [os.path.join(web2py_app_dir, '..','..','bin', 'python3'), os.path.join(web2py_app_dir, '..','..','web2py.py'), '-Q', '-i', ip, '-p', port, '-a', 'pass'] if appconfig_file is not None: cmd += ['--args', appconfig_file] self.pid = subprocess.Popen(cmd) diff --git a/web2py-run b/web2py-run new file mode 100755 index 000000000..c17db9934 --- /dev/null +++ b/web2py-run @@ -0,0 +1,8 @@ +#!/bin/sh +set -eu + +SCRIPT="applications/OZtree/$1" +shift + +cd ../../ +exec ./bin/python3 web2py.py -S OZtree -M -R "${SCRIPT}" --args $*