diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..3c9710e --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,39 @@ +name: Tests + +on: + pull_request: + branches: [master] + push: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + test: + runs-on: ubuntu-latest + strategy: + max-parallel: 2 + matrix: + python: ["3.10", "3.11", "3.12", "3.13", "3.14"] + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python + id: setup-python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python }} + + - name: Install dependencies + id: install-dependencies + run: | + python -m pip install --upgrade pip + python -m pip install tox tox-gh-actions + + - name: Test with tox + id: test-with-tox + run: | + tox diff --git a/.gitignore b/.gitignore index 500a111..f1b6f48 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,4 @@ __pycache__ .DS_Store /test_robots.db +venv diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..9049050 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,23 @@ +# Changelog + +## Unreleased + +- Add testing support for Python 3.14 and Wagtail 7.2 +- Add Wagtail 6 to test matrices +- Drop support for Django < 4.2 + +## Other tags + +- See [tags on this repository](https://github.com/torchbox-forks/wagtail-robots/tags) + +## 1.1.0+tbx (2024-03-05) + +- Add Wagtail 5.2 to test matrices + +## 1.0.1 (2023-10-16) + +- Add upgrade considerations for [Wagtail 5.1](https://github.com/torchbox-forks/wagtail-robots/pull/5) + +## 1.0.0 (2023-07-25) + +- Integrate [wagtail upgrades](https://github.com/unexceptable/wagtail-robots/pull/20) diff --git a/Makefile b/Makefile index e6e0619..160280a 100644 --- a/Makefile +++ b/Makefile @@ -11,15 +11,11 @@ user: migrate: @python testmanage.py migrate -tox-215: - tox -e py38-dj32-wt215 +tox-63: + tox -e python3.10-django4.2-wagtail6.3 -tox-216: - tox -e py38-dj32-wt216 +tox-70: + tox -e python3.12-django5.1-wagtail7.0 -tox-3: - tox -e py38-dj40-wt30 - -tox-4: - tox -e py38-dj40-wt40 - tox -e py38-dj41-wt41 +tox-71: + tox -e python3.13-django5.2-wagtail7.1 diff --git a/README.rst b/README.rst index b57643f..08ce3ee 100644 --- a/README.rst +++ b/README.rst @@ -1,3 +1,4 @@ +============== Wagtail Robots ============== @@ -9,11 +10,192 @@ This started as a fork of `Django Robots`_ but because of the differences between the Django Admin and the Wagtail Admin, and other project requirements git history has not been retained. -For installation and configuration instructions, -`check out the docs`_. +For installation and configuration instructions, keep reading. .. _robots exclusion protocol: http://en.wikipedia.org/wiki/Robots_exclusion_standard .. _Django: http://www.djangoproject.com/ .. _Sitemap contrib app: http://docs.djangoproject.com/en/dev/ref/contrib/sitemaps/ .. _Django Robots: https://github.com/jazzband/django-robots -.. _check out the docs: https://wagtail-robots.readthedocs.io + +Contents: + +.. toctree:: + :maxdepth: 1 + + screenshots + +Installation +============ + +Use your favorite Python installer to install it from PyPI:: + + pip install wagtail-robots + +Or get the source from the application site at:: + + http://github.com/adrian-turjak/wagtail-robots/ + +Then follow these steps: + +1. Add ``'wagtail_modeladmin'`` and ``'robots'`` to your INSTALLED_APPS_ setting. +2. Run the ``migrate`` management command + +You may want to additionally setup the `Wagtail sitemap generator`_. + +And if you install or already happen to be using `CondensedInlinePanel`_ this +library will automatically use it in place of InlinePanel for the Rule create +and edit pages. + +.. _INSTALLED_APPS: http://docs.djangoproject.com/en/dev/ref/settings/#installed-apps +.. _TEMPLATES: https://docs.djangoproject.com/en/dev/ref/settings/#templates +.. _Wagtail sitemap generator: http://docs.wagtail.io/en/latest/reference/contrib/sitemaps.html +.. _CondensedInlinePanel: https://github.com/wagtail/wagtail-condensedinlinepanel + +Initialization +============== + +To activate robots.txt generation on your Wagtail site, add this line to your +URLconf_:: + + re_path(r'^robots\.txt', include('robots.urls')), + +This tells Django to build a robots.txt when a robot accesses ``/robots.txt``. +Then, please migrate your database to create the necessary tables and create +``Rule`` objects in the admin interface or via the shell. + +.. _URLconf: http://docs.djangoproject.com/en/dev/topics/http/urls/ + +Rules +===== + +``Rule`` - defines an abstract rule which is used to respond to crawling web +robots, using the `robots exclusion protocol`_, a.k.a. robots.txt. + +You can link multiple URL pattern to allows or disallows the robot identified +by its user agent to access the given URLs. + +The crawl delay field is supported by some search engines and defines the +delay between successive crawler accesses in seconds. If the crawler rate is a +problem for your server, you can set the delay up to 5 or 10 or a comfortable +value for your server, but it's suggested to start with small values (0.5-1), +and increase as needed to an acceptable value for your server. Larger delay +values add more delay between successive crawl accesses and decrease the +maximum crawl rate to your web server. + +The Wagtail sites are used to enable multiple robots.txt per Wagtail instance. +If no rule exists it automatically allows every web robot access to every URL +except Wagtail's admin path (`/admin`). + +Please have a look at the `database of web robots`_ for a full list of +existing web robots user agent strings. + +.. _database of web robots: http://www.robotstxt.org/db.html + +URLs +==== + +``Url`` - defines a case-sensitive and exact URL pattern which is used to +allow or disallow the access for web robots. Case-sensitive. + +A missing trailing slash does also match files which start with the name of +the given pattern, e.g., ``'/admin'`` matches ``/admin.html`` too. + +Some major search engines allow an asterisk (``*``) as a wildcard to match any +sequence of characters and a dollar sign (``$``) to match the end of the URL, +e.g., ``'/*.jpg$'`` can be used to match all jpeg files. + +Caching +======= + +You can optionally cache the generation of the ``robots.txt``. Add or change +the ``ROBOTS_CACHE_TIMEOUT`` setting with a value in seconds in your Django +settings file:: + + ROBOTS_CACHE_TIMEOUT = 60*60*24 + +This tells Django to cache the ``robots.txt`` for 24 hours (86400 seconds). +The default value is ``None`` (no caching). + +If you need to, you can also specify exactly which cache to use:: + + ROBOTS_CACHE_ALIAS="robots" + +Unless specified otherwise it will use the ``default`` cache. + +Sitemaps +======== + +By default a ``Sitemap`` statement is automatically added to the resulting +robots.txt by reverse matching the URL of the installed `Wagtail Sitemap app`_. +This is especially useful if you allow every robot to access your whole site, +since it then gets URLs explicitly instead of searching every link. + +To change the default behaviour to omit the inclusion of a sitemap link, +change the ``ROBOTS_USE_SITEMAP`` setting in your Django settings file to:: + + ROBOTS_USE_SITEMAP = False + +In case you want to use specific sitemap URLs instead of the one that is +automatically discovered, change the ``ROBOTS_SITEMAP_URLS`` setting to:: + + ROBOTS_SITEMAP_URLS = [ + 'http://www.example.com/sitemap.xml', + ] + +If the sitemap is wrapped in a decorator, dotted path reverse to discover +the sitemap URL does not work. +To overcome this, provide a name to the sitemap instance in ``urls.py``:: + + urlpatterns = [ + ... + url(r'^sitemap.xml$', cache_page(60)(sitemap_view), {'sitemaps': [...]}, name='cached-sitemap'), + ... + ] + +and inform django-robots about the view name by adding the following setting:: + + ROBOTS_SITEMAP_VIEW_NAME = 'cached-sitemap' + +Use ``ROBOTS_SITEMAP_VIEW_NAME`` also if you use custom sitemap views. + +.. _Wagtail Sitemap app: http://docs.wagtail.io/en/latest/reference/contrib/sitemaps.html + +Host directive +============== +By default a ``Host`` statement is automatically added to the resulting +robots.txt to avoid mirrors and select the main website properly. + +To change the default behaviour to omit the inclusion of host directive, +change the ``ROBOTS_USE_HOST`` setting in your Django settings file to:: + + ROBOTS_USE_HOST = False + +if you want to prefix the domain with the current request protocol +(**http** or **https** as in ``Host: https://www.mysite.com``) add this setting:: + + ROBOTS_USE_SCHEME_IN_HOST = True + + +Development/Staging Override +============================ +Sometimes when you have duplicate database content in both a production and +staging website, it can be useful to override any and all database entries +for the this application and explicitly disallow all. + +To do that add this setting:: + + ROBOTS_DISALLOW_ALL = True + +The resulting `robots.txt` will look as follows:: + + User-agent: * + Disallow: / + + +Bugs and feature requests +========================= + +As always your mileage may vary, so please don't hesitate to send feature +requests and bug reports: + + https://github.com/adrian-turjak/wagtail-robots/issues \ No newline at end of file diff --git a/docs/source/index.rst b/docs/source/index.rst index bd53879..e9992a8 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -2,6 +2,10 @@ Wagtail Robots ============== +These docs are only relevant if using the latest version available on Pypi. + +If you are using the Torchbox fork, please refer to the README.rst file in the root of the repository. + This is a basic Django application for Wagtail to manage robots.txt files following the `robots exclusion protocol`_, complementing the Django_ `Sitemap contrib app`_. @@ -37,7 +41,14 @@ Or get the source from the application site at:: Then follow these steps: -1. Add ``'wagtail.contrib.modeladmin'`` and ``'robots'`` to your INSTALLED_APPS_ setting. +1. Add ``robots`` and ``wagtail_modeladmin`` to the ``INSTALLED_APPS`` setting in your project settings:: + + INSTALLED_APPS = [ + ... + 'wagtail_modeladmin', + 'robots', + ] + 2. Run the ``migrate`` management command You may want to additionally setup the `Wagtail sitemap generator`_. diff --git a/docs/source/static/1_in_menu.png b/docs/source/static/1_in_menu.png index 2f3f784..e8d769a 100644 Binary files a/docs/source/static/1_in_menu.png and b/docs/source/static/1_in_menu.png differ diff --git a/docs/source/static/2_index.png b/docs/source/static/2_index.png index 0ae956f..886fbc0 100644 Binary files a/docs/source/static/2_index.png and b/docs/source/static/2_index.png differ diff --git a/docs/source/static/3_create_edit.png b/docs/source/static/3_create_edit.png index 2646914..8eafb90 100644 Binary files a/docs/source/static/3_create_edit.png and b/docs/source/static/3_create_edit.png differ diff --git a/docs/source/static/4_create_edit_condensed.png b/docs/source/static/4_create_edit_condensed.png index c8ceb0b..3db1f05 100644 Binary files a/docs/source/static/4_create_edit_condensed.png and b/docs/source/static/4_create_edit_condensed.png differ diff --git a/docs/source/static/5_index_rule.png b/docs/source/static/5_index_rule.png index 359f391..87bff95 100644 Binary files a/docs/source/static/5_index_rule.png and b/docs/source/static/5_index_rule.png differ diff --git a/robots/models.py b/robots/models.py index 76c0364..d81c6bd 100644 --- a/robots/models.py +++ b/robots/models.py @@ -1,4 +1,4 @@ -import django +from six import u from django.db import models from django.utils.text import get_text_list @@ -7,20 +7,10 @@ from modelcluster.models import ClusterableModel from modelcluster.fields import ParentalKey -from robots.panels import WrappedInlinepanel +from wagtail.admin.panels import FieldPanel +from wagtail.models import Site -from wagtail import VERSION as WAGTAIL_VERSION -if WAGTAIL_VERSION >= (3, 0): - from wagtail.models import Site - from wagtail.admin.panels import FieldPanel -else: - from wagtail.core.models import Site - from wagtail.admin.edit_handlers import FieldPanel - -if django.VERSION >= (3, 0): - from six import u -else: - from django.utils.six import u +from robots.panels import WrappedInlinepanel class BaseUrl(models.Model): diff --git a/robots/panels.py b/robots/panels.py index 4cc121f..7aab41c 100644 --- a/robots/panels.py +++ b/robots/panels.py @@ -1,40 +1,36 @@ -from distutils.version import LooseVersion +from packaging.version import parse from django.conf import settings -from wagtail import VERSION as WAGTAIL_VERSION -if WAGTAIL_VERSION >= (3, 0): - from wagtail.admin.panels import InlinePanel -else: - from wagtail.admin.edit_handlers import InlinePanel +from wagtail.admin.panels import InlinePanel -def WrappedInlinepanel(relation_name, heading='', label='', - card_header_from_field=None, new_card_header_text='', - **kwargs): +def WrappedInlinepanel( + relation_name, + heading="", + label="", + card_header_from_field=None, + new_card_header_text="", + **kwargs, +): klass = InlinePanel - if 'condensedinlinepanel' in settings.INSTALLED_APPS: + if "condensedinlinepanel" in settings.INSTALLED_APPS: import condensedinlinepanel from condensedinlinepanel.edit_handlers import CondensedInlinePanel - if LooseVersion( - condensedinlinepanel.__version__) >= LooseVersion('0.3'): + + if parse(condensedinlinepanel.__version__) >= parse("0.3"): klass = CondensedInlinePanel defaults = { - 'heading': heading, - 'label': label, - 'card_header_from_field': card_header_from_field, - 'new_card_header_text': new_card_header_text, + "heading": heading, + "label": label, + "card_header_from_field": card_header_from_field, + "new_card_header_text": new_card_header_text, } else: - if WAGTAIL_VERSION >= (2, 0): - defaults = { - 'heading': heading, - 'label': label, - } - else: - defaults = { - 'label': label, - } + defaults = { + "heading": heading, + "label": label, + } defaults.update(kwargs) return klass(relation_name, **defaults) diff --git a/robots/test/__init__.py b/robots/test/__init__.py index c7b1474..e69de29 100644 --- a/robots/test/__init__.py +++ b/robots/test/__init__.py @@ -1 +0,0 @@ -default_app_config = "robots.test.apps.RobotsTestAppConfig" diff --git a/robots/test/settings.py b/robots/test/settings.py index fa62a0d..6407115 100644 --- a/robots/test/settings.py +++ b/robots/test/settings.py @@ -10,6 +10,8 @@ import os +from wagtail import VERSION as WAGTAIL_VERSION + # Build paths inside the project like this: os.path.join(PROJECT_DIR, ...) PROJECT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) BASE_DIR = os.path.dirname(PROJECT_DIR) @@ -42,11 +44,11 @@ "wagtail.search", "wagtail.admin", "wagtail.api.v2", - "wagtail.contrib.modeladmin", + "wagtail_modeladmin", "wagtail.contrib.routable_page", "wagtail.contrib.styleguide", "wagtail.sites", - "wagtail.core", + "wagtail", "taggit", "rest_framework", "django.contrib.admin", @@ -125,8 +127,6 @@ USE_I18N = True -USE_L10N = True - USE_TZ = True diff --git a/robots/test/urls.py b/robots/test/urls.py index 90a6f9b..d158628 100644 --- a/robots/test/urls.py +++ b/robots/test/urls.py @@ -3,7 +3,7 @@ from wagtail.admin import urls as wagtailadmin_urls from wagtail.documents import urls as wagtaildocs_urls -from wagtail.core import urls as wagtail_urls +from wagtail import urls as wagtail_urls urlpatterns = [ path("django-admin/", admin.site.urls), diff --git a/robots/views.py b/robots/views.py index 0453e94..86acbbb 100644 --- a/robots/views.py +++ b/robots/views.py @@ -1,24 +1,14 @@ -import django from django.db.models import Q from django.views.decorators.cache import cache_page from django.views.generic import ListView +from django.urls import NoReverseMatch, reverse + +from wagtail.contrib.sitemaps.views import sitemap +from wagtail.models import Site from robots import settings from robots.models import Rule -if django.VERSION[:2] >= (2, 0): - from django.urls import NoReverseMatch, reverse -else: - from django.core.urlresolvers import NoReverseMatch, reverse - -from wagtail import VERSION as WAGTAIL_VERSION # noqa -if WAGTAIL_VERSION >= (3, 0): - from wagtail.models import Site -else: - from wagtail.core.models import Site - -from wagtail.contrib.sitemaps.views import sitemap - class RuleList(ListView): """ diff --git a/robots/wagtail_hooks.py b/robots/wagtail_hooks.py index e8b32e1..1c64753 100644 --- a/robots/wagtail_hooks.py +++ b/robots/wagtail_hooks.py @@ -1,5 +1,5 @@ -from wagtail.contrib.modeladmin.options import ModelAdmin, modeladmin_register +from wagtail_modeladmin.options import ModelAdmin, modeladmin_register from robots.models import Rule diff --git a/setup.py b/setup.py index 9db3099..6175bff 100644 --- a/setup.py +++ b/setup.py @@ -4,6 +4,12 @@ with open('README.rst') as file: long_description = file.read() +testing_extras = [ + "flake8>=7.3.0", + "sphinx>=7.4.7", + "sphinx-rtd-theme>=3.0.2", + "doc8", +] setup( name='wagtail-robots', @@ -20,8 +26,12 @@ ], }, install_requires=[ - 'wagtail>=2.15', + 'wagtail>=6.3', + 'wagtail-modeladmin>=2.2', + 'six>=1.17.0', + 'packaging>=25.0', ], + extras_require={"testing": testing_extras}, classifiers=[ 'Development Status :: 3 - Alpha', 'Environment :: Web Environment', @@ -33,16 +43,16 @@ 'Topic :: Internet :: WWW/HTTP :: Dynamic Content', 'Topic :: Software Development', 'Topic :: Software Development :: Libraries :: Application Frameworks', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: 3.8', - 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', + 'Programming Language :: Python :: 3.13', + 'Programming Language :: Python :: 3.14', 'Framework :: Django', - 'Framework :: Django :: 3.2', - 'Framework :: Django :: 4.0', - 'Framework :: Django :: 4.1', - 'Framework :: Wagtail :: 2', - 'Framework :: Wagtail :: 3', - 'Framework :: Wagtail :: 4', + 'Framework :: Django :: 4.2', + 'Framework :: Django :: 5.1', + 'Framework :: Django :: 5.2', + 'Framework :: Wagtail :: 6', + 'Framework :: Wagtail :: 7', ] ) diff --git a/test-requirements.txt b/test-requirements.txt deleted file mode 100644 index 490eb19..0000000 --- a/test-requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -flake8>=3.0.4 -sphinx!=1.6.1,>=1.5.1 -sphinx-rtd-theme>=0.2.4 -doc8 diff --git a/testmanage.py b/testmanage.py index 85c8cdc..5e64582 100755 --- a/testmanage.py +++ b/testmanage.py @@ -56,7 +56,7 @@ def runtests(): try: execute_from_command_line(argv) finally: - from wagtail.tests.settings import STATIC_ROOT, MEDIA_ROOT + from wagtail.test.settings import STATIC_ROOT, MEDIA_ROOT shutil.rmtree(STATIC_ROOT, ignore_errors=True) shutil.rmtree(MEDIA_ROOT, ignore_errors=True) diff --git a/tox.ini b/tox.ini index 1ff205d..91ef4c5 100644 --- a/tox.ini +++ b/tox.ini @@ -1,14 +1,38 @@ [tox] skipsdist = True usedevelop = True +skip_missing_interpreters = True envlist = - py{37}-dj{32}-wt{215,216} - py{38,39,310}-dj{32,40}-wt{216,30,40} - py{39,310}-dj{41}-wt{40,41} + python{3.10,3.11}-django4.2-wagtail{6.3,7.0,7.2} + python{3.11,3.12,3.13}-django{5.1,5.2}-wagtail{6.3,7.0,7.2} + python3.14-django5.2-wagtail7.2 + +[gh-actions] +python = + 3.10: python3.10 + 3.11: python3.11 + 3.12: python3.12 + 3.13: python3.13 + 3.14: python3.14 [testenv] -deps = -r{toxinidir}/test-requirements.txt +deps = + django4.2: Django>=4.2,<4.3 + django5.1: Django>=5.1,<5.2 + django5.2: Django>=5.2,<5.3 + + wagtail6.3: wagtail>=6.3,<6.4 + wagtail7.0: wagtail>=7.0,<7.1 + wagtail7.2: wagtail>=7.2,<7.3 + +basepython = + py310: python3.10 + py311: python3.11 + py312: python3.12 + py313: python3.13 + py314: python3.14 + setenv = VIRTUAL_ENV={envdir} install_command = pip install -e ".[testing]" -U {opts} {packages} commands =