Skip to content

feat: Add setting to add site map to robots.txt #76

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
justinmayer merged 4 commits into from
Apr 25, 2025
Merged

feat: Add setting to add site map to robots.txt #76

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
cee187f
Fixed sitemap
max-pfeiffer Mar 20, 2025
63a2397
Updated docs explaining the new feature
max-pfeiffer Mar 21, 2025
d07b628
Tweak README & CHANGELOG
justinmayer Apr 25, 2025
8b08ca1
Prepare release
justinmayer Apr 25, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ CHANGELOG
1.3.0 - 2025-01-15
------------------

- Add ability to configure the plugin via the Pelican settings file
Add ability to configure the plugin via the Pelican settings file

1.2.2 - 2021-06-21
------------------
Expand Down
7 changes: 7 additions & 0 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,13 @@ $ pelican content --verbose
Done: Processed 1 articles, 0 drafts, 0 pages, 0 hidden pages and 0 draft pages in 0.17 seconds.
```

If you want to include a site map in your `robots.txt` file, which
[Google recommends](https://developers.google.com/search/docs/crawling-indexing/robots/create-robots-txt),
you can specify the appropriate URL for your site map XML file via the following setting:
```python
SEO_ENHANCER_SITEMAP_URL = "https://www.example.com/sitemap.xml"
```

### SEO Report

The SEO plugin analyzes all your articles and pages and generate an SEO HTML report in your Pelican project root: `seo-report.html`
Expand Down
3 changes: 3 additions & 0 deletions RELEASE.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
Release type: minor

Add setting to add site map to `robots.txt`
2 changes: 2 additions & 0 deletions pelican/plugins/seo/seo.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@ def get_plugin_settings(context):
- SEO_ENHANCER
- SEO_ENHANCER_OPEN_GRAPH
- SEO_ENHANCER_TWITTER_CARDS
- SEO_ENHANCER_SITEMAP_URL

:return: Dictionary of settings for the plugin
:rtype: <dict>
Expand Down Expand Up @@ -130,6 +131,7 @@ def run_robots_file(generators):
seo_enhancer.generate_robots(
rules=robots_rules,
output_path=output_path,
sitemap_url=plugin_settings["SEO_ENHANCER_SITEMAP_URL"],
)


Expand Down
4 changes: 3 additions & 1 deletion pelican/plugins/seo/seo_enhancer/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ def populate_robots(self, document):
"disallow": robots_file.get_disallow,
}

def generate_robots(self, rules, output_path):
def generate_robots(self, rules, output_path, sitemap_url=None):
"""Create robots.txt, with noindex and disallow rules for each document URL."""
if not os.path.isdir(output_path):
os.mkdir(output_path)
Expand All @@ -78,6 +78,8 @@ def generate_robots(self, rules, output_path):
robots_file.write("\n" + "Noindex: " + rule.get("document_url"))
if rule.get("disallow"):
robots_file.write("\n" + "Disallow: " + rule.get("document_url"))
if sitemap_url:
robots_file.write("\n" + "Sitemap: " + sitemap_url)

logger.info("SEO plugin - SEO Enhancement: robots.txt file created")

Expand Down
1 change: 1 addition & 0 deletions pelican/plugins/seo/settings.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
SEO_ENHANCER = False
SEO_ENHANCER_OPEN_GRAPH = False
SEO_ENHANCER_TWITTER_CARDS = False
SEO_ENHANCER_SITEMAP_URL = None

SEO_ARTICLES_LIMIT = 10
SEO_PAGES_LIMIT = 10
24 changes: 24 additions & 0 deletions pelican/plugins/seo/tests/test_run_robots_file.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
from seo.seo import run_robots_file
import pytest
from unittest.mock import patch, call, MagicMock


def test_run_robots_file():
class FakeGenerator:
context = {
"SEO_ENHANCER": True,
"SEO_ENHANCER_SITEMAP_URL": "https://www.example.com/sitemap.xml",
}
output_path = "foo"

with patch("seo.seo.SEOEnhancer") as patched_seo_enhancer:
run_robots_file([FakeGenerator()])

assert (
call().generate_robots(
rules=[],
output_path="foo",
sitemap_url="https://www.example.com/sitemap.xml",
)
in patched_seo_enhancer.mock_calls
)
19 changes: 19 additions & 0 deletions pelican/plugins/seo/tests/test_seo_enhancer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
"""Units tests for SEO Enhancer."""

from pathlib import Path
from tempfile import TemporaryDirectory
from unittest.mock import mock_open, patch

import pytest
Expand Down Expand Up @@ -42,6 +44,23 @@ def test_generate_robots_file(self, fake_seo_enhancer, fake_robots_rules):
fake_rule = args[0]
assert "Noindex: fake-title.html" in fake_rule

def test_generate_robots_file_with_sitemap_url(
self, fake_seo_enhancer, fake_robots_rules
):
"""Test if generate_robots create a robots.txt file by mocking open()."""
sitemap_url = "https://www.example.com/sitemap.xml"

with TemporaryDirectory() as tmp_dir_name:
fake_seo_enhancer.generate_robots(
rules=fake_robots_rules,
output_path=tmp_dir_name,
sitemap_url=sitemap_url,
)
robots_txt_path = Path(tmp_dir_name) / "robots.txt"
contents = robots_txt_path.read_text()

assert sitemap_url in contents

@pytest.mark.parametrize("open_graph", (True, False))
def test_launch_html_enhancer_returns_dict(
self, fake_article, fake_seo_enhancer, open_graph
Expand Down
4 changes: 2 additions & 2 deletions pelican/plugins/seo/tests/test_settings.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ def test_get_settings():
settings = get_plugin_settings(context=pelican_context)

assert (
len(settings) == 8
len(settings) == 9
) # 6 in the plugin settings file + 2 from the Pelican context

# Let's define a setting in Pelican context that
Expand All @@ -21,5 +21,5 @@ def test_get_settings():

settings = get_plugin_settings(context={"SEO_REPORT": False})

assert len(settings) == 6
assert len(settings) == 7
assert settings["SEO_REPORT"] is False