Test cases added.

.github/workflows/execute-tests.yml

@@ -2,11 +2,8 @@ name: Execute tests
 on:
   workflow_call:
     inputs:
-      architecture:
-        required: true
-        type: string
       basepath:
-        required: true
+        required: false
         type: string
       os:
         required: true
@@ -29,24 +26,22 @@ jobs:
         working-directory: ${{ inputs.workpath }}
 
     steps:
-      - name: Checkout Unix
-        if: ${{ inputs.os != 'windows-latest' }}
-        run: |
-          mkdir -p ${{ inputs.workpath }}
-          git clone https://github.com/${{ github.repository }} ${{ inputs.workpath }} --depth 1 --branch $GITHUB_REF_NAME
-      - name: Checkout Windows
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Change path on Windows
+        if: ${{ inputs.os == 'windows-latest' }}
         # Cannot start powershell from a path that does not exist, so change
         # working directory for this step only.
         working-directory: ${{ inputs.basepath }}
-        if: ${{ inputs.os == 'windows-latest' }}
         run: |
           mkdir -p ${{ inputs.workpath }}
-          git clone https://github.com/${{ github.repository }} ${{ inputs.workpath }} --depth 1 --branch $env:GITHUB_REF_NAME
+          mv $env:GITHUB_WORKSPACE\* ${{ inputs.workpath }}\ -Force
 
       - uses: actions/setup-python@v2
         with:
           python-version: ${{ matrix.python-version }}
-          architecture: ${{ inputs.architecture }}
+          architecture: x64
 
       - uses: Gr1N/setup-poetry@v7
 
@@ -61,11 +56,29 @@ jobs:
       - name: Install project dependencies
         run: make install
 
-      - name: Run tests
+      # Some tests fails intermittently, likely due to the public runners being
+      # very slow. Especially any client/server tests seems to be problematic.
+      # This is a simple attempt to re-run the tests up to three times if they
+      # fail. Does not add any execution time if successful.
+      - name: Run tests attempt 1
+        run: make test
+      - name: Run tests attempt 2
+        if: ${{ failure() }}
         run: make test
+      - name: Run tests attempt 3
+        if: ${{ failure() }}
+        run: make test
+
+      - name: Upload coverage
+        uses: codecov/codecov-action@v1
+        if: ${{ inputs.os == 'ubuntu-latest' && matrix.python-version == '3.9' && github.repository == 'doorstop-dev/doorstop' }}
+        with:
+          fail_ci_if_error: true
 
       - name: Run checks
         run: make check
+        if: ${{ inputs.os == 'ubuntu-latest' }}
 
       - name: Run demo
         run: make demo
+        if: ${{ inputs.os == 'ubuntu-latest' }}

.github/workflows/test-linux.yml

@@ -1,12 +1,13 @@
 name: Linux
 
-on: push
+on:
+  push:
+  pull_request:
+    branches: [ develop ]
 
 jobs:
   Test:
     uses: ./.github/workflows/execute-tests.yml
     with:
-      architecture: x64
-      basepath: "/home/runner/work"
       os: "ubuntu-latest"
       workpath: "/home/runner/work/doorstop/doorstop"

.github/workflows/test-osx.yml

@@ -1,12 +1,13 @@
 name: macOS
 
-on: push
+on:
+  push:
+  pull_request:
+    branches: [ develop ]
 
 jobs:
   Test:
     uses: ./.github/workflows/execute-tests.yml
     with:
-      architecture: x64
-      basepath: "/Users/runner/work"
       os: "macos-latest"
       workpath: "/Users/runner/work/doorstop/doorstop"

.github/workflows/test-windows.yml

@@ -1,14 +1,14 @@
 name: Windows
 
-on: push
+on:
+  push:
+  pull_request:
+    branches: [ develop ]
 
 jobs:
-
-
   Test:
     uses: ./.github/workflows/execute-tests.yml
     with:
-      architecture: x64
       basepath: 'D:\'
       os: "windows-latest"
       workpath: 'C:\a\doorstop\doorstop'

.gitignore

@@ -22,6 +22,7 @@ Icon*
 /site/
 /*.html
 /docs/*.png
+texput.log
 
 # Google Drive
 *.gdoc
@@ -37,6 +38,7 @@ Icon*
 /pyunit.xml
 /tmp/
 *.tmp
+coverage.xml
 
 # Build and release directories
 /build/

.vscode/settings.json

@@ -18,6 +18,7 @@
     "choco",
     "clevel",
     "cname",
+    "codecov",
     "codeql",
     "cors",
     "curr",

CHANGELOG.md

@@ -3,6 +3,7 @@
 - **BREAKING:** Dropped support for Python 3.6.
 - **BREAKING:** Removed `--no-body-levels` option to `doorstop publish`.
 - Fixed overriding default attributes with `doorstop add`. ([@tangoalx](https://github.com/doorstop-dev/doorstop/pull/544))
+- Fixed encoding error with special characters on Windows. ([@urbasus](https://github.com/doorstop-dev/doorstop/pull/526))
 - Add support for publishing to the LaTeX format. ([@neerdoc](https://github.com/doorstop-dev/doorstop/pull/545))
 
 # 2.2 (2022-01-22)

Makefile

@@ -82,6 +82,9 @@ PYTEST_OPTIONS := --doctest-modules
 ifndef DISABLE_COVERAGE
 PYTEST_OPTIONS += --cov=$(PACKAGE) --cov-report=html --cov-report=term-missing
 endif
+ifdef CI
+PYTEST_OPTIONS += --cov-report=xml
+endif
 
 .PHONY: test
 test: test-all ## Run unit and integration tests
@@ -147,7 +150,7 @@ DOORSTOP := poetry run doorstop
 YAML := $(wildcard */*.yml */*/*.yml */*/*/*/*.yml)
 
 .PHONY: reqs
-reqs: doorstop reqs-html reqs-latex reqs-md reqs-txt
+reqs: doorstop reqs-html reqs-latex reqs-md reqs-pdf reqs-txt
 
 .PHONY: reqs-html
 reqs-html: install docs/gen/*.html
@@ -164,6 +167,10 @@ reqs-md: install docs/gen/*.md
 docs/gen/*.md: $(YAML)
 	$(DOORSTOP) publish all docs/gen --markdown
 
+.PHONY: reqs-pdf
+reqs-pdf: reqs-latex
+	cd docs/gen && ./compile.sh
+
 .PHONY: reqs-txt
 reqs-txt: install docs/gen/*.txt
 docs/gen/*.txt: $(YAML)

README.md

@@ -2,7 +2,7 @@
 [![Mac OS X Test](https://github.com/doorstop-dev/doorstop/actions/workflows/test-osx.yml/badge.svg)](https://github.com/doorstop-dev/doorstop/actions/workflows/test-osx.yml)
 [![Windows Test](https://github.com/doorstop-dev/doorstop/actions/workflows/test-windows.yml/badge.svg)](https://github.com/doorstop-dev/doorstop/actions/workflows/test-windows.yml)
 <br>
-[![Coverage Status](http://img.shields.io/coveralls/doorstop-dev/doorstop/develop.svg)](https://coveralls.io/r/doorstop-dev/doorstop)
+[![Coverage Status](https://img.shields.io/codecov/c/gh/doorstop-dev/doorstop)](https://codecov.io/gh/doorstop-dev/doorstop)
 [![Scrutinizer Code Quality](http://img.shields.io/scrutinizer/g/doorstop-dev/doorstop.svg)](https://scrutinizer-ci.com/g/doorstop-dev/doorstop/?branch=develop)
 [![PyPI Version](http://img.shields.io/pypi/v/Doorstop.svg)](https://pypi.org/project/Doorstop)
 <br>

doorstop/common.py

@@ -143,7 +143,7 @@ def load_yaml(text, path, loader=yaml.SafeLoader):
     return data
 
 
-def write_lines(lines, path, end="\n", encoding="utf-8"):
+def write_lines(lines, path, end="\n", encoding="utf-8", *, executable=False):
     """Write lines of text to a file.
 
     :param lines: iterator of strings
@@ -159,6 +159,8 @@ def write_lines(lines, path, end="\n", encoding="utf-8"):
         for line in lines:
             data = (line + end).encode(encoding)
             stream.write(data)
+    if executable and os.path.isfile(path):
+        os.chmod(path, 0o775)
     return path
 
 

doorstop/core/publisher.py

@@ -222,7 +222,7 @@ def publish(
             log.info("Copied assets from %s to %s", obj.assets, assets_dir)
 
     if ext == ".tex":
-        common.write_lines(compile_files, compile_path)
+        common.write_lines(compile_files, compile_path, executable=True)
         msg = "You can now execute the file 'compile.sh' twice in the exported folder to produce the PDFs!"
         utilities.show(msg, flush=True)
 

doorstop/core/tests/test_fixtures/002-utf8-characters/REQ-CYRILLIC.yml

@@ -0,0 +1,16 @@
+active: true
+derived: false
+header: |
+  UTF-8
+level: 1.7.0
+links: []
+normative: true
+ref: ''
+reviewed: null
+text: |
+  UTF-8 is supported. Verified by this file.
+
+  première is first
+  première is slightly different
+  Кириллица is Cyrillic
+  𐐀 am Deseret

doorstop/core/tests/test_fixtures/002-utf8-characters/REQ-MIT.yml

@@ -0,0 +1,19 @@
+active: true
+derived: false
+header: |
+  MIT licence
+level: 1.7.1
+links: []
+normative: true
+ref: ''
+reviewed: null
+text: |
+  Just verify that the MIT licence is typeset correctly.
+
+  Copyright © 2022 <copyright holders>
+
+  Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+  The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+  THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

doorstop/core/tests/test_item.py

@@ -1050,3 +1050,31 @@ def test_attributes_with_spec(self, mock_warning):
     def test_stamp(self):
         """Verify an unknown item has no stamp."""
         self.assertEqual(Stamp(None), self.item.stamp())
+
+
+class TestUTF8(unittest.TestCase):
+    """Unit tests for reading UTF8 formatted files."""
+
+    def test_load_cyrillic(self):
+        """Verify that cyrillic and other UTF-8 characters are correltly loaded and written."""
+        ITEM = "doorstop/core/tests/test_fixtures/002-utf8-characters/REQ-CYRILLIC.yml"
+        backup = common.read_text(ITEM)
+        item = Item(None, ITEM)
+        item.load()
+        item.save()
+        text = common.read_text(ITEM)
+        self.maxDiff = None
+        common.write_text(backup, ITEM)
+        self.assertEqual(backup, text)
+
+    def test_load_mit(self):
+        """Verify that an MIT licence is correltly loaded and written."""
+        ITEM = "doorstop/core/tests/test_fixtures/002-utf8-characters/REQ-MIT.yml"
+        backup = common.read_text(ITEM)
+        item = Item(None, ITEM)
+        item.load()
+        item.save()
+        text = common.read_text(ITEM)
+        self.maxDiff = None
+        common.write_text(backup, ITEM)
+        self.assertEqual(backup, text)

doorstop/server/tests/test_all.py

@@ -9,6 +9,8 @@
 from multiprocessing import Process
 from unittest.mock import patch
 
+import requests
+
 from doorstop import server
 from doorstop.server import main
 from doorstop.server.tests import ENV, REASON
@@ -24,8 +26,16 @@ def setUpClass(cls):
         if os.getenv(ENV):
             cls.process = Process(target=main.main, kwargs={"args": []})
             cls.process.start()
-            logging.info("delaying for the server to initialize...")
-            time.sleep(3)
+            logging.info("waiting for the server to initialize...")
+            # Check for response!
+            url = "http://localhost:7867/documents"
+            for _ in range(0, 29):
+                try:
+                    _ = requests.head(url)
+                    break
+                except requests.exceptions.RequestException:
+                    time.sleep(1)
+            logging.info("server is answering!")
             assert cls.process.is_alive()
 
     @classmethod