diff --git a/.github/workflows/execute-tests.yml b/.github/workflows/execute-tests.yml index 793ae5196..df09c291e 100644 --- a/.github/workflows/execute-tests.yml +++ b/.github/workflows/execute-tests.yml @@ -2,11 +2,8 @@ name: Execute tests on: workflow_call: inputs: - architecture: - required: true - type: string basepath: - required: true + required: false type: string os: required: true @@ -29,24 +26,22 @@ jobs: working-directory: ${{ inputs.workpath }} steps: - - name: Checkout Unix - if: ${{ inputs.os != 'windows-latest' }} - run: | - mkdir -p ${{ inputs.workpath }} - git clone https://github.com/${{ github.repository }} ${{ inputs.workpath }} --depth 1 --branch $GITHUB_REF_NAME - - name: Checkout Windows + - name: Checkout + uses: actions/checkout@v2 + + - name: Change path on Windows + if: ${{ inputs.os == 'windows-latest' }} # Cannot start powershell from a path that does not exist, so change # working directory for this step only. working-directory: ${{ inputs.basepath }} - if: ${{ inputs.os == 'windows-latest' }} run: | mkdir -p ${{ inputs.workpath }} - git clone https://github.com/${{ github.repository }} ${{ inputs.workpath }} --depth 1 --branch $env:GITHUB_REF_NAME + mv $env:GITHUB_WORKSPACE\* ${{ inputs.workpath }}\ -Force - uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - architecture: ${{ inputs.architecture }} + architecture: x64 - uses: Gr1N/setup-poetry@v7 @@ -61,11 +56,29 @@ jobs: - name: Install project dependencies run: make install - - name: Run tests + # Some tests fails intermittently, likely due to the public runners being + # very slow. Especially any client/server tests seems to be problematic. + # This is a simple attempt to re-run the tests up to three times if they + # fail. Does not add any execution time if successful. + - name: Run tests attempt 1 + run: make test + - name: Run tests attempt 2 + if: ${{ failure() }} run: make test + - name: Run tests attempt 3 + if: ${{ failure() }} + run: make test + + - name: Upload coverage + uses: codecov/codecov-action@v1 + if: ${{ inputs.os == 'ubuntu-latest' && matrix.python-version == '3.9' && github.repository == 'doorstop-dev/doorstop' }} + with: + fail_ci_if_error: true - name: Run checks run: make check + if: ${{ inputs.os == 'ubuntu-latest' }} - name: Run demo run: make demo + if: ${{ inputs.os == 'ubuntu-latest' }} diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index 5779b8590..4bcfc716b 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -1,12 +1,13 @@ name: Linux -on: push +on: + push: + pull_request: + branches: [ develop ] jobs: Test: uses: ./.github/workflows/execute-tests.yml with: - architecture: x64 - basepath: "/home/runner/work" os: "ubuntu-latest" workpath: "/home/runner/work/doorstop/doorstop" diff --git a/.github/workflows/test-osx.yml b/.github/workflows/test-osx.yml index ba8d96288..0ef364b5f 100644 --- a/.github/workflows/test-osx.yml +++ b/.github/workflows/test-osx.yml @@ -1,12 +1,13 @@ name: macOS -on: push +on: + push: + pull_request: + branches: [ develop ] jobs: Test: uses: ./.github/workflows/execute-tests.yml with: - architecture: x64 - basepath: "/Users/runner/work" os: "macos-latest" workpath: "/Users/runner/work/doorstop/doorstop" diff --git a/.github/workflows/test-windows.yml b/.github/workflows/test-windows.yml index 6c0e3acea..fdb26b214 100644 --- a/.github/workflows/test-windows.yml +++ b/.github/workflows/test-windows.yml @@ -1,14 +1,14 @@ name: Windows -on: push +on: + push: + pull_request: + branches: [ develop ] jobs: - - Test: uses: ./.github/workflows/execute-tests.yml with: - architecture: x64 basepath: 'D:\' os: "windows-latest" workpath: 'C:\a\doorstop\doorstop' diff --git a/.gitignore b/.gitignore index 6a379fba4..2650cc9a7 100644 --- a/.gitignore +++ b/.gitignore @@ -22,6 +22,7 @@ Icon* /site/ /*.html /docs/*.png +texput.log # Google Drive *.gdoc @@ -37,6 +38,7 @@ Icon* /pyunit.xml /tmp/ *.tmp +coverage.xml # Build and release directories /build/ diff --git a/.vscode/settings.json b/.vscode/settings.json index c3794bedb..38b54e527 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -18,6 +18,7 @@ "choco", "clevel", "cname", + "codecov", "codeql", "cors", "curr", diff --git a/CHANGELOG.md b/CHANGELOG.md index 25ab6639f..3be494e8a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ - **BREAKING:** Dropped support for Python 3.6. - **BREAKING:** Removed `--no-body-levels` option to `doorstop publish`. - Fixed overriding default attributes with `doorstop add`. ([@tangoalx](https://github.com/doorstop-dev/doorstop/pull/544)) +- Fixed encoding error with special characters on Windows. ([@urbasus](https://github.com/doorstop-dev/doorstop/pull/526)) - Add support for publishing to the LaTeX format. ([@neerdoc](https://github.com/doorstop-dev/doorstop/pull/545)) # 2.2 (2022-01-22) diff --git a/Makefile b/Makefile index f02059d94..7899a6173 100644 --- a/Makefile +++ b/Makefile @@ -82,6 +82,9 @@ PYTEST_OPTIONS := --doctest-modules ifndef DISABLE_COVERAGE PYTEST_OPTIONS += --cov=$(PACKAGE) --cov-report=html --cov-report=term-missing endif +ifdef CI +PYTEST_OPTIONS += --cov-report=xml +endif .PHONY: test test: test-all ## Run unit and integration tests @@ -147,7 +150,7 @@ DOORSTOP := poetry run doorstop YAML := $(wildcard */*.yml */*/*.yml */*/*/*/*.yml) .PHONY: reqs -reqs: doorstop reqs-html reqs-latex reqs-md reqs-txt +reqs: doorstop reqs-html reqs-latex reqs-md reqs-pdf reqs-txt .PHONY: reqs-html reqs-html: install docs/gen/*.html @@ -164,6 +167,10 @@ reqs-md: install docs/gen/*.md docs/gen/*.md: $(YAML) $(DOORSTOP) publish all docs/gen --markdown +.PHONY: reqs-pdf +reqs-pdf: reqs-latex + cd docs/gen && ./compile.sh + .PHONY: reqs-txt reqs-txt: install docs/gen/*.txt docs/gen/*.txt: $(YAML) diff --git a/README.md b/README.md index 4bc57762e..f9f78a6a4 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ [![Mac OS X Test](https://github.com/doorstop-dev/doorstop/actions/workflows/test-osx.yml/badge.svg)](https://github.com/doorstop-dev/doorstop/actions/workflows/test-osx.yml) [![Windows Test](https://github.com/doorstop-dev/doorstop/actions/workflows/test-windows.yml/badge.svg)](https://github.com/doorstop-dev/doorstop/actions/workflows/test-windows.yml)
-[![Coverage Status](http://img.shields.io/coveralls/doorstop-dev/doorstop/develop.svg)](https://coveralls.io/r/doorstop-dev/doorstop) +[![Coverage Status](https://img.shields.io/codecov/c/gh/doorstop-dev/doorstop)](https://codecov.io/gh/doorstop-dev/doorstop) [![Scrutinizer Code Quality](http://img.shields.io/scrutinizer/g/doorstop-dev/doorstop.svg)](https://scrutinizer-ci.com/g/doorstop-dev/doorstop/?branch=develop) [![PyPI Version](http://img.shields.io/pypi/v/Doorstop.svg)](https://pypi.org/project/Doorstop)
diff --git a/doorstop/common.py b/doorstop/common.py index ce335b08f..a7acc3cc3 100644 --- a/doorstop/common.py +++ b/doorstop/common.py @@ -143,7 +143,7 @@ def load_yaml(text, path, loader=yaml.SafeLoader): return data -def write_lines(lines, path, end="\n", encoding="utf-8"): +def write_lines(lines, path, end="\n", encoding="utf-8", *, executable=False): """Write lines of text to a file. :param lines: iterator of strings @@ -159,6 +159,8 @@ def write_lines(lines, path, end="\n", encoding="utf-8"): for line in lines: data = (line + end).encode(encoding) stream.write(data) + if executable and os.path.isfile(path): + os.chmod(path, 0o775) return path diff --git a/doorstop/core/publisher.py b/doorstop/core/publisher.py index 6dc6c4cb8..e87165bd1 100644 --- a/doorstop/core/publisher.py +++ b/doorstop/core/publisher.py @@ -222,7 +222,7 @@ def publish( log.info("Copied assets from %s to %s", obj.assets, assets_dir) if ext == ".tex": - common.write_lines(compile_files, compile_path) + common.write_lines(compile_files, compile_path, executable=True) msg = "You can now execute the file 'compile.sh' twice in the exported folder to produce the PDFs!" utilities.show(msg, flush=True) diff --git a/doorstop/core/tests/test_fixtures/002-utf8-characters/REQ-CYRILLIC.yml b/doorstop/core/tests/test_fixtures/002-utf8-characters/REQ-CYRILLIC.yml new file mode 100644 index 000000000..6417fce96 --- /dev/null +++ b/doorstop/core/tests/test_fixtures/002-utf8-characters/REQ-CYRILLIC.yml @@ -0,0 +1,16 @@ +active: true +derived: false +header: | + UTF-8 +level: 1.7.0 +links: [] +normative: true +ref: '' +reviewed: null +text: | + UTF-8 is supported. Verified by this file. + + première is first + première is slightly different + Кириллица is Cyrillic + 𐐀 am Deseret diff --git a/doorstop/core/tests/test_fixtures/002-utf8-characters/REQ-MIT.yml b/doorstop/core/tests/test_fixtures/002-utf8-characters/REQ-MIT.yml new file mode 100644 index 000000000..52cdd877e --- /dev/null +++ b/doorstop/core/tests/test_fixtures/002-utf8-characters/REQ-MIT.yml @@ -0,0 +1,19 @@ +active: true +derived: false +header: | + MIT licence +level: 1.7.1 +links: [] +normative: true +ref: '' +reviewed: null +text: | + Just verify that the MIT licence is typeset correctly. + + Copyright © 2022 + + Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/doorstop/core/tests/test_item.py b/doorstop/core/tests/test_item.py index c50d1b3e7..db1c0dff3 100644 --- a/doorstop/core/tests/test_item.py +++ b/doorstop/core/tests/test_item.py @@ -1050,3 +1050,31 @@ def test_attributes_with_spec(self, mock_warning): def test_stamp(self): """Verify an unknown item has no stamp.""" self.assertEqual(Stamp(None), self.item.stamp()) + + +class TestUTF8(unittest.TestCase): + """Unit tests for reading UTF8 formatted files.""" + + def test_load_cyrillic(self): + """Verify that cyrillic and other UTF-8 characters are correltly loaded and written.""" + ITEM = "doorstop/core/tests/test_fixtures/002-utf8-characters/REQ-CYRILLIC.yml" + backup = common.read_text(ITEM) + item = Item(None, ITEM) + item.load() + item.save() + text = common.read_text(ITEM) + self.maxDiff = None + common.write_text(backup, ITEM) + self.assertEqual(backup, text) + + def test_load_mit(self): + """Verify that an MIT licence is correltly loaded and written.""" + ITEM = "doorstop/core/tests/test_fixtures/002-utf8-characters/REQ-MIT.yml" + backup = common.read_text(ITEM) + item = Item(None, ITEM) + item.load() + item.save() + text = common.read_text(ITEM) + self.maxDiff = None + common.write_text(backup, ITEM) + self.assertEqual(backup, text) diff --git a/doorstop/server/tests/test_all.py b/doorstop/server/tests/test_all.py index bf70f8109..38ac7bf51 100644 --- a/doorstop/server/tests/test_all.py +++ b/doorstop/server/tests/test_all.py @@ -9,6 +9,8 @@ from multiprocessing import Process from unittest.mock import patch +import requests + from doorstop import server from doorstop.server import main from doorstop.server.tests import ENV, REASON @@ -24,8 +26,16 @@ def setUpClass(cls): if os.getenv(ENV): cls.process = Process(target=main.main, kwargs={"args": []}) cls.process.start() - logging.info("delaying for the server to initialize...") - time.sleep(3) + logging.info("waiting for the server to initialize...") + # Check for response! + url = "http://localhost:7867/documents" + for _ in range(0, 29): + try: + _ = requests.head(url) + break + except requests.exceptions.RequestException: + time.sleep(1) + logging.info("server is answering!") assert cls.process.is_alive() @classmethod diff --git a/poetry.lock b/poetry.lock index ce8c3ee30..afe3259fa 100644 --- a/poetry.lock +++ b/poetry.lock @@ -285,14 +285,14 @@ altgraph = ">=0.15" [[package]] name = "markdown" -version = "3.3.3" +version = "3.3.6" description = "Python implementation of Markdown." category = "main" optional = false python-versions = ">=3.6" [package.dependencies] -importlib-metadata = {version = "*", markers = "python_version < \"3.8\""} +importlib-metadata = {version = ">=4.4", markers = "python_version < \"3.10\""} [package.extras] testing = ["coverage", "pyyaml"] @@ -612,6 +612,19 @@ category = "dev" optional = false python-versions = ">=3.6,<4.0" +[[package]] +name = "pytest-sugar" +version = "0.9.4" +description = "pytest-sugar is a plugin for pytest that changes the default look and feel of pytest (e.g. progressbar, show tests that fail instantly)." +category = "dev" +optional = false +python-versions = "*" + +[package.dependencies] +packaging = ">=14.1" +pytest = ">=2.9" +termcolor = ">=1.1.0" + [[package]] name = "python-dateutil" version = "2.8.1" @@ -727,6 +740,14 @@ category = "dev" optional = false python-versions = "*" +[[package]] +name = "termcolor" +version = "1.1.0" +description = "ANSII Color formatting for output in terminal." +category = "dev" +optional = false +python-versions = "*" + [[package]] name = "toml" version = "0.10.2" @@ -806,7 +827,7 @@ testing = ["pytest (>=3.5,!=3.7.3)", "pytest-checkdocs (>=1.2.3)", "pytest-flake [metadata] lock-version = "1.1" python-versions = "^3.7" -content-hash = "aa2e865ef9910d636a123e651287443a9a4b664ac7edbe4f02f8434298216cdd" +content-hash = "d2a94218852cc9d18515e466b071a8c1757b196eaf39baec7309f981ebdddbdd" [metadata.files] altgraph = [ @@ -997,8 +1018,8 @@ macholib = [ {file = "macholib-1.14.tar.gz", hash = "sha256:0c436bc847e7b1d9bda0560351bf76d7caf930fb585a828d13608839ef42c432"}, ] markdown = [ - {file = "Markdown-3.3.3-py3-none-any.whl", hash = "sha256:c109c15b7dc20a9ac454c9e6025927d44460b85bd039da028d85e2b6d0bcc328"}, - {file = "Markdown-3.3.3.tar.gz", hash = "sha256:5d9f2b5ca24bc4c7a390d22323ca4bad200368612b5aaa7796babf971d2b2f18"}, + {file = "Markdown-3.3.6-py3-none-any.whl", hash = "sha256:9923332318f843411e9932237530df53162e29dc7a4e2b91e35764583c46c9a3"}, + {file = "Markdown-3.3.6.tar.gz", hash = "sha256:76df8ae32294ec39dcf89340382882dfa12975f87f45c3ed1ecdb1e8cefc7006"}, ] markupsafe = [ {file = "MarkupSafe-1.1.1-cp27-cp27m-macosx_10_6_intel.whl", hash = "sha256:09027a7803a62ca78792ad89403b1b7a73a01c8cb65909cd876f7fcebd79b161"}, @@ -1165,6 +1186,9 @@ pytest-expecter = [ {file = "pytest-expecter-2.3.tar.gz", hash = "sha256:8e6a3e565fbc524e5a4988b664d1b748e0a810b33233880aa5d3d78970351e06"}, {file = "pytest_expecter-2.3-py3-none-any.whl", hash = "sha256:6336d7f43221500e392014a949fd77ee2c2a84bcae2be7669acdd0d7c89b89e9"}, ] +pytest-sugar = [ + {file = "pytest-sugar-0.9.4.tar.gz", hash = "sha256:b1b2186b0a72aada6859bea2a5764145e3aaa2c1cfbb23c3a19b5f7b697563d3"}, +] python-dateutil = [ {file = "python-dateutil-2.8.1.tar.gz", hash = "sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c"}, {file = "python_dateutil-2.8.1-py2.py3-none-any.whl", hash = "sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a"}, @@ -1229,6 +1253,9 @@ snowballstemmer = [ {file = "snowballstemmer-2.0.0-py2.py3-none-any.whl", hash = "sha256:209f257d7533fdb3cb73bdbd24f436239ca3b2fa67d56f6ff88e86be08cc5ef0"}, {file = "snowballstemmer-2.0.0.tar.gz", hash = "sha256:df3bac3df4c2c01363f3dd2cfa78cce2840a79b9f1c2d2de9ce8d31683992f52"}, ] +termcolor = [ + {file = "termcolor-1.1.0.tar.gz", hash = "sha256:1d6d69ce66211143803fbc56652b41d73b4a400a2891d7bf7a1cdf4c02de613b"}, +] toml = [ {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, diff --git a/pyproject.toml b/pyproject.toml index 01cf28818..a0713fb25 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [tool.poetry] name = "doorstop" -version = "3.0b1" +version = "3.0b3" description = "Requirements management using version control." license = "LGPLv3" @@ -43,7 +43,7 @@ classifiers = [ python = "^3.7" PyYAML = "^5.1" -Markdown = "^3.3.3" +Markdown = "^3.3.6" bottle = "~0.12.13" requests = "^2.0" python-markdown-math = "~0.6" @@ -66,6 +66,7 @@ pylint = "~2.12" pytest = "^6.2.5" pytest-cov = "*" pytest-expecter = "*" +pytest-sugar = "*" # Reports coveragespace = "^4.0"