chore: update all ci files; switch to hatch; add ty

Author: karlicoss

.ci/run

@@ -1,24 +1,36 @@
-#!/bin/bash -eu
+#!/bin/bash
+set -eu
 
 cd "$(dirname "$0")"
-cd ..
+cd .. # git root
 
 if ! command -v sudo; then
-    # CI or Docker sometimes don't have it, so useful to have a dummy
+    # CI or Docker sometimes doesn't have it, so useful to have a dummy
     function sudo {
         "$@"
     }
 fi
 
-if ! [ -z "$CI" ]; then
+# --parallel-live to show outputs while it's running
+tox_cmd='run-parallel --parallel-live'
+if [ -n "${CI-}" ]; then
     # install OS specific stuff here
-    if [[ "$OSTYPE" == "darwin"* ]]; then
+    case "$OSTYPE" in
+    darwin*) 
         # macos
         :
-    else
+        ;;
+    cygwin* | msys* | win*)
+        # windows
+        # ugh. parallel stuff seems super flaky under windows, some random failures, "file used by other process" and crap like that
+        tox_cmd='run'
+        ;;
+    *)
+        # must be linux?
         :
-    fi
+        ;;
+    esac
 fi
 
-pip3 install --user tox
-tox
+# NOTE: expects uv installed
+uv tool run --with tox-uv tox $tox_cmd "$@"

.github/workflows/main.yml

@@ -6,45 +6,79 @@ on:
     branches: '*'
     tags: 'v[0-9]+.*' # only trigger on 'release' tags for PyPi
     # Ideally I would put this in the pypi job... but github syntax doesn't allow for regexes there :shrug:
-    # P.S. fuck made up yaml DSLs.
+
+  # Needed to trigger on others' PRs.
   # Note that people who fork it need to go to "Actions" tab on their fork and click "I understand my workflows, go ahead and enable them".
-  pull_request: # needed to trigger on others' PRs
-  workflow_dispatch: # needed to trigger workflows manually
-  # todo cron?
+  pull_request:
+
+  # Needed to trigger workflows manually.
+  workflow_dispatch:
+    inputs:
+      debug_enabled:
+        type: boolean
+        description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)'
+        required: false
+        default: false
+
+  schedule:
+    - cron: '31 18 * * 5'  # run every Friday
 
-env:
-  # useful for scripts & sometimes tests to know
-  CI: true
 
 jobs:
   build:
     strategy:
+      fail-fast: false
       matrix:
-        platform: [ubuntu-latest]
-        python-version: [3.7, 3.8, 3.9]
+        platform: [ubuntu-latest, macos-latest, windows-latest]
+        python-version: ['3.9', '3.10', '3.11', '3.12', '3.13']
+        exclude: [
+            # windows runners are pretty scarce, so let's only run lowest and highest python version
+            {platform: windows-latest, python-version: '3.10'},
+            {platform: windows-latest, python-version: '3.11'},
+            {platform: windows-latest, python-version: '3.12'},
+
+            # same, macos is a bit too slow and ubuntu covers python quirks well
+            {platform: macos-latest  , python-version: '3.10'},
+            {platform: macos-latest  , python-version: '3.11'},
+            {platform: macos-latest  , python-version: '3.12'},
+        ]
 
     runs-on: ${{ matrix.platform }}
 
+    # useful for 'optional' pipelines
+    # continue-on-error: ${{ matrix.platform == 'windows-latest' }}
+
     steps:
     # ugh https://github.com/actions/toolkit/blob/main/docs/commands.md#path-manipulation
     - run: echo "$HOME/.local/bin" >> $GITHUB_PATH
 
-    - uses: actions/setup-python@v1
+    - uses: actions/checkout@v4
       with:
-        python-version: ${{ matrix.python-version }}
+        submodules: recursive
+        fetch-depth: 0  # nicer to have all git history when debugging/for tests
 
-    - uses: actions/checkout@v2
+    - uses: actions/setup-python@v5
       with:
-        submodules: recursive
+        python-version: ${{ matrix.python-version }}
+      
+    - uses: astral-sh/setup-uv@v5
+      with:
+        enable-cache: false  # we don't have lock files, so can't use them as cache key
 
-    # uncomment for SSH debugging
-    # - uses: mxschmitt/action-tmate@v2
+    - uses: mxschmitt/action-tmate@v3
+      if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }}
 
-    - run: .ci/run
+    # explicit bash command is necessary for Windows CI runner, otherwise it thinks it's cmd...
+    - run: bash .ci/run
+      env:
+        # only compute lxml coverage on ubuntu; it crashes on windows
+        CI_MYPY_COVERAGE: ${{ matrix.platform == 'ubuntu-latest' && '--cobertura-xml-report .coverage.mypy' || '' }}
 
-    - uses: actions/upload-artifact@v2
+    - if: matrix.platform == 'ubuntu-latest'  # no need to compute coverage for other platforms
+      uses: codecov/codecov-action@v5
       with:
-        name: .coverage.mypy 
-        path: .coverage.mypy/
-      # restrict to a single python version, otherwise uploading fails
-      if: ${{ matrix.python-version == '3.8' }}
+        fail_ci_if_error: true  # default false
+        token: ${{ secrets.CODECOV_TOKEN }}
+        flags: mypy-${{ matrix.python-version }}
+        files: .coverage.mypy/cobertura.xml
+

conftest.py

@@ -0,0 +1,58 @@
+# this is a hack to monkey patch pytest so it handles tests inside namespace packages without __init__.py properly
+# without it, pytest can't discover the package root for some reason
+# also see https://github.com/karlicoss/pytest_namespace_pkgs for more
+
+import os
+import pathlib
+from typing import Optional
+
+import _pytest.main
+import _pytest.pathlib
+
+# we consider all dirs in repo/ to be namespace packages
+root_dir = pathlib.Path(__file__).absolute().parent.resolve() / 'src'
+assert root_dir.exists(), root_dir
+
+# TODO assert it contains package name?? maybe get it via setuptools..
+
+namespace_pkg_dirs = [str(d) for d in root_dir.iterdir() if d.is_dir()]
+
+# resolve_package_path is called from _pytest.pathlib.import_path
+# takes a full abs path to the test file and needs to return the path to the 'root' package on the filesystem
+resolve_pkg_path_orig = _pytest.pathlib.resolve_package_path
+
+
+def resolve_package_path(path: pathlib.Path) -> Optional[pathlib.Path]:
+    result = path  # search from the test file upwards
+    for parent in result.parents:
+        if str(parent) in namespace_pkg_dirs:
+            return parent
+    if os.name == 'nt':
+        # ??? for some reason on windows it is trying to call this against conftest? but not on linux/osx
+        if path.name == 'conftest.py':
+            return resolve_pkg_path_orig(path)
+    raise RuntimeError("Couldn't determine path for ", path)
+
+
+# NOTE: seems like it's not necessary anymore?
+# keeping it for now just in case
+# after https://github.com/pytest-dev/pytest/pull/13426 we should be able to remove the whole conftest
+# _pytest.pathlib.resolve_package_path = resolve_package_path
+
+
+# without patching, the orig function returns just a package name for some reason
+# (I think it's used as a sort of fallback)
+# so we need to point it at the absolute path properly
+# not sure what are the consequences.. maybe it wouldn't be able to run against installed packages? not sure..
+search_pypath_orig = _pytest.main.search_pypath
+
+
+def search_pypath(module_name: str) -> str:
+    mpath = root_dir / module_name.replace('.', os.sep)
+    if not mpath.is_dir():
+        mpath = mpath.with_suffix('.py')
+        assert mpath.exists(), mpath  # just in case
+    return str(mpath)
+
+
+_pytest.main.search_pypath = search_pypath  # ty: ignore[invalid-assignment]

mypy.ini

@@ -1,9 +1,17 @@
 [mypy]
 pretty = True
 show_error_context = True
-show_error_codes = True
+show_column_numbers = True
+show_error_end = True
+
 check_untyped_defs = True
-namespace_packages = True
+
+# see https://mypy.readthedocs.io/en/stable/error_code_list2.html
+warn_redundant_casts = True
+strict_equality = True
+warn_unused_ignores = True
+enable_error_code = deprecated,redundant-expr,possibly-undefined,truthy-bool,truthy-iterable,ignore-without-code,unused-awaitable
+
 
 # an example of suppressing
 # [mypy-my.config.repos.pdfannots.pdfannots]

pyproject.toml

@@ -0,0 +1,62 @@
+# see https://github.com/karlicoss/pymplate for up-to-date reference
+[project]
+dynamic = ["version"]  # version is managed by build backend
+name = "goodrexport"
+dependencies = [
+    "goodrexport[dal]",  # TODO backwards compatibility -- remove later?
+]
+requires-python = ">=3.9"
+
+## these need to be set if you're planning to upload to pypi
+# description = "TODO"
+# license = {file = "LICENSE"}
+# authors = [
+#     {name = "Dima Gerasimov (@karlicoss)", email = "karlicoss@gmail.com"},
+# ]
+# maintainers = [
+#     {name = "Dima Gerasimov (@karlicoss)", email = "karlicoss@gmail.com"},
+# ]
+#
+# [project.urls]
+# Homepage = "https://github.com/karlicoss/pymplate"
+##
+
+
+[project.optional-dependencies]
+dal = ["lxml"]
+export = []
+optional = [
+    "orjson",  # faster json processing
+    "colorlog",
+    "ijson",  # faster iterative json processing
+]
+[dependency-groups]
+# TODO: not sure, on the one hand could just use 'standard' dev dependency group
+# On the other hand, it's a bit annoying that it's always included by default? 
+# To make sure it's not included, need to use `uv run --exact --no-default-groups ...`
+testing = [
+    "pytest",
+    "ruff",
+    "mypy",
+    "lxml", # for mypy html coverage
+    "ty>=0.0.1a15",
+
+    "lxml-stubs",
+]
+
+
+[build-system]
+requires = ["hatchling", "hatch-vcs"]
+build-backend = "hatchling.build"
+
+# unfortunately have to duplicate project name here atm, see https://github.com/pypa/hatch/issues/1894
+[tool.hatch.build.targets.wheel]
+packages = ["src/goodrexport"]
+
+[tool.hatch.version]
+source = "vcs"
+
+[tool.hatch.version.raw-options]
+version_scheme = "python-simplified-semver"
+local_scheme = "dirty-tag"
+

pytest.ini

@@ -1,8 +1,20 @@
 [pytest]
 # discover files that don't follow test_ naming. Useful to keep tests along with the source code
 python_files = *.py
+
+# this setting only impacts package/module naming under pytest, not the discovery
+consider_namespace_packages = true
+
 addopts =
+  # prevent pytest cache from being created... it craps into project dir and I never use it anyway
+  -p no:cacheprovider
+
+  # -rap to print tests summary even when they are successful
+  -rap
   --verbose
 
   # otherwise it won't discover doctests
   --doctest-modules
+
+  # show all test durations (unless they are too short)
+  --durations=0