Add SCIP support

.github/workflows/tests.yml

@@ -13,32 +13,19 @@ jobs:
       matrix:
         os: [ubuntu-latest, macos-latest]
         python-version: ["3.11"]
-        ctags: [true]
         include:
           - os: windows-latest
             python-version: "3.7"
-            ctags: false
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v3
         with:
           python-version: ${{ matrix.python-version }}
-      - run: sudo apt-get update && sudo apt-get install -y universal-ctags
-        if: matrix.ctags && matrix.os == 'ubuntu-latest'
-      - run: brew install universal-ctags
-        if: matrix.ctags && matrix.os == 'macos-latest'
       - name: Run tests
         run: |
           git config --global user.email "you@example.com"
           git config --global user.name "Your Name"
-          if ${{ matrix.ctags }}; then
-            pip install -r test_requirements.txt
-            pip install -e .
-            bash ./runtests.sh -v tests
-          else
-            grep -v ctags test_requirements.txt > /tmp/test_requirements.txt
-            pip install -r /tmp/test_requirements.txt
-            pip install -e .
-            bash ./runtests.sh -v tests -k "not ctags"
-          fi
+          pip install -r test_requirements.txt
+          pip install -e .
+          bash ./runtests.sh -v tests
         shell: bash -ex {0}

.gitignore

@@ -9,3 +9,5 @@ build/
 dist/
 .DS_Store
 .mypy_cache
+klaus/scip_pb2.py
+klaus/scip_pb2.pyi

CHANGELOG.rst

@@ -1,6 +1,21 @@
 Changelog
 =========
 
+UNRELEASED
+----------
+- Replace ctags-based code navigation with SCIP. klaus now reads a SCIP
+  index from ``<repo>/.scip/<sha>.scip`` (or ``HEAD.scip``) and uses it for
+  syntax classes and cross-reference links. Files not covered by SCIP fall
+  back to Pygments syntax highlighting. The ``--ctags`` CLI flag and
+  ``KLAUS_CTAGS_POLICY`` env var are gone.
+- Add ``--scip`` / ``KLAUS_SCIP_POLICY`` ({none, tags-and-branches, ALL},
+  default none) for on-demand SCIP generation. When enabled, the first
+  request for an un-indexed revision triggers a background job that runs
+  an indexer inside a ``git worktree`` and writes the dump to
+  ``<repo>/.scip/<sha>.scip``; subsequent requests pick it up. The default
+  indexer is ``scip-python``; the indexer list is pluggable via
+  ``klaus.scip_generate.INDEXERS``.
+
 3.0.1 (Jun 17, 2024)
 --------------------
 - #330: Fix startup with ctags (Louis Sautier)

Dockerfile

@@ -1,12 +1,9 @@
 FROM alpine:edge
 
 RUN echo http://dl-cdn.alpinelinux.org/alpine/edge/testing >> /etc/apk/repositories && \
-    apk add --no-cache uwsgi-python3 git ctags  py3-markupsafe py3-pygments \
-                       py3-dulwich py3-humanize py3-flask py3-flask-markdown py3-docutils
-
-RUN apk add --no-cache python3-dev py3-pip gcc musl-dev && \
-    pip3 install --break-system-packages python-ctags3 && \
-    apk del python3-dev gcc musl-dev
+    apk add --no-cache uwsgi-python3 git py3-markupsafe py3-pygments \
+                       py3-dulwich py3-humanize py3-flask py3-flask-markdown \
+                       py3-docutils py3-protobuf
 
 COPY . /klaus
 RUN pip3 install --break-system-packages /klaus && rm -rf /klaus

MANIFEST.in

@@ -1,3 +1,4 @@
 recursive-include klaus/static *
 recursive-include klaus/templates *
 include klaus.1
+include klaus/scip.proto

README.rst

@@ -16,7 +16,7 @@ klaus: a simple, easy-to-set-up Git web viewer that Just Works™.
 * Syntax highlighting
 * Markdown + RestructuredText rendering support
 * Pull + push support (Git Smart HTTP)
-* Code navigation using Exuberant ctags
+* Code navigation using SCIP indexes (drop ``index.scip`` files into ``<repo>/.scip/``, or use ``--scip`` to generate on demand)
 
 :Demo: https://github.com/jonashaag/klaus/wiki/Sites-using-klaus
 :On PyPI: http://pypi.python.org/pypi/klaus/

klaus.1

@@ -35,8 +35,8 @@ open klaus in a browser on server start
 \fB\-B\fR BROWSER, \fB\-\-with\-browser\fR BROWSER
 specify which browser to use with \fB\-\-browser\fR
 .TP
-\fB\-\-ctags\fR {none,tags\-and\-branches,ALL}
-enable ctags for which revisions? default: none.
+\fB\-\-scip\fR {none,tags\-and\-branches,ALL}
+generate SCIP indexes on demand for which revisions? default: none.
 WARNING: Don't use 'ALL' for public servers!
 .SS "Git Smart HTTP:"
 .TP

klaus/__init__.py

@@ -23,11 +23,11 @@ class Klaus(flask.Flask):
         "undefined": jinja2.StrictUndefined,
     }
 
-    def __init__(self, repo_paths, site_name, use_smarthttp, ctags_policy="none"):
+    def __init__(self, repo_paths, site_name, use_smarthttp, scip_policy="none"):
         """(See `make_app` for parameter descriptions.)"""
         self.site_name = site_name
         self.use_smarthttp = use_smarthttp
-        self.ctags_policy = ctags_policy
+        self.scip_policy = scip_policy
 
         valid_repos, invalid_repos = self.load_repos(repo_paths)
         self.valid_repos = {repo.namespaced_name: repo for repo in valid_repos}
@@ -85,15 +85,14 @@ def setup_routes(self):
                 )
         # fmt: on
 
-    def should_use_ctags(self, git_repo, git_commit):
-        if self.ctags_policy == "none":
+    def should_generate_scip(self, git_repo, git_commit):
+        if self.scip_policy == "none":
             return False
-        elif self.ctags_policy == "ALL":
+        if self.scip_policy == "ALL":
             return True
-        elif self.ctags_policy == "tags-and-branches":
+        if self.scip_policy == "tags-and-branches":
             return git_commit.id in git_repo.get_tag_and_branch_shas()
-        else:
-            raise ValueError("Unknown ctags policy %r" % self.ctags_policy)
+        raise ValueError(f"Unknown scip policy {self.scip_policy!r}")
 
     def load_repos(self, repo_paths):
         valid_repos = []
@@ -115,7 +114,7 @@ def make_app(
     require_browser_auth=False,
     disable_push=False,
     unauthenticated_push=False,
-    ctags_policy="none",
+    scip_policy="none",
 ):
     """
     Returns a WSGI app with all the features (smarthttp, authentication)
@@ -140,11 +139,19 @@ def make_app(
         are set, but push should not be supported.
     :param htdigest_file: A *file-like* object that contains the HTTP auth credentials.
     :param unauthenticated_push: Allow push'ing without authentication. DANGER ZONE!
-    :param ctags_policy: The ctags policy to use, may be one of:
-        - 'none': never use ctags
-        - 'tags-and-branches': use ctags for revisions that are the HEAD of
-          a tag or branc
-        - 'ALL': use ctags for all revisions, may result in high server load!
+    :param scip_policy: When to generate SCIP indexes on demand for revisions
+        that don't yet have one.  Pre-generated dumps under
+        ``<repo>/.scip/<sha>.scip`` are always used.  When no dump exists:
+
+        - ``'none'``: never generate; render with Pygments.
+        - ``'tags-and-branches'``: generate for revisions that are the HEAD of
+          a tag or branch.
+        - ``'ALL'``: generate for every revision.  May result in high server
+          load; don't use for public servers.
+
+        Generation happens in a background thread and uses ``git worktree``,
+        so the first request renders with Pygments and subsequent requests
+        for the same revision pick up the SCIP index.
     """
     if unauthenticated_push:
         if not use_smarthttp:
@@ -167,7 +174,7 @@ def make_app(
         repo_paths,
         site_name,
         use_smarthttp,
-        ctags_policy,
+        scip_policy=scip_policy,
     )
     app.wsgi_app = utils.ProxyFix(app.wsgi_app)
 

klaus/cli.py

@@ -1,4 +1,5 @@
 import argparse
+import logging
 import os
 import sys
 import webbrowser
@@ -44,9 +45,9 @@ def make_parser():
         default=None,
     )
     parser.add_argument(
-        "--ctags",
-        help="enable ctags for which revisions? default: none. "
-        "WARNING: Don't use 'ALL' for public servers!",
+        "--scip",
+        help="generate SCIP indexes on demand for which revisions? "
+        "default: none. WARNING: Don't use 'ALL' for public servers!",
         choices=["none", "tags-and-branches", "ALL"],
         default="none",
     )
@@ -81,6 +82,11 @@ def make_parser():
 def main():
     args = make_parser().parse_args()
 
+    logging.basicConfig(
+        level=logging.DEBUG if args.debug else logging.INFO,
+        format="%(asctime)s %(levelname)s %(name)s: %(message)s",
+    )
+
     if args.version:
         print(KLAUS_VERSION)
         return 0
@@ -101,26 +107,12 @@ def main():
     if not args.site_name:
         args.site_name = "%s:%d" % (args.host, args.port)
 
-    if args.ctags != "none":
-        from klaus.ctagsutils import check_have_compatible_ctags
-
-        if not check_have_compatible_ctags():
-            print(
-                "ERROR: Exuberant ctags not installed (or 'ctags' binary isn't *Exuberant* ctags)",
-                file=sys.stderr,
-            )
-            return 1
-        try:
-            pass
-        except ImportError:
-            raise ImportError("Please install 'python-ctags3' to enable ctags support.")
-
     app = make_app(
         args.repos,
         force_unicode(args.site_name or args.host),
         args.smarthttp,
         args.htdigest,
-        ctags_policy=args.ctags,
+        scip_policy=args.scip,
     )
 
     if args.browser:

klaus/contrib/app_args.py

@@ -26,6 +26,6 @@ def get_args_from_env():
         unauthenticated_push=strtobool(
             os.environ.get("KLAUS_UNAUTHENTICATED_PUSH", "0")
         ),
-        ctags_policy=os.environ.get("KLAUS_CTAGS_POLICY", "none"),
+        scip_policy=os.environ.get("KLAUS_SCIP_POLICY", "none"),
     )
     return args, kwargs