From aef3263b630818b12f585a08424a45ca0018ff6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Wed, 27 May 2026 13:23:00 +0100 Subject: [PATCH 1/5] Replace ctags with SCIP for code intelligence --- .github/workflows/tests.yml | 19 +- .gitignore | 2 + CHANGELOG.rst | 8 + Dockerfile | 9 +- MANIFEST.in | 1 + README.rst | 2 +- klaus.1 | 4 - klaus/__init__.py | 25 +- klaus/cli.py | 23 - klaus/contrib/app_args.py | 1 - klaus/ctagscache.py | 178 ------- klaus/ctagsutils.py | 45 -- klaus/diff.py | 10 +- klaus/highlighting.py | 273 +++++++---- klaus/scip.proto | 897 ++++++++++++++++++++++++++++++++++ klaus/scip_index.py | 159 ++++++ klaus/static/scip.css | 39 ++ klaus/templates/skeleton.html | 1 + klaus/views.py | 55 +-- pyproject.toml | 10 +- setup.py | 1 + test_requirements.txt | 1 - tests/test_contrib.py | 3 - tests/test_make_app.py | 99 ++-- 24 files changed, 1398 insertions(+), 467 deletions(-) delete mode 100644 klaus/ctagscache.py delete mode 100644 klaus/ctagsutils.py create mode 100644 klaus/scip.proto create mode 100644 klaus/scip_index.py create mode 100644 klaus/static/scip.css diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index cfd3bda6..dc86c876 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -13,32 +13,19 @@ jobs: matrix: os: [ubuntu-latest, macos-latest] python-version: ["3.11"] - ctags: [true] include: - os: windows-latest python-version: "3.7" - ctags: false steps: - uses: actions/checkout@v3 - uses: actions/setup-python@v3 with: python-version: ${{ matrix.python-version }} - - run: sudo apt-get update && sudo apt-get install -y universal-ctags - if: matrix.ctags && matrix.os == 'ubuntu-latest' - - run: brew install universal-ctags - if: matrix.ctags && matrix.os == 'macos-latest' - name: Run tests run: | git config --global user.email "you@example.com" git config --global user.name "Your Name" - if ${{ matrix.ctags }}; then - pip install -r test_requirements.txt - pip install -e . - bash ./runtests.sh -v tests - else - grep -v ctags test_requirements.txt > /tmp/test_requirements.txt - pip install -r /tmp/test_requirements.txt - pip install -e . - bash ./runtests.sh -v tests -k "not ctags" - fi + pip install -r test_requirements.txt + pip install -e . + bash ./runtests.sh -v tests shell: bash -ex {0} diff --git a/.gitignore b/.gitignore index 4d8834d3..7eb09a2a 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,5 @@ build/ dist/ .DS_Store .mypy_cache +klaus/scip_pb2.py +klaus/scip_pb2.pyi diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 223619cd..b3cf8b1e 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,14 @@ Changelog ========= +UNRELEASED +---------- +- Replace ctags-based code navigation with SCIP. klaus now reads a SCIP + index from ``/.scip/.scip`` (or ``HEAD.scip``) and uses it for + syntax classes and cross-reference links. Files not covered by SCIP fall + back to Pygments syntax highlighting. The ``--ctags`` CLI flag and + ``KLAUS_CTAGS_POLICY`` env var are gone. + 3.0.1 (Jun 17, 2024) -------------------- - #330: Fix startup with ctags (Louis Sautier) diff --git a/Dockerfile b/Dockerfile index cd48e9b2..9352e80e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,12 +1,9 @@ FROM alpine:edge RUN echo http://dl-cdn.alpinelinux.org/alpine/edge/testing >> /etc/apk/repositories && \ - apk add --no-cache uwsgi-python3 git ctags py3-markupsafe py3-pygments \ - py3-dulwich py3-humanize py3-flask py3-flask-markdown py3-docutils - -RUN apk add --no-cache python3-dev py3-pip gcc musl-dev && \ - pip3 install --break-system-packages python-ctags3 && \ - apk del python3-dev gcc musl-dev + apk add --no-cache uwsgi-python3 git py3-markupsafe py3-pygments \ + py3-dulwich py3-humanize py3-flask py3-flask-markdown \ + py3-docutils py3-protobuf COPY . /klaus RUN pip3 install --break-system-packages /klaus && rm -rf /klaus diff --git a/MANIFEST.in b/MANIFEST.in index ddeea761..7e0b08d4 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,3 +1,4 @@ recursive-include klaus/static * recursive-include klaus/templates * include klaus.1 +include klaus/scip.proto diff --git a/README.rst b/README.rst index e6e3c3ec..f47c2615 100644 --- a/README.rst +++ b/README.rst @@ -16,7 +16,7 @@ klaus: a simple, easy-to-set-up Git web viewer that Just Works™. * Syntax highlighting * Markdown + RestructuredText rendering support * Pull + push support (Git Smart HTTP) -* Code navigation using Exuberant ctags +* Code navigation using SCIP indexes (drop ``index.scip`` files into ``/.scip/``) :Demo: https://github.com/jonashaag/klaus/wiki/Sites-using-klaus :On PyPI: http://pypi.python.org/pypi/klaus/ diff --git a/klaus.1 b/klaus.1 index 2b205204..4b92e795 100644 --- a/klaus.1 +++ b/klaus.1 @@ -34,10 +34,6 @@ open klaus in a browser on server start .TP \fB\-B\fR BROWSER, \fB\-\-with\-browser\fR BROWSER specify which browser to use with \fB\-\-browser\fR -.TP -\fB\-\-ctags\fR {none,tags\-and\-branches,ALL} -enable ctags for which revisions? default: none. -WARNING: Don't use 'ALL' for public servers! .SS "Git Smart HTTP:" .TP \fB\-\-smarthttp\fR diff --git a/klaus/__init__.py b/klaus/__init__.py index 96caddb2..709e587b 100644 --- a/klaus/__init__.py +++ b/klaus/__init__.py @@ -23,11 +23,10 @@ class Klaus(flask.Flask): "undefined": jinja2.StrictUndefined, } - def __init__(self, repo_paths, site_name, use_smarthttp, ctags_policy="none"): + def __init__(self, repo_paths, site_name, use_smarthttp): """(See `make_app` for parameter descriptions.)""" self.site_name = site_name self.use_smarthttp = use_smarthttp - self.ctags_policy = ctags_policy valid_repos, invalid_repos = self.load_repos(repo_paths) self.valid_repos = {repo.namespaced_name: repo for repo in valid_repos} @@ -85,16 +84,6 @@ def setup_routes(self): ) # fmt: on - def should_use_ctags(self, git_repo, git_commit): - if self.ctags_policy == "none": - return False - elif self.ctags_policy == "ALL": - return True - elif self.ctags_policy == "tags-and-branches": - return git_commit.id in git_repo.get_tag_and_branch_shas() - else: - raise ValueError("Unknown ctags policy %r" % self.ctags_policy) - def load_repos(self, repo_paths): valid_repos = [] invalid_repos = [] @@ -115,7 +104,6 @@ def make_app( require_browser_auth=False, disable_push=False, unauthenticated_push=False, - ctags_policy="none", ): """ Returns a WSGI app with all the features (smarthttp, authentication) @@ -140,11 +128,11 @@ def make_app( are set, but push should not be supported. :param htdigest_file: A *file-like* object that contains the HTTP auth credentials. :param unauthenticated_push: Allow push'ing without authentication. DANGER ZONE! - :param ctags_policy: The ctags policy to use, may be one of: - - 'none': never use ctags - - 'tags-and-branches': use ctags for revisions that are the HEAD of - a tag or branc - - 'ALL': use ctags for all revisions, may result in high server load! + + Code intelligence (cross-references and syntax classes) is enabled + automatically when a SCIP index is present at ``/.scip/.scip`` + (with ``HEAD.scip`` as a fallback). When no index is present, files are + rendered with Pygments syntax highlighting. """ if unauthenticated_push: if not use_smarthttp: @@ -167,7 +155,6 @@ def make_app( repo_paths, site_name, use_smarthttp, - ctags_policy, ) app.wsgi_app = utils.ProxyFix(app.wsgi_app) diff --git a/klaus/cli.py b/klaus/cli.py index aa92f002..c157bc1e 100644 --- a/klaus/cli.py +++ b/klaus/cli.py @@ -43,14 +43,6 @@ def make_parser(): metavar="BROWSER", default=None, ) - parser.add_argument( - "--ctags", - help="enable ctags for which revisions? default: none. " - "WARNING: Don't use 'ALL' for public servers!", - choices=["none", "tags-and-branches", "ALL"], - default="none", - ) - parser.add_argument( "repos", help="repositories to serve", @@ -101,26 +93,11 @@ def main(): if not args.site_name: args.site_name = "%s:%d" % (args.host, args.port) - if args.ctags != "none": - from klaus.ctagsutils import check_have_compatible_ctags - - if not check_have_compatible_ctags(): - print( - "ERROR: Exuberant ctags not installed (or 'ctags' binary isn't *Exuberant* ctags)", - file=sys.stderr, - ) - return 1 - try: - pass - except ImportError: - raise ImportError("Please install 'python-ctags3' to enable ctags support.") - app = make_app( args.repos, force_unicode(args.site_name or args.host), args.smarthttp, args.htdigest, - ctags_policy=args.ctags, ) if args.browser: diff --git a/klaus/contrib/app_args.py b/klaus/contrib/app_args.py index ba02e53e..a849ca93 100644 --- a/klaus/contrib/app_args.py +++ b/klaus/contrib/app_args.py @@ -26,6 +26,5 @@ def get_args_from_env(): unauthenticated_push=strtobool( os.environ.get("KLAUS_UNAUTHENTICATED_PUSH", "0") ), - ctags_policy=os.environ.get("KLAUS_CTAGS_POLICY", "none"), ) return args, kwargs diff --git a/klaus/ctagscache.py b/klaus/ctagscache.py deleted file mode 100644 index 0c242b9d..00000000 --- a/klaus/ctagscache.py +++ /dev/null @@ -1,178 +0,0 @@ -"""A cache for tagsfiles generated by the 'ctags' command line tool. - -We don't want to run the 'ctags' command line tool on each request as it may -take a lot of time. The following steps are necessary in order to create a -ctags tagsfile that be read by Pygments: - -1. Clone the repository to a temporary location and check out the branch/commit - the user is browsing, unless the branch is already checked out. (*) -2. Run 'ctags -R' on the temporary repository checkout. -3. Delete the temporary repository checkout. - -To avoid going through these steps on each request, we cache the tagsfile -generated in step 2. The cache is on-disk and non-persistent, i.e. cleared -whenever the Python interpreter running klaus is shut down. - -For large projects, the ctags tagsfiles may grow to sizes of multiple MiB, so -we have to set an upper limit on the size of the cache. Since tagsfiles are -represented as uncompressed ASCII files, we can increase the number of tagsfiles -we can cache by using compression. Of course, 'python-ctags', which is used by -Pygments to read the tagsfiles, can't deal with compressed tagsfiles, so we have -to uncompress them before actually using them. To avoid decompressing tagsfiles -on each request, we keep the tagsfiles that are most likely to be used (**) in -uncompressed form. - -(*) We always create a clone in the current implementation; - this could be optimized in the future. -(**) "most likely": currently implemented as "most recently used" -""" -import atexit -import gzip -import os -import shutil -import tempfile -import threading - -from dulwich.lru_cache import LRUSizeCache - -from klaus.ctagsutils import create_tagsfile, delete_tagsfile - -# Good compression while taking only 10% more time than level 1 -COMPRESSION_LEVEL = 4 - - -def compress_tagsfile(uncompressed_tagsfile_path): - """Compress an uncompressed tagsfile. - - :return: path to the compressed version of the tagsfile - """ - _, compressed_tagsfile_path = tempfile.mkstemp() - with open(uncompressed_tagsfile_path, "rb") as uncompressed: - with gzip.open(compressed_tagsfile_path, "wb", COMPRESSION_LEVEL) as compressed: - shutil.copyfileobj(uncompressed, compressed) - return compressed_tagsfile_path - - -def uncompress_tagsfile(compressed_tagsfile_path): - """Uncompress an compressed tagsfile. - - :return: path to the uncompressed version of the tagsfile - """ - _, uncompressed_tagsfile_path = tempfile.mkstemp() - with gzip.open(compressed_tagsfile_path, "rb") as compressed: - with open(uncompressed_tagsfile_path, "wb") as uncompressed: - shutil.copyfileobj(compressed, uncompressed) - return uncompressed_tagsfile_path - - -MiB = 1024 * 1024 - - -class CTagsCache: - """A ctags cache. Both uncompressed and compressed entries are kept in - temporary files created by `tempfile.mkstemp` which are deleted from disk - when the Python interpreter is shut down. - - :param uncompressed_max_bytes: Maximum size of the uncompressed cache sector - :param compressed_max_bytes: Maximum size of the compressed cache sector - - The lifecycle of a cache entry is as follows. - - - When first created, a tagsfile is put into the uncompressed cache sector. - - When free space is required for other uncompressed tagsfiles, it may be - moved to the compressed cache sector. Gzip is used to compress the tagsfile. - - When free space is required for other compressed tagsfiles, it may be - evicted from the cache entirely. - - When the tagsfile is requested and it's in the compressed cache sector, - it is moved back to the uncompressed sector prior to using it. - """ - - def __init__(self, uncompressed_max_bytes=30 * MiB, compressed_max_bytes=20 * MiB): - self.uncompressed_max_bytes = uncompressed_max_bytes - self.compressed_max_bytes = compressed_max_bytes - # Note: We use dulwich's LRU cache to store the tagsfile paths here, - # but we could easily replace it by any other (LRU) cache implementation. - self._uncompressed_cache = LRUSizeCache( - uncompressed_max_bytes, compute_size=os.path.getsize - ) - self._compressed_cache = LRUSizeCache( - compressed_max_bytes, compute_size=os.path.getsize - ) - self._clearing = False - self._lock = threading.Lock() - - atexit.register(self.clear) - - def __del__(self): - self.clear() - - def clear(self): - """Clear both the uncompressed and compressed caches.""" - # Don't waste time moving tagsfiles from uncompressed to compressed cache, - # but remove them directly instead: - self._clearing = True - self._uncompressed_cache.clear() - self._compressed_cache.clear() - self._clearing = False - - def get_tagsfile(self, git_repo_path, git_rev): - """Get the ctags tagsfile for the given Git repository and revision. - - - If the tagsfile is still in cache, and in uncompressed form, return it - without any further cost. - - If the tagsfile is still in cache, but in compressed form, uncompress - it, put it into uncompressed space, and return the uncompressed version. - - If the tagsfile isn't in cache at all, create it, put it into - uncompressed cache and return the newly created version. - """ - # Always require full SHAs - assert len(git_rev) == 40 - - # Avoiding race conditions, The Sledgehammer Way - with self._lock: - if git_rev in self._uncompressed_cache: - return self._uncompressed_cache[git_rev] - - if git_rev in self._compressed_cache: - compressed_tagsfile_path = self._compressed_cache[git_rev] - uncompressed_tagsfile_path = uncompress_tagsfile( - compressed_tagsfile_path - ) - self._compressed_cache._remove_node( - self._compressed_cache._cache[git_rev] - ) - else: - # Not in cache. - uncompressed_tagsfile_path = create_tagsfile(git_repo_path, git_rev) - self._uncompressed_cache.add( - git_rev, uncompressed_tagsfile_path, self._clear_uncompressed_entry - ) - return uncompressed_tagsfile_path - - def _clear_uncompressed_entry(self, git_rev, uncompressed_tagsfile_path): - """Called by LRUSizeCache whenever an entry is to be evicted from - uncompressed cache. - - Most of the times this happens when space is needed - in uncompressed cache, in which case we move the tagsfile to compressed - cache. When clearing the cache, we don't bother moving entries to - uncompressed space; we delete them directly instead. - """ - if not self._clearing: - # If we're clearing the whole cache, don't waste time moving tagsfiles - # from uncompressed to compressed cache, but remove them directly instead. - self._compressed_cache.add( - git_rev, - compress_tagsfile(uncompressed_tagsfile_path), - self._clear_compressed_entry, - ) - delete_tagsfile(uncompressed_tagsfile_path) - - def _clear_compressed_entry(self, git_rev, compressed_tagsfile_path): - """Called by LRUSizeCache whenever an entry to be evicted from - compressed cache. - - This happens when space is needed for new compressed - tagsfiles. We delete the evictee from the cache entirely. - """ - delete_tagsfile(compressed_tagsfile_path) diff --git a/klaus/ctagsutils.py b/klaus/ctagsutils.py deleted file mode 100644 index 8dfefc1a..00000000 --- a/klaus/ctagsutils.py +++ /dev/null @@ -1,45 +0,0 @@ -import os -import shutil -import subprocess -import tempfile - - -def check_have_compatible_ctags(): - """Check that the 'ctags' binary is a compatible ctags (Universal or Exuberant, not etags etc)""" - try: - out = subprocess.check_output(["ctags", "--version"], stderr=subprocess.PIPE) - return b"Universal" in out or b"Exuberant" in out - except subprocess.CalledProcessError: - return False - - -def create_tagsfile(git_repo_path, git_rev): - """Create a ctags tagsfile for the given Git repository and revision. - - This creates a temporary clone of the repository, checks out the revision, - runs 'ctags -R' and deletes the temporary clone. - - :return: path to the generated tagsfile - """ - assert ( - check_have_compatible_ctags() - ), "'ctags' binary is missing or not *Universal* (or *Exuberant*) ctags" - - _, target_tagsfile = tempfile.mkstemp() - checkout_tmpdir = tempfile.mkdtemp() - try: - subprocess.check_call( - ["git", "clone", "-q", "--shared", git_repo_path, checkout_tmpdir] - ) - subprocess.check_call(["git", "checkout", "-q", git_rev], cwd=checkout_tmpdir) - subprocess.check_call( - ["ctags", "--fields=+l", "-Rno", target_tagsfile], cwd=checkout_tmpdir - ) - finally: - shutil.rmtree(checkout_tmpdir) - return target_tagsfile - - -def delete_tagsfile(tagsfile_path): - """Delete a tagsfile.""" - os.remove(tagsfile_path) diff --git a/klaus/diff.py b/klaus/diff.py index a263d0f5..a4e9c396 100644 --- a/klaus/diff.py +++ b/klaus/diff.py @@ -1,11 +1,11 @@ """ - lodgeit.lib.diff - ~~~~~~~~~~~~~~~~ +lodgeit.lib.diff +~~~~~~~~~~~~~~~~ - Render a nice diff between two things. +Render a nice diff between two things. - :copyright: 2007 by Armin Ronacher. - :license: BSD +:copyright: 2007 by Armin Ronacher. +:license: BSD """ from difflib import SequenceMatcher diff --git a/klaus/highlighting.py b/klaus/highlighting.py index 3793f602..954fe092 100644 --- a/klaus/highlighting.py +++ b/klaus/highlighting.py @@ -1,110 +1,208 @@ +"""Source-code rendering for klaus. + +When a SCIP document is available for the file being rendered, syntax classes +and cross-reference links come from SCIP. Otherwise we fall back to Pygments +(without ctags, since cross-references in the Pygments path are gone). +""" + +from html import escape +from typing import Any, Iterator, Optional, Tuple + from pygments import highlight from pygments.formatters import HtmlFormatter from pygments.lexers import ( ClassNotFound, TextLexer, - get_lexer_by_name, get_lexer_for_filename, guess_lexer, ) from klaus import markup - -CTAGS_SUPPORTED_LANGUAGES = ( - "Asm Awk Basic C C# C++ Cobol DosBatch Eiffel Erlang Fortran HTML Java " - "JavaScript Lisp Lua Make Makefile MatLab OCaml PHP Pascal Perl Python " - "REXX Ruby SML SQL Scheme Sh Tcl Tex VHDL Verilog Vim" - # Not supported by Pygments: Asp Ant BETA Flex SLang Vera YACC -).split() -PYGMENTS_CTAGS_LANGUAGE_MAP = dict( - (get_lexer_by_name(lexer).name, lexer) for lexer in CTAGS_SUPPORTED_LANGUAGES # type: ignore -) - - -class KlausDefaultFormatter(HtmlFormatter): - def __init__(self, language, ctags, **kwargs): - HtmlFormatter.__init__( - self, +from klaus.scip_index import Document, Index, Occurrence + +# Map SCIP SyntaxKind enum value -> CSS class suffix. Numbers match +# klaus.scip_pb2.SyntaxKind; using ints avoids importing the proto module +# at top level just for the names. +_SCIP_SYNTAX_CLASSES = { + 1: "comment", + 2: "punctuation-delimiter", + 3: "punctuation-bracket", + 4: "keyword", # also IdentifierKeyword + 5: "identifier-operator", + 6: "identifier", + 7: "identifier-builtin", + 8: "identifier-null", + 9: "identifier-constant", + 10: "identifier-mutable-global", + 11: "identifier-parameter", + 12: "identifier-local", + 13: "identifier-shadowed", + 14: "identifier-namespace", # also IdentifierModule + 15: "identifier-function", + 16: "identifier-function-definition", + 17: "identifier-macro", + 18: "identifier-macro-definition", + 19: "identifier-type", + 20: "identifier-builtin-type", + 21: "identifier-attribute", + 22: "regex-escape", + 23: "regex-repeated", + 24: "regex-wildcard", + 25: "regex-delimiter", + 26: "regex-join", + 27: "string-literal", + 28: "string-literal-escape", + 29: "string-literal-special", + 30: "string-literal-key", + 31: "character-literal", + 32: "numeric-literal", + 33: "boolean-literal", + 34: "tag", + 35: "tag-attribute", + 36: "tag-delimiter", +} + + +class KlausHtmlFormatter(HtmlFormatter): + """Pygments HTML formatter wired up to klaus's CSS and link conventions.""" + + def __init__(self, **kwargs: Any) -> None: + super().__init__( linenos="table", lineanchors="L", linespans="L", anchorlinenos=True, **kwargs, ) - self.language = language - if ctags: - # Use Pygments' ctags system but provide our own CTags instance - self.tagsfile = True # some trueish object - self._ctags = ctags - - def _format_lines(self, tokensource): - for tag, line in HtmlFormatter._format_lines(self, tokensource): + + def _format_lines(self, tokensource: Any) -> Iterator[Tuple[int, str]]: + for tag, line in super()._format_lines(tokensource): # type: ignore[misc] if tag == 1: - # sourcecode line - line = "%s" % line + line = f"{line}" yield tag, line - def _lookup_ctag(self, token): - matches = list(self._get_all_ctags_matches(token)) - best_matches = list(self.get_best_ctags_matches(matches)) - if not best_matches: - return None, None - else: - return ( - best_matches[0]["file"].decode("utf-8"), - best_matches[0]["lineNumber"], - ) - - def _get_all_ctags_matches(self, token): - FIELDS = ("file", "lineNumber", "kind", b"language") - from ctags import TagEntry - - entry = TagEntry() # target "buffer" for ctags - if self._ctags.find(entry, token.encode("utf-8"), 0): - yield dict((k, entry[k]) for k in FIELDS) - while self._ctags.findNext(entry): - yield dict((k, entry[k]) for k in FIELDS) - - def get_best_ctags_matches(self, matches): - if self.language is None: - return matches - else: - return filter( - lambda match: match[b"language"] == self.language.encode("utf-8"), - matches, - ) - - -class KlausPythonFormatter(KlausDefaultFormatter): - def get_best_ctags_matches(self, matches): - # The first ctags match may be an import, which ctags sees as a - # definition of the tag -- even though it might very well have found - # the "real" definition of the tag. Import matches aren't very helpful: - # In the best case, we are brought to the line where the tag is imported - # in the same file. But it may also bring us to some completely unrelated - # import of the tag in some other file. We change the tag lookup mechanics - # so that non-import matches are always preferred over import matches. - return filter( - lambda match: match["kind"] != b"i", - super().get_best_ctags_matches(matches), + +def _render_scip_lines( + source: str, document: Document, index: Index, scip_baseurl: str +) -> str: + """Render ``source`` as HTML using occurrences from ``document``. + + Produces the same outer structure as Pygments' ``linenos='table'`` mode so + klaus's CSS and JS keep working: ``
...`` with linenos column and code column. + """ + lines = source.splitlines(keepends=False) + linenos_parts = [] + code_parts = [] + for i, line_text in enumerate(lines, start=1): + anchor = f"L-{i}" + linenos_parts.append(f'{i}') + rendered_line = _render_scip_line( + line_text, i - 1, document, index, scip_baseurl + ) + code_parts.append( + f'' + f'' + f"{rendered_line}\n" + f"" ) + return ( + '
' + '" + '
'
+        + "\n".join(linenos_parts)
+        + "
'
+        + "".join(code_parts)
+        + "
" + ) + + +def _render_scip_line( + line_text: str, + line_index: int, + document: Document, + index: Index, + scip_baseurl: str, +) -> str: + """Render a single source line, splicing in spans for each occurrence.""" + occurrences = [ + o + for o in document.occurrences_on_line(line_index) + if o.range.end_line == line_index # skip occurrences spanning lines + ] + if not occurrences: + return escape(line_text) + + out = [] + cursor = 0 + for occ in occurrences: + start = occ.range.start_char + end = occ.range.end_char + if start < cursor or end > len(line_text) or start >= end: + continue + if start > cursor: + out.append(escape(line_text[cursor:start])) + out.append(_render_occurrence(line_text[start:end], occ, index, scip_baseurl)) + cursor = end + if cursor < len(line_text): + out.append(escape(line_text[cursor:])) + return "".join(out) + + +def _render_occurrence( + text: str, occ: Occurrence, index: Index, scip_baseurl: str +) -> str: + classes = [] + css_class = _SCIP_SYNTAX_CLASSES.get(occ.syntax_kind) + if css_class: + classes.append(f"scip-{css_class}") + if occ.is_definition: + classes.append("scip-definition") + + body = escape(text) + if occ.symbol and not occ.is_definition: + defn = index.get_definition(occ.symbol) + if defn is not None: + href = f"{scip_baseurl}{defn.relative_path}#L-{defn.line + 1}" + body = f'{body}' + if classes: + body = f'{body}' + return body + def highlight_or_render( - code, filename, render_markup=True, ctags=None, ctags_baseurl=None -): - """Render code using Pygments, markup (markdown, rst, ...) using the - corresponding renderer, if available. - - :param code: the program code to highlight, str - :param filename: name of the source file the code is taken from, str - :param render_markup: whether to render markup if possible, bool - :param ctags: tagsfile obj used for source code hyperlinks, ``ctags.CTags`` - :param ctags_baseurl: base url used for source code hyperlinks, str + code: str, + filename: str, + render_markup: bool = True, + scip_document: Optional[Document] = None, + scip_index: Optional[Index] = None, + scip_baseurl: Optional[str] = None, +) -> str: + """Render code as HTML. + + :param code: source text + :param filename: name of the source file (used to pick a Pygments lexer + and decide whether the file is markup) + :param render_markup: whether to render markup (Markdown, reST, ...) when + the filename matches a known markup format + :param scip_document: SCIP ``Document`` for this file, if available. When + provided, syntax highlighting and cross-reference links come from + SCIP. Without it we fall back to Pygments. + :param scip_index: the SCIP ``Index`` that ``scip_document`` came from. + Required when ``scip_document`` is provided. + :param scip_baseurl: base URL used to build cross-reference hyperlinks. + Required when ``scip_document`` is provided. """ if render_markup and markup.can_render(filename): return markup.render(filename, code) + if scip_document is not None: + assert scip_index is not None and scip_baseurl is not None, ( + "scip_index and scip_baseurl are required with scip_document" + ) + return _render_scip_lines(code, scip_document, scip_index, scip_baseurl) + try: lexer = get_lexer_for_filename(filename, code) except ClassNotFound: @@ -113,17 +211,4 @@ def highlight_or_render( except ClassNotFound: lexer = TextLexer() - formatter_cls = { - "Python": KlausPythonFormatter, - }.get(lexer.name, KlausDefaultFormatter) - if ctags: - ctags_urlscheme = ctags_baseurl + "%(path)s%(fname)s%(fext)s" - else: - ctags_urlscheme = None - formatter = formatter_cls( - language=PYGMENTS_CTAGS_LANGUAGE_MAP.get(lexer.name), - ctags=ctags, - tagurlformat=ctags_urlscheme, - ) - - return highlight(code, lexer, formatter) + return highlight(code, lexer, KlausHtmlFormatter()) diff --git a/klaus/scip.proto b/klaus/scip.proto new file mode 100644 index 00000000..46f23f0e --- /dev/null +++ b/klaus/scip.proto @@ -0,0 +1,897 @@ +// An index contains one or more pieces of information about a given piece of +// source code or software artifact. Complementary information can be merged +// together from multiple sources to provide a unified code intelligence +// experience. +// +// Programs producing a file of this format is an "indexer" and may operate +// somewhere on the spectrum between precision, such as indexes produced by +// compiler-backed indexers, and heurstics, such as indexes produced by local +// syntax-directed analysis for scope rules. + +syntax = "proto3"; + +package scip; + +option go_package = "github.com/scip-code/scip/bindings/go/scip/"; + +// Index represents a complete SCIP index for a workspace this is rooted at a +// single directory. An Index message payload can have a large memory footprint +// and it's therefore recommended to emit and consume an Index payload one field +// value at a time. To permit streaming consumption of an Index payload, the +// `metadata` field must appear at the start of the stream and must only appear +// once in the stream. Other field values may appear in any order. +message Index { + // Metadata about this index. + Metadata metadata = 1; + // Documents that belong to this index. + repeated Document documents = 2; + // (optional) Symbols that are referenced from this index but are defined in + // an external package (a separate `Index` message). Leave this field empty + // if you assume the external package will get indexed separately. If the + // external package won't get indexed for some reason then you can use this + // field to provide hover documentation for those external symbols. + repeated SymbolInformation external_symbols = 3; + // IMPORTANT: When adding a new field to `Index` here, add a matching + // function in `IndexVisitor` and update `ParseStreaming`. +} + +message Metadata { + // Which version of this protocol was used to generate this index? + ProtocolVersion version = 1; + // Information about the tool that produced this index. + ToolInfo tool_info = 2; + // URI-encoded absolute path to the root directory of this index. All + // documents in this index must appear in a subdirectory of this root + // directory. + string project_root = 3; + // Text encoding of the source files on disk that are referenced from + // `Document.relative_path`. This value is unrelated to the `Document.text` + // field, which is a Protobuf string and hence must be UTF-8 encoded. + TextEncoding text_document_encoding = 4; +} + +enum ProtocolVersion { + UnspecifiedProtocolVersion = 0; +} + +enum TextEncoding { + UnspecifiedTextEncoding = 0; + UTF8 = 1; + UTF16 = 2; +} + +message ToolInfo { + // Name of the indexer that produced this index. + string name = 1; + // Version of the indexer that produced this index. + string version = 2; + // Command-line arguments that were used to invoke this indexer. + repeated string arguments = 3; +} + +// Document defines the metadata about a source file on disk. +message Document { + // The string ID for the programming language this file is written in. + // The `Language` enum contains the names of most common programming languages. + // This field is typed as a string to permit any programming language, including + // ones that are not specified by the `Language` enum. + string language = 4; + // (Required) Unique path to the text document. + // + // 1. The path must be relative to the directory supplied in the associated + // `Metadata.project_root`. + // 2. The path must not begin with a leading '/'. + // 3. The path must point to a regular file, not a symbolic link. + // 4. The path must use '/' as the separator, including on Windows. + // 5. The path must be canonical; it cannot include empty components ('//'), + // or '.' or '..'. + string relative_path = 1; + // Occurrences that appear in this file. + repeated Occurrence occurrences = 2; + // Symbols that are "defined" within this document. + // + // This should include symbols which technically do not have any definition, + // but have a reference and are defined by some other symbol (see + // Relationship.is_definition). + repeated SymbolInformation symbols = 3; + + // (optional) Text contents of this document. Indexers are not expected to + // include the text by default. It's preferable that clients read the text + // contents from the file system by resolving the absolute path from joining + // `Index.metadata.project_root` and `Document.relative_path`. This field + // can be useful for testing or when working with virtual/in-memory documents. + string text = 5; + + // Specifies the encoding used for source ranges in this Document. + // + // Usually, this will match the type used to index the string type + // in the indexer's implementation language in O(1) time. + // - For an indexer implemented in JVM/.NET language or JavaScript/TypeScript, + // use UTF16CodeUnitOffsetFromLineStart. + // - For an indexer implemented in Python, + // use UTF32CodeUnitOffsetFromLineStart. + // - For an indexer implemented in Go, Rust or C++, + // use UTF8ByteOffsetFromLineStart. + PositionEncoding position_encoding = 6; +} + +// Encoding used to interpret the 'character' value in source ranges. +enum PositionEncoding { + // Default value. This value should not be used by new SCIP indexers + // so that a consumer can process the SCIP index without ambiguity. + UnspecifiedPositionEncoding = 0; + // The 'character' value is interpreted as an offset in terms + // of UTF-8 code units (i.e. bytes). + // + // Example: For the string "🚀 Woo" in UTF-8, the bytes are + // [240, 159, 154, 128, 32, 87, 111, 111], so the offset for 'W' + // would be 5. + UTF8CodeUnitOffsetFromLineStart = 1; + // The 'character' value is interpreted as an offset in terms + // of UTF-16 code units (each is 2 bytes). + // + // Example: For the string "🚀 Woo", the UTF-16 code units are + // ['\ud83d', '\ude80', ' ', 'W', 'o', 'o'], so the offset for 'W' + // would be 3. + UTF16CodeUnitOffsetFromLineStart = 2; + // The 'character' value is interpreted as an offset in terms + // of UTF-32 code units (each is 4 bytes). + // + // Example: For the string "🚀 Woo", the UTF-32 code units are + // ['🚀', ' ', 'W', 'o', 'o'], so the offset for 'W' would be 2. + UTF32CodeUnitOffsetFromLineStart = 3; +} + +// Symbol is similar to a URI, it identifies a class, method, or a local +// variable. `SymbolInformation` contains rich metadata about symbols such as +// the docstring. +// +// Symbol has a standardized string representation, which can be used +// interchangeably with `Symbol`. The syntax for Symbol is the following: +// ``` +// # ()+ stands for one or more repetitions of +// # ()? stands for zero or one occurrence of +// ::= ' ' ' ' ()+ | 'local ' +// ::= ' ' ' ' +// ::= any UTF-8, escape spaces with double space. Must not be empty nor start with 'local' +// ::= any UTF-8, escape spaces with double space. Use the placeholder '.' to indicate an empty value +// ::= same as above +// ::= same as above +// ::= | | | | | | | +// ::= '/' +// ::= '#' +// ::= '.' +// ::= ':' +// ::= '!' +// ::= '(' ()? ').' +// ::= '[' ']' +// ::= '(' ')' +// ::= +// ::= +// ::= | +// ::= ()+ +// ::= '_' | '+' | '-' | '$' | ASCII letter or digit +// ::= '`' ()+ '`', must contain at least one non- +// ::= any UTF-8, escape backticks with double backtick. +// ::= +// ``` +// +// The list of descriptors for a symbol should together form a fully +// qualified name for the symbol. That is, it should serve as a unique +// identifier across the package. Typically, it will include one descriptor +// for every node in the AST (along the ancestry path) between the root of +// the file and the node corresponding to the symbol. +// +// Local symbols MUST only be used for entities which are local to a Document, +// and cannot be accessed from outside the Document. +message Symbol { + string scheme = 1; + Package package = 2; + repeated Descriptor descriptors = 3; +} + +// Unit of packaging and distribution. +// +// NOTE: This corresponds to a module in Go and JVM languages. +message Package { + string manager = 1; + string name = 2; + string version = 3; +} + +message Descriptor { + enum Suffix { + option allow_alias = true; + UnspecifiedSuffix = 0; + // Unit of code abstraction and/or namespacing. + // + // NOTE: This corresponds to a package in Go and JVM languages. + Namespace = 1; + // Use Namespace instead. + Package = 1 [deprecated = true]; + Type = 2; + Term = 3; + Method = 4; + TypeParameter = 5; + Parameter = 6; + // Can be used for any purpose. + Meta = 7; + Local = 8; + Macro = 9; + } + string name = 1; + string disambiguator = 2; + Suffix suffix = 3; + // NOTE: If you add new fields here, make sure to update the prepareSlot() + // function responsible for parsing symbols. +} + +// Signature represents the signature of a symbol as it's displayed in API +// documentation or hover tooltips. It uses a subset of Document's fields with +// the same field numbers for wire compatibility with older indexes that encoded +// signatures using the Document message type. +message Signature { + // The language of the signature, e.g. "java", "go", "python". + string language = 4; + // The text content of the signature, e.g. "void add(int a, int b)". + string text = 5; + // (optional) Occurrences within the signature text that reference other + // symbols, enabling hyperlinking of types in the signature. Ranges are + // relative to the `text` field. + repeated Occurrence occurrences = 2; + + // Reserved field numbers from the Document message to prevent accidental + // reuse, which would break wire compatibility with older indexes. + reserved 1, 3, 6; +} + +// SymbolInformation defines metadata about a symbol, such as the symbol's +// docstring or what package it's defined it. +message SymbolInformation { + // Identifier of this symbol, which can be referenced from `Occurence.symbol`. + // The string must be formatted according to the grammar in `Symbol`. + string symbol = 1; + // (optional, but strongly recommended) The markdown-formatted documentation + // for this symbol. Use `SymbolInformation.signature_documentation` to + // document the method/class/type signature of this symbol. + // Due to historical reasons, indexers may include signature documentation in + // this field by rendering markdown code blocks. New indexers should only + // include non-code documentation in this field, for example docstrings. + repeated string documentation = 3; + // (optional) Relationships to other symbols (e.g., implements, type definition). + repeated Relationship relationships = 4; + // The kind of this symbol. Use this field instead of + // `SymbolDescriptor.Suffix` to determine whether something is, for example, a + // class or a method. + Kind kind = 5; + // (optional) Kind represents the fine-grained category of a symbol, suitable for presenting + // information about the symbol's meaning in the language. + // + // For example: + // - A Java method would have the kind `Method` while a Go function would + // have the kind `Function`, even if the symbols for these use the same + // syntax for the descriptor `SymbolDescriptor.Suffix.Method`. + // - A Go struct has the symbol kind `Struct` while a Java class has + // the symbol kind `Class` even if they both have the same descriptor: + // `SymbolDescriptor.Suffix.Type`. + // + // Since Kind is more fine-grained than Suffix: + // - If two symbols have the same Kind, they should share the same Suffix. + // - If two symbols have different Suffixes, they should have different Kinds. + enum Kind { + UnspecifiedKind = 0; + // A method which may or may not have a body. For Java, Kotlin etc. + AbstractMethod = 66; + // For Ruby's attr_accessor + Accessor = 72; + Array = 1; + // For Alloy + Assertion = 2; + AssociatedType = 3; + // For C++ + Attribute = 4; + // For Lean + Axiom = 5; + Boolean = 6; + Class = 7; + // For C++ + Concept = 86; + Constant = 8; + Constructor = 9; + // For Solidity + Contract = 62; + // For Haskell + DataFamily = 10; + // For C# and F# + Delegate = 73; + Enum = 11; + EnumMember = 12; + Error = 63; + Event = 13; + // For Dart + Extension = 84; + // For Alloy + Fact = 14; + Field = 15; + File = 16; + Function = 17; + // For 'get' in Swift, 'attr_reader' in Ruby + Getter = 18; + // For Raku + Grammar = 19; + // For Purescript and Lean + Instance = 20; + Interface = 21; + Key = 22; + // For Racket + Lang = 23; + // For Lean + Lemma = 24; + // For solidity + Library = 64; + Macro = 25; + Method = 26; + // For Ruby + MethodAlias = 74; + // Analogous to 'ThisParameter' and 'SelfParameter', but for languages + // like Go where the receiver doesn't have a conventional name. + MethodReceiver = 27; + // Analogous to 'AbstractMethod', for Go. + MethodSpecification = 67; + // For Protobuf + Message = 28; + // For Dart + Mixin = 85; + // For Solidity + Modifier = 65; + Module = 29; + Namespace = 30; + Null = 31; + Number = 32; + Object = 33; + Operator = 34; + Package = 35; + PackageObject = 36; + Parameter = 37; + ParameterLabel = 38; + // For Haskell's PatternSynonyms + Pattern = 39; + // For Alloy + Predicate = 40; + Property = 41; + // Analogous to 'Trait' and 'TypeClass', for Swift and Objective-C + Protocol = 42; + // Analogous to 'AbstractMethod', for Swift and Objective-C. + ProtocolMethod = 68; + // Analogous to 'AbstractMethod', for C++. + PureVirtualMethod = 69; + // For Haskell + Quasiquoter = 43; + // 'self' in Python, Rust, Swift etc. + SelfParameter = 44; + // For 'set' in Swift, 'attr_writer' in Ruby + Setter = 45; + // For Alloy, analogous to 'Struct'. + Signature = 46; + // For Ruby + SingletonClass = 75; + // Analogous to 'StaticMethod', for Ruby. + SingletonMethod = 76; + // Analogous to 'StaticField', for C++ + StaticDataMember = 77; + // For C# + StaticEvent = 78; + // For C# + StaticField = 79; + // For Java, C#, C++ etc. + StaticMethod = 80; + // For C#, TypeScript etc. + StaticProperty = 81; + // For C, C++ + StaticVariable = 82; + String = 48; + Struct = 49; + // For Swift + Subscript = 47; + // For Lean + Tactic = 50; + // For Lean + Theorem = 51; + // Method receiver for languages + // 'this' in JavaScript, C++, Java etc. + ThisParameter = 52; + // Analogous to 'Protocol' and 'TypeClass', for Rust, Scala etc. + Trait = 53; + // Analogous to 'AbstractMethod', for Rust, Scala etc. + TraitMethod = 70; + // Data type definition for languages like OCaml which use `type` + // rather than separate keywords like `struct` and `enum`. + Type = 54; + TypeAlias = 55; + // Analogous to 'Trait' and 'Protocol', for Haskell, Purescript etc. + TypeClass = 56; + // Analogous to 'AbstractMethod', for Haskell, Purescript etc. + TypeClassMethod = 71; + // For Haskell + TypeFamily = 57; + TypeParameter = 58; + // For C, C++, Capn Proto + Union = 59; + Value = 60; + Variable = 61; + // Next = 87; + // Feel free to open a PR proposing new language-specific kinds. + } + // (optional) The name of this symbol as it should be displayed to the user. + // For example, the symbol "com/example/MyClass#myMethod(+1)." should have the + // display name "myMethod". The `symbol` field is not a reliable source of + // the display name for several reasons: + // + // - Local symbols don't encode the name. + // - Some languages have case-insensitive names, so the symbol is all-lowercase. + // - The symbol may encode names with special characters that should not be + // displayed to the user. + string display_name = 6; + // (optional) The signature of this symbol as it's displayed in API + // documentation or in hover tooltips. For example, a Java method that adds + // two numbers would have `Signature.language = "java"` and + // `Signature.text = "void add(int a, int b)"`. The `language` and `text` + // fields are required while `occurrences` can be optionally included to + // support hyperlinking referenced symbols in the signature. + Signature signature_documentation = 7; + // (optional) The enclosing symbol if this is a local symbol. For non-local + // symbols, the enclosing symbol should be parsed from the `symbol` field + // using the `Descriptor` grammar. + // + // The primary use-case for this field is to allow local symbol to be displayed + // in a symbol hierarchy for API documentation. It's OK to leave this field + // empty for local variables since local variables usually don't belong in API + // documentation. However, in the situation that you wish to include a local + // symbol in the hierarchy, then you can use `enclosing_symbol` to locate the + // "parent" or "owner" of this local symbol. For example, a Java indexer may + // choose to use local symbols for private class fields while providing an + // `enclosing_symbol` to reference the enclosing class to allow the field to + // be part of the class documentation hierarchy. From the perspective of an + // author of an indexer, the decision to use a local symbol or global symbol + // should exclusively be determined whether the local symbol is accessible + // outside the document, not by the capability to find the enclosing + // symbol. + string enclosing_symbol = 8; +} + +message Relationship { + string symbol = 1; + // When resolving "Find references", this field documents what other symbols + // should be included together with this symbol. For example, consider the + // following TypeScript code that defines two symbols `Animal#sound()` and + // `Dog#sound()`: + // ```ts + // interface Animal { + // ^^^^^^ definition Animal# + // sound(): string + // ^^^^^ definition Animal#sound() + // } + // class Dog implements Animal { + // ^^^ definition Dog#, relationships = [{symbol: "Animal#", is_implementation: true}] + // public sound(): string { return "woof" } + // ^^^^^ definition Dog#sound(), references_symbols = Animal#sound(), relationships = [{symbol: "Animal#sound()", is_implementation:true, is_reference: true}] + // } + // const animal: Animal = new Dog() + // ^^^^^^ reference Animal# + // console.log(animal.sound()) + // ^^^^^ reference Animal#sound() + // ``` + // Doing "Find references" on the symbol `Animal#sound()` should return + // references to the `Dog#sound()` method as well. Vice-versa, doing "Find + // references" on the `Dog#sound()` method should include references to the + // `Animal#sound()` method as well. + bool is_reference = 2; + // Similar to `is_reference` but for "Find implementations". + // It's common for `is_implementation` and `is_reference` to both be true but + // it's not always the case. + // In the TypeScript example above, observe that `Dog#` has an + // `is_implementation` relationship with `"Animal#"` but not `is_reference`. + // This is because "Find references" on the "Animal#" symbol should not return + // "Dog#". We only want "Dog#" to return as a result for "Find + // implementations" on the "Animal#" symbol. + bool is_implementation = 3; + // Similar to `references_symbols` but for "Go to type definition". + bool is_type_definition = 4; + // Allows overriding the behavior of "Go to definition" and "Find references" + // for symbols which do not have a definition of their own or could + // potentially have multiple definitions. + // + // For example, in a language with single inheritance and no field overriding, + // inherited fields can reuse the same symbol as the ancestor which declares + // the field. In such a situation, is_definition is not needed. + // + // On the other hand, in languages with single inheritance and some form + // of mixins, you can use is_definition to relate the symbol to the + // matching symbol in ancestor classes, and is_reference to relate the + // symbol to the matching symbol in mixins. + bool is_definition = 5; + // Update registerInverseRelationships on adding a new field here. +} + +// SymbolRole declares what "role" a symbol has in an occurrence. A role is +// encoded as a bitset where each bit represents a different role. For example, +// to determine if the `Import` role is set, test whether the second bit of the +// enum value is defined. In pseudocode, this can be implemented with the +// logic: `const isImportRole = (role.value & SymbolRole.Import.value) > 0`. +enum SymbolRole { + // This case is not meant to be used; it only exists to avoid an error + // from the Protobuf code generator. + UnspecifiedSymbolRole = 0; + // Is the symbol defined here? If not, then this is a symbol reference. + Definition = 0x1; + // Is the symbol imported here? + Import = 0x2; + // Is the symbol written here? + WriteAccess = 0x4; + // Is the symbol read here? + ReadAccess = 0x8; + // Is the symbol in generated code? + Generated = 0x10; + // Is the symbol in test code? + Test = 0x20; + // Is this a signature for a symbol that is defined elsewhere? + // + // Applies to forward declarations for languages like C, C++ + // and Objective-C, as well as `val` declarations in interface + // files in languages like SML and OCaml. + ForwardDefinition = 0x40; +} + +enum SyntaxKind { + option allow_alias = true; + + UnspecifiedSyntaxKind = 0; + + // Comment, including comment markers and text + Comment = 1; + + // `;` `.` `,` + PunctuationDelimiter = 2; + // (), {}, [] when used syntactically + PunctuationBracket = 3; + + // `if`, `else`, `return`, `class`, etc. + Keyword = 4; + IdentifierKeyword = 4 [deprecated = true]; + + // `+`, `*`, etc. + IdentifierOperator = 5; + + // non-specific catch-all for any identifier not better described elsewhere + Identifier = 6; + // Identifiers builtin to the language: `min`, `print` in Python. + IdentifierBuiltin = 7; + // Identifiers representing `null`-like values: `None` in Python, `nil` in Go. + IdentifierNull = 8; + // `xyz` in `const xyz = "hello"` + IdentifierConstant = 9; + // `var X = "hello"` in Go + IdentifierMutableGlobal = 10; + // Parameter definition and references + IdentifierParameter = 11; + // Identifiers for variable definitions and references within a local scope + IdentifierLocal = 12; + // Identifiers that shadow other identifiers in an outer scope + IdentifierShadowed = 13; + // Identifier representing a unit of code abstraction and/or namespacing. + // + // NOTE: This corresponds to a package in Go and JVM languages, + // and a module in languages like Python and JavaScript. + IdentifierNamespace = 14; + IdentifierModule = 14 [deprecated = true]; + + // Function references, including calls + IdentifierFunction = 15; + // Function definition only + IdentifierFunctionDefinition = 16; + + // Macro references, including invocations + IdentifierMacro = 17; + // Macro definition only + IdentifierMacroDefinition = 18; + + // non-builtin types + IdentifierType = 19; + // builtin types only, such as `str` for Python or `int` in Go + IdentifierBuiltinType = 20; + + // Python decorators, c-like __attribute__ + IdentifierAttribute = 21; + + // `\b` + RegexEscape = 22; + // `*`, `+` + RegexRepeated = 23; + // `.` + RegexWildcard = 24; + // `(`, `)`, `[`, `]` + RegexDelimiter = 25; + // `|`, `-` + RegexJoin = 26; + + // Literal strings: "Hello, world!" + StringLiteral = 27; + // non-regex escapes: "\t", "\n" + StringLiteralEscape = 28; + // datetimes within strings, special words within a string, `{}` in format strings + StringLiteralSpecial = 29; + // "key" in { "key": "value" }, useful for example in JSON + StringLiteralKey = 30; + // 'c' or similar, in languages that differentiate strings and characters + CharacterLiteral = 31; + // Literal numbers, both floats and integers + NumericLiteral = 32; + // `true`, `false` + BooleanLiteral = 33; + + // Used for XML-like tags + Tag = 34; + // Attribute name in XML-like tags + TagAttribute = 35; + // Delimiters for XML-like tags + TagDelimiter = 36; +} + +// Occurrence associates a source position with a symbol and/or highlighting +// information. +// +// If possible, indexers should try to bundle logically related information +// across occurrences into a single occurrence to reduce payload sizes. +message Occurrence { + // Half-open [start, end) range of this occurrence. Must be exactly three or four + // elements: + // + // - Four elements: `[startLine, startCharacter, endLine, endCharacter]` + // - Three elements: `[startLine, startCharacter, endCharacter]`. The end line + // is inferred to have the same value as the start line. + // + // It is allowed for the range to be empty (i.e. start==end). + // + // Line numbers and characters are always 0-based. Make sure to increment the + // line/character values before displaying them in an editor-like UI because + // editors conventionally use 1-based numbers. + // + // The 'character' value is interpreted based on the PositionEncoding for + // the Document. + // + // Historical note: the original draft of this schema had a `Range` message + // type with `start` and `end` fields of type `Position`, mirroring LSP. + // Benchmarks revealed that this encoding was inefficient and that we could + // reduce the total payload size of an index by 50% by using `repeated int32` + // instead. The `repeated int32` encoding is admittedly more embarrassing to + // work with in some programming languages but we hope the performance + // improvements make up for it. + repeated int32 range = 1; + // (optional) The symbol that appears at this position. See + // `SymbolInformation.symbol` for how to format symbols as strings. + string symbol = 2; + // (optional) Bitset containing `SymbolRole`s in this occurrence. + // See `SymbolRole`'s documentation for how to read and write this field. + int32 symbol_roles = 3; + // (optional) CommonMark-formatted documentation for this specific range. If + // empty, the `Symbol.documentation` field is used instead. One example + // where this field might be useful is when the symbol represents a generic + // function (with abstract type parameters such as `List`) and at this + // occurrence we know the exact values (such as `List`). + // + // This field can also be used for dynamically or gradually typed languages, + // which commonly allow for type-changing assignment. + repeated string override_documentation = 4; + // (optional) What syntax highlighting class should be used for this range? + SyntaxKind syntax_kind = 5; + // (optional) Diagnostics that have been reported for this specific range. + repeated Diagnostic diagnostics = 6; + // (optional) Using the same encoding as the sibling `range` field, half-open + // source range of the nearest non-trivial enclosing AST node. This range must + // enclose the `range` field. Example applications that make use of the + // enclosing_range field: + // + // - Call hierarchies: to determine what symbols are references from the body + // of a function + // - Symbol outline: to display breadcrumbs from the cursor position to the + // root of the file + // - Expand selection: to select the nearest enclosing AST node. + // - Highlight range: to indicate the AST expression that is associated with a + // hover popover + // + // For definition occurrences, the enclosing range should indicate the + // start/end bounds of the entire definition AST node, including + // documentation. + // ``` + // const n = 3 + // ^ range + // ^^^^^^^^^^^ enclosing_range + // + // /** Parses the string into something */ + // ^ enclosing_range start --------------------------------------| + // function parse(input string): string { | + // ^^^^^ range | + // return input.slice(n) | + // } | + // ^ enclosing_range end <---------------------------------------| + // ``` + // + // Any attributes/decorators/attached macros should also be part of the + // enclosing range. + // + // ```python + // @cache + // ^ enclosing_range start---------------------| + // def factorial(n): | + // return n * factorial(n-1) if n else 1 | + // < enclosing_range end-----------------------| + // + // ``` + // + // For reference occurrences, the enclosing range should indicate the start/end + // bounds of the parent expression. + // ``` + // const a = a.b + // ^ range + // ^^^ enclosing_range + // const b = a.b(41).f(42).g(43) + // ^ range + // ^^^^^^^^^^^^^ enclosing_range + // ``` + repeated int32 enclosing_range = 7; +} + +// Represents a diagnostic, such as a compiler error or warning, which should be +// reported for a document. +message Diagnostic { + // Should this diagnostic be reported as an error, warning, info, or hint? + Severity severity = 1; + // (optional) Code of this diagnostic, which might appear in the user interface. + string code = 2; + // Message of this diagnostic. + string message = 3; + // (optional) Human-readable string describing the source of this diagnostic, e.g. + // 'typescript' or 'super lint'. + string source = 4; + repeated DiagnosticTag tags = 5; +} + +enum Severity { + UnspecifiedSeverity = 0; + Error = 1; + Warning = 2; + Information = 3; + Hint = 4; +} + +enum DiagnosticTag { + UnspecifiedDiagnosticTag = 0; + Unnecessary = 1; + Deprecated = 2; +} + +// Language standardises names of common programming languages that can be used +// for the `Document.language` field. The primary purpose of this enum is to +// prevent a situation where we have a single programming language ends up with +// multiple string representations. For example, the C++ language uses the name +// "CPP" in this enum and other names such as "cpp" are incompatible. +// Feel free to send a pull-request to add missing programming languages. +enum Language { + UnspecifiedLanguage = 0; + ABAP = 60; + Apex = 96; + APL = 49; + Ada = 39; + Agda = 45; + AsciiDoc = 86; + Assembly = 58; + Awk = 66; + Bat = 68; + BibTeX = 81; + C = 34; + COBOL = 59; + CPP = 35; // C++ (the name "CPP" was chosen for consistency with LSP) + CSS = 26; + CSharp = 1; + Clojure = 8; + Coffeescript = 21; + CommonLisp = 9; + Coq = 47; + CUDA = 97; + Dart = 3; + Delphi = 57; + Diff = 88; + Dockerfile = 80; + Dyalog = 50; + Elixir = 17; + Erlang = 18; + FSharp = 42; + Fish = 65; + Flow = 24; + Fortran = 56; + Git_Commit = 91; + Git_Config = 89; + Git_Rebase = 92; + Go = 33; + GraphQL = 98; + Groovy = 7; + HTML = 30; + Hack = 20; + Handlebars = 90; + Haskell = 44; + Idris = 46; + Ini = 72; + J = 51; + JSON = 75; + Java = 6; + JavaScript = 22; + JavaScriptReact = 93; + Jsonnet = 76; + Julia = 55; + Justfile = 109; + Kotlin = 4; + LaTeX = 83; + Lean = 48; + Less = 27; + Lua = 12; + Luau = 108; + Makefile = 79; + Markdown = 84; + Matlab = 52; + Nickel = 110; // https://nickel-lang.org/ + Nix = 77; + OCaml = 41; + Objective_C = 36; + Objective_CPP = 37; + Pascal = 99; + PHP = 19; + PLSQL = 70; + Perl = 13; + PowerShell = 67; + Prolog = 71; + Protobuf = 100; + Python = 15; + R = 54; + Racket = 11; + Raku = 14; + Razor = 62; + Repro = 102; // Internal language for testing SCIP + ReST = 85; + Ruby = 16; + Rust = 40; + SAS = 61; + SCSS = 29; + SML = 43; + SQL = 69; + Sass = 28; + Scala = 5; + Scheme = 10; + ShellScript = 64; // Bash + Skylark = 78; + Slang = 107; + Solidity = 95; + Svelte = 106; + Swift = 2; + Tcl = 101; + TOML = 73; + TeX = 82; + Thrift = 103; + TypeScript = 23; + TypeScriptReact = 94; + Verilog = 104; + VHDL = 105; + VisualBasic = 63; + Vue = 25; + Wolfram = 53; + XML = 31; + XSL = 32; + YAML = 74; + Zig = 38; + // NextLanguage = 111; + // Steps add a new language: + // 1. Copy-paste the "NextLanguage = N" line above + // 2. Increment "NextLanguage = N" to "NextLanguage = N+1" + // 3. Replace "NextLanguage = N" with the name of the new language. + // 4. Move the new language to the correct line above using alphabetical order + // 5. (optional) Add a brief comment behind the language if the name is not self-explanatory +} diff --git a/klaus/scip_index.py b/klaus/scip_index.py new file mode 100644 index 00000000..a6a333fd --- /dev/null +++ b/klaus/scip_index.py @@ -0,0 +1,159 @@ +"""Loader for SCIP (Sourcegraph Code Intelligence Protocol) indexes. + +SCIP dumps live outside the git tree, in a per-repo cache directory: +``/.scip/.scip`` for a specific commit, with an +``HEAD.scip`` fallback. + +The loader returns an :class:`Index` wrapper that exposes per-document +occurrence lookup and per-symbol definition lookup. Indexes are cached +in-memory keyed by ``(repo_path, sha)``. +""" + +import os +import threading +from collections import OrderedDict +from dataclasses import dataclass +from typing import Dict, List, Optional, Tuple + +from klaus import scip_pb2 # type: ignore[attr-defined] + +_DEFINITION_ROLE: int = scip_pb2.SymbolRole.Definition # type: ignore[attr-defined] + +_CACHE_LOCK = threading.Lock() +_CACHE: "OrderedDict[Tuple[str, str], Optional[Index]]" = OrderedDict() +_CACHE_MAX_ENTRIES = 16 + + +@dataclass(frozen=True) +class Range: + """Half-open [start, end) range in a document, 0-based.""" + + start_line: int + start_char: int + end_line: int + end_char: int + + @classmethod + def from_proto(cls, raw: List[int]) -> "Range": + if len(raw) == 4: + return cls(raw[0], raw[1], raw[2], raw[3]) + if len(raw) == 3: + return cls(raw[0], raw[1], raw[0], raw[2]) + raise ValueError(f"SCIP range must have 3 or 4 elements, got {len(raw)}") + + +@dataclass(frozen=True) +class Occurrence: + range: Range + symbol: str + syntax_kind: int + is_definition: bool + + +@dataclass(frozen=True) +class Definition: + """Location of a symbol's definition within the index.""" + + relative_path: str + line: int # 0-based + + +class Document: + """Wrapper around a SCIP ``Document`` with sorted occurrences.""" + + def __init__(self, relative_path: str, occurrences: List[Occurrence]): + self.relative_path = relative_path + self.occurrences = sorted( + occurrences, + key=lambda o: (o.range.start_line, o.range.start_char), + ) + + def occurrences_on_line(self, line: int) -> List[Occurrence]: + # Linear scan is fine for typical line counts; occurrences are sorted. + return [o for o in self.occurrences if o.range.start_line == line] + + +class Index: + """Parsed SCIP index providing document and symbol lookup.""" + + def __init__(self, proto: "scip_pb2.Index"): # type: ignore[name-defined] + self._documents: Dict[str, Document] = {} + self._definitions: Dict[str, Definition] = {} + + for doc in proto.documents: + occurrences = [ + Occurrence( + range=Range.from_proto(list(occ.range)), + symbol=occ.symbol, + syntax_kind=occ.syntax_kind, + is_definition=bool(occ.symbol_roles & _DEFINITION_ROLE), + ) + for occ in doc.occurrences + ] + self._documents[doc.relative_path] = Document( + doc.relative_path, occurrences + ) + for occ in occurrences: + if ( + occ.is_definition + and occ.symbol + and occ.symbol not in self._definitions + ): + self._definitions[occ.symbol] = Definition( + relative_path=doc.relative_path, + line=occ.range.start_line, + ) + + def get_document(self, relative_path: str) -> Optional[Document]: + return self._documents.get(relative_path) + + def get_definition(self, symbol: str) -> Optional[Definition]: + return self._definitions.get(symbol) + + +def _scip_path_for(repo_path: str, sha: str) -> Optional[str]: + scip_dir = os.path.join(repo_path, ".scip") + if not os.path.isdir(scip_dir): + return None + candidate = os.path.join(scip_dir, f"{sha}.scip") + if os.path.isfile(candidate): + return candidate + fallback = os.path.join(scip_dir, "HEAD.scip") + if os.path.isfile(fallback): + return fallback + return None + + +def load_index(repo_path: str, sha: str) -> Optional[Index]: + """Return the SCIP index for ``sha`` in ``repo_path``, or ``None``. + + Returns ``None`` (cached) when no SCIP dump is available, so callers + don't probe the filesystem on every request. + """ + key = (repo_path, sha) + with _CACHE_LOCK: + if key in _CACHE: + _CACHE.move_to_end(key) + return _CACHE[key] + + path = _scip_path_for(repo_path, sha) + index: Optional[Index] + if path is None: + index = None + else: + proto = scip_pb2.Index() # type: ignore[attr-defined] + with open(path, "rb") as f: + proto.ParseFromString(f.read()) + index = Index(proto) + + with _CACHE_LOCK: + _CACHE[key] = index + _CACHE.move_to_end(key) + while len(_CACHE) > _CACHE_MAX_ENTRIES: + _CACHE.popitem(last=False) + return index + + +def clear_cache() -> None: + with _CACHE_LOCK: + _CACHE.clear() diff --git a/klaus/static/scip.css b/klaus/static/scip.css new file mode 100644 index 00000000..6da3b55e --- /dev/null +++ b/klaus/static/scip.css @@ -0,0 +1,39 @@ +/* Syntax classes emitted by the SCIP-based code renderer. Mirrors the + * Trac-ish look of pygments.css so the two paths blend visually. */ + +.code .scip-comment { color: #999988; font-style: italic; } +.code .scip-punctuation-delimiter { color: inherit; } +.code .scip-punctuation-bracket { color: inherit; } +.code .scip-keyword { font-weight: bold; } +.code .scip-identifier-operator { font-weight: bold; } +.code .scip-identifier { color: inherit; } +.code .scip-identifier-builtin { color: #999999; } +.code .scip-identifier-null { font-weight: bold; } +.code .scip-identifier-constant { color: #008080; } +.code .scip-identifier-mutable-global { color: #008080; } +.code .scip-identifier-parameter { color: inherit; } +.code .scip-identifier-local { color: inherit; } +.code .scip-identifier-shadowed { color: inherit; } +.code .scip-identifier-namespace { color: #555555; } +.code .scip-identifier-function { color: #990000; font-weight: bold; } +.code .scip-identifier-function-definition{ color: #990000; font-weight: bold; } +.code .scip-identifier-macro { color: #990000; font-weight: bold; } +.code .scip-identifier-macro-definition { color: #990000; font-weight: bold; } +.code .scip-identifier-type { color: #445588; font-weight: bold; } +.code .scip-identifier-builtin-type { color: #445588; font-weight: bold; } +.code .scip-identifier-attribute { color: #008080; } +.code .scip-regex-escape { color: #808000; } +.code .scip-regex-repeated { color: #808000; } +.code .scip-regex-wildcard { color: #808000; } +.code .scip-regex-delimiter { color: #808000; } +.code .scip-regex-join { color: #808000; } +.code .scip-string-literal { color: #bb8844; } +.code .scip-string-literal-escape { color: #bb8844; font-weight: bold; } +.code .scip-string-literal-special { color: #bb8844; font-weight: bold; } +.code .scip-string-literal-key { color: #bb8844; } +.code .scip-character-literal { color: #bb8844; } +.code .scip-numeric-literal { color: #009999; } +.code .scip-boolean-literal { font-weight: bold; } +.code .scip-tag { color: #000080; } +.code .scip-tag-attribute { color: #008080; } +.code .scip-tag-delimiter { color: inherit; } diff --git a/klaus/templates/skeleton.html b/klaus/templates/skeleton.html index 1768733c..6fd6bf4f 100644 --- a/klaus/templates/skeleton.html +++ b/klaus/templates/skeleton.html @@ -2,6 +2,7 @@ + {% if base_href %} diff --git a/klaus/views.py b/klaus/views.py index 466a3f9d..a3f3766c 100644 --- a/klaus/views.py +++ b/klaus/views.py @@ -1,5 +1,4 @@ import os -import sys from io import BytesIO import dulwich.archive @@ -12,16 +11,7 @@ from werkzeug.exceptions import NotFound from werkzeug.wrappers import Response -try: - import ctags -except ImportError: - ctags = None -else: - from klaus import ctagscache - - CTAGS_CACHE = ctagscache.CTagsCache() - -from klaus import markup +from klaus import markup, scip_index from klaus.highlighting import highlight_or_render from klaus.utils import ( encode_for_git, @@ -416,35 +406,30 @@ class BaseFileView(TreeViewMixin, BaseBlobView): """Base for FileView and BlameView.""" def render_code(self, render_markup): - should_use_ctags = current_app.should_use_ctags( - self.context["repo"], self.context["commit"] - ) - if should_use_ctags: - if ctags is None: - raise ImportError("Ctags enabled but python-ctags not installed") - ctags_base_url = url_for( - self.view_name, - repo=self.context["repo"].namespaced_name, - rev=self.context["rev"], - path="", - ) - ctags_tagsfile = CTAGS_CACHE.get_tagsfile( - self.context["repo"].path, self.context["commit"].id - ) - ctags_args = { - "ctags": ctags.CTags( - ctags_tagsfile.encode(sys.getfilesystemencoding()) - ), - "ctags_baseurl": ctags_base_url, - } - else: - ctags_args = {} + repo = self.context["repo"] + commit = self.context["commit"] + + scip_args = {} + index = scip_index.load_index(repo.path, commit.id.decode("ascii")) + if index is not None: + document = index.get_document(self.context["path"]) + if document is not None: + scip_args = { + "scip_document": document, + "scip_index": index, + "scip_baseurl": url_for( + self.view_name, + repo=repo.namespaced_name, + rev=self.context["rev"], + path="", + ), + } return highlight_or_render( force_unicode(self.context["blob_or_tree"].data), self.context["filename"], render_markup, - **ctags_args, + **scip_args, ) def make_template_context(self, *args): diff --git a/pyproject.toml b/pyproject.toml index e4736f2f..e065adbb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,12 +1,18 @@ [build-system] -requires = ["setuptools"] +requires = ["setuptools", "setuptools-protobuf>=0.1.13"] build-backend = "setuptools.build_meta" +[tool.setuptools-protobuf] +protobufs = ["klaus/scip.proto"] + [tool.mypy] ignore_missing_imports = true [tool.ruff] target-version = "py37" +line-length = 88 +extend-exclude = ["klaus/scip_pb2.py"] + +[tool.ruff.lint] select = ["E", "F", "I", "UP"] ignore = ["F405", "F403", "E501"] -line-length = 88 diff --git a/setup.py b/setup.py index 2d1f594c..afcd8e28 100644 --- a/setup.py +++ b/setup.py @@ -11,6 +11,7 @@ "httpauth>=0.4", "humanize", "dulwich>=0.19.3", + "protobuf>=3.20", ] setup( diff --git a/test_requirements.txt b/test_requirements.txt index da43373b..bdceb699 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -1,4 +1,3 @@ pytest requests -python-ctags3>=1.2.1 mock; python_version < '3' diff --git a/tests/test_contrib.py b/tests/test_contrib.py index ff731e36..1452f373 100644 --- a/tests/test_contrib.py +++ b/tests/test_contrib.py @@ -29,7 +29,6 @@ def test_minimum_env(monkeypatch): require_browser_auth=False, disable_push=False, unauthenticated_push=False, - ctags_policy="none", ), ) @@ -46,7 +45,6 @@ def test_complete_env(monkeypatch): "KLAUS_REQUIRE_BROWSER_AUTH": "1", "KLAUS_DISABLE_PUSH": "false", "KLAUS_UNAUTHENTICATED_PUSH": "0", - "KLAUS_CTAGS_POLICY": "ALL", }, ([TEST_REPO_NO_NAMESPACE], TEST_SITE_NAME), dict( @@ -55,7 +53,6 @@ def test_complete_env(monkeypatch): require_browser_auth=True, disable_push=False, unauthenticated_push=False, - ctags_policy="ALL", ), ) diff --git a/tests/test_make_app.py b/tests/test_make_app.py index 7a149b15..ce043ba3 100644 --- a/tests/test_make_app.py +++ b/tests/test_make_app.py @@ -1,4 +1,4 @@ -import re +import os import shutil import subprocess import tempfile @@ -8,6 +8,7 @@ import requests.auth import klaus +from klaus import scip_index, scip_pb2 from .utils import * @@ -108,17 +109,6 @@ def test(): }, ) -test_ctags_disabled = options_test( - {}, {"ctags_tags_and_branches": False, "ctags_all": False} -) -test_ctags_tags_and_branches = options_test( - {"ctags_policy": "tags-and-branches"}, - {"ctags_tags_and_branches": True, "ctags_all": False}, -) -test_ctags_all = options_test( - {"ctags_policy": "ALL"}, {"ctags_tags_and_branches": True, "ctags_all": True} -) - # Reach def can_reach_unauth(): @@ -180,30 +170,71 @@ def _can_push(http_get, url): ) -# Ctags -def ctags_tags_and_branches(): - return all( - _ctags_enabled(ref, f) - for ref in ["master", "tag1"] - for f in ["test.c", "test.js"] - ) - - -def ctags_all(): - all_refs = re.findall( - 'href=".+/commit/([a-z0-9]{40})/">', requests.get(UNAUTH_TEST_REPO_URL).text - ) - assert len(all_refs) == 3 - return all( - _ctags_enabled(ref, f) for ref in all_refs for f in ["test.c", "test.js"] - ) +# SCIP +def _write_test_scip_dump(): + """Build a small SCIP dump for the test repo covering test.c and test.js.""" + index = scip_pb2.Index() + index.metadata.version = scip_pb2.UnspecifiedProtocolVersion + index.metadata.text_document_encoding = scip_pb2.UTF8 + index.metadata.project_root = "file://" + TEST_REPO + + c_doc = index.documents.add() + c_doc.relative_path = "test.c" + c_doc.language = "C" + c_int = c_doc.occurrences.add() + c_int.range.extend([0, 0, 0, 3]) + c_int.syntax_kind = scip_pb2.IdentifierBuiltinType + c_a = c_doc.occurrences.add() + c_a.range.extend([0, 4, 0, 5]) + c_a.symbol = "scip-test c/a." + c_a.symbol_roles = scip_pb2.Definition + c_a.syntax_kind = scip_pb2.IdentifierConstant + + js_doc = index.documents.add() + js_doc.relative_path = "test.js" + js_doc.language = "JavaScript" + js_kw = js_doc.occurrences.add() + js_kw.range.extend([0, 0, 0, 8]) + js_kw.syntax_kind = scip_pb2.IdentifierKeyword + js_def = js_doc.occurrences.add() + js_def.range.extend([0, 9, 0, 13]) + js_def.symbol = "scip-test js/test()." + js_def.symbol_roles = scip_pb2.Definition + js_def.syntax_kind = scip_pb2.IdentifierFunctionDefinition + + scip_dir = os.path.join(TEST_REPO, ".scip") + os.makedirs(scip_dir, exist_ok=True) + with open(os.path.join(scip_dir, "HEAD.scip"), "wb") as f: + f.write(index.SerializeToString()) + scip_index.clear_cache() + + +def _remove_test_scip_dump(): + scip_dir = os.path.join(TEST_REPO, ".scip") + shutil.rmtree(scip_dir, ignore_errors=True) + scip_index.clear_cache() + + +def test_scip_renders_syntax_classes(): + _write_test_scip_dump() + try: + with serve(): + response = requests.get(UNAUTH_TEST_REPO_URL + "blob/master/test.c") + assert response.status_code == 200, response.text + assert "scip-identifier-builtin-type" in response.text + assert "scip-identifier-constant" in response.text + finally: + _remove_test_scip_dump() -def _ctags_enabled(ref, filename): - response = requests.get(UNAUTH_TEST_REPO_URL + f"blob/{ref}/{filename}") - assert response.status_code == 200, response.text - href = f'' - return href in response.text +def test_scip_falls_back_to_pygments_when_no_dump(): + scip_index.clear_cache() + with serve(): + response = requests.get(UNAUTH_TEST_REPO_URL + "blob/master/test.c") + assert response.status_code == 200, response.text + # Pygments emits a with linenos but no scip- classes. + assert "scip-" not in response.text + assert 'class="highlighttable"' in response.text def _GET_unauth(url=""): From 1028945ec9f783c5a84aec93a748d2892cf2b7fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Wed, 27 May 2026 13:41:03 +0100 Subject: [PATCH 2/5] Add on-demand SCIP generation New flag --scip {none,tags-and-branches,ALL} (and KLAUS_SCIP_POLICY env var) controls when klaus should generate a SCIP index on demand for an un-indexed revision. When enabled, the first request kicks off a background job that: 1. Adds a git worktree at a tempdir checked out to the requested commit. 2. Runs the first matching indexer (scip-python is the built-in). 3. Moves index.scip to /.scip/.scip. 4. Removes the worktree. Subsequent requests for the same revision pick up the dump. Concurrent requests for the same (repo, sha) share a single in-process thread via a lock. Indexers are pluggable via klaus.scip_generate.INDEXERS. --- CHANGELOG.rst | 7 ++ README.rst | 2 +- klaus.1 | 4 + klaus/__init__.py | 32 +++++-- klaus/cli.py | 15 ++++ klaus/contrib/app_args.py | 1 + klaus/scip_generate.py | 183 ++++++++++++++++++++++++++++++++++++++ klaus/views.py | 7 +- pyproject.toml | 2 +- tests/test_contrib.py | 3 + tests/test_make_app.py | 76 +++++++++++++++- 11 files changed, 321 insertions(+), 11 deletions(-) create mode 100644 klaus/scip_generate.py diff --git a/CHANGELOG.rst b/CHANGELOG.rst index b3cf8b1e..a3d00770 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -8,6 +8,13 @@ UNRELEASED syntax classes and cross-reference links. Files not covered by SCIP fall back to Pygments syntax highlighting. The ``--ctags`` CLI flag and ``KLAUS_CTAGS_POLICY`` env var are gone. +- Add ``--scip`` / ``KLAUS_SCIP_POLICY`` ({none, tags-and-branches, ALL}, + default none) for on-demand SCIP generation. When enabled, the first + request for an un-indexed revision triggers a background job that runs + an indexer inside a ``git worktree`` and writes the dump to + ``/.scip/.scip``; subsequent requests pick it up. The default + indexer is ``scip-python``; the indexer list is pluggable via + ``klaus.scip_generate.INDEXERS``. 3.0.1 (Jun 17, 2024) -------------------- diff --git a/README.rst b/README.rst index f47c2615..235e6c57 100644 --- a/README.rst +++ b/README.rst @@ -16,7 +16,7 @@ klaus: a simple, easy-to-set-up Git web viewer that Just Works™. * Syntax highlighting * Markdown + RestructuredText rendering support * Pull + push support (Git Smart HTTP) -* Code navigation using SCIP indexes (drop ``index.scip`` files into ``/.scip/``) +* Code navigation using SCIP indexes (drop ``index.scip`` files into ``/.scip/``, or use ``--scip`` to generate on demand) :Demo: https://github.com/jonashaag/klaus/wiki/Sites-using-klaus :On PyPI: http://pypi.python.org/pypi/klaus/ diff --git a/klaus.1 b/klaus.1 index 4b92e795..33218ca3 100644 --- a/klaus.1 +++ b/klaus.1 @@ -34,6 +34,10 @@ open klaus in a browser on server start .TP \fB\-B\fR BROWSER, \fB\-\-with\-browser\fR BROWSER specify which browser to use with \fB\-\-browser\fR +.TP +\fB\-\-scip\fR {none,tags\-and\-branches,ALL} +generate SCIP indexes on demand for which revisions? default: none. +WARNING: Don't use 'ALL' for public servers! .SS "Git Smart HTTP:" .TP \fB\-\-smarthttp\fR diff --git a/klaus/__init__.py b/klaus/__init__.py index 709e587b..d4d39b6b 100644 --- a/klaus/__init__.py +++ b/klaus/__init__.py @@ -23,10 +23,11 @@ class Klaus(flask.Flask): "undefined": jinja2.StrictUndefined, } - def __init__(self, repo_paths, site_name, use_smarthttp): + def __init__(self, repo_paths, site_name, use_smarthttp, scip_policy="none"): """(See `make_app` for parameter descriptions.)""" self.site_name = site_name self.use_smarthttp = use_smarthttp + self.scip_policy = scip_policy valid_repos, invalid_repos = self.load_repos(repo_paths) self.valid_repos = {repo.namespaced_name: repo for repo in valid_repos} @@ -84,6 +85,15 @@ def setup_routes(self): ) # fmt: on + def should_generate_scip(self, git_repo, git_commit): + if self.scip_policy == "none": + return False + if self.scip_policy == "ALL": + return True + if self.scip_policy == "tags-and-branches": + return git_commit.id in git_repo.get_tag_and_branch_shas() + raise ValueError(f"Unknown scip policy {self.scip_policy!r}") + def load_repos(self, repo_paths): valid_repos = [] invalid_repos = [] @@ -104,6 +114,7 @@ def make_app( require_browser_auth=False, disable_push=False, unauthenticated_push=False, + scip_policy="none", ): """ Returns a WSGI app with all the features (smarthttp, authentication) @@ -128,11 +139,19 @@ def make_app( are set, but push should not be supported. :param htdigest_file: A *file-like* object that contains the HTTP auth credentials. :param unauthenticated_push: Allow push'ing without authentication. DANGER ZONE! - - Code intelligence (cross-references and syntax classes) is enabled - automatically when a SCIP index is present at ``/.scip/.scip`` - (with ``HEAD.scip`` as a fallback). When no index is present, files are - rendered with Pygments syntax highlighting. + :param scip_policy: When to generate SCIP indexes on demand for revisions + that don't yet have one. Pre-generated dumps under + ``/.scip/.scip`` are always used. When no dump exists: + + - ``'none'``: never generate; render with Pygments. + - ``'tags-and-branches'``: generate for revisions that are the HEAD of + a tag or branch. + - ``'ALL'``: generate for every revision. May result in high server + load; don't use for public servers. + + Generation happens in a background thread and uses ``git worktree``, + so the first request renders with Pygments and subsequent requests + for the same revision pick up the SCIP index. """ if unauthenticated_push: if not use_smarthttp: @@ -155,6 +174,7 @@ def make_app( repo_paths, site_name, use_smarthttp, + scip_policy=scip_policy, ) app.wsgi_app = utils.ProxyFix(app.wsgi_app) diff --git a/klaus/cli.py b/klaus/cli.py index c157bc1e..cad95d98 100644 --- a/klaus/cli.py +++ b/klaus/cli.py @@ -1,4 +1,5 @@ import argparse +import logging import os import sys import webbrowser @@ -43,6 +44,14 @@ def make_parser(): metavar="BROWSER", default=None, ) + parser.add_argument( + "--scip", + help="generate SCIP indexes on demand for which revisions? " + "default: none. WARNING: Don't use 'ALL' for public servers!", + choices=["none", "tags-and-branches", "ALL"], + default="none", + ) + parser.add_argument( "repos", help="repositories to serve", @@ -73,6 +82,11 @@ def make_parser(): def main(): args = make_parser().parse_args() + logging.basicConfig( + level=logging.DEBUG if args.debug else logging.INFO, + format="%(asctime)s %(levelname)s %(name)s: %(message)s", + ) + if args.version: print(KLAUS_VERSION) return 0 @@ -98,6 +112,7 @@ def main(): force_unicode(args.site_name or args.host), args.smarthttp, args.htdigest, + scip_policy=args.scip, ) if args.browser: diff --git a/klaus/contrib/app_args.py b/klaus/contrib/app_args.py index a849ca93..4ece84c5 100644 --- a/klaus/contrib/app_args.py +++ b/klaus/contrib/app_args.py @@ -26,5 +26,6 @@ def get_args_from_env(): unauthenticated_push=strtobool( os.environ.get("KLAUS_UNAUTHENTICATED_PUSH", "0") ), + scip_policy=os.environ.get("KLAUS_SCIP_POLICY", "none"), ) return args, kwargs diff --git a/klaus/scip_generate.py b/klaus/scip_generate.py new file mode 100644 index 00000000..b8f2d155 --- /dev/null +++ b/klaus/scip_generate.py @@ -0,0 +1,183 @@ +"""On-demand SCIP index generation. + +When ``scip_policy`` allows it (see :func:`klaus.Klaus.should_generate_scip`), +:func:`request_index` schedules a background job that: + +1. Adds a ``git worktree`` at a tempdir checked out to the requested commit. +2. Runs the first matching indexer (see :data:`INDEXERS`) inside the + worktree. +3. Moves the resulting ``index.scip`` to ``/.scip/.scip``. +4. Tears the worktree down. + +Subsequent requests for the same revision pick up the dump via +:mod:`klaus.scip_index`. Concurrent requests for the same ``(repo, sha)`` +share a single background job via an in-process lock. +""" + +import logging +import os +import shutil +import subprocess +import tempfile +import threading +import time +from collections.abc import Callable +from dataclasses import dataclass + +from klaus import scip_index + +logger = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class Indexer: + """A SCIP indexer for some language family. + + :param name: human-readable name shown in logs. + :param applies: predicate run against a checked-out worktree path; returns + ``True`` if this indexer should handle the project. + :param command: argv to invoke inside the worktree. Must write + ``index.scip`` to the current working directory. + """ + + name: str + applies: Callable[[str], bool] + command: list[str] + + +def _looks_like_python(worktree: str) -> bool: + for marker in ("pyproject.toml", "setup.py", "setup.cfg"): + if os.path.isfile(os.path.join(worktree, marker)): + return True + for root, _, files in os.walk(worktree): + # Skip hidden dirs (notably .git, .scip) + if os.path.basename(root).startswith("."): + continue + if any(f.endswith(".py") for f in files): + return True + return False + + +#: Built-in indexers, tried in order. External integrators can append to this +#: list before constructing the app. +INDEXERS: list[Indexer] = [ + Indexer( + name="scip-python", + applies=_looks_like_python, + command=["scip-python", "index", "."], + ), +] + + +_LOCK = threading.Lock() +_INFLIGHT: dict[tuple[str, str], threading.Thread] = {} + + +def request_index(repo_path: str, sha: str) -> None: + """Schedule background generation of ``/.scip/.scip``. + + Returns immediately. Idempotent: if a generation job is already running + for ``(repo_path, sha)``, or if the dump already exists, this is a no-op. + """ + sha = sha.lower() + if os.path.isfile(_dump_path(repo_path, sha)): + return + key = (repo_path, sha) + with _LOCK: + if key in _INFLIGHT and _INFLIGHT[key].is_alive(): + logger.debug( + "scip: %s@%s already being indexed; skipping", + repo_path, + sha[:7], + ) + return + thread = threading.Thread( + target=_run, + args=(repo_path, sha), + name=f"scip-index {os.path.basename(repo_path)}@{sha[:7]}", + daemon=True, + ) + _INFLIGHT[key] = thread + logger.info("scip: scheduled index for %s@%s", repo_path, sha[:7]) + thread.start() + + +def _dump_path(repo_path: str, sha: str) -> str: + return os.path.join(repo_path, ".scip", f"{sha}.scip") + + +def _run(repo_path: str, sha: str) -> None: + started = time.monotonic() + logger.info("scip: starting index for %s@%s", repo_path, sha[:7]) + try: + _generate(repo_path, sha) + except Exception: + elapsed = time.monotonic() - started + logger.exception( + "scip: index for %s@%s failed after %.1fs", repo_path, sha[:7], elapsed + ) + else: + elapsed = time.monotonic() - started + logger.info( + "scip: finished index for %s@%s in %.1fs", repo_path, sha[:7], elapsed + ) + finally: + with _LOCK: + _INFLIGHT.pop((repo_path, sha), None) + scip_index.clear_cache() + + +def _generate(repo_path: str, sha: str) -> None: + dump_path = _dump_path(repo_path, sha) + if os.path.isfile(dump_path): + return + + worktree = tempfile.mkdtemp(prefix="klaus-scip-") + # `git worktree add` will fail if the destination directory already + # exists; mkdtemp created it, so remove it first. + os.rmdir(worktree) + try: + subprocess.check_call( + ["git", "worktree", "add", "--detach", worktree, sha], + cwd=repo_path, + ) + try: + indexer = _pick_indexer(worktree) + if indexer is None: + logger.info( + "scip: no matching indexer for %s@%s; skipping", + repo_path, + sha[:7], + ) + return + logger.info("scip: running %s for %s@%s", indexer.name, repo_path, sha[:7]) + subprocess.check_call(indexer.command, cwd=worktree) + produced = os.path.join(worktree, "index.scip") + if not os.path.isfile(produced): + raise RuntimeError( + f"{indexer.name} did not produce index.scip in {worktree}" + ) + os.makedirs(os.path.dirname(dump_path), exist_ok=True) + shutil.move(produced, dump_path) + logger.debug("scip: wrote %s", dump_path) + finally: + subprocess.call( + ["git", "worktree", "remove", "--force", worktree], + cwd=repo_path, + ) + finally: + # `git worktree remove` should have cleaned up, but if it didn't + # (e.g. it was never created), drop the directory ourselves. + if os.path.isdir(worktree): + shutil.rmtree(worktree, ignore_errors=True) + + +def _pick_indexer(worktree: str) -> Indexer | None: + for indexer in INDEXERS: + try: + applies = indexer.applies(worktree) + except Exception: + applies = False + if applies: + return indexer + return None diff --git a/klaus/views.py b/klaus/views.py index a3f3766c..7bbaa36d 100644 --- a/klaus/views.py +++ b/klaus/views.py @@ -11,7 +11,7 @@ from werkzeug.exceptions import NotFound from werkzeug.wrappers import Response -from klaus import markup, scip_index +from klaus import markup, scip_generate, scip_index from klaus.highlighting import highlight_or_render from klaus.utils import ( encode_for_git, @@ -408,9 +408,10 @@ class BaseFileView(TreeViewMixin, BaseBlobView): def render_code(self, render_markup): repo = self.context["repo"] commit = self.context["commit"] + sha = commit.id.decode("ascii") scip_args = {} - index = scip_index.load_index(repo.path, commit.id.decode("ascii")) + index = scip_index.load_index(repo.path, sha) if index is not None: document = index.get_document(self.context["path"]) if document is not None: @@ -424,6 +425,8 @@ def render_code(self, render_markup): path="", ), } + elif current_app.should_generate_scip(repo, commit): + scip_generate.request_index(repo.path, sha) return highlight_or_render( force_unicode(self.context["blob_or_tree"].data), diff --git a/pyproject.toml b/pyproject.toml index e065adbb..c7d53b03 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ protobufs = ["klaus/scip.proto"] ignore_missing_imports = true [tool.ruff] -target-version = "py37" +target-version = "py310" line-length = 88 extend-exclude = ["klaus/scip_pb2.py"] diff --git a/tests/test_contrib.py b/tests/test_contrib.py index 1452f373..3243bf9d 100644 --- a/tests/test_contrib.py +++ b/tests/test_contrib.py @@ -29,6 +29,7 @@ def test_minimum_env(monkeypatch): require_browser_auth=False, disable_push=False, unauthenticated_push=False, + scip_policy="none", ), ) @@ -45,6 +46,7 @@ def test_complete_env(monkeypatch): "KLAUS_REQUIRE_BROWSER_AUTH": "1", "KLAUS_DISABLE_PUSH": "false", "KLAUS_UNAUTHENTICATED_PUSH": "0", + "KLAUS_SCIP_POLICY": "ALL", }, ([TEST_REPO_NO_NAMESPACE], TEST_SITE_NAME), dict( @@ -53,6 +55,7 @@ def test_complete_env(monkeypatch): require_browser_auth=True, disable_push=False, unauthenticated_push=False, + scip_policy="ALL", ), ) diff --git a/tests/test_make_app.py b/tests/test_make_app.py index ce043ba3..df23ae69 100644 --- a/tests/test_make_app.py +++ b/tests/test_make_app.py @@ -1,14 +1,16 @@ import os import shutil import subprocess +import sys import tempfile +import time import pytest import requests import requests.auth import klaus -from klaus import scip_index, scip_pb2 +from klaus import scip_generate, scip_index, scip_pb2 from .utils import * @@ -237,6 +239,78 @@ def test_scip_falls_back_to_pygments_when_no_dump(): assert 'class="highlighttable"' in response.text +def test_scip_policy_gating(): + app = klaus.Klaus({None: [TEST_REPO]}, TEST_SITE_NAME, False, scip_policy="none") + repo = next(iter(app.valid_repos.values())) + commit = repo.get_commit("master") + assert app.should_generate_scip(repo, commit) is False + + app = klaus.Klaus({None: [TEST_REPO]}, TEST_SITE_NAME, False, scip_policy="ALL") + assert app.should_generate_scip(repo, commit) is True + + app = klaus.Klaus( + {None: [TEST_REPO]}, TEST_SITE_NAME, False, scip_policy="tags-and-branches" + ) + # master is a branch HEAD; tag1 is a tag. Both should be eligible. + assert app.should_generate_scip(repo, commit) is True + tag_commit = repo.get_commit("tag1") + assert app.should_generate_scip(repo, tag_commit) is True + # A commit that isn't the HEAD of any ref shouldn't be eligible. The + # root commit (master~~) has no tag or branch pointing at it. + root = repo.get_commit(repo.get_commit("tag1").parents[0].decode("ascii")) + assert app.should_generate_scip(repo, root) is False + + +def test_scip_generate_via_worktree(monkeypatch): + """Background generation should produce /.scip/.scip via a + fake indexer running in a `git worktree add` tempdir.""" + + scip_index.clear_cache() + _remove_test_scip_dump() + + fake_index = scip_pb2.Index() + fake_doc = fake_index.documents.add() + fake_doc.relative_path = "test.c" + fake_occ = fake_doc.occurrences.add() + fake_occ.range.extend([0, 0, 0, 3]) + fake_occ.syntax_kind = scip_pb2.IdentifierBuiltinType + payload = fake_index.SerializeToString().hex() + fake_indexer = scip_generate.Indexer( + name="fake", + applies=lambda _: True, + command=[ + sys.executable, + "-c", + f"open('index.scip','wb').write(bytes.fromhex('{payload}'))", + ], + ) + monkeypatch.setattr(scip_generate, "INDEXERS", [fake_indexer]) + + import dulwich.repo + + sha = dulwich.repo.Repo(TEST_REPO).head().decode("ascii") + dump_path = os.path.join(TEST_REPO, ".scip", f"{sha}.scip") + + try: + scip_generate.request_index(TEST_REPO, sha) + # Wait for the background thread to finish. + for _ in range(200): + with scip_generate._LOCK: + thread = scip_generate._INFLIGHT.get((TEST_REPO, sha)) + if thread is None or not thread.is_alive(): + break + time.sleep(0.05) + else: + raise AssertionError("indexer thread didn't finish") + + assert os.path.isfile(dump_path), "expected dump at %s" % dump_path + index = scip_index.load_index(TEST_REPO, sha) + assert index is not None + assert index.get_document("test.c") is not None + finally: + _remove_test_scip_dump() + + def _GET_unauth(url=""): return requests.get( UNAUTH_TEST_SERVER + url, From e68813cd16a0deb57bbe8ed091eba4b2c61d7d18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Wed, 27 May 2026 14:06:54 +0100 Subject: [PATCH 3/5] Restore Pygments syntax highlighting when SCIP is present --- klaus/highlighting.py | 232 ++++++++++++++++++++++++++++-------------- 1 file changed, 156 insertions(+), 76 deletions(-) diff --git a/klaus/highlighting.py b/klaus/highlighting.py index 954fe092..6e05f692 100644 --- a/klaus/highlighting.py +++ b/klaus/highlighting.py @@ -5,11 +5,13 @@ (without ctags, since cross-references in the Pygments path are gone). """ +from collections.abc import Iterator from html import escape -from typing import Any, Iterator, Optional, Tuple +from typing import Any -from pygments import highlight +from pygments import highlight, lex from pygments.formatters import HtmlFormatter +from pygments.formatters.html import _get_ttype_class from pygments.lexers import ( ClassNotFound, TextLexer, @@ -75,35 +77,166 @@ def __init__(self, **kwargs: Any) -> None: **kwargs, ) - def _format_lines(self, tokensource: Any) -> Iterator[Tuple[int, str]]: + def _format_lines(self, tokensource: Any) -> Iterator[tuple[int, str]]: for tag, line in super()._format_lines(tokensource): # type: ignore[misc] if tag == 1: line = f"{line}" yield tag, line +def _tokenize_per_line(source: str, lexer: Any) -> list[list[tuple[int, int, str]]]: + """Tokenize ``source`` with Pygments and produce, for each line, a list of + ``(start_col, end_col, css_class)`` segments covering the full line. + + Segments are non-overlapping and end-exclusive; they together cover the + whole line including whitespace (whitespace segments may have an empty + css_class). + """ + lines: list[list[tuple[int, int, str]]] = [[]] + col = 0 + for ttype, text in lex(source, lexer): + if not text: + continue + css_class = _get_ttype_class(ttype) or "" + # A token's text may contain newlines; split so each segment stays on + # one line. + parts = text.split("\n") + for i, part in enumerate(parts): + if part: + lines[-1].append((col, col + len(part), css_class)) + col += len(part) + if i < len(parts) - 1: + lines.append([]) + col = 0 + # If the source ended with a newline, splitlines drops the trailing empty + # line; mirror that for consistency with the caller's view. + if lines and not lines[-1] and source and not source.endswith("\n"): + lines.pop() + elif source.endswith("\n") and lines and not lines[-1]: + lines.pop() + return lines + + +def _splice_occurrences( + segments: list[tuple[int, int, str]], + occurrences: list[Occurrence], +) -> list[tuple[int, int, str, Occurrence | None]]: + """Layer SCIP occurrences over Pygments-derived segments. + + The result is a list of ``(start, end, css_class, occurrence)`` tuples + where ``occurrence`` is set when the segment lies inside a SCIP + occurrence's range. Pygments segments are split at occurrence boundaries + as needed so each output segment is fully inside or outside an occurrence. + """ + if not occurrences: + return [(s, e, c, None) for s, e, c in segments] + + # Sort occurrences by start position; assume they don't overlap (SCIP + # normally doesn't emit overlapping ranges on a line). + occurrences = sorted(occurrences, key=lambda o: o.range.start_char) + out: list[tuple[int, int, str, Occurrence | None]] = [] + occ_idx = 0 + for seg_start, seg_end, css in segments: + cursor = seg_start + while cursor < seg_end: + # Skip occurrences entirely before the cursor. + while ( + occ_idx < len(occurrences) + and occurrences[occ_idx].range.end_char <= cursor + ): + occ_idx += 1 + if occ_idx >= len(occurrences): + out.append((cursor, seg_end, css, None)) + cursor = seg_end + break + occ = occurrences[occ_idx] + occ_start = occ.range.start_char + occ_end = occ.range.end_char + if occ_start >= seg_end: + out.append((cursor, seg_end, css, None)) + cursor = seg_end + break + if cursor < occ_start: + out.append((cursor, occ_start, css, None)) + cursor = occ_start + slice_end = min(seg_end, occ_end) + out.append((cursor, slice_end, css, occ)) + cursor = slice_end + return out + + +def _render_segment( + text: str, + pygments_class: str, + occ: Occurrence | None, + index: Index, + scip_baseurl: str, +) -> str: + """Render one slice of a line as HTML.""" + classes: list[str] = [] + if pygments_class: + classes.append(pygments_class) + if occ is not None: + scip_class = _SCIP_SYNTAX_CLASSES.get(occ.syntax_kind) + if scip_class: + classes.append(f"scip-{scip_class}") + if occ.is_definition: + classes.append("scip-definition") + + body = escape(text) + if occ is not None and occ.symbol and not occ.is_definition: + defn = index.get_definition(occ.symbol) + if defn is not None: + href = f"{scip_baseurl}{defn.relative_path}#L-{defn.line + 1}" + body = f'{body}' + if classes: + body = f'{body}' + return body + + def _render_scip_lines( - source: str, document: Document, index: Index, scip_baseurl: str + source: str, + document: Document, + index: Index, + scip_baseurl: str, + lexer: Any, ) -> str: - """Render ``source`` as HTML using occurrences from ``document``. + """Render ``source`` as HTML using Pygments tokens overlaid with SCIP + occurrences. Produces the same outer structure as Pygments' ``linenos='table'`` mode so - klaus's CSS and JS keep working: ``
...`` with linenos column and code column. + klaus's CSS and JS keep working. """ - lines = source.splitlines(keepends=False) - linenos_parts = [] - code_parts = [] - for i, line_text in enumerate(lines, start=1): + raw_lines = source.splitlines(keepends=False) + pygments_segments = _tokenize_per_line(source, lexer) + # Make sure we have one segment list per source line. + while len(pygments_segments) < len(raw_lines): + pygments_segments.append([]) + + linenos_parts: list[str] = [] + code_parts: list[str] = [] + for i, line_text in enumerate(raw_lines, start=1): anchor = f"L-{i}" linenos_parts.append(f'{i}') - rendered_line = _render_scip_line( - line_text, i - 1, document, index, scip_baseurl + segments = pygments_segments[i - 1] + if not segments and line_text: + # Fallback: lexer produced nothing for this line; treat the whole + # line as plain text. + segments = [(0, len(line_text), "")] + occurrences = [ + o + for o in document.occurrences_on_line(i - 1) + if o.range.end_line == i - 1 # skip multi-line ranges + ] + spliced = _splice_occurrences(segments, occurrences) + rendered = "".join( + _render_segment(line_text[s:e], css, occ, index, scip_baseurl) + for s, e, css, occ in spliced ) code_parts.append( f'' f'' - f"{rendered_line}\n" + f"{rendered}\n" f"" ) @@ -118,66 +251,13 @@ def _render_scip_lines( ) -def _render_scip_line( - line_text: str, - line_index: int, - document: Document, - index: Index, - scip_baseurl: str, -) -> str: - """Render a single source line, splicing in spans for each occurrence.""" - occurrences = [ - o - for o in document.occurrences_on_line(line_index) - if o.range.end_line == line_index # skip occurrences spanning lines - ] - if not occurrences: - return escape(line_text) - - out = [] - cursor = 0 - for occ in occurrences: - start = occ.range.start_char - end = occ.range.end_char - if start < cursor or end > len(line_text) or start >= end: - continue - if start > cursor: - out.append(escape(line_text[cursor:start])) - out.append(_render_occurrence(line_text[start:end], occ, index, scip_baseurl)) - cursor = end - if cursor < len(line_text): - out.append(escape(line_text[cursor:])) - return "".join(out) - - -def _render_occurrence( - text: str, occ: Occurrence, index: Index, scip_baseurl: str -) -> str: - classes = [] - css_class = _SCIP_SYNTAX_CLASSES.get(occ.syntax_kind) - if css_class: - classes.append(f"scip-{css_class}") - if occ.is_definition: - classes.append("scip-definition") - - body = escape(text) - if occ.symbol and not occ.is_definition: - defn = index.get_definition(occ.symbol) - if defn is not None: - href = f"{scip_baseurl}{defn.relative_path}#L-{defn.line + 1}" - body = f'{body}' - if classes: - body = f'{body}' - return body - - def highlight_or_render( code: str, filename: str, render_markup: bool = True, - scip_document: Optional[Document] = None, - scip_index: Optional[Index] = None, - scip_baseurl: Optional[str] = None, + scip_document: Document | None = None, + scip_index: Index | None = None, + scip_baseurl: str | None = None, ) -> str: """Render code as HTML. @@ -197,12 +277,6 @@ def highlight_or_render( if render_markup and markup.can_render(filename): return markup.render(filename, code) - if scip_document is not None: - assert scip_index is not None and scip_baseurl is not None, ( - "scip_index and scip_baseurl are required with scip_document" - ) - return _render_scip_lines(code, scip_document, scip_index, scip_baseurl) - try: lexer = get_lexer_for_filename(filename, code) except ClassNotFound: @@ -211,4 +285,10 @@ def highlight_or_render( except ClassNotFound: lexer = TextLexer() + if scip_document is not None: + assert scip_index is not None and scip_baseurl is not None, ( + "scip_index and scip_baseurl are required with scip_document" + ) + return _render_scip_lines(code, scip_document, scip_index, scip_baseurl, lexer) + return highlight(code, lexer, KlausHtmlFormatter()) From 4e109e8321aaab7d84d4402c65be8ea5ce5a6a81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Wed, 27 May 2026 14:14:28 +0100 Subject: [PATCH 4/5] Cross-reference navigation: anchors + hover references popover - SCIP definitions get a stable id (sym-) so deep links jump to the actual occurrence rather than the line number. - Non-definition occurrences now render as pointing at the definition's anchor, with data-sym, data-display, data-defhref / data-defloc, and data-refs attributes carrying the symbol's references inline. - scip.js shows a hover popover with the display name, definition link, and reference list (up to 10, with overflow indicator). - scip.css styles the popover and gives definitions a subtle underline + target highlight. - Index now also exposes get_references and get_display_name, built from each Document's symbols list. --- klaus/highlighting.py | 74 ++++++++++++++++++-- klaus/scip_index.py | 66 ++++++++++++------ klaus/static/scip.css | 49 ++++++++++++++ klaus/static/scip.js | 123 ++++++++++++++++++++++++++++++++++ klaus/templates/skeleton.html | 1 + tests/test_make_app.py | 36 ++++++++++ 6 files changed, 322 insertions(+), 27 deletions(-) create mode 100644 klaus/static/scip.js diff --git a/klaus/highlighting.py b/klaus/highlighting.py index 6e05f692..8e48189a 100644 --- a/klaus/highlighting.py +++ b/klaus/highlighting.py @@ -5,6 +5,8 @@ (without ctags, since cross-references in the Pygments path are gone). """ +import hashlib +import json from collections.abc import Iterator from html import escape from typing import Any @@ -165,32 +167,86 @@ def _splice_occurrences( return out +def _symbol_id(symbol: str) -> str: + """Stable short hash usable as an HTML ``id`` for a SCIP symbol.""" + return "sym-" + hashlib.sha1(symbol.encode("utf-8")).hexdigest()[:12] + + def _render_segment( text: str, pygments_class: str, occ: Occurrence | None, index: Index, scip_baseurl: str, + anchored_definitions: set[str], ) -> str: - """Render one slice of a line as HTML.""" + """Render one slice of a line as HTML. + + ``anchored_definitions`` tracks symbol IDs that already received an + ``id=...`` attribute in this document, so duplicate definitions (e.g. + forward declarations) don't produce duplicate HTML IDs. The set is + updated in place. + """ classes: list[str] = [] if pygments_class: classes.append(pygments_class) + has_symbol = occ is not None and bool(occ.symbol) + if has_symbol: + classes.append("scip-occurrence") if occ is not None: scip_class = _SCIP_SYNTAX_CLASSES.get(occ.syntax_kind) if scip_class: classes.append(f"scip-{scip_class}") if occ.is_definition: classes.append("scip-definition") + elif has_symbol: + classes.append("scip-reference") body = escape(text) - if occ is not None and occ.symbol and not occ.is_definition: + attrs: list[str] = [] + href: str | None = None + if has_symbol: + assert occ is not None + sym_id = _symbol_id(occ.symbol) + attrs.append(f'data-sym="{sym_id}"') + display = index.get_display_name(occ.symbol) + if display: + attrs.append(f'data-display="{escape(display, quote=True)}"') defn = index.get_definition(occ.symbol) if defn is not None: - href = f"{scip_baseurl}{defn.relative_path}#L-{defn.line + 1}" - body = f'{body}' - if classes: - body = f'{body}' + def_href = f"{scip_baseurl}{defn.relative_path}#{sym_id}" + attrs.append(f'data-defhref="{escape(def_href, quote=True)}"') + attrs.append( + f'data-defloc="{escape(defn.relative_path, quote=True)}:{defn.line + 1}"' + ) + if not occ.is_definition: + href = def_href + refs = index.get_references(occ.symbol) + if refs: + refs_json = json.dumps( + [ + [ + r.relative_path, + r.line + 1, + f"{scip_baseurl}{r.relative_path}#L-{r.line + 1}", + ] + for r in refs + ], + separators=(",", ":"), + ) + attrs.append(f"data-refs='{escape(refs_json, quote=True)}'") + if occ.is_definition and sym_id not in anchored_definitions: + anchored_definitions.add(sym_id) + attrs.append(f'id="{sym_id}"') + + if href: + attrs_str = (" " + " ".join(attrs)) if attrs else "" + cls = " ".join(classes) + return f'{body}' + if classes or attrs: + attrs_str = (" " + " ".join(attrs)) if attrs else "" + cls = " ".join(classes) + return f'{body}' return body @@ -213,6 +269,8 @@ def _render_scip_lines( while len(pygments_segments) < len(raw_lines): pygments_segments.append([]) + anchored_definitions: set[str] = set() + linenos_parts: list[str] = [] code_parts: list[str] = [] for i, line_text in enumerate(raw_lines, start=1): @@ -230,7 +288,9 @@ def _render_scip_lines( ] spliced = _splice_occurrences(segments, occurrences) rendered = "".join( - _render_segment(line_text[s:e], css, occ, index, scip_baseurl) + _render_segment( + line_text[s:e], css, occ, index, scip_baseurl, anchored_definitions + ) for s, e, css, occ in spliced ) code_parts.append( diff --git a/klaus/scip_index.py b/klaus/scip_index.py index a6a333fd..1fccaac0 100644 --- a/klaus/scip_index.py +++ b/klaus/scip_index.py @@ -13,14 +13,13 @@ import threading from collections import OrderedDict from dataclasses import dataclass -from typing import Dict, List, Optional, Tuple from klaus import scip_pb2 # type: ignore[attr-defined] _DEFINITION_ROLE: int = scip_pb2.SymbolRole.Definition # type: ignore[attr-defined] _CACHE_LOCK = threading.Lock() -_CACHE: "OrderedDict[Tuple[str, str], Optional[Index]]" = OrderedDict() +_CACHE: "OrderedDict[tuple[str, str], Index | None]" = OrderedDict() _CACHE_MAX_ENTRIES = 16 @@ -34,7 +33,7 @@ class Range: end_char: int @classmethod - def from_proto(cls, raw: List[int]) -> "Range": + def from_proto(cls, raw: list[int]) -> "Range": if len(raw) == 4: return cls(raw[0], raw[1], raw[2], raw[3]) if len(raw) == 3: @@ -58,17 +57,25 @@ class Definition: line: int # 0-based +@dataclass(frozen=True) +class Reference: + """Location of one non-definition occurrence of a symbol.""" + + relative_path: str + line: int # 0-based + + class Document: """Wrapper around a SCIP ``Document`` with sorted occurrences.""" - def __init__(self, relative_path: str, occurrences: List[Occurrence]): + def __init__(self, relative_path: str, occurrences: list[Occurrence]): self.relative_path = relative_path self.occurrences = sorted( occurrences, key=lambda o: (o.range.start_line, o.range.start_char), ) - def occurrences_on_line(self, line: int) -> List[Occurrence]: + def occurrences_on_line(self, line: int) -> list[Occurrence]: # Linear scan is fine for typical line counts; occurrences are sorted. return [o for o in self.occurrences if o.range.start_line == line] @@ -77,8 +84,10 @@ class Index: """Parsed SCIP index providing document and symbol lookup.""" def __init__(self, proto: "scip_pb2.Index"): # type: ignore[name-defined] - self._documents: Dict[str, Document] = {} - self._definitions: Dict[str, Definition] = {} + self._documents: dict[str, Document] = {} + self._definitions: dict[str, Definition] = {} + self._references: dict[str, list[Reference]] = {} + self._display_names: dict[str, str] = {} for doc in proto.documents: occurrences = [ @@ -94,24 +103,41 @@ def __init__(self, proto: "scip_pb2.Index"): # type: ignore[name-defined] doc.relative_path, occurrences ) for occ in occurrences: - if ( - occ.is_definition - and occ.symbol - and occ.symbol not in self._definitions - ): - self._definitions[occ.symbol] = Definition( - relative_path=doc.relative_path, - line=occ.range.start_line, + if not occ.symbol: + continue + if occ.is_definition: + if occ.symbol not in self._definitions: + self._definitions[occ.symbol] = Definition( + relative_path=doc.relative_path, + line=occ.range.start_line, + ) + else: + self._references.setdefault(occ.symbol, []).append( + Reference( + relative_path=doc.relative_path, + line=occ.range.start_line, + ) + ) + for sym_info in doc.symbols: + if sym_info.symbol and sym_info.display_name: + self._display_names.setdefault( + sym_info.symbol, sym_info.display_name ) - def get_document(self, relative_path: str) -> Optional[Document]: + def get_document(self, relative_path: str) -> Document | None: return self._documents.get(relative_path) - def get_definition(self, symbol: str) -> Optional[Definition]: + def get_definition(self, symbol: str) -> Definition | None: return self._definitions.get(symbol) + def get_references(self, symbol: str) -> list[Reference]: + return self._references.get(symbol, []) + + def get_display_name(self, symbol: str) -> str | None: + return self._display_names.get(symbol) + -def _scip_path_for(repo_path: str, sha: str) -> Optional[str]: +def _scip_path_for(repo_path: str, sha: str) -> str | None: scip_dir = os.path.join(repo_path, ".scip") if not os.path.isdir(scip_dir): return None @@ -124,7 +150,7 @@ def _scip_path_for(repo_path: str, sha: str) -> Optional[str]: return None -def load_index(repo_path: str, sha: str) -> Optional[Index]: +def load_index(repo_path: str, sha: str) -> Index | None: """Return the SCIP index for ``sha`` in ``repo_path``, or ``None``. Returns ``None`` (cached) when no SCIP dump is available, so callers @@ -137,7 +163,7 @@ def load_index(repo_path: str, sha: str) -> Optional[Index]: return _CACHE[key] path = _scip_path_for(repo_path, sha) - index: Optional[Index] + index: Index | None if path is None: index = None else: diff --git a/klaus/static/scip.css b/klaus/static/scip.css index 6da3b55e..b9721573 100644 --- a/klaus/static/scip.css +++ b/klaus/static/scip.css @@ -37,3 +37,52 @@ .code .scip-tag { color: #000080; } .code .scip-tag-attribute { color: #008080; } .code .scip-tag-delimiter { color: inherit; } + +/* Cross-reference styling */ +.code .scip-occurrence { cursor: help; } +.code a.scip-reference { color: inherit; text-decoration: none; } +.code a.scip-reference:hover { text-decoration: underline; text-decoration-style: dotted; } +.code .scip-definition { text-decoration: underline; text-decoration-color: #999; text-decoration-style: dotted; } +.code .scip-definition:target { background-color: #fefed0; } + +/* Hover popover */ +.scip-popover { + position: absolute; + z-index: 100; + background-color: #fff; + border: 1px solid #ccc; + box-shadow: 0 2px 6px rgba(0,0,0,0.15); + padding: 8px 10px; + font-size: 90%; + font-family: sans-serif; + max-width: 480px; + min-width: 200px; +} +.scip-popover-name { + font-family: monospace; + font-weight: bold; + margin-bottom: 4px; + word-break: break-all; +} +.scip-popover-label { + color: #666; + font-size: 90%; + margin-right: 2px; +} +.scip-popover-def { margin-bottom: 6px; font-family: monospace; } +.scip-popover-refs-header { + color: #666; + font-size: 90%; + border-top: 1px solid #eee; + padding-top: 4px; +} +.scip-popover-refs { + margin: 4px 0 0 0; + padding: 0; + list-style: none; + font-family: monospace; + max-height: 240px; + overflow-y: auto; +} +.scip-popover-refs li { padding: 1px 0; } +.scip-popover-more { color: #999; font-style: italic; } diff --git a/klaus/static/scip.js b/klaus/static/scip.js new file mode 100644 index 00000000..5906e429 --- /dev/null +++ b/klaus/static/scip.js @@ -0,0 +1,123 @@ +/* Hover popover for SCIP occurrences. + * + * Reads data-sym / data-display / data-defhref / data-defloc / data-refs + * attributes emitted by klaus.highlighting and shows a small panel listing + * the symbol's display name, definition link, and references. + */ +(function () { + function makePopover() { + var el = document.createElement('div'); + el.className = 'scip-popover'; + el.style.display = 'none'; + document.body.appendChild(el); + return el; + } + + var popover = null; + var activeTarget = null; + var hideTimer = null; + + function hide() { + if (popover) popover.style.display = 'none'; + activeTarget = null; + } + + function scheduleHide() { + clearTimeout(hideTimer); + hideTimer = setTimeout(hide, 200); + } + + function cancelHide() { + clearTimeout(hideTimer); + } + + function escapeHtml(s) { + return String(s) + .replace(/&/g, '&') + .replace(//g, '>') + .replace(/"/g, '"'); + } + + function show(target) { + if (!popover) popover = makePopover(); + cancelHide(); + activeTarget = target; + + var display = target.getAttribute('data-display') || ''; + var defhref = target.getAttribute('data-defhref'); + var defloc = target.getAttribute('data-defloc'); + var refsAttr = target.getAttribute('data-refs'); + var refs = []; + if (refsAttr) { + try { refs = JSON.parse(refsAttr); } catch (e) { refs = []; } + } + + var html = ''; + if (display) { + html += '
' + escapeHtml(display) + '
'; + } + if (defhref) { + html += '
' + + 'definition ' + + '' + + escapeHtml(defloc || 'definition') + '
'; + } + if (refs.length) { + html += '
' + + refs.length + ' reference' + (refs.length === 1 ? '' : 's') + + '
    '; + var max = 10; + for (var i = 0; i < Math.min(refs.length, max); i++) { + var r = refs[i]; // [path, line, href] + html += '
  • ' + + escapeHtml(r[0]) + ':' + r[1] + '
    • '; + } + if (refs.length > max) { + html += '
  • ' + + (refs.length - max) + ' more…
    • '; + } + html += '
'; + } + if (!html) { + hide(); + return; + } + popover.innerHTML = html; + popover.style.display = 'block'; + + var rect = target.getBoundingClientRect(); + var top = rect.bottom + window.scrollY + 4; + var left = rect.left + window.scrollX; + // Keep popover on-screen horizontally + var maxLeft = window.scrollX + document.documentElement.clientWidth - + popover.offsetWidth - 8; + if (left > maxLeft) left = Math.max(8, maxLeft); + popover.style.top = top + 'px'; + popover.style.left = left + 'px'; + } + + document.addEventListener('mouseover', function (e) { + var target = e.target.closest && e.target.closest('[data-sym]'); + if (target) { + if (target !== activeTarget) show(target); + cancelHide(); + } else if (e.target.closest && e.target.closest('.scip-popover')) { + cancelHide(); + } + }); + + document.addEventListener('mouseout', function (e) { + if (e.target.closest && (e.target.closest('[data-sym]') || + e.target.closest('.scip-popover'))) { + scheduleHide(); + } + }); + + document.addEventListener('click', function (e) { + if (!e.target.closest) return; + if (!e.target.closest('[data-sym]') && !e.target.closest('.scip-popover')) { + hide(); + } + }); +})(); diff --git a/klaus/templates/skeleton.html b/klaus/templates/skeleton.html index 6fd6bf4f..91f95c94 100644 --- a/klaus/templates/skeleton.html +++ b/klaus/templates/skeleton.html @@ -11,6 +11,7 @@ {% block title %}{% endblock %} - {{ SITE_NAME }} +
diff --git a/tests/test_make_app.py b/tests/test_make_app.py index df23ae69..676f35d3 100644 --- a/tests/test_make_app.py +++ b/tests/test_make_app.py @@ -203,6 +203,15 @@ def _write_test_scip_dump(): js_def.symbol = "scip-test js/test()." js_def.symbol_roles = scip_pb2.Definition js_def.syntax_kind = scip_pb2.IdentifierFunctionDefinition + # A reference from test.js to the `a` symbol defined in test.c, so we can + # assert cross-reference attrs on a non-definition occurrence. + js_ref = js_doc.occurrences.add() + js_ref.range.extend([0, 14, 0, 18]) + js_ref.symbol = "scip-test c/a." + + a_info = js_doc.symbols.add() + a_info.symbol = "scip-test c/a." + a_info.display_name = "a" scip_dir = os.path.join(TEST_REPO, ".scip") os.makedirs(scip_dir, exist_ok=True) @@ -229,6 +238,33 @@ def test_scip_renders_syntax_classes(): _remove_test_scip_dump() +def test_scip_emits_cross_reference_attrs(): + """Definitions get a stable id; references get data-sym/data-refs and + link to the definition's anchor.""" + _write_test_scip_dump() + try: + with serve(): + r_def = requests.get(UNAUTH_TEST_REPO_URL + "blob/master/test.c") + assert r_def.status_code == 200, r_def.text + # `a` is defined in test.c; its definition span should have an + # id="sym-..." and class scip-definition. + assert "scip-definition" in r_def.text + assert 'data-sym="sym-' in r_def.text + + r_ref = requests.get(UNAUTH_TEST_REPO_URL + "blob/master/test.js") + assert r_ref.status_code == 200, r_ref.text + # test.js has a reference to `a` (defined in test.c) at col 14-18. + assert "scip-reference" in r_ref.text + # The reference link should point at the definition's sym-... anchor. + assert "test.c#sym-" in r_ref.text + # data-refs should be a JSON array. + assert "data-refs=" in r_ref.text + # data-display should carry the human-friendly symbol name. + assert 'data-display="a"' in r_ref.text + finally: + _remove_test_scip_dump() + + def test_scip_falls_back_to_pygments_when_no_dump(): scip_index.clear_cache() with serve(): From 2aa185d95a995ed40f483f6e716cce4f2256dad9 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 27 May 2026 14:05:12 +0000 Subject: [PATCH 5/5] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- klaus/highlighting.py | 6 +++--- klaus/markup.py | 5 +++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/klaus/highlighting.py b/klaus/highlighting.py index 8e48189a..90b7aca1 100644 --- a/klaus/highlighting.py +++ b/klaus/highlighting.py @@ -346,9 +346,9 @@ def highlight_or_render( lexer = TextLexer() if scip_document is not None: - assert scip_index is not None and scip_baseurl is not None, ( - "scip_index and scip_baseurl are required with scip_document" - ) + assert ( + scip_index is not None and scip_baseurl is not None + ), "scip_index and scip_baseurl are required with scip_document" return _render_scip_lines(code, scip_document, scip_index, scip_baseurl, lexer) return highlight(code, lexer, KlausHtmlFormatter()) diff --git a/klaus/markup.py b/klaus/markup.py index 1974c902..5d0620e2 100644 --- a/klaus/markup.py +++ b/klaus/markup.py @@ -1,7 +1,8 @@ import os -from typing import Callable, List, Optional, Tuple +from collections.abc import Callable +from typing import Optional -LANGUAGES: List[Tuple[List[str], Callable[[str], str]]] = [] +LANGUAGES: list[tuple[list[str], Callable[[str], str]]] = [] def get_renderer(filename: str) -> Optional[Callable[[str], str]]: