scanner.py

import ast
import json
import argparse
import hashlib
import tokenize
from io import BytesIO
from pathlib import Path
from db_utils import save_snapshot

# ---------------------------------------------------
# Konfiguration
# ---------------------------------------------------
SEPARATOR = "!"  # festes Trennzeichen für DNA-Sequenzen

# ---------------------------------------------------
# Glossar laden
# ---------------------------------------------------
def load_glossar(path="glossar.json"):
    with open(path, "r", encoding="utf-8") as f:
        return json.load(f)

# ---------------------------------------------------
# Hilfsfunktionen für Fingerprints
# ---------------------------------------------------
def make_fingerprint(base_id: int, props: dict, length=8) -> int:
    """Erzeugt eine DNA-artige Zahl aus Glossar-ID + Eigenschaften"""
    raw = str(base_id) + json.dumps(props, sort_keys=True)
    h = hashlib.blake2b(raw.encode(), digest_size=16).hexdigest()
    return int(h, 16) % (10**length)

def encode_sequence(seq):
    """Konvertiert eine Zahlenliste in eine String-Sequenz mit festem Separator"""
    return SEPARATOR.join(str(x) for x in seq)

def sequence_to_fingerprint(seq, length=32):
    """Erzeugt stabilen Hash-Fingerprint aus der ganzen Sequenz"""
    raw = "-".join(str(x) for x in seq)
    h = hashlib.blake2b(raw.encode(), digest_size=16).hexdigest()
    num = int(h, 16) % (10**length)
    return str(num).zfill(length)  # immer gleich lang (mit führenden Nullen)

# ---------------------------------------------------
# AST-Scanner (semantisch)
# ---------------------------------------------------
class CodeScanner(ast.NodeVisitor):
    def __init__(self, glossar, dna=False):
        self.glossar = glossar
        self.sequence = []
        self.dna = dna

    def add(self, category, key, variant=None, props=None):
        try:
            entry = self.glossar[category][key]
            if isinstance(entry, dict):
                if variant and "variants" in entry and variant in entry["variants"]:
                    base_id = entry["variants"][variant]
                else:
                    base_id = entry["id"]
            else:
                base_id = entry

            props = props or {}
            if self.dna:
                val = make_fingerprint(base_id, props)
            else:
                val = base_id

            self.sequence.append(val)
        except KeyError:
            pass

    def visit_FunctionDef(self, node):
        props = {"name_len": len(node.name), "args": len(node.args.args)}
        self.add("structure", "FUNC_DEF", props=props)
        self.generic_visit(node)

    def visit_If(self, node):
        props = {"orelse": bool(node.orelse)}
        if node.orelse:
            self.add("control_flow", "IF", "WITH_ELSE", props)
        else:
            self.add("control_flow", "IF", "SIMPLE", props)
        self.generic_visit(node)

    def visit_Return(self, node):
        props = {"has_value": bool(node.value)}
        if node.value:
            self.add("control_flow", "RETURN", "WITH_VALUE", props)
        else:
            self.add("control_flow", "RETURN", "VOID", props)
        self.generic_visit(node)

    def visit_BinOp(self, node):
        op_type = type(node.op).__name__
        props = {"op": op_type}
        self.add("operators", "ARITHMETIC", props=props)
        self.generic_visit(node)

    def visit_Compare(self, node):
        ops = [type(op).__name__ for op in node.ops]
        props = {"ops": ops}
        self.add("operators", "COMPARISON", props=props)
        self.generic_visit(node)

    def visit_Name(self, node):
        props = {"id": node.id}
        self.add("data_types", "VAR_ASSIGN", props=props)
        self.generic_visit(node)

    def visit_Constant(self, node):
        props = {"value": str(node.value)}
        if isinstance(node.value, (int, float)):
            self.add("data_types", "NUMBER", props=props)
        elif isinstance(node.value, str):
            self.add("data_types", "STRING", props=props)
        elif isinstance(node.value, bool):
            self.add("data_types", "BOOL", props=props)
        elif node.value is None:
            self.add("data_types", "NULL", props=props)
        self.generic_visit(node)

# ---------------------------------------------------
# Token-Scanner (alles, inkl. Kommentare)
# ---------------------------------------------------
def tokenize_file(filepath):
    code = Path(filepath).read_text(encoding="utf-8")
    tokens = tokenize.tokenize(BytesIO(code.encode("utf-8")).readline)

    sequence = []
    for tok in tokens:
        if tok.type == tokenize.ENCODING or tok.type == tokenize.ENDMARKER:
            continue
        if tok.type == tokenize.COMMENT:
            # Kommentar → Hash speichern
            val = make_fingerprint(9999, {"comment": tok.string})
            sequence.append(val)
        elif tok.type == tokenize.STRING:
            val = make_fingerprint(8888, {"string": tok.string})
            sequence.append(val)
        elif tok.type == tokenize.NUMBER:
            sequence.append(int(hashlib.blake2b(tok.string.encode(), digest_size=4).hexdigest(), 16))
        else:
            # Alles andere → auf Zeichenebene speichern
            for ch in tok.string:
                sequence.append(ord(ch))
    return sequence

# ---------------------------------------------------
# Analysefunktion
# ---------------------------------------------------
def analyze_file(filepath, glossar, dna=False):
    # AST-Analyse
    code = Path(filepath).read_text(encoding="utf-8")
    tree = ast.parse(code)
    scanner = CodeScanner(glossar, dna=dna)
    scanner.visit(tree)

    ast_seq = scanner.sequence
    token_seq = tokenize_file(filepath)

    # Hybrid: AST + Tokens zusammenführen
    return ast_seq + token_seq

def scan_folder(folder_path: str, exts=None, dna=True):
    """Scant alle Dateien in einem Ordner und gibt Fingerprints zurück."""
    exts = exts or [".py"]
    glossar = load_glossar()
    results = {}
    for ext in exts:
        for filepath in Path(folder_path).rglob(f"*{ext}"):
            try:
                seq = analyze_file(filepath, glossar, dna=dna)
                fingerprint = sequence_to_fingerprint(seq)
                results[str(filepath)] = fingerprint
            except Exception as e:
                print(f"[SCAN_FOLDER] Fehler bei {filepath}: {e}")
    return results


# ---------------------------------------------------
# Chunking
# ---------------------------------------------------
def get_chunk_size(num_lines: int) -> int:
    """Bestimme dynamische Chunkgröße anhand der Dateilänge."""
    if num_lines <= 1000:
        return num_lines  # Full-Scan
    elif num_lines <= 10000:
        return 300
    elif num_lines <= 50000:
        return 500
    else:
        return 1000

def split_into_chunks(filepath: str):
    """Datei nach Zeilen in Chunks aufteilen."""
    lines = Path(filepath).read_text(encoding="utf-8").splitlines()
    num_lines = len(lines)
    chunk_size = get_chunk_size(num_lines)

    chunks = []
    for i in range(0, num_lines, chunk_size):
        chunk_lines = lines[i:i + chunk_size]
        chunks.append({
            "chunk_index": len(chunks),
            "line_start": i + 1,
            "line_end": i + len(chunk_lines),
            "code": "\n".join(chunk_lines)
        })
    return chunks

def analyze_code_chunk(code: str, glossar, dna=False):
    """Analysiere nur einen Code-Chunk (AST + Tokens)."""
    try:
        tree = ast.parse(code)
        scanner = CodeScanner(glossar, dna=dna)
        scanner.visit(tree)
        ast_seq = scanner.sequence
    except SyntaxError:
        ast_seq = []

    tokens = tokenize.tokenize(BytesIO(code.encode("utf-8")).readline)
    token_seq = []
    for tok in tokens:
        if tok.type == tokenize.ENCODING or tok.type == tokenize.ENDMARKER:
            continue
        if tok.type == tokenize.COMMENT:
            val = make_fingerprint(9999, {"comment": tok.string})
            token_seq.append(val)
        elif tok.type == tokenize.STRING:
            val = make_fingerprint(8888, {"string": tok.string})
            token_seq.append(val)
        elif tok.type == tokenize.NUMBER:
            token_seq.append(int(hashlib.blake2b(tok.string.encode(), digest_size=4).hexdigest(), 16))
        else:
            for ch in tok.string:
                token_seq.append(ord(ch))

    return ast_seq + token_seq

def analyze_file_chunked(filepath, glossar, dna=False):
    """Komplette Datei analysieren, aber chunkweise."""
    chunks = split_into_chunks(filepath)
    results = []
    for c in chunks:
        seq = analyze_code_chunk(c["code"], glossar, dna=dna)
        encoded = encode_sequence(seq)
        fingerprint = sequence_to_fingerprint(seq)
        results.append({
            "chunk_index": c["chunk_index"],
            "line_start": c["line_start"],
            "line_end": c["line_end"],
            "dna": encoded,
            "fingerprint": fingerprint,
            "code": c["code"]
        })
    return results


# ---------------------------------------------------
# CLI
# ---------------------------------------------------
if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="CoNum – Code→Zahlen Scanner")
    parser.add_argument("file", help="Pfad zu einer Python-Datei")
    parser.add_argument("--dna", action="store_true", help="DNA-artige Fingerprint-Zahlen ausgeben")
    args = parser.parse_args()

    glossar = load_glossar()
    seq = analyze_file(args.file, glossar, dna=args.dna)

    encoded = encode_sequence(seq)
    fingerprint = sequence_to_fingerprint(seq)

    print("DNA-Sequenz:", encoded)
    print("Fingerprint:", fingerprint)