diff --git a/agent_file_system/AGENT.md b/agent_file_system/AGENT.md index 2036db75..bcc340a4 100644 --- a/agent_file_system/AGENT.md +++ b/agent_file_system/AGENT.md @@ -770,12 +770,11 @@ You're blocked when you don't know what to do next AND retrying won't help. The - Full input schema: [app/data/action/read_file.py](app/data/action/read_file.py). ### grep_files -Three output modes: -- `files_with_matches`: returns file paths only. Use for discovery ("which files contain X"). -- `content`: returns matching lines with line numbers. Use for investigation. -- `count`: returns match counts per file. Use for frequency checks. +Runs ripgrep and returns matching lines as `path:line: text`. When you search a folder, paths are relative to it, and the message names the folder. With `context`, surrounding lines read `path-line- text` and separate groups are split by `--`. -Supported parameters: `glob`, `file_type`, `before_context` / `after_context`, `case_insensitive`, `multiline`. +Parameters: `pattern` (ripgrep regex), `path`, `glob`, `ignore_case`, `context`, `limit` (default 100 matches; the message says when the limit cut the results short). + +A folder search skips files ignored by `.gitignore`, hidden files and binary files. To search one of those, pass its file path. Full input schema: [app/data/action/grep_files.py](app/data/action/grep_files.py). diff --git a/app/data/action/grep_files.py b/app/data/action/grep_files.py index 78822a8d..bc8d2469 100644 --- a/app/data/action/grep_files.py +++ b/app/data/action/grep_files.py @@ -4,62 +4,32 @@ "pattern": { "type": "string", "example": "def \\w+\\(", - "description": "Regex pattern to search for. Supports full regex syntax (e.g., 'def \\w+\\(' to find function definitions, 'TODO:.*' to find TODOs). For literal text search, just use the plain text (special regex chars will need escaping).", + "description": "Regex to search for (ripgrep syntax). Escape regex characters such as ( [ . to match them literally.", }, "path": { "type": "string", "example": "/workspace/project", - "description": "File or directory path to search in. If a directory, searches all files recursively. If a file, searches only that file. Defaults to current working directory if not provided.", + "description": "File or folder to search. Defaults to the current working directory.", }, "glob": { "type": "string", - "example": "*.py", - "description": "Glob filter (e.g. '*.py', '*.{js,ts}', 'test_*.py', 'src/**/*.ts'). Brace alternatives are expanded. A pattern with no '/' matches the FILENAME anywhere in the tree; one with a '/' matches the path and is also tried unanchored, so 'src/**/*.ts' finds 'frontend/src/app/x.ts'. Only applies when path is a directory.", + "example": "*.{ts,tsx}", + "description": "Optional. Only search files matching this glob, e.g. '*.py', '*.{ts,tsx}' or 'src/**/*.ts'.", }, - "file_type": { - "type": "string", - "example": "py", - "description": "Filter by file extension. Accepts one ('py') or a comma-separated list ('ts,tsx'). NOTE: a single extension is exact — 'ts' does NOT include .tsx, so use 'ts,tsx' for a React codebase. Shorthand for glob; if both are given, glob wins.", - }, - "output_mode": { - "type": "string", - "example": "content", - "description": "Controls what is returned. 'files_with_matches' (default): returns only file paths that contain matches. 'content': returns matching lines with line numbers and optional context. 'count': returns the number of matches per file.", - }, - "case_insensitive": { - "type": "boolean", - "example": True, - "description": "If true, search is case-insensitive. Default is false (case-sensitive).", - }, - "before_context": { - "type": "integer", - "example": 2, - "description": "Number of lines to show BEFORE each match. Only applies when output_mode is 'content'. Default is 0.", - }, - "after_context": { - "type": "integer", - "example": 2, - "description": "Number of lines to show AFTER each match. Only applies when output_mode is 'content'. Default is 0.", - }, - "context": { - "type": "integer", - "example": 3, - "description": "Number of context lines to show both before AND after each match (shorthand for setting before_context and after_context to the same value). Only applies when output_mode is 'content'. Overridden by explicit before_context/after_context if provided.", - }, - "multiline": { + "ignore_case": { "type": "boolean", "example": False, - "description": "If true, enables multiline mode where '.' matches newlines and patterns can span across lines. Default is false.", + "description": "Case-insensitive search. Default false.", }, - "head_limit": { + "context": { "type": "integer", - "example": 50, - "description": "Maximum number of results to return. For 'files_with_matches': max file paths. For 'content': max output lines. For 'count': max file entries. Default is 250. Pass 0 for unlimited results (no truncation). If results are truncated, the applied_limit field in the response tells you it happened — use offset to paginate through the rest. Note: 'content' output is ALSO byte-capped independently of this (each line trimmed to 500 chars, whole payload to 40000 chars) so a file with very long lines cannot flood the context; the message field says so when it happens.", + "example": 0, + "description": "Lines to show before and after each match. Default 0.", }, - "offset": { + "limit": { "type": "integer", - "example": 0, - "description": "Number of results to skip before returning. Use with head_limit for pagination. Default is 0.", + "example": 100, + "description": "Maximum number of matches to return. Default 100.", }, } @@ -71,48 +41,28 @@ }, "message": { "type": "string", - "example": "Found matches in 5 files", - "description": "Summary message or error description.", - }, - "mode": { - "type": "string", - "example": "content", - "description": "The output mode that was used.", - }, - "num_files": { - "type": "integer", - "example": 5, - "description": "Number of files that contained matches.", - }, - "filenames": { - "type": "array", - "example": ["/workspace/project/main.py", "/workspace/project/utils.py"], - "description": "List of file paths that contained matches.", + "example": "Found 2 matches in 2 files under /workspace/project; paths are relative to it.", + "description": "What was searched and found, including why a result may be incomplete.", }, "content": { "type": "string", - "example": "File: /workspace/main.py\n10:def hello():\n11- pass\n--\n25:def world():\n26- return 1\n", - "description": "Matching lines with line numbers. Match lines use ':' after the line number (e.g., '10:matched line'), context lines use '-' (e.g., '11-context line'). Non-contiguous groups are separated by '--'. For single-file searches, the filepath is shown once at the top to save tokens. For multi-file searches, each file section is prefixed with 'File: path'. Only populated when output_mode is 'content'.", - }, - "num_lines": { - "type": "integer", - "example": 15, - "description": "Number of content lines returned. Only populated when output_mode is 'content'.", + "example": "src/main.py:10: def hello():\nsrc/util.py:4: def world():", + "description": "Matching lines as 'path:line: text'. Context lines read 'path-line- text' and non-adjacent groups are separated by '--'.", }, "num_matches": { "type": "integer", - "example": 42, - "description": "Total number of matches across all files. Only populated when output_mode is 'count'.", + "example": 2, + "description": "Number of matching lines in content.", }, - "applied_limit": { + "num_files": { "type": "integer", - "example": 250, - "description": "The head_limit that was applied, or null if unlimited (head_limit=0). If your results were truncated to this limit, use offset to paginate through the rest.", + "example": 2, + "description": "Number of files those matches come from.", }, - "applied_offset": { - "type": "integer", - "example": 0, - "description": "The offset that was applied.", + "truncated": { + "type": "boolean", + "example": False, + "description": "True when more matches exist than content shows.", }, } @@ -120,16 +70,11 @@ @action( name="grep_files", description=( - "Searches files for a regex pattern and returns results. " - "Supports searching a single file or an entire directory recursively. " - "Three output modes: " - "'files_with_matches' (default) returns file paths containing matches — use for discovery. " - "'content' returns matching lines with line numbers and optional before/after context — use to read matched code. " - "In content mode, match lines use ':' after line number (e.g., '10:matched line'), " - "context lines use '-' (e.g., '11-context line'), and non-contiguous groups are separated by '--'. " - "'count' returns match counts per file — use for quick frequency checks. " - "Supports glob and file_type filtering, case-insensitive search, and multiline patterns. " - "Use with read_file: first grep_files to find relevant line numbers, then read_file with offset to read that section." + "Search file contents with a ripgrep regex. Returns matching lines as " + "'path:line: text', with paths relative to the searched folder. " + "Searching a folder skips files ignored by .gitignore, hidden files and " + "binary files. Returns at most `limit` matches. Use read_file on a " + "matched file when you need more of it." ), mode="CLI", platforms=["linux", "windows", "darwin"], @@ -138,465 +83,262 @@ output_schema=_OUTPUT_SCHEMA, test_payload={ "pattern": "Mt\\. Fuji|visibility", - "path": "/path/to/input.txt", - "output_mode": "content", - "case_insensitive": True, - "head_limit": 50, + "path": "/path/to", + "ignore_case": True, + "limit": 50, "simulated_mode": True, }, ) def grep_files(input_data: dict) -> dict: - """Searches files for a regex pattern and returns results.""" + """Search files with ripgrep and return the matching lines.""" import os - import re - import fnmatch - - # Byte caps on the returned payload. head_limit bounds the number of LINES, - # which is no bound at all when a "line" is a 160KB MIME header blob (raw - # Received/DKIM/ARC headers in an externalized get_gmail dump). Without these - # a single grep can land a ~77k-token event in the event stream, which blows - # the summarization threshold in one shot. Both are applied AFTER pagination - # so head_limit/offset still mean what they say. + import subprocess + import threading + + from agent_core.core.impl.action.cancellation import ( + register_process, + unregister_process, + ) + from app import ripgrep_runtime + + # One matched line of a minified bundle can be megabytes, and the whole + # result lands in the event stream: both caps keep one search from + # flooding the context. MAX_LINE_CHARS = 500 MAX_CONTENT_CHARS = 40000 + DEFAULT_LIMIT = 100 - # --- Helper functions (must be inside for sandboxed execution) --- - - def make_error(message): + def make_result( + status, message, content="", num_matches=0, num_files=0, truncated=False + ): return { - "status": "error", + "status": status, "message": message, - "mode": None, - "num_files": 0, - "filenames": [], - "content": None, - "num_lines": None, - "num_matches": None, - "applied_limit": None, - "applied_offset": None, + "content": content, + "num_matches": num_matches, + "num_files": num_files, + "truncated": truncated, } - def expand_braces(pat): - """Expand {a,b} alternatives into separate patterns. - - fnmatch has no brace support, but THIS TOOL'S OWN SCHEMA advertises - "*.{js,ts}" as a valid glob. Every agent that followed the docs got a - silent 0 results — indistinguishable from "the string is not there". - """ - out = [pat] - while True: - grown = [] - changed = False - for item in out: - i = item.find("{") - j = item.find("}", i + 1) if i >= 0 else -1 - if i >= 0 and j > i: - head, body, tail = item[:i], item[i + 1 : j], item[j + 1 :] - for alt in body.split(","): - grown.append(head + alt.strip() + tail) - changed = True - else: - grown.append(item) - out = grown - if not changed: - return out - - def glob_hit(rel_path, fname, patterns): - """Match a file against expanded patterns. - - A pattern with no separator is a NAME pattern ("*.py" anywhere in the - tree); one with a separator is a PATH pattern ("src/**/*.ts"). The - old code compared every pattern against the BASENAME only, so any - path-shaped glob matched nothing, silently. - """ - for pat in patterns: - if "/" not in pat and os.sep not in pat: - if fnmatch.fnmatch(fname, pat): - return True - continue - norm = rel_path.replace(os.sep, "/") - if fnmatch.fnmatch(norm, pat): - return True - # "**/" must also match ZERO directories: "src/**/*.ts" has to - # find "src/x.ts", not only "src/a/x.ts". - flat = pat.replace("**/", "") - if "**/" in pat and fnmatch.fnmatch(norm, flat): - return True - # Unanchored fallback: "src/**/*.ts" should also find - # "frontend/src/app/x.ts". Strict root-anchoring is defensible, - # but a false negative here costs far more than a false positive - # — the caller reads it as "this code does not exist". - if fnmatch.fnmatch(norm, "*/" + pat) or fnmatch.fnmatch(norm, "*/" + flat): - return True - return False - - def collect_files(directory, glob_pat=None, max_files=10000): - SKIP_DIRS = { - ".git", - ".svn", - ".hg", - "__pycache__", - "node_modules", - ".venv", - "venv", - ".env", - ".tox", - ".mypy_cache", - ".pytest_cache", - "dist", - "build", - ".idea", - ".vscode", - } - patterns = expand_braces(glob_pat) if glob_pat else None - collected = [] - stats = {"seen": 0, "skipped_by_glob": 0, "hidden": 0} - for root, dirs, files in os.walk(directory): - dirs[:] = [d for d in dirs if d not in SKIP_DIRS and not d.startswith(".")] - for fname in files: - if fname.startswith("."): - stats["hidden"] += 1 - continue - stats["seen"] += 1 - if patterns: - rel = os.path.relpath(os.path.join(root, fname), directory) - if not glob_hit(rel, fname, patterns): - stats["skipped_by_glob"] += 1 - continue - collected.append(os.path.join(root, fname)) - if len(collected) >= max_files: - return collected, stats - return collected, stats - - def format_content_lines( - fpath, lines, sorted_indices, display_map, single_file, first_file - ): - result = [] - if single_file: - if first_file: - result.append(f"File: {fpath}") - else: - if not first_file: - result.append("--") - result.append(f"File: {fpath}") - - prev_ln = None - for ln in sorted_indices: - if ln >= len(lines): - continue - if prev_ln is not None and ln > prev_ln + 1: - result.append("--") - separator = ":" if display_map[ln] else "-" - result.append(f"{ln + 1}{separator}{lines[ln]}") - prev_ln = ln - return result - - # --- Main logic --- - - simulated_mode = input_data.get("simulated_mode", False) - - if simulated_mode: - return { - "status": "success", - "message": "Found matches in 2 files", - "mode": "content", - "num_files": 2, - "filenames": ["/path/to/input.txt", "/path/to/other.txt"], - "content": "File: /path/to/input.txt\n10:Mt. Fuji is visible today\n11-The mountain was clear\n--\nFile: /path/to/other.txt\n5:visibility is low\n", - "num_lines": 5, - "num_matches": None, - "applied_limit": 50, - "applied_offset": 0, - } + def as_int(value, default): + try: + return int(value) + except (TypeError, ValueError): + return default + + if input_data.get("simulated_mode", False): + return make_result( + "success", + "Found 2 matches in 2 files under /path/to; paths are relative to it.", + "input.txt:10: Mt. Fuji is visible today\nother.txt:5: visibility is low", + num_matches=2, + num_files=2, + ) - # --- Parse and validate inputs --- - pattern_str = input_data.get("pattern") - if not pattern_str: - return make_error("pattern is required.") + pattern = input_data.get("pattern") + if not isinstance(pattern, str) or pattern == "": + return make_result("error", "pattern is required.") - search_path = input_data.get("path") or os.getcwd() - output_mode = input_data.get("output_mode", "files_with_matches") - if output_mode not in ("files_with_matches", "content", "count"): - output_mode = "files_with_matches" + root = os.path.abspath(os.path.expanduser(input_data.get("path") or os.getcwd())) + if not os.path.exists(root): + return make_result("error", f"Path does not exist: {root}") + root_is_dir = os.path.isdir(root) - case_insensitive = bool(input_data.get("case_insensitive", False)) - multiline_mode = bool(input_data.get("multiline", False)) - glob_pattern = input_data.get("glob") - file_type = input_data.get("file_type") + glob = input_data.get("glob") or None + ignore_case = bool(input_data.get("ignore_case", False)) + context = max(0, as_int(input_data.get("context"), 0)) + limit = max(1, as_int(input_data.get("limit"), DEFAULT_LIMIT)) - # Context lines (only for content mode) try: - ctx = int(input_data.get("context", 0)) - except (TypeError, ValueError): - ctx = 0 - try: - before_ctx = int(input_data.get("before_context", ctx)) - except (TypeError, ValueError): - before_ctx = ctx - try: - after_ctx = int(input_data.get("after_context", ctx)) - except (TypeError, ValueError): - after_ctx = ctx - before_ctx = max(0, before_ctx) - after_ctx = max(0, after_ctx) + rg = ripgrep_runtime.resolve() + except RuntimeError as e: + return make_result("error", f"Cannot search: {e}") + + # --no-require-git: a project's .gitignore applies whether or not the + # folder sits inside a git checkout, so a search skips the same files in + # a dev checkout and in an installed copy. + # --no-messages: unreadable files are skipped silently; a bad regex is + # still reported on stderr. + args = [rg, "--no-config", "--no-messages", "--no-require-git"] + # One line per result: path NUL line number, ':' for a match or '-' for + # context, then the text. Not --json: it ignores --max-columns and lists + # every occurrence on a matched line, so one minified line could load + # megabytes into memory. + args += [ + "--color=never", + "--no-heading", + "--with-filename", + "--line-number", + "--null", + "--no-context-separator", + ] + # rg trims long lines itself. A UTF-8 character is at most 4 bytes and rg + # counts the line ending, so any line it trims is still longer than + # MAX_LINE_CHARS and its "[... omitted]" marker falls in the part cut + # below. + args += [f"--max-columns={4 * MAX_LINE_CHARS + 2}", "--max-columns-preview"] + # No file can contribute more than `limit` matches. + args.append(f"--max-count={limit}") + if not root_is_dir: + # A file named directly is searched even when it looks binary; + # otherwise rg prints only a "binary file matches" notice for it. + args.append("--text") + if ignore_case: + args.append("--ignore-case") + if context: + args += ["--context", str(context)] + if glob and root_is_dir: + args += ["--glob", glob] + args += ["--regexp", pattern, "--", root] + + def display(path): + if not root_is_dir: + return os.path.basename(path) + normalized = os.path.normpath(path) + try: + relative = os.path.relpath(normalized, root) + except ValueError: # another drive on Windows + return normalized.replace(os.sep, "/") + if relative.startswith(".."): + return normalized.replace(os.sep, "/") + return relative.replace(os.sep, "/") + + # Content is built while rg's output is read, so memory stays within + # MAX_CONTENT_CHARS however many matches a search turns up. + lines = [] + used = 0 + shown_matches = 0 + shown_files = set() + lines_cut = 0 + capped = False + previous = None + limit_line = None + stderr_parts = [] + session_id = input_data.get("_session_id") or "" - # Pagination - raw_limit = input_data.get("head_limit") try: - head_limit = int(raw_limit) if raw_limit is not None else 250 - except (TypeError, ValueError): - head_limit = 250 - try: - offset = int(input_data.get("offset", 0)) - except (TypeError, ValueError): - offset = 0 - if head_limit < 0: - head_limit = 250 - unlimited = head_limit == 0 - if offset < 0: - offset = 0 - - # --- Compile regex --- - flags = 0 - if case_insensitive: - flags |= re.IGNORECASE - if multiline_mode: - flags |= re.DOTALL | re.MULTILINE - + proc = subprocess.Popen( + args, + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + creationflags=subprocess.CREATE_NO_WINDOW if os.name == "nt" else 0, + ) + except OSError as e: + return make_result("error", f"Could not start ripgrep at {rg}: {e}") + + # A user force-stop kills rg; reading stdout then simply ends. + register_process(session_id, proc) + # stderr is drained on its own thread: a full pipe would block rg while + # this thread waits on stdout. + drain = threading.Thread( + target=lambda: stderr_parts.append(proc.stderr.read()), daemon=True + ) + drain.start() try: - regex = re.compile(pattern_str, flags) - except re.error as e: - return make_error(f"Invalid regex pattern: {e}") - - # --- Collect files to search --- - if not os.path.exists(search_path): - return make_error(f"Path does not exist: {search_path}") - - active_glob = None - collect_stats = None - if os.path.isfile(search_path): - files_to_search = [search_path] - else: - if glob_pattern: - active_glob = glob_pattern - elif file_type: - # file_type now accepts a list ("ts,tsx"). A bare "ts" silently - # excludes every .tsx file — i.e. most of a React codebase — and - # the caller only ever saw "0 matches" for it. - exts = [ - t.strip().lstrip(".") for t in str(file_type).split(",") if t.strip() - ] - if len(exts) > 1: - active_glob = "*.{" + ",".join(exts) + "}" - elif exts: - active_glob = f"*.{exts[0]}" - files_to_search, collect_stats = collect_files(search_path, active_glob) - - # --- Search each file --- - matched_filenames = [] - content_lines = [] - total_match_count = 0 - count_entries = [] - is_single_file = len(files_to_search) == 1 - - for fpath in files_to_search: - try: - with open(fpath, "r", encoding="utf-8", errors="ignore") as f: - file_content = f.read() - except (OSError, IOError): - continue - - if not file_content: - continue - - lines = file_content.split("\n") - - if multiline_mode: - matches = list(regex.finditer(file_content)) - if not matches: + for raw in proc.stdout: + path_bytes, nul, rest = raw.partition(b"\0") + digits = len(rest) - len(rest.lstrip(b"0123456789")) + # rg's notices, such as a binary file it stopped searching after a + # match, carry no NUL and no line number. + if not nul or not digits: continue - matched_line_nums = set() - for m in matches: - start_line = file_content[: m.start()].count("\n") - end_line = file_content[: m.end()].count("\n") - for ln in range(start_line, end_line + 1): - matched_line_nums.add(ln) - else: - matched_line_nums = set() - for i, line in enumerate(lines): - if regex.search(line): - matched_line_nums.add(i) - - if not matched_line_nums: - continue - - matched_filenames.append(fpath) - match_count = len(matched_line_nums) - total_match_count += match_count - - if output_mode == "count": - count_entries.append(f"{fpath}: {match_count}") - elif output_mode == "content": - display_map = {} - for ln in matched_line_nums: - display_map[ln] = True - for ctx_ln in range( - max(0, ln - before_ctx), min(len(lines), ln + after_ctx + 1) - ): - if ctx_ln not in display_map: - display_map[ctx_ln] = False - - sorted_indices = sorted(display_map.keys()) - file_lines = format_content_lines( - fpath, - lines, - sorted_indices, - display_map, - is_single_file, - first_file=(len(content_lines) == 0), - ) - content_lines.extend(file_lines) - - # --- Apply pagination and build output --- - def paginate(items): - after_offset = items[offset:] - if unlimited: - return after_offset - return after_offset[:head_limit] - - def clamp_line(line): - """Trim one output line to MAX_LINE_CHARS, keeping the 'NN:' prefix.""" - if len(line) <= MAX_LINE_CHARS: - return line, 0 - dropped = len(line) - MAX_LINE_CHARS - return ( - f"{line[:MAX_LINE_CHARS]}… [line truncated, {dropped} chars dropped]", - dropped, - ) - - def clamp_content(lines): - """Apply the per-line and total byte caps. Returns (lines, note).""" - clamped = [] - truncated_lines = 0 - used = 0 - stopped_at = None - for i, line in enumerate(lines): - text, dropped = clamp_line(line) - if dropped: - truncated_lines += 1 - if used + len(text) + 1 > MAX_CONTENT_CHARS: - stopped_at = i + path = path_bytes.decode("utf-8", "replace") + line_no = int(rest[:digits]) + is_match = rest[digits : digits + 1] == b":" + if limit_line is not None: + # Past the limit, keep only the trailing context of the last + # match, never the lead-in of the next one. + if is_match or path != previous[0] or line_no > limit_line + context: + break + text = rest[digits + 1 :].decode("utf-8", "replace").rstrip("\r\n") + block = [] + if ( + context + and previous is not None + and (previous[0] != path or line_no != previous[1] + 1) + ): + block.append("--") + if len(text) > MAX_LINE_CHARS: + text = text[:MAX_LINE_CHARS] + " [line cut]" + lines_cut += 1 + sep = ":" if is_match else "-" + block.append(f"{display(path)}{sep}{line_no}{sep} {text}") + size = sum(len(b) + 1 for b in block) + if used + size > MAX_CONTENT_CHARS: + capped = True break - clamped.append(text) - used += len(text) + 1 - - notes = [] - if truncated_lines: - notes.append( - f"{truncated_lines} line(s) were trimmed to {MAX_LINE_CHARS} chars" - ) - if stopped_at is not None: - notes.append( - f"output capped at {MAX_CONTENT_CHARS} chars after {stopped_at} of " - f"{len(lines)} line(s) — narrow the pattern or use offset={offset + stopped_at} " - "to continue" - ) - return clamped, "; ".join(notes) - - def _with_note(msg): - note = search_note() - return f"{msg} — {note}" if note else msg - - def search_note(): - """Explain an empty result instead of just reporting one. + lines.extend(block) + used += size + previous = (path, line_no) + if is_match: + shown_matches += 1 + shown_files.add(path) + if shown_matches >= limit: + limit_line = line_no + if not context: + break + except Exception as e: + proc.kill() + return make_result("error", f"Search failed: {e}") + finally: + if proc.poll() is None: + proc.kill() + returncode = proc.wait() + drain.join(timeout=5) + unregister_process(session_id, proc) + + limit_hit = limit_line is not None + stderr_text = ( + b"".join(p for p in stderr_parts if p).decode("utf-8", "replace").strip() + ) + + # rg exits 2 on any error, including one unreadable file in a tree that + # otherwise matched; it is only a failure when nothing usable came back. + if not lines and not limit_hit and returncode not in (0, 1): + return make_result( + "error", + f"Search failed: {stderr_text or f'ripgrep exited with code {returncode}'}", + ) - A 0 from a filter that ate every candidate used to read exactly like - a 0 from a genuine absence. Observed twice in production: an agent - turned "0 matches" into a confident structural claim ("the handler - must live in the frontend", "the code must be bundled elsewhere") and - acted on it for half an hour. The file it wanted was four lines long - and sat in the directory it had just searched. - """ - if matched_filenames or collect_stats is None: - return "" - skipped = collect_stats.get("skipped_by_glob", 0) - seen = collect_stats.get("seen", 0) - hidden = collect_stats.get("hidden", 0) - if not files_to_search: - which = "glob" if glob_pattern else "file_type" - return ( - f"NOTHING WAS SEARCHED: the {which} filter {active_glob!r} " - f"excluded all {seen} file(s) under {search_path}. This is " - "NOT evidence the pattern is absent — widen or drop the " - "filter and search again." + if not lines: + if root_is_dir: + message = ( + f"No matches under {root}. Searching a folder skips files ignored " + f"by .gitignore, hidden files and binary files." ) - parts = [f"searched {len(files_to_search)} file(s)"] - if skipped: - parts.append( - f"{skipped} more were excluded by {active_glob!r} and never looked at" - ) - if hidden: - parts.append(f"{hidden} dot-file(s) are always skipped") - return "no match after " + "; ".join(parts) + "." - - effective_limit = None if unlimited else head_limit - - if output_mode == "files_with_matches": - total = len(matched_filenames) - paginated = paginate(matched_filenames) - return { - "status": "success", - "message": _with_note(f"Found matches in {total} file(s)"), - "mode": "files_with_matches", - "num_files": total, - "filenames": paginated, - "content": None, - "num_lines": None, - "num_matches": None, - "applied_limit": effective_limit, - "applied_offset": offset, - } - - elif output_mode == "content": - paginated, cap_note = clamp_content(paginate(content_lines)) - content_str = "\n".join(paginated) - if paginated: - content_str += "\n" - message = ( - f"Found {total_match_count} match(es) in {len(matched_filenames)} file(s)" + else: + message = f"No matches in {root}." + return make_result("success", message) + + where = f"under {root}; paths are relative to it" if root_is_dir else f"in {root}" + notes = [ + f"Found {shown_matches} match{'' if shown_matches == 1 else 'es'} in " + f"{len(shown_files)} file{'' if len(shown_files) == 1 else 's'} {where}." + ] + if limit_hit: + notes.append( + f"Stopped at the limit of {limit} matches; more may exist. Raise " + f"limit or narrow the pattern, path or glob." ) - if cap_note: - message += f" ({cap_note})" - message = _with_note(message) - return { - "status": "success", - "message": message, - "mode": "content", - "num_files": len(matched_filenames), - # Content mode already carries each path inline in `content`; echoing an - # unbounded filename list on top of it is pure token cost on a wide search. - "filenames": matched_filenames[:100], - "content": content_str, - "num_lines": len(paginated), - "num_matches": None, - "applied_limit": effective_limit, - "applied_offset": offset, - } - - else: # count - paginated = paginate(count_entries) - return { - "status": "success", - "message": _with_note( - f"Total: {total_match_count} match(es) in " - f"{len(matched_filenames)} file(s)" - ), - "mode": "count", - "num_files": len(matched_filenames), - "filenames": matched_filenames, - "content": "\n".join(paginated) + "\n" if paginated else "", - "num_lines": None, - "num_matches": total_match_count, - "applied_limit": effective_limit, - "applied_offset": offset, - } + if capped: + notes.append( + f"Output capped at {MAX_CONTENT_CHARS} characters; narrow the search " + f"to see the rest." + ) + if lines_cut: + notes.append( + f"{lines_cut} long line{'' if lines_cut == 1 else 's'} cut to " + f"{MAX_LINE_CHARS} characters; read_file shows them in full." + ) + if returncode == 2 and not limit_hit: + notes.append("Some files could not be read.") + + return make_result( + "success", + " ".join(notes), + "\n".join(lines), + num_matches=shown_matches, + num_files=len(shown_files), + truncated=limit_hit or capped, + ) diff --git a/app/data/agent_file_system_template/AGENT.md b/app/data/agent_file_system_template/AGENT.md index 2036db75..bcc340a4 100644 --- a/app/data/agent_file_system_template/AGENT.md +++ b/app/data/agent_file_system_template/AGENT.md @@ -770,12 +770,11 @@ You're blocked when you don't know what to do next AND retrying won't help. The - Full input schema: [app/data/action/read_file.py](app/data/action/read_file.py). ### grep_files -Three output modes: -- `files_with_matches`: returns file paths only. Use for discovery ("which files contain X"). -- `content`: returns matching lines with line numbers. Use for investigation. -- `count`: returns match counts per file. Use for frequency checks. +Runs ripgrep and returns matching lines as `path:line: text`. When you search a folder, paths are relative to it, and the message names the folder. With `context`, surrounding lines read `path-line- text` and separate groups are split by `--`. -Supported parameters: `glob`, `file_type`, `before_context` / `after_context`, `case_insensitive`, `multiline`. +Parameters: `pattern` (ripgrep regex), `path`, `glob`, `ignore_case`, `context`, `limit` (default 100 matches; the message says when the limit cut the results short). + +A folder search skips files ignored by `.gitignore`, hidden files and binary files. To search one of those, pass its file path. Full input schema: [app/data/action/grep_files.py](app/data/action/grep_files.py). diff --git a/app/provision/__init__.py b/app/provision/__init__.py index 0580b8d8..3a39b14e 100644 --- a/app/provision/__init__.py +++ b/app/provision/__init__.py @@ -19,6 +19,7 @@ node → the one Node runtime frontend → browser UI's npm tree (default UI, so required) whatsapp → bridge's npm tree + ripgrep → the bundled search binary grep_files runs, checked not fetched playwright → chromium for browser-automation actions Node comes after Python deps because nothing before it needs Node, and a slow @@ -39,7 +40,7 @@ WhatsAppBridgeStage, ) from app.provision.pipeline import format_report, run -from app.provision.runtimes import NodeStage, PythonStage +from app.provision.runtimes import NodeStage, PythonStage, RipgrepStage from app.provision.types import ( Context, LogFn, @@ -79,6 +80,7 @@ def default_stages() -> List[Stage]: NodeStage(), FrontendStage(), WhatsAppBridgeStage(), + RipgrepStage(), PlaywrightStage(), ] diff --git a/app/provision/runtimes.py b/app/provision/runtimes.py index 238715de..458e79e7 100644 --- a/app/provision/runtimes.py +++ b/app/provision/runtimes.py @@ -78,6 +78,62 @@ def apply(self, ctx: Context, log: LogFn) -> StageResult: ) +class RipgrepStage: + """The rg binary bundled under app/vendor/ripgrep/ runs. + + Nothing to download: the binary ships in the source. Unzipping the install + payload drops the executable bit on macOS and Linux, so apply() restores + it. Paths come from ctx.code_root because the installer runs this stage + from its own copy of the code against the tree it just extracted. + """ + + name = "ripgrep" + description = "ripgrep (file search)" + # Only grep_files needs it; the rest of the agent runs without it. + optional = True + + def check(self, ctx: Context) -> StageResult: + from app import ripgrep_runtime + + try: + path = ripgrep_runtime.binary_path(Path(ctx.code_root)) + except RuntimeError as e: + return StageResult(Status.MISSING, str(e)) + if not path.is_file(): + return StageResult(Status.MISSING, f"bundled binary missing: {path}") + try: + res = subprocess.run( + [str(path), "--version"], capture_output=True, text=True, timeout=30 + ) + except OSError as e: + return StageResult( + Status.DEGRADED, + f"bundled binary will not run: {e}", + {"path": str(path)}, + ) + if res.returncode != 0: + return StageResult( + Status.DEGRADED, + f"bundled binary exited with {res.returncode}", + {"path": str(path)}, + ) + version = (res.stdout or "").splitlines()[0] if res.stdout else "" + return StageResult(Status.SATISFIED, version, {"path": str(path)}) + + def apply(self, ctx: Context, log: LogFn) -> StageResult: + from app import ripgrep_runtime + + try: + path = ripgrep_runtime.binary_path(Path(ctx.code_root)) + except RuntimeError as e: + return StageResult(Status.FAILED, str(e)) + if not path.is_file(): + return StageResult(Status.FAILED, f"bundled binary missing: {path}") + if sys.platform != "win32": + os.chmod(path, 0o755) + return self.check(ctx) + + def _probe_python(exe: str) -> Optional[tuple]: """(major, minor) of an interpreter, or None if it will not run. diff --git a/app/ripgrep_runtime.py b/app/ripgrep_runtime.py new file mode 100644 index 00000000..adc4e729 --- /dev/null +++ b/app/ripgrep_runtime.py @@ -0,0 +1,73 @@ +"""The ripgrep binary bundled with CraftBot, which grep_files runs. + +The official release binary for every platform CraftBot ships is committed +under app/vendor/ripgrep//; scripts/vendor_ripgrep.py puts it there +after checking each archive against the checksum the release publishes. + +The path comes from the code tree, never from user data, so the binary moves +and upgrades together with the code that parses its `--json` output. Nothing +is downloaded at runtime. The install pipeline only checks that the binary +runs (RipgrepStage): unzipping the install payload drops the executable bit +on macOS and Linux, and the stage restores it. + +Stdlib-only: install.py imports the provisioning stages before any +dependencies exist. +""" + +import platform +import sys +from pathlib import Path +from typing import List, Optional + +# (sys.platform, machine) -> ripgrep release target, one per platform +# CraftBot ships a dependency lock for. +TARGETS = { + ("win32", "x86_64"): "x86_64-pc-windows-msvc", + ("darwin", "aarch64"): "aarch64-apple-darwin", + ("linux", "x86_64"): "x86_64-unknown-linux-musl", +} + +# platform.machine() spellings -> the names TARGETS uses. +_MACHINES = { + "amd64": "x86_64", + "x86_64": "x86_64", + "arm64": "aarch64", + "aarch64": "aarch64", +} + +_CODE_ROOT = Path(__file__).resolve().parents[1] + + +def binary_name(plat: str = sys.platform) -> str: + return "rg.exe" if plat == "win32" else "rg" + + +def bundled_files() -> List[str]: + """Code-root-relative path of every bundled binary.""" + return [ + f"app/vendor/ripgrep/{target}/{binary_name(plat)}" + for (plat, _machine), target in TARGETS.items() + ] + + +def binary_path(code_root: Optional[Path] = None) -> Path: + """Where this machine's rg sits under code_root (default: this code tree). + + Raises RuntimeError when CraftBot bundles no ripgrep for this platform. + """ + machine = _MACHINES.get(platform.machine().lower(), platform.machine()) + target = TARGETS.get((sys.platform, machine)) + if target is None: + raise RuntimeError( + f"ripgrep is not bundled for {sys.platform} on {platform.machine()}" + ) + root = code_root if code_root is not None else _CODE_ROOT + return root / "app" / "vendor" / "ripgrep" / target / binary_name() + + +def resolve(code_root: Optional[Path] = None) -> str: + """Path of the rg binary grep_files runs. Raises RuntimeError with the reason.""" + path = binary_path(code_root) + if not path.is_file(): + raise RuntimeError(f"the bundled ripgrep is missing from this install: {path}") + return str(path) diff --git a/app/ui_layer/browser/frontend/src/components/activity/mascotFormatters.ts b/app/ui_layer/browser/frontend/src/components/activity/mascotFormatters.ts index d9c20a56..94c16b8b 100644 --- a/app/ui_layer/browser/frontend/src/components/activity/mascotFormatters.ts +++ b/app/ui_layer/browser/frontend/src/components/activity/mascotFormatters.ts @@ -5,7 +5,7 @@ // you add a new action name to SUPPORTED_ACTION_NAMES, TypeScript will // force you to add the matching formatter to FORMATTER_REGISTRY here. -import { basename, strField, arrField, dictField } from './parse' +import { basename, strField, arrField, dictField, numField } from './parse' import { extractTodos, isSupportedActionName, normalizeActionName, type SupportedActionName } from './actionNames' import i18n from '../../i18n/config' import { formatNumber } from '../../i18n/format' @@ -382,7 +382,7 @@ const grep_files: MascotActionFormatter = { const verb = s === 'error' ? i18n.t('activity:action.grepFiles.failed') : i18n.t('activity:action.grepFiles.cancelled') return { status: s, label: verb, body: p || undefined, bodyMono: true } } - const count = (arrField(o, 'filenames') ?? []).length + const count = numField(o, 'num_files') ?? 0 return { status: s, label: tCount('activity:action.grepFiles.foundIn', count), body: p || undefined, bodyMono: true } }, } diff --git a/app/ui_layer/browser/frontend/src/components/activity/parse.ts b/app/ui_layer/browser/frontend/src/components/activity/parse.ts index b97e1cb0..14e68858 100644 --- a/app/ui_layer/browser/frontend/src/components/activity/parse.ts +++ b/app/ui_layer/browser/frontend/src/components/activity/parse.ts @@ -151,6 +151,11 @@ export function arrField(obj: Record | null, key: string): unkn return parseArray(obj?.[key]) } +export function numField(obj: Record | null, key: string): number | undefined { + const v = obj?.[key] + return typeof v === 'number' ? v : undefined +} + export function dictField(obj: Record | null, key: string): Record | null { const v = obj?.[key] if (v && typeof v === 'object' && !Array.isArray(v)) { diff --git a/app/vendor/ripgrep/aarch64-apple-darwin/rg b/app/vendor/ripgrep/aarch64-apple-darwin/rg new file mode 100755 index 00000000..687bc365 Binary files /dev/null and b/app/vendor/ripgrep/aarch64-apple-darwin/rg differ diff --git a/app/vendor/ripgrep/x86_64-pc-windows-msvc/rg.exe b/app/vendor/ripgrep/x86_64-pc-windows-msvc/rg.exe new file mode 100644 index 00000000..92007cf7 Binary files /dev/null and b/app/vendor/ripgrep/x86_64-pc-windows-msvc/rg.exe differ diff --git a/app/vendor/ripgrep/x86_64-unknown-linux-musl/rg b/app/vendor/ripgrep/x86_64-unknown-linux-musl/rg new file mode 100755 index 00000000..4630d48b Binary files /dev/null and b/app/vendor/ripgrep/x86_64-unknown-linux-musl/rg differ diff --git a/scripts/package_source.py b/scripts/package_source.py index 807f2b9f..725843bd 100644 --- a/scripts/package_source.py +++ b/scripts/package_source.py @@ -37,6 +37,7 @@ from typing import List REPO_ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(REPO_ROOT)) # Gitignored build outputs that ARE required at runtime. Kept short and # explicit: every entry here is a hole in the "git decides" guarantee. @@ -126,9 +127,11 @@ def build(output: Path) -> int: size_mb = output.stat().st_size / (1024 * 1024) print(f"wrote {output} — {written} files, {size_mb:.1f} MB") + from app import ripgrep_runtime + # A payload without these is installable and then broken at runtime, in # ways that surface far from the cause. Fail the build instead. - required = [ + required = ripgrep_runtime.bundled_files() + [ "run.py", "main.py", "requirements.txt", diff --git a/scripts/vendor_ripgrep.py b/scripts/vendor_ripgrep.py new file mode 100644 index 00000000..0b9ced70 --- /dev/null +++ b/scripts/vendor_ripgrep.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 +"""Refresh the ripgrep binaries committed under app/vendor/ripgrep/. + +What this is for + grep_files runs a ripgrep (rg) binary that ships inside CraftBot, one per + platform listed in app.ripgrep_runtime.TARGETS. Those binaries are already + committed, so nothing runs this script at install time or at runtime. It + exists for the two moments the committed binaries have to change, and as a + record of exactly where they came from. + +When to run it + * Upgrading ripgrep: change VERSION below, then run it. + * Supporting another platform: add the platform to + app.ripgrep_runtime.TARGETS (its release target name is in the asset + list of the GitHub release), then run it. + +How + 1. python scripts/vendor_ripgrep.py + 2. python -m pytest tests/test_ripgrep_runtime.py tests/test_grep_files.py + 3. Stage the binaries. Git on Windows does not record the executable bit + (core.fileMode=false), so mark the macOS and Linux ones explicitly or a + git clone or Docker image will get files rg cannot be run from: + git add app/vendor/ripgrep + git update-index --chmod=+x app/vendor/ripgrep/*/rg + Check with `git ls-files -s app/vendor/ripgrep`: rg entries must read 100755. + +What it guarantees + Every binary comes from the official release at + github.com/BurntSushi/ripgrep and matches the .sha256 file that release + publishes next to the archive. A mismatch stops the script before anything + is written. The checksum comes from the same release, so it proves the + download is intact, not that the release itself is trustworthy. +""" + +from __future__ import annotations + +import hashlib +import io +import re +import sys +import tarfile +import urllib.request +import zipfile +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] +# Lets the script import app.* when run from anywhere as a plain file. +sys.path.insert(0, str(REPO_ROOT)) + +# The ripgrep release the committed binaries are taken from. Bumping this is +# the whole upgrade; grep_files parses `rg --json`, so run the tests after. +VERSION = "15.2.0" + +_RELEASE = "https://github.com/BurntSushi/ripgrep/releases/download/{version}/{asset}" + + +def _fetch(url: str) -> bytes: + """Download url into memory. Archives are about 2 MB each.""" + # CraftBot's shared SSL context: it prefers certifi's roots because the + # Windows certificate store fails to load under some OpenSSL builds. + from app import downloads + + req = urllib.request.Request(url, headers={"User-Agent": "CraftBot"}) + with urllib.request.urlopen( + req, timeout=120, context=downloads.ssl_context() + ) as resp: + return resp.read() + + +def _published_sha256(asset_url: str) -> str: + """The sha256 the release publishes for asset_url, from .sha256.""" + # The two formats differ: Unix assets publish " ", Windows ones + # publish CertUtil output with the hex on its own line. Both contain exactly + # one 64-character hex string, and anything else is refused rather than + # guessed at. + text = _fetch(asset_url + ".sha256").decode("utf-8", "replace") + found = re.findall(r"\b[0-9a-fA-F]{64}\b", text) + if len(found) != 1: + raise RuntimeError(f"unexpected checksum file for {asset_url}: {text!r}") + return found[0].lower() + + +def _binary_from_archive(archive: bytes, ext: str, member: str) -> bytes: + """Read one file out of a release archive (.zip on Windows, .tar.gz elsewhere).""" + if ext == ".zip": + with zipfile.ZipFile(io.BytesIO(archive)) as zf: + return zf.read(member) + with tarfile.open(fileobj=io.BytesIO(archive), mode="r:gz") as tf: + handle = tf.extractfile(member) + if handle is None: + raise RuntimeError(f"{member} is not a file in the archive") + return handle.read() + + +def main() -> int: + # TARGETS is the single list of bundled platforms: grep_files resolves its + # binary from it, the release build (scripts/package_source.py) refuses a + # payload missing any of them, and this script fills exactly those folders. + from app import ripgrep_runtime + + for (plat, _machine), target in ripgrep_runtime.TARGETS.items(): + ext = ".zip" if plat == "win32" else ".tar.gz" + name = ripgrep_runtime.binary_name(plat) + url = _RELEASE.format(version=VERSION, asset=f"ripgrep-{VERSION}-{target}{ext}") + + print(f"{target}:") + archive = _fetch(url) + expected = _published_sha256(url) + actual = hashlib.sha256(archive).hexdigest() + if actual != expected: + print( + f" sha256 {actual} does not match the published {expected}", + file=sys.stderr, + ) + return 1 + print(f" verified sha256 {actual}") + + # Release archives keep everything under ripgrep--/; + # only the rg binary is kept (docs and shell completions are not needed). + binary = _binary_from_archive( + archive, ext, f"ripgrep-{VERSION}-{target}/{name}" + ) + dest = REPO_ROOT / "app" / "vendor" / "ripgrep" / target / name + dest.parent.mkdir(parents=True, exist_ok=True) + dest.write_bytes(binary) + # Takes effect on macOS and Linux only. On Windows the bit has to be set + # in git instead (see step 3 at the top). + if plat != "win32": + dest.chmod(0o755) + print(f" wrote {dest.relative_to(REPO_ROOT)} ({len(binary) / 1048576:.1f} MB)") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_grep_files.py b/tests/test_grep_files.py new file mode 100644 index 00000000..b4c8bf7d --- /dev/null +++ b/tests/test_grep_files.py @@ -0,0 +1,226 @@ +"""grep_files runs ripgrep: line results, root-relative paths, .gitignore, limit. + +These drive the real bundled rg binary; they skip only on a platform CraftBot +bundles no ripgrep for. +""" + +import tracemalloc + +import pytest + +from app import ripgrep_runtime +from app.data.action.grep_files import grep_files + + +@pytest.fixture(scope="module", autouse=True) +def _ripgrep(): + try: + ripgrep_runtime.binary_path() + except RuntimeError as e: # pragma: no cover - platform dependent + pytest.skip(str(e)) + + +def _write(path, text): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + + +def test_returns_matching_lines_with_paths_relative_to_the_folder(tmp_path): + _write(tmp_path / "src" / "a.py", "import os\ndef hello():\n pass\n") + _write(tmp_path / "src" / "b.py", "def world():\n") + + res = grep_files({"pattern": r"def \w+\(", "path": str(tmp_path)}) + + assert res["status"] == "success" + lines = sorted(res["content"].splitlines()) + assert lines == ["src/a.py:2: def hello():", "src/b.py:1: def world():"] + assert res["num_matches"] == 2 + assert res["num_files"] == 2 + assert res["truncated"] is False + assert str(tmp_path) in res["message"] + + +def test_folder_search_honours_gitignore_outside_a_git_checkout(tmp_path): + _write(tmp_path / ".gitignore", "generated/\n") + _write(tmp_path / "generated" / "big.json", '{"needle": 1}\n') + _write(tmp_path / "src" / "keep.ts", "const needle = 1\n") + + res = grep_files({"pattern": "needle", "path": str(tmp_path)}) + + assert res["content"] == "src/keep.ts:1: const needle = 1" + + +def test_an_ignored_file_is_still_searched_when_named_directly(tmp_path): + _write(tmp_path / ".gitignore", "generated/\n") + target = tmp_path / "generated" / "big.json" + _write(target, '{"needle": 1}\n') + + res = grep_files({"pattern": "needle", "path": str(target)}) + + assert res["content"] == 'big.json:1: {"needle": 1}' + assert res["num_files"] == 1 + + +def test_stops_at_limit_and_says_so(tmp_path): + _write(tmp_path / "log.txt", "".join(f"hit {i}\n" for i in range(500))) + + res = grep_files({"pattern": "hit", "path": str(tmp_path), "limit": 10}) + + assert res["status"] == "success" + assert res["num_matches"] == 10 + assert len(res["content"].splitlines()) == 10 + assert res["truncated"] is True + assert "limit of 10" in res["message"] + + +def test_context_lines_and_group_separators(tmp_path): + body = "\n".join(["a", "b", "MATCH one", "c", "d", "e", "f", "g", "MATCH two", "h"]) + _write(tmp_path / "f.txt", body + "\n") + + res = grep_files({"pattern": "MATCH", "path": str(tmp_path), "context": 1}) + + assert res["content"].splitlines() == [ + "f.txt-2- b", + "f.txt:3: MATCH one", + "f.txt-4- c", + "--", + "f.txt-8- g", + "f.txt:9: MATCH two", + "f.txt-10- h", + ] + assert res["num_matches"] == 2 + + +def test_limit_with_context_keeps_trailing_context_but_not_the_next_lead_in(tmp_path): + body = "\n".join(["MATCH one", "after one", "x", "before two", "MATCH two"]) + _write(tmp_path / "f.txt", body + "\n") + + res = grep_files( + {"pattern": "MATCH", "path": str(tmp_path), "context": 1, "limit": 1} + ) + + assert res["content"].splitlines() == ["f.txt:1: MATCH one", "f.txt-2- after one"] + assert res["truncated"] is True + + +def test_glob_filters_files(tmp_path): + _write(tmp_path / "a.py", "needle\n") + _write(tmp_path / "a.md", "needle\n") + + res = grep_files({"pattern": "needle", "path": str(tmp_path), "glob": "*.py"}) + + assert res["content"] == "a.py:1: needle" + + +def test_ignore_case(tmp_path): + _write(tmp_path / "a.txt", "Needle\n") + + assert grep_files({"pattern": "needle", "path": str(tmp_path)})["num_matches"] == 0 + res = grep_files({"pattern": "needle", "path": str(tmp_path), "ignore_case": True}) + assert res["content"] == "a.txt:1: Needle" + + +def test_long_lines_are_cut(tmp_path): + _write(tmp_path / "bundle.js", "needle" + "x" * 5000 + "\n") + + res = grep_files({"pattern": "needle", "path": str(tmp_path)}) + + line = res["content"] + assert line.startswith("bundle.js:1: needle") + assert line.endswith("[line cut]") + assert len(line) < 600 + assert "cut to 500 characters" in res["message"] + + +def _peak_bytes(call): + tracemalloc.start() + try: + result = call() + return result, tracemalloc.get_traced_memory()[1] + finally: + tracemalloc.stop() + + +def test_a_minified_line_with_many_occurrences_stays_small(tmp_path): + # 250,000 occurrences on one line: rg's --json described each of them and + # took over 100 MB to parse. + _write(tmp_path / "bundle.min.js", "a=1;" * 250_000 + "\n") + + res, peak = _peak_bytes( + lambda: grep_files({"pattern": "a", "path": str(tmp_path), "limit": 1}) + ) + + assert res["content"].startswith("bundle.min.js:1: a=1;") + assert res["content"].endswith("[line cut]") + assert peak < 2_000_000 + + +def test_content_budget_applies_while_reading(tmp_path): + _write(tmp_path / "log.txt", "hit\n" * 300_000) + + res, peak = _peak_bytes( + lambda: grep_files( + {"pattern": "hit", "path": str(tmp_path), "limit": 1_000_000} + ) + ) + + assert res["truncated"] is True + assert "Output capped at 40000 characters" in res["message"] + assert len(res["content"]) <= 40000 + assert peak < 2_000_000 + + +def test_a_line_of_exactly_500_wide_characters_is_not_cut(tmp_path): + # 2,000 bytes plus a CRLF ending: rg counts the ending against its byte + # cap, so the cap leaves room for it. + line = "\U0001f600" * 500 + target = tmp_path / "wide.txt" + target.write_bytes((line + "\r\n").encode("utf-8")) + + res = grep_files({"pattern": ".", "path": str(target)}) + + assert res["content"] == f"wide.txt:1: {line}" + assert "cut to 500 characters" not in res["message"] + + +def test_a_binary_file_named_directly_is_still_searched(tmp_path): + target = tmp_path / "data.bin" + target.write_bytes(b"needle one\n\x00\nneedle two\n") + + res = grep_files({"pattern": "needle", "path": str(target)}) + + assert res["content"].splitlines() == [ + "data.bin:1: needle one", + "data.bin:3: needle two", + ] + + +def test_no_match_explains_what_a_folder_search_skips(tmp_path): + _write(tmp_path / "a.txt", "nothing here\n") + + res = grep_files({"pattern": "needle", "path": str(tmp_path)}) + + assert res["status"] == "success" + assert res["content"] == "" + assert res["num_matches"] == 0 + assert ".gitignore" in res["message"] + + +def test_invalid_regex_is_an_error(tmp_path): + _write(tmp_path / "a.txt", "x\n") + + res = grep_files({"pattern": "foo(", "path": str(tmp_path)}) + + assert res["status"] == "error" + assert "regex parse error" in res["message"] + + +def test_missing_path_is_an_error(tmp_path): + res = grep_files({"pattern": "x", "path": str(tmp_path / "nope")}) + + assert res["status"] == "error" + assert "does not exist" in res["message"] + + +def test_pattern_is_required(tmp_path): + assert grep_files({"path": str(tmp_path)})["status"] == "error" diff --git a/tests/test_ripgrep_runtime.py b/tests/test_ripgrep_runtime.py new file mode 100644 index 00000000..7e06ee9a --- /dev/null +++ b/tests/test_ripgrep_runtime.py @@ -0,0 +1,87 @@ +"""The ripgrep binary ships in the code tree and the install stage keeps it runnable.""" + +import os +import shutil +import stat +import subprocess +import sys +from pathlib import Path + +import pytest + +from app import ripgrep_runtime +from app.provision.runtimes import RipgrepStage +from app.provision.types import Context, Status + +REPO_ROOT = Path(__file__).resolve().parents[1] + + +def _bundled_here() -> bool: + try: + ripgrep_runtime.binary_path() + except RuntimeError: + return False + return True + + +def test_every_bundled_binary_is_in_the_tree(): + for rel in ripgrep_runtime.bundled_files(): + path = REPO_ROOT / rel + assert path.is_file(), rel + assert path.stat().st_size > 1_000_000, rel + + +@pytest.mark.skipif( + not _bundled_here(), reason="ripgrep is not bundled for this platform" +) +def test_resolves_to_a_binary_that_runs(): + path = ripgrep_runtime.resolve() + + assert Path(path).is_relative_to(REPO_ROOT / "app" / "vendor" / "ripgrep") + out = subprocess.run([path, "--version"], capture_output=True, text=True) + assert out.returncode == 0 + assert out.stdout.startswith("ripgrep ") + + +def test_unbundled_platform_is_named_in_the_error(monkeypatch): + monkeypatch.setattr(ripgrep_runtime.platform, "machine", lambda: "sparc64") + + with pytest.raises(RuntimeError, match="not bundled for .* on sparc64"): + ripgrep_runtime.resolve() + + +@pytest.mark.skipif( + not _bundled_here(), reason="ripgrep is not bundled for this platform" +) +def test_missing_binary_is_an_error(tmp_path): + with pytest.raises(RuntimeError, match="missing from this install"): + ripgrep_runtime.resolve(tmp_path) + + +def _ctx(code_root: Path) -> Context: + return Context(code_root=str(code_root), state_root=str(code_root)) + + +@pytest.mark.skipif( + not _bundled_here(), reason="ripgrep is not bundled for this platform" +) +def test_stage_checks_the_tree_it_is_given(tmp_path): + assert RipgrepStage().check(_ctx(REPO_ROOT)).status is Status.SATISFIED + assert RipgrepStage().check(_ctx(tmp_path)).status is Status.MISSING + + +@pytest.mark.skipif( + sys.platform == "win32" or not _bundled_here(), + reason="the executable bit only exists on macOS and Linux", +) +def test_stage_restores_the_executable_bit_an_unzip_dropped(tmp_path): + rel = ripgrep_runtime.binary_path().relative_to(REPO_ROOT) + copy = tmp_path / rel + copy.parent.mkdir(parents=True) + shutil.copyfile(REPO_ROOT / rel, copy) + os.chmod(copy, 0o644) + stage = RipgrepStage() + + assert stage.check(_ctx(tmp_path)).status is Status.DEGRADED + assert stage.apply(_ctx(tmp_path), lambda _m: None).status is Status.SATISFIED + assert copy.stat().st_mode & stat.S_IXUSR