grep improvement update using ripgrep

agent_file_system/AGENT.md

@@ -770,12 +770,11 @@ You're blocked when you don't know what to do next AND retrying won't help. The
 - Full input schema: [app/data/action/read_file.py](app/data/action/read_file.py).
 
 ### grep_files
-Three output modes:
-- `files_with_matches`: returns file paths only. Use for discovery ("which files contain X").
-- `content`: returns matching lines with line numbers. Use for investigation.
-- `count`: returns match counts per file. Use for frequency checks.
+Runs ripgrep and returns matching lines as `path:line: text`. When you search a folder, paths are relative to it, and the message names the folder. With `context`, surrounding lines read `path-line- text` and separate groups are split by `--`.
 
-Supported parameters: `glob`, `file_type`, `before_context` / `after_context`, `case_insensitive`, `multiline`.
+Parameters: `pattern` (ripgrep regex), `path`, `glob`, `ignore_case`, `context`, `limit` (default 100 matches; the message says when the limit cut the results short).
+
+A folder search skips files ignored by `.gitignore`, hidden files and binary files. To search one of those, pass its file path.
 
 Full input schema: [app/data/action/grep_files.py](app/data/action/grep_files.py).
 

app/data/agent_file_system_template/AGENT.md

@@ -770,12 +770,11 @@ You're blocked when you don't know what to do next AND retrying won't help. The
 - Full input schema: [app/data/action/read_file.py](app/data/action/read_file.py).
 
 ### grep_files
-Three output modes:
-- `files_with_matches`: returns file paths only. Use for discovery ("which files contain X").
-- `content`: returns matching lines with line numbers. Use for investigation.
-- `count`: returns match counts per file. Use for frequency checks.
+Runs ripgrep and returns matching lines as `path:line: text`. When you search a folder, paths are relative to it, and the message names the folder. With `context`, surrounding lines read `path-line- text` and separate groups are split by `--`.
 
-Supported parameters: `glob`, `file_type`, `before_context` / `after_context`, `case_insensitive`, `multiline`.
+Parameters: `pattern` (ripgrep regex), `path`, `glob`, `ignore_case`, `context`, `limit` (default 100 matches; the message says when the limit cut the results short).
+
+A folder search skips files ignored by `.gitignore`, hidden files and binary files. To search one of those, pass its file path.
 
 Full input schema: [app/data/action/grep_files.py](app/data/action/grep_files.py).
 

app/provision/__init__.py

@@ -19,6 +19,7 @@
   node        → the one Node runtime
   frontend    → browser UI's npm tree (default UI, so required)
   whatsapp    → bridge's npm tree
+  ripgrep     → the bundled search binary grep_files runs, checked not fetched
   playwright  → chromium for browser-automation actions
 
 Node comes after Python deps because nothing before it needs Node, and a slow
@@ -39,7 +40,7 @@
     WhatsAppBridgeStage,
 )
 from app.provision.pipeline import format_report, run
-from app.provision.runtimes import NodeStage, PythonStage
+from app.provision.runtimes import NodeStage, PythonStage, RipgrepStage
 from app.provision.types import (
     Context,
     LogFn,
@@ -79,6 +80,7 @@ def default_stages() -> List[Stage]:
         NodeStage(),
         FrontendStage(),
         WhatsAppBridgeStage(),
+        RipgrepStage(),
         PlaywrightStage(),
     ]
 

app/provision/runtimes.py

@@ -78,6 +78,62 @@ def apply(self, ctx: Context, log: LogFn) -> StageResult:
         )
 
 
+class RipgrepStage:
+    """The rg binary bundled under app/vendor/ripgrep/ runs.
+
+    Nothing to download: the binary ships in the source. Unzipping the install
+    payload drops the executable bit on macOS and Linux, so apply() restores
+    it. Paths come from ctx.code_root because the installer runs this stage
+    from its own copy of the code against the tree it just extracted.
+    """
+
+    name = "ripgrep"
+    description = "ripgrep (file search)"
+    # Only grep_files needs it; the rest of the agent runs without it.
+    optional = True
+
+    def check(self, ctx: Context) -> StageResult:
+        from app import ripgrep_runtime
+
+        try:
+            path = ripgrep_runtime.binary_path(Path(ctx.code_root))
+        except RuntimeError as e:
+            return StageResult(Status.MISSING, str(e))
+        if not path.is_file():
+            return StageResult(Status.MISSING, f"bundled binary missing: {path}")
+        try:
+            res = subprocess.run(
+                [str(path), "--version"], capture_output=True, text=True, timeout=30
+            )
+        except OSError as e:
+            return StageResult(
+                Status.DEGRADED,
+                f"bundled binary will not run: {e}",
+                {"path": str(path)},
+            )
+        if res.returncode != 0:
+            return StageResult(
+                Status.DEGRADED,
+                f"bundled binary exited with {res.returncode}",
+                {"path": str(path)},
+            )
+        version = (res.stdout or "").splitlines()[0] if res.stdout else ""
+        return StageResult(Status.SATISFIED, version, {"path": str(path)})
+
+    def apply(self, ctx: Context, log: LogFn) -> StageResult:
+        from app import ripgrep_runtime
+
+        try:
+            path = ripgrep_runtime.binary_path(Path(ctx.code_root))
+        except RuntimeError as e:
+            return StageResult(Status.FAILED, str(e))
+        if not path.is_file():
+            return StageResult(Status.FAILED, f"bundled binary missing: {path}")
+        if sys.platform != "win32":
+            os.chmod(path, 0o755)
+        return self.check(ctx)
+
+
 def _probe_python(exe: str) -> Optional[tuple]:
     """(major, minor) of an interpreter, or None if it will not run.
 

app/ripgrep_runtime.py

@@ -0,0 +1,73 @@
+"""The ripgrep binary bundled with CraftBot, which grep_files runs.
+
+The official release binary for every platform CraftBot ships is committed
+under app/vendor/ripgrep/<target>/; scripts/vendor_ripgrep.py puts it there
+after checking each archive against the checksum the release publishes.
+
+The path comes from the code tree, never from user data, so the binary moves
+and upgrades together with the code that parses its `--json` output. Nothing
+is downloaded at runtime. The install pipeline only checks that the binary
+runs (RipgrepStage): unzipping the install payload drops the executable bit
+on macOS and Linux, and the stage restores it.
+
+Stdlib-only: install.py imports the provisioning stages before any
+dependencies exist.
+"""
+
+import platform
+import sys
+from pathlib import Path
+from typing import List, Optional
+
+# (sys.platform, machine) -> ripgrep release target, one per platform
+# CraftBot ships a dependency lock for.
+TARGETS = {
+    ("win32", "x86_64"): "x86_64-pc-windows-msvc",
+    ("darwin", "aarch64"): "aarch64-apple-darwin",
+    ("linux", "x86_64"): "x86_64-unknown-linux-musl",
+}
+
+# platform.machine() spellings -> the names TARGETS uses.
+_MACHINES = {
+    "amd64": "x86_64",
+    "x86_64": "x86_64",
+    "arm64": "aarch64",
+    "aarch64": "aarch64",
+}
+
+_CODE_ROOT = Path(__file__).resolve().parents[1]
+
+
+def binary_name(plat: str = sys.platform) -> str:
+    return "rg.exe" if plat == "win32" else "rg"
+
+
+def bundled_files() -> List[str]:
+    """Code-root-relative path of every bundled binary."""
+    return [
+        f"app/vendor/ripgrep/{target}/{binary_name(plat)}"
+        for (plat, _machine), target in TARGETS.items()
+    ]
+
+
+def binary_path(code_root: Optional[Path] = None) -> Path:
+    """Where this machine's rg sits under code_root (default: this code tree).
+
+    Raises RuntimeError when CraftBot bundles no ripgrep for this platform.
+    """
+    machine = _MACHINES.get(platform.machine().lower(), platform.machine())
+    target = TARGETS.get((sys.platform, machine))
+    if target is None:
+        raise RuntimeError(
+            f"ripgrep is not bundled for {sys.platform} on {platform.machine()}"
+        )
+    root = code_root if code_root is not None else _CODE_ROOT
+    return root / "app" / "vendor" / "ripgrep" / target / binary_name()
+
+
+def resolve(code_root: Optional[Path] = None) -> str:
+    """Path of the rg binary grep_files runs. Raises RuntimeError with the reason."""
+    path = binary_path(code_root)
+    if not path.is_file():
+        raise RuntimeError(f"the bundled ripgrep is missing from this install: {path}")
+    return str(path)

app/ui_layer/browser/frontend/src/components/activity/mascotFormatters.ts

@@ -5,7 +5,7 @@
 // you add a new action name to SUPPORTED_ACTION_NAMES, TypeScript will
 // force you to add the matching formatter to FORMATTER_REGISTRY here.
 
-import { basename, strField, arrField, dictField } from './parse'
+import { basename, strField, arrField, dictField, numField } from './parse'
 import { extractTodos, isSupportedActionName, normalizeActionName, type SupportedActionName } from './actionNames'
 import i18n from '../../i18n/config'
 import { formatNumber } from '../../i18n/format'
@@ -382,7 +382,7 @@ const grep_files: MascotActionFormatter = {
       const verb = s === 'error' ? i18n.t('activity:action.grepFiles.failed') : i18n.t('activity:action.grepFiles.cancelled')
       return { status: s, label: verb, body: p || undefined, bodyMono: true }
     }
-    const count = (arrField(o, 'filenames') ?? []).length
+    const count = numField(o, 'num_files') ?? 0
     return { status: s, label: tCount('activity:action.grepFiles.foundIn', count), body: p || undefined, bodyMono: true }
   },
 }

app/ui_layer/browser/frontend/src/components/activity/parse.ts

@@ -151,6 +151,11 @@ export function arrField(obj: Record<string, unknown> | null, key: string): unkn
   return parseArray(obj?.[key])
 }
 
+export function numField(obj: Record<string, unknown> | null, key: string): number | undefined {
+  const v = obj?.[key]
+  return typeof v === 'number' ? v : undefined
+}
+
 export function dictField(obj: Record<string, unknown> | null, key: string): Record<string, unknown> | null {
   const v = obj?.[key]
   if (v && typeof v === 'object' && !Array.isArray(v)) {

scripts/package_source.py

@@ -37,6 +37,7 @@
 from typing import List
 
 REPO_ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(REPO_ROOT))
 
 # Gitignored build outputs that ARE required at runtime. Kept short and
 # explicit: every entry here is a hole in the "git decides" guarantee.
@@ -126,9 +127,11 @@ def build(output: Path) -> int:
     size_mb = output.stat().st_size / (1024 * 1024)
     print(f"wrote {output} — {written} files, {size_mb:.1f} MB")
 
+    from app import ripgrep_runtime
+
     # A payload without these is installable and then broken at runtime, in
     # ways that surface far from the cause. Fail the build instead.
-    required = [
+    required = ripgrep_runtime.bundled_files() + [
         "run.py",
         "main.py",
         "requirements.txt",

scripts/vendor_ripgrep.py

@@ -0,0 +1,135 @@
+#!/usr/bin/env python3
+"""Refresh the ripgrep binaries committed under app/vendor/ripgrep/.
+
+What this is for
+    grep_files runs a ripgrep (rg) binary that ships inside CraftBot, one per
+    platform listed in app.ripgrep_runtime.TARGETS. Those binaries are already
+    committed, so nothing runs this script at install time or at runtime. It
+    exists for the two moments the committed binaries have to change, and as a
+    record of exactly where they came from.
+
+When to run it
+    * Upgrading ripgrep: change VERSION below, then run it.
+    * Supporting another platform: add the platform to
+      app.ripgrep_runtime.TARGETS (its release target name is in the asset
+      list of the GitHub release), then run it.
+
+How
+    1. python scripts/vendor_ripgrep.py
+    2. python -m pytest tests/test_ripgrep_runtime.py tests/test_grep_files.py
+    3. Stage the binaries. Git on Windows does not record the executable bit
+       (core.fileMode=false), so mark the macOS and Linux ones explicitly or a
+       git clone or Docker image will get files rg cannot be run from:
+           git add app/vendor/ripgrep
+           git update-index --chmod=+x app/vendor/ripgrep/*/rg
+       Check with `git ls-files -s app/vendor/ripgrep`: rg entries must read 100755.
+
+What it guarantees
+    Every binary comes from the official release at
+    github.com/BurntSushi/ripgrep and matches the .sha256 file that release
+    publishes next to the archive. A mismatch stops the script before anything
+    is written. The checksum comes from the same release, so it proves the
+    download is intact, not that the release itself is trustworthy.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import io
+import re
+import sys
+import tarfile
+import urllib.request
+import zipfile
+from pathlib import Path
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+# Lets the script import app.* when run from anywhere as a plain file.
+sys.path.insert(0, str(REPO_ROOT))
+
+# The ripgrep release the committed binaries are taken from. Bumping this is
+# the whole upgrade; grep_files parses `rg --json`, so run the tests after.
+VERSION = "15.2.0"
+
+_RELEASE = "https://github.com/BurntSushi/ripgrep/releases/download/{version}/{asset}"
+
+
+def _fetch(url: str) -> bytes:
+    """Download url into memory. Archives are about 2 MB each."""
+    # CraftBot's shared SSL context: it prefers certifi's roots because the
+    # Windows certificate store fails to load under some OpenSSL builds.
+    from app import downloads
+
+    req = urllib.request.Request(url, headers={"User-Agent": "CraftBot"})
+    with urllib.request.urlopen(
+        req, timeout=120, context=downloads.ssl_context()
+    ) as resp:
+        return resp.read()
+
+
+def _published_sha256(asset_url: str) -> str:
+    """The sha256 the release publishes for asset_url, from <asset>.sha256."""
+    # The two formats differ: Unix assets publish "<hex>  <name>", Windows ones
+    # publish CertUtil output with the hex on its own line. Both contain exactly
+    # one 64-character hex string, and anything else is refused rather than
+    # guessed at.
+    text = _fetch(asset_url + ".sha256").decode("utf-8", "replace")
+    found = re.findall(r"\b[0-9a-fA-F]{64}\b", text)
+    if len(found) != 1:
+        raise RuntimeError(f"unexpected checksum file for {asset_url}: {text!r}")
+    return found[0].lower()
+
+
+def _binary_from_archive(archive: bytes, ext: str, member: str) -> bytes:
+    """Read one file out of a release archive (.zip on Windows, .tar.gz elsewhere)."""
+    if ext == ".zip":
+        with zipfile.ZipFile(io.BytesIO(archive)) as zf:
+            return zf.read(member)
+    with tarfile.open(fileobj=io.BytesIO(archive), mode="r:gz") as tf:
+        handle = tf.extractfile(member)
+        if handle is None:
+            raise RuntimeError(f"{member} is not a file in the archive")
+        return handle.read()
+
+
+def main() -> int:
+    # TARGETS is the single list of bundled platforms: grep_files resolves its
+    # binary from it, the release build (scripts/package_source.py) refuses a
+    # payload missing any of them, and this script fills exactly those folders.
+    from app import ripgrep_runtime
+
+    for (plat, _machine), target in ripgrep_runtime.TARGETS.items():
+        ext = ".zip" if plat == "win32" else ".tar.gz"
+        name = ripgrep_runtime.binary_name(plat)
+        url = _RELEASE.format(version=VERSION, asset=f"ripgrep-{VERSION}-{target}{ext}")
+
+        print(f"{target}:")
+        archive = _fetch(url)
+        expected = _published_sha256(url)
+        actual = hashlib.sha256(archive).hexdigest()
+        if actual != expected:
+            print(
+                f"  sha256 {actual} does not match the published {expected}",
+                file=sys.stderr,
+            )
+            return 1
+        print(f"  verified sha256 {actual}")
+
+        # Release archives keep everything under ripgrep-<version>-<target>/;
+        # only the rg binary is kept (docs and shell completions are not needed).
+        binary = _binary_from_archive(
+            archive, ext, f"ripgrep-{VERSION}-{target}/{name}"
+        )
+        dest = REPO_ROOT / "app" / "vendor" / "ripgrep" / target / name
+        dest.parent.mkdir(parents=True, exist_ok=True)
+        dest.write_bytes(binary)
+        # Takes effect on macOS and Linux only. On Windows the bit has to be set
+        # in git instead (see step 3 at the top).
+        if plat != "win32":
+            dest.chmod(0o755)
+        print(f"  wrote {dest.relative_to(REPO_ROOT)} ({len(binary) / 1048576:.1f} MB)")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())