Feature : RTK compress

open-sse/rtk/filters/dedupLog.js

@@ -0,0 +1,44 @@
+// Generic fallback: collapse consecutive duplicate lines + blank-line dedupe + hard line cap
+import { DEDUP_LINE_MAX } from "../constants.js";
+
+export function dedupLog(input) {
+  const lines = input.split("\n");
+  const out = [];
+  let prev = null;
+  let runCount = 0;
+  let blankStreak = 0;
+
+  const flushRun = () => {
+    if (prev !== null && runCount > 1) {
+      out.push(`  ... (${runCount - 1} duplicate lines)`);
+    }
+  };
+
+  for (const line of lines) {
+    if (line.trim() === "") {
+      if (blankStreak < 1) out.push(line);
+      blankStreak += 1;
+      flushRun();
+      prev = null;
+      runCount = 0;
+      continue;
+    }
+    blankStreak = 0;
+    if (line === prev) {
+      runCount += 1;
+      continue;
+    }
+    flushRun();
+    out.push(line);
+    prev = line;
+    runCount = 1;
+    if (out.length >= DEDUP_LINE_MAX) {
+      out.push(`... (truncated at ${DEDUP_LINE_MAX} lines)`);
+      return out.join("\n");
+    }
+  }
+  flushRun();
+  return out.join("\n");
+}
+
+dedupLog.filterName = "dedup-log";

open-sse/rtk/filters/find.js

@@ -0,0 +1,49 @@
+// Port of find_wrapper (rtk/src/cmds/system/pipe_cmd.rs:89-128)
+// Group by parent dir, show basenames, cap 10/dir and 20 dirs total
+import { FIND_PER_DIR_MAX, FIND_TOTAL_DIR_MAX } from "../constants.js";
+
+export function find(input) {
+  const lines = input.split("\n").filter(l => l.trim());
+  if (lines.length === 0) return input;
+
+  const byDir = new Map();
+
+  for (const path of lines) {
+    const lastSlash = path.lastIndexOf("/");
+    let dir;
+    let basename;
+    if (lastSlash === -1) {
+      dir = ".";
+      basename = path;
+    } else {
+      // Rust: PathBuf::from(path).parent().display() + file_name().display()
+      dir = path.slice(0, lastSlash) || "/";
+      basename = path.slice(lastSlash + 1);
+    }
+    if (!byDir.has(dir)) byDir.set(dir, []);
+    byDir.get(dir).push(basename);
+  }
+
+  // Rust: dirs.sort_by_key(|(d, _)| d.clone())
+  const dirs = Array.from(byDir.keys()).sort();
+  let out = `${lines.length} files in ${dirs.length} dirs:\n\n`;
+
+  const showDirs = dirs.slice(0, FIND_TOTAL_DIR_MAX);
+  for (const dir of showDirs) {
+    const files = byDir.get(dir);
+    out += `${dir}/ (${files.length}):\n`;
+    const showFiles = files.slice(0, FIND_PER_DIR_MAX);
+    for (const f of showFiles) out += `  ${f}\n`;
+    if (files.length > FIND_PER_DIR_MAX) {
+      out += `  +${files.length - FIND_PER_DIR_MAX}\n`;
+    }
+    out += "\n";
+  }
+  if (dirs.length > FIND_TOTAL_DIR_MAX) {
+    out += `+${dirs.length - FIND_TOTAL_DIR_MAX} more dirs\n`;
+  }
+
+  return out;
+}
+
+find.filterName = "find";

open-sse/rtk/filters/gitDiff.js

@@ -0,0 +1,92 @@
+// Port of Rust git::compact_diff (src/cmds/git/git.rs L325-413)
+// Compacts unified diff: file headers, hunk-level truncation at 100 lines, +/-/context counting
+import { GIT_DIFF_HUNK_MAX_LINES } from "../constants.js";
+
+export function gitDiff(diff, maxLines = 500) {
+  const result = [];
+  let currentFile = "";
+  let added = 0;
+  let removed = 0;
+  let inHunk = false;
+  let hunkShown = 0;
+  let hunkSkipped = 0;
+  let wasTruncated = false;
+  const maxHunkLines = GIT_DIFF_HUNK_MAX_LINES;
+
+  const lines = diff.split("\n");
+
+  outer: for (const line of lines) {
+    if (line.startsWith("diff --git")) {
+      if (hunkSkipped > 0) {
+        result.push(`  ... (${hunkSkipped} lines truncated)`);
+        wasTruncated = true;
+        hunkSkipped = 0;
+      }
+      if (currentFile && (added > 0 || removed > 0)) {
+        result.push(`  +${added} -${removed}`);
+      }
+      const parts = line.split(" b/");
+      currentFile = parts.length > 1 ? parts.slice(1).join(" b/") : "unknown";
+      result.push(`\n${currentFile}`);
+      added = 0;
+      removed = 0;
+      inHunk = false;
+      hunkShown = 0;
+    } else if (line.startsWith("@@")) {
+      if (hunkSkipped > 0) {
+        result.push(`  ... (${hunkSkipped} lines truncated)`);
+        wasTruncated = true;
+        hunkSkipped = 0;
+      }
+      inHunk = true;
+      hunkShown = 0;
+      result.push(`  ${line}`);
+    } else if (inHunk) {
+      if (line.startsWith("+") && !line.startsWith("+++")) {
+        added += 1;
+        if (hunkShown < maxHunkLines) {
+          result.push(`  ${line}`);
+          hunkShown += 1;
+        } else {
+          hunkSkipped += 1;
+        }
+      } else if (line.startsWith("-") && !line.startsWith("---")) {
+        removed += 1;
+        if (hunkShown < maxHunkLines) {
+          result.push(`  ${line}`);
+          hunkShown += 1;
+        } else {
+          hunkSkipped += 1;
+        }
+      } else if (hunkShown < maxHunkLines && !line.startsWith("\\")) {
+        if (hunkShown > 0) {
+          result.push(`  ${line}`);
+          hunkShown += 1;
+        }
+      }
+    }
+
+    if (result.length >= maxLines) {
+      result.push("\n... (more changes truncated)");
+      wasTruncated = true;
+      break outer;
+    }
+  }
+
+  if (hunkSkipped > 0) {
+    result.push(`  ... (${hunkSkipped} lines truncated)`);
+    wasTruncated = true;
+  }
+
+  if (currentFile && (added > 0 || removed > 0)) {
+    result.push(`  +${added} -${removed}`);
+  }
+
+  if (wasTruncated) {
+    result.push("[full diff: rtk git diff --no-compact]");
+  }
+
+  return result.join("\n");
+}
+
+gitDiff.filterName = "git-diff";

open-sse/rtk/filters/gitStatus.js

@@ -0,0 +1,117 @@
+// Port of git::format_status_output (rtk/src/cmds/git/git.rs:619-730)
+// Output format:
+//   * <branch>
+//   + Staged: N files
+//      path1
+//      ... +K more
+//   ~ Modified: N files
+//   ? Untracked: N files
+//   conflicts: N files
+//   clean — nothing to commit
+import { STATUS_MAX_FILES, STATUS_MAX_UNTRACKED } from "../constants.js";
+
+export function gitStatus(input) {
+  const lines = input.split("\n");
+  if (lines.length === 0 || (lines.length === 1 && !lines[0].trim())) {
+    return "Clean working tree";
+  }
+
+  let branch = "";
+  const stagedFiles = [];
+  const modifiedFiles = [];
+  const untrackedFiles = [];
+  let staged = 0;
+  let modified = 0;
+  let untracked = 0;
+  let conflicts = 0;
+
+  for (const raw of lines) {
+    if (!raw.trim()) continue;
+
+    // Long-form branch detection (LLM usually sends this, not porcelain)
+    const longBranch = raw.match(/^On branch (\S+)/);
+    if (longBranch) { branch = longBranch[1]; continue; }
+
+    // Porcelain branch header: "## main...origin/main"
+    if (raw.startsWith("##")) { branch = raw.replace(/^##\s*/, ""); continue; }
+
+    // Porcelain status (2 chars + space + path)
+    if (raw.length >= 3 && /^[ MADRCU?!][ MADRCU?!] /.test(raw)) {
+      const x = raw[0];
+      const y = raw[1];
+      const file = raw.slice(3);
+
+      if (raw.slice(0, 2) === "??") {
+        untracked++;
+        untrackedFiles.push(file);
+        continue;
+      }
+
+      if ("MADRC".includes(x)) {
+        staged++;
+        stagedFiles.push(file);
+      } else if (x === "U") {
+        conflicts++;
+      }
+
+      if (y === "M" || y === "D") {
+        modified++;
+        modifiedFiles.push(file);
+      }
+      continue;
+    }
+
+    // Long form fallback ("modified:   path", "new file:   path", ...)
+    const longMatch = raw.match(/^\s*(modified|new file|deleted|renamed|both modified):\s+(.+)$/);
+    if (longMatch) {
+      const kind = longMatch[1];
+      const path = longMatch[2].trim();
+      if (kind === "both modified") { conflicts++; }
+      else if (kind === "modified" || kind === "deleted") { modified++; modifiedFiles.push(path); }
+      else if (kind === "new file" || kind === "renamed") { staged++; stagedFiles.push(path); }
+      continue;
+    }
+
+    // "Untracked files:" section — gather bare paths after this marker
+    // Handled implicitly: plain paths without markers are skipped (safer).
+  }
+
+  let out = "";
+  if (branch) out += `* ${branch}\n`;
+
+  if (staged > 0) {
+    out += `+ Staged: ${staged} files\n`;
+    for (const f of stagedFiles.slice(0, STATUS_MAX_FILES)) out += `   ${f}\n`;
+    if (stagedFiles.length > STATUS_MAX_FILES) {
+      out += `   ... +${stagedFiles.length - STATUS_MAX_FILES} more\n`;
+    }
+  }
+
+  if (modified > 0) {
+    out += `~ Modified: ${modified} files\n`;
+    for (const f of modifiedFiles.slice(0, STATUS_MAX_FILES)) out += `   ${f}\n`;
+    if (modifiedFiles.length > STATUS_MAX_FILES) {
+      out += `   ... +${modifiedFiles.length - STATUS_MAX_FILES} more\n`;
+    }
+  }
+
+  if (untracked > 0) {
+    out += `? Untracked: ${untracked} files\n`;
+    for (const f of untrackedFiles.slice(0, STATUS_MAX_UNTRACKED)) out += `   ${f}\n`;
+    if (untrackedFiles.length > STATUS_MAX_UNTRACKED) {
+      out += `   ... +${untrackedFiles.length - STATUS_MAX_UNTRACKED} more\n`;
+    }
+  }
+
+  if (conflicts > 0) {
+    out += `conflicts: ${conflicts} files\n`;
+  }
+
+  if (staged === 0 && modified === 0 && untracked === 0 && conflicts === 0) {
+    out += "clean — nothing to commit\n";
+  }
+
+  return out.replace(/\n+$/, "");
+}
+
+gitStatus.filterName = "git-status";

open-sse/rtk/filters/grep.js

@@ -0,0 +1,48 @@
+// Port of grep_wrapper (rtk/src/cmds/system/pipe_cmd.rs:50-86)
+// Input format: "file:lineno:content" — splitn(3, ':') in Rust
+import { GREP_PER_FILE_MAX } from "../constants.js";
+
+export function grep(input) {
+  const byFile = new Map();
+  let total = 0;
+
+  for (const line of input.split("\n")) {
+    // splitn(3, ':') — only split on first 2 colons
+    const first = line.indexOf(":");
+    if (first === -1) continue;
+    const second = line.indexOf(":", first + 1);
+    if (second === -1) continue;
+    const file = line.slice(0, first);
+    const lineNumStr = line.slice(first + 1, second);
+    const content = line.slice(second + 1);
+    // Rust: parts[1].parse::<usize>().is_ok()
+    if (!/^\d+$/.test(lineNumStr)) continue;
+    total++;
+    if (!byFile.has(file)) byFile.set(file, []);
+    byFile.get(file).push([lineNumStr, content]);
+  }
+
+  if (total === 0) return input;
+
+  // Rust: files.sort_by_key(|(f, _)| *f)
+  const files = Array.from(byFile.keys()).sort();
+  let out = `${total} matches in ${files.length}F:\n\n`;
+
+  for (const file of files) {
+    const matches = byFile.get(file);
+    out += `[file] ${file} (${matches.length}):\n`;
+    const show = matches.slice(0, GREP_PER_FILE_MAX);
+    for (const [lineNum, content] of show) {
+      // Rust: format!("  {:>4}: {}", line_num, content.trim())
+      out += `  ${lineNum.padStart(4)}: ${content.trim()}\n`;
+    }
+    if (matches.length > GREP_PER_FILE_MAX) {
+      out += `  +${matches.length - GREP_PER_FILE_MAX}\n`;
+    }
+    out += "\n";
+  }
+
+  return out;
+}
+
+grep.filterName = "grep";

open-sse/rtk/filters/ls.js

@@ -0,0 +1,79 @@
+// Port of compact_ls (rtk/src/cmds/system/ls.rs:154-232)
+// Input: `ls -la` style output. Output: compact "name/  (dirs)\nname  size"
+import { LS_EXT_SUMMARY_TOP, LS_NOISE_DIRS } from "../constants.js";
+
+// Rust LS_DATE_RE: month + day + (year|HH:MM)
+const LS_DATE_RE = /\s+(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s+\d{1,2}\s+(\d{4}|\d{2}:\d{2})\s+/;
+
+function humanSize(bytes) {
+  if (bytes >= 1_048_576) return `${(bytes / 1_048_576).toFixed(1)}M`;
+  if (bytes >= 1024) return `${(bytes / 1024).toFixed(1)}K`;
+  return `${bytes}B`;
+}
+
+function parseLsLine(line) {
+  const m = LS_DATE_RE.exec(line);
+  if (!m) return null;
+  const name = line.slice(m.index + m[0].length);
+  const beforeDate = line.slice(0, m.index);
+  const beforeParts = beforeDate.split(/\s+/).filter(Boolean);
+  if (beforeParts.length < 4) return null;
+
+  const perms = beforeParts[0];
+  const fileType = perms.charAt(0);
+
+  // size = rightmost parseable number before the date
+  let size = 0;
+  for (let i = beforeParts.length - 1; i >= 0; i--) {
+    const n = Number(beforeParts[i]);
+    if (Number.isInteger(n) && String(n) === beforeParts[i]) { size = n; break; }
+  }
+  return { fileType, size, name };
+}
+
+export function ls(input) {
+  const dirs = [];
+  const files = [];      // [name, sizeStr]
+  const byExt = new Map();
+
+  for (const line of input.split("\n")) {
+    if (line.startsWith("total ") || line.length === 0) continue;
+    const parsed = parseLsLine(line);
+    if (!parsed) continue;
+    if (parsed.name === "." || parsed.name === "..") continue;
+
+    // Rust ls.rs: show_all flag respected — for LLM context always skip noise
+    if (LS_NOISE_DIRS.includes(parsed.name)) continue;
+
+    if (parsed.fileType === "d") {
+      dirs.push(parsed.name);
+    } else if (parsed.fileType === "-" || parsed.fileType === "l") {
+      const dot = parsed.name.lastIndexOf(".");
+      const ext = dot > 0 ? parsed.name.slice(dot) : "no ext";
+      byExt.set(ext, (byExt.get(ext) || 0) + 1);
+      files.push([parsed.name, humanSize(parsed.size)]);
+    }
+  }
+
+  if (dirs.length === 0 && files.length === 0) return input;
+
+  let out = "";
+  for (const d of dirs) out += `${d}/\n`;
+  for (const [name, size] of files) out += `${name}  ${size}\n`;
+
+  // Summary line (Rust port)
+  let summary = `\nSummary: ${files.length} files, ${dirs.length} dirs`;
+  if (byExt.size > 0) {
+    const ext = Array.from(byExt.entries()).sort((a, b) => b[1] - a[1]);
+    const parts = ext.slice(0, LS_EXT_SUMMARY_TOP).map(([e, c]) => `${c} ${e}`);
+    summary += ` (${parts.join(", ")}`;
+    if (ext.length > LS_EXT_SUMMARY_TOP) {
+      summary += `, +${ext.length - LS_EXT_SUMMARY_TOP} more`;
+    }
+    summary += ")";
+  }
+
+  return out + summary;
+}
+
+ls.filterName = "ls";

open-sse/rtk/filters/readNumbered.js

@@ -0,0 +1,27 @@
+// Handles Cursor/Codex read_file output: "  1|content\n  2|content".
+// Strategy mirrors Rust filter::smart_truncate (filter.rs): keep head+tail, drop middle.
+import { SMART_TRUNCATE_HEAD, SMART_TRUNCATE_TAIL, SMART_TRUNCATE_MIN_LINES } from "../constants.js";
+
+const LINE_RE = /^\s*\d+\|/;
+
+export function readNumbered(input) {
+  const lines = input.split("\n");
+  if (lines.length < SMART_TRUNCATE_MIN_LINES) return input;
+
+  // Count how many lines match "N|content" to verify shape (hit ratio check
+  // already done by autodetect; here we just truncate).
+  const head = lines.slice(0, SMART_TRUNCATE_HEAD);
+  const tail = lines.slice(lines.length - SMART_TRUNCATE_TAIL);
+  const cut = lines.length - head.length - tail.length;
+
+  return [
+    ...head,
+    `... +${cut} lines truncated (file continues)`,
+    ...tail
+  ].join("\n");
+}
+
+readNumbered.filterName = "read-numbered";
+
+// Exposed for autodetect
+export const READ_NUMBERED_LINE_RE = LINE_RE;

open-sse/rtk/filters/searchList.js

@@ -0,0 +1,52 @@
+// Compact "Result of search in '...' (total N files):\n- path\n- path" output
+// (Cursor Glob tool). Groups by parent dir like find, shows basenames.
+import { SEARCH_LIST_PER_DIR_MAX, SEARCH_LIST_TOTAL_DIR_MAX } from "../constants.js";
+
+const HEADER_RE = /^Result of search in '[^']*' \(total (\d+) files?\):/;
+
+export function searchList(input) {
+  const lines = input.split("\n");
+  if (lines.length === 0) return input;
+
+  // First line must be the header (validated by autodetect too)
+  const header = lines[0] || "";
+  const rest = lines.slice(1);
+
+  const paths = [];
+  for (const raw of rest) {
+    const t = raw.trim();
+    if (!t.startsWith("- ")) continue;
+    paths.push(t.slice(2));
+  }
+  if (paths.length === 0) return input;
+
+  const byDir = new Map();
+  for (const p of paths) {
+    const slash = p.lastIndexOf("/");
+    const dir = slash === -1 ? "." : (p.slice(0, slash) || "/");
+    const name = slash === -1 ? p : p.slice(slash + 1);
+    if (!byDir.has(dir)) byDir.set(dir, []);
+    byDir.get(dir).push(name);
+  }
+
+  const dirs = Array.from(byDir.keys()).sort();
+  let out = `${header}\n${paths.length} files in ${dirs.length} dirs:\n\n`;
+
+  for (const dir of dirs.slice(0, SEARCH_LIST_TOTAL_DIR_MAX)) {
+    const names = byDir.get(dir);
+    out += `${dir}/ (${names.length}):\n`;
+    for (const n of names.slice(0, SEARCH_LIST_PER_DIR_MAX)) out += `  ${n}\n`;
+    if (names.length > SEARCH_LIST_PER_DIR_MAX) {
+      out += `  +${names.length - SEARCH_LIST_PER_DIR_MAX}\n`;
+    }
+    out += "\n";
+  }
+  if (dirs.length > SEARCH_LIST_TOTAL_DIR_MAX) {
+    out += `+${dirs.length - SEARCH_LIST_TOTAL_DIR_MAX} more dirs\n`;
+  }
+
+  return out.replace(/\n+$/, "");
+}
+
+searchList.filterName = "search-list";
+export const SEARCH_LIST_HEADER_RE = HEADER_RE;

open-sse/rtk/filters/smartTruncate.js

@@ -0,0 +1,15 @@
+// Port concept of filter::smart_truncate (rtk/src/core/filter.rs).
+// Keep HEAD + TAIL lines, replace middle with "... +N lines truncated".
+import { SMART_TRUNCATE_HEAD, SMART_TRUNCATE_TAIL, SMART_TRUNCATE_MIN_LINES } from "../constants.js";
+
+export function smartTruncate(input) {
+  const lines = input.split("\n");
+  if (lines.length < SMART_TRUNCATE_MIN_LINES) return input;
+
+  const head = lines.slice(0, SMART_TRUNCATE_HEAD);
+  const tail = lines.slice(lines.length - SMART_TRUNCATE_TAIL);
+  const cut = lines.length - head.length - tail.length;
+  return [...head, `... +${cut} lines truncated`, ...tail].join("\n");
+}
+
+smartTruncate.filterName = "smart-truncate";

open-sse/rtk/filters/tree.js

@@ -0,0 +1,32 @@
+// Port of filter_tree_output (rtk/src/cmds/system/tree.rs:65-94)
+// Removes summary line (e.g. "5 directories, 23 files") and trailing blanks.
+import { TREE_MAX_LINES } from "../constants.js";
+
+export function tree(input) {
+  const lines = input.split("\n");
+  if (lines.length === 0) return input;
+
+  const filtered = [];
+  for (const line of lines) {
+    // Drop "X directories, Y files" summary
+    if (line.includes("director") && line.includes("file")) continue;
+    // Drop leading blanks
+    if (line.trim() === "" && filtered.length === 0) continue;
+    filtered.push(line);
+  }
+
+  // Drop trailing blanks
+  while (filtered.length > 0 && filtered[filtered.length - 1].trim() === "") {
+    filtered.pop();
+  }
+
+  // Cap overly long trees (JS-only safeguard; Rust has no cap)
+  if (filtered.length > TREE_MAX_LINES) {
+    const cut = filtered.length - TREE_MAX_LINES;
+    return filtered.slice(0, TREE_MAX_LINES).join("\n") + `\n... +${cut} more lines`;
+  }
+
+  return filtered.join("\n");
+}
+
+tree.filterName = "tree";