`ghost_commit` tool creates large session that kills codex

Potential duplicates detected. Please review them and close your issue if it is a duplicate.

• #7369
• #7313
• #7133
• #7067
• #6530

Powered by Codex Action

import argparse
import json
import sys
from pathlib import Path
from tempfile import NamedTemporaryFile

def clear_preexisting_arrays(node: object) -> bool:
    """Recursively blank every preexisting_untracked_files array in the JSON object."""
    changed = False
    if isinstance(node, dict):
        for key, value in list(node.items()):
            if key == "preexisting_untracked_files":
                if not isinstance(value, list) or value:
                    node[key] = []
                    changed = True
                value = node[key]
            if clear_preexisting_arrays(value):
                changed = True
    elif isinstance(node, list):
        for item in node:
            if clear_preexisting_arrays(item):
                changed = True
    return changed


def iter_objects(stream):
    """Yield (raw_text, parsed_object, parsed_ok) tuples from the stream."""
    buffer: list[str] = []
    depth = 0
    in_string = False
    escape = False
    chunk_start_line = 0

    for line_number, raw_line in enumerate(stream, start=1):
        if not buffer and not raw_line.strip():
            # Preserve whitespace between entries.
            yield raw_line, None, False
            continue

        if not buffer:
            chunk_start_line = line_number

        buffer.append(raw_line)

        for char in raw_line:
            if in_string:
                if escape:
                    escape = False
                    continue
                if char == "\\":
                    escape = True
                    continue
                if char == '"':
                    in_string = False
                continue

            if char == '"':
                in_string = True
            elif char in "{[":
                depth += 1
            elif char in "}]":
                depth -= 1

        if depth == 0 and buffer:
            chunk = "".join(buffer)
            try:
                parsed = json.loads(chunk)
            except json.JSONDecodeError as exc:  # pragma: no cover - defensive
                print(
                    f"Failed to parse JSON starting on line {chunk_start_line}: {exc}",
                    file=sys.stderr,
                )
                yield chunk, None, False
            else:
                yield chunk, parsed, True
            buffer = []

    if buffer:
        chunk = "".join(buffer)
        try:
            parsed = json.loads(chunk)
        except json.JSONDecodeError as exc:  # pragma: no cover - defensive
            print(
                f"Failed to parse JSON starting on line {chunk_start_line}: {exc}",
                file=sys.stderr,
            )
            yield chunk, None, False
        else:
            yield chunk, parsed, True


def filter_stream(input_path: Path, output_path: Path | None, in_place: bool) -> None:
    source = input_path.open("r", encoding="utf-8") if input_path else sys.stdin
    if in_place:
        temp_file = NamedTemporaryFile("w", encoding="utf-8", delete=False)
        destination = temp_file
    elif output_path:
        destination = output_path.open("w", encoding="utf-8")
    else:
        destination = sys.stdout

    try:
        kept = 0
        sanitized = 0
        passthrough = 0
        for raw_chunk, parsed, parsed_ok in iter_objects(source):
            if parsed_ok:
                kept += 1
                if clear_preexisting_arrays(parsed):
                    sanitized += 1
                    stripped = raw_chunk.rstrip("\n")
                    newline_suffix = raw_chunk[len(stripped) :]
                    if not newline_suffix:
                        newline_suffix = "\n"
                    serialized = json.dumps(
                        parsed,
                        ensure_ascii=False,
                        indent=2,
                    ) + newline_suffix
                    destination.write(serialized)
                    continue
            else:
                passthrough += 1
            destination.write(raw_chunk)
        if in_place:
            destination.flush()
            Path(destination.name).replace(input_path)
        summary = f"Processed {kept} JSON entr{'y' if kept == 1 else 'ies'}."
        if sanitized:
            summary += (
                f" Cleared preexisting_untracked_files in {sanitized} ghost snapshot "
                f"entr{'y' if sanitized == 1 else 'ies'}."
            )
        if passthrough:
            summary += (
                f" Forwarded {passthrough} non-JSON line"
                f"{'' if passthrough == 1 else 's'} unchanged."
            )
        print(summary, file=sys.stderr)
    finally:
        if source is not sys.stdin:
            source.close()
        if destination not in (sys.stdout,):
            destination.close()


def main() -> None:
    parser = argparse.ArgumentParser(
        description="Sanitize ghost snapshot entries inside JSONL session logs.",
    )
    parser.add_argument(
        "input",
        nargs="?",
        type=Path,
        help="Path to the source JSONL file (defaults to stdin).",
    )
    parser.add_argument(
        "-o",
        "--output",
        type=Path,
        help="Path for the filtered JSONL (defaults to stdout).",
    )
    parser.add_argument(
        "--in-place",
        action="store_true",
        help="Rewrite the input file after removing ghost snapshots.",
    )
    parser.add_argument(
        "--all-sessions",
        type=Path,
        help="Process every *.jsonl file within the specified directory tree (requires --in-place).",
    )
    args = parser.parse_args()

    if args.all_sessions:
        if args.output:
            parser.error("--output cannot be combined with --all-sessions.")
        if args.input:
            parser.error("Positional input cannot be used with --all-sessions.")
        if not args.in_place:
            parser.error("--all-sessions requires --in-place to rewrite each file.")
        directory = args.all_sessions
        if not directory.is_dir():
            parser.error(f"{directory} is not a directory.")
        files = sorted(p for p in directory.rglob("*.jsonl") if p.is_file())
        if not files:
            print(f"No JSONL files found under {directory}", file=sys.stderr)
            return
        for path in files:
            print(f"Processing {path} ...", file=sys.stderr)
            filter_stream(path, None, True)
        return

    if args.in_place and args.output:
        parser.error("--in-place cannot be combined with --output.")
    if args.in_place and not args.input:
        parser.error("--in-place requires specifying an input file.")

    filter_stream(args.input, args.output, args.in_place)


if __name__ == "__main__":
    main()

This script will simply remove all untracked files from the arrays in ghost_snapshot. Doing this reduced my problem jsonl from 3.9gb all the way down to 150mb.

• python3 fix.py input.jsonl -o cleaned.jsonl – read one JSONL file, write a filtered copy, leaving the source untouched.

• python3 fix.py input.jsonl --in-place – sanitize the file directly (uses a temp file + rename for safety).

• cat input.jsonl | python3 fix.py > cleaned.jsonl – stream from stdin/stdout when you don’t want to pass filenames.

• python3 fix.py --all-sessions /path/to/sessions --in-place – recurse through that directory tree, rewriting every *.jsonl file in place; progress + per-file stats are printed to stderr.

running python3 fix.py --all-sessions ~/.codex --in-place will fix your codex until it happens again. (fix.py name obviously not necessary) running this will clear all untracked file arrays, after its done restart vs code, or re run codex cli, and it will work again. make sure you back up your sessions folder just in case.

https://github.com/openai/codex/pull/7483