#!/usr/bin/env python3 """ MCRO Language Analytics Report (v2) ---------------------------------- Focus: for a chosen focal case, ONLY analyze terms that appear in the focal case AND in >= 1 other case/cluster. Terms unique to the focal case are excluded. Scoring: - score_case_inv = 1 / others_case_count (max = 1.0 when 1 other case) - score_case_pct = 100 / others_case_count (max = 100 when 1 other case) - score_cluster_inv = 1 / others_cluster_count (analogous for clusters) - score_cluster_pct = 100 / others_cluster_count Outputs: - focal_term_compare_summary.csv - focal_term_links_by_case.csv - focal_term_links_by_cluster.csv - focal_term_heatmap_long.csv (term-level, case-based score) - focal_term_heatmap_clusters_long.csv (term-level, cluster-based score) - terms_by_case.csv (context) - terms_by_cluster.csv (context) - overview_metrics.csv - README.txt Usage: python mcro_language_report_v2.py --input /path/to/json_dir --outdir /path/to/report_language_v2 [--recurse] [--pattern *.json] [--focus_case 27-CR-23-1886] """ import argparse import json from pathlib import Path import sys import pandas as pd def as_list(x): if x is None: return [] if isinstance(x, list): return x return [x] def list_join(vals): vals = [v for v in vals if pd.notna(v)] return "; ".join(sorted(set(map(str, vals)))) def main(): ap = argparse.ArgumentParser() ap.add_argument("--input", required=True, help="Directory with JSON files") ap.add_argument("--outdir", default="report_language_v2", help="Output folder") ap.add_argument("--pattern", default="*.json", help="Glob for JSON files") ap.add_argument("--recurse", action="store_true", help="Recurse into subdirs") ap.add_argument("--focus_case", default="27-CR-23-1886", help="Focal case for comparative analysis") args = ap.parse_args() in_dir = Path(args.input) files = sorted(in_dir.rglob(args.pattern) if args.recurse else in_dir.glob(args.pattern)) if not files: print(f"[warn] No JSON files matched {args.pattern} in {in_dir}") sys.exit(0) outdir = Path(args.outdir) outdir.mkdir(parents=True, exist_ok=True) # ---- Extract minimal doc, case, language tables ---- docs_rows = [] terms_rows = [] inst_rows = [] for fp in files: try: data = json.loads(fp.read_text(encoding="utf-8")) except Exception: continue doc_sha256 = data.get("doc_sha256") or data.get("sha256") or "" filename = data.get("filename", "") filing_type = data.get("filing_type", "") filing_date = data.get("filing_date", "") case = data.get("case") or {} case_id = case.get("case_id") cluster_id = case.get("cluster_id") cluster_name = case.get("cluster_name") docs_rows.append({ "doc_sha256": doc_sha256, "filename": filename, "filing_type": filing_type, "filing_date": filing_date, "case_id": case_id, "cluster_id": cluster_id, "cluster_name": cluster_name, }) lang = data.get("language") or {} for t in as_list(lang.get("terms")): terms_rows.append({ "doc_sha256": doc_sha256, "search_group": t.get("search_group"), "search_term": t.get("search_term"), "quantity": t.get("quantity"), }) for inst in as_list(lang.get("instances")): inst_rows.append({ "doc_sha256": doc_sha256, "search_group": inst.get("search_group"), "search_term": inst.get("search_term"), "page_num": inst.get("page_num"), "num_pages": inst.get("num_pages"), "full_text_row": inst.get("full_text_row"), "search_term_url": inst.get("search_term_url"), }) docs = pd.DataFrame(docs_rows) terms = pd.DataFrame(terms_rows) if terms.empty: (outdir/"README.txt").write_text("No language terms found.\n", encoding="utf-8") sys.exit(0) for col in ["search_group","search_term"]: terms[col] = terms[col].astype("string") terms["quantity"] = pd.to_numeric(terms["quantity"], errors="coerce").fillna(0).astype(int) t = terms.merge(docs, on="doc_sha256", how="left") # Context tables (helpful for joins/inspection) terms_by_case = ( t.groupby(["case_id","cluster_id","cluster_name","search_group","search_term"], dropna=False, as_index=False) .agg(total_quantity=("quantity","sum"), doc_count=("doc_sha256","nunique"), filenames=("filename", list_join)) ) terms_by_cluster = ( t.groupby(["cluster_id","cluster_name","search_group","search_term"], dropna=False, as_index=False) .agg(total_quantity=("quantity","sum"), doc_count=("doc_sha256","nunique"), case_count=("case_id","nunique"), filenames=("filename", list_join)) ) # ----- FOCAL: terms present in focus case ----- focus = args.focus_case t_focus = t[t["case_id"] == focus].copy() focal_terms = ( t_focus.groupby(["case_id","cluster_id","cluster_name","search_group","search_term"], as_index=False) .agg(in_case_quantity=("quantity","sum"), in_case_doc_count=("doc_sha256","nunique"), focal_filenames=("filename", list_join)) ) # For the focal terms, compute how many OTHER cases/clusters contain each term t_others = t[t["case_id"] != focus].copy() others_case_counts = ( t_others.groupby(["search_group","search_term"], as_index=False) .agg(others_case_count=("case_id","nunique")) ) others_cluster_counts = ( t_others.groupby(["search_group","search_term"], as_index=False) .agg(others_cluster_count=("cluster_id","nunique")) ) # Merge onto focal terms focal_cmp = (focal_terms .merge(others_case_counts, on=["search_group","search_term"], how="left") .merge(others_cluster_counts, on=["search_group","search_term"], how="left")) focal_cmp["others_case_count"] = focal_cmp["others_case_count"].fillna(0).astype(int) focal_cmp["others_cluster_count"] = focal_cmp["others_cluster_count"].fillna(0).astype(int) # Keep ONLY terms with at least one other case focal_cmp = focal_cmp[focal_cmp["others_case_count"] >= 1].copy() # Scoring (highest when exactly one other case/cluster) focal_cmp["score_case_inv"] = 1.0 / focal_cmp["others_case_count"] focal_cmp["score_case_pct"] = 100.0 / focal_cmp["others_case_count"] # For clusters, only score if there's at least one other cluster focal_cmp["score_cluster_inv"] = focal_cmp.apply( lambda r: (1.0 / r["others_cluster_count"]) if r["others_cluster_count"] >= 1 else 0.0, axis=1) focal_cmp["score_cluster_pct"] = focal_cmp.apply( lambda r: (100.0 / r["others_cluster_count"]) if r["others_cluster_count"] >= 1 else 0.0, axis=1) # Sort: hottest (fewest other cases) first, then by in_case_quantity desc focal_cmp = focal_cmp.sort_values( ["score_case_inv","score_cluster_inv","in_case_quantity"], ascending=[False, False, False] ) # Expand: for each focal term, list other cases/clusters & filenames t_focus_terms = focal_cmp[["search_group","search_term"]].drop_duplicates() links = t_others.merge(t_focus_terms, on=["search_group","search_term"], how="inner") focal_links_by_case = ( links.groupby(["search_group","search_term","case_id","cluster_id","cluster_name"], as_index=False) .agg(other_case_quantity=("quantity","sum"), other_case_doc_count=("doc_sha256","nunique"), other_filenames=("filename", list_join)) .sort_values(["search_group","search_term","other_case_quantity"], ascending=[True, True, False]) ) focal_links_by_cluster = ( links.groupby(["search_group","search_term","cluster_id","cluster_name"], as_index=False) .agg(other_cluster_quantity=("quantity","sum"), other_cluster_doc_count=("doc_sha256","nunique"), other_case_ids=("case_id", list_join)) .sort_values(["search_group","search_term","other_cluster_quantity"], ascending=[True, True, False]) ) # Heatmap-friendly long tables (term -> score) focal_heatmap_long = focal_cmp[[ "search_group","search_term","others_case_count","score_case_inv","score_case_pct", "in_case_quantity","focal_filenames","case_id","cluster_id","cluster_name" ]].copy() focal_heatmap_clusters_long = focal_cmp[[ "search_group","search_term","others_cluster_count","score_cluster_inv","score_cluster_pct", "in_case_quantity","focal_filenames","case_id","cluster_id","cluster_name" ]].copy() # Overview metrics = pd.DataFrame([{ "focus_case": focus, "focus_distinct_terms": int(focal_terms.shape[0]), "focus_terms_with_others": int(focal_cmp.shape[0]), "distinct_other_cases_linked": int(focal_links_by_case["case_id"].nunique()) if not focal_links_by_case.empty else 0, "distinct_other_clusters_linked": int(focal_links_by_cluster["cluster_id"].nunique()) if not focal_links_by_cluster.empty else 0, }]) # Write outputs outdir.mkdir(parents=True, exist_ok=True) terms_by_case.to_csv(outdir/"terms_by_case.csv", index=False) terms_by_cluster.to_csv(outdir/"terms_by_cluster.csv", index=False) focal_cmp.to_csv(outdir/"focal_term_compare_summary.csv", index=False) focal_links_by_case.to_csv(outdir/"focal_term_links_by_case.csv", index=False) focal_links_by_cluster.to_csv(outdir/"focal_term_links_by_cluster.csv", index=False) focal_heatmap_long.to_csv(outdir/"focal_term_heatmap_long.csv", index=False) focal_heatmap_clusters_long.to_csv(outdir/"focal_term_heatmap_clusters_long.csv", index=False) metrics.to_csv(outdir/"overview_metrics.csv", index=False) readme = f""" MCRO Language Analytics (v2 — comparative) ========================================= Focal case: {args.focus_case} This report EXCLUDES terms unique to the focal case. Only terms that also appear in at least one other case are analyzed and scored for uniqueness. Outputs: - focal_term_compare_summary.csv One row per focal-case term, including: * in_case_quantity, in_case_doc_count, focal_filenames * others_case_count, others_cluster_count * score_case_inv (1/others_case_count), score_case_pct (100/others_case_count) * score_cluster_inv, score_cluster_pct - focal_term_links_by_case.csv For each focal-case term, which other cases contain it, with quantities and filenames. - focal_term_links_by_cluster.csv Same as above but grouped by cluster. - focal_term_heatmap_long.csv Heatmap-ready long table: term -> case-based uniqueness scores. - focal_term_heatmap_clusters_long.csv Heatmap-ready long table: term -> cluster-based uniqueness scores. - terms_by_case.csv / terms_by_cluster.csv Context tables for broader term exploration. - overview_metrics.csv Counts of focal terms and linked other cases/clusters. """.strip() + "\n" (outdir/"README.txt").write_text(readme, encoding="utf-8") print(f"[ok] Wrote comparative language reports to: {outdir.resolve()}") if __name__ == "__main__": main()