#!/usr/bin/env python3 # See docstring in previous attempt; same full script body repeated. import argparse import json from pathlib import Path import sys import pandas as pd ALLOWED_CLUSTERS = { "1581": "MATTHEW GUERTIN", # focal "680": "MUAD ABDULKADIR", "292": "ADRIAN WESLEY", "698": "PETER LEHMEYER", } FOCAL_CLUSTER_ID = "1581" OTHER_CLUSTER_IDS = {"680", "292", "698"} def as_list(x): if x is None: return [] if isinstance(x, list): return x return [x] def list_join(vals): vals = [v for v in vals if pd.notna(v)] return "; ".join(sorted(set(map(str, vals)))) def ensure_string_cols(df, cols): for c in cols: if c in df.columns: df[c] = df[c].astype("string") return df def main(): ap = argparse.ArgumentParser() ap.add_argument("--input", required=True, help="Directory with JSON files") ap.add_argument("--outdir", required=True, help="Output folder") ap.add_argument("--pattern", default="*.json", help="Glob for JSON files") ap.add_argument("--recurse", action="store_true", help="Recurse into subdirs") args = ap.parse_args() in_dir = Path(args.input) files = sorted(in_dir.rglob(args.pattern) if args.recurse else in_dir.glob(args.pattern)) if not files: print(f"[warn] No JSON files matched {args.pattern} in {in_dir}") sys.exit(0) outdir = Path(args.outdir) outdir.mkdir(parents=True, exist_ok=True) # ---- Extract minimal doc, case, language tables from ALL JSONS (for exclusivity check) ---- docs_rows, terms_rows, inst_rows = [], [], [] for fp in files: try: data = json.loads(fp.read_text(encoding="utf-8")) except Exception: continue doc_sha256 = data.get("doc_sha256") or data.get("sha256") or "" filename = data.get("filename", "") filing_type = data.get("filing_type", "") filing_date = data.get("filing_date", "") case = data.get("case") or {} case_id = case.get("case_id") cluster_id = case.get("cluster_id") cluster_name = case.get("cluster_name") docs_rows.append({ "doc_sha256": doc_sha256, "filename": filename, "filing_type": filing_type, "filing_date": filing_date, "case_id": case_id, "cluster_id": str(cluster_id) if cluster_id is not None else None, "cluster_name": cluster_name, }) lang = data.get("language") or {} for t in as_list(lang.get("terms")): terms_rows.append({ "doc_sha256": doc_sha256, "search_group": t.get("search_group"), "search_term": t.get("search_term"), "quantity": t.get("quantity"), }) for inst in as_list(lang.get("instances")): inst_rows.append({ "doc_sha256": doc_sha256, "search_group": inst.get("search_group"), "search_term": inst.get("search_term"), "page_num": inst.get("page_num"), "num_pages": inst.get("num_pages"), "full_text_row": inst.get("full_text_row"), "search_term_url": inst.get("search_term_url"), }) docs = pd.DataFrame(docs_rows) terms = pd.DataFrame(terms_rows) instances = pd.DataFrame(inst_rows) # Standardize types terms = ensure_string_cols(terms, ["search_group","search_term"]) instances = ensure_string_cols(instances, ["search_group","search_term"]) if "quantity" in terms.columns: terms["quantity"] = pd.to_numeric(terms["quantity"], errors="coerce").fillna(0).astype(int) # Join doc context t = terms.merge(docs, on="doc_sha256", how="left") t = ensure_string_cols(t, ["search_group","search_term"]) t["cluster_id"] = t["cluster_id"].astype("string") # ---- Determine term eligibility across the FULL corpus ---- presence = ( t.dropna(subset=["cluster_id"]) .groupby(["search_group","search_term"], as_index=False)["cluster_id"] .agg(lambda s: sorted(set(map(str, s.dropna().tolist())))) ) allowed_set = set(ALLOWED_CLUSTERS.keys()) def check_elig(clusters): clusters_set = set(clusters) return (FOCAL_CLUSTER_ID in clusters_set) and clusters_set.issubset(allowed_set) and (len(clusters_set - {FOCAL_CLUSTER_ID}) >= 1) presence["eligible"] = presence["cluster_id"].apply(check_elig) term_elig = presence[presence["eligible"]].copy() term_elig = term_elig.rename(columns={"cluster_id": "clusters_present"}) term_elig["others_clusters"] = term_elig["clusters_present"].apply(lambda lst: [c for c in lst if c != FOCAL_CLUSTER_ID]) term_elig["others_cluster_names"] = term_elig["others_clusters"].apply(lambda lst: "; ".join(ALLOWED_CLUSTERS.get(c, c) for c in lst)) term_elig.to_csv(outdir/"term_eligibility.csv", index=False) if term_elig.empty: (outdir/"README.txt").write_text("No terms matched the 4‑cluster criteria.\n", encoding="utf-8") print("[ok] No eligible terms; wrote README.") sys.exit(0) eligible_keys = term_elig[["search_group","search_term"]].drop_duplicates() t_allowed = t[t["cluster_id"].isin(allowed_set)].merge(eligible_keys, on=["search_group","search_term"], how="inner") # Split focal vs others t_focal = t_allowed[t_allowed["cluster_id"] == FOCAL_CLUSTER_ID].copy() t_others = t_allowed[t_allowed["cluster_id"] != FOCAL_CLUSTER_ID].copy() # FOCAL summary focal_term_summary = ( t_focal.groupby(["search_group","search_term"], as_index=False) .agg(in_focal_quantity=("quantity","sum"), in_focal_doc_count=("doc_sha256","nunique"), in_focal_case_count=("case_id","nunique"), focal_case_ids=("case_id", list_join), focal_filenames=("filename", list_join)) ) focal_term_summary.to_csv(outdir/"focal_term_summary.csv", index=False) # OTHERS rollups others_by_cluster = ( t_others.groupby(["search_group","search_term","cluster_id","cluster_name"], as_index=False) .agg(other_cluster_quantity=("quantity","sum"), other_cluster_doc_count=("doc_sha256","nunique"), other_cluster_case_count=("case_id","nunique"), other_cluster_case_ids=("case_id", list_join), other_filenames=("filename", list_join)) ) others_by_cluster.to_csv(outdir/"others_by_cluster.csv", index=False) others_by_case = ( t_others.groupby(["search_group","search_term","case_id","cluster_id","cluster_name"], as_index=False) .agg(other_case_quantity=("quantity","sum"), other_case_doc_count=("doc_sha256","nunique"), other_filenames=("filename", list_join), other_filing_types=("filing_type", lambda s: "; ".join(sorted(set([v for v in s if pd.notna(v)])))), first_seen_date=("filing_date", lambda s: s.dropna().min() if len(s.dropna()) else pd.NA), last_seen_date=("filing_date", lambda s: s.dropna().max() if len(s.dropna()) else pd.NA)) ) others_by_case.to_csv(outdir/"others_by_case.csv", index=False) others_by_doc = ( t_others.groupby(["search_group","search_term","doc_sha256","filename","filing_type","filing_date","case_id","cluster_id","cluster_name"], as_index=False) .agg(other_doc_quantity=("quantity","sum")) ) others_by_doc.to_csv(outdir/"others_by_doc.csv", index=False) # Compare summary others_cluster_counts = ( t_others.groupby(["search_group","search_term"], as_index=False) .agg(others_cluster_count=("cluster_id","nunique"), others_clusters=("cluster_id", lambda s: sorted(set(s))), others_cluster_names=("cluster_name", lambda s: "; ".join(sorted(set([v for v in s if pd.notna(v)]))))) ) others_case_counts = ( t_others.groupby(["search_group","search_term"], as_index=False) .agg(others_case_count=("case_id","nunique")) ) term_compare = (focal_term_summary .merge(others_cluster_counts, on=["search_group","search_term"], how="left") .merge(others_case_counts, on=["search_group","search_term"], how="left")) term_compare["others_cluster_count"] = term_compare["others_cluster_count"].fillna(0).astype(int) term_compare["others_case_count"] = term_compare["others_case_count"].fillna(0).astype(int) term_compare["weight_clusters"] = term_compare["others_cluster_count"].apply(lambda k: (1.0/k) if k>=1 else 0.0) term_compare = term_compare.sort_values(["weight_clusters","in_focal_quantity"], ascending=[False, False]) term_compare.to_csv(outdir/"term_compare_summary.csv", index=False) # Matrices by_case_qty = ( t_allowed.groupby(["search_group","search_term","case_id"], as_index=False) .agg(total_quantity=("quantity","sum")) ) matrix_cases_qty = by_case_qty.pivot_table( index=["search_group","search_term"], columns="case_id", values="total_quantity", fill_value=0, aggfunc="sum" ).reset_index() matrix_cases_presence = matrix_cases_qty.copy() matrix_cases_presence.iloc[:, 2:] = (matrix_cases_presence.iloc[:, 2:] > 0).astype(int) matrix_cases_qty.to_csv(outdir/"matrix_cases_quantity.csv", index=False) matrix_cases_presence.to_csv(outdir/"matrix_cases_presence.csv", index=False) by_cluster_qty = ( t_allowed.groupby(["search_group","search_term","cluster_id"], as_index=False) .agg(total_quantity=("quantity","sum")) ) matrix_clusters_qty = by_cluster_qty.pivot_table( index=["search_group","search_term"], columns="cluster_id", values="total_quantity", fill_value=0, aggfunc="sum" ).reset_index() matrix_clusters_presence = matrix_clusters_qty.copy() matrix_clusters_presence.iloc[:, 2:] = (matrix_clusters_presence.iloc[:, 2:] > 0).astype(int) matrix_clusters_qty.to_csv(outdir/"matrix_clusters_quantity.csv", index=False) matrix_clusters_presence.to_csv(outdir/"matrix_clusters_presence.csv", index=False) # Weight matrix for clusters (apply only to other clusters; focal column = 0) weights_series = term_compare.set_index(["search_group","search_term"])["weight_clusters"] core = matrix_clusters_presence.set_index(["search_group","search_term"]).astype(float) weighted = core.copy() for col in core.columns: if col == FOCAL_CLUSTER_ID: weighted[col] = 0.0 else: weighted[col] = core[col].where(core[col]==0, weights_series.reindex(core.index).fillna(0).astype(float)) matrix_clusters_weight = weighted.reset_index() matrix_clusters_weight.to_csv(outdir/"matrix_clusters_weight.csv", index=False) # Instances if not instances.empty: inst_all = instances.merge(docs[["doc_sha256","filename","filing_type","filing_date","case_id","cluster_id","cluster_name"]], on="doc_sha256", how="left") inst_all = ensure_string_cols(inst_all, ["search_group","search_term"]) inst_all["cluster_id"] = inst_all["cluster_id"].astype("string") inst_all = inst_all.merge(eligible_keys, on=["search_group","search_term"], how="inner") inst_all = inst_all[inst_all["cluster_id"].isin(ALLOWED_CLUSTERS.keys())].copy() instances_all = inst_all[[ "filename","filing_type","filing_date","case_id","cluster_id","cluster_name", "search_group","search_term","page_num","num_pages","full_text_row","search_term_url" ]].sort_values(["cluster_id","case_id","filename","search_group","search_term","page_num"]) instances_focal = instances_all[instances_all["cluster_id"] == FOCAL_CLUSTER_ID].copy() instances_others = instances_all[instances_all["cluster_id"] != FOCAL_CLUSTER_ID].copy() instances_all.to_csv(outdir/"instances_all.csv", index=False) instances_focal.to_csv(outdir/"instances_focal.csv", index=False) instances_others.to_csv(outdir/"instances_others.csv", index=False) else: for name in ["instances_all.csv","instances_focal.csv","instances_others.csv"]: (outdir/name).write_text("", encoding="utf-8") metrics = pd.DataFrame([{ "eligible_terms": int(term_elig.shape[0]), "focal_cluster_terms": int(focal_term_summary.shape[0]), "other_clusters_terms": int(others_by_cluster[["search_group","search_term"]].drop_duplicates().shape[0]), "cases_included": int(t_allowed["case_id"].nunique()), "docs_included": int(t_allowed["doc_sha256"].nunique()), }]) metrics.to_csv(outdir/"overview_metrics.csv", index=False) readme = f""" MCRO Language: Hard‑wired 4‑cluster comparative report ====================================================== Clusters considered: 1581 — {ALLOWED_CLUSTERS['1581']} (FOCAL) 680 — {ALLOWED_CLUSTERS['680']} 292 — {ALLOWED_CLUSTERS['292']} 698 — {ALLOWED_CLUSTERS['698']} Inclusion rule for terms: • The term MUST appear in the focal cluster (1581), AND • It MUST appear in >=1 of the other three clusters, AND • It MUST NOT appear in any cluster outside these four. Weight: • For a given term, let K be the number of other clusters among {{680,292,698}} where it appears (K>=1). • weight = 1 / K (highest = 1.0 when the term appears in exactly one other cluster). See CSV files in this folder for per-term focal totals, cross-cluster links by case/doc, presence/quantity matrices, weight matrix, and page-level instances. """.strip() + "\n" (outdir/"README.txt").write_text(readme, encoding="utf-8") print(f"[ok] Wrote 4‑cluster language reports to: {outdir.resolve()}") if __name__ == "__main__": main()