#!/usr/bin/env python3 """ MCRO Flags Fix (with punctuation normalization) =============================================== Reset and repopulate the "flags" section for all MCRO JSON files according to flags_new.csv, and normalize “smart” punctuation (curly quotes, smart dashes, etc.) into plain ASCII for consistent searching. BEHAVIOR -------- • For EVERY *.json file in --json-dir: 1. Wipe ALL existing flag subkeys so that only a clean "flags" structure is written based on flags_new.csv. 2. Use flags_new.csv (column 'json_filename') to decide which JSONs get which flags. • flags_new.csv MUST contain at least: json_filename, flag_label, data_1, data_2, data_3, object_sha256_1, object_sha256_2, data_url_1, data_url_2 (Additional columns will also be carried through automatically.) • For each row in flags_new.csv: json_filename -> target JSON file name on disk That row becomes ONE flag entry for that JSON, with all columns (except 'json_filename') stored in a dictionary. • Resulting JSON structure per file: "flags": { "labels": [ { "flag_label": "...", "data_1": "...", "data_2": "...", "data_3": "...", "object_sha256_1": "...", "object_sha256_2": "...", "data_url_1": "...", "data_url_2": "...", ... any other CSV columns ... }, ... ] } - If a JSON has no rows in flags_new.csv: "flags": {} (empty object) PUNCTUATION NORMALIZATION ------------------------- Before writing any flag value (flag_label, data_*, etc.), we normalize: • “ ” „ ” → " • ‘ ’ ‛ → ' • « » → " • – — ‒ − → - • … → ... • non-breaking space (U+00A0) → regular space So e.g.: "Document “CreateDate” is dated AFTER document e-file date" becomes: "Document "CreateDate" is dated AFTER document e-file date" USAGE ----- python3 mcro_flags_fix.py \ --json-dir /path/to/json_dir \ --csv flags_new.csv """ import argparse import csv import json from pathlib import Path from typing import Dict, Any, List import sys # ------------------ Normalization helpers ------------------ # def normalize_punctuation(s: str) -> str: """ Normalize 'smart' punctuation to plain ASCII equivalents so searches aren't broken by curly quotes, smart dashes, etc. """ if s is None: return s # Ensure string s = str(s) mapping = { # single quotes / apostrophes "\u2018": "'", # left single quotation mark "\u2019": "'", # right single quotation mark "\u201b": "'", # single high-reversed-9 quotation mark "\u2032": "'", # prime (often used like apostrophe) # double quotes "\u201c": '"', # left double quotation mark "\u201d": '"', # right double quotation mark "\u201f": '"', # double high-reversed-9 quotation mark "\u00ab": '"', # left-pointing double angle quotation mark "\u00bb": '"', # right-pointing double angle quotation mark # dashes / hyphens "\u2010": "-", # hyphen "\u2011": "-", # non-breaking hyphen "\u2012": "-", # figure dash "\u2013": "-", # en dash "\u2014": "-", # em dash "\u2015": "-", # horizontal bar "\u2212": "-", # minus sign # ellipsis "\u2026": "...", # non-breaking space "\u00a0": " ", } for bad, good in mapping.items(): s = s.replace(bad, good) return s # ------------------ CSV loader ------------------ # def load_flags_csv(csv_path: Path): """ Load flags_new.csv into: - mapping: json_filename -> list of row dicts - cols: list of column names (excluding 'json_filename') Each row in the CSV corresponds to ONE flag definition for that JSON file. """ if not csv_path.is_file(): raise FileNotFoundError(f"flags CSV not found: {csv_path}") with csv_path.open("r", newline="", encoding="utf-8") as f: reader = csv.DictReader(f) fieldnames = reader.fieldnames or [] if "json_filename" not in fieldnames: raise ValueError("flags_new.csv must contain a 'json_filename' column") rows = list(reader) mapping: Dict[str, List[Dict[str, Any]]] = {} for row in rows: jf = (row.get("json_filename") or "").strip() if not jf: continue mapping.setdefault(jf, []).append(row) # All CSV columns except 'json_filename' are written into each flag dict flag_cols = [c for c in fieldnames if c != "json_filename"] return mapping, flag_cols # ------------------ Main ------------------ # def main(): parser = argparse.ArgumentParser( description="Reset and repopulate flags.* in MCRO JSON files based on flags_new.csv (with punctuation normalization)." ) parser.add_argument( "--json-dir", default=".", help="Directory containing JSON files (default: current directory)", ) parser.add_argument( "--csv", default="flags_new.csv", help="Path to flags_new.csv (default: ./flags_new.csv)", ) args = parser.parse_args() json_dir = Path(args.json_dir) csv_path = Path(args.csv) if not json_dir.is_dir(): raise NotADirectoryError(f"JSON directory does not exist: {json_dir}") try: mapping, flag_cols = load_flags_csv(csv_path) except Exception as e: print(f"[error] Failed to load flags CSV: {e}", file=sys.stderr) sys.exit(1) print(f"[info] Loaded flag rows for {len(mapping)} distinct json_filenames from {csv_path}") print(f"[info] Flag columns to store in each label: {flag_cols}") json_files = sorted(json_dir.glob("*.json")) print(f"[info] Found {len(json_files)} JSON files in {json_dir}") n_with_flags = 0 n_without_flags = 0 total_labels = 0 flag_label_counts: Dict[str, int] = {} for jp in json_files: try: text = jp.read_text(encoding="utf-8") data = json.loads(text) except Exception as e: print(f"[warn] Skipping {jp.name}: failed to parse JSON ({e})") continue if not isinstance(data, dict): print(f"[warn] Skipping {jp.name}: JSON root is not a dict") continue # Determine JSON filename key (we expect mapping keys to be this file name) json_filename = jp.name # 1. Wipe flags subtree data["flags"] = {} flags_obj = data["flags"] row_list = mapping.get(json_filename, []) if row_list: labels: List[Dict[str, Any]] = [] for row in row_list: label: Dict[str, Any] = {} for col in flag_cols: val = row.get(col) if val is None: continue # normalize punctuation on all stored values val_str = normalize_punctuation(val) if val_str == "": continue label[col] = val_str if label: labels.append(label) total_labels += 1 fl = label.get("flag_label") if fl: flag_label_counts[fl] = flag_label_counts.get(fl, 0) + 1 if labels: flags_obj["labels"] = labels n_with_flags += 1 else: # No non-empty labels for this file n_without_flags += 1 else: # No rows in CSV for this JSON n_without_flags += 1 # Write back JSON (in-place) try: jp.write_text( json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8", ) except Exception as e: print(f"[warn] Failed to write updated JSON for {jp.name}: {e}") continue # ---- Summary ---- print("\n[summary] Flags rewrite complete.") print(f" JSON files processed : {len(json_files)}") print(f" JSON files with flags : {n_with_flags}") print(f" JSON files without flags : {n_without_flags}") print(f" Total flag label entries : {total_labels}") if flag_label_counts: print("\n[summary] flag_label distribution:") for fl in sorted(flag_label_counts.keys()): print(f" {fl!r}: {flag_label_counts[fl]} entry(ies)") else: print("\n[summary] No flag_label values were set for any JSON files.") print("[ok] Finished updating flags.* for all JSON files.") if __name__ == "__main__": main()