{
  "corpus": {
    "harvested": 15288,
    "usable": 13358,
    "excluded": 1930,
    "usable_share": 0.8737571951857667,
    "ocr_pages": 77634,
    "by_language": [
      {
        "lang": "ben",
        "name": "Bengali",
        "harvested": 2000,
        "usable": 1930,
        "median_conf": 73.38808139534883
      },
      {
        "lang": "eng",
        "name": "English",
        "harvested": 2000,
        "usable": 1976,
        "median_conf": 91.86714013803508
      },
      {
        "lang": "guj",
        "name": "Gujarati",
        "harvested": 2000,
        "usable": 1886,
        "median_conf": 71.50042411244695
      },
      {
        "lang": "hin",
        "name": "Hindi",
        "harvested": 2000,
        "usable": 1806,
        "median_conf": 77.06009389671361
      },
      {
        "lang": "kan",
        "name": "Kannada",
        "harvested": 1608,
        "usable": 1460,
        "median_conf": 77.10147601476015
      },
      {
        "lang": "san",
        "name": "Sanskrit",
        "harvested": 2000,
        "usable": 1562,
        "median_conf": 60.99824114557839
      },
      {
        "lang": "tam",
        "name": "Tamil",
        "harvested": 1766,
        "usable": 1416,
        "median_conf": 63.62144214109851
      },
      {
        "lang": "tel",
        "name": "Telugu",
        "harvested": 1914,
        "usable": 1322,
        "median_conf": 55.403534609720175
      }
    ],
    "splits": {
      "train": 8129,
      "val": 2638,
      "test": 2591
    },
    "text_source": {
      "hocr": 10581,
      "djvu": 1682,
      "tesseract_pages": 1095
    },
    "languages": 8
  },
  "encoder": {
    "feature_set": "V",
    "name": "Vertex AI Multilingual embeddings",
    "dims": 100,
    "pca_explained": 0.7860354781150818
  },
  "selection": {
    "rule_met": false,
    "rule": "highest silhouette among feature sets and valid settings whose groups follow script with NMI <= 0.5 and whose median bootstrap ARI >= 0.5",
    "chosen_on": "train books (docs/evaluation_plan.md)"
  },
  "hierarchical": {
    "linkage": "ward",
    "metric": "euclidean",
    "k": 3,
    "k_actual": 3,
    "silhouette": 0.03505982047172203,
    "largest_share": 0.4605479862254829,
    "smallest": 1714,
    "valid": true,
    "nmi_script": 0.019299247199408814,
    "boot_median": 0.2595734183748013,
    "bootstrap": {
      "median": 0.2595734183748013,
      "p10": 0.21368650260941158,
      "p90": 0.3977653721259586,
      "values": [
        0.2575,
        0.2616,
        0.295,
        0.2368,
        0.3654,
        0.2184,
        0.2295,
        0.4534,
        0.3751,
        0.1816,
        0.2844,
        0.2357,
        0.329,
        0.2034,
        0.2148,
        0.2521,
        0.3916,
        0.4553,
        0.2949,
        0.2333
      ]
    },
    "neighbours": [
      {
        "k": 2,
        "ari": 0.7892323432120867
      },
      {
        "k": 4,
        "ari": 0.8145053677871947
      }
    ],
    "effects": {
      "nmi_script": 0.019299247199408814,
      "ari_script": 0.009436437545795397,
      "nmi_era": 0.016633098380156565,
      "nmi_classification": 0.08835885550650598,
      "classified_books": 6825
    }
  },
  "distance": {
    "chosen": "euclidean",
    "wins": {
      "euclidean": 3,
      "cityblock": 1
    }
  },
  "dbscan": {
    "min_pts": 15,
    "eps": 0.866087,
    "eps_is_knee": false,
    "n_clusters": 2,
    "noise_share": 0.30700703698158405,
    "silhouette": 0.006201213247436124,
    "largest_share": 0.6919449019314269,
    "valid": true,
    "grid_settings": 24,
    "bootstrap": {
      "median": 0.8551046515086571,
      "p10": 0.8481791282838121,
      "p90": 0.8681911852073579,
      "values": [
        0.8611,
        0.8593,
        0.8672,
        0.8541,
        0.8599,
        0.8514,
        0.8426,
        0.8539,
        0.8488,
        0.843,
        0.8616,
        0.8773,
        0.8596,
        0.856,
        0.8542,
        0.877,
        0.8645,
        0.8495,
        0.8515,
        0.8535
      ]
    },
    "min_pts_tried": [
      5,
      10,
      15,
      20
    ],
    "eps_range": [
      0.680584,
      1.060916
    ]
  },
  "hier_vs_dbscan": {
    "ari_non_noise": 0.0004042178030626395,
    "nmi_non_noise": 0.003120985505452889,
    "ari_all_noise_as_group": 0.007319349515005668
  },
  "pca": [
    0.1163,
    0.0872
  ],
  "extras": {
    "within_script_silhouette": {
      "by_script": {
        "Bengali": 0.046501465849948014,
        "Devanagari": 0.05313585566305425,
        "Gujarati": 0.060245259626958446,
        "Kannada": 0.044424279578117426,
        "Latin": 0.029903237533052834,
        "Tamil": 0.05967744788301882,
        "Telugu": 0.07068273380137644
      },
      "books": {
        "Bengali": 2007,
        "Devanagari": 2687,
        "Gujarati": 1767,
        "Kannada": 1441,
        "Latin": 2732,
        "Tamil": 1400,
        "Telugu": 1324
      },
      "weighted_mean": 0.04981294043194523
    },
    "high_conf_refit": {
      "books": 8099,
      "share": 0.6063033388231771,
      "ari_vs_full": 0.3532607422603123
    },
    "hdbscan": {
      "min_cluster_size": 134,
      "n_clusters": 2,
      "noise_share": 0.9413834406348256,
      "silhouette": 0.28212043753997934,
      "ari_vs_hierarchical_non_noise": 0.6644934822485945
    }
  },
  "evaluation": {
    "val": {
      "feature_set": "V",
      "setting": {
        "linkage": "ward",
        "metric": "euclidean",
        "k": 3,
        "silhouette": 0.03836288071761263,
        "nmi_script": 0.017158644063467024
      },
      "held_out_books": 2638,
      "t1_agreement": 0.757663734950047,
      "t1_chance": 0.35892903944439647,
      "t1_abstain_rate": 0.030947455523376077,
      "t1_tagged": 0.8568012979250278,
      "t1_books": 2417,
      "t1_left_out_own_pages": 221,
      "t2_accuracy": 0.4583333333333333,
      "t2_baseline_script": 0.4166666666666667,
      "t2_baseline_common": 0.3,
      "t2_points": 4.1666666666666625,
      "t2_p": 0.0024987506246876563,
      "t2_labelled_train": 351,
      "t2_labelled_heldout": 120,
      "group_to_subject": {
        "0": "literature",
        "1": "religion and philosophy",
        "2": "religion and philosophy"
      },
      "t3_bootstrap": 0.3573638085687529,
      "targets": {
        "T1_page_agreement": 0.8,
        "T2_points_above_baseline": 10.0,
        "T3_bootstrap_ari": 0.6
      },
      "met": {
        "T1": false,
        "T2": false,
        "T3": false
      },
      "t1_agreement_ci95": [
        0.7463783042085622,
        0.7678802388540787
      ],
      "t2_accuracy_ci95": [
        0.36666666666666664,
        0.5416666666666666
      ],
      "t2_points_ci95": [
        -6.666666666666665,
        14.187499999999892
      ],
      "catalogue": {
        "labelled_train": 3559,
        "labelled_heldout": 1133,
        "accuracy_groups": 0.4836716681376876,
        "baseline_most_common": 0.4836716681376876,
        "baseline_script_only": 0.5057369814651368,
        "points_above_best_baseline": -2.206531332744921,
        "accuracy_groups_ci95": [
          0.45366284201235657,
          0.5136804942630185
        ],
        "points_above_best_baseline_ci95": [
          -3.089143865842897,
          -1.3239187996469504
        ],
        "permutation_p": 1.0,
        "group_to_subject": {
          "0": "language and literature",
          "1": "language and literature",
          "2": "language and literature"
        },
        "books_with_a_code": 7799,
        "books_mapped": 6689
      }
    },
    "test": {
      "feature_set": "V",
      "setting": {
        "linkage": "ward",
        "metric": "euclidean",
        "k": 3,
        "silhouette": 0.03836288071761263,
        "nmi_script": 0.017158644063467024
      },
      "held_out_books": 2591,
      "t1_agreement": 0.7517385257301809,
      "t1_chance": 0.35892903944439647,
      "t1_abstain_rate": 0.02591024555461474,
      "t1_tagged": 0.8565716272600834,
      "t1_books": 2362,
      "t1_left_out_own_pages": 229,
      "t2_accuracy": 0.5299145299145299,
      "t2_baseline_script": 0.38461538461538464,
      "t2_baseline_common": 0.3076923076923077,
      "t2_points": 14.529914529914528,
      "t2_p": 0.0004997501249375312,
      "t2_labelled_train": 351,
      "t2_labelled_heldout": 117,
      "group_to_subject": {
        "0": "literature",
        "1": "religion and philosophy",
        "2": "religion and philosophy"
      },
      "t3_bootstrap": 0.3573638085687529,
      "targets": {
        "T1_page_agreement": 0.8,
        "T2_points_above_baseline": 10.0,
        "T3_bootstrap_ari": 0.6
      },
      "met": {
        "T1": false,
        "T2": true,
        "T3": false
      },
      "t1_agreement_ci95": [
        0.7409208594073156,
        0.7627139677275171
      ],
      "t2_accuracy_ci95": [
        0.4358974358974359,
        0.6153846153846154
      ],
      "t2_points_ci95": [
        5.128205128205127,
        24.78632478632478
      ],
      "catalogue": {
        "labelled_train": 3559,
        "labelled_heldout": 1168,
        "accuracy_groups": 0.4777397260273973,
        "baseline_most_common": 0.4777397260273973,
        "baseline_script_only": 0.5179794520547946,
        "points_above_best_baseline": -4.023972602739728,
        "accuracy_groups_ci95": [
          0.449486301369863,
          0.5059931506849316
        ],
        "points_above_best_baseline_ci95": [
          -5.222602739726029,
          -2.910958904109584
        ],
        "permutation_p": 1.0,
        "group_to_subject": {
          "0": "language and literature",
          "1": "language and literature",
          "2": "language and literature"
        },
        "books_with_a_code": 7799,
        "books_mapped": 6689
      }
    }
  },
  "groups": [
    {
      "group": 0,
      "books": 6152,
      "share": 0.4605479862254829,
      "majority_subject": "literature",
      "languages": {
        "eng": 0.188,
        "ben": 0.16,
        "hin": 0.133,
        "guj": 0.129,
        "kan": 0.12,
        "san": 0.11,
        "tel": 0.081,
        "tam": 0.078
      },
      "top_language_share": 0.18774382314694407
    },
    {
      "group": 1,
      "books": 1714,
      "share": 0.12831262164994758,
      "majority_subject": "religion and philosophy",
      "languages": {
        "hin": 0.221,
        "san": 0.186,
        "guj": 0.18,
        "ben": 0.131,
        "tel": 0.1,
        "tam": 0.097,
        "kan": 0.068,
        "eng": 0.019
      },
      "top_language_share": 0.220536756126021
    },
    {
      "group": 2,
      "books": 5492,
      "share": 0.41113939212456957,
      "majority_subject": "religion and philosophy",
      "languages": {
        "eng": 0.144,
        "guj": 0.143,
        "tam": 0.141,
        "ben": 0.131,
        "tel": 0.119,
        "hin": 0.111,
        "kan": 0.11,
        "san": 0.103
      },
      "top_language_share": 0.14366351056081572
    }
  ],
  "centring_effect": {
    "feature_set": "V",
    "k": 3,
    "train_books": 8129,
    "linkage": "ward",
    "metric": "euclidean",
    "uncentred": {
      "nmi_script": 0.6416368308689485,
      "knn10_same_script": 0.9726657645466847,
      "largest_group_share": 0.49108131381473735
    },
    "centred": {
      "nmi_script": 0.0652169098696686,
      "knn10_same_script": 0.6276417763562554,
      "largest_group_share": 0.4192397588879321
    }
  },
  "features": {
    "editions": {
      "precision": 1.0,
      "recall": 0.36363636363636365,
      "precision_ci95": [
        1.0,
        1.0
      ],
      "recall_ci95": [
        0.08333333333333333,
        0.6666666666666666
      ],
      "groups_heldout": 11,
      "threshold": 0.044659557213287036,
      "heldout_books": 2591,
      "train_books": 8129,
      "metric": "euclidean",
      "split": "test"
    },
    "related": {
      "knn_other_script_share": {
        "centred": 0.3723582236437446,
        "uncentred": 0.02733423545331529
      },
      "k": 10,
      "books": 8129,
      "split": "train"
    },
    "suggest": {
      "top1": 0.586472602739726,
      "top3": 0.916095890410959,
      "top1_ci95": [
        0.5582191780821918,
        0.6147260273972602
      ],
      "top3_ci95": [
        0.9006849315068494,
        0.9306506849315068
      ],
      "baseline_most_common": 0.4777397260273973,
      "k": 25,
      "metric": "euclidean",
      "heldout_books": 1168,
      "train_coded_books": 3559,
      "method": "distance-weighted vote of the k nearest coded train books; top-1 / top-3 = the held-out book's own catalogue class is the first / among the first three suggestions; CI by resampling held-out books",
      "split": "test",
      "feature_set": "V"
    },
    "triage": {
      "books": 341,
      "signal": "agreement of our OCR with the archive's OCR",
      "rule_thresholds": {
        "min_conf": 50.0,
        "min_tokens": 100,
        "weights": {
          "low_conf": 0.5,
          "few_tokens": 0.3,
          "dbscan_noise": 0.2
        }
      },
      "auc": 0.49867561059511517,
      "auc_ci95": [
        0.4515130228642174,
        0.5464765358817654
      ]
    }
  }
}