{
  "title": "Exploratory prompted-model evaluation on VIPeR",
  "version": 1,
  "completed_date": "2026-10-14",
  "date_timezone": "America/Los_Angeles",
  "scope": "Clothing and scene appearance retrieval, followed by listwise reranking. This is an informal baseline, not a controlled leaderboard or a human-ceiling study.",
  "dataset": {
    "name": "VIPeR v1.0",
    "identities": 632,
    "archive_sha256": "857e49928a0c74d2da6fbeef372ea2e42b2bce357f87307d74c6601f1a97f267",
    "input_image_width": 96,
    "input_image_height": 256,
    "image_preprocessing": "Bilinear resize from the original 48 by 128 pixel crops."
  },
  "evaluation": {
    "repetitions": 1,
    "test_identity_count": 316,
    "gallery_size": 316,
    "directions": ["camera A to camera B", "camera B to camera A"],
    "total_queries": 632,
    "other_identity_half": "Not evaluated and not used for weight training.",
    "partition_seed": 20261013,
    "test_half": 1,
    "ranking_seed": 20261014,
    "original_prompt_development_identity_count": 50,
    "development_identities_in_this_test_half": 13,
    "prompt_held_out_identity_count": 303,
    "prompt_held_out_query_count": 606,
    "tie_handling": "Fixed seeded ordering, shared by the retrieval and reranked comparison.",
    "invalid_output_handling": "A malformed ranking counts as unsuccessful at every CMC rank; no ranking is guessed or silently retried."
  },
  "retrieval": {
    "model": "claude-haiku-5-5",
    "effort": "low",
    "thinking": "adaptive",
    "max_output_tokens": 1024,
    "system_prompt": "You score visual appearance compatibility for a pedestrian retrieval research evaluation.",
    "prompt": "Compare these pedestrian crops as a possible cross-camera match. Use non-facial appearance: clothing, patterns, accessories, carried objects, body outline, and scene context. Allow changes in viewpoint and lighting. Score the overall evidence for a match from 0 to 1000: 0 is clearly incompatible and 1000 is strongly compatible. Use only the two attached images. Do not use tools or inspect files. Return only one JSON object: {\"score\": <integer from 0 to 1000>}.",
    "score_matrix_shape": [632, 632],
    "reverse_direction": "Ranks the transpose of the existing A-first/B-second pair scores, rather than independently rescoring reversed prompts."
  },
  "reranking": {
    "model": "claude-opus-5-5",
    "effort": "high",
    "max_output_tokens": 8192,
    "shortlist_size": 30,
    "system_prompt": "You rank clothing and scene appearance for a cross-camera retrieval evaluation.",
    "prompt": "Rank the candidate crops by their visual compatibility with the probe crop. Base your ranking only on clothing, patterns, accessories, carried objects, outfit outline, and scene context. Allow changes in viewpoint and lighting. Give highest priority to putting the most compatible candidate first, then order the remaining candidates from best to worst. Use only the attached images; do not use tools or inspect files. Return only one JSON array of the supplied candidate labels in ranked order. Include every candidate label exactly once. Do not output scores, explanations, Markdown, or any other text. The labels and their presentation order carry no information about compatibility.",
    "presentation": "One probe and 30 anonymously labeled, shuffled candidates per fresh session, with no tools or conversation history. The correct counterpart is not inserted if absent from the shortlist.",
    "ranking_beyond_shortlist": "Preserves the original Haiku order after the reranked top 30.",
    "completed_queries": 632,
    "valid_rankings": 629,
    "invalid_rankings": 3,
    "refusals": 0
  },
  "results": {
    "prompt_held_out": {
      "queries": 606,
      "haiku_rank_1_correct": 296,
      "opus_reranked_rank_1_correct": 355,
      "haiku_rank_5_correct": 412,
      "opus_reranked_rank_5_correct": 452,
      "haiku_rank_10_correct": 466,
      "opus_reranked_rank_10_correct": 484,
      "rank_1_corrections": 94,
      "rank_1_regressions": 35
    },
    "all_test_queries": {
      "queries": 632,
      "haiku_rank_1_correct": 310,
      "opus_reranked_rank_1_correct": 368,
      "rank_1_corrections": 96,
      "rank_1_regressions": 38
    }
  },
  "limitations": [
    "Only one frozen identity split was evaluated with Opus, not five repetitions.",
    "Prompt selection used 50 development identities; the headline subset excludes those identities, but earlier failure diagnostics inspected some other evaluation images. This is not a pristine blind test.",
    "No task-specific weight training was performed, but the models' original training data and any prior exposure to VIPeR are unknown.",
    "Published specialist results use different source training datasets, split averaging, and sometimes different fitting or reranking procedures.",
    "The prompts deliberately score clothing and scene appearance; the experiment does not establish an unrestricted capability ceiling for either model.",
    "This file records aggregate measurements and the protocol, not a complete reproduction bundle."
  ]
}
