Merge remote-tracking branch 'origin/main' into qwen

app.py

@@ -2,13 +2,19 @@ import streamlit as st
 import json
 import os
 import time
-from io import BytesIO
-from PIL import Image
+import re
 from pathlib import Path
 
-from geo_bot import GeoBot, AGENT_PROMPT_TEMPLATE
+from geo_bot import GeoBot
 from benchmark import MapGuesserBenchmark
-from config import MODELS_CONFIG, get_data_paths, SUCCESS_THRESHOLD_KM, get_model_class
+from config import (
+    MODELS_CONFIG,
+    get_data_paths,
+    SUCCESS_THRESHOLD_KM,
+    get_model_class,
+    DEFAULT_MODEL,
+    DEFAULT_TEMPERATURE,
+)
 
 
 # Simple API key setup
@@ -22,6 +28,24 @@ if "HF_TOKEN" in st.secrets:
     os.environ["HF_TOKEN"] = st.secrets["HF_TOKEN"]
 
 
+def convert_google_to_mapcrunch_url(google_url):
+    """Convert Google Maps URL to MapCrunch URL format."""
+    try:
+        # Extract coordinates using regex
+        match = re.search(r"@(-?\d+\.\d+),(-?\d+\.\d+)", google_url)
+        if not match:
+            return None
+
+        lat, lon = match.groups()
+        # MapCrunch format: lat_lon_heading_pitch_zoom
+        # Using default values for heading (317.72), pitch (0.86), and zoom (0)
+        mapcrunch_url = f"http://www.mapcrunch.com/p/{lat}_{lon}_317.72_0.86_0"
+        return mapcrunch_url
+    except Exception as e:
+        st.error(f"Error converting URL: {str(e)}")
+        return None
+
+
 def get_available_datasets():
     datasets_dir = Path("datasets")
     if not datasets_dir.exists():
@@ -36,54 +60,164 @@ def get_available_datasets():
 
 
 # UI Setup
-st.set_page_config(page_title="🧠 Omniscient - AI Geographic Analysis", layout="wide")
+st.set_page_config(
+    page_title="🧠 Omniscient - Multiturn Geographic Intelligence", layout="wide"
+)
 st.title("🧠 Omniscient")
-st.markdown("### *The all-knowing AI that sees everything, knows everything*")
+st.markdown("""
+### *An all-seeing AI agent for geographic analysis and deduction*
 
+Omniscient engages in a multi-turn reasoning process — collecting visual clues, asking intelligent questions, and narrowing down locations step by step.  
+Whether it's identifying terrain, interpreting signs, or tracing road patterns, this AI agent learns, adapts, and solves like a true geo-detective.
+""")
 # Sidebar
 with st.sidebar:
     st.header("Configuration")
 
-    # Get available datasets and ensure we have a valid default
-    available_datasets = get_available_datasets()
-    default_dataset = available_datasets[0] if available_datasets else "default"
-
-    dataset_choice = st.selectbox("Dataset", available_datasets, index=0)
-    model_choice = st.selectbox("Model", list(MODELS_CONFIG.keys()))
-    steps_per_sample = st.slider("Max Steps", 3, 20, 10)
-    temperature = st.slider(
-        "Temperature",
-        0.0,
-        2.0,
-        0.0,
-        0.1,
-        help="Controls randomness in AI responses. 0.0 = deterministic, higher = more creative",
-    )
+    # Mode selection
+    mode = st.radio("Mode", ["Dataset Mode", "Online Mode"], index=0)
 
-    # Load dataset with error handling
-    data_paths = get_data_paths(dataset_choice)
-    try:
-        with open(data_paths["golden_labels"], "r") as f:
-            golden_labels = json.load(f).get("samples", [])
+    if mode == "Dataset Mode":
+        # Get available datasets and ensure we have a valid default
+        available_datasets = get_available_datasets()
+        default_dataset = available_datasets[0] if available_datasets else "default"
 
-        st.info(f"Dataset '{dataset_choice}' has {len(golden_labels)} samples")
-        if len(golden_labels) == 0:
-            st.error(f"Dataset '{dataset_choice}' contains no samples!")
+        dataset_choice = st.selectbox("Dataset", available_datasets, index=0)
+        model_choice = st.selectbox(
+            "Model",
+            list(MODELS_CONFIG.keys()),
+            index=list(MODELS_CONFIG.keys()).index(DEFAULT_MODEL),
+        )
+        steps_per_sample = st.slider("Max Steps", 1, 20, 10)
+        temperature = st.slider(
+            "Temperature",
+            0.0,
+            2.0,
+            DEFAULT_TEMPERATURE,
+            0.1,
+            help="Controls randomness in AI responses. 0.0 = deterministic, higher = more creative",
+        )
+
+        # Load dataset with error handling
+        data_paths = get_data_paths(dataset_choice)
+        try:
+            with open(data_paths["golden_labels"], "r") as f:
+                golden_labels = json.load(f).get("samples", [])
+
+            st.info(f"Dataset '{dataset_choice}' has {len(golden_labels)} samples")
+            if len(golden_labels) == 0:
+                st.error(f"Dataset '{dataset_choice}' contains no samples!")
+                st.stop()
+
+        except FileNotFoundError:
+            st.error(
+                f"❌ Dataset '{dataset_choice}' not found at {data_paths['golden_labels']}"
+            )
+            st.info("💡 Available datasets: " + ", ".join(available_datasets))
+            st.stop()
+        except Exception as e:
+            st.error(f"❌ Error loading dataset '{dataset_choice}': {str(e)}")
             st.stop()
 
-    except FileNotFoundError:
-        st.error(
-            f"❌ Dataset '{dataset_choice}' not found at {data_paths['golden_labels']}"
+        num_samples = st.slider(
+            "Samples to Test", 1, len(golden_labels), min(3, len(golden_labels))
         )
-        st.info("💡 Available datasets: " + ", ".join(available_datasets))
-        st.stop()
-    except Exception as e:
-        st.error(f"❌ Error loading dataset '{dataset_choice}': {str(e)}")
-        st.stop()
+    else:  # Online Mode
+        st.info("Enter a URL to analyze a specific location")
 
-    num_samples = st.slider(
-        "Samples to Test", 1, len(golden_labels), min(3, len(golden_labels))
-    )
+        # Add example URLs
+        example_google_url = "https://www.google.com/maps/@37.8728123,-122.2445339,3a,75y,3.36h,90t/data=!3m7!1e1!3m5!1s4DTABKOpCL6hdNRgnAHTgw!2e0!6shttps:%2F%2Fstreetviewpixels-pa.googleapis.com%2Fv1%2Fthumbnail%3Fcb_client%3Dmaps_sv.tactile%26w%3D900%26h%3D600%26pitch%3D0%26panoid%3D4DTABKOpCL6hdNRgnAHTgw%26yaw%3D3.3576431!7i13312!8i6656?entry=ttu"
+        example_mapcrunch_url = (
+            "http://www.mapcrunch.com/p/37.882284_-122.269626_293.91_-6.63_0"
+        )
+
+        # Create tabs for different URL types
+        input_tab1, input_tab2 = st.tabs(["Google Maps URL", "MapCrunch URL"])
+
+        google_url = ""
+        mapcrunch_url = ""
+        golden_labels = None
+        num_samples = None
+
+        with input_tab1:
+            url_col1, url_col2 = st.columns([3, 1])
+            with url_col1:
+                google_url = st.text_input(
+                    "Google Maps URL",
+                    placeholder="https://www.google.com/maps/@37.5851338,-122.1519467,9z?entry=ttu",
+                    key="google_maps_url",
+                )
+            st.markdown(
+                f"💡 **Example Location:** [View in Google Maps]({example_google_url})"
+            )
+            if google_url:
+                mapcrunch_url_converted = convert_google_to_mapcrunch_url(google_url)
+                if mapcrunch_url_converted:
+                    st.success(f"Converted to MapCrunch URL: {mapcrunch_url_converted}")
+                    try:
+                        match = re.search(r"@(-?\d+\.\d+),(-?\d+\.\d+)", google_url)
+                        if not match:
+                            st.error("Invalid Google Maps URL format")
+                            st.stop()
+
+                        lat, lon = match.groups()
+
+                        golden_labels = [
+                            {
+                                "id": "online",
+                                "lat": float(lat),
+                                "lng": float(lon),
+                                "url": mapcrunch_url_converted,
+                            }
+                        ]
+                        num_samples = 1
+                    except Exception as e:
+                        st.error(f"Invalid Google Maps URL format: {str(e)}")
+                else:
+                    st.error("Invalid Google Maps URL format")
+
+        with input_tab2:
+            st.markdown("💡 **Example Location:**")
+            st.markdown(f"[View in MapCrunch]({example_mapcrunch_url})")
+            st.code(example_mapcrunch_url, language="text")
+            mapcrunch_url = st.text_input(
+                "MapCrunch URL", placeholder=example_mapcrunch_url, key="mapcrunch_url"
+            )
+            if mapcrunch_url:
+                try:
+                    coords = mapcrunch_url.split("/")[-1].split("_")
+                    lat, lon = float(coords[0]), float(coords[1])
+                    golden_labels = [
+                        {"id": "online", "lat": lat, "lng": lon, "url": mapcrunch_url}
+                    ]
+                    num_samples = 1
+                except Exception as e:
+                    st.error(f"Invalid MapCrunch URL format: {str(e)}")
+
+        # Only stop if neither input is provided
+        if not google_url and not mapcrunch_url:
+            st.warning(
+                "Please enter a Google Maps URL or MapCrunch URL, or use the example above."
+            )
+            st.stop()
+        if golden_labels is None or num_samples is None:
+            st.warning("Please enter a valid URL.")
+            st.stop()
+
+        model_choice = st.selectbox(
+            "Model",
+            list(MODELS_CONFIG.keys()),
+            index=list(MODELS_CONFIG.keys()).index(DEFAULT_MODEL),
+        )
+        steps_per_sample = st.slider("Max Steps", 1, 20, 10)
+        temperature = st.slider(
+            "Temperature",
+            0.0,
+            2.0,
+            DEFAULT_TEMPERATURE,
+            0.1,
+            help="Controls randomness in AI responses. 0.0 = deterministic, higher = more creative",
+        )
 
     start_button = st.button("🚀 Start", type="primary")
 
@@ -93,7 +227,9 @@ if start_button:
     config = MODELS_CONFIG[model_choice]
     model_class = get_model_class(config["class"])
 
-    benchmark_helper = MapGuesserBenchmark(dataset_name=dataset_choice)
+    benchmark_helper = MapGuesserBenchmark(
+        dataset_name=dataset_choice if mode == "Dataset Mode" else "online"
+    )
     all_results = []
 
     progress_bar = st.progress(0)
@@ -106,10 +242,16 @@ if start_button:
     ) as bot:
         for i, sample in enumerate(test_samples):
             st.divider()
-            st.header(f"Sample {i + 1}/{num_samples} - ID: {sample.get('id', 'N/A')}")
+            st.header(f"Sample {i + 1}/{num_samples}")
+
+            if mode == "Online Mode":
+                # Load the MapCrunch URL directly
+                bot.controller.load_url(sample["url"])
+            else:
+                # Load from dataset as before
+                bot.controller.load_location_from_data(sample)
 
-            # Load the sample location
-            bot.controller.load_location_from_data(sample)
+            bot.controller.setup_clean_environment()
 
             # Create containers for UI updates
             sample_container = st.container()
@@ -290,3 +432,11 @@ if start_button:
             file_name=f"geo_results_{dataset_choice}_{model_choice}_{num_samples}samples.json",
             mime="application/json",
         )
+
+
+def handle_tab_completion():
+    """Handle tab completion for the Google Maps URL input."""
+    if st.session_state.google_maps_url == "":
+        st.session_state.google_maps_url = (
+            "https://www.google.com/maps/@37.5851338,-122.1519467,9z?entry=ttu"
+        )

config.py

@@ -31,6 +31,10 @@ BENCHMARK_CONFIG = {
 # MapCrunch options
 MAPCRUNCH_OPTIONS = {}
 
+# Default settings
+DEFAULT_MODEL = "gemini-2.5-pro"
+DEFAULT_TEMPERATURE = 1.0
+
 # Model configurations
 MODELS_CONFIG = {
     "gpt-4o": {

datasets/example2/golden_labels.json

@@ -0,0 +1,309 @@
+{
+  "metadata": {
+    "dataset_name": "example2",
+    "collection_date": "2025-06-12T15:26:27.073898",
+    "collection_options": {}
+  },
+  "samples": [
+    {
+      "id": "e958ec37-0b30-4b30-b508-df614acb489f",
+      "timestamp": "2025-06-12T15:25:22.208434",
+      "lat": 41.52785178236295,
+      "lng": 22.061235302695696,
+      "address": "Unknown",
+      "pano_id": "hB2IrvH7f4iapBJZ_jNMJg",
+      "pov": {
+        "heading": -201.27999999999997,
+        "pitch": 5,
+        "zoom": 1.0000039848465483
+      },
+      "url_slug": "41.527852_22.061235_-201.28_-5.00_0",
+      "thumbnail_path": "e958ec37-0b30-4b30-b508-df614acb489f.jpg"
+    },
+    {
+      "id": "8c047700-2775-4161-bfa5-9c5f966ee929",
+      "timestamp": "2025-06-12T15:25:25.564230",
+      "lat": 35.51112041872643,
+      "lng": 138.76733244706546,
+      "address": "Unknown",
+      "pano_id": "KSCuXhlMyISBw9Poh1hG8w",
+      "pov": {
+        "heading": 61.72000000000003,
+        "pitch": 5,
+        "zoom": 1.0000046104496552
+      },
+      "url_slug": "35.511120_138.767332_61.72_-5.00_0",
+      "thumbnail_path": "8c047700-2775-4161-bfa5-9c5f966ee929.jpg"
+    },
+    {
+      "id": "c0beb43d-e95d-4352-9d36-56aa6c903e7d",
+      "timestamp": "2025-06-12T15:25:28.972905",
+      "lat": 43.42554082467874,
+      "lng": -89.09498316077553,
+      "address": "Unknown",
+      "pano_id": "H3QYXtsn7NBQxjZep7whsQ",
+      "pov": {
+        "heading": 164.72000000000003,
+        "pitch": 5,
+        "zoom": 1.0000093225833766
+      },
+      "url_slug": "43.425541_-89.094983_164.72_-5.00_0",
+      "thumbnail_path": "c0beb43d-e95d-4352-9d36-56aa6c903e7d.jpg"
+    },
+    {
+      "id": "f0af2553-81cf-48a7-8557-ed4b437a0c9d",
+      "timestamp": "2025-06-12T15:25:32.389314",
+      "lat": 47.48390126215672,
+      "lng": 34.659670566007904,
+      "address": "Unknown",
+      "pano_id": "bfFQMztfnvuffg-_9NkXjQ",
+      "pov": {
+        "heading": -337.28,
+        "pitch": 5,
+        "zoom": 1.0000017337781582
+      },
+      "url_slug": "47.483901_34.659671_-337.28_-5.00_0",
+      "thumbnail_path": "f0af2553-81cf-48a7-8557-ed4b437a0c9d.jpg"
+    },
+    {
+      "id": "06c2d9c2-f765-4f2b-b9b1-5b93229dc0fc",
+      "timestamp": "2025-06-12T15:25:35.813919",
+      "lat": -34.63754161327788,
+      "lng": 118.09156096189146,
+      "address": "Unknown",
+      "pano_id": "PzTOFAXRN2xbwgjUqN4bpg",
+      "pov": {
+        "heading": -148.27999999999997,
+        "pitch": 5,
+        "zoom": 1.0000041033706084
+      },
+      "url_slug": "-34.637542_118.091561_-148.28_-5.00_0",
+      "thumbnail_path": "06c2d9c2-f765-4f2b-b9b1-5b93229dc0fc.jpg"
+    },
+    {
+      "id": "92beabfc-c50d-421e-a847-fd2cb145aac8",
+      "timestamp": "2025-06-12T15:25:39.191588",
+      "lat": 64.57213289684468,
+      "lng": -21.97630482318329,
+      "address": "Unknown",
+      "pano_id": "UojsE8YlVZ0N_TsBzTXkHw",
+      "pov": {
+        "heading": 90.72000000000003,
+        "pitch": 5,
+        "zoom": 1.0000091919376193
+      },
+      "url_slug": "64.572133_-21.976305_90.72_-5.00_0",
+      "thumbnail_path": "92beabfc-c50d-421e-a847-fd2cb145aac8.jpg"
+    },
+    {
+      "id": "fae54d62-755c-447f-ace2-7d7d9dcbab0a",
+      "timestamp": "2025-06-12T15:25:42.516009",
+      "lat": 32.34919407912315,
+      "lng": -64.71096298763197,
+      "address": "Unknown",
+      "pano_id": "CAoSF0NJSE0wb2dLRUlDQWdJREV1Tl9YMmdF",
+      "pov": {
+        "heading": 260.72,
+        "pitch": 5,
+        "zoom": 1.0000030548533054
+      },
+      "url_slug": "32.349194_-64.710963_260.72_-5.00_0",
+      "thumbnail_path": "fae54d62-755c-447f-ace2-7d7d9dcbab0a.jpg"
+    },
+    {
+      "id": "a086d72f-5c18-4895-a728-4262d565005f",
+      "timestamp": "2025-06-12T15:25:45.919449",
+      "lat": 34.880191512475136,
+      "lng": 138.1990659116743,
+      "address": "Unknown",
+      "pano_id": "CAoSFkNJSE0wb2dLRUlDQWdJRHE4NDJsWnc.",
+      "pov": {
+        "heading": 358.72,
+        "pitch": 5,
+        "zoom": 1.0000054942816385
+      },
+      "url_slug": "34.880192_138.199066_358.72_-5.00_0",
+      "thumbnail_path": "a086d72f-5c18-4895-a728-4262d565005f.jpg"
+    },
+    {
+      "id": "ea8fae71-8acd-4ba5-a016-3fb9d051c8ff",
+      "timestamp": "2025-06-12T15:25:49.337047",
+      "lat": 22.130721057935787,
+      "lng": 113.57874365739633,
+      "address": "Unknown",
+      "pano_id": "IZjDP8WGT4O3zju2hFiFCg",
+      "pov": {
+        "heading": -249.27999999999997,
+        "pitch": 5,
+        "zoom": 1.0000003495222285
+      },
+      "url_slug": "22.130721_113.578744_-249.28_-5.00_0",
+      "thumbnail_path": "ea8fae71-8acd-4ba5-a016-3fb9d051c8ff.jpg"
+    },
+    {
+      "id": "0730a5f2-a68b-4e7a-9f8d-877bd65b04ba",
+      "timestamp": "2025-06-12T15:25:52.704884",
+      "lat": 18.6634455460649,
+      "lng": -96.76220059003958,
+      "address": "Unknown",
+      "pano_id": "J9ZTNGHVQf4EBW2rCgaHaQ",
+      "pov": {
+        "heading": -5.279999999999973,
+        "pitch": 5,
+        "zoom": 1.0000057047491013
+      },
+      "url_slug": "18.663446_-96.762201_-5.28_-5.00_0",
+      "thumbnail_path": "0730a5f2-a68b-4e7a-9f8d-877bd65b04ba.jpg"
+    },
+    {
+      "id": "992ba13b-0ced-4a8f-b8ee-3e2f9ac14464",
+      "timestamp": "2025-06-12T15:25:56.187109",
+      "lat": 64.51234002155906,
+      "lng": -22.13983357174534,
+      "address": "Unknown",
+      "pano_id": "nayAVqmoufS1IQHDOtXK5g",
+      "pov": {
+        "heading": 204.72000000000003,
+        "pitch": 5,
+        "zoom": 1.0000082656900817
+      },
+      "url_slug": "64.512340_-22.139834_204.72_-5.00_0",
+      "thumbnail_path": "992ba13b-0ced-4a8f-b8ee-3e2f9ac14464.jpg"
+    },
+    {
+      "id": "83f086e9-2b3c-47c8-9611-557853313d73",
+      "timestamp": "2025-06-12T15:25:59.649986",
+      "lat": 49.18855106875811,
+      "lng": 18.715493968155048,
+      "address": "Unknown",
+      "pano_id": "X4_z3fyEkFf1TURoTEtTRw",
+      "pov": {
+        "heading": -285.28,
+        "pitch": 5,
+        "zoom": 1.0000014961409551
+      },
+      "url_slug": "49.188551_18.715494_-285.28_-5.00_0",
+      "thumbnail_path": "83f086e9-2b3c-47c8-9611-557853313d73.jpg"
+    },
+    {
+      "id": "11e0d5df-607f-4673-9075-6cb784949469",
+      "timestamp": "2025-06-12T15:26:03.006109",
+      "lat": 47.566063195512115,
+      "lng": 117.66840235174527,
+      "address": "Unknown",
+      "pano_id": "pmbqMP3wUN-y4UNF29-ijg",
+      "pov": {
+        "heading": -53.27999999999997,
+        "pitch": 5,
+        "zoom": 1.0000074643993226
+      },
+      "url_slug": "47.566063_117.668402_-53.28_-5.00_0",
+      "thumbnail_path": "11e0d5df-607f-4673-9075-6cb784949469.jpg"
+    },
+    {
+      "id": "35aad182-1f46-4ded-a7c0-f41663c1dbc6",
+      "timestamp": "2025-06-12T15:26:06.382239",
+      "lat": 39.85636978265199,
+      "lng": 38.22670052745333,
+      "address": "Unknown",
+      "pano_id": "YX0qC_q239NxWkz2rjYO2A",
+      "pov": {
+        "heading": 194.72000000000003,
+        "pitch": 5,
+        "zoom": 1.0000085095965745
+      },
+      "url_slug": "39.856370_38.226701_194.72_-5.00_0",
+      "thumbnail_path": "35aad182-1f46-4ded-a7c0-f41663c1dbc6.jpg"
+    },
+    {
+      "id": "6668b3fd-eefd-4d25-aa4a-7021a19d19c3",
+      "timestamp": "2025-06-12T15:26:09.783272",
+      "lat": 22.456568266194985,
+      "lng": 114.09923601594565,
+      "address": "Unknown",
+      "pano_id": "DM221xFUpWG9lY53LbL--Q",
+      "pov": {
+        "heading": -355.28,
+        "pitch": 5,
+        "zoom": 1.0000096482864165
+      },
+      "url_slug": "22.456568_114.099236_-355.28_-5.00_0",
+      "thumbnail_path": "6668b3fd-eefd-4d25-aa4a-7021a19d19c3.jpg"
+    },
+    {
+      "id": "2df0138c-dacf-4d89-9f23-9b095226f70f",
+      "timestamp": "2025-06-12T15:26:13.164542",
+      "lat": 38.46383428233303,
+      "lng": 35.16450386289184,
+      "address": "Unknown",
+      "pano_id": "yYdcfT5d7wr1sFM0be7AbQ",
+      "pov": {
+        "heading": -101.27999999999997,
+        "pitch": 5,
+        "zoom": 1.0000053534557005
+      },
+      "url_slug": "38.463834_35.164504_-101.28_-5.00_0",
+      "thumbnail_path": "2df0138c-dacf-4d89-9f23-9b095226f70f.jpg"
+    },
+    {
+      "id": "7a1c2e5a-8cde-4a44-9fdb-5142a26a001d",
+      "timestamp": "2025-06-12T15:26:16.508455",
+      "lat": -6.370108870806767,
+      "lng": -76.28528690259334,
+      "address": "Unknown",
+      "pano_id": "bDGZjPoLOS0J4FaKbCQi3Q",
+      "pov": {
+        "heading": 13.720000000000027,
+        "pitch": 5,
+        "zoom": 1.0000056924769267
+      },
+      "url_slug": "-6.370109_-76.285287_13.72_-5.00_0",
+      "thumbnail_path": "7a1c2e5a-8cde-4a44-9fdb-5142a26a001d.jpg"
+    },
+    {
+      "id": "add1ffdf-22ca-4e15-9061-12d4e4b70638",
+      "timestamp": "2025-06-12T15:26:19.889548",
+      "lat": -31.647905825412163,
+      "lng": -60.705529014729464,
+      "address": "Unknown",
+      "pano_id": "LfewarY8AHpdsRryIvxjVQ",
+      "pov": {
+        "heading": 169.72000000000003,
+        "pitch": 5,
+        "zoom": 1.0000043082285572
+      },
+      "url_slug": "-31.647906_-60.705529_169.72_-5.00_0",
+      "thumbnail_path": "add1ffdf-22ca-4e15-9061-12d4e4b70638.jpg"
+    },
+    {
+      "id": "0c93acf3-75cc-43c1-be9e-28e034c139ff",
+      "timestamp": "2025-06-12T15:26:23.261201",
+      "lat": 43.95405778584408,
+      "lng": 22.298469861096923,
+      "address": "Unknown",
+      "pano_id": "q_NGYfvHdH9mL4cyp3UjEQ",
+      "pov": {
+        "heading": 305.72,
+        "pitch": 5,
+        "zoom": 1.0000030120206056
+      },
+      "url_slug": "43.954058_22.298470_305.72_-5.00_0",
+      "thumbnail_path": "0c93acf3-75cc-43c1-be9e-28e034c139ff.jpg"
+    },
+    {
+      "id": "016d1183-d7a6-46a4-98f2-a97f62445b89",
+      "timestamp": "2025-06-12T15:26:26.696698",
+      "lat": 34.59326187516724,
+      "lng": 135.74317546771286,
+      "address": "Unknown",
+      "pano_id": "iiCKjrr3Zib0M5OYS5BV8g",
+      "pov": {
+        "heading": -279.28,
+        "pitch": 5,
+        "zoom": 1.0000047776936738
+      },
+      "url_slug": "34.593262_135.743175_-279.28_-5.00_0",
+      "thumbnail_path": "016d1183-d7a6-46a4-98f2-a97f62445b89.jpg"
+    }
+  ]
+}

datasets/example3/golden_labels.json

@@ -0,0 +1,309 @@
+{
+  "metadata": {
+    "dataset_name": "example3",
+    "collection_date": "2025-06-12T15:57:06.206331",
+    "collection_options": {}
+  },
+  "samples": [
+    {
+      "id": "197a5345-a9da-41e1-93c5-b46ad24e5363",
+      "timestamp": "2025-06-12T15:56:00.069960",
+      "lat": 32.450617168194135,
+      "lng": -114.69399063957216,
+      "address": "Libramiento Golfo de Sta Clara-Ejido La Islita, San Luis R\u00edo Colorado, Sonora",
+      "pano_id": "RwuCf43rrr5vnlEgH7uMHw",
+      "pov": {
+        "heading": -265.28,
+        "pitch": 5,
+        "zoom": 1.000000033789912
+      },
+      "url_slug": "32.450617_-114.693991_-265.28_-5.00_0",
+      "thumbnail_path": "197a5345-a9da-41e1-93c5-b46ad24e5363.jpg"
+    },
+    {
+      "id": "66f91dcf-3fe6-4962-bfe6-2de8a349d6f6",
+      "timestamp": "2025-06-12T15:56:03.857510",
+      "lat": -37.78214235746083,
+      "lng": 144.12300194368797,
+      "address": "1476 Ballan-Meredith Rd, Morrisons, Victoria",
+      "pano_id": "hXyohzqEx6F3Q7eSQ089Mw",
+      "pov": {
+        "heading": -22.279999999999973,
+        "pitch": 5,
+        "zoom": 1.0000055105094043
+      },
+      "url_slug": "-37.782142_144.123002_-22.28_-5.00_0",
+      "thumbnail_path": "66f91dcf-3fe6-4962-bfe6-2de8a349d6f6.jpg"
+    },
+    {
+      "id": "7c22a7fc-fa66-40cc-88fd-626b826e14af",
+      "timestamp": "2025-06-12T15:56:07.240175",
+      "lat": 61.90332655530213,
+      "lng": 25.33606097079652,
+      "address": "98 Tervaj\u00e4rventie, Central Finland",
+      "pano_id": "yYWVkowafmuAlOFKWCV1TA",
+      "pov": {
+        "heading": 156.72000000000003,
+        "pitch": 5,
+        "zoom": 1.0000044280956983
+      },
+      "url_slug": "61.903327_25.336061_156.72_-5.00_0",
+      "thumbnail_path": "7c22a7fc-fa66-40cc-88fd-626b826e14af.jpg"
+    },
+    {
+      "id": "0facf8d6-5bcf-41e0-931e-2585e177fb68",
+      "timestamp": "2025-06-12T15:56:10.614004",
+      "lat": -18.13971017041426,
+      "lng": -63.937036497382664,
+      "address": "7, Santa Cruz Department",
+      "pano_id": "ILV43CvgWyxYt5HTTieZ4Q",
+      "pov": {
+        "heading": 290.72,
+        "pitch": 5,
+        "zoom": 1.0000061861010834
+      },
+      "url_slug": "-18.139710_-63.937036_290.72_-5.00_0",
+      "thumbnail_path": "0facf8d6-5bcf-41e0-931e-2585e177fb68.jpg"
+    },
+    {
+      "id": "9b372f4e-ca87-481c-a4d0-c3a6c90dff12",
+      "timestamp": "2025-06-12T15:56:14.031494",
+      "lat": 39.28408711718923,
+      "lng": 22.84441276717771,
+      "address": "EO30, Decentralized Administration of Thessaly and Central Greece",
+      "pano_id": "TnZ8AiIDWLWyoLMtWvThPg",
+      "pov": {
+        "heading": -320.28,
+        "pitch": 5,
+        "zoom": 1.0000078610298728
+      },
+      "url_slug": "39.284087_22.844413_-320.28_-5.00_0",
+      "thumbnail_path": "9b372f4e-ca87-481c-a4d0-c3a6c90dff12.jpg"
+    },
+    {
+      "id": "6558ba29-9edc-4c66-9213-edb82e77cf77",
+      "timestamp": "2025-06-12T15:56:17.391786",
+      "lat": 57.728373123146845,
+      "lng": 11.654417090381687,
+      "address": "1 Tjolmenv\u00e4gen, \u00d6cker\u00f6, V\u00e4stra G\u00f6taland County",
+      "pano_id": "FUCNS8h6b1KdjbkqekrpKw",
+      "pov": {
+        "heading": -114.27999999999997,
+        "pitch": 5,
+        "zoom": 1.0000089987016267
+      },
+      "url_slug": "57.728373_11.654417_-114.28_-5.00_0",
+      "thumbnail_path": "6558ba29-9edc-4c66-9213-edb82e77cf77.jpg"
+    },
+    {
+      "id": "ed167213-4364-40ae-80ee-78bf63480b9c",
+      "timestamp": "2025-06-12T15:56:20.754401",
+      "lat": -14.333745187624173,
+      "lng": -170.720756044093,
+      "address": "Rte 014, Tafuna, Western District",
+      "pano_id": "bUeapV3I_9VGBhumXlk2Tw",
+      "pov": {
+        "heading": 151.72000000000003,
+        "pitch": 5,
+        "zoom": 1.000005537590412
+      },
+      "url_slug": "-14.333745_-170.720756_151.72_-5.00_0",
+      "thumbnail_path": "ed167213-4364-40ae-80ee-78bf63480b9c.jpg"
+    },
+    {
+      "id": "41734c39-d109-4c5c-a939-82fcadfadbdf",
+      "timestamp": "2025-06-12T15:56:24.184911",
+      "lat": 45.86505410116751,
+      "lng": 15.519019308411098,
+      "address": "Pokopali\u0161\u010de Bu\u0161e\u010da vas",
+      "pano_id": "CAoSFkNJSE0wb2dLRUlDQWdJQ0VpYWVtWlE.",
+      "pov": {
+        "heading": 326.72,
+        "pitch": 5,
+        "zoom": 1.0000074031092103
+      },
+      "url_slug": "45.865054_15.519019_326.72_-5.00_0",
+      "thumbnail_path": "41734c39-d109-4c5c-a939-82fcadfadbdf.jpg"
+    },
+    {
+      "id": "04f77afb-4b62-4d27-ae04-69f72b47cd30",
+      "timestamp": "2025-06-12T15:56:28.265166",
+      "lat": 53.210286076785295,
+      "lng": -114.86431799288032,
+      "address": "AB-22, Buck Creek, Alberta",
+      "pano_id": "RfHJ9zAVgKGNzIAqHfk8rA",
+      "pov": {
+        "heading": -134.27999999999997,
+        "pitch": 5,
+        "zoom": 1.0000034294812554
+      },
+      "url_slug": "53.210286_-114.864318_-134.28_-5.00_0",
+      "thumbnail_path": "04f77afb-4b62-4d27-ae04-69f72b47cd30.jpg"
+    },
+    {
+      "id": "e3141a38-fb2d-45a8-8f3a-4c26018720e4",
+      "timestamp": "2025-06-12T15:56:31.850499",
+      "lat": 57.66292577014665,
+      "lng": 27.24427273260464,
+      "address": "M\u00f6ldre, V\u00f5ru County",
+      "pano_id": "vvKr0wNXF5WfS1lOq1IIxw",
+      "pov": {
+        "heading": 44.72000000000003,
+        "pitch": 5,
+        "zoom": 1.000000124614534
+      },
+      "url_slug": "57.662926_27.244273_44.72_-5.00_0",
+      "thumbnail_path": "e3141a38-fb2d-45a8-8f3a-4c26018720e4.jpg"
+    },
+    {
+      "id": "1eea0ce4-3332-4096-982b-37e97772e105",
+      "timestamp": "2025-06-12T15:56:35.316086",
+      "lat": -3.681535588587605,
+      "lng": -80.19932740757173,
+      "address": "24250 TU-101, Matapalo, Tumbes",
+      "pano_id": "48w450euvqIfAVQ7R3HLng",
+      "pov": {
+        "heading": 295.72,
+        "pitch": 5,
+        "zoom": 1.0000097574696656
+      },
+      "url_slug": "-3.681536_-80.199327_295.72_-5.00_0",
+      "thumbnail_path": "1eea0ce4-3332-4096-982b-37e97772e105.jpg"
+    },
+    {
+      "id": "e49be483-24e8-47ef-bf01-77dcfcb5d2d4",
+      "timestamp": "2025-06-12T15:56:38.725810",
+      "lat": -14.279133596130174,
+      "lng": -170.55687001529134,
+      "address": "Route 119, Aunu\u02bbu, AS 96799",
+      "pano_id": "TpeB7isFtiuumh6Eh15ZcA",
+      "pov": {
+        "heading": -265.28,
+        "pitch": 5,
+        "zoom": 1.0000006336487888
+      },
+      "url_slug": "-14.279134_-170.556870_-265.28_-5.00_0",
+      "thumbnail_path": "e49be483-24e8-47ef-bf01-77dcfcb5d2d4.jpg"
+    },
+    {
+      "id": "c5a611d5-29d2-4a83-b00d-1ed7df76869f",
+      "timestamp": "2025-06-12T15:56:42.149170",
+      "lat": 31.800383498252334,
+      "lng": 35.24268946547305,
+      "address": "Moqatel, Jerusalem, Jerusalem District",
+      "pano_id": "1VidmzHWGOrCkkidLLWF3Q",
+      "pov": {
+        "heading": -159.27999999999997,
+        "pitch": 5,
+        "zoom": 1.0000062558576592
+      },
+      "url_slug": "31.800383_35.242689_-159.28_-5.00_0",
+      "thumbnail_path": "c5a611d5-29d2-4a83-b00d-1ed7df76869f.jpg"
+    },
+    {
+      "id": "6dd669f4-2f13-4909-a480-9708a23cd80f",
+      "timestamp": "2025-06-12T15:56:45.579437",
+      "lat": 51.29578341353695,
+      "lng": 12.500988457395495,
+      "address": "Thensa",
+      "pano_id": "CAoSF0NJSE0wb2dLRUlDQWdJREQyZUtxbVFF",
+      "pov": {
+        "heading": -42.27999999999997,
+        "pitch": 5,
+        "zoom": 1.0000073719182492
+      },
+      "url_slug": "51.295783_12.500988_-42.28_-5.00_0",
+      "thumbnail_path": "6dd669f4-2f13-4909-a480-9708a23cd80f.jpg"
+    },
+    {
+      "id": "4f76d721-c2ec-4dc4-9485-e542d70ccf5c",
+      "timestamp": "2025-06-12T15:56:48.982692",
+      "lat": 58.32519714026791,
+      "lng": 26.777231622290483,
+      "address": "5 Palupealse, Aardlapalu, Tartu County",
+      "pano_id": "WUUzxDv8tWsNfGv28OGIeA",
+      "pov": {
+        "heading": 206.72000000000003,
+        "pitch": 5,
+        "zoom": 1.0000007116201164
+      },
+      "url_slug": "58.325197_26.777232_206.72_-5.00_0",
+      "thumbnail_path": "4f76d721-c2ec-4dc4-9485-e542d70ccf5c.jpg"
+    },
+    {
+      "id": "23a2f011-371a-48ed-bed7-254e8beefff3",
+      "timestamp": "2025-06-12T15:56:52.355563",
+      "lat": 27.53069417681522,
+      "lng": 90.1727287722609,
+      "address": "Trashigang-Semtokha Hwy, Wangdue Phodrang",
+      "pano_id": "1xZ7zVnYVh1f-hN4Xo_xFA",
+      "pov": {
+        "heading": 304.72,
+        "pitch": 5,
+        "zoom": 1.0000081048518055
+      },
+      "url_slug": "27.530694_90.172729_304.72_-5.00_0",
+      "thumbnail_path": "23a2f011-371a-48ed-bed7-254e8beefff3.jpg"
+    },
+    {
+      "id": "af61d989-8751-4c07-820c-8b35ffd0f2ae",
+      "timestamp": "2025-06-12T15:56:55.722548",
+      "lat": 5.287167713224647,
+      "lng": 103.06979234400693,
+      "address": "Lorong Bunga Raya, Kuala Terengganu, Terengganu",
+      "pano_id": "Pktsz31ZAWkJ-R1Ixesabw",
+      "pov": {
+        "heading": -178.27999999999997,
+        "pitch": 5,
+        "zoom": 1.0000068048105129
+      },
+      "url_slug": "5.287168_103.069792_-178.28_-5.00_0",
+      "thumbnail_path": "af61d989-8751-4c07-820c-8b35ffd0f2ae.jpg"
+    },
+    {
+      "id": "484ba34d-e072-4cb1-81b8-b1a4af8f36e9",
+      "timestamp": "2025-06-12T15:56:59.179328",
+      "lat": 45.95418096151833,
+      "lng": 5.379404630232549,
+      "address": "Amb\u00e9rieu-en-Bugey, Auvergne-Rh\u00f4ne-Alpes",
+      "pano_id": "AhkPs6oryerDdiRpLzRrDg",
+      "pov": {
+        "heading": 90.72000000000003,
+        "pitch": 5,
+        "zoom": 1.000005463895491
+      },
+      "url_slug": "45.954181_5.379405_90.72_-5.00_0",
+      "thumbnail_path": "484ba34d-e072-4cb1-81b8-b1a4af8f36e9.jpg"
+    },
+    {
+      "id": "720334f9-6332-4438-bcd2-68e0b464bb65",
+      "timestamp": "2025-06-12T15:57:02.582946",
+      "lat": 25.590547334531223,
+      "lng": 55.77315301790442,
+      "address": "E311, Al Raafah, Ras Al Khaimah",
+      "pano_id": "XNQCCNZ7aSxDPKeOoBaBYA",
+      "pov": {
+        "heading": 338.72,
+        "pitch": 5,
+        "zoom": 1.000001329133932
+      },
+      "url_slug": "25.590547_55.773153_338.72_-5.00_0",
+      "thumbnail_path": "720334f9-6332-4438-bcd2-68e0b464bb65.jpg"
+    },
+    {
+      "id": "d19dbe06-9b1a-4639-8d02-b21d2f87e8f4",
+      "timestamp": "2025-06-12T15:57:05.925602",
+      "lat": 69.97748674684166,
+      "lng": 19.105915802473216,
+      "address": "32 Fv302, Troms",
+      "pano_id": "CLIgvpLaAivFTsJqHVhsZg",
+      "pov": {
+        "heading": -129.27999999999997,
+        "pitch": 5,
+        "zoom": 1.0000075163842848
+      },
+      "url_slug": "69.977487_19.105916_-129.28_-5.00_0",
+      "thumbnail_path": "d19dbe06-9b1a-4639-8d02-b21d2f87e8f4.jpg"
+    }
+  ]
+}

geo_bot.py

@@ -6,13 +6,7 @@ from typing import Tuple, List, Optional, Dict, Any, Type
 
 from PIL import Image
 from langchain_core.messages import HumanMessage, BaseMessage
-from langchain_core.language_models.chat_models import BaseChatModel
-from langchain_openai import ChatOpenAI
-from langchain_anthropic import ChatAnthropic
-from langchain_google_genai import ChatGoogleGenerativeAI
-
 from hf_chat import HuggingFaceChat
-
 from mapcrunch_controller import MapCrunchController
 
 # The "Golden" Prompt (v7): add more descprtions in context and task
@@ -43,20 +37,20 @@ AGENT_PROMPT_TEMPLATE = """
 
 4.  **Be Decisive:** A unique, definitive clue (full address, rare town name, etc.) ⇒ `GUESS` immediately.
 
-5.  **Final-Step Rule:** If **Remaining Steps = 1**, you **MUST** `GUESS`.
+5.  **Final-Step Rule:** If **Remaining Steps = 1**, you **MUST** `GUESS` and you should carefully check the image and the surroundings.
 
 ────────────────────────────────
+**Context & Task:**
+Analyze your full journey history and current view, apply the Core Principles, and decide your next action in the required JSON format.
+
 **Action History**
 {history_text}
 
 ────────────────────────────────
-**OUTPUT FORMAT**
-
-Return **one** JSON object wrapped in ```json … ```:
-
-**JSON Output Format:**
-Your response MUST be a valid JSON object wrapped in json ... .
-{{"reasoning": "...", "action_details": {{"action": "GUESS", "lat": <float>, "lon": <float>}} }}
+**JSON Output Format:**More actions
+Your response MUST be a valid JSON object wrapped in ```json ... ```.
+- For exploration: `{{"reasoning": "...", "action_details": {{"action": "ACTION_NAME"}} }}`
+- For the final guess: `{{"reasoning": "...", "action_details": {{"action": "GUESS", "lat": <float>, "lon": <float>}} }}`
 """
 
 BENCHMARK_PROMPT = """

main.py

@@ -1,11 +1,5 @@
 import argparse
 import json
-import random
-from typing import Dict, Optional, List
-
-from langchain_openai import ChatOpenAI
-from langchain_anthropic import ChatAnthropic
-from langchain_google_genai import ChatGoogleGenerativeAI
 
 from geo_bot import GeoBot
 from benchmark import MapGuesserBenchmark

mapcrunch_controller.py

@@ -179,6 +179,22 @@ class MapCrunchController:
             base_actions.extend(["MOVE_FORWARD", "MOVE_BACKWARD"])
         return base_actions
 
+    def get_current_address(self) -> Optional[str]:
+        try:
+            address_element = self.wait.until(
+                EC.visibility_of_element_located(
+                    (By.CSS_SELECTOR, SELECTORS["address_element"])
+                )
+            )
+            address_text = address_element.text.strip()
+            address_title = address_element.get_attribute("title") or ""
+            return (
+                address_title
+                if len(address_title) > len(address_text)
+                else address_text
+            )
+        except Exception:
+            return "Stealth Mode"
     def pan_view(self, direction: str, degrees: int = 45):
         """Pans the view using a direct JS call."""
         pov = self.driver.execute_script("return window.panorama.getPov();")
@@ -275,3 +291,13 @@ class MapCrunchController:
 
     def __exit__(self, exc_type, exc_val, exc_tb):
         self.close()
+
+    def load_url(self, url):
+        """Load a specific MapCrunch URL."""
+        try:
+            self.driver.get(url)
+            time.sleep(2)  # Wait for the page to load
+            return True
+        except Exception as e:
+            print(f"Error loading URL: {e}")
+            return False

requirements.txt

@@ -116,6 +116,7 @@ pytweening==1.2.0
 pytz==2025.2
 pyyaml==6.0.2
 pyzmq==26.2.0
+pyperclip==1.8.2
 referencing==0.36.2
 regex==2024.9.11
 requests==2.32.3