Reading a run¶

Everything downstream of a run reads one file: the web UI, every metric, and every flow that rewinds a finished run.

This notebook generates its own run rather than pointing at one of yours, so it works in a fresh clone before you have API keys. The run is real: real MCP server, real game clock, real event log, with the model replaced by a script.

In [1]:

import logging
import os
import tempfile
from pathlib import Path

from pytest import MonkeyPatch

# These notebooks generate their own run, so nothing here reaches a provider.
# Clearing the keys makes that a fact rather than a claim: if any cell below
# tried to call a model, it would fail here rather than spend.
for _key in ("ANTHROPIC_API_KEY", "OPENAI_API_KEY", "HF_TOKEN"):
    os.environ.pop(_key, None)

# The platform and the MCP server each log a line per tool call at INFO, which
# buries a notebook's own output in several hundred lines of it.
for _logger_name in ("glossogen", "mcp"):
    logging.getLogger(_logger_name).setLevel(logging.WARNING)

SCENARIO = "warehouse_robot_recovery"
PRESET = "knobs_default"

import logging import os import tempfile from pathlib import Path from pytest import MonkeyPatch # These notebooks generate their own run, so nothing here reaches a provider. # Clearing the keys makes that a fact rather than a claim: if any cell below # tried to call a model, it would fail here rather than spend. for _key in ("ANTHROPIC_API_KEY", "OPENAI_API_KEY", "HF_TOKEN"): os.environ.pop(_key, None) # The platform and the MCP server each log a line per tool call at INFO, which # buries a notebook's own output in several hundred lines of it. for _logger_name in ("glossogen", "mcp"): logging.getLogger(_logger_name).setLevel(logging.WARNING) SCENARIO = "warehouse_robot_recovery" PRESET = "knobs_default"

In [2]:

from glossogen.testing import run_rounds

ROUND_COUNT = 4


async def generate_run(round_count, overrides):
    """Run the real round loop with the model replaced by a script.

    Every part of the platform is real here: the MCP server, the tool dispatch,
    the game clock, the world and the event logger. Only the LLM is scripted, so
    this costs nothing and needs no key. Each send sits behind a round gate, so
    every message lands in the same round with the same text on every execution;
    only the incidental cycle count varies with scheduling.
    """
    with MonkeyPatch.context() as patch:
        return await run_rounds(
            scenario_name=SCENARIO,
            preset_name=PRESET,
            round_count=round_count,
            overrides=overrides,
            tmp_path=Path(tempfile.mkdtemp()),
            monkeypatch=patch,
        )


result = await generate_run(round_count=ROUND_COUNT, overrides={})
print(f"{len(result.events)} events at {result.log_path}")

from glossogen.testing import run_rounds ROUND_COUNT = 4 async def generate_run(round_count, overrides): """Run the real round loop with the model replaced by a script. Every part of the platform is real here: the MCP server, the tool dispatch, the game clock, the world and the event logger. Only the LLM is scripted, so this costs nothing and needs no key. Each send sits behind a round gate, so every message lands in the same round with the same text on every execution; only the incidental cycle count varies with scheduling. """ with MonkeyPatch.context() as patch: return await run_rounds( scenario_name=SCENARIO, preset_name=PRESET, round_count=round_count, overrides=overrides, tmp_path=Path(tempfile.mkdtemp()), monkeypatch=patch, ) result = await generate_run(round_count=ROUND_COUNT, overrides={}) print(f"{len(result.events)} events at {result.log_path}")

genai-prices has no price for model 'scripted::floor_associate' under provider <Provider.ANTHROPIC: 'anthropic'>; its tokens are not costed
Traceback (most recent call last):
  File "/home/runner/work/GlossoGen/GlossoGen/src/glossogen/token_pricing.py", line 62, in _find_catalog_model
    calculation = calc_price(
        usage=Usage(input_tokens=0, output_tokens=0),
        model_ref=model,
        provider_id=provider,
    )
  File "/home/runner/work/GlossoGen/GlossoGen/.venv/lib/python3.14/site-packages/genai_prices/__init__.py", line 62, in calc_price
    return data_snapshot.get_snapshot().calc(
           ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^
        usage, model_ref, provider_id, provider_api_url, genai_request_timestamp, price_context
        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    )
    ^
  File "/home/runner/work/GlossoGen/GlossoGen/.venv/lib/python3.14/site-packages/genai_prices/data_snapshot.py", line 68, in calc
    provider, model = self.find_provider_model(model_ref, None, provider_id, provider_api_url)
                      ~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/runner/work/GlossoGen/GlossoGen/.venv/lib/python3.14/site-packages/genai_prices/data_snapshot.py", line 128, in find_provider_model
    raise LookupError(f'Unable to find model with {model_ref=!r} in {provider.id}')
LookupError: Unable to find model with model_ref='scripted::floor_associate' in anthropic

genai-prices has no price for model 'scripted::robotics_engineer' under provider <Provider.ANTHROPIC: 'anthropic'>; its tokens are not costed
Traceback (most recent call last):
  File "/home/runner/work/GlossoGen/GlossoGen/src/glossogen/token_pricing.py", line 62, in _find_catalog_model
    calculation = calc_price(
        usage=Usage(input_tokens=0, output_tokens=0),
        model_ref=model,
        provider_id=provider,
    )
  File "/home/runner/work/GlossoGen/GlossoGen/.venv/lib/python3.14/site-packages/genai_prices/__init__.py", line 62, in calc_price
    return data_snapshot.get_snapshot().calc(
           ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^
        usage, model_ref, provider_id, provider_api_url, genai_request_timestamp, price_context
        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    )
    ^
  File "/home/runner/work/GlossoGen/GlossoGen/.venv/lib/python3.14/site-packages/genai_prices/data_snapshot.py", line 68, in calc
    provider, model = self.find_provider_model(model_ref, None, provider_id, provider_api_url)
                      ~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/runner/work/GlossoGen/GlossoGen/.venv/lib/python3.14/site-packages/genai_prices/data_snapshot.py", line 128, in find_provider_model
    raise LookupError(f'Unable to find model with {model_ref=!r} in {provider.id}')
LookupError: Unable to find model with model_ref='scripted::robotics_engineer' in anthropic

genai-prices has no price for model 'scripted::fleet_safety_coordinator' under provider <Provider.ANTHROPIC: 'anthropic'>; its tokens are not costed
Traceback (most recent call last):
  File "/home/runner/work/GlossoGen/GlossoGen/src/glossogen/token_pricing.py", line 62, in _find_catalog_model
    calculation = calc_price(
        usage=Usage(input_tokens=0, output_tokens=0),
        model_ref=model,
        provider_id=provider,
    )
  File "/home/runner/work/GlossoGen/GlossoGen/.venv/lib/python3.14/site-packages/genai_prices/__init__.py", line 62, in calc_price
    return data_snapshot.get_snapshot().calc(
           ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^
        usage, model_ref, provider_id, provider_api_url, genai_request_timestamp, price_context
        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    )
    ^
  File "/home/runner/work/GlossoGen/GlossoGen/.venv/lib/python3.14/site-packages/genai_prices/data_snapshot.py", line 68, in calc
    provider, model = self.find_provider_model(model_ref, None, provider_id, provider_api_url)
                      ~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/runner/work/GlossoGen/GlossoGen/.venv/lib/python3.14/site-packages/genai_prices/data_snapshot.py", line 128, in find_provider_model
    raise LookupError(f'Unable to find model with {model_ref=!r} in {provider.id}')
LookupError: Unable to find model with model_ref='scripted::fleet_safety_coordinator' in anthropic

497 events at /tmp/tmpundcaox_/smoke.jsonl

What a run is made of¶

One line per event, appended and never rewritten. That is what makes a byte offset into this file a stable address, and it is why a finished run can be replayed from any round.

In [3]:

import collections

kinds = collections.Counter(event.event_type for event in result.events)
for kind, count in kinds.most_common():
    print(f"{count:5}  {kind}")

import collections kinds = collections.Counter(event.event_type for event in result.events) for kind, count in kinds.most_common(): print(f"{count:5} {kind}")

  141  llm_response_received
   75  tool_call_invoked
   75  tool_result_received
   63  wait_registered
   63  agent_resumed
   24  injection_delivered
   12  message_sent
   12  world_event_delivered
    4  round_advanced
    4  warehouse_case_started
    4  round_ended
    4  round_result_recorded
    4  postmortem_started
    4  postmortem_ended
    3  agent_registered
    3  agent_connected
    1  simulation_started
    1  simulation_ended

Most of a run is the agents thinking rather than speaking: llm_response_received and tool_call_invoked dwarf message_sent. An agent spends most of its turns reading notifications and calling tools.

The events that carry the scenario are the other two. injection_delivered is the scenario telling an agent what happened this round; world_event_delivered is the world answering what an agent did.

Loading it the way the platform does¶

load_events parses each line into a typed event and backfills round_number on older logs, so downstream code can read that field on any event.

In [4]:

from glossogen.evaluation.log_reader import load_events

events = await load_events(log_path=result.log_path)
print(f"{len(events)} typed events")
print(f"rounds recorded: {len({e.round_number for e in events if e.round_number})}")

from glossogen.evaluation.log_reader import load_events events = await load_events(log_path=result.log_path) print(f"{len(events)} typed events") print(f"rounds recorded: {len({e.round_number for e in events if e.round_number})}")

497 typed events
rounds recorded: 4

One row per message¶

The primary channel is the one the throughput and language metrics read, and the scenario declares it rather than the analysis guessing. Ask the scenario, so this cell works unchanged on a scenario with two competing teams.

In [5]:

import pandas as pd

from glossogen.models.event import MessageSent

primary = {channel.channel_id for channel in result.scenario.get_primary_channels()}
print(f"primary channel(s): {sorted(primary)}")

rows = [
    {
        "round": event.message.round_number,
        "sender": event.message.sender_agent_id,
        "channel": event.message.channel_id,
        "chars": len(event.message.text),
        "text": event.message.text,
    }
    for event in result.of_type(event_type=MessageSent)
    if event.message.channel_id in primary
]
messages = pd.DataFrame(rows)
messages

import pandas as pd from glossogen.models.event import MessageSent primary = {channel.channel_id for channel in result.scenario.get_primary_channels()} print(f"primary channel(s): {sorted(primary)}") rows = [ { "round": event.message.round_number, "sender": event.message.sender_agent_id, "channel": event.message.channel_id, "chars": len(event.message.text), "text": event.message.text, } for event in result.of_type(event_type=MessageSent) if event.message.channel_id in primary ] messages = pd.DataFrame(rows) messages

primary channel(s): ['radio']

Out[5]:

	round	sender	channel	chars	text
0	1	floor_associate	radio	25	floor_associate reporting
1	1	robotics_engineer	radio	27	robotics_engineer reporting
2	1	fleet_safety_coordinator	radio	34	fleet_safety_coordinator reporting
3	2	fleet_safety_coordinator	radio	34	fleet_safety_coordinator reporting
4	2	floor_associate	radio	25	floor_associate reporting
5	2	robotics_engineer	radio	27	robotics_engineer reporting
6	3	fleet_safety_coordinator	radio	34	fleet_safety_coordinator reporting
7	3	floor_associate	radio	25	floor_associate reporting
8	3	robotics_engineer	radio	27	robotics_engineer reporting
9	4	fleet_safety_coordinator	radio	34	fleet_safety_coordinator reporting
10	4	floor_associate	radio	25	floor_associate reporting
11	4	robotics_engineer	radio	27	robotics_engineer reporting

One row per agent per round: the harness paces each script with a round gate, so a message's round is the script's statement rather than the scheduler's. Assert it rather than trusting the prose: if the harness stops pacing, this cell fails instead of the notebook quietly describing a different run.

In [6]:

from glossogen.testing import assert_agents_chatted_every_round

assert_agents_chatted_every_round(result=result, round_count=ROUND_COUNT)
print(f"every agent sent exactly one message in each of the {ROUND_COUNT} rounds")

from glossogen.testing import assert_agents_chatted_every_round assert_agents_chatted_every_round(result=result, round_count=ROUND_COUNT) print(f"every agent sent exactly one message in each of the {ROUND_COUNT} rounds")

every agent sent exactly one message in each of the 4 rounds

Characters per round¶

In this scenario a character is a cost: the agents spend a budget by talking. mean_chars_per_round is this number, computed here by hand.

In [7]:

import matplotlib.pyplot as plt

per_round = messages.groupby("round")["chars"].agg(["sum", "count"])
per_round.columns = ["chars", "messages"]
print(per_round)
print(f"\nmean chars/round: {per_round['chars'].mean():.1f}")

figure, axis = plt.subplots(figsize=(6, 3))
axis.bar(per_round.index, per_round["chars"], color="#0e6b5c")
axis.set_xlabel("round")
axis.set_ylabel("characters sent")
axis.set_title(f"{SCENARIO}: characters per round")
axis.set_xticks(list(per_round.index))
figure.tight_layout()

import matplotlib.pyplot as plt per_round = messages.groupby("round")["chars"].agg(["sum", "count"]) per_round.columns = ["chars", "messages"] print(per_round) print(f"\nmean chars/round: {per_round['chars'].mean():.1f}") figure, axis = plt.subplots(figsize=(6, 3)) axis.bar(per_round.index, per_round["chars"], color="#0e6b5c") axis.set_xlabel("round") axis.set_ylabel("characters sent") axis.set_title(f"{SCENARIO}: characters per round") axis.set_xticks(list(per_round.index)) figure.tight_layout()

       chars  messages
round                 
1         86         3
2         86         3
3         86         3
4         86         3

mean chars/round: 86.0

The scripted agents send the same short message every round, so this plot is flat. Against a real run it is not, and the shape of it is the finding: agents under budget pressure send less per round as they converge on a shorter code.

Round verdicts¶

judge_round_result is a required hook, and the game clock writes what it returns into round_result_recorded. That event is the only thing round_success reads, which is why the metric works on any scenario without knowing anything about robots.

In [8]:

from glossogen.models.event import RoundResultRecorded

verdicts = result.of_type(event_type=RoundResultRecorded)
for verdict in verdicts:
    print(f"round {verdict.round_number}: success={verdict.success}")

passed = sum(1 for verdict in verdicts if verdict.success)
print(f"\nround_success would report {passed}/{len(verdicts)}")

from glossogen.models.event import RoundResultRecorded verdicts = result.of_type(event_type=RoundResultRecorded) for verdict in verdicts: print(f"round {verdict.round_number}: success={verdict.success}") passed = sum(1 for verdict in verdicts if verdict.success) print(f"\nround_success would report {passed}/{len(verdicts)}")

round 1: success=False
round 2: success=False
round 3: success=False
round 4: success=False

round_success would report 0/4

Next: 02_score_a_run.ipynb runs the real metric layer over a run instead of computing numbers by hand.