Jev and System One models

    /// <summary>
    /// POST one System One request; return (response, seconds). This is the whole API:
    ///
    ///     POST {base}/v1/systemone
    ///     Authorization: Bearer {TYPESAFE_API_KEY}
    ///     {"model": "jev-latest", "state": "...", "questions": {...}}
    ///
    /// There is no official C# SDK, and none is needed: HttpClient is enough.
    /// A key is only ever sent to TypeSafe or to this machine: anything else is refused,
    /// so a typo in TYPESAFE_BASE_URL cannot hand the key (or the calls) to a stranger.
    /// </summary>
    public async Task<(Dict Response, double Seconds)> PostAsync(string baseUrl, JsonObject body, string apiKey)
    {
        var root = baseUrl.TrimEnd('/');
        // The destination rule comes first, before a request object even exists.
        if (!(root == SystemOne.TypeSafeUrl || SystemOne.IsLoopback(baseUrl)))
            throw new ArgumentException($"refusing {baseUrl}: only {SystemOne.TypeSafeUrl} or a loopback address");
        using var req = new HttpRequestMessage(HttpMethod.Post, root + SystemOne.Path)
        {
            Content = new StringContent(body.ToJsonString(), Encoding.UTF8, "application/json"),
        };
        // The local adapter needs no key; TypeSafe does.
        if (apiKey.Length > 0) req.Headers.Authorization = new AuthenticationHeaderValue("Bearer", apiKey);
        var started = Stopwatch.StartNew();
        string text;
        try
        {
            using var resp = await Http.SendAsync(req);
            text = await resp.Content.ReadAsStringAsync();
            if (!resp.IsSuccessStatusCode)
            {
                // 401 bad key, 422 a malformed question, 429 rate limit: show what the server said.
                var detail = text.Length > 300 ? text[..300] : text;
                throw new InvalidOperationException($"HTTP {(int)resp.StatusCode} from {baseUrl}: {detail}");
            }
        }
        catch (Exception err) when (err is HttpRequestException or TaskCanceledException)
        {
            throw new InvalidOperationException($"cannot reach {baseUrl}: {err.Message}");
        }
        if (Py.Loads(text) is not Dict response) throw new InvalidOperationException($"{baseUrl}: response is not a JSON object");
        return (response, started.Elapsed.TotalSeconds);
    }

    /// <summary>
    /// Turn a System One response into {question: Answer}, plus what was missing or malformed.
    ///
    /// The three answer types arrive in three shapes. This gives them one: Pick is always a
    /// label from Options() ("yes"/"no" for a noul, the level name for a score) and Probs maps
    /// every label to its probability. The policy and the scorecard only see this shape, so
    /// they cannot tell TypeSafe's Jev from a local adapter or the keyword rules.
    /// With a real System One model the list of problems stays empty.
    /// </summary>
    public static (Dictionary<string, Answer> Answers, List<string> Problems) ReadAnswers(Dict questions, object? response)
    {
        var answers = new Dictionary<string, Answer>();
        var problems = new List<string>();
        // A reply that is not an object (or has no "answers" object) answers nothing.
        var got = response is Dict r0 && r0.Get("answers") is Dict g ? g : new Dict();
        foreach (var (name, value) in questions.Items)
        {
            var q = (Dict)value!;
            var kind = (string)q["type"]!;
            var labels = Options(q);
            // The answer must be of the type that was asked: a noul for a noul, and so on.
            if (got.Get(name) is not Dict a || a.Get("type") as string != kind)
            {
                problems.Add($"{name}: no {kind} answer");
                continue;
            }
            string pick;
            List<KeyValuePair<string, double>> probs;
            double confidence;
            if (kind == "noul")
            {
                // noul: one number, P(yes). Split it into yes/no so it looks like the other types.
                var p = a.Get("noul");
                if (!Py.IsProbability(p))
                {
                    problems.Add($"{name}: noul must be a number in [0, 1]");
                    continue;
                }
                double x = Py.Num(p);
                probs = new() { new("yes", x), new("no", 1.0 - x) };
                pick = x >= 0.5 ? "yes" : "no";
                confidence = Math.Max(x, 1 - x);
            }
            else
            {
                // choice and score: a probability per option.
                var raw = a.Get("probabilities") is Dict r ? r : new Dict();
                if (kind == "score")
                {
                    // A score's keys are level numbers ("0", "1", ...): translate them to level names.
                    var mapped = new Dict();
                    foreach (var (k, v) in raw.Items)
                    {
                        if (Regex.IsMatch(k, @"\A[0-9]+\z") && k.TrimStart('0').Length <= 3 && int.Parse(k, Py.Inv) < labels.Count)
                            mapped[labels[int.Parse(k, Py.Inv)]] = v;
                    }
                    raw = mapped;
                }
                // Valid means: exactly the question's options, each a real probability, summing to 1.
                bool numeric = raw.Items.All(kv => Py.IsProbability(kv.Value));
                bool sameSet = raw.Count == labels.Distinct().Count() && raw.Keys.All(labels.Contains);
                if (!sameSet || !numeric || Math.Abs(Py.FSum(raw.Items.Select(kv => kv.Value)) - 1.0) > 0.01)
                {
                    problems.Add($"{name}: probabilities must cover {Py.Repr(labels)} and sum to 1");
                    continue;
                }
                probs = labels.Select(l => new KeyValuePair<string, double>(l, Py.Num(raw[l]))).ToList();
                object? chosen;
                if (kind == "choice")
                {
                    chosen = a.Get("choice"); // the server names its pick
                }
                else
                {
                    var best = probs[0]; // a score has no "choice": take the most likely level
                    foreach (var kv in probs) if (kv.Value > best.Value) best = kv;
                    chosen = best.Key;
                }
                if (chosen is not string c || !labels.Contains(c))
                {
                    problems.Add($"{name}: {Py.Repr(chosen)} is not an option");
                    continue;
                }
                pick = c;
                // No usable confidence in the reply: fall back to the top probability.
                confidence = Py.IsProbability(a.Get("confidence")) ? Py.Num(a["confidence"]) : probs.Max(kv => kv.Value);
            }
            answers[name] = new Answer(pick, probs, confidence);
        }
        return (answers, problems);
    }

    /// <summary>
    /// Ask a System One server about one call and print every probability and the action.
    ///
    /// With TYPESAFE_API_KEY set this is the real Jev, called with HttpClient: TypeSafe
    /// publishes SDKs for Python and JavaScript only, and the API is one POST, so C# needs
    /// none. With TYPESAFE_BASE_URL=http://127.0.0.1:8765 the same code calls the local
    /// adapter instead (start it with ./run -l 10 serve).
    /// </summary>
    static async Task<int> Ask(string text, string dataset, TextWriter output)
    {
        var (questions, _) = LoadDataset(dataset);
        var model = Environment.GetEnvironmentVariable("TYPESAFE_DEFAULT_MODEL") is { Length: > 0 } m ? m : "jev-latest";
        var url = Environment.GetEnvironmentVariable("TYPESAFE_BASE_URL") ?? SystemOne.TypeSafeUrl;
        var key = Environment.GetEnvironmentVariable("TYPESAFE_API_KEY") ?? "";
        // TypeSafe needs a key; a server on this machine does not.
        if (key.Length == 0 && !SystemOne.IsLoopback(url))
            throw new Exit(1, "TYPESAFE_API_KEY is not set (get one at https://console.typesafe.ai)");
        // One request carries the state and every question: {model, state, questions}.
        var body = SystemOne.BuildRequest(text, questions, model);
        Dict response;
        double seconds;
        try
        {
            (response, seconds) = await new SystemOneClient().PostAsync(url, (JsonObject)SystemOne.ToJsonNode(body)!, key);
        }
        catch (Exception err) when (err is ArgumentException or InvalidOperationException or JsonDecodeError or UriFormatException)
        {
            throw new Exit(1, err.Message);
        }
        var run = ToRun(questions, new Dict { ["response"] = response, ["seconds"] = seconds, ["calls"] = 1L });
        // never call a stand-in on this machine "Jev"
        var label = SystemOne.IsLoopback(url) ? "local System One server (simulated)" : LabelFor("typesafe", null, model);
        PrintRun(output, label, questions, run, dataset);
        return 0;
    }

    /// <summary>
    /// The chat messages for one question: (system, user). The option texts are the criteria.
    ///
    /// The transcript is fenced between &lt;&lt;&lt; and &gt;&gt;&gt; and the system message says to treat it
    /// as data. Callers can and do read instructions to "the AI" down the phone (Lesson 4);
    /// the fence is the first line of defence, the policy's thresholds are the second.
    /// </summary>
    public static (string System, string User) PromptFor(string state, Dict question)
    {
        var labels = SystemOne.Options(question);
        var criteria = question.Get("criteria");
        List<string> texts;
        switch ((string)question["type"]!)
        {
            case "noul": // yes/no: say what counts as each, if the question gave criteria
                var c = criteria as Dict;
                texts = new() { $"yes - {c?.Get("true") ?? "yes"}", $"no - {c?.Get("false") ?? "no"}" };
                break;
            case "choice": // label - what the label means
                texts = ((Dict)criteria!).Items
                    .Select(kv => kv.Value is string { Length: > 0 } desc ? $"{kv.Key} - {desc}" : kv.Key).ToList();
                break;
            default: // a score's levels are already words: none, minor, moderate ...
                texts = labels;
                break;
        }
        var lines = string.Join("\n", texts.Select((t, i) => $"{Letters[i]}) {t}"));
        var user = $"Input:\n<<<\n{state}\n>>>\n\nQuestion: {question["instructions"]}\n{lines}\n\nAnswer with one letter.";
        return (SystemPrompt, user);
    }

    /// <summary>
    /// Probability per option, from the candidates Ollama reports for the first token.
    ///
    /// Each candidate is (token, logprob): what the model considered writing, and how likely
    /// it was as a natural logarithm, so Math.Exp turns it back into a probability.
    /// "A", " A" and "a" are the same answer and their probabilities add up. A letter that
    /// is not among the candidates gets 0. If no option letter appears at all, every option
    /// gets an equal share: the honest reading of "the model said something else".
    /// </summary>
    public static double[] LetterProbabilities(IEnumerable<(string Token, double Logprob)> candidates, int n)
    {
        var mass = new double[n];
        foreach (var (token, logprob) in candidates)
        {
            var t = token.Trim().ToUpperInvariant();
            int index = t.Length == 1 ? Letters[..n].IndexOf(t[0]) : -1;
            if (index >= 0) mass[index] += Math.Exp(logprob);
        }
        double total = mass.Sum();
        if (total <= 0) return Enumerable.Repeat(1.0 / n, n).ToArray();
        return mass.Select(m => m / total).ToArray(); // renormalise over the letters that are options
    }

    /// <summary>
    /// One action per insurance call, from the answers to the six questions.
    ///
    /// The model never chooses the action. It says P(fraud signals) = 0.83; this method
    /// decides that 0.83 is enough. Moving a threshold changes what the business does without
    /// retraining or re-prompting anything. The order of the rules is the priority.
    /// </summary>
    static string Decide(Dictionary<string, Answer> answers, double siu = Siu)
    {
        answers.TryGetValue("intent", out var intent);
        // 1. Someone needs help now: that comes first, whatever else the call is about.
        if (Prob(answers, "emergency", "yes") >= Emergency) return "dispatch emergency help";
        // 2. No usable answer, or the model is unsure what the caller wants: a person, never a guess.
        if (!answers.ContainsKey("emergency") || intent is null || intent.Confidence < Confident) return "human agent";
        // 3. Complaints and cancellations are conversations, not forms.
        if (intent.Pick is "complaint" or "cancel_policy") return "human agent";
        // 4. Questions about a claim or about cover can be answered without a claim handler.
        if (intent.Pick is "claim_status" or "coverage_question") return "self-service answer";
        // 5. What is left is a new claim. A missing fraud answer counts as suspicious (default 1.0).
        double pFraud = Prob(answers, "fraud_signals", "yes", 1.0);
        if (pFraud >= siu) return "special investigations";
        // 6. Pay without a human only when the loss is small, nobody needs to inspect it,
        //    and the claim is clean enough: P(no fraud) >= FastTrack.
        bool small = Prob(answers, "severity", "none") + Prob(answers, "severity", "minor") > 0.5;
        if (small && Prob(answers, "needs_adjuster", "yes", 1.0) < 0.5 && 1 - pFraud >= FastTrack) return "fast-track payout";
        // 7. Everything else is looked at by an adjuster.
        return "assign adjuster";
    }

    /// <summary>
    /// Read what a chat model wrote when asked for JSON, the way most code does it today.
    ///
    /// Nothing guarantees the text is JSON, or that a value is one of the options. A field
    /// the model got wrong, or left out, is a type error: it is reported and that question
    /// stays unanswered. Every valid answer gets probability 1.0 - a written answer carries
    /// no probability to put a threshold on.
    /// </summary>
    static (Dict Response, List<string> Errors) ParseLlmJson(Dict questions, string text)
    {
        var errors = new List<string>();
        // Models wrap JSON in prose or code fences: take the outermost {...} and try to parse it.
        var match = System.Text.RegularExpressions.Regex.Match(text, @"\{.*\}", System.Text.RegularExpressions.RegexOptions.Singleline);
        object? written;
        try
        {
            written = match.Success ? Py.Loads(match.Value) : null;
        }
        catch (JsonDecodeError)
        {
            written = null;
        }
        if (written is not Dict w)
            return (new Dict { ["model"] = "llm-json", ["answers"] = new Dict(), ["usage"] = new Dict() }, new() { "not a JSON object" });
        var answers = new Dict();
        foreach (var (name, qv) in questions.Items)
        {
            var q = (Dict)qv!;
            var value = w.Get(name);
            if (value is bool b) value = b ? "yes" : "no"; // {"emergency": true} is a fair reading
            // "very bad" for a question whose levels are none..catastrophic is not an answer.
            if (value is not string s || !SystemOne.Options(q).Contains(Py.Lower(Py.Strip(s))))
            {
                errors.Add($"{name}={Py.Repr(value)}");
                continue;
            }
            answers[name] = Certain(q, Py.Lower(Py.Strip(s))); // all the weight on its pick
        }
        return (new Dict { ["model"] = "llm-json", ["answers"] = answers, ["usage"] = new Dict() }, errors);
    }

    /// <summary>
    /// The Brier score of one answer: the squared distance between the probabilities and the truth.
    ///
    /// 0 is perfect. Certain and wrong costs 2 (1 on the wrong option, 1 on the right one).
    /// An unanswered question is scored as "every option equally likely". Accuracy only asks
    /// whether the top answer was right; this also asks whether the engine knew how sure to be,
    /// which is what a threshold in the policy relies on.
    /// </summary>
    static double Brier(Dict q, Answer? answer, string truth)
    {
        var labels = SystemOne.Options(q);
        return Py.FSum(labels.Select(label =>
        {
            double p = answer is null ? 1.0 / labels.Count : answer.Prob(label, double.NaN);
            double d = p - (label == truth ? 1.0 : 0.0); // the truth is 1 on the right label, 0 elsewhere
            return (object?)(d * d);
        }));
    }

$ ./run -l 10 --lang csharp demo

$ ./run -l 10 --lang csharp ask "Caller: I hit a deer. The car still drives."

$ ./run -l 10 --lang csharp install-sdk   # nothing to install: this only builds

$ ./run -l 10 --lang csharp sdk "Caller: someone dented my parked car, I have a photo"

$ ./run -l 10 --lang csharp test