pip install ai-evaluation

undefined

npm install @traceai/fi-core
npm install @traceai/openai

git clone https://github.com/future-agi/future-agi.git
cd future-agi
cp futureagi/.env.example futureagi/.env

undefined

undefined

---

undefined

undefined

import os
from fi_instrumentation import register

---

from fi_instrumentation import register
from traceai_openai import OpenAIInstrumentor
from openai import OpenAI

undefined

from fi_instrumentation import register
from traceai_langchain import LangChainInstrumentor
from langchain_openai import ChatOpenAI
from langchain.schema import HumanMessage

register(project_name="langchain-agent")
LangChainInstrumentor().instrument()

llm = ChatOpenAI(model="gpt-4o")
response = llm.invoke([HumanMessage(content="Explain quantum computing")])
print(response.content)

from fi_instrumentation import register
from traceai_llamaindex import LlamaIndexInstrumentor
from llama_index.core import VectorStoreIndex, SimpleDirectoryReader

register(project_name="llamaindex-rag")
LlamaIndexInstrumentor().instrument()

documents = SimpleDirectoryReader("./data").load_data()
index = VectorStoreIndex.from_documents(documents)
query_engine = index.as_query_engine()
response = query_engine.query("What did the author do growing up?")
print(response)

from fi_instrumentation import register
from opentelemetry import trace

register(project_name="custom-agent")
tracer = trace.get_tracer(__name__)

def process_user_query(query: str) -> str:
    with tracer.start_as_current_span("process_query") as span:
        span.set_attribute("query", query)
        span.set_attribute("model", "gpt-4o")
        
        # Your LLM call here
        result = call_llm(query)
        
        span.set_attribute("response_length", len(result))
        return result

import { register } from "@traceai/fi-core";
import { OpenAIInstrumentation } from "@traceai/openai";
import OpenAI from "openai";

// Register at app startup
register({
  projectName: "my-ts-agent",
  // fiApiKey: process.env.FI_API_KEY,  // auto-read from env
  // fiBaseUrl: process.env.FI_BASE_URL,
});
new OpenAIInstrumentation().instrument();

const client = new OpenAI(); // OPENAI_API_KEY from env

const response = await client.chat.completions.create({
  model: "gpt-4o",
  messages: [{ role: "user", content: "Hello, world!" }],
});
console.log(response.choices[0].message.content);

from fi.evals import evaluate
from fi.evals.metrics import Hallucination, Groundedness, ResponseRelevance

undefined

from fi.evals import evaluate
from fi.evals.metrics import (
    Hallucination,
    Groundedness,
    ResponseRelevance,
    ToneCheck,
    PIICheck,
    ToolCallAccuracy,
)

result = evaluate(
    metrics=[
        Hallucination(),
        Groundedness(),
        ResponseRelevance(),
        ToneCheck(expected_tone="professional"),
        PIICheck(),
    ],
    query="Explain the benefits of exercise.",
    response="Exercise reduces the risk of heart disease and improves mental health.",
    context="Regular physical activity has numerous health benefits including cardiovascular health improvement.",
)

for metric_name, metric_result in result.items():
    print(f"{metric_name}: {metric_result['score']} — {metric_result.get('label', '')}")

from fi.evals import batch_evaluate
from fi.evals.metrics import Hallucination, Groundedness

dataset = [
    {
        "query": "What year was Python created?",
        "response": "Python was created in 1991.",
        "context": "Python is a programming language created by Guido van Rossum. It was first released in 1991.",
    },
    {
        "query": "Who wrote Hamlet?",
        "response": "Hamlet was written by Charles Dickens.",
        "context": "Hamlet is a tragedy written by William Shakespeare, believed to have been written around 1600.",
    },
]

results = batch_evaluate(
    metrics=[Hallucination(), Groundedness()],
    data=dataset,
    project_name="batch-eval-demo",
)

for i, result in enumerate(results):
    print(f"Item {i}: {result}")

from fi.evals import evaluate
from fi.evals.metrics import CustomRubric

result = evaluate(
    metrics=[
        CustomRubric(
            criteria="Does the response correctly answer the question without making up facts?",
            rubric={
                1: "Response is completely correct and factual",
                0: "Response contains fabricated or incorrect information",
            },
        )
    ],
    query="What is 2 + 2?",
    response="2 + 2 equals 4.",
)
print(result)

from fi.evals import evaluate
from fi.evals.metrics import ToolCallAccuracy

result = evaluate(
    metrics=[ToolCallAccuracy()],
    query="What's the weather in New York?",
    response="The weather in New York is 72°F and sunny.",
    expected_tool_calls=[
        {"name": "get_weather", "arguments": {"location": "New York"}}
    ],
    actual_tool_calls=[
        {"name": "get_weather", "arguments": {"location": "New York, NY"}}
    ],
)
print(result)

from fi.simulate import Simulation, Persona, Scenario

---

from fi.protect import Guard, Scanner
from fi.protect.scanners import (
    PIIScanner,
    JailbreakScanner,
    PromptInjectionScanner,
    ToxicityScanner,
)

# Scan output before returning to user
output_result = guard.scan_output(response_text)
safe_response = output_result.sanitized_text
print(safe_response)

undefined

from openai import OpenAI

---

from openai import OpenAI
import os

---

from fi.optimize import PromptOptimizer, OptimizationAlgorithm

optimizer = PromptOptimizer(
    project_name="my-agent",
    algorithm=OptimizationAlgorithm.GEPA,  # or PROMPT_WIZARD, PROTEGI, BAYESIAN, META_PROMPT
)

---

import os
from fi_instrumentation import register
from traceai_openai import OpenAIInstrumentor
from fi.evals import evaluate
from fi.evals.metrics import Hallucination, ResponseRelevance
from openai import OpenAI

# Evaluate the response
eval_result = evaluate(
    metrics=[Hallucination(), ResponseRelevance()],
    query=query,
    response=response_text,
    context=context,
)

return {
    "response": response_text,
    "evaluation": eval_result,
    "safe_to_return": eval_result.get("hallucination", {}).get("score", 1.0) < 0.5,
}

undefined

from fi_instrumentation import register
from traceai_langchain import LangChainInstrumentor
from langchain_openai import ChatOpenAI, OpenAIEmbeddings
from langchain_community.vectorstores import FAISS
from langchain.chains import RetrievalQA
from langchain.schema import Document

register(project_name="rag-pipeline")
LangChainInstrumentor().instrument()

undefined

from fi_instrumentation import register
from traceai_crewai import CrewAIInstrumentor
from crewai import Agent, Task, Crew

register(project_name="crewai-demo")
CrewAIInstrumentor().instrument()

researcher = Agent(
    role="Research Analyst",
    goal="Research and summarize topics accurately",
    backstory="Expert at gathering and synthesizing information",
    verbose=True,
)

writer = Agent(
    role="Content Writer",
    goal="Write clear, engaging content based on research",
    backstory="Skilled at turning research into compelling narratives",
    verbose=True,
)

research_task = Task(
    description="Research the history of artificial intelligence",
    agent=researcher,
    expected_output="A comprehensive summary of AI history",
)

writing_task = Task(
    description="Write a blog post based on the research",
    agent=writer,
    expected_output="A 500-word blog post about AI history",
    context=[research_task],
)

crew = Crew(agents=[researcher, writer], tasks=[research_task, writing_task])
result = crew.kickoff()

undefined

import json
from fi.evals import batch_evaluate
from fi.evals.metrics import Hallucination, Groundedness, ResponseRelevance
from fi.datasets import Dataset

---

undefined

undefined

undefined

undefined

from fi.evals import evaluate
from fi.evals.metrics import Hallucination

undefined

undefined

undefined

undefined

---