Evaluation¶

def evaluate(  # noqa: PLR0913
    model: ModelMeta | MTEBModels | SentenceTransformerModels,
    tasks: AbsTask | Iterable[AbsTask],
    *,
    co2_tracker: bool | None = None,
    raise_error: bool = True,
    encode_kwargs: EncodeKwargs | None = None,
    cache: ResultCache | None = _DEFAULT_CACHE,
    overwrite_strategy: str | OverwriteStrategy = "only-missing",
    prediction_folder: Path | str | None = None,
    show_progress_bar: bool = True,
    public_only: bool | None = None,
    num_proc: int | None = None,
    timer: TimingStack | None = None,
) -> ModelResult:
    """This function runs a model on a given task and returns the results.

    Args:
        model: The model to use for encoding.
        tasks: A task to run.
        co2_tracker: If True, track the CO₂ emissions of the evaluation, required codecarbon to be installed, which can be installed using
            `pip install mteb[codecarbon]`. If none is passed co2 tracking will only be run if codecarbon is installed.
        encode_kwargs: Additional keyword arguments passed to the models `encode` and `load_data` methods;
        raise_error: If True, raise an error if the task fails. If False, return an empty list.
        cache: The cache to use for loading the results. If None, then no cache will be used. The default cache saved the cache in the
            `~/.cache/mteb` directory. It can be overridden by setting the `MTEB_CACHE` environment variable to a different directory or by directly
            passing a `ResultCache` object.
        overwrite_strategy: The strategy to use for run a task and overwrite the results. Can be:
            - "always": Always run the task, overwriting the results
            - "never": Run the task only if the results are not found in the cache. If the results are found, it will not run the task.
            - "only-missing": Only rerun the missing splits of a task. It will not rerun the splits if the dataset revision or mteb version has
                changed.
            - "only-cache": Only load the results from the cache folder and do not run the task. Useful if you just want to load the results from the
                cache.
        prediction_folder: Optional folder in which to save model predictions for the task. Predictions of the tasks will be saved in `prediction_folder/{task_name}_predictions.json`
        show_progress_bar: Whether to show a progress bar when running the evaluation. Default is True. Setting this to False will also set the
            `encode_kwargs['show_progress_bar']` to False if encode_kwargs is unspecified.
        public_only: Run only public tasks. If None, it will attempt to run the private task.
        num_proc: Number of processes to use during data loading and transformation. Defaults to None (single process).
        timer: A context manager that tracks the timing of evaluation phases.

    Returns:
        The results of the evaluation.

    Examples:
        >>> import mteb
        >>> model_meta = mteb.get_model_meta("sentence-transformers/all-MiniLM-L6-v2")
        >>> task = mteb.get_task("STS12")
        >>> result = mteb.evaluate(ModelMeta, task)
        >>>
        >>> # with CO2 tracking
        >>> result = mteb.evaluate(model_meta, task, co2_tracker=True)
        >>>
        >>> # with encode kwargs
        >>> result = mteb.evaluate(model_meta, task, encode_kwargs={"batch_size": 16})
        >>>
        >>> # with online cache
        >>> cache = mteb.ResultCache(cache_path="~/.cache/mteb")
        >>>
        >>> cache.download_from_remote()
        >>> result = mteb.evaluate(model_meta, task, cache=cache)
    """
    if isinstance(prediction_folder, str):
        prediction_folder = Path(prediction_folder)

    if encode_kwargs is None:
        encode_kwargs = (
            {"show_progress_bar": False} if show_progress_bar is False else {}
        )
    if "batch_size" not in encode_kwargs:
        encode_kwargs["batch_size"] = 32
        logger.info(
            "No batch size defined in encode_kwargs. Setting `encode_kwargs['batch_size'] = 32`. Explicitly set the batch size to silence this message."
        )

    if num_proc is None and _has_non_text_task(tasks):
        logger.info(
            "`num_proc` is not set, so data loading runs in a single process. Consider setting `num_proc` (e.g. `mteb.evaluate(..., num_proc=4)`) to speed up data loading. "
            "Explicitly set `num_proc` to silence this message."
        )

    # encoding runs through torch dataloaders, so check before doing any work
    _requires_full_installation("torch", "Evaluating a model")

    model, meta, model_name, model_revision = _sanitize_model(model)
    meta = _apply_precision_to_meta(meta, encode_kwargs)
    _check_model_modalities(meta, tasks)
    overwrite_strategy = OverwriteStrategy.from_str(overwrite_strategy)

    return _evaluate_resolved(
        model,
        meta,
        model_name,
        model_revision,
        tasks,
        co2_tracker=co2_tracker,
        raise_error=raise_error,
        encode_kwargs=encode_kwargs,
        cache=cache,
        overwrite_strategy=overwrite_strategy,
        prediction_folder=prediction_folder,
        show_progress_bar=show_progress_bar,
        public_only=public_only,
        num_proc=num_proc,
        timer=timer,
    )