Shortcuts

Source code for catalyst.metrics._map

from typing import Iterable

from catalyst.metrics._topk_metric import TopKMetric
from catalyst.metrics.functional._average_precision import mean_average_precision


[docs]class MAPMetric(TopKMetric): """ Calculates the Mean Average Precision (MAP) for RecSys. The precision metric summarizes the fraction of relevant items out of the whole the recommendation list. Computes mean value of MAP and it's approximate std value Args: topk_args: list of `topk` for map@topk computing compute_on_call: if True, computes and returns metric value during metric call prefix: metric prefix suffix: metric suffix Examples: .. code-block:: python import torch from catalyst import metrics outputs = torch.tensor([ [9, 8, 7, 6, 5, 4, 3, 2, 1, 0], [9, 8, 7, 6, 5, 4, 3, 2, 1, 0], ]) targets = torch.tensor([ [1.0, 0.0, 1.0, 0.0, 0.0, 1.0, 0.0, 0.0, 1.0, 1.0], [0.0, 1.0, 0.0, 0.0, 1.0, 0.0, 1.0, 0.0, 0.0, 0.0], ]) metric = metrics.MAPMetric(topk_args=[1, 3, 5, 10]) metric.reset() metric.update(outputs, targets) metric.compute() # ( # # mean for @01, @03, @05, @10 # (0.5, 0.6666666865348816, 0.6416666507720947, 0.5325397253036499), # # std for @01, @03, @05, @10 # (0.0, 0.0, 0.0, 0.0) # ) metric.compute_key_value() # { # 'map01': 0.5, # 'map01/std': 0.0, # 'map03': 0.6666666865348816, # 'map03/std': 0.0, # 'map05': 0.6416666507720947, # 'map05/std': 0.0, # 'map10': 0.5325397253036499, # 'map10/std': 0.0 # } metric.reset() metric(outputs, targets) # ( # # mean for @01, @03, @05, @10 # (0.5, 0.6666666865348816, 0.6416666507720947, 0.5325397253036499), # # std for @01, @03, @05, @10 # (0.0, 0.0, 0.0, 0.0) # ) .. code-block:: python import torch from torch.utils.data import DataLoader, TensorDataset from catalyst import dl # sample data num_users, num_features, num_items = int(1e4), int(1e1), 10 X = torch.rand(num_users, num_features) y = (torch.rand(num_users, num_items) > 0.5).to(torch.float32) # pytorch loaders dataset = TensorDataset(X, y) loader = DataLoader(dataset, batch_size=32, num_workers=1) loaders = {"train": loader, "valid": loader} # model, criterion, optimizer, scheduler model = torch.nn.Linear(num_features, num_items) criterion = torch.nn.BCEWithLogitsLoss() optimizer = torch.optim.Adam(model.parameters()) scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer, [2]) # model training runner = dl.SupervisedRunner( input_key="features", output_key="logits", target_key="targets", loss_key="loss" ) runner.train( model=model, criterion=criterion, optimizer=optimizer, scheduler=scheduler, loaders=loaders, num_epochs=3, verbose=True, callbacks=[ dl.BatchTransformCallback( transform=torch.sigmoid, scope="on_batch_end", input_key="logits", output_key="scores" ), dl.CriterionCallback( input_key="logits", target_key="targets", metric_key="loss" ), dl.AUCCallback(input_key="scores", target_key="targets"), dl.HitrateCallback( input_key="scores", target_key="targets", topk_args=(1, 3, 5) ), dl.MRRCallback(input_key="scores", target_key="targets", topk_args=(1, 3, 5)), dl.MAPCallback(input_key="scores", target_key="targets", topk_args=(1, 3, 5)), dl.NDCGCallback(input_key="scores", target_key="targets", topk_args=(1, 3, 5)), dl.OptimizerCallback(metric_key="loss"), dl.SchedulerCallback(), dl.CheckpointCallback( logdir="./logs", loader_key="valid", metric_key="loss", minimize=True ), ] ) .. note:: Metric names depending on input parameters: - ``topk_args = (1,) or None`` ---> ``"map01"`` - ``topk_args = (1, 3)`` ---> ``"map01"``, ``"map03"`` - ``topk_args = (1, 3, 5)`` ---> ``"map01"``, ``"map03"``, ``"map05"`` You can find them in ``runner.batch_metrics``, ``runner.loader_metrics`` or ``runner.epoch_metrics``. .. note:: Please follow the `minimal examples`_ sections for more use cases. .. _`minimal examples`: https://github.com/catalyst-team/catalyst#minimal-examples """ def __init__( self, topk_args: Iterable[int] = None, compute_on_call: bool = True, prefix: str = None, suffix: str = None, ): """Init MAPMetric""" super().__init__( metric_name="map", metric_function=mean_average_precision, topk_args=topk_args, compute_on_call=compute_on_call, prefix=prefix, suffix=suffix, )
__all__ = ["MAPMetric"]