Shortcuts

Source code for catalyst.metrics.functional._average_precision

from typing import List, Optional

import torch

from catalyst.metrics.functional._misc import (
    process_multilabel_components,
    process_recsys_components,
)


[docs]def binary_average_precision( outputs: torch.Tensor, targets: torch.Tensor, weights: Optional[torch.Tensor] = None ) -> torch.Tensor: """Computes the binary average precision. Args: outputs: NxK tensor that for each of the N examples indicates the probability of the example belonging to each of the K classes, according to the model. targets: binary NxK tensort that encodes which of the K classes are associated with the N-th input (eg: a row [0, 1, 0, 1] indicates that the example is associated with classes 2 and 4) weights: importance for each sample Returns: torch.Tensor: tensor of [K; ] shape, with average precision for K classes Example: .. code-block:: python import torch from catalyst import metrics metrics.binary_average_precision( outputs=torch.Tensor([0.1, 0.4, 0.35, 0.8]), targets=torch.Tensor([0, 0, 1, 1]), ) # tensor([0.8333]) """ # outputs - [bs; num_classes] with scores # targets - [bs; num_classes] with binary labels outputs, targets, weights, _ = process_multilabel_components( outputs=outputs, targets=targets, weights=weights ) if outputs.numel() == 0: return torch.zeros(1) ap = torch.zeros(targets.size(1)) # compute average precision for each class for class_i in range(targets.size(1)): # sort scores class_scores = outputs[:, class_i] class_targets = targets[:, class_i] _, sortind = torch.sort(class_scores, dim=0, descending=True) correct = class_targets[sortind] # compute true positive sums if weights is not None: class_weight = weights[sortind] weighted_correct = correct.float() * class_weight tp = weighted_correct.cumsum(0) rg = class_weight.cumsum(0) else: tp = correct.float().cumsum(0) rg = torch.arange(1, targets.size(0) + 1).float() # compute precision curve precision = tp.div(rg) # compute average precision ap[class_i] = precision[correct.bool()].sum() / max(float(correct.sum()), 1) return ap
[docs]def average_precision( outputs: torch.Tensor, targets: torch.Tensor, k: int ) -> torch.Tensor: """ Calculate the Average Precision for RecSys. The precision metric summarizes the fraction of relevant items out of the whole the recommendation list. To compute the precision at k set the threshold rank k, compute the percentage of relevant items in topK, ignoring the documents ranked lower than k. The average precision at k (AP at k) summarizes the average precision for relevant items up to the k-th one. Wikipedia entry for the Average precision <https://en.wikipedia.org/w/index.php?title=Information_retrieval& oldid=793358396#Average_precision> If a relevant document never gets retrieved, we assume the precision corresponding to that relevant doc to be zero Args: outputs (torch.Tensor): Tensor with predicted score size: [batch_size, slate_length] model outputs, logits targets (torch.Tensor): Binary tensor with ground truth. 1 means the item is relevant and 0 not relevant size: [batch_szie, slate_length] ground truth, labels k: Parameter for evaluation on top-k items Returns: ap_score (torch.Tensor): The map score for each batch. size: [batch_size, 1] Example: .. code-block:: python import torch from catalyst import metrics metrics.average_precision( outputs=torch.tensor([ [9, 8, 7, 6, 5, 4, 3, 2, 1, 0], [9, 8, 7, 6, 5, 4, 3, 2, 1, 0], ]), targets=torch.tensor([ [1.0, 0.0, 1.0, 0.0, 0.0, 1.0, 0.0, 0.0, 1.0, 1.0], [0.0, 1.0, 0.0, 0.0, 1.0, 0.0, 1.0, 0.0, 0.0, 0.0], ]), k=10, ) # tensor([0.6222, 0.4429]) """ targets_sort_by_outputs = process_recsys_components(outputs, targets)[:, :k] precisions = torch.zeros_like(targets_sort_by_outputs) for index in range(k): precisions[:, index] = torch.sum( targets_sort_by_outputs[:, : (index + 1)], dim=1 ) / float(index + 1) precisions[:, index] = torch.sum( targets_sort_by_outputs[:, : (index + 1)], dim=1 ) / float(index + 1) only_relevant_precision = precisions * targets_sort_by_outputs ap_score = only_relevant_precision.sum(dim=1) / ( (only_relevant_precision != 0).sum(dim=1) ) ap_score[torch.isnan(ap_score)] = 0 return ap_score
[docs]def mean_average_precision( outputs: torch.Tensor, targets: torch.Tensor, topk: List[int] ) -> List[torch.Tensor]: """ Calculate the mean average precision (MAP) for RecSys. The metrics calculate the mean of the AP across all batches MAP amplifies the interest in finding many relevant items for each query Args: outputs (torch.Tensor): Tensor with predicted score size: [batch_size, slate_length] model outputs, logits targets (torch.Tensor): Binary tensor with ground truth. 1 means the item is relevant and 0 not relevant size: [batch_szie, slate_length] ground truth, labels topk (List[int]): List of parameter for evaluation topK items Returns: map_at_k (Tuple[float]): The map score for every k. size: len(top_k) Example: .. code-block:: python import torch from catalyst import metrics metrics.mean_average_precision( outputs=torch.tensor([ [9, 8, 7, 6, 5, 4, 3, 2, 1, 0], [9, 8, 7, 6, 5, 4, 3, 2, 1, 0], ]), targets=torch.tensor([ [1.0, 0.0, 1.0, 0.0, 0.0, 1.0, 0.0, 0.0, 1.0, 1.0], [0.0, 1.0, 0.0, 0.0, 1.0, 0.0, 1.0, 0.0, 0.0, 0.0], ]), topk=[1, 3, 5, 10], ) # [tensor(0.5000), tensor(0.6667), tensor(0.6417), tensor(0.5325)] """ results = [] for k in topk: k = min(outputs.size(1), k) results.append(torch.mean(average_precision(outputs, targets, k))) return results
__all__ = [ "binary_average_precision", "mean_average_precision", "average_precision", ]