Pytorch中Softmax和LogSoftmax的使用詳解

class Softmax(Module):
    r"""Applies the Softmax function to an n-dimensional input Tensor
    rescaling them so that the elements of the n-dimensional output Tensor
    lie in the range [0,1] and sum to 1.
    Softmax is defined as:
    .. math::
        \text{Softmax}(x_{i}) = \frac{\exp(x_i)}{\sum_j \exp(x_j)}
    Shape:
        - Input: :math:`(*)` where `*` means, any number of additional
          dimensions
        - Output: :math:`(*)`, same shape as the input
    Returns:
        a Tensor of the same dimension and shape as the input with
        values in the range [0, 1]
    Arguments:
        dim (int): A dimension along which Softmax will be computed (so every slice
            along dim will sum to 1).
    .. note::
        This module doesn't work directly with NLLLoss,
        which expects the Log to be computed between the Softmax and itself.
        Use `LogSoftmax` instead (it's faster and has better numerical properties).
    Examples::
        >>> m = nn.Softmax(dim=1)
        >>> input = torch.randn(2, 3)
        >>> output = m(input)
    """
    __constants__ = ['dim']
 
    def __init__(self, dim=None):
        super(Softmax, self).__init__()
        self.dim = dim
 
    def __setstate__(self, state):
        self.__dict__.update(state)
        if not hasattr(self, 'dim'):
            self.dim = None
 
    def forward(self, input):
        return F.softmax(input, self.dim, _stacklevel=5)
 
    def extra_repr(self):
        return 'dim={dim}'.format(dim=self.dim)

class LogSoftmax(Module):
    r"""Applies the :math:`\log(\text{Softmax}(x))` function to an n-dimensional
    input Tensor. The LogSoftmax formulation can be simplified as:
    .. math::
        \text{LogSoftmax}(x_{i}) = \log\left(\frac{\exp(x_i) }{ \sum_j \exp(x_j)} \right)
    Shape:
        - Input: :math:`(*)` where `*` means, any number of additional
          dimensions
        - Output: :math:`(*)`, same shape as the input
    Arguments:
        dim (int): A dimension along which LogSoftmax will be computed.
    Returns:
        a Tensor of the same dimension and shape as the input with
        values in the range [-inf, 0)
    Examples::
        >>> m = nn.LogSoftmax()
        >>> input = torch.randn(2, 3)
        >>> output = m(input)
    """
    __constants__ = ['dim']
 
    def __init__(self, dim=None):
        super(LogSoftmax, self).__init__()
        self.dim = dim
 
    def __setstate__(self, state):
        self.__dict__.update(state)
        if not hasattr(self, 'dim'):
            self.dim = None
 
    def forward(self, input):
        return F.log_softmax(input, self.dim, _stacklevel=5)

import torch
import torch.nn as nn
import numpy as np
 
batch_size = 4
class_num = 6
inputs = torch.randn(batch_size, class_num)
for i in range(batch_size):
    for j in range(class_num):
        inputs[i][j] = (i + 1) * (j + 1)
 
print("inputs:", inputs)

Softmax = nn.Softmax(dim=1)
probs = Softmax(inputs)
print("probs:\n", probs)

LogSoftmax = nn.LogSoftmax(dim=1)
log_probs = LogSoftmax(inputs)
print("log_probs:\n", log_probs)

# probs_sum in dim=1
probs_sum = [0 for i in range(batch_size)]
 
for i in range(batch_size):
    for j in range(class_num):
        probs_sum[i] += probs[i][j]
    print(i, "row probs sum:", probs_sum[i])

# to numpy
np_probs = probs.data.numpy()
print("numpy probs:\n", np_probs)
 
# np.log()
log_np_probs = np.log(np_probs)
print("log numpy probs:\n", log_np_probs)

import torch
import torch.nn as nn
import numpy as np
 
batch_size = 4
class_num = 6
inputs = torch.randn(batch_size, class_num)
for i in range(batch_size):
    for j in range(class_num):
        inputs[i][j] = (i + 1) * (j + 1)
 
print("inputs:", inputs)
Softmax = nn.Softmax(dim=1)
probs = Softmax(inputs)
print("probs:\n", probs)
 
LogSoftmax = nn.LogSoftmax(dim=1)
log_probs = LogSoftmax(inputs)
print("log_probs:\n", log_probs)
 
# probs_sum in dim=1
probs_sum = [0 for i in range(batch_size)]
 
for i in range(batch_size):
    for j in range(class_num):
        probs_sum[i] += probs[i][j]
    print(i, "row probs sum:", probs_sum[i])
 
# to numpy
np_probs = probs.data.numpy()
print("numpy probs:\n", np_probs)
 
# np.log()
log_np_probs = np.log(np_probs)
print("log numpy probs:\n", log_np_probs)

import torch
import numpy as np
input = torch.autograd.Variable(torch.rand(1, 3))

print(input)
print('softmax={}'.format(torch.nn.functional.softmax(input, dim=1)))
print('logsoftmax={}'.format(np.log(torch.nn.functional.softmax(input, dim=1))))