123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899

# -*- coding: utf8 -*-#import math

import torchfrom torch.functional import F

print('sigmoid'.center(60, '-'))

def sigmoid(x):    return 1 / (1 + math.exp(-x))

print(torch.sigmoid(torch.tensor([1., 2.])))print([sigmoid(1.), sigmoid(2.)])

print('softmax'.center(60, '-'))

def softmax(xs):    z_exp = [math.exp(x) for x in xs]    sum_z_exp = sum(z_exp)    return [_z_exp / sum_z_exp for _z_exp in z_exp]

print(torch.softmax(torch.tensor([1., 2., 7.]), dim=-1))print(softmax([1., 2., 7.]))

print('tanh'.center(60, '-'))

def tanh(x):    return (math.exp(x) - math.exp(-x)) / (math.exp(x) + math.exp(-x))

print(torch.tanh(torch.tensor([1., 2.])))print([tanh(1.), tanh(2.)])

print('cross_entropy'.center(60, '-'))import torch

def my_cross_entropy(input, target, reduction="mean"):    # input.shape: torch.size([-1, class])    # target.shape: torch.size([-1])    # reduction = "mean" or "sum"    # input是模型输出的结果，与target求loss    # target的长度和input第一维的长度一致    # target的元素值为目标class    # reduction默认为mean，即对loss求均值    # 还有另一种为sum，对loss求和

    # 这里对input所有元素求exp    exp = torch.exp(input)    # 根据target的索引，在exp第一维取出元素值，这是softmax的分子    tmp1 = exp.gather(1, target.unsqueeze(-1)).squeeze()    # 在exp第一维求和，这是softmax的分母    tmp2 = exp.sum(1)    # softmax公式：ei / sum(ej)    softmax = tmp1 / tmp2    # cross-entropy公式： -yi * log(pi)    # 因为target的yi为1，其余为0，所以在tmp1直接把目标拿出来，    # 公式中的pi就是softmax的结果    log = -torch.log(softmax)    # 官方实现中，reduction有mean/sum及none    # 只是对交叉熵后处理的差别    if reduction == "mean":        return log.mean()    else:        return log.sum()

# example1# input = torch.randn(3, 5,)# target = torch.randint(5, (3,), dtype=torch.int64)# example2input = torch.tensor([[0.1, 0.9], [0.9, 0.1]])target = torch.tensor([1, 0])# example3# input = torch.tensor([[0., 1.], [1., 0.]])# target = torch.tensor([1, 0])loss1_mean = F.cross_entropy(input, target)loss2_mean = my_cross_entropy(input, target)print(loss1_mean)print(loss2_mean)

loss1_sum = F.cross_entropy(input, target, reduction="sum")loss2_sum = my_cross_entropy(input, target, reduction="sum")print(loss1_sum)print(loss2_sum)

def bce_loss_with_logit(y_pred, y_true, reduction='mean'):    y_pred = sigmoid(y_pred)

    loss = -y_true * torch.log(y_pred) - (1 - y_true) * torch.log(1 - y_pred)    if reduction == 'mean':        return torch.mean(loss)    raise NotImplementedError