add mish activation function (#1068)

tensorflow · Sep 2, 2020 · c1d8225 · c1d8225
1 parent 897bac9
commit c1d8225
Show file tree

Hide file tree

Showing 2 changed files with 19 additions and 0 deletions.
diff --git a/Sources/TensorFlow/Operators/Math.swift b/Sources/TensorFlow/Operators/Math.swift
@@ -1518,6  1518,17 @@ public func hardSwish<T: TensorFlowFloatingPoint>(_ x: Tensor<T>) -> Tensor<T> {
   x * hardSigmoid(x)
 }
 
 /// Returns a tensor by applying the mish activation function, namely
 /// `x * tanh(softplus(x))`.
 ///
 /// Source: "Mish: A Self Regularized Non-Monotonic Neural Activation Function"
 /// https://arxiv.org/abs/1908.08681
 @inlinable
 @differentiable
 public func mish<T: TensorFlowFloatingPoint>(_ x: Tensor<T>) -> Tensor<T> {
   x * tanh(softplus(x))
 }
 
 extension Tensor where Scalar: TensorFlowFloatingPoint {
   /// Returns a boolean tensor indicating which elements of `x` are finite.
   @inlinable public var isFinite: Tensor<Bool> { _Raw.isFinite(self) }

diff --git a/Tests/TensorFlowTests/OperatorTests/MathTests.swift b/Tests/TensorFlowTests/OperatorTests/MathTests.swift
@@ -284,6  284,13 @@ final class MathOperatorTests: XCTestCase {
     let expectedY = Tensor<Float>([0.0, -0.33333334, 0.0, 1.6666666, 4.0])
     assertEqual(y, expectedY, accuracy: 1e-5)
   }
 
   func testMish() {
     let x = Tensor<Float>([-4, -2, 0, 2, 4])
     let y = mish(x)
     let expectedY = Tensor<Float>([-0.07259174, -0.25250146, 0.0, 1.943959, 3.9974122])
     assertEqual(y, expectedY, accuracy: 1e-5)
   }
 
   func testIsFinite() {
     let x = Tensor<Float>([1, 2, 3, 4, -Float.infinity])
@@ -651,6  658,7 @@ final class MathOperatorTests: XCTestCase {
     ("testSwish", testSwish),
     ("testHardSigmoid", testHardSigmoid),
     ("testHardSwish", testHardSwish),
     ("testMish", testMish),
     ("testIsFinite", testIsFinite),
     ("testIsInfinite", testIsInfinite),
     ("testIsNaN", testIsNaN),