@@ -32,36 +32,4 @@ def backward(self, gradients):
3232
3333 # Unscale gradients (ensure result is float32)
3434 unscaled_gradients = gradients_fp32 / self .loss_scale
35- return unscaled_gradients .astype (np .float32 )
36-
37-
38- tests = [
39- {
40- "input" : "import numpy as np\n mp = MixedPrecision(loss_scale=1024.0)\n weights = np.array([0.5, -0.3], dtype=np.float32)\n inputs = np.array([[1.0, 2.0], [3.0, 4.0]], dtype=np.float32)\n targets = np.array([1.0, 0.0], dtype=np.float32)\n loss = mp.forward(weights, inputs, targets)\n print(f\" Loss: {loss:.4f}\" )\n print(f\" Loss dtype: {type(loss).__name__}\" )\n grads = np.array([512.0, -256.0], dtype=np.float32)\n result = mp.backward(grads)\n print(f\" Gradients: {result}\" )\n print(f\" Grad dtype: {result.dtype}\" )" ,
41- "output" : "Loss: 1638.4000\n Loss dtype: float\n Gradients: [0.5 -0.25]\n Grad dtype: float32"
42- },
43- {
44- "input" : "import numpy as np\n mp = MixedPrecision(loss_scale=512.0)\n weights = np.array([1.0, 0.5], dtype=np.float64)\n inputs = np.array([[2.0, 1.0]], dtype=np.float64)\n targets = np.array([3.0], dtype=np.float64)\n loss = mp.forward(weights, inputs, targets)\n print(f\" Loss: {loss:.1f}\" )\n print(f\" Loss dtype: {type(loss).__name__}\" )\n grads = np.array([1024.0, 512.0], dtype=np.float16)\n result = mp.backward(grads)\n print(f\" Gradients: [{result[0]:.0f} {result[1]:.0f}]\" )\n print(f\" Grad dtype: {result.dtype}\" )" ,
45- "output" : "Loss: 256.0\n Loss dtype: float\n Gradients: [2 1]\n Grad dtype: float32"
46- },
47- {
48- "input" : "import numpy as np\n mp = MixedPrecision(loss_scale=100.0)\n weights = np.array([0.1, 0.2], dtype=np.float32)\n inputs = np.array([[1.0, 1.0]], dtype=np.float32)\n targets = np.array([0.5], dtype=np.float32)\n loss = mp.forward(weights, inputs, targets)\n print(f\" Loss: {loss:.1f}\" )\n print(f\" Loss dtype: {type(loss).__name__}\" )\n grads = np.array([200.0, 100.0], dtype=np.float64)\n result = mp.backward(grads)\n print(f\" Gradients: [{result[0]:.0f} {result[1]:.0f}]\" )\n print(f\" Grad dtype: {result.dtype}\" )" ,
49- "output" : "Loss: 4.0\n Loss dtype: float\n Gradients: [2 1]\n Grad dtype: float32"
50- },
51- {
52- "input" : "import numpy as np\n mp = MixedPrecision(loss_scale=2048.0)\n weights = np.array([0.25], dtype=np.float64)\n inputs = np.array([[4.0]], dtype=np.float64)\n targets = np.array([2.0], dtype=np.float64)\n loss = mp.forward(weights, inputs, targets)\n print(f\" Loss: {loss:.1f}\" )\n print(f\" Loss dtype: {type(loss).__name__}\" )\n grads = np.array([np.nan], dtype=np.float16)\n result = mp.backward(grads)\n print(f\" Gradients: [{result[0]:.0f}]\" )\n print(f\" Grad dtype: {result.dtype}\" )" ,
53- "output" : "Loss: 2048.0\n Loss dtype: float\n Gradients: [0]\n Grad dtype: float32"
54- },
55- {
56- "input" : "import numpy as np\n mp = MixedPrecision(loss_scale=256.0)\n weights = np.array([1.0], dtype=np.float16)\n inputs = np.array([[2.0]], dtype=np.float16)\n targets = np.array([3.0], dtype=np.float16)\n loss = mp.forward(weights, inputs, targets)\n print(f\" Loss: {loss:.1f}\" )\n print(f\" Loss dtype: {type(loss).__name__}\" )\n grads = np.array([np.inf], dtype=np.float64)\n result = mp.backward(grads)\n print(f\" Gradients: [{result[0]:.0f}]\" )\n print(f\" Grad dtype: {result.dtype}\" )" ,
57- "output" : "Loss: 256.0\n Loss dtype: float\n Gradients: [0]\n Grad dtype: float32"
58- }
59- ]
60-
61- for i , test in enumerate (tests ):
62- print (f"Test #{ i + 1 } ." )
63- print ("Got:" )
64- exec (test ['input' ])
65- print (f"\n Expected: { test ['output' ]} " )
66- print ("--------" * 3 )
67-
35+ return unscaled_gradients .astype (np .float32 )
0 commit comments