activation_functions / gradient_magnitudes.json
AmberLJC's picture
Upload gradient_magnitudes.json with huggingface_hub
4ca9543 verified
{
"linear": [
0.0004009805852547288,
0.00040036847349256277,
0.00036985811311751604,
0.000359987752744928,
0.0003289323067292571,
0.00030929772765375674,
0.000437420851085335,
0.000557888182811439,
0.0006172899156808853,
0.0007439448963850737
],
"sigmoid": [
1.6582196726844245e-10,
8.183660415994609e-10,
5.867596630793059e-09,
3.754411892487042e-08,
2.397275977727986e-07,
1.497530092819943e-06,
1.265296850760933e-05,
7.97040993347764e-05,
0.0005029253661632538,
0.0036848573945462704
],
"relu": [
1.2022654118482023e-05,
9.013649105327204e-06,
7.954162356327288e-06,
6.0182464949321e-06,
6.1193031797301956e-06,
7.427526725223288e-06,
1.2612316822924186e-05,
1.5091778550413437e-05,
2.7222169592278078e-05,
3.230436777812429e-05
],
"leaky_relu": [
2.784265234367922e-06,
2.4898686206142884e-06,
2.8182541882415535e-06,
2.0921370378346182e-06,
5.041565600549802e-06,
7.597795047331601e-06,
1.537061507406179e-05,
3.3493703085696325e-05,
0.00011857690697070211,
0.0003166203387081623
],
"gelu": [
4.0977013782139693e-07,
4.266817086318042e-07,
4.2514167830631777e-07,
4.776728701472166e-07,
7.016578251750616e-07,
1.13078033336933e-06,
4.0528016143071e-06,
1.2373106073937379e-05,
5.07820732309483e-05,
0.00014970538904890418
]
}