Mathf.Absかますと?

Angle += oldAngle-Mathf.Abs(Handle.transform.localRotation.eulerAngles.x);
Debug.Log(Angle);

oldAngle=Mathf.Abs(Handle.transform.localRotation.eulerAngles.x);