Fix rounding issues in ratpak that result from using large precisions.

src/CalcManager/Ratpack/conv.cpp

@@ -1259,20 +1259,7 @@ wstring NumberToString(_Inout_ PNUMBER& pnum, int format, uint32_t radix, int32_
 //-----------------------------------------------------------------------------
 wstring RatToString(_Inout_ PRAT& prat, int format, uint32_t radix, int32_t precision)
 {
-    // Convert p and q of rational form from internal base to requested base.
+    PNUMBER p = RatToNumber(prat, radix, precision);
-    // Scale by largest power of BASEX possible.
-    long scaleby = min(prat->pp->exp, prat->pq->exp);
-    scaleby = max(scaleby, 0);
-
-    prat->pp->exp -= scaleby;
-    prat->pq->exp -= scaleby;
-
-    PNUMBER p = nRadixxtonum(prat->pp, radix, precision);
-    PNUMBER q = nRadixxtonum(prat->pq, radix, precision);
-
-    // finally take the time hit to actually divide.
-    divnum(&p, q, radix, precision);
-    destroynum(q);
 
     wstring result = NumberToString(p, format, radix, precision);
     destroynum(p);
@@ -1280,6 +1267,40 @@ wstring RatToString(_Inout_ PRAT& prat, int format, uint32_t radix, int32_t prec
     return result;
 }
 
+PNUMBER RatToNumber(_In_ PRAT prat, uint32_t radix, int32_t precision)
+{
+    PRAT temprat = nullptr;
+    DUPRAT(temprat, prat);
+    // Convert p and q of rational form from internal base to requested base.
+    // Scale by largest power of BASEX possible.
+    long scaleby = min(temprat->pp->exp, temprat->pq->exp);
+    scaleby = max(scaleby, 0);
+
+    temprat->pp->exp -= scaleby;
+    temprat->pq->exp -= scaleby;
+
+    PNUMBER p = nRadixxtonum(temprat->pp, radix, precision);
+    PNUMBER q = nRadixxtonum(temprat->pq, radix, precision);
+
+    destroyrat(temprat);
+
+    // finally take the time hit to actually divide.
+    divnum(&p, q, radix, precision);
+    destroynum(q);
+
+    return p;
+}
+
+// Converts a PRAT to a PNUMBER and back to a PRAT, flattening/simplifying the rational in the process
+void flatrat(_Inout_ PRAT& prat, uint32_t radix, int32_t precision)
+{
+    PNUMBER pnum = RatToNumber(prat, radix, precision);
+
+    destroyrat(prat);
+    prat = numtorat(pnum, radix);
+    destroynum(pnum);
+}
+
 //-----------------------------------------------------------------------------
 //
 //  FUNCTION: gcd

src/CalcManager/Ratpack/rat.cpp

@@ -73,16 +73,11 @@ void gcdrat( PRAT *pa, uint32_t radix, int32_t precision)
 
 void fracrat( PRAT *pa , uint32_t radix, int32_t precision)
 {
-    // Only do the intrat operation if number is nonzero.
+    // Only do the flatrat operation if number is nonzero.
     // and only if the bottom part is not one.
     if ( !zernum( (*pa)->pp ) && !equnum( (*pa)->pq, num_one ) )
     {
-        wstring ratStr = RatToString(*pa, FMT_FLOAT, radix, precision);
+        flatrat(*pa, radix, precision);
-        PNUMBER pnum = StringToNumber(ratStr, radix, precision);
-
-        destroyrat( *pa );
-        *pa = numtorat( pnum, radix);
-        destroynum( pnum );
     }
 
     remnum( &((*pa)->pp), (*pa)->pq, BASEX );

src/CalcManager/Ratpack/ratpak.h

@@ -1,4 +1,4 @@
-// Copyright (c) Microsoft Corporation. All rights reserved.
+// Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT License.
 
 #pragma once
@@ -316,6 +316,10 @@ extern std::wstring NumberToString(_Inout_ PNUMBER& pnum, int format, uint32_t r
 
 // returns a text representation of a PRAT
 extern std::wstring RatToString(_Inout_ PRAT& prat, int format, uint32_t radix, int32_t precision);
+// converts a PRAT into a PNUMBER
+extern PNUMBER RatToNumber(_In_ PRAT prat, uint32_t radix, int32_t precision);
+// flattens a PRAT by converting it to a PNUMBER and back to a PRAT
+extern void flatrat(_Inout_ PRAT& prat, uint32_t radix, int32_t precision);
 
 extern long numtolong(_In_ PNUMBER pnum, uint32_t radix );
 extern long rattolong(_In_ PRAT prat, uint32_t radix, int32_t precision);

src/CalcManager/Ratpack/support.cpp

@@ -291,19 +291,18 @@ void intrat( PRAT *px, uint32_t radix, int32_t precision)
     // and only if the bottom part is not one.
     if ( !zernum( (*px)->pp ) && !equnum( (*px)->pq, num_one ) )
     {
-        wstring ratStr = RatToString(*px, FMT_FLOAT, radix, precision);
+        flatrat(*px, radix, precision);
-        PNUMBER pnum = StringToNumber(ratStr, radix, precision);
-
-        destroyrat( *px );
-        *px = numtorat( pnum, radix);
-        destroynum( pnum );
 
+        // Subtract the fractional part of the rational
         PRAT pret = nullptr;
         DUPRAT(pret,*px);
         modrat( &pret, rat_one );
 
         subrat( px, pret, precision);
         destroyrat( pret );
+
+        // Simplify the value if possible to resolve rounding errors
+        flatrat(*px, radix, precision);
     }
 }