github/msft-calculator
1
0
Fork 0
mirror of https://github.com/Microsoft/calculator.git synced 2025-08-22 06:13:14 -07:00
Code Issues Projects Releases Packages Wiki Activity Actions

Fix rounding issues in ratpak that result from using large precisions.

Browse source
This commit is contained in:
Josh Koon 2019-02-22 15:50:19 -08:00
commit 2dc3455fd6
4 changed files with 48 additions and 29 deletions
Download patch file Download diff file Expand all files Collapse all files

49
src/CalcManager/Ratpack/conv.cpp
View file

@ -1259,20 +1259,7 @@ wstring NumberToString(_Inout_ PNUMBER& pnum, int format, uint32_t radix, int32_
//-----------------------------------------------------------------------------
wstring RatToString(_Inout_ PRAT& prat, int format, uint32_t radix, int32_t precision)
{
// Convert p and q of rational form from internal base to requested base.
// Scale by largest power of BASEX possible.
long scaleby = min(prat->pp->exp, prat->pq->exp);
scaleby = max(scaleby, 0);
prat->pp->exp -= scaleby;
prat->pq->exp -= scaleby;
PNUMBER p = nRadixxtonum(prat->pp, radix, precision);
PNUMBER q = nRadixxtonum(prat->pq, radix, precision);
// finally take the time hit to actually divide.
divnum(&p, q, radix, precision);
destroynum(q);
PNUMBER p = RatToNumber(prat, radix, precision);
wstring result = NumberToString(p, format, radix, precision);
destroynum(p);
@ -1280,6 +1267,40 @@ wstring RatToString(_Inout_ PRAT& prat, int format, uint32_t radix, int32_t prec
return result;
}
PNUMBER RatToNumber(_In_ PRAT prat, uint32_t radix, int32_t precision)
{
PRAT temprat = nullptr;
DUPRAT(temprat, prat);
// Convert p and q of rational form from internal base to requested base.
// Scale by largest power of BASEX possible.
long scaleby = min(temprat->pp->exp, temprat->pq->exp);
scaleby = max(scaleby, 0);
temprat->pp->exp -= scaleby;
temprat->pq->exp -= scaleby;
PNUMBER p = nRadixxtonum(temprat->pp, radix, precision);
PNUMBER q = nRadixxtonum(temprat->pq, radix, precision);
destroyrat(temprat);
// finally take the time hit to actually divide.
divnum(&p, q, radix, precision);
destroynum(q);
return p;
}
// Converts a PRAT to a PNUMBER and back to a PRAT, flattening/simplifying the rational in the process
void flatrat(_Inout_ PRAT& prat, uint32_t radix, int32_t precision)
{
PNUMBER pnum = RatToNumber(prat, radix, precision);
destroyrat(prat);
prat = numtorat(pnum, radix);
destroynum(pnum);
}
//-----------------------------------------------------------------------------
//
// FUNCTION: gcd

9
src/CalcManager/Ratpack/rat.cpp
View file

@ -73,16 +73,11 @@ void gcdrat( PRAT *pa, uint32_t radix, int32_t precision)
void fracrat( PRAT *pa , uint32_t radix, int32_t precision)
{
// Only do the intrat operation if number is nonzero.
// Only do the flatrat operation if number is nonzero.
// and only if the bottom part is not one.
if ( !zernum( (*pa)->pp ) && !equnum( (*pa)->pq, num_one ) )
{
wstring ratStr = RatToString(*pa, FMT_FLOAT, radix, precision);
PNUMBER pnum = StringToNumber(ratStr, radix, precision);
destroyrat( *pa );
*pa = numtorat( pnum, radix);
destroynum( pnum );
flatrat(*pa, radix, precision);
}
remnum( &((*pa)->pp), (*pa)->pq, BASEX );

6
src/CalcManager/Ratpack/ratpak.h
View file

@ -1,4 +1,4 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#pragma once
@ -316,6 +316,10 @@ extern std::wstring NumberToString(_Inout_ PNUMBER& pnum, int format, uint32_t r
// returns a text representation of a PRAT
extern std::wstring RatToString(_Inout_ PRAT& prat, int format, uint32_t radix, int32_t precision);
// converts a PRAT into a PNUMBER
extern PNUMBER RatToNumber(_In_ PRAT prat, uint32_t radix, int32_t precision);
// flattens a PRAT by converting it to a PNUMBER and back to a PRAT
extern void flatrat(_Inout_ PRAT& prat, uint32_t radix, int32_t precision);
extern long numtolong(_In_ PNUMBER pnum, uint32_t radix );
extern long rattolong(_In_ PRAT prat, uint32_t radix, int32_t precision);

13
src/CalcManager/Ratpack/support.cpp
View file

@ -291,19 +291,18 @@ void intrat( PRAT *px, uint32_t radix, int32_t precision)
// and only if the bottom part is not one.
if ( !zernum( (*px)->pp ) && !equnum( (*px)->pq, num_one ) )
{
wstring ratStr = RatToString(*px, FMT_FLOAT, radix, precision);
PNUMBER pnum = StringToNumber(ratStr, radix, precision);
destroyrat( *px );
*px = numtorat( pnum, radix);
destroynum( pnum );
flatrat(*px, radix, precision);
// Subtract the fractional part of the rational
PRAT pret = nullptr;
DUPRAT(pret,*px);
modrat( &pret, rat_one );
subrat( px, pret, precision);
destroyrat( pret );
// Simplify the value if possible to resolve rounding errors
flatrat(*px, radix, precision);
}
}