mirror of
https://github.com/shopspring/decimal.git
synced 2024-11-22 12:30:49 +01:00
fix the rounding bug in roundShortest method (#161)
* fix: fix rounding in FormatFloat fallback path
This commit is contained in:
parent
6fe01c1abd
commit
867ed12000
2 changed files with 56 additions and 12 deletions
|
@ -57,6 +57,9 @@ var testTable = []*testEnt{
|
|||
{2e250, "20000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", "", ""},
|
||||
{math.MaxInt64, strconv.FormatFloat(float64(math.MaxInt64), 'f', -1, 64), "", strconv.FormatInt(math.MaxInt64, 10)},
|
||||
{1.29067116156722e-309, "0.00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000129067116156722", "", "0.000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001290671161567218558822290567835270536800098852722416870074139002112543896676308448335063375297788379444685193974290737962187240854947838776604607190387984577130572928111657710645015086812756013489109884753559084166516937690932698276436869274093950997935137476803610007959500457935217950764794724766740819156974617155861568214427828145972181876775307023388139991104942469299524961281641158436752347582767153796914843896176260096039358494077706152272661453132497761307744086665088096215425146090058519888494342944692629602847826300550628670375451325582843627504604013541465361435761965354140678551369499812124085312128659002910905639984075064968459581691226705666561364681985266583563078466180095375402399087817404368974165082030458595596655868575908243656158447265625000000000000000000000000000000000000004440000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"},
|
||||
// go Issue 29491.
|
||||
{498484681984085570, "498484681984085570", "", ""},
|
||||
{5.8339553793802237e+23, "583395537938022370000000", "", ""},
|
||||
}
|
||||
|
||||
var testTableScientificNotation = map[string]string{
|
||||
|
|
65
rounding.go
65
rounding.go
|
@ -80,39 +80,80 @@ func roundShortest(d *decimal, mant uint64, exp int, flt *floatInfo) {
|
|||
// would round to the original mantissa and not the neighbors.
|
||||
inclusive := mant%2 == 0
|
||||
|
||||
// As we walk the digits we want to know whether rounding up would fall
|
||||
// within the upper bound. This is tracked by upperdelta:
|
||||
//
|
||||
// If upperdelta == 0, the digits of d and upper are the same so far.
|
||||
//
|
||||
// If upperdelta == 1, we saw a difference of 1 between d and upper on a
|
||||
// previous digit and subsequently only 9s for d and 0s for upper.
|
||||
// (Thus rounding up may fall outside the bound, if it is exclusive.)
|
||||
//
|
||||
// If upperdelta == 2, then the difference is greater than 1
|
||||
// and we know that rounding up falls within the bound.
|
||||
var upperdelta uint8
|
||||
|
||||
// Now we can figure out the minimum number of digits required.
|
||||
// Walk along until d has distinguished itself from upper and lower.
|
||||
for i := 0; i < d.nd; i++ {
|
||||
l := byte('0') // lower digit
|
||||
if i < lower.nd {
|
||||
l = lower.d[i]
|
||||
for ui := 0; ; ui++ {
|
||||
// lower, d, and upper may have the decimal points at different
|
||||
// places. In this case upper is the longest, so we iterate from
|
||||
// ui==0 and start li and mi at (possibly) -1.
|
||||
mi := ui - upper.dp + d.dp
|
||||
if mi >= d.nd {
|
||||
break
|
||||
}
|
||||
li := ui - upper.dp + lower.dp
|
||||
l := byte('0') // lower digit
|
||||
if li >= 0 && li < lower.nd {
|
||||
l = lower.d[li]
|
||||
}
|
||||
m := byte('0') // middle digit
|
||||
if mi >= 0 {
|
||||
m = d.d[mi]
|
||||
}
|
||||
m := d.d[i] // middle digit
|
||||
u := byte('0') // upper digit
|
||||
if i < upper.nd {
|
||||
u = upper.d[i]
|
||||
if ui < upper.nd {
|
||||
u = upper.d[ui]
|
||||
}
|
||||
|
||||
// Okay to round down (truncate) if lower has a different digit
|
||||
// or if lower is inclusive and is exactly the result of rounding
|
||||
// down (i.e., and we have reached the final digit of lower).
|
||||
okdown := l != m || inclusive && i+1 == lower.nd
|
||||
okdown := l != m || inclusive && li+1 == lower.nd
|
||||
|
||||
switch {
|
||||
case upperdelta == 0 && m+1 < u:
|
||||
// Example:
|
||||
// m = 12345xxx
|
||||
// u = 12347xxx
|
||||
upperdelta = 2
|
||||
case upperdelta == 0 && m != u:
|
||||
// Example:
|
||||
// m = 12345xxx
|
||||
// u = 12346xxx
|
||||
upperdelta = 1
|
||||
case upperdelta == 1 && (m != '9' || u != '0'):
|
||||
// Example:
|
||||
// m = 1234598x
|
||||
// u = 1234600x
|
||||
upperdelta = 2
|
||||
}
|
||||
// Okay to round up if upper has a different digit and either upper
|
||||
// is inclusive or upper is bigger than the result of rounding up.
|
||||
okup := m != u && (inclusive || m+1 < u || i+1 < upper.nd)
|
||||
okup := upperdelta > 0 && (inclusive || upperdelta > 1 || ui+1 < upper.nd)
|
||||
|
||||
// If it's okay to do either, then round to the nearest one.
|
||||
// If it's okay to do only one, do it.
|
||||
switch {
|
||||
case okdown && okup:
|
||||
d.Round(i + 1)
|
||||
d.Round(mi + 1)
|
||||
return
|
||||
case okdown:
|
||||
d.RoundDown(i + 1)
|
||||
d.RoundDown(mi + 1)
|
||||
return
|
||||
case okup:
|
||||
d.RoundUp(i + 1)
|
||||
d.RoundUp(mi + 1)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
|
Loading…
Reference in a new issue