Open main menu

Changes

Module:Math

403 bytes added, 20:33, 21 February 2013
problem with lost precision
local lang = mw.getContentLanguage();
local value value_string = tonumber( frame.args[1] or '0 ' ) local value = tonumber( value_string );
local precision = tonumber( frame.args[2] or 0 );
local current_precision = z._precision( value );
 
local order = z._order( value );
-- Due to round-off effects it is neccesary to limit the returned precision under
-- some circumstances because the terminal digits will be inaccurately reported.
if order + precision >= 14 then
local orig_precision = z._precision( value_string );
if order + orig_precision >= 14 then
precision = 13 - order;
end
end
 
-- If rounding off, truncate extra digits
if precision < current_precision then
sign = '';
end
local order;
-- Handle cases requiring scientific notation
order = z._order( value );
if string.find( formatted_num, 'E', 1, true ) ~= nil or math.abs(order) >= 9 then
value = value * math.pow( 10, -order );
Anonymous user