diff --git a/runtime/bytecode.h b/runtime/bytecode.h index 828226d59..ec320e673 100644 --- a/runtime/bytecode.h +++ b/runtime/bytecode.h @@ -59,12 +59,12 @@ namespace py { V(UNUSED_BYTECODE_39, 39, doInvalidBytecode) \ V(UNUSED_BYTECODE_40, 40, doInvalidBytecode) \ V(UNUSED_BYTECODE_41, 41, doInvalidBytecode) \ - V(UNUSED_BYTECODE_42, 42, doInvalidBytecode) \ - V(UNUSED_BYTECODE_43, 43, doInvalidBytecode) \ - V(UNUSED_BYTECODE_44, 44, doInvalidBytecode) \ - V(UNUSED_BYTECODE_45, 45, doInvalidBytecode) \ - V(UNUSED_BYTECODE_46, 46, doInvalidBytecode) \ - V(UNUSED_BYTECODE_47, 47, doInvalidBytecode) \ + V(INPLACE_ADD_FLOAT, 42, doInplaceAddFloat) \ + V(INPLACE_SUB_FLOAT, 43, doInplaceSubFloat) \ + V(BINARY_ADD_FLOAT, 44, doBinaryAddFloat) \ + V(BINARY_SUB_FLOAT, 45, doBinarySubFloat) \ + V(BINARY_MUL_FLOAT, 46, doBinaryMulFloat) \ + V(BINARY_POWER_FLOAT, 47, doBinaryPowerFloat) \ V(LOAD_BOOL, 48, doLoadBool) \ V(UNUSED_BYTECODE_49, 49, doInvalidBytecode) \ V(GET_AITER, 50, doGetAiter) \ diff --git a/runtime/interpreter-gen-x64.cpp b/runtime/interpreter-gen-x64.cpp index 2e560de1f..0dd118f32 100644 --- a/runtime/interpreter-gen-x64.cpp +++ b/runtime/interpreter-gen-x64.cpp @@ -635,8 +635,9 @@ void emitJumpIfNotHasLayoutId(EmitEnv* env, Register r_obj, LayoutId layout_id, } void emitJumpIfNotHeapObjectWithLayoutId(EmitEnv* env, Register r_obj, - LayoutId layout_id, Label* target) { - emitJumpIfImmediate(env, r_obj, target, Assembler::kNearJump); + LayoutId layout_id, Label* target, + bool is_near = Assembler::kNearJump) { + emitJumpIfImmediate(env, r_obj, target, is_near); // It is a HeapObject. emitJumpIfNotHasLayoutId(env, r_obj, layout_id, target); @@ -733,6 +734,32 @@ void emitPushBoundMethod(EmitEnv* env, Label* slow_path, Register r_self, __ pushq(r_scratch); } +// Allocate and push a Float on the stack. If the heap is full and a GC +// is needed, jump to slow_path instead. r_value will be used to populate the +// Float. r_space and r_scratch are used as scratch registers. +// +// Writes to r_space and r_scratch. +void emitPushFloat(EmitEnv* env, Label* slow_path, XmmRegister r_value) { + ScratchReg r_scratch(env); + ScratchReg r_space(env); + __ movq(r_space, Address(env->thread, Thread::runtimeOffset())); + __ movq(r_space, + Address(r_space, Runtime::heapOffset() + Heap::spaceOffset())); + + __ movq(r_scratch, Address(r_space, Space::fillOffset())); + __ addq(r_scratch, Immediate(Float::allocationSize())); + __ cmpq(r_scratch, Address(r_space, Space::endOffset())); + __ jcc(GREATER, slow_path, Assembler::kFarJump); + __ xchgq(r_scratch, Address(r_space, Space::fillOffset())); + RawHeader header = Header::from(Float::kSize, /*hash=*/0, LayoutId::kFloat, + ObjectFormat::kData); + __ movq(Address(r_scratch, 0), Immediate(header.raw())); + __ leaq(r_scratch, Address(r_scratch, -RawFloat::kHeaderOffset + + Object::kHeapObjectTag)); + __ movsd(Address(r_scratch, heapObjectDisp(RawFloat::kValueOffset)), r_value); + __ pushq(r_scratch); +} + // Given a RawObject in r_obj and its LayoutId (as a SmallInt) in r_layout_id, // load its overflow RawTuple into r_dst. // @@ -824,6 +851,60 @@ void emitHandler(EmitEnv* env) { emitHandleContinue(env, kGenericHandler); } +static void emitBinaryOpFloat(EmitEnv* env, + void (Assembler::*asm_op)(XmmRegister left, + XmmRegister right)) { + ScratchReg r_right(env); + ScratchReg r_left(env); + Label slow_path; + + __ popq(r_right); + __ popq(r_left); + emitJumpIfNotHeapObjectWithLayoutId(env, r_left, LayoutId::kFloat, &slow_path, + Assembler::kFarJump); + emitJumpIfNotHeapObjectWithLayoutId(env, r_right, LayoutId::kFloat, + &slow_path, Assembler::kNearJump); + __ movsd(XMM0, Address(r_left, heapObjectDisp(Float::kValueOffset))); + __ movsd(XMM1, Address(r_right, heapObjectDisp(Float::kValueOffset))); + (env->as.*asm_op)(XMM0, XMM1); + emitPushFloat(env, &slow_path, XMM0); + emitNextOpcode(env); + + __ bind(&slow_path); + __ pushq(r_left); + __ pushq(r_right); + if (env->in_jit) { + emitJumpToDeopt(env); + return; + } + emitJumpToGenericHandler(env); +} + +template <> +void emitHandler(EmitEnv* env) { + emitBinaryOpFloat(env, &Assembler::mulsd); +} + +template <> +void emitHandler(EmitEnv* env) { + emitBinaryOpFloat(env, &Assembler::addsd); +} + +template <> +void emitHandler(EmitEnv* env) { + emitBinaryOpFloat(env, &Assembler::addsd); +} + +template <> +void emitHandler(EmitEnv* env) { + emitBinaryOpFloat(env, &Assembler::subsd); +} + +template <> +void emitHandler(EmitEnv* env) { + emitBinaryOpFloat(env, &Assembler::subsd); +} + template <> void emitHandler(EmitEnv* env) { ScratchReg r_right(env); @@ -2445,13 +2526,35 @@ void emitHandler(EmitEnv* env) { emitNextOpcodeFallthrough(env); } +template <> +void emitHandler(EmitEnv* env) { + __ pushq(Address(RSP, kPointerSize)); + __ pushq(Address(RSP, kPointerSize)); + emitNextOpcodeFallthrough(env); +} + template <> void emitHandler(EmitEnv* env) { ScratchReg r_scratch(env); __ popq(r_scratch); __ pushq(Address(RSP, 0)); - __ movq(Address(RSP, 8), r_scratch); + __ movq(Address(RSP, kPointerSize), r_scratch); + emitNextOpcodeFallthrough(env); +} + +template <> +void emitHandler(EmitEnv* env) { + ScratchReg r_top(env); + ScratchReg r_second(env); + ScratchReg r_third(env); + + __ popq(r_top); + __ popq(r_second); + __ popq(r_third); + __ pushq(r_top); + __ pushq(r_third); + __ pushq(r_second); emitNextOpcodeFallthrough(env); } diff --git a/runtime/interpreter.cpp b/runtime/interpreter.cpp index f4cbf396f..9e1a0784c 100644 --- a/runtime/interpreter.cpp +++ b/runtime/interpreter.cpp @@ -1,6 +1,7 @@ // Copyright (c) Facebook, Inc. and its affiliates. (http://www.facebook.com) #include "interpreter.h" +#include #include #include #include @@ -5812,6 +5813,23 @@ Continue Interpreter::doInplaceAddSmallInt(Thread* thread, word arg) { return inplaceOpUpdateCache(thread, arg, cache); } +HANDLER_INLINE +Continue Interpreter::doInplaceAddFloat(Thread* thread, word arg) { + RawObject left = thread->stackPeek(1); + RawObject right = thread->stackPeek(0); + if (left.isFloat() && right.isFloat()) { + double left_value = Float::cast(left).value(); + double right_value = Float::cast(right).value(); + double result_value = left_value + right_value; + thread->stackDrop(1); + thread->stackSetTop(thread->runtime()->newFloat(result_value)); + return Continue::NEXT; + } + EVENT_CACHE(INPLACE_ADD_FLOAT); + word cache = currentCacheIndex(thread->currentFrame()); + return inplaceOpUpdateCache(thread, arg, cache); +} + HANDLER_INLINE Continue Interpreter::doInplaceSubSmallInt(Thread* thread, word arg) { RawObject left = thread->stackPeek(1); @@ -5831,6 +5849,23 @@ Continue Interpreter::doInplaceSubSmallInt(Thread* thread, word arg) { return inplaceOpUpdateCache(thread, arg, cache); } +HANDLER_INLINE +Continue Interpreter::doInplaceSubFloat(Thread* thread, word arg) { + RawObject left = thread->stackPeek(1); + RawObject right = thread->stackPeek(0); + if (left.isFloat() && right.isFloat()) { + double left_value = Float::cast(left).value(); + double right_value = Float::cast(right).value(); + double result_value = left_value - right_value; + thread->stackDrop(1); + thread->stackSetTop(thread->runtime()->newFloat(result_value)); + return Continue::NEXT; + } + EVENT_CACHE(INPLACE_SUB_FLOAT); + word cache = currentCacheIndex(thread->currentFrame()); + return inplaceOpUpdateCache(thread, arg, cache); +} + HANDLER_INLINE Continue Interpreter::doInplaceOpAnamorphic(Thread* thread, word arg) { Frame* frame = thread->currentFrame(); @@ -5848,6 +5883,20 @@ Continue Interpreter::doInplaceOpAnamorphic(Thread* thread, word arg) { } } } + if (thread->stackPeek(0).isFloat() && thread->stackPeek(1).isFloat()) { + switch (static_cast(arg)) { + case BinaryOp::ADD: + rewriteCurrentBytecode(frame, INPLACE_ADD_FLOAT); + return doInplaceAddFloat(thread, arg); + case BinaryOp::SUB: + rewriteCurrentBytecode(frame, INPLACE_SUB_FLOAT); + return doInplaceSubFloat(thread, arg); + default: { + word cache = currentCacheIndex(frame); + return inplaceOpUpdateCache(thread, arg, cache); + } + } + } word cache = currentCacheIndex(frame); return inplaceOpUpdateCache(thread, arg, cache); } @@ -5974,6 +6023,23 @@ Continue Interpreter::doBinaryAddSmallInt(Thread* thread, word arg) { return binaryOpUpdateCache(thread, arg, cache); } +HANDLER_INLINE +Continue Interpreter::doBinaryAddFloat(Thread* thread, word arg) { + RawObject left = thread->stackPeek(1); + RawObject right = thread->stackPeek(0); + if (left.isFloat() && right.isFloat()) { + double left_value = Float::cast(left).value(); + double right_value = Float::cast(right).value(); + double result_value = left_value + right_value; + thread->stackDrop(1); + thread->stackSetTop(thread->runtime()->newFloat(result_value)); + return Continue::NEXT; + } + EVENT_CACHE(BINARY_ADD_FLOAT); + word cache = currentCacheIndex(thread->currentFrame()); + return binaryOpUpdateCache(thread, arg, cache); +} + HANDLER_INLINE Continue Interpreter::doBinaryAndSmallInt(Thread* thread, word arg) { RawObject left = thread->stackPeek(1); @@ -6011,6 +6077,23 @@ Continue Interpreter::doBinaryMulSmallInt(Thread* thread, word arg) { return binaryOpUpdateCache(thread, arg, cache); } +HANDLER_INLINE +Continue Interpreter::doBinaryMulFloat(Thread* thread, word arg) { + RawObject left = thread->stackPeek(1); + RawObject right = thread->stackPeek(0); + if (left.isFloat() && right.isFloat()) { + double left_value = Float::cast(left).value(); + double right_value = Float::cast(right).value(); + double result_value = left_value * right_value; + thread->stackDrop(1); + thread->stackSetTop(thread->runtime()->newFloat(result_value)); + return Continue::NEXT; + } + EVENT_CACHE(BINARY_MUL_FLOAT); + word cache = currentCacheIndex(thread->currentFrame()); + return binaryOpUpdateCache(thread, arg, cache); +} + HANDLER_INLINE Continue Interpreter::doBinaryFloordivSmallInt(Thread* thread, word arg) { RawObject left = thread->stackPeek(1); @@ -6053,6 +6136,23 @@ Continue Interpreter::doBinarySubSmallInt(Thread* thread, word arg) { return binaryOpUpdateCache(thread, arg, cache); } +HANDLER_INLINE +Continue Interpreter::doBinarySubFloat(Thread* thread, word arg) { + RawObject left = thread->stackPeek(1); + RawObject right = thread->stackPeek(0); + if (left.isFloat() && right.isFloat()) { + double left_value = Float::cast(left).value(); + double right_value = Float::cast(right).value(); + double result_value = left_value - right_value; + thread->stackDrop(1); + thread->stackSetTop(thread->runtime()->newFloat(result_value)); + return Continue::NEXT; + } + EVENT_CACHE(BINARY_SUB_FLOAT); + word cache = currentCacheIndex(thread->currentFrame()); + return binaryOpUpdateCache(thread, arg, cache); +} + HANDLER_INLINE Continue Interpreter::doBinaryOrSmallInt(Thread* thread, word arg) { RawObject left = thread->stackPeek(1); @@ -6071,6 +6171,23 @@ Continue Interpreter::doBinaryOrSmallInt(Thread* thread, word arg) { return binaryOpUpdateCache(thread, arg, cache); } +HANDLER_INLINE +Continue Interpreter::doBinaryPowerFloat(Thread* thread, word arg) { + RawObject left = thread->stackPeek(1); + RawObject right = thread->stackPeek(0); + if (left.isFloat() && right.isFloat()) { + double left_value = Float::cast(left).value(); + double right_value = Float::cast(right).value(); + double result_value = std::pow(left_value, right_value); + thread->stackDrop(1); + thread->stackSetTop(thread->runtime()->newFloat(result_value)); + return Continue::NEXT; + } + EVENT_CACHE(BINARY_POWER_FLOAT); + word cache = currentCacheIndex(thread->currentFrame()); + return binaryOpUpdateCache(thread, arg, cache); +} + HANDLER_INLINE Continue Interpreter::doBinaryOpAnamorphic(Thread* thread, word arg) { Frame* frame = thread->currentFrame(); @@ -6100,6 +6217,30 @@ Continue Interpreter::doBinaryOpAnamorphic(Thread* thread, word arg) { } } } + if (thread->stackPeek(0).isFloat() && thread->stackPeek(1).isFloat()) { + switch (static_cast(arg)) { + case BinaryOp::ADD: + rewriteCurrentBytecode(frame, BINARY_ADD_FLOAT); + return doBinaryAddFloat(thread, arg); + break; + case BinaryOp::SUB: + rewriteCurrentBytecode(frame, BINARY_SUB_FLOAT); + return doBinarySubFloat(thread, arg); + break; + case BinaryOp::MUL: + rewriteCurrentBytecode(frame, BINARY_MUL_FLOAT); + return doBinaryMulFloat(thread, arg); + break; + case BinaryOp::POW: + rewriteCurrentBytecode(frame, BINARY_POWER_FLOAT); + return doBinaryPowerFloat(thread, arg); + break; + default: { + word cache = currentCacheIndex(frame); + return binaryOpUpdateCache(thread, arg, cache); + } + } + } word cache = currentCacheIndex(frame); return binaryOpUpdateCache(thread, arg, cache); } diff --git a/runtime/interpreter.h b/runtime/interpreter.h index 25f554434..446db696b 100644 --- a/runtime/interpreter.h +++ b/runtime/interpreter.h @@ -360,6 +360,10 @@ class Interpreter { static Continue doBinarySubSmallInt(Thread* thread, word arg); static Continue doBinaryOrSmallInt(Thread* thread, word arg); static Continue doBinaryOpAnamorphic(Thread* thread, word arg); + static Continue doBinaryAddFloat(Thread* thread, word arg); + static Continue doBinarySubFloat(Thread* thread, word arg); + static Continue doBinaryMulFloat(Thread* thread, word arg); + static Continue doBinaryPowerFloat(Thread* thread, word arg); static Continue doBinaryOr(Thread* thread, word arg); static Continue doBinaryPower(Thread* thread, word arg); static Continue doBinaryRshift(Thread* thread, word arg); @@ -433,6 +437,7 @@ class Interpreter { static Continue doImportName(Thread* thread, word arg); static Continue doInplaceAdd(Thread* thread, word arg); static Continue doInplaceAddSmallInt(Thread* thread, word arg); + static Continue doInplaceAddFloat(Thread* thread, word arg); static Continue doInplaceAnd(Thread* thread, word arg); static Continue doInplaceFloorDivide(Thread* thread, word arg); static Continue doInplaceLshift(Thread* thread, word arg); @@ -447,6 +452,7 @@ class Interpreter { static Continue doInplaceRshift(Thread* thread, word arg); static Continue doInplaceSubtract(Thread* thread, word arg); static Continue doInplaceSubSmallInt(Thread* thread, word arg); + static Continue doInplaceSubFloat(Thread* thread, word arg); static Continue doInplaceTrueDivide(Thread* thread, word arg); static Continue doInplaceXor(Thread* thread, word arg); static Continue doInvalidBytecode(Thread* thread, word arg);