Rewrite BUILD_SLICE to LOAD_SLICE_CACHED when it's constant

runtime/bytecode.cpp

@@ -25,7 +25,7 @@ BytecodeOp nextBytecodeOp(const MutableBytes& bytecode, word* index) {
   }
   DCHECK(i - *index <= 8, "EXTENDED_ARG-encoded arg must fit in int32_t");
   *index = i;
-  return BytecodeOp{bc, arg, cache};
+  return BytecodeOp{bc, arg, cache, i - 1};
 }
 
 const word kOpcodeOffset = 0;
@@ -233,6 +233,28 @@ static RewrittenOp rewriteOperation(const Function& function, BytecodeOp op) {
         }
       }
     } break;
+    case BUILD_SLICE: {
+      DCHECK(op.arg == 2 || op.arg == 3,
+             "BUILD_SLICE oparg must be 2 or 3; found %d", op.arg);
+      Thread* thread = Thread::current();
+      HandleScope scope(thread);
+      Bytes bytecode(&scope, Code::cast(function.code()).code());
+      if (op.index > 4 &&
+          bytecodeOpAt(bytecode, op.index - 4) == EXTENDED_ARG) {
+        // LOAD_CONST, LOAD_CONST, LOAD_CONST, BUILD_SLICE
+        break;
+      }
+      if (bytecodeOpAt(bytecode, op.index - 2) != LOAD_CONST) {
+        break;
+      }
+      if (bytecodeOpAt(bytecode, op.index - 1) != LOAD_CONST) {
+        break;
+      }
+      if (op.arg == 3 && bytecodeOpAt(bytecode, op.index - 3) != LOAD_CONST) {
+        break;
+      }
+      return RewrittenOp{LOAD_SLICE_CACHED, op.arg, true};
+    }
     case BINARY_OP_ANAMORPHIC:
     case COMPARE_OP_ANAMORPHIC:
     case FOR_ITER_ANAMORPHIC:
@@ -265,6 +287,21 @@ RawObject expandBytecode(Thread* thread, const Bytes& bytecode) {
 
 static const word kMaxCaches = 65536;
 
+static RawObject constAtOp(const MutableBytes& bytecode, const Tuple& consts,
+                           word index) {
+  Bytecode bc = rewrittenBytecodeOpAt(bytecode, index);
+  switch (bc) {
+    case LOAD_CONST:
+      return consts.at(rewrittenBytecodeArgAt(bytecode, index));
+    case LOAD_BOOL:
+      return Bool::fromBool(rewrittenBytecodeArgAt(bytecode, index));
+    case LOAD_IMMEDIATE:
+      return objectFromOparg(rewrittenBytecodeArgAt(bytecode, index));
+    default:
+      UNIMPLEMENTED("unexpected opcode %s", kBytecodeNames[bc]);
+  }
+}
+
 void rewriteBytecode(Thread* thread, const Function& function) {
   HandleScope scope(thread);
   Runtime* runtime = thread->runtime();
@@ -308,12 +345,16 @@ void rewriteBytecode(Thread* thread, const Function& function) {
     return;
   }
   word cache = num_global_caches;
+  bool rewrite_build_slice = false;
   for (word i = 0; i < num_opcodes;) {
     BytecodeOp op = nextBytecodeOp(bytecode, &i);
     word previous_index = i - 1;
     RewrittenOp rewritten = rewriteOperation(function, op);
     if (rewritten.bc == UNUSED_BYTECODE_0) continue;
     if (rewritten.needs_inline_cache) {
+      if (rewritten.bc == LOAD_SLICE_CACHED) {
+        rewrite_build_slice = true;
+      }
       rewrittenBytecodeOpAtPut(bytecode, previous_index, rewritten.bc);
       rewrittenBytecodeArgAtPut(bytecode, previous_index,
                                 static_cast<byte>(rewritten.arg));
@@ -332,6 +373,37 @@ void rewriteBytecode(Thread* thread, const Function& function) {
                         runtime->newMutableTuple(cache * kIcPointersPerEntry));
     caches.fill(NoneType::object());
     function.setCaches(*caches);
+
+    if (rewrite_build_slice) {
+      // Rewrite LOAD_CONSTs before BUILD_SLICE. We need not worry about
+      // EXTENDED_ARG in between because we checked above; we can directly index
+      // backwards.
+      Tuple consts(&scope, Code::cast(function.code()).consts());
+      Object start(&scope, NoneType::object());
+      Object stop(&scope, NoneType::object());
+      Object step(&scope, NoneType::object());
+      Slice slice(&scope, runtime->emptySlice());
+      for (word i = 0; i < num_opcodes;) {
+        BytecodeOp op = nextBytecodeOp(bytecode, &i);
+        if (op.bc == LOAD_SLICE_CACHED) {
+          if (op.arg == 2) {
+            start = constAtOp(bytecode, consts, op.index - 2);
+            stop = constAtOp(bytecode, consts, op.index - 1);
+            step = NoneType::object();
+          } else {
+            start = constAtOp(bytecode, consts, op.index - 3);
+            stop = constAtOp(bytecode, consts, op.index - 2);
+            step = constAtOp(bytecode, consts, op.index - 1);
+            rewrittenBytecodeOpAtPut(bytecode, op.index - 3, NOP);
+          }
+          rewrittenBytecodeOpAtPut(bytecode, op.index - 1, NOP);
+          rewrittenBytecodeOpAtPut(bytecode, op.index - 2, NOP);
+          slice = runtime->newSlice(start, stop, step);
+          word index = op.cache * kIcPointersPerEntry;
+          caches.atPut(index + kIcEntryValueOffset, *slice);
+        }
+      }
+    }
   }
 }
 

runtime/bytecode.h

@@ -151,7 +151,7 @@ namespace py {
   V(CALL_FUNCTION, 131, doCallFunction)                                        \
   V(MAKE_FUNCTION, 132, doMakeFunction)                                        \
   V(BUILD_SLICE, 133, doBuildSlice)                                            \
-  V(UNUSED_BYTECODE_134, 134, doInvalidBytecode)                               \
+  V(LOAD_SLICE_CACHED, 134, doLoadSliceCached)                                 \
   V(LOAD_CLOSURE, 135, doLoadClosure)                                          \
   V(LOAD_DEREF, 136, doLoadDeref)                                              \
   V(STORE_DEREF, 137, doStoreDeref)                                            \
@@ -334,6 +334,7 @@ struct BytecodeOp {
   Bytecode bc;
   int32_t arg;
   uint16_t cache;
+  word index;
 };
 
 BytecodeOp nextBytecodeOp(const MutableBytes& bytecode, word* index);

runtime/interpreter-gen-x64.cpp

@@ -791,6 +791,11 @@ void emitAttrWithOffset(EmitEnv* env, void (Assembler::*asm_op)(Address),
   __ jmp(next, Assembler::kNearJump);
 }
 
+template <>
+void emitHandler<NOP>(EmitEnv* env) {
+  emitNextOpcodeFallthrough(env);
+}
+
 template <>
 void emitHandler<BINARY_ADD_SMALLINT>(EmitEnv* env) {
   ScratchReg r_right(env);

runtime/interpreter.cpp

@@ -4638,6 +4638,17 @@ HANDLER_INLINE Continue Interpreter::doBuildSlice(Thread* thread, word arg) {
   return Continue::NEXT;
 }
 
+HANDLER_INLINE Continue Interpreter::doLoadSliceCached(Thread* thread, word) {
+  Frame* frame = thread->currentFrame();
+  RawMutableTuple caches = MutableTuple::cast(frame->caches());
+  word cache = currentCacheIndex(frame);
+  word index = cache * kIcPointersPerEntry;
+  RawObject slice = caches.at(index + kIcEntryValueOffset);
+  DCHECK(slice.isSlice(), "expected to have a slice in the cache");
+  thread->stackPush(slice);
+  return Continue::NEXT;
+}
+
 HANDLER_INLINE Continue Interpreter::doLoadClosure(Thread* thread, word arg) {
   Frame* frame = thread->currentFrame();
   RawCode code = Code::cast(frame->code());

runtime/interpreter.h

@@ -496,6 +496,7 @@ class Interpreter {
   static Continue doBuildConstKeyMap(Thread* thread, word arg);
   static Continue doBuildList(Thread* thread, word arg);
   static Continue doBuildSlice(Thread* thread, word arg);
+  static Continue doLoadSliceCached(Thread* thread, word arg);
   static Continue doBuildString(Thread* thread, word arg);
   static Continue doBuildTuple(Thread* thread, word arg);
   static Continue doDeleteDeref(Thread* thread, word arg);