[ETCM-1166] Pass ETS GeneralStateTests for Berlin (#1120)

* Adds accessed address even in case of failure

* Adds accessed address and storage in call result

* Fix refund calculation

* avoid adding access list after an error

* update ets tests to 8.0.5 (latest for Berlin)

* add created contract to accessedAddress list

* activate Berlin tests

* [ETCM-1166] Fix AccessedStorageKeys for CREATE/CREATE2 opcodes

* [ETCM-1166] Add gasUsed data in executed Opcodes logs

* Add accessed address in create only if the call is not invalid

* formatAll

Co-authored-by: Aurélien Richez <aurelien.richez@iohk.io>

Author: strauss-m

ets/config/mantis/config

@@ -14,6 +14,7 @@
       "Constantinople",
       "ConstantinopleFix",
       "Istanbul",
+      "Berlin",
       "London"
     ],
     "additionalForks" : [

ets/tests

@@ -113,7 +113,10 @@ class VMClient(externalVmConfig: VmConfig.ExternalConfig, messageHandler: Messag
       resultMsg.logs.map(l => TxLogEntry(l.address, l.topics.map(t => t: ByteString), l.data)),
       Nil,
       resultMsg.gasRefund,
-      if (resultMsg.error) Some(OutOfGas) else None
+      if (resultMsg.error) Some(OutOfGas) else None,
+      // FIXME handle accessed addresses and storage in extVM
+      Set.empty,
+      Set.empty
     )
   }
 

src/main/scala/io/iohk/ethereum/extvm/VMClient.scala

@@ -712,7 +712,7 @@ case object SSTORE extends OpCode(0x55, 2, 0, _.G_zero) {
 
           val reset = if (originalValue == newValue.toBigInt) {
             if (UInt256(originalValue).isZero)
-              state.config.feeSchedule.R_sclear + state.config.feeSchedule.G_sreset - state.config.feeSchedule.G_sload
+              state.config.feeSchedule.G_sset - state.config.feeSchedule.G_sload
             else
               state.config.feeSchedule.G_sreset - state.config.feeSchedule.G_sload
           } else BigInt(0)
@@ -986,16 +986,16 @@ abstract class CreateOp(code: Int, delta: Int) extends OpCode(code, delta, 1, _.
     }
 
     result.error match {
-      case Some(err) =>
-        val world2 = if (err == InvalidCall) state.world else world1
+      case Some(error) =>
+        val world2 = if (error == InvalidCall) state.world else world1
         val resultStack = stack2.push(UInt256.Zero)
-        val returnData = if (err == RevertOccurs) result.returnData else ByteString.empty
+        val returnData = if (error == RevertOccurs) result.returnData else ByteString.empty
         state
           .spendGas(startGas - result.gasRemaining)
           .withWorld(world2)
           .withStack(resultStack)
           .withReturnData(returnData)
-          .addAccessedAddress(newAddress)
+          .addAccessedAddresses(if (error == InvalidCall) Set.empty else Set(newAddress))
           .step()
 
       case None =>
@@ -1013,7 +1013,8 @@ abstract class CreateOp(code: Int, delta: Int) extends OpCode(code, delta, 1, _.
           .withMemory(memory1)
           .withInternalTxs(internalTx +: result.internalTxs)
           .withReturnData(ByteString.empty)
-          .addAccessedAddress(newAddress)
+          .addAccessedStorageKeys(result.accessedStorageKeys)
+          .addAccessedAddresses(result.accessedAddresses + newAddress)
           .step()
     }
   }
@@ -1106,6 +1107,7 @@ abstract class CallOp(code: Int, delta: Int, alpha: Int) extends OpCode(code, de
           .withWorld(world1)
           .spendGas(gasAdjustment)
           .withReturnData(result.returnData)
+          .addAccessedAddress(toAddr)
           .step()
 
       case None =>
@@ -1122,7 +1124,8 @@ abstract class CallOp(code: Int, delta: Int, alpha: Int) extends OpCode(code, de
           .withInternalTxs(internalTx +: result.internalTxs)
           .withLogs(result.logs)
           .withReturnData(result.returnData)
-          .addAccessedAddress(toAddr)
+          .addAccessedStorageKeys(result.accessedStorageKeys)
+          .addAccessedAddresses(result.accessedAddresses + toAddr)
           .step()
     }
   }

src/main/scala/io/iohk/ethereum/vm/OpCode.scala

@@ -107,7 +107,9 @@ object PrecompiledContracts {
         Nil,
         Nil,
         0,
-        error
+        error,
+        Set.empty,
+        Set.empty
       )
     }
   }

src/main/scala/io/iohk/ethereum/vm/PrecompiledContracts.scala

@@ -22,5 +22,7 @@ case class ProgramResult[W <: WorldStateProxy[W, S], S <: Storage[S]](
     logs: Seq[TxLogEntry],
     internalTxs: Seq[InternalTransaction],
     gasRefund: BigInt,
-    error: Option[ProgramError]
+    error: Option[ProgramError],
+    accessedAddresses: Set[Address],
+    accessedStorageKeys: Set[(Address, BigInt)]
 )

src/main/scala/io/iohk/ethereum/vm/ProgramResult.scala

@@ -142,6 +142,12 @@ case class ProgramState[W <: WorldStateProxy[W, S], S <: Storage[S]](
   def addAccessedStorageKey(addr: Address, storageKey: BigInt): ProgramState[W, S] =
     copy(accessedStorageKeys = accessedStorageKeys + ((addr, storageKey)))
 
+  def addAccessedAddresses(addresses: Set[Address]): ProgramState[W, S] =
+    copy(accessedAddresses = accessedAddresses ++ addresses)
+
+  def addAccessedStorageKeys(storageKeys: Set[(Address, BigInt)]): ProgramState[W, S] =
+    copy(accessedStorageKeys = accessedStorageKeys ++ storageKeys)
+
   def toResult: ProgramResult[W, S] =
     ProgramResult[W, S](
       returnData,
@@ -151,6 +157,8 @@ case class ProgramState[W <: WorldStateProxy[W, S], S <: Storage[S]](
       logs,
       internalTxs,
       gasRefund,
-      error
+      error,
+      accessedAddresses,
+      accessedStorageKeys
     )
 }

src/main/scala/io/iohk/ethereum/vm/ProgramState.scala

@@ -42,7 +42,7 @@ class VM[W <: WorldStateProxy[W, S], S <: Storage[S]] extends Logger {
     */
   private[vm] def call(context: PC, ownerAddr: Address): PR =
     if (!isValidCall(context))
-      invalidCallResult(context)
+      invalidCallResult(context, Set.empty, Set.empty)
     else {
       require(context.recipientAddr.isDefined, "Recipient address must be defined for message call")
 
@@ -69,7 +69,7 @@ class VM[W <: WorldStateProxy[W, S], S <: Storage[S]] extends Logger {
       salt: Option[UInt256] = None
   ): (PR, Address) =
     if (!isValidCall(context))
-      (invalidCallResult(context), Address(0))
+      (invalidCallResult(context, Set.empty, Set.empty), Address(0))
     else {
       require(context.recipientAddr.isEmpty, "recipient address must be empty for contract creation")
       require(context.doTransfer, "contract creation will alwyas transfer funds")
@@ -96,15 +96,16 @@ class VM[W <: WorldStateProxy[W, S], S <: Storage[S]] extends Logger {
         */
       val originInitialisedAccount = context.originalWorld.initialiseAccount(newAddress)
 
-      val world1 =
+      val world1: W =
         context.world.initialiseAccount(newAddress).transfer(context.callerAddr, newAddress, context.endowment)
 
       val code = if (conflict) ByteString(INVALID.code) else context.inputData
 
       val env = ExecEnv(context, code, newAddress).copy(inputData = ByteString.empty)
 
       val initialState: PS =
-        ProgramState(this, context.copy(world = world1, originalWorld = originInitialisedAccount), env)
+        ProgramState(this, context.copy(world = world1, originalWorld = originInitialisedAccount): PC, env)
+          .addAccessedAddress(newAddress)
 
       val execResult = exec(initialState).toResult
 
@@ -119,7 +120,9 @@ class VM[W <: WorldStateProxy[W, S], S <: Storage[S]] extends Logger {
       case Some(opCode) =>
         val newState = opCode.execute(state)
         import newState._
-        log.trace(s"$opCode | pc: $pc | depth: ${env.callDepth} | gas: $gas | stack: $stack")
+        log.trace(
+          s"$opCode | pc: $pc | depth: ${env.callDepth} | gasUsed: ${state.gas - gas} | gas: $gas | stack: $stack"
+        )
         if (newState.halted)
           newState
         else
@@ -134,8 +137,23 @@ class VM[W <: WorldStateProxy[W, S], S <: Storage[S]] extends Logger {
     context.endowment <= context.world.getBalance(context.callerAddr) &&
       context.callDepth <= EvmConfig.MaxCallDepth
 
-  private def invalidCallResult(context: PC): PR =
-    ProgramResult(ByteString.empty, context.startGas, context.world, Set(), Nil, Nil, 0, Some(InvalidCall))
+  private def invalidCallResult(
+      context: PC,
+      accessedAddresses: Set[Address],
+      accessedStorageKeys: Set[(Address, BigInt)]
+  ): PR =
+    ProgramResult(
+      ByteString.empty,
+      context.startGas,
+      context.world,
+      Set(),
+      Nil,
+      Nil,
+      0,
+      Some(InvalidCall),
+      accessedAddresses,
+      accessedStorageKeys
+    )
 
   private def exceedsMaxContractSize(context: PC, config: EvmConfig, contractCode: ByteString): Boolean = {
     lazy val maxCodeSizeExceeded = config.maxCodeSize.exists(codeSizeLimit => contractCode.size > codeSizeLimit)

src/main/scala/io/iohk/ethereum/vm/VM.scala

@@ -36,7 +36,9 @@ object Mocks {
       logs = Nil,
       internalTxs = Nil,
       gasRefund = 20000,
-      error = None
+      error = None,
+      Set.empty,
+      Set.empty
     )
 
   class MockVM(runFn: PC => PR = defaultProgramResult) extends VMImpl {

src/test/scala/io/iohk/ethereum/Mocks.scala

@@ -141,7 +141,9 @@ trait TestSetup extends SecureRandomBuilder with EphemBlockchainTestSetup {
     logs = logs,
     internalTxs = Nil,
     gasRefund = gasRefund,
-    error = error
+    error = error,
+    Set.empty,
+    Set.empty
   )
 
   sealed trait Changes