NODE-2362: New data entries limit calculation (wavesplatform#3588)

node-it/src/test/scala/com/wavesplatform/it/api/AsyncGrpcApi.scala

@@ -1,40 +1,41 @@
 package com.wavesplatform.it.api
 
+import java.util.NoSuchElementException
+
+import scala.concurrent.Future
+import scala.concurrent.duration.*
+
 import com.google.protobuf.ByteString
 import com.google.protobuf.empty.Empty
 import com.wavesplatform.account.{AddressScheme, Alias, KeyPair}
+import com.wavesplatform.api.grpc.{TransactionStatus as PBTransactionStatus, *}
 import com.wavesplatform.api.grpc.BalanceResponse.WavesBalances
-import com.wavesplatform.api.grpc.{TransactionStatus => PBTransactionStatus, _}
 import com.wavesplatform.common.utils.{Base58, EitherExt2}
 import com.wavesplatform.crypto
 import com.wavesplatform.it.Node
 import com.wavesplatform.it.sync.invokeExpressionFee
-import com.wavesplatform.it.util.GlobalTimer.{instance => timer}
-import com.wavesplatform.it.util._
+import com.wavesplatform.it.util.*
+import com.wavesplatform.it.util.GlobalTimer.instance as timer
+import com.wavesplatform.lang.script.Script as Scr
 import com.wavesplatform.lang.script.v1.ExprScript
-import com.wavesplatform.lang.script.{Script => Scr}
 import com.wavesplatform.lang.v1.Serde
 import com.wavesplatform.lang.v1.compiler.Terms.FUNCTION_CALL
 import com.wavesplatform.protobuf.Amount
 import com.wavesplatform.protobuf.block.PBBlocks
 import com.wavesplatform.serialization.Deser
+import com.wavesplatform.transaction.{Asset, TxVersion}
 import com.wavesplatform.transaction.Asset.Waves
 import com.wavesplatform.transaction.assets.exchange.Order
-import com.wavesplatform.transaction.{Asset, TxVersion}
 import io.grpc.stub.StreamObserver
 import monix.eval.Task
 import monix.execution.Scheduler
 import monix.reactive.subjects.ConcurrentSubject
 import play.api.libs.json.Json
 
-import java.util.NoSuchElementException
-import scala.concurrent.Future
-import scala.concurrent.duration._
-
 object AsyncGrpcApi {
   implicit class NodeAsyncGrpcApi(val n: Node) {
 
-    import com.wavesplatform.protobuf.transaction.{Transaction => PBTransaction, _}
+    import com.wavesplatform.protobuf.transaction.{Transaction as PBTransaction, *}
     import monix.execution.Scheduler.Implicits.global
 
     private[this] lazy val assets       = AssetsApiGrpc.stub(n.grpcChannel)
@@ -56,7 +57,7 @@ object AsyncGrpcApi {
       val (obs, result) = createCallObserver[TransactionResponse]
       transactions.getTransactions(request, obs)
       result.runToFuture.map { r =>
-        import com.wavesplatform.state.{InvokeScriptResult => VISR}
+        import com.wavesplatform.state.InvokeScriptResult as VISR
         r.map { r =>
           val tx = PBTransactions.vanillaUnsafe(r.getTransaction)
           assert(r.getInvokeScriptResult.transfers.forall(_.address.size() == 20))
@@ -196,10 +197,11 @@ object AsyncGrpcApi {
         version,
         PBTransaction.Data.DataTransaction(DataTransactionData.of(data))
       )
-      if (PBTransactions.vanilla(SignedTransaction(SignedTransaction.Transaction.WavesTransaction(unsigned)), unsafe = false).isLeft) {
+      val safeCreated = PBTransactions.vanilla(SignedTransaction(SignedTransaction.Transaction.WavesTransaction(unsigned)), unsafe = false)
+      if (safeCreated.isLeft) {
         transactions.broadcast(SignedTransaction.of(SignedTransaction.Transaction.WavesTransaction(unsigned), Seq(ByteString.EMPTY)))
       } else {
-        val proofs = crypto.sign(source.privateKey, PBTransactions.vanilla(SignedTransaction(SignedTransaction.Transaction.WavesTransaction(unsigned)), unsafe = false).explicitGet().bodyBytes())
+        val proofs = crypto.sign(source.privateKey, safeCreated.explicitGet().bodyBytes())
         transactions.broadcast(SignedTransaction.of(SignedTransaction.Transaction.WavesTransaction(unsigned), Seq(ByteString.copyFrom(proofs.arr))))
       }
     }

node-it/src/test/scala/com/wavesplatform/it/sync/grpc/DataTransactionGrpcSuite.scala

@@ -251,14 +251,14 @@ class DataTransactionGrpcSuite extends GrpcBaseTransactionSuite {
         Code.INVALID_ARGUMENT
       )
       val largeBinData =
-        List.tabulate(5)(n => DataEntry(tooBigKey, DataEntry.Value.BinaryValue(ByteString.copyFrom(Array.fill(maxValueSize)(n.toByte)))))
+        List.tabulate(5)(n => DataEntry(tooBigKey + n.toString, DataEntry.Value.BinaryValue(ByteString.copyFrom(Array.fill(maxValueSize)(n.toByte)))))
       assertGrpcError(
         sender.putData(firstAcc, largeBinData, calcDataFee(largeBinData, v), version = v),
         "Too big sequence requested",
         Code.INVALID_ARGUMENT
       )
 
-      val largeStrData = List.tabulate(5)(n => DataEntry(tooBigKey, DataEntry.Value.StringValue("A" * maxValueSize)))
+      val largeStrData = List.tabulate(5)(n => DataEntry(tooBigKey + n.toString, DataEntry.Value.StringValue("A" * maxValueSize)))
       assertGrpcError(
         sender.putData(firstAcc, largeStrData, calcDataFee(largeStrData, v), version = v),
         "Too big sequence requested",
@@ -297,7 +297,10 @@ class DataTransactionGrpcSuite extends GrpcBaseTransactionSuite {
       val txIds = dataSet
         .grouped(100)
         .map(_.toList)
-        .map(data => PBTransactions.vanilla(sender.putData(fourthAcc, data, calcDataFee(data, v), version = v), unsafe = false).explicitGet().id().toString)
+        .map(
+          data =>
+            PBTransactions.vanilla(sender.putData(fourthAcc, data, calcDataFee(data, v), version = v), unsafe = false).explicitGet().id().toString
+        )
       txIds.foreach(tx => sender.waitForTransaction(tx))
       val r = scala.util.Random.nextInt(199)
       sender.getDataByKey(fourthAddress, s"int$r") shouldBe List(DataEntry(s"int$r", DataEntry.Value.IntValue(1000 + r)))

node-it/src/test/scala/com/wavesplatform/it/sync/smartcontract/DataTransactionBodyBytesByteVectorSuite.scala

@@ -7,7 +7,7 @@ import com.wavesplatform.common.utils.EitherExt2
 import com.wavesplatform.it.NodeConfigs
 import com.wavesplatform.it.api.SyncHttpApi._
 import com.wavesplatform.it.api.TransactionInfo
-import com.wavesplatform.it.sync.{setScriptFee, _}
+import com.wavesplatform.it.sync._
 import com.wavesplatform.it.transactions.BaseTransactionSuite
 import com.wavesplatform.lang.v1.compiler.Terms
 import com.wavesplatform.lang.v1.estimator.v2.ScriptEstimatorV2
@@ -96,7 +96,7 @@ class DataTransactionBodyBytesByteVectorSuite extends BaseTransactionSuite {
     val increasedData = data.head.copy(value = data.head.value ++ ByteStr.fromBytes(1)) :: data.tail
     assertBadRequestAndMessage(
       sender.putData(address, increasedData, version = version, fee = calcDataFee(data, version) + smartFee),
-      "Too big sequence requested"
+      "Transaction is not allowed by account-script"
     )
   }
 }

node-it/src/test/scala/com/wavesplatform/it/sync/transactions/DataTransactionSuite.scala

@@ -312,9 +312,9 @@ class DataTransactionSuite extends BaseTransactionSuite with EitherValues {
     val keys      = Seq("int", "bool", "int", "blob", "?&$#^123\\/.a:;'\"\r\n\t\u0000|%è&", "str", "inexisted_key", tooBigKey)
     val values    = Seq[Any](-127, false, -127, ByteStr(Array[Byte](127.toByte, 0, 1, 1)), "specïal", "BBBB")
 
-    val list     = sender.getDataList(secondAddress, keys*).map(_.value)
-    val jsonList = sender.getDataListJson(secondAddress, keys*).map(_.value)
-    val postList = sender.getDataListPost(secondAddress, keys*).map(_.value)
+    val list     = sender.getDataList(secondAddress, keys *).map(_.value)
+    val jsonList = sender.getDataListJson(secondAddress, keys *).map(_.value)
+    val postList = sender.getDataListPost(secondAddress, keys *).map(_.value)
 
     list shouldBe values
     jsonList shouldBe list
@@ -434,14 +434,14 @@ class DataTransactionSuite extends BaseTransactionSuite with EitherValues {
         "fee"             -> fee,
         "version"         -> version,
         "data"            -> data,
-        "proofs"          -> JsArray(),
+        "proofs"          -> Json.arr(JsString("")),
         "timestamp"       -> System.currentTimeMillis()
       )
     )
 
   test("try to send tx above limits of key, value and entries count") {
     for (v <- dataTxSupportedVersions) {
-      val maxKeySize    = if (v < 2) 100 else 400
+      val maxKeySize    = 400
       val maxValueSize  = Short.MaxValue
       val maxEntryCount = 100
       val TooBig        = "Too big sequence requested"
@@ -458,13 +458,13 @@ class DataTransactionSuite extends BaseTransactionSuite with EitherValues {
       val extraValueData = List(BinaryDataEntry("key", ByteStr(Array.fill(maxValueSize + 1)(1.toByte))))
       assertBadRequestAndResponse(postDataTxJson(firstKeyPair, extraValueData, 1.waves, version = v), TooBig)
 
-      val largeBinData = List.tabulate(5)(n => BinaryDataEntry(extraKey, ByteStr(Array.fill(maxValueSize)(n.toByte))))
+      val largeBinData = List.tabulate(5)(n => BinaryDataEntry(extraKey + n.toString, ByteStr(Array.fill(maxValueSize)(n.toByte))))
       assertBadRequestAndResponse(postDataTxJson(firstKeyPair, largeBinData, 1.waves, version = v), TooBig)
 
-      val largeStrData = List.tabulate(5)(n => StringDataEntry(extraKey, "A" * maxValueSize))
+      val largeStrData = List.tabulate(5)(n => StringDataEntry(extraKey + n.toString, "A" * maxValueSize))
       assertBadRequestAndResponse(postDataTxJson(firstKeyPair, largeStrData, 1.waves, version = v), TooBig)
 
-      val tooManyEntriesData = List.fill(maxEntryCount + 1)(IntegerDataEntry("key", 88))
+      val tooManyEntriesData = List.tabulate(maxEntryCount + 1)(n => IntegerDataEntry("key" + n.toString, 88))
       assertBadRequestAndResponse(postDataTxJson(firstKeyPair, tooManyEntriesData, 1.waves, version = v), TooBig)
     }
   }

node/src/main/scala/com/wavesplatform/network/BasicMessagesRepo.scala

@@ -205,7 +205,7 @@ object TransactionSpec extends MessageSpec[Transaction] {
   override val messageCode: MessageCode = 25: Byte
 
   // Modeled after Data Transaction https://wavesplatform.atlassian.net/wiki/spaces/MAIN/pages/119734321/Data+Transaction
-  override val maxLength: Int = 150 * 1024
+  override val maxLength: Int = (DataTransaction.MaxBytes * 1.2).toInt // 150 * 1024
 
   override def deserializeData(bytes: Array[Byte]): Try[Transaction] =
     TransactionParsers.parseBytes(bytes)
@@ -295,8 +295,8 @@ object PBMicroBlockSpec extends MessageSpec[MicroBlockResponse] {
 object PBTransactionSpec extends MessageSpec[Transaction] {
   override val messageCode: MessageCode = 31: Byte
 
-  // Signed (8 proofs) PBTransaction + max DataTransaction.DataEntry + max proto serialization meta + gap
-  override val maxLength: Int = 624 + DataTransaction.MaxProtoBytes + 5 + 100
+  //624 + DataTransaction.MaxProtoBytes + 5 + 100 // Signed (8 proofs) PBTransaction + max DataTransaction.DataEntry + max proto serialization meta + gap
+  override val maxLength: Int = (DataTransaction.MaxBytes * 1.2).toInt
 
   override def deserializeData(bytes: Array[MessageCode]): Try[Transaction] =
     PBTransactions.tryToVanilla(PBSignedTransaction.parseFrom(bytes))

node/src/main/scala/com/wavesplatform/state/DataEntry.scala

@@ -4,10 +4,10 @@ import com.google.common.primitives.{Bytes, Longs, Shorts}
 import com.wavesplatform.common.state.ByteStr
 import com.wavesplatform.lang.v1.traits.domain.{DataItem, DataOp}
 import com.wavesplatform.serialization.Deser
-import com.wavesplatform.state.DataEntry._
+import com.wavesplatform.state.DataEntry.*
 import com.wavesplatform.transaction.TxVersion
-import com.wavesplatform.utils._
-import play.api.libs.json._
+import com.wavesplatform.utils.*
+import play.api.libs.json.*
 
 sealed abstract class DataEntry[T](val `type`: String, val key: String, val value: T) {
   def valueBytes: Array[Byte]

node/src/main/scala/com/wavesplatform/state/diffs/DataTransactionDiff.scala

@@ -1,19 +1,21 @@
 package com.wavesplatform.state.diffs
 
+import cats.syntax.either._
 import com.wavesplatform.lang.ValidationError
 import com.wavesplatform.state._
 import com.wavesplatform.transaction.DataTransaction
+import com.wavesplatform.transaction.validation.impl.DataTxValidator
 
 object DataTransactionDiff {
-
   def apply(blockchain: Blockchain)(tx: DataTransaction): Either[ValidationError, Diff] = {
     val sender = tx.sender.toAddress
-    Right(
-      Diff(
-        portfolios = Map(sender -> Portfolio(-tx.fee, LeaseBalance.empty, Map.empty)),
-        accountData = Map(sender -> AccountDataInfo(tx.data.map(item => item.key -> item).toMap)),
-        scriptsRun = DiffsCommon.countScriptRuns(blockchain, tx)
-      )
+    for {
+      // Validate data size
+      _ <- DataTxValidator.payloadSizeValidation(blockchain, tx).toEither.leftMap(_.head)
+    } yield Diff(
+      portfolios = Map(sender  -> Portfolio(-tx.fee, LeaseBalance.empty, Map.empty)),
+      accountData = Map(sender -> AccountDataInfo(tx.data.map(item => item.key -> item).toMap)),
+      scriptsRun = DiffsCommon.countScriptRuns(blockchain, tx)
     )
   }
 }

node/src/main/scala/com/wavesplatform/state/diffs/invoke/InvokeDiffsCommon.scala

@@ -1,5 +1,7 @@
 package com.wavesplatform.state.diffs.invoke
 
+import scala.util.{Failure, Right, Success, Try}
+
 import cats.Id
 import cats.instances.list._
 import cats.instances.map._
@@ -23,28 +25,26 @@ import com.wavesplatform.lang.v1.ContractLimits
 import com.wavesplatform.lang.v1.compiler.Terms.{FUNCTION_CALL, _}
 import com.wavesplatform.lang.v1.evaluator.{Log, RejectException, ScriptResult, ScriptResultV4}
 import com.wavesplatform.lang.v1.traits.Environment
-import com.wavesplatform.lang.v1.traits.domain.Tx.{BurnPseudoTx, ReissuePseudoTx, ScriptTransfer, SponsorFeePseudoTx}
 import com.wavesplatform.lang.v1.traits.domain._
+import com.wavesplatform.lang.v1.traits.domain.Tx.{BurnPseudoTx, ReissuePseudoTx, ScriptTransfer, SponsorFeePseudoTx}
 import com.wavesplatform.state._
 import com.wavesplatform.state.diffs.DiffsCommon
 import com.wavesplatform.state.diffs.FeeValidation._
 import com.wavesplatform.state.reader.CompositeBlockchain
+import com.wavesplatform.transaction.{Asset, AssetIdLength, ERC20Address, PBSince, TransactionType}
 import com.wavesplatform.transaction.Asset.{IssuedAsset, Waves}
 import com.wavesplatform.transaction.TxValidationError._
 import com.wavesplatform.transaction.assets.IssueTransaction
 import com.wavesplatform.transaction.smart._
 import com.wavesplatform.transaction.smart.script.ScriptRunner
 import com.wavesplatform.transaction.smart.script.ScriptRunner.TxOrd
+import com.wavesplatform.transaction.smart.script.trace.{AssetVerifierTrace, TracedResult}
 import com.wavesplatform.transaction.smart.script.trace.AssetVerifierTrace.AssetContext
 import com.wavesplatform.transaction.smart.script.trace.TracedResult.Attribute
-import com.wavesplatform.transaction.smart.script.trace.{AssetVerifierTrace, TracedResult}
-import com.wavesplatform.transaction.validation.impl.{LeaseCancelTxValidator, LeaseTxValidator, SponsorFeeTxValidator}
-import com.wavesplatform.transaction.{Asset, AssetIdLength, ERC20Address, PBSince, TransactionType}
+import com.wavesplatform.transaction.validation.impl.{DataTxValidator, LeaseCancelTxValidator, LeaseTxValidator, SponsorFeeTxValidator}
 import com.wavesplatform.utils._
 import shapeless.Coproduct
 
-import scala.util.{Failure, Right, Success, Try}
-
 object InvokeDiffsCommon {
   def txFeeDiff(blockchain: Blockchain, tx: InvokeScriptTransactionLike): Either[GenericError, (Long, Map[Address, Portfolio])] = {
     val attachedFee = tx.fee
@@ -184,7 +184,7 @@ object InvokeDiffsCommon {
     val dataEntries     = actionsByType(classOf[DataOp]).asInstanceOf[List[DataOp]].map(dataItemToEntry)
 
     for {
-      _ <- TracedResult(checkDataEntries(tx, dataEntries, version)).leftMap(FailedTransactionError.dAppExecution(_, storingComplexity))
+      _ <- TracedResult(checkDataEntries(blockchain, tx, dataEntries, version)).leftMap(FailedTransactionError.dAppExecution(_, storingComplexity))
       _ <- TracedResult(checkLeaseCancels(leaseCancelList)).leftMap(FailedTransactionError.dAppExecution(_, storingComplexity))
       _ <- TracedResult(
         Either.cond(
@@ -340,11 +340,7 @@ object InvokeDiffsCommon {
     else
       Right(())
 
-  private[this] def checkDataEntries(
-      tx: InvokeScriptLike,
-      dataEntries: Seq[DataEntry[_]],
-      stdLibVersion: StdLibVersion
-  ): Either[String, Unit] =
+  private def checkDataEntries(blockchain: Blockchain, tx: InvokeScriptLike, dataEntries: Seq[DataEntry[_]], stdLibVersion: StdLibVersion) =
     for {
       _ <- Either.cond(
         dataEntries.length <= ContractLimits.MaxWriteSetSize(stdLibVersion),
@@ -377,12 +373,7 @@ object InvokeDiffsCommon {
         }
         .toLeft(())
 
-      totalDataBytes = dataEntries.map(_.toBytes.length).sum
-      _ <- Either.cond(
-        totalDataBytes <= ContractLimits.MaxWriteSetSizeInBytes,
-        (),
-        s"WriteSet size can't exceed ${ContractLimits.MaxWriteSetSizeInBytes} bytes, actual: $totalDataBytes bytes"
-      )
+      _ <- DataTxValidator.verifyInvokeWriteSet(blockchain, dataEntries)
     } yield ()
 
   private def checkLeaseCancels(leaseCancels: Seq[LeaseCancel]): Either[String, Unit] = {