Add rocksdb_block_cache_numshardbits for issue 1336 (#1339)

Summary:
This fixes #1336

This adds the my.cnf options: rocksdb_block_cache_numshardbits

This option can be set so that RocksDB to fix the number of block cache shards.

The default value is -1 to match existing behavior. When -1 RocksDB code will determine the number of block cache shards as min(6, rocksdb_block_cache_size / min_shard_size) and today min_shard_size is 512K for LRU and 32M for Hyper.

The math above frequently results in a block cache with too many small shards when rocksdb_block_cache_size is not too big (a few GB is not too big) and there will be perf problems that are hard to debug in such a case.

Pull Request resolved: #1339

Differential Revision: D47635762

fbshipit-source-id: 7ca759f9a001dbe1a20978ded5b614c209bd5b1f

mysql-test/r/mysqld--help-notwin.result

@@ -1855,6 +1855,8 @@ The following options may be given as the first argument:
  Deleting rows by primary key lookup, without reading rows
  (Blind Deletes). Blind delete is disabled if the table
  has secondary key
+ --rocksdb-block-cache-numshardbits=# 
+ Block cache numshardbits for RocksDB
  --rocksdb-block-cache-size=# 
  block_cache size for RocksDB
  --rocksdb-block-restart-interval=# 
@@ -3557,6 +3559,7 @@ rocksdb-binlog-ttl FALSE
 rocksdb-binlog-ttl-compaction-ts-interval-secs 3600
 rocksdb-binlog-ttl-compaction-ts-offset-secs 60
 rocksdb-blind-delete-primary-key FALSE
+rocksdb-block-cache-numshardbits -1
 rocksdb-block-cache-size 536870912
 rocksdb-block-restart-interval 16
 rocksdb-block-size 4096

mysql-test/suite/rocksdb/r/rocksdb.result

@@ -907,6 +907,7 @@ rocksdb_binlog_ttl	OFF
 rocksdb_binlog_ttl_compaction_ts_interval_secs	3600
 rocksdb_binlog_ttl_compaction_ts_offset_secs	60
 rocksdb_blind_delete_primary_key	OFF
+rocksdb_block_cache_numshardbits	-1
 rocksdb_block_cache_size	536870912
 rocksdb_block_restart_interval	16
 rocksdb_block_size	4096

mysql-test/suite/rocksdb_sys_vars/r/rocksdb_block_cache_numshardbits_basic.result

@@ -0,0 +1,7 @@
+SET @start_global_value = @@global.ROCKSDB_BLOCK_CACHE_NUMSHARDBITS;
+SELECT @start_global_value;
+@start_global_value
+-1
+"Trying to set variable @@global.ROCKSDB_BLOCK_CACHE_NUMSHARDBITS to 444. It should fail because it is readonly."
+SET @@global.ROCKSDB_BLOCK_CACHE_NUMSHARDBITS   = 444;
+ERROR HY000: Variable 'rocksdb_block_cache_numshardbits' is a read only variable

mysql-test/suite/rocksdb_sys_vars/t/rocksdb_block_cache_numshardbits_basic.test

@@ -0,0 +1,6 @@
+--source include/have_rocksdb.inc
+
+--let $sys_var=ROCKSDB_BLOCK_CACHE_NUMSHARDBITS
+--let $read_only=1
+--let $session=0
+--source ../include/rocksdb_sys_var.inc

storage/rocksdb/ha_rocksdb.cc

@@ -920,6 +920,7 @@ static long long rocksdb_compaction_sequential_deletes_file_size = 0l;
 static uint32_t rocksdb_validate_tables = 1;
 char *rocksdb_datadir;
 static uint32_t rocksdb_max_bottom_pri_background_compactions = 0;
+static int rocksdb_block_cache_numshardbits = -1;
 static uint32_t rocksdb_table_stats_sampling_pct;
 static uint32_t rocksdb_table_stats_recalc_threshold_pct = 10;
 static unsigned long long rocksdb_table_stats_recalc_threshold_count = 100ul;
@@ -2057,6 +2058,13 @@ static MYSQL_SYSVAR_INT(table_cache_numshardbits,
                         // fails to create a cache and returns a nullptr
                         /* min */ 0, /* max */ 19, 0);
 
+static MYSQL_SYSVAR_INT(block_cache_numshardbits,
+                        rocksdb_block_cache_numshardbits,
+                        PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY,
+                        "Block cache numshardbits for RocksDB", nullptr,
+                        nullptr,
+                        /* default */ -1, /* min */ -1, /* max */ 8, 0);
+
 static MYSQL_SYSVAR_UINT64_T(wal_ttl_seconds,
                              rocksdb_db_options->WAL_ttl_seconds,
                              PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY,
@@ -3045,6 +3053,7 @@ static struct SYS_VAR *rocksdb_system_variables[] = {
     MYSQL_SYSVAR(keep_log_file_num),
     MYSQL_SYSVAR(max_manifest_file_size),
     MYSQL_SYSVAR(table_cache_numshardbits),
+    MYSQL_SYSVAR(block_cache_numshardbits),
     MYSQL_SYSVAR(wal_ttl_seconds),
     MYSQL_SYSVAR(wal_size_limit_mb),
     MYSQL_SYSVAR(manifest_preallocation_size),
@@ -7958,13 +7967,12 @@ static int rocksdb_init_internal(void *const p) {
         rocksdb_use_hyper_clock_cache
             ? rocksdb::HyperClockCacheOptions(
                   rocksdb_block_cache_size, rocksdb_tbl_options->block_size,
-                  -1
-                  /* num_shard_bits */,
+                  rocksdb_block_cache_numshardbits,
                   false /* strict_capacity_limit */, memory_allocator)
                   .MakeSharedCache()
 
             : rocksdb::NewLRUCache(
-                  rocksdb_block_cache_size, -1 /*num_shard_bits*/,
+                  rocksdb_block_cache_size, rocksdb_block_cache_numshardbits,
                   false /*strict_capcity_limit*/,
                   rocksdb_cache_high_pri_pool_ratio, memory_allocator);
     if (rocksdb_sim_cache_size > 0) {