Skip to content

Commit

Permalink
Merge "Redis API in Scylla"
Browse files Browse the repository at this point in the history 
Merged patch series from Peng Jian, adding optionally-enabled Redis API
support to Scylla. This feature is experimental, and partial - the extent
of this support is detailed in docs/redis/redis.md.

Patches:
   Document: add docs/redis/redis.md
   redis: Redis API in Scylla
   Redis API: graft redis module to Scylla
   redis-test: add test cases for Redis API
  • Loading branch information
@nyh
nyh committed Nov 20, 2019
2 parents 086e744 + e6adddd commit 89d6d66
Show file tree
Hide file tree
Showing 44 changed files with 3,293 additions and 1 deletion.
42 changes: 41 additions & 1 deletion configure.py
View file
Expand Up @@ -456,6 +456,8 @@ def find_headers(repodir, excluded_dirs):
help='enable allocation failure injection')
arg_parser.add_argument('--with-antlr3', dest='antlr3_exec', action='store', default=None,
help='path to antlr3 executable')
arg_parser.add_argument('--with-ragel', dest='ragel_exec', action='store', default='ragel',
help='path to ragel executable')
args = arg_parser.parse_args()

defines = ['XXH_PRIVATE_API',
Expand Down Expand Up @@ -802,6 +804,26 @@ def find_headers(repodir, excluded_dirs):
'alternator/auth.cc',
]

redis = [
'redis/service.cc',
'redis/server.cc',
'redis/query_processor.cc',
'redis/protocol_parser.rl',
'redis/keyspace_utils.cc',
'redis/options.cc',
'redis/stats.cc',
'redis/mutation_utils.cc',
'redis/query_utils.cc',
'redis/abstract_command.cc',
'redis/command_factory.cc',
'redis/commands/unknown.cc',
'redis/commands/ping.cc',
'redis/commands/set.cc',
'redis/commands/get.cc',
'redis/commands/del.cc',
'redis/commands/select.cc',
]

idls = ['idl/gossip_digest.idl.hh',
'idl/uuid.idl.hh',
'idl/range.idl.hh',
Expand Down Expand Up @@ -847,7 +869,7 @@ def find_headers(repodir, excluded_dirs):
]

deps = {
'scylla': idls + ['main.cc', 'release.cc'] + scylla_core + api + alternator,
'scylla': idls + ['main.cc', 'release.cc'] + scylla_core + api + alternator + redis,
}

pure_boost_tests = set([
Expand Down Expand Up @@ -1237,6 +1259,11 @@ def configure_zstd(build_dir, mode):
else:
antlr3_exec = "antlr3"

if args.ragel_exec:
ragel_exec = args.ragel_exec
else:
ragel_exec = "ragel"

for mode in build_modes:
configure_zstd(outdir, mode)

Expand Down Expand Up @@ -1271,6 +1298,11 @@ def configure_zstd(build_dir, mode):
command = {ninja} -C $subdir $target
restat = 1
description = NINJA $out
rule ragel
# sed away a bug in ragel 7 that emits some extraneous _nfa* variables
# (the $$ is collapsed to a single one by ninja)
command = {ragel_exec} -G2 -o $out $in && sed -i -e '1h;2,$$H;$$!d;g' -re 's/static const char _nfa[^;]*;//g' $out
description = RAGEL $out
rule run
command = $in > $out
description = GEN $out
Expand Down Expand Up @@ -1335,6 +1367,7 @@ def configure_zstd(build_dir, mode):
swaggers = {}
serializers = {}
thrifts = set()
ragels = {}
antlr3_grammars = set()
seastar_dep = 'build/{}/seastar/libseastar.a'.format(mode)
for binary in build_artifacts:
Expand Down Expand Up @@ -1392,6 +1425,9 @@ def configure_zstd(build_dir, mode):
elif src.endswith('.json'):
hh = '$builddir/' + mode + '/gen/' + src + '.hh'
swaggers[hh] = src
elif src.endswith('.rl'):
hh = '$builddir/' + mode + '/gen/' + src.replace('.rl', '.hh')
ragels[hh] = src
elif src.endswith('.thrift'):
thrifts.add(src)
elif src.endswith('.g'):
Expand Down Expand Up @@ -1420,6 +1456,7 @@ def configure_zstd(build_dir, mode):
gen_headers += g.headers('$builddir/{}/gen'.format(mode))
gen_headers += list(swaggers.keys())
gen_headers += list(serializers.keys())
gen_headers += list(ragels.keys())
gen_headers_dep = ' '.join(gen_headers)

for obj in compiles:
Expand All @@ -1433,6 +1470,9 @@ def configure_zstd(build_dir, mode):
for hh in serializers:
src = serializers[hh]
f.write('build {}: serializer {} | idl-compiler.py\n'.format(hh, src))
for hh in ragels:
src = ragels[hh]
f.write('build {}: ragel {}\n'.format(hh, src))
for thrift in thrifts:
outs = ' '.join(thrift.generated('$builddir/{}/gen'.format(mode)))
f.write('build {}: thrift.{} {}\n'.format(outs, mode, thrift.source))
Expand Down
15 changes: 15 additions & 0 deletions db/config.cc
View file
Expand Up @@ -713,6 +713,21 @@ db::config::config(std::shared_ptr<db::extensions> exts)
, alternator_address(this, "alternator_address", value_status::Used, "0.0.0.0", "Alternator API listening address")
, alternator_enforce_authorization(this, "alternator_enforce_authorization", value_status::Used, false, "Enforce checking the authorization header for every request in Alternator")
, abort_on_ebadf(this, "abort_on_ebadf", value_status::Used, true, "Abort the server on incorrect file descriptor access. Throws exception when disabled.")
, redis_transport_port(this, "redis_transport_port", value_status::Used, 6379, "Port on which the REDIS transport listens for clients.")
// FIXME: 9142 has been used by native_transport_port_ssl
, redis_transport_port_ssl(this, "redis_transport_port_ssl", value_status::Used, 9142, "Port on which the REDIS TLS native transport listens for clients.")
, enable_redis_protocol(this, "enable_redis_protocol", value_status::Used, false, "Enable redis protocol; Scylla will process redis protocol as a redis cluster if enable.")
, redis_read_consistency_level(this, "redis_read_consistency_level", value_status::Used, "ONE", "Consistency level for read operations for redis.")
, redis_write_consistency_level(this, "redis_write_consistency_level", value_status::Used, "ANY", "Consistency level for write operations for redis.")
, redis_default_database_count(this, "redis_default_database_count", value_status::Used, 16, "Default database count for the redis cluster.")
, redis_keyspace_options(this, "redis_keyspace_options", value_status::Used, {},
"Enable redis protocol, list of properties of replication of redis keyspace. The available options are:\n"
"\n"
"\tclass : (Default: SimpleStrategy ).\n"
"\treplication_factor: (Default: 1) If the class is SimpleStrategy, set the replication factor for the strategy.\n"
"\tdatacenter_i: (Default: 1) If the class is NetworkTopologyStrategy, set the replication factor per datacenter.\n"
"\n"
"The properties of replication for redis keyspace.")
, default_log_level(this, "default_log_level", value_status::Used)
, logger_log_level(this, "logger_log_level", value_status::Used)
Expand Down
8 changes: 8 additions & 0 deletions db/config.hh
View file
Expand Up @@ -297,6 +297,14 @@ public:
named_value<bool> alternator_enforce_authorization;
named_value<bool> abort_on_ebadf;

named_value<uint16_t> redis_transport_port;
named_value<uint16_t> redis_transport_port_ssl;
named_value<bool> enable_redis_protocol;
named_value<sstring> redis_read_consistency_level;
named_value<sstring> redis_write_consistency_level;
named_value<uint16_t> redis_default_database_count;
named_value<string_map> redis_keyspace_options;

seastar::logging_settings logging_settings(const boost::program_options::variables_map&) const;

boost::program_options::options_description_easy_init&
Expand Down

0 comments on commit 89d6d66

Please sign in to comment.