package io.vertx.redis.client.impl;
import io.vertx.codegen.annotations.Nullable;
import io.vertx.core.*;
import io.vertx.core.impl.VertxInternal;
import io.vertx.core.impl.logging.Logger;
import io.vertx.core.impl.logging.LoggerFactory;
import io.vertx.redis.client.*;
import io.vertx.redis.client.impl.types.ErrorType;
import java.nio.charset.StandardCharsets;
import java.util.*;
import java.util.function.Function;
import static io.vertx.redis.client.Command.ASKING;
import static io.vertx.redis.client.Command.AUTH;
import static io.vertx.redis.client.Request.cmd;
public class RedisClusterConnection implements RedisConnection {
private static final Logger LOG = LoggerFactory.getLogger(RedisClusterConnection.class);
private static final SplittableRandom RANDOM = new SplittableRandom();
private static final int RETRIES = 16;
private static final Map<Command, String> UNSUPPORTEDCOMMANDS = new HashMap<>();
private static final Map<Command, Function<List<Response>, Response>> REDUCERS = new HashMap<>();
private static final List<Command> MASTER_ONLY_COMMANDS = new ArrayList<>();
public static void addReducer(Command command, Function<List<Response>, Response> fn) {
REDUCERS.put(command, fn);
}
public static void addUnSupportedCommand(Command command, String error) {
if (error == null || error.isEmpty()) {
UNSUPPORTEDCOMMANDS.put(command, "RedisClusterClient does not handle command " +
new String(command.getBytes(), StandardCharsets.ISO_8859_1).split("\r\n")[1] + ", use non cluster client on the right node.");
} else {
UNSUPPORTEDCOMMANDS.put(command, error);
}
}
public static void addMasterOnlyCommand(Command command) {
MASTER_ONLY_COMMANDS.add(command);
}
private final VertxInternal vertx;
private final RedisOptions options;
private final Slots slots;
private final Map<String, RedisConnection> connections;
RedisClusterConnection(Vertx vertx, RedisOptions options, Slots slots, Map<String, RedisConnection> connections) {
this.vertx = (VertxInternal) vertx;
this.options = options;
this.slots = slots;
this.connections = connections;
}
@Override
public RedisConnection exceptionHandler(Handler<Throwable> handler) {
for (RedisConnection conn : connections.values()) {
if (conn != null) {
conn.exceptionHandler(handler);
}
}
return this;
}
@Override
public RedisConnection handler(Handler<Response> handler) {
for (RedisConnection conn : connections.values()) {
if (conn != null) {
conn.handler(handler);
}
}
return this;
}
@Override
public RedisConnection pause() {
for (RedisConnection conn : connections.values()) {
if (conn != null) {
conn.pause();
}
}
return this;
}
@Override
public RedisConnection resume() {
for (RedisConnection conn : connections.values()) {
if (conn != null) {
conn.resume();
}
}
return this;
}
@Override
public RedisConnection fetch(long amount) {
for (RedisConnection conn : connections.values()) {
if (conn != null) {
conn.fetch(amount);
}
}
return this;
}
@Override
public RedisConnection endHandler(@Nullable Handler<Void> handler) {
for (RedisConnection conn : connections.values()) {
if (conn != null) {
conn.endHandler(handler);
}
}
return this;
}
@Override
public Future<Response> send(Request request) {
final Promise<Response> promise = vertx.promise();
final RequestImpl req = (RequestImpl) request;
final Command cmd = req.command();
final boolean forceMasterEndpoint = MASTER_ONLY_COMMANDS.contains(cmd);
if (UNSUPPORTEDCOMMANDS.containsKey(cmd)) {
promise.fail(UNSUPPORTEDCOMMANDS.get(cmd));
return promise.future();
}
if (cmd.isMovable()) {
final byte[][] keys = KeyExtractor.extractMovableKeys(req);
int hashSlot = ZModem.generateMulti(keys);
if (hashSlot == -1) {
promise.fail(buildCrossslotFailureMsg(req));
return promise.future();
}
String[] endpoints = slots.endpointsForKey(hashSlot);
send(selectMasterOrReplicaEndpoint(req.command().isReadOnly(), endpoints, forceMasterEndpoint), RETRIES, req, promise);
return promise.future();
}
if (cmd.isKeyless() && REDUCERS.containsKey(cmd)) {
final List<Future> responses = new ArrayList<>(slots.size());
for (int i = 0; i < slots.size(); i++) {
String[] endpoints = slots.endpointsForSlot(i);
final Promise<Response> p = Promise.promise();
send(selectMasterOrReplicaEndpoint(req.command().isReadOnly(), endpoints, forceMasterEndpoint), RETRIES, req, p);
responses.add(p.future());
}
CompositeFuture.all(responses).onComplete(composite -> {
if (composite.failed()) {
promise.fail(composite.cause());
} else {
promise.complete(REDUCERS.get(cmd).apply(composite.result().list()));
}
});
return promise.future();
}
if (cmd.isKeyless()) {
send(selectEndpoint(-1, cmd.isReadOnly(), forceMasterEndpoint), RETRIES, req, promise);
return promise.future();
}
final List<byte[]> args = req.getArgs();
if (cmd.isMultiKey()) {
int currentSlot = -1;
int start = cmd.getFirstKey() - 1;
int end = cmd.getLastKey();
if (end > 0) {
end--;
}
if (end < 0) {
end = args.size() + (end + 1);
}
int step = cmd.getInterval();
for (int i = start; i < end; i += step) {
int slot = ZModem.generate(args.get(i));
if (currentSlot == -1) {
currentSlot = slot;
continue;
}
if (currentSlot != slot) {
if (!REDUCERS.containsKey(cmd)) {
promise.fail(buildCrossslotFailureMsg(req));
return promise.future();
}
final Map<Integer, Request> requests = splitRequest(cmd, args, start, end, step);
final List<Future> responses = new ArrayList<>(requests.size());
for (Map.Entry<Integer, Request> kv : requests.entrySet()) {
final Promise<Response> p = Promise.promise();
send(selectEndpoint(kv.getKey(), cmd.isReadOnly(), forceMasterEndpoint), RETRIES, kv.getValue(), p);
responses.add(p.future());
}
CompositeFuture.all(responses).onComplete(composite -> {
if (composite.failed()) {
promise.fail(composite.cause());
} else {
promise.complete(REDUCERS.get(cmd).apply(composite.result().list()));
}
});
return promise.future();
}
}
send(selectEndpoint(currentSlot, cmd.isReadOnly(), forceMasterEndpoint), RETRIES, req, promise);
return promise.future();
}
int start = cmd.getFirstKey() - 1;
send(selectEndpoint(ZModem.generate(args.get(start)), cmd.isReadOnly(), forceMasterEndpoint), RETRIES, req, promise);
return promise.future();
}
private Map<Integer, Request> splitRequest(Command cmd, List<byte[]> args, int start, int end, int step) {
final Map<Integer, Request> map = new IdentityHashMap<>();
for (int i = start; i < end; i += step) {
int slot = ZModem.generate(args.get(i));
Request request = map.get(slot);
if (request == null) {
request = Request.cmd(cmd);
for (int j = 0; j < start; j++) {
request.arg(args.get(j));
}
map.put(slot, request);
}
request.arg(args.get(i));
for (int j = i + 1; j < i + step; j++) {
request.arg(args.get(j));
}
}
final Collection<Request> col = map.values();
col.forEach(req -> {
for (int j = end; j < args.size(); j++) {
req.arg(args.get(j));
}
});
return map;
}
private void send(String endpoint, int retries, Request command, Handler<AsyncResult<Response>> handler) {
final RedisConnection connection = connections.get(endpoint);
if (connection == null) {
handler.handle(Future.failedFuture("Missing connection to: " + endpoint));
return;
}
connection.send(command, send -> {
if (send.failed() && send.cause() instanceof ErrorType && retries >= 0) {
final ErrorType cause = (ErrorType) send.cause();
if (cause.is("MOVED")) {
handler.handle(Future.failedFuture(cause));
return;
}
if (cause.is("ASK")) {
connection.send(cmd(ASKING), asking -> {
if (asking.failed()) {
handler.handle(Future.failedFuture(asking.cause()));
return;
}
String addr = cause.slice(' ', 2);
if (addr == null) {
handler.handle(Future.failedFuture(cause));
return;
}
final RedisURI uri = new RedisURI(endpoint);
send(uri.protocol() + "://" + uri.userinfo() + addr, retries - 1, command, handler);
});
return;
}
if (cause.is("TRYAGAIN") || cause.is("CLUSTERDOWN")) {
long backoff = (long) (Math.pow(2, 16 - Math.max(retries, 9)) * 10);
vertx.setTimer(backoff, t -> send(endpoint, retries - 1, command, handler));
return;
}
if (cause.is("NOAUTH") && options.getPassword() != null) {
connection.send(cmd(AUTH).arg(options.getPassword()), auth -> {
if (auth.failed()) {
handler.handle(Future.failedFuture(auth.cause()));
return;
}
send(endpoint, retries - 1, command, handler);
});
return;
}
}
try {
handler.handle(send);
} catch (RuntimeException e) {
LOG.error("Handler failure", e);
}
});
}
@Override
public Future<List<Response>> batch(List<Request> requests) {
final Promise<List<Response>> promise = vertx.promise();
int currentSlot = -1;
boolean readOnly = false;
boolean forceMasterEndpoint = false;
for (Request request : requests) {
final RequestImpl req = (RequestImpl) request;
final Command cmd = req.command();
if (UNSUPPORTEDCOMMANDS.containsKey(cmd)) {
promise.fail(UNSUPPORTEDCOMMANDS.get(cmd));
return promise.future();
}
readOnly |= cmd.isReadOnly();
forceMasterEndpoint |= MASTER_ONLY_COMMANDS.contains(cmd);
if (cmd.isKeyless()) {
continue;
}
if (cmd.isMovable()) {
final byte[][] keys = KeyExtractor.extractMovableKeys(req);
int slot = ZModem.generateMulti(keys);
if (slot == -1 || (currentSlot != -1 && currentSlot != slot)) {
promise.fail(buildCrossslotFailureMsg(req));
return promise.future();
}
currentSlot = slot;
continue;
}
final List<byte[]> args = req.getArgs();
if (cmd.isMultiKey()) {
int start = cmd.getFirstKey() - 1;
int end = cmd.getLastKey();
if (end > 0) {
end--;
}
if (end < 0) {
end = args.size() + (end + 1);
}
int step = cmd.getInterval();
for (int j = start; j < end; j += step) {
int slot = ZModem.generate(args.get(j));
if (currentSlot == -1) {
currentSlot = slot;
continue;
}
if (currentSlot != slot) {
promise.fail(buildCrossslotFailureMsg(req));
return promise.future();
}
}
continue;
}
final int start = cmd.getFirstKey() - 1;
final int slot = ZModem.generate(args.get(start));
if (currentSlot == -1) {
currentSlot = slot;
continue;
}
if (currentSlot != slot) {
promise.fail(buildCrossslotFailureMsg(req));
return promise.future();
}
}
batch(selectEndpoint(currentSlot, readOnly, forceMasterEndpoint), RETRIES, requests, promise);
return promise.future();
}
private void batch(String endpoint, int retries, List<Request> commands, Handler<AsyncResult<List<Response>>> handler) {
final RedisConnection connection = connections.get(endpoint);
if (connection == null) {
handler.handle(Future.failedFuture("Missing connection to: " + endpoint));
return;
}
connection.batch(commands, send -> {
if (send.failed() && send.cause() instanceof ErrorType && retries >= 0) {
final ErrorType cause = (ErrorType) send.cause();
if (cause.is("MOVED")) {
handler.handle(Future.failedFuture(cause));
return;
}
if (cause.is("ASK")) {
connection.send(cmd(ASKING), asking -> {
if (asking.failed()) {
handler.handle(Future.failedFuture(asking.cause()));
return;
}
String addr = cause.slice(' ', 2);
if (addr == null) {
handler.handle(Future.failedFuture(cause));
return;
}
final RedisURI uri = new RedisURI(endpoint);
batch(uri.protocol() + "://" + uri.userinfo() + addr, retries - 1, commands, handler);
});
return;
}
if (cause.is("TRYAGAIN") || cause.is("CLUSTERDOWN")) {
long backoff = (long) (Math.pow(2, 16 - Math.max(retries, 9)) * 10);
vertx.setTimer(backoff, t -> batch(endpoint, retries - 1, commands, handler));
return;
}
if (cause.is("NOAUTH") && options.getPassword() != null) {
connection.send(cmd(AUTH).arg(options.getPassword()), auth -> {
if (auth.failed()) {
handler.handle(Future.failedFuture(auth.cause()));
return;
}
batch(endpoint, retries - 1, commands, handler);
});
return;
}
}
try {
handler.handle(send);
} catch (RuntimeException e) {
LOG.error("Handler failure", e);
}
});
}
@Override
public void close() {
for (RedisConnection conn : connections.values()) {
if (conn != null) {
conn.close();
}
}
}
@Override
public boolean pendingQueueFull() {
for (RedisConnection conn : connections.values()) {
if (conn != null) {
if (conn.pendingQueueFull()) {
return true;
}
}
}
return false;
}
private String selectEndpoint(int keySlot, boolean readOnly, boolean forceMasterEndpoint) {
if (keySlot == -1) {
return slots.randomEndPoint(forceMasterEndpoint);
}
String[] endpoints = slots.endpointsForKey(keySlot);
if (endpoints == null || endpoints.length == 0) {
return options.getEndpoint();
}
return selectMasterOrReplicaEndpoint(readOnly, endpoints, forceMasterEndpoint);
}
private String selectMasterOrReplicaEndpoint(boolean readOnly, String[] endpoints, boolean forceMasterEndpoint) {
if (forceMasterEndpoint) {
return endpoints[0];
}
RedisReplicas useReplicas = options.getUseReplicas();
if (readOnly && useReplicas != RedisReplicas.NEVER && endpoints.length > 1) {
switch (useReplicas) {
case ALWAYS:
return endpoints[1 + RANDOM.nextInt(endpoints.length - 1)];
case SHARE:
return endpoints[RANDOM.nextInt(endpoints.length)];
}
}
return endpoints[0];
}
private String buildCrossslotFailureMsg(RequestImpl req) {
return "Keys of command or batch: \"" + req.toString() + "\" targets not all in the same hash slot (CROSSSLOT) and client side resharding is not supported";
}
}