WIP: Sync changes from twitter/twemproxy

.clang-format.example

@@ -0,0 +1,45 @@
+# This may be useful with https://clang.llvm.org/docs/ClangFormat.html#script-for-patch-reformatting
+# but some of clang's reformattings may conflict with notes/c-style-guide.txt, which takes precedence.
+#
+# This is deliberately named .clang-format.example instead of .clang-format to avoid editors 
+# unexpectedly using this to reformatting entire files because of these shortcomings.
+# It can be copied into .clang-format to use this for local development selectively.
+#
+# Note: this clang format file has many shortcomings.
+# Attempting to apply this automatically to everything may make code less readable.
+# However, this may be useful for spot checking new code,
+# or if you're not certain of indentation style or general spacing/wrapping rules
+#
+# Known shortcomings:
+# - Some places exceed the 80 column limit deliberately for readability, e.g. help strings or error messages or function prototypes. (BreakStringLiterals helps preserve some of those)
+# - Some places deliberately put blocks on a single line when there are a lot of similar blocks.
+#   AllowShortBlocksOnASingleLine is not useful.
+# - Some places deliberately put blocks on a single line when there are a lot of similar blocks.
+# - No good way to eliminate space before and after PRIu64 and other macros for adjacent string literal concatenation
+# - clang-format is not aware of macros, some of which have different styles from functions.
+# - Function declarations are not typically aligned
+# - Some variable declarations are aligned and others aren't on a case by case basis
+# - The choice of function argument grouping should depends on which function arguments are semantically related,
+#   not just on fitting within 80 columns.
+
+AllowShortBlocksOnASingleLine: true
+BasedOnStyle: LLVM
+AlwaysBreakAfterDefinitionReturnType: All
+UseTab: Never
+IndentWidth: 4
+TabWidth: 4
+ColumnLimit: 80
+BreakBeforeBraces: Linux
+SortIncludes: false
+
+BreakStringLiterals: false
+# BitFieldColonSpacing is too new to work in clang-format 11
+# https://releases.llvm.org/11.0.0/tools/clang/docs/ClangFormatStyleOptions.html
+# Latest: https://clang.llvm.org/docs/ClangFormatStyleOptions.html
+#
+# BitFieldColonSpacing: None
+#
+# XXX no way to treat the `*` indicating a value is a pointer as part of the aligned name for the declaration
+# XXX function declarations are not typically aligned
+AlignConsecutiveDeclarations: true
+AlignConsecutiveMacros: true

.dockerignore

@@ -51,7 +51,7 @@ core*
 nutcracker
 
 # extracted yaml
-!/contrib/yaml-0.1.4.tar.gz
+!/contrib/yaml-0.2.5.tar.gz
 
 # Autotools
 .deps

.editorconfig

@@ -0,0 +1,19 @@
+# https://editorconfig.org/
+
+root = true
+
+[*]
+trim_trailing_whitespace = true
+insert_final_newline     = true
+end_of_line              = lf
+charset                  = utf-8
+
+# See twemproxy/notes/c-styleguide.txt
+[*.c,*.h]
+tab_width                = 4
+indent_size              = 4
+indent_style             = space
+# indent_brace_style depends on function vs conditional
+max_line_length          = 80
+spaces_around_brackets   = none
+spaces_around_operators  = true

.gitignore

@@ -41,7 +41,7 @@ core*
 nutcracker
 
 # extracted yaml
-!/contrib/yaml-0.1.4.tar.gz
+!/contrib/yaml-0.2.5.tar.gz
 
 # Autotools
 .deps
@@ -71,3 +71,11 @@ nutcracker
 
 Makefile
 Makefile.in
+
+# The .clang-format.example file may be copied here for use with -style=file
+.clang-format
+
+test_all
+*.trs
+tags
+

ChangeLog

@@ -9,7 +9,6 @@
 	  Fix memory corruption seen when all hosts are ejected from a pool with
 	  auto_eject_hosts: true (tyson)
 	  Update tests to work with python 3 (tyson)
-	  Ensure file permissions aren't chosen based on uninitialized memory. (tyson)
 	  Add a heartbeat patch to detect dead servers (charsyam)
 	  Add improvements to the heartbeat patch (tyson)
 	  Decrease the verbosity of logging clients establishing/closing connections to
@@ -32,15 +31,41 @@
 	  server when reconnecting to a server. Reduce concurrent reconnection attempts. (tyson)
 
 2016-04-27  Misha Nasledov <[email protected]>
-    * twemproxy: version 0.5.1 release
+    * twemproxy: version 0.5.1 release (ifwe fork on github)
       updated to latest master from twitter/twemproxy
       updated to latest version of andyqzb's sentinel patch
 
 2015-07-14  Misha Nasledov <[email protected]>
-    * twemproxy: version 0.5.0 release
+    * twemproxy: version 0.5.0 release (ifwe fork on github)
       redis-sentinel support (andyqzb)
 
-2015-22-06  Manju Rajashekhar  <[email protected]>
+ 2021-??-??  Tyson Andre
+    * twemproxy: version 0.5.0 release (dev)
+	  Fix parsing of redis error response for error type with no space,
+	  add tests (tyson, tom dalton)
+	  Update integration tests, add C unit test suite for 'make check' (tyson)
+	  Increase the maximum host length+port+identifier to 273
+	  in ketama_update (李广博)
+	  Always initialize file permissions field when listening on a unix domain
+	  socket (tyson)
+	  Use number of servers instead of number of points on the continuum when
+	  sharding requests to backend services to improve sharding performance
+	  and fix potential invalid memory access when all hosts were ejected
+	  from a pool. (tyson)
+	  Don't fragment memcache/redis get commands when they only have a single
+	  key (improves performance and error handling of single key case) (tyson)
+	  Don't let requests hang when there is a dns error when processing a
+	  fragmented request (e.g. multiget with multiple keys) (tyson)
+	  Allow extra parameters for redis spop (charsyam)
+	  Update documentation and README (various)
+	  Fix memory leak bug for redis mset (deep011)
+	  Support arbitrarily deep nested redis multi-bulk
+	  responses (nested arrays) (qingping209, tyson)
+	  Upgrade from libyaml 0.1.4 to 0.2.5 (tyson)
+	  Fix compiler warnings about wrong conversion specifiers in format
+	  strings for logging (tyson)
+
+ 2015-22-06  Manju Rajashekhar  <[email protected]>
     * twemproxy: version 0.4.1 release
       backend server hostnames are resolved lazily
       redis_auth is only valid for a redis pool

NOTICE

@@ -101,7 +101,7 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-The source also includes libyaml (yaml-0.1.4) in contrib/ directory
+The source also includes libyaml (yaml-0.2.5) in contrib/ directory
 
 Copyright (c) 2006 Kirill Simonov
 

README.md

@@ -83,23 +83,23 @@ Twemproxy can be configured through a YAML file specified by the -c or --conf-fi
 + **listen**: The listening address and port (name:port or ip:port) or an absolute path to sock file (e.g. /var/run/nutcracker.sock) for this server pool.
 + **client_connections**: The maximum number of connections allowed from redis clients. Unlimited by default, though OS-imposed limitations will still apply.
 + **hash**: The name of the hash function. Possible values are:
- + one_at_a_time
- + md5
- + crc16
- + crc32 (crc32 implementation compatible with [libmemcached](http://libmemcached.org/))
- + crc32a (correct crc32 implementation as per the spec)
- + fnv1_64
- + fnv1a_64
- + fnv1_32
- + fnv1a_32
- + hsieh
- + murmur
- + jenkins
+  + one_at_a_time
+  + md5
+  + crc16
+  + crc32 (crc32 implementation compatible with [libmemcached](http://libmemcached.org/))
+  + crc32a (correct crc32 implementation as per the spec)
+  + fnv1_64
+  + fnv1a_64
+  + fnv1_32
+  + fnv1a_32
+  + hsieh
+  + murmur
+  + jenkins
 + **hash_tag**: A two character string that specifies the part of the key used for hashing. Eg "{}" or "$$". [Hash tag](notes/recommendation.md#hash-tags) enable mapping different keys to the same server as long as the part of the key within the tag is the same.
 + **distribution**: The key distribution mode. Possible values are:
- + ketama (default, recommended. An implementation of https://en.wikipedia.org/wiki/Consistent_hashing)
- + modula (use hash modulo number of servers to choose the backend)
- + random (choose a random backend)
+  + ketama (default, recommended. An implementation of https://en.wikipedia.org/wiki/Consistent_hashing)
+  + modula (use hash modulo number of servers to choose the backend)
+  + random (choose a random backend)
 + **timeout**: The timeout value in msec that we wait for to establish a connection to the server or receive a response from a server. By default, we wait indefinitely.
 + **backlog**: The TCP backlog argument. Defaults to 512.
 + **preconnect**: A boolean value that controls if twemproxy should preconnect to all the servers in this pool on process start. Defaults to false.
@@ -201,13 +201,13 @@ For example, the configuration file in [conf/nutcracker.yml](conf/nutcracker.yml
         - 127.0.0.1:26380:1
         - 127.0.0.1:26381:1
 
-Finally, to make writing a syntactically correct configuration file easier, twemproxy provides a command-line argument -t or --test-conf that can be used to test the YAML configuration file for any syntax error.
+Finally, to make writing a syntactically correct configuration file easier, twemproxy provides a command-line argument `-t` or `--test-conf` that can be used to test the YAML configuration file for any syntax error.
 
 ## Observability
 
 Observability in twemproxy is through logs and stats.
 
-Twemproxy exposes stats at the granularity of server pool and servers per pool through the stats monitoring port. The stats are essentially JSON formatted key-value pairs, with the keys corresponding to counter names. By default stats are exposed on port 22222 and aggregated every 30 seconds. Both these values can be configured on program start using the -c or --conf-file and -i or --stats-interval command-line arguments respectively. You can print the description of all stats exported by  using the -D or --describe-stats command-line argument.
+Twemproxy exposes stats at the granularity of server pool and servers per pool through the stats monitoring port. The stats are essentially JSON formatted key-value pairs, with the keys corresponding to counter names. By default stats are exposed on port 22222 and aggregated every 30 seconds. Both these values can be configured on program start using the `-c` or `--conf-file` and `-i` or `--stats-interval` command-line arguments respectively. You can print the description of all stats exported by  using the `-D` or `--describe-stats` command-line argument.
 
     $ nutcracker --describe-stats
 
@@ -233,13 +233,13 @@ Twemproxy exposes stats at the granularity of server pool and servers per pool t
       out_queue           "# requests in outgoing queue"
       out_queue_bytes     "current request bytes in outgoing queue"
 
-Logging in twemproxy is only available when twemproxy is built with logging enabled. By default logs are written to stderr. Twemproxy can also be configured to write logs to a specific file through the -o or --output command-line argument. On a running twemproxy, we can turn log levels up and down by sending it SIGTTIN and SIGTTOU signals respectively and reopen log files by sending it SIGHUP signal.
+Logging in twemproxy is only available when twemproxy is built with logging enabled. By default logs are written to stderr. Twemproxy can also be configured to write logs to a specific file through the `-o` or `--output` command-line argument. On a running twemproxy, we can turn log levels up and down by sending it SIGTTIN and SIGTTOU signals respectively and reopen log files by sending it SIGHUP signal.
 
 ## Pipelining
 
 Twemproxy enables proxying multiple client connections onto one or few server connections. This architectural setup makes it ideal for pipelining requests and responses and hence saving on the round trip time.
 
-For example, if twemproxy is proxying three client connections onto a single server and we get requests - 'get key\r\n', 'set key 0 0 3\r\nval\r\n' and 'delete key\r\n' on these three connections respectively, twemproxy would try to batch these requests and send them as a single message onto the server connection as 'get key\r\nset key 0 0 3\r\nval\r\ndelete key\r\n'.
+For example, if twemproxy is proxying three client connections onto a single server and we get requests - `get key\r\n`, `set key 0 0 3\r\nval\r\n` and `delete key\r\n` on these three connections respectively, twemproxy would try to batch these requests and send them as a single message onto the server connection as `get key\r\nset key 0 0 3\r\nval\r\ndelete key\r\n`.
 
 Pipelining is the reason why twemproxy ends up doing better in terms of throughput even though it introduces an extra hop between the client and server.
 

ci/Dockerfile

@@ -30,7 +30,7 @@ RUN pip3.6 install nose && \
 # RUN yum install -y redis redis-sentinel
 
 ARG REDIS_VER=3.2.11
-RUN wget https://github.com/antirez/redis/archive/$REDIS_VER.tar.gz && \
+RUN wget https://github.com/redis/redis/archive/$REDIS_VER.tar.gz && \
 	tar zxvf $REDIS_VER.tar.gz && \
 	pushd redis-$REDIS_VER && \
 		make install && \

ci/build-nutcracker.sh

@@ -9,9 +9,12 @@ trap cleanup INT
 cleanup
 
 export CFLAGS="-O3 -fno-strict-aliasing -I/usr/lib/x86_64-redhat-linux6E/include -B /usr/lib/x86_64-redhat-linux6E/lib64"
+# TODO: Figure out how to make this apply only to the contrib/ directory. Maybe override the yaml directory's Makefile.am after extracting it.
+# https://gcc.gnu.org/bugzilla/show_bug.cgi?id=53119
+CFLAGS+=" -Werror -Wall -Wno-pointer-sign -Wno-sign-conversion -Wno-missing-braces -Wno-unused-value -Wno-builtin-declaration-mismatch -Wno-maybe-uninitialized"
 export LDFLAGS="-lc_nonshared"
 cd /usr/src/twemproxy
 autoreconf -fvi
-./configure --enable-debug=log --prefix=/usr/src/twemproxy/work/usr
+./configure --enable-debug=yes --prefix=/usr/src/twemproxy/work/usr
 make -j5
 make install

configure.ac

@@ -199,11 +199,11 @@ AS_IF([test "x$disable_stats" = xyes],
   [AC_DEFINE([HAVE_STATS], [1], [Define to 1 if stats is not disabled])])
 AC_MSG_RESULT($disable_stats)
 
-# Untar the yaml-0.1.4 in contrib/ before config.status is rerun
-AC_CONFIG_COMMANDS_PRE([tar xvfz contrib/yaml-0.1.4.tar.gz -C contrib])
+# Untar the yaml-0.2.5 in contrib/ before config.status is rerun
+AC_CONFIG_COMMANDS_PRE([tar xvfz contrib/yaml-0.2.5.tar.gz -C contrib])
 
-# Call yaml-0.1.4 ./configure recursively
-AC_CONFIG_SUBDIRS([contrib/yaml-0.1.4])
+# Call yaml-0.2.5 ./configure recursively
+AC_CONFIG_SUBDIRS([contrib/yaml-0.2.5])
 
 # Define Makefiles
 AC_CONFIG_FILES([Makefile

contrib/Makefile.am

@@ -1,3 +1,3 @@
-SUBDIRS = yaml-0.1.4
+SUBDIRS = yaml-0.2.5
 
-EXTRA_DIST = yaml-0.1.4.tar.gz
+EXTRA_DIST = yaml-0.2.5.tar.gz