diff --git a/Makefile b/Makefile index 02585f0e1..12699f026 100644 --- a/Makefile +++ b/Makefile @@ -4,6 +4,7 @@ PKG_REVISION ?= $(shell git describe --tags 2>/dev/null) PKG_BUILD = 1 BASE_DIR = $(shell pwd) ERLANG_BIN = $(shell dirname $(shell which erl 2>/dev/null) 2>/dev/null) +OTP_VER = $(shell echo $(ERLANG_BIN) | rev | cut -d "/" -f 2 | rev) REBAR ?= $(BASE_DIR)/rebar3 OVERLAY_VARS ?= TEST_IGNORE ?= lager riak basho_bench @@ -219,7 +220,7 @@ MAJOR_VERSION ?= $(shell echo $(REVISION) | sed -e 's/\([0-9.]*\)-.*/\1/') # This enables the toplevel repository package to change names # when underlying dependencies change. NAME_HASH = $(shell git hash-object distdir/$(CLONEDIR)/$(MANIFEST_FILE) 2>/dev/null | cut -c 1-8) -PKG_ID := $(REPO_TAG) +PKG_ID := "$(REPO_TAG)-OTP$(OTP_VER)" ## ## Packaging targets diff --git a/README.md b/README.md new file mode 100644 index 000000000..d94f062f6 --- /dev/null +++ b/README.md @@ -0,0 +1,15 @@ +# Riak - a distributed, decentralised data storage system. + +To build riak, Erlang OTP 22 or higher is required. + +`make rel` will build a release which can be run via `rel/riak/bin/riak start`. Riak is primarily configured via `rel/riak/etc/riak.conf` + +To make a package, install appropriate build tools for your operating system and run `make package`. + +To create a local multi-node build environment use `make devclean; make devrel`. + +To test Riak use [Riak Test](https://github.com/basho/riak_test/blob/develop-3.0/doc/SIMPLE_SETUP.md). + +[legacy documentation](https://docs.riak.com/riak/kv/latest/index.html) is still generally relevant. + +Issues and PRs can be tracked via [Riak Github](https://github.com/basho/riak/issues) or [Riak KV Github](https://github.com/basho/riak_kv/issues). diff --git a/README.org b/README.org deleted file mode 100644 index 7a507caac..000000000 --- a/README.org +++ /dev/null @@ -1,22 +0,0 @@ -#+SETUPFILE: "doc/basho-doc-style.iorg" - -Welcome to Riak. - -* Overview - Riak is a distributed, decentralized data storage system. - - In the wiki, you will find the "quick start" directions for setting up and - using Riak. For more information, browse the following files: - - - README: this file - - LICENSE: the license under which Riak is released - - doc/ - - admin.org: Riak Administration Guide - - architecture.txt: details about the underlying design of Riak - - basic-client.txt: slightly more detail on using Riak - - basic-setup.txt: slightly more detail on setting up Riak - - man/riak.1.gz: manual page for the riak(1) command - - man/riak-admin.1.gz manual page for the riak-admin(1) command - - raw-http-howto.txt: using the Riak HTTP interface - -Archived Branches: https://github.com/basho-archive/riak diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md index 9b5c52493..2b89deaeb 100644 --- a/RELEASE-NOTES.md +++ b/RELEASE-NOTES.md @@ -1,4 +1,215 @@ -# Riak KV 3.0 Release Notes +# Riak KV 3.0.16 Release Notes + +This release includes the following updates: + +- A significant new [riak_core cluster claim algorithm](https://github.com/basho/riak_core/pull/1008) has been added - `choose_claim_v4`. The default cluster claim algorithm (`choose_claim_v2`) is unchanged, but the `choose_claim_fun` configuration option in riak.conf can be used to enable the new algorithm. The new algorithm offers significant improvements in the claim process, especially when location awareness is enabled, with the algorithm now more likely to find valid solutions and with fewer required transfers. The trade-off when compared to `choose_claim_v2` is that in some situations computational overhead of claim may increase by multiple orders of magnitude, leading to long cluster plan times. + +- The leveled backend has been updated to reduce the memory required for stores with a large number of keys, and also reduce volatility in the memory demanded, by [optimising the memory footprint of the SST process working heap](https://github.com/martinsumner/leveled/pull/403). + +- A collection of fixes have been added. A fix to timeouts in [leveled hot backups](https://github.com/martinsumner/leveled/pull/406). A fix to reap and erase queries to [prevent a backlog from overloading riak_kv_eraser or riak_kv_reaper mailboxes](https://github.com/basho/riak_kv/pull/1862) (creating rapidly increasing memory footprint). A fix to an issue with [nextgen replication when node_confirms is enabled](https://github.com/basho/riak_kv/pull/1858). The [R/W value used by nextgen replication is now configurable](https://github.com/basho/riak_kv/pull/1860), and this may be useful for preventing a backlog of replication events from overloading a cluster. + +As a result of the memory management improvements made in 3.0.16, the recommendation to consider altering the eheap single-block carrier threshold made in [Riak 3.0.12](#riak-kv-3012-release-notes) has been deprecated. With the leveled backend, memory management should now be efficient on default settings, although with a small overhead on the latency of HEAD operations in that backend. + +# Riak KV 3.0.15 Release Notes + +Fix to an issue introduced with the `auto_check` feature for TictacAAE full-sync in [Riak KV 3.0.10](#riak-kv-3010-release-notes). + +# Riak KV 3.0.14 Release Notes + +This release [fixes an issue](https://github.com/martinsumner/leveled/issues/393) whereby a failure to signal and handle back-pressure correctly by the leveled backend can cause a backlog within the store. In particular this can be triggered by handoffs (e.g. due to cluster admin operations), and lead to partition transfers stalling almost completely. The issue existed in previous releases, but may have been exacerbated by refactoring in [Riak KV 3.0.13](#riak-kv-3013-release-notes). + +An additional [minor improvement has been made to handoffs](https://github.com/basho/riak_kv/pull/1851). Previously requests to reap tombstones after deletions (where the delete_mode is not `keep`), would not be forwarded during handoffs. These tombstones would then need to be corrected by AAE (which may result in a permanent tombstone). There is now a configuration option `handoff_deletes` which can be enabled to ensure these reap requests are forwarded, reducing the AAE work required on handoff completion. + +Desipite the handoff improvements in [Riak KV 3.0.13](#riak-kv-3013-release-notes), handoff timeouts are still possible. If handoff timeouts do occur, then the first stage should be to reduce the [handoff batch threshold count](https://github.com/basho/riak_core/blob/riak_kv-3.0.14/priv/riak_core.schema#L47-L55) to a lower number than that of [the item_count in the handoff sender log](https://github.com/basho/riak_core/blob/riak_kv-3.0.14/src/riak_core_handoff_sender.erl#L474-L486). + +# Riak KV 3.0.13 Release Notes + +This release is focused on improving the reliability of handoffs. The speed of handoffs is critical to the recovery times of nodes following failure, and also to the time necessary to expand or contract the cluster. Controlling the speed can be managed by increasing concurrency (using `riak admin transfer-limit `), but this can often lead to handoff unreliability due to timeouts. + +- The main [handoff process has been refactored](https://github.com/basho/riak_core/pull/995) to remove some deprecated messages, simplify the naming of configuration items, improve the logging of handoffs and increase the frequency of handoff sync messages. More frequent sync messages should make the flow of handoffs from the sender more responsive to pressure at the receiver. + +- On joining a node to a cluster, there will now be an attempt to exchange metadata with the joining node before the join is staged. This should reduce the probability of events failing immediately after join, as bucket types had not yet been replicated to the joining node. + +- A [new configuration option has been added](https://github.com/basho/riak_kv/blob/fbb53630645e53af053228d526caa3c86f304066/priv/riak_kv.schema#L1469-L1487) to change Riak to only commit read repair on primary (not fallback) vnodes. In short-term failures, enabling this option will reduce the time taken for hinted handoff to complete following recovery of a failed node - as now the handoff will contain only objects changed for that partition during the outage. With the default setting of `disabled`; fallback nodes will also contain each object fetched during the outage, in-line with the behaviour in previous releases. + +- A [fix has been implemented in leveled](https://github.com/martinsumner/leveled/pull/390) to reduce failures and inefficiencies when re-building the ledger key store from the object Journal, in situations where there has been a high volume of object churn. + +- Some [helper functions have been added to riak_client](https://github.com/basho/riak_kv/blob/fbb53630645e53af053228d526caa3c86f304066/src/riak_client.erl#L949-L993), to simplify some operational tasks. These functions can be called from `riak remote_console`, e.g. `riak_client:repair_node().` - which replaces the series of commands previously required to run partition repair across all partitions on a node. `riak_client:tictacaae_suspend_node().` may be used to suspend Tictacaae AAE exchanges on a node following a failure, so that they can be re-enabled using `riak_client:tictacaae_resume_node().` once handoffs have been completed. + +There are still [outstanding](https://github.com/basho/riak_kv/issues/1846) [issues](https://github.com/basho/riak_core/issues/996) related to handoffs. + +The release also includes a [significant change to the HTTP API](https://github.com/basho/riak_kv/issues/1849). In previous releases PUT, POST and DELETE requests would all GET the object prior to starting the PUT process. This is in contrast to the Protocol Buffers API which would only GET the object in case where conditions were passed in the put (e.g. `if_none_match` or `if_not_modified`). These two APIs now have the same non-functional behaviour, the HTTP API will no longer request a GET before the PUT if the request does not contain a condition (e.g. using `If-None-Match`, `If-Match`, `If-Modified-Since` as well as a new bespoke condition `X-Riak-If-Not-Modified`). + +A vector clock being passed on a PUT using the `X-Riak-If-Not-Modified` header, will return a `409:Conflict` should the passed vector clock not match the clock found prior to updating the object. This will work as the PB API `if_not_modified` option. This is still an eventually consistent condition, parallel updates may still lead to siblings when `{allow_mult, true}`. + +# Riak KV 3.0.12 Release Notes + +This is a general release of changes and fixes: + +- A [fix to a critical issue with the use of PR variables](https://github.com/basho/riak_kv/pull/1836) in Riak KV (when the PR value is set via bucket properties). + +- Changes to the leveled backend with the aim of further improving memory management, by simplifying the [summary tree within each leveled_sst file](https://github.com/martinsumner/leveled/pull/383). + +- An [update to leveldb snappy compression](https://github.com/basho/eleveldb/pull/267), to make Riak compatible with a broader set of platforms (including AWS Graviton). Because of this change, anyone building from source will now require `cmake` to be installed. + +- A [new `reip_manual` console command](https://github.com/basho/riak/pull/1122) has been added to use as an alternative to `reip`, which does not currently work in the 3.0 release stream. Use `riak admin reip_manual` in place of `riak admin reip` when performing the reip operation on a node. This now requires that the path to the ring file, and the cluster name, to be known by the operator. + +- Some [improvements to logging within the leveled backend](https://github.com/martinsumner/leveled/pull/385) have been made to allow for better statistics monitoring of low-level operations, and reduce the cost of writing a log within leveled. + +- A fix for a deadlock issue in [riak_repl AAE fullsync](https://github.com/basho/riak_repl/pull/818) to help when clusters with high object counts have large deltas. + +- [Two](https://github.com/basho/riak_kv/pull/1839) [other](https://github.com/basho/riak_kv/pull/1837) fixes within Riak KV. + +As part of this release, further testing of the new memory configuration options added in Riak 3.0.10 has been undertaken. It is now recommended when using the leveled backend, that if memory growth in the Riak process is a signifcant concern, then the following configuration option may be tested: `erlang.eheap_memory.sbct = 128`. This has been shown to reduce the memory footprint of Riak, with a small performance overhead. + +## 3.0.12 Addendum + +Although not originally advertised in the release, the version of eleveldb used in 3.0.12 includes an upgrade of the [`snappy` compression algorithm used within eleveldb](https://github.com/basho/eleveldb/pull/267). This version of snappy is incompatible with the previous version used, and so any update to 3.0.12 and beyond where the eleveldb backend is deployed with `snappy` compression [will fail](https://github.com/basho/eleveldb/issues/273). + +To update to 3.0.12 in this case, the change should be rolled into the cluster using the ` riak admin cluster replace` command. + +# Riak KV 3.0.11 Release Notes + +A simple change to [release a bottleneck](https://github.com/martinsumner/leveled/issues/379) in 2i queries with the leveled backend. Should only be relevant to those using leveled, and attempting o(1000) 2i queries per second. + +# Riak KV 3.0.10 Release Notes + +This release is focused on improving memory management, especially with the leveled backend, and improving the efficiency and ease of configuration of tictacaae full-sync. + +- Improved [memory management of leveled](https://github.com/martinsumner/leveled/pull/371) SST files that contain [rarely accessed data](https://github.com/martinsumner/leveled/pull/371) + +- Fix a bug whereby leveled_sst files could spend an [extended time in the delete_pending state](https://github.com/martinsumner/leveled/pull/377), causing significant short-term increases in memory usage when there are work backlogs in the penciller. + +- Change the queue for reapers and erasers so that [they overflow to disk](https://github.com/basho/riak_kv/issues/1807), rather than simply consuming more and more memory. + +- Change the replrtq (nextgenrepl) queue to use the same [overflow queue mechanism](https://github.com/basho/riak_kv/issues/1817) as used by the reaper and erasers. + +- Change the default full-sync mechanism for tictacaae (nextgenrepl) full-sync to `auto_check`, which attempts to [automatically learn and use information about modified date-ranges](https://github.com/basho/riak_kv/issues/1815) in full-sync checks. The related changes also make full-sync by default bi-directional, reducing the amount of wasted effort in full-sync queries. + +- Add [a peer discovery feature](https://github.com/basho/riak_kv/issues/1804) for replrtq (nextgenrepl) so that new nodes added to the cluster can be automatically recognised without configuration changes. By default this is disabled, and should only be enabled once both clusters have been upgraded to at least 3.0.10. + +- Allow for underlying beam memory management and scheduler configuration to be exposed via riak.conf to allow for further performance tests on these settings. Note initial tests indicate the potential for [significant improvements when using the leveled backend](https://github.com/basho/riak_kv/issues/1826). + +- Fix a potential issue whereby corrupted objects would prevent AAE (either legacy or nextgenrepl) [tree rebuilds](https://github.com/basho/riak_kv/issues/1824) from completing. + +- Improved [handling of key amnesia](https://github.com/basho/riak_kv/issues/1813), to prevent rebounding of objects, and also introduce a reader process (like reaper and eraser) to which read repairs can be queued with overflow to disk. + +Some caveats for this release exist: + +- The release does not support OTP 20, only OTP 22 is supported. Updating some long out-of-date components have led to a requirement for the OTP version to be lifted. + +- Volume and performance testing with the leveled backend now uses the following non-default settings: + +``` +erlang.schedulers_busywait = none +erlang.schedulers_busywait_dirtycpu = none +erlang.schedulers_busywait_dirtyio = none +erlang.async_threads = 4 +erlang.schedulers.force_wakeup_interval = 0 +erlang.schedulers.compaction_of_load = true +leveled_reload_recalc = enabled +``` + +- To maintain backwards compatibility with older linux versions, the [latest version of basho's leveldb](https://github.com/basho/leveldb/releases/tag/2.0.37) is not yet supported. This is likely to change in the next release, where support for older linux versions will be dropped. + +- The release process has [exposed an issue](https://github.com/basho/riak_kv/issues/1831) via a recently extended test. This issue is pre-existing, and not specific to this release. + +# Riak KV 3.0.9 Release Notes + +This release contains stability, monitoring and performance improvements. + +- Fix to the [riak_core coverage planner](https://github.com/basho/riak_kv/issues/1801) to significantly reduce the CPU time required to produce a coverage plan, especially with larger ring sizes. This improves both the mean and tail-latency of secondary index queries. As a result of this change, it is recommended that larger ring sizes should be used by default, even when running relatively small clusters - for example in standard volume tests a ring size of 512 is outperforming lower ring sizes even on small (8-node) clusters. + +- Further monitoring stats have been added to track the performance of coverage queries, in particular secondary index queries. For each worker queue (e.g. vnode_worker_pool, af1_pool etc) the queue_time and work_time is now monitored with results available via riak stats. The result counts, and overall query time for secondary index queries are now also monitored via riak stats. See the [PR](https://github.com/basho/riak_kv/pull/1802#issuecomment-966160925) for a full list of stats added in this release. + +- Change to some default settings to be better adapted to running with higher concentrations of vnodes per nodes. The per-vnode cache sizes in leveled are reduced, and the [default size of the vnode_worker_pool has been reduced from 10 to 5](https://github.com/basho/riak_kv/blob/0af857d57d0206b100db59043a926df2ada8f7db/priv/riak_kv.schema#L1210-L1233) and is now configurable via riak.conf. Exceptionally heavy users of secondary index queries (i.e. > 1% of transactions), should consider monitoring the [new queue_time and work_time statistics](https://github.com/basho/riak_kv/pull/1802#issuecomment-966160925) before accepting this new default. + +- Fix to an issue in the leveled backend when a key and (internal) sequence number would [hash to 0](https://github.com/martinsumner/leveled/issues/361). It is recommended that users of leveled uplift to this version as soon as possible to resolve this issue. The risk is mitigated in Riak as it can generally be expected that most keys will have different sequence numbers in different vnodes, and will always have different sequence numbers when re-written - so normal anti-entropy process will recover from any localised data loss. + +- More time is now given to the legacy AAE `kv_index_hashtree` process to [shut down](https://github.com/basho/riak_kv/pull/1803), to handle delays as multiple vnodes are shutdown concurrently and contend for disk and CPU resources. + +# Riak KV 3.0.8 Release Notes + +This release contains a number of stability improvements. + +- Fix to critical issue in leveled when using (non-default, but recommended, option): [leveled_reload_recalc = enabled](https://github.com/basho/riak_kv/blob/33add2a29b6880b680a407dc91828736f54c7911/priv/riak_kv.schema#L1156-L1174). If using this option, it is recommended to rebuild the ledger on each vnode at some stage after updating. + +- [Fix to an issue with cluster leave](https://github.com/basho/riak_core/issues/970) operations that could leave clusters unnecessarily unbalanced after Stage 1 of leave, and also cause unexpected safety violations in Stage 1 (with either a simple transfer or a rebalance). An option to force a rebalance in Stage 1 is also now supported. + +- The ability to [set an environment variable](https://github.com/basho/riak/pull/1079) to remove the risk of atom table exhaustion due to repeated calls to `riak status` (and other) command-line functions. + +- The default setting of the object_hash_version environment variable to reduce the opportunity for the [riak_core_capability system to falsely request downgrading to legacy](https://github.com/basho/riak_kv/issues/1656), especially when concurrently restarting many nodes. + +- An update to the versions of `recon` and `redbug` used in Riak. + +- The fixing of an issue with [connection close handling](https://github.com/basho/riak-erlang-client/pull/402) in the Riak erlang client. + +This release also contains two new features: + +- A [new aae_fold operation](https://github.com/basho/riak_kv/issues/1793) has been added which will support the prompting of read-repair for a range of keys e.g. all the keys in a bucket after a given last modified date. This is intended to allow an operator to accelerate full recovery of data following a known node outage. + +- The addition of the [`sync_on_write` property for write operations](https://github.com/basho/riak_kv/blob/develop-3.0/docs/Sync-On-Write.md). Some Riak users require flushing of writes to disk to protect against data loss in disaster scenarios, such as mass loss of power across a DC. This can have a significant impact on throughput even with hardware acceleration (e.g. flash-backed write caches). The decision to flush was previously all or nothing. It can now be set as a bucket property (and even determined on individual writes), and can be set to flush on `all` vnodes or just `one` (the coordinator), or to simply respect the `backend` configuration. If `one` is used the single flush will occur only on client-initiated writes - writes due to handoffs or replication will not be flushed. + +# Riak KV 3.0.7 Release Notes + +The primary change in 3.0.7 is that Riak will now run the [erlang runtime system in interactive mode, not embedded mode](http://erlang.org/doc/man/code.html). This returns Riak to the default behaviour prior to Riak KV 3.0, in order to resolve a number of problems which occurred post 3.0 when trying to dynamically load code. + +The mode used is controlled in a [pre-start script](https://github.com/basho/riak/blob/develop-3.0/rel/files/erl_codeloadingmode), changing this script or overriding the referenced environment variable for the riak user should allow the runtime to be reverted to using `embedded` mode, should this be required. + +This release also extends a prototype API to support for the use of the `nextgenrepl` API by external applications, for example to reconcile replication to an external non-riak database. The existing `fetch` api function has been extended to allow for a new response format that includes the Active Anti-Entropy Segment ID and Segment Hash for the object (e.g. to be used when recreating the Riak merkle tree in external databases). A new `push` function has been added to the api, this will push a list of object references to be queued for replication. + +# Riak KV 3.0.6 Release Notes + +Release 3.0.5 adds [location-awareness to Riak cluster management](https://github.com/basho/riak_core/blob/riak_kv-3.0.5/docs/rack-awareness.md). The broad aim is to improve data diversity across locations (e.g. racks) to reduce the probability of data-loss should a set of nodes fail concurrently within a location. The location-awareness does not provide firm guarantees of data diversity that will always be maintained across all cluster changes - but [testing](https://github.com/basho/riak_test/pull/1353) has indicated it will generally find a cluster arrangement which is close to optimal in terms of data protection. + +If location information is not added to the cluster, there will be no change in behaviour from previous releases. + +There may be some performance advantages when using the location-awareness feature in conjunction with the leveled backend when handling GET requests. With location awareness, when responding to a GET request, a cluster is more likely to be able to fetch an object from a vnode within the location that received the GET request, without having to fetch that object from another location (only HEAD request/response will commonly travel between locations). + +This release is tested with OTP 20 and OTP 22; but optimal performance is likely to be achieved when using OTP 22. + +# Riak KV 3.0.5 Release Notes + +Release version skipped due to tagging error + +# Riak KV 3.0.4 Release Notes + +There are two fixes provided in Release 3.0.4: + +- An issue with [leveled application dependencies](https://github.com/basho/riak/issues/1059) has been resolved, and so lz4 can now again be used as the compression method. + +- The [riak clients are now compatible with systems that require semantic versioning](https://github.com/basho/riak-erlang-client/issues/400). + +This release is tested with OTP 20, OTP 21 and OTP 22; but optimal performance is likely to be achieved when using OTP 22. + + +# Riak KV 3.0.3 Release Notes + +There are two fixes provided in Release 3.0.3: + +- A [performance issue with OTP 22 and leveled](https://github.com/martinsumner/leveled/issues/326) has been corrected. This generally did not have a significant impact when running Riak, but there were some potential cases with Tictac AAE and AAE Folds where there could have been a noticeable slowdown. + +- An [issue with console commands for bucket types](https://github.com/basho/riak/issues/1043) has now been fully corrected, having been partially mitigated in 3.0.2. + +This release is tested with OTP 20, OTP 21 and OTP 22; but optimal performance is likely to be achieved when using OTP 22. + + +# Riak KV 3.0.2 Release Notes + +There are four changes made in Release 3.0.2: + +- Inclusion of backend fixes introduced in [2.9.8](#riak-kv-298-release-notes). + +- The addition of the [`range_check` in the Tictac AAE based full-sync replication](https://github.com/basho/riak_kv/blob/riak_kv-3.0.2/docs/NextGenREPL-GettingStarted.md#configure-full-sync-replication), when replicating between entire clusters. This, along with the backend performance improvements delivered in 2.9.8, can significantly improve the stability of Riak clusters when resolving large deltas. + +- A number of issues with command-line functions and packaging related to the switch from `node_package` to `relx` have now been resolved. + +- Riak tombstones, [empty objects used by Riak to replace deleted objects](https://riak.com/posts/technical/riaks-config-behaviors-part-3/index.html), will now have a last_modified_date added to the metadata, although this will not be visible externally via the API. + +This release is tested with OTP 20, OTP 21 and OTP 22; but optimal performance is likely to be achieved when using OTP 22. + + +# Riak KV 3.0.1 Release Notes This major release allows Riak to run on OTP versions 20, 21 and 22 - but is not fully backwards-compatible with previous releases. Some limitations and key changes should be noted: @@ -16,6 +227,39 @@ This major release allows Riak to run on OTP versions 20, 21 and 22 - but is not Other than the limitations listed above, the release should be functionally identical to Riak KV 2.9.7. Throughput improvements may be seen as a result of the OTP 20 upgrade on some CPU-bound workloads. For disk-bound workloads, additional benefit may be achieved by upgrading further to OTP 22. +# Riak KV 2.9.10 Release Notes + +Fix to critical issue in leveled when using (non-default, but recommended, option): [leveled_reload_recalc = enabled](https://github.com/basho/riak_kv/blob/33add2a29b6880b680a407dc91828736f54c7911/priv/riak_kv.schema#L1156-L1174). + +If using this option, it is recommended to rebuild the ledger on each vnode at some stage after updating. + +# Riak KV 2.9.9 Release Notes + +Minor stability improvements to leveled backend - [see leveled release notes](https://github.com/martinsumner/leveled/releases/tag/0.9.24) for further details. + +# Riak KV 2.9.8 Release Notes + +This release improves the performance and stability of the leveled backend and of AAE folds. These performance improvements are based on feedback from deployments with > 1bn keys per cluster. + +The particular improvements are: + +- In leveled, caching of individual file scores so not all files are required to be scored each journal compaction run. + +- In leveled, a change to the default journal compaction scoring percentages to make longer runs more likely (i.e. achieve more compaction per scoring run). + +- In leveled, a change to the caching of SST file block-index in the ledger, that makes repeated folds with a last modified date range an order of magnitude faster through improved computational efficiency. + +- In leveled, a fix to prevent very long list-buckets queries when buckets have just been deleted (by erasing all keys). + +- In kv_index_tictcatree, improved logging and exchange controls to make exchanges easier to monitor and less likely to prompt unnecessary work. + +- In kv_index_tictcatree, a change to speed-up the necessary rebuilds of aae tree-caches following a node crash, by only testing journal presence in scheduled rebuilds. + +- In riak_kv_ttaaefs_manager, some essential fixes to prevent excessive CPU load when comparing large volumes of keys and clocks, due to a failure to decode clocks correctly before passing to the exchange. + +Further significant improvements have been made to Tictac AAE full-sync, to greatly improve the efficiency of operation when there exists relatively large deltas between relatively large clusters (in terms of key counts). Those changes, which introduce the use of 'day_check', 'hour_check' and 'range_check' options to nval-based full-sync will be available in a future 3.0.2 release of Riak. For those wishing to use Tictac AAE full-sync at a non-trivial scale, it is recommended moving straight to 3.0.2 when it is available. + + # Riak KV 2.9.7 Release Notes This release improves the stability of Riak when running with Tictac AAE in parallel mode: @@ -34,20 +278,24 @@ This release improves the stability of Riak when running with Tictac AAE in para The system is now stable under specific load tests designed to trigger AAE failure. However, parallel mode should still not be used in production systems unless it has been subject to environment-specific load testing. + # Riak KV 2.9.6 Release Notes Withdrawn. + # Riak KV 2.9.5 Release Notes Withdrawn. + # Riak KV 2.9.4 Release Notes This release replaces the Riak KV 2.9.3 release, extending the issue resolution in kv_index_tictactree to detect other files where file truncation means the CRC is not present. This release has a key [outstanding issue](https://github.com/basho/riak_kv/issues/1765) when Tictac AAE is used in parallel mode. On larger clusters, this has been seen to cause significant issues, and so this feature should not be used other than in native mode. + # Riak KV 2.9.3 Release Notes - NOT TO BE RELEASED This release is focused on fixing a number of non-critical issues: @@ -75,6 +323,7 @@ This release includes: - An improvement to the efficiency of [compaction in the leveled LSM-tree based ledger](https://github.com/martinsumner/leveled/issues/311) with large numbers of tombstones (or modified index entries), by using a `grooming` selection strategy 50% of the time when selecting files to merge rather than selecting files at random each time. The `grooming` selection, will take a sample of files and merge the one with the most tombstones. The use of the grooming strategy is not configurable, and will have no impact until the vast majority of SST files have been re-written under this release. + # Riak KV 2.9.1 Release Notes This release adds a number of features built on top of the Tictac AAE feature made available in 2.9.0. The new features depend on Tictac AAE being enabled, but are backend independent. The primary features of the release are: @@ -89,6 +338,7 @@ This release adds a number of features built on top of the Tictac AAE feature ma Detail of volume testing related to the replication uplift can be found [here](https://github.com/martinsumner/riak_testing_notes/blob/master/Release%202.9.1%20-%20Volume%20Tests.md). + # Riak KV 2.9.0 Release Notes - Patch 5 This patch release is primarily required to improve two scenarios where leveled's handling of file-system corruption was unsatisfactory. One scenario related to [corrupted blocks that fail CRC checks](https://github.com/martinsumner/leveled/issues/298), another related to failing to consistently check that [a file write had been flushed to disk](https://github.com/martinsumner/leveled/issues/301). @@ -103,6 +353,7 @@ The patch changes the capability request for `HEAD` to be bucket-dependent, so t Finally, the aae_fold features of 2.9.0 were previously only available via the HTTP API. Such folds can now also be [requested via the PB API](https://github.com/basho/riak_kv/pull/1732). + # Riak KV 2.9.0 Release Notes - Patch 4 There are a number of fixes in this patch: @@ -119,6 +370,7 @@ There are a number of fixes in this patch: It is recommended that any 2.9.0 installations be upgraded to include this path, although if Tictac AAE is not used there is no immediate urgency to making the change. + # Riak KV 2.9.0 Release Notes - Patch 3 An [issue](https://github.com/martinsumner/leveled/issues/287) was discovered in leveled, whereby following a restart of Riak and a workload of fetch requests, the backend demanded excess amounts of binary heap references. Underlying was an issue with the use of sub-binary references during the lazy load of slot header information after a SST file process restart. This has been resolved, and with [greater control added](https://github.com/martinsumner/leveled/blob/0.9.18/priv/leveled.schema#L86-L93) to force the ledger contents into the page cache at startup. @@ -127,18 +379,22 @@ A further [issue](https://github.com/martinsumner/leveled/issues/289) was discov The issues resolved in this patch impact only the use of leveled backend, either directly or via the use of Tictac AAE. + # Riak KV 2.9.0 Release Notes - Patch 2 An [issue](https://github.com/martinsumner/leveled/issues/285) with leveled holding references to binaries what could cause severe memory depletion, when a consecutive series of very large objects are received by a vnode. + # Riak KV 2.9.0 Release Notes - Patch 1 An [issue](https://github.com/basho/riak_kv/issues/1699) was discovered whereby leveled would leak file descriptors under heavy write pressure (e.g. handoffs). + # Riak KV 2.9.0 Release Notes See [here for notes on 2.9.0](doc/Release%202.9%20Series%20-%20Overview.md) + # Riak KV 2.2.5 Release Notes > This release is dedicated to the memory of Andy Gross. Thank you and RIP. diff --git a/THANKS b/THANKS index 440c68fff..2dbfb1856 100644 --- a/THANKS +++ b/THANKS @@ -37,6 +37,7 @@ Anders Nygren Andre Graf Andreas Hasselberg Andreas Pfotenhauer +Andrei Zavada Andrew Gopienko Andrew J. Stone Andrew Matheny @@ -188,6 +189,7 @@ Eli Klein Endre H Engel A. Sanchez Enrique +Enrique Fernández Eric Eric Cestari Eric Merritt @@ -469,10 +471,12 @@ Pavlo Baron Pedram Nimreezi Pedro Larroy Pentium10 +Peter Clark Peter Garbers Peter Gribanov Peter Lemenkov Peter Teichman +Peter Tihanyi Petr A. Korobeinikov Petr Gotthard Phil Pirozhkov @@ -486,6 +490,7 @@ Przemysław Dąbek Péter Gömöri Ragil Rajiv M. Ranganath +Dr Ramen Sen Ramón Lastres Randy Secrist Ransom Richardson @@ -559,6 +564,7 @@ Steve Pulec Steve Strong Steve Vinoski Steve Wight +Steve Wood Steven Gravell Susan Potter Takahiko Sato @@ -570,6 +576,7 @@ Ted Nyman Thiago Avelino Thiago Carvalho Thomas +Thomas Arts Thomas Guyot-Sionnest Thomas Järvstrand Tiago Peczenyj diff --git a/apps/riak/src/etop_txt.erl b/apps/riak/src/etop_txt.erl deleted file mode 100644 index aad011677..000000000 --- a/apps/riak/src/etop_txt.erl +++ /dev/null @@ -1,135 +0,0 @@ -%% -%% %CopyrightBegin% -%% -%% Copyright Ericsson AB 2002-2009. All Rights Reserved. -%% -%% The contents of this file are subject to the Erlang Public License, -%% Version 1.1, (the "License"); you may not use this file except in -%% compliance with the License. You should have received a copy of the -%% Erlang Public License along with this software. If not, it can be -%% retrieved online at http://www.erlang.org/. -%% -%% Software distributed under the License is distributed on an "AS IS" -%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See -%% the License for the specific language governing rights and limitations -%% under the License. -%% -%% %CopyrightEnd% -%% --module(etop_txt). --author('siri@erix.ericsson.se'). --author('scott@basho.com'). - -%%-compile(export_all). --export([init/1,stop/1]). --export([do_update/3]). - -%%-include("etop.hrl"). --record(etop_proc_info, - {pid, - mem=0, - reds=0, - name, - runtime=0, - cf, - mq=0}). --record(etop_info, - {now = {0, 0, 0}, - n_procs = 0, - wall_clock = {0, 0}, - runtime = {0, 0}, - run_queue = 0, - alloc_areas = [], - memi = [{total, 0}, - {processes, 0}, - {ets, 0}, - {atom, 0}, - {code, 0}, - {binary, 0}], - procinfo = [] - }). -%%-include("etop_defs.hrl"). --define(SYSFORM, - " ~-72w~10s~n" - " Load: cpu ~8w Memory: total ~8w binary ~8w~n" - " procs~8w processes~8w code ~8w~n" - " runq ~8w atom ~8w ets ~8w~n"). --record(opts, {node=node(), port = 8415, accum = false, intv = 5000, lines = 10, - width = 700, height = 340, sort = runtime, tracing = on, - %% Other state information - out_mod=etop_gui, out_proc, server, host, tracer, store, - accum_tab, remote}). - --import(etop,[loadinfo/1,meminfo/2]). --import(etop_gui,[formatmfa/1,to_list/1]). - --define(PROCFORM,"~-20w~-25s~8w~11w~11w~11w ~-40s~n"). - -stop(Pid) -> Pid ! stop. - -init(Config) -> - loop(Config). - -loop(Config) -> - Info = do_update(Config), - receive - stop -> stopped; - {dump,Fd} -> do_update(Fd,Info,Config), loop(Config); - {config,_,Config1} -> loop(Config1) - after Config#opts.intv-500 -> loop(Config) - end. - -do_update(Config) -> - Info = etop:update(Config), - do_update(standard_io,Info,Config). - -do_update(Fd,Info,Config) -> - {Cpu,NProcs,RQ,Clock} = loadinfo(Info), - io:nl(Fd), - writedoubleline(Fd), - case Info#etop_info.memi of - undefined -> - io:fwrite(Fd, " ~-72w~10s~n" - " Load: cpu ~8w~n" - " procs~8w~n" - " runq ~8w~n", - [Config#opts.node,Clock, - Cpu,NProcs,RQ]); - Memi -> - [Tot,Procs,Atom,Bin,Code,Ets] = - meminfo(Memi, [total,processes,atom,binary,code,ets]), - io:fwrite(Fd, ?SYSFORM, - [Config#opts.node,Clock, - Cpu,Tot,Bin, - NProcs,Procs,Code, - RQ,Atom,Ets]) - end, - io:nl(Fd), - writepinfo_header(Fd), - writesingleline(Fd), - writepinfo(Fd,Info#etop_info.procinfo), - %%writedoubleline(Fd), - %%io:nl(Fd), - Info. - -writepinfo_header(Fd) -> - io:fwrite(Fd,"Pid Name or Initial Func Time Reds Memory MsgQ Current Function~n",[]). - -writesingleline(Fd) -> - io:fwrite(Fd,"-------------------------------------------------------------------------------------------------------------------------------~n",[]). -writedoubleline(Fd) -> - io:fwrite(Fd,"===============================================================================================================================~n",[]). - -writepinfo(Fd,[#etop_proc_info{pid=Pid, - mem=Mem, - reds=Reds, - name=Name, - runtime=Time, - cf=MFA, - mq=MQ} - |T]) -> - io:fwrite(Fd,?PROCFORM,[Pid,to_list(Name),Time,Reds,Mem,MQ,formatmfa(MFA)]), - writepinfo(Fd,T); -writepinfo(_Fd,[]) -> - ok. - diff --git a/apps/riak/src/riak.app.src b/apps/riak/src/riak.app.src index 4b049872c..541e742f1 100644 --- a/apps/riak/src/riak.app.src +++ b/apps/riak/src/riak.app.src @@ -2,11 +2,11 @@ [{description, "The Riak Database"}, {vsn, "3.0"}, {registered, []}, - {modules, [etop_txt]}, {applications, [ kernel, stdlib, lager, + tools, lager_syslog, sasl, public_key, diff --git a/eunit.log b/eunit.log deleted file mode 100644 index 89887af3f..000000000 --- a/eunit.log +++ /dev/null @@ -1,46 +0,0 @@ -Eunit: basho_stats PASSED -Eunit: bear PASSED -Eunit: bitcask PASSED -Eunit: canola PASSED -Eunit: clique PASSED -Eunit: cluster_info PASSED -Eunit: cuttlefish PASSED -Eunit: ebloom PASSED -Eunit: eleveldb PASSED -Eunit: exometer_core PASSED -Eunit: folsom PASSED -Eunit: getopt PASSED -Eunit: goldrush PASSED -Eunit: hut PASSED -Eunit: hyper PASSED -Eunit: ibrowse PASSED -Eunit: kv_index_tictactree PASSED -Eunit: lager_syslog PASSED -Eunit: leveled PASSED -Eunit: lz4 PASSED -Eunit: meck PASSED -Eunit: mochiweb PASSED -Eunit: parse_trans PASSED -Eunit: pbkdf2 PASSED -Eunit: poolboy PASSED -Eunit: protobuffs PASSED -Eunit: ranch PASSED -Eunit: recon PASSED -Eunit: redbug PASSED -Eunit: riak_api PASSED -Eunit: riak_auth_mods PASSED -Eunit: riak_core PASSED -Eunit: riak_dt PASSED -Eunit: riak_ensemble PASSED -Eunit: riak_kv PASSED -Eunit: riak_pb PASSED -Eunit: riak_pipe PASSED -Eunit: riak_repl PASSED -Eunit: riak_sysmon PASSED -Eunit: riakc PASSED -Eunit: riakhttpc PASSED -Eunit: setup PASSED -Eunit: sext PASSED -Eunit: sidejob PASSED -Eunit: syslog PASSED -Eunit: webmachine PASSED diff --git a/priv/riak.schema b/priv/riak.schema index 8407c39d5..ba72c948f 100644 --- a/priv/riak.schema +++ b/priv/riak.schema @@ -205,25 +205,226 @@ merge ]}. -%% VM scheduler collapse, part 1 of 2 +%% @doc Riak changes the VM default wakeup interval in order to reduce the +%% risk of scheduler collapse, prior to the availability of Dirty NIFs in +%% later OTP versions. When using the leveled backend exclusively (either for +%% AAE or object storage) this change is likely unnecessary, and the VM default +%% of 0 can be used. {mapping, "erlang.schedulers.force_wakeup_interval", "vm_args.+sfwi", [ {default, 500}, {datatype, integer}, merge ]}. -%% VM scheduler collapse, part 2 of 2 +%% @doc Riak changes the compaction_of_load default from true to false. This +%% is part of the strategy for preventing scheduler collapse in older VMs. +%% When using the leveled backend exclusively (either for AAE or object +%% storage), this change from the standard BEAM defaults is likely unnecessary +%% - and compaction_of_load can be re-enabled. {mapping, "erlang.schedulers.compaction_of_load", "vm_args.+scl", [ {default, "false"}, merge ]}. +%% @doc Sets the number of threads in async thread pool, valid range +%% is 0-1024. If thread support is available, the default is 64. +%% +%% More information at: http://erlang.org/doc/man/erl.html +%% +%% Large async_thread pools are likely now unnecessary if exclusively using +%% the leveled backend due to dirty NIFs, and so can be set to a much smaller +%% value (potentially 1). +{mapping, "erlang.async_threads", "vm_args.+A", [ + {default, 64}, + {datatype, integer}, + {validators, ["range:0-1024"]}, + merge +]}. + %% VM emulator ignore break signal (prevent ^C / ^Gq) {mapping, "erlang.vm.ignore_break_signal", "vm_args.+Bi", [ {default, "true"}, merge ]}. +%% @doc The VM single block carrier threshold (KB) for process heap +{mapping, "erlang.eheap_memory.sbct", "vm_args.+MHsbct", [ + {commented, 512}, + {datatype, integer}, + merge +]}. + +%% @doc The VM single block carrier threshold (KB) for binary heap +{mapping, "erlang.binary_memory.sbct", "vm_args.+MBsbct", [ + {commented, 512}, + {datatype, integer}, + merge +]}. + +%% @doc The VM multi block carrier large size for process heap +{mapping, "erlang.eheap_memory.lmbcs", "vm_args.+MHlmbcs", [ + {commented, 5120}, + {datatype, integer}, + merge +]}. + +%% @doc The VM multi block carrier large size for binary heap +{mapping, "erlang.binary_memory.lmbcs", "vm_args.+MBlmbcs", [ + {commented, 5120}, + {datatype, integer}, + merge +]}. + +%% @doc The VM multi block carrier small size for process heap +{mapping, "erlang.eheap_memory.smbcs", "vm_args.+MHsmbcs", [ + {commented, 256}, + {datatype, integer}, + merge +]}. + +%% @doc The VM multi block carrier small size for binary heap +{mapping, "erlang.binary_memory.smbcs", "vm_args.+MBsmbcs", [ + {commented, 256}, + {datatype, integer}, + merge +]}. + + %% @doc Set allocation strategy for binary multiblock carriers. Default is +%% not predictable - do not rely on aoffcbf being the default. For more info +%% see: +%% https://github.com/erlang/otp/blob/master/erts/emulator/internal_doc/CarrierMigration.md +{mapping, "erlang.binary_memory.as", "vm_args.+MBas", [ + {commented, "aoffcbf"}, + {datatype, {enum, [bf, aobf, aoff, aoffcbf, aoffcaobf, ageffcaoff, ageffcbf, ageffcaobf, gf]}}, + merge +]}. + +%% @doc Set allocation strategy for process multiblock carriers. Default is +%% not predictable - do not rely on aoffcbf being the default. For more info +%% see: +%% https://github.com/erlang/otp/blob/master/erts/emulator/internal_doc/CarrierMigration.md +{mapping, "erlang.eheap_memory.as", "vm_args.+MHas", [ + {commented, "aoffcbf"}, + {datatype, {enum, [bf, aobf, aoff, aoffcbf, aoffcaobf, ageffcaoff, ageffcbf, ageffcaobf, gf]}}, + merge +]}. + +%% @doc Set scheduler binding. This is either unbound (default - u) or can be +%% set to whatever the default binding condition is, in the deployed release of +%% OTP (db). +%% For more info see: https://www.erlang.org/doc/man/erl.html#+sbt +%% Note that if non-Riak work is activated on the same node - e.g. as part +%% of batch operational jobs, or monitoring - allowing schedulers to be bound +%% can result in significant and unpredictable negative outcomes. There may be +%% other ways of achieving similar performance improvements - e.g. by +%% right-sizing scheduler counts - that are lower risk than scheduler binding. +%% If a CPU topology cannot be determined, the binding will default to unbound +%% even when a binding is configured. To confirm binding, use `remote_console` +%% and view: +%% `erlang:system_info(scheduler_bindings).` +{mapping, "erlang.schedulers_binding", "vm_args.+stbt", [ + {commented, "u"}, + {datatype, {enum, [u, db]}}, + merge +]}. + +%% @doc Busy wait of schedulers +%% Sets scheduler busy wait threshold. Defaults to medium. The threshold +%% determines how long schedulers are to busy wait when running out of work +%% before going to sleep. +%% Significant improvements in efficiency may be gained by disabling busy +%% waiting +{mapping, "erlang.schedulers_busywait", "vm_args.+sbwt", [ + {commented, "none"}, + {datatype, {enum, [none, very_short, short, medium, long, very_long]}}, + merge +]}. + +%% @doc Busy wait of dirty cpu schedulers +%% Sets scheduler busy wait threshold. Defaults to short. The threshold +%% determines how long schedulers are to busy wait when running out of work +%% before going to sleep. +%% Significant improvements in efficiency may be gained by disabling busy +%% waiting +{mapping, "erlang.schedulers_busywait_dirtycpu", "vm_args.+sbwtdcpu", [ + {commented, "none"}, + {datatype, {enum, [none, very_short, short, medium, long, very_long]}}, + merge +]}. + +%% @doc Busy wait of dirty io schedulers +%% Sets scheduler busy wait threshold. Defaults to short. The threshold +%% determines how long schedulers are to busy wait when running out of work +%% before going to sleep. +%% Significant improvements in efficiency may be gained by disabling busy +%% waiting +{mapping, "erlang.schedulers_busywait_dirtyio", "vm_args.+sbwtdio", [ + {commented, "none"}, + {datatype, {enum, [none, very_short, short, medium, long, very_long]}}, + merge +]}. + +%% @doc Set the Percentage of Schedulers to be online +%% For every vCPU in the system, what percentage should have a scheduler, and +%% what percentage of those schedulers should be online by default. +%% Do not set unless guided by perfomance tests for the specific setup and +%% workload. +{mapping, "erlang.schedulers_online_percentage", "vm_args.+SP",[ + {commented, "100:75"}, + {validators, ["scheduler_percentage"]}, + merge +]}. + +%% @doc Set the Percentage of Dirty CPU Schedulers to be online +%% When using the leveled backend a relatievly low number of dirty schedulers +%% (e.g. 25%) are likely to be required due to the low proportion of NIFs in +%% use. +%% The percentages cannot exceed those of the schedulers_online_percentage +%% which will default to 100% of CPU. +%% Do not set unless guided by perfomance tests for the specific setup and +%% workload. +{mapping, "erlang.schedulers_dirtycpu_online_percentage", "vm_args.+SDPcpu",[ + {commented, "50:25"}, + {validators, ["scheduler_percentage"]}, + merge +]}. + +%% @doc Set the absolute limit of Dirty IO Schedulers to be online +%% When using the leveled backend a relatievly high number of dirty schedulers +%% may be required relative to the CPU count, depending on the concurrent disk +%% throughput possible. +%% Unlike the scheduler percentages, this is set as an abolute number between +%% 1 and 1024 (default is 10). +%% Do not set unless guided by perfomance tests for the specific setup and +%% workload. +{mapping, "erlang.schedulers_dirtyio_online", "vm_args.+SDio",[ + {commented, 10}, + {datatype, integer}, + {validators, ["scheduler_absolute"]}, + merge +]}. + +{validator, + "scheduler_percentage", + "must be A:B when B =< A and both A and B 1 < x =< 100", + fun(PercPerc) -> + case string:tokens(PercPerc, ":") of + [A, B] -> + AV = list_to_integer(A), + BV = list_to_integer(B), + AV =< 100 andalso AV > 0 andalso BV =< 100 andalso BV > 0; + _ -> + false + end + end}. + +{validator, + "scheduler_absolute", + "must be 1 to 1024", + fun(Value) -> + is_integer(Value) andalso Value =< 1024 andalso Value >= 1 + end}. + {{#devrel}} %% Because of the 'merge' keyword in the proplist below, the docs and datatype %% are pulled from the leveldb schema. diff --git a/rebar.config b/rebar.config index a0600d558..0bf6d152e 100644 --- a/rebar.config +++ b/rebar.config @@ -1,7 +1,7 @@ %% -*- mode: erlang;erlang-indent-level: 4;indent-tabs-mode: nil -*- {sub_dirs, ["rel", "apps/riak"]}. -{minimum_otp_vsn, "20.0"}. +{minimum_otp_vsn, "22.0"}. {cover_enabled, true}. @@ -9,18 +9,19 @@ {eunit_opts, [nowarn_export_all, verbose]}. {deps, [ - {lager_syslog, {git, "git://github.com/basho/lager_syslog.git", {tag, "3.1.0"}}}, - {cluster_info, {git, "git://github.com/basho/cluster_info.git", {tag, "2.1.0"}}}, - {riak_kv, {git, "git://github.com/basho/riak_kv.git", {tag, "riak_kv-3.0.1"}}}, - {riak_auth_mods, {git, "git://github.com/basho/riak_auth_mods.git", {tag, "riak_kv-3.0.0"}}}, + {lager_syslog, {git, "https://github.com/basho/lager_syslog.git", {tag, "3.1.1"}}}, + {cluster_info, {git, "https://github.com/basho/cluster_info.git", {tag, "2.1.0"}}}, + {riak_kv, {git, "https://github.com/basho/riak_kv.git", {tag, "riak_kv-3.0.16"}}}, + {riak_auth_mods, {git, "https://github.com/basho/riak_auth_mods.git", {tag, "riak_kv-3.0.10"}}}, + {riaknostic, {git, "https://github.com/basho/riaknostic.git", {tag, "riak_kv-3.0.10"}}}, - % {yokozuna, {git, "git://github.com/basho/yokozuna.git", {branch, "develop-3.0"}}}, + % {yokozuna, {git, "https://github.com/basho/yokozuna.git", {branch, "develop-3.0"}}}, - {riak_repl, {git, "git://github.com/basho/riak_repl.git", {tag, "riak_kv-3.0.1"}}} + {riak_repl, {git, "https://github.com/basho/riak_repl.git", {tag, "riak_kv-3.0.16"}}} ]}. {project_plugins, [ - {rebar3_cuttlefish, {git, "https://github.com/basho/rebar3_cuttlefish", {tag, "0.2.0"}}} + {rebar3_cuttlefish, {git, "https://github.com/basho/rebar3_cuttlefish", {tag, "0.2.1"}}} ]}. {cuttlefish, [ @@ -45,7 +46,7 @@ ]} ]}. -{relx, [{release, {riak, "3.0"}, +{relx, [{release, {riak, "3.0.16"}, [kernel, stdlib, lager, @@ -57,6 +58,7 @@ riak_sysmon, os_mon, crypto, + observer, runtime_tools, xmerl, mochiweb, @@ -71,6 +73,7 @@ riak_repl, cluster_info, % yokozuna, + riaknostic, riak_auth_mods]}, {dev_mode, false}, @@ -83,16 +86,17 @@ {template, "rel/files/advanced.config", "etc/advanced.config"}, %% Copy additional bin scripts - {template, "rel/files/riak-admin", "bin/riak-admin"}, - {template, "rel/files/riak-debug", "bin/riak-debug"}, - {template, "rel/files/riak-chkconfig", "bin/riak-chkconfig"}, - {template, "rel/files/riak-repl", "bin/riak-repl"}, - - {template, "rel/files/riak", "usr/bin/riak"}, - - {copy, "rel/files/check_ulimit", "bin/hooks/check_ulimit"}, - {copy, "rel/files/erl_maxlogsize", "bin/hooks/erl_maxlogsize"}, - {copy, "rel/files/riak_not_running", "bin/hooks/riak_not_running"} + {template, "rel/files/riak-admin", "bin/riak-admin"}, + {template, "rel/files/riak-debug", "bin/riak-debug"}, + {template, "rel/files/riak-chkconfig", "bin/riak-chkconfig"}, + {template, "rel/files/riak-repl", "bin/riak-repl"}, + {template, "rel/files/riak", "usr/bin/riak"}, + + {copy, "rel/files/check_ulimit", "bin/hooks/check_ulimit"}, + {copy, "rel/files/erl_maxlogsize", "bin/hooks/erl_maxlogsize"}, + {copy, "rel/files/erl_codeloadingmode","bin/hooks/erl_codeloadingmode"}, + {copy, "rel/files/riak_not_running", "bin/hooks/riak_not_running"}, + {copy, "rel/files/app_epath.sh", "lib/app_epath.sh"} ]}, @@ -100,7 +104,6 @@ {extended_start_script, true}, {extended_start_script_extensions, [ {admin, "riak-admin"}, - {debug, "riak-debug"}, {repl, "riak-repl"}, {chkconfig, "riak-chkconfig"} ]} @@ -117,7 +120,8 @@ [{pre_start, [{custom, "hooks/riak_not_running"}, {custom, "hooks/check_ulimit"}, - {custom, "hooks/erl_maxlogsize"}]}, + {custom, "hooks/erl_maxlogsize"}, + {custom, "hooks/erl_codeloadingmode"}]}, {post_start, [{wait_for_process, riak_core_node_watcher}]} ]} @@ -129,7 +133,8 @@ [{pre_start, [{custom, "hooks/riak_not_running"}, {custom, "hooks/check_ulimit"}, - {custom, "hooks/erl_maxlogsize"}]}, + {custom, "hooks/erl_maxlogsize"}, + {custom, "hooks/erl_codeloadingmode"}]}, {post_start, [{wait_for_process, riak_core_node_watcher}]} ]} @@ -146,7 +151,8 @@ [{pre_start, [{custom, "hooks/riak_not_running"}, {custom, "hooks/check_ulimit"}, - {custom, "hooks/erl_maxlogsize"}]}, + {custom, "hooks/erl_maxlogsize"}, + {custom, "hooks/erl_codeloadingmode"}]}, {post_start, [{pid, "/var/run/riak/riak.pid"}, {wait_for_process, riak_core_node_watcher}]} @@ -163,7 +169,8 @@ [{pre_start, [{custom, "hooks/riak_not_running"}, {custom, "hooks/check_ulimit"}, - {custom, "hooks/erl_maxlogsize"}]}, + {custom, "hooks/erl_maxlogsize"}, + {custom, "hooks/erl_codeloadingmode"}]}, {post_start, [wait_for_vm_start, {pid, "/run/riak/riak.pid"}, diff --git a/rebar.lock b/rebar.lock index 624c0fcd3..b14805352 100644 --- a/rebar.lock +++ b/rebar.lock @@ -1,87 +1,84 @@ {"1.1.0", [{<<"basho_stats">>, - {git,"git://github.com/basho/basho_stats.git", + {git,"https://github.com/basho/basho_stats.git", {ref,"c5b7ecb92af7b050bc62d08c6fbf1aafd6bdeff0"}}, 2}, - {<<"bear">>,{pkg,<<"bear">>,<<"0.8.7">>},4}, + {<<"bear">>,{pkg,<<"bear">>,<<"1.0.0">>},4}, {<<"bitcask">>, - {git,"git://github.com/basho/bitcask.git", + {git,"https://github.com/basho/bitcask.git", {ref,"dd96f6dd84a1cad68ccc21959ba56b0426434d85"}}, 1}, {<<"canola">>, - {git,"git://github.com/basho/canola.git", + {git,"https://github.com/basho/canola.git", {ref,"2cd39378442f3dc727de22289243b3c4d56f25fd"}}, 1}, {<<"clique">>, - {git,"git://github.com/basho/clique.git", - {ref,"4014357e4e677164b890fdad08a45cfa54b3e8f3"}}, + {git,"https://github.com/basho/clique.git", + {ref,"9ad8515dfeebf7d324b6d56eab8670c0b8995ce0"}}, 2}, {<<"cluster_info">>, - {git,"git://github.com/basho/cluster_info.git", + {git,"https://github.com/basho/cluster_info.git", {ref,"389d43af7ac1550b3c01cd55b8147bcc0e20022f"}}, 0}, {<<"cuttlefish">>, {git,"https://github.com/basho/cuttlefish.git", - {ref,"e84070b6cf08196ece2ae54083eefae4dc4e4ecc"}}, + {ref,"a70377b83c4859f418f2cae901c577ba4c085670"}}, 3}, {<<"ebloom">>, - {git,"git://github.com/basho/ebloom.git", + {git,"https://github.com/basho/ebloom.git", {ref,"3c25f9ef61a73d5f1d2c2601da464dd03975721d"}}, 1}, {<<"eleveldb">>, - {git,"git://github.com/basho/eleveldb.git", - {ref,"3dee1dd7173507a4ad29df1ceb99cb70d6142478"}}, + {git,"https://github.com/basho/eleveldb.git", + {ref,"edfd3924abbb2c5780d2c4641dcc0592a221ac43"}}, 2}, - {<<"exometer_core">>, - {git,"git://github.com/Feuerlabs/exometer_core.git", - {ref,"547f28bc93c7cb3d3f1174fb4c510667a4ebb645"}}, - 2}, - {<<"folsom">>,{pkg,<<"folsom">>,<<"0.8.7">>},3}, + {<<"exometer_core">>,{pkg,<<"exometer_core">>,<<"1.6.1">>},2}, + {<<"folsom">>,{pkg,<<"folsom">>,<<"1.0.0">>},3}, {<<"getopt">>, - {git,"git://github.com/jcomellas/getopt.git", + {git,"https://github.com/jcomellas/getopt.git", {ref,"fbd210550677114a6c96cdf66e340eb248908d93"}}, - 4}, + 1}, {<<"goldrush">>,{pkg,<<"goldrush">>,<<"0.1.9">>},2}, - {<<"hut">>,{pkg,<<"hut">>,<<"1.2.1">>},3}, + {<<"hut">>,{pkg,<<"hut">>,<<"1.3.0">>},3}, {<<"hyper">>, - {git,"git://github.com/basho/hyper", + {git,"https://github.com/basho/hyper", {ref,"d0bf3d67fd17b8ed2610b266aebd54b1027ea916"}}, 1}, {<<"ibrowse">>, - {git,"git://github.com/basho/ibrowse.git", - {ref,"aee5b93411b0a853494025005661623b13bf43f4"}}, + {git,"https://github.com/basho/ibrowse.git", + {ref,"cdde6fe5e80ce71e7b5570e40a45d9f862721499"}}, 2}, {<<"kv_index_tictactree">>, {git,"https://github.com/martinsumner/kv_index_tictactree.git", - {ref,"fe623784cc3e3185238924bc6b1786f2d79edb36"}}, + {ref,"50302987d33689fb2066c831ec7376f11d4304c9"}}, 1}, {<<"lager">>, - {git,"git://github.com/erlang-lager/lager.git", + {git,"https://github.com/erlang-lager/lager.git", {ref,"22e62f28e5afabe90a6f31bcde367a2b5799fc94"}}, 1}, {<<"lager_syslog">>, - {git,"git://github.com/basho/lager_syslog.git", - {ref,"152bb8eb254c225ebd8c579f9603f44b50df44ec"}}, + {git,"https://github.com/basho/lager_syslog.git", + {ref,"5c8eb6ec82fa45ff0f4e456747e095654e342746"}}, 0}, {<<"leveled">>, - {git,"https://github.com/martinsumner/leveled.git", - {ref,"bf591a5aa988ef4c3699152d129fc81ca0896030"}}, - 1}, + {git,"https://github.com/martinsumner/leveled", + {ref,"4142914e3fc32090747a4a7ea63087e18a74a042"}}, + 2}, {<<"lz4">>, {git,"https://github.com/martinsumner/erlang-lz4", {ref,"d407af5de71303b8e381df718d2e5d05eb775c59"}}, - 2}, + 3}, {<<"meck">>, {git,"https://github.com/eproxus/meck.git", {ref,"4ecc1ae9089edc6977e8c8c4cd41081513cc5590"}}, - 3}, + 4}, {<<"mochiweb">>, - {git,"git://github.com/basho/mochiweb.git", - {ref,"fb7bc696357b77fd8075de796d3687dfb5ed0d0b"}}, + {git,"https://github.com/basho/mochiweb.git", + {ref,"8471d2197afd412fcec239dd9ad5cbe90e64eefa"}}, 2}, - {<<"parse_trans">>,{pkg,<<"parse_trans">>,<<"3.3.0">>},3}, + {<<"parse_trans">>,{pkg,<<"parse_trans">>,<<"3.4.1">>},3}, {<<"pbkdf2">>, - {git,"git://github.com/basho/erlang-pbkdf2.git", + {git,"https://github.com/basho/erlang-pbkdf2.git", {ref,"681494c3ea03625c909a16b1b3ca295edf892057"}}, 2}, {<<"poolboy">>, @@ -91,90 +88,95 @@ {<<"protobuffs">>, {git,"https://github.com/basho/erlang_protobuffs.git", {ref,"098efad8f85dfe556d64e2cf6ce31f2075808f67"}}, - 2}, + 3}, {<<"ranch">>, - {git,"git://github.com/ninenines/ranch.git", - {ref,"d0fad24657711cf584da61b5163c29fdafbd9201"}}, + {git,"https://github.com/ninenines/ranch.git", + {ref,"9b8ed47d789412b0021bfc1f94f1c17c387c721c"}}, 1}, {<<"recon">>, {git,"https://github.com/ferd/recon", - {ref,"75d70c7c08926d2f24f1ee6de14ee50fe8a52763"}}, + {ref,"34194da6d9f8ed25f274e0ebb098dc9e95bcf547"}}, 1}, {<<"redbug">>, {git,"https://github.com/massemanet/redbug", - {ref,"c583d4567d1e2b8a540593ad0512559798d42565"}}, + {ref,"c466ddc4f05a34c47bcf0666899c50e40c5223d2"}}, 1}, {<<"riak_api">>, - {git,"git://github.com/basho/riak_api.git", - {ref,"0aec475c38ea17232b2e80d9c6c684a6ad9cd226"}}, + {git,"https://github.com/basho/riak_api.git", + {ref,"1ccf72103a154493e952e7a544ea3429cb342b20"}}, 1}, {<<"riak_auth_mods">>, - {git,"git://github.com/basho/riak_auth_mods.git", - {ref,"1eccaef8e9e6200f0b75ea9312f0a731c8d572bf"}}, + {git,"https://github.com/basho/riak_auth_mods.git", + {ref,"5e4a240791f6bbadc9b3bce4653ba08cb9aeb0ce"}}, 0}, {<<"riak_core">>, - {git,"git://github.com/basho/riak_core.git", - {ref,"3903efc8c975c47125babe06f2a4cc674616b658"}}, + {git,"https://github.com/basho/riak_core.git", + {ref,"7b04e2c4731c16ffb9cfbe1ac085cb4056cb0422"}}, 1}, {<<"riak_dt">>, - {git,"git://github.com/basho/riak_dt.git", + {git,"https://github.com/basho/riak_dt.git", {ref,"d0de4694f1423dc942844fbc494604a3c6df17d6"}}, 1}, {<<"riak_ensemble">>, {git,"https://github.com/basho/riak_ensemble", - {ref,"573a271aa546dcdcaeaaf09b3de6949a3130e021"}}, + {ref,"163bafa1a29db14f242e164dbe79b3fa84c3173d"}}, 2}, {<<"riak_kv">>, - {git,"git://github.com/basho/riak_kv.git", - {ref,"d17495741ed001dcb54fa9e75e0dff026baaefde"}}, + {git,"https://github.com/basho/riak_kv.git", + {ref,"e80a0699ffc5bc004b93e06c09197b79109d6844"}}, 0}, {<<"riak_pb">>, - {git,"git://github.com/basho/riak_pb.git", - {ref,"64cabe25eec68acd6cc38035cb91814dff3c5f14"}}, - 1}, + {git,"https://github.com/basho/riak_pb.git", + {ref,"04e8cc7daa081c23be137b1eff2a6a3fb827f8e4"}}, + 2}, {<<"riak_pipe">>, - {git,"git://github.com/basho/riak_pipe.git", - {ref,"811ae70fc258e4c3e8d65ff0593ea06989b6dec1"}}, + {git,"https://github.com/basho/riak_pipe.git", + {ref,"abedf61218d4ed61a3494a12e95bd4de0277468d"}}, 1}, {<<"riak_repl">>, - {git,"git://github.com/basho/riak_repl.git", - {ref,"90c317286fdc8cff3709e8dcd6d99633d89201ce"}}, + {git,"https://github.com/basho/riak_repl.git", + {ref,"dab42494f35b5102925138b28e8a1b0492947287"}}, 0}, {<<"riak_sysmon">>, - {git,"git://github.com/basho/riak_sysmon.git", - {ref,"726df1f4c31108bb9366fb767b480f286e51f7fc"}}, + {git,"https://github.com/basho/riak_sysmon.git", + {ref,"bb75c2b8fd2e1074ffded60054f3a17b745957aa"}}, 2}, {<<"riakc">>, - {git,"git://github.com/basho/riak-erlang-client", - {ref,"3c6c0d28507b87026f99b7de583eedda851338f7"}}, + {git,"https://github.com/basho/riak-erlang-client", + {ref,"3b6ad6e92dcb55833e9add829b2871be8468a68e"}}, 2}, {<<"riakhttpc">>, - {git,"git://github.com/basho/riak-erlang-http-client", - {ref,"762c7cb5a1ca440207817dd452840431697430ab"}}, + {git,"https://github.com/basho/riak-erlang-http-client", + {ref,"20cfdf4eb1f2e2f8521fd6c974d7b3e2bd9db728"}}, 1}, - {<<"setup">>,{pkg,<<"setup">>,<<"2.0.2">>},3}, + {<<"riaknostic">>, + {git,"https://github.com/basho/riaknostic.git", + {ref,"c86a7a943f6c5b5b743968d183f03caa98c78cf9"}}, + 0}, + {<<"setup">>,{pkg,<<"setup">>,<<"2.1.0">>},3}, {<<"sext">>, - {git,"git://github.com/uwiger/sext.git", + {git,"https://github.com/uwiger/sext.git", {ref,"615eebcf975ec4b4561c6f2b2bc433dabdb2be0f"}}, 1}, {<<"sidejob">>, - {git,"git://github.com/basho/sidejob.git", + {git,"https://github.com/basho/sidejob.git", {ref,"d5e8f8450b92ab30610496ffb58f560d6a8bad6b"}}, 1}, {<<"syslog">>, - {git,"git://github.com/Vagabond/erlang-syslog", - {ref,"4a6c6f2c996483e86c1320e9553f91d337bcb6aa"}}, + {git,"https://github.com/basho/erlang-syslog", + {ref,"b6d4db6fcaf5ce1031c77a1582e3ccd24b9ae0e8"}}, 1}, {<<"webmachine">>, - {git,"git://github.com/webmachine/webmachine.git", + {git,"https://github.com/webmachine/webmachine.git", {ref,"92225b82fc702f78cf9d23248023ec841272df80"}}, 2}]}. [ {pkg_hash,[ - {<<"bear">>, <<"16264309AE5D005D03718A5C82641FCC259C9E8F09ADEB6FD79CA4271168656F">>}, - {<<"folsom">>, <<"A885F0AEEE4C84270954C88A55A5A473D6B2C7493E32FFDC5765412DD555A951">>}, + {<<"bear">>, <<"430419C1126B477686CDE843E88BA0F2C7DC5CDF0881C677500074F704339A99">>}, + {<<"exometer_core">>, <<"742ED00E1F10F8BEFF61A0F43E105C78C2F982E10F1C4E740D59DF89924C65EA">>}, + {<<"folsom">>, <<"50ECC998D2149939F1D5E0AA3E32788F8ED16A58E390D81B5C0BE4CC4EF25589">>}, {<<"goldrush">>, <<"F06E5D5F1277DA5C413E84D5A2924174182FB108DABB39D5EC548B27424CD106">>}, - {<<"hut">>, <<"08D46679523043424870723923971889E8A34D63B2F946A35B46CF921D1236E7">>}, - {<<"parse_trans">>, <<"09765507A3C7590A784615CFD421D101AEC25098D50B89D7AA1D66646BC571C1">>}, - {<<"setup">>, <<"1203F4CDA11306C2E34434244576DED0A7BBFB0908D9A572356C809BD0CDF085">>}]} + {<<"hut">>, <<"71F2F054E657C03F959CF1ACC43F436EA87580696528CA2A55C8AFB1B06C85E7">>}, + {<<"parse_trans">>, <<"6E6AA8167CB44CC8F39441D05193BE6E6F4E7C2946CB2759F015F8C56B76E5FF">>}, + {<<"setup">>, <<"05F69185A5EB71474C9BC6BA892565651EC7507791F85632B7B914DBFE130510">>}]} ]. diff --git a/rel/files/app_epath.sh b/rel/files/app_epath.sh new file mode 100755 index 000000000..a3de910b0 --- /dev/null +++ b/rel/files/app_epath.sh @@ -0,0 +1,173 @@ +#!/bin/sh + +## --------------------------------------------------------------------- +## +## app_epath.sh: Parse Erlang app.config with POSIX tools for automation +## +## Copyright (c) 2013 Basho Technologies, Inc. All Rights Reserved. +## +## This file is provided to you under the Apache License, +## Version 2.0 (the "License"); you may not use this file +## except in compliance with the License. You may obtain +## a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, +## software distributed under the License is distributed on an +## "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +## KIND, either express or implied. See the License for the +## specific language governing permissions and limitations +## under the License. +## +## --------------------------------------------------------------------- + +## Example usage: +# +# #!/bin/sh +# +# # Load the functions +# . path/to/app_epath.sh +# +# # Build the path info +# epaths=`make_app_epaths path/to/app.config` +# +# # View all of the settings. Quotes are important. +# echo "$epaths" +# +# # Grep around in the paths for various items. +# echo "$epaths" | grep 'riak_core ring_creation_size' +# echo "$epaths" | grep "lager handlers lager_file_backend" | grep info +# +# # Use the epath function to get directly at settings +# epath 'riak_core ring_creation_size' "$epaths" +# epath 'riak_core platform_bin_dir' "$epaths" +# epath 'riak_kv storage_backend' "$epaths" +# +# # Use epath to view all of the riak_core settings +# epath riak_core +# +## End example + +## Here is a command you can put in your path for general cli use. +# +# #!/bin/sh +# +# # $1 quoted eterm path for search: 'riak_core ring_creation_size' +# # $2 path/to/app.config +# +# . path/to/app_epath.sh +# +# epaths=`make_app_epaths "$2"` +# epath "$1" "$epaths" +# +## End epath command + +# make_app_epaths takes a path to an app.config file as its argument and +# and returns (prints) a flattened text structure of configuration settings. + +make_app_epaths () { + # Remove all lines containing comments + # Remove all blank lines + # Remove the first [ + # Remove the last ]. + # Remove all blank lines again (just in case) + appconfig=`cat $1 \ + | sed '/^[ \t]*%/d' \ + | sed '/^[ \t]*$/d' \ + | sed -e '/\[/{s///;:a' -e '$!N;$!ba' -e '}' \ + | sed '$ s/\]\.//g' \ + | sed '/^[ \t]*$/d'` + + STACK= + INQUOTE=0 + # The awk puts each char, spaces included, on their own line for parsing. + echo "$appconfig" | awk '{gsub(//,"\n");print}' | while read i; do + case "x${i}x" in + "x\"x") + # Flip the INQUOTE state and echo the quote + if [ 1 -eq $INQUOTE ]; then + INQUOTE=0 + else + INQUOTE=1 + fi + STACK="${STACK}${i}" + ;; + "xx") + if [ 1 -eq $INQUOTE ]; then + # If in quotes, keep this space + STACK="${STACK} " + fi + ;; + "x,x") + if [ 1 -eq $INQUOTE ]; then + # If in quotes, keep this comma + STACK="${STACK}," + else + # Commas outside quotes means a new item is next + STACK="${STACK} " + fi + ;; + "x{x") + if [ 1 -eq $INQUOTE ]; then + # If in quotes, keep this bracket + STACK="${STACK}{" + else + # Add brace to the stack; will pop off from here on next } + STACK="${STACK} __EBRACE__ " + fi + ;; + "x}x") + if [ 1 -eq $INQUOTE ]; then + # If in quotes, keep this bracket + STACK="${STACK}}" + else + # We're only interested in printing leaves, not + # intermediates like 'riak_core http', which contain lists. + # See if the current stack ends with ] (end of list). + echo $STACK | grep -E "__EBRACKET__$" >/dev/null 2>&1 + if [ 1 -eq $? ]; then + # If not, print the stack without all of the mess. + echo "$STACK" | \ + sed 's/ *__EBRACE__//g;s/ *__EBRACKET__//g;s/^ *//' + fi + # Pop off everything from the last brace on. + STACK=`echo "$STACK" | sed 's/\(.*\) __EBRACE__.*/\1/g'` + fi + ;; + "x[x") + if [ 1 -eq $INQUOTE ]; then + # If in quotes, keep this bracket + STACK="${STACK}[" + else + # Add a placeholder to aid in determining whether or not to + # print. That is, we don't want to print 'riak_core http'. + STACK="${STACK} __EBRACKET__ " + fi + ;; + "x]x") + if [ 1 -eq $INQUOTE ]; then + # If in quotes, keep this bracket + STACK="${STACK}]" + fi + # Don't actually do anything with ], as the starting brackets + # are instead removed with }. + ;; + *) + # Anything else is just pushed. + STACK="${STACK}${i}" + ;; + esac + done +} + +epath () { + # arg1 - a pattern to search for + # arg2 - output of make_app_epaths, passed in quoted + # output - search of arg2 for arg1, trimming arg1 from the beginning + # Note: there may be multiple lines of output. + pat=$1 + shift + echo "$*" | grep "$pat " | sed "s/^${pat} *//" +} + diff --git a/rel/files/erl_codeloadingmode b/rel/files/erl_codeloadingmode new file mode 100644 index 000000000..b6c8c6716 --- /dev/null +++ b/rel/files/erl_codeloadingmode @@ -0,0 +1,3 @@ +#!/bin/sh +CODE_LOADING_MODE="${CODE_LOADING_MODE:-interactive}" +export CODE_LOADING_MODE diff --git a/rel/files/riak b/rel/files/riak index b41b97c2f..a04281509 100755 --- a/rel/files/riak +++ b/rel/files/riak @@ -4,6 +4,7 @@ RUNNER_GEN_DIR="${RUNNER_GEN_DIR:-{{ platform_gen_dir }}}" RELEASE_ROOT_DIR="${RELEASE_ROOT_DIR:-{{ runner_base_dir }}}" PID_DIR={{pid_dir}} COMMAND={{platform_bin_dir}}/riak +DEBUG_COMMAND={{platform_bin_dir}}/riak-debug RUNNER_LOG_DIR={{platform_log_dir}} RELX_CONFIG_PATH=${RUNNER_GEN_DIR}/sys.config VMARGS_PATH=${RUNNER_GEN_DIR}/vm.args @@ -38,10 +39,17 @@ else fi case "$1" in start|console|foreground) - su - riak -c "RELX_CONFIG_PATH=${RELX_CONFIG_PATH} VMARGS_PATH=${VMARGS_PATH} RUNNER_LOG_DIR=${RUNNER_LOG_DIR} PIPE_DIR=${PIPE_DIR} ${COMMAND} ${*} -pa {{platform_lib_dir}}/patches" + su - riak -c "NODETOOL_NODE_PREFIX=${NODETOOL_NODE_PREFIX} RELX_CONFIG_PATH=${RELX_CONFIG_PATH} VMARGS_PATH=${VMARGS_PATH} RUNNER_LOG_DIR=${RUNNER_LOG_DIR} PIPE_DIR=${PIPE_DIR} ${COMMAND} ${*} -pa {{platform_lib_dir}}/patches" + ;; + debug) + # Drop the "debug" from the args as we're going to directly call riak-debug now + shift + # Debug may fail if run via relx script due to use of relative start location, and also need for root access + NODETOOL_NODE_PREFIX=${NODETOOL_NODE_PREFIX} RELX_CONFIG_PATH=${RELX_CONFIG_PATH} VMARGS_PATH=${VMARGS_PATH} RUNNER_LOG_DIR=${RUNNER_LOG_DIR} PIPE_DIR=${PIPE_DIR} ${DEBUG_COMMAND} ${*} ;; *) - su - riak -c "RELX_CONFIG_PATH=${RELX_CONFIG_PATH} VMARGS_PATH=${VMARGS_PATH} RUNNER_LOG_DIR=${RUNNER_LOG_DIR} PIPE_DIR=${PIPE_DIR} ${COMMAND} ${*}" + ESCAPED_ARGS=`echo "$@" | sed -e 's/\([\\\(\\\){}"\x27]\)/\\\\\1/g'` + su - riak -c "NODETOOL_NODE_PREFIX=${NODETOOL_NODE_PREFIX} RELX_CONFIG_PATH=${RELX_CONFIG_PATH} VMARGS_PATH=${VMARGS_PATH} RUNNER_LOG_DIR=${RUNNER_LOG_DIR} PIPE_DIR=${PIPE_DIR} ${COMMAND} ${ESCAPED_ARGS}" ;; esac fi diff --git a/rel/files/riak-admin b/rel/files/riak-admin index 6f4cdc2ce..59b94f1d8 100755 --- a/rel/files/riak-admin +++ b/rel/files/riak-admin @@ -11,9 +11,22 @@ cd $RUNNER_BASE_DIR # Identify the script name SCRIPT="riak-admin" +# Setup required due to relx not node-package +ERTS_PATH=$BINDIR +NAME_PARAM="-name" +BOOT_FILE="start_clean" + +# Function to validate the node is down +node_down_check() { + if relx_nodetool "ping" > /dev/null; then + echo "Node is already running!" + exit 1 + fi +} + usage() { echo "Usage: $SCRIPT { cluster | join | leave | backup | restore | test | " - echo " reip | js-reload | erl-reload | wait-for-service | " + echo " reip | reip_manual | erl-reload | wait-for-service | " echo " ringready | transfers | force-remove | down |" echo " cluster-info | member-status | ring-status | vnode-status |" echo " aae-status | diag | stat | status | transfer-limit | reformat-indexes |" @@ -78,7 +91,7 @@ Usage: $SCRIPT stat The following commands display, enable/disable and reset statistics. A statistics entry is given either as a 'dotted' exometer name - Identifiers separated by periods, '.', e.g. riak.riak_kv.node.gets, -or as a 'legacy' name (same as in riak-admin status) - e.g. node_gets. +or as a 'legacy' name (same as in riak admin status) - e.g. node_gets. When a legacy name is listed, the corresponding exometer name is shown as well. Two kinds of wildcard are suppored: @@ -117,7 +130,7 @@ will match all stats entries. All Riak stat entry names start with 'riak', so \`stat show riak.**\` will match all riak stat entries. Example: -\$ bin/riak-admin stat show riak.riak_kv.node.gets +\$ bin/riak admin stat show riak.riak_kv.node.gets [riak,riak_kv,node,gets]: [{count,0},{one,0}] - 'Legacy name': The stat names used e.g. in \`$SCRIPT status\` can be used @@ -125,7 +138,7 @@ here, but also with wildcard support. The corresponding Exometer name and datapoint will be shown as well. Example: -\$ bin/riak-admin stat show node_gets +\$ bin/riak admin stat show node_gets == node_gets (Legacy pattern): == node_gets: 0 ([riak,riak_kv,node,gets]/one) @@ -136,22 +149,22 @@ be present, so '*.**' would work as a catch-all expression.) Each Exometer entry has a type and a set of datapoints. A filter can be given on the command line, selecting only a subset of datapoints: -\$ bin/riak-admin stat show riak.riak_kv.node.gets/one +\$ bin/riak admin stat show riak.riak_kv.node.gets/one [riak,riak_kv,node,gets]: [{one,0}] The type can also be restricted: -\$ bin/riak-admin stat show *.**/type=duration/mean,max +\$ bin/riak admin stat show *.**/type=duration/mean,max [riak,riak_core,converge_delay]: [{mean,0},{max,0}] [riak,riak_core,rebalance_delay]: [{mean,0},{max,0}] Note how multiple datapoints are separated by comma (no space). Showing disabled entries: -\$ bin/riak-admin stat show riak.riak_kv.node.gets +\$ bin/riak admin stat show riak.riak_kv.node.gets No matching stats -\$ bin/riak-admin stat show riak.riak_kv.node.gets/status=* +\$ bin/riak admin stat show riak.riak_kv.node.gets/status=* [riak,riak_kv,node,gets]: disabled -\$ bin/riak-admin stat show riak.riak_kv.node.gets/status=disabled +\$ bin/riak admin stat show riak.riak_kv.node.gets/status=disabled [riak,riak_kv,node,gets]: disabled " } @@ -166,16 +179,16 @@ updated, and have no value. The same syntax can be used as in \`stat show\`. The requested action will be performed on the matching entries. -\$ bin/riak-admin stat disable node_gets +\$ bin/riak admin stat disable node_gets == node_gets (Legacy pattern): == [riak,riak_kv,node,gets]: disabled -\$ bin/riak-admin stat enable node_gets +\$ bin/riak admin stat enable node_gets == node_gets (Legacy pattern): == [riak,riak_kv,node,gets]: enabled Wildcards can be used: -\$ bin/riak-admin stat disable riak.riak_kv.node.* +\$ bin/riak admin stat disable riak.riak_kv.node.* [riak,riak_kv,node,gets]: disabled [riak,riak_kv,node,puts]: disabled " @@ -197,7 +210,7 @@ with options: The same entry formats can be used as for all other stat subcommands. Example: -\$ bin/riak-admin stat info riak.riak_kv.node.gets +\$ bin/riak admin stat info riak.riak_kv.node.gets [riak,riak_kv,node,gets]: name = [riak,riak_kv,node,gets] type = spiral module = exometer_spiral @@ -207,7 +220,7 @@ Example: timestamp = undefined options = [{status,disabled}] -\$ bin/riak-admin stat info -type -status riak.riak_kv.node.gets +\$ bin/riak admin stat info -type -status riak.riak_kv.node.gets [riak,riak_kv,node,gets]: type = spiral status = disabled " @@ -290,6 +303,10 @@ cluster_admin() shift relx_nodetool rpc riak_core_console command $SCRIPT cluster partition-count $@ ;; + location) + shift + relx_nodetool rpc riak_core_console command $SCRIPT cluster location $@ + ;; *) echo "\ Usage: $SCRIPT cluster @@ -312,6 +329,8 @@ plan must be committed to take effect: force-replace Reassign all partitions owned by to without first handing off data, and remove from the cluster. + + location Assign a location to the current node Staging commands: plan Display the staged changes to the cluster @@ -578,12 +597,12 @@ btype_admin() # bucket-type create {"props": { ... }}" # if [ $# -le 3 ]; then - json=${3:-} + json=`echo ${3:-}|tr -d " "` relx_nodetool rpc riak_kv_console bucket_type_create "$2" "$json" else # greater than 3 parameters two=$2 shift 2 - three=$@ + three=`echo $@|tr -d " "` relx_nodetool rpc riak_kv_console bucket_type_create "$two" "$three" fi @@ -599,11 +618,12 @@ btype_admin() # bucket-type update {"props": { ... }}" # if [ $# -eq 3 ]; then - relx_nodetool rpc riak_kv_console bucket_type_update "$2" "$3" + json=`echo $3|tr -d " "` + relx_nodetool rpc riak_kv_console bucket_type_update "$2" "$json" else # greater than 3 parameters two=$2 shift 2 - three=$@ + three=`echo $@|tr -d " "` relx_nodetool rpc riak_kv_console bucket_type_update "$two" "$three" fi ;; @@ -869,16 +889,6 @@ case "$1" in done ;; - js[_-]reload) - # Reload all Javascript VMs - - shift #optional names come after 'js_reload' - - # Make sure the local node is running - - relx_nodetool rpc riak_kv_js_manager reload "$@" - ;; - erl[_-]reload) # Reload user Erlang code # Make sure the local node is running @@ -897,22 +907,63 @@ case "$1" in # Make sure the local node is not running node_down_check - # Sanity check the app.config file - check_config - ES=$? - if [ "$ES" -ne 0 ]; then - exit $ES + OLDNODE=$1 + NEWNODE=$2 + + echo "" + echo "Usage $SCRIPT $ACTION is not currently supported in Riak 3.0" + echo "" + echo "As an alternative use reip_manual:" + echo "riak admin reip_manual $OLDNODE $NEWNODE '/absolute_path_to/riak/data/ring' 'cluster_name'" + echo "" + echo "Where the path should be the absolute path to the ring directory in platform_data_dir containing filenames such as:" + echo "riak_core_ring.default.20221122164111" + echo "" + echo "The cluster_name should be set to the middle part of the ring file name (e.g. 'default' in this case)," + echo "or can be found when the node is active using app_helper:get_env(riak_core, cluster_name) from remote_console." + echo "" + + exit 1;; + + # check_config + # ES=$? + # if [ "$ES" -ne 0 ]; then + # exit $ES + # fi + + # OLDNODE=$1 + # NEWNODE=$2 + # $ERTS_PATH/erl -noshell \ + # -pa $RUNNER_LIB_DIR/basho-patches \ + # -boot $BOOT_FILE \ + # $CONFIG_ARGS \ + # -eval "riak_kv_console:$ACTION(['$OLDNODE', '$NEWNODE'])" \ + # -s init stop + # ;; + + reip_manual) + ACTION=$1 + shift + if [ $# -lt 4 ]; then + echo "Usage $SCRIPT $ACTION " + exit 1 fi + # Make sure the local node is not running + node_down_check + OLDNODE=$1 NEWNODE=$2 + RINGDIR=$3 + CLUSTERNAME=$4 $ERTS_PATH/erl -noshell \ -pa $RUNNER_LIB_DIR/basho-patches \ - $CONFIG_ARGS \ - -eval "riak_kv_console:$ACTION(['$OLDNODE', '$NEWNODE'])" \ + -boot $BOOT_FILE \ + -eval "riak_kv_console:$ACTION(['$OLDNODE', '$NEWNODE', '$RINGDIR', '$CLUSTERNAME'])" \ -s init stop ;; + restore) ACTION=$1 shift @@ -928,6 +979,7 @@ case "$1" in $ERTS_PATH/erl -noshell $NAME_PARAM riak_kv_backup$NAME_HOST -setcookie $COOKIE \ -pa $RUNNER_LIB_DIR/basho-patches \ + -boot $BOOT_FILE \ -eval "riak_kv_backup:$ACTION('$NODE', \"$FILENAME\")" -s init stop ;; @@ -947,6 +999,7 @@ case "$1" in $ERTS_PATH/erl -noshell $NAME_PARAM riak_kv_backup$NAME_HOST -setcookie $COOKIE \ -pa $RUNNER_LIB_DIR/basho-patches \ + -boot $BOOT_FILE \ -eval "riak_kv_backup:$ACTION('$NODE', \"$FILENAME\", \"$TYPE\")" -s init stop ;; @@ -960,6 +1013,7 @@ case "$1" in $ERTS_PATH/erl -noshell $NAME_PARAM riak_test$NAME_HOST $COOKIE_ARG \ -pa $RUNNER_LIB_DIR/basho-patches \ + -boot $BOOT_FILE \ -eval "case catch(riak:client_test(\"$NODE_NAME\")) of \ ok -> init:stop(); \ _ -> init:stop(1) \ @@ -969,7 +1023,7 @@ case "$1" in diag) case "$2" in --help|-h) - echo "Usage: riak-admin diag [-d ] [-l] [-h] [--export] [check_name ...]" + echo "Usage: riak admin diag [-d ] [-l] [-h] [--export] [check_name ...]" echo "" echo "-h, --help Display help/usage dialogue" echo "-d, --level Minimum message severity level (default: notice)" @@ -995,6 +1049,7 @@ case "$1" in # Using np_etop instead of riak_etop to follow node_package convention $ERTS_PATH/erl -noshell -noinput \ -pa $RUNNER_LIB_DIR/basho-patches \ + -boot $BOOT_FILE \ -hidden $NAME_PARAM np_etop$RAND$NAME_HOST $COOKIE_ARG \ -s etop -s erlang halt -output text \ -node $NODE_NAME \ diff --git a/rel/files/riak-chkconfig b/rel/files/riak-chkconfig index 9d521460e..649e6e8e0 100644 --- a/rel/files/riak-chkconfig +++ b/rel/files/riak-chkconfig @@ -21,3 +21,6 @@ CODE=" try $BINDIR/erl -noshell -boot start_clean -eval "$CODE" echo $CUTTLE_CONF + +# No explicit exit, assumed to have succeeded +exit 0 diff --git a/rel/files/riak-debug b/rel/files/riak-debug index b64bfe4d4..1de29390b 100755 --- a/rel/files/riak-debug +++ b/rel/files/riak-debug @@ -23,7 +23,7 @@ ## ------------------------------------------------------------------- # If you start to think "We should execute some Erlang in here", then go work -# on Riaknostic, which is called with `riak-admin diag` below. +# on Riaknostic, which is called with `riak admin diag` below. # /bin/sh on Solaris is not a POSIX compatible shell, but /usr/bin/ksh is. if [ `uname -s` = 'SunOS' -a "${POSIX_SHELL}" != "true" ]; then @@ -140,6 +140,8 @@ RUNNER_BASE_DIR={{runner_base_dir}} riak_base_dir={{runner_base_dir}} riak_bin_dir={{runner_script_dir}} riak_etc_dir={{runner_etc_dir}} +riak_sbin_dir=/usr/sbin +curdir=`pwd` get_cfgs=0 get_ssl_certs=0 @@ -153,7 +155,7 @@ verbose_output=0 ## Use riak config generate to set $riak_app_config and ## $riak_vm_args -gen_result=`"$riak_bin_dir"/riak config generate | cut -d' ' -f 3,5` +gen_result=`"$riak_bin_dir"/riak chkconfig | grep -v "OK" | cut -d' ' -f 2,4` riak_app_config=`echo $gen_result | cut -d' ' -f 1` generated_config_dir=`dirname "$riak_app_config"` riak_vm_args=`echo $gen_result | cut -d' ' -f 2` @@ -333,7 +335,7 @@ fi if [ -z "$outfile" ]; then # If output file not specified, output to the default - outfile="`pwd`"/"${debug_dir}".tar.gz + outfile="${curdir}"/"${debug_dir}".tar.gz fi if [ '-' != "$outfile" ] && [ -f "$outfile" ]; then @@ -356,7 +358,8 @@ if [ 1 -eq $get_syscmds ]; then dump last last dump hostname hostname dump uname uname -a - dump lsb_release lsb_release + dump lsb_release lsb_release -a + dump hostnamectl hostnamectl dump ps ps aux dump vmstat vmstat 1 5 dump free free -m @@ -388,7 +391,7 @@ if [ 1 -eq $get_syscmds ]; then BLOCKDEV=/sbin/blockdev if [ -e $BLOCKDEV ]; then - for mount_point in `mount | egrep '^/' | awk '{print $1}'`; do + for mount_point in `mount | egrep '^/' | grep -v /dev/shm | grep -v /var/lib/snapd | awk '{print $1}'`; do flat_point=`echo $mount_point | sed 's:/:_:g'` dump blockdev.$flat_point $BLOCKDEV --getra $mount_point done @@ -428,6 +431,13 @@ if [ 1 -eq $get_syscmds ]; then [ -f /etc/release ] && dump release cat /etc/release [ -f /etc/redhat-release ] && dump redhat_release cat /etc/redhat-release [ -f /etc/debian_version ] && dump debian_version cat /etc/debian_version + [ -f /etc/amazon-release ] && dump amazon_release cat /etc/amazon-release + [ -f /etc/ec2_version ] && dump ec2_version cat /etc/ec2_version + [ -f /etc/centos-release ] && dump centos_release cat /etc/centos-release + [ -f /etc/oracle-release ] && dump oracle_release cat /etc/oracle-release + [ -f /etc/os-release ] && dump os_release cat /etc/os-release + [ -f /etc/issue ] && dump issue cat /etc/issue + [ -f /etc/system-release ] && dump system_release cat /etc/system-release [ -f /etc/security/limits.conf ] && dump limits.conf cat /etc/security/limits.conf [ -f /var/log/messages ] && dump messages cat /var/log/messages [ -f /var/log/syslog ] && dump messages cat /var/log/syslog @@ -467,27 +477,32 @@ if [ 1 -eq $get_riakcmds ]; then mkdir_or_die "${start_dir}"/"${debug_dir}"/commands/.info cd "${start_dir}"/"${debug_dir}"/commands - # Note that 'riak-admin status' and 'riak-admin transfers' are heavy + # Note that 'riak admin status' and 'riak admin transfers' are heavy # commands on Riak<=1.2.0 and should not be executed in a loop against all # nodes in a cluster. - dump riak_ping "$riak_bin_dir"/riak ping - dump riak_version "$riak_bin_dir"/riak version - dump riak_member_status "$riak_bin_dir"/riak-admin member-status - dump riak_ring_status "$riak_bin_dir"/riak-admin ring-status - dump riak_status "$riak_bin_dir"/riak-admin status - dump riak_transfers "$riak_bin_dir"/riak-admin transfers - dump riak_aae_status "$riak_bin_dir"/riak-admin aae-status - dump riak_search_aae_status "$riak_bin_dir"/riak-admin search aae-status - dump riak_diag "$riak_bin_dir"/riak-admin diag - dump riak_repl_status "$riak_bin_dir"/riak-repl status - dump riak_repl_connections "$riak_bin_dir"/riak-repl connections - dump riak_repl_clusterstats "$riak_bin_dir"/riak-repl clusterstats - dump riak_repl_modes "$riak_bin_dir"/riak-repl modes + dump riak_ping "$riak_sbin_dir"/riak ping + dump riak_version "$riak_sbin_dir"/riak versions + dump riak_member_status "$riak_sbin_dir"/riak admin member-status + dump riak_ring_status "$riak_sbin_dir"/riak admin ring-status + dump riak_status "$riak_sbin_dir"/riak admin status + dump riak_transfers "$riak_sbin_dir"/riak admin transfers + dump riak_diag "$riak_sbin_dir"/riak admin diag + dump riak_repl_status "$riak_sbin_dir"/riak repl status + dump riak_repl_connections "$riak_sbin_dir"/riak repl connections + dump riak_repl_clusterstats "$riak_sbin_dir"/riak repl clusterstats + dump riak_repl_modes "$riak_sbin_dir"/riak repl modes # Make a flat, easily searchable version of the app.config settings riak_epaths=`make_app_epaths "${riak_app_config}"` + # If Yokozuna is not installed, $yz_test will be null. + yz_test="`epath 'yokozuna root_dir' "$riak_epaths" | sed -e 's/^\"//' -e 's/\".*$//'`" + # Only execute the search aae_statis command if $yz_test is not null. + if [ -n "$yz_test" ]; then + dump riak_search_aae_status "$riak_sbin_dir"/riak admin search aae-status + fi + # Get one http listener (epath might output a newline-separated list). riak_api_http="`epath 'riak_api http' "$riak_epaths" | sed -e 's/^\"//' -e 's/\" /:/' | head -n1`" @@ -532,55 +547,59 @@ if [ 1 -eq $get_yzcmds ]; then [ -z "$riak_epaths" ] && riak_epaths=`make_app_epaths "${riak_app_config}"` yz_dir="`epath 'yokozuna root_dir' "$riak_epaths" | sed -e 's/^\"//' -e 's/\".*$//'`" - if [ '/' != `echo "$yz_dir" | cut -c1` ]; then - # relative path. prepend base dir - yz_dir="$riak_base_dir"/"$yz_dir" - fi + # If Yokozuna is not installed, $yz_dir will be null and cause errors. + # Only execute the Yokozuna commands if $yz_dir is not null. + if [ -n "$yz_dir" ]; then + if [ '/' != `echo "$yz_dir" | cut -c1` ]; then + # relative path. prepend base dir + yz_dir="$riak_base_dir"/"$yz_dir" + fi - yz_aae_dir="`epath 'yokozuna anti_entropy_data_dir' "$riak_epaths" | sed -e 's/^\"//' -e 's/\".*$//'`" - if [ '/' != `echo "$yz_aae_dir" | cut -c1` ]; then - # relative path. prepend base dir - yz_aae_dir="$riak_base_dir"/"$yz_aae_dir" - fi + yz_aae_dir="`epath 'yokozuna anti_entropy_data_dir' "$riak_epaths" | sed -e 's/^\"//' -e 's/\".*$//'`" + if [ '/' != `echo "$yz_aae_dir" | cut -c1` ]; then + # relative path. prepend base dir + yz_aae_dir="$riak_base_dir"/"$yz_aae_dir" + fi - if [ -d "$yz_dir" ]; then - mkdir_or_die "${start_dir}"/"${debug_dir}"/yokozuna/.info - cd "${start_dir}"/"${debug_dir}"/yokozuna + if [ -d "$yz_dir" ]; then + mkdir_or_die "${start_dir}"/"${debug_dir}"/yokozuna/.info + cd "${start_dir}"/"${debug_dir}"/yokozuna - # grab a listing of the yokozuna directory - dump ls_yokozuna_dir ls -lhR "$yz_dir" + # grab a listing of the yokozuna directory + dump ls_yokozuna_dir ls -lhR "$yz_dir" - # Take a du listing of the yokozuna directory for size checking. - # This info is included in the ls, but parsing ls is not recommended. - dump du_yokozuna_dir du "$yz_dir"/* - - # tar up every /conf directory. This will assure we capture the - # schema (regardless of name/extension). - cd "$yz_dir" - for f in `ls -1 .`; do - if [ -d "$f" -a -d "$f"/conf ]; then - tar -czf ""${start_dir}"/"${debug_dir}"/yokozuna/${f}_conf.tar.gz" "$f"/conf - fi - done - fi + # Take a du listing of the yokozuna directory for size checking. + # This info is included in the ls, but parsing ls is not recommended. + dump du_yokozuna_dir du "$yz_dir"/* - if [ -d "$yz_aae_dir" ]; then - mkdir_or_die "${start_dir}"/"${debug_dir}"/yokozuna/anti_entropy/.info - cd "${start_dir}"/"${debug_dir}"/yokozuna/anti_entropy + # tar up every /conf directory. This will assure we capture the + # schema (regardless of name/extension). + cd "$yz_dir" + for f in `ls -1 .`; do + if [ -d "$f" -a -d "$f"/conf ]; then + tar -czf ""${start_dir}"/"${debug_dir}"/yokozuna/${f}_conf.tar.gz" "$f"/conf + fi + done + fi - # Take a listing of the yz_aae_dir directory for reference - dump ls_yokozuna_anti_entropy_dir ls -lhR "$yz_aae_dir" + if [ -d "$yz_aae_dir" ]; then + mkdir_or_die "${start_dir}"/"${debug_dir}"/yokozuna/anti_entropy/.info + cd "${start_dir}"/"${debug_dir}"/yokozuna/anti_entropy - # Take a du listing of the yz_aae_dir directory for size checking. - # This info is included in the ls, but parsing ls is not recommended. - dump du_yokozuna_anti_entropy_dir du "$yz_aae_dir"/* + # Take a listing of the yz_aae_dir directory for reference + dump ls_yokozuna_anti_entropy_dir ls -lhR "$yz_aae_dir" - # Mirror the directory, only copying files that match the pattern - cd "$yz_aae_dir" - find . -type f -name 'LOG*' -exec sh -c ' - mkdir -p "$0/${1%/*}"; - cp "$1" "$0/$1" - ' "${start_dir}"/"${debug_dir}"/yokozuna/anti_entropy {} \; + # Take a du listing of the yz_aae_dir directory for size checking. + # This info is included in the ls, but parsing ls is not recommended. + dump du_yokozuna_anti_entropy_dir du "$yz_aae_dir"/* + + # Mirror the directory, only copying files that match the pattern + cd "$yz_aae_dir" + find . -type f -name 'LOG*' -exec sh -c ' + mkdir -p "$0/${1%/*}"; + cp "$1" "$0/$1" + ' "${start_dir}"/"${debug_dir}"/yokozuna/anti_entropy {} \; + fi fi fi @@ -591,7 +610,7 @@ fi if [ 1 -eq $get_extracmds ]; then mkdir_or_die "${start_dir}"/"${debug_dir}"/commands/.info cd "${start_dir}"/"${debug_dir}"/commands - dump riak_vnode_status "$riak_bin_dir"/riak-admin vnode-status + dump riak_vnode_status "$riak_sbin_dir"/riak admin vnode-status fi ### @@ -716,6 +735,30 @@ if [ 1 -eq $get_logs ]; then # This info is included in the ls, but parsing ls is not recommended. dump du_bitcask_dir du "$bitcask_dir"/* + elif [ 'riak_kv_leveled_backend' = "$backend" ]; then + leveled_dir="`epath 'leveled data_root' "$riak_epaths" | sed -e 's/^\"//' -e 's/\".*$//'`" + + if [ '/' != `echo "$leveled_dir" | cut -c1` ]; then + # relative path. prepend base dir + leveled_dir="$riak_base_dir"/"$leveled_dir" + fi + + if [ ! -d "$leveled_dir" ]; then + echoerr "Unable to locate Leveled data directory. Aborting." + echoerr "Using leveled data_root: $leveled_dir" + exit 1 + fi + + mkdir_or_die "${start_dir}"/"${debug_dir}"/logs/leveled/.info + cd "${start_dir}"/"${debug_dir}"/logs/leveled + + # Take a listing of the bitcask directory for reference + dump ls_leveled_dir ls -lhR "$leveled_dir" + + # Take a du listing of the bitcask directory for size checking. + # This info is included in the ls, but parsing ls is not recommended. + dump du_leveled_dir du "$leveled_dir"/* + # Walk multi-backends and collect whatever information is there to collect. elif [ 'riak_kv_multi_backend' = "$backend" ] || [ 'riak_cs_kv_multi_backend' = "$backend" ] ; then @@ -777,6 +820,30 @@ if [ 1 -eq $get_logs ]; then # Take a du listing of the bitcask directory for size checking. # This info is included in the ls, but parsing ls is not recommended. dump du_bitcask_dir du "$dr"/* + elif [ 'riak_kv_leveled_backend' = "$backend" ]; then + dr="`epath "riak_kv multi_backend $b riak_kv_leveled_backend data_root" "$riak_epaths" | + sed -e 's/^\"//' -e 's/\".*$//'`" + + if [ '/' != `echo "$dr" | cut -c1` ]; then + # relative path. prepend base dir + dr="$riak_base_dir"/"$dr" + fi + + if [ ! -d "${dr}" ]; then + echoerr "Unable to locate $b Leveled data directory. Aborting." + echoerr "Using riak_kv_leveled_backend data_root: $dr" + exit 1 + fi + + mkdir_or_die "${start_dir}"/"${debug_dir}"/logs/leveled-${b}/.info + cd "${start_dir}"/"${debug_dir}"/logs/leveled-${b} + + # Take a listing of the bitcask directory for reference + dump ls_leveled_dir ls -lhR "$dr" + + # Take a du listing of the bitcask directory for size checking. + # This info is included in the ls, but parsing ls is not recommended. + dump du_leveled_dir du "$dr"/* fi done fi @@ -811,27 +878,57 @@ if [ 1 -eq $get_logs ]; then cp "$1" "$0/$1" ' "${start_dir}"/"${debug_dir}"/logs/anti_entropy {} \; fi + # Gather AAE logs, listing, sizing information, etc.. + # Calling `epath 'riak_kv tictacaae' "$riak_epaths. . .` will result in + # an empty variable regardless of the value set in the configuration + # settings. We're going to grab the `tictacaae_dataroot` instead, and + # assume it's presence on disk indicates activity. + tictacaae_dir="`epath 'riak_kv tictacaae_dataroot' "$riak_epaths" | sed -e 's/^\"//' -e 's/\".*$//'`" + + if [ '/' != `echo "$tictacaae_dir" | cut -c1` ]; then + # relative path. prepend base dir + tictacaae_dir="$riak_base_dir"/"$tictacaae_dir" + fi + + if [ -d "$tictacaae_dir" ]; then + mkdir_or_die "${start_dir}"/"${debug_dir}"/logs/tictacaae/.info + cd "${start_dir}"/"${debug_dir}"/logs/tictacaae + + # Take a listing of the anti_entropy_dir directory for reference + dump ls_tictacaae_dir ls -lhR "$tictacaae_dir" + + # Take a du listing of the anti_entropy_dir directory for size checking. + # This info is included in the ls, but parsing ls is not recommended. + dump du_tictacaae_dir du "$tictacaae_dir"/* + + # Mirror the directory, only copying files that match the pattern + cd "$tictacaae_dir" + find . -type f -name 'LOG*' -exec sh -c ' + mkdir -p "$0/${1%/*}"; + cp "$1" "$0/$1" + ' "${start_dir}"/"${debug_dir}"/logs/tictacaae {} \; + fi fi ### -### Gather Riak's basho-patches directory +### Gather Riak's patches directory ### if [ 1 -eq $get_patches ]; then - mkdir_or_die "${start_dir}"/"${debug_dir}"/basho-patches/.info - cd "${start_dir}"/"${debug_dir}"/basho-patches + mkdir_or_die "${start_dir}"/"${debug_dir}"/patches/.info + cd "${start_dir}"/"${debug_dir}"/patches # As per the below link, this patch should be based off the base_dir. # http://docs.basho.com/riakee/latest/cookbooks/Rolling-Upgrade-to-Enterprise/#Basho-Patches - riak_lib_dir="${riak_base_dir}/lib/basho-patches" + riak_lib_dir="${riak_base_dir}/lib/patches" # Use dump to execute the copy. This will provide a progress dot and # capture any error messages. - dump cp_basho_patches cp -R "$riak_lib_dir"/* . + dump cp_patches cp -R "$riak_lib_dir"/* . # If the copy succeeded, then the output will be empty and it is unneeded. if [ 0 -eq $? ]; then - rm -f cp_basho_patches + rm -f cp_patches fi fi diff --git a/rel/pkg/rpm/Makefile b/rel/pkg/rpm/Makefile index fed36bf7a..da19ab270 100644 --- a/rel/pkg/rpm/Makefile +++ b/rel/pkg/rpm/Makefile @@ -6,7 +6,7 @@ PWD = $(shell pwd) PKG_VERSION_NO_H ?= $(shell echo $(PKG_VERSION) | tr - .) default: - rpmbuild --define "_rpmfilename %%{NAME}-%%{VERSION}-%%{RELEASE}$(DISTRO)%%{ARCH}.rpm" \ + rpmbuild --define "_rpmfilename %%{NAME}-%%{VERSION}-%%{RELEASE}$(DISTRO).%%{ARCH}.rpm" \ --define '_topdir $(BASE_DIR)/rel/pkg/out/' \ --define '_sourcedir $(BASE_DIR)/rel/pkg/out/' \ --define '_specdir $(BASE_DIR)/rel/pkg/out/' \ diff --git a/rel/pkg/rpm/specfile b/rel/pkg/rpm/specfile index 05ab83604..58a3a3d1f 100644 --- a/rel/pkg/rpm/specfile +++ b/rel/pkg/rpm/specfile @@ -122,7 +122,6 @@ fi %post # Post Installation Script - # For distros with SELinux (RHEL/Fedora) if [ `which selinuxenabled > /dev/null 2>&1` ] ; then # Fixup perms for SELinux (if it is enabled)