diff --git a/AUTHORS b/AUTHORS new file mode 100644 index 0000000..4dadd72 --- /dev/null +++ b/AUTHORS @@ -0,0 +1 @@ +see README diff --git a/COPYING b/COPYING new file mode 100644 index 0000000..d511905 --- /dev/null +++ b/COPYING @@ -0,0 +1,339 @@ + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Lesser General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + , 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. diff --git a/ChangeLog b/ChangeLog new file mode 100644 index 0000000..ea31d6a --- /dev/null +++ b/ChangeLog @@ -0,0 +1,2 @@ +0.1 - Fri Dec 21 22:23:51 JST 2007 + - initial release diff --git a/INSTALL b/INSTALL new file mode 100644 index 0000000..4dadd72 --- /dev/null +++ b/INSTALL @@ -0,0 +1 @@ +see README diff --git a/Makefile.am b/Makefile.am new file mode 100644 index 0000000..8db637c --- /dev/null +++ b/Makefile.am @@ -0,0 +1,2 @@ +SUBDIRS= src +EXTRA_DIST= config/ac_mysql.m4 diff --git a/NEWS b/NEWS new file mode 100644 index 0000000..16db6ac --- /dev/null +++ b/NEWS @@ -0,0 +1 @@ +Please read the ChangeLog diff --git a/README b/README new file mode 100644 index 0000000..694b988 --- /dev/null +++ b/README @@ -0,0 +1,75 @@ +Q4M - a Message Queue for MySQL + +Dec. 21 2007 +Kazuho Oku +Cybozu Labs, Inc. + + +Q4M is a message queue that works as a pluggable storage engine of MySQL +5.1. + +Since its development is still in very early alpha-stage, NEVER USE THE +SOFTWARE IN PRODUCTION ENVIRONMENT. + + +* LICENSE and COPYRIGHT + +Please refer to each file. The engine was built from the Skeleton engine +and the copyright of the build scripts mostly belong to their authors. +Copyright of the source code of the queue engine belongs to Cybozu Labs, +Inc., and is licensed under GPLv2. + + +* LIMITATIONS + +- table contents are not saved onto disk + - wait for updates +- no indexes support + - simplicity for speed + - is it neressary, when we can create any number of tables=queues? + + +* INSTALLATION + +To install the software, follow the steps below. + + % ./configure --with-mysql= --libdir= + % make + % make install + % mysql -u root + > INSTALL PLUGIN queue SONAME 'libqueue_engine.so'; + > CREATE FUNCTION queue_wait RETURN INT SONAME 'libqueue_engine.so'; + > CREATE FUNCTION queue_end RETURN INT SONAME 'libqueue_engine.so'; + > CREATE FUNCTION queue_abort RETURN INT SONAME 'libqueue_engine.so'; + +If you are installing Q4M against a debug build of MySQL, it might be +necessary to set the -DSAFE_MUTEX compile option. + + % CFLAGS=-DSAFE_MUTEX CXXFLAGS=-DSAFE_MUTEX ./configure ... + + +* USAGE + +creating queues: + > CREATE TABLE my_queue (id INT UNSIGNED NOT NULL) ENGINE=queue; + +sending messages to queue: + > INSERT INTO my_queue (id) values (2),(3),(4); + +receiving messages from queue (shown in perl code): + + while (1) { + # wait until any data becomes available + $dbh->do('select queue_wait("dbname.tblname")'); + # receive data (only one row becomes ready at once) + if (my @ary = $dbh->selectrow_array('select * from tblname')) { + # process data + unless (process_row(@ary)) { + # abort if failed, return data to queue + $dbh->do('select queue_abort()'); + last; + } + } + } + # tidy up + $dbh->do('select queue_end()'); diff --git a/TODO b/TODO new file mode 100644 index 0000000..4dadd72 --- /dev/null +++ b/TODO @@ -0,0 +1 @@ +see README diff --git a/config/ac_mysql.m4 b/config/ac_mysql.m4 new file mode 100644 index 0000000..3c78184 --- /dev/null +++ b/config/ac_mysql.m4 @@ -0,0 +1,25 @@ +dnl --------------------------------------------------------------------------- +dnl Macro: MYSQL_SRC +dnl --------------------------------------------------------------------------- +AC_DEFUN([MYSQL_SRC_TEST], [ + AC_MSG_CHECKING(for mysql source code) + AC_ARG_WITH(mysql, + [[ --with-mysql[=mysql src directory] + Source requir to build engine.]], + [ + if test -d "$withval/sql"; then + MYSQL_SRC="$i/mysql_config" + fi + + AC_DEFINE([MYSQL_SRC], [1], [Source directory for MySQL]) + MYSQL_INC="-I$withval/sql -I$withval/include -I$withval/regex -I$withval" + AC_MSG_RESULT(["$withval"]) + ], + [ + AC_MSG_ERROR(["no mysql source provided"]) + ]) +]) + +dnl --------------------------------------------------------------------------- +dnl Macro: MYSQL_SRC +dnl --------------------------------------------------------------------------- diff --git a/configure.in b/configure.in new file mode 100644 index 0000000..e837f4c --- /dev/null +++ b/configure.in @@ -0,0 +1,35 @@ +dnl For Queue for MySQL - a mysql pluggable storage engine +AC_INIT(src/ha_queue.cc) +AC_CONFIG_AUX_DIR(config) +AM_CONFIG_HEADER(src/queue_config.h) +AM_INIT_AUTOMAKE("q4m", 0.1) + +AC_PROG_CC +AC_PROG_CXX +AC_PROG_CPP +AC_PROG_GCC_TRADITIONAL +AC_PROG_LIBTOOL +LIBTOOL="$LIBTOOL --preserve-dup-deps" +AC_SUBST(LIBTOOL)dnl + +sinclude(config/ac_mysql.m4) +sinclude(config/ac_system.m4) +sinclude(config/dtrace.m4) + +MYSQL_SRC_TEST +AC_SUBST(MYSQL_INC) + +# We only support GCC and Sun's forte at the moment +if test "$GCC" = "yes" +then + CXXFLAGS="$CXXFLAGS -fno-exceptions -fno-rtti" +else + CFLAGS="-Xa -xstrconst -mt -D_FORTEC_ -fast -m64" + CXXFLAGS="$CXXFLAGS -noex -mt -D_FORTEC_ -fast -m64" + DTRACEFLAGS="-64" +fi + +AC_C_CONST +AC_TYPE_SIZE_T +AC_CHECK_HEADERS(limits.h syslimits.h) +AC_OUTPUT(Makefile src/Makefile) diff --git a/src/Makefile.am b/src/Makefile.am new file mode 100644 index 0000000..aeb1662 --- /dev/null +++ b/src/Makefile.am @@ -0,0 +1,11 @@ +EXTRA_DIST= ha_queue.h +INCLUDES= $(MYSQL_INC) +noinst_HEADER= ha_queue.h + +lib_LTLIBRARIES= libqueue_engine.la +libqueue_engine_la_SOURCES= ha_queue.cc +libqueue_engine_la_LIBADD= + +libqueue_engine_la_LDFLAGS= -module +libqueue_engine_la_CFLAGS= $(AM_CFLAGS) -DMYSQL_DYNAMIC_PLUGIN +libqueue_engine_la_CXXFLAGS= $(AM_CFLAGS) -DMYSQL_DYNAMIC_PLUGIN diff --git a/src/ha_queue.cc b/src/ha_queue.cc new file mode 100644 index 0000000..1bd2fb8 --- /dev/null +++ b/src/ha_queue.cc @@ -0,0 +1,838 @@ +/* + * Copyright (C) 2007 Cybozu Labs, Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +extern "C" { +#include +#include +} +#include +#include +#include +#include + +#include "mysql_priv.h" +#undef PACKAGE +#undef VERSION +#undef HAVE_DTRACE +#undef _DTRACE_VERSION + +#include "queue_config.h" + +#include "ha_queue.h" +#include + +extern uint build_table_filename(char *buff, size_t bufflen, const char *db, + const char *table, const char *ext, + uint flags); + + +using namespace std; + +#define EXPAND_BY (65536) +#define Q4M ".Q4M" + +static HASH open_tables; +static pthread_mutex_t g_mutex; + +/* if non-NULL, access is restricted to the rows owned, and it points + * to queue_share_t + */ +static pthread_key_t share_key; + + +queue_file_header_t::queue_file_header_t() + : _magic(MAGIC), _padding1(0), _eod(sizeof(queue_file_header_t)) +{ + memset(_padding2, 0, sizeof(_padding2)); +} + +int queue_file_header_t::set_eod(int fd, off_t eod) +{ + if (pwrite(fd, &eod, sizeof(eod), offsetof(queue_file_header_t, _eod)) + != sizeof(eod)) { + return -1; + } + if (fsync(fd) != 0) { + return -1; + } + _eod = eod; + return 0; +} + +uchar* queue_share_t::get_share_key(queue_share_t *share, size_t *length, + my_bool not_used __attribute__((unused))) +{ + *length = share->table_name_length; + return reinterpret_cast(share->table_name); +} + +queue_share_t *queue_share_t::get_share(const char *table_name) +{ + queue_share_t *share; + uint table_name_length; + char *tmp_name; + char filename[FN_REFLEN]; + + pthread_mutex_lock(&g_mutex); + + table_name_length = strlen(table_name); + + /* return the one, if found (after incrementing refcount) */ + if ((share = reinterpret_cast(hash_search(&open_tables, reinterpret_cast(table_name), table_name_length))) + != NULL) { + ++share->use_count; + pthread_mutex_unlock(&g_mutex); + return share; + } + + /* alloc */ + if (my_multi_malloc(MYF(MY_WME | MY_ZEROFILL), &share, sizeof(queue_share_t), + &tmp_name, table_name_length + 1, NullS) + == NULL) { + goto ERR_RETURN; + } + + /* init members that would always succeed in doing so */ + share->use_count = 1; + share->table_name = tmp_name; + strmov(share->table_name, table_name); + share->table_name_length = table_name_length; + pthread_mutex_init(&share->mutex, MY_MUTEX_INIT_FAST); + thr_lock_init(&share->store_lock); + share->cache.off = 0; + new (&share->write_buf) vector(); + new (&share->rows_owned) queue_rows_owned_t(); + pthread_cond_init(&share->queue_cond, NULL); + share->num_readers = 0; + /* open file */ + fn_format(filename, share->table_name, "", Q4M, + MY_REPLACE_EXT | MY_UNPACK_FILENAME); + if ((share->fd = open(filename, O_RDWR, 0)) == -1) { + goto ERR_ON_FILEOPEN; + } + /* load header */ + if (read(share->fd, &share->_header, sizeof(share->_header)) + != sizeof(share->_header)) { + goto ERR_AFTER_FILEOPEN; + } + if (share->header()->magic() != queue_file_header_t::MAGIC) { + goto ERR_AFTER_FILEOPEN; + } + /* determine first row position */ + if ((share->first_row = sizeof(queue_file_header_t)) + == share->header()->eod()) { + queue_row_t* row = + reinterpret_cast(share->read_cache(share->first_row, 4, + true)); + if (row == NULL) { + goto ERR_AFTER_FILEOPEN; + } + if (row->is_removed() && share->next(&share->first_row) != 0) { + goto ERR_AFTER_FILEOPEN; + } + } + + /* add to open_tables */ + if (my_hash_insert(&open_tables, reinterpret_cast(share))) { + goto ERR_AFTER_FILEOPEN; + } + + /* success */ + pthread_mutex_unlock(&g_mutex); + return share; + + ERR_AFTER_FILEOPEN: + close(share->fd); + ERR_ON_FILEOPEN: + pthread_cond_destroy(&share->queue_cond); + share->rows_owned.~list(); + share->write_buf.~vector(); + thr_lock_delete(&share->store_lock); + pthread_mutex_destroy(&share->mutex); + my_free(reinterpret_cast(share), MYF(0)); + ERR_RETURN: + pthread_mutex_unlock(&g_mutex); + return NULL; +} + +void queue_share_t::release() +{ + pthread_mutex_lock(&g_mutex); + + if (--use_count == 0) { + close(fd); + pthread_cond_destroy(&queue_cond); + rows_owned.~list(); + write_buf.~vector(); + thr_lock_delete(&store_lock); + pthread_mutex_destroy(&mutex); + my_free(reinterpret_cast(this), MYF(0)); + } + + pthread_mutex_unlock(&g_mutex); +} + +off_t queue_share_t::reset_owner(pthread_t owner) +{ + off_t off = 0; + lock(); + + for (queue_rows_owned_t::iterator i = rows_owned.begin(); + i != rows_owned.end(); + ++i) { + if (i->first == owner) { + off = i->second; + rows_owned.erase(i); + break; + } + } + + unlock(); + return off; +} + +ssize_t queue_share_t::read_direct(void *data, off_t off, size_t size) +{ + return pread(fd, data, size, off); +} + +void *queue_share_t::read_cache(off_t off, size_t size, bool use_syscall) +{ + if (size > sizeof(cache.buf)) { + return NULL; + } + if (cache.off <= off && off + size <= cache.off + sizeof(cache.buf)) { + return cache.buf + off - cache.off; + } + if (! use_syscall) { + return NULL; + } + if (pread(fd, cache.buf, sizeof(cache.buf), off) < size) { + cache.off = 0; // invalidate + return NULL; + } + cache.off = off; + return cache.buf; +} + +int queue_share_t::write_file(const void *data, off_t off, size_t size) +{ + if (pwrite(fd, data, size, off) != size) { + return -1; + } + if (cache.off + sizeof(cache.buf) <= off || off + size <= cache.off) { + // direct write + } else if (cache.off <= off) { + memcpy(cache.buf + off - cache.off, + data, + min(size, sizeof(cache.off) - (off - cache.off))); + } else { + memcpy(cache.buf, + static_cast(data) + cache.off - off, + min(size - (cache.off - off), sizeof(cache.off))); + } + return 0; +} + +int queue_share_t::next(off_t *off) +{ + if (*off == header()->eod()) { + // eof + } else { + queue_row_t *row = + reinterpret_cast(read_cache(*off, + queue_row_t::header_size(), + true)); + if (row == NULL) { + return -1; + } + *off += queue_row_t::header_size() + row->size(); + while (1) { + if (*off == header()->eod()) { + break; + } + row = + reinterpret_cast(read_cache(*off, + queue_row_t::header_size(), + true)); + if (row == NULL) { + return -1; + } + if (! row->is_removed()) { + break; + } + *off += row->size(); + } + } + + return 0; +} + +off_t queue_share_t::get_owned_row(pthread_t owner) +{ + for (queue_rows_owned_t::iterator i = rows_owned.begin(); + i != rows_owned.end(); + ++i) { + if (i->first == owner) { + return i->second; + } + } + return 0; +} + +void queue_share_t::write_begin() +{ + write_buf.resize(queue_row_t::header_size()); +} + +void queue_share_t::write_append(const void* data, size_t size) +{ + size_t t = write_buf.size(); + write_buf.resize(t + size); + copy(static_cast(data), + static_cast(data) + size, + write_buf.begin() + t); +} + +int queue_share_t::write_commit() +{ + /* align */ + if (write_buf.size() & 3 != 0) { + size_t cplen = 4 - (write_buf.size() & 3); + const char* s = "\0\0"; + write_buf.resize(write_buf.size() + cplen); + copy(s, s + cplen, write_buf.end() - cplen); + } + /* setup header */ + queue_row_t *row = reinterpret_cast(&write_buf.front()); + row->init_header(write_buf.size() - queue_row_t::header_size()); + /* extend the file by certain amount for speed */ + if (_header.eod() / EXPAND_BY + != (_header.eod() + write_buf.size()) / EXPAND_BY) { + if (lseek(fd, (_header.eod() / EXPAND_BY + 1) * EXPAND_BY, SEEK_SET) + == -1) { + return -1; + } + } + /* write */ + write_file(row, _header.eod(), write_buf.size()); + write_buf.clear(); + /* sync data */ + if (fsync(fd) != 0) { + return -1; + } + /* update eod */ + return _header.set_eod(fd, + _header.eod() + row->size() + + queue_row_t::header_size()); +} + +int queue_share_t::erase_row(off_t off) +{ + queue_row_t *row = + static_cast(read_cache(off, queue_row_t::header_size(), + true)); + if (row == NULL) { + return -1; + } + row->set_is_removed(); + if (write_file(row, off, queue_row_t::header_size()) != 0) { + return -1; + } + if (fsync(fd) != 0) { + return -1; + } + if (off == first_row) { + if (next(&off) != 0) { + return -1; + } + first_row = off; + } + return 0; +} + +pthread_t queue_share_t::find_owner(off_t off) +{ + for (queue_rows_owned_t::const_iterator j = rows_owned.begin(); + j != rows_owned.end(); + ++j) { + if (off == j->second) { + return j->first; + } + } + return 0; +} + +off_t queue_share_t::assign_owner(pthread_t owner) +{ + off_t off = begin(); + while (off != end()) { + if (find_owner(off) == 0) { + rows_owned.push_back(queue_rows_owned_t::value_type(owner, off)); + return off; + } + if (next(&off) != 0) { + return 0; + } + } + return 0; +} + +static queue_share_t* get_share_check(const char* db_table_name) +{ + char buf[FN_REFLEN]; + char path[FN_REFLEN]; + + // copy to buf, split to db name and table name, and build filename + // FIXME: creates bogus name if db_table_name is too long (but no overruns) + strncpy(buf, db_table_name, FN_REFLEN - 1); + buf[FN_REFLEN - 1] = '\0'; + char* tbl = strchr(buf, '.'); + if (tbl == NULL) + return NULL; + *tbl++ = '\0'; + if (*tbl == '\0') + return NULL; + build_table_filename(path, FN_REFLEN - 1, buf, tbl, "", 0); + + return queue_share_t::get_share(path); +} + +static void erase_owned() +{ + queue_share_t *share; + + if ((share = static_cast(pthread_getspecific(share_key))) + != NULL) { + share->lock(); + off_t off = share->get_owned_row(pthread_self()); + share->erase_row(off); + share->unlock(); + share->release(); + pthread_setspecific(share_key, NULL); + } +} + +static int close_connection_handler(handlerton *hton, THD *thd) +{ + queue_share_t *share; + + if ((share = static_cast(pthread_getspecific(share_key))) + != NULL) { + if (share->reset_owner(pthread_self()) != 0) { + share->wake_listener(); + } + share->release(); + pthread_setspecific(share_key, NULL); + } + + return 0; +} + +handler *create_handler(handlerton *hton, TABLE_SHARE *table, + MEM_ROOT *mem_root) +{ + return new (mem_root) ha_queue(hton, table); +} + +static int init(void *p) +{ + handlerton* queue_hton = (handlerton *)p; + + pthread_mutex_init(&g_mutex, MY_MUTEX_INIT_FAST); + pthread_key_create(&share_key, NULL); + hash_init(&open_tables, system_charset_info, 32, 0, 0, + reinterpret_cast(queue_share_t::get_share_key), 0, 0); + + queue_hton->state = SHOW_OPTION_YES; + queue_hton->close_connection = close_connection_handler; + queue_hton->create = create_handler; + queue_hton->flags = HTON_CAN_RECREATE; + + return 0; +} + +static int deinit(void *p) +{ + hash_free(&open_tables); + pthread_mutex_destroy(&g_mutex); + + return 0; +} + +ha_queue::ha_queue(handlerton *hton, TABLE_SHARE *table_arg) + :handler(hton, table_arg), + share(NULL), + pos() +{ +} + +static const char *ha_queue_exts[] = { + Q4M, + NullS +}; + +const char **ha_queue::bas_ext() const +{ + return ha_queue_exts; +} + +int ha_queue::open(const char *name, int mode, uint test_if_locked) +{ + if ((share = queue_share_t::get_share(name)) == NULL) { + return 1; + } + thr_lock_data_init(share->get_store_lock(), &lock, NULL); + return 0; +} + +int ha_queue::close() +{ + share->release(); + return 0; +} + +int ha_queue::rnd_init(bool scan) +{ + share->lock_reader(); + pos = 0; + return 0; +} + +int ha_queue::rnd_end() +{ + share->unlock_reader(); + return 0; +} + +int ha_queue::rnd_next(uchar *buf) +{ + int err = HA_ERR_END_OF_FILE; + share->lock(); + + if (pthread_getspecific(share_key)) { + if (pos == 0 && (pos = share->get_owned_row(pthread_self())) != 0) { + // ok + } else { + goto EXIT; + } + } else { + if (pos == 0) { + if ((pos = share->begin()) == share->end()) { + goto EXIT; + } + } else { + if (share->next(&pos) != 0) { + err = HA_ERR_GENERIC; // what's the appropriate error code? + goto EXIT; + } else if (pos == share->end()) { + goto EXIT; + } + } + while (share->find_owner(pos) != 0) { + if (share->next(&pos) != 0) { + err = HA_ERR_GENERIC; // ???? + goto EXIT; + } + if (pos == share->end()) { + goto EXIT; + } + } + } + + { /* read data */ + const void *src; + if ((src = share->read_cache(pos + queue_row_t::header_size(), + table->s->reclength, false)) + != NULL) { + memcpy(buf, src, table->s->reclength); + } else { + if (share->read_direct(buf, pos + queue_row_t::header_size(), + table->s->reclength) + != table->s->reclength) { + err = HA_ERR_GENERIC; // ???? + goto EXIT; + } + } + } + + err = 0; + + EXIT: + share->unlock(); + return err; +} + +void ha_queue::position(const uchar *record) +{ +} + +int ha_queue::rnd_pos(uchar * buf, uchar *pos) +{ + return HA_ERR_WRONG_COMMAND; +} + +int ha_queue::info(uint flag) +{ + // records = share->rows.size(); + // deleted = 0; + + return 0; +} + +THR_LOCK_DATA **ha_queue::store_lock(THD *thd, + THR_LOCK_DATA **to, + enum thr_lock_type lock_type) +{ + *ha_data(thd) = reinterpret_cast(1); // so that close_conn gets called + + if (lock_type != TL_IGNORE && lock.type == TL_UNLOCK) { + lock.type=lock_type; + } + + *to++= &lock; + return to; +} + +int ha_queue::create(const char *name, TABLE *table_arg, + HA_CREATE_INFO *create_info) +{ + char filename[FN_REFLEN]; + int fd; + + fn_format(filename, name, "", Q4M, MY_REPLACE_EXT | MY_UNPACK_FILENAME); + if ((fd = ::open(filename, O_WRONLY | O_CREAT | O_TRUNC | O_EXLOCK, 0660)) + == -1) { + return HA_ERR_GENERIC; + } + queue_file_header_t header; + if (write(fd, &header, sizeof(header)) != sizeof(header)) { + goto ERROR; + } + if (lseek(fd, EXPAND_BY - 1, SEEK_SET) == -1 + || write(fd, "", 1) != 1) { + goto ERROR; + } + ::close(fd); + return 0; + + ERROR: + ::close(fd); + unlink(filename); + return HA_ERR_GENERIC; +} + +int ha_queue::write_row(uchar *buf) +{ + unsigned link_to; + int ret = 0; + + share->lock(); + share->write_begin(); + share->write_append(buf, table->s->reclength); + if (share->write_commit() != 0) { + ret = HA_ERR_GENERIC; // ???? + } + share->unlock(); + + if (ret == 0) { + share->wake_listener(); + } + return ret; +} + +int ha_queue::update_row(const uchar *old_data __attribute__((unused)), + uchar *new_data) +{ + int ret = 0; + share->lock(); + + pthread_t owner = share->find_owner(pos); + if (owner != 0 && owner != pthread_self()) { + share->unlock(); + return HA_ERR_RECORD_DELETED; + } + + /* write code */ + if (share->write_file(new_data, pos + queue_row_t::header_size(), + table->s->reclength) + != 0) { + ret = HA_ERR_GENERIC; // ???? + } + + share->unlock(); + return ret; +} + +int ha_queue::delete_row(const uchar *buf __attribute__((unused))) +{ + share->lock(); + + pthread_t owner = share->find_owner(pos); + if (owner != 0 && owner != pthread_self()) { + share->unlock(); + return HA_ERR_RECORD_DELETED; + } + share->erase_row(pos); + + share->unlock(); + return 0; +} + +struct st_mysql_storage_engine queue_storage_engine = { + MYSQL_HANDLERTON_INTERFACE_VERSION +}; + +mysql_declare_plugin(queue) +{ + MYSQL_STORAGE_ENGINE_PLUGIN, + &queue_storage_engine, + "QUEUE", + "Kazuho Oku at Cybozu Labs, Inc.", + "Queue storage engine for MySQL", + PLUGIN_LICENSE_GPL, + init, + deinit, + 0x0001, + NULL, /* status variables */ + NULL, /* system variables */ + NULL /* config options */ +} +mysql_declare_plugin_end; + +struct queue_wait_t { + queue_share_t *share; + time_t return_at; + queue_wait_t(queue_share_t *s, time_t r) + : share(s), return_at(r) + {} +}; + +my_bool queue_wait_init(UDF_INIT *initid, UDF_ARGS *args, char *message) +{ + queue_share_t *share; + time_t return_at; + + if (args->arg_count < 1) { + strcpy(message, "queue_wait(): missing table name"); + return 1; + } + if (args->arg_type[0] != STRING_RESULT + || (share = get_share_check(args->args[0])) == NULL) { + strcpy(message, "queue_wait(): table not found"); + return 1; + } + if (args->arg_count >= 2) { + if (args->arg_type[1] != INT_RESULT) { + share->release(); + strcpy(message, "queue_wait(): timeout not an integer"); + return 1; + } + return_at = time(NULL) + *reinterpret_cast(args->args[1]); + } else { + return_at = time(NULL) + 60; + } + + initid->maybe_null = 0; + initid->ptr = reinterpret_cast(new queue_wait_t(share, return_at)); + + return 0; +} + +void queue_wait_deinit(UDF_INIT *initid __attribute__((unused))) +{ + queue_wait_t *info = reinterpret_cast(initid->ptr); + + delete info; +} + +long long queue_wait(UDF_INIT *initid, UDF_ARGS *args __attribute__((unused)), + char *is_null, char *error) +{ + erase_owned(); + + queue_wait_t *info = reinterpret_cast(initid->ptr); + int ret = 0; + + info->share->lock(); + do { + if (info->share->assign_owner(pthread_self()) != 0) { + ret = 1; + break; + } + } while (info->share->wait(info->return_at) == 0); + pthread_setspecific(share_key, info->share); + info->share->unlock(); + + *is_null = 0; + return ret; +} + +my_bool queue_end_init(UDF_INIT *initid, + UDF_ARGS *args __attribute__((unused)), + char *message __attribute__((unused))) +{ + initid->maybe_null = 0; + return 0; +} + +void queue_end_deinit(UDF_INIT *initid __attribute__((unused))) +{ +} + +long long queue_end(UDF_INIT *initid __attribute__((unused)), + UDF_ARGS *args __attribute__((unused)), + char *is_null, char *error __attribute__((unused))) +{ + erase_owned(); + + *is_null = 0; + return 1; +} + +my_bool queue_abort_init(UDF_INIT *initid, + UDF_ARGS *args __attribute__((unused)), + char *message) +{ + initid->maybe_null = 0; + if ((initid->ptr = reinterpret_cast(pthread_getspecific(share_key))) + == NULL) { + strcpy(message, "queue_abort(): not in queue access mode"); + return 1; + } + return 0; +} + +void queue_abort_deinit(UDF_INIT *initid) +{ + queue_share_t *share = reinterpret_cast(initid->ptr); + share->release(); + pthread_setspecific(share_key, NULL); +} + +long long queue_abort(UDF_INIT *initid, UDF_ARGS *args __attribute__((unused)), + char *is_null, char *error) +{ + queue_share_t *share = reinterpret_cast(initid->ptr); + + if (share->reset_owner(pthread_self()) != 0) { + share->wake_listener(); + } + + *is_null = 0; + return 1; +} diff --git a/src/ha_queue.h b/src/ha_queue.h new file mode 100644 index 0000000..43673f7 --- /dev/null +++ b/src/ha_queue.h @@ -0,0 +1,192 @@ +/* + * Copyright (C) 2007 Cybozu Labs, Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#ifndef HA_QUEUE_H +#define HA_QUEUE_H + +class queue_share_t; + +class queue_row_t { + unsigned _size; /* lower 2 bits used for flags, removed, and reserved */ + uchar _bytes[1]; +public: + unsigned size() const { + return _size & 0xfffffffc; + } + bool is_removed() const { + return (_size & 1) != 0; + } + void set_is_removed() { + _size |= 1; + } + uchar *bytes() { return _bytes; } + void init_header(unsigned size) { + assert((size & 3) == 0); + _size = size; + } + static size_t header_size() { + return offsetof(queue_row_t, _bytes[0]); + } +private: + queue_row_t(); + queue_row_t(const queue_row_t&); + ~queue_row_t(); + queue_row_t& operator=(const queue_row_t&); +}; + +class queue_file_header_t { +public: + static const unsigned MAGIC = 0x6d393031; +private: + unsigned _magic; + unsigned _padding1; + off_t _eod; + unsigned _padding2[(4096 - sizeof(unsigned) * 2 - sizeof(off_t)) / sizeof(unsigned)]; +public: + queue_file_header_t(); + unsigned magic() const { return _magic; } + off_t eod() const { return _eod; } + int set_eod(int fd, off_t e); +}; + +typedef std::list > queue_rows_owned_t; + +class queue_share_t { + uint use_count; + char *table_name; + uint table_name_length; + + pthread_mutex_t mutex; + THR_LOCK store_lock; + + int fd; + off_t first_row; + queue_file_header_t _header; + + struct { + off_t off; + char buf[1024]; // should be smaller than queue_file_header_t for using off==0 for invalidation + } cache; + + std::vector write_buf; + + queue_rows_owned_t rows_owned; + + pthread_cond_t queue_cond; + int num_readers; + +public: + static uchar *get_share_key(queue_share_t *share, size_t *length, + my_bool not_used); + static queue_share_t *get_share(const char* table_name); + void release(); + void lock() { pthread_mutex_lock(&mutex); } + void unlock() { pthread_mutex_unlock(&mutex); } + void lock_reader() { lock(); ++num_readers; unlock(); } + void unlock_reader() { lock(); --num_readers; unlock(); } + void wake_listener() { pthread_cond_signal(&queue_cond); } + int wait(time_t t) { + timespec ts = { t, 0 }; + return pthread_cond_timedwait(&queue_cond, &mutex, &ts); + } + THR_LOCK *get_store_lock() { return &store_lock; } + const queue_file_header_t *header() const { return &_header; } + off_t reset_owner(pthread_t owner); + /* functions below requires lock */ + ssize_t read_direct(void *data, off_t off, size_t size); + void *read_cache(off_t off, size_t size, bool use_syscall); + int write_file(const void *data, off_t off, size_t size); + off_t begin() { return first_row; } + off_t end() { return header()->eod(); } + int next(off_t *off); + off_t get_owned_row(pthread_t owner); + void write_begin(); + void write_append(const void* data, size_t size); + int write_commit(); + int erase_row(off_t row); + pthread_t find_owner(off_t off); + off_t assign_owner(pthread_t owner); +private: + queue_share_t(); + ~queue_share_t(); + queue_share_t(const queue_share_t&); + queue_share_t& operator=(const queue_share_t&); +}; + +class ha_queue: public handler +{ + THR_LOCK_DATA lock; + queue_share_t *share; + + off_t pos; + + public: + ha_queue(handlerton *hton, TABLE_SHARE *table_arg); + ~ha_queue() {} + + const char *table_type() const { + return "QUEUE"; + } + const char *index_type(uint) { + return "NONE"; + } + const char **bas_ext() const; + ulonglong table_flags() const { + return 0; + } + + ulong index_flags(uint, uint, bool) const { + return 0; + } + + int open(const char *name, int mode, uint test_if_locked); + int close(); + int rnd_init(bool scan); + int rnd_end(); + int rnd_next(uchar *buf); + int rnd_pos(uchar *buf, uchar *pos); + void position(const uchar *record); + + int info(uint); + int create(const char *name, TABLE *form, HA_CREATE_INFO *create_info); + + THR_LOCK_DATA **store_lock(THD *thd, THR_LOCK_DATA **to, + enum thr_lock_type lock_type); ///< required + + int write_row(uchar *buf); + int update_row(const uchar *old_data, uchar *new_data); + int delete_row(const uchar *buf); +}; + +#undef queue_end + +extern "C" { + my_bool queue_wait_init(UDF_INIT *initid, UDF_ARGS *args, char *message); + void queue_wait_deinit(UDF_INIT *initid); + long long queue_wait(UDF_INIT *initid, UDF_ARGS *args, char *is_null, + char *error); + my_bool queue_end_init(UDF_INIT *initid, UDF_ARGS *args, char *message); + void queue_end_deinit(UDF_INIT *initid); + long long queue_end(UDF_INIT *initid, UDF_ARGS *args, char *is_null, + char *error); + my_bool queue_abort_init(UDF_INIT *initid, UDF_ARGS *args, char *message); + void queue_abort_deinit(UDF_INIT *initid); + long long queue_abort(UDF_INIT *initid, UDF_ARGS *args, char *is_null, + char *error); +}; + +#endif