Master

dev-support/git-jira-release-audit/.gitignore

@@ -2,3 +2,4 @@
 *.log
 *.svg
 venv
+new_for_*.csv

dev-support/git-jira-release-audit/fallback_actions.csv

@@ -27,8 +27,10 @@ hexsha,action,jira_id
 05f8e94191ef6a63baadf56d6114d7d0317796f2,SKIP,
 0791b878422eadf00b55076338f09bf059f39f0c,SKIP,
 07f9f3d38cf4d0d01044ab28d90a50a1a009f6b8,SKIP,
+0bff1305134b9c3a0bcad21900f5af68a8aedb4a,SKIP,
 10f00547627076d79d77cf58dd2deaece2287084,ADD,HBASE-22330
 10f3b77748a02a2c11635c33964929c0474e890d,SKIP,
+1196e42362312080d3c523c107b5e8fefef9e57e,SKIP,
 1404d5a97331ecc63db53971f5cb7329cb40ce67,ADD,HBASE-15203
 14a869828fe481697d29b2d6e4135e8026039a38,SKIP,
 1546613e76b1013a08ebc179c2c22bfeb44f3a4a,SKIP,
@@ -42,6 +44,7 @@ hexsha,action,jira_id
 1b3557649c9ee682c7f135ca52a0e3cd10cb9219,SKIP,
 1c46250bef9ef9be9c255d61bda69ff7792ed551,SKIP,
 1cb7d0e82ad64f37fbd6de950b74081b0d5eddf3,SKIP,
+1d988afc9d2065a51fe74d0553f0943ef540dfaa,SKIP,
 1eaef185327171b3dd3edb303e08cfe85186e745,SKIP,
 1eb8ac6fe9dd0c15cdb52f66ced4136316c06465,SKIP,
 2068804d7510e8c1f822b5db3cd4585455f6e7e7,SKIP,
@@ -52,6 +55,7 @@ hexsha,action,jira_id
 259d12f7397679c6b0d0a4788e5a37f65fd49f20,SKIP,
 267bce0590c39570ddb935921e34bda35e3aa44c,SKIP,
 278828333c44493ccbaa7db26a788b2756632034,SKIP,
+27c1f2f978142b7bb4135e53c3ae067c5608c9fb,SKIP,
 288794d68ba5bd4d1fd8d5c315cee972019dcb3d,ADD,HBASE-22330
 28f07451a5dddf0ab3988b32b8672654fdbc5b58,SKIP,
 2ba542d74c2d9e78332c8c94289d1295752d8072,SKIP,
@@ -97,7 +101,9 @@ hexsha,action,jira_id
 4eb84651a2b6d02d2074143308cef5d0f4b856a3,SKIP,
 4f5b22bc19cb8d24ced5d42ebd9794cfd83bae85,SKIP,
 54337870eda5649ab7bb81ed01c9dd25d59204f2,SKIP,
+558ee079fd04dfab8e61eca10ee98ab5bac89dfa,SKIP,
 58ab201be341f02829286f036a7401d0806eb999,SKIP,
+58b63e04c4af99a5730efb0a7e553be4d950e6a5,SKIP,
 5a16c15d7f51087a50511a2e0730f547c97a033f,SKIP,
 5b5ff1d8b2cc43f78acaf9bc960be382dc6c34f7,SKIP,
 5fa15dd7488433ea610ff5e92161409d20565690,SKIP,
@@ -170,6 +176,7 @@ b3d55441b8174c704ada4585603f6bcfca298843,SKIP,
 b65231d04dbc565a578ce928e809aa51f5439857,SKIP,
 b6549007b313e8f3aa993d5c1ebd29c84ccb7b7b,SKIP,
 b6d4fc955fe0fc41f5225f1cc2e3e4b92029251c,SKIP,
+b9c676cdc048c52f927cfa906fd18ff412e4ca20,SKIP,
 b9f5c6b065ebd572193c1fdc9d38557320b42fe6,SKIP,
 bcadcef21048e4764f7ae8dec3ce52884f20c02c,SKIP,
 bcdc56ac76e4a26e53faa8301a441e94ee8614d7,SKIP,
@@ -178,9 +185,11 @@ bd4e14db07ea32a45c3ef734e06d195a405da67c,SKIP,
 bd4eba2b53b7af738fd9584511d737c4393d0855,SKIP,
 bef0616ef33306afca3060b96c2cba5f9762035d,SKIP,
 c100fb835a54be6002fe9704349e726f27b15b7a,SKIP,
+c5e0a1397b3c6a14612e4c5b66f995c02de4310b,SKIP,
 c71da858ada94e1b93065f0b7caf3558942bc4da,SKIP,
 c89cfd3406823cf05fa83464c5ddee16bf0d473f,ADD,HBASE-17248
 c89cfd3406823cf05fa83464c5ddee16bf0d473f,ADD,HBASE-17248
+c8c2a875056f27c9af81293a504d75634cbc1fa5,SKIP,
 c97905a962b88a0c68ca8a51c2e507daec81ca6d,SKIP,
 c9f506a2973e0acbd0d2df7b9353c9291f6c94a8,SKIP,
 cbb2c7e00d0c0b3f641250d981b9c87286d31058,ADD,HBASE-23069
@@ -209,6 +218,7 @@ e40fcee6b54712b76d702af6937c3320c60df2b9,SKIP,
 e501fe1a296be8fec0890e7e15414683aa3d933b,SKIP,
 e5349d589c000e395e12340e003aa9e2153afea6,SKIP,
 e5fb8214b2bfd6396539a4e8b6cf5f3cc5e9c06f,REVERT,HBASE-21874
+e869a20123afe326e198d35d110f5c0360ea244f,SKIP,
 e8e45ef8f2fb91a870399636b492d5cee58a4c39,SKIP,
 e92a147e1961366e36a39577816994566e1e21c5,SKIP,
 eacf3cb29641af1a68978d9bd7654f643a3aa3a1,SKIP,

dev-support/git-jira-release-audit/git_jira_release_audit.py

@@ -30,12 +30,16 @@
 import re
 import sqlite3
 import time
+import os
 
 import enlighten
 import git
 import jira
 
 
+LOG = logging.getLogger(os.path.basename(__file__))
+
+
 class _DB:
     """Manages an instance of Sqlite on behalf of the application.
 
@@ -46,6 +50,9 @@ class _DB:
     Attributes:
         conn (:obj:`sqlite3.db2api.Connection`): The underlying connection object.
     """
+
+    SQL_LOG = LOG.getChild("sql")
+
     class Action(enum.Enum):
         """Describes an action to be taken against the database."""
         ADD = 'ADD'
@@ -54,6 +61,7 @@ class Action(enum.Enum):
 
     def __init__(self, db_path, initialize_db, **_kwargs):
         self._conn = sqlite3.connect(db_path)
+        self._conn.set_trace_callback(_DB.log_query)
 
         if initialize_db:
             for table in 'git_commits', 'jira_versions':
@@ -81,6 +89,10 @@ def __enter__(self):
     def __exit__(self, exc_type, exc_val, exc_tb):
         self._conn.close()
 
+    @staticmethod
+    def log_query(query):
+        _DB.SQL_LOG.debug(re.sub(r'\s+', ' ', query).strip())
+
     @property
     def conn(self):
         """:obj:`sqlite3.db2api.Connection`: Underlying database handle."""
@@ -324,7 +336,7 @@ def _set_release_tag(self, branch, tag, shas):
 
     def _resolve_ambiguity(self, commit):
         if commit.hexsha not in self._fallback_actions:
-            logging.warning('Unable to resolve action for %s: %s', commit.hexsha, commit.summary)
+            LOG.warning('Unable to resolve action for %s: %s', commit.hexsha, commit.summary)
             return _DB.Action.SKIP, None
         action, jira_id = self._fallback_actions[commit.hexsha]
         if not jira_id:
@@ -354,7 +366,7 @@ def populate_db_release_branch(self, origin_commit, release_branch):
         global MANAGER
         commits = list(self._repo.iter_commits(
             "%s...%s" % (origin_commit.hexsha, release_branch), reverse=True))
-        logging.info("%s has %d commits since its origin at %s.", release_branch, len(commits),
+        LOG.info("%s has %d commits since its origin at %s.", release_branch, len(commits),
                      origin_commit)
         counter = MANAGER.counter(total=len(commits), desc=release_branch, unit='commit')
         commits_since_release = list()
@@ -404,7 +416,7 @@ def populate_db(self):
         database."""
         global MANAGER
         jira_ids = self._db.unique_jira_ids_from_git()
-        logging.info("retrieving %s jira_ids from the issue tracker", len(jira_ids))
+        LOG.info("retrieving %s jira_ids from the issue tracker", len(jira_ids))
         counter = MANAGER.counter(total=len(jira_ids), desc='fetch from Jira', unit='issue')
         chunk_size = 50
         chunks = [jira_ids[i:i + chunk_size] for i in range(0, len(jira_ids), chunk_size)]
@@ -429,7 +441,7 @@ def populate_db(self):
     def fetch_issues(self, jira_ids):
         """Retrieve the specified jira Ids."""
         global MANAGER
-        logging.info("retrieving %s jira_ids from the issue tracker", len(jira_ids))
+        LOG.info("retrieving %s jira_ids from the issue tracker", len(jira_ids))
         counter = MANAGER.counter(total=len(jira_ids), desc='fetch from Jira', unit='issue')
         chunk_size = 50
         chunks = [jira_ids[i:i + chunk_size] for i in range(0, len(jira_ids), chunk_size)]
@@ -501,17 +513,17 @@ def _write_report(filename, issues):
             writer.writeheader()
             for issue in issues:
                 writer.writerow(issue)
-        logging.info('generated report at %s', filename)
+        LOG.info('generated report at %s', filename)
 
     def report_new_for_release_line(self, release_line):
         """Builds a report of the Jira issues that are new on the target release line, not present
         on any of the associated release branches. (i.e., on branch-2 but not
         branch-{2.0,2.1,...})"""
         matches = [x for x in self._repo_reader.release_line_refs
-                   if x.name == release_line or x.name.endswith('/%s' % release_line)]
+                   if x.name == release_line or x.remote_head == release_line]
         release_line_ref = next(iter(matches), None)
         if not release_line_ref:
-            logging.error('release line %s not found. available options are %s.',
+            LOG.error('release line %s not found. available options are %s.',
                           release_line, [x.name for x in self._repo_reader.release_line_refs])
             return
         cursor = self._db.conn.execute("""
@@ -525,6 +537,31 @@ def report_new_for_release_line(self, release_line):
         filename = 'new_for_%s.csv' % release_line.replace('/', '-')
         Auditor._write_report(filename, issues)
 
+    def report_new_for_release_branch(self, release_branch):
+        """Builds a report of the Jira issues that are new on the target release branch, not present
+        on any of the previous release branches. (i.e., on branch-2.3 but not
+        branch-{2.0,2.1,...})"""
+        matches = [x for x in self._repo_reader.release_branch_refs
+                   if x.name == release_branch or x.remote_head == release_branch]
+        release_branch_ref = next(iter(matches), None)
+        if not release_branch_ref:
+            LOG.error('release branch %s not found. available options are %s.',
+                      release_branch, [x.name for x in self._repo_reader.release_branch_refs])
+            return
+        previous_branches = [x.name for x in self._repo_reader.release_branch_refs
+                             if x.remote_head != release_branch_ref.remote_head]
+        query = (
+            "SELECT distinct jira_id FROM git_commits"
+            " WHERE branch = ?"
+            " EXCEPT SELECT distinct jira_id FROM git_commits"
+            f" WHERE branch IN ({','.join('?' for _ in previous_branches)})"
+        )
+        cursor = self._db.conn.execute(query, tuple([release_branch_ref.name] + previous_branches))
+        jira_ids = [x[0] for x in cursor.fetchall()]
+        issues = self._jira_reader.fetch_issues(jira_ids)
+        filename = 'new_for_%s.csv' % release_branch.replace('/', '-')
+        Auditor._write_report(filename, issues)
+
     @staticmethod
     def _str_to_bool(val):
         if not val:
@@ -548,7 +585,7 @@ def _build_first_pass_parser():
         building_group.add_argument(
             '--db-path',
             help='Path to the database file, or leave unspecified for a transient db.',
-            default=':memory:')
+            default='audit.db')
         building_group.add_argument(
             '--initialize-db',
             help='When true, initialize the database tables. This is destructive to the contents'
@@ -561,6 +598,11 @@ def _build_first_pass_parser():
             help=Auditor.report_new_for_release_line.__doc__,
             type=str,
             default=None)
+        report_group.add_argument(
+            '--report-new-for-release-branch',
+            help=Auditor.report_new_for_release_branch.__doc__,
+            type=str,
+            default=None)
         git_repo_group = parser.add_argument_group('Interactions with the Git repo')
         git_repo_group.add_argument(
             '--git-repo-path',
@@ -580,7 +622,7 @@ def _build_first_pass_parser():
         git_repo_group.add_argument(
             '--development-branch-fix-version',
             help='The Jira fixVersion used to indicate an issue is committed to the development'
-            + ' branch. Default: \'3.0.0\'',
+            + ' branch.',
             default='3.0.0')
         git_repo_group.add_argument(
             '--release-line-regexp',
@@ -612,7 +654,10 @@ def _build_second_pass_parser(repo_reader, parent_parser, git_repo_group):
                 help='The Jira fixVersion used to indicate an issue is committed to the specified '
                 + 'release line branch',
                 required=True)
-        return argparse.ArgumentParser(parents=[parent_parser])
+        return argparse.ArgumentParser(
+            parents=[parent_parser],
+            formatter_class=argparse.ArgumentDefaultsHelpFormatter
+        )
 
 
 MANAGER = None
@@ -621,11 +666,11 @@ def _build_second_pass_parser(repo_reader, parent_parser, git_repo_group):
 def main():
     global MANAGER
 
+    logging.basicConfig(level=logging.INFO)
     first_pass_parser, git_repo_group = Auditor._build_first_pass_parser()
     first_pass_args, extras = first_pass_parser.parse_known_args()
     first_pass_args_dict = vars(first_pass_args)
     with _DB(**first_pass_args_dict) as db:
-        logging.basicConfig(level=logging.INFO)
         repo_reader = _RepoReader(db, **first_pass_args_dict)
         jira_reader = _JiraReader(db, **first_pass_args_dict)
         second_pass_parser = Auditor._build_second_pass_parser(
@@ -641,6 +686,9 @@ def main():
             if second_pass_args.report_new_for_release_line:
                 release_line = second_pass_args.report_new_for_release_line
                 auditor.report_new_for_release_line(release_line)
+            if second_pass_args.report_new_for_release_branch:
+                release_branch = second_pass_args.report_new_for_release_branch
+                auditor.report_new_for_release_branch(release_branch)
 
 
 if __name__ == '__main__':

hbase-common/src/main/java/org/apache/hadoop/hbase/PrivateCellUtil.java

@@ -915,7 +915,7 @@ public static List<Tag> getTags(Cell cell) {
    * Retrieve Cell's first tag, matching the passed in type
    * @param cell The Cell
    * @param type Type of the Tag to retrieve
-   * @return null if there is no tag of the passed in tag type
+   * @return Optional, empty if there is no tag of the passed in tag type
    */
   public static Optional<Tag> getTag(Cell cell, byte type) {
     boolean bufferBacked = cell instanceof ByteBufferExtendedCell;

hbase-common/src/main/java/org/apache/hadoop/hbase/TableName.java

@@ -24,9 +24,12 @@
 import java.util.Set;
 import java.util.concurrent.CopyOnWriteArraySet;
 
+import org.apache.commons.lang3.ArrayUtils;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.yetus.audience.InterfaceAudience;
 
+import org.apache.hbase.thirdparty.com.google.common.base.Preconditions;
+
 /**
  * Immutable POJO class for representing a table name.
  * Which is of the form:
@@ -146,9 +149,7 @@ public static boolean isMetaTableName(final TableName tn) {
       throw new IllegalArgumentException("Name is null or empty");
     }
 
-    int namespaceDelimIndex =
-      org.apache.hbase.thirdparty.com.google.common.primitives.Bytes.lastIndexOf(tableName,
-        (byte) NAMESPACE_DELIM);
+    int namespaceDelimIndex = ArrayUtils.lastIndexOf(tableName, (byte) NAMESPACE_DELIM);
     if (namespaceDelimIndex < 0){
       isLegalTableQualifierName(tableName);
     } else {
@@ -433,33 +434,73 @@ public static TableName valueOf(String namespaceAsString, String qualifierAsStri
 
 
   /**
+   * @param fullName will use the entire byte array
    * @throws IllegalArgumentException if fullName equals old root or old meta. Some code
    *  depends on this. The test is buried in the table creation to save on array comparison
    *  when we're creating a standard table object that will be in the cache.
    */
   public static TableName valueOf(byte[] fullName) throws IllegalArgumentException{
+    return valueOf(fullName, 0, fullName.length);
+  }
+
+  /**
+   * @param fullName byte array to look into
+   * @param offset within said array
+   * @param length within said array
+   * @throws IllegalArgumentException if fullName equals old root or old meta.
+   */
+  public static TableName valueOf(byte[] fullName, int offset, int length)
+      throws IllegalArgumentException {
+    Preconditions.checkArgument(offset >= 0, "offset must be non-negative but was %s", offset);
+    Preconditions.checkArgument(offset < fullName.length, "offset (%s) must be < array length (%s)",
+        offset, fullName.length);
+    Preconditions.checkArgument(length <= fullName.length,
+        "length (%s) must be <= array length (%s)", length, fullName.length);
     for (TableName tn : tableCache) {
-      if (Arrays.equals(tn.getName(), fullName)) {
+      final byte[] tnName = tn.getName();
+      if (Bytes.equals(tnName, 0, tnName.length, fullName, offset, length)) {
         return tn;
       }
     }
 
-    int namespaceDelimIndex =
-      org.apache.hbase.thirdparty.com.google.common.primitives.Bytes.lastIndexOf(fullName,
-        (byte) NAMESPACE_DELIM);
+    int namespaceDelimIndex = ArrayUtils.lastIndexOf(fullName, (byte) NAMESPACE_DELIM,
+        offset + length - 1);
 
-    if (namespaceDelimIndex < 0) {
+    if (namespaceDelimIndex < offset) {
       return createTableNameIfNecessary(
           ByteBuffer.wrap(NamespaceDescriptor.DEFAULT_NAMESPACE_NAME),
-          ByteBuffer.wrap(fullName));
+          ByteBuffer.wrap(fullName, offset, length));
     } else {
       return createTableNameIfNecessary(
-          ByteBuffer.wrap(fullName, 0, namespaceDelimIndex),
-          ByteBuffer.wrap(fullName, namespaceDelimIndex + 1,
-              fullName.length - (namespaceDelimIndex + 1)));
+          ByteBuffer.wrap(fullName, offset, namespaceDelimIndex),
+          ByteBuffer.wrap(fullName, namespaceDelimIndex + 1, length - (namespaceDelimIndex + 1)));
     }
   }
 
+  /**
+   * @param fullname of a table, possibly with a leading namespace and ':' as delimiter.
+   * @throws IllegalArgumentException if fullName equals old root or old meta.
+   */
+  public static TableName valueOf(ByteBuffer fullname) {
+    fullname = fullname.duplicate();
+    fullname.mark();
+    boolean miss = true;
+    while (fullname.hasRemaining() && miss) {
+      miss = ((byte) NAMESPACE_DELIM) != fullname.get();
+    }
+    if (miss) {
+      fullname.reset();
+      return valueOf(null, fullname);
+    } else {
+      ByteBuffer qualifier = fullname.slice();
+      int delimiterIndex = fullname.position() - 1;
+      fullname.reset();
+      // changing variable name for clarity
+      ByteBuffer namespace = fullname.duplicate();
+      namespace.limit(delimiterIndex);
+      return valueOf(namespace, qualifier);
+    }
+  }
 
   /**
    * @throws IllegalArgumentException if fullName equals old root or old meta. Some code