find_static_candidates.py

#!/usr/bin/env python3

"""Tool to find static libraries that maybe should be shared libraries and shared libraries that maybe should be static libraries.

This tool only looks at the module-info.json for the current target.

Example of "class" types for each of the modules in module-info.json
  "EXECUTABLES": 2307,
  "ETC": 9094,
  "NATIVE_TESTS": 10461,
  "APPS": 2885,
  "JAVA_LIBRARIES": 5205,
  "EXECUTABLES/JAVA_LIBRARIES": 119,
  "FAKE": 553,
  "SHARED_LIBRARIES/STATIC_LIBRARIES": 7591,
  "STATIC_LIBRARIES": 11535,
  "SHARED_LIBRARIES": 10852,
  "HEADER_LIBRARIES": 1897,
  "DYLIB_LIBRARIES": 1262,
  "RLIB_LIBRARIES": 3413,
  "ROBOLECTRIC": 39,
  "PACKAGING": 5,
  "PROC_MACRO_LIBRARIES": 36,
  "RENDERSCRIPT_BITCODE": 17,
  "DYLIB_LIBRARIES/RLIB_LIBRARIES": 8,
  "ETC/FAKE": 1

None of the "SHARED_LIBRARIES/STATIC_LIBRARIES" are double counted in the
modules with one class
RLIB/

All of these classes have shared_libs and/or static_libs
    "EXECUTABLES",
    "SHARED_LIBRARIES",
    "STATIC_LIBRARIES",
    "SHARED_LIBRARIES/STATIC_LIBRARIES", # cc_library
    "HEADER_LIBRARIES",
    "NATIVE_TESTS", # test modules
    "DYLIB_LIBRARIES", # rust
    "RLIB_LIBRARIES", # rust
    "ETC", # rust_bindgen
"""

from collections import defaultdict

import json, os, argparse

ANDROID_PRODUCT_OUT = os.environ.get("ANDROID_PRODUCT_OUT")
# If a shared library is used less than MAX_SHARED_INCLUSIONS times in a target,
# then it will likely save memory by changing it to a static library
# This move will also use less storage
MAX_SHARED_INCLUSIONS = 2
# If a static library is used more than MAX_STATIC_INCLUSIONS times in a target,
# then it will likely save memory by changing it to a shared library
# This move will also likely use less storage
MIN_STATIC_INCLUSIONS = 3


def parse_args():
  parser = argparse.ArgumentParser(
      description=(
          "Parse module-info.jso and display information about static and"
          " shared library dependencies."
      )
  )
  parser.add_argument(
      "--module", dest="module", help="Print the info for the module."
  )
  parser.add_argument(
      "--shared",
      dest="print_shared",
      action=argparse.BooleanOptionalAction,
      help=(
          "Print the list of libraries that are shared_libs for fewer than {}"
          " modules.".format(MAX_SHARED_INCLUSIONS)
      ),
  )
  parser.add_argument(
      "--static",
      dest="print_static",
      action=argparse.BooleanOptionalAction,
      help=(
          "Print the list of libraries that are static_libs for more than {}"
          " modules.".format(MIN_STATIC_INCLUSIONS)
      ),
  )
  parser.add_argument(
      "--recursive",
      dest="recursive",
      action=argparse.BooleanOptionalAction,
      default=True,
      help=(
          "Gather all dependencies of EXECUTABLES recursvily before calculating"
          " the stats. This eliminates duplicates from multiple libraries"
          " including the same dependencies in a single binary."
      ),
  )
  parser.add_argument(
      "--both",
      dest="both",
      action=argparse.BooleanOptionalAction,
      default=False,
      help=(
          "Print a list of libraries that are including libraries as both"
          " static and shared"
      ),
  )
  return parser.parse_args()


class TransitiveHelper:

  def __init__(self):
    # keep a list of already expanded libraries so we don't end up in a cycle
    self.visited = defaultdict(lambda: defaultdict(set))

  # module is an object from the module-info dictionary
  # module_info is the dictionary from module-info.json
  # modify the module's shared_libs and static_libs with all of the transient
  # dependencies required from all of the explicit dependencies
  def flattenDeps(self, module, module_info):
    libs_snapshot = dict(shared_libs = set(module.get("shared_libs",{})), static_libs = set(module.get("static_libs",{})))

    for lib_class in ["shared_libs", "static_libs"]:
      for lib in libs_snapshot[lib_class]:
        if not lib or lib not in module_info or lib_class not in module:
          continue
        if lib in self.visited:
          module[lib_class].update(self.visited[lib][lib_class])
        else:
          res = self.flattenDeps(module_info[lib], module_info)
          module[lib_class].update(res.get(lib_class, {}))
          self.visited[lib][lib_class].update(res.get(lib_class, {}))

    return module

def main():
  module_info = json.load(open(ANDROID_PRODUCT_OUT + "/module-info.json"))

  args = parse_args()

  if args.module:
    if args.module not in module_info:
      print("Module {} does not exist".format(args.module))
      exit(1)

  # turn all of the static_libs and shared_libs lists into sets to make them
  # easier to update
  for _, module in module_info.items():
    module["shared_libs"] = set(module.get("shared_libs", {}))
    module["static_libs"] = set(module.get("static_libs", {}))

  includedStatically = defaultdict(set)
  includedSharedly = defaultdict(set)
  includedBothly = defaultdict(set)
  transitive = TransitiveHelper()
  for name, module in module_info.items():
    if args.recursive:
      # in this recursive mode we only want to see what is included by the executables
      if "EXECUTABLES" not in module["class"]:
        continue
      module = transitive.flattenDeps(module, module_info)
      # filter out fuzzers by their dependency on clang
      if "static_libs" in module:
        if "libclang_rt.fuzzer" in module["static_libs"]:
          continue
    else:
      if "NATIVE_TESTS" in module["class"]:
        # We don't care about how tests are including libraries
        continue

    # count all of the shared and static libs included in this module
    if "shared_libs" in module:
      for lib in module["shared_libs"]:
        includedSharedly[lib].add(name)
    if "static_libs" in module:
      for lib in module["static_libs"]:
        includedStatically[lib].add(name)

    if "shared_libs" in module and  "static_libs" in module:
      intersection = set(module["shared_libs"]).intersection(
          module["static_libs"]
      )
      if intersection:
        includedBothly[name] = intersection

  if args.print_shared:
    print(
        "Shared libraries that are included by fewer than {} modules on a"
        " device:".format(MAX_SHARED_INCLUSIONS)
    )
    for name, libs in includedSharedly.items():
      if len(libs) < MAX_SHARED_INCLUSIONS:
        print("{}: {} included by: {}".format(name, len(libs), libs))

  if args.print_static:
    print(
        "Libraries that are included statically by more than {} modules on a"
        " device:".format(MIN_STATIC_INCLUSIONS)
    )
    for name, libs in includedStatically.items():
      if len(libs) > MIN_STATIC_INCLUSIONS:
        print("{}: {} included by: {}".format(name, len(libs), libs))

  if args.both:
    allIncludedBothly = set()
    for name, libs in includedBothly.items():
      allIncludedBothly.update(libs)

    print(
        "List of libraries used both statically and shared in the same"
        " processes:\n {}\n\n".format("\n".join(sorted(allIncludedBothly)))
    )
    print(
        "List of libraries used both statically and shared in any processes:\n {}".format("\n".join(sorted(includedStatically.keys() & includedSharedly.keys()))))

  if args.module:
    print(json.dumps(module_info[args.module], default=list, indent=2))
    print(
        "{} is included in shared_libs {} times by these modules: {}".format(
            args.module, len(includedSharedly[args.module]),
            includedSharedly[args.module]
        )
    )
    print(
        "{} is included in static_libs {} times by these modules: {}".format(
            args.module, len(includedStatically[args.module]),
            includedStatically[args.module]
        )
    )
    print("Shared libs included by this module that are used in fewer than {} processes:\n{}".format(
        MAX_SHARED_INCLUSIONS, [x for x in module_info[args.module]["shared_libs"] if len(includedSharedly[x]) < MAX_SHARED_INCLUSIONS]))



if __name__ == "__main__":
  main()