Update the create adapter script to dbt 0.16.1

Make the base dbt version in the script managed by bumpversion

.bumpversion.cfg

@@ -26,6 +26,8 @@ first_value = 1
 
 [bumpversion:file:core/dbt/version.py]
 
+[bumpversion:file:core/scripts/create_adapter_plugins.py]
+
 [bumpversion:file:plugins/postgres/setup.py]
 
 [bumpversion:file:plugins/redshift/setup.py]

CHANGELOG.md

@@ -10,6 +10,7 @@
 - Fix a bug where third party plugins that used the default `list_schemas` and `information_schema_name` macros with database quoting enabled double-quoted the database name in their queries ([#2267](https://github.com/fishtown-analytics/dbt/issues/2267), [#2281](https://github.com/fishtown-analytics/dbt/pull/2281))
 - The BigQuery "partitions" config value can now be used in `dbt_project.yml` ([#2256](https://github.com/fishtown-analytics/dbt/issues/2256), [#2280](https://github.com/fishtown-analytics/dbt/pull/2280))
 - dbt deps once again does not require a profile, but if profile-specific fields are accessed users will get an error ([#2231](https://github.com/fishtown-analytics/dbt/issues/2231), [#2290](https://github.com/fishtown-analytics/dbt/pull/2290))
+- The create_adapter_plugin.py script has been updated to support 0.16.X adapters ([#2145](https://github.com/fishtown-analytics/dbt/issues/2145), [#2294](https://github.com/fishtown-analytics/dbt/pull/2294))
 
 ### Under the hood
 - Pin google libraries to higher minimum values, add more dependencies as explicit ([#2233](https://github.com/fishtown-analytics/dbt/issues/2233), [#2249](https://github.com/fishtown-analytics/dbt/pull/2249))

core/scripts/create_adapter_plugins.py

@@ -1,9 +1,7 @@
 #!/usr/bin/env python
 import argparse
-import os
 import sys
-
-pj = os.path.join
+from pathlib import Path
 
 PROJECT_TEMPLATE = '''
 name: dbt_{adapter}
@@ -12,20 +10,15 @@
 macro-paths: ["macros"]
 '''
 
-NAMESPACE_INIT_TEMPLATE = '''
-__path__ = __import__('pkgutil').extend_path(__path__, __name__)
-'''.lstrip()
-
 
-# TODO: make this not default to fishtown for everything!
 SETUP_PY_TEMPLATE = '''
 #!/usr/bin/env python
 from setuptools import find_packages
 from setuptools import setup
 
 package_name = "dbt-{adapter}"
 package_version = "{version}"
-description = """The {adapter} adpter plugin for dbt (data build tool)"""
+description = """The {adapter} adapter plugin for dbt (data build tool)"""
 
 setup(
     name=package_name,
@@ -38,19 +31,15 @@
     packages=find_packages(),
     package_data={{
         'dbt': [
-            'include/{adapter}/dbt_project.yml',
-            'include/{adapter}/macros/*.sql',
+{package_data}
         ]
     }},
     install_requires=[
-        {dbt_core_str},
-        {dependencies}
+        "{dbt_core_str}",{dependencies}
     ]
 )
 '''.lstrip()
 
-
-
 ADAPTER_INIT_TEMPLATE = '''
 from dbt.adapters.{adapter}.connections import {title_adapter}ConnectionManager
 from dbt.adapters.{adapter}.connections import {title_adapter}Credentials
@@ -67,38 +56,28 @@
 '''.lstrip()
 
 
-
 ADAPTER_CONNECTIONS_TEMPLATE = '''
-from contextlib import contextmanager
+from dataclasses import dataclass
 
 from dbt.adapters.base import Credentials
 from dbt.adapters.{adapter_src} import {connection_cls}
 
 
-{upper_adapter}_CREDENTIALS_CONTRACT = {{
-    'type': 'object',
-    'additionalProperties': False,
-    'properties': {{
-        'database': {{
-            'type': 'string',
-        }},
-        'schema': {{
-            'type': 'string',
-        }},
-    }},
-    'required': ['database', 'schema'],
-}}
-
-
+@dataclass
 class {title_adapter}Credentials(Credentials):
-    SCHEMA = {upper_adapter}_CREDENTIALS_CONTRACT
+    # Add credentials members here, like:
+    # host: str
+    # port: int
+    # username: str
+    # password: str
 
     @property
     def type(self):
         return '{adapter}'
 
     def _connection_keys(self):
-        # return an iterator of keys to pretty-print in 'dbt debug'
+        # return an iterator of keys to pretty-print in 'dbt debug'.
+        # Omit fields like 'password'!
         raise NotImplementedError
 
 
@@ -117,74 +96,76 @@ class {title_adapter}Adapter({adapter_cls}):
 '''.lstrip()
 
 
-INCLUDE_INIT_TEMPLATE = '''
+CATALOG_MACRO_TEMPLATE = """
+{{% macro {adapter}__get_catalog(information_schema, schemas) -%}}
+
+  {{% set msg -%}}
+    get_catalog not implemented for {adapter}
+  {{%- endset %}}
+
+  {{{{ exceptions.raise_compiler_error(msg) }}}}
+{{% endmacro %}}
+"""
+
+
+INCLUDE_INIT_TEXT = """
 import os
 PACKAGE_PATH = os.path.dirname(__file__)
-'''.lstrip()
+""".lstrip()
 
 
 class Builder:
     def __init__(self, args):
         self.args = args
-        self.dest = pj(self.args.root, self.args.adapter)
-        self.dbt_dir = pj(self.dest, 'dbt')
-        self.adapters_path = pj(self.dbt_dir, 'adapters', self.args.adapter)
-        self.include_path = pj(self.dbt_dir, 'include', self.args.adapter)
-        if os.path.exists(self.dest):
+        self.adapter = self.args.adapter
+        self.dest = self.args.root / self.adapter
+        # self.dbt_dir = self.dest / 'dbt'
+        self.dbt_dir = Path('dbt')
+        self.adapters = self.dbt_dir / 'adapters' / self.adapter
+        self.include = self.dbt_dir / 'include' / self.adapter
+        if self.dest.exists():
             raise Exception('path exists')
 
     def go(self):
-        self.build_namespace()
         self.write_setup()
-        self.write_include()
         self.write_adapter()
+        self.write_include()
+
+    def include_paths(self):
+        return [
+            self.include / 'macros' / '*.sql',
+            self.include / 'dbt_project.yml',
+        ]
 
-    def build_namespace(self):
-        """Build out the directory skeleton and python namespace files:
-
-            dbt/
-                __init__.py
-                adapters/
-                    ${adapter_name}
-                    __init__.py
-                include/
-                    ${adapter_name}
-                    __init__.py
-        """
-        os.makedirs(self.adapters_path)
-        os.makedirs(pj(self.include_path, 'macros'))
-        with open(pj(self.dbt_dir, '__init__.py'), 'w') as fp:
-            fp.write(NAMESPACE_INIT_TEMPLATE)
-        with open(pj(self.dbt_dir, 'adapters', '__init__.py'), 'w') as fp:
-            fp.write(NAMESPACE_INIT_TEMPLATE)
-        with open(pj(self.dbt_dir, 'include', '__init__.py'), 'w') as fp:
-            fp.write(NAMESPACE_INIT_TEMPLATE)
+    def dest_path(self, *paths):
+        return self.dest.joinpath(*paths)
 
     def write_setup(self):
-        if self.args.dbt_core_version == self.args.package_version:
-            dbt_core_str = "'dbt-core=={}'.format(package_version)"
-        else:
-            dbt_core_str = 'dbt-core=={}'.format(self.args.dbt_core_version)
+        self.dest.mkdir(parents=True, exist_ok=True)
+
+        dbt_core_str = 'dbt-core=={}'.format(self.args.dbt_core_version)
+
+        # 12-space indent, then single-quoted with a trailing comma. The path
+        # should not be the actual path from the root but from the 'dbt' dir
+        # (because this is in the 'dbt' package)
+        package_data = '\n'.join(
+            "{}'{!s}',".format(12*' ', p.relative_to(self.dbt_dir))
+            for p in self.include_paths()
+        )
+
         setup_py_contents = SETUP_PY_TEMPLATE.format(
-            adapter=self.args.adapter,
+            adapter=self.adapter,
             version=self.args.package_version,
             author_name=self.args.author,
             author_email=self.args.email,
             url=self.args.url,
             dbt_core_str=dbt_core_str,
-            dependencies=self.args.dependency
+            dependencies=self.args.dependency,
+            package_data=package_data,
         )
-        with open(pj(self.dest, 'setup.py'), 'w') as fp:
-            fp.write(setup_py_contents)
-
-    def write_adapter(self):
-        adapter_init_contents = ADAPTER_INIT_TEMPLATE.format(
-            adapter=self.args.adapter,
-            title_adapter=self.args.title_case
-        )
-        with open(pj(self.adapters_path, '__init__.py'), 'w') as fp:
-            fp.write(adapter_init_contents)
+        self.dest_path('setup.py').write_text(setup_py_contents)
 
+    def _make_adapter_kwargs(self):
         if self.args.sql:
             kwargs = {
                 'adapter_src': 'sql',
@@ -198,58 +179,86 @@ def write_adapter(self):
                 'connection_cls': 'BaseConnectionManager',
             }
         kwargs.update({
-            'upper_adapter': self.args.adapter.upper(),
+            'upper_adapter': self.adapter.upper(),
             'title_adapter': self.args.title_case,
-            'adapter': self.args.adapter,
+            'adapter': self.adapter,
         })
 
-        adapter_connections_contents = ADAPTER_CONNECTIONS_TEMPLATE.format(
-            **kwargs
-        )
-        with open(pj(self.adapters_path, 'connections.py'), 'w') as fp:
-            fp.write(adapter_connections_contents)
+        return kwargs
+
+    def write_adapter(self):
+        adapters_dest = self.dest_path(self.adapters)
+        adapters_dest.mkdir(parents=True, exist_ok=True)
+
+        kwargs = self._make_adapter_kwargs()
 
-        adapter_impl_contents = ADAPTER_IMPL_TEMPLATE.format(
-            **kwargs
+        init_text = ADAPTER_INIT_TEMPLATE.format(
+            adapter=self.adapter,
+            title_adapter=self.args.title_case
         )
-        with open(pj(self.adapters_path, 'impl.py'), 'w') as fp:
-            fp.write(adapter_impl_contents)
+        connections_text = ADAPTER_CONNECTIONS_TEMPLATE.format(**kwargs)
+        impl_text = ADAPTER_IMPL_TEMPLATE.format(**kwargs)
+
+        (adapters_dest / '__init__.py').write_text(init_text)
+        (adapters_dest / 'connections.py').write_text(connections_text)
+        (adapters_dest / 'impl.py').write_text(impl_text)
 
     def write_include(self):
-        with open(pj(self.include_path, '__init__.py'), 'w') as fp:
-            fp.write(INCLUDE_INIT_TEMPLATE)
+        include_dest = self.dest_path(self.include)
+        include_dest.mkdir(parents=True, exist_ok=True)
+        macros_dest = include_dest / 'macros'
+        macros_dest.mkdir(exist_ok=True)
+
+        dbt_project_text = PROJECT_TEMPLATE.format(
+            adapter=self.adapter,
+            version=self.args.project_version,
+        )
+        catalog_macro_text = CATALOG_MACRO_TEMPLATE.format(
+            adapter=self.adapter
+        )
+
+        (include_dest / '__init__.py').write_text(INCLUDE_INIT_TEXT)
+        (include_dest / 'dbt_project.yml').write_text(dbt_project_text)
+        # make sure something satisfies the 'include/macros/*.sql' in setup.py
+        (macros_dest / 'catalog.sql').write_text(catalog_macro_text)
 
-        with open(pj(self.include_path, 'dbt_project.yml'), 'w') as fp:
-            fp.write(PROJECT_TEMPLATE.format(adapter=self.args.adapter,
-                                             version=self.args.project_version))
 
 def parse_args(argv=None):
     if argv is None:
         argv = sys.argv[1:]
     parser = argparse.ArgumentParser()
-    parser.add_argument('root')
+    parser.add_argument('root', type=Path)
     parser.add_argument('adapter')
     parser.add_argument('--title-case', '-t', default=None)
     parser.add_argument('--dependency', action='append')
-    parser.add_argument('--dbt-core-version', default='0.13.0')
+    parser.add_argument('--dbt-core-version', default='0.16.1a1')
     parser.add_argument('--email')
     parser.add_argument('--author')
     parser.add_argument('--url')
     parser.add_argument('--sql', action='store_true')
     parser.add_argument('--package-version', default='0.0.1')
     parser.add_argument('--project-version', default='1.0')
+    parser.add_argument(
+        '--no-dependency', action='store_false', dest='set_dependency'
+    )
     parsed = parser.parse_args()
 
     if parsed.title_case is None:
         parsed.title_case = parsed.adapter.title()
 
-    if parsed.dependency:
-        #['a', 'b'] => "'a',\n        'b'"; ['a'] -> "'a',"
-        parsed.dependency = '\n        '.join(
-            "'{}',".format(d) for d in parsed.dependency
-        )
+    if parsed.set_dependency:
+        if parsed.dependency:
+            # ['a', 'b'] => "'a',\n        'b'"; ['a'] -> "'a',"
+
+            prefix = '\n        '
+
+            parsed.dependency = prefix + prefix.join(
+                "'{}',".format(d) for d in parsed.dependency
+            )
+        else:
+            parsed.dependency = prefix + '<INSERT DEPENDENCIES HERE>'
     else:
-        parsed.dependency = '<INSERT DEPENDENCIES HERE>'
+        parsed.dependency = ''
 
     if parsed.email is not None:
         parsed.email = "'{}'".format(parsed.email)
@@ -266,7 +275,6 @@ def parse_args(argv=None):
     return parsed
 
 
-
 def main():
     builder = Builder(parse_args())
     builder.go()