Merge pull request #20 from thehappydinoa/adh/certs-v2-update

Update for Censys Certs V2

.gitignore

@@ -1,2 +1,4 @@
 *.pyc
+.venv
+venv
 .env

README.md

@@ -1,6 +1,6 @@
 # Censys subdomain finder
 
-This is a tool to enumerate subdomains using the Certificate Transparency logs stored by [Censys](https://censys.io). It should return any subdomain who has ever been issued a SSL certificate by a public CA.
+This is a tool to enumerate subdomains using the Certificate Transparency logs stored in [Censys Search](https://search.censys.io). It should return any subdomain who has ever been issued a SSL certificate by a public CA.
 
 See it in action:
 
@@ -57,8 +57,8 @@ $ python censys-subdomain-finder.py github.com
 
 ## Setup
 
-1) Register an account (free) on https://censys.io/register
-2) Browse to https://censys.io/account, and set two environment variables with your API ID and API secret:
+1. Register an account (free) on <https://search.censys.io/register>
+2. Browse to <https://search.censys.io/account>, and set two environment variables with your API ID and API secret:
 
     ```shell
     export CENSYS_API_ID=...
@@ -73,13 +73,13 @@ $ python censys-subdomain-finder.py github.com
 
     Then edit the `.env` file and set the values for `CENSYS_API_ID` and `CENSYS_API_SECRET`.
 
-3) Clone the repository:
+3. Clone the repository:
 
     ```shell
     git clone https://github.com/christophetd/censys-subdomain-finder.git
     ```
 
-4) Install the dependencies in a virtualenv:
+4. Install the dependencies in a virtualenv:
 
     ```shell
     cd censys-subdomain-finder
@@ -124,10 +124,9 @@ optional arguments:
                         CENSYS_API_SECRET environment variable (default: None)
 ```
 
-
 ## Compatibility
 
-Should run on Python 2.7 and 3.5.
+Should run on Python 3.7+.
 
 ## Notes
 

censys-subdomain-finder.py

@@ -1,7 +1,11 @@
 #!/usr/bin/env python3
 
-from censys.search import CensysCertificates
-import censys
+from censys.search import CensysCerts
+from censys.common.exceptions import (
+    CensysUnauthorizedException,
+    CensysRateLimitExceededException,
+    CensysException,
+)
 from dotenv import load_dotenv
 import sys
 import cli
@@ -10,95 +14,138 @@
 
 load_dotenv()
 
-NON_COMMERCIAL_API_LIMIT = 1000
+USER_AGENT = f"{CensysCerts.DEFAULT_USER_AGENT} (censys-subdomain-finder; +https://github.com/christophetd/censys-subdomain-finder)"
+
+MAX_PER_PAGE = 100
+COMMUNITY_PAGES = 10
+
 
 # Finds subdomains of a domain using Censys API
 def find_subdomains(domain, api_id, api_secret, limit_results):
     try:
-        censys_certificates = CensysCertificates(api_id=api_id, api_secret=api_secret)
-        certificate_query = 'parsed.names: %s' % domain
+        censys_certificates = CensysCerts(
+            api_id=api_id, api_secret=api_secret, user_agent=USER_AGENT
+        )
+        certificate_query = "names: %s" % domain
+        pages = -1 # unlimited
         if limit_results:
-            certificates_search_results = censys_certificates.search(certificate_query, fields=['parsed.names'], max_records=NON_COMMERCIAL_API_LIMIT)
-        else:
-            certificates_search_results = censys_certificates.search(certificate_query, fields=['parsed.names'])
+            pages = COMMUNITY_PAGES
+        certificates_search_results = censys_certificates.search(
+            certificate_query,
+            per_page=MAX_PER_PAGE,
+            pages=pages
+        )
 
         # Flatten the result, and remove duplicates
         subdomains = []
-        for search_result in certificates_search_results:
-            subdomains.extend(search_result['parsed.names'])
+        for page in certificates_search_results:
+            for search_result in page:
+                subdomains.extend(search_result["names"])
 
         return set(subdomains)
-    except censys.common.exceptions.CensysUnauthorizedException:
-        sys.stderr.write('[-] Your Censys credentials look invalid.\n')
+    except CensysUnauthorizedException:
+        sys.stderr.write("[-] Your Censys credentials look invalid.\n")
         exit(1)
-    except censys.common.exceptions.CensysRateLimitExceededException:
-        sys.stderr.write('[-] Looks like you exceeded your Censys account limits rate. Exiting\n')
+    except CensysRateLimitExceededException:
+        sys.stderr.write(
+            "[-] Looks like you exceeded your Censys account limits rate. Exiting\n"
+        )
         return set(subdomains)
-    except censys.common.exceptions.CensysException as e:
+    except CensysException as e:
         # catch the Censys Base exception, example "only 1000 first results are available"
-        sys.stderr.write('[-] Something bad happened, ' + repr(e))
+        sys.stderr.write("[-] Something bad happened, " + repr(e))
         return set(subdomains)
 
+
 # Filters out uninteresting subdomains
 def filter_subdomains(domain, subdomains):
-	return [ subdomain for subdomain in subdomains if '*' not in subdomain and subdomain.endswith(domain) ]
+    return [
+        subdomain
+        for subdomain in subdomains
+        if "*" not in subdomain and subdomain.endswith(domain) and subdomain != domain
+    ]
+
 
 # Prints the list of found subdomains to stdout
-def print_subdomains(domain, subdomains, time_ellapsed):
+def print_subdomains(domain, subdomains, time_elapsed):
     if len(subdomains) == 0:
-        print('[-] Did not find any subdomain')
+        print("[-] Did not find any subdomain")
         return
 
-    print('[*] Found %d unique subdomain%s of %s in ~%s seconds\n' % (len(subdomains), 's' if len(subdomains) > 1 else '', domain, str(time_ellapsed)))
+    print(
+        "[*] Found %d unique subdomain%s of %s in ~%s seconds\n"
+        % (
+            len(subdomains),
+            "s" if len(subdomains) > 1 else "",
+            domain,
+            str(time_elapsed),
+        )
+    )
     for subdomain in subdomains:
-        print('  - ' + subdomain)
+        print("  - " + subdomain)
+
+    print("")
 
-    print('')
 
 # Saves the list of found subdomains to an output file
 def save_subdomains_to_file(subdomains, output_file):
     if output_file is None or len(subdomains) == 0:
         return
 
     try:
-        with open(output_file, 'w') as f:
+        with open(output_file, "w") as f:
             for subdomain in subdomains:
-                f.write(subdomain + '\n')
+                f.write(subdomain + "\n")
 
-        print('[*] Wrote %d subdomains to %s' % (len(subdomains), os.path.abspath(output_file)))
+        print(
+            "[*] Wrote %d subdomains to %s"
+            % (len(subdomains), os.path.abspath(output_file))
+        )
     except IOError as e:
-        sys.stderr.write('[-] Unable to write to output file %s : %s\n' % (output_file, e))
+        sys.stderr.write(
+            "[-] Unable to write to output file %s : %s\n" % (output_file, e)
+        )
+
 
 def main(domain, output_file, censys_api_id, censys_api_secret, limit_results):
-    print('[*] Searching Censys for subdomains of %s' % domain)
+    print("[*] Searching Censys for subdomains of %s" % domain)
     start_time = time.time()
-    subdomains = find_subdomains(domain, censys_api_id, censys_api_secret, limit_results)
+    subdomains = find_subdomains(
+        domain, censys_api_id, censys_api_secret, limit_results
+    )
     subdomains = filter_subdomains(domain, subdomains)
     end_time = time.time()
-    time_ellapsed = round(end_time - start_time, 1)
-    print_subdomains(domain, subdomains, time_ellapsed)
+    time_elapsed = round(end_time - start_time, 1)
+    print_subdomains(domain, subdomains, time_elapsed)
     save_subdomains_to_file(subdomains, output_file)
 
+
 if __name__ == "__main__":
     args = cli.parser.parse_args()
 
     censys_api_id = None
     censys_api_secret = None
 
-    if 'CENSYS_API_ID' in os.environ and 'CENSYS_API_SECRET' in os.environ:
-        censys_api_id = os.environ['CENSYS_API_ID']
-        censys_api_secret = os.environ['CENSYS_API_SECRET']
+    if "CENSYS_API_ID" in os.environ and "CENSYS_API_SECRET" in os.environ:
+        censys_api_id = os.environ["CENSYS_API_ID"]
+        censys_api_secret = os.environ["CENSYS_API_SECRET"]
 
     if args.censys_api_id and args.censys_api_secret:
         censys_api_id = args.censys_api_id
         censys_api_secret = args.censys_api_secret
 
     limit_results = not args.commercial
     if limit_results:
-        print('[*] Applying non-commerical limits (' + str(NON_COMMERCIAL_API_LIMIT) + ' results at most)')
-
-    if None in [ censys_api_id, censys_api_secret ]:
-        sys.stderr.write('[!] Please set your Censys API ID and secret from your environment (CENSYS_API_ID and CENSYS_API_SECRET) or from the command line.\n')
+        print(
+            f"[*] Applying free plan limits ({MAX_PER_PAGE * COMMUNITY_PAGES} results at most)"
+        )
+    else:
+        print("[*] No limits applied, getting all results")
+
+    if None in [censys_api_id, censys_api_secret]:
+        sys.stderr.write(
+            "[!] Please set your Censys API ID and secret from your environment (CENSYS_API_ID and CENSYS_API_SECRET) or from the command line.\n"
+        )
         exit(1)
 
     main(args.domain, args.output_file, censys_api_id, censys_api_secret, limit_results)

cli.py

@@ -2,33 +2,31 @@
 
 parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
 
-parser.add_argument(
-    'domain',
-    help = 'The domain to scan'
-)
+parser.add_argument("domain", help="The domain to scan")
 
 parser.add_argument(
-    '-o', '--output',
-    help = 'A file to output the list of subdomains to',
-    dest = 'output_file'
+    "-o",
+    "--output",
+    help="A file to output the list of subdomains to",
+    dest="output_file",
 )
 
 parser.add_argument(
-    '--censys-api-id',
-    help = 'Censys API ID. Can also be defined using the CENSYS_API_ID environment variable',
-    dest = 'censys_api_id'
+    "--censys-api-id",
+    help="Censys API ID. Can also be defined using the CENSYS_API_ID environment variable",
+    dest="censys_api_id",
 )
 
 parser.add_argument(
-    '--censys-api-secret',
-    help = 'Censys API secret. Can also be defined using the CENSYS_API_SECRET environment variable',
-    dest = 'censys_api_secret'
+    "--censys-api-secret",
+    help="Censys API secret. Can also be defined using the CENSYS_API_SECRET environment variable",
+    dest="censys_api_secret",
 )
 
 parser.add_argument(
-    '--commercial',
-    help = 'Don\'t limit search results (for commercial accounts)',
-    dest = 'commercial',
-    action='store_true',
-    default=False
+    "--commercial",
+    help="Don't limit search results (for commercial accounts)",
+    dest="commercial",
+    action="store_true",
+    default=False,
 )

requirements.txt

@@ -1,2 +1,2 @@
-censys==2.1.2
+censys==2.2.1
 python-dotenv