Update

Author: rhergenreder

chisel

@@ -81,6 +81,7 @@ def __init__(self, addr, port=53):
         self.debug = False
         self.ttl = 60 * 5
         self.logging = False
+        self.not_found_handler = None
 
     def addEntry(self, type, domain, value):
         if type not in self.entries:
@@ -135,6 +136,8 @@ def dns_response(self, data):
             reply.add_answer(RR(rname=qname, rtype=getattr(QTYPE, rqt), rclass=1, ttl=self.ttl, rdata=entry))
             if self.logging:
                 print(f"Request: {qt} {qn} -> {entry}")
+        elif self.not_found_handler:
+            self.not_found_handler(request, reply)
 
         if self.debug:
             print("DNS RESPONSE:", reply)

dnsserver.py

@@ -68,59 +68,61 @@ def do_OPTIONS(self):
         self.do_GET()
 
     def do_GET(self):
+        try:
+            if not self.server.is_running:
+                self.send_response(200)
+                self.end_headers()
+                return
 
-        if not self.server.is_running:
-            self.send_response(200)
-            self.end_headers()
-            return
-
-        path = self.server.cleanPath(self.path)
-        route = self.find_route(path)
-        result = route(self)
+            path = self.server.cleanPath(self.path)
+            route = self.find_route(path)
+            result = route(self)
 
-        blacklist_headers = ["transfer-encoding", "content-length", "content-encoding", "allow", "connection"]
-        status_code = 200 if len(result) < 1 else result[0]
-        data        = b"" if len(result) < 2 else result[1]
-        headers     = { } if len(result) < 3 else result[2]
+            blacklist_headers = ["transfer-encoding", "content-length", "content-encoding", "allow", "connection"]
+            status_code = 200 if len(result) < 1 else result[0]
+            data        = b"" if len(result) < 2 else result[1]
+            headers     = { } if len(result) < 3 else result[2]
 
-        if path in self.server.dumpRequests:
-            headers["Access-Control-Allow-Origin"] = "*"
+            if path in self.server.dumpRequests:
+                headers["Access-Control-Allow-Origin"] = "*"
 
-        headers["Content-Length"] = len(data)
+            headers["Content-Length"] = len(data)
 
-        if len(headers) == 0:
-            self.send_response(status_code)
-        else:
-            if path != "/dummy":
-                self.log_request(status_code)
-            self.send_response_only(status_code)
+            if len(headers) == 0:
+                self.send_response(status_code)
+            else:
+                if path != "/dummy":
+                    self.log_request(status_code)
+                self.send_response_only(status_code)
 
-            for key, value in headers.items():
-                if key.lower() not in blacklist_headers:
-                    self.send_header(key, value)
+                for key, value in headers.items():
+                    if key.lower() not in blacklist_headers:
+                        self.send_header(key, value)
 
-            if self.command.upper() == "OPTIONS":
-                self.send_header("Allow", "OPTIONS, GET, HEAD, POST")
+                if self.command.upper() == "OPTIONS":
+                    self.send_header("Allow", "OPTIONS, GET, HEAD, POST")
 
-        self.end_headers()
+            self.end_headers()
 
-        if data and self.command.upper() not in ["HEAD","OPTIONS"]:
-            self.wfile.write(data)
+            if data and self.command.upper() not in ["HEAD","OPTIONS"]:
+                self.wfile.write(data)
 
-        if (path in self.server.dumpRequests or "/" in self.server.dumpRequests) and path != "/dummy":
-            contentLength = self.headers.get('Content-Length')
-            body = None
+            if (path in self.server.dumpRequests or "/" in self.server.dumpRequests) and path != "/dummy":
+                contentLength = self.headers.get('Content-Length')
+                body = None
 
-            if contentLength and int(contentLength) > 0:
-                body = self.rfile.read(int(contentLength))
+                if contentLength and int(contentLength) > 0:
+                    body = self.rfile.read(int(contentLength))
 
-            print("===== Connection from:",self.client_address[0])
-            print("%s %s %s" % (self.command, self.path, self.request_version))
-            print(str(self.headers).strip())
-            if body:
-                print()
-                print(body)
-            print("==========")
+                print("===== Connection from:",self.client_address[0])
+                print("%s %s %s" % (self.command, self.path, self.request_version))
+                print(str(self.headers).strip())
+                if body:
+                    print()
+                    print(body)
+                print("==========")
+        except Exception as e:
+            print("Exception on handling http", str(e))
 
     def log_message(self, format, *args):
         if self.server.logRequests:
@@ -148,9 +150,15 @@ def cleanPath(self, path):
         return path.strip()
 
     def addFile(self, name, data, mimeType=None):
+
+        if hasattr(data, "read"):
+            fd = data
+            data = data.read()
+            fd.close()
+
         if isinstance(data, str):
             data = data.encode("UTF-8")
-
+    
         headers = { 
             "Access-Control-Allow-Origin": "*",
         }
@@ -160,6 +168,27 @@ def addFile(self, name, data, mimeType=None):
         # return 200 - OK and data
         self.addRoute(name, lambda req: (200, data, headers))
 
+    def add_file_path(self, path, name=None):
+        def readfile():
+            with open(path, "rb") as f:
+                return f.read()
+
+        if name is None:
+            name = os.path.basename(path)
+        self.addRoute(name, lambda req: (200, readfile()))
+
+    def load_directory(self, path, recursive=True, exclude_ext=[]):
+        if not os.path.isdir(path):
+            print("Not a directory:", path)
+            return
+
+        for dp, dn, filenames in os.walk(path):
+            for f in filenames:
+                file_path = os.path.join(dp, f)
+                if not exclude_ext or os.path.splitext(file_path)[1] not in exclude_ext:
+                    relative_path = file_path[len(path):]
+                    self.add_file_path(file_path, relative_path)
+
     def dumpRequest(self, name):
         self.dumpRequests.append(self.cleanPath(name))
 

fileserver.py

@@ -44,6 +44,8 @@ def get_indexed_files(response):
         if (url.path and
                 url.path != '.' and
                 url.path != '..' and
+                url.path != './' and
+                url.path != '../' and
                 not url.path.startswith('/') and
                 not url.scheme and
                 not url.netloc):
@@ -171,15 +173,15 @@ def process_tasks(initial_tasks, worker, jobs, args=(), tasks_done=None):
 class DownloadWorker(Worker):
     ''' Download a list of files '''
 
-    def init(self, url, directory, retry, timeout, module=None):
+    def init(self, url, directory, retry, timeout, follow_redirects=False, module=None):
         self.session = requests.Session()
         self.session.verify = False
         self.session.mount(url, requests.adapters.HTTPAdapter(max_retries=retry))
         self.module = module
 
-    def do_task(self, filepath, url, directory, retry, timeout, module=None):
+    def do_task(self, filepath, url, directory, retry, timeout, follow_redirects=False, module=None):
         with closing(self.session.get('%s/%s' % (url, filepath),
-                                      allow_redirects=False,
+                                      allow_redirects=follow_redirects,
                                       stream=True,
                                       timeout=timeout,
                                       headers={"User-Agent": USER_AGENT})) as response:
@@ -202,9 +204,9 @@ def do_task(self, filepath, url, directory, retry, timeout, module=None):
 class RecursiveDownloadWorker(DownloadWorker):
     ''' Download a directory recursively '''
 
-    def do_task(self, filepath, url, directory, retry, timeout):
+    def do_task(self, filepath, url, directory, retry, timeout, follow_redirects=False):
         with closing(self.session.get('%s/%s' % (url, filepath),
-                                      allow_redirects=False,
+                                      allow_redirects=follow_redirects,
                                       stream=True,
                                       timeout=timeout,
                                       headers={"User-Agent": USER_AGENT})) as response:
@@ -237,9 +239,9 @@ def do_task(self, filepath, url, directory, retry, timeout):
 class FindRefsWorker(DownloadWorker):
     ''' Find refs/ '''
 
-    def do_task(self, filepath, url, directory, retry, timeout, module):
+    def do_task(self, filepath, url, directory, retry, timeout, follow_redirects=False, module=None):
         response = self.session.get('%s/%s' % (url, filepath),
-                                    allow_redirects=False,
+                                    allow_redirects=follow_redirects,
                                     timeout=timeout,
                                     headers={"User-Agent": USER_AGENT})
         printf('[-] Fetching %s/%s [%d]\n', url, filepath, response.status_code)
@@ -271,11 +273,11 @@ def do_task(self, filepath, url, directory, retry, timeout, module):
 class FindObjectsWorker(DownloadWorker):
     ''' Find objects '''
 
-    def do_task(self, obj, url, directory, retry, timeout, module):
+    def do_task(self, obj, url, directory, retry, timeout, follow_redirects, module):
         # module = ".git" if not url.endswith("/modules") else ""
         filepath = '%s/objects/%s/%s' % (self.module, obj[:2], obj[2:])
         response = self.session.get('%s/%s' % (url, filepath),
-                                    allow_redirects=False,
+                                    allow_redirects=follow_redirects,
                                     timeout=timeout,
                                     headers={"User-Agent": USER_AGENT})
         printf('[-] Fetching %s/%s [%d]\n', url, filepath, response.status_code)
@@ -295,7 +297,7 @@ def do_task(self, obj, url, directory, retry, timeout, module):
         return get_referenced_sha1(obj_file)
 
 
-def fetch_git(url, directory, jobs, retry, timeout, module=".git"):
+def fetch_git(url, directory, jobs, retry, timeout, follow_redirects, module=".git"):
     ''' Dump a git repository into the output directory '''
 
     assert os.path.isdir(directory), '%s is not a directory' % directory
@@ -320,7 +322,7 @@ def fetch_git(url, directory, jobs, retry, timeout, module=".git"):
 
     # check for /.git/HEAD
     printf('[-] Testing %s/%s/HEAD ', url, module)
-    response = requests.get('%s/%s/HEAD' % (url, module), verify=False, allow_redirects=False, headers={"User-Agent": USER_AGENT})
+    response = requests.get('%s/%s/HEAD' % (url, module), verify=False, allow_redirects=follow_redirects, headers={"User-Agent": USER_AGENT})
     printf('[%d]\n', response.status_code)
 
     if response.status_code != 200:
@@ -332,15 +334,15 @@ def fetch_git(url, directory, jobs, retry, timeout, module=".git"):
 
     # check for directory listing
     printf('[-] Testing %s/%s/ ', url, module)
-    response = requests.get('%s/%s/' % (url, module), verify=False, allow_redirects=False, headers={"User-Agent": USER_AGENT})
+    response = requests.get('%s/%s/' % (url, module), verify=False, allow_redirects=follow_redirects, headers={"User-Agent": USER_AGENT})
     printf('[%d]\n', response.status_code)
 
     if response.status_code == 200 and is_html(response) and 'HEAD' in get_indexed_files(response):
         printf('[-] Fetching .git recursively\n')
         process_tasks(['.git/', '.gitignore'],
                       RecursiveDownloadWorker,
                       jobs,
-                      args=(url, directory, retry, timeout))
+                      args=(url, directory, retry, timeout, follow_redirects))
 
         printf('[-] Running git checkout .\n')
         os.chdir(directory)
@@ -378,7 +380,7 @@ def fetch_git(url, directory, jobs, retry, timeout, module=".git"):
     process_tasks(tasks,
                   DownloadWorker,
                   jobs,
-                  args=(url, directory, retry, timeout, module))
+                  args=(url, directory, retry, timeout, follow_redirects, module))
 
     if module == ".git":
         modules_path = os.path.join(directory, '.gitmodules')
@@ -392,7 +394,7 @@ def fetch_git(url, directory, jobs, retry, timeout, module=".git"):
                     printf("[-] Fetching module: %s\n", module_name)
                     # os.makedirs(os.path.abspath(module_dir))
                     module_url = url + "/.git/modules"
-                    fetch_git(module_url, module_dir, jobs, retry, timeout, module=module_name)
+                    fetch_git(module_url, module_dir, jobs, retry, timeout, follow_redirects, module=module_name)
                     printf("[+] Done iterating module\n")
 
     # find refs
@@ -420,7 +422,7 @@ def fetch_git(url, directory, jobs, retry, timeout, module=".git"):
     process_tasks(tasks,
                   FindRefsWorker,
                   jobs,
-                  args=(url, directory, retry, timeout, module))
+                  args=(url, directory, retry, timeout, follow_redirects, module))
 
     # find packs
     printf('[-] Finding packs\n')
@@ -439,7 +441,7 @@ def fetch_git(url, directory, jobs, retry, timeout, module=".git"):
     process_tasks(tasks,
                   DownloadWorker,
                   jobs,
-                  args=(url, directory, retry, timeout))
+                  args=(url, directory, retry, timeout, follow_redirects))
 
     # find objects
     printf('[-] Finding objects\n')
@@ -477,8 +479,12 @@ def fetch_git(url, directory, jobs, retry, timeout, module=".git"):
     if os.path.exists(index_path):
         index = dulwich.index.Index(index_path)
 
-        for entry in index.iterblobs():
-            objs.add(entry[1].decode())
+        # index.iteritems()
+        for entry in index.iteritems():
+            if isinstance(entry[1], dulwich.index.IndexEntry):
+                objs.add(entry[1].sha.decode())
+            elif hasattr(entry[1], "decode"):
+                objs.add(entry[1].decode())
 
     # use packs to find more objects to fetch, and objects that are packed
     pack_file_dir = os.path.join(directory, module, 'objects', 'pack')
@@ -500,7 +506,7 @@ def fetch_git(url, directory, jobs, retry, timeout, module=".git"):
     process_tasks(objs,
                   FindObjectsWorker,
                   jobs,
-                  args=(url, directory, retry, timeout, module),
+                  args=(url, directory, retry, timeout, follow_redirects, module),
                   tasks_done=packed_objs)
 
     # git checkout
@@ -529,6 +535,9 @@ def fetch_git(url, directory, jobs, retry, timeout, module=".git"):
                         help='number of request attempts before giving up')
     parser.add_argument('-t', '--timeout', type=int, default=3,
                         help='maximum time in seconds before giving up')
+    parser.add_argument('-L', '--follow-redirects', default=False,
+                        dest='follow_redirects', action="store_true",
+                        help='follow redirects')
     args = parser.parse_args()
 
     # jobs
@@ -576,7 +585,7 @@ def fetch_git(url, directory, jobs, retry, timeout, module=".git"):
 
     # fetch everything
     path = os.path.realpath(args.directory)
-    code = fetch_git(args.url, args.directory, args.jobs, args.retry, args.timeout)
+    code = fetch_git(args.url, args.directory, args.jobs, args.retry, args.timeout, args.follow_redirects)
     if not os.listdir(path):
         os.rmdir(path)