Merge pull request gitleaks#48 from zricethezav/develop

Develop

Author: zricethezav

CHANGELOG.md

@@ -0,0 +1,26 @@
+CHANGELOG
+=========
+
+0.2.0
+-----
+Version 0.2.0 of Gitleaks is the first version update since this got relatively popular. Based on the issues raised it seems that folks want better support for integration into their pipelines. I hear ya. This is what this update tries to provide. So... what are the changes?
+
+* Additionally regex checking
+* $HOME/.gitleaks/ directory for clones and reports
+* Clone into temp dir option
+* Persistent repos for Orgs and Users (no more re-cloning)
+* Pagination for Org/User list... no more partial repo lists
+* Since commit option
+* Updated README
+* Multi-staged Docker build
+* Travis CI
+
+
+0.1.0
+-----
+
+Version 0.1.0 of Gitleaks demonstrates:
+
+* full git history search
+* regex/entropy checks
+* report generation

Gopkg.lock

@@ -0,0 +1,42 @@
+# Gopkg.toml example
+#
+# Refer to https://golang.github.io/dep/docs/Gopkg.toml.html
+# for detailed Gopkg.toml documentation.
+#
+# required = ["github.com/user/thing/cmd/thing"]
+# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"]
+#
+# [[constraint]]
+#   name = "github.com/user/project"
+#   version = "1.0.0"
+#
+# [[constraint]]
+#   name = "github.com/user/project2"
+#   branch = "dev"
+#   source = "github.com/myfork/project2"
+#
+# [[override]]
+#   name = "github.com/x/y"
+#   version = "2.4.0"
+#
+# [prune]
+#   non-go = false
+#   go-tests = true
+#   unused-packages = true
+
+
+[[constraint]]
+  name = "github.com/google/go-github"
+  version = "15.0.0"
+
+[[constraint]]
+  branch = "master"
+  name = "github.com/mitchellh/go-homedir"
+
+[[constraint]]
+  branch = "master"
+  name = "golang.org/x/oauth2"
+
+[prune]
+  go-tests = true
+  unused-packages = true

Gopkg.toml

@@ -5,11 +5,6 @@
 
 ## Check git repos for secrets and keys
 
-### Features
-
-* Search all commits on all branches in topological order
-* Regex/Entropy checks
-
 #### Installing
 
 ```bash
@@ -24,34 +19,80 @@ go get -u github.com/zricethezav/gitleaks
 ./gitleaks {git url}
 ```
 
-This example will clone the target `{git url}` and run a diff on all commits. A report will be outputted to `{repo_name}_leaks.json`
-Gitleaks scans all lines of all commits and checks if there are any regular expression matches. The regexs are defined in `main.go`. Work largely based on  [https://people.eecs.berkeley.edu/~rohanpadhye/files/key_leaks-msr15.pdf](https://people.eecs.berkeley.edu/~rohanpadhye/files/key_leaks-msr15.pdf) and regexes from https://github.com/dxa4481/truffleHog and https://github.com/anshumanbh/git-all-secrets.
-
-##### gitLeaks User
-```bash
-./gitleaks -u {user git url}
+Gitleaks will clone the target `<git url>` to `$HOME/.gitleaks/clones/<repo name>` and run a regex check against all diffs of all commits on all remotes in topological order. If any leaks are found gitleaks will output the leak in json, Ex:
 ```
-##### gitLeaks Org
+{
+   "line": "-const AWS_KEY = \"AKIALALEMEL33243OLIAE\"",
+   "commit": "eaeffdc65b4c73ccb67e75d96bd8743be2c85973",
+   "string": "AKIALALEMEL33243OLIA",
+   "reason": "AWS",
+   "commitMsg": "remove fake key",
+   "time": "2018-02-04 19:43:28 -0600",
+   "author": "Zachary Rice",
+   "file": "main.go",
+   "repoURL": "https://github.com/zricethezav/gronit"
+}
+``` 
+Gitleaks will not re-clone repos unless the temporary flag is set (see Options section), instead gitleaks will `fetch` all new changes before the scan. This works for users and organization repos as well. Regex's for the scan are defined in `main.go`, feel free to open a PR and contribute if you have additional regex you want included. Work largely based on  [https://people.eecs.berkeley.edu/~rohanpadhye/files/key_leaks-msr15.pdf](https://people.eecs.berkeley.edu/~rohanpadhye/files/key_leaks-msr15.pdf) and regexes from https://github.com/dxa4481/truffleHog and https://github.com/anshumanbh/git-all-secrets.
+
+#### Example with Report
 ```bash
-./gitleaks -o {org git url}
+gitleaks --json https://github.com/zricethezav/gronit
+```
+This will run gitleaks on one of my projects, gronit and create the following structure in `$HOME/.gitleaks`:
 ```
+.
+├── clones
+│   └── zricethezav
+│       └── gronit
+│           ├── README.md
+│           ├── main.go
+│           ├── options.go
+│           ├── server.go
+│           └── utils.go
+└── report
+    └── zricethezav
+        └── gronit_leaks.json
+```
+The clones directory contains the repo owner (me) and any repos gitleaks has scanned. Next time we run gitleaks on gronit again we will `fetch` gronit rather than `clone`. Reports are written out to `$HOME/.gitleaks/report/<owner>/<repo>_leaks.json`
 
-#### Help
+#### Options
 ```
 usage: gitleaks [options] <url>
 
 Options:
- -c                     Concurrency factor (default is 10)
- -u --user              Git user url
- -r --repo              Git repo url
- -o --org               Git organization url
- -s --since             Scan until this commit (SHA)
- -b --b64Entropy        Base64 entropy cutoff (default is 70)
- -x --hexEntropy        Hex entropy cutoff (default is 40)
- -e --entropy           Enable entropy
- --strict               Enables stopwords
- -h --help              Display this message
+ -c --concurrency 	Upper bound on concurrent diffs
+ -u --user 		    Git user url
+ -r --repo 		    Git repo url
+ -o --org 		    Git organization url
+ -s --since 		Commit to stop at
+ -b --b64Entropy 	Base64 entropy cutoff (default is 70)
+ -x --hexEntropy  	Hex entropy cutoff (default is 40)
+ -e --entropy		Enable entropy		
+ -j --json 		    Output gitleaks report
+ --token    		Github API token
+ --strict 		    Enables stopwords
+ -h --help 		    Display this message
+
 ```
+
+##### Options Explained
+
+| Option | Explanation |
+| ------------- | ------------- |
+| -c --concurrency | Set the limit on the number of concurrent diffs. If unbounded, your system would throw a `too many open files` error. Tweak `ulimit` for quicker scans at your own risk. Ex: `gitleaks -c 100 <repo_url>` |
+| -u --user | Target git user. Reports and clones are dumped to `$HOME/.gitleaks/clones/<user>/<user_repos>` and `$HOME/.gitleaks/reports/<user>/<gitleaks_reports>`. Ex: `gitleaks -u <user_git_url>`.
+| -o --org | Target git organization. Reports and clones are dumped to `$HOME/.gitleaks/clones/<org>/<org_repos>` and `$HOME/.gitleaks/reports/<org>/<gitleaks_reports>`. Ex: `gitleaks -o <org_git_url>`
+| -r --repo | Default behavior is to have gitleaks target a specific repo, so this option is unecessary, but... Target git repo. Reports and clones are dumped to `$HOME/.gitleaks/clones/<owner>/<repos>` and `$HOME/.gitleaks/reports/<owner>/<gitleaks_reports>`
+| -s --since  | Since argument accepts a commit hash and will scan the repo history up to and including this hash. Ex: `gitleaks -s <HASH> <repo_url>`
+| -b --b64Entropy | Entropy cutoff for base 64 characters. Ex: `gitleaks -e -b 70 <repo_url>` |
+| -x --hexEntropy | Entropy cutoff for hex characters. Ex: `gitleaks -e -x 70 <repo_url>` |
+| -e --entroy | Enable entropy checks. Ex: `gitleaks -e <repo_url>` |
+| -j --json | Enable report generation. Ex: `gitleaks --json <repo_url>` | 
+| -t --temporary | Cloned repos will be cloned into a temp directory and removed after gitleaks exits. Ex: `gitleaks -t <repo_url>` |
+| --token | NOTE: you should use env var `GITHUB_TOKEN` instead of this flag. Github API token needed for scanning private repos and pagination on repo fetching from github's api. |
+| -- strict | Enable stopwords. Ex: `gitleaks --strict <repo_url>` |
+
 NOTE: your mileage may vary so if you aren't getting the results you expected try updating the regexes to fit your needs or try tweaking the entropy cutoffs and stopwords. Entropy cutoff for base64 alphabets seemed to give good results around 70 and hex alphabets seemed to give good results around 40. Entropy is calculated using [Shannon entropy](http://www.bearcave.com/misl/misl_tech/wavelets/compression/shannon.html).
 
 
@@ -69,4 +110,3 @@ docker build -t gitleaks .
 docker run --rm --name=gitleaks gitleaks https://github.com/zricethezav/gitleaks
 ```
 
-

README.md

@@ -1,32 +1,50 @@
 package main
 
 import (
+	_ "fmt"
 	"math"
 	"strings"
 )
 
+// TODO LOCAL REPO!!!!
+
 // checks Regex and if enabled, entropy and stopwords
-func doChecks(diff string, commit string) []LeakElem {
-	var match string
-	var leaks []LeakElem
-	var leak LeakElem
+func doChecks(diff string, commit Commit, opts *Options, repo RepoDesc) []LeakElem {
+	var (
+		match string
+		leaks []LeakElem
+		leak  LeakElem
+	)
+
 	lines := strings.Split(diff, "\n")
+	file := "unable to determine file"
 	for _, line := range lines {
+		if strings.Contains(line, "diff --git a") {
+			idx := fileDiffRegex.FindStringIndex(line)
+			if len(idx) == 2 {
+				file = line[idx[1]:]
+			}
+		}
+
 		for leakType, re := range regexes {
 			match = re.FindString(line)
 			if len(match) == 0 ||
 				(opts.Strict && containsStopWords(line)) ||
-				(opts.Entropy && !checkShannonEntropy(line)) {
+				(opts.Entropy && !checkShannonEntropy(line, opts)) {
 				continue
 			}
 
 			leak = LeakElem{
 				Line:     line,
-				Commit:   commit,
+				Commit:   commit.Hash,
 				Offender: match,
 				Reason:   leakType,
+				Msg:      commit.Msg,
+				Time:     commit.Time,
+				Author:   commit.Author,
+				File:     file,
+				RepoURL:  repo.url,
 			}
-
 			leaks = append(leaks, leak)
 		}
 	}
@@ -35,7 +53,7 @@ func doChecks(diff string, commit string) []LeakElem {
 }
 
 // checkShannonEntropy checks entropy of target
-func checkShannonEntropy(target string) bool {
+func checkShannonEntropy(target string, opts *Options) bool {
 	var (
 		sum             float64
 		targetBase64Len int

checks.go

@@ -4,24 +4,25 @@ import (
 	"testing"
 )
 
-func init() {
-	opts = &Options{
+func TestCheckRegex(t *testing.T) {
+	var results []LeakElem
+	opts := &Options{
 		Concurrency:      10,
 		B64EntropyCutoff: 70,
 		HexEntropyCutoff: 40,
 		Entropy:          false,
 	}
-}
-
-func TestCheckRegex(t *testing.T) {
-	var results []LeakElem
+	repo := RepoDesc{
+		url: "someurl",
+	}
+	commit := Commit{}
 	checks := map[string]int{
 		"aws=\"AKIALALEMEL33243OLIAE": 1,
 		"aws\"afewafewafewafewaf\"":   0,
 	}
 
 	for k, v := range checks {
-		results = doChecks(k, "commit")
+		results = doChecks(k, commit, opts, repo)
 		if v != len(results) {
 			t.Errorf("regexCheck failed on string %s", k)
 		}
@@ -30,14 +31,20 @@ func TestCheckRegex(t *testing.T) {
 
 func TestEntropy(t *testing.T) {
 	var enoughEntropy bool
+	opts := &Options{
+		Concurrency:      10,
+		B64EntropyCutoff: 70,
+		HexEntropyCutoff: 40,
+		Entropy:          false,
+	}
 	checks := map[string]bool{
 		"reddit_api_secret = settings./.http}":           false,
 		"heroku_client_secret = simple":                  false,
 		"reddit_api_secret = \"4ok1WFf57-EMswEfAFGewa\"": true,
 		"aws_secret= \"AKIAIMNOJVGFDXXFE4OA\"":           true,
 	}
 	for k, v := range checks {
-		enoughEntropy = checkShannonEntropy(k)
+		enoughEntropy = checkShannonEntropy(k, opts)
 		if v != enoughEntropy {
 			t.Errorf("checkEntropy failed for %s. Expected %t, got %t", k, v, enoughEntropy)
 		}