Skip to content

Commit 1101e39

Browse files
EsonhughEsonhugh
authored
Merge pull request #2 from Esonhugh/spider-mutlithreading
Spider mutlithreading
2 parents b098572 + 2da4ceb commit 1101e39

File tree

11 files changed

+296
-37
lines changed

11 files changed

+296
-37
lines changed

‎Makefile

+1-1
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,6 @@ build:
1111

1212
build-static:
1313
GOOS=linux GOARCH=amd64 go build -ldflags="-s -w" -o $(BUILD_DIR)/$(MAIN_PROGRAM_NAME)-linux-static main.go
14-
14+
upx $(BUILD_DIR)/$(MAIN_PROGRAM_NAME)-linux-static
1515
clean:
1616
rm -rf $(BUILD_DIR)

‎cmd/all/all.go

+31-8
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,14 @@
11
package all
22

33
import (
4+
"net"
45
"os"
56

67
command "github.com/esonhugh/k8spider/cmd"
78
"github.com/esonhugh/k8spider/define"
89
"github.com/esonhugh/k8spider/pkg"
10+
"github.com/esonhugh/k8spider/pkg/mutli"
11+
"github.com/esonhugh/k8spider/pkg/scanner"
912
"github.com/miekg/dns"
1013
log "github.com/sirupsen/logrus"
1114
"github.com/spf13/cobra"
@@ -24,23 +27,43 @@ var AllCmd = &cobra.Command{
2427
log.Warn("cidr is required")
2528
return
2629
}
30+
records, err := scanner.DumpAXFR(dns.Fqdn(command.Opts.Zone), "ns.dns."+command.Opts.Zone+":53")
31+
if err == nil {
32+
printResult(records)
33+
}
34+
log.Errorf("Transfer failed: %v", err)
2735
ipNets, err := pkg.ParseStringToIPNet(command.Opts.Cidr)
2836
if err != nil {
2937
log.Warnf("ParseStringToIPNet failed: %v", err)
3038
return
3139
}
32-
var records define.Records = pkg.ScanSubnet(ipNets)
33-
if records == nil || len(records) == 0 {
34-
log.Warnf("ScanSubnet Found Nothing: %v", err)
35-
return
40+
if command.Opts.BatchMode {
41+
RunBatch(ipNets)
42+
} else {
43+
Run(ipNets)
3644
}
37-
records = pkg.ScanSvcForPorts(records)
38-
printResult(records)
39-
records = pkg.DumpAXFR(dns.Fqdn(command.Opts.Zone), "ns.dns."+command.Opts.Zone+":53")
40-
printResult(records)
4145
},
4246
}
4347

48+
func Run(net *net.IPNet) {
49+
var records define.Records = scanner.ScanSubnet(net)
50+
if records == nil || len(records) == 0 {
51+
log.Warnf("ScanSubnet Found Nothing")
52+
return
53+
}
54+
records = scanner.ScanSvcForPorts(records)
55+
printResult(records)
56+
}
57+
58+
func RunBatch(net *net.IPNet) {
59+
scan := mutli.ScanAll(net)
60+
var finalRecord []define.Record
61+
for r := range scan {
62+
finalRecord = append(finalRecord, r...)
63+
}
64+
printResult(finalRecord)
65+
}
66+
4467
func printResult(records define.Records) {
4568
if command.Opts.OutputFile != "" {
4669
f, err := os.OpenFile(command.Opts.OutputFile, os.O_CREATE|os.O_WRONLY, 0644)

‎cmd/axfr/axfr.go

+7-2
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ import (
66

77
command "github.com/esonhugh/k8spider/cmd"
88
"github.com/esonhugh/k8spider/define"
9-
"github.com/esonhugh/k8spider/pkg"
9+
"github.com/esonhugh/k8spider/pkg/scanner"
1010
"github.com/miekg/dns"
1111
log "github.com/sirupsen/logrus"
1212
"github.com/spf13/cobra"
@@ -35,7 +35,12 @@ var AxfrCmd = &cobra.Command{
3535
}
3636

3737
log.Debugf("same command: dig axfr %v @%v", zone, dnsServer)
38-
var records define.Records = pkg.DumpAXFR(zone, dnsServer)
38+
var records define.Records
39+
records, err := scanner.DumpAXFR(zone, dnsServer)
40+
if err != nil {
41+
log.Errorf("Transfer failed: %v", err)
42+
return
43+
}
3944
if command.Opts.OutputFile != "" {
4045
f, err := os.OpenFile(command.Opts.OutputFile, os.O_CREATE|os.O_WRONLY, 0644)
4146
if err != nil {

‎cmd/root.go

+6-3
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,9 @@ var Opts = struct {
1717
SvcDomains []string
1818
Zone string
1919
OutputFile string
20-
LogLevel string
20+
Verbose string
21+
22+
BatchMode bool
2123
}{}
2224

2325
func init() {
@@ -26,15 +28,16 @@ func init() {
2628
RootCmd.PersistentFlags().StringSliceVarP(&Opts.SvcDomains, "svc-domains", "s", []string{}, "service domains, like: kubernetes.default,etcd.default don't add zone like svc.cluster.local")
2729
RootCmd.PersistentFlags().StringVarP(&Opts.Zone, "zone", "z", "cluster.local", "zone")
2830
RootCmd.PersistentFlags().StringVarP(&Opts.OutputFile, "output-file", "o", "", "output file")
29-
RootCmd.PersistentFlags().StringVarP(&Opts.LogLevel, "log-level", "l", "info", "log level")
31+
RootCmd.PersistentFlags().StringVarP(&Opts.Verbose, "verbose", "v", "info", "log level (debug,info,trace,warn,error,fatal,panic)")
32+
RootCmd.PersistentFlags().BoolVarP(&Opts.BatchMode, "batch-mode", "b", false, "batch mode")
3033
}
3134

3235
var RootCmd = &cobra.Command{
3336
Use: "k8spider",
3437
Short: "k8spider is a tool to discover k8s services",
3538
Long: "k8spider is a tool to discover k8s services",
3639
PersistentPreRun: func(cmd *cobra.Command, args []string) {
37-
SetLogLevel(Opts.LogLevel)
40+
SetLogLevel(Opts.Verbose)
3841
if Opts.DnsServer != "" {
3942
pkg.NetResolver = &net.Resolver{
4043
PreferGo: true,

‎cmd/service/service.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ import (
66

77
command "github.com/esonhugh/k8spider/cmd"
88
"github.com/esonhugh/k8spider/define"
9-
"github.com/esonhugh/k8spider/pkg"
9+
"github.com/esonhugh/k8spider/pkg/scanner"
1010
log "github.com/sirupsen/logrus"
1111
"github.com/spf13/cobra"
1212
)
@@ -27,7 +27,7 @@ var ServiceCmd = &cobra.Command{
2727
for _, domain := range command.Opts.SvcDomains {
2828
records = append(records, define.Record{SvcDomain: fmt.Sprintf("%s.svc.%s", domain, command.Opts.Zone)})
2929
}
30-
records = pkg.ScanSvcForPorts(records)
30+
records = scanner.ScanSvcForPorts(records)
3131
if command.Opts.OutputFile != "" {
3232
f, err := os.OpenFile(command.Opts.OutputFile, os.O_CREATE|os.O_WRONLY, 0644)
3333
if err != nil {

‎cmd/subnet/subnet.go

+41-13
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,14 @@
11
package subnet
22

33
import (
4+
"net"
45
"os"
56

67
command "github.com/esonhugh/k8spider/cmd"
78
"github.com/esonhugh/k8spider/define"
89
"github.com/esonhugh/k8spider/pkg"
10+
"github.com/esonhugh/k8spider/pkg/mutli"
11+
"github.com/esonhugh/k8spider/pkg/scanner"
912
log "github.com/sirupsen/logrus"
1013
"github.com/spf13/cobra"
1114
)
@@ -27,20 +30,45 @@ var SubNetCmd = &cobra.Command{
2730
log.Warnf("ParseStringToIPNet failed: %v", err)
2831
return
2932
}
30-
var records define.Records = pkg.ScanSubnet(ipNets)
31-
if records == nil || len(records) == 0 {
32-
log.Warnf("ScanSubnet Found Nothing: %v", err)
33-
return
34-
}
35-
if command.Opts.OutputFile != "" {
36-
f, err := os.OpenFile(command.Opts.OutputFile, os.O_CREATE|os.O_WRONLY, 0644)
37-
if err != nil {
38-
log.Warnf("OpenFile failed: %v", err)
39-
}
40-
defer f.Close()
41-
records.Print(log.StandardLogger().Writer(), f)
33+
if command.Opts.BatchMode {
34+
BatchRun(ipNets)
4235
} else {
43-
records.Print(log.StandardLogger().Writer())
36+
Run(ipNets)
4437
}
4538
},
4639
}
40+
41+
func Run(net *net.IPNet) {
42+
var records define.Records = scanner.ScanSubnet(net)
43+
if records == nil || len(records) == 0 {
44+
log.Warnf("ScanSubnet Found Nothing")
45+
return
46+
}
47+
printResult(records)
48+
}
49+
50+
func BatchRun(net *net.IPNet) {
51+
scan := mutli.NewSubnetScanner()
52+
var finalRecord []define.Record
53+
for r := range scan.ScanSubnet(net) {
54+
finalRecord = append(finalRecord, r...)
55+
}
56+
if len(finalRecord) == 0 {
57+
log.Warn("ScanSubnet Found Nothing")
58+
return
59+
}
60+
printResult(finalRecord)
61+
}
62+
63+
func printResult(records define.Records) {
64+
if command.Opts.OutputFile != "" {
65+
f, err := os.OpenFile(command.Opts.OutputFile, os.O_CREATE|os.O_WRONLY, 0644)
66+
if err != nil {
67+
log.Warnf("OpenFile failed: %v", err)
68+
}
69+
defer f.Close()
70+
records.Print(log.StandardLogger().Writer(), f)
71+
} else {
72+
records.Print(log.StandardLogger().Writer())
73+
}
74+
}

‎pkg/mutli/executor.go

+13
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
package mutli
2+
3+
import (
4+
"net"
5+
6+
"github.com/esonhugh/k8spider/define"
7+
)
8+
9+
func ScanAll(subnet *net.IPNet) (result <-chan []define.Record) {
10+
subs := NewSubnetScanner()
11+
result = ScanServiceWithChan(subs.ScanSubnet(subnet))
12+
return result
13+
}

‎pkg/mutli/subnet.go

+53
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
package mutli
2+
3+
import (
4+
"net"
5+
"sync"
6+
"time"
7+
8+
"github.com/esonhugh/k8spider/define"
9+
"github.com/esonhugh/k8spider/pkg"
10+
"github.com/esonhugh/k8spider/pkg/scanner"
11+
log "github.com/sirupsen/logrus"
12+
)
13+
14+
type SubnetScanner struct {
15+
wg *sync.WaitGroup
16+
}
17+
18+
func NewSubnetScanner() *SubnetScanner {
19+
return &SubnetScanner{
20+
wg: new(sync.WaitGroup),
21+
}
22+
}
23+
24+
func (s *SubnetScanner) ScanSubnet(subnet *net.IPNet) <-chan []define.Record {
25+
if subnet == nil {
26+
log.Debugf("subnet is nil")
27+
return nil
28+
}
29+
out := make(chan []define.Record, 100)
30+
go func() {
31+
log.Debugf("splitting subnet into 16 pices")
32+
if subnets, err := pkg.SubnetShift(subnet, 4); err != nil {
33+
go s.scan(subnet, out)
34+
} else {
35+
for _, sn := range subnets {
36+
go s.scan(sn, out)
37+
}
38+
}
39+
time.Sleep(10 * time.Millisecond) // wait for all goroutines to start
40+
s.wg.Wait()
41+
close(out)
42+
}()
43+
return out
44+
}
45+
46+
func (s *SubnetScanner) scan(subnet *net.IPNet, to chan []define.Record) {
47+
s.wg.Add(1)
48+
// to <- scanner.ScanSubnet(subnet)
49+
for _, ip := range pkg.ParseIPNetToIPs(subnet) {
50+
to <- scanner.ScanSingleIP(ip)
51+
}
52+
s.wg.Done()
53+
}

‎pkg/mutli/svc.go

+17
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
package mutli
2+
3+
import (
4+
"github.com/esonhugh/k8spider/define"
5+
"github.com/esonhugh/k8spider/pkg/scanner"
6+
)
7+
8+
func ScanServiceWithChan(rev <-chan []define.Record) <-chan []define.Record {
9+
out := make(chan []define.Record, 100)
10+
go func() {
11+
for records := range rev {
12+
out <- scanner.ScanSvcForPorts(records)
13+
}
14+
close(out)
15+
}()
16+
return out
17+
}

‎pkg/scanner.go ‎pkg/scanner/scanner.go

+34-8
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,30 @@
1-
package pkg
1+
package scanner
22

33
import (
44
"net"
55
"strings"
66

77
"github.com/esonhugh/k8spider/define"
8+
"github.com/esonhugh/k8spider/pkg"
89
"github.com/miekg/dns"
910
log "github.com/sirupsen/logrus"
1011
)
1112

13+
func ScanSingleIP(subnet net.IP) (records []define.Record) {
14+
ptr := pkg.PTRRecord(subnet)
15+
if len(ptr) > 0 {
16+
for _, domain := range ptr {
17+
log.Infof("PTRrecord %v --> %v", subnet, domain)
18+
r := define.Record{Ip: subnet, SvcDomain: domain}
19+
records = append(records, r)
20+
}
21+
}
22+
return
23+
}
24+
1225
func ScanSubnet(subnet *net.IPNet) (records []define.Record) {
13-
for _, ip := range ParseIPNetToIPs(subnet) {
14-
ptr := PTRRecord(ip)
26+
for _, ip := range pkg.ParseIPNetToIPs(subnet) {
27+
ptr := pkg.PTRRecord(ip)
1528
if len(ptr) > 0 {
1629
for _, domain := range ptr {
1730
log.Infof("PTRrecord %v --> %v", ip, domain)
@@ -25,9 +38,22 @@ func ScanSubnet(subnet *net.IPNet) (records []define.Record) {
2538
return
2639
}
2740

41+
func ScanSingleSvcForPorts(records define.Record) define.Record {
42+
cname, srv, err := pkg.SRVRecord(records.SvcDomain)
43+
if err != nil {
44+
log.Debugf("SRVRecord for %v,failed: %v", records.SvcDomain, err)
45+
return records
46+
}
47+
for _, s := range srv {
48+
log.Infof("SRVRecord: %v --> %v:%v", records.SvcDomain, s.Target, s.Port)
49+
}
50+
records.SetSrvRecord(cname, srv)
51+
return records
52+
}
53+
2854
func ScanSvcForPorts(records []define.Record) []define.Record {
2955
for i, r := range records {
30-
cname, srv, err := SRVRecord(r.SvcDomain)
56+
cname, srv, err := pkg.SRVRecord(r.SvcDomain)
3157
if err != nil {
3258
log.Debugf("SRVRecord for %v,failed: %v", r.SvcDomain, err)
3359
continue
@@ -41,18 +67,18 @@ func ScanSvcForPorts(records []define.Record) []define.Record {
4167
}
4268

4369
// default target should be zone
44-
func DumpAXFR(target string, dnsServer string) []define.Record {
70+
func DumpAXFR(target string, dnsServer string) ([]define.Record, error) {
4571
t := new(dns.Transfer)
4672
m := new(dns.Msg)
4773
m.SetAxfr(target)
4874
ch, err := t.In(m, dnsServer)
4975
if err != nil {
50-
log.Fatalf("Transfer failed: %v", err)
76+
return nil, err
5177
}
5278
var records []define.Record
5379
for rr := range ch {
5480
if rr.Error != nil {
55-
log.Errorf("Error: %v", rr.Error)
81+
log.Debugf("Error: %v", rr.Error)
5682
continue
5783
}
5884
for _, r := range rr.RR {
@@ -63,5 +89,5 @@ func DumpAXFR(target string, dnsServer string) []define.Record {
6389
}
6490
log.Debugf("Record: %v", rr.RR)
6591
}
66-
return records
92+
return records, nil
6793
}

0 commit comments

Comments
 (0)