Skip to content

Commit 68cc66e

Browse files
jmhobbsjmhobbs
committed
Add command to download and cache lists.
1 parent 0f5609e commit 68cc66e

File tree

3 files changed

+167
-0
lines changed

3 files changed

+167
-0
lines changed

‎internal/package/fetcher/all.go

+118
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
package fetcher
2+
3+
import (
4+
"fmt"
5+
"strings"
6+
"sync"
7+
8+
"github.com/umutphp/awesome-cli/internal/package/node"
9+
"github.com/umutphp/awesome-cli/internal/package/parser"
10+
)
11+
12+
const DOWNLOAD_CONCURRENCY = 4
13+
14+
type Progress struct {
15+
Found uint64
16+
Crawled uint64
17+
Errors uint64
18+
}
19+
20+
func FetchAllRepos(update chan Progress) (Progress, error) {
21+
fetched, err := FetchAwsomeRootRepo()
22+
if err != nil {
23+
return Progress{1, 0, 1}, err
24+
}
25+
26+
root := parser.ParseIndex(fetched)
27+
urls := getAllChildrenURLs(root.GetChildren())
28+
29+
progress := Progress{
30+
Found: uint64(1 + len(urls)),
31+
Crawled: 1,
32+
Errors: 0,
33+
}
34+
update <- progress
35+
36+
var wg sync.WaitGroup
37+
queue := make(chan string)
38+
errors := make(chan error)
39+
40+
for i := 0; i < 4; i++ {
41+
go fetchWorker(queue, errors, &wg)
42+
}
43+
44+
allErrors := MultiError{[]error{}}
45+
go func () {
46+
for err := range errors {
47+
if err == nil {
48+
progress.Crawled = progress.Crawled + 1
49+
} else {
50+
progress.Errors = progress.Errors + 1
51+
allErrors.Errors = append(allErrors.Errors, err)
52+
}
53+
update <- progress
54+
}
55+
close(update)
56+
}()
57+
58+
for _, url := range urls {
59+
queue <- url
60+
}
61+
62+
close(queue)
63+
wg.Wait()
64+
close(errors)
65+
66+
if len(allErrors.Errors) == 0 {
67+
return progress, nil
68+
}
69+
return progress, allErrors
70+
}
71+
72+
func getAllChildrenURLs(children []node.Node) []string {
73+
urls := []string{}
74+
for _, child := range children {
75+
if child.GetURL() != "" {
76+
urls = append(urls, child.GetURL())
77+
}
78+
urls = append(urls, getAllChildrenURLs(child.GetChildren())...)
79+
}
80+
return urls
81+
}
82+
83+
type MultiError struct {
84+
Errors []error
85+
}
86+
87+
func (e MultiError) Error() string {
88+
errStrings := []string{}
89+
for _, err := range e.Errors {
90+
errStrings = append(errStrings, err.Error())
91+
}
92+
return strings.Join(errStrings, "\n")
93+
}
94+
95+
type FetchError struct {
96+
URL string
97+
Wrapped error
98+
}
99+
100+
func (e FetchError) Unwrap() error {
101+
return e.Wrapped
102+
}
103+
104+
func (e FetchError) Error() string {
105+
return fmt.Sprintf("failed to fetch %q: %v", e.URL, e.Wrapped)
106+
}
107+
108+
func fetchWorker(queue chan string, errors chan error, wg *sync.WaitGroup) {
109+
wg.Add(1)
110+
defer wg.Done()
111+
for url := range queue {
112+
if _, err := FetchAwsomeRepo(url); err != nil {
113+
errors <- FetchError{url, err}
114+
} else {
115+
errors <- nil
116+
}
117+
}
118+
}

‎internal/package/progress/progress.go

+24
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
package progress
2+
3+
import (
4+
"fmt"
5+
6+
"github.com/umutphp/awesome-cli/internal/package/fetcher"
7+
)
8+
9+
func ProgressBar(done chan struct{}, updates chan fetcher.Progress) {
10+
for progress := range updates {
11+
percent := 100.0 * float64(progress.Crawled + progress.Errors) / float64(progress.Found)
12+
fmt.Print("\n\033[1A\033[K")
13+
fmt.Print("[")
14+
for i := 0; i < 20; i++ {
15+
if float64(i*5) <= percent {
16+
fmt.Print("=")
17+
} else {
18+
fmt.Print(" ")
19+
}
20+
}
21+
fmt.Printf("] % 3.0f%%", percent)
22+
}
23+
close(done)
24+
}

‎main.go

+25
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,9 @@ import (
55
"os"
66

77
"github.com/umutphp/awesome-cli/internal/package/favourite"
8+
"github.com/umutphp/awesome-cli/internal/package/fetcher"
89
"github.com/umutphp/awesome-cli/internal/package/manager"
10+
"github.com/umutphp/awesome-cli/internal/package/progress"
911
"github.com/umutphp/awesome-cli/internal/package/prompter"
1012
"github.com/umutphp/awesome-cli/internal/package/selfupdate"
1113
)
@@ -43,6 +45,7 @@ func DisplayHelp() {
4345
fmt.Printf("%-2v%-10v%-10v\n", "", "profile", "To see your previous choices.")
4446
fmt.Printf("%-2v%-10v%-10v\n", "", "reset", "To clean your choices to start from the beginning.")
4547
fmt.Printf("%-2v%-10v%-10v\n", "", "update", "Update awesome-cli to the latest version.")
48+
fmt.Printf("%-2v%-10v%-10v\n", "", "cache", "Download all repository information to the cache.")
4649

4750
fmt.Println("")
4851
fmt.Println("")
@@ -100,6 +103,23 @@ func DisplayRepoWithPath(url string, path []string) {
100103
prompter.OpenInBrowser(url)
101104
}
102105

106+
func CacheAll() {
107+
fmt.Println("Downloading all reachable lists to cache.")
108+
fmt.Println("This may take some time!")
109+
110+
updates := make(chan fetcher.Progress)
111+
done := make(chan struct{})
112+
113+
go progress.ProgressBar(done, updates)
114+
finalResult, errs := fetcher.FetchAllRepos(updates)
115+
<-done
116+
fmt.Print("\n")
117+
fmt.Printf("Downloaded %d / Failed %d\n", finalResult.Crawled, finalResult.Errors)
118+
if finalResult.Errors > 0 {
119+
fmt.Println(errs)
120+
}
121+
}
122+
103123
func Argumented(param []string, man manager.Manager) {
104124
if param[0] == "random" {
105125
RandomRepo(man)
@@ -133,6 +153,11 @@ func Argumented(param []string, man manager.Manager) {
133153

134154
return
135155
}
156+
157+
if param[0] == "cache" {
158+
CacheAll()
159+
return
160+
}
136161
}
137162

138163
func Walk(man manager.Manager) {

0 commit comments

Comments
 (0)