From a1ccdd2f6aa1a83e649fe60048435ef03b1a2fd8 Mon Sep 17 00:00:00 2001 From: Vincent Boutour Date: Wed, 26 May 2021 20:52:34 +0200 Subject: [PATCH] feat(pypi): Adding pypi support Closes None Signed-off-by: Vincent Boutour --- README.md | 2 +- cmd/ketchup/api.go | 4 +- cmd/ketchup/templates/ketchup.html | 16 +++++- cmd/ketchup/templates/public.html | 4 +- cmd/ketchup/templates/svg.html | 4 ++ cmd/notifier/notifier.go | 4 +- pkg/ketchup/ketchups.go | 9 +--- pkg/model/repository.go | 6 ++- pkg/notifier/releases.go | 2 +- pkg/provider/pypi/pypi.go | 61 +++++++++++++++++++++++ pkg/service/repository/repository.go | 7 ++- pkg/service/repository/repository_test.go | 38 +++++++------- sql/ddl.sql | 2 +- sql/migration_2021-05-26_2.sql | 1 + 14 files changed, 123 insertions(+), 37 deletions(-) create mode 100644 pkg/provider/pypi/pypi.go create mode 100644 sql/migration_2021-05-26_2.sql diff --git a/README.md b/README.md index c970ec92..aa4d15c4 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ Thanks to [OpenEmoji](https://openmoji.org) for favicon. Thanks to [FontAwesome](https://fontawesome.com) for icons. -> Check your GitHub, Helm, Docker or NPM dependencies every day at 8am and send a digest by email. +> Check your GitHub, Helm, Docker, NPM or Pypi dependencies every day at 8am and send a digest by email. ![](ketchup.png) diff --git a/cmd/ketchup/api.go b/cmd/ketchup/api.go index e4986e58..14832de5 100644 --- a/cmd/ketchup/api.go +++ b/cmd/ketchup/api.go @@ -32,6 +32,7 @@ import ( "github.com/ViBiOh/ketchup/pkg/provider/github" "github.com/ViBiOh/ketchup/pkg/provider/helm" "github.com/ViBiOh/ketchup/pkg/provider/npm" + "github.com/ViBiOh/ketchup/pkg/provider/pypi" "github.com/ViBiOh/ketchup/pkg/scheduler" ketchupService "github.com/ViBiOh/ketchup/pkg/service/ketchup" repositoryService "github.com/ViBiOh/ketchup/pkg/service/repository" @@ -108,7 +109,8 @@ func main() { dockerApp := docker.New(dockerConfig) helmApp := helm.New() npmApp := npm.New() - repositoryServiceApp := repositoryService.New(repositoryStore.New(ketchupDb), githubApp, helmApp, dockerApp, npmApp) + pypiApp := pypi.New() + repositoryServiceApp := repositoryService.New(repositoryStore.New(ketchupDb), githubApp, helmApp, dockerApp, npmApp, pypiApp) ketchupServiceApp := ketchupService.New(ketchupStore.New(ketchupDb), repositoryServiceApp) mailerApp, err := mailer.New(mailerConfig) diff --git a/cmd/ketchup/templates/ketchup.html b/cmd/ketchup/templates/ketchup.html index 319e2fd3..189d3d4e 100644 --- a/cmd/ketchup/templates/ketchup.html +++ b/cmd/ketchup/templates/ketchup.html @@ -34,6 +34,13 @@

Create ketchup

+ + + + +

@@ -105,6 +112,13 @@

Create ketchup

nameInput.classList.add("hidden"); } }); + + document.getElementById('create-kind-pypi').addEventListener('change', (e) => { + if (e.target.value === 'pypi') { + repositoryInput.placeholder = 'pip'; + nameInput.classList.add("hidden"); + } + }); {{ end }} @@ -257,7 +271,7 @@

Confirmation

} .create-form { - width: 25rem; + width: 30rem; } .separator { diff --git a/cmd/ketchup/templates/public.html b/cmd/ketchup/templates/public.html index 80ba8d37..3b60eb5d 100644 --- a/cmd/ketchup/templates/public.html +++ b/cmd/ketchup/templates/public.html @@ -1,5 +1,5 @@ {{ define "seo" }} - {{ $description := "Check updates of your GitHub, Helm, Docker or NPM dependencies with ease" }} + {{ $description := "Check updates of your GitHub, Helm, Docker, NPM or Pypi dependencies with ease" }} {{ .Title }} @@ -94,7 +94,7 @@

{{ define "app" }}
-

Receive an email digest of your GitHub, Helm, Docker or NPM dependencies updates every day at 8am.

+

Receive an email digest of your GitHub, Helm, Docker, NPM or Pypi dependencies updates every day at 8am.

No ads, no analytics, no data selling, free. Because being update-to-date must be accessible to everyone. diff --git a/cmd/ketchup/templates/svg.html b/cmd/ketchup/templates/svg.html index a44e4a93..551c7883 100644 --- a/cmd/ketchup/templates/svg.html +++ b/cmd/ketchup/templates/svg.html @@ -34,6 +34,10 @@ {{ end }} +{{ define "svg-pypi" }} + +{{ end }} + {{ define "svg-question" }} {{ end }} diff --git a/cmd/notifier/notifier.go b/cmd/notifier/notifier.go index 61968b7c..4763b6be 100644 --- a/cmd/notifier/notifier.go +++ b/cmd/notifier/notifier.go @@ -12,6 +12,7 @@ import ( "github.com/ViBiOh/ketchup/pkg/provider/github" "github.com/ViBiOh/ketchup/pkg/provider/helm" "github.com/ViBiOh/ketchup/pkg/provider/npm" + "github.com/ViBiOh/ketchup/pkg/provider/pypi" ketchupService "github.com/ViBiOh/ketchup/pkg/service/ketchup" repositoryService "github.com/ViBiOh/ketchup/pkg/service/repository" ketchupStore "github.com/ViBiOh/ketchup/pkg/store/ketchup" @@ -49,7 +50,8 @@ func main() { helmApp := helm.New() npmApp := npm.New() - repositoryServiceApp := repositoryService.New(repositoryStore.New(ketchupDb), github.New(githubConfig, nil), helmApp, docker.New(dockerConfig), npmApp) + pypiApp := pypi.New() + repositoryServiceApp := repositoryService.New(repositoryStore.New(ketchupDb), github.New(githubConfig, nil), helmApp, docker.New(dockerConfig), npmApp, pypiApp) ketchupServiceApp := ketchupService.New(ketchupStore.New(ketchupDb), repositoryServiceApp) notifierApp := notifier.New(notifierConfig, repositoryServiceApp, ketchupServiceApp, mailerApp, helmApp) diff --git a/pkg/ketchup/ketchups.go b/pkg/ketchup/ketchups.go index 45813624..9be98dab 100644 --- a/pkg/ketchup/ketchups.go +++ b/pkg/ketchup/ketchups.go @@ -55,17 +55,10 @@ func (a app) handleCreate(w http.ResponseWriter, r *http.Request) { name := r.FormValue("name") switch repositoryKind { - case model.Github: - repository = model.NewGithubRepository(0, name) case model.Helm: repository = model.NewHelmRepository(0, strings.TrimSuffix(name, "/"), r.FormValue("part")) - case model.Docker: - repository = model.NewDockerRepository(0, name) - case model.NPM: - repository = model.NewNPMRepository(0, name) default: - a.rendererApp.Error(w, httpModel.WrapInternal(fmt.Errorf("unhandled repository kind `%s`", repositoryKind))) - return + repository = model.NewRepository(0, repositoryKind, name, "") } item := model.NewKetchup(r.FormValue("pattern"), r.FormValue("version"), ketchupFrequency, repository) diff --git a/pkg/model/repository.go b/pkg/model/repository.go index 0d8c370c..b5958e62 100644 --- a/pkg/model/repository.go +++ b/pkg/model/repository.go @@ -24,11 +24,13 @@ const ( Docker // NPM repository kind NPM + // Pypi repository kind + Pypi ) var ( // RepositoryKindValues string values - RepositoryKindValues = []string{"github", "helm", "docker", "npm"} + RepositoryKindValues = []string{"github", "helm", "docker", "npm", "pypi"} // NoneRepository is an undefined repository NoneRepository = Repository{} @@ -128,6 +130,8 @@ func (r Repository) URL(pattern string) string { } case NPM: return fmt.Sprintf("https://www.npmjs.com/package/%s/v/%s", r.Name, r.Versions[pattern]) + case Pypi: + return fmt.Sprintf("https://pypi.org/project/%s/%s/", r.Name, r.Versions[pattern]) default: return "#" } diff --git a/pkg/notifier/releases.go b/pkg/notifier/releases.go index f6151c3a..e07be1e9 100644 --- a/pkg/notifier/releases.go +++ b/pkg/notifier/releases.go @@ -29,7 +29,7 @@ func (a app) getNewStandardReleases(ctx context.Context) ([]model.Release, uint6 var lastKey string for { - repositories, _, err := a.repositoryService.ListByKinds(ctx, pageSize, lastKey, model.Github, model.Docker, model.NPM) + repositories, _, err := a.repositoryService.ListByKinds(ctx, pageSize, lastKey, model.Github, model.Docker, model.NPM, model.Pypi) if err != nil { return nil, count, fmt.Errorf("unable to fetch standard repositories: %s", err) } diff --git a/pkg/provider/pypi/pypi.go b/pkg/provider/pypi/pypi.go new file mode 100644 index 00000000..43f19eff --- /dev/null +++ b/pkg/provider/pypi/pypi.go @@ -0,0 +1,61 @@ +package pypi + +import ( + "context" + "fmt" + + "github.com/ViBiOh/httputils/v4/pkg/httpjson" + "github.com/ViBiOh/httputils/v4/pkg/request" + "github.com/ViBiOh/ketchup/pkg/model" + "github.com/ViBiOh/ketchup/pkg/semver" +) + +const ( + registryURL = "https://pypi.org/pypi" +) + +type packageResp struct { + Versions map[string]interface{} `json:"releases"` +} + +// App of package +type App interface { + LatestVersions(string, []string) (map[string]semver.Version, error) +} + +type app struct{} + +// New creates new App from Config +func New() App { + return app{} +} + +func (a app) LatestVersions(name string, patterns []string) (map[string]semver.Version, error) { + ctx := context.Background() + + versions, compiledPatterns, err := model.PreparePatternMatching(patterns) + if err != nil { + return nil, fmt.Errorf("unable to prepare pattern matching: %s", err) + } + + resp, err := request.New().Get(fmt.Sprintf("%s/%s/json", registryURL, name)).Send(ctx, nil) + if err != nil { + return nil, fmt.Errorf("unable to fetch registry: %s", err) + } + + var content packageResp + if err := httpjson.Read(resp, &content); err != nil { + return nil, fmt.Errorf("unable to read versions: %s", err) + } + + for version := range content.Versions { + tagVersion, err := semver.Parse(version) + if err != nil { + continue + } + + model.CheckPatternsMatching(versions, compiledPatterns, tagVersion) + } + + return versions, nil +} diff --git a/pkg/service/repository/repository.go b/pkg/service/repository/repository.go index 212a8b12..674c8a81 100644 --- a/pkg/service/repository/repository.go +++ b/pkg/service/repository/repository.go @@ -13,6 +13,7 @@ import ( "github.com/ViBiOh/ketchup/pkg/provider/github" "github.com/ViBiOh/ketchup/pkg/provider/helm" "github.com/ViBiOh/ketchup/pkg/provider/npm" + "github.com/ViBiOh/ketchup/pkg/provider/pypi" "github.com/ViBiOh/ketchup/pkg/semver" "github.com/ViBiOh/ketchup/pkg/store/repository" ) @@ -38,16 +39,18 @@ type app struct { helmApp helm.App dockerApp docker.App npmApp npm.App + pypiApp pypi.App } // New creates new App from Config -func New(repositoryStore repository.App, githubApp github.App, helmApp helm.App, dockerApp docker.App, npmApp npm.App) App { +func New(repositoryStore repository.App, githubApp github.App, helmApp helm.App, dockerApp docker.App, npmApp npm.App, pypiApp pypi.App) App { return app{ repositoryStore: repositoryStore, githubApp: githubApp, helmApp: helmApp, dockerApp: dockerApp, npmApp: npmApp, + pypiApp: pypiApp, } } @@ -233,6 +236,8 @@ func (a app) LatestVersions(repo model.Repository) (map[string]semver.Version, e return a.dockerApp.LatestVersions(repo.Name, patterns) case model.NPM: return a.npmApp.LatestVersions(repo.Name, patterns) + case model.Pypi: + return a.pypiApp.LatestVersions(repo.Name, patterns) default: return nil, fmt.Errorf("unknown repository kind %d", repo.Kind) } diff --git a/pkg/service/repository/repository_test.go b/pkg/service/repository/repository_test.go index d30a1f43..de3ccfeb 100644 --- a/pkg/service/repository/repository_test.go +++ b/pkg/service/repository/repository_test.go @@ -51,7 +51,7 @@ func TestList(t *testing.T) { New(repositorytest.New().SetList([]model.Repository{ model.NewGithubRepository(1, ketchupRepository).AddVersion(model.DefaultPattern, "1.0.0"), model.NewGithubRepository(2, viwsRepository).AddVersion(model.DefaultPattern, "1.2.3"), - }, 2, nil), nil, nil, nil, nil), + }, 2, nil), nil, nil, nil, nil, nil), args{}, []model.Repository{ model.NewGithubRepository(1, ketchupRepository).AddVersion(model.DefaultPattern, "1.0.0"), @@ -62,7 +62,7 @@ func TestList(t *testing.T) { }, { "error", - New(repositorytest.New().SetList(nil, 0, errors.New("failed")), nil, nil, nil, nil), + New(repositorytest.New().SetList(nil, 0, errors.New("failed")), nil, nil, nil, nil, nil), args{}, nil, 0, @@ -109,7 +109,7 @@ func TestSuggest(t *testing.T) { "simple", New(repositorytest.New().SetSuggest([]model.Repository{ model.NewGithubRepository(1, ketchupRepository).AddVersion(model.DefaultPattern, "1.2.3"), - }, nil), nil, nil, nil, nil), + }, nil), nil, nil, nil, nil, nil), args{ ctx: context.Background(), }, @@ -120,7 +120,7 @@ func TestSuggest(t *testing.T) { }, { "error", - New(repositorytest.New().SetSuggest(nil, errors.New("failed")), nil, nil, nil, nil), + New(repositorytest.New().SetSuggest(nil, errors.New("failed")), nil, nil, nil, nil, nil), args{ ctx: context.Background(), }, @@ -166,7 +166,7 @@ func TestGetOrCreate(t *testing.T) { }{ { "get error", - New(repositorytest.New().SetGetByName(model.NoneRepository, errors.New("failed")), githubtest.New(), nil, nil, nil), + New(repositorytest.New().SetGetByName(model.NoneRepository, errors.New("failed")), githubtest.New(), nil, nil, nil, nil), args{ ctx: context.Background(), name: "error", @@ -178,7 +178,7 @@ func TestGetOrCreate(t *testing.T) { }, { "exists with pattern", - New(repositorytest.New().SetGetByName(model.NewGithubRepository(1, ketchupRepository).AddVersion(model.DefaultPattern, "1.0.0"), nil), githubtest.New(), nil, nil, nil), + New(repositorytest.New().SetGetByName(model.NewGithubRepository(1, ketchupRepository).AddVersion(model.DefaultPattern, "1.0.0"), nil), githubtest.New(), nil, nil, nil, nil), args{ ctx: context.Background(), name: "exist", @@ -190,7 +190,7 @@ func TestGetOrCreate(t *testing.T) { }, { "exists no pattern error", - New(repositorytest.New().SetGetByName(model.NewGithubRepository(1, ketchupRepository), nil), githubtest.New().SetLatestVersions(nil, errors.New("failed")), nil, nil, nil), + New(repositorytest.New().SetGetByName(model.NewGithubRepository(1, ketchupRepository), nil), githubtest.New().SetLatestVersions(nil, errors.New("failed")), nil, nil, nil, nil), args{ ctx: context.Background(), name: "exist", @@ -204,7 +204,7 @@ func TestGetOrCreate(t *testing.T) { "exists pattern not found", New(repositorytest.New().SetGetByName(model.NewGithubRepository(1, ketchupRepository), nil), githubtest.New().SetLatestVersions(map[string]semver.Version{ "latest": safeParse("1.0.0"), - }, nil), nil, nil, nil), + }, nil), nil, nil, nil, nil), args{ ctx: context.Background(), name: "exist", @@ -218,7 +218,7 @@ func TestGetOrCreate(t *testing.T) { "exists but no pattern", New(repositorytest.New().SetGetByName(model.NewGithubRepository(1, ketchupRepository), nil), githubtest.New().SetLatestVersions(map[string]semver.Version{ model.DefaultPattern: safeParse("1.0.0"), - }, nil), nil, nil, nil), + }, nil), nil, nil, nil, nil), args{ ctx: context.Background(), name: "exist", @@ -232,7 +232,7 @@ func TestGetOrCreate(t *testing.T) { "update error", New(repositorytest.New().SetGetByName(model.NewHelmRepository(1, chartRepository, "app"), nil).SetUpdateVersions(errors.New("failed")), githubtest.New(), helmtest.New().SetLatestVersions(map[string]semver.Version{ model.DefaultPattern: safeParse("1.0.0"), - }, nil), nil, nil), + }, nil), nil, nil, nil), args{ ctx: context.Background(), name: "exist", @@ -246,7 +246,7 @@ func TestGetOrCreate(t *testing.T) { "create", New(repositorytest.New().SetCreate(1, nil), githubtest.New().SetLatestVersions(map[string]semver.Version{ model.DefaultPattern: safeParse("1.0.0"), - }, nil), nil, nil, nil), + }, nil), nil, nil, nil, nil), args{ ctx: context.Background(), name: "not found", @@ -381,7 +381,7 @@ func TestUpdate(t *testing.T) { }{ { "start atomic error", - New(repositorytest.New().SetDoAtomic(errAtomicStart), nil, nil, nil, nil), + New(repositorytest.New().SetDoAtomic(errAtomicStart), nil, nil, nil, nil, nil), args{ ctx: context.TODO(), item: model.NoneRepository, @@ -390,7 +390,7 @@ func TestUpdate(t *testing.T) { }, { "fetch error", - New(repositorytest.New().SetGet(model.NoneRepository, errors.New("failed")), nil, nil, nil, nil), + New(repositorytest.New().SetGet(model.NoneRepository, errors.New("failed")), nil, nil, nil, nil, nil), args{ ctx: context.Background(), item: model.NewGithubRepository(0, ""), @@ -399,7 +399,7 @@ func TestUpdate(t *testing.T) { }, { "invalid check", - New(repositorytest.New().SetGet(model.NewGithubRepository(1, ketchupRepository), nil), nil, nil, nil, nil), + New(repositorytest.New().SetGet(model.NewGithubRepository(1, ketchupRepository), nil), nil, nil, nil, nil, nil), args{ ctx: context.Background(), item: model.NewGithubRepository(1, ""), @@ -408,7 +408,7 @@ func TestUpdate(t *testing.T) { }, { "update error", - New(repositorytest.New().SetGet(model.NewGithubRepository(1, ketchupRepository), nil).SetUpdateVersions(errors.New("failed")), nil, nil, nil, nil), + New(repositorytest.New().SetGet(model.NewGithubRepository(1, ketchupRepository), nil).SetUpdateVersions(errors.New("failed")), nil, nil, nil, nil, nil), args{ ctx: context.Background(), item: model.NewGithubRepository(1, "").AddVersion(model.DefaultPattern, "1.2.3"), @@ -417,7 +417,7 @@ func TestUpdate(t *testing.T) { }, { "success", - New(repositorytest.New().SetGet(model.NewGithubRepository(1, ketchupRepository), nil), nil, nil, nil, nil), + New(repositorytest.New().SetGet(model.NewGithubRepository(1, ketchupRepository), nil), nil, nil, nil, nil, nil), args{ ctx: context.Background(), item: model.NewGithubRepository(3, "").AddVersion(model.DefaultPattern, "1.2.3"), @@ -456,7 +456,7 @@ func TestClean(t *testing.T) { }{ { "error", - New(repositorytest.New().SetDeleteUnused(errors.New("failed")), nil, nil, nil, nil), + New(repositorytest.New().SetDeleteUnused(errors.New("failed")), nil, nil, nil, nil, nil), args{ ctx: context.Background(), }, @@ -464,7 +464,7 @@ func TestClean(t *testing.T) { }, { "error versions", - New(repositorytest.New().SetDeleteUnusedVersions(errors.New("failed")), nil, nil, nil, nil), + New(repositorytest.New().SetDeleteUnusedVersions(errors.New("failed")), nil, nil, nil, nil, nil), args{ ctx: context.Background(), }, @@ -472,7 +472,7 @@ func TestClean(t *testing.T) { }, { "success", - New(repositorytest.New(), nil, nil, nil, nil), + New(repositorytest.New(), nil, nil, nil, nil, nil), args{ ctx: context.Background(), }, diff --git a/sql/ddl.sql b/sql/ddl.sql index 3f14b8c4..938863a9 100644 --- a/sql/ddl.sql +++ b/sql/ddl.sql @@ -35,7 +35,7 @@ CREATE UNIQUE INDEX user_login_id ON ketchup.user(login_id); CREATE UNIQUE INDEX user_email ON ketchup.user(email); -- repository_kind -CREATE TYPE ketchup.repository_kind AS ENUM ('github', 'helm', 'docker', 'npm'); +CREATE TYPE ketchup.repository_kind AS ENUM ('github', 'helm', 'docker', 'npm', 'pypi'); -- repository CREATE SEQUENCE ketchup.repository_seq; diff --git a/sql/migration_2021-05-26_2.sql b/sql/migration_2021-05-26_2.sql new file mode 100644 index 00000000..c7c52086 --- /dev/null +++ b/sql/migration_2021-05-26_2.sql @@ -0,0 +1 @@ +ALTER TYPE ketchup.repository_kind ADD VALUE 'pypi';