feat(pypi): Adding pypi support

Closes None

Signed-off-by: Vincent Boutour <[email protected]>

README.md

@@ -8,7 +8,7 @@ Thanks to [OpenEmoji](https://openmoji.org) for favicon.
 
 Thanks to [FontAwesome](https://fontawesome.com) for icons.
 
-> Check your GitHub, Helm, Docker or NPM dependencies every day at 8am and send a digest by email.
+> Check your GitHub, Helm, Docker, NPM or Pypi dependencies every day at 8am and send a digest by email.
 
 ![](ketchup.png)
 

cmd/ketchup/api.go

@@ -32,6 +32,7 @@ import (
 	"github.com/ViBiOh/ketchup/pkg/provider/github"
 	"github.com/ViBiOh/ketchup/pkg/provider/helm"
 	"github.com/ViBiOh/ketchup/pkg/provider/npm"
+	"github.com/ViBiOh/ketchup/pkg/provider/pypi"
 	"github.com/ViBiOh/ketchup/pkg/scheduler"
 	ketchupService "github.com/ViBiOh/ketchup/pkg/service/ketchup"
 	repositoryService "github.com/ViBiOh/ketchup/pkg/service/repository"
@@ -108,7 +109,8 @@ func main() {
 	dockerApp := docker.New(dockerConfig)
 	helmApp := helm.New()
 	npmApp := npm.New()
-	repositoryServiceApp := repositoryService.New(repositoryStore.New(ketchupDb), githubApp, helmApp, dockerApp, npmApp)
+	pypiApp := pypi.New()
+	repositoryServiceApp := repositoryService.New(repositoryStore.New(ketchupDb), githubApp, helmApp, dockerApp, npmApp, pypiApp)
 	ketchupServiceApp := ketchupService.New(ketchupStore.New(ketchupDb), repositoryServiceApp)
 
 	mailerApp, err := mailer.New(mailerConfig)

cmd/ketchup/templates/ketchup.html

@@ -34,6 +34,13 @@ <h2 class="header">Create ketchup</h2>
             </label>
             <input id="create-kind-npm" type="radio" name="kind" value="npm">
           </span>
+
+          <span class="flex-grow center">
+            <label for="create-kind-pypi" class="block">
+              <img class="icon icon-large clickable" src="{{ url "/svg/pypi?fill=silver" }}" alt="Pypi icon">
+            </label>
+            <input id="create-kind-pypi" type="radio" name="kind" value="pypi">
+          </span>
         </p>
 
         <p class="padding no-margin">
@@ -105,6 +112,13 @@ <h2 class="header">Create ketchup</h2>
         nameInput.classList.add("hidden");
       }
     });
+
+    document.getElementById('create-kind-pypi').addEventListener('change', (e) => {
+      if (e.target.value === 'pypi') {
+        repositoryInput.placeholder = 'pip';
+        nameInput.classList.add("hidden");
+      }
+    });
   </script>
 {{ end }}
 
@@ -257,7 +271,7 @@ <h2 class="header">Confirmation</h2>
     }
 
     .create-form {
-      width: 25rem;
+      width: 30rem;
     }
 
     .separator {

cmd/ketchup/templates/public.html

@@ -1,5 +1,5 @@
 {{ define "seo" }}
-  {{ $description := "Check updates of your GitHub, Helm, Docker or NPM dependencies with ease" }}
+  {{ $description := "Check updates of your GitHub, Helm, Docker, NPM or Pypi dependencies with ease" }}
 
   <title>{{ .Title }}</title>
   <meta name="description" content="{{ $description }}">
@@ -94,7 +94,7 @@ <h2 class="center">
 
 {{ define "app" }}
   <article class="padding center">
-    <h2>Receive an email digest of your GitHub, Helm, Docker or NPM dependencies updates every day at 8am.</h2>
+    <h2>Receive an email digest of your GitHub, Helm, Docker, NPM or Pypi dependencies updates every day at 8am.</h2>
 
     <em>
       No ads, no analytics, no data selling, free. Because being update-to-date must be accessible to everyone.

cmd/ketchup/templates/svg.html

@@ -34,6 +34,10 @@
   <svg xmlns="http://www.w3.org/2000/svg" width="540" height="210" viewBox="0 0 18 7"><defs/><path fill="{{ . }}" d="M0 0h18v6H9v1H5V6H0V0zm1 5h2V2h1v3h1V1H1v4zm5-4v5h2V5h2V1H6zm2 1h1v2H8V2zm3-1v4h2V2h1v3h1V2h1v3h1V1h-6z"/><path fill="#FFF" d="M1 5h2V2h1v3h1V1H1zM6 1v5h2V5h2V1H6zm3 3H8V2h1v2zM11 1v4h2V2h1v3h1V2h1v3h1V1z"/></svg>
 {{ end }}
 
+{{ define "svg-pypi" }}
+  <svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.0" viewBox="0 0 110.421 109.846"><defs/><defs><linearGradient id="a"><stop offset="0" stop-color="#ffe052"/><stop offset="1" stop-color="#ffc331"/></linearGradient><linearGradient id="d" x1="89.137" x2="147.777" y1="111.921" y2="168.101" gradientUnits="userSpaceOnUse" xlink:href="#a"/><linearGradient id="b"><stop offset="0" stop-color="#387eb8"/><stop offset="1" stop-color="#366994"/></linearGradient><linearGradient id="c" x1="55.549" x2="110.149" y1="77.07" y2="131.853" gradientUnits="userSpaceOnUse" xlink:href="#b"/></defs><g color="#000"><path fill="{{ . }}" d="M99.75 67.469c-28.032 0-26.281 12.156-26.281 12.156l.031 12.594h26.75V96H62.875s-17.938-2.034-17.938 26.25 15.657 27.281 15.657 27.281h9.343v-13.125s-.503-15.656 15.407-15.656h26.531s14.906.241 14.906-14.406V82.125s2.263-14.656-27.031-14.656zM85 75.938a4.808 4.808 0 014.813 4.812A4.808 4.808 0 0185 85.563a4.808 4.808 0 01-4.813-4.813A4.808 4.808 0 0185 75.937z" overflow="visible" style="marker:none" transform="translate(-44.938 -67.469)"/><path fill="{{ . }}" d="M100.546 177.315c28.032 0 26.281-12.156 26.281-12.156l-.03-12.594h-26.75v-3.781h37.374s17.938 2.034 17.938-26.25c0-28.285-15.657-27.282-15.657-27.282h-9.343v13.125s.503 15.657-15.407 15.657h-26.53s-14.907-.241-14.907 14.406v24.219s-2.263 14.656 27.031 14.656zm14.75-8.469a4.808 4.808 0 01-4.812-4.812 4.808 4.808 0 014.812-4.813 4.808 4.808 0 014.813 4.813 4.808 4.808 0 01-4.813 4.812z" overflow="visible" style="marker:none" transform="translate(-44.938 -67.469)"/></g></svg>
+{{ end }}
+
 {{ define "svg-question" }}
   <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512 512"><defs/><path fill="{{ . }}" d="M504 256c0 136.997-111.043 248-248 248S8 392.997 8 256C8 119.083 119.043 8 256 8s248 111.083 248 248zM262.655 90c-54.497 0-89.255 22.957-116.549 63.758-3.536 5.286-2.353 12.415 2.715 16.258l34.699 26.31c5.205 3.947 12.621 3.008 16.665-2.122 17.864-22.658 30.113-35.797 57.303-35.797 20.429 0 45.698 13.148 45.698 32.958 0 14.976-12.363 22.667-32.534 33.976C247.128 238.528 216 254.941 216 296v4c0 6.627 5.373 12 12 12h56c6.627 0 12-5.373 12-12v-1.333c0-28.462 83.186-29.647 83.186-106.667 0-58.002-60.165-102-116.531-102zM256 338c-25.365 0-46 20.635-46 46 0 25.364 20.635 46 46 46s46-20.636 46-46c0-25.365-20.635-46-46-46z"/></svg>
 {{ end }}

cmd/notifier/notifier.go

@@ -12,6 +12,7 @@ import (
 	"github.com/ViBiOh/ketchup/pkg/provider/github"
 	"github.com/ViBiOh/ketchup/pkg/provider/helm"
 	"github.com/ViBiOh/ketchup/pkg/provider/npm"
+	"github.com/ViBiOh/ketchup/pkg/provider/pypi"
 	ketchupService "github.com/ViBiOh/ketchup/pkg/service/ketchup"
 	repositoryService "github.com/ViBiOh/ketchup/pkg/service/repository"
 	ketchupStore "github.com/ViBiOh/ketchup/pkg/store/ketchup"
@@ -49,7 +50,8 @@ func main() {
 
 	helmApp := helm.New()
 	npmApp := npm.New()
-	repositoryServiceApp := repositoryService.New(repositoryStore.New(ketchupDb), github.New(githubConfig, nil), helmApp, docker.New(dockerConfig), npmApp)
+	pypiApp := pypi.New()
+	repositoryServiceApp := repositoryService.New(repositoryStore.New(ketchupDb), github.New(githubConfig, nil), helmApp, docker.New(dockerConfig), npmApp, pypiApp)
 	ketchupServiceApp := ketchupService.New(ketchupStore.New(ketchupDb), repositoryServiceApp)
 
 	notifierApp := notifier.New(notifierConfig, repositoryServiceApp, ketchupServiceApp, mailerApp, helmApp)

pkg/ketchup/ketchups.go

@@ -55,17 +55,10 @@ func (a app) handleCreate(w http.ResponseWriter, r *http.Request) {
 	name := r.FormValue("name")
 
 	switch repositoryKind {
-	case model.Github:
-		repository = model.NewGithubRepository(0, name)
 	case model.Helm:
 		repository = model.NewHelmRepository(0, strings.TrimSuffix(name, "/"), r.FormValue("part"))
-	case model.Docker:
-		repository = model.NewDockerRepository(0, name)
-	case model.NPM:
-		repository = model.NewNPMRepository(0, name)
 	default:
-		a.rendererApp.Error(w, httpModel.WrapInternal(fmt.Errorf("unhandled repository kind `%s`", repositoryKind)))
-		return
+		repository = model.NewRepository(0, repositoryKind, name, "")
 	}
 
 	item := model.NewKetchup(r.FormValue("pattern"), r.FormValue("version"), ketchupFrequency, repository)

pkg/model/repository.go

@@ -24,11 +24,13 @@ const (
 	Docker
 	// NPM repository kind
 	NPM
+	// Pypi repository kind
+	Pypi
 )
 
 var (
 	// RepositoryKindValues string values
-	RepositoryKindValues = []string{"github", "helm", "docker", "npm"}
+	RepositoryKindValues = []string{"github", "helm", "docker", "npm", "pypi"}
 
 	// NoneRepository is an undefined repository
 	NoneRepository = Repository{}
@@ -128,6 +130,8 @@ func (r Repository) URL(pattern string) string {
 		}
 	case NPM:
 		return fmt.Sprintf("https://www.npmjs.com/package/%s/v/%s", r.Name, r.Versions[pattern])
+	case Pypi:
+		return fmt.Sprintf("https://pypi.org/project/%s/%s/", r.Name, r.Versions[pattern])
 	default:
 		return "#"
 	}

pkg/notifier/releases.go

@@ -29,7 +29,7 @@ func (a app) getNewStandardReleases(ctx context.Context) ([]model.Release, uint6
 	var lastKey string
 
 	for {
-		repositories, _, err := a.repositoryService.ListByKinds(ctx, pageSize, lastKey, model.Github, model.Docker, model.NPM)
+		repositories, _, err := a.repositoryService.ListByKinds(ctx, pageSize, lastKey, model.Github, model.Docker, model.NPM, model.Pypi)
 		if err != nil {
 			return nil, count, fmt.Errorf("unable to fetch standard repositories: %s", err)
 		}

pkg/provider/pypi/pypi.go

@@ -0,0 +1,61 @@
+package pypi
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/ViBiOh/httputils/v4/pkg/httpjson"
+	"github.com/ViBiOh/httputils/v4/pkg/request"
+	"github.com/ViBiOh/ketchup/pkg/model"
+	"github.com/ViBiOh/ketchup/pkg/semver"
+)
+
+const (
+	registryURL = "https://pypi.org/pypi"
+)
+
+type packageResp struct {
+	Versions map[string]interface{} `json:"releases"`
+}
+
+// App of package
+type App interface {
+	LatestVersions(string, []string) (map[string]semver.Version, error)
+}
+
+type app struct{}
+
+// New creates new App from Config
+func New() App {
+	return app{}
+}
+
+func (a app) LatestVersions(name string, patterns []string) (map[string]semver.Version, error) {
+	ctx := context.Background()
+
+	versions, compiledPatterns, err := model.PreparePatternMatching(patterns)
+	if err != nil {
+		return nil, fmt.Errorf("unable to prepare pattern matching: %s", err)
+	}
+
+	resp, err := request.New().Get(fmt.Sprintf("%s/%s/json", registryURL, name)).Send(ctx, nil)
+	if err != nil {
+		return nil, fmt.Errorf("unable to fetch registry: %s", err)
+	}
+
+	var content packageResp
+	if err := httpjson.Read(resp, &content); err != nil {
+		return nil, fmt.Errorf("unable to read versions: %s", err)
+	}
+
+	for version := range content.Versions {
+		tagVersion, err := semver.Parse(version)
+		if err != nil {
+			continue
+		}
+
+		model.CheckPatternsMatching(versions, compiledPatterns, tagVersion)
+	}
+
+	return versions, nil
+}

pkg/service/repository/repository.go

@@ -13,6 +13,7 @@ import (
 	"github.com/ViBiOh/ketchup/pkg/provider/github"
 	"github.com/ViBiOh/ketchup/pkg/provider/helm"
 	"github.com/ViBiOh/ketchup/pkg/provider/npm"
+	"github.com/ViBiOh/ketchup/pkg/provider/pypi"
 	"github.com/ViBiOh/ketchup/pkg/semver"
 	"github.com/ViBiOh/ketchup/pkg/store/repository"
 )
@@ -38,16 +39,18 @@ type app struct {
 	helmApp         helm.App
 	dockerApp       docker.App
 	npmApp          npm.App
+	pypiApp         pypi.App
 }
 
 // New creates new App from Config
-func New(repositoryStore repository.App, githubApp github.App, helmApp helm.App, dockerApp docker.App, npmApp npm.App) App {
+func New(repositoryStore repository.App, githubApp github.App, helmApp helm.App, dockerApp docker.App, npmApp npm.App, pypiApp pypi.App) App {
 	return app{
 		repositoryStore: repositoryStore,
 		githubApp:       githubApp,
 		helmApp:         helmApp,
 		dockerApp:       dockerApp,
 		npmApp:          npmApp,
+		pypiApp:         pypiApp,
 	}
 }
 
@@ -233,6 +236,8 @@ func (a app) LatestVersions(repo model.Repository) (map[string]semver.Version, e
 		return a.dockerApp.LatestVersions(repo.Name, patterns)
 	case model.NPM:
 		return a.npmApp.LatestVersions(repo.Name, patterns)
+	case model.Pypi:
+		return a.pypiApp.LatestVersions(repo.Name, patterns)
 	default:
 		return nil, fmt.Errorf("unknown repository kind %d", repo.Kind)
 	}