From 348b3bbbd68b27f55f35e6918a35a4bd681af540 Mon Sep 17 00:00:00 2001 From: chgl Date: Mon, 11 Jan 2021 15:16:33 +0100 Subject: [PATCH 01/52] fix: Update stable Helm repo URL (#8251) --- docs/usage/configuration-options.md | 2 +- lib/datasource/helm/__fixtures__/index.yaml | 178 +++++++++--------- lib/datasource/helm/index.ts | 4 +- .../__fixtures__/helmRequirements.yml | 6 +- lib/manager/helm-requirements/index.ts | 2 +- .../__snapshots__/extract.spec.ts.snap | 6 +- lib/manager/helmfile/extract.spec.ts | 16 +- lib/manager/helmfile/index.ts | 2 +- .../helmv3/__snapshots__/extract.spec.ts.snap | 6 +- lib/manager/helmv3/extract.spec.ts | 26 +-- lib/manager/helmv3/index.ts | 2 +- .../regex/__snapshots__/index.spec.ts.snap | 4 +- lib/manager/regex/index.spec.ts | 2 +- lib/manager/terraform/__fixtures__/helm.tf | 8 +- .../__snapshots__/extract.spec.ts.snap | 8 +- 15 files changed, 135 insertions(+), 137 deletions(-) diff --git a/docs/usage/configuration-options.md b/docs/usage/configuration-options.md index c10ce7dec5ec7c..efe77cf0d372db 100644 --- a/docs/usage/configuration-options.md +++ b/docs/usage/configuration-options.md @@ -85,7 +85,7 @@ Currently it is needed/supported for the `helm-requirements` manager only. ```json { "aliases": { - "stable": "https://kubernetes-charts.storage.googleapis.com/" + "stable": "https://charts.helm.sh/stable" } } ``` diff --git a/lib/datasource/helm/__fixtures__/index.yaml b/lib/datasource/helm/__fixtures__/index.yaml index c894cecfdd5f4f..b4240618e0099e 100644 --- a/lib/datasource/helm/__fixtures__/index.yaml +++ b/lib/datasource/helm/__fixtures__/index.yaml @@ -13,7 +13,7 @@ entries: sources: - https://github.com/wbuchwalter/Kubernetes-acs-engine-autoscaler urls: - - https://kubernetes-charts.storage.googleapis.com/acs-engine-autoscaler-2.2.2.tgz + - https://charts.helm.sh/stable/packages/acs-engine-autoscaler-2.2.2.tgz version: 2.2.2 - apiVersion: v1 appVersion: 2.1.1 @@ -31,7 +31,7 @@ entries: sources: - https://github.com/wbuchwalter/Kubernetes-acs-engine-autoscaler urls: - - https://kubernetes-charts.storage.googleapis.com/acs-engine-autoscaler-2.2.1.tgz + - https://charts.helm.sh/stable/packages/acs-engine-autoscaler-2.2.1.tgz version: 2.2.1 - apiVersion: v1 appVersion: 2.1.1 @@ -49,7 +49,7 @@ entries: sources: - https://github.com/wbuchwalter/Kubernetes-acs-engine-autoscaler urls: - - https://kubernetes-charts.storage.googleapis.com/acs-engine-autoscaler-2.2.0.tgz + - https://charts.helm.sh/stable/packages/acs-engine-autoscaler-2.2.0.tgz version: 2.2.0 - apiVersion: v1 appVersion: 2.1.1 @@ -67,7 +67,7 @@ entries: sources: - https://github.com/wbuchwalter/Kubernetes-acs-engine-autoscaler urls: - - https://kubernetes-charts.storage.googleapis.com/acs-engine-autoscaler-2.1.5.tgz + - https://charts.helm.sh/stable/packages/acs-engine-autoscaler-2.1.5.tgz version: 2.1.5 - apiVersion: v1 appVersion: 2.1.1 @@ -85,7 +85,7 @@ entries: sources: - https://github.com/wbuchwalter/Kubernetes-acs-engine-autoscaler urls: - - https://kubernetes-charts.storage.googleapis.com/acs-engine-autoscaler-2.1.4.tgz + - https://charts.helm.sh/stable/packages/acs-engine-autoscaler-2.1.4.tgz version: 2.1.4 - apiVersion: v1 appVersion: 2.1.1 @@ -102,7 +102,7 @@ entries: sources: - https://github.com/wbuchwalter/Kubernetes-acs-engine-autoscaler urls: - - https://kubernetes-charts.storage.googleapis.com/acs-engine-autoscaler-2.1.3.tgz + - https://charts.helm.sh/stable/packages/acs-engine-autoscaler-2.1.3.tgz version: 2.1.3 - apiVersion: v1 appVersion: 2.1.1 @@ -119,7 +119,7 @@ entries: sources: - https://github.com/wbuchwalter/Kubernetes-acs-engine-autoscaler urls: - - https://kubernetes-charts.storage.googleapis.com/acs-engine-autoscaler-2.1.2.tgz + - https://charts.helm.sh/stable/packages/acs-engine-autoscaler-2.1.2.tgz version: 2.1.2 - apiVersion: v1 appVersion: 2.1.1 @@ -136,7 +136,7 @@ entries: sources: - https://github.com/wbuchwalter/Kubernetes-acs-engine-autoscaler urls: - - https://kubernetes-charts.storage.googleapis.com/acs-engine-autoscaler-2.1.1.tgz + - https://charts.helm.sh/stable/packages/acs-engine-autoscaler-2.1.1.tgz version: 2.1.1 - apiVersion: v1 appVersion: 2.1.1 @@ -153,7 +153,7 @@ entries: sources: - https://github.com/wbuchwalter/Kubernetes-acs-engine-autoscaler urls: - - https://kubernetes-charts.storage.googleapis.com/acs-engine-autoscaler-2.1.0.tgz + - https://charts.helm.sh/stable/packages/acs-engine-autoscaler-2.1.0.tgz version: 2.1.0 - apiVersion: v1 created: 2017-10-09T19:34:08.355467969Z @@ -169,7 +169,7 @@ entries: sources: - https://github.com/wbuchwalter/Kubernetes-acs-engine-autoscaler urls: - - https://kubernetes-charts.storage.googleapis.com/acs-engine-autoscaler-2.0.0.tgz + - https://charts.helm.sh/stable/packages/acs-engine-autoscaler-2.0.0.tgz version: 2.0.0 - apiVersion: v1 created: 2017-09-10T16:34:04.515113086Z @@ -185,7 +185,7 @@ entries: sources: - https://github.com/wbuchwalter/Kubernetes-acs-engine-autoscaler urls: - - https://kubernetes-charts.storage.googleapis.com/acs-engine-autoscaler-1.0.0.tgz + - https://charts.helm.sh/stable/packages/acs-engine-autoscaler-1.0.0.tgz version: 1.0.0 - apiVersion: v1 created: 2017-08-02T01:33:51.76928059Z @@ -199,7 +199,7 @@ entries: sources: - https://github.com/wbuchwalter/Kubernetes-acs-engine-autoscaler urls: - - https://kubernetes-charts.storage.googleapis.com/acs-engine-autoscaler-0.1.0.tgz + - https://charts.helm.sh/stable/packages/acs-engine-autoscaler-0.1.0.tgz version: 0.1.0 aerospike: - apiVersion: v1 @@ -221,7 +221,7 @@ entries: sources: - https://github.com/aerospike/aerospike-server urls: - - https://kubernetes-charts.storage.googleapis.com/aerospike-0.2.8.tgz + - https://charts.helm.sh/stable/packages/aerospike-0.2.8.tgz version: 0.2.8 - apiVersion: v1 appVersion: v4.5.0.5 @@ -242,7 +242,7 @@ entries: sources: - https://github.com/aerospike/aerospike-server urls: - - https://kubernetes-charts.storage.googleapis.com/aerospike-0.2.7.tgz + - https://charts.helm.sh/stable/packages/aerospike-0.2.7.tgz version: 0.2.7 - apiVersion: v1 appVersion: v4.5.0.5 @@ -263,7 +263,7 @@ entries: sources: - https://github.com/aerospike/aerospike-server urls: - - https://kubernetes-charts.storage.googleapis.com/aerospike-0.2.6.tgz + - https://charts.helm.sh/stable/packages/aerospike-0.2.6.tgz version: 0.2.6 - apiVersion: v1 appVersion: v4.5.0.5 @@ -282,7 +282,7 @@ entries: sources: - https://github.com/aerospike/aerospike-server urls: - - https://kubernetes-charts.storage.googleapis.com/aerospike-0.2.5.tgz + - https://charts.helm.sh/stable/packages/aerospike-0.2.5.tgz version: 0.2.5 - appVersion: v4.5.0.5 created: 2019-05-14T19:56:18.413729387Z @@ -300,7 +300,7 @@ entries: sources: - https://github.com/aerospike/aerospike-server urls: - - https://kubernetes-charts.storage.googleapis.com/aerospike-0.2.4.tgz + - https://charts.helm.sh/stable/packages/aerospike-0.2.4.tgz version: 0.2.4 - appVersion: v4.5.0.5 created: 2019-05-06T20:56:39.728865535Z @@ -318,7 +318,7 @@ entries: sources: - https://github.com/aerospike/aerospike-server urls: - - https://kubernetes-charts.storage.googleapis.com/aerospike-0.2.3.tgz + - https://charts.helm.sh/stable/packages/aerospike-0.2.3.tgz version: 0.2.3 - appVersion: v3.14.1.2 created: 2019-03-18T09:29:12.447335953Z @@ -336,7 +336,7 @@ entries: sources: - https://github.com/aerospike/aerospike-server urls: - - https://kubernetes-charts.storage.googleapis.com/aerospike-0.2.1.tgz + - https://charts.helm.sh/stable/packages/aerospike-0.2.1.tgz version: 0.2.1 - appVersion: v3.14.1.2 created: 2018-12-17T20:55:58.28236604Z @@ -354,7 +354,7 @@ entries: sources: - https://github.com/aerospike/aerospike-server urls: - - https://kubernetes-charts.storage.googleapis.com/aerospike-0.2.0.tgz + - https://charts.helm.sh/stable/packages/aerospike-0.2.0.tgz version: 0.2.0 - appVersion: v3.14.1.2 created: 2018-12-17T18:55:34.258699519Z @@ -372,7 +372,7 @@ entries: sources: - https://github.com/aerospike/aerospike-server urls: - - https://kubernetes-charts.storage.googleapis.com/aerospike-0.1.7.tgz + - https://charts.helm.sh/stable/packages/aerospike-0.1.7.tgz version: 0.1.7 - appVersion: v3.14.1.2 created: 2018-02-25T19:49:20.935510612Z @@ -390,7 +390,7 @@ entries: sources: - https://github.com/aerospike/aerospike-server urls: - - https://kubernetes-charts.storage.googleapis.com/aerospike-0.1.6.tgz + - https://charts.helm.sh/stable/packages/aerospike-0.1.6.tgz version: 0.1.6 - appVersion: v3.14.1.2 created: 2018-01-24T15:49:08.893862195Z @@ -408,7 +408,7 @@ entries: sources: - https://github.com/aerospike/aerospike-server urls: - - https://kubernetes-charts.storage.googleapis.com/aerospike-0.1.5.tgz + - https://charts.helm.sh/stable/packages/aerospike-0.1.5.tgz version: 0.1.5 airflow: - apiVersion: v1 @@ -429,7 +429,7 @@ entries: sources: - https://airflow.apache.org/ urls: - - https://kubernetes-charts.storage.googleapis.com/airflow-2.8.6.tgz + - https://charts.helm.sh/stable/packages/airflow-2.8.6.tgz version: 2.8.6 - apiVersion: v1 appVersion: 1.10.2 @@ -449,7 +449,7 @@ entries: sources: - https://airflow.apache.org/ urls: - - https://kubernetes-charts.storage.googleapis.com/airflow-2.8.5.tgz + - https://charts.helm.sh/stable/packages/airflow-2.8.5.tgz version: 2.8.5 - apiVersion: v1 appVersion: 1.10.2 @@ -469,7 +469,7 @@ entries: sources: - https://airflow.apache.org/ urls: - - https://kubernetes-charts.storage.googleapis.com/airflow-2.8.4.tgz + - https://charts.helm.sh/stable/packages/airflow-2.8.4.tgz version: 2.8.4 - apiVersion: v1 appVersion: 1.10.2 @@ -489,7 +489,7 @@ entries: sources: - https://airflow.apache.org/ urls: - - https://kubernetes-charts.storage.googleapis.com/airflow-2.8.3.tgz + - https://charts.helm.sh/stable/packages/airflow-2.8.3.tgz version: 2.8.3 - apiVersion: v1 appVersion: 1.10.2 @@ -509,7 +509,7 @@ entries: sources: - https://airflow.apache.org/ urls: - - https://kubernetes-charts.storage.googleapis.com/airflow-2.8.2.tgz + - https://charts.helm.sh/stable/packages/airflow-2.8.2.tgz version: 2.8.2 - appVersion: 1.10.2 created: 2019-05-14T20:56:18.316517075Z @@ -528,7 +528,7 @@ entries: sources: - https://airflow.apache.org/ urls: - - https://kubernetes-charts.storage.googleapis.com/airflow-2.8.1.tgz + - https://charts.helm.sh/stable/packages/airflow-2.8.1.tgz version: 2.8.1 - appVersion: 1.10.2 created: 2019-05-10T14:26:52.105929603Z @@ -547,7 +547,7 @@ entries: sources: - https://airflow.apache.org/ urls: - - https://kubernetes-charts.storage.googleapis.com/airflow-2.8.0.tgz + - https://charts.helm.sh/stable/packages/airflow-2.8.0.tgz version: 2.8.0 - appVersion: 1.10.2 created: 2019-05-09T20:57:10.982943956Z @@ -566,7 +566,7 @@ entries: sources: - https://airflow.apache.org/ urls: - - https://kubernetes-charts.storage.googleapis.com/airflow-2.7.1.tgz + - https://charts.helm.sh/stable/packages/airflow-2.7.1.tgz version: 2.7.1 - appVersion: 1.10.2 created: 2019-05-02T12:57:23.336007074Z @@ -585,7 +585,7 @@ entries: sources: - https://airflow.apache.org/ urls: - - https://kubernetes-charts.storage.googleapis.com/airflow-2.7.0.tgz + - https://charts.helm.sh/stable/packages/airflow-2.7.0.tgz version: 2.7.0 - appVersion: 1.10.2 created: 2019-05-01T15:00:20.703005999Z @@ -604,7 +604,7 @@ entries: sources: - https://airflow.apache.org/ urls: - - https://kubernetes-charts.storage.googleapis.com/airflow-2.6.0.tgz + - https://charts.helm.sh/stable/packages/airflow-2.6.0.tgz version: 2.6.0 - appVersion: 1.10.2 created: 2019-04-27T13:29:23.158449514Z @@ -623,7 +623,7 @@ entries: sources: - https://airflow.apache.org/ urls: - - https://kubernetes-charts.storage.googleapis.com/airflow-2.5.0.tgz + - https://charts.helm.sh/stable/packages/airflow-2.5.0.tgz version: 2.5.0 - appVersion: 1.10.0 created: 2019-04-26T15:27:19.503529711Z @@ -642,7 +642,7 @@ entries: sources: - https://airflow.apache.org/ urls: - - https://kubernetes-charts.storage.googleapis.com/airflow-2.4.4.tgz + - https://charts.helm.sh/stable/packages/airflow-2.4.4.tgz version: 2.4.4 - appVersion: 1.10.0 created: 2019-04-06T19:56:20.369848811Z @@ -661,7 +661,7 @@ entries: sources: - https://airflow.apache.org/ urls: - - https://kubernetes-charts.storage.googleapis.com/airflow-2.4.3.tgz + - https://charts.helm.sh/stable/packages/airflow-2.4.3.tgz version: 2.4.3 - appVersion: 1.10.0 created: 2019-04-03T17:56:25.12245166Z @@ -680,7 +680,7 @@ entries: sources: - https://airflow.apache.org/ urls: - - https://kubernetes-charts.storage.googleapis.com/airflow-2.4.2.tgz + - https://charts.helm.sh/stable/packages/airflow-2.4.2.tgz version: 2.4.2 - appVersion: 1.10.0 created: 2019-04-03T15:26:00.331861518Z @@ -699,7 +699,7 @@ entries: sources: - https://airflow.apache.org/ urls: - - https://kubernetes-charts.storage.googleapis.com/airflow-2.4.1.tgz + - https://charts.helm.sh/stable/packages/airflow-2.4.1.tgz version: 2.4.1 - appVersion: 1.10.0 created: 2019-04-03T09:26:11.517169425Z @@ -718,7 +718,7 @@ entries: sources: - https://airflow.apache.org/ urls: - - https://kubernetes-charts.storage.googleapis.com/airflow-2.4.0.tgz + - https://charts.helm.sh/stable/packages/airflow-2.4.0.tgz version: 2.4.0 - appVersion: 1.10.0 created: 2019-03-30T07:56:19.59176903Z @@ -737,7 +737,7 @@ entries: sources: - https://airflow.apache.org/ urls: - - https://kubernetes-charts.storage.googleapis.com/airflow-2.3.0.tgz + - https://charts.helm.sh/stable/packages/airflow-2.3.0.tgz version: 2.3.0 - appVersion: 1.10.0 created: 2019-03-15T13:56:12.86312058Z @@ -756,7 +756,7 @@ entries: sources: - https://airflow.apache.org/ urls: - - https://kubernetes-charts.storage.googleapis.com/airflow-2.2.1.tgz + - https://charts.helm.sh/stable/packages/airflow-2.2.1.tgz version: 2.2.1 - appVersion: 1.10.0 created: 2019-03-14T13:55:58.320346594Z @@ -775,7 +775,7 @@ entries: sources: - https://airflow.apache.org/ urls: - - https://kubernetes-charts.storage.googleapis.com/airflow-2.2.0.tgz + - https://charts.helm.sh/stable/packages/airflow-2.2.0.tgz version: 2.2.0 - appVersion: 1.10.0 created: 2019-03-14T00:55:53.375781592Z @@ -794,7 +794,7 @@ entries: sources: - https://airflow.apache.org/ urls: - - https://kubernetes-charts.storage.googleapis.com/airflow-2.1.0.tgz + - https://charts.helm.sh/stable/packages/airflow-2.1.0.tgz version: 2.1.0 - appVersion: 1.10.0 created: 2019-02-27T15:56:28.248164679Z @@ -813,7 +813,7 @@ entries: sources: - https://airflow.apache.org/ urls: - - https://kubernetes-charts.storage.googleapis.com/airflow-2.0.2.tgz + - https://charts.helm.sh/stable/packages/airflow-2.0.2.tgz version: 2.0.2 - appVersion: 1.10.0 created: 2019-02-26T23:25:43.766793946Z @@ -832,7 +832,7 @@ entries: sources: - https://airflow.apache.org/ urls: - - https://kubernetes-charts.storage.googleapis.com/airflow-2.0.1.tgz + - https://charts.helm.sh/stable/packages/airflow-2.0.1.tgz version: 2.0.1 - appVersion: 1.10.0 created: 2019-02-25T14:56:12.675816282Z @@ -851,7 +851,7 @@ entries: sources: - https://airflow.apache.org/ urls: - - https://kubernetes-charts.storage.googleapis.com/airflow-2.0.0.tgz + - https://charts.helm.sh/stable/packages/airflow-2.0.0.tgz version: 2.0.0 - appVersion: 1.10.0 created: 2019-02-24T20:56:29.55445913Z @@ -870,7 +870,7 @@ entries: sources: - https://airflow.apache.org/ urls: - - https://kubernetes-charts.storage.googleapis.com/airflow-1.0.1.tgz + - https://charts.helm.sh/stable/packages/airflow-1.0.1.tgz version: 1.0.1 - appVersion: 1.10.0 created: 2019-02-23T19:55:57.905294125Z @@ -889,7 +889,7 @@ entries: sources: - https://airflow.apache.org/ urls: - - https://kubernetes-charts.storage.googleapis.com/airflow-1.0.0.tgz + - https://charts.helm.sh/stable/packages/airflow-1.0.0.tgz version: 1.0.0 - appVersion: 1.10.0 created: 2019-02-22T16:28:27.39780529Z @@ -908,7 +908,7 @@ entries: sources: - https://airflow.apache.org/ urls: - - https://kubernetes-charts.storage.googleapis.com/airflow-0.17.4.tgz + - https://charts.helm.sh/stable/packages/airflow-0.17.4.tgz version: 0.17.4 - appVersion: 1.10.0 created: 2019-02-18T16:25:43.319764468Z @@ -927,7 +927,7 @@ entries: sources: - https://airflow.apache.org/ urls: - - https://kubernetes-charts.storage.googleapis.com/airflow-0.17.3.tgz + - https://charts.helm.sh/stable/packages/airflow-0.17.3.tgz version: 0.17.3 - appVersion: 1.10.0 created: 2019-02-14T17:25:50.814935714Z @@ -946,7 +946,7 @@ entries: sources: - https://airflow.apache.org/ urls: - - https://kubernetes-charts.storage.googleapis.com/airflow-0.17.2.tgz + - https://charts.helm.sh/stable/packages/airflow-0.17.2.tgz version: 0.17.2 - appVersion: 1.10.0 created: 2019-02-12T16:55:32.3672359Z @@ -965,7 +965,7 @@ entries: sources: - https://airflow.apache.org/ urls: - - https://kubernetes-charts.storage.googleapis.com/airflow-0.17.1.tgz + - https://charts.helm.sh/stable/packages/airflow-0.17.1.tgz version: 0.17.1 - appVersion: 1.10.0 created: 2019-02-12T16:26:17.092387253Z @@ -984,7 +984,7 @@ entries: sources: - https://airflow.apache.org/ urls: - - https://kubernetes-charts.storage.googleapis.com/airflow-0.17.0.tgz + - https://charts.helm.sh/stable/packages/airflow-0.17.0.tgz version: 0.17.0 - appVersion: 1.10.0 created: 2019-02-11T20:26:21.078586168Z @@ -1003,7 +1003,7 @@ entries: sources: - https://airflow.apache.org/ urls: - - https://kubernetes-charts.storage.googleapis.com/airflow-0.16.0.tgz + - https://charts.helm.sh/stable/packages/airflow-0.16.0.tgz version: 0.16.0 - appVersion: 1.10.0 created: 2019-02-09T16:56:03.198944399Z @@ -1022,7 +1022,7 @@ entries: sources: - https://airflow.apache.org/ urls: - - https://kubernetes-charts.storage.googleapis.com/airflow-0.15.0.tgz + - https://charts.helm.sh/stable/packages/airflow-0.15.0.tgz version: 0.15.0 - appVersion: 1.10.0 created: 2019-01-30T16:25:43.262863813Z @@ -1041,7 +1041,7 @@ entries: sources: - https://airflow.apache.org/ urls: - - https://kubernetes-charts.storage.googleapis.com/airflow-0.14.0.tgz + - https://charts.helm.sh/stable/packages/airflow-0.14.0.tgz version: 0.14.0 - appVersion: 1.10.0 created: 2019-01-25T20:56:10.906599294Z @@ -1060,7 +1060,7 @@ entries: sources: - https://airflow.apache.org/ urls: - - https://kubernetes-charts.storage.googleapis.com/airflow-0.13.0.tgz + - https://charts.helm.sh/stable/packages/airflow-0.13.0.tgz version: 0.13.0 - appVersion: 1.10.0 created: 2019-01-15T20:25:33.538917073Z @@ -1079,7 +1079,7 @@ entries: sources: - https://airflow.apache.org/ urls: - - https://kubernetes-charts.storage.googleapis.com/airflow-0.12.0.tgz + - https://charts.helm.sh/stable/packages/airflow-0.12.0.tgz version: 0.12.0 - appVersion: 1.10.0 created: 2019-01-07T11:55:25.852726039Z @@ -1098,7 +1098,7 @@ entries: sources: - https://airflow.apache.org/ urls: - - https://kubernetes-charts.storage.googleapis.com/airflow-0.11.0.tgz + - https://charts.helm.sh/stable/packages/airflow-0.11.0.tgz version: 0.11.0 - appVersion: 1.10.0 created: 2019-01-03T17:26:21.38369729Z @@ -1117,7 +1117,7 @@ entries: sources: - https://airflow.apache.org/ urls: - - https://kubernetes-charts.storage.googleapis.com/airflow-0.10.0.tgz + - https://charts.helm.sh/stable/packages/airflow-0.10.0.tgz version: 0.10.0 - appVersion: 1.10.0 created: 2018-12-21T14:25:30.471733522Z @@ -1136,7 +1136,7 @@ entries: sources: - https://airflow.apache.org/ urls: - - https://kubernetes-charts.storage.googleapis.com/airflow-0.9.1.tgz + - https://charts.helm.sh/stable/packages/airflow-0.9.1.tgz version: 0.9.1 - appVersion: 1.10.0 created: 2018-12-11T12:55:33.837615422Z @@ -1155,7 +1155,7 @@ entries: sources: - https://airflow.apache.org/ urls: - - https://kubernetes-charts.storage.googleapis.com/airflow-0.9.0.tgz + - https://charts.helm.sh/stable/packages/airflow-0.9.0.tgz version: 0.9.0 ambassador: - apiVersion: v1 @@ -1183,7 +1183,7 @@ entries: - https://github.com/datawire/ambassador - https://github.com/prometheus/statsd_exporter urls: - - https://kubernetes-charts.storage.googleapis.com/ambassador-2.7.0.tgz + - https://charts.helm.sh/stable/packages/ambassador-2.7.0.tgz version: 2.7.0 - apiVersion: v1 appVersion: 0.70.1 @@ -1210,7 +1210,7 @@ entries: - https://github.com/datawire/ambassador - https://github.com/prometheus/statsd_exporter urls: - - https://kubernetes-charts.storage.googleapis.com/ambassador-2.6.2.tgz + - https://charts.helm.sh/stable/packages/ambassador-2.6.2.tgz version: 2.6.2 - apiVersion: v1 appVersion: 0.70.1 @@ -1237,7 +1237,7 @@ entries: - https://github.com/datawire/ambassador - https://github.com/prometheus/statsd_exporter urls: - - https://kubernetes-charts.storage.googleapis.com/ambassador-2.6.1.tgz + - https://charts.helm.sh/stable/packages/ambassador-2.6.1.tgz version: 2.6.1 - apiVersion: v1 appVersion: 0.70.0 @@ -1264,7 +1264,7 @@ entries: - https://github.com/datawire/ambassador - https://github.com/prometheus/statsd_exporter urls: - - https://kubernetes-charts.storage.googleapis.com/ambassador-2.6.0.tgz + - https://charts.helm.sh/stable/packages/ambassador-2.6.0.tgz version: 2.6.0 - apiVersion: v1 appVersion: 0.61.1 @@ -1291,7 +1291,7 @@ entries: - https://github.com/datawire/ambassador - https://github.com/prometheus/statsd_exporter urls: - - https://kubernetes-charts.storage.googleapis.com/ambassador-2.5.1.tgz + - https://charts.helm.sh/stable/packages/ambassador-2.5.1.tgz version: 2.5.1 - apiVersion: v1 appVersion: 0.61.0 @@ -1318,7 +1318,7 @@ entries: - https://github.com/datawire/ambassador - https://github.com/prometheus/statsd_exporter urls: - - https://kubernetes-charts.storage.googleapis.com/ambassador-2.5.0.tgz + - https://charts.helm.sh/stable/packages/ambassador-2.5.0.tgz version: 2.5.0 - apiVersion: v1 appVersion: 0.61.0 @@ -1345,7 +1345,7 @@ entries: - https://github.com/datawire/ambassador - https://github.com/prometheus/statsd_exporter urls: - - https://kubernetes-charts.storage.googleapis.com/ambassador-2.4.1.tgz + - https://charts.helm.sh/stable/packages/ambassador-2.4.1.tgz version: 2.4.1 - apiVersion: v1 appVersion: 0.60.3 @@ -1372,7 +1372,7 @@ entries: - https://github.com/datawire/ambassador - https://github.com/prometheus/statsd_exporter urls: - - https://kubernetes-charts.storage.googleapis.com/ambassador-2.4.0.tgz + - https://charts.helm.sh/stable/packages/ambassador-2.4.0.tgz version: 2.4.0 - apiVersion: v1 appVersion: 0.60.3 @@ -1399,7 +1399,7 @@ entries: - https://github.com/datawire/ambassador - https://github.com/prometheus/statsd_exporter urls: - - https://kubernetes-charts.storage.googleapis.com/ambassador-2.3.1.tgz + - https://charts.helm.sh/stable/packages/ambassador-2.3.1.tgz version: 2.3.1 - apiVersion: v1 appVersion: 0.60.3 @@ -1426,7 +1426,7 @@ entries: - https://github.com/datawire/ambassador - https://github.com/prometheus/statsd_exporter urls: - - https://kubernetes-charts.storage.googleapis.com/ambassador-2.3.0.tgz + - https://charts.helm.sh/stable/packages/ambassador-2.3.0.tgz version: 2.3.0 - apiVersion: v1 appVersion: 0.60.3 @@ -1453,7 +1453,7 @@ entries: - https://github.com/datawire/ambassador - https://github.com/prometheus/statsd_exporter urls: - - https://kubernetes-charts.storage.googleapis.com/ambassador-2.2.5.tgz + - https://charts.helm.sh/stable/packages/ambassador-2.2.5.tgz version: 2.2.5 - apiVersion: v1 appVersion: 0.60.2 @@ -1480,7 +1480,7 @@ entries: - https://github.com/datawire/ambassador - https://github.com/prometheus/statsd_exporter urls: - - https://kubernetes-charts.storage.googleapis.com/ambassador-2.2.4.tgz + - https://charts.helm.sh/stable/packages/ambassador-2.2.4.tgz version: 2.2.4 - apiVersion: v1 appVersion: 0.60.2 @@ -1507,7 +1507,7 @@ entries: - https://github.com/datawire/ambassador - https://github.com/prometheus/statsd_exporter urls: - - https://kubernetes-charts.storage.googleapis.com/ambassador-2.2.3.tgz + - https://charts.helm.sh/stable/packages/ambassador-2.2.3.tgz version: 2.2.3 - apiVersion: v1 appVersion: 0.60.1 @@ -1534,7 +1534,7 @@ entries: - https://github.com/datawire/ambassador - https://github.com/prometheus/statsd_exporter urls: - - https://kubernetes-charts.storage.googleapis.com/ambassador-2.2.2.tgz + - https://charts.helm.sh/stable/packages/ambassador-2.2.2.tgz version: 2.2.2 - apiVersion: v1 appVersion: 0.60.0 @@ -1561,7 +1561,7 @@ entries: - https://github.com/datawire/ambassador - https://github.com/prometheus/statsd_exporter urls: - - https://kubernetes-charts.storage.googleapis.com/ambassador-2.2.1.tgz + - https://charts.helm.sh/stable/packages/ambassador-2.2.1.tgz version: 2.2.1 - apiVersion: v1 appVersion: 0.60.0 @@ -1588,7 +1588,7 @@ entries: - https://github.com/datawire/ambassador - https://github.com/prometheus/statsd_exporter urls: - - https://kubernetes-charts.storage.googleapis.com/ambassador-2.2.0.tgz + - https://charts.helm.sh/stable/packages/ambassador-2.2.0.tgz version: 2.2.0 - apiVersion: v1 appVersion: 0.53.1 @@ -1615,7 +1615,7 @@ entries: - https://github.com/datawire/ambassador - https://github.com/prometheus/statsd_exporter urls: - - https://kubernetes-charts.storage.googleapis.com/ambassador-2.1.0.tgz + - https://charts.helm.sh/stable/packages/ambassador-2.1.0.tgz version: 2.1.0 - apiVersion: v1 appVersion: 0.53.1 @@ -1642,7 +1642,7 @@ entries: - https://github.com/datawire/ambassador - https://github.com/prometheus/statsd_exporter urls: - - https://kubernetes-charts.storage.googleapis.com/ambassador-2.0.2.tgz + - https://charts.helm.sh/stable/packages/ambassador-2.0.2.tgz version: 2.0.2 - apiVersion: v1 appVersion: 0.52.0 @@ -1669,7 +1669,7 @@ entries: - https://github.com/datawire/ambassador - https://github.com/prometheus/statsd_exporter urls: - - https://kubernetes-charts.storage.googleapis.com/ambassador-2.0.1.tgz + - https://charts.helm.sh/stable/packages/ambassador-2.0.1.tgz version: 2.0.1 - apiVersion: v1 appVersion: 0.51.2 @@ -1696,7 +1696,7 @@ entries: - https://github.com/datawire/ambassador - https://github.com/prometheus/statsd_exporter urls: - - https://kubernetes-charts.storage.googleapis.com/ambassador-2.0.0.tgz + - https://charts.helm.sh/stable/packages/ambassador-2.0.0.tgz version: 2.0.0 - apiVersion: v1 appVersion: 0.50.3 @@ -1723,7 +1723,7 @@ entries: - https://github.com/datawire/ambassador - https://github.com/prometheus/statsd_exporter urls: - - https://kubernetes-charts.storage.googleapis.com/ambassador-1.1.5.tgz + - https://charts.helm.sh/stable/packages/ambassador-1.1.5.tgz version: 1.1.5 - apiVersion: v1 appVersion: 0.50.2 @@ -1748,7 +1748,7 @@ entries: - https://github.com/datawire/ambassador - https://github.com/prometheus/statsd_exporter urls: - - https://kubernetes-charts.storage.googleapis.com/ambassador-1.1.4.tgz + - https://charts.helm.sh/stable/packages/ambassador-1.1.4.tgz version: 1.1.4 - apiVersion: v1 appVersion: 0.50.2 @@ -1773,7 +1773,7 @@ entries: - https://github.com/datawire/ambassador - https://github.com/prometheus/statsd_exporter urls: - - https://kubernetes-charts.storage.googleapis.com/ambassador-1.1.3.tgz + - https://charts.helm.sh/stable/packages/ambassador-1.1.3.tgz version: 1.1.3 - apiVersion: v1 appVersion: 0.50.1 @@ -1798,7 +1798,7 @@ entries: - https://github.com/datawire/ambassador - https://github.com/prometheus/statsd_exporter urls: - - https://kubernetes-charts.storage.googleapis.com/ambassador-1.1.2.tgz + - https://charts.helm.sh/stable/packages/ambassador-1.1.2.tgz version: 1.1.2 - apiVersion: v1 appVersion: 0.50.1 @@ -1823,7 +1823,7 @@ entries: - https://github.com/datawire/ambassador - https://github.com/prometheus/statsd_exporter urls: - - https://kubernetes-charts.storage.googleapis.com/ambassador-1.1.1.tgz + - https://charts.helm.sh/stable/packages/ambassador-1.1.1.tgz version: 1.1.1 - apiVersion: v1 appVersion: 0.50.1 @@ -1847,7 +1847,7 @@ entries: - https://github.com/datawire/ambassador - https://github.com/prometheus/statsd_exporter urls: - - https://kubernetes-charts.storage.googleapis.com/ambassador-1.1.0.tgz + - https://charts.helm.sh/stable/packages/ambassador-1.1.0.tgz version: 1.1.0 - apiVersion: v1 appVersion: 0.50.1 @@ -1871,5 +1871,5 @@ entries: - https://github.com/datawire/ambassador - https://github.com/prometheus/statsd_exporter urls: - - https://kubernetes-charts.storage.googleapis.com/ambassador-1.0.0.tgz + - https://charts.helm.sh/stable/packages/ambassador-1.0.0.tgz version: 1.0.0 diff --git a/lib/datasource/helm/index.ts b/lib/datasource/helm/index.ts index b15cac931db49f..8dae5828350a2f 100644 --- a/lib/datasource/helm/index.ts +++ b/lib/datasource/helm/index.ts @@ -12,9 +12,7 @@ export const id = 'helm'; const http = new Http(id); -export const defaultRegistryUrls = [ - 'https://kubernetes-charts.storage.googleapis.com/', -]; +export const defaultRegistryUrls = ['https://charts.helm.sh/stable']; export const registryStrategy = 'first'; export const defaultConfig = { diff --git a/lib/manager/ansible-galaxy/__fixtures__/helmRequirements.yml b/lib/manager/ansible-galaxy/__fixtures__/helmRequirements.yml index fadfba305018ea..47961a9e3c0471 100644 --- a/lib/manager/ansible-galaxy/__fixtures__/helmRequirements.yml +++ b/lib/manager/ansible-galaxy/__fixtures__/helmRequirements.yml @@ -1,13 +1,13 @@ dependencies: - name: etcd version: 0.6.2 - repository: https://kubernetes-charts-incubator.storage.googleapis.com/ + repository: https://charts.helm.sh/incubator condition: etcd.deployChart - name: zookeeper version: 1.0.0 - repository: https://kubernetes-charts-incubator.storage.googleapis.com/ + repository: https://charts.helm.sh/incubator condition: zookeeper.deployChart - name: consul version: 3.6.1 - repository: https://kubernetes-charts.storage.googleapis.com/ + repository: https://charts.helm.sh/stable condition: consul.deployChart diff --git a/lib/manager/helm-requirements/index.ts b/lib/manager/helm-requirements/index.ts index 27dc082a0b3783..e8cf05e2bff2da 100644 --- a/lib/manager/helm-requirements/index.ts +++ b/lib/manager/helm-requirements/index.ts @@ -2,7 +2,7 @@ export { extractPackageFile } from './extract'; export const defaultConfig = { aliases: { - stable: 'https://kubernetes-charts.storage.googleapis.com/', + stable: 'https://charts.helm.sh/stable', }, commitMessageTopic: 'helm chart {{depName}}', fileMatch: ['(^|/)requirements\\.yaml$'], diff --git a/lib/manager/helmfile/__snapshots__/extract.spec.ts.snap b/lib/manager/helmfile/__snapshots__/extract.spec.ts.snap index 5a093a75ea2afb..926ff4b6b78e0f 100644 --- a/lib/manager/helmfile/__snapshots__/extract.spec.ts.snap +++ b/lib/manager/helmfile/__snapshots__/extract.spec.ts.snap @@ -8,7 +8,7 @@ Object { "currentValue": undefined, "depName": "example", "registryUrls": Array [ - "https://kubernetes-charts.storage.googleapis.com/", + "https://charts.helm.sh/stable", ], "skipReason": "local-chart", }, @@ -88,7 +88,7 @@ Object { "currentValue": "1.0.0", "depName": "{{\`{{ .Release.Name }}\`}}", "registryUrls": Array [ - "https://kubernetes-charts.storage.googleapis.com/", + "https://charts.helm.sh/stable", ], "skipReason": "unsupported-chart-type", }, @@ -96,7 +96,7 @@ Object { "currentValue": "1.0.0", "depName": "example", "registryUrls": Array [ - "https://kubernetes-charts.storage.googleapis.com/", + "https://charts.helm.sh/stable", ], }, ], diff --git a/lib/manager/helmfile/extract.spec.ts b/lib/manager/helmfile/extract.spec.ts index e57d7bb80b162a..d27de155f0fe7f 100644 --- a/lib/manager/helmfile/extract.spec.ts +++ b/lib/manager/helmfile/extract.spec.ts @@ -15,7 +15,7 @@ describe('lib/manager/helmfile/extract', () => { const fileName = 'helmfile.yaml'; const result = extractPackageFile(content, fileName, { aliases: { - stable: 'https://kubernetes-charts.storage.googleapis.com/', + stable: 'https://charts.helm.sh/stable', }, }); expect(result).toBeNull(); @@ -32,7 +32,7 @@ describe('lib/manager/helmfile/extract', () => { const fileName = 'helmfile.yaml'; const result = extractPackageFile(content, fileName, { aliases: { - stable: 'https://kubernetes-charts.storage.googleapis.com/', + stable: 'https://charts.helm.sh/stable', }, }); expect(result).toBeNull(); @@ -51,7 +51,7 @@ describe('lib/manager/helmfile/extract', () => { const fileName = 'helmfile.yaml'; const result = extractPackageFile(content, fileName, { aliases: { - stable: 'https://kubernetes-charts.storage.googleapis.com/', + stable: 'https://charts.helm.sh/stable', }, }); expect(result).not.toBeNull(); @@ -75,7 +75,7 @@ describe('lib/manager/helmfile/extract', () => { const fileName = 'helmfile.yaml'; const result = extractPackageFile(content, fileName, { aliases: { - stable: 'https://kubernetes-charts.storage.googleapis.com/', + stable: 'https://charts.helm.sh/stable', }, }); expect(result).not.toBeNull(); @@ -95,7 +95,7 @@ describe('lib/manager/helmfile/extract', () => { const fileName = 'helmfile.yaml'; const result = extractPackageFile(content, fileName, { aliases: { - stable: 'https://kubernetes-charts.storage.googleapis.com/', + stable: 'https://charts.helm.sh/stable', }, }); expect(result).not.toBeNull(); @@ -116,7 +116,7 @@ describe('lib/manager/helmfile/extract', () => { const fileName = 'helmfile.yaml'; const result = extractPackageFile(content, fileName, { aliases: { - stable: 'https://kubernetes-charts.storage.googleapis.com/', + stable: 'https://charts.helm.sh/stable', }, }); expect(result).not.toBeNull(); @@ -140,7 +140,7 @@ describe('lib/manager/helmfile/extract', () => { const fileName = 'helmfile.yaml'; const result = extractPackageFile(content, fileName, { aliases: { - stable: 'https://kubernetes-charts.storage.googleapis.com/', + stable: 'https://charts.helm.sh/stable', }, }); expect(result).not.toBeNull(); @@ -160,7 +160,7 @@ describe('lib/manager/helmfile/extract', () => { const fileName = 'helmfile.yaml'; const result = extractPackageFile(content, fileName, { aliases: { - stable: 'https://kubernetes-charts.storage.googleapis.com/', + stable: 'https://charts.helm.sh/stable', }, }); expect(result).not.toBeNull(); diff --git a/lib/manager/helmfile/index.ts b/lib/manager/helmfile/index.ts index 090f5c15be18b5..d8d3e541962653 100644 --- a/lib/manager/helmfile/index.ts +++ b/lib/manager/helmfile/index.ts @@ -2,7 +2,7 @@ export { extractPackageFile } from './extract'; export const defaultConfig = { aliases: { - stable: 'https://kubernetes-charts.storage.googleapis.com/', + stable: 'https://charts.helm.sh/stable', }, commitMessageTopic: 'helm chart {{depName}}', fileMatch: ['(^|/)helmfile.yaml$'], diff --git a/lib/manager/helmv3/__snapshots__/extract.spec.ts.snap b/lib/manager/helmv3/__snapshots__/extract.spec.ts.snap index 460d146a0ab52d..424bf34f108908 100644 --- a/lib/manager/helmv3/__snapshots__/extract.spec.ts.snap +++ b/lib/manager/helmv3/__snapshots__/extract.spec.ts.snap @@ -8,14 +8,14 @@ Object { "currentValue": "0.9.0", "depName": "redis", "registryUrls": Array [ - "https://kubernetes-charts.storage.googleapis.com/", + "https://charts.helm.sh/stable", ], }, Object { "currentValue": "0.8.1", "depName": "postgresql", "registryUrls": Array [ - "https://kubernetes-charts.storage.googleapis.com/", + "https://charts.helm.sh/stable", ], }, ], @@ -77,7 +77,7 @@ Object { "currentValue": "0.9.0", "depName": "redis", "registryUrls": Array [ - "https://kubernetes-charts.storage.googleapis.com/", + "https://charts.helm.sh/stable", ], }, Object { diff --git a/lib/manager/helmv3/extract.spec.ts b/lib/manager/helmv3/extract.spec.ts index 8a1185a4f74a66..769f61408cb93e 100644 --- a/lib/manager/helmv3/extract.spec.ts +++ b/lib/manager/helmv3/extract.spec.ts @@ -29,7 +29,7 @@ describe('lib/manager/helm-requirements/extract', () => { const fileName = 'Chart.yaml'; const result = extractPackageFile(content, fileName, { aliases: { - stable: 'https://kubernetes-charts.storage.googleapis.com/', + stable: 'https://charts.helm.sh/stable', }, }); expect(result).not.toBeNull(); @@ -46,17 +46,17 @@ describe('lib/manager/helm-requirements/extract', () => { dependencies: - name: redis version: 0.9.0 - repository: https://kubernetes-charts.storage.googleapis.com/ + repository: https://charts.helm.sh/stable enabled: true - name: postgresql version: 0.8.1 - repository: https://kubernetes-charts.storage.googleapis.com/ + repository: https://charts.helm.sh/stable condition: postgresql.enabled `; const fileName = 'Chart.yaml'; const result = extractPackageFile(content, fileName, { aliases: { - stable: 'https://kubernetes-charts.storage.googleapis.com/', + stable: 'https://charts.helm.sh/stable', }, }); expect(result).not.toBeNull(); @@ -93,7 +93,7 @@ describe('lib/manager/helm-requirements/extract', () => { const fileName = 'Chart.yaml'; const result = extractPackageFile(content, fileName, { aliases: { - stable: 'https://kubernetes-charts.storage.googleapis.com/', + stable: 'https://charts.helm.sh/stable', }, }); expect(result).toBeNull(); @@ -108,7 +108,7 @@ describe('lib/manager/helm-requirements/extract', () => { dependencies: - name: redis version: 0.9.0 - repository: https://kubernetes-charts.storage.googleapis.com/ + repository: https://charts.helm.sh/stable - name: postgresql version: 0.8.1 repository: file:///some/local/path/ @@ -116,7 +116,7 @@ describe('lib/manager/helm-requirements/extract', () => { const fileName = 'Chart.yaml'; const result = extractPackageFile(content, fileName, { aliases: { - stable: 'https://kubernetes-charts.storage.googleapis.com/', + stable: 'https://charts.helm.sh/stable', }, }); expect(result).not.toBeNull(); @@ -136,7 +136,7 @@ describe('lib/manager/helm-requirements/extract', () => { const fileName = 'Chart.yaml'; const result = extractPackageFile(content, fileName, { aliases: { - stable: 'https://kubernetes-charts.storage.googleapis.com/', + stable: 'https://charts.helm.sh/stable', }, }); expect(result).toBeNull(); @@ -155,7 +155,7 @@ describe('lib/manager/helm-requirements/extract', () => { const fileName = 'Chart.yaml'; const result = extractPackageFile(content, fileName, { aliases: { - stable: 'https://kubernetes-charts.storage.googleapis.com/', + stable: 'https://charts.helm.sh/stable', }, }); expect(result).toBeNull(); @@ -174,7 +174,7 @@ describe('lib/manager/helm-requirements/extract', () => { const fileName = 'Chart.yaml'; const result = extractPackageFile(content, fileName, { aliases: { - stable: 'https://kubernetes-charts.storage.googleapis.com/', + stable: 'https://charts.helm.sh/stable', }, }); expect(result).toBeNull(); @@ -184,7 +184,7 @@ describe('lib/manager/helm-requirements/extract', () => { const fileName = 'Chart.yaml'; const result = extractPackageFile(content, fileName, { aliases: { - stable: 'https://kubernetes-charts.storage.googleapis.com/', + stable: 'https://charts.helm.sh/stable', }, }); expect(result).toBeNull(); @@ -200,7 +200,7 @@ describe('lib/manager/helm-requirements/extract', () => { const fileName = 'Chart.yaml'; const result = extractPackageFile(content, fileName, { aliases: { - stable: 'https://kubernetes-charts.storage.googleapis.com/', + stable: 'https://charts.helm.sh/stable', }, }); expect(result).toBeNull(); @@ -220,7 +220,7 @@ describe('lib/manager/helm-requirements/extract', () => { const fileName = 'Chart.yaml'; const result = extractPackageFile(content, fileName, { aliases: { - stable: 'https://kubernetes-charts.storage.googleapis.com/', + stable: 'https://charts.helm.sh/stable', }, }); expect(result).toBeNull(); diff --git a/lib/manager/helmv3/index.ts b/lib/manager/helmv3/index.ts index a2a9b0a841b3a1..21a0ccd14f9e75 100644 --- a/lib/manager/helmv3/index.ts +++ b/lib/manager/helmv3/index.ts @@ -6,7 +6,7 @@ export const supportsLockFileMaintenance = true; export const defaultConfig = { aliases: { - stable: 'https://kubernetes-charts.storage.googleapis.com/', + stable: 'https://charts.helm.sh/stable', }, commitMessageTopic: 'helm chart {{depName}}', fileMatch: ['(^|/)Chart.yaml$'], diff --git a/lib/manager/regex/__snapshots__/index.spec.ts.snap b/lib/manager/regex/__snapshots__/index.spec.ts.snap index 45d0d301e8079c..38786955f94a87 100644 --- a/lib/manager/regex/__snapshots__/index.spec.ts.snap +++ b/lib/manager/regex/__snapshots__/index.spec.ts.snap @@ -131,10 +131,10 @@ Object { "datasource": "helm", "depName": "prometheus-operator", "registryUrls": Array [ - "https://kubernetes-charts.storage.googleapis.com/", + "https://charts.helm.sh/stable", ], "replaceString": "chart: - repository: https://kubernetes-charts.storage.googleapis.com/ + repository: https://charts.helm.sh/stable name: prometheus-operator version: 8.12.13 ", diff --git a/lib/manager/regex/index.spec.ts b/lib/manager/regex/index.spec.ts index da6a424572ab0a..a6c47a4d032b3e 100644 --- a/lib/manager/regex/index.spec.ts +++ b/lib/manager/regex/index.spec.ts @@ -106,7 +106,7 @@ describe(getName(__filename), () => { spec: releaseName: prometheus-operator chart: - repository: https://kubernetes-charts.storage.googleapis.com/ + repository: https://charts.helm.sh/stable name: prometheus-operator version: 8.12.13 `, diff --git a/lib/manager/terraform/__fixtures__/helm.tf b/lib/manager/terraform/__fixtures__/helm.tf index 01d0dd84f7c28e..2173d06d1dc40c 100644 --- a/lib/manager/terraform/__fixtures__/helm.tf +++ b/lib/manager/terraform/__fixtures__/helm.tf @@ -2,7 +2,7 @@ ## complete example resource "helm_release" "example" { name = "my-redis-release" - repository = "https://kubernetes-charts.storage.googleapis.com" + repository = "https://charts.helm.sh/stable" chart = "redis" version = "1.0.1" } @@ -10,7 +10,7 @@ resource "helm_release" "example" { ## example without version, this will default to latest in Terraform resource "helm_release" "example" { name = "my-redis-release" - repository = "https://kubernetes-charts.storage.googleapis.com" + repository = "https://charts.helm.sh/stable" chart = "redis" } @@ -23,12 +23,12 @@ resource "helm_release" "local" { ## malformed examples resource "helm_release" "example" { name = "my-redis-release" - repository = "https://kubernetes-charts.storage.googleapis.com" + repository = "https://charts.helm.sh/stable" version = "4.0.1" } resource "helm_release" "example" { - repository = "https://kubernetes-charts.storage.googleapis.com" + repository = "https://charts.helm.sh/stable" chart = "redis" version = "5.0.1" } diff --git a/lib/manager/terraform/__snapshots__/extract.spec.ts.snap b/lib/manager/terraform/__snapshots__/extract.spec.ts.snap index 5436bdb0b46db3..fd1718a2e269c4 100644 --- a/lib/manager/terraform/__snapshots__/extract.spec.ts.snap +++ b/lib/manager/terraform/__snapshots__/extract.spec.ts.snap @@ -10,7 +10,7 @@ Object { "depNameShort": "redis", "depType": "helm", "registryUrls": Array [ - "https://kubernetes-charts.storage.googleapis.com", + "https://charts.helm.sh/stable", ], }, Object { @@ -19,7 +19,7 @@ Object { "depNameShort": "redis", "depType": "helm", "registryUrls": Array [ - "https://kubernetes-charts.storage.googleapis.com", + "https://charts.helm.sh/stable", ], }, Object { @@ -39,7 +39,7 @@ Object { "depNameShort": undefined, "depType": "helm", "registryUrls": Array [ - "https://kubernetes-charts.storage.googleapis.com", + "https://charts.helm.sh/stable", ], "skipReason": "invalid-name", }, @@ -50,7 +50,7 @@ Object { "depNameShort": "redis", "depType": "helm", "registryUrls": Array [ - "https://kubernetes-charts.storage.googleapis.com", + "https://charts.helm.sh/stable", ], }, Object { From 0fa01062f9a766fdc122e87f34bbc5af5646a9e0 Mon Sep 17 00:00:00 2001 From: Michael Kriese Date: Mon, 11 Jan 2021 15:17:11 +0100 Subject: [PATCH 02/52] fix(github-actions): escape forward slash in fileMatch (#8252) --- lib/manager/github-actions/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/manager/github-actions/index.ts b/lib/manager/github-actions/index.ts index f006b087260b56..1cb7cf9e930d02 100644 --- a/lib/manager/github-actions/index.ts +++ b/lib/manager/github-actions/index.ts @@ -6,5 +6,5 @@ const language = LANGUAGE_DOCKER; export { extractPackageFile, language }; export const defaultConfig = { - fileMatch: ['^\\.github/workflows/[^/]+\\.ya?ml$'], + fileMatch: ['^\\.github\\/workflows\\/[^/]+\\.ya?ml$'], }; From f886f30a1a2428e35df3b0e19030a7be22e566ad Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 11 Jan 2021 23:01:05 +0000 Subject: [PATCH 03/52] chore(deps): update dependency @types/node to v12.19.12 (#8255) Co-authored-by: Renovate Bot --- package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index b5e152ce89c424..ba8619ba74542e 100644 --- a/package.json +++ b/package.json @@ -219,7 +219,7 @@ "@types/markdown-table": "2.0.0", "@types/moo": "0.5.3", "@types/nock": "10.0.3", - "@types/node": "12.19.11", + "@types/node": "12.19.12", "@types/node-emoji": "1.8.1", "@types/parse-link-header": "1.0.0", "@types/registry-auth-token": "4.2.0", diff --git a/yarn.lock b/yarn.lock index e4101a7f4cca43..011d8d4f85842e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1772,10 +1772,10 @@ resolved "https://registry.yarnpkg.com/@types/node/-/node-14.14.20.tgz#f7974863edd21d1f8a494a73e8e2b3658615c340" integrity sha512-Y93R97Ouif9JEOWPIUyU+eyIdyRqQR0I8Ez1dzku4hDx34NWh4HbtIc3WNzwB1Y9ULvNGeu5B8h8bVL5cAk4/A== -"@types/node@12.19.11": - version "12.19.11" - resolved "https://registry.yarnpkg.com/@types/node/-/node-12.19.11.tgz#9220ab4b20d91169eb78f456dbfcbabee89dfb50" - integrity sha512-bwVfNTFZOrGXyiQ6t4B9sZerMSShWNsGRw8tC5DY1qImUNczS9SjT4G6PnzjCnxsu5Ubj6xjL2lgwddkxtQl5w== +"@types/node@12.19.12": + version "12.19.12" + resolved "https://registry.yarnpkg.com/@types/node/-/node-12.19.12.tgz#04793c2afa4ce833a9972e4c476432e30f9df47b" + integrity sha512-UwfL2uIU9arX/+/PRcIkT08/iBadGN2z6ExOROA2Dh5mAuWTBj6iJbQX4nekiV5H8cTrEG569LeX+HRco9Cbxw== "@types/node@^13.7.0": version "13.13.39" From fa3f1c134b66b2b8f106712c14a99f54addb5006 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 12 Jan 2021 02:01:10 +0000 Subject: [PATCH 04/52] build(deps): update dependency handy-redis to v2.2.1 (#8256) Co-authored-by: Renovate Bot --- package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index ba8619ba74542e..d1dec09bae9315 100644 --- a/package.json +++ b/package.json @@ -142,7 +142,7 @@ "global-agent": "2.1.12", "got": "11.8.1", "handlebars": "4.7.6", - "handy-redis": "2.1.0", + "handy-redis": "2.2.1", "hasha": "5.2.2", "ignore": "5.1.8", "ini": "1.3.8", diff --git a/yarn.lock b/yarn.lock index 011d8d4f85842e..ce958ea3c3c299 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4989,10 +4989,10 @@ handlebars@4.7.6, handlebars@^4.7.6: optionalDependencies: uglify-js "^3.1.4" -handy-redis@2.1.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/handy-redis/-/handy-redis-2.1.0.tgz#783596960d48c3a81db9996f98ae2db78838d961" - integrity sha512-XEv5k1/vCkvASfzOTNqW4Ft1+QrMbY274uB9KSpBLw1EtzVefXoO4SgdAlUddMHQd6ftv7KWJgr7kb5TXRMk5w== +handy-redis@2.2.1: + version "2.2.1" + resolved "https://registry.yarnpkg.com/handy-redis/-/handy-redis-2.2.1.tgz#33f4293dffba7e0c2b198c4a4ce7cd4a832b5ecc" + integrity sha512-Kmz9HhdAA9tGV1b1R6m3AJ8GcC+r/0uzyiTia39Q8tseJJc4kOwGGQr7cfPppD/xlO2ENE5IK+ToRahLuX1JVg== dependencies: "@types/redis" "^2.8.27" From 9a9a2c176be75767f013493626c803dd5ccfd22b Mon Sep 17 00:00:00 2001 From: Michael Kriese Date: Tue, 12 Jan 2021 07:21:05 +0100 Subject: [PATCH 05/52] feat(docker): add metadata (#8258) --- lib/datasource/metadata.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/datasource/metadata.ts b/lib/datasource/metadata.ts index 42943d9932fa4b..fbc243ef6cdf44 100644 --- a/lib/datasource/metadata.ts +++ b/lib/datasource/metadata.ts @@ -65,6 +65,7 @@ const manualSourceUrls = { 'https://github.com/hyper-expanse/library-release-workflows', }, docker: { + 'amd64/traefik': 'https://github.com/containous/traefik', 'docker/compose': 'https://github.com/docker/compose', 'drone/drone': 'https://github.com/drone/drone', 'drone/drone-runner-docker': From 08a187a8a1b241b4d094184bb0dbe7a65ee82d5e Mon Sep 17 00:00:00 2001 From: Rhys Arkins Date: Tue, 12 Jan 2021 09:26:00 +0100 Subject: [PATCH 06/52] =?UTF-8?q?fix(npm):=20don=E2=80=99t=20pick=20node?= =?UTF-8?q?=2015=20unnecessarily=20(#8254)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes #8253 --- .../npm/post-update/node-version.spec.ts | 13 ++++++++++++ lib/manager/npm/post-update/node-version.ts | 20 ++++++++++++++++--- 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/lib/manager/npm/post-update/node-version.spec.ts b/lib/manager/npm/post-update/node-version.spec.ts index 36702afb450e8d..9f045c63f1891e 100644 --- a/lib/manager/npm/post-update/node-version.spec.ts +++ b/lib/manager/npm/post-update/node-version.spec.ts @@ -1,4 +1,5 @@ import { fs } from '../../../../test/util'; +import { isStable } from '../../../versioning/node'; import { getNodeConstraint } from './node-version'; jest.mock('../../../util/fs'); @@ -15,6 +16,18 @@ describe('getNodeConstraint', () => { const res = await getNodeConstraint(config); expect(res).toEqual('^12.16.0'); }); + it('augments to avoid node 15', async () => { + fs.readLocalFile = jest.fn(); + fs.readLocalFile.mockResolvedValueOnce(null); + fs.readLocalFile.mockResolvedValueOnce(null); + const res = await getNodeConstraint({ + ...config, + constraints: { node: '>= 12.16.0' }, + }); + const isAugmentedRange = res === '>= 12.16.0 <15'; + const node16IsStable = isStable('16.100.0'); + expect(isAugmentedRange || node16IsStable).toBe(true); + }); it('returns .node-version value', async () => { fs.readLocalFile = jest.fn(); fs.readLocalFile.mockResolvedValueOnce(null); diff --git a/lib/manager/npm/post-update/node-version.ts b/lib/manager/npm/post-update/node-version.ts index adaf0dff9c9fde..8cbbc0b301265f 100644 --- a/lib/manager/npm/post-update/node-version.ts +++ b/lib/manager/npm/post-update/node-version.ts @@ -1,6 +1,7 @@ -import { validRange } from 'semver'; +import { satisfies, validRange } from 'semver'; import { logger } from '../../../logger'; import { getSiblingFileName, readLocalFile } from '../../../util/fs'; +import { isStable } from '../../../versioning/node'; import { PostUpdateConfig } from '../../common'; async function getNodeFile(filename: string): Promise | null { @@ -31,11 +32,24 @@ export async function getNodeConstraint( config: PostUpdateConfig ): Promise | null { const { packageFile } = config; - const constraint = + let constraint = (await getNodeFile(getSiblingFileName(packageFile, '.nvmrc'))) || (await getNodeFile(getSiblingFileName(packageFile, '.node-version'))) || getPackageJsonConstraint(config); - if (!constraint) { + // Avoid using node 15 if node 14 also satisfies the same constraint + // Remove this once node 16 is LTS + if (constraint) { + if ( + validRange(constraint) && + satisfies('14.100.0', constraint) && + satisfies('15.100.0', constraint) && + !isStable('16.100.0') && // this should return false as soon as Node 16 is LTS + validRange(`${constraint} <15`) + ) { + logger.debug('Augmenting constraint to avoid node 15'); + constraint = `${constraint} <15`; + } + } else { logger.debug('No node constraint found - using latest'); } return constraint; From 489bd2c39415e1285846145934137193e77b870f Mon Sep 17 00:00:00 2001 From: Rhys Arkins Date: Tue, 12 Jan 2021 11:18:28 +0100 Subject: [PATCH 07/52] docs: status labels --- docs/development/issue-labeling.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/docs/development/issue-labeling.md b/docs/development/issue-labeling.md index 9c255d431edf85..56127ce645ed43 100644 --- a/docs/development/issue-labeling.md +++ b/docs/development/issue-labeling.md @@ -16,6 +16,22 @@ Most issues should have a label relating to either a platform, manager, datasour ## Label categories +### Status + +
+ Status of issue + + status:requirements + status:blocked + status:ready + status:in-progress + +
+ +Use these to label the status of an issue. +For example, use `status:requirements` to mean that an issue is not yet ready for development to begin. +All open issues should have some `status:*` label applied. + ### Type of issue
From ec104ac19e350c4df7cf94849aa412b2e126642c Mon Sep 17 00:00:00 2001 From: Sergei Zharinov Date: Tue, 12 Jan 2021 17:42:02 +0400 Subject: [PATCH 08/52] fix(gradle-lite): Upgrade deps in reverse order for grouped dependencies (#8263) --- lib/manager/gradle-lite/extract.ts | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/lib/manager/gradle-lite/extract.ts b/lib/manager/gradle-lite/extract.ts index cab54c04aa7d80..de218363413cdf 100644 --- a/lib/manager/gradle-lite/extract.ts +++ b/lib/manager/gradle-lite/extract.ts @@ -13,6 +13,21 @@ import { toAbsolutePath, } from './utils'; +// Enables reverse sorting in generateBranchConfig() +// +// Required for grouped dependencies to be upgraded +// correctly in single branch. +// +// https://github.com/renovatebot/renovate/issues/8224 +function elevateFileReplacePositionField( + deps: PackageDependency[] +): PackageDependency[] { + return deps.map((dep) => ({ + ...dep, + fileReplacePosition: dep?.managerData?.fileReplacePosition, + })); +} + export async function extractAllPackageFiles( config: ExtractConfig, packageFiles: string[] @@ -57,7 +72,7 @@ export async function extractAllPackageFiles( return null; } - extractedDeps.forEach((dep) => { + elevateFileReplacePositionField(extractedDeps).forEach((dep) => { const key = dep.managerData.packageFile; const pkgFile: PackageFile = packageFilesByName[key]; const { deps } = pkgFile; From afe2ccb1a5ebb5ab68f0b61f57f4501fb2fa007a Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 13 Jan 2021 02:01:08 +0000 Subject: [PATCH 09/52] chore(deps): update dependency @types/js-yaml to v3.12.6 (#8270) Co-authored-by: Renovate Bot --- package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index d1dec09bae9315..ee5d0fadcf473c 100644 --- a/package.json +++ b/package.json @@ -209,7 +209,7 @@ "@types/global-agent": "2.1.0", "@types/ini": "1.3.30", "@types/jest": "26.0.19", - "@types/js-yaml": "3.12.5", + "@types/js-yaml": "3.12.6", "@types/jsdom": "16.2.5", "@types/json-dup-key-validator": "1.0.0", "@types/json5": "0.0.30", diff --git a/yarn.lock b/yarn.lock index ce958ea3c3c299..e6ea3490d0cbc6 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1664,10 +1664,10 @@ jest-diff "^26.0.0" pretty-format "^26.0.0" -"@types/js-yaml@3.12.5": - version "3.12.5" - resolved "https://registry.yarnpkg.com/@types/js-yaml/-/js-yaml-3.12.5.tgz#136d5e6a57a931e1cce6f9d8126aa98a9c92a6bb" - integrity sha512-JCcp6J0GV66Y4ZMDAQCXot4xprYB+Zfd3meK9+INSJeVZwJmHAW30BBEEkPzXswMXuiyReUGOP3GxrADc9wPww== +"@types/js-yaml@3.12.6": + version "3.12.6" + resolved "https://registry.yarnpkg.com/@types/js-yaml/-/js-yaml-3.12.6.tgz#7f10c926aa41e189a2755c4c7fcf8e4573bd7ac1" + integrity sha512-cK4XqrLvP17X6c0C8n4iTbT59EixqyXL3Fk8/Rsk4dF3oX4dg70gYUXrXVUUHpnsGMPNlTQMqf+TVmNPX6FmSQ== "@types/jsdom@16.2.5": version "16.2.5" From b8e9a1367f2b607127cad985b8c1ab1ce9b474b0 Mon Sep 17 00:00:00 2001 From: Sergei Zharinov Date: Wed, 13 Jan 2021 14:41:49 +0400 Subject: [PATCH 10/52] fix(gradle-lite): Parse default registry urls configured explicitly (#8267) --- lib/manager/gradle-lite/common.ts | 5 ++++ lib/manager/gradle-lite/parser.spec.ts | 12 +++++++-- lib/manager/gradle-lite/parser.ts | 35 +++++++++++++++++++++++--- 3 files changed, 47 insertions(+), 5 deletions(-) diff --git a/lib/manager/gradle-lite/common.ts b/lib/manager/gradle-lite/common.ts index 4c47de178afb72..354f64c0d58a8c 100644 --- a/lib/manager/gradle-lite/common.ts +++ b/lib/manager/gradle-lite/common.ts @@ -1,5 +1,10 @@ import { PackageDependency } from '../common'; +export { MAVEN_REPO } from '../../datasource/maven/common'; + +export const JCENTER_REPO = 'https://jcenter.bintray.com/'; +export const GOOGLE_REPO = 'https://dl.google.com/android/maven2/'; + export interface ManagerData { fileReplacePosition: number; packageFile?: string; diff --git a/lib/manager/gradle-lite/parser.spec.ts b/lib/manager/gradle-lite/parser.spec.ts index be5c69c1b12a79..437a77ceefeb8f 100644 --- a/lib/manager/gradle-lite/parser.spec.ts +++ b/lib/manager/gradle-lite/parser.spec.ts @@ -1,5 +1,6 @@ import { readFileSync } from 'fs'; import path from 'path'; +import { GOOGLE_REPO, JCENTER_REPO, MAVEN_REPO } from './common'; import { parseGradle, parseProps } from './parser'; function getGradleFile(fileName: string): string { @@ -45,8 +46,15 @@ describe('manager/gradle-lite/parser', () => { ({ urls } = parseGradle('uri "https://example.com"')); expect(urls).toStrictEqual(['https://example.com']); - ({ urls } = parseGradle('uri("https://example.com")')); - expect(urls).toStrictEqual(['https://example.com']); + ({ urls } = parseGradle( + 'mavenCentral(); uri("https://example.com"); jcenter(); google();' + )); + expect(urls).toStrictEqual([ + MAVEN_REPO, + 'https://example.com', + JCENTER_REPO, + GOOGLE_REPO, + ]); }); it('parses long form deps', () => { let deps; diff --git a/lib/manager/gradle-lite/parser.ts b/lib/manager/gradle-lite/parser.ts index c14f8137c380ae..e570e523b53049 100644 --- a/lib/manager/gradle-lite/parser.ts +++ b/lib/manager/gradle-lite/parser.ts @@ -4,6 +4,9 @@ import { logger } from '../../logger'; import { regEx } from '../../util/regex'; import { PackageDependency } from '../common'; import { + GOOGLE_REPO, + JCENTER_REPO, + MAVEN_REPO, ManagerData, PackageVariables, StringInterpolation, @@ -188,7 +191,7 @@ function processPlugin({ return { deps: [dep] }; } -function processRegistryUrl({ +function processCustomRegistryUrl({ tokenMap, }: SyntaxHandlerInput): SyntaxHandlerOutput { const registryUrl = tokenMap.registryUrl?.value; @@ -205,6 +208,18 @@ function processRegistryUrl({ return null; } +function processPredefinedRegistryUrl({ + tokenMap, +}: SyntaxHandlerInput): SyntaxHandlerOutput { + const registryName = tokenMap.registryName?.value; + const registryUrl = { + mavenCentral: MAVEN_REPO, + jcenter: JCENTER_REPO, + google: GOOGLE_REPO, + }[registryName]; + return { urls: [registryUrl] }; +} + function processLongFormDep({ tokenMap, variables, @@ -296,6 +311,20 @@ const matcherConfigs: SyntaxMatchConfig[] = [ ], handler: processPlugin, }, + { + // mavenCentral() + matchers: [ + { + matchType: TokenType.Word, + matchValue: ['mavenCentral', 'jcenter', 'google'], + tokenMapKey: 'registryName', + }, + { matchType: TokenType.LeftParen }, + { matchType: TokenType.RightParen }, + endOfInstruction, + ], + handler: processPredefinedRegistryUrl, + }, { // url 'https://repo.spring.io/snapshot/' matchers: [ @@ -303,7 +332,7 @@ const matcherConfigs: SyntaxMatchConfig[] = [ { matchType: TokenType.String, tokenMapKey: 'registryUrl' }, endOfInstruction, ], - handler: processRegistryUrl, + handler: processCustomRegistryUrl, }, { // url('https://repo.spring.io/snapshot/') @@ -314,7 +343,7 @@ const matcherConfigs: SyntaxMatchConfig[] = [ { matchType: TokenType.RightParen }, endOfInstruction, ], - handler: processRegistryUrl, + handler: processCustomRegistryUrl, }, { // group: "com.example", name: "my.dependency", version: "1.2.3" From ed892071fb9a11ed1efeed3e2972dfe1d1807e53 Mon Sep 17 00:00:00 2001 From: HonkingGoose <34918129+HonkingGoose@users.noreply.github.com> Date: Thu, 14 Jan 2021 08:46:44 +0100 Subject: [PATCH 11/52] docs: add duplicate label to issue labeling guide (#8284) --- docs/development/issue-labeling.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/development/issue-labeling.md b/docs/development/issue-labeling.md index 56127ce645ed43..f4908c5d56331f 100644 --- a/docs/development/issue-labeling.md +++ b/docs/development/issue-labeling.md @@ -132,6 +132,7 @@ Apply these labels when somebody opens a `feature` type issue requesting a new d help wanted reproduced reproduction needed + duplicate
@@ -143,6 +144,8 @@ Add the label `help wanted` to indicate that we need the original poster or some Add a label `reproduction needed` if nobody's reproduced it in a public repo yet and such a reproduction is necessary before further work can be done. Add the label `reproduced` once there is a public reproduction. +Add a label `duplicate` to issues/PRs that are a duplicate of an earlier issue/PR. + ### Self hosted
From 2d7a0dad2fce1a997a88be6c8846b4cebefd1478 Mon Sep 17 00:00:00 2001 From: Sergei Zharinov Date: Thu, 14 Jan 2021 11:47:42 +0400 Subject: [PATCH 12/52] fix(sbt): Ignore http4s digest-based milestone releases (#8282) --- lib/config/presets/internal/workarounds.ts | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/lib/config/presets/internal/workarounds.ts b/lib/config/presets/internal/workarounds.ts index f09bb7a68c185e..04c17d2cb5aa4d 100644 --- a/lib/config/presets/internal/workarounds.ts +++ b/lib/config/presets/internal/workarounds.ts @@ -9,6 +9,7 @@ export const presets: Record = { 'workarounds:mavenCommonsAncientVersion', 'workarounds:ignoreSbtLatestIntegration', 'workarounds:ignoreSpringCloudNumeric', + 'workarounds:ignoreHttp4sDigestMilestones', ], }, mavenCommonsAncientVersion: { @@ -40,4 +41,14 @@ export const presets: Record = { }, ], }, + ignoreHttp4sDigestMilestones: { + description: 'Ignore http4s digest-based 1.x milestones', + packageRules: [ + { + managers: ['sbt'], + packagePatterns: ['^org\\.http4s:'], + allowedVersions: `!/^1\\.0-\\d+-[a-fA-F0-9]{7}$/`, + }, + ], + }, }; From 8084ea3d9c16bcade02a5332a2ae12f59c7fc082 Mon Sep 17 00:00:00 2001 From: Mariano Wahlmann Date: Thu, 14 Jan 2021 02:01:42 -0600 Subject: [PATCH 13/52] fix: cleanSimpleHtml function regexp greedyness (issue #8285) (#8286) * Fixes issue #8285 * Remove unnecesary escapes --- lib/datasource/pypi/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/datasource/pypi/index.ts b/lib/datasource/pypi/index.ts index 2e98b5e5b96d88..19bee498cf3445 100644 --- a/lib/datasource/pypi/index.ts +++ b/lib/datasource/pypi/index.ts @@ -172,11 +172,11 @@ function cleanSimpleHtml(html: string): string { .replace(/<\/?pre>/, '') // Certain simple repositories like artifactory don't escape > and < .replace( - /data-requires-python="(.*?)>(.*?)"/g, + /data-requires-python="([^"]*?)>([^"]*?)"/g, 'data-requires-python="$1>$2"' ) .replace( - /data-requires-python="(.*?)<(.*?)"/g, + /data-requires-python="([^"]*?)<([^"]*?)"/g, 'data-requires-python="$1<$2"' ) ); From acc3690fb257f0e69e8355d00fd692a10553aa4d Mon Sep 17 00:00:00 2001 From: HonkingGoose <34918129+HonkingGoose@users.noreply.github.com> Date: Thu, 14 Jan 2021 09:02:26 +0100 Subject: [PATCH 14/52] docs: use one sentence per line for helm values readme (#8283) --- lib/manager/helm-values/readme.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/manager/helm-values/readme.md b/lib/manager/helm-values/readme.md index acb203598f51ca..031f4bb8850ad5 100644 --- a/lib/manager/helm-values/readme.md +++ b/lib/manager/helm-values/readme.md @@ -1,6 +1,5 @@ -Renovate supports updating of Docker dependencies within Helm Chart `values.yaml` files or other YAML -files that use the same format (via `fileMatch` configuration). Updates are performed if the files -follow the conventional format used in most of the `stable` Helm charts: +Renovate supports updating of Docker dependencies within Helm Chart `values.yaml` files or other YAML files that use the same format (via `fileMatch` configuration). +Updates are performed if the files follow the conventional format used in most of the `stable` Helm charts: ```yaml image: From 30f8f33c5cf33a2f4afac423c4f3311362e3510b Mon Sep 17 00:00:00 2001 From: HonkingGoose <34918129+HonkingGoose@users.noreply.github.com> Date: Thu, 14 Jan 2021 09:03:26 +0100 Subject: [PATCH 15/52] docs: create triage guide (#8268) Co-authored-by: Rhys Arkins --- docs/development/triage-guide.md | 93 ++++++++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 docs/development/triage-guide.md diff --git a/docs/development/triage-guide.md b/docs/development/triage-guide.md new file mode 100644 index 00000000000000..10f42c79bdc8b0 --- /dev/null +++ b/docs/development/triage-guide.md @@ -0,0 +1,93 @@ +# Triage guide + +## What is triage? + +Triage is basically filtering the issues/discussions, and categorizing them with the proper labels. + +## What a triagist is allowed to do + +If you've been given triage rights, you are allowed to do the following things: + +- Apply labels to issues +- Close, reopen, and assign all issues and pull requests +- Mark duplicate issues and pull requests +- Request pull request reviews +- Lock and unlock discussions +- Individually convert issues to discussions + +**Note:** We don't use milestones or project boards. + +## Guidelines for triage workflow + +The following are guidelines as we cannot cover all situations. +Use common sense, and just do your best, and you'll do all right. +Don't be afraid to ask for help. + +### Apply labels to issues + +All issues should have labels attached to them. +Read the [issue-labeling guide](https://github.com/renovatebot/renovate/blob/master/docs/development/issue-labeling.md) to get all the necessary info. + +In general try to make a good-faith effort to label things correctly. + +### Closing issues + +You can close a issue yourself if it's: + +- Spam +- Obviously fixed + +For really old issues, it's probably a good idea to ask the maintainers to decide if they want to keep or close the issue. + +### Closing pull requests + +It's not very often that you'll need to close a PR, but you can certainly do it in case of spam or malicious content in the PR diff. + +### Reopen issues + +Sometimes a bug is fixed with a PR that links to a issue. +When the PR is merged, the issue is automatically closed. +Sometimes the bug was not really fixed, and someone says: "Hey this is still broken for me." +In that case, re-open the issue only if it's definitely the same problem (users often associate different problems together incorrectly). +Otherwise, ask the user to open a new issue if it seems like it is different. + +### Assign issues + +You can assign a issue to yourself, so that others know you're going to work on the issue. +GitHub allows issues to be assigned to any project collaborator or to any non-collaborator who has created or commented on the issue, so you can also assign in either of those cases if it makes sense. + +### Mark duplicate issues and pull requests + +If you see an issue that's an obvious duplicate: + +1. Attach a `duplicate` label +1. Use the "Duplicate of" functionality [GitHub docs, about duplicate issues and pull requests](https://docs.github.com/en/free-pro-team@latest/github/managing-your-work-on-github/about-duplicate-issues-and-pull-requests) +1. Close the issue + +Follow the same workflow to mark duplicate PRs. + +### Request PR reviews + +You can request a review from one of the maintainers, in case this is needed to get the PR review process rolling. + +### Lock and unlock discussions + +Sometimes a discussion can go sour, like when people call each other names, or post spam, or veer off-topic. +In those cases you can lock the discussion to prevent further escalation. + +### Individually convert issues to discussions + +Sometimes a issue that's raised at the Renovate repository is not really a bug or a feature request. +This happens most often because a user files a bug for things that are really a misstake in the Renovate configuration. +Those "configuration help" issues are then moved to the discussions board for further help. + +### Moving issues from `status:requirements` to `status:ready` + +One of the most important non-code contributions people can do is help features and fixes go from `status:requirements` to `status:ready`. +We use the label `status:requirements` to mean "more information or research is needed before someone could start coding this". + +It can sometimes be an oversight of the maintainers, but more often it's because there are requirements or edge cases to consider and the user hasn't got an opinion or time to think about them and contribute enough. +Sometimes it can be because there's a need for some research and "design" decisions to be made, which may require maintainers to do, but it's not high enough priority to justify the time yet. + +In a way `status:requirements` means "someone's going to need to put more thought into this before it can move forward to development". +It can also mean "don't start this now because you might do something which can't be accepted into the code base". From 987d37400b392433383d79c254524654f3370259 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 14 Jan 2021 10:01:30 +0000 Subject: [PATCH 16/52] chore(deps): update dependency husky to v4.3.7 (#8287) Co-authored-by: Renovate Bot --- package.json | 2 +- yarn.lock | 33 ++++++++++++++------------------- 2 files changed, 15 insertions(+), 20 deletions(-) diff --git a/package.json b/package.json index ee5d0fadcf473c..5a2459f8eab348 100644 --- a/package.json +++ b/package.json @@ -244,7 +244,7 @@ "eslint-plugin-promise": "4.2.1", "glob": "7.1.6", "graphql": "15.4.0", - "husky": "4.3.6", + "husky": "4.3.7", "jest": "26.6.3", "jest-circus": "26.6.3", "jest-extended": "0.11.5", diff --git a/yarn.lock b/yarn.lock index e6ea3490d0cbc6..47aab212d48445 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4514,7 +4514,7 @@ find-npm-prefix@^1.0.2: resolved "https://registry.yarnpkg.com/find-npm-prefix/-/find-npm-prefix-1.0.2.tgz#8d8ce2c78b3b4b9e66c8acc6a37c231eb841cfdf" integrity sha512-KEftzJ+H90x6pcKtdXZEPsQse8/y/UnvzRKrOSQFprnrGaFuJ62fVkP34Iu2IYuMvyauCyoLTNkJZgrrGA2wkA== -find-up@5.0.0: +find-up@5.0.0, find-up@^5.0.0: version "5.0.0" resolved "https://registry.yarnpkg.com/find-up/-/find-up-5.0.0.tgz#4c92819ecb7083561e4f4a240a86be5198f536fc" integrity sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng== @@ -4544,13 +4544,6 @@ find-up@^4.0.0, find-up@^4.1.0: locate-path "^5.0.0" path-exists "^4.0.0" -find-versions@^3.2.0: - version "3.2.0" - resolved "https://registry.yarnpkg.com/find-versions/-/find-versions-3.2.0.tgz#10297f98030a786829681690545ef659ed1d254e" - integrity sha512-P8WRou2S+oe222TOCHitLy8zj+SIsVJh52VP4lvXkaFVnOFFdoWv1H1Jjvel1aI6NCFOAaeAVm8qrI0odiLcww== - dependencies: - semver-regex "^2.0.0" - find-versions@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/find-versions/-/find-versions-4.0.0.tgz#3c57e573bf97769b8cb8df16934b627915da4965" @@ -5198,18 +5191,18 @@ humanize-ms@^1.2.1: dependencies: ms "^2.0.0" -husky@4.3.6: - version "4.3.6" - resolved "https://registry.yarnpkg.com/husky/-/husky-4.3.6.tgz#ebd9dd8b9324aa851f1587318db4cccb7665a13c" - integrity sha512-o6UjVI8xtlWRL5395iWq9LKDyp/9TE7XMOTvIpEVzW638UcGxTmV5cfel6fsk/jbZSTlvfGVJf2svFtybcIZag== +husky@4.3.7: + version "4.3.7" + resolved "https://registry.yarnpkg.com/husky/-/husky-4.3.7.tgz#ca47bbe6213c1aa8b16bbd504530d9600de91e88" + integrity sha512-0fQlcCDq/xypoyYSJvEuzbDPHFf8ZF9IXKJxlrnvxABTSzK1VPT2RKYQKrcgJ+YD39swgoB6sbzywUqFxUiqjw== dependencies: chalk "^4.0.0" ci-info "^2.0.0" compare-versions "^3.6.0" cosmiconfig "^7.0.0" - find-versions "^3.2.0" + find-versions "^4.0.0" opencollective-postinstall "^2.0.2" - pkg-dir "^4.2.0" + pkg-dir "^5.0.0" please-upgrade-node "^3.2.0" slash "^3.0.0" which-pm-runs "^1.0.0" @@ -8499,6 +8492,13 @@ pkg-dir@^4.2.0: dependencies: find-up "^4.0.0" +pkg-dir@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/pkg-dir/-/pkg-dir-5.0.0.tgz#a02d6aebe6ba133a928f74aec20bafdfe6b8e760" + integrity sha512-NPE8TDbzl/3YQYY7CSS228s3g2ollTFnc+Qi3tqmqJp9Vg2ovUpixcJEo2HJScN2Ez+kEaal6y70c0ehqJBJeA== + dependencies: + find-up "^5.0.0" + please-upgrade-node@^3.2.0: version "3.2.0" resolved "https://registry.yarnpkg.com/please-upgrade-node/-/please-upgrade-node-3.2.0.tgz#aeddd3f994c933e4ad98b99d9a556efa0e2fe942" @@ -9466,11 +9466,6 @@ semver-diff@^3.1.1: dependencies: semver "^6.3.0" -semver-regex@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/semver-regex/-/semver-regex-2.0.0.tgz#a93c2c5844539a770233379107b38c7b4ac9d338" - integrity sha512-mUdIBBvdn0PLOeP3TEkMH7HHeUP3GjsXCwKarjv/kGmUFOYg1VqEemKhoQpWMu6X2I8kHeuVdGibLGkVK+/5Qw== - semver-regex@^3.1.2: version "3.1.2" resolved "https://registry.yarnpkg.com/semver-regex/-/semver-regex-3.1.2.tgz#34b4c0d361eef262e07199dbef316d0f2ab11807" From 9ebf1a9f4da78aa612c23a9e2b40db981b92cdb1 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 14 Jan 2021 11:30:50 +0100 Subject: [PATCH 17/52] build(deps): update dependency aws-sdk to v2.827.0 (#8290) Co-authored-by: Renovate Bot --- package.json | 2 +- yarn.lock | 23 ++++------------------- 2 files changed, 5 insertions(+), 20 deletions(-) diff --git a/package.json b/package.json index 5a2459f8eab348..47ed5ac4c961c6 100644 --- a/package.json +++ b/package.json @@ -119,7 +119,7 @@ "@sindresorhus/is": "4.0.0", "@yarnpkg/core": "2.4.0", "@yarnpkg/parsers": "2.3.0", - "aws-sdk": "2.792.0", + "aws-sdk": "2.827.0", "azure-devops-node-api": "10.2.0", "bunyan": "1.8.14", "cacache": "15.0.5", diff --git a/yarn.lock b/yarn.lock index 47aab212d48445..3ed341408af262 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2494,25 +2494,10 @@ aws-sdk-mock@5.1.0: sinon "^9.0.1" traverse "^0.6.6" -aws-sdk@2.792.0: - version "2.792.0" - resolved "https://registry.yarnpkg.com/aws-sdk/-/aws-sdk-2.792.0.tgz#d124a6074244a4675e0416887734e8f6934bdd30" - integrity sha512-h7oSlrCDtZkW5qNw/idKmMjjNJaaPlXFY+NbqtaTjejpCyVuIonUmFvm8GW16V58Avj/hujJfhpX9q0BMCg+VQ== - dependencies: - buffer "4.9.2" - events "1.1.1" - ieee754 "1.1.13" - jmespath "0.15.0" - querystring "0.2.0" - sax "1.2.1" - url "0.10.3" - uuid "3.3.2" - xml2js "0.4.19" - -aws-sdk@^2.637.0: - version "2.824.0" - resolved "https://registry.yarnpkg.com/aws-sdk/-/aws-sdk-2.824.0.tgz#a67747d4d0b53d09c6c121e93f44d8f6e76fc44b" - integrity sha512-9KNRQBkIMPn+6DWb4gR+RzqTMNyGLEwOgXbE4dDehOIAflfLnv3IFwLnzrhxJnleB4guYrILIsBroJFBzjiekg== +aws-sdk@2.827.0, aws-sdk@^2.637.0: + version "2.827.0" + resolved "https://registry.yarnpkg.com/aws-sdk/-/aws-sdk-2.827.0.tgz#372e37cabf0e8351de6d8f07a8f519bfaaeeae2a" + integrity sha512-71PWS1dqJ65/SeNgDQWEgbJ6oKCuB+Ypq30TM3EyzbAHaxl69WjQRK71oJ2bjhdIHfGQJtOV0G9wg4zpge4Erg== dependencies: buffer "4.9.2" events "1.1.1" From 60cb127b43fc1760eef5368b3ef31b006ac17ad0 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 14 Jan 2021 11:01:15 +0000 Subject: [PATCH 18/52] chore(deps): update dependency @types/jest to v26.0.20 (#8292) Co-authored-by: Renovate Bot --- package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index 47ed5ac4c961c6..39405b7b2167bd 100644 --- a/package.json +++ b/package.json @@ -208,7 +208,7 @@ "@types/github-url-from-git": "1.5.0", "@types/global-agent": "2.1.0", "@types/ini": "1.3.30", - "@types/jest": "26.0.19", + "@types/jest": "26.0.20", "@types/js-yaml": "3.12.6", "@types/jsdom": "16.2.5", "@types/json-dup-key-validator": "1.0.0", diff --git a/yarn.lock b/yarn.lock index 3ed341408af262..27a02d7eb26e17 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1656,10 +1656,10 @@ dependencies: "@types/istanbul-lib-report" "*" -"@types/jest@26.0.19": - version "26.0.19" - resolved "https://registry.yarnpkg.com/@types/jest/-/jest-26.0.19.tgz#e6fa1e3def5842ec85045bd5210e9bb8289de790" - integrity sha512-jqHoirTG61fee6v6rwbnEuKhpSKih0tuhqeFbCmMmErhtu3BYlOZaXWjffgOstMM4S/3iQD31lI5bGLTrs97yQ== +"@types/jest@26.0.20": + version "26.0.20" + resolved "https://registry.yarnpkg.com/@types/jest/-/jest-26.0.20.tgz#cd2f2702ecf69e86b586e1f5223a60e454056307" + integrity sha512-9zi2Y+5USJRxd0FsahERhBwlcvFh6D2GLQnY2FH2BzK8J9s9omvNHIbvABwIluXa0fD8XVKMLTO0aOEuUfACAA== dependencies: jest-diff "^26.0.0" pretty-format "^26.0.0" From c3e11ea46ab2acf7a0c54d59c70a5554247b498e Mon Sep 17 00:00:00 2001 From: Michael Kriese Date: Thu, 14 Jan 2021 12:37:09 +0100 Subject: [PATCH 19/52] test: raise branch coverage (#8294) --- jest.config.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jest.config.js b/jest.config.js index d0fc8e9a53e96f..d147716cef5268 100644 --- a/jest.config.js +++ b/jest.config.js @@ -14,7 +14,7 @@ module.exports = { : ['html', 'text-summary'], coverageThreshold: { global: { - branches: 95, + branches: 96, functions: 100, lines: 100, statements: 100, From 76ada5e7d5ec4c28dc7cf7b158bf29584f33fa5b Mon Sep 17 00:00:00 2001 From: Michael Kriese Date: Thu, 14 Jan 2021 13:11:23 +0100 Subject: [PATCH 20/52] feat: deprecated logLevel option (#8293) * feat: deprecated logLevel option * feat: deprecated logLevel option * docs: update logging --- docs/development/configuration.md | 3 ++- docs/usage/self-hosted-configuration.md | 2 ++ docs/usage/self-hosting.md | 2 -- lib/config/config/__fixtures__/file.js | 1 - lib/config/config/__fixtures__/with-force.js | 1 - lib/config/definitions.ts | 3 +-- lib/config/file.spec.ts | 1 - lib/config/index.ts | 10 ++++++++-- 8 files changed, 13 insertions(+), 10 deletions(-) diff --git a/docs/development/configuration.md b/docs/development/configuration.md index 8f50e1ba6a0f77..549c44712c0c32 100644 --- a/docs/development/configuration.md +++ b/docs/development/configuration.md @@ -38,7 +38,6 @@ e.g. apply one set of labels for `backend/package.json` and a different set of l module.exports = { npmrc: '//registry.npmjs.org/:_authToken=abc123', baseDir: '/tmp/renovate', - logLevel: 'debug', includeForks: true, gradle: { enabled: false }, }; @@ -53,6 +52,8 @@ $ node renovate --help To configure any `` items, separate with commas. E.g. `renovate --labels=renovate,dependency`. +To enable debug logging export `LOG_LEVEL=debug` to your environment. + ### renovate.json If you add a `renovate.json` file to the root of your repository, you can use this to override default settings. diff --git a/docs/usage/self-hosted-configuration.md b/docs/usage/self-hosted-configuration.md index d3f0ed28e127ac..fe78ebafa235de 100644 --- a/docs/usage/self-hosted-configuration.md +++ b/docs/usage/self-hosted-configuration.md @@ -168,6 +168,8 @@ By configuring using the environment it means that debug logging starts from the Additionally, if you configure `LOG_FORMAT=json` in env then logging will be done in JSON format instead of "pretty" format, which is usually better if you're doing any ingestion or parsing of the logs. +Warning: Configuring `logLevel` config option or `--log-level` cli option is deprecated and will be removed in a major version. + ## onboarding Set this to `false` if (a) you configure Renovate entirely on the bot side (i.e. empty `renovate.json` in repositories) and (b) you wish to run Renovate on every repository the bot has access to, and (c) you wish to skip the onboarding PRs. diff --git a/docs/usage/self-hosting.md b/docs/usage/self-hosting.md index cf97f61382179b..91624d10f64ecf 100644 --- a/docs/usage/self-hosting.md +++ b/docs/usage/self-hosting.md @@ -99,7 +99,6 @@ metadata: data: config.json: |- { - "logLevel" : "debug", "repositories": ["orgname/repo","username/repo"], "dryRun" : "true" } @@ -274,7 +273,6 @@ module.exports = { endpoint: 'https://self-hosted.gitlab/api/v4/', token: '**gitlab_token**', platform: 'gitlab', - logLevel: 'debug', onboardingConfig: { extends: ['config:base'], }, diff --git a/lib/config/config/__fixtures__/file.js b/lib/config/config/__fixtures__/file.js index 61600196f14b7d..e4fe0d85493f41 100644 --- a/lib/config/config/__fixtures__/file.js +++ b/lib/config/config/__fixtures__/file.js @@ -1,5 +1,4 @@ // @ts-ignore module.exports = { token: 'abcdefg', - logLevel: 'error' }; diff --git a/lib/config/config/__fixtures__/with-force.js b/lib/config/config/__fixtures__/with-force.js index d16cae37e45284..ad2b1074fb23d4 100644 --- a/lib/config/config/__fixtures__/with-force.js +++ b/lib/config/config/__fixtures__/with-force.js @@ -1,7 +1,6 @@ // @ts-ignore module.exports = { token: 'abcdefg', - logLevel: 'error', force: { schedule: null, } diff --git a/lib/config/definitions.ts b/lib/config/definitions.ts index f4c66a57d99fca..eef3b2ce37e613 100644 --- a/lib/config/definitions.ts +++ b/lib/config/definitions.ts @@ -342,11 +342,10 @@ const options: RenovateOptions[] = [ // Log options { name: 'logLevel', - description: 'Logging level', + description: 'Logging level. Deprecated, use `LOG_LEVEL` environment.', stage: 'global', type: 'string', allowedValues: ['fatal', 'error', 'warn', 'info', 'debug', 'trace'], - default: 'info', env: 'LOG_LEVEL', }, { diff --git a/lib/config/file.spec.ts b/lib/config/file.spec.ts index 4eeed34655d5fd..f5eb41da982383 100644 --- a/lib/config/file.spec.ts +++ b/lib/config/file.spec.ts @@ -37,7 +37,6 @@ describe('config/file', () => { "platform": "github", "token":"abcdef", "logFileLevel": "warn", - "logLevel": "info", "onboarding": false, "gitAuthor": "Renovate Bot " "onboardingConfig": { diff --git a/lib/config/index.ts b/lib/config/index.ts index 0f4699091c0fa9..44c803cd2c4ecc 100644 --- a/lib/config/index.ts +++ b/lib/config/index.ts @@ -80,8 +80,14 @@ export async function parseConfigs( delete config.privateKeyPath; } - // Set log level - levels('stdout', config.logLevel); + // Deprecated set log level: https://github.com/renovatebot/renovate/issues/8291 + // istanbul ignore if + if (config.logLevel) { + logger.warn( + 'Configuring logLevel is deprecated. Use LOG_LEVEL environment variable instead' + ); + levels('stdout', config.logLevel); + } if (config.logContext) { // This only has an effect if logContext was defined via file or CLI, otherwise it would already have been detected in env From d56ab63989f6fca53b4e9168d9849883ce852072 Mon Sep 17 00:00:00 2001 From: Rhys Arkins Date: Thu, 14 Jan 2021 13:26:49 +0100 Subject: [PATCH 21/52] fix: pass versioning for docker getPkgReleases (#8261) Co-authored-by: Michael Kriese --- lib/util/exec/docker/index.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/util/exec/docker/index.ts b/lib/util/exec/docker/index.ts index ebce1214a4690c..532c95dff49f4f 100644 --- a/lib/util/exec/docker/index.ts +++ b/lib/util/exec/docker/index.ts @@ -84,7 +84,11 @@ async function getDockerTag( { constraint }, `Found ${scheme} version constraint - checking for a compatible ${depName} image to use` ); - const imageReleases = await getPkgReleases({ datasource: 'docker', depName }); + const imageReleases = await getPkgReleases({ + datasource: 'docker', + depName, + versioning: scheme, + }); if (imageReleases?.releases) { let versions = imageReleases.releases.map((release) => release.version); versions = versions.filter( From e77253208e8fef6e030d408513ede56f164c3554 Mon Sep 17 00:00:00 2001 From: Rhys Arkins Date: Thu, 14 Jan 2021 15:12:53 +0100 Subject: [PATCH 22/52] fix: Revert "feat: deprecated logLevel option (#8293)" This reverts commit 76ada5e7d5ec4c28dc7cf7b158bf29584f33fa5b. --- docs/development/configuration.md | 3 +-- docs/usage/self-hosted-configuration.md | 2 -- docs/usage/self-hosting.md | 2 ++ lib/config/config/__fixtures__/file.js | 1 + lib/config/config/__fixtures__/with-force.js | 1 + lib/config/definitions.ts | 3 ++- lib/config/file.spec.ts | 1 + lib/config/index.ts | 10 ++-------- 8 files changed, 10 insertions(+), 13 deletions(-) diff --git a/docs/development/configuration.md b/docs/development/configuration.md index 549c44712c0c32..8f50e1ba6a0f77 100644 --- a/docs/development/configuration.md +++ b/docs/development/configuration.md @@ -38,6 +38,7 @@ e.g. apply one set of labels for `backend/package.json` and a different set of l module.exports = { npmrc: '//registry.npmjs.org/:_authToken=abc123', baseDir: '/tmp/renovate', + logLevel: 'debug', includeForks: true, gradle: { enabled: false }, }; @@ -52,8 +53,6 @@ $ node renovate --help To configure any `` items, separate with commas. E.g. `renovate --labels=renovate,dependency`. -To enable debug logging export `LOG_LEVEL=debug` to your environment. - ### renovate.json If you add a `renovate.json` file to the root of your repository, you can use this to override default settings. diff --git a/docs/usage/self-hosted-configuration.md b/docs/usage/self-hosted-configuration.md index fe78ebafa235de..d3f0ed28e127ac 100644 --- a/docs/usage/self-hosted-configuration.md +++ b/docs/usage/self-hosted-configuration.md @@ -168,8 +168,6 @@ By configuring using the environment it means that debug logging starts from the Additionally, if you configure `LOG_FORMAT=json` in env then logging will be done in JSON format instead of "pretty" format, which is usually better if you're doing any ingestion or parsing of the logs. -Warning: Configuring `logLevel` config option or `--log-level` cli option is deprecated and will be removed in a major version. - ## onboarding Set this to `false` if (a) you configure Renovate entirely on the bot side (i.e. empty `renovate.json` in repositories) and (b) you wish to run Renovate on every repository the bot has access to, and (c) you wish to skip the onboarding PRs. diff --git a/docs/usage/self-hosting.md b/docs/usage/self-hosting.md index 91624d10f64ecf..cf97f61382179b 100644 --- a/docs/usage/self-hosting.md +++ b/docs/usage/self-hosting.md @@ -99,6 +99,7 @@ metadata: data: config.json: |- { + "logLevel" : "debug", "repositories": ["orgname/repo","username/repo"], "dryRun" : "true" } @@ -273,6 +274,7 @@ module.exports = { endpoint: 'https://self-hosted.gitlab/api/v4/', token: '**gitlab_token**', platform: 'gitlab', + logLevel: 'debug', onboardingConfig: { extends: ['config:base'], }, diff --git a/lib/config/config/__fixtures__/file.js b/lib/config/config/__fixtures__/file.js index e4fe0d85493f41..61600196f14b7d 100644 --- a/lib/config/config/__fixtures__/file.js +++ b/lib/config/config/__fixtures__/file.js @@ -1,4 +1,5 @@ // @ts-ignore module.exports = { token: 'abcdefg', + logLevel: 'error' }; diff --git a/lib/config/config/__fixtures__/with-force.js b/lib/config/config/__fixtures__/with-force.js index ad2b1074fb23d4..d16cae37e45284 100644 --- a/lib/config/config/__fixtures__/with-force.js +++ b/lib/config/config/__fixtures__/with-force.js @@ -1,6 +1,7 @@ // @ts-ignore module.exports = { token: 'abcdefg', + logLevel: 'error', force: { schedule: null, } diff --git a/lib/config/definitions.ts b/lib/config/definitions.ts index eef3b2ce37e613..f4c66a57d99fca 100644 --- a/lib/config/definitions.ts +++ b/lib/config/definitions.ts @@ -342,10 +342,11 @@ const options: RenovateOptions[] = [ // Log options { name: 'logLevel', - description: 'Logging level. Deprecated, use `LOG_LEVEL` environment.', + description: 'Logging level', stage: 'global', type: 'string', allowedValues: ['fatal', 'error', 'warn', 'info', 'debug', 'trace'], + default: 'info', env: 'LOG_LEVEL', }, { diff --git a/lib/config/file.spec.ts b/lib/config/file.spec.ts index f5eb41da982383..4eeed34655d5fd 100644 --- a/lib/config/file.spec.ts +++ b/lib/config/file.spec.ts @@ -37,6 +37,7 @@ describe('config/file', () => { "platform": "github", "token":"abcdef", "logFileLevel": "warn", + "logLevel": "info", "onboarding": false, "gitAuthor": "Renovate Bot " "onboardingConfig": { diff --git a/lib/config/index.ts b/lib/config/index.ts index 44c803cd2c4ecc..0f4699091c0fa9 100644 --- a/lib/config/index.ts +++ b/lib/config/index.ts @@ -80,14 +80,8 @@ export async function parseConfigs( delete config.privateKeyPath; } - // Deprecated set log level: https://github.com/renovatebot/renovate/issues/8291 - // istanbul ignore if - if (config.logLevel) { - logger.warn( - 'Configuring logLevel is deprecated. Use LOG_LEVEL environment variable instead' - ); - levels('stdout', config.logLevel); - } + // Set log level + levels('stdout', config.logLevel); if (config.logContext) { // This only has an effect if logContext was defined via file or CLI, otherwise it would already have been detected in env From bb67552c35f182ba4aed6d4b1da57080d046b09c Mon Sep 17 00:00:00 2001 From: Bastian Eicher Date: Thu, 14 Jan 2021 15:40:42 +0100 Subject: [PATCH 23/52] docs: improve Azure DevOps setup documentation (#8269) Co-authored-by: Jamie Magee Co-authored-by: Michael Kriese --- docs/usage/setup-azure-devops.md | 98 ++++++++++++++++---------------- 1 file changed, 49 insertions(+), 49 deletions(-) diff --git a/docs/usage/setup-azure-devops.md b/docs/usage/setup-azure-devops.md index c68074db36d793..a6614953bc8430 100644 --- a/docs/usage/setup-azure-devops.md +++ b/docs/usage/setup-azure-devops.md @@ -3,74 +3,74 @@ title: Azure DevOps setup description: How to setup Renovate for Azure DevOps --- -# Azure DevOps setup (for npm/Yarn) +# Azure DevOps setup 1. Create a brand new pipeline within Azure DevOps, and select your source ![Azure DevOps create new pipeline](assets/images/azure-devops-setup-1.png) 1. Select your repository 1. Within _Configure your pipeline_ select: **Starter pipeline file** ![Azure DevOps starter pipeline template](assets/images/azure-devops-setup-2.png) -1. Replace all contents with next npm/Yarn template (change all _PLACEHOLDER_ strings with your values): +1. Replace all contents with: ``` schedules: - cron: '0 3 * * *' displayName: 'Every day at 3am' branches: - include: - - master - always: true + include: [master] trigger: none - jobs: - - job: - pool: - vmImage: 'ubuntu-latest' - steps: - - checkout: none + pool: + vmImage: ubuntu-latest - - task: Bash@3 - displayName: Create .npmrc file - inputs: - targetType: inline - script: | - cat > .npmrc << EOF - PLACEHOLDER(.NPMRC CONTENTS) - EOF + steps: + - task: npmAuthenticate@0 + inputs: + workingFile: .npmrc - - task: npmAuthenticate@0 - displayName: npm Authenticate - inputs: - workingFile: .npmrc + - bash: | + git config --global user.email 'bot@renovateapp.com' + git config --global user.name 'Renovate Bot' + npx --userconfig .npmrc renovate + env: + TOKEN: $(System.AccessToken) + ``` + +1. Add a file named `.npmrc` to your repository with the following contents: + (replacing `YOUR-ORG` with your Azure DevOps organization and `YOUR-FEED` with your Azure Artifacts feed) + + ``` + registry=https://pkgs.dev.azure.com/YOUR-ORG/_packaging/YOUR-FEED/npm/registry/ + always-auth=true + ``` - - task: Bash@3 - displayName: Create renovate config - inputs: - targetType: inline - script: | - cat > config.js << EOF - module.exports = { - platform: 'azure', - endpoint: 'https://dev.azure.com/PLACEHOLDER(ORGANIZATION)/', - token: '$(System.AccessToken)', - npmrc: - '$(sed ':a;N;$!ba;s/\n/\\n/g' .npmrc)', - ignoreNpmrcFile: true, - repositories: ['PLACEHOLDER(PROJECT)/PLACEHOLDER(REPO NAME)'] - }; - EOF +1. Add a file named `config.js` to your repository with the following contents: + (replacing `YOUR-ORG` with your Azure DevOps organization and `YOUR-PROJECT/YOUR-REPO` with your Azure DevOps project and repository) - - task: Bash@3 - displayName: Run renovate - inputs: - targetType: inline - script: | - # Git credentials - git config --global user.email 'bot@renovateapp.com' - git config --global user.name 'Renovate Bot' - # Run renovate - npx --userconfig .npmrc renovate + ```javascript + module.exports = { + platform: 'azure', + endpoint: 'https://dev.azure.com/YOUR-ORG/', + token: process.env.TOKEN, + hostRules: [ + { + hostName: 'pkgs.dev.azure.com', + username: 'apikey', + password: process.env.TOKEN, + }, + ], + repositories: ['YOUR-PROJECT/YOUR-REPO'], + }; ``` 1. Additionally, you can add `renovate.json` with Renovate configurations in the root of the repo. [Read more about configurations options](https://docs.renovatebot.com/configuration-options/) + +If you wish to use a single Renovate pipeline to update multiple repositories: + +1. Add the names of the repositories to `config.js`. + +1. Make sure that the "Project Collection Build Service (YOUR-PROJECT)" user has the following permissions on the repositores: + - Contribute + - Contribute to pull requests + - Create branch From a72fb3cf74e20e1f7e985c73293a308cb0d6dc83 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sat, 16 Jan 2021 02:01:21 +0000 Subject: [PATCH 24/52] build(deps): update dependency bunyan to v1.8.15 (#8318) Co-authored-by: Renovate Bot --- package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index 39405b7b2167bd..d0be090d10ea1d 100644 --- a/package.json +++ b/package.json @@ -121,7 +121,7 @@ "@yarnpkg/parsers": "2.3.0", "aws-sdk": "2.827.0", "azure-devops-node-api": "10.2.0", - "bunyan": "1.8.14", + "bunyan": "1.8.15", "cacache": "15.0.5", "chalk": "4.1.0", "changelog-filename-regex": "2.0.1", diff --git a/yarn.lock b/yarn.lock index 27a02d7eb26e17..b90937164a6070 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2786,10 +2786,10 @@ builtins@^1.0.3: resolved "https://registry.yarnpkg.com/builtins/-/builtins-1.0.3.tgz#cb94faeb61c8696451db36534e1422f94f0aee88" integrity sha1-y5T662HIaWRR2zZTThQi+U8K7og= -bunyan@1.8.14: - version "1.8.14" - resolved "https://registry.yarnpkg.com/bunyan/-/bunyan-1.8.14.tgz#3d8c1afea7de158a5238c7cb8a66ab6b38dd45b4" - integrity sha512-LlahJUxXzZLuw/hetUQJmRgZ1LF6+cr5TPpRj6jf327AsiIq2jhYEH4oqUUkVKTor+9w2BT3oxVwhzE5lw9tcg== +bunyan@1.8.15: + version "1.8.15" + resolved "https://registry.yarnpkg.com/bunyan/-/bunyan-1.8.15.tgz#8ce34ca908a17d0776576ca1b2f6cbd916e93b46" + integrity sha512-0tECWShh6wUysgucJcBAoYegf3JJoZWibxdqhTm7OHPeT42qdjkZ29QCMcKwbgU1kiH+auSIasNRXMLWXafXig== optionalDependencies: dtrace-provider "~0.8" moment "^2.19.3" From bc0bc71d8386a1e5f47dc4d56daf2abae6f6b654 Mon Sep 17 00:00:00 2001 From: Sergei Zharinov Date: Sat, 16 Jan 2021 10:26:30 +0400 Subject: [PATCH 25/52] feat(gradle-lite): Support method-based variable assignment via "set()" (#8315) --- lib/manager/gradle-lite/parser.spec.ts | 18 +++++++++++++++++- lib/manager/gradle-lite/parser.ts | 13 +++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/lib/manager/gradle-lite/parser.spec.ts b/lib/manager/gradle-lite/parser.spec.ts index 437a77ceefeb8f..072126bdc08645 100644 --- a/lib/manager/gradle-lite/parser.spec.ts +++ b/lib/manager/gradle-lite/parser.spec.ts @@ -16,7 +16,23 @@ describe('manager/gradle-lite/parser', () => { let deps; ({ deps } = parseGradle( - '\nversion = "1.2.3"\n"foo:bar:$version"\nversion = "3.2.1"' + ['version = "1.2.3"', '"foo:bar:$version"', 'version = "3.2.1"'].join( + '\n' + ) + )); + expect(deps).toMatchObject([ + { + depName: 'foo:bar', + currentValue: '1.2.3', + }, + ]); + + ({ deps } = parseGradle( + [ + 'set("version", "1.2.3")', + '"foo:bar:$version"', + 'set("version", "3.2.1")', + ].join('\n') )); expect(deps).toMatchObject([ { diff --git a/lib/manager/gradle-lite/parser.ts b/lib/manager/gradle-lite/parser.ts index e570e523b53049..4e3ab1f64b8cc8 100644 --- a/lib/manager/gradle-lite/parser.ts +++ b/lib/manager/gradle-lite/parser.ts @@ -259,6 +259,19 @@ const matcherConfigs: SyntaxMatchConfig[] = [ ], handler: handleAssignment, }, + { + // set('foo', 'bar') + matchers: [ + { matchType: TokenType.Word, matchValue: 'set' }, + { matchType: TokenType.LeftParen }, + { matchType: TokenType.String, tokenMapKey: 'keyToken' }, + { matchType: TokenType.Comma }, + { matchType: TokenType.String, tokenMapKey: 'valToken' }, + { matchType: TokenType.RightParen }, + endOfInstruction, + ], + handler: handleAssignment, + }, { // 'foo.bar:baz:1.2.3' matchers: [ From af35a32152bde38152bc8e4ac7b406281d80cde2 Mon Sep 17 00:00:00 2001 From: Sergei Zharinov Date: Sat, 16 Jan 2021 20:42:53 +0400 Subject: [PATCH 26/52] feat(gradle-lite): Recognize platform dependencies (#8322) --- lib/manager/gradle-lite/parser.spec.ts | 10 ++++++++++ lib/manager/gradle-lite/parser.ts | 20 ++++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/lib/manager/gradle-lite/parser.spec.ts b/lib/manager/gradle-lite/parser.spec.ts index 072126bdc08645..1be7ddbf29e719 100644 --- a/lib/manager/gradle-lite/parser.spec.ts +++ b/lib/manager/gradle-lite/parser.spec.ts @@ -84,6 +84,16 @@ describe('manager/gradle-lite/parser', () => { }, ]); + ({ deps } = parseGradle( + "implementation platform(group: 'foo', name: 'bar', version: '1.2.3')" + )); + expect(deps).toMatchObject([ + { + depName: 'foo:bar', + currentValue: '1.2.3', + }, + ]); + ({ deps } = parseGradle( 'group: "com.example", name: "my.dependency", version: depVersion' )); diff --git a/lib/manager/gradle-lite/parser.ts b/lib/manager/gradle-lite/parser.ts index 4e3ab1f64b8cc8..6c6659279bfda9 100644 --- a/lib/manager/gradle-lite/parser.ts +++ b/lib/manager/gradle-lite/parser.ts @@ -376,6 +376,26 @@ const matcherConfigs: SyntaxMatchConfig[] = [ ], handler: processLongFormDep, }, + { + // (group: "com.example", name: "my.dependency", version: "1.2.3") + matchers: [ + { matchType: TokenType.LeftParen }, + { matchType: TokenType.Word, matchValue: 'group' }, + { matchType: TokenType.Colon }, + { matchType: potentialStringTypes, tokenMapKey: 'groupId' }, + { matchType: TokenType.Comma }, + { matchType: TokenType.Word, matchValue: 'name' }, + { matchType: TokenType.Colon }, + { matchType: potentialStringTypes, tokenMapKey: 'artifactId' }, + { matchType: TokenType.Comma }, + { matchType: TokenType.Word, matchValue: 'version' }, + { matchType: TokenType.Colon }, + { matchType: potentialStringTypes, tokenMapKey: 'version' }, + { matchType: TokenType.RightParen }, + endOfInstruction, + ], + handler: processLongFormDep, + }, { // ("com.example", "my.dependency", "1.2.3") matchers: [ From 48a0127a9628158253f2f46eb329d05e6c4ed042 Mon Sep 17 00:00:00 2001 From: Marius Carp Date: Sun, 17 Jan 2021 09:48:00 +0200 Subject: [PATCH 27/52] feat(sbt): support bumpVersion (#8063) Closes #7962 --- docs/usage/configuration-options.md | 2 +- .../__fixtures__/missing-scala-version.sbt | 2 + lib/manager/sbt/__fixtures__/sample.sbt | 2 + .../__fixtures__/scala-version-variable.sbt | 2 + .../sbt/__snapshots__/extract.spec.ts.snap | 7 +++ .../sbt/__snapshots__/update.spec.ts.snap | 15 ++++++ lib/manager/sbt/extract.ts | 19 ++++++- lib/manager/sbt/index.ts | 1 + lib/manager/sbt/update.spec.ts | 49 +++++++++++++++++++ lib/manager/sbt/update.ts | 32 ++++++++++++ 10 files changed, 128 insertions(+), 3 deletions(-) create mode 100644 lib/manager/sbt/__snapshots__/update.spec.ts.snap create mode 100644 lib/manager/sbt/update.spec.ts create mode 100644 lib/manager/sbt/update.ts diff --git a/docs/usage/configuration-options.md b/docs/usage/configuration-options.md index efe77cf0d372db..d778065ade4787 100644 --- a/docs/usage/configuration-options.md +++ b/docs/usage/configuration-options.md @@ -251,7 +251,7 @@ This is an advance field and it's recommend you seek a config review before appl ## bumpVersion -Currently this setting supports `helmv3` and `npm` only, so raise a feature request if you have a use for it with other package managers. +Currently this setting supports `helmv3`, `npm` and `sbt` only, so raise a feature request if you have a use for it with other package managers. Its purpose is if you want Renovate to update the `version` field within your file's `package.json` any time it updates dependencies within. Usually this is for automatic release purposes, so that you don't need to add another step after Renovate before you can release a new version. diff --git a/lib/manager/sbt/__fixtures__/missing-scala-version.sbt b/lib/manager/sbt/__fixtures__/missing-scala-version.sbt index decd7f9a5dfc87..d75a13876a7d3e 100644 --- a/lib/manager/sbt/__fixtures__/missing-scala-version.sbt +++ b/lib/manager/sbt/__fixtures__/missing-scala-version.sbt @@ -1,3 +1,5 @@ +version := "1.0.1" + libraryDependencies ++= Seq( "org.scalatest" %% "scalatest" % "3.0.0" ) diff --git a/lib/manager/sbt/__fixtures__/sample.sbt b/lib/manager/sbt/__fixtures__/sample.sbt index 38a9378500cae5..a2f4e94dcdca27 100644 --- a/lib/manager/sbt/__fixtures__/sample.sbt +++ b/lib/manager/sbt/__fixtures__/sample.sbt @@ -1,3 +1,5 @@ +version := "1.0" + scalaVersion := "2.9.10" // libraryDependencies += "org.example" % "foo" % "0.0.0" diff --git a/lib/manager/sbt/__fixtures__/scala-version-variable.sbt b/lib/manager/sbt/__fixtures__/scala-version-variable.sbt index fd0abe2efd43aa..0d260be8e438f5 100644 --- a/lib/manager/sbt/__fixtures__/scala-version-variable.sbt +++ b/lib/manager/sbt/__fixtures__/scala-version-variable.sbt @@ -1,6 +1,8 @@ val ScalaVersion = "2.12.10" val versionExample = "0.0.8" +version := "3.2.1" + scalaVersion := ScalaVersion // libraryDependencies += "org.example" % "foo" % "0.0.0" diff --git a/lib/manager/sbt/__snapshots__/extract.spec.ts.snap b/lib/manager/sbt/__snapshots__/extract.spec.ts.snap index 37f113d6615339..4c70a362425226 100644 --- a/lib/manager/sbt/__snapshots__/extract.spec.ts.snap +++ b/lib/manager/sbt/__snapshots__/extract.spec.ts.snap @@ -33,6 +33,7 @@ Object { ], }, ], + "packageFileVersion": undefined, } `; @@ -69,6 +70,7 @@ Object { ], }, ], + "packageFileVersion": undefined, } `; @@ -238,6 +240,7 @@ Object { ], }, ], + "packageFileVersion": "1.0", } `; @@ -377,6 +380,7 @@ Object { ], }, ], + "packageFileVersion": "3.2.1", } `; @@ -393,6 +397,7 @@ Object { ], }, ], + "packageFileVersion": undefined, } `; @@ -419,6 +424,7 @@ Object { ], }, ], + "packageFileVersion": undefined, } `; @@ -447,5 +453,6 @@ Object { ], }, ], + "packageFileVersion": "1.0.1", } `; diff --git a/lib/manager/sbt/__snapshots__/update.spec.ts.snap b/lib/manager/sbt/__snapshots__/update.spec.ts.snap new file mode 100644 index 00000000000000..f98fb1492b88bf --- /dev/null +++ b/lib/manager/sbt/__snapshots__/update.spec.ts.snap @@ -0,0 +1,15 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`lib/manager/sbt/update .bumpPackageVersion() increments 1`] = ` +"name := \\"test\\" +organization := \\"test-org\\" +version := \\"0.0.3\\" +" +`; + +exports[`lib/manager/sbt/update .bumpPackageVersion() updates 1`] = ` +"name := \\"test\\" +organization := \\"test-org\\" +version := \\"0.1.0\\" +" +`; diff --git a/lib/manager/sbt/extract.ts b/lib/manager/sbt/extract.ts index d6c85c200ae047..6743b2e5d423a8 100644 --- a/lib/manager/sbt/extract.ts +++ b/lib/manager/sbt/extract.ts @@ -25,6 +25,15 @@ const isScalaVersion = (str: string): boolean => const getScalaVersion = (str: string): string => str.replace(/^\s*scalaVersion\s*:=\s*"/, '').replace(/"[\s,]*$/, ''); +const isPackageFileVersion = (str: string): boolean => + /^(version\s*:=\s*).*$/.test(str); + +const getPackageFileVersion = (str: string): string => + str + .replace(/^\s*version\s*:=\s*/, '') + .replace(/[\s,]*$/, '') + .replace(/"/g, ''); + /* https://www.scala-sbt.org/release/docs/Cross-Build.html#Publishing+conventions */ @@ -201,7 +210,7 @@ function parseSbtLine( ): (PackageFile & ParseOptions) | null { const { deps, registryUrls, variables } = acc; - let { isMultiDeps, scalaVersion } = acc; + let { isMultiDeps, scalaVersion, packageFileVersion } = acc; const ctx: ParseContext = { scalaVersion, @@ -225,6 +234,8 @@ function parseSbtLine( } else if (isScalaVersionVariable(line)) { isMultiDeps = false; scalaVersionVariable = getScalaVersionVariable(line); + } else if (isPackageFileVersion(line)) { + packageFileVersion = getPackageFileVersion(line); } else if (isResolver(line)) { isMultiDeps = false; const url = getResolverUrl(line); @@ -292,10 +303,14 @@ function parseSbtLine( (scalaVersionVariable && variables[scalaVersionVariable] && normalizeScalaVersion(variables[scalaVersionVariable].val)), + packageFileVersion, }; } if (deps.length) { - return { deps }; + return { + deps, + packageFileVersion, + }; } return null; } diff --git a/lib/manager/sbt/index.ts b/lib/manager/sbt/index.ts index e587990399cc5a..017a1a8acb28c1 100644 --- a/lib/manager/sbt/index.ts +++ b/lib/manager/sbt/index.ts @@ -1,6 +1,7 @@ import * as ivyVersioning from '../../versioning/ivy'; export { extractPackageFile } from './extract'; +export { bumpPackageVersion } from './update'; export const defaultConfig = { fileMatch: ['\\.sbt$', 'project/[^/]*.scala$'], diff --git a/lib/manager/sbt/update.spec.ts b/lib/manager/sbt/update.spec.ts new file mode 100644 index 00000000000000..d88e94494f1267 --- /dev/null +++ b/lib/manager/sbt/update.spec.ts @@ -0,0 +1,49 @@ +import * as sbtUpdater from './update'; + +describe('lib/manager/sbt/update', () => { + describe('.bumpPackageVersion()', () => { + const content = + 'name := "test"\n' + + 'organization := "test-org"\n' + + 'version := "0.0.2"\n'; + + it('increments', () => { + const { bumpedContent } = sbtUpdater.bumpPackageVersion( + content, + '0.0.2', + 'patch' + ); + + expect(bumpedContent).toMatchSnapshot(); + expect(bumpedContent).not.toEqual(content); + }); + it('no ops', () => { + const { bumpedContent } = sbtUpdater.bumpPackageVersion( + content, + '0.0.1', + 'patch' + ); + + expect(bumpedContent).toEqual(content); + }); + it('updates', () => { + const { bumpedContent } = sbtUpdater.bumpPackageVersion( + content, + '0.0.1', + 'minor' + ); + + expect(bumpedContent).toMatchSnapshot(); + expect(bumpedContent).not.toEqual(content); + }); + it('returns content if bumping errors', () => { + const { bumpedContent } = sbtUpdater.bumpPackageVersion( + content, + '0.0.2', + true as any + ); + + expect(bumpedContent).toEqual(content); + }); + }); +}); diff --git a/lib/manager/sbt/update.ts b/lib/manager/sbt/update.ts new file mode 100644 index 00000000000000..11949283b0e6c5 --- /dev/null +++ b/lib/manager/sbt/update.ts @@ -0,0 +1,32 @@ +import { ReleaseType, inc } from 'semver'; +import { logger } from '../../logger'; +import { BumpPackageVersionResult } from '../common'; + +export function bumpPackageVersion( + content: string, + currentValue: string, + bumpVersion: ReleaseType | string +): BumpPackageVersionResult { + logger.debug( + { bumpVersion, currentValue }, + 'Checking if we should bump build.sbt version' + ); + let bumpedContent = content; + const newVersion = inc(currentValue, bumpVersion as ReleaseType); + if (!newVersion) { + logger.warn('Version incremental failed'); + return { bumpedContent }; + } + bumpedContent = content.replace( + /^(version\s*:=\s*).*$/m, + `$1"${newVersion}"` + ); + + if (bumpedContent === content) { + logger.debug('Version was already bumped'); + } else { + logger.debug({ newVersion }, 'Bumped build.sbt version'); + } + + return { bumpedContent }; +} From b21e8a1909ead567aa183616c3e4c54a8d7ad054 Mon Sep 17 00:00:00 2001 From: Amos Wenger Date: Sun, 17 Jan 2021 09:18:32 +0100 Subject: [PATCH 28/52] feat(rust): Private crate registry support (rust/cargo) (#8083) Co-authored-by: Rhys Arkins --- docs/usage/rust.md | 29 ++ lib/datasource/common.ts | 1 + lib/datasource/crate/__fixtures__/mypkg | 2 + .../crate/__snapshots__/index.spec.ts.snap | 30 ++ lib/datasource/crate/index.spec.ts | 177 ++++++++++- lib/datasource/crate/index.ts | 278 ++++++++++++++---- lib/manager/cargo/__fixtures__/Cargo.6.toml | 11 + .../cargo/__fixtures__/cargo.6.config.toml | 5 + .../cargo/__snapshots__/extract.spec.ts.snap | 159 ++++++++++ lib/manager/cargo/extract.spec.ts | 131 +++++++-- lib/manager/cargo/extract.ts | 114 +++++-- lib/manager/cargo/types.ts | 34 ++- lib/util/fs/index.ts | 9 + lib/workers/repository/index.ts | 7 +- lib/workers/repository/init/cache.ts | 5 + 15 files changed, 884 insertions(+), 108 deletions(-) create mode 100644 docs/usage/rust.md create mode 100644 lib/datasource/crate/__fixtures__/mypkg create mode 100644 lib/manager/cargo/__fixtures__/Cargo.6.toml create mode 100644 lib/manager/cargo/__fixtures__/cargo.6.config.toml diff --git a/docs/usage/rust.md b/docs/usage/rust.md new file mode 100644 index 00000000000000..0bab1c332912a4 --- /dev/null +++ b/docs/usage/rust.md @@ -0,0 +1,29 @@ +--- +title: Rust crates +description: Rust crates support in Renovate +--- + +# Automated Dependency Updates for Rust crates + +Renovate supports upgrading dependencies in `Cargo.toml` files and their accompanying `Cargo.lock` checksums. + +## How it works + +1. Renovate searches in each repository for any `Cargo.toml` files +1. Renovate extracts existing dependencies from `[dependencies]`, `[dev-dependencies]` and `[build-dependencies]` +1. Renovate attempts to find and parse a `.cargo/config.toml` file to discover index URLs for private registries +1. Renovate resolves the dependency's version using the crates.io API or by cloning the index URL +1. If Renovate finds an update, Renovate will use `cargo update` to update both `Cargo.toml` and `Cargo.lock` + +## Enabling Rust Modules Updating + +Renovate updates Rust crates by default. + +## Private crate registries and private git dependencies + +If any dependencies are hosted in private git repositories, [Git Authentication for cargo](https://doc.rust-lang.org/cargo/appendix/git-authentication.html) must be set up. + +If any dependencies are hosted on private crate registries (ie., not on `crates.io`), then credentials should be set up in such a way that the Git command-line is able to clone the registry index. +Third-party crate registries usually provide instructions to achieve this. + +Both of these are currently only possible when running Renovate self-hosted. diff --git a/lib/datasource/common.ts b/lib/datasource/common.ts index bd89e77d591c7e..a8f26f015aa30b 100644 --- a/lib/datasource/common.ts +++ b/lib/datasource/common.ts @@ -13,6 +13,7 @@ interface ReleasesConfigBase { constraints?: Record; npmrc?: string; registryUrls?: string[]; + trustLevel?: 'low' | 'high'; } export interface GetReleasesConfig extends ReleasesConfigBase { diff --git a/lib/datasource/crate/__fixtures__/mypkg b/lib/datasource/crate/__fixtures__/mypkg new file mode 100644 index 00000000000000..e8341b12070a6d --- /dev/null +++ b/lib/datasource/crate/__fixtures__/mypkg @@ -0,0 +1,2 @@ +{"name":"mypkg","vers":"0.1.0","deps":[{"name":"time","req":"^0.1","features":[],"optional":false,"default_features":true,"target":null,"kind":"normal"},{"name":"threadpool","req":"^0.1.4","features":[],"optional":false,"default_features":true,"target":null,"kind":"normal"}],"cksum":"2884e8cf8d78d9a6de8bbc1898603edb2b625eb8f64e371919906d0fec5660e7","features":{},"yanked":false} +{"name":"mypkg","vers":"0.1.1","deps":[{"name":"threadpool","req":"^0.1.4","features":[],"optional":false,"default_features":true,"target":null,"kind":"normal"},{"name":"time","req":"^0.1","features":[],"optional":false,"default_features":true,"target":null,"kind":"normal"}],"cksum":"878a28535503732a2a6394c1f4e6a310e181bfe2fa8dd39f3ae926e91f8666cf","features":{},"yanked":false} diff --git a/lib/datasource/crate/__snapshots__/index.spec.ts.snap b/lib/datasource/crate/__snapshots__/index.spec.ts.snap index 12358777c1acce..1030faab7da9cf 100644 --- a/lib/datasource/crate/__snapshots__/index.spec.ts.snap +++ b/lib/datasource/crate/__snapshots__/index.spec.ts.snap @@ -1,5 +1,33 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP +exports[`datasource/crate getReleases clones cloudsmith private registry 1`] = ` +Object { + "dependencyUrl": "https://cloudsmith.io/~myorg/repos/myrepo/packages/detail/cargo/mypkg", + "releases": Array [ + Object { + "version": "0.1.0", + }, + Object { + "version": "0.1.1", + }, + ], +} +`; + +exports[`datasource/crate getReleases clones other private registry 1`] = ` +Object { + "dependencyUrl": "https://github.com/mcorbin/testregistry/mypkg", + "releases": Array [ + Object { + "version": "0.1.0", + }, + Object { + "version": "0.1.1", + }, + ], +} +`; + exports[`datasource/crate getReleases processes real data: amethyst 1`] = ` Object { "dependencyUrl": "https://crates.io/crates/amethyst", @@ -416,6 +444,8 @@ Array [ ] `; +exports[`datasource/crate getReleases refuses to clone if trustLevel is not high 1`] = `null`; + exports[`datasource/crate getReleases returns null for 404 1`] = ` Array [ Object { diff --git a/lib/datasource/crate/index.spec.ts b/lib/datasource/crate/index.spec.ts index 2b83ea81e75d2a..7a4dcf9fea0a16 100644 --- a/lib/datasource/crate/index.spec.ts +++ b/lib/datasource/crate/index.spec.ts @@ -1,38 +1,106 @@ -import fs from 'fs'; +import fs from 'fs-extra'; +import _simpleGit from 'simple-git'; +import { DirectoryResult, dir } from 'tmp-promise'; +import { dirname, join } from 'upath'; import { getPkgReleases } from '..'; import * as httpMock from '../../../test/http-mock'; +import * as memCache from '../../util/cache/memory'; +import { setFsConfig } from '../../util/fs'; +import { + RegistryFlavor, + RegistryInfo, + id as datasource, + fetchCrateRecordsPayload, + getIndexSuffix, +} from '.'; -import { id as datasource, getIndexSuffix } from '.'; +jest.mock('simple-git'); +const simpleGit: any = _simpleGit; const res1 = fs.readFileSync('lib/datasource/crate/__fixtures__/libc', 'utf8'); const res2 = fs.readFileSync( 'lib/datasource/crate/__fixtures__/amethyst', 'utf8' ); +const res3 = fs.readFileSync('lib/datasource/crate/__fixtures__/mypkg', 'utf8'); const baseUrl = 'https://raw.githubusercontent.com/rust-lang/crates.io-index/master/'; +function setupGitMocks(): { mockClone: jest.Mock } { + const mockClone = jest + .fn() + .mockName('clone') + .mockImplementation((_registryUrl: string, clonePath: string, _opts) => { + const path = `${clonePath}/my/pk/mypkg`; + fs.mkdirSync(dirname(path), { recursive: true }); + fs.writeFileSync(path, res3, { encoding: 'utf8' }); + }); + + simpleGit.mockReturnValue({ + clone: mockClone, + }); + + return { mockClone }; +} + describe('datasource/crate', () => { describe('getIndexSuffix', () => { it('returns correct suffixes', () => { - expect(getIndexSuffix('a')).toBe('1/a'); - expect(getIndexSuffix('1')).toBe('1/1'); - expect(getIndexSuffix('1234567')).toBe('12/34/1234567'); - expect(getIndexSuffix('ab')).toBe('2/ab'); - expect(getIndexSuffix('abc')).toBe('3/a/abc'); - expect(getIndexSuffix('abcd')).toBe('ab/cd/abcd'); - expect(getIndexSuffix('abcde')).toBe('ab/cd/abcde'); + expect(getIndexSuffix('a')).toStrictEqual(['1', 'a']); + expect(getIndexSuffix('1')).toStrictEqual(['1', '1']); + expect(getIndexSuffix('1234567')).toStrictEqual(['12', '34', '1234567']); + expect(getIndexSuffix('ab')).toStrictEqual(['2', 'ab']); + expect(getIndexSuffix('abc')).toStrictEqual(['3', 'a', 'abc']); + expect(getIndexSuffix('abcd')).toStrictEqual(['ab', 'cd', 'abcd']); + expect(getIndexSuffix('abcde')).toStrictEqual(['ab', 'cd', 'abcde']); }); }); describe('getReleases', () => { + let tmpDir: DirectoryResult | null; + let localDir: string | null; + let cacheDir: string | null; + beforeEach(async () => { + tmpDir = await dir(); + localDir = join(tmpDir.path, 'local'); + cacheDir = join(tmpDir.path, 'cache'); + setFsConfig({ + localDir, + cacheDir, + }); + simpleGit.mockReset(); + memCache.init(); + }); + afterEach(() => { + fs.rmdirSync(tmpDir.path, { recursive: true }); + tmpDir = null; + }); + it('returns null for missing registry url', async () => { + expect( + await getPkgReleases({ + datasource, + depName: 'non_existent_crate', + registryUrls: [], + }) + ).toBeNull(); + }); + it('returns null for invalid registry url', async () => { + expect( + await getPkgReleases({ + datasource, + depName: 'non_existent_crate', + registryUrls: ['3'], + }) + ).toBeNull(); + }); it('returns null for empty result', async () => { httpMock.scope(baseUrl).get('/no/n_/non_existent_crate').reply(200, {}); expect( await getPkgReleases({ datasource, depName: 'non_existent_crate', + registryUrls: ['https://crates.io'], }) ).toBeNull(); expect(httpMock.getTrace()).toMatchSnapshot(); @@ -46,6 +114,7 @@ describe('datasource/crate', () => { await getPkgReleases({ datasource, depName: 'non_existent_crate', + registryUrls: ['https://crates.io'], }) ).toBeNull(); expect(httpMock.getTrace()).toMatchSnapshot(); @@ -56,6 +125,7 @@ describe('datasource/crate', () => { await getPkgReleases({ datasource, depName: 'non_existent_crate', + registryUrls: ['https://crates.io'], }) ).toBeNull(); expect(httpMock.getTrace()).toMatchSnapshot(); @@ -63,7 +133,11 @@ describe('datasource/crate', () => { it('returns null for 404', async () => { httpMock.scope(baseUrl).get('/so/me/some_crate').reply(404); expect( - await getPkgReleases({ datasource, depName: 'some_crate' }) + await getPkgReleases({ + datasource, + depName: 'some_crate', + registryUrls: ['https://crates.io'], + }) ).toBeNull(); expect(httpMock.getTrace()).toMatchSnapshot(); }); @@ -71,7 +145,11 @@ describe('datasource/crate', () => { httpMock.scope(baseUrl).get('/so/me/some_crate').reply(502); let e; try { - await getPkgReleases({ datasource, depName: 'some_crate' }); + await getPkgReleases({ + datasource, + depName: 'some_crate', + registryUrls: ['https://crates.io'], + }); } catch (err) { e = err; } @@ -82,7 +160,11 @@ describe('datasource/crate', () => { it('returns null for unknown error', async () => { httpMock.scope(baseUrl).get('/so/me/some_crate').replyWithError(''); expect( - await getPkgReleases({ datasource, depName: 'some_crate' }) + await getPkgReleases({ + datasource, + depName: 'some_crate', + registryUrls: ['https://crates.io'], + }) ).toBeNull(); expect(httpMock.getTrace()).toMatchSnapshot(); }); @@ -91,6 +173,7 @@ describe('datasource/crate', () => { const res = await getPkgReleases({ datasource, depName: 'libc', + registryUrls: ['https://crates.io'], }); expect(res).toMatchSnapshot(); expect(res).not.toBeNull(); @@ -102,11 +185,81 @@ describe('datasource/crate', () => { const res = await getPkgReleases({ datasource, depName: 'amethyst', + registryUrls: ['https://crates.io'], }); expect(res).toMatchSnapshot(); expect(res).not.toBeNull(); expect(res).toBeDefined(); expect(httpMock.getTrace()).toMatchSnapshot(); }); + it('refuses to clone if trustLevel is not high', async () => { + const { mockClone } = setupGitMocks(); + + const url = 'https://dl.cloudsmith.io/basic/myorg/myrepo/cargo/index.git'; + const res = await getPkgReleases({ + datasource, + depName: 'mypkg', + registryUrls: [url], + }); + expect(mockClone).toHaveBeenCalledTimes(0); + expect(res).toMatchSnapshot(); + expect(res).toBeNull(); + }); + it('clones cloudsmith private registry', async () => { + const { mockClone } = setupGitMocks(); + + const url = 'https://dl.cloudsmith.io/basic/myorg/myrepo/cargo/index.git'; + const res = await getPkgReleases({ + datasource, + depName: 'mypkg', + registryUrls: [url], + trustLevel: 'high', + }); + expect(mockClone).toHaveBeenCalled(); + expect(res).toMatchSnapshot(); + expect(res).not.toBeNull(); + expect(res).toBeDefined(); + }); + it('clones other private registry', async () => { + const { mockClone } = setupGitMocks(); + + const url = 'https://github.com/mcorbin/testregistry'; + const res = await getPkgReleases({ + datasource, + depName: 'mypkg', + registryUrls: [url], + trustLevel: 'high', + }); + expect(mockClone).toHaveBeenCalled(); + expect(res).toMatchSnapshot(); + expect(res).not.toBeNull(); + expect(res).toBeDefined(); + }); + it('clones once then reuses the cache', async () => { + const { mockClone } = setupGitMocks(); + + const url = 'https://github.com/mcorbin/othertestregistry'; + await getPkgReleases({ + datasource, + depName: 'mypkg', + registryUrls: [url], + trustLevel: 'high', + }); + await getPkgReleases({ + datasource, + depName: 'mypkg', + registryUrls: [url], + }); + expect(mockClone).toHaveBeenCalledTimes(1); + }); + }); + + describe('fetchCrateRecordsPayload', () => { + it('rejects if it has neither clonePath nor crates.io flavor', async () => { + const info: RegistryInfo = { + flavor: RegistryFlavor.Cloudsmith, + }; + await expect(fetchCrateRecordsPayload(info, 'benedict')).toReject(); + }); }); }); diff --git a/lib/datasource/crate/index.ts b/lib/datasource/crate/index.ts index 198e781a9d1295..807a28c0f0196d 100644 --- a/lib/datasource/crate/index.ts +++ b/lib/datasource/crate/index.ts @@ -1,31 +1,61 @@ +import hasha from 'hasha'; +import Git from 'simple-git'; +import { join } from 'upath'; +import { logger } from '../../logger'; import { ExternalHostError } from '../../types/errors/external-host-error'; +import * as memCache from '../../util/cache/memory'; import * as packageCache from '../../util/cache/package'; +import { privateCacheDir, readFile } from '../../util/fs'; import { Http } from '../../util/http'; import { GetReleasesConfig, Release, ReleaseResult } from '../common'; export const id = 'crate'; +export const defaultRegistryUrls = ['https://crates.io']; +export const registryStrategy = 'first'; const http = new Http(id); -const BASE_URL = +const CRATES_IO_BASE_URL = 'https://raw.githubusercontent.com/rust-lang/crates.io-index/master/'; -export function getIndexSuffix(lookupName: string): string { +export enum RegistryFlavor { + /** https://crates.io, supports rawgit access */ + CratesIo, + + /** https://cloudsmith.io, needs git clone */ + Cloudsmith, + + /** unknown, assuming private git repository */ + Other, +} + +export interface RegistryInfo { + flavor: RegistryFlavor; + + /** raw URL of the registry, as specified in cargo config */ + rawUrl?: string; + + /** parsed URL of the registry */ + url?: URL; + + /** path where the registry is cloned */ + clonePath?: string; +} + +export function getIndexSuffix(lookupName: string): string[] { const len = lookupName.length; if (len === 1) { - return '1/' + lookupName; + return ['1', lookupName]; } if (len === 2) { - return '2/' + lookupName; + return ['2', lookupName]; } if (len === 3) { - return '3/' + lookupName[0] + '/' + lookupName; + return ['3', lookupName[0], lookupName]; } - return ( - lookupName.slice(0, 2) + '/' + lookupName.slice(2, 4) + '/' + lookupName - ); + return [lookupName.slice(0, 2), lookupName.slice(2, 4), lookupName]; } interface CrateRecord { @@ -33,56 +63,200 @@ interface CrateRecord { yanked: boolean; } -export async function getReleases({ - lookupName, -}: GetReleasesConfig): Promise { - const cacheNamespace = 'datasource-crate'; - const cacheKey = lookupName; - const cachedResult = await packageCache.get( - cacheNamespace, - cacheKey - ); - // istanbul ignore if - if (cachedResult) { - return cachedResult; +export async function fetchCrateRecordsPayload( + info: RegistryInfo, + lookupName: string +): Promise { + if (info.clonePath) { + const path = join(info.clonePath, ...getIndexSuffix(lookupName)); + return readFile(path, 'utf8'); + } + + if (info.flavor === RegistryFlavor.CratesIo) { + const crateUrl = CRATES_IO_BASE_URL + getIndexSuffix(lookupName).join('/'); + try { + return (await http.get(crateUrl)).body; + } catch (err) { + if ( + err.statusCode === 429 || + (err.statusCode >= 500 && err.statusCode < 600) + ) { + throw new ExternalHostError(err); + } + } } - const crateUrl = BASE_URL + getIndexSuffix(lookupName); - const dependencyUrl = `https://crates.io/crates/${lookupName}`; + throw new Error(`unsupported crate registry flavor: ${info.flavor}`); +} + +/** + * Computes the dependency URL for a crate, given + * registry information + */ +function getDependencyUrl(info: RegistryInfo, lookupName: string): string { + switch (info.flavor) { + case RegistryFlavor.CratesIo: + return `https://crates.io/crates/${lookupName}`; + case RegistryFlavor.Cloudsmith: { + // input: https://dl.cloudsmith.io/basic/$org/$repo/cargo/index.git + const tokens = info.url.pathname.split('/'); + const org = tokens[2]; + const repo = tokens[3]; + return `https://cloudsmith.io/~${org}/repos/${repo}/packages/detail/cargo/${lookupName}`; + } + default: + return `${info.rawUrl}/${lookupName}`; + } +} + +/** + * Given a Git URL, computes a semi-human-readable name for a folder in which to + * clone the repository. + */ +function cacheDirFromUrl(url: URL): string { + const proto = url.protocol.replace(/:$/, ''); + const host = url.hostname; + const hash = hasha(url.pathname, { + algorithm: 'sha256', + }).substr(0, 7); + + return `crate-registry-${proto}-${host}-${hash}`; +} + +/** + * Fetches information about a registry, by url. + * If no url is given, assumes crates.io. + * If an url is given, assumes it's a valid Git repository + * url and clones it to cache. + */ +async function fetchRegistryInfo( + config: GetReleasesConfig, + registryUrl: string +): Promise { + let url: URL; try { - const lines = (await http.get(crateUrl)).body - .split('\n') // break into lines - .map((line) => line.trim()) // remove whitespace - .filter((line) => line.length !== 0) // remove empty lines - .map((line) => JSON.parse(line) as CrateRecord); // parse - const result: ReleaseResult = { - dependencyUrl, - releases: [], - }; - result.releases = lines - .map((version) => { - const release: Release = { - version: version.vers, - }; - if (version.yanked) { - release.isDeprecated = true; - } - return release; - }) - .filter((release) => release.version); - if (!result.releases.length) { + url = new URL(registryUrl); + } catch (err) { + logger.debug({ registryUrl }, 'could not parse registry URL'); + return null; + } + + let flavor: RegistryFlavor; + if (url.hostname === 'crates.io') { + flavor = RegistryFlavor.CratesIo; + } else if (url.hostname === 'dl.cloudsmith.io') { + flavor = RegistryFlavor.Cloudsmith; + } else { + flavor = RegistryFlavor.Other; + } + + const registry: RegistryInfo = { + flavor, + rawUrl: registryUrl, + url, + }; + + if (flavor !== RegistryFlavor.CratesIo) { + if (config.trustLevel !== 'high') { + logger.warn( + 'crate datasource: trustLevel=high is required for registries other than crates.io, bailing out' + ); return null; } + + const cacheKey = `crate-datasource/registry-clone-path/${registryUrl}`; + + let clonePath: string = memCache.get(cacheKey); + if (!clonePath) { + clonePath = join(privateCacheDir(), cacheDirFromUrl(url)); + logger.info({ clonePath, registryUrl }, `Cloning private cargo registry`); + { + const git = Git(); + await git.clone(registryUrl, clonePath, { + '--depth': 1, + }); + } + memCache.set(cacheKey, clonePath); + } + + registry.clonePath = clonePath; + } + + return registry; +} + +export function areReleasesCacheable(registryUrl: string): boolean { + // We only cache public releases, we don't want to cache private + // cloned data between runs. + return registryUrl === 'https://crates.io'; +} + +export async function getReleases( + config: GetReleasesConfig +): Promise { + const { lookupName, registryUrl } = config; + + // istanbul ignore if + if (!registryUrl) { + logger.warn( + 'crate datasource: No registryUrl specified, cannot perform getReleases' + ); + return null; + } + + const cacheable = areReleasesCacheable(registryUrl); + const cacheNamespace = 'datasource-crate'; + const cacheKey = `${registryUrl}/${lookupName}`; + + if (cacheable) { + const cachedResult = await packageCache.get( + cacheNamespace, + cacheKey + ); + // istanbul ignore if + if (cachedResult) { + logger.debug({ cacheKey }, 'Returning cached resource'); + return cachedResult; + } + } + + const registryInfo = await fetchRegistryInfo(config, registryUrl); + if (!registryInfo) { + logger.debug({ registryUrl }, 'Could not fetch registry info'); + return null; + } + + const dependencyUrl = getDependencyUrl(registryInfo, lookupName); + + const payload = await fetchCrateRecordsPayload(registryInfo, lookupName); + const lines = payload + .split('\n') // break into lines + .map((line) => line.trim()) // remove whitespace + .filter((line) => line.length !== 0) // remove empty lines + .map((line) => JSON.parse(line) as CrateRecord); // parse + const result: ReleaseResult = { + dependencyUrl, + releases: [], + }; + result.releases = lines + .map((version) => { + const release: Release = { + version: version.vers, + }; + if (version.yanked) { + release.isDeprecated = true; + } + return release; + }) + .filter((release) => release.version); + if (!result.releases.length) { + return null; + } + + if (cacheable) { const cacheMinutes = 10; await packageCache.set(cacheNamespace, cacheKey, result, cacheMinutes); - return result; - } catch (err) { - if ( - err.statusCode === 429 || - (err.statusCode >= 500 && err.statusCode < 600) - ) { - throw new ExternalHostError(err); - } - throw err; } + + return result; } diff --git a/lib/manager/cargo/__fixtures__/Cargo.6.toml b/lib/manager/cargo/__fixtures__/Cargo.6.toml new file mode 100644 index 00000000000000..18c8bd973d4486 --- /dev/null +++ b/lib/manager/cargo/__fixtures__/Cargo.6.toml @@ -0,0 +1,11 @@ +[package] +name = "renovate-test" +version = "0.1.0" +authors = ["John Doe "] +edition = "2018" + +[dependencies] +proprietary-crate = { version = "0.1.0", registry = "private-crates" } +mcorbin-test = { version = "3.0.0", registry = "mcorbin" } +tokio = "0.2" + diff --git a/lib/manager/cargo/__fixtures__/cargo.6.config.toml b/lib/manager/cargo/__fixtures__/cargo.6.config.toml new file mode 100644 index 00000000000000..16df4b86719734 --- /dev/null +++ b/lib/manager/cargo/__fixtures__/cargo.6.config.toml @@ -0,0 +1,5 @@ +[registries] +private-crates = { index = "https://dl.cloudsmith.io/basic/my-org/my-repo/cargo/index.git" } + +[registries.mcorbin] +index = "https://github.com/mcorbin/testregistry" \ No newline at end of file diff --git a/lib/manager/cargo/__snapshots__/extract.spec.ts.snap b/lib/manager/cargo/__snapshots__/extract.spec.ts.snap index 680911d6b39a17..03708bbbd228b3 100644 --- a/lib/manager/cargo/__snapshots__/extract.spec.ts.snap +++ b/lib/manager/cargo/__snapshots__/extract.spec.ts.snap @@ -437,6 +437,116 @@ Array [ ] `; +exports[`lib/manager/cargo/extract extractPackageFile() extracts registry urls from .cargo/config (legacy path) 1`] = ` +Array [ + Object { + "currentValue": "0.1.0", + "datasource": "crate", + "depName": "proprietary-crate", + "depType": "dependencies", + "managerData": Object { + "nestedVersion": true, + }, + "registryUrls": Array [ + "https://dl.cloudsmith.io/basic/my-org/my-repo/cargo/index.git", + ], + }, + Object { + "currentValue": "3.0.0", + "datasource": "crate", + "depName": "mcorbin-test", + "depType": "dependencies", + "managerData": Object { + "nestedVersion": true, + }, + "registryUrls": Array [ + "https://github.com/mcorbin/testregistry", + ], + }, + Object { + "currentValue": "0.2", + "datasource": "crate", + "depName": "tokio", + "depType": "dependencies", + "managerData": Object { + "nestedVersion": false, + }, + }, +] +`; + +exports[`lib/manager/cargo/extract extractPackageFile() extracts registry urls from .cargo/config.toml 1`] = ` +Array [ + Object { + "currentValue": "0.1.0", + "datasource": "crate", + "depName": "proprietary-crate", + "depType": "dependencies", + "managerData": Object { + "nestedVersion": true, + }, + "registryUrls": Array [ + "https://dl.cloudsmith.io/basic/my-org/my-repo/cargo/index.git", + ], + }, + Object { + "currentValue": "3.0.0", + "datasource": "crate", + "depName": "mcorbin-test", + "depType": "dependencies", + "managerData": Object { + "nestedVersion": true, + }, + "registryUrls": Array [ + "https://github.com/mcorbin/testregistry", + ], + }, + Object { + "currentValue": "0.2", + "datasource": "crate", + "depName": "tokio", + "depType": "dependencies", + "managerData": Object { + "nestedVersion": false, + }, + }, +] +`; + +exports[`lib/manager/cargo/extract extractPackageFile() fails to parse cargo config with invalid TOML 1`] = ` +Array [ + Object { + "currentValue": "0.1.0", + "datasource": "crate", + "depName": "proprietary-crate", + "depType": "dependencies", + "managerData": Object { + "nestedVersion": true, + }, + "skipReason": "unknown-registry", + }, + Object { + "currentValue": "3.0.0", + "datasource": "crate", + "depName": "mcorbin-test", + "depType": "dependencies", + "managerData": Object { + "nestedVersion": true, + }, + "skipReason": "unknown-registry", + }, + Object { + "currentValue": "0.2", + "datasource": "crate", + "depName": "tokio", + "depType": "dependencies", + "managerData": Object { + "nestedVersion": false, + }, + }, +] +`; + exports[`lib/manager/cargo/extract extractPackageFile() handles inline tables 1`] = ` Array [ Object { @@ -583,3 +693,52 @@ Array [ }, ] `; + +exports[`lib/manager/cargo/extract extractPackageFile() ignore cargo config registries with missing index 1`] = ` +Array [ + Object { + "currentValue": "0.1.0", + "datasource": "crate", + "depName": "proprietary-crate", + "depType": "dependencies", + "managerData": Object { + "nestedVersion": true, + }, + "skipReason": "unknown-registry", + }, + Object { + "currentValue": "3.0.0", + "datasource": "crate", + "depName": "mcorbin-test", + "depType": "dependencies", + "managerData": Object { + "nestedVersion": true, + }, + "skipReason": "unknown-registry", + }, + Object { + "currentValue": "0.2", + "datasource": "crate", + "depName": "tokio", + "depType": "dependencies", + "managerData": Object { + "nestedVersion": false, + }, + }, +] +`; + +exports[`lib/manager/cargo/extract extractPackageFile() skips unknown registries 1`] = ` +Array [ + Object { + "currentValue": "0.1.0", + "datasource": "crate", + "depName": "foobar", + "depType": "dependencies", + "managerData": Object { + "nestedVersion": true, + }, + "skipReason": "unknown-registry", + }, +] +`; diff --git a/lib/manager/cargo/extract.spec.ts b/lib/manager/cargo/extract.spec.ts index 54419a9cd32a7a..db7c269816b3e2 100644 --- a/lib/manager/cargo/extract.spec.ts +++ b/lib/manager/cargo/extract.spec.ts @@ -1,4 +1,7 @@ import { readFileSync } from 'fs'; +import { dir } from 'tmp-promise'; +import { join } from 'upath'; +import { setFsConfig, writeLocalFile } from '../../util/fs'; import { extractPackageFile } from './extract'; const cargo1toml = readFileSync( @@ -22,51 +25,143 @@ const cargo5toml = readFileSync( 'utf8' ); +const cargo6configtoml = readFileSync( + 'lib/manager/cargo/__fixtures__/cargo.6.config.toml', + 'utf8' +); +const cargo6toml = readFileSync( + 'lib/manager/cargo/__fixtures__/Cargo.6.toml', + 'utf8' +); + describe('lib/manager/cargo/extract', () => { describe('extractPackageFile()', () => { let config; beforeEach(() => { config = {}; }); - it('returns null for invalid toml', () => { - expect(extractPackageFile('invalid toml', config)).toBeNull(); + it('returns null for invalid toml', async () => { + expect( + await extractPackageFile('invalid toml', 'Cargo.toml', config) + ).toBeNull(); }); - it('returns null for empty dependencies', () => { + it('returns null for empty dependencies', async () => { const cargotoml = '[dependencies]\n'; - expect(extractPackageFile(cargotoml, config)).toBeNull(); + expect( + await extractPackageFile(cargotoml, 'Cargo.toml', config) + ).toBeNull(); }); - it('returns null for empty dev-dependencies', () => { + it('returns null for empty dev-dependencies', async () => { const cargotoml = '[dev-dependencies]\n'; - expect(extractPackageFile(cargotoml, config)).toBeNull(); + expect( + await extractPackageFile(cargotoml, 'Cargo.toml', config) + ).toBeNull(); }); - it('returns null for empty custom target', () => { + it('returns null for empty custom target', async () => { const cargotoml = '[target."foo".dependencies]\n'; - expect(extractPackageFile(cargotoml, config)).toBeNull(); + expect( + await extractPackageFile(cargotoml, 'Cargo.toml', config) + ).toBeNull(); }); - it('extracts multiple dependencies simple', () => { - const res = extractPackageFile(cargo1toml, config); + it('extracts multiple dependencies simple', async () => { + const res = await extractPackageFile(cargo1toml, 'Cargo.toml', config); expect(res.deps).toMatchSnapshot(); expect(res.deps).toHaveLength(15); }); - it('extracts multiple dependencies advanced', () => { - const res = extractPackageFile(cargo2toml, config); + it('extracts multiple dependencies advanced', async () => { + const res = await extractPackageFile(cargo2toml, 'Cargo.toml', config); expect(res.deps).toMatchSnapshot(); expect(res.deps).toHaveLength(18 + 6 + 1); }); - it('handles inline tables', () => { - const res = extractPackageFile(cargo3toml, config); + it('handles inline tables', async () => { + const res = await extractPackageFile(cargo3toml, 'Cargo.toml', config); expect(res.deps).toMatchSnapshot(); expect(res.deps).toHaveLength(8); }); - it('handles standard tables', () => { - const res = extractPackageFile(cargo4toml, config); + it('handles standard tables', async () => { + const res = await extractPackageFile(cargo4toml, 'Cargo.toml', config); expect(res.deps).toMatchSnapshot(); expect(res.deps).toHaveLength(6); }); - it('extracts platform specific dependencies', () => { - const res = extractPackageFile(cargo5toml, config); + it('extracts platform specific dependencies', async () => { + const res = await extractPackageFile(cargo5toml, 'Cargo.toml', config); expect(res.deps).toMatchSnapshot(); expect(res.deps).toHaveLength(4); }); + it('extracts registry urls from .cargo/config.toml', async () => { + const tmpDir = await dir(); + const localDir = join(tmpDir.path, 'local'); + const cacheDir = join(tmpDir.path, 'cache'); + setFsConfig({ + localDir, + cacheDir, + }); + await writeLocalFile('.cargo/config.toml', cargo6configtoml); + + const res = await extractPackageFile(cargo6toml, 'Cargo.toml', { + ...config, + localDir, + }); + expect(res.deps).toMatchSnapshot(); + expect(res.deps).toHaveLength(3); + }); + it('extracts registry urls from .cargo/config (legacy path)', async () => { + const tmpDir = await dir(); + const localDir = join(tmpDir.path, 'local'); + const cacheDir = join(tmpDir.path, 'cache'); + setFsConfig({ + localDir, + cacheDir, + }); + await writeLocalFile('.cargo/config', cargo6configtoml); + + const res = await extractPackageFile(cargo6toml, 'Cargo.toml', { + ...config, + localDir, + }); + expect(res.deps).toMatchSnapshot(); + expect(res.deps).toHaveLength(3); + }); + it('skips unknown registries', async () => { + const cargotoml = + '[dependencies]\nfoobar = { version = "0.1.0", registry = "not-listed" }'; + const res = await extractPackageFile(cargotoml, 'Cargo.toml', config); + expect(res.deps).toMatchSnapshot(); + expect(res.deps).toHaveLength(1); + }); + it('fails to parse cargo config with invalid TOML', async () => { + const tmpDir = await dir(); + const localDir = join(tmpDir.path, 'local'); + const cacheDir = join(tmpDir.path, 'cache'); + setFsConfig({ + localDir, + cacheDir, + }); + await writeLocalFile('.cargo/config', '[registries'); + + const res = await extractPackageFile(cargo6toml, 'Cargo.toml', { + ...config, + localDir, + }); + expect(res.deps).toMatchSnapshot(); + expect(res.deps).toHaveLength(3); + }); + it('ignore cargo config registries with missing index', async () => { + const tmpDir = await dir(); + const localDir = join(tmpDir.path, 'local'); + const cacheDir = join(tmpDir.path, 'cache'); + setFsConfig({ + localDir, + cacheDir, + }); + await writeLocalFile('.cargo/config', '[registries.mine]\nfoo = "bar"'); + + const res = await extractPackageFile(cargo6toml, 'Cargo.toml', { + ...config, + localDir, + }); + expect(res.deps).toMatchSnapshot(); + expect(res.deps).toHaveLength(3); + }); }); }); diff --git a/lib/manager/cargo/extract.ts b/lib/manager/cargo/extract.ts index 17fcb1ac5aa5e9..052858d717e3f6 100644 --- a/lib/manager/cargo/extract.ts +++ b/lib/manager/cargo/extract.ts @@ -2,12 +2,19 @@ import { parse } from '@iarna/toml'; import * as datasourceCrate from '../../datasource/crate'; import { logger } from '../../logger'; import { SkipReason } from '../../types'; -import { PackageDependency, PackageFile } from '../common'; -import { CargoConfig, CargoSection } from './types'; +import { readLocalFile } from '../../util/fs'; +import { ExtractConfig, PackageDependency, PackageFile } from '../common'; +import { + CargoConfig, + CargoManifest, + CargoRegistries, + CargoSection, +} from './types'; function extractFromSection( parsedContent: CargoSection, section: keyof CargoSection, + cargoRegistries: CargoRegistries, target?: string ): PackageDependency[] { const deps: PackageDependency[] = []; @@ -16,16 +23,27 @@ function extractFromSection( return []; } Object.keys(sectionContent).forEach((depName) => { - let skipReason: SkipReason; + let skipReason: SkipReason | undefined; let currentValue = sectionContent[depName]; let nestedVersion = false; + let registryUrls: string[] | undefined; + if (typeof currentValue !== 'string') { const version = currentValue.version; const path = currentValue.path; const git = currentValue.git; + const registryName = currentValue.registry; if (version) { currentValue = version; nestedVersion = true; + if (registryName) { + const registryUrl = cargoRegistries[registryName]; + if (registryUrl) { + registryUrls = [registryUrl]; + } else { + skipReason = SkipReason.UnknownRegistry; + } + } if (path) { skipReason = SkipReason.PathDependency; } @@ -50,6 +68,9 @@ function extractFromSection( managerData: { nestedVersion }, datasource: datasourceCrate.id, }; + if (registryUrls) { + dep.registryUrls = registryUrls; + } if (skipReason) { dep.skipReason = skipReason; } @@ -61,15 +82,59 @@ function extractFromSection( return deps; } -export function extractPackageFile( +/** Reads `.cargo/config.toml`, or, if not found, `.cargo/config` */ +async function readCargoConfig(): Promise { + for (const configName of ['config.toml', 'config']) { + const path = `.cargo/${configName}`; + const payload = await readLocalFile(path, 'utf8'); + if (payload) { + try { + return parse(payload) as CargoConfig; + } catch (err) { + logger.debug({ err }, `Error parsing ${path}`); + } + break; + } + } + + logger.debug('Neither .cargo/config nor .cargo/config.toml found'); + return null; +} + +/** Extracts a map of cargo registries from a CargoConfig */ +function extractCargoRegistries(config: CargoConfig | null): CargoRegistries { + const result: CargoRegistries = {}; + if (!config?.registries) { + return result; + } + + const { registries } = config; + + for (const registryName of Object.keys(registries)) { + const registry = registries[registryName]; + if (registry.index) { + result[registryName] = registry.index; + } else { + logger.debug({ registryName }, 'cargo registry is missing index'); + } + } + + return result; +} + +export async function extractPackageFile( content: string, - fileName: string -): PackageFile | null { + fileName: string, + _config?: ExtractConfig +): Promise { logger.trace(`cargo.extractPackageFile(${fileName})`); - let parsedContent: CargoConfig; + + const cargoConfig = await readCargoConfig(); + const cargoRegistries = extractCargoRegistries(cargoConfig); + + let cargoManifest: CargoManifest; try { - // TODO: fix type - parsedContent = parse(content) as any; + cargoManifest = parse(content); } catch (err) { logger.debug({ err }, 'Error parsing Cargo.toml file'); return null; @@ -81,26 +146,41 @@ export function extractPackageFile( [build-dependencies] [target.*.dependencies] */ - const targetSection = parsedContent.target; + const targetSection = cargoManifest.target; // An array of all dependencies in the target section let targetDeps: PackageDependency[] = []; if (targetSection) { const targets = Object.keys(targetSection); targets.forEach((target) => { - const targetContent = parsedContent.target[target]; + const targetContent = targetSection[target]; // Dependencies for `${target}` const deps = [ - ...extractFromSection(targetContent, 'dependencies', target), - ...extractFromSection(targetContent, 'dev-dependencies', target), - ...extractFromSection(targetContent, 'build-dependencies', target), + ...extractFromSection( + targetContent, + 'dependencies', + cargoRegistries, + target + ), + ...extractFromSection( + targetContent, + 'dev-dependencies', + cargoRegistries, + target + ), + ...extractFromSection( + targetContent, + 'build-dependencies', + cargoRegistries, + target + ), ]; targetDeps = targetDeps.concat(deps); }); } const deps = [ - ...extractFromSection(parsedContent, 'dependencies'), - ...extractFromSection(parsedContent, 'dev-dependencies'), - ...extractFromSection(parsedContent, 'build-dependencies'), + ...extractFromSection(cargoManifest, 'dependencies', cargoRegistries), + ...extractFromSection(cargoManifest, 'dev-dependencies', cargoRegistries), + ...extractFromSection(cargoManifest, 'build-dependencies', cargoRegistries), ...targetDeps, ]; if (!deps.length) { diff --git a/lib/manager/cargo/types.ts b/lib/manager/cargo/types.ts index e22022b7c7bebf..27dc73f5ee7072 100644 --- a/lib/manager/cargo/types.ts +++ b/lib/manager/cargo/types.ts @@ -1,17 +1,35 @@ export interface CargoDep { - path: any; - git: any; - version: any; + /** Path on disk to the crate sources */ + path?: string; + /** Git URL for the dependency */ + git?: string; + /** Semver version */ + version?: string; + /** Name of a registry whose URL is configured in `.cargo/config.toml` */ + registry?: string; } export type CargoDeps = Record; export interface CargoSection { - dependencies: CargoDeps; - 'dev-dependencies': CargoDeps; - 'build-dependencies': CargoDeps; + dependencies?: CargoDeps; + 'dev-dependencies'?: CargoDeps; + 'build-dependencies'?: CargoDeps; } -export interface CargoConfig extends CargoSection { - target: Record; +export interface CargoManifest extends CargoSection { + target?: Record; +} + +export interface CargoConfig { + registries?: Record; +} + +export interface CargoRegistry { + index?: string; +} + +export interface CargoRegistries { + // maps registry names to URLs + [key: string]: string; } diff --git a/lib/util/fs/index.ts b/lib/util/fs/index.ts index 75d3fc12468104..da73e844283bc3 100644 --- a/lib/util/fs/index.ts +++ b/lib/util/fs/index.ts @@ -80,6 +80,15 @@ export async function ensureCacheDir( return cacheDirName; } +/** + * Return the path of the private cache directory. This directory is wiped + * between repositories, so they can be used to store private registries' index + * without risk of that information leaking to other repositories/users. + */ +export function privateCacheDir(): string { + return join(cacheDir, '__renovate-private-cache'); +} + export function localPathExists(pathName: string): Promise { // Works for both files as well as directories return fs diff --git a/lib/workers/repository/index.ts b/lib/workers/repository/index.ts index 770d20fd71995e..370e1d3d14dfae 100644 --- a/lib/workers/repository/index.ts +++ b/lib/workers/repository/index.ts @@ -1,7 +1,7 @@ import fs from 'fs-extra'; import { RenovateConfig } from '../../config'; import { logger, setMeta } from '../../logger'; -import { deleteLocalFile } from '../../util/fs'; +import { deleteLocalFile, privateCacheDir } from '../../util/fs'; import * as queue from '../../util/http/queue'; import { addSplit, getSplits, splitInit } from '../../util/split'; import { setBranchCache } from './cache'; @@ -62,6 +62,11 @@ export async function renovateRepository( logger.warn({ err }, 'localDir deletion error'); } } + try { + await fs.remove(privateCacheDir()); + } catch (err) /* istanbul ignore if */ { + logger.warn({ err }, 'privateCacheDir deletion error'); + } const splits = getSplits(); logger.debug(splits, 'Repository timing splits (milliseconds)'); printRequestStats(); diff --git a/lib/workers/repository/init/cache.ts b/lib/workers/repository/init/cache.ts index d2f4587f6693f1..40995f8b18c253 100644 --- a/lib/workers/repository/init/cache.ts +++ b/lib/workers/repository/init/cache.ts @@ -1,8 +1,13 @@ +import * as fs from 'fs-extra'; import { RenovateConfig } from '../../../config'; import * as memCache from '../../../util/cache/memory'; import * as repositoryCache from '../../../util/cache/repository'; +import { privateCacheDir } from '../../../util/fs'; export async function initializeCaches(config: RenovateConfig): Promise { memCache.init(); await repositoryCache.initialize(config); + + await fs.remove(privateCacheDir()); + await fs.ensureDir(privateCacheDir()); } From 12cc30779a74ce29935dc4c66499e0126d0a0cf2 Mon Sep 17 00:00:00 2001 From: Sergei Zharinov Date: Mon, 18 Jan 2021 01:20:17 +0400 Subject: [PATCH 29/52] feat(maven): Parse maven dependency.scope into depType (#8325) * feat(maven): Parse maven dependency.scope into depType * Fix snapshot --- lib/manager/maven/__fixtures__/simple.pom.xml | 1 + lib/manager/maven/__snapshots__/extract.spec.ts.snap | 9 +++++---- lib/manager/maven/__snapshots__/index.spec.ts.snap | 9 +++++---- lib/manager/maven/extract.ts | 9 ++++++++- 4 files changed, 19 insertions(+), 9 deletions(-) diff --git a/lib/manager/maven/__fixtures__/simple.pom.xml b/lib/manager/maven/__fixtures__/simple.pom.xml index cbebf202795081..138cc1e7b3e61f 100644 --- a/lib/manager/maven/__fixtures__/simple.pom.xml +++ b/lib/manager/maven/__fixtures__/simple.pom.xml @@ -107,6 +107,7 @@ org.example quuz 1.2.3 + test org.example diff --git a/lib/manager/maven/__snapshots__/extract.spec.ts.snap b/lib/manager/maven/__snapshots__/extract.spec.ts.snap index 05e299a0d32981..6bae8e26b7386f 100644 --- a/lib/manager/maven/__snapshots__/extract.spec.ts.snap +++ b/lib/manager/maven/__snapshots__/extract.spec.ts.snap @@ -98,6 +98,7 @@ Object { "currentValue": "1.2.3", "datasource": "maven", "depName": "org.example:quuz", + "depType": "test", "fileReplacePosition": 2832, "registryUrls": Array [ "https://repo.maven.apache.org/maven2", @@ -108,7 +109,7 @@ Object { "currentValue": "it's not a version", "datasource": "maven", "depName": "org.example:quuuz", - "fileReplacePosition": 2972, + "fileReplacePosition": 2998, "registryUrls": Array [ "https://repo.maven.apache.org/maven2", "https://maven.atlassian.com/content/repositories/atlassian-public/", @@ -118,7 +119,7 @@ Object { "currentValue": "[1.0.0]", "datasource": "maven", "depName": "org.example:hard-range", - "fileReplacePosition": 3130, + "fileReplacePosition": 3156, "registryUrls": Array [ "https://repo.maven.apache.org/maven2", "https://maven.atlassian.com/content/repositories/atlassian-public/", @@ -128,7 +129,7 @@ Object { "currentValue": "\${profile-placeholder}", "datasource": "maven", "depName": "org.example:profile-artifact", - "fileReplacePosition": 3392, + "fileReplacePosition": 3418, "registryUrls": Array [ "https://repo.maven.apache.org/maven2", "https://maven.atlassian.com/content/repositories/atlassian-public/", @@ -138,7 +139,7 @@ Object { "currentValue": "2.17", "datasource": "maven", "depName": "org.apache.maven.plugins:maven-checkstyle-plugin", - "fileReplacePosition": 3668, + "fileReplacePosition": 3694, "registryUrls": Array [ "https://repo.maven.apache.org/maven2", "https://maven.atlassian.com/content/repositories/atlassian-public/", diff --git a/lib/manager/maven/__snapshots__/index.spec.ts.snap b/lib/manager/maven/__snapshots__/index.spec.ts.snap index b34f116b8b1a3c..f1606d81add56b 100644 --- a/lib/manager/maven/__snapshots__/index.spec.ts.snap +++ b/lib/manager/maven/__snapshots__/index.spec.ts.snap @@ -103,6 +103,7 @@ Array [ "currentValue": "1.2.3", "datasource": "maven", "depName": "org.example:quuz", + "depType": "test", "fileReplacePosition": 2832, "registryUrls": Array [ "https://repo.maven.apache.org/maven2", @@ -113,7 +114,7 @@ Array [ "currentValue": "it's not a version", "datasource": "maven", "depName": "org.example:quuuz", - "fileReplacePosition": 2972, + "fileReplacePosition": 2998, "registryUrls": Array [ "https://repo.maven.apache.org/maven2", "https://maven.atlassian.com/content/repositories/atlassian-public/", @@ -123,7 +124,7 @@ Array [ "currentValue": "[1.0.0]", "datasource": "maven", "depName": "org.example:hard-range", - "fileReplacePosition": 3130, + "fileReplacePosition": 3156, "registryUrls": Array [ "https://repo.maven.apache.org/maven2", "https://maven.atlassian.com/content/repositories/atlassian-public/", @@ -133,7 +134,7 @@ Array [ "currentValue": "\${profile-placeholder}", "datasource": "maven", "depName": "org.example:profile-artifact", - "fileReplacePosition": 3392, + "fileReplacePosition": 3418, "registryUrls": Array [ "https://repo.maven.apache.org/maven2", "https://maven.atlassian.com/content/repositories/atlassian-public/", @@ -144,7 +145,7 @@ Array [ "currentValue": "2.17", "datasource": "maven", "depName": "org.apache.maven.plugins:maven-checkstyle-plugin", - "fileReplacePosition": 3668, + "fileReplacePosition": 3694, "registryUrls": Array [ "https://repo.maven.apache.org/maven2", "https://maven.atlassian.com/content/repositories/atlassian-public/", diff --git a/lib/manager/maven/extract.ts b/lib/manager/maven/extract.ts index b1d2a11ce3428d..3e46e09438969b 100644 --- a/lib/manager/maven/extract.ts +++ b/lib/manager/maven/extract.ts @@ -59,13 +59,20 @@ function depFromNode(node: XmlElement): PackageDependency | null { const fileReplacePosition = versionNode.position; const datasource = datasourceMaven.id; const registryUrls = [MAVEN_REPO]; - return { + const result: PackageDependency = { datasource, depName, currentValue, fileReplacePosition, registryUrls, }; + + const depType = node.valueWithPath('scope'); + if (depType) { + result.depType = depType; + } + + return result; } return null; } From 9bffbb6e622849ddb92131bd16e136ae98946c8f Mon Sep 17 00:00:00 2001 From: Michael Kriese Date: Mon, 18 Jan 2021 08:53:53 +0100 Subject: [PATCH 30/52] fix(nuget): use last version if no stable exists (#8327) --- lib/datasource/nuget/v3.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lib/datasource/nuget/v3.ts b/lib/datasource/nuget/v3.ts index 0d937ef318dfe1..5375806f7b9151 100644 --- a/lib/datasource/nuget/v3.ts +++ b/lib/datasource/nuget/v3.ts @@ -165,6 +165,13 @@ export async function getReleases( return null; } + // istanbul ignore if: only happens when no stable version exists + if (latestStable === null) { + const last = catalogEntries.pop(); + latestStable = removeBuildMeta(last.version); + homepage ??= last.projectUrl; + } + const dep: ReleaseResult = { pkgName, releases, From 1bd51a3740b3240eb15da42c7a946117d1566156 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89tienne=20L=C3=A9vesque?= Date: Mon, 18 Jan 2021 03:14:45 -0500 Subject: [PATCH 31/52] fix(hex): Preserve minor versions when in bump mode (#8326) * feat(hex): Preserve minor versions when in bump mode * Fix linting issue * Add minor test Co-authored-by: Rhys Arkins --- lib/versioning/hex/index.spec.ts | 10 +++++++++- lib/versioning/hex/index.ts | 12 +++++++++--- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/lib/versioning/hex/index.spec.ts b/lib/versioning/hex/index.spec.ts index 03b77d3c66e14a..b6579867251fba 100644 --- a/lib/versioning/hex/index.spec.ts +++ b/lib/versioning/hex/index.spec.ts @@ -128,7 +128,15 @@ describe('lib/versioning/hex', () => { fromVersion: '1.2.3', toVersion: '2.0.7', }) - ).toEqual('~> 2'); + ).toEqual('~> 2.0'); + expect( + hexScheme.getNewValue({ + currentValue: '~> 1.2', + rangeStrategy: 'bump', + fromVersion: '1.2.3', + toVersion: '1.3.1', + }) + ).toEqual('~> 1.3'); expect( hexScheme.getNewValue({ currentValue: '~> 1.2.0', diff --git a/lib/versioning/hex/index.ts b/lib/versioning/hex/index.ts index c1a6d61263dde2..b5219e6318901d 100644 --- a/lib/versioning/hex/index.ts +++ b/lib/versioning/hex/index.ts @@ -69,10 +69,16 @@ const getNewValue = ({ toVersion, }); newSemver = npm2hex(newSemver); - if (/~>\s*(\d+\.\d+)$/.test(currentValue)) { + if (/~>\s*(\d+\.\d+\.\d+)$/.test(currentValue)) { newSemver = newSemver.replace( - /\^\s*(\d+\.\d+(\.\d)?)/, - (_str, p1: string) => `~> ${p1.slice(0, -2)}` + /[\^~]\s*(\d+\.\d+\.\d+)/, + (_str, p1: string) => `~> ${p1}` + ); + } else if (/~>\s*(\d+\.\d+)$/.test(currentValue)) { + newSemver = newSemver.replace( + /\^\s*(\d+\.\d+)/, + (_str, p1: string) => + `~> ${rangeStrategy !== 'bump' ? p1.slice(0, -2) : p1}` ); } else { newSemver = newSemver.replace(/~\s*(\d+\.\d+\.\d)/, '~> $1'); From ec2839dc29dae87ed9a614894e3091ee524eed6f Mon Sep 17 00:00:00 2001 From: Julien Tanay Date: Mon, 18 Jan 2021 09:22:24 +0100 Subject: [PATCH 32/52] docs: add hostrules example for private helm charts repo (#8316) --- docs/usage/docker.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/docs/usage/docker.md b/docs/usage/docker.md index a6fd6ea834397c..2ade7bf3c7946a 100644 --- a/docs/usage/docker.md +++ b/docs/usage/docker.md @@ -199,4 +199,18 @@ module.exports = { }; ``` +You might be hosting your own chartmuseum somewhere to manage your private Helm Charts. Here is how you would connect to a private Helm repository : + +```js +module.exports = { + hostRules: [ + { + hostName: 'your.host.io', + username: '', + password: 'process.env.SELF_HOSTED_HELM_CHARTS_PASSWORD', + }, + ], +}; +``` + If you need to configure per-repository credentials then you can also configure the above within a repository's Renovate config (e.g. `renovate.json`). From 75b1e0c7edf17d83335565879dc48c4251aaa876 Mon Sep 17 00:00:00 2001 From: HonkingGoose <34918129+HonkingGoose@users.noreply.github.com> Date: Mon, 18 Jan 2021 09:50:42 +0100 Subject: [PATCH 33/52] docs: improve self-hosting documentation (#8079) --- docs/usage/self-hosting.md | 61 +++++++++++++++++++++----------------- 1 file changed, 34 insertions(+), 27 deletions(-) diff --git a/docs/usage/self-hosting.md b/docs/usage/self-hosting.md index cf97f61382179b..82c9cd183ee8be 100644 --- a/docs/usage/self-hosting.md +++ b/docs/usage/self-hosting.md @@ -2,25 +2,26 @@ ## Installing Renovate OSS CLI -#### npmjs +### npmjs ```sh $ npm install -g renovate ``` -Since renovate v20 `npm`, `pnpm` and `yarn` are no longer embedded, so you need to install them globally if you need to update lockfiles. +Renovate does not embed `npm`, `pnpm` and `yarn` as its own dependencies. +If you want to use these package managers to update your lockfiles, you must ensure that the correct versions are already installed globally. ```sh $ npm install -g yarn pnpm ``` -The same goes for any other third party binary tool that may be needed, such as `gradle` or `poetry` - you need to make sure they are installed and the appropriate version you need before running Renovate. +The same goes for any other third party binary tool like `gradle` or `poetry` - you need to make sure they are installed and the appropriate version before running Renovate. -#### Docker +### Docker Renovate is available for Docker via an automated build [renovate/renovate](https://hub.docker.com/r/renovate/renovate/). It builds `latest` based on the `master` branch and all semver tags are published too. -All the following are valid: +For example, all the following are valid tags: ```sh $ docker run --rm renovate/renovate @@ -29,16 +30,17 @@ $ docker run --rm renovate/renovate:24.14 $ docker run --rm renovate/renovate:24 ``` -(Please look up what the latest actual tags are though, do not use the above literally). +Do not use the example tags listed above, as they will be out-of-date. +Go to [renovate/renovate tags](https://hub.docker.com/r/renovate/renovate/tags) to grab the latest tagged release from Renovate. -If you wish to configure Renovate using a `config.js` file then map it to `/usr/src/app/config.js` using Docker volumes. +If you want to configure Renovate using a `config.js` file then map it to `/usr/src/app/config.js` using Docker volumes. For example: ```sh $ docker run --rm -v "/path/to/your/config.js:/usr/src/app/config.js" renovate/renovate ``` -#### Kubernetes +### Kubernetes Renovate's official Docker image is compatible with Kubernetes. The following is an example manifest of running Renovate against a GitHub Enterprise server. @@ -80,7 +82,7 @@ metadata: type: Opaque stringData: GITHUB_COM_TOKEN: 'any-personal-user-token-for-github-com-for-fetching-changelogs' - # set to true to run on all repos you have push access to + # You can set RENOVATE_AUTODISCOVER to true to run Renovate on all repos you have push access to RENOVATE_AUTODISCOVER: 'false' RENOVATE_ENDPOINT: 'https://github.company.com/api/v3' RENOVATE_GIT_AUTHOR: 'Renovate Bot ' @@ -88,7 +90,7 @@ stringData: RENOVATE_TOKEN: 'your-github-enterprise-renovate-user-token' ``` -A `config.js` file can be added to the manifest using a `ConfigMap` as shown in the following example (using a "dry run" in github.com) +A `config.js` file can be added to the manifest using a `ConfigMap` as shown in the following example (using a "dry run" in github.com): ```yaml --- @@ -144,7 +146,7 @@ spec: emptyDir: {} ``` -#### CircleCI +### CircleCI If you are using CircleCI, you can use the third-party [daniel-shuy/renovate](https://circleci.com/developer/orbs/orb/daniel-shuy/renovate) orb to run a self-hosted instance of Renovate on CircleCI. @@ -160,7 +162,7 @@ The following example runs Renovate hourly, and looks for the self-hosted config ```yml version: '2.1' orbs: - renovate: daniel-shuy/renovate@2.1 + renovate: daniel-shuy/renovate@2.1.1 workflows: renovate: jobs: @@ -176,7 +178,7 @@ workflows: - master ``` -#### GitLab CI/CD pipeline +### GitLab CI/CD pipeline For GitLab pipelines we recommend you use the [renovate-runner project on GitLab](https://gitlab.com/renovate-bot/renovate-runner). We've prepared some pipeline templates to run Renovate on pipeline schedules in an easy way. @@ -189,8 +191,8 @@ For self-hosted GitLab clone/import the [renovate-runner](https://gitlab.com/ren Self-hosted Renovate can be configured using any of the following (or a combination): - A `config.js` file (can also be named `config.json`, but you can't have both at the same time) -- CLI params -- Environment params +- CLI parameters +- Environment parameters Note that some Renovate configuration options are _only_ available for self-hosting, and so can only be configured using one of the above methods. These are described in the [Self-hosted Configuration](./self-hosted-configuration.md) doc. @@ -201,32 +203,32 @@ Regardless of platform, you need to select a user account for `renovate` to assu It is recommended to be `@renovate-bot` if you are using a self-hosted server with free choice of usernames. It is also recommended that you configure `config.gitAuthor` with the same identity as your Renovate user, e.g. like `"gitAuthor": "Renovate Bot "`. -#### GitHub Enterprise +### GitHub Enterprise First, [create a personal access token](https://help.github.com/articles/creating-an-access-token-for-command-line-use/) for the bot account (select "repo" permissions). Configure it either as `token` in your `config.js` file, or in environment variable `RENOVATE_TOKEN`, or via CLI `--token=`. -#### GitLab CE/EE +### GitLab CE/EE First, [create a personal access token](https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html) for the bot account (select `read_user`, `api` and `write_repository` scopes). Configure it either as `token` in your `config.js` file, or in environment variable `RENOVATE_TOKEN`, or via CLI `--token=`. Don't forget to configure `platform=gitlab` somewhere in config. -#### Bitbucket Cloud +### Bitbucket Cloud First, [create an AppPassword](https://confluence.atlassian.com/bitbucket/app-passwords-828781300.html) for the bot account. Configure it as `password` in your `config.js` file, or in environment variable `RENOVATE_PASSWORD`, or via CLI `--password=`. Also be sure to configure the `username` for your bot account too. Don't forget to configure `platform=bitbucket` somewhere in config. -#### Bitbucket Server +### Bitbucket Server Create a [Personal Access Token](https://confluence.atlassian.com/bitbucketserver/personal-access-tokens-939515499.html) for your bot account. Configure it as `password` in your `config.js` file, or in environment variable `RENOVATE_PASSWORD`, or via CLI `--password=`. Also configure the `username` for your bot account too, if you decided not to name it `@renovate-bot`. Don't forget to configure `platform=bitbucket-server` somewhere in config. -#### Azure DevOps +### Azure DevOps First, [create a personal access token](https://docs.microsoft.com/en-us/azure/devops/integrate/get-started/authentication/pats) for the bot account. Configure it either as `token` in your `config.js` file, or in environment variable `RENOVATE_TOKEN`, or via CLI `--token=`. @@ -245,19 +247,23 @@ This account can actually be _any_ account on GitHub, and needs only read-only a It's used when fetching release notes for repositories in order to increase the hourly API limit. It's also OK to configure the same as a host rule instead, if you prefer that. -**Note:** If you're using renovate in a project where dependencies are loaded from github.com (such as Go m=Modules hosted on GitHub) it is highly recommended to add a token as you will run in the rate limit from the github.com API, which will lead to renovate closing and reopening PRs because it could not get reliable info on updated dependencies. +**Note:** If you're using Renovate in a project where dependencies are loaded from github.com (such as Go modules hosted on GitHub) it is highly recommended to add a token as you will run in the rate limit from the github.com API, which will lead to Renovate closing and reopening PRs because it could not get reliable info on updated dependencies. ## File/directory usage -By default, Renovate will store all files within a `renovate/` subdirectory of the operating system's temporary directory, e.g. `/tmp/renovate/`. +By default, Renovate stores all files in the `renovate/` subdirectory of the operating system's temporary directory, e.g. `/tmp/renovate/`. -Repository data will be copied or cloned into unique subdirectories under `repos/`, e.g. `/tmp/renovate/repos/github/owner1/repo-a/`. +Repository data is copied or cloned into unique subdirectories under `repos/`, e.g. `/tmp/renovate/repos/github/owner1/repo-a/`. -Cache data - such as Renovate's own cache as well as that for npm, Yarn, Composer, etc - will be stored in `/tmp/renovate/cache`. +Renovate's own cache, as well as the caches(s) for npm, Yarn, Composer etc, is stored in `/tmp/renovate/cache`. -If you wish to override the base directory to be used (e.g. instead of `/tmp/renovate/`) then configure a value for `baseDir` in `config.js`, or via env (`RENOVATE_BASE_DIR`) or via CLI (`--base-dir=`). +To use another directory as the base directory, instead of `tmp/renovate`: -If you wish to override the cache location specifically then configure a value for `cacheDir` instead. +- Configure a value for `baseDir` in `config.js` +- Use an environment variable `RENOVATE_BASE_DIR` +- Use the CLI to pass a base directory: `--base-dir=` + +If you want to override the cache location then configure a value for `cacheDir` instead. ## Usage @@ -289,7 +295,8 @@ If running against GitHub Enterprise, change the above `gitlab` values to the eq You can save this file as anything you want and then use `RENOVATE_CONFIG_FILE` env variable to tell Renovate where to find it. -Most people will run Renovate via cron, e.g. once per hour. Here is an example bash script that you can point `cron` to: +Most people will run Renovate via cron, e.g. once per hour. +Here is an example bash script that you can point `cron` to: ```sh #!/bin/bash From 6da2f1d2f0a96986eeb9d3d04f43c8767a28b486 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Halil=20=C4=B0brahim=20=C5=9Eener?= Date: Mon, 18 Jan 2021 14:11:29 +0100 Subject: [PATCH 34/52] fix(pip_setup): Run pip setup extract in the correct directory (#8320) --- data/extract.py | 6 ++-- .../__snapshots__/index.spec.ts.snap | 36 +++++++++---------- lib/manager/pip_setup/extract.ts | 9 +++-- lib/manager/pip_setup/index.spec.ts | 9 +++-- lib/manager/pip_setup/util.ts | 20 ++++++----- 5 files changed, 42 insertions(+), 38 deletions(-) diff --git a/data/extract.py b/data/extract.py index d384f544831872..9a2a137474697a 100644 --- a/data/extract.py +++ b/data/extract.py @@ -2,7 +2,7 @@ import json import os import distutils.core -from os.path import dirname, realpath +from os.path import basename if sys.version_info[:2] >= (3, 3): from importlib.machinery import SourceFileLoader @@ -33,10 +33,8 @@ def setup(): @mock.patch.object(setuptools, 'setup') @mock.patch.object(distutils.core, 'setup') def invoke(mock1, mock2): - # Inserting the parent directory of the target setup.py in Python import path: - sys.path.append(dirname(realpath(sys.argv[-1]))) # This is setup.py which calls setuptools.setup - load_source('_target_setup_', sys.argv[-1]) + load_source('_target_setup_', basename(sys.argv[-1])) # called arguments are in `mock_setup.call_args` call_args = mock1.call_args or mock2.call_args args, kwargs = call_args diff --git a/lib/manager/pip_setup/__snapshots__/index.spec.ts.snap b/lib/manager/pip_setup/__snapshots__/index.spec.ts.snap index c869d5aec3aa28..b6ddf4d852c4e8 100644 --- a/lib/manager/pip_setup/__snapshots__/index.spec.ts.snap +++ b/lib/manager/pip_setup/__snapshots__/index.spec.ts.snap @@ -5,7 +5,7 @@ Array [ Object { "cmd": "python --version", "options": Object { - "cwd": null, + "cwd": "/tmp/github/some/repo", "encoding": "utf-8", "env": Object { "HOME": "/home/user", @@ -23,7 +23,7 @@ Array [ Object { "cmd": "python3 --version", "options": Object { - "cwd": null, + "cwd": "/tmp/github/some/repo", "encoding": "utf-8", "env": Object { "HOME": "/home/user", @@ -41,7 +41,7 @@ Array [ Object { "cmd": "python3.8 --version", "options": Object { - "cwd": null, + "cwd": "/tmp/github/some/repo", "encoding": "utf-8", "env": Object { "HOME": "/home/user", @@ -57,9 +57,9 @@ Array [ }, }, Object { - "cmd": " \\"/tmp/folders/foobar.py\\"", + "cmd": " \\"folders/foobar.py\\"", "options": Object { - "cwd": "/tmp/github/some/repo", + "cwd": "/tmp/github/some/repo/folders", "encoding": "utf-8", "env": Object { "HOME": "/home/user", @@ -338,7 +338,7 @@ Array [ Object { "cmd": "python --version", "options": Object { - "cwd": null, + "cwd": "/tmp/github/some/repo", "encoding": "utf-8", "env": Object { "HOME": "/home/user", @@ -356,7 +356,7 @@ Array [ Object { "cmd": "python3 --version", "options": Object { - "cwd": null, + "cwd": "/tmp/github/some/repo", "encoding": "utf-8", "env": Object { "HOME": "/home/user", @@ -374,7 +374,7 @@ Array [ Object { "cmd": "python3.8 --version", "options": Object { - "cwd": null, + "cwd": "/tmp/github/some/repo", "encoding": "utf-8", "env": Object { "HOME": "/home/user", @@ -392,7 +392,7 @@ Array [ Object { "cmd": " \\"lib/manager/pip_setup/__fixtures__/setup.py\\"", "options": Object { - "cwd": "/tmp/github/some/repo", + "cwd": "/tmp/github/some/repo/lib/manager/pip_setup/__fixtures__", "encoding": "utf-8", "env": Object { "HOME": "/home/user", @@ -415,7 +415,7 @@ Array [ Object { "cmd": "python --version", "options": Object { - "cwd": null, + "cwd": "/tmp/github/some/repo", "encoding": "utf-8", "env": Object { "HOME": "/home/user", @@ -433,7 +433,7 @@ Array [ Object { "cmd": "python3 --version", "options": Object { - "cwd": null, + "cwd": "/tmp/github/some/repo", "encoding": "utf-8", "env": Object { "HOME": "/home/user", @@ -451,7 +451,7 @@ Array [ Object { "cmd": "python3.8 --version", "options": Object { - "cwd": null, + "cwd": "/tmp/github/some/repo", "encoding": "utf-8", "env": Object { "HOME": "/home/user", @@ -469,7 +469,7 @@ Array [ Object { "cmd": " \\"lib/manager/pip_setup/__fixtures__/setup.py\\"", "options": Object { - "cwd": "/tmp/github/some/repo", + "cwd": "/tmp/github/some/repo/lib/manager/pip_setup/__fixtures__", "encoding": "utf-8", "env": Object { "HOME": "/home/user", @@ -492,7 +492,7 @@ Array [ Object { "cmd": "python --version", "options": Object { - "cwd": null, + "cwd": "/tmp/github/some/repo", "encoding": "utf-8", "env": Object { "HOME": "/home/user", @@ -510,7 +510,7 @@ Array [ Object { "cmd": "python3 --version", "options": Object { - "cwd": null, + "cwd": "/tmp/github/some/repo", "encoding": "utf-8", "env": Object { "HOME": "/home/user", @@ -528,7 +528,7 @@ Array [ Object { "cmd": "python3.8 --version", "options": Object { - "cwd": null, + "cwd": "/tmp/github/some/repo", "encoding": "utf-8", "env": Object { "HOME": "/home/user", @@ -544,9 +544,9 @@ Array [ }, }, Object { - "cmd": " \\"/tmp/folders/foobar.py\\"", + "cmd": " \\"folders/foobar.py\\"", "options": Object { - "cwd": "/tmp/github/some/repo", + "cwd": "/tmp/github/some/repo/folders", "encoding": "utf-8", "env": Object { "HOME": "/home/user", diff --git a/lib/manager/pip_setup/extract.ts b/lib/manager/pip_setup/extract.ts index 7719eee5e1157d..7fccd05ade7226 100644 --- a/lib/manager/pip_setup/extract.ts +++ b/lib/manager/pip_setup/extract.ts @@ -6,7 +6,7 @@ import { BinarySource } from '../../util/exec/common'; import { isSkipComment } from '../../util/ignore'; import { ExtractConfig, PackageDependency, PackageFile } from '../common'; import { dependencyPattern } from '../pip_requirements/extract'; -import { PythonSetup, copyExtractFile, parseReport } from './util'; +import { PythonSetup, getExtractFile, parseReport } from './util'; export const pythonVersions = ['python', 'python3', 'python3.8']; let pythonAlias: string | null = null; @@ -44,9 +44,8 @@ export async function extractSetupFile( packageFile: string, config: ExtractConfig ): Promise { - const cwd = config.localDir; let cmd = 'python'; - const extractPy = await copyExtractFile(); + const extractPy = await getExtractFile(); const args = [`"${extractPy}"`, `"${packageFile}"`]; if (config.binarySource !== BinarySource.Docker) { logger.debug('Running python via global command'); @@ -54,7 +53,7 @@ export async function extractSetupFile( } logger.debug({ cmd, args }, 'python command'); const res = await exec(`${cmd} ${args.join(' ')}`, { - cwd, + cwdFile: packageFile, timeout: 30000, docker: { image: 'renovate/pip', @@ -69,7 +68,7 @@ export async function extractSetupFile( logger.warn({ stdout: res.stdout, stderr }, 'Error in read setup file'); } } - return parseReport(); + return parseReport(packageFile); } export async function extractPackageFile( diff --git a/lib/manager/pip_setup/index.spec.ts b/lib/manager/pip_setup/index.spec.ts index d826bdf09180a9..09d58f3d5a7d9b 100644 --- a/lib/manager/pip_setup/index.spec.ts +++ b/lib/manager/pip_setup/index.spec.ts @@ -7,6 +7,7 @@ import { mockExecSequence, } from '../../../test/exec-util'; import { env, getName } from '../../../test/util'; +import { setUtilConfig } from '../../util'; import { BinarySource } from '../../util/exec/common'; import * as fs from '../../util/fs'; import * as extract from './extract'; @@ -20,6 +21,7 @@ const jsonContent = readFileSync(packageFileJson, 'utf8'); const config = { localDir: '/tmp/github/some/repo', + cacheDir: '/tmp/renovate/cache', }; jest.mock('child_process'); @@ -40,11 +42,12 @@ const fixSnapshots = (snapshots: ExecSnapshots): ExecSnapshots => describe(getName(__filename), () => { describe('extractPackageFile()', () => { - beforeEach(() => { + beforeEach(async () => { jest.resetAllMocks(); jest.resetModules(); extract.resetModule(); + await setUtilConfig(config); env.getChildProcessEnv.mockReturnValue(envMock.basic); // do not copy extract.py @@ -105,7 +108,7 @@ describe(getName(__filename), () => { expect( await extractPackageFile( 'raise Exception()', - '/tmp/folders/foobar.py', + 'folders/foobar.py', config ) ).toBeNull(); @@ -117,7 +120,7 @@ describe(getName(__filename), () => { expect( await extractPackageFile( 'raise Exception()', - '/tmp/folders/foobar.py', + 'folders/foobar.py', config ) ).toBeNull(); diff --git a/lib/manager/pip_setup/util.ts b/lib/manager/pip_setup/util.ts index 40d57e4bc6b78a..dae67ed378c21d 100644 --- a/lib/manager/pip_setup/util.ts +++ b/lib/manager/pip_setup/util.ts @@ -1,5 +1,7 @@ +import { dirname } from 'path'; +import { join } from 'upath'; import dataFiles from '../../data-files.generated'; -import { readLocalFile, writeLocalFile } from '../../util/fs'; +import { ensureCacheDir, outputFile, readLocalFile } from '../../util/fs'; // need to match filename in `data/extract.py` const REPORT = 'renovate-pip_setup-report.json'; @@ -7,14 +9,16 @@ const EXTRACT = 'renovate-pip_setup-extract.py'; let extractPy: string | undefined; -export async function copyExtractFile(): Promise { - if (extractPy === undefined) { - extractPy = dataFiles.get('extract.py'); +export async function getExtractFile(): Promise { + if (extractPy) { + return extractPy; } - await writeLocalFile(EXTRACT, extractPy); + const cacheDir = await ensureCacheDir('./others/pip_setup'); + extractPy = join(cacheDir, EXTRACT); + await outputFile(extractPy, dataFiles.get('extract.py')); - return EXTRACT; + return extractPy; } export interface PythonSetup { @@ -22,7 +26,7 @@ export interface PythonSetup { install_requires: string[]; } -export async function parseReport(): Promise { - const data = await readLocalFile(REPORT, 'utf8'); +export async function parseReport(packageFile: string): Promise { + const data = await readLocalFile(join(dirname(packageFile), REPORT), 'utf8'); return JSON.parse(data); } From 8a38e443b2821a21fcf84679ee2fb7a99911725d Mon Sep 17 00:00:00 2001 From: HonkingGoose <34918129+HonkingGoose@users.noreply.github.com> Date: Mon, 18 Jan 2021 18:30:30 +0100 Subject: [PATCH 35/52] docs: improve docker usage documentation (#8334) --- docs/usage/docker.md | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/docs/usage/docker.md b/docs/usage/docker.md index 2ade7bf3c7946a..c4f4cbf56d92b2 100644 --- a/docs/usage/docker.md +++ b/docs/usage/docker.md @@ -165,6 +165,11 @@ If you want Renovate to commit directly to your base branch without opening a PR ### Registry authentication +There are many different registries, and many ways to authenticate to those registries. +We will explain how to authenticate for the most common registries. + +#### DockerHub + Here is an example of configuring a default Docker username/password in `config.js`. The Docker Hub password is stored in a process environment variable. @@ -182,6 +187,8 @@ module.exports = { You can add additional host rules, read the [hostrules documentation](https://docs.renovatebot.com/configuration-options/#hostrules) for more information. +#### Self-hosted Docker registry + Say you host some Docker images yourself, and use a password to access your self-hosted Docker images. In addition to self-hosting, you also pull images from Docker Hub, without a password. In this example you would configure a specific Docker host like this: @@ -199,7 +206,10 @@ module.exports = { }; ``` -You might be hosting your own chartmuseum somewhere to manage your private Helm Charts. Here is how you would connect to a private Helm repository : +#### ChartMuseum + +Maybe you're running your own ChartMuseum server to host your private Helm Charts. +This is how you connect to a private Helm repository: ```js module.exports = { From 207b177d60a226f4989f88bed83519698f0ad774 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Halil=20=C4=B0brahim=20=C5=9Eener?= Date: Tue, 19 Jan 2021 09:11:45 +0100 Subject: [PATCH 36/52] feat: customEnvVariables (#8321) Introduces a new option to pass custom environment variables to child processes. So that users can provide/override environment variables to influence executions, e.g. setting MAVEN_OPTS or passing credentials for private registries for the locking process. --- docs/usage/self-hosted-configuration.md | 4 + lib/config/common.ts | 1 + lib/config/definitions.ts | 8 ++ lib/util/exec/common.ts | 1 + lib/util/exec/exec.spec.ts | 114 ++++++++++++++++++++++++ lib/util/exec/index.ts | 4 +- 6 files changed, 131 insertions(+), 1 deletion(-) diff --git a/docs/usage/self-hosted-configuration.md b/docs/usage/self-hosted-configuration.md index d3f0ed28e127ac..2ccacf6d5f0aa0 100644 --- a/docs/usage/self-hosted-configuration.md +++ b/docs/usage/self-hosted-configuration.md @@ -92,6 +92,10 @@ If you configure this to be different to the `baseDir`, it means you can have on Set to `false` to prevent usage of `--ignore-platform-reqs` in the Composer package manager. +## customEnvVariables + +This configuration will be applied after all other environment variables so that it can be used to override defaults. + ## dockerImagePrefix Override the default renovate sidecar Docker containers image prefix from `docker.io/renovate` to a custom value, so renovate will pull images from a custom Docker registry. diff --git a/lib/config/common.ts b/lib/config/common.ts index f841fd90fda3b5..366579184edfb5 100644 --- a/lib/config/common.ts +++ b/lib/config/common.ts @@ -78,6 +78,7 @@ export interface RenovateAdminConfig { cacheDir?: string; configWarningReuseIssue?: boolean; + customEnvVariables?: Record; dockerImagePrefix?: string; dockerUser?: string; diff --git a/lib/config/definitions.ts b/lib/config/definitions.ts index f4c66a57d99fca..b53af3cd45e547 100644 --- a/lib/config/definitions.ts +++ b/lib/config/definitions.ts @@ -308,6 +308,14 @@ const options: RenovateOptions[] = [ admin: true, type: 'string', }, + { + name: 'customEnvVariables', + description: + 'Custom environment variables for child processes and sidecar Docker containers.', + admin: true, + type: 'object', + default: false, + }, { name: 'dockerMapDotfiles', description: diff --git a/lib/util/exec/common.ts b/lib/util/exec/common.ts index 4ae8f395da4fdb..b3a4e4ff9524cf 100644 --- a/lib/util/exec/common.ts +++ b/lib/util/exec/common.ts @@ -14,6 +14,7 @@ export enum BinarySource { export interface ExecConfig { binarySource: Opt; + customEnvVariables: Opt>; dockerImagePrefix: Opt; dockerUser: Opt; localDir: Opt; diff --git a/lib/util/exec/exec.spec.ts b/lib/util/exec/exec.spec.ts index 9877ce8a208ebc..c83246fa624dca 100644 --- a/lib/util/exec/exec.spec.ts +++ b/lib/util/exec/exec.spec.ts @@ -535,6 +535,120 @@ describe(`Child process execution wrapper`, () => { ], }, ], + + [ + 'Custom environment variables for child', + { + execConfig: { + ...execConfig, + customEnvVariables: { + CUSTOM_KEY: 'CUSTOM_VALUE', + }, + }, + processEnv: envMock.basic, + inCmd, + inOpts: {}, + outCmd, + outOpts: [ + { + cwd, + encoding, + env: { ...envMock.basic, CUSTOM_KEY: 'CUSTOM_VALUE' }, + timeout: 900000, + maxBuffer: 10485760, + }, + ], + }, + ], + + [ + 'Custom environment variables for child should override', + { + execConfig: { + ...execConfig, + customEnvVariables: { + CUSTOM_KEY: 'CUSTOM_OVERRIDEN_VALUE', + }, + }, + processEnv: { ...envMock.basic, CUSTOM_KEY: 'CUSTOM_VALUE' }, + inCmd, + inOpts: {}, + outCmd, + outOpts: [ + { + cwd, + encoding, + env: { ...envMock.basic, CUSTOM_KEY: 'CUSTOM_OVERRIDEN_VALUE' }, + timeout: 900000, + maxBuffer: 10485760, + }, + ], + }, + ], + + [ + 'Custom environment variables for child (Docker)', + { + execConfig: { + ...execConfig, + binarySource: BinarySource.Docker, + customEnvVariables: { + CUSTOM_KEY: 'CUSTOM_VALUE', + }, + }, + processEnv, + inCmd, + inOpts: { docker, cwd }, + outCmd: [ + dockerPullCmd, + dockerRemoveCmd, + `docker run --rm --name=${name} --label=renovate_child ${defaultVolumes} -e CUSTOM_KEY ${defaultCwd} ${image} bash -l -c "${inCmd}"`, + ], + outOpts: [ + dockerPullOpts, + dockerRemoveOpts, + { + cwd, + encoding, + env: { ...envMock.basic, CUSTOM_KEY: 'CUSTOM_VALUE' }, + timeout: 900000, + maxBuffer: 10485760, + }, + ], + }, + ], + + [ + 'Custom environment variables for child should override (Docker)', + { + execConfig: { + ...execConfig, + binarySource: BinarySource.Docker, + customEnvVariables: { + CUSTOM_KEY: 'CUSTOM_OVERRIDEN_VALUE', + }, + }, + processEnv: { ...envMock.basic, CUSTOM_KEY: 'CUSTOM_VALUE' }, + inCmd, + inOpts: { docker, cwd }, + outCmd: [ + dockerPullCmd, + dockerRemoveCmd, + `docker run --rm --name=${name} --label=renovate_child ${defaultVolumes} -e CUSTOM_KEY ${defaultCwd} ${image} bash -l -c "${inCmd}"`, + ], + outOpts: [ + dockerPullOpts, + dockerRemoveOpts, + { + cwd, + encoding, + env: { ...envMock.basic, CUSTOM_KEY: 'CUSTOM_OVERRIDEN_VALUE' }, + timeout: 900000, + maxBuffer: 10485760, + }, + ], + }, + ], ]; test.each(testInputs)('%s', async (_msg, testOpts) => { diff --git a/lib/util/exec/index.ts b/lib/util/exec/index.ts index 7a8ec6b18e694f..e6d40915a4089c 100644 --- a/lib/util/exec/index.ts +++ b/lib/util/exec/index.ts @@ -20,6 +20,7 @@ import { getChildProcessEnv } from './env'; const execConfig: ExecConfig = { binarySource: null, + customEnvVariables: null, dockerImagePrefix: null, dockerUser: null, localDir: null, @@ -97,7 +98,8 @@ export async function exec( cmd: string | string[], opts: ExecOptions = {} ): Promise { - const { env, extraEnv, docker, cwdFile } = opts; + const { env, docker, cwdFile } = opts; + const extraEnv = { ...opts.extraEnv, ...execConfig.customEnvVariables }; let cwd; // istanbul ignore if if (cwdFile) { From 52c70f0b2bece0d4c32b746e8ef380bb13fcaebb Mon Sep 17 00:00:00 2001 From: Amos Wenger Date: Tue, 19 Jan 2021 09:35:48 +0100 Subject: [PATCH 37/52] feat(rust): Find and update Cargo.lock for cargo workspaces (#8338) --- .../__snapshots__/artifacts.spec.ts.snap | 23 +++++++ lib/manager/cargo/artifacts.spec.ts | 27 ++++++++ lib/manager/cargo/artifacts.ts | 14 +++- lib/util/fs/index.spec.ts | 66 ++++++++++++++++++- lib/util/fs/index.ts | 31 ++++++++- 5 files changed, 156 insertions(+), 5 deletions(-) diff --git a/lib/manager/cargo/__snapshots__/artifacts.spec.ts.snap b/lib/manager/cargo/__snapshots__/artifacts.spec.ts.snap index 7e6b944d3ccdca..4c0587bb002097 100644 --- a/lib/manager/cargo/__snapshots__/artifacts.spec.ts.snap +++ b/lib/manager/cargo/__snapshots__/artifacts.spec.ts.snap @@ -114,3 +114,26 @@ Array [ }, ] `; + +exports[`.updateArtifacts() returns updated workspace Cargo.lock 1`] = ` +Array [ + Object { + "cmd": "cargo update --manifest-path crates/one/Cargo.toml --package dep1", + "options": Object { + "cwd": "/tmp/github/some/repo", + "encoding": "utf-8", + "env": Object { + "HOME": "/home/user", + "HTTPS_PROXY": "https://example.com", + "HTTP_PROXY": "http://example.com", + "LANG": "en_US.UTF-8", + "LC_ALL": "en_US", + "NO_PROXY": "localhost", + "PATH": "/tmp/path", + }, + "maxBuffer": 10485760, + "timeout": 900000, + }, + }, +] +`; diff --git a/lib/manager/cargo/artifacts.spec.ts b/lib/manager/cargo/artifacts.spec.ts index 537d324b27a17f..50bba66fa8bdc5 100644 --- a/lib/manager/cargo/artifacts.spec.ts +++ b/lib/manager/cargo/artifacts.spec.ts @@ -35,6 +35,7 @@ describe('.updateArtifacts()', () => { docker.resetPrefetchedImages(); }); it('returns null if no Cargo.lock found', async () => { + fs.stat.mockRejectedValue(new Error('not found!')); const updatedDeps = ['dep1']; expect( await cargo.updateArtifacts({ @@ -56,9 +57,11 @@ describe('.updateArtifacts()', () => { ).toBeNull(); }); it('returns null if unchanged', async () => { + fs.stat.mockResolvedValueOnce({ name: 'Cargo.lock' } as any); fs.readFile.mockResolvedValueOnce('Current Cargo.lock' as any); const execSnapshots = mockExecAll(exec); fs.readFile.mockResolvedValueOnce('Current Cargo.lock' as any); + const updatedDeps = ['dep1']; expect( await cargo.updateArtifacts({ @@ -71,6 +74,7 @@ describe('.updateArtifacts()', () => { expect(execSnapshots).toMatchSnapshot(); }); it('returns updated Cargo.lock', async () => { + fs.stat.mockResolvedValueOnce({ name: 'Cargo.lock' } as any); git.getFile.mockResolvedValueOnce('Old Cargo.lock'); const execSnapshots = mockExecAll(exec); fs.readFile.mockResolvedValueOnce('New Cargo.lock' as any); @@ -86,7 +90,28 @@ describe('.updateArtifacts()', () => { expect(execSnapshots).toMatchSnapshot(); }); + it('returns updated workspace Cargo.lock', async () => { + fs.stat.mockRejectedValueOnce(new Error('crates/one/Cargo.lock not found')); + fs.stat.mockRejectedValueOnce(new Error('crates/Cargo.lock not found')); + fs.stat.mockResolvedValueOnce({ name: 'Cargo.lock' } as any); + + git.getFile.mockResolvedValueOnce('Old Cargo.lock'); + const execSnapshots = mockExecAll(exec); + fs.readFile.mockResolvedValueOnce('New Cargo.lock' as any); + const updatedDeps = ['dep1']; + expect( + await cargo.updateArtifacts({ + packageFileName: 'crates/one/Cargo.toml', + updatedDeps, + newPackageFileContent: '{}', + config, + }) + ).not.toBeNull(); + expect(execSnapshots).toMatchSnapshot(); + }); + it('returns updated Cargo.lock for lockfile maintenance', async () => { + fs.stat.mockResolvedValueOnce({ name: 'Cargo.lock' } as any); git.getFile.mockResolvedValueOnce('Old Cargo.lock'); const execSnapshots = mockExecAll(exec); fs.readFile.mockResolvedValueOnce('New Cargo.lock' as any); @@ -102,6 +127,7 @@ describe('.updateArtifacts()', () => { }); it('returns updated Cargo.lock with docker', async () => { + fs.stat.mockResolvedValueOnce({ name: 'Cargo.lock' } as any); jest.spyOn(docker, 'removeDanglingContainers').mockResolvedValueOnce(); await setExecConfig({ ...config, binarySource: BinarySource.Docker }); git.getFile.mockResolvedValueOnce('Old Cargo.lock'); @@ -119,6 +145,7 @@ describe('.updateArtifacts()', () => { expect(execSnapshots).toMatchSnapshot(); }); it('catches errors', async () => { + fs.stat.mockResolvedValueOnce({ name: 'Cargo.lock' } as any); fs.readFile.mockResolvedValueOnce('Current Cargo.lock' as any); fs.outputFile.mockImplementationOnce(() => { throw new Error('not found'); diff --git a/lib/manager/cargo/artifacts.ts b/lib/manager/cargo/artifacts.ts index 9d08e90dbac3ef..857a61b8522376 100644 --- a/lib/manager/cargo/artifacts.ts +++ b/lib/manager/cargo/artifacts.ts @@ -2,7 +2,7 @@ import { quote } from 'shlex'; import { logger } from '../../logger'; import { ExecOptions, exec } from '../../util/exec'; import { - getSiblingFileName, + findLocalSiblingOrParent, readLocalFile, writeLocalFile, } from '../../util/fs'; @@ -64,8 +64,16 @@ export async function updateArtifacts({ return null; } - const lockFileName = getSiblingFileName(packageFileName, 'Cargo.lock'); - const existingLockFileContent = await readLocalFile(lockFileName); + // For standalone package crates, the `Cargo.lock` will be in the same + // directory as `Cargo.toml` (ie. a sibling). For cargo workspaces, it + // will be further up. + const lockFileName = await findLocalSiblingOrParent( + packageFileName, + 'Cargo.lock' + ); + const existingLockFileContent = lockFileName + ? await readLocalFile(lockFileName) + : null; if (!existingLockFileContent) { logger.debug('No Cargo.lock found'); return null; diff --git a/lib/util/fs/index.spec.ts b/lib/util/fs/index.spec.ts index 7614d83516cf6f..81fe579b984dd3 100644 --- a/lib/util/fs/index.spec.ts +++ b/lib/util/fs/index.spec.ts @@ -1,5 +1,13 @@ +import { withDir } from 'tmp-promise'; import { getName } from '../../../test/util'; -import { getSubDirectory, localPathExists, readLocalFile } from '.'; +import { + findLocalSiblingOrParent, + getSubDirectory, + localPathExists, + readLocalFile, + setFsConfig, + writeLocalFile, +} from '.'; describe(getName(__filename), () => { describe('readLocalFile', () => { @@ -32,3 +40,59 @@ describe(getName(__filename), () => { }); }); }); + +describe(getName(__filename), () => { + describe('findLocalSiblingOrParent', () => { + it('returns path for file', async () => { + await withDir( + async (localDir) => { + setFsConfig({ + localDir: localDir.path, + }); + + await writeLocalFile('crates/one/Cargo.toml', ''); + await writeLocalFile('Cargo.lock', ''); + + expect( + await findLocalSiblingOrParent( + 'crates/one/Cargo.toml', + 'Cargo.lock' + ) + ).toBe('Cargo.lock'); + expect( + await findLocalSiblingOrParent( + 'crates/one/Cargo.toml', + 'Cargo.mock' + ) + ).toBeNull(); + + await writeLocalFile('crates/one/Cargo.lock', ''); + + expect( + await findLocalSiblingOrParent( + 'crates/one/Cargo.toml', + 'Cargo.lock' + ) + ).toBe('crates/one/Cargo.lock'); + expect( + await findLocalSiblingOrParent('crates/one', 'Cargo.lock') + ).toBe('Cargo.lock'); + expect( + await findLocalSiblingOrParent( + 'crates/one/Cargo.toml', + 'Cargo.mock' + ) + ).toBeNull(); + }, + { + unsafeCleanup: true, + } + ); + }); + + it('immediately returns null when either path is absolute', async () => { + expect(await findLocalSiblingOrParent('/etc/hosts', 'other')).toBeNull(); + expect(await findLocalSiblingOrParent('other', '/etc/hosts')).toBeNull(); + }); + }); +}); diff --git a/lib/util/fs/index.ts b/lib/util/fs/index.ts index da73e844283bc3..d94cef1f8420aa 100644 --- a/lib/util/fs/index.ts +++ b/lib/util/fs/index.ts @@ -1,5 +1,5 @@ import * as fs from 'fs-extra'; -import { join, parse } from 'upath'; +import { isAbsolute, join, parse } from 'upath'; import { RenovateConfig } from '../../config/common'; import { logger } from '../../logger'; @@ -96,3 +96,32 @@ export function localPathExists(pathName: string): Promise { .then((s) => !!s) .catch(() => false); } + +/** + * Tries to find `otherFileName` in the directory where + * `existingFileNameWithPath` is, then in its parent directory, then in the + * grandparent, until we reach the top-level directory. All paths + * must be relative to `localDir`. + */ +export async function findLocalSiblingOrParent( + existingFileNameWithPath: string, + otherFileName: string +): Promise { + if (isAbsolute(existingFileNameWithPath)) { + return null; + } + if (isAbsolute(otherFileName)) { + return null; + } + + let current = existingFileNameWithPath; + while (current !== '') { + current = getSubDirectory(current); + const candidate = join(current, otherFileName); + if (await localPathExists(candidate)) { + return candidate; + } + } + + return null; +} From 91d30caed720191b72685a95defc3f35e8b7d6e3 Mon Sep 17 00:00:00 2001 From: Stefan <5585744+casdevs@users.noreply.github.com> Date: Tue, 19 Jan 2021 09:36:27 +0100 Subject: [PATCH 38/52] fix: fetch changelog from self-hosted gitlab independent from url (#8336) --- .../__snapshots__/gitlab.spec.ts.snap | 43 +++++++++++++++++++ lib/workers/pr/changelog/gitlab.spec.ts | 17 ++++++++ lib/workers/pr/changelog/index.ts | 6 ++- 3 files changed, 65 insertions(+), 1 deletion(-) diff --git a/lib/workers/pr/changelog/__snapshots__/gitlab.spec.ts.snap b/lib/workers/pr/changelog/__snapshots__/gitlab.spec.ts.snap index e2150863d8873c..be55515daa4d9f 100644 --- a/lib/workers/pr/changelog/__snapshots__/gitlab.spec.ts.snap +++ b/lib/workers/pr/changelog/__snapshots__/gitlab.spec.ts.snap @@ -190,6 +190,49 @@ Object { } `; +exports[`workers/pr/changelog/gitlab getChangeLogJSON supports self-hosted gitlab changelog 1`] = ` +Object { + "hasReleaseNotes": false, + "project": Object { + "apiBaseUrl": "https://git.test.com/api/v4/", + "baseUrl": "https://git.test.com/", + "depName": "renovate", + "gitlab": "meno/dropzone", + "repository": "https://git.test.com/meno/dropzone/", + }, + "versions": Array [ + Object { + "changes": Array [], + "compare": Object {}, + "date": undefined, + "releaseNotes": null, + "version": "5.6.1", + }, + Object { + "changes": Array [], + "compare": Object {}, + "date": "2020-02-13T15:37:00.000Z", + "releaseNotes": null, + "version": "5.6.0", + }, + Object { + "changes": Array [], + "compare": Object {}, + "date": undefined, + "releaseNotes": null, + "version": "5.5.0", + }, + Object { + "changes": Array [], + "compare": Object {}, + "date": "2018-08-24T14:23:00.000Z", + "releaseNotes": null, + "version": "5.4.0", + }, + ], +} +`; + exports[`workers/pr/changelog/gitlab getChangeLogJSON uses GitLab tags 1`] = ` Object { "hasReleaseNotes": true, diff --git a/lib/workers/pr/changelog/gitlab.spec.ts b/lib/workers/pr/changelog/gitlab.spec.ts index ea83bb69133a15..1ba89095e1fa7e 100644 --- a/lib/workers/pr/changelog/gitlab.spec.ts +++ b/lib/workers/pr/changelog/gitlab.spec.ts @@ -189,5 +189,22 @@ describe(getName(__filename), () => { }) ).toMatchSnapshot(); }); + it('supports self-hosted gitlab changelog', async () => { + httpMock.scope('https://git.test.com').persist().get(/.*/).reply(200, []); + hostRules.add({ + hostType: PLATFORM_TYPE_GITLAB, + baseUrl: 'https://git.test.com/', + token: 'abc', + }); + process.env.GITHUB_ENDPOINT = ''; + expect( + await getChangeLogJSON({ + ...upgrade, + platform: PLATFORM_TYPE_GITLAB, + sourceUrl: 'https://git.test.com/meno/dropzone/', + endpoint: 'https://git.test.com/api/v4/', + }) + ).toMatchSnapshot(); + }); }); }); diff --git a/lib/workers/pr/changelog/index.ts b/lib/workers/pr/changelog/index.ts index c6b45557c29f24..efb383a005a62b 100644 --- a/lib/workers/pr/changelog/index.ts +++ b/lib/workers/pr/changelog/index.ts @@ -27,7 +27,11 @@ export async function getChangeLogJSON( let res: ChangeLogResult | null = null; - if (args.sourceUrl?.includes('gitlab')) { + if ( + args.sourceUrl?.includes('gitlab') || + (args.platform === 'gitlab' && + new URL(args.sourceUrl).hostname === new URL(args.endpoint).hostname) + ) { res = await sourceGitlab.getChangeLogJSON({ ...args, releases }); } else { res = await sourceGithub.getChangeLogJSON({ ...args, releases }); From 139ff4f2e5c5be686f9c7e365a7dd93d0fcb49bc Mon Sep 17 00:00:00 2001 From: Stefan <5585744+casdevs@users.noreply.github.com> Date: Tue, 19 Jan 2021 09:37:23 +0100 Subject: [PATCH 39/52] fix(dockerfile): improve FROM/COPY regex matching (#8331) --- .../dockerfile/__fixtures__/Dockerfile2 | 49 +++++++++ .../__snapshots__/extract.spec.ts.snap | 104 ++++++++++++++++++ lib/manager/dockerfile/extract.spec.ts | 10 ++ lib/manager/dockerfile/extract.ts | 99 +++++++++-------- 4 files changed, 215 insertions(+), 47 deletions(-) create mode 100644 lib/manager/dockerfile/__fixtures__/Dockerfile2 diff --git a/lib/manager/dockerfile/__fixtures__/Dockerfile2 b/lib/manager/dockerfile/__fixtures__/Dockerfile2 new file mode 100644 index 00000000000000..2c2320d27793d5 --- /dev/null +++ b/lib/manager/dockerfile/__fixtures__/Dockerfile2 @@ -0,0 +1,49 @@ +# different FROM syntaxes + FROM image1 as name1 + +FROM image2:1.0.0@sha256:abcdef \ + as name2 + +# FROM image3 + +FROM\ + --platform=linux \ + # comment1 + image4 + + FROM \ + \ + image5 \ + #comment5 + as name3 + +# different COPY --from syntaxes +COPY --from=image6 /path/1 /path/2 + + COPY \ + --from=image7:1.0.0@sha256:abcdef \ + /path/1 \ + /path/2 + +# COPY --from=image8 + +COPY --from=image9 + +COPY\ + --chown=root \ + # comment1 +# + #comment2 + --from=image10 + # comment2 + path1 path2 + +COPY --chown=root --from=image11 / ./ + + COPY \ + \ + --from=image12 a \ + #comment5 + b + +COPY --from=image13 --chown=root: a b diff --git a/lib/manager/dockerfile/__snapshots__/extract.spec.ts.snap b/lib/manager/dockerfile/__snapshots__/extract.spec.ts.snap index 19960ee8cc275f..7c0f2969618d24 100644 --- a/lib/manager/dockerfile/__snapshots__/extract.spec.ts.snap +++ b/lib/manager/dockerfile/__snapshots__/extract.spec.ts.snap @@ -33,6 +33,110 @@ Array [ ] `; +exports[`lib/manager/dockerfile/extract extractPackageFile() extracts images from all sorts of (maybe multiline) FROM and COPY --from statements 1`] = ` +Array [ + Object { + "autoReplaceStringTemplate": "{{depName}}{{#if newValue}}:{{newValue}}{{/if}}{{#if newDigest}}@{{newDigest}}{{/if}}", + "currentDigest": undefined, + "currentValue": undefined, + "datasource": "docker", + "depName": "image1", + "depType": "stage", + "replaceString": "image1", + }, + Object { + "autoReplaceStringTemplate": "{{depName}}{{#if newValue}}:{{newValue}}{{/if}}{{#if newDigest}}@{{newDigest}}{{/if}}", + "currentDigest": "sha256:abcdef", + "currentValue": "1.0.0", + "datasource": "docker", + "depName": "image2", + "depType": "stage", + "replaceString": "image2:1.0.0@sha256:abcdef", + }, + Object { + "autoReplaceStringTemplate": "{{depName}}{{#if newValue}}:{{newValue}}{{/if}}{{#if newDigest}}@{{newDigest}}{{/if}}", + "currentDigest": undefined, + "currentValue": undefined, + "datasource": "docker", + "depName": "image4", + "depType": "stage", + "replaceString": "image4", + }, + Object { + "autoReplaceStringTemplate": "{{depName}}{{#if newValue}}:{{newValue}}{{/if}}{{#if newDigest}}@{{newDigest}}{{/if}}", + "currentDigest": undefined, + "currentValue": undefined, + "datasource": "docker", + "depName": "image5", + "depType": "stage", + "replaceString": "image5", + }, + Object { + "autoReplaceStringTemplate": "{{depName}}{{#if newValue}}:{{newValue}}{{/if}}{{#if newDigest}}@{{newDigest}}{{/if}}", + "currentDigest": undefined, + "currentValue": undefined, + "datasource": "docker", + "depName": "image6", + "depType": "stage", + "replaceString": "image6", + }, + Object { + "autoReplaceStringTemplate": "{{depName}}{{#if newValue}}:{{newValue}}{{/if}}{{#if newDigest}}@{{newDigest}}{{/if}}", + "currentDigest": "sha256:abcdef", + "currentValue": "1.0.0", + "datasource": "docker", + "depName": "image7", + "depType": "stage", + "replaceString": "image7:1.0.0@sha256:abcdef", + }, + Object { + "autoReplaceStringTemplate": "{{depName}}{{#if newValue}}:{{newValue}}{{/if}}{{#if newDigest}}@{{newDigest}}{{/if}}", + "currentDigest": undefined, + "currentValue": undefined, + "datasource": "docker", + "depName": "image9", + "depType": "stage", + "replaceString": "image9", + }, + Object { + "autoReplaceStringTemplate": "{{depName}}{{#if newValue}}:{{newValue}}{{/if}}{{#if newDigest}}@{{newDigest}}{{/if}}", + "currentDigest": undefined, + "currentValue": undefined, + "datasource": "docker", + "depName": "image10", + "depType": "stage", + "replaceString": "image10", + }, + Object { + "autoReplaceStringTemplate": "{{depName}}{{#if newValue}}:{{newValue}}{{/if}}{{#if newDigest}}@{{newDigest}}{{/if}}", + "currentDigest": undefined, + "currentValue": undefined, + "datasource": "docker", + "depName": "image11", + "depType": "stage", + "replaceString": "image11", + }, + Object { + "autoReplaceStringTemplate": "{{depName}}{{#if newValue}}:{{newValue}}{{/if}}{{#if newDigest}}@{{newDigest}}{{/if}}", + "currentDigest": undefined, + "currentValue": undefined, + "datasource": "docker", + "depName": "image12", + "depType": "stage", + "replaceString": "image12", + }, + Object { + "autoReplaceStringTemplate": "{{depName}}{{#if newValue}}:{{newValue}}{{/if}}{{#if newDigest}}@{{newDigest}}{{/if}}", + "currentDigest": undefined, + "currentValue": undefined, + "datasource": "docker", + "depName": "image13", + "depType": "final", + "replaceString": "image13", + }, +] +`; + exports[`lib/manager/dockerfile/extract extractPackageFile() extracts images on adjacent lines 1`] = ` Array [ Object { diff --git a/lib/manager/dockerfile/extract.spec.ts b/lib/manager/dockerfile/extract.spec.ts index 93d0af0dacf2ac..347664ebde61e6 100644 --- a/lib/manager/dockerfile/extract.spec.ts +++ b/lib/manager/dockerfile/extract.spec.ts @@ -6,6 +6,11 @@ const d1 = readFileSync( 'utf8' ); +const d2 = readFileSync( + 'lib/manager/dockerfile/__fixtures__/Dockerfile2', + 'utf8' +); + describe('lib/manager/dockerfile/extract', () => { describe('extractPackageFile()', () => { it('handles no FROM', () => { @@ -142,6 +147,11 @@ describe('lib/manager/dockerfile/extract', () => { expect(res).toMatchSnapshot(); expect(res).toHaveLength(2); }); + it('extracts images from all sorts of (maybe multiline) FROM and COPY --from statements', () => { + const res = extractPackageFile(d2).deps; + expect(res).toMatchSnapshot(); + expect(res).toHaveLength(11); + }); it('handles calico/node', () => { const res = extractPackageFile('FROM calico/node\n').deps; expect(res).toMatchSnapshot(); diff --git a/lib/manager/dockerfile/extract.ts b/lib/manager/dockerfile/extract.ts index aaba5097229e5d..6705b9ea2aed8d 100644 --- a/lib/manager/dockerfile/extract.ts +++ b/lib/manager/dockerfile/extract.ts @@ -65,56 +65,61 @@ export function getDep( export function extractPackageFile(content: string): PackageFile | null { const deps: PackageDependency[] = []; const stageNames: string[] = []; - let lineNumber = 0; - for (const fromLine of content.split('\n')) { - const fromMatch = /^FROM /i.test(fromLine); - if (fromMatch) { - logger.trace({ lineNumber, fromLine }, 'FROM line'); - const [, currentFrom, ...fromRest] = fromLine.match(/\S+/g); - if (fromRest.length === 2 && fromRest[0].toLowerCase() === 'as') { - logger.debug('Found a multistage build stage name'); - stageNames.push(fromRest[1]); - } - if (currentFrom === 'scratch') { - logger.debug('Skipping scratch'); - } else if (stageNames.includes(currentFrom)) { - logger.debug({ currentFrom }, 'Skipping alias FROM'); - } else { - const dep = getDep(currentFrom); - logger.trace( - { - depName: dep.depName, - currentValue: dep.currentValue, - currentDigest: dep.currentDigest, - }, - 'Dockerfile FROM' - ); - deps.push(dep); - } + + const fromMatches = content.matchAll( + /^[ \t]*FROM(?:\\\r?\n| |\t|#.*\r?\n|-\S+)+(?\S+)(?:(?:\\\r?\n| |\t|#.*\r?\n)+as[ \t]+(?\S+))?/gim + ); + + for (const fromMatch of fromMatches) { + if (fromMatch.groups.name) { + logger.debug('Found a multistage build stage name'); + stageNames.push(fromMatch.groups.name); + } + if (fromMatch.groups.image === 'scratch') { + logger.debug('Skipping scratch'); + } else if (stageNames.includes(fromMatch.groups.image)) { + logger.debug({ image: fromMatch.groups.image }, 'Skipping alias FROM'); + } else { + const dep = getDep(fromMatch.groups.image); + logger.trace( + { + depName: dep.depName, + currentValue: dep.currentValue, + currentDigest: dep.currentDigest, + }, + 'Dockerfile FROM' + ); + deps.push(dep); } + } + + const copyFromMatches = content.matchAll( + /^[ \t]*COPY(?:\\\r?\n| |\t|#.*\r?\n|-\S+)+--from=(?\S+)/gim + ); - const copyFromMatch = /^(COPY --from=)([^\s]+)\s+(.*)$/i.exec(fromLine); - if (copyFromMatch) { - const [, , currentFrom] = copyFromMatch; - logger.trace({ lineNumber, fromLine }, 'COPY --from line'); - if (stageNames.includes(currentFrom)) { - logger.debug({ currentFrom }, 'Skipping alias COPY --from'); - } else if (!Number.isNaN(Number(currentFrom))) { - logger.debug({ currentFrom }, 'Skipping index reference COPY --from'); - } else { - const dep = getDep(currentFrom); - logger.debug( - { - depName: dep.depName, - currentValue: dep.currentValue, - currentDigest: dep.currentDigest, - }, - 'Dockerfile COPY --from' - ); - deps.push(dep); - } + for (const copyFromMatch of copyFromMatches) { + if (stageNames.includes(copyFromMatch.groups.image)) { + logger.debug( + { image: copyFromMatch.groups.image }, + 'Skipping alias COPY --from' + ); + } else if (!Number.isNaN(Number(copyFromMatch.groups.image))) { + logger.debug( + { image: copyFromMatch.groups.image }, + 'Skipping index reference COPY --from' + ); + } else { + const dep = getDep(copyFromMatch.groups.image); + logger.debug( + { + depName: dep.depName, + currentValue: dep.currentValue, + currentDigest: dep.currentDigest, + }, + 'Dockerfile COPY --from' + ); + deps.push(dep); } - lineNumber += 1; } if (!deps.length) { return null; From 22245ceff7a61e0fbf1f196f07ba36bc240421aa Mon Sep 17 00:00:00 2001 From: Rhys Arkins Date: Tue, 19 Jan 2021 10:15:31 +0100 Subject: [PATCH 40/52] tests: fix coverage (#8340) --- lib/config/migration.spec.ts | 20 ++++++++++++++++++++ lib/util/exec/index.ts | 13 +++++-------- 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/lib/config/migration.spec.ts b/lib/config/migration.spec.ts index cb62c8b6963967..70931e10c9de4b 100644 --- a/lib/config/migration.spec.ts +++ b/lib/config/migration.spec.ts @@ -557,5 +557,25 @@ describe('config/migration', () => { extends: [':unpublishSafeDisabled', 'npm:unpublishSafe'], }); }); + it('migrates combinations of packageRules', () => { + let config: RenovateConfig; + let res: MigratedConfig; + + config = { + packages: [{ packagePatterns: ['*'] }], + packageRules: { packageNames: [] }, + } as never; + res = configMigration.migrateConfig(config); + expect(res.isMigrated).toBe(true); + expect(res.migratedConfig.packageRules).toHaveLength(2); + + config = { + packageRules: { packageNames: [] }, + packages: [{ packagePatterns: ['*'] }], + } as never; + res = configMigration.migrateConfig(config); + expect(res.isMigrated).toBe(true); + expect(res.migratedConfig.packageRules).toHaveLength(2); + }); }); }); diff --git a/lib/util/exec/index.ts b/lib/util/exec/index.ts index e6d40915a4089c..1ede34ce6353d7 100644 --- a/lib/util/exec/index.ts +++ b/lib/util/exec/index.ts @@ -62,14 +62,11 @@ function createChildEnv( }); const extraEnvKeys = Object.keys(extraEnvEntries); - const childEnv = - env || extraEnv - ? { - ...extraEnv, - ...getChildProcessEnv(extraEnvKeys), - ...env, - } - : getChildProcessEnv(); + const childEnv = { + ...extraEnv, + ...getChildProcessEnv(extraEnvKeys), + ...env, + }; const result: ExtraEnv = {}; Object.entries(childEnv).forEach(([key, val]) => { From 868208bfef9f6fb0730ea39c03942ea546af78bc Mon Sep 17 00:00:00 2001 From: Andrey Kuznetsov Date: Tue, 19 Jan 2021 12:19:07 +0300 Subject: [PATCH 41/52] fix: configure signing for each repo (#8314) --- lib/util/git/index.ts | 5 +++-- lib/util/git/private-key.spec.ts | 17 ++++++++++++----- lib/util/git/private-key.ts | 13 ++++++++++--- 3 files changed, 25 insertions(+), 10 deletions(-) diff --git a/lib/util/git/index.ts b/lib/util/git/index.ts index aa4e9de0e49522..4f6dd12c0c429a 100644 --- a/lib/util/git/index.ts +++ b/lib/util/git/index.ts @@ -20,7 +20,7 @@ import { logger } from '../../logger'; import { ExternalHostError } from '../../types/errors/external-host-error'; import { GitOptions, GitProtocol } from '../../types/git'; import { Limit, incLimitedValue } from '../../workers/global/limits'; -import { writePrivateKey } from './private-key'; +import { configSigningKey, writePrivateKey } from './private-key'; export * from './private-key'; @@ -580,9 +580,10 @@ export async function commitFiles({ await syncGit(); logger.debug(`Committing files to branch ${branchName}`); if (!privateKeySet) { - await writePrivateKey(config.localDir); + await writePrivateKey(); privateKeySet = true; } + await configSigningKey(config.localDir); try { await git.reset(ResetMode.HARD); await git.raw(['clean', '-fd']); diff --git a/lib/util/git/private-key.spec.ts b/lib/util/git/private-key.spec.ts index 61a9a32b03b7e7..ca3e61f77fbfcf 100644 --- a/lib/util/git/private-key.spec.ts +++ b/lib/util/git/private-key.spec.ts @@ -1,6 +1,10 @@ import { getName, mocked } from '../../../test/util'; import * as exec_ from '../exec'; -import { setPrivateKey, writePrivateKey } from './private-key'; +import { + configSigningKey, + setPrivateKey, + writePrivateKey, +} from './private-key'; jest.mock('fs-extra'); jest.mock('../exec'); @@ -10,7 +14,8 @@ const exec = mocked(exec_); describe(getName(__filename), () => { describe('writePrivateKey()', () => { it('returns if no private key', async () => { - await expect(writePrivateKey('/tmp/some-repo')).resolves.not.toThrow(); + await expect(writePrivateKey()).resolves.not.toThrow(); + await expect(configSigningKey('/tmp/some-repo')).resolves.not.toThrow(); }); it('throws error if failing', async () => { setPrivateKey('some-key'); @@ -18,7 +23,7 @@ describe(getName(__filename), () => { stderr: `something wrong`, stdout: '', }); - await expect(writePrivateKey('/tmp/some-repo')).rejects.toThrow(); + await expect(writePrivateKey()).rejects.toThrow(); }); it('imports the private key', async () => { setPrivateKey('some-key'); @@ -26,10 +31,12 @@ describe(getName(__filename), () => { stderr: `gpg: key BADC0FFEE: secret key imported\nfoo\n`, stdout: '', }); - await expect(writePrivateKey('/tmp/some-repo')).resolves.not.toThrow(); + await expect(writePrivateKey()).resolves.not.toThrow(); + await expect(configSigningKey('/tmp/some-repo')).resolves.not.toThrow(); }); it('does not import the key again', async () => { - await expect(writePrivateKey('/tmp/some-repo')).resolves.not.toThrow(); + await expect(writePrivateKey()).resolves.not.toThrow(); + await expect(configSigningKey('/tmp/some-repo')).resolves.not.toThrow(); }); }); }); diff --git a/lib/util/git/private-key.ts b/lib/util/git/private-key.ts index a0e927bbe151a3..1b5a911b84b522 100644 --- a/lib/util/git/private-key.ts +++ b/lib/util/git/private-key.ts @@ -29,17 +29,24 @@ async function importKey(): Promise { await fs.remove(keyFileName); } -export async function writePrivateKey(cwd: string): Promise { +export async function writePrivateKey(): Promise { if (!gitPrivateKey) { return; } logger.debug('Setting git private key'); try { await importKey(); - await exec(`git config user.signingkey ${keyId}`, { cwd }); - await exec(`git config commit.gpgsign true`, { cwd }); } catch (err) { logger.warn({ err }, 'Error writing git private key'); throw new Error(PLATFORM_GPG_FAILED); } } + +export async function configSigningKey(cwd: string): Promise { + if (!gitPrivateKey) { + return; + } + logger.debug('Configuring commits signing'); + await exec(`git config user.signingkey ${keyId}`, { cwd }); + await exec(`git config commit.gpgsign true`, { cwd }); +} From 434fdbd4be9969bc0a8fb3f7f9fa0e26efe8da9f Mon Sep 17 00:00:00 2001 From: Michael Kriese Date: Tue, 19 Jan 2021 10:29:45 +0100 Subject: [PATCH 42/52] feat: deprecated logLevel option (#8301) * feat: deprecated logLevel option * feat: deprecated logLevel option * docs: update logging * fix: remove log level env option * Update warn Co-authored-by: Rhys Arkins --- docs/development/configuration.md | 3 ++- docs/usage/self-hosted-configuration.md | 2 ++ docs/usage/self-hosting.md | 2 -- lib/config/config/__fixtures__/file.js | 1 - lib/config/config/__fixtures__/with-force.js | 1 - lib/config/definitions.ts | 4 +--- lib/config/file.spec.ts | 1 - lib/config/index.ts | 10 ++++++++-- 8 files changed, 13 insertions(+), 11 deletions(-) diff --git a/docs/development/configuration.md b/docs/development/configuration.md index 8f50e1ba6a0f77..549c44712c0c32 100644 --- a/docs/development/configuration.md +++ b/docs/development/configuration.md @@ -38,7 +38,6 @@ e.g. apply one set of labels for `backend/package.json` and a different set of l module.exports = { npmrc: '//registry.npmjs.org/:_authToken=abc123', baseDir: '/tmp/renovate', - logLevel: 'debug', includeForks: true, gradle: { enabled: false }, }; @@ -53,6 +52,8 @@ $ node renovate --help To configure any `` items, separate with commas. E.g. `renovate --labels=renovate,dependency`. +To enable debug logging export `LOG_LEVEL=debug` to your environment. + ### renovate.json If you add a `renovate.json` file to the root of your repository, you can use this to override default settings. diff --git a/docs/usage/self-hosted-configuration.md b/docs/usage/self-hosted-configuration.md index 2ccacf6d5f0aa0..30876451ee6851 100644 --- a/docs/usage/self-hosted-configuration.md +++ b/docs/usage/self-hosted-configuration.md @@ -172,6 +172,8 @@ By configuring using the environment it means that debug logging starts from the Additionally, if you configure `LOG_FORMAT=json` in env then logging will be done in JSON format instead of "pretty" format, which is usually better if you're doing any ingestion or parsing of the logs. +Warning: Configuring `logLevel` config option or `--log-level` cli option is deprecated and will be removed in a major version. + ## onboarding Set this to `false` if (a) you configure Renovate entirely on the bot side (i.e. empty `renovate.json` in repositories) and (b) you wish to run Renovate on every repository the bot has access to, and (c) you wish to skip the onboarding PRs. diff --git a/docs/usage/self-hosting.md b/docs/usage/self-hosting.md index 82c9cd183ee8be..76218dd17c5e53 100644 --- a/docs/usage/self-hosting.md +++ b/docs/usage/self-hosting.md @@ -101,7 +101,6 @@ metadata: data: config.json: |- { - "logLevel" : "debug", "repositories": ["orgname/repo","username/repo"], "dryRun" : "true" } @@ -280,7 +279,6 @@ module.exports = { endpoint: 'https://self-hosted.gitlab/api/v4/', token: '**gitlab_token**', platform: 'gitlab', - logLevel: 'debug', onboardingConfig: { extends: ['config:base'], }, diff --git a/lib/config/config/__fixtures__/file.js b/lib/config/config/__fixtures__/file.js index 61600196f14b7d..e4fe0d85493f41 100644 --- a/lib/config/config/__fixtures__/file.js +++ b/lib/config/config/__fixtures__/file.js @@ -1,5 +1,4 @@ // @ts-ignore module.exports = { token: 'abcdefg', - logLevel: 'error' }; diff --git a/lib/config/config/__fixtures__/with-force.js b/lib/config/config/__fixtures__/with-force.js index d16cae37e45284..ad2b1074fb23d4 100644 --- a/lib/config/config/__fixtures__/with-force.js +++ b/lib/config/config/__fixtures__/with-force.js @@ -1,7 +1,6 @@ // @ts-ignore module.exports = { token: 'abcdefg', - logLevel: 'error', force: { schedule: null, } diff --git a/lib/config/definitions.ts b/lib/config/definitions.ts index b53af3cd45e547..deb1c21d4c7f5c 100644 --- a/lib/config/definitions.ts +++ b/lib/config/definitions.ts @@ -350,12 +350,10 @@ const options: RenovateOptions[] = [ // Log options { name: 'logLevel', - description: 'Logging level', + description: 'Logging level. Deprecated, use `LOG_LEVEL` environment.', stage: 'global', type: 'string', allowedValues: ['fatal', 'error', 'warn', 'info', 'debug', 'trace'], - default: 'info', - env: 'LOG_LEVEL', }, { name: 'logFile', diff --git a/lib/config/file.spec.ts b/lib/config/file.spec.ts index 4eeed34655d5fd..f5eb41da982383 100644 --- a/lib/config/file.spec.ts +++ b/lib/config/file.spec.ts @@ -37,7 +37,6 @@ describe('config/file', () => { "platform": "github", "token":"abcdef", "logFileLevel": "warn", - "logLevel": "info", "onboarding": false, "gitAuthor": "Renovate Bot " "onboardingConfig": { diff --git a/lib/config/index.ts b/lib/config/index.ts index 0f4699091c0fa9..8c8eae307a8082 100644 --- a/lib/config/index.ts +++ b/lib/config/index.ts @@ -80,8 +80,14 @@ export async function parseConfigs( delete config.privateKeyPath; } - // Set log level - levels('stdout', config.logLevel); + // Deprecated set log level: https://github.com/renovatebot/renovate/issues/8291 + // istanbul ignore if + if (config.logLevel) { + logger.warn( + 'Configuring logLevel in CLI or file is deprecated. Use LOG_LEVEL environment variable instead' + ); + levels('stdout', config.logLevel); + } if (config.logContext) { // This only has an effect if logContext was defined via file or CLI, otherwise it would already have been detected in env From 13f5bb9d08921394fca85f6c16114bf007a26ddf Mon Sep 17 00:00:00 2001 From: Rhys Arkins Date: Tue, 19 Jan 2021 10:58:30 +0100 Subject: [PATCH 43/52] fix: revert YARN_CACHE_FOLDER Closes #8247 --- lib/manager/npm/post-update/lerna.ts | 1 - lib/manager/npm/post-update/yarn.ts | 1 - 2 files changed, 2 deletions(-) diff --git a/lib/manager/npm/post-update/lerna.ts b/lib/manager/npm/post-update/lerna.ts index c9377076813d29..bbb95de91afda5 100644 --- a/lib/manager/npm/post-update/lerna.ts +++ b/lib/manager/npm/post-update/lerna.ts @@ -81,7 +81,6 @@ export async function generateLockFiles( extraEnv: { NPM_CONFIG_CACHE: env.NPM_CONFIG_CACHE, npm_config_store: env.npm_config_store, - YARN_CACHE_FOLDER: env.YARN_CACHE_FOLDER, }, docker: { image: 'renovate/node', diff --git a/lib/manager/npm/post-update/yarn.ts b/lib/manager/npm/post-update/yarn.ts index 6f17b1d9ea0983..addc60925fdc23 100644 --- a/lib/manager/npm/post-update/yarn.ts +++ b/lib/manager/npm/post-update/yarn.ts @@ -64,7 +64,6 @@ export async function generateLockFile( const extraEnv: ExecOptions['extraEnv'] = { NPM_CONFIG_CACHE: env.NPM_CONFIG_CACHE, npm_config_store: env.npm_config_store, - YARN_CACHE_FOLDER: env.YARN_CACHE_FOLDER, CI: 'true', }; From 74b7038907bf60e97350aa27f5a7d9815ce8f734 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 19 Jan 2021 14:54:46 +0000 Subject: [PATCH 44/52] build(deps): update dependency azure-devops-node-api to v10.2.1 (#8345) Co-authored-by: Renovate Bot --- package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index d0be090d10ea1d..1905e37920d8d1 100644 --- a/package.json +++ b/package.json @@ -120,7 +120,7 @@ "@yarnpkg/core": "2.4.0", "@yarnpkg/parsers": "2.3.0", "aws-sdk": "2.827.0", - "azure-devops-node-api": "10.2.0", + "azure-devops-node-api": "10.2.1", "bunyan": "1.8.15", "cacache": "15.0.5", "chalk": "4.1.0", diff --git a/yarn.lock b/yarn.lock index b90937164a6070..992015bf981613 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2519,10 +2519,10 @@ aws4@^1.8.0: resolved "https://registry.yarnpkg.com/aws4/-/aws4-1.11.0.tgz#d61f46d83b2519250e2784daf5b09479a8b41c59" integrity sha512-xh1Rl34h6Fi1DC2WWKfxUTVqRsNnr6LsKz2+hfwDxQJWmrx8+c7ylaqBMcHfl1U1r2dsifOvKX3LQuLNZ+XSvA== -azure-devops-node-api@10.2.0: - version "10.2.0" - resolved "https://registry.yarnpkg.com/azure-devops-node-api/-/azure-devops-node-api-10.2.0.tgz#3faea4ac40bb62e554ec5174a8cc6d19cc36ae73" - integrity sha512-5yXGb0kBszCiLw1qZNtS/y3EqGI/nVtkndX5n9qsdU26pJjFwyg9pAOjixni34R+IX4u243p34oXQwXaKbyD8g== +azure-devops-node-api@10.2.1: + version "10.2.1" + resolved "https://registry.yarnpkg.com/azure-devops-node-api/-/azure-devops-node-api-10.2.1.tgz#835080164f8c30cec6506c47198b044c053f1f36" + integrity sha512-XuSiUaYpk0tQpd9fD8qfRa5y1IdavupKNVmwxy0w/RhmxG2Wl8uAYnNJchUoWd3Rn9On0mYTCCZSn+UlYdYFSg== dependencies: tunnel "0.0.6" typed-rest-client "^1.8.0" From ebb40b3adb5e0160468dd0b3528d75b0bc1380c8 Mon Sep 17 00:00:00 2001 From: Rhys Arkins Date: Tue, 19 Jan 2021 23:07:12 +0100 Subject: [PATCH 45/52] fix(github): pass up repo not found in init (#8355) --- lib/platform/github/index.ts | 3 ++- lib/workers/repository/result.ts | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/platform/github/index.ts b/lib/platform/github/index.ts index cfe193aa7c4fb8..7739a43ec4528d 100644 --- a/lib/platform/github/index.ts +++ b/lib/platform/github/index.ts @@ -248,7 +248,8 @@ export async function initRepo({ logger.debug('Caught initRepo error'); if ( err.message === REPOSITORY_ARCHIVED || - err.message === REPOSITORY_RENAMED + err.message === REPOSITORY_RENAMED || + err.message === REPOSITORY_NOT_FOUND ) { throw err; } diff --git a/lib/workers/repository/result.ts b/lib/workers/repository/result.ts index 9156dcd15e715b..5858ce99c34270 100644 --- a/lib/workers/repository/result.ts +++ b/lib/workers/repository/result.ts @@ -11,6 +11,7 @@ import { REPOSITORY_EMPTY, REPOSITORY_FORKED, REPOSITORY_MIRRORED, + REPOSITORY_NOT_FOUND, REPOSITORY_RENAMED, REPOSITORY_UNINITIATED, } from '../../constants/error-messages'; @@ -40,6 +41,7 @@ export function processResult( REPOSITORY_RENAMED, REPOSITORY_UNINITIATED, REPOSITORY_EMPTY, + REPOSITORY_NOT_FOUND, ]; const enabledStatuses = [CONFIG_SECRETS_EXPOSED, CONFIG_VALIDATION]; let status: ProcessStatus; From 880de74dc03a37579d130b5e00e292139b8d1611 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 19 Jan 2021 23:43:20 +0000 Subject: [PATCH 46/52] chore(deps): update dependency @types/jsdom to v16.2.6 (#8352) Co-authored-by: Renovate Bot --- package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index 1905e37920d8d1..aed17177ddf00b 100644 --- a/package.json +++ b/package.json @@ -210,7 +210,7 @@ "@types/ini": "1.3.30", "@types/jest": "26.0.20", "@types/js-yaml": "3.12.6", - "@types/jsdom": "16.2.5", + "@types/jsdom": "16.2.6", "@types/json-dup-key-validator": "1.0.0", "@types/json5": "0.0.30", "@types/linkify-markdown": "1.0.0", diff --git a/yarn.lock b/yarn.lock index 992015bf981613..88a234d6ed6d23 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1669,10 +1669,10 @@ resolved "https://registry.yarnpkg.com/@types/js-yaml/-/js-yaml-3.12.6.tgz#7f10c926aa41e189a2755c4c7fcf8e4573bd7ac1" integrity sha512-cK4XqrLvP17X6c0C8n4iTbT59EixqyXL3Fk8/Rsk4dF3oX4dg70gYUXrXVUUHpnsGMPNlTQMqf+TVmNPX6FmSQ== -"@types/jsdom@16.2.5": - version "16.2.5" - resolved "https://registry.yarnpkg.com/@types/jsdom/-/jsdom-16.2.5.tgz#74ebad438741d249ecb416c5486dcde4217eb66c" - integrity sha512-k/ZaTXtReAjwWu0clU0KLS53dyqZnA8mm+jwKFeFrvufXgICp+VNbskETFxKKAguv0pkaEKTax5MaRmvalM+TA== +"@types/jsdom@16.2.6": + version "16.2.6" + resolved "https://registry.yarnpkg.com/@types/jsdom/-/jsdom-16.2.6.tgz#9ddf0521e49be5365797e690c3ba63148e562c29" + integrity sha512-yQA+HxknGtW9AkRTNyiSH3OKW5V+WzO8OPTdne99XwJkYC+KYxfNIcoJjeiSqP3V00PUUpFP6Myoo9wdIu78DQ== dependencies: "@types/node" "*" "@types/parse5" "*" From d0c3be38bfaff19befef8745d4e13ce1925f753f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Goetz?= Date: Wed, 20 Jan 2021 10:07:26 +0100 Subject: [PATCH 47/52] feat: Allow to specify pypi registry for other datasources (#8348) --- lib/manager/pip_requirements/extract.spec.ts | 6 ++++++ lib/manager/pip_requirements/extract.ts | 2 +- lib/manager/poetry/extract.spec.ts | 6 ++++++ lib/manager/poetry/extract.ts | 2 +- 4 files changed, 14 insertions(+), 2 deletions(-) diff --git a/lib/manager/pip_requirements/extract.spec.ts b/lib/manager/pip_requirements/extract.spec.ts index 1619e3b98c3349..271443a0fa4f57 100644 --- a/lib/manager/pip_requirements/extract.spec.ts +++ b/lib/manager/pip_requirements/extract.spec.ts @@ -45,8 +45,14 @@ describe('lib/manager/pip_requirements/extract', () => { }); describe('extractPackageFile()', () => { let config; + const OLD_ENV = process.env; beforeEach(() => { config = { registryUrls: ['AnExistingDefaultUrl'] }; + process.env = { ...OLD_ENV }; + delete process.env.PIP_INDEX_URL; + }); + afterEach(() => { + process.env = OLD_ENV; }); it('returns null for empty', () => { expect( diff --git a/lib/manager/pip_requirements/extract.ts b/lib/manager/pip_requirements/extract.ts index 0ec21b5437c9c9..448d90b4d655b3 100644 --- a/lib/manager/pip_requirements/extract.ts +++ b/lib/manager/pip_requirements/extract.ts @@ -44,7 +44,7 @@ export function extractPackageFile( registryUrls = registryUrls.concat(config.registryUrls); } else if (extraUrls.length) { // Use default registry first if extra URLs are present and index URL is not - registryUrls.push('https://pypi.org/pypi/'); + registryUrls.push(process.env.PIP_INDEX_URL || 'https://pypi.org/pypi/'); } registryUrls = registryUrls.concat(extraUrls); diff --git a/lib/manager/poetry/extract.spec.ts b/lib/manager/poetry/extract.spec.ts index f0b812696e0359..b91ca400507a3f 100644 --- a/lib/manager/poetry/extract.spec.ts +++ b/lib/manager/poetry/extract.spec.ts @@ -49,8 +49,14 @@ const pyproject9toml = readFileSync( describe('lib/manager/poetry/extract', () => { describe('extractPackageFile()', () => { let filename: string; + const OLD_ENV = process.env; beforeEach(() => { filename = ''; + process.env = { ...OLD_ENV }; + delete process.env.PIP_INDEX_URL; + }); + afterEach(() => { + process.env = OLD_ENV; }); it('returns null for empty', () => { expect(extractPackageFile('nothing here', filename)).toBeNull(); diff --git a/lib/manager/poetry/extract.ts b/lib/manager/poetry/extract.ts index eff864a62cb678..fb2bc1485326e3 100644 --- a/lib/manager/poetry/extract.ts +++ b/lib/manager/poetry/extract.ts @@ -82,7 +82,7 @@ function extractRegistries(pyprojectfile: PoetryFile): string[] { registryUrls.add(source.url); } } - registryUrls.add('https://pypi.org/pypi/'); + registryUrls.add(process.env.PIP_INDEX_URL || 'https://pypi.org/pypi/'); return Array.from(registryUrls); } From 2d9d64d1bc644edc0672448d9293658b1ad29c98 Mon Sep 17 00:00:00 2001 From: Sergei Zharinov Date: Wed, 20 Jan 2021 13:10:20 +0400 Subject: [PATCH 48/52] fix(sbt): Parse dependency the more precise way (#8353) --- lib/manager/sbt/__fixtures__/sample.sbt | 2 ++ lib/manager/sbt/__snapshots__/extract.spec.ts.snap | 14 ++++++++++++++ lib/manager/sbt/extract.ts | 4 +++- 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/lib/manager/sbt/__fixtures__/sample.sbt b/lib/manager/sbt/__fixtures__/sample.sbt index a2f4e94dcdca27..9c82f1186c60e9 100644 --- a/lib/manager/sbt/__fixtures__/sample.sbt +++ b/lib/manager/sbt/__fixtures__/sample.sbt @@ -33,3 +33,5 @@ resolvers ++= Seq( resolvers ++= Seq("Repo #5" at "https://example.com/repos/5/") addSbtPlugin("org.example" % "waldo" % "0.0.9") + +libraryDependencies += "org.example" % "fred" % "(,8.4.0]" diff --git a/lib/manager/sbt/__snapshots__/extract.spec.ts.snap b/lib/manager/sbt/__snapshots__/extract.spec.ts.snap index 4c70a362425226..770910ed1e034b 100644 --- a/lib/manager/sbt/__snapshots__/extract.spec.ts.snap +++ b/lib/manager/sbt/__snapshots__/extract.spec.ts.snap @@ -239,6 +239,20 @@ Object { "https://dl.bintray.com/sbt/sbt-plugin-releases", ], }, + Object { + "currentValue": "(,8.4.0]", + "datasource": "sbt-package", + "depName": "org.example:fred", + "lookupName": "org.example:fred", + "registryUrls": Array [ + "https://repo.maven.apache.org/maven2", + "https://example.com/repos/1/", + "https://example.com/repos/2/", + "https://example.com/repos/3/", + "https://example.com/repos/4/", + "https://example.com/repos/5/", + ], + }, ], "packageFileVersion": "1.0", } diff --git a/lib/manager/sbt/extract.ts b/lib/manager/sbt/extract.ts index 6743b2e5d423a8..57f9a0a41d5cfa 100644 --- a/lib/manager/sbt/extract.ts +++ b/lib/manager/sbt/extract.ts @@ -125,7 +125,9 @@ function parseDepExpr( const tokens = expr .trim() - .replace(/[()]/g, '') + .split(/("[^"]*")/g) + .map((x) => (/"[^"]*"/.test(x) ? x : x.replace(/[()]+/g, ''))) + .join('') .split(/\s*(%%?)\s*|\s*classifier\s*/); const [ From dbd7a0d73f5d9d523647e73dd1b0d37fe36f2d2e Mon Sep 17 00:00:00 2001 From: HonkingGoose <34918129+HonkingGoose@users.noreply.github.com> Date: Wed, 20 Jan 2021 10:10:45 +0100 Subject: [PATCH 49/52] chore: remove old TODO comment (#8351) --- lib/config/migrate-validate.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/config/migrate-validate.ts b/lib/config/migrate-validate.ts index 63568b5b9c0811..294a6856cf5af1 100644 --- a/lib/config/migrate-validate.ts +++ b/lib/config/migrate-validate.ts @@ -38,7 +38,6 @@ export async function migrateAndValidate( } massagedConfig.errors = (config.errors || []).concat(errors); if (!config.repoIsOnboarded) { - // TODO #556 - enable warnings in real PRs massagedConfig.warnings = (config.warnings || []).concat(warnings); } return massagedConfig; From 3756c7c9419086c497aabaf6e75fbe621a248097 Mon Sep 17 00:00:00 2001 From: Rhys Arkins Date: Wed, 20 Jan 2021 15:05:09 +0100 Subject: [PATCH 50/52] docs: automergeType elaboration --- docs/usage/configuration-options.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/usage/configuration-options.md b/docs/usage/configuration-options.md index d778065ade4787..c4cf9e8abda92f 100644 --- a/docs/usage/configuration-options.md +++ b/docs/usage/configuration-options.md @@ -165,7 +165,9 @@ Example use: This setting is only applicable if you opt in to configure `automerge` to `true` for any of your dependencies. Automerging defaults to using Pull Requests (`automergeType="pr"`). -In that case Renovate first creates a branch, then an associated Pull Request, and then automerges the first time it detects that the Pull Requests status checks are "green". +In that case Renovate first creates a branch and associated Pull Request, and then automerges the PR on a subsequent run once it detects the PR's status checks are "green". +If by the next run the PR is already behind master branch then it will be automatically rebased, because Renovate only automerges branches which are up-to-date and green. +If Renovate is scheduled for hourly runs on the repository but commits are made every 15 minutes to the main branch, then an automerge like this will keep getting deferred with every rebase. Note: if you have no tests but still want Renovate to automerge, you need to add `"requiredStatusChecks": null` to your configuration. From 6cf87352048b562c713bce39b2f411cb661dcea0 Mon Sep 17 00:00:00 2001 From: Rhys Arkins Date: Thu, 21 Jan 2021 08:28:41 +0100 Subject: [PATCH 51/52] build: add junit5 monorepo --- lib/config/presets/internal/monorepo.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/config/presets/internal/monorepo.ts b/lib/config/presets/internal/monorepo.ts index e03998a947dcc3..9c8f4791b3d987 100644 --- a/lib/config/presets/internal/monorepo.ts +++ b/lib/config/presets/internal/monorepo.ts @@ -82,6 +82,7 @@ const repoGroups = { jasmine: 'https://github.com/jasmine/jasmine', jersey: 'https://github.com/eclipse-ee4j/jersey', jest: 'https://github.com/facebook/jest', + junit5: 'https://github.com/junit-team/junit5', lerna: 'https://github.com/lerna/lerna', lingui: 'https://github.com/lingui/js-lingui', lodash: 'https://github.com/lodash/', From d0531f52fbfe8e508f83ee0040678eb522d41d63 Mon Sep 17 00:00:00 2001 From: Rhys Arkins Date: Thu, 21 Jan 2021 10:04:22 +0100 Subject: [PATCH 52/52] refactor: remove appendRegistryUrls --- lib/datasource/common.ts | 1 - lib/datasource/index.ts | 6 +++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/lib/datasource/common.ts b/lib/datasource/common.ts index a8f26f015aa30b..03653f42340fc4 100644 --- a/lib/datasource/common.ts +++ b/lib/datasource/common.ts @@ -75,7 +75,6 @@ export interface DatasourceApi { getDigest?(config: DigestConfig, newValue?: string): Promise; getReleases(config: GetReleasesConfig): Promise; defaultRegistryUrls?: string[]; - appendRegistryUrls?: string[]; defaultConfig?: Record; registryStrategy?: 'first' | 'hunt' | 'merge'; } diff --git a/lib/datasource/index.ts b/lib/datasource/index.ts index 2aa1554f0d5886..6169abc8d646fd 100644 --- a/lib/datasource/index.ts +++ b/lib/datasource/index.ts @@ -151,13 +151,13 @@ function resolveRegistryUrls( datasource: Datasource, extractedUrls: string[] ): string[] { - const { defaultRegistryUrls = [], appendRegistryUrls = [] } = datasource; + const { defaultRegistryUrls = [] } = datasource; const customUrls = extractedUrls?.filter(Boolean); let registryUrls: string[]; if (is.nonEmptyArray(customUrls)) { - registryUrls = [...extractedUrls, ...appendRegistryUrls]; + registryUrls = [...customUrls]; } else { - registryUrls = [...defaultRegistryUrls, ...appendRegistryUrls]; + registryUrls = [...defaultRegistryUrls]; } return registryUrls.filter(Boolean); }