diff --git a/dev/archery/archery/crossbow/core.py b/dev/archery/archery/crossbow/core.py index c8ea8a13a4b..a83c190d121 100644 --- a/dev/archery/archery/crossbow/core.py +++ b/dev/archery/archery/crossbow/core.py @@ -738,14 +738,16 @@ class Target(Serializable): (currently only an email address where the notification should be sent). """ - def __init__(self, head, branch, remote, version, email=None): + def __init__(self, head, branch, remote, version, r_version, email=None): self.head = head self.email = email self.branch = branch self.remote = remote self.github_repo = "/".join(_parse_github_user_repo(remote)) self.version = version + self.r_version = r_version self.no_rc_version = re.sub(r'-rc\d+\Z', '', version) + self.no_rc_r_version = re.sub(r'-rc\d+\Z', '', r_version) # TODO(ARROW-17552): Remove "master" from default_branch after # migration to "main". self.default_branch = ['main', 'master'] @@ -791,8 +793,39 @@ def from_repo(cls, repo, head=None, branch=None, remote=None, version=None, if email is None: email = repo.user_email + version_dev_match = re.match(r".*\.dev(\d+)$", version) + if version_dev_match: + with open(f"{repo.path}/r/DESCRIPTION") as description_file: + description = description_file.read() + r_version_pattern = re.compile(r"^Version:\s*(.*)$", + re.MULTILINE) + r_version = re.findall(r_version_pattern, description)[0] + if r_version: + version_dev = int(version_dev_match[1]) + # "1_0000_00_00 +" is for generating a greater version + # than YYYYMMDD. For example, 1_0000_00_01 + # (version_dev == 1 case) is greater than 2022_10_16. + # + # Why do we need a greater version than YYYYMMDD? It's + # for keeping backward compatibility. We used + # MAJOR.MINOR.PATCH.YYYYMMDD as our nightly package + # version. (See also ARROW-16403). If we use "9000 + + # version_dev" here, a developer that used + # 9.0.0.20221016 can't upgrade to the later nightly + # package unless we release 10.0.0. Because 9.0.0.9234 + # or something is less than 9.0.0.20221016. + r_version_dev = 1_0000_00_00 + version_dev + # version: 10.0.0.dev234 + # r_version: 9.0.0.9000 + # -> 9.0.0.100000234 + r_version = re.sub(r"\.9000\Z", f".{r_version_dev}", r_version) + else: + r_version = version + else: + r_version = version + return cls(head=head, email=email, branch=branch, remote=remote, - version=version) + version=version, r_version=r_version) def is_default_branch(self): # TODO(ARROW-17552): Switch the condition to "is" instead of "in" @@ -1105,7 +1138,10 @@ def from_config(cls, config, target, tasks=None, groups=None, params=None): 'version': target.version, 'no_rc_version': target.no_rc_version, 'no_rc_semver_version': target.no_rc_semver_version, - 'no_rc_snapshot_version': target.no_rc_snapshot_version} + 'no_rc_snapshot_version': target.no_rc_snapshot_version, + 'r_version': target.r_version, + 'no_rc_r_version': target.no_rc_r_version, + } for task_name, task in task_definitions.items(): task = task.copy() artifacts = task.pop('artifacts', None) or [] # because of yaml @@ -1260,6 +1296,7 @@ def validate(self): branch='master', remote='https://github.com/apache/arrow', version='1.0.0dev123', + r_version='0.13.0.100000123', email='dummy@example.ltd' ) job = Job.from_config(config=self, diff --git a/dev/archery/archery/crossbow/tests/fixtures/crossbow-job-no-failure.yaml b/dev/archery/archery/crossbow/tests/fixtures/crossbow-job-no-failure.yaml index 15e8ca3ff5e..eb03bbee0bd 100644 --- a/dev/archery/archery/crossbow/tests/fixtures/crossbow-job-no-failure.yaml +++ b/dev/archery/archery/crossbow/tests/fixtures/crossbow-job-no-failure.yaml @@ -5,7 +5,9 @@ target: !Target branch: refs/pull/4435/merge remote: https://github.com/apache/arrow version: 0.13.0.dev306 + r_version: 0.12.0.100000306 no_rc_version: 0.13.0.dev306 + no_rc_r_version: 0.12.0.100000306 tasks: docker-cpp-cmake32: !Task ci: circle @@ -64,4 +66,4 @@ branch: ursabot-1 _queue: !Queue path: the_path github_token: xxxxxxxxx - _remote_url: https://github.com/apache/crossbow \ No newline at end of file + _remote_url: https://github.com/apache/crossbow diff --git a/dev/archery/archery/crossbow/tests/fixtures/crossbow-job.yaml b/dev/archery/archery/crossbow/tests/fixtures/crossbow-job.yaml index 90eab704988..f6de07dd456 100644 --- a/dev/archery/archery/crossbow/tests/fixtures/crossbow-job.yaml +++ b/dev/archery/archery/crossbow/tests/fixtures/crossbow-job.yaml @@ -5,7 +5,9 @@ target: !Target branch: refs/pull/4435/merge remote: https://github.com/apache/arrow version: 0.13.0.dev306 + r_version: 0.12.0.100000306 no_rc_version: 0.13.0.dev306 + no_rc_r_version: 0.12.0.100000306 tasks: docker-cpp-cmake32: !Task ci: circle @@ -64,4 +66,4 @@ branch: ursabot-1 _queue: !Queue path: the_path github_token: xxxxxxxxx - _remote_url: https://github.com/apache/crossbow \ No newline at end of file + _remote_url: https://github.com/apache/crossbow diff --git a/dev/tasks/macros.jinja b/dev/tasks/macros.jinja index 3bec472bcf6..bd3358e0733 100644 --- a/dev/tasks/macros.jinja +++ b/dev/tasks/macros.jinja @@ -269,7 +269,7 @@ on: rm -f apache-arrow*.rb.bak {% endmacro %} -{%- macro github_change_r_pkg_version(is_fork, version = '\\2.\'\"$(date +%Y%m%d)\"\'' ) -%} +{%- macro github_change_r_pkg_version(is_fork, version) -%} - name: Modify version shell: bash run: | @@ -339,12 +339,16 @@ on: # tree not available in git-bash on windows run: | ls -R repo - - name: Add dev repo to .Rprofile + - name: Add repos to .Rprofile shell: Rscript {0} run: | - str <- paste0("options(arrow.dev_repo ='file://", getwd(), "/repo' )") - print(str) profile_path <- file.path(getwd(), ".Rprofile") + repo <- paste0("file://", getwd(), "/repo") + str <- paste0("options(arrow.repo = '", repo, "' )") + print(str) + write(str, file = profile_path, append = TRUE) + str <- paste0("options(arrow.dev_repo = '", repo, "' )") + print(str) write(str, file = profile_path, append = TRUE) # Set envvar for later steps by appending to $GITHUB_ENV write(paste0("R_PROFILE_USER=", profile_path), file = Sys.getenv("GITHUB_ENV"), append = TRUE) diff --git a/dev/tasks/r/github.packages.yml b/dev/tasks/r/github.packages.yml index efe071a766a..e53579f7416 100644 --- a/dev/tasks/r/github.packages.yml +++ b/dev/tasks/r/github.packages.yml @@ -17,10 +17,6 @@ {% import 'macros.jinja' as macros with context %} -# This allows us to set a custom version via param: -# crossbow submit --param custom_version=8.5.3 r-binary-packages -# if the param is unset defaults to the usual Ymd naming scheme -{% set package_version = custom_version|replace("Unset", "\\2.\'\"$(date +%Y%m%d)\"\'") %} {% set is_fork = macros.is_fork %} {{ macros.github_header() }} @@ -35,7 +31,7 @@ jobs: pkg_version: {{ '${{ steps.save-version.outputs.pkg_version }}' }} steps: {{ macros.github_checkout_arrow()|indent }} - {{ macros.github_change_r_pkg_version(is_fork, package_version)|indent }} + {{ macros.github_change_r_pkg_version(is_fork, arrow.no_rc_r_version)|indent }} - name: Save Version id: save-version shell: bash @@ -281,7 +277,7 @@ jobs: {{ macros.github_test_r_src_pkg()|indent(8) }} ' - name: Upload binary artifact - if: matrix.config.devtoolset + if: matrix.config.devtoolset uses: actions/upload-artifact@v3 with: name: r-pkg_centos7 diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index 81a2c24bba1..bdf53ff1dac 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -957,17 +957,17 @@ tasks: params: custom_version: Unset artifacts: - - r-lib__libarrow__bin__windows__arrow-[0-9\.]+\.zip - - r-lib__libarrow__bin__centos-7__arrow-[0-9\.]+\.zip - - r-lib__libarrow__bin__ubuntu-18.04__arrow-[0-9\.]+\.zip - - r-lib__libarrow__bin__ubuntu-22.04__arrow-[0-9\.]+\.zip - - r-pkg__bin__windows__contrib__4.1__arrow_[0-9\.]+\.zip - - r-pkg__bin__windows__contrib__4.2__arrow_[0-9\.]+\.zip - - r-pkg__bin__macosx__contrib__4.1__arrow_[0-9\.]+\.tgz - - r-pkg__bin__macosx__contrib__4.2__arrow_[0-9\.]+\.tgz - - r-pkg__bin__macosx__big-sur-arm64__contrib__4.1__arrow_[0-9\.]+\.tgz - - r-pkg__bin__macosx__big-sur-arm64__contrib__4.2__arrow_[0-9\.]+\.tgz - - r-pkg__src__contrib__arrow_[0-9\.]+\.tar\.gz + - r-lib__libarrow__bin__windows__arrow-{no_rc_r_version}\.zip + - r-lib__libarrow__bin__centos-7__arrow-{no_rc_r_version}\.zip + - r-lib__libarrow__bin__ubuntu-18.04__arrow-{no_rc_r_version}\.zip + - r-lib__libarrow__bin__ubuntu-22.04__arrow-{no_rc_r_version}\.zip + - r-pkg__bin__windows__contrib__4.1__arrow_{no_rc_r_version}\.zip + - r-pkg__bin__windows__contrib__4.2__arrow_{no_rc_r_version}\.zip + - r-pkg__bin__macosx__contrib__4.1__arrow_{no_rc_r_version}\.tgz + - r-pkg__bin__macosx__contrib__4.2__arrow_{no_rc_r_version}\.tgz + - r-pkg__bin__macosx__big-sur-arm64__contrib__4.1__arrow_{no_rc_r_version}\.tgz + - r-pkg__bin__macosx__big-sur-arm64__contrib__4.2__arrow_{no_rc_r_version}\.tgz + - r-pkg__src__contrib__arrow_{no_rc_r_version}\.tar\.gz ########################### Release verification ############################ diff --git a/r/tools/nixlibs.R b/r/tools/nixlibs.R index f066e4494d3..817563b85cc 100644 --- a/r/tools/nixlibs.R +++ b/r/tools/nixlibs.R @@ -32,7 +32,7 @@ dev_version <- package_version(VERSION)[1, 4] # Small dev versions are added for R-only changes during CRAN submission. if (is.na(dev_version) || dev_version < 100) { VERSION <- package_version(VERSION)[1, 1:3] - arrow_repo <- sprintf("https://apache.jfrog.io/artifactory/arrow/r/%s/libarrow/", VERSION) + arrow_repo <- paste0(getOption("arrow.repo", sprintf("https://apache.jfrog.io/artifactory/arrow/r/%s", VERSION)), "/libarrow/") } else { arrow_repo <- paste0(getOption("arrow.dev_repo", "https://nightlies.apache.org/arrow/r"), "/libarrow/") } diff --git a/r/tools/winlibs.R b/r/tools/winlibs.R index 165c98da5ea..d941da4baa6 100644 --- a/r/tools/winlibs.R +++ b/r/tools/winlibs.R @@ -44,7 +44,10 @@ if (!file.exists(sprintf("windows/arrow-%s/include/arrow/api.h", VERSION))) { "/libarrow/bin/windows/arrow-%s.zip" ) # %1$s uses the first variable for both substitutions - artifactory <- "https://apache.jfrog.io/artifactory/arrow/r/%1$s/libarrow/bin/windows/arrow-%1$s.zip" + artifactory <- paste0( + getOption("arrow.repo", "https://apache.jfrog.io/artifactory/arrow/r/%1$s"), + "/libarrow/bin/windows/arrow-%1$s.zip" + ) rwinlib <- "https://github.com/rwinlib/arrow/archive/v%s.zip" dev_version <- package_version(VERSION)[1, 4]