Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix python::pip installing $editable VCS packages every Puppet run #491

Merged
merged 3 commits into from
Jun 10, 2019
Merged

Fix python::pip installing $editable VCS packages every Puppet run #491

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
19f3f3f
Fix python::pip installing $editable VCS packages every puppet run
mlow Jun 9, 2019
8372013
Acceptance test: works with editable=>true
mlow Jun 9, 2019
96770aa
Acceptance test: works with == in pkgname
mlow Jun 9, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
130 changes: 49 additions & 81 deletions manifests/pip.pp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -135,14 +135,23 @@
fail('python::pip cannot provide uninstall_args with ensure => present')
}

if $pkgname =~ /==/ {
$parts = split($pkgname, '==')
$real_pkgname = $parts[0]
mlow marked this conversation as resolved.
Show resolved Hide resolved
$_ensure = $ensure ? {
'absent' => 'absent',
default => $parts[1],
}
} else {
$real_pkgname = $pkgname
$_ensure = $ensure
}

# Check if searching by explicit version.
if $ensure =~ /^((19|20)[0-9][0-9]-(0[1-9]|1[1-2])-([0-2][1-9]|3[0-1])|[0-9]+\.\w+\+?\w*(\.\w+)*)$/ {
$grep_regex = "^${pkgname}==${ensure}\$"
if $_ensure =~ /^((19|20)[0-9][0-9]-(0[1-9]|1[1-2])-([0-2][1-9]|3[0-1])|[0-9]+\.\w+\+?\w*(\.\w+)*)$/ {
$grep_regex = "^${real_pkgname}[[:space:]]\\+(\\?${_ensure}\\()$\\|$\\|, \\|[[:space:]]\\)"
} else {
$grep_regex = $pkgname ? {
/==/ => "^${pkgname}\$",
default => "^${pkgname}==",
}
$grep_regex = "^${real_pkgname}[[:space:]].*$"
}

$extras_string = empty($extras) ? {
Expand All @@ -151,12 +160,12 @@
}

$egg_name = $egg ? {
false => "${pkgname}${extras_string}",
false => "${real_pkgname}${extras_string}",
default => $egg
}

$source = $url ? {
false => "${pkgname}${extras_string}",
false => "${real_pkgname}${extras_string}",
/^(\/|[a-zA-Z]\:)/ => "'${url}'",
/^(git\+|hg\+|bzr\+|svn\+)(http|https|ssh|svn|sftp|ftp|lp|git)(:\/\/).+$/ => "'${url}'",
default => "'${url}#egg=${egg_name}'",
Expand All @@ -165,111 +174,70 @@
$pip_install = "${pip_env} --log ${log}/pip.log install"
$pip_common_args = "${pypi_index} ${proxy_flag} ${install_args} ${install_editable} ${source}"

$pip_exec_name = "pip_install_${name}"

# Explicit version out of VCS when PIP supported URL is provided
if $source =~ /^'(git\+|hg\+|bzr\+|svn\+)(http|https|ssh|svn|sftp|ftp|lp|git)(:\/\/).+'$/ {
if $ensure != present and $ensure != latest {
exec { "pip_install_${name}":
command => "${pip_install} ${install_args} ${pip_common_args}@${ensure}#egg=${egg_name}",
unless => "${pip_env} freeze --all | grep -i -e ${grep_regex}",
user => $owner,
group => $group,
umask => $umask,
cwd => $cwd,
environment => $environment,
timeout => $timeout,
path => $_path,
}
if $_ensure != present and $_ensure != latest {
$command = "${pip_install} ${install_args} ${pip_common_args}@${_ensure}#egg=${egg_name}"
$unless_command = "${pip_env} list | grep -i -e '${grep_regex}'"
} else {
exec { "pip_install_${name}":
command => "${pip_install} ${install_args} ${pip_common_args}",
unless => "${pip_env} freeze --all | grep -i -e ${grep_regex}",
user => $owner,
group => $group,
umask => $umask,
cwd => $cwd,
environment => $environment,
timeout => $timeout,
path => $_path,
}
$command = "${pip_install} ${install_args} ${pip_common_args}"
$unless_command = "${pip_env} list | grep -i -e '${grep_regex}'"
}
} else {
case $ensure {
case $_ensure {
/^((19|20)[0-9][0-9]-(0[1-9]|1[1-2])-([0-2][1-9]|3[0-1])|[0-9]+\.\w+\+?\w*(\.\w+)*)$/: {
# Version formats as per http://guide.python-distribute.org/specification.html#standard-versioning-schemes
# Explicit version.
exec { "pip_install_${name}":
command => "${pip_install} ${install_args} ${pip_common_args}==${ensure}",
unless => "${pip_env} freeze --all | grep -i -e ${grep_regex} || ${pip_env} list | sed -e 's/[ ]\\+/==/' -e 's/[()]//g' | grep -i -e ${grep_regex}",
user => $owner,
group => $group,
umask => $umask,
cwd => $cwd,
environment => $environment,
timeout => $timeout,
path => $_path,
}
$command = "${pip_install} ${install_args} ${pip_common_args}==${_ensure}"
$unless_command = "${pip_env} list | grep -i -e '${grep_regex}'"
}
#

'present': {
# Whatever version is available.
exec { "pip_install_${name}":
command => "${pip_install} ${pip_common_args}",
unless => "${pip_env} freeze --all | grep -i -e ${grep_regex} || ${pip_env} list | sed -e 's/[ ]\\+/==/' -e 's/[()]//g' | grep -i -e ${grep_regex}",
user => $owner,
group => $group,
umask => $umask,
cwd => $cwd,
environment => $environment,
timeout => $timeout,
path => $_path,
}
$command = "${pip_install} ${pip_common_args}"
$unless_command = "${pip_env} list | grep -i -e '${grep_regex}'"
}

'latest': {
# Unfortunately this is the smartest way of getting the latest available package version with pip as of now
# Note: we DO need to repeat ourselves with "from version" in both grep and sed as on some systems pip returns
# more than one line with paretheses.
$latest_version = join(["${pip_install} ${pypi_index} ${proxy_flag} ${install_args} ${install_editable} ${pkgname}==notreallyaversion 2>&1",
$latest_version = join(["${pip_install} ${pypi_index} ${proxy_flag} ${install_args} ${install_editable} ${real_pkgname}==notreallyaversion 2>&1",
' | grep -oP "\(from versions: .*\)" | sed -E "s/\(from versions: (.*?, )*(.*)\)/\2/g"',
' | tr -d "[:space:]"'])

# Packages with underscores in their names are listed with dashes in their place in `pip freeze` output
$pkgname_with_dashes = regsubst($pkgname, '_', '-', 'G')
$pkgname_with_dashes = regsubst($real_pkgname, '_', '-', 'G')
$grep_regex_pkgname_with_dashes = "^${pkgname_with_dashes}=="
$installed_version = join(["${pip_env} freeze --all",
" | grep -i -e ${grep_regex_pkgname_with_dashes} | cut -d= -f3",
" | tr -d '[:space:]'"])

$command = "${pip_install} --upgrade ${pip_common_args}"
$unless_command = "[ \$(${latest_version}) = \$(${installed_version}) ]"

# Latest version.
exec { "pip_install_${name}":
command => "${pip_install} --upgrade ${pip_common_args}",
unless => $unless_command,
user => $owner,
group => $group,
umask => $umask,
cwd => $cwd,
environment => $environment,
timeout => $timeout,
path => $_path,
}
}

default: {
# Anti-action, uninstall.
exec { "pip_uninstall_${name}":
command => "echo y | ${pip_env} uninstall ${uninstall_args} ${proxy_flag} ${name}",
onlyif => "${pip_env} freeze --all | grep -i -e ${grep_regex}",
user => $owner,
group => $group,
umask => $umask,
cwd => $cwd,
environment => $environment,
timeout => $timeout,
path => $_path,
}
$pip_exec_name = "pip_uninstall_${name}"
$command = "echo y | ${pip_env} uninstall ${uninstall_args} ${proxy_flag} ${name}"
$unless_command = "! ${pip_env} list | grep -i -e '${grep_regex}'"
}
}
}

exec { $pip_exec_name:
command => $command,
unless => $unless_command,
user => $owner,
group => $group,
umask => $umask,
cwd => $cwd,
environment => $environment,
timeout => $timeout,
path => $_path,
}

}
81 changes: 65 additions & 16 deletions spec/acceptance/virtualenv_spec.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,16 +10,14 @@ class { 'python' :
pip => 'present',
virtualenv => 'present',
}
->
python::virtualenv { 'venv' :
-> python::virtualenv { 'venv' :
ensure => 'present',
systempkgs => false,
venv_dir => '/opt/venv',
owner => 'root',
group => 'root',
}
->
python::pip { 'rpyc' :
-> python::pip { 'rpyc' :
ensure => '3.2.3',
virtualenv => '/opt/venv',
}
Expand All @@ -29,23 +27,22 @@ class { 'python' :
apply_manifest(pp, catch_failures: true)
apply_manifest(pp, catch_changes: true)
end

it 'maintains pip version' do
pp = <<-EOS
class { 'python' :
version => 'system',
pip => 'present',
virtualenv => 'present',
}
->
python::virtualenv { 'venv' :
-> python::virtualenv { 'venv' :
ensure => 'present',
systempkgs => false,
venv_dir => '/opt/venv2',
owner => 'root',
group => 'root',
}
->
python::pip { 'pip' :
-> python::pip { 'pip' :
ensure => '18.0',
virtualenv => '/opt/venv2',
}
Expand All @@ -55,23 +52,22 @@ class { 'python' :
apply_manifest(pp, catch_failures: true)
apply_manifest(pp, catch_changes: true)
end

it 'works with ensure=>latest' do
pp = <<-EOS
class { 'python' :
version => 'system',
pip => 'present',
virtualenv => 'present',
}
->
python::virtualenv { 'venv' :
-> python::virtualenv { 'venv' :
ensure => 'present',
systempkgs => false,
venv_dir => '/opt/venv3',
owner => 'root',
group => 'root',
}
->
python::pip { 'rpyc' :
-> python::pip { 'rpyc' :
ensure => 'latest',
virtualenv => '/opt/venv3',
}
Expand All @@ -83,23 +79,22 @@ class { 'python' :
# but probability of this happening is minimal, so it should be acceptable.
apply_manifest(pp, catch_changes: true)
end

it 'works with ensure=>latest for package with underscore in its name' do
pp = <<-EOS
class { 'python' :
version => 'system',
pip => 'present',
virtualenv => 'present',
}
->
python::virtualenv { 'venv' :
-> python::virtualenv { 'venv' :
ensure => 'present',
systempkgs => false,
venv_dir => '/opt/venv4',
owner => 'root',
group => 'root',
}
->
python::pip { 'Randomized_Requests' :
-> python::pip { 'Randomized_Requests' :
ensure => 'latest',
virtualenv => '/opt/venv4',
}
Expand All @@ -111,5 +106,59 @@ class { 'python' :
# but probability of this happening is minimal, so it should be acceptable.
apply_manifest(pp, catch_changes: true)
end

it 'works with editable=>true' do
pp = <<-EOS
package{ 'git' :
ensure => 'present',
}
-> class { 'python' :
version => 'system',
pip => 'present',
virtualenv => 'present',
}
-> python::virtualenv { 'venv' :
ensure => 'present',
systempkgs => false,
venv_dir => '/opt/venv5',
owner => 'root',
group => 'root',
}
-> python::pip { 'rpyc' :
ensure => '4.1.0',
url => 'git+https://github.com/tomerfiliba/rpyc.git',
editable => true,
virtualenv => '/opt/venv5',
}
EOS

# Run it twice and test for idempotency
apply_manifest(pp, catch_failures: true)
apply_manifest(pp, catch_changes: true)
end

it 'works with == in pkgname' do
pp = <<-EOS
class { 'python' :
version => 'system',
pip => 'present',
virtualenv => 'present',
}
-> python::virtualenv { 'venv' :
ensure => 'present',
systempkgs => false,
venv_dir => '/opt/venv6',
owner => 'root',
group => 'root',
}
-> python::pip { 'rpyc==4.1.0' :
virtualenv => '/opt/venv6',
}
EOS

# Run it twice and test for idempotency
apply_manifest(pp, catch_failures: true)
apply_manifest(pp, catch_changes: true)
end
end
end