Skip to content

Commit a730a90

Browse files
JimSuplizioJimSuplizio
authored
Update whitelist includes (#9912)
* Update whitelist includes * Lock whitelist entries to a specific version * Rebase to get checkstyle update and fix a grammar issue * rebase to get pom restructure changes
1 parent e49f16c commit a730a90

File tree

18 files changed

+197
-80
lines changed

18 files changed

+197
-80
lines changed

‎common/perf-test-core/pom.xml‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -66,11 +66,10 @@
6666
</excludes>
6767
<includes>
6868
<include>com.azure:*</include>
69-
<include>org.slf4j</include>
70-
<include>com.fasterxml.jackson.*</include>
69+
<include>com.fasterxml.jackson.core:jackson-databind:[2.10.1]</include> <!-- {x-include-update;com.fasterxml.jackson.core:jackson-databind;external_dependency} -->
7170

7271
<!-- special allowance for perf-test-core as it is not a shipping library: -->
73-
<include>com.beust:jcommander</include>
72+
<include>com.beust:jcommander:[1.58]</include> <!-- {x-include-update;com.beust:jcommander;external_dependency} -->
7473
</includes>
7574
</bannedDependencies>
7675
</rules>

‎eng/versioning/pom_file_version_scanner.ps1‎

Lines changed: 83 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,11 @@ $DependencyTypeForError = "$($DependencyTypeCurrent)|$($DependencyTypeDependency
4444
$UpdateTagFormat = "{x-version-update;<groupId>:<artifactId>;$($DependencyTypeForError)}"
4545
$StartTime = $(get-date)
4646

47+
# This is the for the bannedDependencies include exceptions. All <include> entries need to be of the
48+
# form <include>groupId:artifactId:[version]</include> which locks to a specific version. The exception
49+
# to this is the blanket, wildcard include for com.azure libraries.
50+
$ComAzureWhitelistInclude = "com.azure:*"
51+
4752
function Write-Error-With-Color([string]$msg)
4853
{
4954
Write-Host "$($msg)" -ForegroundColor Red
@@ -492,7 +497,84 @@ Get-ChildItem -Path $Path -Filter pom*.xml -Recurse -File | ForEach-Object {
492497
$script:FoundError = $true
493498
Write-Error-With-Color "Error: Missing plugin version update tag for groupId=$($groupId), artifactId=$($artifactId). The tag should be <!-- {x-version-update;$($groupId):$($artifactId);current|dependency|external_dependency<select one>} -->"
494499
}
495-
}
500+
}
501+
502+
# This is for the whitelist dependencies. Fetch the banned dependencies
503+
foreach($bannedDependencies in $xmlPomFile.GetElementsByTagName("bannedDependencies"))
504+
{
505+
# Include nodes will look like the following:
506+
# <include>groupId:artifactId:[version]</include> <!-- {x-include-update;groupId:artifactId;external_dependency} -->
507+
foreach($includeNode in $bannedDependencies.GetElementsByTagName("include"))
508+
{
509+
$rawIncludeText = $includeNode.InnerText.Trim()
510+
$split = $rawIncludeText.Split(":")
511+
if ($split.Count -eq 3)
512+
{
513+
$groupId = $split[0]
514+
$artifactId = $split[1]
515+
$version = $split[2]
516+
# The groupId match has to be able to deal with <area>_ for external dependency exceptions
517+
if (!$includeNode.NextSibling -or $includeNode.NextSibling.NodeType -ne "Comment")
518+
{
519+
$script:FoundError = $true
520+
Write-Error-With-Color "Error: <include> is missing the update tag which should be <!-- {x-include-update;$($groupId):$($artifactId);external_dependency} -->"
521+
}
522+
elseif ($includeNode.NextSibling.Value.Trim() -notmatch "{x-include-update;(\w+)?$($groupId):$($artifactId);external_dependency}")
523+
{
524+
$script:FoundError = $true
525+
Write-Error-With-Color "Error: <include> version update tag for $($includeNode.InnerText) should be <!-- {x-include-update;$($groupId):$($artifactId);external_dependency} -->"
526+
}
527+
else
528+
{
529+
# verify that the version is formatted correctly
530+
if (!$version.StartsWith("[") -or !$version.EndsWith("]"))
531+
{
532+
$script:FoundError = $true
533+
Write-Error-With-Color "Error: the version entry '$($version)' for <include> '$($rawIncludeText)' is not formatted correctly. The include version needs to of the form '[<version>]', the braces lock the include to a specific version for these entries. -->"
534+
}
535+
# verify the version has the correct value
536+
else
537+
{
538+
$versionWithoutBraces = $version.Substring(1, $version.Length -2)
539+
# the key into the dependency has needs to be created from the tag's group/artifact
540+
# entries in case it's an external dependency entry. Because this has already
541+
# been validated for format, grab the group:artifact
542+
$depKey = $includeNode.NextSibling.Value.Trim().Split(";")[1]
543+
if ($extDepHash.ContainsKey($depKey))
544+
{
545+
if ($versionWithoutBraces -ne $extDepHash[$depKey].ver)
546+
{
547+
$script:FoundError = $true
548+
Write-Error-With-Color "Error: $($depKey)'s version is '$($versionWithoutBraces)' but the external_dependency version is listed as $($extDepHash[$depKey].ver)"
549+
}
550+
}
551+
else
552+
{
553+
$script:FoundError = $true
554+
Write-Error-With-Color "Error: the groupId:artifactId entry '$($depKey)' for <include> '$($rawIncludeText)' is not a valid external dependency. Please verify the entry exists in the external_dependencies.txt file. -->"
555+
}
556+
}
557+
}
558+
}
559+
# The only time a split count of 2 is allowed is in the following case.
560+
# <include>com.azure:*</include>
561+
# These entries will not and should not have an update tag
562+
elseif ($split.Count -eq 2)
563+
{
564+
if ($rawIncludeText -ne $ComAzureWhitelistInclude)
565+
{
566+
$script:FoundError = $true
567+
Write-Error-With-Color "Error: $($rawIncludeText) is not a valid <include> entry. With the exception of the $($ComAzureWhitelistInclude), every <include> entry must be of the form <include>groupId:artifactId:[version]<include>"
568+
}
569+
}
570+
else
571+
{
572+
# At this point the include entry is wildly incorrect.
573+
$script:FoundError = $true
574+
Write-Error-With-Color "Error: $($rawIncludeText) is not a valid <include> entry. Every <include> entry must be of the form <include>groupId:artifactId:[version]<include>"
575+
}
576+
}
577+
}
496578
}
497579
$ElapsedTime = $(get-date) - $StartTime
498580
$TotalRunTime = "{0:HH:mm:ss}" -f ([datetime]$ElapsedTime.Ticks)

‎eng/versioning/update_versions.py‎

Lines changed: 23 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -40,8 +40,10 @@
4040
from utils import BuildType
4141
from utils import CodeModule
4242
from utils import external_dependency_version_regex
43+
from utils import external_dependency_include_regex
4344
from utils import run_check_call
4445
from utils import UpdateType
46+
from utils import include_update_marker
4547
from utils import version_regex_str_no_anchor
4648
from utils import version_update_start_marker
4749
from utils import version_update_end_marker
@@ -51,16 +53,22 @@
5153
def update_versions(update_type, version_map, ext_dep_map, target_file, skip_readme, auto_version_increment):
5254

5355
newlines = []
54-
repl_open, repl_thisline, file_changed = False, False, False
56+
repl_open, repl_thisline, file_changed, is_include = False, False, False, False
5557
print('processing: ' + target_file)
5658
try:
5759
with open(target_file, encoding='utf-8') as f:
5860
for line in f:
61+
is_include = False
5962
repl_thisline = repl_open
6063
match = version_update_marker.search(line)
6164
if match and not target_file.endswith('.md'):
6265
module_name, version_type = match.group(1), match.group(2)
6366
repl_thisline = True
67+
elif include_update_marker.search(line):
68+
match = include_update_marker.search(line)
69+
module_name, version_type = match.group(1), match.group(2)
70+
repl_thisline = True
71+
is_include = True
6472
else:
6573
match = version_update_start_marker.search(line)
6674
if match:
@@ -100,12 +108,20 @@ def update_versions(update_type, version_map, ext_dep_map, target_file, skip_rea
100108
if update_type == UpdateType.library:
101109
newlines.append(line)
102110
continue
103-
try:
104-
module = ext_dep_map[module_name]
105-
new_version = module.external_dependency
106-
newline = re.sub(external_dependency_version_regex, new_version, line)
107-
except AttributeError:
108-
raise ValueError('Module: {0} does not have an external dependency version.\nFile={1}\nLine={2}'.format(module_name, target_file, line))
111+
if is_include:
112+
try:
113+
module = ext_dep_map[module_name]
114+
new_include_version = module.string_for_whitelist_include()
115+
newline = re.sub(external_dependency_include_regex, new_include_version, line)
116+
except AttributeError:
117+
raise ValueError('Module: {0} does not have an external dependency version.\nFile={1}\nLine={2}'.format(module_name, target_file, line))
118+
else:
119+
try:
120+
module = ext_dep_map[module_name]
121+
new_version = module.external_dependency
122+
newline = re.sub(external_dependency_version_regex, new_version, line)
123+
except AttributeError:
124+
raise ValueError('Module: {0} does not have an external dependency version.\nFile={1}\nLine={2}'.format(module_name, target_file, line))
109125
else:
110126
raise ValueError('Invalid version type: {} for module: {}.\nFile={}\nLine={}'.format(version_type, module_name, target_file, line))
111127

‎eng/versioning/utils.py‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
import re
99
from subprocess import check_call, CalledProcessError
1010

11+
include_update_marker = re.compile(r'\{x-include-update;([^;]+);([^}]+)\}')
1112
version_update_start_marker = re.compile(r'\{x-version-update-start;([^;]+);([^}]+)\}')
1213
version_update_end_marker = re.compile(r'\{x-version-update-end\}')
1314
version_update_marker = re.compile(r'\{x-version-update;([^;]+);([^}]+)\}')
@@ -19,6 +20,10 @@
1920
# https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string
2021
version_regex_str_no_anchor = r'(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?'
2122

23+
# External dependency versions do not have to match semver format and the semver regular expressions
24+
# will partially match and produce some hilarious results.
25+
external_dependency_include_regex = r'(?<=<include>).+?(?=</include>)'
26+
2227
# External dependency versions do not have to match semver format and the semver regular expressions
2328
# will partially match and produce some hilarious results.
2429
external_dependency_version_regex = r'(?<=<version>).+?(?=</version>)'
@@ -99,6 +104,24 @@ def string_for_version_file(self):
99104
except AttributeError:
100105
return self.name + ';' + self.dependency + '\n'
101106

107+
# return the CodeModule string formatted for a whitelist include entry
108+
# note: for whitelist includes the version needs to be braces in order for
109+
# the version to be an explicit version. Without the braces a version
110+
# would be treated as that version and above. For example:
111+
# <groupId>:<artifactId>:1.2 would be treated as 1.2 and above or equivalent to [1.2,)
112+
def string_for_whitelist_include(self):
113+
if hasattr(self, 'external_dependency'):
114+
temp = self.name
115+
# This is necessary to deal with the fact that external_dependencies can have
116+
# '_' in them if they're an external dependency exception. Since the whitelist
117+
# name needs to be the actual dependency, take everything after the _ which is
118+
# the actual name
119+
if '_' in temp:
120+
temp = temp.split('_')[1]
121+
return temp + ':[' + self.external_dependency + ']'
122+
else:
123+
raise ValueError('string_for_whitelist_include called on non-external_dependency: ' + self.name)
124+
102125
def run_check_call(
103126
command_array,
104127
working_directory,

‎sdk/core/azure-core-amqp/pom.xml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -128,8 +128,8 @@
128128
</excludes>
129129
<includes>
130130
<include>com.azure:*</include>
131-
<include>org.apache.qpid:proton-j</include>
132-
<include>com.microsoft.azure:qpid-proton-j-extensions</include>
131+
<include>com.microsoft.azure:qpid-proton-j-extensions:[1.2.2]</include> <!-- {x-include-update;com.microsoft.azure:qpid-proton-j-extensions;external_dependency} -->
132+
<include>org.apache.qpid:proton-j:[0.33.2]</include> <!-- {x-include-update;org.apache.qpid:proton-j;external_dependency} -->
133133
</includes>
134134
</bannedDependencies>
135135
</rules>

‎sdk/core/azure-core-http-netty/pom.xml‎

Lines changed: 8 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -193,16 +193,14 @@
193193
</excludes>
194194
<includes>
195195
<include>com.azure:*</include>
196-
197-
<include>io.projectreactor.netty</include>
198-
199-
<include>io.netty:netty-buffer</include>
200-
<include>io.netty:netty-codec-http</include>
201-
<include>io.netty:netty-codec-http2</include>
202-
<include>io.netty:netty-handler</include>
203-
<include>io.netty:netty-handler-proxy</include>
204-
<include>io.netty:netty-transport-native-unix-common</include>
205-
<include>io.netty:netty-transport-native-epoll</include>
196+
<include>io.netty:netty-buffer:[4.1.45.Final]</include> <!-- {x-include-update;io.netty:netty-buffer;external_dependency} -->
197+
<include>io.netty:netty-codec-http:[4.1.45.Final]</include> <!-- {x-include-update;io.netty:netty-codec-http;external_dependency} -->
198+
<include>io.netty:netty-codec-http2:[4.1.45.Final]</include> <!-- {x-include-update;io.netty:netty-codec-http2;external_dependency} -->
199+
<include>io.netty:netty-handler:[4.1.45.Final]</include> <!-- {x-include-update;io.netty:netty-handler;external_dependency} -->
200+
<include>io.netty:netty-handler-proxy:[4.1.45.Final]</include> <!-- {x-include-update;io.netty:netty-handler-proxy;external_dependency} -->
201+
<include>io.netty:netty-transport-native-unix-common:[4.1.45.Final]</include> <!-- {x-include-update;io.netty:netty-transport-native-unix-common;external_dependency} -->
202+
<include>io.netty:netty-transport-native-epoll:[4.1.45.Final]</include> <!-- {x-include-update;io.netty:netty-transport-native-epoll;external_dependency} -->
203+
<include>io.projectreactor.netty:reactor-netty:[0.9.5.RELEASE]</include> <!-- {x-include-update;io.projectreactor.netty:reactor-netty;external_dependency} -->
206204
</includes>
207205
</bannedDependencies>
208206
</rules>

‎sdk/core/azure-core-http-okhttp/pom.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -155,7 +155,7 @@
155155
</excludes>
156156
<includes>
157157
<include>com.azure:*</include>
158-
<include>com.squareup.okhttp3:okhttp</include>
158+
<include>com.squareup.okhttp3:okhttp:[4.2.2]</include> <!-- {x-include-update;com.squareup.okhttp3:okhttp;external_dependency} -->
159159
</includes>
160160
</bannedDependencies>
161161
</rules>

‎sdk/core/azure-core-test/pom.xml‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -124,11 +124,9 @@
124124
</excludes>
125125
<includes>
126126
<include>com.azure:*</include>
127-
<include>org.slf4j</include>
128-
<include>io.projectreactor</include>
129-
127+
<include>io.projectreactor:reactor-test:[3.3.3.RELEASE]</include> <!-- {x-include-update;io.projectreactor:reactor-test;external_dependency} -->
130128
<!-- special allowance for azure-core-test as it is not a shipping library: -->
131-
<include>org.junit.jupiter</include>
129+
<include>org.junit.jupiter:junit-jupiter-api:[5.4.2]</include> <!-- {x-include-update;org.junit.jupiter:junit-jupiter-api;external_dependency} -->
132130
</includes>
133131
</bannedDependencies>
134132
</rules>

‎sdk/core/azure-core-tracing-opentelemetry/pom.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,7 @@
9595
</excludes>
9696
<includes>
9797
<include>com.azure:*</include>
98-
<include>io.opentelemetry</include>
98+
<include>io.opentelemetry:opentelemetry-api:[0.2.4]</include> <!-- {x-include-update;io.opentelemetry:opentelemetry-api;external_dependency} -->
9999
</includes>
100100
</bannedDependencies>
101101
</rules>

‎sdk/core/azure-core/pom.xml‎

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -186,14 +186,12 @@
186186
<exclude>*:*:*:*:provided</exclude>
187187
</excludes>
188188
<includes>
189-
<include>org.slf4j:slf4j-api</include>
190-
<include>io.projectreactor:reactor-core</include>
191-
<include>io.netty:netty-tcnative-boringssl-static</include>
192-
193-
<include>com.fasterxml.jackson.dataformat:jackson-dataformat-xml</include>
194-
<include>com.fasterxml.jackson.datatype:jackson-datatype-jsr310</include>
195-
196-
<include>com.google.code.findbugs:jsr305</include>
189+
<include>io.netty:netty-tcnative-boringssl-static:[2.0.27.Final]</include> <!-- {x-include-update;io.netty:netty-tcnative-boringssl-static;external_dependency} -->
190+
<include>io.projectreactor:reactor-core:[3.3.3.RELEASE]</include> <!-- {x-include-update;io.projectreactor:reactor-core;external_dependency} -->
191+
<include>com.fasterxml.jackson.dataformat:jackson-dataformat-xml:[2.10.1]</include> <!-- {x-include-update;com.fasterxml.jackson.dataformat:jackson-dataformat-xml;external_dependency} -->
192+
<include>com.fasterxml.jackson.datatype:jackson-datatype-jsr310:[2.10.1]</include> <!-- {x-include-update;com.fasterxml.jackson.datatype:jackson-datatype-jsr310;external_dependency} -->
193+
<include>com.google.code.findbugs:jsr305:[3.0.2]</include> <!-- {x-include-update;com.google.code.findbugs:jsr305;external_dependency} -->
194+
<include>org.slf4j:slf4j-api:[1.7.28]</include> <!-- {x-include-update;org.slf4j:slf4j-api;external_dependency} -->
197195
</includes>
198196
</bannedDependencies>
199197
</rules>

0 commit comments

Comments
 (0)