Fix/7.0 ga code generation (#3664)

* Add XPack download location
* ask to generate code again as well, allows you to only download the specs without regenerating all the code
* move patch files to explicit _Patches folder so they are easier to spot and delete when no longer needed
* Create patch files for deprecated paths
* Download spec files for 7.0
* skip ccr.forget_follower
* patch required on parts automatically and warn if the spec is wrong
* Don't download REST API specs by default

Author: Mpdreamz

src/CodeGeneration/ApiGenerator/ApiGenerator.cs

@@ -133,13 +133,27 @@ private static KeyValuePair<string, ApiEndpoint> CreateApiEndpoint(string jsonFi
 			var endpoint = officialJsonSpec.ToObject<Dictionary<string, ApiEndpoint>>().First();
 			endpoint.Value.RestSpecName = endpoint.Key;
 			endpoint.Value.CsharpMethodName = CreateMethodName(endpoint.Key);
+
+			PatchUrlParts(jsonFile, endpoint.Value.Url);
 			return endpoint;
 		}
 
+		private static void PatchUrlParts(string jsonFile, ApiUrl url)
+		{
+			if (url.IsPartless) return;
+			foreach (var kv in url.Parts)
+			{
+				var required = url.ExposedApiPaths.All(p => p.Path.Contains($"{{{kv.Key}}}"));
+				if (kv.Value.Required != required)
+					Warnings.Add($"{jsonFile} has part: {kv.Key} listed as {kv.Value.Required} but should be {required}");
+				kv.Value.Required = required;
+			}
+		}
+
 		private static void PatchOfficialSpec(JObject original, string jsonFile)
 		{
 			var directory = Path.GetDirectoryName(jsonFile);
-			var patchFile = Path.Combine(directory, Path.GetFileNameWithoutExtension(jsonFile)) + ".patch.json";
+			var patchFile = Path.Combine(directory,"..", "_Patches", Path.GetFileNameWithoutExtension(jsonFile)) + ".patch.json";
 			if (!File.Exists(patchFile)) return;
 
 			var patchedJson = JObject.Parse(File.ReadAllText(patchFile));

src/CodeGeneration/ApiGenerator/ApiGenerator.csproj

@@ -12,7 +12,7 @@
   </PropertyGroup>
   <ItemGroup>
     <PackageReference Include="Newtonsoft.Json" Version="12.0.1" />
-    <PackageReference Include="ShellProgressBar" Version="4.1.1" />
+    <PackageReference Include="ShellProgressBar" Version="4.2.0" />
     <PackageReference Include="CsQuery.Core" Version="2.0.1" />
     <!-- https://github.com/toddams/RazorLight/issues/172 -->
     <PackageReference Include="RazorLight.Unofficial" Version="2.0.0-beta1.1" />

src/CodeGeneration/ApiGenerator/CodeConfiguration.cs

@@ -56,6 +56,7 @@ public static class CodeConfiguration
 			"indices.unfreeze.json",
 
 			"ccr.follow_info.json",
+			"ccr.forget_follower.json"
 		};
 
 

src/CodeGeneration/ApiGenerator/Domain/ApiUrl.cs

@@ -20,7 +20,7 @@ public IEnumerable<string> Paths
 			set => _paths = (value ?? Enumerable.Empty<string>()).ToList();
 		}
 
-		public IDictionary<string, ApiUrlPart> Parts { private get; set; }
+		public IDictionary<string, ApiUrlPart> Parts { get; set; }
 
 		public IEnumerable<ApiPath> ExposedApiPaths
 		{

src/CodeGeneration/ApiGenerator/Program.cs

@@ -10,14 +10,14 @@ public static class Program
 		private static void Main(string[] args)
 		{
 			var redownloadCoreSpecification = false;
+			var generateCode = false;
 			var downloadBranch = DownloadBranch;
 
 			var answer = "invalid";
 			while (answer != "y" && answer != "n" && answer != "")
 			{
-				Console.Write("Download online rest specifications? [Y/N] (default N): ");
-				//answer = Console.ReadLine()?.Trim().ToLowerInvariant();
-				answer = "n";
+				Console.Write("Download online rest specifications? [y/N] (default N): ");
+				answer = Console.ReadLine()?.Trim().ToLowerInvariant();
 				redownloadCoreSpecification = answer == "y";
 			}
 
@@ -40,7 +40,15 @@ private static void Main(string[] args)
 			if (redownloadCoreSpecification)
 				RestSpecDownloader.Download(downloadBranch);
 
-			ApiGenerator.Generate(downloadBranch, "Core", "XPack");
+			answer = "invalid";
+			while (answer != "y" && answer != "n" && answer != "")
+			{
+				Console.Write("Generate code from the specification files on disk? [Y/n] (default Y): ");
+				answer = Console.ReadLine()?.Trim().ToLowerInvariant();
+				generateCode = answer == "y" || answer == "";
+			}
+			if (generateCode)
+				ApiGenerator.Generate(downloadBranch, "Core", "XPack");
 		}
 	}
 }

src/CodeGeneration/ApiGenerator/RestSpecDownloader.cs

@@ -15,6 +15,7 @@ public class RestSpecDownloader
 		private static readonly Dictionary<string, string> OnlineSpecifications = new Dictionary<string, string>
 		{
 			{ "Core", "https://github.com/elastic/elasticsearch/tree/{version}/rest-api-spec/src/main/resources/rest-api-spec/api" },
+			{ "XPack", "https://github.com/elastic/elasticsearch/tree/{version}/x-pack/plugin/src/test/resources/rest-api-spec/api"}
 		};
 
 		private static readonly ProgressBarOptions SubProgressBarOptions = new ProgressBarOptions
@@ -32,7 +33,6 @@ private RestSpecDownloader(string branch)
 					let url = kv.Value.Replace("{version}", branch)
 					select new Specification { FolderOnDisk = kv.Key, Branch = branch, GithubListingUrl = url }).ToList();
 
-
 			using (var pbar = new ProgressBar(specifications.Count, "Downloading specifications", MainProgressBarOptions))
 			{
 				foreach (var spec in specifications)

src/CodeGeneration/ApiGenerator/RestSpecification/Core/bulk.json

@@ -4,14 +4,7 @@
     "methods": ["POST", "PUT"],
     "url": {
       "path": "/_bulk",
-      "paths": ["/_bulk", "/{index}/_bulk"],
-      "deprecated_paths" : [
-        {
-          "version" : "7.0",
-          "path" : "/{index}/{type}/_bulk",
-          "description" : "Specifying types in urls has been deprecated"
-        }
-      ],
+      "paths": ["/_bulk", "/{index}/_bulk", "/{index}/{type}/_bulk"],
       "parts": {
         "index": {
           "type" : "string",

src/CodeGeneration/ApiGenerator/RestSpecification/Core/clear_scroll.json

@@ -3,15 +3,8 @@
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/search-request-scroll.html",
     "methods": ["DELETE"],
     "url": {
-      "path": "/_search/scroll",
-      "paths": [ "/_search/scroll"],
-      "deprecated_paths" : [
-        {
-          "version" : "7.0",
-          "path" : "/_search/scroll/{scroll_id}",
-          "description" : "A scroll id can be quite large and should be specified as part of the body"
-        }
-      ],
+      "path": "/_search/scroll/{scroll_id}",
+      "paths": ["/_search/scroll/{scroll_id}", "/_search/scroll"],
       "parts": {
         "scroll_id": {
           "type" : "list",

src/CodeGeneration/ApiGenerator/RestSpecification/Core/count.json

@@ -4,14 +4,7 @@
     "methods": ["POST", "GET"],
     "url": {
       "path": "/_count",
-      "paths": ["/_count", "/{index}/_count"],
-      "deprecated_paths" : [
-        {
-          "version" : "7.0",
-          "path" : "/{index}/{type}/_count",
-          "description" : "Specifying types in urls has been deprecated"
-        }
-      ],
+      "paths": ["/_count", "/{index}/_count", "/{index}/{type}/_count"],
       "parts": {
         "index": {
           "type" : "list",

src/CodeGeneration/ApiGenerator/RestSpecification/Core/create.json

@@ -4,14 +4,7 @@
     "methods": ["PUT","POST"],
     "url": {
       "path": "/{index}/_create/{id}",
-      "paths": ["/{index}/_create/{id}"],
-      "deprecated_paths" : [
-        {
-          "version" : "7.0",
-          "path" : "/{index}/{type}/{id}/_create",
-          "description" : "Specifying types in urls has been deprecated"
-        }
-      ],
+      "paths": ["/{index}/_create/{id}", "/{index}/{type}/{id}/_create"],
       "parts": {
         "id": {
           "type" : "string",