diff --git a/buildbot/osuosl/master/config/builders.py b/buildbot/osuosl/master/config/builders.py index 1233a51b6..06d5ad11e 100644 --- a/buildbot/osuosl/master/config/builders.py +++ b/buildbot/osuosl/master/config/builders.py @@ -2691,6 +2691,16 @@ def collapseRequestsDoxygen(master, builder, req1, req2): # We have to have a long timeout here. timeout=172800)}, + {'name' : "publish-runtimes-doxygen-docs", + 'tags' : ["doc"], + 'workernames' : ["as-worker-4"], + 'builddir': "publish-runtimes-doxygen-docs", + 'factory' : DoxygenDocsBuilder.getLLVMRuntimesDocsBuildFactory( + # Doxygen builds the final result for really + # long time without any output. + # We have to have a long timeout here. + timeout=172800)}, + # CUDA builders. {'name' : "clang-cuda-l4", diff --git a/zorg/buildbot/builders/DoxygenDocsBuilder.py b/zorg/buildbot/builders/DoxygenDocsBuilder.py index 958eaf6a3..27f8c62d2 100644 --- a/zorg/buildbot/builders/DoxygenDocsBuilder.py +++ b/zorg/buildbot/builders/DoxygenDocsBuilder.py @@ -13,14 +13,14 @@ reload(CmakeCommand) llvm_docs = OrderedDict([ - # Project Build target Local path Remote path - ("llvm", ("doxygen", "docs/doxygen/html/", "llvm")), - ("clang", (None, "tools/clang/docs/doxygen/html/", "cfe")), - ("clang-tools-extra", (None, "tools/clang/tools/extra/docs/doxygen/html/", "cfe-extra")), - ("flang", (None, "tools/flang/docs/doxygen/html/", "flang")), - ("polly", (None, "tools/polly/docs/doxygen/html/", "polly")), - ("openmp", (None, "projects/openmp/docs/doxygen/html/", "openmp")), - ("lldb", ("lldb-cpp-doc", "tools/lldb/docs/cpp_reference/", "lldb/cpp_reference")), + # Project Build target Local path Remote path + ("llvm", ("doxygen", "docs/doxygen/html/", "llvm")), + ("clang", (None, "tools/clang/docs/doxygen/html/", "cfe")), + ("clang-tools-extra", (None, "tools/clang/tools/extra/docs/doxygen/html/", "cfe-extra")), + ("flang", (None, "tools/flang/docs/doxygen/html/", "flang")), + ("polly", (None, "tools/polly/docs/doxygen/html/", "polly")), + ("openmp", ("doxygen-openmp", "openmp/docs/doxygen/html/", "openmp")), + ("lldb", ("lldb-cpp-doc", "tools/lldb/docs/cpp_reference/", "lldb/cpp_reference")), # NOTE: 5/9/2020 lldb-python-doc fails to build. Disabled till be fixed. #(None, ("lldb-python-doc", "tools/lldb/docs/python_reference/", "lldb")), ]) @@ -34,9 +34,15 @@ def getLLVMDocsBuildFactory( **kwargs): if depends_on_projects is None: - # All the projects from llvm_docs, and remove all duplicates. - _depends_on_projects=list(set( - [project for project in llvm_docs if project])) + # All the projects by default. + _depends_on_projects=[ + "llvm", + "clang", + "clang-tools-extra", + "lldb", + "flang", + "polly", + ] else: # Make a local copy of depends_on_projects, as we are going to modify # that. @@ -69,6 +75,12 @@ def getLLVMDocsBuildFactory( ("-DCMAKE_BUILD_TYPE=", "Release"), ]) + # Build only docs for each of the projects this builder depends on + docs = { + project: info for project,info in llvm_docs.items() + if project in _depends_on_projects + } + f = UnifiedTreeBuilder.getCmakeBuildFactory( clean=clean, depends_on_projects=_depends_on_projects, @@ -78,7 +90,7 @@ def getLLVMDocsBuildFactory( **kwargs) # Pass through all the extra arguments. # Build the documentation for all the projects. - for project in llvm_docs: + for project in docs: target = llvm_docs[project][0] # Build only those with specifies targets. @@ -95,7 +107,7 @@ def getLLVMDocsBuildFactory( **kwargs) # Publish just built documentation - for project in llvm_docs: + for project in docs: target, local_path, remote_path = llvm_docs[project] f.addStep( @@ -120,3 +132,106 @@ def getLLVMDocsBuildFactory( ) return f + +def getLLVMRuntimesDocsBuildFactory( + clean = True, + depends_on_runtimes = None, + extra_configure_args = None, + timeout=10800, + env = None, + **kwargs): + + if depends_on_runtimes is None: + # All the projects by default. + _depends_on_runtimes=[ + "openmp" + ] + else: + # Make a local copy of depends_on_runtimes, dependencies may require + # adding additional runtimes. + _depends_on_runtimes=depends_on_runtimes[:] + + # Make a local copy of the configure args, as we are going to modify that. + if extra_configure_args: + cmake_args = extra_configure_args[:] + else: + cmake_args = list() + + # Prepare environmental variables. Set here all env we want everywhere. + merged_env = { + 'TERM' : 'dumb' # Be cautious and disable color output from all tools. + } + if env is not None: + # Overwrite pre-set items with the given ones, so user can set anything. + merged_env.update(env) + + CmakeCommand.CmakeCommand.applyDefaultOptions(cmake_args, [ + ("-G", "Ninja"), + ("-DLLVM_ENABLE_DOXYGEN=", "ON"), + ("-DLLVM_BUILD_DOCS=", "ON"), + ("-DCMAKE_BUILD_TYPE=", "Release"), + ]) + + # Build docs for each of the runtimes this builder depends on + docs = { + project: info for project,info in llvm_docs.items() + if project in _depends_on_runtimes + } + + cleanBuildRequested = lambda step: step.build.getProperty("clean") or step.build.getProperty("clean_obj") or clean + + f = UnifiedTreeBuilder.getLLVMBuildFactoryAndSourcecodeSteps( + depends_on_projects=_depends_on_runtimes, + enable_runtimes=_depends_on_runtimes, + src_to_build_dir='runtimes', + cleanBuildRequested=cleanBuildRequested, + **kwargs) # Pass through all the extra arguments. + + UnifiedTreeBuilder.addCmakeSteps( + f, + cleanBuildRequested=cleanBuildRequested, + obj_dir=f.obj_dir, + install_dir=f.install_dir, + extra_configure_args=cmake_args, + env=merged_env, + **kwargs) + + # Build the documentation for all the runtimes. + for target, local_path, remote_path in docs.values(): + # Build only those with specifies targets. + if target: + UnifiedTreeBuilder.addNinjaSteps( + f, + # Doxygen builds the final result for really + # long time without any output. + # We have to have a long timeout at this step. + timeout=timeout, + targets=[target], + checks=[], + env=merged_env, + **kwargs) + + # Publish just built documentation + for project, (target, local_path, remote_path) in docs.items(): + f.addStep( + ShellCommand( + name="Publish {}".format(project or target), + description=[ + "Publish", "just", "built", "documentation", "for", + "{}".format(project or target) + ], + command=[ + 'rsync', + '-vrl', + '--delete', '--force', '--delay-updates', '--delete-delay', + '--ignore-times', + '--checksum', + '-p', '--chmod=Du=rwx,Dg=rwx,Do=rx,Fu=rw,Fg=rw,Fo=r', + "{}".format(local_path), + "lists.llvm.org:web/doxygen/{}".format(remote_path), + ], + env=merged_env, + ) + ) + + return f