From 0da28c404c77c58347cdb10fcf98251bbb0c5345 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Holcroft Date: May 24 2021 17:10:08 +0000 Subject: [PATCH 1/11] allow single branch and detect default branch --- diff --git a/build-scripts/rebuild-site.py b/build-scripts/rebuild-site.py index 7755326..e69e516 100755 --- a/build-scripts/rebuild-site.py +++ b/build-scripts/rebuild-site.py @@ -127,18 +127,23 @@ def prepare_translated_sources(translated_sources, site_yml, languages, config): # Get a list of the original English repos repos = [] + default_branch = site_yml["content"].get("branches", ["master"]) + if type(default_branch) is str: + default_branch = [default_branch] + for repo_data in site_yml["content"]["sources"]: - if "branches" not in repo_data: - repo_data["branches"] = ["master"] - if "start_path" not in repo_data: - repo_data["start_path"] = "" + branches = repo_data.get("branches", default_branch) + if type(branches) is str: + branches = [branches] + + start_path = repo_data.get("start_path", "") - for branch in repo_data["branches"]: + for branch in branches: repo = {} repo["url"] = repo_data["url"] repo["branch"] = branch - repo["start_path"] = repo_data["start_path"] + repo["start_path"] = start_path repos.append(repo) From 54725f449dfaba11f10855044f5f1b50ca7680d7 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Holcroft Date: May 24 2021 17:19:40 +0000 Subject: [PATCH 2/11] run hardlink at the end as asked in https://pagure.io/fedora-infrastructure/issue/8964 --- diff --git a/build-scripts/rebuild-site.py b/build-scripts/rebuild-site.py index e69e516..10e0537 100755 --- a/build-scripts/rebuild-site.py +++ b/build-scripts/rebuild-site.py @@ -182,11 +182,11 @@ def prepare_translated_sources(translated_sources, site_yml, languages, config): original_module_dir = os.path.join(repo_dir, "modules", module) module_dir = os.path.join(en_sources, component, version, "modules", module) - subprocess.run(["cp", "-r", original_module_dir, module_dir]) + subprocess.run(["cp", "-a", original_module_dir, module_dir]) if module == "ROOT" or "nav" in antora_yml: # if this is the main antora.yml file - subprocess.run(["cp", "-r", antora_yml_file, os.path.join(en_sources, component, version, "antora.yml")]) + subprocess.run(["cp", "-a", antora_yml_file, os.path.join(en_sources, component, version, "antora.yml")]) log("----- copying antora.yml for {component} {module} {version}".format( component=component, module=module, version=version)) else: @@ -207,13 +207,13 @@ def prepare_translated_sources(translated_sources, site_yml, languages, config): lang_dir = os.path.join(translated_sources, language) for component in components: en_component_dir = os.path.join(en_sources, component) - subprocess.run(["cp", "-r", en_component_dir, lang_dir]) + subprocess.run(["cp", "-a", en_component_dir, lang_dir]) # And finally copy the original translated sources # into translated_sources for language in languages: src = os.path.join(translated_sources_original, language) - subprocess.run(["cp", "-r", src, translated_sources + "/"]) + subprocess.run(["cp", "-a", src, translated_sources + "/"]) return components @@ -304,7 +304,7 @@ def main(): target_dir_copied = "/antora/output/en-US.building/en-US" target_dir_final = "/antora/output/en-US" shutil.rmtree(target_dir_copying, ignore_errors=True) - subprocess.run(["cp", "-r", source_dir, target_dir_copying]) + subprocess.run(["cp", "-a", source_dir, target_dir_copying]) shutil.rmtree(target_dir_final, ignore_errors=True) shutil.move(target_dir_copied, target_dir_final) shutil.rmtree(target_dir_copying, ignore_errors=True) @@ -376,7 +376,7 @@ def main(): target_dir_copied = "/antora/output/{lang}.building/{lang}".format(lang=lang) target_dir_final = "/antora/output/{lang}".format(lang=lang) shutil.rmtree(target_dir_copying, ignore_errors=True) - subprocess.run(["cp", "-r", source_dir, target_dir_copying]) + subprocess.run(["cp", "-a", source_dir, target_dir_copying]) shutil.rmtree(target_dir_final, ignore_errors=True) shutil.move(target_dir_copied, target_dir_final) shutil.rmtree(target_dir_copying, ignore_errors=True) From 9af6dfc60f2b8fdb4d5a94fc5ee4cf246dac37fa Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Holcroft Date: May 24 2021 17:21:31 +0000 Subject: [PATCH 3/11] replace cp -r by cp -a & run hardlink at the end as asked in https://pagure.io/fedora-infrastructure/issue/8964 --- diff --git a/build-scripts/rebuild-site.py b/build-scripts/rebuild-site.py index 10e0537..99734aa 100755 --- a/build-scripts/rebuild-site.py +++ b/build-scripts/rebuild-site.py @@ -368,18 +368,48 @@ def main(): sys.exit(1) # Copying all the translated sites + log("") + log("=== Copying translated sites ===") + + results_dir = os.path.join(docs_repo, "public") + publish_dir = "/antora/output" + copying_dir = os.path.join(publish_dir, "copying.tmp") + + # I have: + # docs_repo/public <- results_dir (local partition) + # /antora/output/copying.tmp <- copying_dir (mounted partition) + # /antora/output <- publish_dir (mounted partition) + # + # I need to: + # 1/ copy from local to mounted + # 2/ remove old in mounted + # 3/ move new within mounted + # 4/ remove the copying dir + + # Make sure copying_dir exists and is empty + shutil.rmtree(copying_dir, ignore_errors=True) + subprocess.run(["mkdir", copying_dir]) + + # Copy results from local partition to a mounted partition + log("Copying between partitions...") + subprocess.run(["cp", "-a", results_dir, "{}/".format(copying_dir)]) + + # Swap the old tree for the new one for each language + log("Moving languages to the final place:") for lang in languages: - log("") - log("===== Copying the {lang} site =====".format(lang=lang)) - source_dir = os.path.join(docs_repo, "public") - target_dir_copying = "/antora/output/{lang}.building".format(lang=lang) - target_dir_copied = "/antora/output/{lang}.building/{lang}".format(lang=lang) - target_dir_final = "/antora/output/{lang}".format(lang=lang) - shutil.rmtree(target_dir_copying, ignore_errors=True) - subprocess.run(["cp", "-a", source_dir, target_dir_copying]) - shutil.rmtree(target_dir_final, ignore_errors=True) - shutil.move(target_dir_copied, target_dir_final) - shutil.rmtree(target_dir_copying, ignore_errors=True) + log(" - {}".format(lang)) + shutil.rmtree(os.path.join(publish_dir, lang), ignore_errors=True) + subprocess.run(["mv", os.path.join(copying_dir, "public", lang), "{}/".format(publish_dir)]) + + # https://pagure.io/fedora-infrastructure/issue/8964 + log("Consolidate files with hardlink...") + subprocess.run(["hardlink", "-v", "{}/".format(publish_dir)]) + + # And finally remove the copying dir + log("Removing the copying dir...") + shutil.rmtree(copying_dir, ignore_errors=True) + + log("DONE!") From 91dcafcbe4229bd9b1a93fd541b9c1917ce1a8db Mon Sep 17 00:00:00 2001 From: Adam Samalik Date: May 24 2021 17:44:11 +0000 Subject: [PATCH 4/11] add qa docs --- diff --git a/site.yml b/site.yml index 66f9e47..8c7050d 100644 --- a/site.yml +++ b/site.yml @@ -91,6 +91,7 @@ content: - url: https://pagure.io/cpe/rawhide-gating-docs.git - url: https://pagure.io/cpe/docs.git - url: https://pagure.io/fedora-docs/websites.git + - url: https://pagure.io/fedora-qa/qa-docs.git - url: https://pagure.io/fedora-pgm/pgm_docs.git branches: main - url: https://pagure.io/Ask-Fedora-SOP-docs.git From 437c329d044b3433adbd3c84fe60333f9be8aaff Mon Sep 17 00:00:00 2001 From: Justin W. Flory (he/him) Date: May 24 2021 17:44:11 +0000 Subject: [PATCH 5/11] Remove "Site" from site title I finally figured out where this info is stored. This is been a mild bother for some time. It makes the title shorter, and it doesn't appear in file names or other things if I take screenshots of a docs.fp.o page. Purely cosmetic, using this change as an excuse to make my first change to the production docs.fp.o builder. Signed-off-by: Justin W. Flory (he/him) --- diff --git a/site.yml b/site.yml index 8c7050d..d8a05c8 100644 --- a/site.yml +++ b/site.yml @@ -1,5 +1,5 @@ site: - title: Fedora Docs Site + title: Fedora Docs url: https://docs.fedoraproject.org/en-US/ start_page: docs::index.adoc content: From 36abbd76740e90282e06e8adf81f424a93d36e1e Mon Sep 17 00:00:00 2001 From: Petr Bokoc Date: May 24 2021 17:44:11 +0000 Subject: [PATCH 6/11] Add Fedora Gaming docs to Engineering --- diff --git a/site.yml b/site.yml index d8a05c8..c74ec73 100644 --- a/site.yml +++ b/site.yml @@ -98,6 +98,10 @@ content: - url: https://pagure.io/fedora-join/fedora-join-docs.git - url: https://pagure.io/fedora-docs/localization.git - url: https://pagure.io/fedora-l10n/docs.git + - url: https://github.com/fedora-eln/eln-docs.git + - url: https://pagure.io/i3-sig/docs.git + - url: https://github.com/fedora-infra/fedora-accounts-docs.git + - url: https://pagure.io/gaming/documentation.git - url: https://pagure.io/fedora-server.git branches: main start_path: docs From a5cc276c84805b186141f7160e9b1e2ce7a908fa Mon Sep 17 00:00:00 2001 From: Ben Cotton Date: May 24 2021 17:44:11 +0000 Subject: [PATCH 7/11] Gaming switched default branch to main Signed-off-by: Ben Cotton --- diff --git a/site.yml b/site.yml index c74ec73..4b03c63 100644 --- a/site.yml +++ b/site.yml @@ -102,6 +102,7 @@ content: - url: https://pagure.io/i3-sig/docs.git - url: https://github.com/fedora-infra/fedora-accounts-docs.git - url: https://pagure.io/gaming/documentation.git + branches: main - url: https://pagure.io/fedora-server.git branches: main start_path: docs From a6cc1469d960551a8644a650c1d6661b0070c2b4 Mon Sep 17 00:00:00 2001 From: Timothée Ravier Date: May 24 2021 17:44:11 +0000 Subject: [PATCH 8/11] site.yaml: Add Fedora Kinoite docs --- diff --git a/site.yml b/site.yml index 4b03c63..9ab6ef8 100644 --- a/site.yml +++ b/site.yml @@ -65,6 +65,8 @@ content: branches: main - url: https://pagure.io/fedora-docs/modularity.git - url: https://github.com/fedora-silverblue/silverblue-docs.git + - url: https://pagure.io/fedora-kde/kinoite-docs + branches: main - url: https://pagure.io/fedora-docs/documentation-contributors-guide.git - url: https://pagure.io/fedora-docs/flatpak.git - url: https://pagure.io/fedora-diversity.git From de05d1bf912bc5decde1df5dcaf407e893a91c0a Mon Sep 17 00:00:00 2001 From: Petr Bokoc Date: May 24 2021 17:44:11 +0000 Subject: [PATCH 9/11] Change Accounts branch to main --- diff --git a/site.yml b/site.yml index 9ab6ef8..e657c94 100644 --- a/site.yml +++ b/site.yml @@ -103,6 +103,7 @@ content: - url: https://github.com/fedora-eln/eln-docs.git - url: https://pagure.io/i3-sig/docs.git - url: https://github.com/fedora-infra/fedora-accounts-docs.git + branches: main - url: https://pagure.io/gaming/documentation.git branches: main - url: https://pagure.io/fedora-server.git From e92a1efd7b165d15148e9a98ac4af8bc3e31f006 Mon Sep 17 00:00:00 2001 From: Petr Bokoc Date: May 24 2021 17:44:11 +0000 Subject: [PATCH 10/11] Publish F34 docs --- diff --git a/site.yml b/site.yml index e657c94..6e2e0a7 100644 --- a/site.yml +++ b/site.yml @@ -14,6 +14,8 @@ content: - url: https://pagure.io/fedora-docs/release-docs-home.git branches: - master + - f34 + - f33 - f32 - f31 - f30 @@ -24,6 +26,8 @@ content: - url: https://pagure.io/fedora-docs/install-guide.git branches: - master + - f34 + - f33 - f32 - f31 - f30 @@ -34,6 +38,8 @@ content: - url: https://pagure.io/fedora-docs/system-administrators-guide.git branches: - master + - f34 + - f33 - f32 - f31 - f30 @@ -44,6 +50,8 @@ content: - url: https://pagure.io/fedora-docs/release-notes.git branches: - master + - f34 + - f33 - f32 - f31 - f30 From 825039c0e57bb2346eb7f8e28b7f38d62d94cf19 Mon Sep 17 00:00:00 2001 From: Dusty Mabe Date: May 24 2021 17:44:11 +0000 Subject: [PATCH 11/11] Switch Fedora CoreOS docs build to main branch --- diff --git a/site.yml b/site.yml index 6e2e0a7..05c2030 100644 --- a/site.yml +++ b/site.yml @@ -98,6 +98,7 @@ content: - url: https://pagure.io/fedora-magazine.git - url: https://pagure.io/minimization.git - url: https://github.com/coreos/fedora-coreos-docs.git + branches: main - url: https://pagure.io/cpe/rawhide-gating-docs.git - url: https://pagure.io/cpe/docs.git - url: https://pagure.io/fedora-docs/websites.git