From 90c1263d59989363ce47468adbc288d4b1a44d27 Mon Sep 17 00:00:00 2001 From: Anatoli Babenia Date: Jun 05 2021 19:59:33 +0000 Subject: [PATCH 1/10] Add progress bar to repo fetcher --- diff --git a/Dockerfile b/Dockerfile index 74f8976..0c76849 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,6 +10,7 @@ RUN dnf -y upgrade \ python3-requests \ python3-jinja2 \ python3-defusedxml \ + python3-tqdm \ npm \ rsync diff --git a/README.md b/README.md index 5757296..97638d8 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,7 @@ The scripts contained in this repository depend on: * `python3-requests` * `python3-jinja2` * `python3-defusedxml` +* `python3-tqdm` ## Usage diff --git a/bin/fetch-repository-dbs.py b/bin/fetch-repository-dbs.py index d891360..faf25a9 100755 --- a/bin/fetch-repository-dbs.py +++ b/bin/fetch-repository-dbs.py @@ -9,6 +9,7 @@ import os import argparse import hashlib import sys +import tqdm repomd_xml_namespace = { 'repo': 'http://linux.duke.edu/metadata/repo', @@ -47,10 +48,15 @@ def needs_update(local_file, remote_sha, sha_type): def download_db(name, repomd_url, archive): print(f'{name.ljust(padding)} Downloading file: {repomd_url} to {archive}') - response = requests.get(repomd_url, verify=DL_VERIFY) + response = requests.get(repomd_url, verify=DL_VERIFY, stream=True) response.raise_for_status() - with open(archive, 'wb') as stream: - stream.write(response.content) + with tqdm.tqdm.wrapattr( + open(archive, 'wb'), + "write", + desc=repomd_url.split('/')[-1], + total=int(response.headers.get('content-length', 0))) as stream: + for chunk in response.iter_content(chunk_size=1024*1024): + stream.write(chunk) def decompress_db(name, archive, location): ''' Decompress the given XZ archive at the specified location. ''' From 2256d42d25cc00cbec2547dfe44934a696cfa9aa Mon Sep 17 00:00:00 2001 From: Anatoli Babenia Date: Jun 05 2021 19:59:33 +0000 Subject: [PATCH 2/10] Add static html index for packages --- diff --git a/bin/generate-html.py b/bin/generate-html.py index 785ce87..0b72d43 100755 --- a/bin/generate-html.py +++ b/bin/generate-html.py @@ -242,19 +242,33 @@ def main(): pkg.parent_not_exist = True print(">>> {} packages have been extracted.".format(len(packages))) + # Generate main user entrypoint. - print("Generating generic pages...") + pkgs_list = sorted(packages.keys()) + print("Generating index pages...") + + # {"aa": ["aaargh", ...]} + prefix_index = {} + for pkg_name in pkgs_list: + prefix_index.setdefault(pkg_name[:2].lower(), []).append(pkg_name) + search = env.get_template('search.html.j2') - search_html = search.render(date=date.today().isoformat(), package_count=len(packages)) + search_html = search.render(date=date.today().isoformat(), + package_count=len(packages), + prefix_index=prefix_index) save_to(os.path.join(output_dir, 'index.html'), search_html) - # Import assets. + index_tpl = env.get_template('index-prefix.html.j2') + for prefix, names in prefix_index.items(): + html = index_tpl.render(prefix=prefix, packages=names) + save_to(os.path.join(output_dir, f'index-{prefix}.html'), html) + + # Copy styles and images. assets_output = os.path.join(output_dir, 'assets') - if not os.path.exists(assets_output): - shutil.copytree(ASSETS_DIR, os.path.join(output_dir, 'assets')) + shutil.rmtree(assets_output, ignore_errors=True) + shutil.copytree(ASSETS_DIR, assets_output) # Generate sitemaps - pkgs_list = list(packages.keys()) sitemap_list = [] sitemap_dir = os.path.join(output_dir, 'sitemaps') shutil.rmtree(sitemap_dir, ignore_errors=True) diff --git a/templates/index-prefix.html.j2 b/templates/index-prefix.html.j2 new file mode 100644 index 0000000..9febb7e --- /dev/null +++ b/templates/index-prefix.html.j2 @@ -0,0 +1,41 @@ + + +Fedora packages + + + + + + +
+ +
+
+
+ +
+
+ +
+
+
+ +

+ Back to the packages index ↵ +

+ +

Packages that start with "{{ prefix }}".

+ +
    + {% for package in packages %} +
  • {{ package }}
    • + {% endfor %} +
+ +
+ Sources on Pagure.
+
+ + diff --git a/templates/search.html.j2 b/templates/search.html.j2 index 91d7641..93e47a1 100644 --- a/templates/search.html.j2 +++ b/templates/search.html.j2 @@ -28,10 +28,16 @@ Back to the application index ↵

-

- Last refreshed on {{ date }} ({{ package_count }} packages). Sources on - Pagure. +

+ {% for prefix in prefix_index %} + {{ prefix }} + {% endfor %}

+ +
+
Last refreshed on {{ date }} ({{ package_count }} packages).
+
Sources on Pagure.
+
From 700a86b2c31a0f14399dcb01580c6a33df7bb493 Mon Sep 17 00:00:00 2001 From: Anatoli Babenia Date: Jun 05 2021 19:59:33 +0000 Subject: [PATCH 3/10] Remove link to Fedora Applications --- diff --git a/templates/search.html.j2 b/templates/search.html.j2 index 93e47a1..2ff666a 100644 --- a/templates/search.html.j2 +++ b/templates/search.html.j2 @@ -24,9 +24,7 @@ -

- Back to the application index ↵ -

+

Static Index

{% for prefix in prefix_index %} From 18b7c59b389e2463760e167631a77c3f84ee7474 Mon Sep 17 00:00:00 2001 From: Anatoli Babenia Date: Jun 05 2021 19:59:33 +0000 Subject: [PATCH 4/10] Remove search form if no search backend is enabled Remove from main, static index and package pages --- diff --git a/bin/generate-html.py b/bin/generate-html.py index 0b72d43..1213b0b 100755 --- a/bin/generate-html.py +++ b/bin/generate-html.py @@ -26,6 +26,7 @@ ASSETS_DIR='assets' SCM_MAINTAINER_MAPPING=os.environ.get('MAINTAINER_MAPPING') or "pagure_owner_alias.json" PRODUCT_VERSION_MAPPING=os.environ.get('PRODUCT_VERSION_MAPPING') or "product_version_mapping.json" SITEMAP_URL = os.environ.get('SITEMAP_URL') or 'https://localhost:8080' +SEARCH_BACKEND = os.environ.get('SEARCH_BACKEND', False) class Package: def __init__(self, name): @@ -255,12 +256,14 @@ def main(): search = env.get_template('search.html.j2') search_html = search.render(date=date.today().isoformat(), package_count=len(packages), - prefix_index=prefix_index) + prefix_index=prefix_index, + search_backend=SEARCH_BACKEND) save_to(os.path.join(output_dir, 'index.html'), search_html) index_tpl = env.get_template('index-prefix.html.j2') for prefix, names in prefix_index.items(): - html = index_tpl.render(prefix=prefix, packages=names) + html = index_tpl.render(prefix=prefix, packages=names, + search_backend=SEARCH_BACKEND) save_to(os.path.join(output_dir, f'index-{prefix}.html'), html) # Copy styles and images. @@ -308,7 +311,7 @@ def main(): html_path = os.path.join(pkg_dir, 'index.html') html_template = env.get_template('package.html.j2') - html_content = html_template.render(pkg=pkg) + html_content = html_template.render(pkg=pkg, search_backend=SEARCH_BACKEND) save_to(html_path, html_content) # Simple way to display progress. @@ -416,7 +419,7 @@ def main(): html_path = os.path.join(pkg_dir, release_branch + ".html") html_template = env.get_template("package-details.html.j2") - html_content = html_template.render(pkg=pkg, release=release, branch=branch, changelog=changelog, files=files, provides=provides, requires=requires) + html_content = html_template.render(pkg=pkg, release=release, branch=branch, changelog=changelog, files=files, provides=provides, requires=requires, search_backend=SEARCH_BACKEND) save_to(html_path, html_content) print("DONE.") diff --git a/templates/index-prefix.html.j2 b/templates/index-prefix.html.j2 index 9febb7e..0977f6e 100644 --- a/templates/index-prefix.html.j2 +++ b/templates/index-prefix.html.j2 @@ -11,6 +11,8 @@

+ + {% if search_backend %}
@@ -21,6 +23,7 @@
+ {% endif %}

Back to the packages index ↵ diff --git a/templates/package-details.html.j2 b/templates/package-details.html.j2 index d7cc587..ce09dc1 100644 --- a/templates/package-details.html.j2 +++ b/templates/package-details.html.j2 @@ -15,6 +15,7 @@

+ {% if search_backend %}
@@ -25,6 +26,7 @@
+ {% endif %}
diff --git a/templates/package.html.j2 b/templates/package.html.j2 index c44dfd4..442ba92 100644 --- a/templates/package.html.j2 +++ b/templates/package.html.j2 @@ -15,6 +15,7 @@
+ {% if search_backend %}
@@ -25,6 +26,7 @@
+ {% endif %}
diff --git a/templates/search.html.j2 b/templates/search.html.j2 index 2ff666a..4eaf0bb 100644 --- a/templates/search.html.j2 +++ b/templates/search.html.j2 @@ -13,6 +13,7 @@
+ {% if search_backend %}
@@ -23,6 +24,7 @@
+ {% endif %}

Static Index

From b8f2bea32edbd80956ef17b3ccd21a4440d49ed7 Mon Sep 17 00:00:00 2001 From: Anatoli Babenia Date: Jun 05 2021 19:59:33 +0000 Subject: [PATCH 5/10] Rename template search.html.j2 -> index.html.j2 --- diff --git a/bin/generate-html.py b/bin/generate-html.py index 1213b0b..37eb3d3 100755 --- a/bin/generate-html.py +++ b/bin/generate-html.py @@ -253,7 +253,7 @@ def main(): for pkg_name in pkgs_list: prefix_index.setdefault(pkg_name[:2].lower(), []).append(pkg_name) - search = env.get_template('search.html.j2') + search = env.get_template('index.html.j2') search_html = search.render(date=date.today().isoformat(), package_count=len(packages), prefix_index=prefix_index, diff --git a/templates/index.html.j2 b/templates/index.html.j2 new file mode 100644 index 0000000..4eaf0bb --- /dev/null +++ b/templates/index.html.j2 @@ -0,0 +1,43 @@ + + + + Fedora Packages + + + + + + + +
+
+ +
+ {% if search_backend %} +
+
+
+ +
+
+ +
+
+
+ {% endif %} + +

Static Index

+ +

+ {% for prefix in prefix_index %} + {{ prefix }} + {% endfor %} +

+ +
+
Last refreshed on {{ date }} ({{ package_count }} packages).
+
Sources on Pagure.
+
+
+ + diff --git a/templates/search.html.j2 b/templates/search.html.j2 deleted file mode 100644 index 4eaf0bb..0000000 --- a/templates/search.html.j2 +++ /dev/null @@ -1,43 +0,0 @@ - - - - Fedora Packages - - - - - - - -
-
- -
- {% if search_backend %} -
-
-
- -
-
- -
-
-
- {% endif %} - -

Static Index

- -

- {% for prefix in prefix_index %} - {{ prefix }} - {% endfor %} -

- -
-
Last refreshed on {{ date }} ({{ package_count }} packages).
-
Sources on Pagure.
-
-
- - From 478dce6c0e9c6cc50015ae11b44948dbd193008a Mon Sep 17 00:00:00 2001 From: Anatoli Babenia Date: Jun 05 2021 19:59:33 +0000 Subject: [PATCH 6/10] Fix for new style changes --- diff --git a/templates/index-prefix.html.j2 b/templates/index-prefix.html.j2 index 0977f6e..ea7b66e 100644 --- a/templates/index-prefix.html.j2 +++ b/templates/index-prefix.html.j2 @@ -1,44 +1,42 @@ - Fedora packages + - -
- +
+
+ +
- {% if search_backend %} -
-
-
- -
-
- -
+ {% if search_backend %} + +
+
+
- - {% endif %} +
+ +
+
+ + {% endif %} -

- Back to the packages index ↵ -

+

+ Back to the packages index ↵ +

-

Packages that start with "{{ prefix }}".

+

Packages that start with "{{ prefix }}".

-
    - {% for package in packages %} -
  • {{ package }}
    • - {% endfor %} -
+
    + {% for package in packages %} +
  • {{ package }}
    • + {% endfor %} +
-
- Sources on Pagure.
-
+
+ Sources on Pagure.
- +
From 680682622ed373ae6fd98b568c3606835910cb94 Mon Sep 17 00:00:00 2001 From: Anatoli Babenia Date: Jun 05 2021 19:59:33 +0000 Subject: [PATCH 7/10] Make primary header black again --- diff --git a/templates/index.html.j2 b/templates/index.html.j2 index 4eaf0bb..eafc4e3 100644 --- a/templates/index.html.j2 +++ b/templates/index.html.j2 @@ -26,7 +26,7 @@ {% endif %} -

Static Index

+

Static Index

{% for prefix in prefix_index %} From 6be5515cddf4e0cf25a126666fa0cdec78076ea4 Mon Sep 17 00:00:00 2001 From: Anatoli Babenia Date: Jun 05 2021 19:59:33 +0000 Subject: [PATCH 8/10] Link logo to website root from index pages --- diff --git a/templates/index-prefix.html.j2 b/templates/index-prefix.html.j2 index ea7b66e..1e9d083 100644 --- a/templates/index-prefix.html.j2 +++ b/templates/index-prefix.html.j2 @@ -8,7 +8,9 @@

- + + +
{% if search_backend %} From 4fc33210bed9566f0b11a19ae5244e7bbf46901e Mon Sep 17 00:00:00 2001 From: Anatoli Babenia Date: Jun 05 2021 19:59:33 +0000 Subject: [PATCH 9/10] Add `layout.html.j2` and use it for static indexes Other changes to prefix index pages. - prefix is now shown in page title to distinguish browser tabs - search form is removed from prefix indexes - link that sends back to title is moved below page header --- diff --git a/templates/index-prefix.html.j2 b/templates/index-prefix.html.j2 index 1e9d083..88c0fc8 100644 --- a/templates/index-prefix.html.j2 +++ b/templates/index-prefix.html.j2 @@ -1,44 +1,17 @@ - -Fedora packages - - - - - +{% extends "layout.html.j2" %} -
-
- - - -
+{% block title %}"{{ prefix }}" Index - Fedora Packages{% endblock %} - {% if search_backend %} -
-
-
- -
-
- -
-
-
- {% endif %} +{% block content %} +

Packages that start with "{{ prefix }}".

Back to the packages index ↵

-

Packages that start with "{{ prefix }}".

- - -
- Sources on Pagure.
-
-
+{% endblock %} diff --git a/templates/layout.html.j2 b/templates/layout.html.j2 new file mode 100644 index 0000000..0ba272d --- /dev/null +++ b/templates/layout.html.j2 @@ -0,0 +1,25 @@ + +{% block title %}Fedora Packages{% endblock %} + + + + + + +
+
+ + + +
+ + {% block content %} + {% endblock %} + +
+ {% block footer %} + Sources on Pagure.
+ {% endblock %} +
+ + From a2930d48d30b47162ca73145d1495e5547db3ba3 Mon Sep 17 00:00:00 2001 From: Anatoli Babenia Date: Jun 05 2021 19:59:33 +0000 Subject: [PATCH 10/10] Generate static index on a separate page --- diff --git a/bin/generate-html.py b/bin/generate-html.py index 37eb3d3..66435d0 100755 --- a/bin/generate-html.py +++ b/bin/generate-html.py @@ -253,6 +253,12 @@ def main(): for pkg_name in pkgs_list: prefix_index.setdefault(pkg_name[:2].lower(), []).append(pkg_name) + static_index_html = env.get_template('index-static.html.j2').render( + date=date.today().isoformat(), + package_count=len(packages), + prefix_index=prefix_index) + save_to(os.path.join(output_dir, 'index-static.html'), static_index_html) + search = env.get_template('index.html.j2') search_html = search.render(date=date.today().isoformat(), package_count=len(packages), diff --git a/templates/index-prefix.html.j2 b/templates/index-prefix.html.j2 index 88c0fc8..1bf306c 100644 --- a/templates/index-prefix.html.j2 +++ b/templates/index-prefix.html.j2 @@ -6,7 +6,7 @@

Packages that start with "{{ prefix }}".

- Back to the packages index ↵ + Back to the packages index ↵

    diff --git a/templates/index-static.html.j2 b/templates/index-static.html.j2 new file mode 100644 index 0000000..1182db1 --- /dev/null +++ b/templates/index-static.html.j2 @@ -0,0 +1,22 @@ +{% extends "layout.html.j2" %} + +{% block title %}Static Index - Fedora Packages{% endblock %} + +{% block content %} +

    Static Index

    + +

    + Back to the main page ↵ +

    + +

    + {% for prefix in prefix_index %} + {{ prefix }} + {% endfor %} +

    +{% endblock %} + +{% block footer %} +
    Last refreshed on {{ date }} ({{ package_count }} packages).
    +
    Sources on Pagure.
    +{% endblock %} diff --git a/templates/index.html.j2 b/templates/index.html.j2 index eafc4e3..9292d8d 100644 --- a/templates/index.html.j2 +++ b/templates/index.html.j2 @@ -13,7 +13,7 @@
    - {% if search_backend %} +
    @@ -24,14 +24,12 @@
    - {% endif %} - -

    Static Index

    -

    - {% for prefix in prefix_index %} - {{ prefix }} - {% endfor %} +

    + Inspect Static Packages Index +

    +

    + Back to Fedora Apps ↵