From 0f91badd86f7852bbf25bc227632824f7c7a6ee6 Mon Sep 17 00:00:00 2001 From: Brendan Early Date: Jan 09 2021 01:11:26 +0000 Subject: [PATCH 1/4] generate diff table in primary db files --- diff --git a/bin/fetch-repository-dbs.py b/bin/fetch-repository-dbs.py index 0bbb6ef..afa80a1 100755 --- a/bin/fetch-repository-dbs.py +++ b/bin/fetch-repository-dbs.py @@ -83,6 +83,70 @@ def index_db(name, tempdb): conn.commit() conn.close() +# Adds a table named 'changes' listing if certian packages were changed, +# added, or deleted. +def gen_db_diff(name, new, old): + if not os.path.isfile(old) or not new.endswith('primary.sqlite'): + return + + print(f'{name.ljust(padding)} Creating diff for file: {old}') + conn = sqlite3.connect(new) + conn.execute(f'ATTACH DATABASE \'{old}\' as old') + # changes table schema: + # name - package name + # arch - package arch + # version - {epoch}:{package version}-{package release} + # change - 'updated', 'removed', or 'added' + conn.execute(''' + CREATE TABLE changes ( + name TEXT NOT NULL, + arch TEXT NOT NULL, + version TEXT, + change TEXT NOT NULL, + UNIQUE(name, arch) + ) + ''') + # Insert added packages list to changes table + conn.execute(''' + INSERT INTO changes (name, arch, version, change) + SELECT main.packages.name, main.packages.arch, + IIF(main.packages.epoch IS NOT NULL, main.packages.epoch || ':', '') || + main.packages.version || '-' || main.packages.release || '.' || main.packages.arch, + 'added' + FROM main.packages LEFT JOIN old.packages ON main.packages.name = old.packages.name + AND main.packages.arch = old.packages.arch + WHERE old.packages.name IS NULL + ''') + # Insert removed packages list to changes table + conn.execute(''' + INSERT INTO changes (name, arch, version, change) + SELECT old.packages.name, old.packages.arch, + IIF(old.packages.epoch IS NOT NULL, old.packages.epoch || ':', '') || + old.packages.version || '-' || old.packages.release, + 'removed' + FROM old.packages LEFT JOIN main.packages ON main.packages.name = old.packages.name + AND main.packages.arch = old.packages.arch + WHERE main.packages.name IS NULL + ''') + # Insert changed packages list to changes table + conn.execute(''' + INSERT INTO changes (name, arch, change) + SELECT name, arch, 'changed' FROM + (SELECT main.packages.name, main.packages.arch, + IIF(main.packages.epoch IS NOT NULL, main.packages.epoch || ':', '') || + main.packages.version || '-' || main.packages.release || '.' || main.packages.arch + FROM main.packages + UNION + SELECT old.packages.name, old.packages.arch, + IIF(old.packages.epoch IS NOT NULL, old.packages.epoch || ':', '') || + old.packages.version || '-' || old.packages.release || '.' || old.packages.arch + FROM old.packages) + GROUP BY name, arch + HAVING COUNT(*) > 1 + ''') + conn.commit() + conn.close() + def install_db(name, src, dest): print(f'{name.ljust(padding)} Installing {src} to {dest}.') shutil.move(src, dest) @@ -148,6 +212,7 @@ def handle(repo, target_dir): download_db(name, repomd_url, archive) decompress_db(name, archive, tempdb) index_db(name, tempdb) + gen_db_diff(name, tempdb, destfile) install_db(name, tempdb, destfile) def get_repository_urls_for(product, version): From 1931856833386eceabdec91530fae0447d485ace Mon Sep 17 00:00:00 2001 From: Brendan Early Date: Jan 10 2021 21:30:07 +0000 Subject: [PATCH 2/4] * Only generate files for packages in changes table if it exists * Add env var for SITEMAP_URL * Clean package folder when updating it --- diff --git a/.gitignore b/.gitignore index a7ee933..6847570 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,5 @@ repositories/ pagure_owner_alias.json vue/node_modules vue/dist +.vscode +bin/__pycache__ \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 9be8acb..42f8935 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,6 +24,7 @@ ENV DB_DIR /etc/packages/repositories ENV MAINTAINER_MAPPING /etc/packages/pagure_owner_alias.json ENV SOLR_CORE packages ENV SOLR_URL http://127.0.0.1:8983/ +ENV SITEMAP_URL https://localhost:8080 COPY . . RUN chmod -R o+rx assets @@ -32,7 +33,7 @@ RUN make setup-js \ && make js COPY container/nginx.conf /etc/nginx/nginx.conf -COPY container/update-packages.sh /etc/cron.weekly/ +COPY container/update-packages.sh /etc/cron.daily/ COPY container/supervisord.conf /etc/supervisord.conf # TODO: Figure out how to use a read-write volume for diff --git a/bin/fetch-repository-dbs.py b/bin/fetch-repository-dbs.py index afa80a1..9778bd7 100755 --- a/bin/fetch-repository-dbs.py +++ b/bin/fetch-repository-dbs.py @@ -95,7 +95,7 @@ def gen_db_diff(name, new, old): # changes table schema: # name - package name # arch - package arch - # version - {epoch}:{package version}-{package release} + # version (optional) - {epoch}:{package version}-{package release} # change - 'updated', 'removed', or 'added' conn.execute(''' CREATE TABLE changes ( @@ -131,7 +131,7 @@ def gen_db_diff(name, new, old): # Insert changed packages list to changes table conn.execute(''' INSERT INTO changes (name, arch, change) - SELECT name, arch, 'changed' FROM + SELECT name, arch, 'updated' FROM (SELECT main.packages.name, main.packages.arch, IIF(main.packages.epoch IS NOT NULL, main.packages.epoch || ':', '') || main.packages.version || '-' || main.packages.release || '.' || main.packages.arch @@ -147,6 +147,14 @@ def gen_db_diff(name, new, old): conn.commit() conn.close() +def clear_diff_table(db): + conn = sqlite3.connect(db) + result = conn.execute("SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'changes'") + if result.fetchone() is not None: + conn.execute("DELETE FROM changes") + conn.commit() + conn.close() + def install_db(name, src, dest): print(f'{name.ljust(padding)} Installing {src} to {dest}.') shutil.move(src, dest) @@ -200,6 +208,7 @@ def handle(repo, target_dir): # Have we downloaded this before? Did it change? destfile = os.path.join(target_dir, db) if not needs_update(destfile, shasum, shatype): + clear_diff_table(destfile) print(f'{name.ljust(padding)} No change of {repomd_url}') continue diff --git a/bin/generate-html.py b/bin/generate-html.py index 0b56330..42b2548 100755 --- a/bin/generate-html.py +++ b/bin/generate-html.py @@ -12,6 +12,7 @@ import json import shutil import sqlite3 import argparse +import glob from datetime import date from collections import defaultdict @@ -22,7 +23,7 @@ TEMPLATE_DIR='../templates' DBS_DIR=os.environ.get('DB_DIR') or "repositories" ASSETS_DIR='assets' SCM_MAINTAINER_MAPPING=os.environ.get('MAINTAINER_MAPPING') or "pagure_owner_alias.json" -SITEMAP_URL = 'https://pkgs.fedoraproject.org' +SITEMAP_URL = os.environ.get('SITEMAP_URL') or 'https://localhost:8080' class Package: def __init__(self, name): @@ -63,6 +64,15 @@ def open_db(db): return (conn, c) +def clean_dir(path): + files = glob.glob(os.path.join(path, '*.html')) + for file in files: + try: + os.remove(file) + except: + print("Error cleaning directory!") + print(sys.exc_info()[0]) + def save_to(path, content): with open(path, 'w') as fh: fh.write(content) @@ -81,7 +91,6 @@ def main(): os.makedirs(output_dir, exist_ok=True) # Initialize templating system. - db = os.path.join(DBS_DIR, "koji-primary.sqlite") env = Environment( loader=PackageLoader('generate-html', TEMPLATE_DIR), autoescape=select_autoescape(['html']) @@ -109,6 +118,7 @@ def main(): # Build internal package metadata structure / cache. packages = {} + partial_update = False srpm_pattern = re.compile("^(.+)-.+-.+.src.rpm$") changelog_mail_pattern = re.compile("<(.+@.+)>") release_branch_pattern = re.compile("^([fedora|epel]+-[\w|\d]+)-?([a-z|-]+)?$") @@ -123,6 +133,16 @@ def main(): (_, filelist) = open_db(databases[release_branch]["filelists"]) (_, other) = open_db(databases[release_branch]["other"]) + if not partial_update: + primary.execute("SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'changes'") + partial_update = primary.fetchone() is not None + + partial_update_packages = [] + if partial_update: + primary.execute('SELECT name FROM changes') + for row in primary.fetchall(): + partial_update_packages.append(row['name']) + for raw in primary.execute('SELECT * FROM packages'): pkg = packages.get(raw["name"]) revision = "{}-{}".format(raw["version"], raw["release"]) @@ -142,6 +162,12 @@ def main(): pkg.license = raw["rpm_license"] pkg.maintainers = maintainer_mapping["rpms"].get(pkg.name, []) + # Check if package should be updated during a partial update + if partial_update and pkg.name in partial_update_packages: + pkg.should_update = True + elif first_pkg_encounter and partial_update: + pkg.should_update = False + # Handle subpackage specific case. (srpm_name) = srpm_pattern.findall(raw["rpm_sourcerpm"])[0] if pkg.name != srpm_name: @@ -214,7 +240,10 @@ def main(): # Generate main pages. print(">>> Index pages...") for pkg in packages.values(): + if pkg.should_update == False: + continue pkg_dir = os.path.join(output_dir, 'pkgs', pkg.name) + clean_dir(pkg_dir) os.makedirs(pkg_dir, exist_ok=True) html_path = os.path.join(pkg_dir, 'index.html') @@ -232,6 +261,8 @@ def main(): db_conns = {} detailed_page_count = 0 for pkg in packages.values(): + if pkg.should_update == False: + continue pkg_dir = os.path.join(output_dir, 'pkgs', pkg.name) for release in pkg.releases.keys(): for branch in pkg.get_release(release).keys(): @@ -299,7 +330,7 @@ def main(): print("Processed {}/{} pages.".format(detailed_page_count, max_page_count)) print("DONE.") - print("> {} packages processed.".format(len(packages))) + print("> {} packages processed.".format(page_count)) if __name__ == '__main__': main() diff --git a/container/update-packages.sh b/container/update-packages.sh index 7e25db1..dafa998 100644 --- a/container/update-packages.sh +++ b/container/update-packages.sh @@ -1,7 +1,4 @@ #!/bin/sh -OUTPUT_DIR=/tmp/packages/ cd /usr/local/src/packages -make all -rsync -av --progress --delete $OUTPUT_DIR /srv/packages -make update-solr -rm -rf $OUTPUT_DIR +make html-only +make update-solr \ No newline at end of file From e0430721d581190ebfa6287065758242e1b721c2 Mon Sep 17 00:00:00 2001 From: Brendan Early Date: Jan 13 2021 19:12:45 +0000 Subject: [PATCH 3/4] remove packages from target dir if it has been removed from all repoistories during a partial update --- diff --git a/bin/generate-html.py b/bin/generate-html.py index 42b2548..c0e403e 100755 --- a/bin/generate-html.py +++ b/bin/generate-html.py @@ -119,6 +119,7 @@ def main(): # Build internal package metadata structure / cache. packages = {} partial_update = False + removed_packages = set() srpm_pattern = re.compile("^(.+)-.+-.+.src.rpm$") changelog_mail_pattern = re.compile("<(.+@.+)>") release_branch_pattern = re.compile("^([fedora|epel]+-[\w|\d]+)-?([a-z|-]+)?$") @@ -183,6 +184,17 @@ def main(): branch = "base" pkg.set_release(release, raw["pkgKey"], branch, revision) + + # Get removed packages to determine if folder needs to be deleted later + if partial_update: + for removed in primary.execute("SELECT name FROM changes WHERE change = 'removed'"): + removed_packages.add(removed["name"]) + + # If a package was removed and it was not in any repository, attempt to + # delete the folder from the target directory + for removed_package in removed_packages: + if removed_package not in packages: + shutil.rmtree(os.path.join(output_dir, 'pkgs', removed_package), True) # Set license and maintainers for subpackages. We have to wait for all # packages to have been processed since subpackage might have been From b73a9e644be2b8aa7408b1fbc234b5f2bd74a674 Mon Sep 17 00:00:00 2001 From: Brendan Early Date: Jan 13 2021 19:27:09 +0000 Subject: [PATCH 4/4] always update packages if a changes table does not exist --- diff --git a/bin/generate-html.py b/bin/generate-html.py index c0e403e..10bab4d 100755 --- a/bin/generate-html.py +++ b/bin/generate-html.py @@ -134,9 +134,9 @@ def main(): (_, filelist) = open_db(databases[release_branch]["filelists"]) (_, other) = open_db(databases[release_branch]["other"]) - if not partial_update: - primary.execute("SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'changes'") - partial_update = primary.fetchone() is not None + # Check if this db has a changes table + primary.execute("SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'changes'") + partial_update = primary.fetchone() is not None partial_update_packages = [] if partial_update: @@ -168,6 +168,11 @@ def main(): pkg.should_update = True elif first_pkg_encounter and partial_update: pkg.should_update = False + + # If a changes table does not exist, then the package should + # always be updated. + if not partial_update: + pkg.should_update = True # Handle subpackage specific case. (srpm_name) = srpm_pattern.findall(raw["rpm_sourcerpm"])[0]