From f9908f489e64c100e87f0612198ad1b00c6edfeb Mon Sep 17 00:00:00 2001 From: Brendan Early Date: Oct 02 2021 20:55:57 +0000 Subject: [PATCH 1/2] run black on all files --- diff --git a/bin/fetch-repository-dbs.py b/bin/fetch-repository-dbs.py index 2b50dbf..88ce8a4 100755 --- a/bin/fetch-repository-dbs.py +++ b/bin/fetch-repository-dbs.py @@ -15,20 +15,21 @@ from dnf.subject import Subject import hawkey repomd_xml_namespace = { - 'repo': 'http://linux.duke.edu/metadata/repo', - 'rpm': 'http://linux.duke.edu/metadata/rpm', + "repo": "http://linux.duke.edu/metadata/repo", + "rpm": "http://linux.duke.edu/metadata/rpm", } padding = 22 -MIRROR = 'https://dl.fedoraproject.org' -KOJI_REPO = 'https://kojipkgs.fedoraproject.org/repos' +MIRROR = "https://dl.fedoraproject.org" +KOJI_REPO = "https://kojipkgs.fedoraproject.org/repos" # Enforce, or not, checking the SSL certs DL_VERIFY = True + def needs_update(local_file, remote_sha, sha_type): - ''' Compare sha of a local and remote file. + """Compare sha of a local and remote file. Return True if our local file needs to be updated. - ''' + """ if not os.path.isfile(local_file): # If we have never downloaded this before, then obviously it has @@ -36,11 +37,11 @@ def needs_update(local_file, remote_sha, sha_type): return True # Old old epel5 doesn't even know which sha it is using.. - if sha_type == 'sha': - sha_type = 'sha1' + if sha_type == "sha": + sha_type = "sha1" hash = getattr(hashlib, sha_type)() - with open(local_file, 'rb') as f: + with open(local_file, "rb") as f: hash.update(f.read()) local_sha = hash.hexdigest() @@ -49,76 +50,89 @@ def needs_update(local_file, remote_sha, sha_type): return False + def download_db(name, repomd_url, archive): - print(f'{name.ljust(padding)} Downloading file: {repomd_url} to {archive}') + print(f"{name.ljust(padding)} Downloading file: {repomd_url} to {archive}") response = requests.get(repomd_url, verify=DL_VERIFY, stream=True) response.raise_for_status() with tqdm.tqdm.wrapattr( - open(archive, 'wb'), - "write", - desc=repomd_url.split('/')[-1], - total=int(response.headers.get('content-length', 0))) as stream: - for chunk in response.iter_content(chunk_size=1024*1024): + open(archive, "wb"), + "write", + desc=repomd_url.split("/")[-1], + total=int(response.headers.get("content-length", 0)), + ) as stream: + for chunk in response.iter_content(chunk_size=1024 * 1024): stream.write(chunk) + def decompress_db(name, archive, location): - ''' Decompress the given XZ archive at the specified location. ''' - print(f'{name.ljust(padding)} Extracting {archive} to {location}') - if archive.endswith('.xz'): + """Decompress the given XZ archive at the specified location.""" + print(f"{name.ljust(padding)} Extracting {archive} to {location}") + if archive.endswith(".xz"): import lzma - with lzma.open(archive) as inp, open(location, 'wb') as out: + + with lzma.open(archive) as inp, open(location, "wb") as out: out.write(inp.read()) - elif archive.endswith('.tar.gz'): + elif archive.endswith(".tar.gz"): import tarfile + with tarfile.open(archive) as tar: tar.extractall(path=location) - elif archive.endswith('.gz'): + elif archive.endswith(".gz"): import gzip - with gzip.open(archive, 'rb') as inp, open(location, 'wb') as out: + + with gzip.open(archive, "rb") as inp, open(location, "wb") as out: out.write(inp.read()) - elif archive.endswith('.bz2'): + elif archive.endswith(".bz2"): import bz2 - with bz2.open(archive) as inp, open(location, 'wb') as out: + + with bz2.open(archive) as inp, open(location, "wb") as out: out.write(inp.read()) else: raise NotImplementedError(archive) + def index_db(name, tempdb): - print(f'{name.ljust(padding)} Indexing file: {tempdb}') + print(f"{name.ljust(padding)} Indexing file: {tempdb}") - if tempdb.endswith('primary.sqlite'): + if tempdb.endswith("primary.sqlite"): conn = sqlite3.connect(tempdb) conn.row_factory = sqlite3.Row - conn.execute('CREATE INDEX packageSource ON packages (rpm_sourcerpm)') + conn.execute("CREATE INDEX packageSource ON packages (rpm_sourcerpm)") conn.commit() # Insert source package name field for diff creation - conn.execute('ALTER TABLE packages ADD rpm_sourcerpm_name TEXT') - for package_info in conn.execute('SELECT * FROM packages'): + conn.execute("ALTER TABLE packages ADD rpm_sourcerpm_name TEXT") + for package_info in conn.execute("SELECT * FROM packages"): subject = Subject(package_info["rpm_sourcerpm"]) nevra = subject.get_nevra_possibilities(forms=hawkey.FORM_NEVRA) - conn.execute("UPDATE packages SET rpm_sourcerpm_name = ? WHERE pkgKey = ?", [nevra[0].name, package_info["pkgKey"]]) + conn.execute( + "UPDATE packages SET rpm_sourcerpm_name = ? WHERE pkgKey = ?", + [nevra[0].name, package_info["pkgKey"]], + ) conn.commit() conn.close() + # Adds a table named 'changes' listing if certian packages were changed, # added, or deleted. def gen_db_diff(name, new, old, regen_all): - if not os.path.isfile(old) or not new.endswith('primary.sqlite'): + if not os.path.isfile(old) or not new.endswith("primary.sqlite"): return # If we want to regen all, then just don't make changes tables if regen_all: return - print(f'{name.ljust(padding)} Creating diff for file: {old}') + print(f"{name.ljust(padding)} Creating diff for file: {old}") conn = sqlite3.connect(new) - conn.execute(f'ATTACH DATABASE \'{old}\' as old') + conn.execute(f"ATTACH DATABASE '{old}' as old") # changes table schema: # name - package name # arch - package arch # version (optional) - {epoch}:{package version}-{package release} # change - 'updated', 'removed', or 'added' - conn.execute(''' + conn.execute( + """ CREATE TABLE changes ( name TEXT NOT NULL, arch TEXT NOT NULL, @@ -127,9 +141,11 @@ def gen_db_diff(name, new, old, regen_all): change TEXT NOT NULL, UNIQUE(name, arch, rpm_sourcerpm_name) ON CONFLICT IGNORE ) - ''') + """ + ) # Insert added packages list to changes table - conn.execute(''' + conn.execute( + """ INSERT INTO changes (name, arch, rpm_sourcerpm_name, version, change) SELECT main.packages.name, main.packages.arch, main.packages.rpm_sourcerpm_name, IIF(main.packages.epoch IS NOT NULL, main.packages.epoch || ':', '') || @@ -138,9 +154,11 @@ def gen_db_diff(name, new, old, regen_all): FROM main.packages LEFT JOIN old.packages ON main.packages.name = old.packages.name AND main.packages.arch = old.packages.arch AND main.packages.rpm_sourcerpm_name = old.packages.rpm_sourcerpm_name WHERE old.packages.name IS NULL - ''') + """ + ) # Insert removed packages list to changes table - conn.execute(''' + conn.execute( + """ INSERT INTO changes (name, arch, rpm_sourcerpm_name, version, change) SELECT old.packages.name, old.packages.arch, old.packages.rpm_sourcerpm_name, IIF(old.packages.epoch IS NOT NULL, old.packages.epoch || ':', '') || @@ -149,9 +167,11 @@ def gen_db_diff(name, new, old, regen_all): FROM old.packages LEFT JOIN main.packages ON main.packages.name = old.packages.name AND main.packages.arch = old.packages.arch AND main.packages.rpm_sourcerpm_name = old.packages.rpm_sourcerpm_name WHERE main.packages.name IS NULL - ''') + """ + ) # Insert changed packages list to changes table - conn.execute(''' + conn.execute( + """ INSERT INTO changes (name, arch, rpm_sourcerpm_name, change) SELECT name, arch, rpm_sourcerpm_name, 'updated' FROM (SELECT main.packages.name, main.packages.arch, main.packages.rpm_sourcerpm_name, @@ -165,47 +185,57 @@ def gen_db_diff(name, new, old, regen_all): FROM old.packages) GROUP BY name, arch, rpm_sourcerpm_name HAVING COUNT(*) > 1 - ''') + """ + ) conn.commit() conn.close() + def clear_diff_table(db): conn = sqlite3.connect(db) - result = conn.execute("SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'changes'") + result = conn.execute( + "SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'changes'" + ) if result.fetchone() is not None: conn.execute("DELETE FROM changes") conn.commit() conn.close() + def install_db(name, src, dest): - print(f'{name.ljust(padding)} Installing {src} to {dest}.') + print(f"{name.ljust(padding)} Installing {src} to {dest}.") shutil.move(src, dest) + def handle(repo, target_dir, db_removed): url, name = repo - repomd_url = f'{url}/repomd.xml' + repomd_url = f"{url}/repomd.xml" response = requests.get(repomd_url, verify=DL_VERIFY) if not response: - print(f'{name.ljust(padding)} !! Failed to get {repomd_url!r} {response!r}') + print(f"{name.ljust(padding)} !! Failed to get {repomd_url!r} {response!r}") return # Parse the xml doc and get a list of locations and their shasum. - files = (( - node.find('repo:location', repomd_xml_namespace), - node.find('repo:open-checksum', repomd_xml_namespace), - ) for node in ET.fromstring(response.text)) + files = ( + ( + node.find("repo:location", repomd_xml_namespace), + node.find("repo:open-checksum", repomd_xml_namespace), + ) + for node in ET.fromstring(response.text) + ) # Extract out the attributes that we're really interested in. files = ( - (f.attrib['href'].replace('repodata/', ''), s.text, s.attrib['type']) - for f, s in files if f is not None and s is not None + (f.attrib["href"].replace("repodata/", ""), s.text, s.attrib["type"]) + for f, s in files + if f is not None and s is not None ) # Filter down to only sqlite dbs - files = ((f, s, t) for f, s, t in files if '.sqlite' in f) + files = ((f, s, t) for f, s, t in files if ".sqlite" in f) # We need to ensure the primary db comes first so we can build a pkey cache - primary_first = lambda item: 'primary' not in item[0] + primary_first = lambda item: "primary" not in item[0] files = sorted(files, key=primary_first) # Primary-key caches built from the primary dbs so we can make sense @@ -213,29 +243,29 @@ def handle(repo, target_dir, db_removed): cache1, cache2 = {}, {} if not files: - print(f'No sqlite database could be found in {url}') + print(f"No sqlite database could be found in {url}") for filename, shasum, shatype in files: - repomd_url = f'{url}/{filename}' + repomd_url = f"{url}/{filename}" # First, determine if the file has changed by comparing hash db = None - if 'primary.sqlite' in filename: - db = f'{name}_primary.sqlite' - elif 'filelists.sqlite' in filename: - db = f'{name}_filelists.sqlite' - elif 'other.sqlite' in filename: - db = f'{name}_other.sqlite' + if "primary.sqlite" in filename: + db = f"{name}_primary.sqlite" + elif "filelists.sqlite" in filename: + db = f"{name}_filelists.sqlite" + elif "other.sqlite" in filename: + db = f"{name}_other.sqlite" # Have we downloaded this before? Did it change? destfile = os.path.join(target_dir, db) if not needs_update(destfile, shasum, shatype): clear_diff_table(destfile) - print(f'{name.ljust(padding)} No change of {repomd_url}') + print(f"{name.ljust(padding)} No change of {repomd_url}") continue # If it has changed, then download it and move it into place. - tempargs = dict(prefix='mdapi-') + tempargs = dict(prefix="mdapi-") with tempfile.TemporaryDirectory(**tempargs) as working_dir: tempdb = os.path.join(working_dir, db) archive = os.path.join(working_dir, filename) @@ -243,55 +273,89 @@ def handle(repo, target_dir, db_removed): try: download_db(name, repomd_url, archive) except HTTPError as err: - print(f'{name.ljust(padding)} ERROR Downloading DB file: {err}') - print(f'{name.ljust(padding)} will be skipped.') + print(f"{name.ljust(padding)} ERROR Downloading DB file: {err}") + print(f"{name.ljust(padding)} will be skipped.") continue decompress_db(name, archive, tempdb) index_db(name, tempdb) gen_db_diff(name, tempdb, destfile, db_removed) install_db(name, tempdb, destfile) + def get_repository_urls_for(product, version): release = "{}-{}".format(product, version) if product == "fedora": if version == "rawhide": - return [(f'{KOJI_REPO}/rawhide/latest/x86_64/repodata', release)] + return [(f"{KOJI_REPO}/rawhide/latest/x86_64/repodata", release)] return [ - ('{}/pub/fedora/linux/releases/{}/Everything/x86_64/os/repodata'.format(MIRROR, version), release), - ('{}/pub/fedora/linux/updates/{}/Everything/x86_64/repodata'.format(MIRROR, version), release + "-updates"), - ('{}/pub/fedora/linux/updates/testing/{}/Everything/x86_64/repodata'.format(MIRROR, version), release + "-updates-testing"), - ] + ( + "{}/pub/fedora/linux/releases/{}/Everything/x86_64/os/repodata".format( + MIRROR, version + ), + release, + ), + ( + "{}/pub/fedora/linux/updates/{}/Everything/x86_64/repodata".format( + MIRROR, version + ), + release + "-updates", + ), + ( + "{}/pub/fedora/linux/updates/testing/{}/Everything/x86_64/repodata".format( + MIRROR, version + ), + release + "-updates-testing", + ), + ] elif product == "epel" and int(version) < 8: return [ - ('{}/pub/epel/{}/x86_64/repodata/'.format(MIRROR, version), release), - ('{}/pub/epel/testing/{}/x86_64/repodata'.format(MIRROR, version), release + "-testing"), - ] + ("{}/pub/epel/{}/x86_64/repodata/".format(MIRROR, version), release), + ( + "{}/pub/epel/testing/{}/x86_64/repodata".format(MIRROR, version), + release + "-testing", + ), + ] elif product == "epel" and int(version) >= 8: return [ - ('{}/pub/epel/{}/Everything/x86_64/repodata/'.format(MIRROR, version), release), - ('{}/pub/epel/testing/{}/Everything/x86_64/repodata'.format(MIRROR, version), release + "-testing"), - ] + ( + "{}/pub/epel/{}/Everything/x86_64/repodata/".format(MIRROR, version), + release, + ), + ( + "{}/pub/epel/testing/{}/Everything/x86_64/repodata".format( + MIRROR, version + ), + release + "-testing", + ), + ] else: sys.exit("Unknown product: {}".format(product)) + def main(): # Handle command-line arguments. parser = argparse.ArgumentParser( - description='Fetch SQL metadata databases of Fedora/EPEL repositories') + description="Fetch SQL metadata databases of Fedora/EPEL repositories" + ) parser.add_argument( - '--target-dir', dest='target_dir', action='store', required=True) + "--target-dir", dest="target_dir", action="store", required=True + ) args = parser.parse_args() # Get active releases from PDC. print("Fetching active releases from PDC...") r = requests.get( - "https://pdc.fedoraproject.org/rest_api/v1/product-versions/", - params={'active': 'true'}) - if (r.status_code != 200): - sys.exit("Failed to fetch active releases from PDC (request returned {})" - .format(r.status_code)) + "https://pdc.fedoraproject.org/rest_api/v1/product-versions/", + params={"active": "true"}, + ) + if r.status_code != 200: + sys.exit( + "Failed to fetch active releases from PDC (request returned {})".format( + r.status_code + ) + ) # Generate repository URLs. repositories = [] @@ -320,5 +384,6 @@ def main(): for repo in repositories: handle(repo, args.target_dir, db_removed) -if __name__ == '__main__': + +if __name__ == "__main__": main() diff --git a/bin/generate-html.py b/bin/generate-html.py index a501de4..2999b72 100755 --- a/bin/generate-html.py +++ b/bin/generate-html.py @@ -19,13 +19,18 @@ from datetime import date from jinja2 import Environment, PackageLoader -TEMPLATE_DIR='../templates' -DBS_DIR=os.environ.get('DB_DIR') or "repositories" -ASSETS_DIR='assets' -SCM_MAINTAINER_MAPPING=os.environ.get('MAINTAINER_MAPPING') or "pagure_owner_alias.json" -PRODUCT_VERSION_MAPPING=os.environ.get('PRODUCT_VERSION_MAPPING') or "product_version_mapping.json" -SITEMAP_URL = os.environ.get('SITEMAP_URL') or 'https://localhost:8080' -SEARCH_BACKEND = os.environ.get('SEARCH_BACKEND', False) +TEMPLATE_DIR = "../templates" +DBS_DIR = os.environ.get("DB_DIR") or "repositories" +ASSETS_DIR = "assets" +SCM_MAINTAINER_MAPPING = ( + os.environ.get("MAINTAINER_MAPPING") or "pagure_owner_alias.json" +) +PRODUCT_VERSION_MAPPING = ( + os.environ.get("PRODUCT_VERSION_MAPPING") or "product_version_mapping.json" +) +SITEMAP_URL = os.environ.get("SITEMAP_URL") or "https://localhost:8080" +SEARCH_BACKEND = os.environ.get("SEARCH_BACKEND", False) + class Package: def __init__(self, name): @@ -43,19 +48,20 @@ class Package: if name not in self.releases: self.releases[name] = {} - if 'branches' not in self.releases[name]: - self.releases[name]['branches'] = {} + if "branches" not in self.releases[name]: + self.releases[name]["branches"] = {} if branch not in self.releases[name]: - self.releases[name]['branches'][branch] = {} + self.releases[name]["branches"][branch] = {} - self.releases[name]['branches'][branch]['revision'] = revision - self.releases[name]['branches'][branch]['pkg_key'] = pkgKey - self.releases[name]['branches'][branch]['arch'] = arch - self.releases[name]['human_name'] = human_name or name + self.releases[name]["branches"][branch]["revision"] = revision + self.releases[name]["branches"][branch]["pkg_key"] = pkgKey + self.releases[name]["branches"][branch]["arch"] = arch + self.releases[name]["human_name"] = human_name or name def get_release(self, name): - return self.releases[name]['branches'] + return self.releases[name]["branches"] + def open_db(db): conn = sqlite3.connect(os.path.join(DBS_DIR, db)) @@ -64,8 +70,9 @@ def open_db(db): return (conn, c) + def clean_dir(path): - files = glob.glob(os.path.join(path, '*.html')) + files = glob.glob(os.path.join(path, "*.html")) for file in files: try: os.remove(file) @@ -73,34 +80,40 @@ def clean_dir(path): print("Error cleaning directory!") print(sys.exc_info()[0]) + def save_to(path, content): - with open(path, 'w') as fh: + with open(path, "w") as fh: fh.write(content) + def gen_file_array(dir_representation, data=None): if not data: data = [] for dir in sorted(dir_representation): if type(dir_representation[dir]) is str: - data.append({ "name": dir, "control": "file" }) + data.append({"name": dir, "control": "file"}) else: - data.append({ "name": dir, "control": "dir" }) + data.append({"name": dir, "control": "dir"}) data = gen_file_array(dir_representation[dir], data) if len(data) != 0: - data.append({ "control": "exit-list" }) + data.append({"control": "exit-list"}) return data + def do_regex(pattern, string): (result) = pattern.findall(string)[0] return result + def main(): # Handle command-line arguments. parser = argparse.ArgumentParser( - description='Generate static pages for Fedora packages') + description="Generate static pages for Fedora packages" + ) parser.add_argument( - '--target-dir', dest='target_dir', action='store', required=True) + "--target-dir", dest="target_dir", action="store", required=True + ) args = parser.parse_args() @@ -110,9 +123,8 @@ def main(): # Initialize templating system. env = Environment( - loader=PackageLoader('generate-html', TEMPLATE_DIR), - autoescape=True - ) + loader=PackageLoader("generate-html", TEMPLATE_DIR), autoescape=True + ) # Load maintainer mapping (imported from dist-git). # TODO: check that mapping exist / error. @@ -127,9 +139,11 @@ def main(): # Group databases files. databases = {} - db_pattern = re.compile('^(fedora|epel)-([\w|-]+)_(primary|filelists|other).sqlite$') + db_pattern = re.compile( + "^(fedora|epel)-([\w|-]+)_(primary|filelists|other).sqlite$" + ) for db in os.listdir(DBS_DIR): - if (not db_pattern.match(db)): + if not db_pattern.match(db): sys.exit("Invalid object in {}: {}".format(DBS_DIR, db)) (product, branch, db_type) = db_pattern.findall(db)[0] @@ -137,7 +151,7 @@ def main(): if release_branch in databases: databases[release_branch][db_type] = db else: - databases[release_branch] = { db_type: db } + databases[release_branch] = {db_type: db} # Build internal package metadata structure / cache. # { "src_pkg": { "subpackage": pkg, ... } } @@ -161,20 +175,22 @@ def main(): db_conns[release_branch] = { "filelist": filelist, "other": other, - "primary": primary + "primary": primary, } # Check if this db has a changes table - primary.execute("SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'changes'") + primary.execute( + "SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'changes'" + ) partial_update = primary.fetchone() is not None partial_update_packages = [] if partial_update: - primary.execute('SELECT name, rpm_sourcerpm_name FROM changes') + primary.execute("SELECT name, rpm_sourcerpm_name FROM changes") for row in primary.fetchall(): - partial_update_packages.append((row["rpm_sourcerpm_name"], row['name'])) + partial_update_packages.append((row["rpm_sourcerpm_name"], row["name"])) - for raw in primary.execute('SELECT * FROM packages'): + for raw in primary.execute("SELECT * FROM packages"): # Get source rpm name srpm_name = raw["rpm_sourcerpm_name"] @@ -224,11 +240,20 @@ def main(): if branch == "": branch = "base" - pkg.set_release(release, raw["pkgKey"], branch, raw["arch"], revision, release_mapping.get(release)) + pkg.set_release( + release, + raw["pkgKey"], + branch, + raw["arch"], + revision, + release_mapping.get(release), + ) # Get removed packages to determine if folder needs to be deleted later if partial_update: - for removed in primary.execute("SELECT name, rpm_sourcerpm_name FROM changes WHERE change = 'removed'"): + for removed in primary.execute( + "SELECT name, rpm_sourcerpm_name FROM changes WHERE change = 'removed'" + ): removed_packages.add((removed["rpm_sourcerpm_name"], removed["name"])) # If a package was removed and it was not in any repository, attempt to @@ -236,10 +261,18 @@ def main(): for removed_package in removed_packages: # If the source package is gone, delete all data if removed_package[0] not in packages: - shutil.rmtree(os.path.join(output_dir, 'pkgs', removed_package[0]), True) + shutil.rmtree(os.path.join(output_dir, "pkgs", removed_package[0]), True) # If only a subpackage of the source package is gone, then just delete that. - elif removed_package[0] in packages and removed_package[1] not in packages[removed_package[0]]: - shutil.rmtree(os.path.join(output_dir, 'pkgs', removed_package[0], removed_package[1]), True) + elif ( + removed_package[0] in packages + and removed_package[1] not in packages[removed_package[0]] + ): + shutil.rmtree( + os.path.join( + output_dir, "pkgs", removed_package[0], removed_package[1] + ), + True, + ) # Otherwise, a branch was removed but it's still in others so just update the package. # This isn't caught by above logic because release with changed data will not process a package that doesn't exist. elif packages[removed_package[0]][removed_package[1]].should_update == False: @@ -258,56 +291,68 @@ def main(): for pkg_name in packages[src_pkg]: tmp_pkg = packages[src_pkg][pkg_name] pkgs_list.append((tmp_pkg.source, tmp_pkg.name)) - prefix_index.setdefault(tmp_pkg.name[:2].lower(), []).append((tmp_pkg.source, tmp_pkg.name)) + prefix_index.setdefault(tmp_pkg.name[:2].lower(), []).append( + (tmp_pkg.source, tmp_pkg.name) + ) max_page_count = len(pkgs_list) # Sort the indexes prefix_index = dict(sorted(prefix_index.items())) for prefix_group in prefix_index: - prefix_index[prefix_group] = sorted(prefix_index[prefix_group], key=lambda x : x[1]) - - static_index_html = env.get_template('index-static.html.j2').render( - date=date.today().isoformat(), - package_count=max_page_count, - prefix_index=prefix_index) - save_to(os.path.join(output_dir, 'index-static.html'), static_index_html) - - search = env.get_template('index.html.j2') - search_html = search.render(date=date.today().isoformat(), - package_count=max_page_count, - search_backend=SEARCH_BACKEND) - save_to(os.path.join(output_dir, 'index.html'), search_html) - - index_tpl = env.get_template('index-prefix.html.j2') - index_dir = os.path.join(output_dir, 'index') + prefix_index[prefix_group] = sorted( + prefix_index[prefix_group], key=lambda x: x[1] + ) + + static_index_html = env.get_template("index-static.html.j2").render( + date=date.today().isoformat(), + package_count=max_page_count, + prefix_index=prefix_index, + ) + save_to(os.path.join(output_dir, "index-static.html"), static_index_html) + + search = env.get_template("index.html.j2") + search_html = search.render( + date=date.today().isoformat(), + package_count=max_page_count, + search_backend=SEARCH_BACKEND, + ) + save_to(os.path.join(output_dir, "index.html"), search_html) + + index_tpl = env.get_template("index-prefix.html.j2") + index_dir = os.path.join(output_dir, "index") os.makedirs(index_dir, exist_ok=True) for prefix, names in prefix_index.items(): - html = index_tpl.render(prefix=prefix, packages=names, - search_backend=SEARCH_BACKEND) - save_to(os.path.join(index_dir, f'{prefix}.html'), html) + html = index_tpl.render( + prefix=prefix, packages=names, search_backend=SEARCH_BACKEND + ) + save_to(os.path.join(index_dir, f"{prefix}.html"), html) # Generate sitemaps sitemap_list = [] - sitemap_dir = os.path.join(output_dir, 'sitemaps') + sitemap_dir = os.path.join(output_dir, "sitemaps") shutil.rmtree(sitemap_dir, ignore_errors=True) os.makedirs(sitemap_dir, exist_ok=True) i = 0 # Number of pkgs in one sitemap. Should not be above 50,000 # https://www.sitemaps.org/protocol.html#index sitemap_amount = 10000 - sitemap_pkgs = pkgs_list[i * sitemap_amount:(i + 1) * sitemap_amount] + sitemap_pkgs = pkgs_list[i * sitemap_amount : (i + 1) * sitemap_amount] while sitemap_pkgs: - crawler_sitemap = env.get_template('sitemap.xml.j2') - crawler_sitemap_xml = crawler_sitemap.render(packages=sitemap_pkgs, url=SITEMAP_URL) - save_to(os.path.join(sitemap_dir, 'sitemap{}.xml'.format(i)), crawler_sitemap_xml) - sitemap_list.append('/sitemaps/sitemap{}.xml'.format(i)) + crawler_sitemap = env.get_template("sitemap.xml.j2") + crawler_sitemap_xml = crawler_sitemap.render( + packages=sitemap_pkgs, url=SITEMAP_URL + ) + save_to( + os.path.join(sitemap_dir, "sitemap{}.xml".format(i)), crawler_sitemap_xml + ) + sitemap_list.append("/sitemaps/sitemap{}.xml".format(i)) i = i + 1 - sitemap_pkgs = pkgs_list[i * sitemap_amount:(i + 1) * sitemap_amount] + sitemap_pkgs = pkgs_list[i * sitemap_amount : (i + 1) * sitemap_amount] - sitemap_sitemap = env.get_template('sitemap-index.xml.j2') + sitemap_sitemap = env.get_template("sitemap-index.xml.j2") sitemap_sitemap_xml = sitemap_sitemap.render(sitemaps=sitemap_list, url=SITEMAP_URL) - save_to(os.path.join(output_dir, 'sitemap.xml'), sitemap_sitemap_xml) + save_to(os.path.join(output_dir, "sitemap.xml"), sitemap_sitemap_xml) # Generate package pages from Rawhide. print("> Generating package pages...") @@ -316,7 +361,7 @@ def main(): # Generate package index and version pages for src_pkg in packages: - src_dir = os.path.join(output_dir, 'pkgs', src_pkg) + src_dir = os.path.join(output_dir, "pkgs", src_pkg) related_pkg_list = [] should_update_src = False for pkg in packages[src_pkg]: @@ -330,8 +375,10 @@ def main(): os.makedirs(src_dir, exist_ok=True) source_package_index = env.get_template("source-package.html.j2") - source_package_index_html = source_package_index.render(name=src_pkg, children=packages[src_pkg], search_backend=SEARCH_BACKEND) - save_to(os.path.join(src_dir, 'index.html'), source_package_index_html) + source_package_index_html = source_package_index.render( + name=src_pkg, children=packages[src_pkg], search_backend=SEARCH_BACKEND + ) + save_to(os.path.join(src_dir, "index.html"), source_package_index_html) # Process subpackages for pkg in packages[src_pkg].values(): @@ -341,15 +388,19 @@ def main(): clean_dir(pkg_dir) os.makedirs(pkg_dir, exist_ok=True) - html_path = os.path.join(pkg_dir, 'index.html') - html_template = env.get_template('package.html.j2') - html_content = html_template.render(pkg=pkg, related_pkgs=related_pkg_list, search_backend=SEARCH_BACKEND) + html_path = os.path.join(pkg_dir, "index.html") + html_template = env.get_template("package.html.j2") + html_content = html_template.render( + pkg=pkg, related_pkgs=related_pkg_list, search_backend=SEARCH_BACKEND + ) save_to(html_path, html_content) # Simple way to display progress. page_count += 1 - if (page_count % 100 == 0 or page_count == max_page_count): - print("Processed {}/{} package pages.".format(page_count, max_page_count)) + if page_count % 100 == 0 or page_count == max_page_count: + print( + "Processed {}/{} package pages.".format(page_count, max_page_count) + ) for release in pkg.releases.keys(): for branch in pkg.get_release(release).keys(): @@ -358,8 +409,8 @@ def main(): else: release_branch = "{}-{}".format(release, branch) - pkg_key = pkg.get_release(release)[branch]['pkg_key'] - revision = pkg.get_release(release)[branch]['revision'] + pkg_key = pkg.get_release(release)[branch]["pkg_key"] + revision = pkg.get_release(release)[branch]["revision"] filelist = db_conns[release_branch]["filelist"] other = db_conns[release_branch]["other"] @@ -368,25 +419,27 @@ def main(): # Generate files page for pkg. # Create a nested object to represent the file tree files = {} - for entry in filelist.execute('SELECT * FROM filelist WHERE pkgKey = ?', (pkg_key,)): - filenames = entry["filenames"].split('/') + for entry in filelist.execute( + "SELECT * FROM filelist WHERE pkgKey = ?", (pkg_key,) + ): + filenames = entry["filenames"].split("/") filetype_index = 0 for filename in filenames: try: filetype = entry["filetypes"][filetype_index] except Exception: - filetype = '?' + filetype = "?" current = files - for dir in entry["dirname"].split('/'): + for dir in entry["dirname"].split("/"): if dir != "": if dir not in current or type(current[dir]) == str: current[dir] = {} current = current[dir] - if filetype == 'd' and not filename in current: + if filetype == "d" and not filename in current: current[filename] = {} - elif filetype != 'd': + elif filetype != "d": current[filename] = filetype filetype_index += 1 # Flatten and sort the files structure for jinja @@ -394,25 +447,36 @@ def main(): # Generate changelog page for pkg. changelog = [] - for change in other.execute('SELECT * FROM changelog WHERE pkgKey = ?', (pkg_key,)): + for change in other.execute( + "SELECT * FROM changelog WHERE pkgKey = ?", (pkg_key,) + ): # Make addresses less obvious to spot for spam bots. author = change["author"] if changelog_mail_pattern.search(change["author"]): addr = changelog_mail_pattern.findall(change["author"])[0] - obfuscated_addr = addr.replace('@', ' at ').replace('.', ' dot ').replace('-', ' dash ') + obfuscated_addr = ( + addr.replace("@", " at ") + .replace(".", " dot ") + .replace("-", " dash ") + ) author = author.replace(addr, obfuscated_addr) - changelog += [{ - "author": author, - "timestamp": change["date"], - "date": date.fromtimestamp(change["date"]), - "change": change["changelog"] - }] + changelog += [ + { + "author": author, + "timestamp": change["date"], + "date": date.fromtimestamp(change["date"]), + "change": change["changelog"], + } + ] # Generate provides list for pkg. provides = [] try: - for provide in primary.execute('SELECT name FROM provides where pkgkey = ? GROUP BY name', (pkg_key,)): + for provide in primary.execute( + "SELECT name FROM provides where pkgkey = ? GROUP BY name", + (pkg_key,), + ): provides.append(provide["name"]) except Exception as e: print(e) @@ -424,19 +488,24 @@ def main(): for row in rows: print(row[0], row[1], row[2]) - primary.execute("SELECT name FROM sqlite_master WHERE type='table';") + primary.execute( + "SELECT name FROM sqlite_master WHERE type='table';" + ) print(primary.fetchall()) sys.exit(1) # Generate dependencies for pkg requires = [] - for require in primary.execute(""" + for require in primary.execute( + """ SELECT requires.flags, requires.version, requires.release, packages.rpm_sourcerpm_name, packages.name AS provides FROM requires INNER JOIN provides ON requires.name=provides.name INNER JOIN packages ON provides.pkgkey=packages.pkgkey WHERE requires.pkgkey = ? GROUP BY packages.name - """, (pkg_key,)): + """, + (pkg_key,), + ): flags = "" if require["flags"] == "EQ": flags = "=" @@ -449,15 +518,36 @@ def main(): elif require["flags"] == "LT": flags = "<" require_srpm_name = require["rpm_sourcerpm_name"] - requires.append({ "requirement": require["provides"], "flags": flags, "version": require["version"], "release": require["release"], "can_link": bool(packages[require_srpm_name].get(require["provides"])), "srpm_name": require_srpm_name }) + requires.append( + { + "requirement": require["provides"], + "flags": flags, + "version": require["version"], + "release": require["release"], + "can_link": bool( + packages[require_srpm_name].get(require["provides"]) + ), + "srpm_name": require_srpm_name, + } + ) html_path = os.path.join(pkg_dir, release_branch + ".html") html_template = env.get_template("package-details.html.j2") - html_content = html_template.render(pkg=pkg, release=release, branch=branch, changelog=changelog, files=files, provides=provides, requires=requires, search_backend=SEARCH_BACKEND) + html_content = html_template.render( + pkg=pkg, + release=release, + branch=branch, + changelog=changelog, + files=files, + provides=provides, + requires=requires, + search_backend=SEARCH_BACKEND, + ) save_to(html_path, html_content) print("DONE.") print("> {} packages processed.".format(page_count)) -if __name__ == '__main__': + +if __name__ == "__main__": main() diff --git a/bin/get-product-names.py b/bin/get-product-names.py index 8237896..7f0d388 100755 --- a/bin/get-product-names.py +++ b/bin/get-product-names.py @@ -7,14 +7,18 @@ from typing import final import requests import json -PRODUCT_VERSION_MAPPING=os.environ.get('PRODUCT_VERSION_MAPPING') or "product_version_mapping.json" -PDC_URI="https://pdc.fedoraproject.org/rest_api/v1/product-versions" +PRODUCT_VERSION_MAPPING = ( + os.environ.get("PRODUCT_VERSION_MAPPING") or "product_version_mapping.json" +) +PDC_URI = "https://pdc.fedoraproject.org/rest_api/v1/product-versions" + def get_name(name): name = name.title() name = name.replace("Epel", "EPEL") return name + def get_data(URI, previous_data=None): formatted_data = previous_data or {} raw_data = requests.get(URI).json() @@ -28,10 +32,12 @@ def get_data(URI, previous_data=None): else: return formatted_data + def main(): final_data = get_data(PDC_URI) - with open(PRODUCT_VERSION_MAPPING, 'w') as outfile: + with open(PRODUCT_VERSION_MAPPING, "w") as outfile: json.dump(final_data, outfile) -if __name__ == '__main__': + +if __name__ == "__main__": main() diff --git a/bin/search-uwsgi.py b/bin/search-uwsgi.py index d2f9890..ab95ff3 100755 --- a/bin/search-uwsgi.py +++ b/bin/search-uwsgi.py @@ -9,26 +9,25 @@ from sys import exc_info from requests import get from copy import deepcopy -SOLR_URL=environ.get('SOLR_URL') -SOLR_CORE=environ.get('SOLR_CORE') -TEMPLATE_DIR='../templates' +SOLR_URL = environ.get("SOLR_URL") +SOLR_CORE = environ.get("SOLR_CORE") +TEMPLATE_DIR = "../templates" + +env = Environment(loader=PackageLoader("search-uwsgi", TEMPLATE_DIR), autoescape=True) +search_results = env.get_template("search-results.html.j2") -env = Environment( - loader=PackageLoader('search-uwsgi', TEMPLATE_DIR), - autoescape=True) -search_results = env.get_template('search-results.html.j2') def application(params, start_response): - query_str = params.get('QUERY_STRING') + query_str = params.get("QUERY_STRING") if query_str == None: - start_response('500 Internal Server Error', [('Content-Type','text/html')]) - return [b'Error: No query string'] + start_response("500 Internal Server Error", [("Content-Type", "text/html")]) + return [b"Error: No query string"] d = parse_qs(query_str) - query = d.get('query', [''])[0] + query = d.get("query", [""])[0] try: - start = d.get('start', [0])[0] + start = d.get("start", [0])[0] start = int(start) except: start = 0 @@ -42,28 +41,33 @@ def application(params, start_response): "start": start, "q": query, "qf": "name^2 srcName^1.5 summary^0.75", - "fq": [] + "fq": [], } if not d.get("show_related", False): query_params["fq"].append("{!collapse field=srcName_string}") - for release in d.get('releases', []): - query_params["fq"].append(f"releases:\"{release}\"") + for release in d.get("releases", []): + query_params["fq"].append(f'releases:"{release}"') - query_res = get(f"{SOLR_URL}solr/{SOLR_CORE}/select?{urlencode(query_params, True)}") + query_res = get( + f"{SOLR_URL}solr/{SOLR_CORE}/select?{urlencode(query_params, True)}" + ) except: print("Solr request error: ", str(exc_info()[0])) - start_response('500 Internal Server Error', [('Content-Type','text/html')]) - return [b'Error communicating with Solr'] + start_response("500 Internal Server Error", [("Content-Type", "text/html")]) + return [b"Error communicating with Solr"] if query_res.ok: - results_html = search_results.render(results=query_res.json(), qdict=d, modify_query=modify_query) - start_response('200 OK', [('Content-Type','text/html')]) - return [bytes(results_html, 'utf-8')] + results_html = search_results.render( + results=query_res.json(), qdict=d, modify_query=modify_query + ) + start_response("200 OK", [("Content-Type", "text/html")]) + return [bytes(results_html, "utf-8")] else: - start_response('500 Internal Server Error', [('Content-Type','text/html')]) - return [b'Solr query error'] + start_response("500 Internal Server Error", [("Content-Type", "text/html")]) + return [b"Solr query error"] + def modify_query(qdict, **new_values): finaldict = deepcopy(qdict) diff --git a/bin/update-solr.py b/bin/update-solr.py index c3aa728..7c67bd1 100755 --- a/bin/update-solr.py +++ b/bin/update-solr.py @@ -16,14 +16,19 @@ import time # This is used to encode xml, not parse it. Security warning is irrelevant. # defusedxml does not have an Element import and defuse_stdlib() is called anyway for caution's sake. -from xml.etree.ElementTree import Element, tostring # nosec +from xml.etree.ElementTree import Element, tostring # nosec + +SOLR_URL = os.environ.get("SOLR_URL") +SOLR_CORE = os.environ.get("SOLR_CORE") +SOLR_CONF_SET = "packages" +DBS_DIR = os.environ.get("DB_DIR") or "repositories" +SCM_MAINTAINER_MAPPING = ( + os.environ.get("MAINTAINER_MAPPING") or "pagure_owner_alias.json" +) +PRODUCT_VERSION_MAPPING = ( + os.environ.get("PRODUCT_VERSION_MAPPING") or "product_version_mapping.json" +) -SOLR_URL=os.environ.get('SOLR_URL') -SOLR_CORE=os.environ.get('SOLR_CORE') -SOLR_CONF_SET="packages" -DBS_DIR=os.environ.get('DB_DIR') or "repositories" -SCM_MAINTAINER_MAPPING=os.environ.get('MAINTAINER_MAPPING') or "pagure_owner_alias.json" -PRODUCT_VERSION_MAPPING=os.environ.get('PRODUCT_VERSION_MAPPING') or "product_version_mapping.json" class Package: def __init__(self, name): @@ -41,19 +46,20 @@ class Package: if name not in self.releases: self.releases[name] = {} - if 'branches' not in self.releases[name]: - self.releases[name]['branches'] = {} + if "branches" not in self.releases[name]: + self.releases[name]["branches"] = {} if branch not in self.releases[name]: - self.releases[name]['branches'][branch] = {} + self.releases[name]["branches"][branch] = {} - self.releases[name]['branches'][branch]['revision'] = revision - self.releases[name]['branches'][branch]['pkg_key'] = pkgKey - self.releases[name]['branches'][branch]['arch'] = arch - self.releases[name]['human_name'] = human_name or name + self.releases[name]["branches"][branch]["revision"] = revision + self.releases[name]["branches"][branch]["pkg_key"] = pkgKey + self.releases[name]["branches"][branch]["arch"] = arch + self.releases[name]["human_name"] = human_name or name def get_release(self, name): - return self.releases[name]['branches'] + return self.releases[name]["branches"] + def open_db(db): conn = sqlite3.connect(os.path.join(DBS_DIR, db)) @@ -62,10 +68,12 @@ def open_db(db): return (conn, c) + def do_regex(pattern, string): (result) = pattern.findall(string)[0] return result + def main(): # Load maintainer mapping (imported from dist-git). # TODO: check that mapping exist / error. @@ -80,9 +88,11 @@ def main(): # Group databases files. databases = {} - db_pattern = re.compile('^(fedora|epel)-([\w|-]+)_(primary|filelists|other).sqlite$') + db_pattern = re.compile( + "^(fedora|epel)-([\w|-]+)_(primary|filelists|other).sqlite$" + ) for db in os.listdir(DBS_DIR): - if (not db_pattern.match(db)): + if not db_pattern.match(db): sys.exit("Invalid object in {}: {}".format(DBS_DIR, db)) (product, branch, db_type) = db_pattern.findall(db)[0] @@ -90,7 +100,7 @@ def main(): if release_branch in databases: databases[release_branch][db_type] = db else: - databases[release_branch] = { db_type: db } + databases[release_branch] = {db_type: db} # Build internal package metadata structure / cache. # { "src_pkg": { "subpackage": pkg, ... } } @@ -110,7 +120,7 @@ def main(): (_, filelist) = open_db(databases[release_branch]["filelists"]) (_, other) = open_db(databases[release_branch]["other"]) - for raw in primary.execute('SELECT * FROM packages'): + for raw in primary.execute("SELECT * FROM packages"): # Get source rpm name srpm_name = do_regex(srpm_pattern, raw["rpm_sourcerpm"]) @@ -150,7 +160,14 @@ def main(): if branch == "": branch = "base" - pkg.set_release(release, raw["pkgKey"], branch, raw["arch"], revision, release_mapping.get(release)) + pkg.set_release( + release, + raw["pkgKey"], + branch, + raw["arch"], + revision, + release_mapping.get(release), + ) print(">>> {} packages have been extracted.".format(packages_count)) @@ -158,62 +175,75 @@ def main(): # Create a tmp solr index tmp_idx = f"solr_{int(time.time())}" - req = requests.get(f"{SOLR_URL}solr/admin/cores?action=CREATE&name={tmp_idx}&instanceDir=/var/solr/data/{tmp_idx}&configSet={SOLR_CONF_SET}") + req = requests.get( + f"{SOLR_URL}solr/admin/cores?action=CREATE&name={tmp_idx}&instanceDir=/var/solr/data/{tmp_idx}&configSet={SOLR_CONF_SET}" + ) req.raise_for_status() # Start submitting to the index - pkg_xml = Element('add') + pkg_xml = Element("add") pkg_count = 0 max_pkg_count = packages_count for src_pkg in packages.values(): for pkg in src_pkg.values(): # Submit packages to Solr index, 500 at a time. - pkg_el = Element('doc') + pkg_el = Element("doc") - pkg_el_id = Element('field', { "name": "name" }) + pkg_el_id = Element("field", {"name": "name"}) pkg_el_id.text = pkg.name pkg_el.append(pkg_el_id) - pkg_el_src_name = Element('field', { "name": "srcName" }) + pkg_el_src_name = Element("field", {"name": "srcName"}) pkg_el_src_name.text = pkg.source pkg_el.append(pkg_el_src_name) - pkg_el_summary = Element('field', { "name": "summary" }) + pkg_el_summary = Element("field", {"name": "summary"}) pkg_el_summary.text = pkg.summary pkg_el.append(pkg_el_summary) for release in pkg.releases.values(): - pkg_el_release = Element('field', { "name": "releases" }) - pkg_el_release.text = release['human_name'] + pkg_el_release = Element("field", {"name": "releases"}) + pkg_el_release.text = release["human_name"] pkg_el.append(pkg_el_release) pkg_xml.append(pkg_el) pkg_count += 1 - if (pkg_count % 500 == 0 or pkg_count == max_pkg_count): - req = requests.post(f"{SOLR_URL}solr/{tmp_idx}/update?commit={str(pkg_count == max_pkg_count).lower()}&update.chain=uuid", data=tostring(pkg_xml), headers={'Content-Type': 'application/xml'}) + if pkg_count % 500 == 0 or pkg_count == max_pkg_count: + req = requests.post( + f"{SOLR_URL}solr/{tmp_idx}/update?commit={str(pkg_count == max_pkg_count).lower()}&update.chain=uuid", + data=tostring(pkg_xml), + headers={"Content-Type": "application/xml"}, + ) req.raise_for_status() pkg_xml.clear() # print("Submitted {}/{} packages.".format(pkg_count, max_pkg_count)) - + # Create default core if it does not exist req = requests.get(f"{SOLR_URL}solr/admin/cores?action=STATUS&core={SOLR_CORE}") status = req.json() if len(status["status"]["packages"]) == 0: - requests.get(f"{SOLR_URL}solr/admin/cores?action=CREATE&name={SOLR_CORE}&instanceDir=/var/solr/data/{SOLR_CORE}&configSet={SOLR_CONF_SET}") + requests.get( + f"{SOLR_URL}solr/admin/cores?action=CREATE&name={SOLR_CORE}&instanceDir=/var/solr/data/{SOLR_CORE}&configSet={SOLR_CONF_SET}" + ) # Swap production core with our temporary core - req = requests.get(f"{SOLR_URL}solr/admin/cores?action=SWAP&core={tmp_idx}&other={SOLR_CORE}") + req = requests.get( + f"{SOLR_URL}solr/admin/cores?action=SWAP&core={tmp_idx}&other={SOLR_CORE}" + ) req.raise_for_status() # Delete the old core that was swapped out - req = requests.get(f"{SOLR_URL}solr/admin/cores?action=UNLOAD&core={tmp_idx}&deleteInstanceDir=true&deleteDataDir=true&deleteIndex=true") + req = requests.get( + f"{SOLR_URL}solr/admin/cores?action=UNLOAD&core={tmp_idx}&deleteInstanceDir=true&deleteDataDir=true&deleteIndex=true" + ) req.raise_for_status() print("DONE.") print("> {} packages submitted to solr.".format(packages_count)) -if __name__ == '__main__': + +if __name__ == "__main__": defusedxml.defuse_stdlib() main() From 035f7b454df31f4ad7497f528d8bc82e55c10596 Mon Sep 17 00:00:00 2001 From: Brendan Early Date: Oct 02 2021 20:57:17 +0000 Subject: [PATCH 2/2] Setup azure pipelines CI --- diff --git a/.pylintrc b/.pylintrc new file mode 100644 index 0000000..8248184 --- /dev/null +++ b/.pylintrc @@ -0,0 +1,593 @@ +[MASTER] + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code. +extension-pkg-whitelist= + +# Specify a score threshold to be exceeded before program exits with error. +fail-under=10.0 + +# Add files or directories to the blacklist. They should be base names, not +# paths. +ignore=CVS + +# Add files or directories matching the regex patterns to the blacklist. The +# regex matches against base names, not paths. +ignore-patterns= + +# Python code to execute, usually for sys.path manipulation such as +# pygtk.require(). +#init-hook= + +# Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the +# number of processors available to use. +jobs=1 + +# Control the amount of potential inferred values when inferring a single +# object. This can help the performance when dealing with large functions or +# complex, nested conditions. +limit-inference-results=100 + +# List of plugins (as comma separated values of python module names) to load, +# usually to register additional checkers. +load-plugins= + +# Pickle collected data for later comparisons. +persistent=yes + +# When enabled, pylint would attempt to guess common misconfiguration and emit +# user-friendly hints instead of false-positive error messages. +suggestion-mode=yes + +# Allow loading of arbitrary C extensions. Extensions are imported into the +# active Python interpreter and may run arbitrary code. +unsafe-load-any-extension=no + + +[MESSAGES CONTROL] + +# Only show warnings with the listed confidence levels. Leave empty to show +# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED. +confidence= + +# Disable the message, report, category or checker with the given id(s). You +# can either give multiple identifiers separated by comma (,) or put this +# option multiple times (only on the command line, not in the configuration +# file where it should appear only once). You can also use "--disable=all" to +# disable everything first and then reenable specific checks. For example, if +# you want to run only the similarities checker, you can use "--disable=all +# --enable=similarities". If you want to run only the classes checker, but have +# no Warning level messages displayed, use "--disable=all --enable=classes +# --disable=W". +disable=print-statement, + parameter-unpacking, + unpacking-in-except, + old-raise-syntax, + backtick, + long-suffix, + old-ne-operator, + old-octal-literal, + import-star-module-level, + non-ascii-bytes-literal, + raw-checker-failed, + bad-inline-option, + locally-disabled, + file-ignored, + suppressed-message, + useless-suppression, + deprecated-pragma, + use-symbolic-message-instead, + apply-builtin, + basestring-builtin, + buffer-builtin, + cmp-builtin, + coerce-builtin, + execfile-builtin, + file-builtin, + long-builtin, + raw_input-builtin, + reduce-builtin, + standarderror-builtin, + unicode-builtin, + xrange-builtin, + coerce-method, + delslice-method, + getslice-method, + setslice-method, + no-absolute-import, + old-division, + dict-iter-method, + dict-view-method, + next-method-called, + metaclass-assignment, + indexing-exception, + raising-string, + reload-builtin, + oct-method, + hex-method, + nonzero-method, + cmp-method, + input-builtin, + round-builtin, + intern-builtin, + unichr-builtin, + map-builtin-not-iterating, + zip-builtin-not-iterating, + range-builtin-not-iterating, + filter-builtin-not-iterating, + using-cmp-argument, + eq-without-hash, + div-method, + idiv-method, + rdiv-method, + exception-message-attribute, + invalid-str-codec, + sys-max-int, + bad-python3-import, + deprecated-string-function, + deprecated-str-translate-call, + deprecated-itertools-function, + deprecated-types-field, + next-method-defined, + dict-items-not-iterating, + dict-keys-not-iterating, + dict-values-not-iterating, + deprecated-operator-function, + deprecated-urllib-function, + xreadlines-attribute, + deprecated-sys-function, + exception-escape, + comprehension-escape + +# Enable the message, report, category or checker with the given id(s). You can +# either give multiple identifier separated by comma (,) or put this option +# multiple time (only on the command line, not in the configuration file where +# it should appear only once). See also the "--disable" option for examples. +enable=c-extension-no-member + + +[REPORTS] + +# Python expression which should return a score less than or equal to 10. You +# have access to the variables 'error', 'warning', 'refactor', and 'convention' +# which contain the number of messages in each category, as well as 'statement' +# which is the total number of statements analyzed. This score is used by the +# global evaluation report (RP0004). +evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) + +# Template used to display messages. This is a python new-style format string +# used to format the message information. See doc for all details. +#msg-template= + +# Set the output format. Available formats are text, parseable, colorized, json +# and msvs (visual studio). You can also give a reporter class, e.g. +# mypackage.mymodule.MyReporterClass. +output-format=colorized + +# Tells whether to display a full report or only the messages. +reports=no + +# Activate the evaluation score. +score=yes + + +[REFACTORING] + +# Maximum number of nested blocks for function / method body +max-nested-blocks=5 + +# Complete name of functions that never returns. When checking for +# inconsistent-return-statements if a never returning function is called then +# it will be considered as an explicit return statement and no message will be +# printed. +never-returning-functions=sys.exit + + +[BASIC] + +# Naming style matching correct argument names. +argument-naming-style=snake_case + +# Regular expression matching correct argument names. Overrides argument- +# naming-style. +#argument-rgx= + +# Naming style matching correct attribute names. +attr-naming-style=snake_case + +# Regular expression matching correct attribute names. Overrides attr-naming- +# style. +#attr-rgx= + +# Bad variable names which should always be refused, separated by a comma. +bad-names=foo, + bar, + baz, + toto, + tutu, + tata + +# Bad variable names regexes, separated by a comma. If names match any regex, +# they will always be refused +bad-names-rgxs= + +# Naming style matching correct class attribute names. +class-attribute-naming-style=any + +# Regular expression matching correct class attribute names. Overrides class- +# attribute-naming-style. +#class-attribute-rgx= + +# Naming style matching correct class names. +class-naming-style=PascalCase + +# Regular expression matching correct class names. Overrides class-naming- +# style. +#class-rgx= + +# Naming style matching correct constant names. +const-naming-style=UPPER_CASE + +# Regular expression matching correct constant names. Overrides const-naming- +# style. +#const-rgx= + +# Minimum line length for functions/classes that require docstrings, shorter +# ones are exempt. +docstring-min-length=-1 + +# Naming style matching correct function names. +function-naming-style=snake_case + +# Regular expression matching correct function names. Overrides function- +# naming-style. +#function-rgx= + +# Good variable names which should always be accepted, separated by a comma. +good-names=i, + j, + k, + ex, + Run, + _ + +# Good variable names regexes, separated by a comma. If names match any regex, +# they will always be accepted +good-names-rgxs= + +# Include a hint for the correct naming format with invalid-name. +include-naming-hint=no + +# Naming style matching correct inline iteration names. +inlinevar-naming-style=any + +# Regular expression matching correct inline iteration names. Overrides +# inlinevar-naming-style. +#inlinevar-rgx= + +# Naming style matching correct method names. +method-naming-style=snake_case + +# Regular expression matching correct method names. Overrides method-naming- +# style. +#method-rgx= + +# Naming style matching correct module names. +module-naming-style=snake_case + +# Regular expression matching correct module names. Overrides module-naming- +# style. +#module-rgx= + +# Colon-delimited sets of names that determine each other's naming style when +# the name regexes allow several styles. +name-group= + +# Regular expression which should only match function or class names that do +# not require a docstring. +no-docstring-rgx=^_ + +# List of decorators that produce properties, such as abc.abstractproperty. Add +# to this list to register other decorators that produce valid properties. +# These decorators are taken in consideration only for invalid-name. +property-classes=abc.abstractproperty + +# Naming style matching correct variable names. +variable-naming-style=snake_case + +# Regular expression matching correct variable names. Overrides variable- +# naming-style. +#variable-rgx= + + +[FORMAT] + +# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. +expected-line-ending-format= + +# Regexp for a line that is allowed to be longer than the limit. +ignore-long-lines=^\s*(# )??$ + +# Number of spaces of indent required inside a hanging or continued line. +indent-after-paren=4 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 +# tab). +indent-string=' ' + +# Maximum number of characters on a single line. +max-line-length=100 + +# Maximum number of lines in a module. +max-module-lines=1000 + +# Allow the body of a class to be on the same line as the declaration if body +# contains single statement. +single-line-class-stmt=no + +# Allow the body of an if to be on the same line as the test if there is no +# else. +single-line-if-stmt=no + + +[LOGGING] + +# The type of string formatting that logging methods do. `old` means using % +# formatting, `new` is for `{}` formatting. +logging-format-style=old + +# Logging modules to check that the string format arguments are in logging +# function parameter format. +logging-modules=logging + + +[MISCELLANEOUS] + +# List of note tags to take in consideration, separated by a comma. +notes=FIXME, + XXX, + TODO + +# Regular expression of note tags to take in consideration. +#notes-rgx= + + +[SIMILARITIES] + +# Ignore comments when computing similarities. +ignore-comments=yes + +# Ignore docstrings when computing similarities. +ignore-docstrings=yes + +# Ignore imports when computing similarities. +ignore-imports=no + +# Minimum lines number of a similarity. +min-similarity-lines=4 + + +[SPELLING] + +# Limits count of emitted suggestions for spelling mistakes. +max-spelling-suggestions=4 + +# Spelling dictionary name. Available dictionaries: en_AG (hunspell), en_AU +# (hunspell), en_BS (hunspell), en_BW (hunspell), en_BZ (hunspell), en_CA +# (hunspell), en_DK (hunspell), en_GB (hunspell), en_GH (hunspell), en_HK +# (hunspell), en_IE (hunspell), en_IN (hunspell), en_JM (hunspell), en_MW +# (hunspell), en_NA (hunspell), en_NG (hunspell), en_NZ (hunspell), en_PH +# (hunspell), en_SG (hunspell), en_TT (hunspell), en_US (hunspell), en_ZA +# (hunspell), en_ZM (hunspell), en_ZW (hunspell). +spelling-dict= + +# List of comma separated words that should not be checked. +spelling-ignore-words= + +# A path to a file that contains the private dictionary; one word per line. +spelling-private-dict-file= + +# Tells whether to store unknown words to the private dictionary (see the +# --spelling-private-dict-file option) instead of raising a message. +spelling-store-unknown-words=no + + +[STRING] + +# This flag controls whether inconsistent-quotes generates a warning when the +# character used as a quote delimiter is used inconsistently within a module. +check-quote-consistency=no + +# This flag controls whether the implicit-str-concat should generate a warning +# on implicit string concatenation in sequences defined over several lines. +check-str-concat-over-line-jumps=no + + +[TYPECHECK] + +# List of decorators that produce context managers, such as +# contextlib.contextmanager. Add to this list to register other decorators that +# produce valid context managers. +contextmanager-decorators=contextlib.contextmanager + +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E1101 when accessed. Python regular +# expressions are accepted. +generated-members= + +# Tells whether missing members accessed in mixin class should be ignored. A +# mixin class is detected if its name ends with "mixin" (case insensitive). +ignore-mixin-members=yes + +# Tells whether to warn about missing members when the owner of the attribute +# is inferred to be None. +ignore-none=yes + +# This flag controls whether pylint should warn about no-member and similar +# checks whenever an opaque object is returned when inferring. The inference +# can return multiple potential results while evaluating a Python object, but +# some branches might not be evaluated, which results in partial inference. In +# that case, it might be useful to still emit no-member and other checks for +# the rest of the inferred objects. +ignore-on-opaque-inference=yes + +# List of class names for which member attributes should not be checked (useful +# for classes with dynamically set attributes). This supports the use of +# qualified names. +ignored-classes=optparse.Values,thread._local,_thread._local + +# List of module names for which member attributes should not be checked +# (useful for modules/projects where namespaces are manipulated during runtime +# and thus existing member attributes cannot be deduced by static analysis). It +# supports qualified module names, as well as Unix pattern matching. +ignored-modules= + +# Show a hint with possible names when a member name was not found. The aspect +# of finding the hint is based on edit distance. +missing-member-hint=yes + +# The minimum edit distance a name should have in order to be considered a +# similar match for a missing member name. +missing-member-hint-distance=1 + +# The total number of similar names that should be taken in consideration when +# showing a hint for a missing member. +missing-member-max-choices=1 + +# List of decorators that change the signature of a decorated function. +signature-mutators= + + +[VARIABLES] + +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid defining new builtins when possible. +additional-builtins= + +# Tells whether unused global variables should be treated as a violation. +allow-global-unused-variables=yes + +# List of strings which can identify a callback function by name. A callback +# name must start or end with one of those strings. +callbacks=cb_, + _cb + +# A regular expression matching the name of dummy variables (i.e. expected to +# not be used). +dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ + +# Argument names that match this expression will be ignored. Default to name +# with leading underscore. +ignored-argument-names=_.*|^ignored_|^unused_ + +# Tells whether we should check for unused import in __init__ files. +init-import=no + +# List of qualified module names which can have objects that can redefine +# builtins. +redefining-builtins-modules=six.moves,past.builtins,future.builtins,builtins,io + + +[CLASSES] + +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods=__init__, + __new__, + setUp, + __post_init__ + +# List of member names, which should be excluded from the protected access +# warning. +exclude-protected=_asdict, + _fields, + _replace, + _source, + _make + +# List of valid names for the first argument in a class method. +valid-classmethod-first-arg=cls + +# List of valid names for the first argument in a metaclass class method. +valid-metaclass-classmethod-first-arg=cls + + +[DESIGN] + +# Maximum number of arguments for function / method. +max-args=5 + +# Maximum number of attributes for a class (see R0902). +max-attributes=7 + +# Maximum number of boolean expressions in an if statement (see R0916). +max-bool-expr=5 + +# Maximum number of branch for function / method body. +max-branches=12 + +# Maximum number of locals for function / method body. +max-locals=15 + +# Maximum number of parents for a class (see R0901). +max-parents=7 + +# Maximum number of public methods for a class (see R0904). +max-public-methods=20 + +# Maximum number of return / yield for function / method body. +max-returns=6 + +# Maximum number of statements in function / method body. +max-statements=50 + +# Minimum number of public methods for a class (see R0903). +min-public-methods=2 + + +[IMPORTS] + +# List of modules that can be imported at any level, not just the top level +# one. +allow-any-import-level= + +# Allow wildcard imports from modules that define __all__. +allow-wildcard-with-all=no + +# Analyse import fallback blocks. This can be used to support both Python 2 and +# 3 compatible code, which means that the block might have code that exists +# only in one or another interpreter, leading to false positives when analysed. +analyse-fallback-blocks=no + +# Deprecated modules which should not be used, separated by a comma. +deprecated-modules=optparse,tkinter.tix + +# Create a graph of external dependencies in the given file (report RP0402 must +# not be disabled). +ext-import-graph= + +# Create a graph of every (i.e. internal and external) dependencies in the +# given file (report RP0402 must not be disabled). +import-graph= + +# Create a graph of internal dependencies in the given file (report RP0402 must +# not be disabled). +int-import-graph= + +# Force import order to recognize a module as part of the standard +# compatibility libraries. +known-standard-library= + +# Force import order to recognize a module as part of a third party library. +known-third-party=enchant + +# Couples of modules and preferred modules, separated by a comma. +preferred-modules= + + +[EXCEPTIONS] + +# Exceptions that will emit a warning when being caught. Defaults to +# "BaseException, Exception". +overgeneral-exceptions=BaseException, + Exception diff --git a/README.md b/README.md index 9dc762f..5f34759 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# fedora-packages-static +# fedora-packages-static [![Azure CI Status](https://dev.azure.com/fedora-packages/Fedora%20Packages%20Static%20CI/_apis/build/status/Fedora%20Packages%20Static%20CI)](https://dev.azure.com/fedora-packages/Fedora%20Packages%20Static%20CI/_build?definitionId=1) This project replaces the former Fedora [packages app](https://apps.fedoraproject.org/packages/) which is built atop now dead