From 8b532e7838fb5b632f24cfc17788c4556087be0c Mon Sep 17 00:00:00 2001 From: James Antill Date: Apr 13 2023 17:41:12 +0000 Subject: [PATCH 1/4] Add two new commands, fix-dbs and weekly-totals. --- diff --git a/README.md b/README.md index f37d8c3..bbbc1dc 100644 --- a/README.md +++ b/README.md @@ -210,6 +210,7 @@ In practice, this is a three-part process: 3. Publish updated `totals.db` and `totals.csv` * See https://data-analysis.fedoraproject.org/csv-reports/countme/ * Might end up in different places/forms in the future + * Can also run countme-weekly-totals-ui.sh command to see text data easily. [^IPvBeefy]: Don't worry, 240.159.140.173 is a fake IP address. Actually, it's the 4-byte UTF-8 encoding for 🌭, U+1F32D HOT DOG. diff --git a/scripts/countme-fix-dbs.sh b/scripts/countme-fix-dbs.sh new file mode 100755 index 0000000..d67bb78 --- /dev/null +++ b/scripts/countme-fix-dbs.sh @@ -0,0 +1,75 @@ +#! /bin/sh -e + +cd /var/lib/countme/ + +conf_LOGDIR="/mnt/fedora_stats/combined-http" + +conf_LOGNAME="mirrors.fedoraproject.org-access.log" + +conf_progress=--progress +# conf_progress= + +conf_RW=true + +if $conf_RW && [ "x$(whoami)" != "xcountme" ]; then + echo "Need to be run as countme." + exit 1 +fi + +# Reload this month and the previous N ... put them in order so +# they come out backwards (and are loaded oldest to newest). +months="$(date +'%Y/%m')" +for i in 1 2 3; do + d="$(date +'%Y/%m' --date=$i' month ago')" + months="$d $months" +done + +echo "Reload: $months" +echo "Dir: $conf_LOGDIR" +echo "Pkg: $(rpm -q python3-mirrors-countme)" +echo "RW: $conf_RW" +sleep 5 + + +rawdb="/var/lib/countme/raw.db" +totsdb="/var/lib/countme/totals.db" +totscsv="/var/lib/countme/totals.csv" + +od="$(date -I)" + +if $conf_RW; then + if [ ! -d $od ] ; then + echo "Moving old DB files to: $od" + mkdir $od + mv raw.db totals.db $od || true + cp totals.csv $od || true + else + echo "Old dir already exists: $od" + echo " ** Keeping DB files." + fi +fi + +# This is a quick way of doing "recent" countme-update-rawdb.sh runs: +imported=false +for month in $months; do + echo "Importing: ${month}" + for day in $(seq -w 31); do + logfile="$conf_LOGDIR/${month}/${day}/$conf_LOGNAME" + if [ -f ${logfile}* ]; then + if $conf_RW; then + imported=true + parse-access-log.py $conf_progress --sqlite ${rawdb} ${logfile}* + else + ls -asF ${logfile}* + fi + fi + done +done + +if $imported; then +bash countme-update-totals.sh --rawdb ${rawdb} --totals-db ${totsdb} --totals-csv ${totscsv} $conf_progress +else +echo "Nothing imported!" +fi + + diff --git a/scripts/countme-weekly-totals-ui.sh b/scripts/countme-weekly-totals-ui.sh new file mode 100755 index 0000000..f4ff258 --- /dev/null +++ b/scripts/countme-weekly-totals-ui.sh @@ -0,0 +1,68 @@ +#!/bin/bash + +DISTROS=('Red Hat Enterprise' + 'CentOS Linux' + 'CentOS Stream' + 'Oracle Linux' + 'AlmaLinux' + 'Rocky' + ); +WEEKS=2 + +if [ "x$1" = "x--help" -o "x$1" = "xhelp" ]; then + echo "$0: [WEEKS]" + echo " Show CSV data in a more usable form for N weeks ago. Default: 2" + echo " 0/1 works but may be incomplete." + exit 0 +fi + +if [ "x$1" != "x" ]; then + WEEKS="$1" +fi + +DAYS=$((7 * $WEEKS)) +DATE=$( date -d "last monday - ${DAYS} days" -I ); + +ARCHES=('x86_64' 'aarch64' 'ppc64le' 's390x'); +RELEASES=('epel-8' 'epel-9'); +RELEASE3=('centos-baseos-9-stream'); +FILE=/var/lib/countme/totals.csv +## Countme minimum age +AGE=2 + +IFS="" + +echo "===== Fedora Base Stats =====" +for arch in ${ARCHES[@]}; do + grep "${DATE}.*fedora-3.,${arch}" ${FILE} | awk -F, -vW=${DATE} -vR="fedora-3." -vA=${arch} -vX=${AGE} 'BEGIN{x=0}; ($8>=X){x=x+$3}; END{printf("%-12s %-10s %-10s %9d\n",W,R,A,x);}'; +done + +echo "===== EPEL Base Stats =====" + +grep "${DATE}" /var/www/html/csv-reports/mirrors/mirrorsdata-all.csv | awk -F, -vW=${DATE} -vR="epel-7" -vA="all" '{x=$5; printf("%-12s %-10s %-10s %9d ( %9d )\n",W,R,A,x,(x*1.95));}' +grep "${DATE}" /var/www/html/csv-reports/mirrors/mirrorsdata-all.csv | awk -F, -vW=${DATE} -vR="epel-8" -vA="all" '{x=$72; printf("%-12s %-10s %-10s %9d ( %9d )\n",W,R,A,x,(x*1.95));}' +grep "${DATE}" /var/www/html/csv-reports/mirrors/mirrorsdata-all.csv | awk -F, -vW=${DATE} -vR="epel-9" -vA="all" '{x=$73; printf("%-12s %-10s %-10s %9d ( %9d )\n",W,R,A,x,(x*1.3));}' +for release in ${RELEASES[@]}; do + for arch in ${ARCHES[@]}; do + grep "${DATE}.*${release},${arch}" ${FILE} | awk -F, -vW=${DATE} -vR=${release} -vA=${arch} -vX=${AGE} 'BEGIN{x=0}; ($8>=X){x=x+$3}; END{printf("%-12s %-10s %-10s %9d\n",W,R,A,x);}'; + done +done + +echo "===== EPEL OS Stats ======" +for distro in ${DISTROS[@]}; do + for release in ${RELEASES[@]}; do + for arch in ${ARCHES[@]}; do + grep "${DATE}.*${distro}.*${release},${arch}" ${FILE} | awk -F, -vW=${DATE} -vD=${distro} -vR=${release} -vA=${arch} -vX=${AGE} 'BEGIN{x=0}; ($8>=X){x=x+$3}; END{printf("%-12s %-18s %-10s %-10s %9d\n",W,D,R,A,x);}'; + done + done +done + +echo "===== CS9 Base Stats =====" +FILE=/var/lib/countme/totals-centos.csv +for distro in 'CentOS Stream'; do + for release in ${RELEASE3[@]}; do + for arch in ${ARCHES[@]}; do + grep "${DATE}.*${distro}.*${release},${arch}" ${FILE} | awk -F, -vW=${DATE} -vD=${distro} -vR=${release} -vA=${arch} -vX=${AGE} 'BEGIN{x=0}; ($8>=X){x=x+$3}; END{printf("%-12s %-18s %-10s %-10s %9d\n",W,D,R,A,x);}'; + done + done +done diff --git a/setup.cfg b/setup.cfg index 3868e88..5c626af 100644 --- a/setup.cfg +++ b/setup.cfg @@ -24,6 +24,8 @@ scripts = scripts/countme-update-totals.sh scripts/countme-csv2sqlite.sh scripts/countme-sqlite2csv.sh + scripts/countme-fix-dbs.sh + scripts/countme-weekly-totals-ui.sh [options.extras_require] # Not sure what the _minimum_ tqdm version is but this should be fine From 13def48448f53b42acf51de67ff46dbced45a8ef Mon Sep 17 00:00:00 2001 From: James Antill Date: Apr 13 2023 17:43:29 +0000 Subject: [PATCH 2/4] Change the pytest.skip method. --- diff --git a/tests/test_integrate.py b/tests/test_integrate.py index 03ee4d6..ff14b33 100644 --- a/tests/test_integrate.py +++ b/tests/test_integrate.py @@ -156,8 +156,9 @@ def log_data(draw): @settings(suppress_health_check=(HealthCheck.too_slow,)) @given(log_data()) -@pytest.mark.skip(reason="Zuul doesn't like this") def test_log(loglines): + if True: # A way to see if we are running in Zuul/CI would be good. + pytest.skip("Zuul doesn't like this") with tempfile.TemporaryDirectory() as tmp_dir: matcher = CountmeMatcher args = Args( From cdbb21f62d08c15e40abebda6cd670dcedd5bc3d Mon Sep 17 00:00:00 2001 From: James Antill Date: Apr 15 2023 00:10:19 +0000 Subject: [PATCH 3/4] Add a new command countme-trim-raw, to remove old entries from raw.db. --- diff --git a/countme-trim-raw.py b/countme-trim-raw.py new file mode 100755 index 0000000..3a2a631 --- /dev/null +++ b/countme-trim-raw.py @@ -0,0 +1,126 @@ +#!/usr/bin/python3 +# countme-trim-raw.py - Trim the raw.db file to the next week start. +# +# Copyright (C) 2023, Red Hat Inc. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# Author: James Antill +# +# The main point of this script is to remove all the data from raw.db +# upto the next week start. If you then run it again it'll remove the next week. +# Uses mindate() but then runs direct SQL. + +import sys +import time +import argparse + +import sqlite3 + +import countme + +# =========================================================================== +# ====== CLI parser & main() ================================================ +# =========================================================================== + + +def parse_args(argv=None): + p = argparse.ArgumentParser( + description="Parse Fedora access.log files.", + ) + p.add_argument("-V", "--version", action="version", version="%(prog)s 0.0.1") + + p.add_argument( + "--sqlite", + metavar="DBFILE", + help="write to a sqlite database", + ) + p.add_argument( + "--noop", + dest="rw", + default=True, + action="store_false", + help="Skip deleting the entries.", + ) + + args = p.parse_args(argv) + + return args + +# Mostly borrowed from countme/__init__ +def mintime(cur): + cur = cur.execute("SELECT MIN(timestamp) FROM countme_raw") + return cur.fetchone()[0] + +# Find the next week to trim, given the earliest timestamp. +def next_week(mintime): + beg = countme.COUNTME_EPOCH + while beg <= mintime: + beg += countme.WEEK_LEN + # Now beg is the first week _after_ the mintime. + return beg + +def _num_entries_before(cur, timestamp): + cur = cur.execute("SELECT COUNT(*) FROM countme_raw WHERE timestamp < ?", (timestamp,)) + return cur.fetchone()[0] + +def _num_entries(cur): + cur = cur.execute("SELECT COUNT(*) FROM countme_raw") + return cur.fetchone()[0] + +def _del_entries_before(con, timestamp): + con.execute("DELETE FROM countme_raw WHERE timestamp < ?", (timestamp,)) + con.commit() + +def tm2ui(timestamp): + tm = time.gmtime(timestamp) + return time.strftime("%Y-%m-%d %H:%M:%S", tm) + +import locale +locale.setlocale(locale.LC_ALL, '') +def num2ui(num): + ret = locale.format_string('%d', num, grouping=True) + mlen = len("100,000,000") + if len(ret) < mlen: + ret = " " * (mlen-len(ret)) + ret + return ret + + +def get_trim_data(args): + data = {} + filename = args.sqlite + data['sql'] = sqlite3.connect(f"file:{filename}?mode=rwc", uri=True) + data['mintime'] = mintime(data['sql']) + data['week'] = next_week(data['mintime']) + + print("First timestamp:", tm2ui(data['mintime'])) + print("Next week :", tm2ui(data['week'])) + print("Entries :", num2ui(_num_entries(data['sql']))) + print("Entries to trim:", num2ui(_num_entries_before(data['sql'], data['week']))) + + return data + +def trim_data(data): + print(" ** About to DELETE data. **") + time.sleep(5) + _del_entries_before(data['sql'], data['week']) + +if __name__ == "__main__": + try: + args = parse_args() + data = get_trim_data(args) + if args.rw: + trim_data(data) + except KeyboardInterrupt: + raise SystemExit(3) # sure, 3 is good, why not diff --git a/setup.cfg b/setup.cfg index 5c626af..6d6cb06 100644 --- a/setup.cfg +++ b/setup.cfg @@ -20,6 +20,7 @@ packages = countme scripts = parse-access-log.py countme-totals.py + countme-trim-raw.py scripts/countme-update-rawdb.sh scripts/countme-update-totals.sh scripts/countme-csv2sqlite.sh From f2c9b419707a517ec9e8e74ae75b4ea077a3d617 Mon Sep 17 00:00:00 2001 From: James Antill Date: Apr 15 2023 02:10:51 +0000 Subject: [PATCH 4/4] Add a new command countme-delete-totals, to remove latest week from totals.db. --- diff --git a/countme-delete-totals.py b/countme-delete-totals.py new file mode 100755 index 0000000..7f70911 --- /dev/null +++ b/countme-delete-totals.py @@ -0,0 +1,119 @@ +#!/usr/bin/python3 +# countme-delete-totals.py - Delete the last week from the totals.db file. +# +# Copyright (C) 2023, Red Hat Inc. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# Author: James Antill +# +# The main point of this script is to remove the last weeknum of data from +# totals.db. If you then run it again it'll remove the next week. + +import sys +import time +import argparse + +import sqlite3 + +import countme +import countme.totals + +# =========================================================================== +# ====== CLI parser & main() ================================================ +# =========================================================================== + + +def parse_args(argv=None): + p = argparse.ArgumentParser( + description="Parse Fedora access.log files.", + ) + p.add_argument("-V", "--version", action="version", version="%(prog)s 0.0.1") + + p.add_argument( + "--sqlite", + metavar="DBFILE", + help="write to a sqlite database", + ) + p.add_argument( + "--noop", + dest="rw", + default=True, + action="store_false", + help="Skip deleting the entries.", + ) + + args = p.parse_args(argv) + + return args + +def last_week(cur): + cur = cur.execute("SELECT MAX(weeknum) FROM countme_totals") + return cur.fetchone()[0] + +def _num_entries_for(cur, weeknum): + cur = cur.execute("SELECT COUNT(*) FROM countme_totals WHERE weeknum = ?", (weeknum,)) + return cur.fetchone()[0] + +def _num_entries(cur): + cur = cur.execute("SELECT COUNT(*) FROM countme_totals") + return cur.fetchone()[0] + +def _del_entries_for(con, weeknum): + con.execute("DELETE FROM countme_totals WHERE weeknum = ?", (weeknum,)) + con.commit() + +def tm2ui(timestamp): + tm = time.gmtime(timestamp) + return time.strftime("%Y-%m-%d %H:%M:%S", tm) + +def weeknum2tm(weeknum): + ret = countme.COUNTME_EPOCH + return ret + int(weeknum)*countme.WEEK_LEN + +import locale +locale.setlocale(locale.LC_ALL, '') +def num2ui(num): + ret = locale.format_string('%d', num, grouping=True) + mlen = len("100,000,000") + if len(ret) < mlen: + ret = " " * (mlen-len(ret)) + ret + return ret + + +def get_trim_data(args): + data = {} + filename = args.sqlite + data['sql'] = sqlite3.connect(f"file:{filename}?mode=rwc", uri=True) + data['week'] = last_week(data['sql']) + + print("Next week :", data['week'], tm2ui(weeknum2tm(data['week']))) + print("Entries :", num2ui(_num_entries(data['sql']))) + print("Entries to del:", num2ui(_num_entries_for(data['sql'], data['week']))) + + return data + +def trim_data(data): + print(" ** About to DELETE data. **") + time.sleep(5) + _del_entries_for(data['sql'], data['week']) + +if __name__ == "__main__": + try: + args = parse_args() + data = get_trim_data(args) + if args.rw: + trim_data(data) + except KeyboardInterrupt: + raise SystemExit(3) # sure, 3 is good, why not diff --git a/setup.cfg b/setup.cfg index 6d6cb06..c4fddd3 100644 --- a/setup.cfg +++ b/setup.cfg @@ -19,6 +19,7 @@ python_requires = >=3.5 packages = countme scripts = parse-access-log.py + countme-delete-totals.py countme-totals.py countme-trim-raw.py scripts/countme-update-rawdb.sh